From 25a1e8d1b71d1f877bec09cb92af0caf7c28db2a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 5 Feb 2016 13:15:41 -0800 Subject: [PATCH 001/920] Initial commit This is the initial commit of the repo that will track development against distro kernels. This is an import of a prototype branch in the upstream kernel that only had a few initial commits. It needed to move to the old readdir interface and use find_or_create_page() instead of pagecache_get_page() to build in older distro kernels. --- kmod/.gitignore | 7 + kmod/Makefile | 4 + kmod/src/Kconfig | 10 + kmod/src/Makefile | 3 + kmod/src/dir.c | 551 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/dir.h | 10 + kmod/src/format.h | 122 ++++++++++ kmod/src/inode.c | 272 +++++++++++++++++++++++ kmod/src/inode.h | 32 +++ kmod/src/item.c | 423 +++++++++++++++++++++++++++++++++++ kmod/src/item.h | 37 ++++ kmod/src/key.h | 43 ++++ kmod/src/lsm.c | 330 +++++++++++++++++++++++++++ kmod/src/lsm.h | 6 + kmod/src/mkfs.c | 52 +++++ kmod/src/mkfs.h | 6 + kmod/src/super.c | 103 +++++++++ kmod/src/super.h | 22 ++ 18 files changed, 2033 insertions(+) create mode 100644 kmod/.gitignore create mode 100644 kmod/Makefile create mode 100644 kmod/src/Kconfig create mode 100644 kmod/src/Makefile create mode 100644 kmod/src/dir.c create mode 100644 kmod/src/dir.h create mode 100644 kmod/src/format.h create mode 100644 kmod/src/inode.c create mode 100644 kmod/src/inode.h create mode 100644 kmod/src/item.c create mode 100644 kmod/src/item.h create mode 100644 kmod/src/key.h create mode 100644 kmod/src/lsm.c create mode 100644 kmod/src/lsm.h create mode 100644 kmod/src/mkfs.c create mode 100644 kmod/src/mkfs.h create mode 100644 kmod/src/super.c create mode 100644 kmod/src/super.h diff --git a/kmod/.gitignore b/kmod/.gitignore new file mode 100644 index 00000000..9d66c4e8 --- /dev/null +++ b/kmod/.gitignore @@ -0,0 +1,7 @@ +src/*.o +src/*.ko +src/*.mod.c +src/*.cmd +src/.tmp_versions/ +src/Module.symvers +src/modules.order diff --git a/kmod/Makefile b/kmod/Makefile new file mode 100644 index 00000000..07fbc001 --- /dev/null +++ b/kmod/Makefile @@ -0,0 +1,4 @@ +ALL: module + +module: + make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src diff --git a/kmod/src/Kconfig b/kmod/src/Kconfig new file mode 100644 index 00000000..eb097405 --- /dev/null +++ b/kmod/src/Kconfig @@ -0,0 +1,10 @@ +config SCOUTFS_FS + tristate "scoutfs filesystem" + help + scoutfs is a clustered file system that stores data in large + blocks in shared block storage. + + To compile this file system support as a module, choose M here. The + module will be called scoutfs. + + If unsure, say N. diff --git a/kmod/src/Makefile b/kmod/src/Makefile new file mode 100644 index 00000000..239e8aef --- /dev/null +++ b/kmod/src/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o + +scoutfs-y += dir.o inode.o item.o lsm.o mkfs.o super.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c new file mode 100644 index 00000000..744759a9 --- /dev/null +++ b/kmod/src/dir.c @@ -0,0 +1,551 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "format.h" +#include "dir.h" +#include "inode.h" +#include "key.h" +#include "item.h" +#include "super.h" + +/* + * Directory entries are stored in items whose offset is determined by + * the hash of the entry's name. This was primarily chosen to minimize + * the amount of data stored for each entry. + * + * Because we're hashing the name we need to worry about collisions. We + * store all the entries with the same hash value in the item. This was + * done so that create works with one specific item. + * + * readdir iterates over these items in hash order. The high bits of + * the entry's readdir f_pos come from the item offset while the low + * bits come from a collision number in the entry. + * + * The full readdir position, and thus the absolute max number of + * entries in a directory, is limited to 2^31 to avoid the risk of + * breaking legacy environments. Even with a relatively small 27bit + * item offset allowing 16 colliding entries gets well into hundreds of + * millions of entries before an item fills up and we return a premature + * ENOSPC. Hundreds of millions in a single dir ought to be, wait for + * it, good enough for anybody. + * + * Each item's contents are protected by the dir inode's i_mutex that + * callers acquire before calling our dir operations. If we wanted more + * fine grained concurrency, and we might, we'd have to be careful to + * manage the shared items. + */ + +static unsigned int mode_to_type(umode_t mode) +{ +#define S_SHIFT 12 + static unsigned char mode_types[S_IFMT >> S_SHIFT] = { + [S_IFIFO >> S_SHIFT] = SCOUTFS_DT_FIFO, + [S_IFCHR >> S_SHIFT] = SCOUTFS_DT_CHR, + [S_IFDIR >> S_SHIFT] = SCOUTFS_DT_DIR, + [S_IFBLK >> S_SHIFT] = SCOUTFS_DT_BLK, + [S_IFREG >> S_SHIFT] = SCOUTFS_DT_REG, + [S_IFLNK >> S_SHIFT] = SCOUTFS_DT_LNK, + [S_IFSOCK >> S_SHIFT] = SCOUTFS_DT_SOCK, + }; + + return mode_types[(mode & S_IFMT) >> S_SHIFT]; +#undef S_SHIFT +} + +#if 0 +static unsigned int dentry_type(unsigned int type) +{ + static unsigned char types[] = { + [SCOUTFS_DT_FIFO] = DT_FIFO, + [SCOUTFS_DT_CHR] = DT_CHR, + [SCOUTFS_DT_DIR] = DT_DIR, + [SCOUTFS_DT_BLK] = DT_BLK, + [SCOUTFS_DT_REG] = DT_REG, + [SCOUTFS_DT_LNK] = DT_LNK, + [SCOUTFS_DT_SOCK] = DT_SOCK, + [SCOUTFS_DT_WHT] = DT_WHT, + }; + + if (type < ARRAY_SIZE(types)) + return types[type]; + + return DT_UNKNOWN; +} +#endif + +static int names_equal(const char *name_a, int len_a, const char *name_b, + int len_b) +{ + return (len_a == len_b) && !memcmp(name_a, name_b, len_a); +} + +/* + * Return the offset portion of a dirent key from the hash of the name. + * + * XXX This crc nonsense is a quick hack. We'll want something a + * lot stronger like siphash. + */ +static u32 name_hash(struct inode *dir, const char *name, unsigned int len) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(dir); + + return crc32c(ci->salt, name, len) >> (32 - SCOUTFS_DIRENT_OFF_BITS); +} + +static unsigned int dent_bytes(unsigned int name_len) +{ + return sizeof(struct scoutfs_dirent) + name_len; +} + +static unsigned int dent_val_off(struct scoutfs_item *item, + struct scoutfs_dirent *dent) +{ + return (char *)dent - (char *)item->val; +} + +static inline struct scoutfs_dirent *next_dent(struct scoutfs_item *item, + struct scoutfs_dirent *dent) +{ + unsigned int next_off; + + next_off = dent_val_off(item, dent) + dent_bytes(dent->name_len); + if (next_off == item->val_len) + return NULL; + + return item->val + next_off; +} + +#define for_each_item_dent(item, dent) \ + for (dent = item->val; dent; dent = next_dent(item, dent)) + +struct dentry_info { + /* + * The key offset and collision nr are stored so that we don't + * have to either hash the name to find the item or compare + * names to find the dirent in the item. + */ + u32 key_offset; + u8 coll_nr; +}; + +static struct kmem_cache *scoutfs_dentry_cachep; + +static struct dentry_info *alloc_dentry_info(struct dentry *dentry) +{ + struct dentry_info *di; + + /* XXX read mb? */ + if (dentry->d_fsdata) + return dentry->d_fsdata; + + di = kmem_cache_zalloc(scoutfs_dentry_cachep, GFP_NOFS); + if (!di) + return ERR_PTR(-ENOMEM); + + spin_lock(&dentry->d_lock); + if (!dentry->d_fsdata) + dentry->d_fsdata = di; + spin_unlock(&dentry->d_lock); + + if (di != dentry->d_fsdata) + kmem_cache_free(scoutfs_dentry_cachep, di); + + return dentry->d_fsdata; +} + +/* + * Lookup searches for an entry for the given name amongst the entries + * stored in the item at the name's hash. + */ +static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct scoutfs_dirent *dent; + struct scoutfs_item *item; + struct dentry_info *di; + struct scoutfs_key key; + struct inode *inode; + u64 ino = 0; + u32 h = 0; + u32 nr = 0; + int ret; + + di = alloc_dentry_info(dentry); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + goto out; + } + + if (dentry->d_name.len > SCOUTFS_NAME_LEN) { + ret = -ENAMETOOLONG; + goto out; + } + + h = name_hash(dir, dentry->d_name.name, dentry->d_name.len); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + + item = scoutfs_item_lookup(sb, &key); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + goto out; + } + + ret = -ENOENT; + for_each_item_dent(item, dent) { + if (names_equal(dentry->d_name.name, dentry->d_name.len, + dent->name, dent->name_len)) { + ino = le64_to_cpu(dent->ino); + nr = dent->coll_nr; + ret = 0; + break; + } + } + + scoutfs_item_put(item); +out: + if (ret == -ENOENT) { + inode = NULL; + } else if (ret) { + inode = ERR_PTR(ret); + } else { + di->key_offset = h; + di->coll_nr = nr; + inode = scoutfs_iget(sb, ino); + } + + return d_splice_alias(inode, dentry); +} + +/* this exists upstream so we can just delete it in a forward port */ +static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) +{ + struct dentry *dentry = file->f_path.dentry; + struct inode *inode = dentry->d_inode; + struct inode *parent = dentry->d_parent->d_inode; + + if (file->f_pos == 0) { + if (!filldir(dirent, ".", 1, 1, scoutfs_ino(inode), DT_DIR)) + return 0; + file->f_pos = 1; + } + + if (file->f_pos == 1) { + if (!filldir(dirent, "..", 2, 1, scoutfs_ino(parent), DT_DIR)) + return 0; + file->f_pos = 2; + } + + return 1; +} + +/* + * readdir finds the next entry at or past the hash|coll_nr stored in + * the ctx->pos (f_pos). + * + * It will need to be careful not to read past the region of the dirent + * hash offset keys that it has access to. + */ +static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_dirent *dent; + struct scoutfs_key last_key; + struct scoutfs_item *item; + struct scoutfs_key key; + u32 nr; + u32 off; + u64 pos; + int ret = 0; + + if (!dir_emit_dots(file, dirent, filldir)) + return 0; + + scoutfs_set_key(&last_key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + SCOUTFS_DIRENT_OFF_MASK); + + do { + off = file->f_pos >> SCOUTFS_DIRENT_COLL_BITS; + nr = file->f_pos & SCOUTFS_DIRENT_COLL_MASK; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + off); + item = scoutfs_item_next(sb, &key); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + if (ret == -ENOENT) + ret = 0; + break; + } + + if (scoutfs_key_cmp(&item->key, &last_key) > 0) { + scoutfs_item_put(item); + break; + } + + /* reset nr to 0 if we found the next item */ + if (scoutfs_key_offset(&item->key) != off) + nr = 0; + + pos = scoutfs_key_offset(&item->key) + << SCOUTFS_DIRENT_COLL_BITS; + for_each_item_dent(item, dent) { + if (dent->coll_nr < nr) + continue; + + if (!filldir(dirent, dent->name, dent->name_len, pos, + le64_to_cpu(dent->ino), dent->type)) + break; + + file->f_pos = (pos | dent->coll_nr) + 1; + } + + scoutfs_item_put(item); + + /* advance to the next hash value if we finished item */ + if (dent == NULL) + file->f_pos = pos + (1 << SCOUTFS_DIRENT_COLL_BITS); + + } while (dent == NULL); + + return ret; +} + +static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t rdev) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = NULL; + struct scoutfs_dirent *dent; + struct scoutfs_item *item; + struct dentry_info *di; + struct scoutfs_key key; + int bytes; + int ret; + int off; + u64 nr; + u64 h; + + di = alloc_dentry_info(dentry); + if (IS_ERR(di)) + return PTR_ERR(di); + + if (dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + + inode = scoutfs_new_inode(sb, dir, mode, rdev); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + h = name_hash(dir, dentry->d_name.name, dentry->d_name.len); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + bytes = dent_bytes(dentry->d_name.len); + + item = scoutfs_item_lookup(sb, &key); + if (item == ERR_PTR(-ENOENT)) { + item = scoutfs_item_create(sb, &key, bytes); + if (!IS_ERR(item)) { + /* mark a newly created item */ + dent = item->val; + dent->name_len = 0; + } + } + if (IS_ERR(item)) { + ret = PTR_ERR(item); + goto out; + } + + ret = 0; + nr = 0; + for_each_item_dent(item, dent) { + /* the common case of a newly created item */ + if (!dent->name_len) + break; + + /* XXX check for eexist? can't happen? */ + + /* found a free coll nr, insert here */ + if (nr < dent->coll_nr) { + off = dent_val_off(item, dent); + ret = scoutfs_item_expand(item, off, bytes); + if (!ret) + dent = item->val + off; + break; + } + + /* the item's full */ + if (nr++ == SCOUTFS_DIRENT_COLL_MASK) { + ret = -ENOSPC; + break; + } + } + + if (!ret) { + dent->ino = cpu_to_le64(scoutfs_ino(inode)); + dent->type = mode_to_type(inode->i_mode); + dent->coll_nr = nr; + dent->name_len = dentry->d_name.len; + memcpy(dent->name, dentry->d_name.name, dent->name_len); + di->key_offset = h; + di->coll_nr = nr; + } + + scoutfs_item_put(item); + + if (ret) + goto out; + + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + + if (S_ISDIR(mode)) { + inc_nlink(inode); + inc_nlink(dir); + } + + mark_inode_dirty(inode); + mark_inode_dirty(dir); + + insert_inode_hash(inode); + d_instantiate(dentry, inode); +out: + /* XXX delete the inode item here */ + if (ret && !IS_ERR_OR_NULL(inode)) + iput(inode); + return ret; +} + +/* XXX hmm, do something with excl? */ +static int scoutfs_create(struct inode *dir, struct dentry *dentry, + umode_t mode, bool excl) +{ + return scoutfs_mknod(dir, dentry, mode | S_IFREG, 0); +} + +static int scoutfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + return scoutfs_mknod(dir, dentry, mode | S_IFDIR, 0); +} + +/* + * Unlink removes the entry from its item and removes the item if ours + * was the only remaining entry. + */ +static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = dentry->d_inode; + struct timespec ts = current_kernel_time(); + struct scoutfs_dirent *dent; + struct scoutfs_item *item; + struct dentry_info *di; + struct scoutfs_key key; + int ret = 0; + + if (WARN_ON_ONCE(!dentry->d_fsdata)) + return -EINVAL; + di = dentry->d_fsdata; + + trace_printk("dir size %llu entry k_off nr %u %u\n", + i_size_read(inode), di->key_offset, di->coll_nr); + + if (S_ISDIR(inode->i_mode) && i_size_read(inode)) + return -ENOTEMPTY; + + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, + di->key_offset); + + item = scoutfs_item_lookup(sb, &key); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + goto out; + } + + /* XXX error to not find the coll nr we were looking for? */ + for_each_item_dent(item, dent) { + if (dent->coll_nr != di->coll_nr) + continue; + + /* XXX compare names and eio? */ + + if (item->val_len == dent_bytes(dent->name_len)) { + scoutfs_item_delete(sb, item); + ret = 0; + } else { + ret = scoutfs_item_shrink(item, + dent_val_off(item, dent), + dent_bytes(dent->name_len)); + } + dent = NULL; + break; + } + + scoutfs_item_put(item); + + if (ret) + goto out; + + dir->i_ctime = ts; + dir->i_mtime = ts; + i_size_write(dir, i_size_read(dir) - dentry->d_name.len); + + inode->i_ctime = ts; + drop_nlink(inode); + if (S_ISDIR(inode->i_mode)) { + drop_nlink(dir); + drop_nlink(inode); + } + mark_inode_dirty(inode); + mark_inode_dirty(dir); + +out: + return ret; +} + +const struct file_operations scoutfs_dir_fops = { + .readdir = scoutfs_readdir, +}; + +const struct inode_operations scoutfs_dir_iops = { + .lookup = scoutfs_lookup, + .mknod = scoutfs_mknod, + .create = scoutfs_create, + .mkdir = scoutfs_mkdir, + .unlink = scoutfs_unlink, + .rmdir = scoutfs_unlink, +}; + +void scoutfs_dir_exit(void) +{ + if (scoutfs_dentry_cachep) { + kmem_cache_destroy(scoutfs_dentry_cachep); + scoutfs_dentry_cachep = NULL; + } +} + +int scoutfs_dir_init(void) +{ + scoutfs_dentry_cachep = kmem_cache_create("scoutfs_dentry_info", + sizeof(struct dentry_info), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (!scoutfs_dentry_cachep) + return -ENOMEM; + + return 0; +} diff --git a/kmod/src/dir.h b/kmod/src/dir.h new file mode 100644 index 00000000..3ee15f0f --- /dev/null +++ b/kmod/src/dir.h @@ -0,0 +1,10 @@ +#ifndef _SCOUTFS_DIR_H_ +#define _SCOUTFS_DIR_H_ + +extern const struct file_operations scoutfs_dir_fops; +extern const struct inode_operations scoutfs_dir_iops; + +int scoutfs_dir_init(void); +void scoutfs_dir_exit(void); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h new file mode 100644 index 00000000..67958f12 --- /dev/null +++ b/kmod/src/format.h @@ -0,0 +1,122 @@ +#ifndef _SCOUTFS_FORMAT_H_ +#define _SCOUTFS_FORMAT_H_ + +#define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ + +#define SCOUTFS_BLOCK_SHIFT 22 +#define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) + +/* + * This bloom size is chosen to have a roughly 1% false positive rate + * for ~90k items which is roughly the worst case for a block full of + * dirents with reasonably small names. Pathologically smaller items + * could be even more dense. + */ +#define SCOUTFS_BLOOM_FILTER_BYTES (128 * 1024) +#define SCOUTFS_BLOOM_FILTER_BITS (SCOUTFS_BLOOM_FILTER_BYTES * 8) +#define SCOUTFS_BLOOM_INDEX_BITS (ilog2(SCOUTFS_BLOOM_FILTER_BITS)) +#define SCOUTFS_BLOOM_INDEX_MASK ((1 << SCOUTFS_BLOOM_INDEX_BITS) - 1) +#define SCOUTFS_BLOOM_INDEX_NR 7 + +/* + * We should be able to make the offset smaller if neither dirents nor + * data items use the full 64 bits. + */ +struct scoutfs_key { + __le64 inode; + u8 type; + __le64 offset; +} __packed; + +#define SCOUTFS_INODE_KEY 128 +#define SCOUTFS_DIRENT_KEY 192 + +struct scoutfs_lsm_block { + struct scoutfs_key first; + struct scoutfs_key last; + __le32 nr_items; + /* u8 bloom[SCOUTFS_BLOOM_BYTES]; */ + /* struct scoutfs_item_header items[0] .. */ +} __packed; + +struct scoutfs_item_header { + struct scoutfs_key key; + __le16 val_len; +} __packed; + + +struct scoutfs_timespec { + __le64 sec; + __le32 nsec; +} __packed; + +/* + * XXX + * - otime? + * - compat flags? + * - version? + * - generation? + * - be more careful with rdev? + */ +struct scoutfs_inode { + __le64 size; + __le64 blocks; + __le32 nlink; + __le32 uid; + __le32 gid; + __le32 mode; + __le32 rdev; + __le32 salt; + struct scoutfs_timespec atime; + struct scoutfs_timespec ctime; + struct scoutfs_timespec mtime; +} __packed; + +#define SCOUTFS_ROOT_INO 1 + +/* + * Dirents are stored in items with an offset of the hash of their name. + * Colliding names are packed into the value. + */ +struct scoutfs_dirent { + __le64 ino; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 type:4, + coll_nr:4; +#else + __u8 coll_nr:4, + type:4; +#endif + __u8 name_len; + __u8 name[0]; +} __packed; + +#define SCOUTFS_NAME_LEN 255 + +/* + * We only use 31 bits for readdir positions so that we don't confuse + * old signed 32bit f_pos applications or those on the other side of + * network protocols that have limited readir positions. + */ + +#define SCOUTFS_DIRENT_OFF_BITS 27 +#define SCOUTFS_DIRENT_OFF_MASK ((1 << SCOUTFS_DIRENT_OFF_BITS) - 1) +#define SCOUTFS_DIRENT_COLL_BITS 4 +#define SCOUTFS_DIRENT_COLL_MASK ((1 << SCOUTFS_DIRENT_COLL_BITS) - 1) + +/* getdents returns the *next* pos with each entry. so we can't return ~0 */ +#define SCOUTFS_DIRENT_MAX_POS \ + (((1 << (SCOUTFS_DIRENT_OFF_BITS + SCOUTFS_DIRENT_COLL_BITS)) - 1) - 1) + +enum { + SCOUTFS_DT_FIFO = 0, + SCOUTFS_DT_CHR, + SCOUTFS_DT_DIR, + SCOUTFS_DT_BLK, + SCOUTFS_DT_REG, + SCOUTFS_DT_LNK, + SCOUTFS_DT_SOCK, + SCOUTFS_DT_WHT, +}; + +#endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c new file mode 100644 index 00000000..02446332 --- /dev/null +++ b/kmod/src/inode.c @@ -0,0 +1,272 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "key.h" +#include "inode.h" +#include "item.h" +#include "dir.h" + +/* + * XXX + * - worry about i_ino trunctation, not sure if we do anything + * - use inode item value lengths for forward/back compat + */ + +static struct kmem_cache *scoutfs_inode_cachep; + +static void scoutfs_inode_ctor(void *obj) +{ + struct scoutfs_inode_info *ci = obj; + + inode_init_once(&ci->inode); +} + +struct inode *scoutfs_alloc_inode(struct super_block *sb) +{ + struct scoutfs_inode_info *ci; + + ci = kmem_cache_alloc(scoutfs_inode_cachep, GFP_NOFS); + if (!ci) + return NULL; + + return &ci->inode; +} + +static void scoutfs_i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + + trace_printk("freeing inode %p\n", inode); + kmem_cache_free(scoutfs_inode_cachep, SCOUTFS_I(inode)); +} + +void scoutfs_destroy_inode(struct inode *inode) +{ + call_rcu(&inode->i_rcu, scoutfs_i_callback); +} + +/* + * Called once new inode allocation or inode reading has initialized + * enough of the inode for us to set the ops based on the mode. + */ +static void set_inode_ops(struct inode *inode) +{ + switch (inode->i_mode & S_IFMT) { + case S_IFREG: +// inode->i_mapping->a_ops = &scoutfs_file_aops; +// inode->i_op = &scoutfs_file_iops; +// inode->i_fop = &scoutfs_file_fops; + break; + case S_IFDIR: + inode->i_op = &scoutfs_dir_iops; + inode->i_fop = &scoutfs_dir_fops; + break; + case S_IFLNK: +// inode->i_op = &scoutfs_symlink_iops; + break; + default: +// inode->i_op = &scoutfs_special_iops; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + break; + } +} + +static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + + i_size_write(inode, le64_to_cpu(cinode->size)); + set_nlink(inode, le32_to_cpu(cinode->nlink)); + i_uid_write(inode, le32_to_cpu(cinode->uid)); + i_gid_write(inode, le32_to_cpu(cinode->gid)); + inode->i_mode = le32_to_cpu(cinode->mode); + inode->i_rdev = le32_to_cpu(cinode->rdev); + inode->i_atime.tv_sec = le64_to_cpu(cinode->atime.sec); + inode->i_atime.tv_nsec = le32_to_cpu(cinode->atime.nsec); + inode->i_mtime.tv_sec = le64_to_cpu(cinode->mtime.sec); + inode->i_mtime.tv_nsec = le32_to_cpu(cinode->mtime.nsec); + inode->i_ctime.tv_sec = le64_to_cpu(cinode->ctime.sec); + inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); + + ci->salt = le32_to_cpu(cinode->salt); +} + +static int scoutfs_read_locked_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_item *item; + struct scoutfs_key key; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + + item = scoutfs_item_lookup(sb, &key); + if (IS_ERR(item)) + return PTR_ERR(item); + + load_inode(inode, item->val); + scoutfs_item_put(item); + + return 0; +} + +static int scoutfs_iget_test(struct inode *inode, void *arg) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 *ino = arg; + + return ci->ino == *ino; +} + +static int scoutfs_iget_set(struct inode *inode, void *arg) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 *ino = arg; + + inode->i_ino = *ino; + ci->ino = *ino; + + return 0; +} + +struct inode *scoutfs_iget(struct super_block *sb, u64 ino) +{ + struct inode *inode; + int ret; + + inode = iget5_locked(sb, ino, scoutfs_iget_test, scoutfs_iget_set, + &ino); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode->i_state & I_NEW) { + ret = scoutfs_read_locked_inode(inode); + if (ret) { + iget_failed(inode); + inode = ERR_PTR(ret); + } else { + set_inode_ops(inode); + unlock_new_inode(inode); + } + } + + return inode; +} + +static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + + cinode->size = cpu_to_le64(i_size_read(inode)); + cinode->nlink = cpu_to_le32(inode->i_nlink); + cinode->uid = cpu_to_le32(i_uid_read(inode)); + cinode->gid = cpu_to_le32(i_gid_read(inode)); + cinode->mode = cpu_to_le32(inode->i_mode); + cinode->rdev = cpu_to_le32(inode->i_rdev); + cinode->atime.sec = cpu_to_le64(inode->i_atime.tv_sec); + cinode->atime.nsec = cpu_to_le32(inode->i_atime.tv_nsec); + cinode->ctime.sec = cpu_to_le64(inode->i_ctime.tv_sec); + cinode->ctime.nsec = cpu_to_le32(inode->i_ctime.tv_nsec); + cinode->mtime.sec = cpu_to_le64(inode->i_mtime.tv_sec); + cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); + + cinode->salt = cpu_to_le32(ci->salt); +} + +/* + * Every time we modify the inode in memory we copy it to its inode + * item. This lets us write out blocks of items without having to track + * down dirty vfs inodes and safely copy them into items before writing. + */ +int scoutfs_inode_update(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_item *item; + struct scoutfs_key key; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + + item = scoutfs_item_lookup(sb, &key); + if (IS_ERR(item)) + return PTR_ERR(item); + + store_inode(item->val, inode); + scoutfs_item_put(item); + + return 0; +} + +/* + * Allocate and initialize a new inode. The caller is responsible for + * creating links to it and updating it. @dir can be null. + */ +struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode, dev_t rdev) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_inode_info *ci; + struct scoutfs_item *item; + struct scoutfs_key key; + struct inode *inode; + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + ci = SCOUTFS_I(inode); + ci->ino = atomic64_inc_return(&sbi->next_ino); + get_random_bytes(&ci->salt, sizeof(ci->salt)); + + inode->i_ino = ci->ino; + inode_init_owner(inode, dir, mode); + inode_set_bytes(inode, 0); + inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; + inode->i_rdev = rdev; + set_inode_ops(inode); + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + + item = scoutfs_item_create(inode->i_sb, &key, + sizeof(struct scoutfs_inode)); + if (IS_ERR(item)) { + iput(inode); + inode = ERR_CAST(item); + } + return inode; +} + +void scoutfs_inode_exit(void) +{ + if (scoutfs_inode_cachep) { + rcu_barrier(); + kmem_cache_destroy(scoutfs_inode_cachep); + scoutfs_inode_cachep = NULL; + } +} + +int scoutfs_inode_init(void) +{ + scoutfs_inode_cachep = kmem_cache_create("scoutfs_inode_info", + sizeof(struct scoutfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT, + scoutfs_inode_ctor); + if (!scoutfs_inode_cachep) + return -ENOMEM; + + return 0; +} diff --git a/kmod/src/inode.h b/kmod/src/inode.h new file mode 100644 index 00000000..bb9a6149 --- /dev/null +++ b/kmod/src/inode.h @@ -0,0 +1,32 @@ +#ifndef _SCOUTFS_INODE_H_ +#define _SCOUTFS_INODE_H_ + +struct scoutfs_inode_info { + u64 ino; + u32 salt; + + struct inode inode; +}; + +static inline struct scoutfs_inode_info *SCOUTFS_I(struct inode *inode) +{ + return container_of(inode, struct scoutfs_inode_info, inode); +} + +static inline u64 scoutfs_ino(struct inode *inode) +{ + return SCOUTFS_I(inode)->ino; +} + +struct inode *scoutfs_alloc_inode(struct super_block *sb); +void scoutfs_destroy_inode(struct inode *inode); + +struct inode *scoutfs_iget(struct super_block *sb, u64 ino); +int scoutfs_inode_update(struct inode *inode); +struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode, dev_t rdev); + +void scoutfs_inode_exit(void); +int scoutfs_inode_init(void); + +#endif diff --git a/kmod/src/item.c b/kmod/src/item.c new file mode 100644 index 00000000..d5c8f204 --- /dev/null +++ b/kmod/src/item.c @@ -0,0 +1,423 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "key.h" +#include "item.h" + +/* + * describe: + * - tracks per-item dirty state for writing + * - decouples vfs cache lifetimes from item lifetimes + * - item-granular cache for things vfs doesn't cache (readdir, xattr) + * + * XXX: + * - warnings for invalid keys/lens + * - memory pressure + */ + +enum { + ITW_NEXT = 1, + ITW_PREV, +}; + +static inline struct scoutfs_item *node_item(struct super_block *sb, + struct rb_root *root, + struct rb_node *node) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + unsigned long off; + + if (root == &sbi->item_root) + off = offsetof(struct scoutfs_item, node); + else + off = offsetof(struct scoutfs_item, dirty_node); + + return (void *)((char *)node - off); +} + +static inline struct rb_node *item_node(struct super_block *sb, + struct rb_root *root, + struct scoutfs_item *item) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + unsigned long off; + + if (root == &sbi->item_root) + off = offsetof(struct scoutfs_item, node); + else + off = offsetof(struct scoutfs_item, dirty_node); + + return (void *)((char *)item + off); +} + +/* + * Insert a new item in the tree. The caller must have done a lookup to + * ensure that the key is not already present. + */ +static void insert_item(struct super_block *sb, struct rb_root *root, + struct scoutfs_item *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_item *item; + int cmp; + + while (*node) { + parent = *node; + item = node_item(sb, root, *node); + + cmp = scoutfs_key_cmp(&ins->key, &item->key); + BUG_ON(cmp == 0); + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(item_node(sb, root, ins), parent, node); + rb_insert_color(item_node(sb, root, ins), root); +} + +enum { + FI_NEXT = 1, + FI_PREV, +}; + +/* + * Walk the tree looking for an item. + * + * If NEXT or PREV are specified then those will be returned + * if the specific item isn't found. + */ +static struct scoutfs_item *find_item(struct super_block *sb, + struct rb_root *root, + struct scoutfs_key *key, int np) +{ + struct rb_node *node = root->rb_node; + struct scoutfs_item *found = NULL; + struct scoutfs_item *item; + int cmp; + + while (node) { + item = node_item(sb, root, node); + + cmp = scoutfs_key_cmp(key, &item->key); + if (cmp < 0) { + if (np == FI_NEXT) + found = item; + node = node->rb_left; + } else if (cmp > 0) { + if (np == FI_PREV) + found = item; + node = node->rb_right; + } else { + found = item; + break; + } + } + + return found; +} + +static struct scoutfs_item *alloc_item(struct scoutfs_key *key, + unsigned int val_len) +{ + struct scoutfs_item *item; + void *val; + + item = kmalloc(sizeof(struct scoutfs_item), GFP_NOFS); + val = kmalloc(val_len, GFP_NOFS); + if (!item || !val) { + kfree(item); + kfree(val); + return ERR_PTR(-ENOMEM); + } + + RB_CLEAR_NODE(&item->node); + RB_CLEAR_NODE(&item->dirty_node); + atomic_set(&item->refcount, 1); + item->key = *key; + item->val_len = val_len; + item->val = val; + + return item; +} + +/* + * Create a new item stored at the given key. Return it with a reference. + * return an ERR_PTR with ENOMEM or EEXIST. + * + * The caller is responsible for initializing the item's value. + */ +struct scoutfs_item *scoutfs_item_create(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_item *existing; + struct scoutfs_item *item; + unsigned long flags; + + item = alloc_item(key, val_len); + if (IS_ERR(item)) + return item; + + spin_lock_irqsave(&sbi->item_lock, flags); + + existing = find_item(sb, &sbi->item_root, key, 0); + if (!existing) { + insert_item(sb, &sbi->item_root, item); + insert_item(sb, &sbi->dirty_item_root, item); + atomic_add(2, &item->refcount); + } + spin_unlock_irqrestore(&sbi->item_lock, flags); + + if (existing) { + scoutfs_item_put(item); + item = ERR_PTR(-EEXIST); + } + + trace_printk("item %p key "CKF" val_len %d\n", item, CKA(key), val_len); + + return item; +} + +/* + * The caller is still responsible for unlocking and putting the item. + * + * We don't try and optimize away the lock for items that are already + * removed from the tree. The caller's locking and item behaviour means + * that racing to remove an item is extremely rare. + * + * XXX for now we're just removing it from the rbtree. We'd need to leave + * behind a deletion record for lsm. + */ +void scoutfs_item_delete(struct super_block *sb, struct scoutfs_item *item) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + unsigned long flags; + + spin_lock_irqsave(&sbi->item_lock, flags); + + if (!RB_EMPTY_NODE(&item->dirty_node)) { + rb_erase(&item->dirty_node, &sbi->dirty_item_root); + RB_CLEAR_NODE(&item->dirty_node); + scoutfs_item_put(item); + } + + if (!RB_EMPTY_NODE(&item->node)) { + rb_erase(&item->node, &sbi->item_root); + RB_CLEAR_NODE(&item->node); + scoutfs_item_put(item); + } + + spin_unlock_irqrestore(&sbi->item_lock, flags); +} + +static struct scoutfs_item *item_lookup(struct super_block *sb, + struct scoutfs_key *key, int np) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_item *item; + unsigned long flags; + + spin_lock_irqsave(&sbi->item_lock, flags); + + item = find_item(sb, &sbi->item_root, key, np); + if (item) + atomic_inc(&item->refcount); + else + item = ERR_PTR(-ENOENT); + + spin_unlock_irqrestore(&sbi->item_lock, flags); + + return item; +} + +struct scoutfs_item *scoutfs_item_lookup(struct super_block *sb, + struct scoutfs_key *key) +{ + return item_lookup(sb, key, 0); +} + +struct scoutfs_item *scoutfs_item_next(struct super_block *sb, + struct scoutfs_key *key) +{ + return item_lookup(sb, key, FI_NEXT); +} + +struct scoutfs_item *scoutfs_item_prev(struct super_block *sb, + struct scoutfs_key *key) +{ + return item_lookup(sb, key, FI_PREV); +} + +/* + * Expand the item's value by inserting bytes at the given offset. The + * new bytes are not initialized. + */ +int scoutfs_item_expand(struct scoutfs_item *item, int off, int bytes) +{ + void *val; + + /* XXX bytes too big */ + if (WARN_ON_ONCE(off < 0 || off > item->val_len)) + return -EINVAL; + + val = kmalloc(item->val_len + bytes, GFP_NOFS); + if (!val) + return -ENOMEM; + + memcpy(val, item->val, off); + memcpy(val + off + bytes, item->val + off, item->val_len - off); + + kfree(item->val); + item->val = val; + item->val_len += bytes; + + return 0; +} + +/* + * Shrink the item's value by remove bytes at the given offset. + */ +int scoutfs_item_shrink(struct scoutfs_item *item, int off, int bytes) +{ + void *val; + + if (WARN_ON_ONCE(off < 0 || off >= item->val_len || + bytes <= 0 || (off + bytes) > item->val_len || + bytes == item->val_len)) + return -EINVAL; + + val = kmalloc(item->val_len - bytes, GFP_NOFS); + if (!val) + return -ENOMEM; + + memcpy(val, item->val, off); + memcpy(val + off, item->val + off + bytes, + item->val_len - (off + bytes)); + + kfree(item->val); + item->val = val; + item->val_len -= bytes; + + return 0; +} + +void scoutfs_item_mark_dirty(struct super_block *sb, struct scoutfs_item *item) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + unsigned long flags; + + spin_lock_irqsave(&sbi->item_lock, flags); + + if (RB_EMPTY_NODE(&item->dirty_node)) { + insert_item(sb, &sbi->dirty_item_root, item); + atomic_inc(&item->refcount); + } + + spin_unlock_irqrestore(&sbi->item_lock, flags); +} + +/* + * Mark all the dirty items clean by emptying the dirty rbtree. The + * caller should be preventing writes from dirtying new items. + * + * We erase leaf nodes with no children to minimize rotation + * overhead during erase. Dirty items must be in the main rbtree if + * they're in the dirty rbtree so the puts here shouldn't free the + * items. + */ +void scoutfs_item_all_clean(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct rb_root *root = &sbi->dirty_item_root; + struct scoutfs_item *item; + struct rb_node *node; + unsigned long flags; + + spin_lock_irqsave(&sbi->item_lock, flags); + + node = sbi->dirty_item_root.rb_node; + while (node) { + if (node->rb_left) + node = node->rb_left; + else if (node->rb_right) + node = node->rb_right; + else { + item = node_item(sb, root, node); + node = rb_parent(node); + + trace_printk("item %p key "CKF"\n", + item, CKA(&item->key)); + rb_erase(&item->dirty_node, root); + RB_CLEAR_NODE(&item->dirty_node); + scoutfs_item_put(item); + } + } + + spin_unlock_irqrestore(&sbi->item_lock, flags); +} + +/* + * If the item is null then the first dirty item is returned. If an + * item is given then the next dirty item is returned. NULL is returned + * if there are no more dirty items. + * + * The caller is given a reference that it has to put. The given item + * will always have its item dropped including if it returns NULL. + */ +struct scoutfs_item *scoutfs_item_next_dirty(struct super_block *sb, + struct scoutfs_item *item) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_item *next_item; + struct rb_node *node; + unsigned long flags; + + spin_lock_irqsave(&sbi->item_lock, flags); + + if (item) + node = rb_next(&item->dirty_node); + else + node = rb_first(&sbi->dirty_item_root); + + if (node) { + next_item = node_item(sb, &sbi->dirty_item_root, node); + atomic_inc(&next_item->refcount); + } else { + next_item = NULL; + } + + spin_unlock_irqrestore(&sbi->item_lock, flags); + + scoutfs_item_put(item); + + return next_item; +} + +void scoutfs_item_put(struct scoutfs_item *item) +{ + if (!IS_ERR_OR_NULL(item) && atomic_dec_and_test(&item->refcount)) { + WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); + WARN_ON_ONCE(!RB_EMPTY_NODE(&item->dirty_node)); + kfree(item); + } +} diff --git a/kmod/src/item.h b/kmod/src/item.h new file mode 100644 index 00000000..27c8fe0d --- /dev/null +++ b/kmod/src/item.h @@ -0,0 +1,37 @@ +#ifndef _SCOUTFS_ITEM_H_ +#define _SCOUTFS_ITEM_H_ + +#include "format.h" + +struct scoutfs_item { + struct rb_node node; + struct rb_node dirty_node; + atomic_t refcount; + + /* the key is constant for the life of the item */ + struct scoutfs_key key; + + /* the value can be changed by expansion or shrinking */ + unsigned int val_len; + void *val; +}; + +struct scoutfs_item *scoutfs_item_create(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len); +struct scoutfs_item *scoutfs_item_lookup(struct super_block *sb, + struct scoutfs_key *key); +struct scoutfs_item *scoutfs_item_next(struct super_block *sb, + struct scoutfs_key *key); +struct scoutfs_item *scoutfs_item_prev(struct super_block *sb, + struct scoutfs_key *key); +int scoutfs_item_expand(struct scoutfs_item *item, int off, int bytes); +int scoutfs_item_shrink(struct scoutfs_item *item, int off, int bytes); +void scoutfs_item_delete(struct super_block *sb, struct scoutfs_item *item); +void scoutfs_item_mark_dirty(struct super_block *sb, struct scoutfs_item *item); +struct scoutfs_item *scoutfs_item_next_dirty(struct super_block *sb, + struct scoutfs_item *item); +void scoutfs_item_all_clean(struct super_block *sb); +void scoutfs_item_put(struct scoutfs_item *item); + +#endif diff --git a/kmod/src/key.h b/kmod/src/key.h new file mode 100644 index 00000000..342a0529 --- /dev/null +++ b/kmod/src/key.h @@ -0,0 +1,43 @@ +#ifndef _SCOUTFS_KEY_H_ +#define _SCOUTFS_KEY_H_ + +#include +#include "format.h" + +#define CKF "%llu.%u.%llu" +#define CKA(key) \ + le64_to_cpu((key)->inode), (key)->type, le64_to_cpu((key)->offset) + +static inline u64 scoutfs_key_inode(struct scoutfs_key *key) +{ + return le64_to_cpu(key->inode); +} + +static inline u64 scoutfs_key_offset(struct scoutfs_key *key) +{ + return le64_to_cpu(key->offset); +} + +static inline int le64_cmp(__le64 a, __le64 b) +{ + return le64_to_cpu(a) < le64_to_cpu(b) ? -1 : + le64_to_cpu(a) > le64_to_cpu(b) ? 1 : 0; +} + +static inline int scoutfs_key_cmp(struct scoutfs_key *a, struct scoutfs_key *b) +{ + return le64_cmp(a->inode, b->inode) ?: + ((short)a->type - (short)b->type) ?: + le64_cmp(a->offset, b->offset); +} + + +static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, + u64 offset) +{ + key->inode = cpu_to_le64(inode); + key->type = type; + key->offset = cpu_to_le64(offset); +} + +#endif diff --git a/kmod/src/lsm.c b/kmod/src/lsm.c new file mode 100644 index 00000000..da1758bf --- /dev/null +++ b/kmod/src/lsm.c @@ -0,0 +1,330 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "dir.h" +#include "inode.h" +#include "key.h" +#include "item.h" +#include "super.h" +#include "lsm.h" + +#define PAGE_CACHE_PAGE_BITS (PAGE_CACHE_SIZE * 8) + +/* XXX garbage hack until we have siphash */ +static u64 bloom_hash(struct scoutfs_key *key, __le64 *hash_key) +{ + __le32 *salts = (void *)hash_key; + + return ((u64)crc32c(le32_to_cpu(salts[0]), key, sizeof(*key)) << 32) | + crc32c(le32_to_cpu(salts[1]), key, sizeof(*key)); +} + +/* + * Set the caller's bloom indices for their item key. + */ +static void get_bloom_indices(struct super_block *sb, + struct scoutfs_key *key, u32 *ind) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + __le64 *hash_key = sbi->bloom_hash_keys; + u64 hash; + int h; + int i; + + for (i = 0; ; ) { + hash = bloom_hash(key, hash_key); + hash_key += 2; + + for (h = 0; h < 64 / SCOUTFS_BLOOM_INDEX_BITS; h++) { + ind[i++] = hash & SCOUTFS_BLOOM_INDEX_MASK; + if (i == SCOUTFS_BLOOM_INDEX_NR) + return; + + hash >>= SCOUTFS_BLOOM_INDEX_BITS; + } + } +} + +struct pages { + /* fixed for the group of pages */ + struct address_space *mapping; + struct page **pages; + pgoff_t pgoff; + + /* number of pages stored in the pages array */ + int nr; + /* byte offset of the free space at end of current page */ + int off; + /* bytes remaining in the ovarall large block */ + int remaining; +}; + +/* + * The caller has our fixed-size bloom filter in the locked pages + * starting at the given byte offset in the first page. Our job is to + * hash the key and set its bits in the bloom filter. + */ +static void set_bloom_bits(struct super_block *sb, struct page **pages, + unsigned int offset, struct scoutfs_key *key) +{ + u32 inds[SCOUTFS_BLOOM_INDEX_NR]; + struct page *page; + int offset_bits = offset * 8; + int full_bit; + int page_bit; + void *addr; + int i; + + get_bloom_indices(sb, key, inds); + + for (i = 0; i < SCOUTFS_BLOOM_INDEX_NR; i++) { + full_bit = offset_bits + inds[i]; + page = pages[full_bit / PAGE_CACHE_PAGE_BITS]; + page_bit = full_bit % PAGE_CACHE_PAGE_BITS; + + addr = kmap_atomic(page); + set_bit_le(page_bit, addr); + kunmap_atomic(addr); + } +} + +/* + * XXX the zeroing here is unreliable. We'll want to zero the bloom but + * not all the pages that are about to be overwritten. Bleh. + * + * Returns the number of bytes copied if there was room. Returns 0 if + * there wasn't. Returns -errno on a hard failure. + */ +static int copy_to_pages(struct pages *pgs, void *ptr, size_t count) +{ + struct page *page; + int ret = count; + void *addr; + int bytes; + + if (count > pgs->remaining) + return 0; + + while (count) { + if (pgs->off == PAGE_CACHE_SIZE) { + page = find_or_create_page(pgs->mapping, + pgs->pgoff + pgs->nr, + GFP_NOFS | __GFP_ZERO); + trace_printk("page %p\n", page); + if (!page) { + ret = -ENOMEM; + break; + } + + pgs->pages[pgs->nr++] = page; + pgs->off = 0; + } else { + page = pgs->pages[pgs->nr - 1]; + } + + bytes = min(PAGE_CACHE_SIZE - pgs->off, count); + + trace_printk("page %p off %d ptr %p count %zu bytes %d remaining %d\n", + page, pgs->off, ptr, count, bytes, pgs->remaining); + + if (ptr) { + addr = kmap_atomic(page); + memcpy(addr + pgs->off, ptr, bytes); + kunmap_atomic(addr); + ptr += bytes; + } + count -= bytes; + pgs->off += bytes; + pgs->remaining -= bytes; + } + + return ret; +} + +static void drop_pages(struct pages *pgs, bool dirty) +{ + struct page *page; + int i; + + if (!pgs->pages) + return; + + for (i = 0; i < pgs->nr; i++) { + page = pgs->pages[i]; + + SetPageUptodate(page); + if (dirty) + set_page_dirty(page); + unlock_page(page); + page_cache_release(page); + } +} + +/* + * Write dirty items from the given item into dirty page cache pages in + * the block device at the given large block number. + * + * All the page cache pages are locked and pinned while they're being + * dirtied. The intent is to have a single large IO leave once they're + * all ready. This is an easy way to do that while maintaining + * consistency with the block device page cache. But it might not work :). + * + * We do one sweep over the items. The item's aren't indexed. We might + * want to change that. + * + * Even though we're doing one sweep over the items we're holding the + * bloom filter and header pinned until the items are done. If we didn't + * mind the risk of the blocks going out of order we wouldn't need the + * allocated array of page pointers. + */ +static struct scoutfs_item *dirty_block_pages(struct super_block *sb, + struct scoutfs_item *item, u64 blkno) +{ + struct scoutfs_item_header ihdr; + struct scoutfs_lsm_block lblk; + struct pages pgs; + void *addr; + int ret; + + /* assuming header starts page, and pgoff shift calculation */ + BUILD_BUG_ON(SCOUTFS_BLOCK_SHIFT < PAGE_CACHE_SHIFT); + + if (WARN_ON_ONCE(!item)) + return item; + + /* XXX not super thrilled with this allocation */ + pgs.pages = kmalloc_array(SCOUTFS_BLOCK_SIZE / PAGE_CACHE_SIZE, + sizeof(struct page *), GFP_NOFS); + if (!pgs.pages) { + ret = -ENOMEM; + goto out; + } + + pgs.mapping = sb->s_bdev->bd_inode->i_mapping; + pgs.pgoff = blkno >> (SCOUTFS_BLOCK_SHIFT - PAGE_CACHE_SHIFT); + pgs.nr = 0; + pgs.off = PAGE_CACHE_SIZE, + pgs.remaining = SCOUTFS_BLOCK_SIZE; + + /* reserve space at the start of the block for header and bloom */ + ret = copy_to_pages(&pgs, NULL, sizeof(lblk)); + if (ret > 0) + ret = copy_to_pages(&pgs, NULL, SCOUTFS_BLOOM_FILTER_BYTES); + if (ret <= 0) + goto out; + + lblk.first = item->key; + lblk.nr_items = 0; + do { + trace_printk("item %p key "CKF"\n", item, CKA(&item->key)); + + ihdr.key = item->key; + ihdr.val_len = cpu_to_le16(item->val_len); + ret = copy_to_pages(&pgs, &ihdr, sizeof(ihdr)); + if (ret > 0) + ret = copy_to_pages(&pgs, item->val, item->val_len); + if (ret <= 0) + goto out; + + lblk.last = item->key; + le32_add_cpu(&lblk.nr_items, 1); + + /* set each item's bloom bits */ + set_bloom_bits(sb, pgs.pages, sizeof(lblk), &item->key); + + item = scoutfs_item_next_dirty(sb, item); + } while (item); + + /* copy the filled in header to the start of the block */ + addr = kmap_atomic(pgs.pages[0]); + memcpy(addr, &lblk, sizeof(lblk)); + kunmap_atomic(addr); + +out: + /* dirty if no error (null ok!), unlock, and release */ + drop_pages(&pgs, !IS_ERR(item)); + kfree(pgs.pages); + if (ret < 0) { + scoutfs_item_put(item); + item = ERR_PTR(ret); + } + return item; +} + +/* + * Sync dirty data by writing all the dirty items into a series of level + * 0 blocks. + * + * This is an initial first pass, the full method will need to: + * - wait for pending writers + * - block future writers + * - update our manifest regardless of server communication + * - communicate blocks and key ranges to server + * - ensure that racing sync/dirty don't livelock + */ +int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_item *item; + u64 blknos[16]; /* XXX */ + u64 blkno; + int ret = 0; + int i; + + item = scoutfs_item_next_dirty(sb, NULL); + if (!item) + return 0; + + for (i = 0; i < ARRAY_SIZE(blknos); i++) { + blkno = atomic64_inc_return(&sbi->next_blkno); + + item = dirty_block_pages(sb, item, blkno); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + goto out; + } + + /* start each block's IO */ + ret = filemap_flush(mapping); + if (ret) + goto out; + + if (!item) + break; + } + /* dirty items should have been limited */ + WARN_ON_ONCE(i >= ARRAY_SIZE(blknos)); + + /* then wait for all block IO to finish */ + if (wait) { + ret = filemap_write_and_wait(mapping); + if (ret) + goto out; + } + + /* mark everything clean */ + scoutfs_item_all_clean(sb); + ret = 0; +out: + trace_printk("ret %d\n", ret); + WARN_ON_ONCE(ret); + return ret; +} diff --git a/kmod/src/lsm.h b/kmod/src/lsm.h new file mode 100644 index 00000000..efed64e9 --- /dev/null +++ b/kmod/src/lsm.h @@ -0,0 +1,6 @@ +#ifndef _SCOUTFS_LSM_H_ +#define _SCOUTFS_LSM_H_ + +int scoutfs_sync_fs(struct super_block *sb, int wait); + +#endif diff --git a/kmod/src/mkfs.c b/kmod/src/mkfs.c new file mode 100644 index 00000000..2a1df169 --- /dev/null +++ b/kmod/src/mkfs.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include "super.h" +#include "item.h" +#include "key.h" +#include "mkfs.h" + +/* + * For now a file system system only exists in the item cache for the + * duration of the mount. This "mkfs" hack creates a root dir inode in + * the item cache on mount so that we can run tests in memory and not + * worry about user space or persistent storage. + */ +int scoutfs_mkfs(struct super_block *sb) +{ + const struct timespec ts = current_kernel_time(); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_inode *cinode; + struct scoutfs_item *item; + struct scoutfs_key key; + int i; + + atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); + atomic64_set(&sbi->next_blkno, 2); + + for (i = 0; i < ARRAY_SIZE(sbi->bloom_hash_keys); i++) { + get_random_bytes(&sbi->bloom_hash_keys[i], + sizeof(sbi->bloom_hash_keys[i])); + } + + scoutfs_set_key(&key, SCOUTFS_ROOT_INO, SCOUTFS_INODE_KEY, 0); + + item = scoutfs_item_create(sb, &key, sizeof(struct scoutfs_inode)); + if (IS_ERR(item)) + return PTR_ERR(item); + + cinode = item->val; + memset(cinode, 0, sizeof(struct scoutfs_inode)); + cinode->nlink = cpu_to_le32(2); + cinode->mode = cpu_to_le32(S_IFDIR | 0755); + cinode->atime.sec = cpu_to_le64(ts.tv_sec); + cinode->atime.nsec = cpu_to_le32(ts.tv_nsec); + cinode->ctime = cinode->atime; + cinode->mtime = cinode->atime; + get_random_bytes(&cinode->salt, sizeof(cinode->salt)); + + scoutfs_item_put(item); + return 0; +} diff --git a/kmod/src/mkfs.h b/kmod/src/mkfs.h new file mode 100644 index 00000000..51679417 --- /dev/null +++ b/kmod/src/mkfs.h @@ -0,0 +1,6 @@ +#ifndef _SCOUTFS_MKFS_H_ +#define _SCOUTFS_MKFS_H_ + +int scoutfs_mkfs(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c new file mode 100644 index 00000000..27336d03 --- /dev/null +++ b/kmod/src/super.c @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "mkfs.h" +#include "inode.h" +#include "dir.h" +#include "lsm.h" + +static const struct super_operations scoutfs_super_ops = { + .alloc_inode = scoutfs_alloc_inode, + .destroy_inode = scoutfs_destroy_inode, + .sync_fs = scoutfs_sync_fs, +}; + +static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct scoutfs_sb_info *sbi; + struct inode *inode; + int ret; + + sb->s_magic = SCOUTFS_SUPER_MAGIC; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_op = &scoutfs_super_ops; + + sbi = kzalloc(sizeof(struct scoutfs_sb_info), GFP_KERNEL); + sb->s_fs_info = sbi; + if (!sbi) + return -ENOMEM; + + spin_lock_init(&sbi->item_lock); + sbi->item_root = RB_ROOT; + sbi->dirty_item_root = RB_ROOT; + + ret = scoutfs_mkfs(sb); + if (ret) + return ret; + + inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; + + return 0; +} + +static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, scoutfs_fill_super); +} + +static void scoutfs_kill_sb(struct super_block *sb) +{ + kill_block_super(sb); + kfree(sb->s_fs_info); +} + +static struct file_system_type scoutfs_fs_type = { + .owner = THIS_MODULE, + .name = "scoutfs", + .mount = scoutfs_mount, + .kill_sb = scoutfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; + +static int __init scoutfs_module_init(void) +{ + return scoutfs_inode_init() ?: + scoutfs_dir_init() ?: + register_filesystem(&scoutfs_fs_type); +} +module_init(scoutfs_module_init) + +static void __exit scoutfs_module_exit(void) +{ + unregister_filesystem(&scoutfs_fs_type); + scoutfs_dir_exit(); + scoutfs_inode_exit(); +} +module_exit(scoutfs_module_exit) + +MODULE_AUTHOR("Zach Brown "); +MODULE_LICENSE("GPL"); diff --git a/kmod/src/super.h b/kmod/src/super.h new file mode 100644 index 00000000..1b6f0be0 --- /dev/null +++ b/kmod/src/super.h @@ -0,0 +1,22 @@ +#ifndef _SCOUTFS_SUPER_H_ +#define _SCOUTFS_SUPER_H_ + +#include + +struct scoutfs_sb_info { + atomic64_t next_ino; + atomic64_t next_blkno; + + __le64 bloom_hash_keys[6]; /* XXX */ + + spinlock_t item_lock; + struct rb_root item_root; + struct rb_root dirty_item_root; +}; + +static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +#endif From eb4694e4013441ac1842713a705437c394e2e21c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Feb 2016 19:28:03 -0800 Subject: [PATCH 002/920] Add simple message printing Add a message printing function whose output includes the device and major:minor and which handles the kernel level string prefix. Signed-off-by: Zach Brown --- kmod/src/msg.c | 20 ++++++++++++++++++++ kmod/src/msg.h | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 kmod/src/msg.c create mode 100644 kmod/src/msg.h diff --git a/kmod/src/msg.c b/kmod/src/msg.c new file mode 100644 index 00000000..e177af07 --- /dev/null +++ b/kmod/src/msg.c @@ -0,0 +1,20 @@ +#include +#include + +void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk("%sscoutfs (%s %u:%u)%s: %pV\n", prefix, + sb->s_id, MAJOR(sb->s_bdev->bd_dev), MINOR(sb->s_bdev->bd_dev), + str, &vaf); + + va_end(args); +} diff --git a/kmod/src/msg.h b/kmod/src/msg.h new file mode 100644 index 00000000..64376f9a --- /dev/null +++ b/kmod/src/msg.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_MSG_H_ +#define _SCOUTFS_MSG_H_ + +void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, + const char *fmt, ...); + +#define scoutfs_err(sb, fmt, args...) \ + scoutfs_msg(sb, KERN_ERR, " error", fmt, ##args) + +#define scoutfs_warn(sb, fmt, args...) \ + scoutfs_msg(sb, KERN_WARNING, " warning", fmt, ##args) + +#define scoutfs_info(sb, fmt, args...) \ + scoutfs_msg(sb, KERN_INFO, "", fmt, ##args) + +#endif From 82ec91d1e022bc286b978aab7e5b8e0b2287ad0a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Feb 2016 19:32:53 -0800 Subject: [PATCH 003/920] Update format to recent utils changes The format was updated while implementing mkfs and print in scoutfs-utils. Bring the kernel code up to speed. For some reason I changed the name of the item length in the item header struct. Who knows. Signed-off-by: Zach Brown --- kmod/src/format.h | 144 ++++++++++++++++++++++++++++++++++++++++------ kmod/src/lsm.c | 2 +- 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 67958f12..bff0b7cb 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -1,11 +1,136 @@ #ifndef _SCOUTFS_FORMAT_H_ #define _SCOUTFS_FORMAT_H_ -#define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ +/* statfs(2) f_type */ +#define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ +/* super block id */ +#define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ +/* + * Some fs structures are stored in smaller fixed size 4k bricks. + */ +#define SCOUTFS_BRICK_SHIFT 12 +#define SCOUTFS_BRICK_SIZE (1 << SCOUTFS_BRICK_SHIFT) + +/* + * A large block size reduces the amount of per-block overhead throughout + * the system: block IO, manifest communications and storage, etc. + */ #define SCOUTFS_BLOCK_SHIFT 22 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) +/* for shifting between brick and block numbers */ +#define SCOUTFS_BLOCK_BRICK (SCOUTFS_BLOCK_SHIFT - SCOUTFS_BRICK_SHIFT) + +/* + * The super bricks leave a bunch of room at the start of the first + * block for platform structures like boot loaders. + */ +#define SCOUTFS_SUPER_BRICK 16 + +/* + * This header is found at the start of every brick and block + * so that we can verify that it's what we were looking for. + */ +struct scoutfs_header { + __le32 crc; + __le64 fsid; + __le64 seq; + __le64 nr; +} __packed; + +#define SCOUTFS_UUID_BYTES 16 + +/* + * The super is stored in a pair of bricks in the first block. + */ +struct scoutfs_super { + struct scoutfs_header hdr; + __le64 id; + __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 total_blocks; + __le64 ring_layout_block; + __le64 ring_layout_seq; + __le64 last_ring_brick; + __le64 last_ring_seq; + __le64 last_block_seq; +} __packed; + +/* + * We should be able to make the offset smaller if neither dirents nor + * data items use the full 64 bits. + */ +struct scoutfs_key { + __le64 inode; + u8 type; + __le64 offset; +} __packed; + +#define SCOUTFS_ROOT_INO 1 + +#define SCOUTFS_INODE_KEY 128 +#define SCOUTFS_DIRENT_KEY 192 + +struct scoutfs_ring_layout { + struct scoutfs_header hdr; + __le32 nr_blocks; + __le64 blocks[0]; +} __packed; + +struct scoutfs_ring_entry { + u8 type; + __le16 len; +} __packed; + +/* + * Ring blocks are 4k blocks stored inside the large ring blocks + * referenced by the ring descriptor block. + * + * The manifest entries describe the position of a given block in the + * manifest. They're keyed by the block number so that we can log + * movement of a block in the manifest with one log entry and we can log + * deletion with just the block number. + */ +struct scoutfs_ring_brick { + struct scoutfs_header hdr; + __le16 nr_entries; +} __packed; + +enum { + SCOUTFS_RING_REMOVE_MANIFEST = 0, + SCOUTFS_RING_ADD_MANIFEST, + SCOUTFS_RING_BITMAP, +}; + +/* + * Manifest entries are logged by their block number. This lets us log + * a change with one entry and a removal with a tiny block number + * without the key. + */ +struct scoutfs_ring_remove_manifest { + __le64 block; +} __packed; + +/* + * Including both keys might make the manifest too large. It might be + * better to only include one key and infer a block's range from the + * neighbour's key. The downside of that is that we assume that there + * isn't unused key space between blocks in a level. We might search + * blocks when we didn't need to. + */ +struct scoutfs_ring_add_manifest { + __le64 block; + __le64 seq; + __u8 level; + struct scoutfs_key first; + struct scoutfs_key last; +} __packed; + +struct scoutfs_ring_bitmap { + __le32 offset; + __le64 bits[2]; +} __packed; + /* * This bloom size is chosen to have a roughly 1% false positive rate * for ~90k items which is roughly the worst case for a block full of @@ -18,20 +143,8 @@ #define SCOUTFS_BLOOM_INDEX_MASK ((1 << SCOUTFS_BLOOM_INDEX_BITS) - 1) #define SCOUTFS_BLOOM_INDEX_NR 7 -/* - * We should be able to make the offset smaller if neither dirents nor - * data items use the full 64 bits. - */ -struct scoutfs_key { - __le64 inode; - u8 type; - __le64 offset; -} __packed; - -#define SCOUTFS_INODE_KEY 128 -#define SCOUTFS_DIRENT_KEY 192 - struct scoutfs_lsm_block { + struct scoutfs_header hdr; struct scoutfs_key first; struct scoutfs_key last; __le32 nr_items; @@ -41,10 +154,9 @@ struct scoutfs_lsm_block { struct scoutfs_item_header { struct scoutfs_key key; - __le16 val_len; + __le16 len; } __packed; - struct scoutfs_timespec { __le64 sec; __le32 nsec; diff --git a/kmod/src/lsm.c b/kmod/src/lsm.c index da1758bf..f9aa3001 100644 --- a/kmod/src/lsm.c +++ b/kmod/src/lsm.c @@ -236,7 +236,7 @@ static struct scoutfs_item *dirty_block_pages(struct super_block *sb, trace_printk("item %p key "CKF"\n", item, CKA(&item->key)); ihdr.key = item->key; - ihdr.val_len = cpu_to_le16(item->val_len); + ihdr.len = cpu_to_le16(item->val_len); ret = copy_to_pages(&pgs, &ihdr, sizeof(ihdr)); if (ret > 0) ret = copy_to_pages(&pgs, item->val, item->val_len); From 3483133cdf25f86f80530b573f0435e09300ed7d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Feb 2016 19:37:48 -0800 Subject: [PATCH 004/920] Read super brick instead of mkfs Now that we have a working userspace mkfs we can read the supers on mount instead of always initializing a new file system. We still don't know how to read items from blocks so mount fails when it can't find the root dir inode. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/mkfs.c | 52 ------------------------------- kmod/src/mkfs.h | 6 ---- kmod/src/super.c | 79 +++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/super.h | 3 ++ 5 files changed, 81 insertions(+), 61 deletions(-) delete mode 100644 kmod/src/mkfs.c delete mode 100644 kmod/src/mkfs.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 239e8aef..d067f9d4 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += dir.o inode.o item.o lsm.o mkfs.o super.o +scoutfs-y += dir.o inode.o item.o lsm.o msg.o super.o diff --git a/kmod/src/mkfs.c b/kmod/src/mkfs.c deleted file mode 100644 index 2a1df169..00000000 --- a/kmod/src/mkfs.c +++ /dev/null @@ -1,52 +0,0 @@ -#include -#include -#include -#include - -#include "super.h" -#include "item.h" -#include "key.h" -#include "mkfs.h" - -/* - * For now a file system system only exists in the item cache for the - * duration of the mount. This "mkfs" hack creates a root dir inode in - * the item cache on mount so that we can run tests in memory and not - * worry about user space or persistent storage. - */ -int scoutfs_mkfs(struct super_block *sb) -{ - const struct timespec ts = current_kernel_time(); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_inode *cinode; - struct scoutfs_item *item; - struct scoutfs_key key; - int i; - - atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); - atomic64_set(&sbi->next_blkno, 2); - - for (i = 0; i < ARRAY_SIZE(sbi->bloom_hash_keys); i++) { - get_random_bytes(&sbi->bloom_hash_keys[i], - sizeof(sbi->bloom_hash_keys[i])); - } - - scoutfs_set_key(&key, SCOUTFS_ROOT_INO, SCOUTFS_INODE_KEY, 0); - - item = scoutfs_item_create(sb, &key, sizeof(struct scoutfs_inode)); - if (IS_ERR(item)) - return PTR_ERR(item); - - cinode = item->val; - memset(cinode, 0, sizeof(struct scoutfs_inode)); - cinode->nlink = cpu_to_le32(2); - cinode->mode = cpu_to_le32(S_IFDIR | 0755); - cinode->atime.sec = cpu_to_le64(ts.tv_sec); - cinode->atime.nsec = cpu_to_le32(ts.tv_nsec); - cinode->ctime = cinode->atime; - cinode->mtime = cinode->atime; - get_random_bytes(&cinode->salt, sizeof(cinode->salt)); - - scoutfs_item_put(item); - return 0; -} diff --git a/kmod/src/mkfs.h b/kmod/src/mkfs.h deleted file mode 100644 index 51679417..00000000 --- a/kmod/src/mkfs.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _SCOUTFS_MKFS_H_ -#define _SCOUTFS_MKFS_H_ - -int scoutfs_mkfs(struct super_block *sb); - -#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 27336d03..e21c3238 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -15,13 +15,16 @@ #include #include #include +#include +#include +#include #include "super.h" #include "format.h" -#include "mkfs.h" #include "inode.h" #include "dir.h" #include "lsm.h" +#include "msg.h" static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, @@ -29,6 +32,73 @@ static const struct super_operations scoutfs_super_ops = { .sync_fs = scoutfs_sync_fs, }; +static int read_supers(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buffer_head *bh = NULL; + struct scoutfs_super *super; + int found = -1; + u32 crc; + int i; + + for (i = 0; i < 2; i++) { + if (bh) + brelse(bh); + bh = sb_bread(sb, SCOUTFS_SUPER_BRICK + i); + if (!bh) { + scoutfs_warn(sb, "couldn't read super brick %u", i); + continue; + } + + super = (void *)bh->b_data; + + if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { + scoutfs_warn(sb, "super brick %u has invalid id %llx", + i, le64_to_cpu(super->id)); + continue; + } + + crc = crc32c(~0, (char *)&super->hdr.crc + sizeof(crc), + SCOUTFS_BRICK_SIZE - sizeof(crc)); + if (crc != le32_to_cpu(super->hdr.crc)) { + scoutfs_warn(sb, "super brick %u has bad crc %x (expected %x)", + i, crc, le32_to_cpu(super->hdr.crc)); + continue; + } + + if (found < 0 || (le64_to_cpu(super->hdr.seq) > + le64_to_cpu(sbi->super.hdr.seq))) { + memcpy(&sbi->super, super, + sizeof(struct scoutfs_super)); + found = i; + } + } + + if (bh) + brelse(bh); + + if (found < 0) { + scoutfs_err(sb, "unable to read valid super brick"); + return -EINVAL; + } + + scoutfs_info(sb, "using super %u with seq %llu", + found, le64_to_cpu(sbi->super.hdr.seq)); + + /* + * XXX These don't exist in the super yet. They should soon. + */ + atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); + atomic64_set(&sbi->next_blkno, 2); + + for (i = 0; i < ARRAY_SIZE(sbi->bloom_hash_keys); i++) { + get_random_bytes(&sbi->bloom_hash_keys[i], + sizeof(sbi->bloom_hash_keys[i])); + } + + return 0; +} + static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; @@ -48,7 +118,12 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->item_root = RB_ROOT; sbi->dirty_item_root = RB_ROOT; - ret = scoutfs_mkfs(sb); + if (!sb_set_blocksize(sb, SCOUTFS_BRICK_SIZE)) { + printk(KERN_ERR "couldn't set blocksize\n"); + return -EINVAL; + } + + ret = read_supers(sb); if (ret) return ret; diff --git a/kmod/src/super.h b/kmod/src/super.h index 1b6f0be0..fd1fe36d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -2,8 +2,11 @@ #define _SCOUTFS_SUPER_H_ #include +#include "format.h" struct scoutfs_sb_info { + struct scoutfs_super super; + atomic64_t next_ino; atomic64_t next_blkno; From 6686ca191afaf3ef0f8573666be2a5daa1b68d3c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Feb 2016 19:33:56 -0800 Subject: [PATCH 005/920] scoutfs: remove the prototype log writing The sync implementation was a quick demonstration of packing items in to large log blocks. We'll be doing things very differently in the actual system. So tear this code out so we can build up more functional structures. It'll still be in revision control so we'll be able to reuse the parts that make sense in the new code. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/lsm.c | 330 ---------------------------------------------- kmod/src/lsm.h | 6 - kmod/src/super.c | 2 - 4 files changed, 1 insertion(+), 339 deletions(-) delete mode 100644 kmod/src/lsm.c delete mode 100644 kmod/src/lsm.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index d067f9d4..14e8d5a4 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += dir.o inode.o item.o lsm.o msg.o super.o +scoutfs-y += dir.o inode.o item.o msg.o super.o diff --git a/kmod/src/lsm.c b/kmod/src/lsm.c deleted file mode 100644 index f9aa3001..00000000 --- a/kmod/src/lsm.c +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include - -#include "format.h" -#include "dir.h" -#include "inode.h" -#include "key.h" -#include "item.h" -#include "super.h" -#include "lsm.h" - -#define PAGE_CACHE_PAGE_BITS (PAGE_CACHE_SIZE * 8) - -/* XXX garbage hack until we have siphash */ -static u64 bloom_hash(struct scoutfs_key *key, __le64 *hash_key) -{ - __le32 *salts = (void *)hash_key; - - return ((u64)crc32c(le32_to_cpu(salts[0]), key, sizeof(*key)) << 32) | - crc32c(le32_to_cpu(salts[1]), key, sizeof(*key)); -} - -/* - * Set the caller's bloom indices for their item key. - */ -static void get_bloom_indices(struct super_block *sb, - struct scoutfs_key *key, u32 *ind) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - __le64 *hash_key = sbi->bloom_hash_keys; - u64 hash; - int h; - int i; - - for (i = 0; ; ) { - hash = bloom_hash(key, hash_key); - hash_key += 2; - - for (h = 0; h < 64 / SCOUTFS_BLOOM_INDEX_BITS; h++) { - ind[i++] = hash & SCOUTFS_BLOOM_INDEX_MASK; - if (i == SCOUTFS_BLOOM_INDEX_NR) - return; - - hash >>= SCOUTFS_BLOOM_INDEX_BITS; - } - } -} - -struct pages { - /* fixed for the group of pages */ - struct address_space *mapping; - struct page **pages; - pgoff_t pgoff; - - /* number of pages stored in the pages array */ - int nr; - /* byte offset of the free space at end of current page */ - int off; - /* bytes remaining in the ovarall large block */ - int remaining; -}; - -/* - * The caller has our fixed-size bloom filter in the locked pages - * starting at the given byte offset in the first page. Our job is to - * hash the key and set its bits in the bloom filter. - */ -static void set_bloom_bits(struct super_block *sb, struct page **pages, - unsigned int offset, struct scoutfs_key *key) -{ - u32 inds[SCOUTFS_BLOOM_INDEX_NR]; - struct page *page; - int offset_bits = offset * 8; - int full_bit; - int page_bit; - void *addr; - int i; - - get_bloom_indices(sb, key, inds); - - for (i = 0; i < SCOUTFS_BLOOM_INDEX_NR; i++) { - full_bit = offset_bits + inds[i]; - page = pages[full_bit / PAGE_CACHE_PAGE_BITS]; - page_bit = full_bit % PAGE_CACHE_PAGE_BITS; - - addr = kmap_atomic(page); - set_bit_le(page_bit, addr); - kunmap_atomic(addr); - } -} - -/* - * XXX the zeroing here is unreliable. We'll want to zero the bloom but - * not all the pages that are about to be overwritten. Bleh. - * - * Returns the number of bytes copied if there was room. Returns 0 if - * there wasn't. Returns -errno on a hard failure. - */ -static int copy_to_pages(struct pages *pgs, void *ptr, size_t count) -{ - struct page *page; - int ret = count; - void *addr; - int bytes; - - if (count > pgs->remaining) - return 0; - - while (count) { - if (pgs->off == PAGE_CACHE_SIZE) { - page = find_or_create_page(pgs->mapping, - pgs->pgoff + pgs->nr, - GFP_NOFS | __GFP_ZERO); - trace_printk("page %p\n", page); - if (!page) { - ret = -ENOMEM; - break; - } - - pgs->pages[pgs->nr++] = page; - pgs->off = 0; - } else { - page = pgs->pages[pgs->nr - 1]; - } - - bytes = min(PAGE_CACHE_SIZE - pgs->off, count); - - trace_printk("page %p off %d ptr %p count %zu bytes %d remaining %d\n", - page, pgs->off, ptr, count, bytes, pgs->remaining); - - if (ptr) { - addr = kmap_atomic(page); - memcpy(addr + pgs->off, ptr, bytes); - kunmap_atomic(addr); - ptr += bytes; - } - count -= bytes; - pgs->off += bytes; - pgs->remaining -= bytes; - } - - return ret; -} - -static void drop_pages(struct pages *pgs, bool dirty) -{ - struct page *page; - int i; - - if (!pgs->pages) - return; - - for (i = 0; i < pgs->nr; i++) { - page = pgs->pages[i]; - - SetPageUptodate(page); - if (dirty) - set_page_dirty(page); - unlock_page(page); - page_cache_release(page); - } -} - -/* - * Write dirty items from the given item into dirty page cache pages in - * the block device at the given large block number. - * - * All the page cache pages are locked and pinned while they're being - * dirtied. The intent is to have a single large IO leave once they're - * all ready. This is an easy way to do that while maintaining - * consistency with the block device page cache. But it might not work :). - * - * We do one sweep over the items. The item's aren't indexed. We might - * want to change that. - * - * Even though we're doing one sweep over the items we're holding the - * bloom filter and header pinned until the items are done. If we didn't - * mind the risk of the blocks going out of order we wouldn't need the - * allocated array of page pointers. - */ -static struct scoutfs_item *dirty_block_pages(struct super_block *sb, - struct scoutfs_item *item, u64 blkno) -{ - struct scoutfs_item_header ihdr; - struct scoutfs_lsm_block lblk; - struct pages pgs; - void *addr; - int ret; - - /* assuming header starts page, and pgoff shift calculation */ - BUILD_BUG_ON(SCOUTFS_BLOCK_SHIFT < PAGE_CACHE_SHIFT); - - if (WARN_ON_ONCE(!item)) - return item; - - /* XXX not super thrilled with this allocation */ - pgs.pages = kmalloc_array(SCOUTFS_BLOCK_SIZE / PAGE_CACHE_SIZE, - sizeof(struct page *), GFP_NOFS); - if (!pgs.pages) { - ret = -ENOMEM; - goto out; - } - - pgs.mapping = sb->s_bdev->bd_inode->i_mapping; - pgs.pgoff = blkno >> (SCOUTFS_BLOCK_SHIFT - PAGE_CACHE_SHIFT); - pgs.nr = 0; - pgs.off = PAGE_CACHE_SIZE, - pgs.remaining = SCOUTFS_BLOCK_SIZE; - - /* reserve space at the start of the block for header and bloom */ - ret = copy_to_pages(&pgs, NULL, sizeof(lblk)); - if (ret > 0) - ret = copy_to_pages(&pgs, NULL, SCOUTFS_BLOOM_FILTER_BYTES); - if (ret <= 0) - goto out; - - lblk.first = item->key; - lblk.nr_items = 0; - do { - trace_printk("item %p key "CKF"\n", item, CKA(&item->key)); - - ihdr.key = item->key; - ihdr.len = cpu_to_le16(item->val_len); - ret = copy_to_pages(&pgs, &ihdr, sizeof(ihdr)); - if (ret > 0) - ret = copy_to_pages(&pgs, item->val, item->val_len); - if (ret <= 0) - goto out; - - lblk.last = item->key; - le32_add_cpu(&lblk.nr_items, 1); - - /* set each item's bloom bits */ - set_bloom_bits(sb, pgs.pages, sizeof(lblk), &item->key); - - item = scoutfs_item_next_dirty(sb, item); - } while (item); - - /* copy the filled in header to the start of the block */ - addr = kmap_atomic(pgs.pages[0]); - memcpy(addr, &lblk, sizeof(lblk)); - kunmap_atomic(addr); - -out: - /* dirty if no error (null ok!), unlock, and release */ - drop_pages(&pgs, !IS_ERR(item)); - kfree(pgs.pages); - if (ret < 0) { - scoutfs_item_put(item); - item = ERR_PTR(ret); - } - return item; -} - -/* - * Sync dirty data by writing all the dirty items into a series of level - * 0 blocks. - * - * This is an initial first pass, the full method will need to: - * - wait for pending writers - * - block future writers - * - update our manifest regardless of server communication - * - communicate blocks and key ranges to server - * - ensure that racing sync/dirty don't livelock - */ -int scoutfs_sync_fs(struct super_block *sb, int wait) -{ - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *item; - u64 blknos[16]; /* XXX */ - u64 blkno; - int ret = 0; - int i; - - item = scoutfs_item_next_dirty(sb, NULL); - if (!item) - return 0; - - for (i = 0; i < ARRAY_SIZE(blknos); i++) { - blkno = atomic64_inc_return(&sbi->next_blkno); - - item = dirty_block_pages(sb, item, blkno); - if (IS_ERR(item)) { - ret = PTR_ERR(item); - goto out; - } - - /* start each block's IO */ - ret = filemap_flush(mapping); - if (ret) - goto out; - - if (!item) - break; - } - /* dirty items should have been limited */ - WARN_ON_ONCE(i >= ARRAY_SIZE(blknos)); - - /* then wait for all block IO to finish */ - if (wait) { - ret = filemap_write_and_wait(mapping); - if (ret) - goto out; - } - - /* mark everything clean */ - scoutfs_item_all_clean(sb); - ret = 0; -out: - trace_printk("ret %d\n", ret); - WARN_ON_ONCE(ret); - return ret; -} diff --git a/kmod/src/lsm.h b/kmod/src/lsm.h deleted file mode 100644 index efed64e9..00000000 --- a/kmod/src/lsm.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _SCOUTFS_LSM_H_ -#define _SCOUTFS_LSM_H_ - -int scoutfs_sync_fs(struct super_block *sb, int wait); - -#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index e21c3238..82d0bd98 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -23,13 +23,11 @@ #include "format.h" #include "inode.h" #include "dir.h" -#include "lsm.h" #include "msg.h" static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, - .sync_fs = scoutfs_sync_fs, }; static int read_supers(struct super_block *sb) From 71df879f0705d25065d87cb69716d583f52cf956 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Feb 2016 19:39:02 -0800 Subject: [PATCH 006/920] scoutfs: update format.h to remove bricks Update to the format.h from the recent -utils changes that moved from the clumsy 'brick' terminology to the more reasonable 'block/chunk/segment' terminology. Signed-off-by: Zach Brown --- kmod/src/format.h | 120 +++++++++++++++++++++++----------------------- kmod/src/super.c | 25 ++++------ kmod/src/super.h | 4 +- 3 files changed, 72 insertions(+), 77 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index bff0b7cb..d27748c0 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -7,53 +7,69 @@ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ /* - * Some fs structures are stored in smaller fixed size 4k bricks. + * Everything is stored in and addressed as 4k fixed size blocks. This + * avoids having to manage contiguous cpu mappings of larger blocks. + * Larger structures are read and written as multiple blocks. */ -#define SCOUTFS_BRICK_SHIFT 12 -#define SCOUTFS_BRICK_SIZE (1 << SCOUTFS_BRICK_SHIFT) - -/* - * A large block size reduces the amount of per-block overhead throughout - * the system: block IO, manifest communications and storage, etc. - */ -#define SCOUTFS_BLOCK_SHIFT 22 +#define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) -/* for shifting between brick and block numbers */ -#define SCOUTFS_BLOCK_BRICK (SCOUTFS_BLOCK_SHIFT - SCOUTFS_BRICK_SHIFT) +/* + * The allocator works on larger chunks. Smaller metadata structures + * like the super blocks and the ring are stored in chunks. + * + * A log segment is a collection of smaller blocks (bloom filter, item blocks) + * stored in a chunk. + */ +#define SCOUTFS_CHUNK_SHIFT 22 +#define SCOUTFS_CHUNK_SIZE (1 << SCOUTFS_CHUNK_SHIFT) +#define SCOUTFS_CHUNK_BLOCK_SHIFT (SCOUTFS_CHUNK_SHIFT - SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_BLOCKS_PER_CHUNK (1 << SCOUTFS_CHUNK_BLOCK_SHIFT) /* - * The super bricks leave a bunch of room at the start of the first - * block for platform structures like boot loaders. + * The super blocks leave some room at the start of the first block for + * platform structures like boot loaders. */ -#define SCOUTFS_SUPER_BRICK 16 +#define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_SUPER_NR 2 /* - * This header is found at the start of every brick and block - * so that we can verify that it's what we were looking for. + * This header is found at the start of every block so that we can + * verify that it's what we were looking for. The crc and padding + * starts the block so that its calculation operations on a nice 64bit + * aligned region. */ -struct scoutfs_header { +struct scoutfs_block_header { __le32 crc; + __le32 _pad; __le64 fsid; __le64 seq; - __le64 nr; + __le64 blkno; } __packed; #define SCOUTFS_UUID_BYTES 16 /* - * The super is stored in a pair of bricks in the first block. + * The super is stored in a pair of blocks in the first chunk on the + * device. + * + * The ring map blocks describe the chunks that make up the ring. + * + * The rest of the ring fields describe the state of the ring blocks + * that are stored in their chunks. The active portion of the ring + * describes the current state of the system and is replayed on mount. */ -struct scoutfs_super { - struct scoutfs_header hdr; +struct scoutfs_super_block { + struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; - __le64 total_blocks; - __le64 ring_layout_block; - __le64 ring_layout_seq; - __le64 last_ring_brick; - __le64 last_ring_seq; - __le64 last_block_seq; + __le64 total_chunks; + __le64 ring_map_blkno; + __le64 ring_map_seq; + __le64 ring_first_block; + __le64 ring_active_blocks; + __le64 ring_total_blocks; + __le64 ring_seq; } __packed; /* @@ -71,10 +87,10 @@ struct scoutfs_key { #define SCOUTFS_INODE_KEY 128 #define SCOUTFS_DIRENT_KEY 192 -struct scoutfs_ring_layout { - struct scoutfs_header hdr; - __le32 nr_blocks; - __le64 blocks[0]; +struct scoutfs_ring_map_block { + struct scoutfs_block_header hdr; + __le32 nr_chunks; + __le64 blknos[0]; } __packed; struct scoutfs_ring_entry { @@ -83,16 +99,15 @@ struct scoutfs_ring_entry { } __packed; /* - * Ring blocks are 4k blocks stored inside the large ring blocks - * referenced by the ring descriptor block. + * Ring blocks are stored in chunks described by the ring map blocks. * - * The manifest entries describe the position of a given block in the - * manifest. They're keyed by the block number so that we can log - * movement of a block in the manifest with one log entry and we can log - * deletion with just the block number. + * The manifest entries describe the position of a given log segment in + * the manifest. They're keyed by the block number so that we can + * record movement of a log segment in the manifest with one ring entry + * and we can record deletion with just the block number. */ -struct scoutfs_ring_brick { - struct scoutfs_header hdr; +struct scoutfs_ring_block { + struct scoutfs_block_header hdr; __le16 nr_entries; } __packed; @@ -102,13 +117,8 @@ enum { SCOUTFS_RING_BITMAP, }; -/* - * Manifest entries are logged by their block number. This lets us log - * a change with one entry and a removal with a tiny block number - * without the key. - */ struct scoutfs_ring_remove_manifest { - __le64 block; + __le64 blkno; } __packed; /* @@ -119,7 +129,7 @@ struct scoutfs_ring_remove_manifest { * blocks when we didn't need to. */ struct scoutfs_ring_add_manifest { - __le64 block; + __le64 blkno; __le64 seq; __u8 level; struct scoutfs_key first; @@ -132,23 +142,15 @@ struct scoutfs_ring_bitmap { } __packed; /* - * This bloom size is chosen to have a roughly 1% false positive rate - * for ~90k items which is roughly the worst case for a block full of - * dirents with reasonably small names. Pathologically smaller items - * could be even more dense. + * To start the log segments are a trivial single item block. We'll + * flesh this out into larger blocks once the rest of the architecture + * is in place. */ -#define SCOUTFS_BLOOM_FILTER_BYTES (128 * 1024) -#define SCOUTFS_BLOOM_FILTER_BITS (SCOUTFS_BLOOM_FILTER_BYTES * 8) -#define SCOUTFS_BLOOM_INDEX_BITS (ilog2(SCOUTFS_BLOOM_FILTER_BITS)) -#define SCOUTFS_BLOOM_INDEX_MASK ((1 << SCOUTFS_BLOOM_INDEX_BITS) - 1) -#define SCOUTFS_BLOOM_INDEX_NR 7 - -struct scoutfs_lsm_block { - struct scoutfs_header hdr; +struct scoutfs_item_block { + struct scoutfs_block_header hdr; struct scoutfs_key first; struct scoutfs_key last; __le32 nr_items; - /* u8 bloom[SCOUTFS_BLOOM_BYTES]; */ /* struct scoutfs_item_header items[0] .. */ } __packed; diff --git a/kmod/src/super.c b/kmod/src/super.c index 82d0bd98..b14f495d 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -33,33 +33,33 @@ static const struct super_operations scoutfs_super_ops = { static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super; struct buffer_head *bh = NULL; - struct scoutfs_super *super; int found = -1; u32 crc; int i; - for (i = 0; i < 2; i++) { + for (i = 0; i < SCOUTFS_SUPER_NR; i++) { if (bh) brelse(bh); - bh = sb_bread(sb, SCOUTFS_SUPER_BRICK + i); + bh = sb_bread(sb, SCOUTFS_SUPER_BLKNO + i); if (!bh) { - scoutfs_warn(sb, "couldn't read super brick %u", i); + scoutfs_warn(sb, "couldn't read super block %u", i); continue; } super = (void *)bh->b_data; if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { - scoutfs_warn(sb, "super brick %u has invalid id %llx", + scoutfs_warn(sb, "super block %u has invalid id %llx", i, le64_to_cpu(super->id)); continue; } crc = crc32c(~0, (char *)&super->hdr.crc + sizeof(crc), - SCOUTFS_BRICK_SIZE - sizeof(crc)); + SCOUTFS_BLOCK_SIZE - sizeof(crc)); if (crc != le32_to_cpu(super->hdr.crc)) { - scoutfs_warn(sb, "super brick %u has bad crc %x (expected %x)", + scoutfs_warn(sb, "super block %u has bad crc %x (expected %x)", i, crc, le32_to_cpu(super->hdr.crc)); continue; } @@ -67,7 +67,7 @@ static int read_supers(struct super_block *sb) if (found < 0 || (le64_to_cpu(super->hdr.seq) > le64_to_cpu(sbi->super.hdr.seq))) { memcpy(&sbi->super, super, - sizeof(struct scoutfs_super)); + sizeof(struct scoutfs_super_block)); found = i; } } @@ -76,7 +76,7 @@ static int read_supers(struct super_block *sb) brelse(bh); if (found < 0) { - scoutfs_err(sb, "unable to read valid super brick"); + scoutfs_err(sb, "unable to read valid super block"); return -EINVAL; } @@ -89,11 +89,6 @@ static int read_supers(struct super_block *sb) atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); atomic64_set(&sbi->next_blkno, 2); - for (i = 0; i < ARRAY_SIZE(sbi->bloom_hash_keys); i++) { - get_random_bytes(&sbi->bloom_hash_keys[i], - sizeof(sbi->bloom_hash_keys[i])); - } - return 0; } @@ -116,7 +111,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->item_root = RB_ROOT; sbi->dirty_item_root = RB_ROOT; - if (!sb_set_blocksize(sb, SCOUTFS_BRICK_SIZE)) { + if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { printk(KERN_ERR "couldn't set blocksize\n"); return -EINVAL; } diff --git a/kmod/src/super.h b/kmod/src/super.h index fd1fe36d..538dd773 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,13 +5,11 @@ #include "format.h" struct scoutfs_sb_info { - struct scoutfs_super super; + struct scoutfs_super_block super; atomic64_t next_ino; atomic64_t next_blkno; - __le64 bloom_hash_keys[6]; /* XXX */ - spinlock_t item_lock; struct rb_root item_root; struct rb_root dirty_item_root; From 28521e8c45b858117eebee265e7932741e2932af Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Feb 2016 21:13:56 -0800 Subject: [PATCH 007/920] scoutfs: add block read helper Add a trivial helper function which verifies the block header in metadata blocks. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/block.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/block.h | 6 +++++ kmod/src/crc.c | 22 +++++++++++++++++ kmod/src/crc.h | 6 +++++ kmod/src/super.c | 13 ++-------- 6 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 kmod/src/block.c create mode 100644 kmod/src/block.h create mode 100644 kmod/src/crc.c create mode 100644 kmod/src/crc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 14e8d5a4..5f23e87d 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += dir.o inode.o item.o msg.o super.o +scoutfs-y += block.o dir.o inode.o item.o msg.o super.o diff --git a/kmod/src/block.c b/kmod/src/block.c new file mode 100644 index 00000000..cb403465 --- /dev/null +++ b/kmod/src/block.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include + +#include "super.h" +#include "format.h" +#include "block.h" +#include "crc.h" + +#define BH_Private_Verified BH_PrivateStart + +BUFFER_FNS(Private_Verified, private_verified) + + +/* + * A quick metadata read wrapper which knows how to validate the + * block header. + */ +struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_block_header *hdr; + struct buffer_head *bh; + u32 crc; + + bh = sb_bread(sb, blkno); + if (!bh || buffer_private_verified(bh)) + return bh; + + hdr = (void *)bh->b_data; + crc = scoutfs_crc_block(hdr); + + if (le32_to_cpu(hdr->crc) != crc) { + printk("blkno %llu hdr crc %x != calculated %x\n", blkno, + le32_to_cpu(hdr->crc), crc); + } else if (super->hdr.fsid && hdr->fsid != super->hdr.fsid) { + printk("blkno %llu fsid %llx != super fsid %llx\n", blkno, + le64_to_cpu(hdr->fsid), le64_to_cpu(super->hdr.fsid)); + } else if (le64_to_cpu(hdr->blkno) != blkno) { + printk("blkno %llu invalid hdr blkno %llx\n", blkno, + le64_to_cpu(hdr->blkno)); + } else { + set_buffer_private_verified(bh); + return bh; + } + + brelse(bh); + return NULL; +} diff --git a/kmod/src/block.h b/kmod/src/block.h new file mode 100644 index 00000000..c87fb6b8 --- /dev/null +++ b/kmod/src/block.h @@ -0,0 +1,6 @@ +#ifndef _SCOUTFS_BLOCK_H_ +#define _SCOUTFS_BLOCK_H_ + +struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno); + +#endif diff --git a/kmod/src/crc.c b/kmod/src/crc.c new file mode 100644 index 00000000..9869cbd1 --- /dev/null +++ b/kmod/src/crc.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include + +#include "format.h" +#include "crc.h" + +u32 scoutfs_crc_block(struct scoutfs_block_header *hdr) +{ + return crc32c(~0, (char *)hdr + sizeof(hdr->crc), + SCOUTFS_BLOCK_SIZE - sizeof(hdr->crc)); +} diff --git a/kmod/src/crc.h b/kmod/src/crc.h new file mode 100644 index 00000000..7f1fbf56 --- /dev/null +++ b/kmod/src/crc.h @@ -0,0 +1,6 @@ +#ifndef _SCOUTFS_CRC_H_ +#define _SCOUTFS_CRC_H_ + +u32 scoutfs_crc_block(struct scoutfs_block_header *hdr); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index b14f495d..27e12c52 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "super.h" @@ -24,6 +23,7 @@ #include "inode.h" #include "dir.h" #include "msg.h" +#include "block.h" static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, @@ -36,13 +36,12 @@ static int read_supers(struct super_block *sb) struct scoutfs_super_block *super; struct buffer_head *bh = NULL; int found = -1; - u32 crc; int i; for (i = 0; i < SCOUTFS_SUPER_NR; i++) { if (bh) brelse(bh); - bh = sb_bread(sb, SCOUTFS_SUPER_BLKNO + i); + bh = scoutfs_read_block(sb, SCOUTFS_SUPER_BLKNO + i); if (!bh) { scoutfs_warn(sb, "couldn't read super block %u", i); continue; @@ -56,14 +55,6 @@ static int read_supers(struct super_block *sb) continue; } - crc = crc32c(~0, (char *)&super->hdr.crc + sizeof(crc), - SCOUTFS_BLOCK_SIZE - sizeof(crc)); - if (crc != le32_to_cpu(super->hdr.crc)) { - scoutfs_warn(sb, "super block %u has bad crc %x (expected %x)", - i, crc, le32_to_cpu(super->hdr.crc)); - continue; - } - if (found < 0 || (le64_to_cpu(super->hdr.seq) > le64_to_cpu(sbi->super.hdr.seq))) { memcpy(&sbi->super, super, From 8604c854863df56ff314394be0ad0aa3c0238bd2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Feb 2016 17:00:19 -0800 Subject: [PATCH 008/920] scoutfs: add basic reing replay on mount Read the ring described by the super block and replay its entries to rebuild the in-memory state of the chunk allocator and log segment manifest. We add just enough of the chunk allocator to set the free bits to the contents of the ring bitmap entries. We start to build out the basic manifest data structure. It'll certainly evolve when we later add code to actually query it. Signed-off-by: Zach Brown --- kmod/src/Makefile | 3 +- kmod/src/chunk.c | 39 +++++++++ kmod/src/chunk.h | 7 ++ kmod/src/format.h | 22 +++-- kmod/src/manifest.c | 207 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/manifest.h | 11 +++ kmod/src/ring.c | 128 +++++++++++++++++++++++++++ kmod/src/ring.h | 6 ++ kmod/src/super.c | 22 +++++ kmod/src/super.h | 6 ++ 10 files changed, 443 insertions(+), 8 deletions(-) create mode 100644 kmod/src/chunk.c create mode 100644 kmod/src/chunk.h create mode 100644 kmod/src/manifest.c create mode 100644 kmod/src/manifest.h create mode 100644 kmod/src/ring.c create mode 100644 kmod/src/ring.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 5f23e87d..f4b293ed 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += block.o dir.o inode.o item.o msg.o super.o +scoutfs-y += block.o chunk.o crc.o dir.o inode.o item.o manifest.o msg.o \ + ring.o super.o diff --git a/kmod/src/chunk.c b/kmod/src/chunk.c new file mode 100644 index 00000000..6b5758af --- /dev/null +++ b/kmod/src/chunk.c @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "inode.h" +#include "dir.h" +#include "msg.h" +#include "block.h" + +void scoutfs_set_chunk_alloc_bits(struct super_block *sb, + struct scoutfs_ring_bitmap *bm) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 off = le64_to_cpu(bm->offset); + + /* XXX check for corruption */ + + sbi->chunk_alloc_bits[off] = bm->bits[0]; + sbi->chunk_alloc_bits[off + 1] = bm->bits[1]; + +} diff --git a/kmod/src/chunk.h b/kmod/src/chunk.h new file mode 100644 index 00000000..b2cb6ff7 --- /dev/null +++ b/kmod/src/chunk.h @@ -0,0 +1,7 @@ +#ifndef _SCOUTFS_CHUNK_H_ +#define _SCOUTFS_CHUNK_H_ + +void scoutfs_set_chunk_alloc_bits(struct super_block *sb, + struct scoutfs_ring_bitmap *bm); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index d27748c0..bafaef80 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -24,6 +24,7 @@ #define SCOUTFS_CHUNK_SHIFT 22 #define SCOUTFS_CHUNK_SIZE (1 << SCOUTFS_CHUNK_SHIFT) #define SCOUTFS_CHUNK_BLOCK_SHIFT (SCOUTFS_CHUNK_SHIFT - SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_CHUNK_BLOCK_MASK ((1 << SCOUTFS_CHUNK_BLOCK_SHIFT) - 1) #define SCOUTFS_BLOCKS_PER_CHUNK (1 << SCOUTFS_CHUNK_BLOCK_SHIFT) /* @@ -93,6 +94,10 @@ struct scoutfs_ring_map_block { __le64 blknos[0]; } __packed; +#define SCOUTFS_RING_MAP_BLOCKS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_ring_map_block)) / \ + sizeof(__le64)) + struct scoutfs_ring_entry { u8 type; __le16 len; @@ -112,15 +117,11 @@ struct scoutfs_ring_block { } __packed; enum { - SCOUTFS_RING_REMOVE_MANIFEST = 0, - SCOUTFS_RING_ADD_MANIFEST, + SCOUTFS_RING_ADD_MANIFEST = 0, + SCOUTFS_RING_DEL_MANIFEST, SCOUTFS_RING_BITMAP, }; -struct scoutfs_ring_remove_manifest { - __le64 blkno; -} __packed; - /* * Including both keys might make the manifest too large. It might be * better to only include one key and infer a block's range from the @@ -128,7 +129,7 @@ struct scoutfs_ring_remove_manifest { * isn't unused key space between blocks in a level. We might search * blocks when we didn't need to. */ -struct scoutfs_ring_add_manifest { +struct scoutfs_ring_manifest_entry { __le64 blkno; __le64 seq; __u8 level; @@ -136,6 +137,13 @@ struct scoutfs_ring_add_manifest { struct scoutfs_key last; } __packed; +struct scoutfs_ring_del_manifest { + __le64 blkno; +} __packed; + +/* 2^22 * 10^13 > 2^64 */ +#define SCOUTFS_MAX_LEVEL 13 + struct scoutfs_ring_bitmap { __le32 offset; __le64 bits[2]; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c new file mode 100644 index 00000000..48fc7999 --- /dev/null +++ b/kmod/src/manifest.c @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "manifest.h" +#include "key.h" + +/* + * The manifest organizes log segment blocks into a tree structure. + * + * Each level of the tree contains an ordered list of log segments whose + * item keys don't overlap. The first level (level 0) of the tree is + * the exception whose segments can have key ranges that overlap. + * + * We also store pointers to the manifest entries in a radix tree + * indexed by their block number so that we can easily find existing + * entries for deletion. + * + * Level 0 segments are stored in the list with the most recent at the + * head of the list. Level 0's rb tree will always be empty. + */ +struct scoutfs_manifest { + spinlock_t lock; + + struct radix_tree_root blkno_radix; + struct list_head level_zero; + + struct scoutfs_level { + struct rb_root root; + } levels[SCOUTFS_MAX_LEVEL + 1]; +}; + +struct scoutfs_manifest_node { + struct rb_node node; + struct list_head head; + + struct scoutfs_ring_manifest_entry ment; +}; + +static void insert_mnode(struct rb_root *root, + struct scoutfs_manifest_node *ins) +{ + struct rb_node **node = &root->rb_node; + struct scoutfs_manifest_node *mnode; + struct rb_node *parent = NULL; + int cmp; + + while (*node) { + parent = *node; + mnode = rb_entry(*node, struct scoutfs_manifest_node, node); + + cmp = scoutfs_key_cmp(&ins->ment.first, &mnode->ment.first); + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); +} + +static struct scoutfs_manifest_node *delete_mnode(struct scoutfs_manifest *mani, + u64 blkno) + +{ + struct scoutfs_manifest_node *mnode; + + mnode = radix_tree_lookup(&mani->blkno_radix, blkno); + if (mnode) { + if (!list_empty(&mnode->head)) + list_del_init(&mnode->head); + if (!RB_EMPTY_NODE(&mnode->node)) { + rb_erase(&mnode->node, + &mani->levels[mnode->ment.level].root); + RB_CLEAR_NODE(&mnode->node); + } + } + + return mnode; +} + +/* + * This is called during ring replay. Because of the way the ring works + * we can get deletion entries for segments that we don't yet have + * in the replayed ring state. + */ +void scoutfs_delete_manifest(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnode; + + spin_lock(&mani->lock); + mnode = delete_mnode(mani, blkno); + spin_unlock(&mani->lock); + if (mnode) + kfree(mnode); +} + +/* + * This is called during ring replay to reconstruct the manifest state + * from the ring entries. Moving segments between levels is recorded + * with a single ring entry so we always try to look up the segment in + * the manifest before we add it to the manifest. + */ +int scoutfs_add_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnode; + + spin_lock(&mani->lock); + + mnode = delete_mnode(mani, le64_to_cpu(ment->blkno)); + if (!mnode) { + spin_unlock(&mani->lock); + mnode = kmalloc(sizeof(struct scoutfs_manifest_node), + GFP_NOFS); + if (!mnode) + return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ + + INIT_LIST_HEAD(&mnode->head); + RB_CLEAR_NODE(&mnode->node); + spin_lock(&mani->lock); + } + + mnode->ment = *ment; + if (ment->level) + insert_mnode(&mani->levels[ment->level].root, mnode); + else + list_add(&mnode->head, &mani->level_zero); + + spin_unlock(&mani->lock); + + return 0; +} + +int scoutfs_setup_manifest(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani; + int i; + + mani = kmalloc(sizeof(struct scoutfs_manifest), GFP_KERNEL); + if (!mani) + return -ENOMEM; + + spin_lock_init(&mani->lock); + INIT_RADIX_TREE(&mani->blkno_radix, GFP_NOFS); + INIT_LIST_HEAD(&mani->level_zero); + + for (i = 0; i < ARRAY_SIZE(mani->levels); i++) + mani->levels[i].root = RB_ROOT; + + sbi->mani = mani; + + return 0; +} + +/* + * This is called once the manifest will no longer be used. We iterate + * over the blkno radix deleting radix entries and freeing manifest + * nodes. + */ +void scoutfs_destroy_manifest(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnodes[16]; + unsigned long first_index = 0; + int ret; + int i; + + for (;;) { + ret = radix_tree_gang_lookup(&mani->blkno_radix, + (void **)mnodes, first_index, + ARRAY_SIZE(mnodes)); + if (!ret) + break; + + for (i = 0; i < ret; i++) { + first_index = le64_to_cpu(mnodes[i]->ment.blkno); + radix_tree_delete(&mani->blkno_radix, first_index); + kfree(mnodes[i]); + } + first_index++; + } + + kfree(sbi->mani); + sbi->mani = NULL; +} diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h new file mode 100644 index 00000000..a7685c5d --- /dev/null +++ b/kmod/src/manifest.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_MANIFEST_H_ +#define _SCOUTFS_MANIFEST_H_ + +int scoutfs_setup_manifest(struct super_block *sb); +void scoutfs_destroy_manifest(struct super_block *sb); + +int scoutfs_add_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment); +void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); + +#endif diff --git a/kmod/src/ring.c b/kmod/src/ring.c new file mode 100644 index 00000000..aeee472b --- /dev/null +++ b/kmod/src/ring.c @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "format.h" +#include "dir.h" +#include "inode.h" +#include "key.h" +#include "item.h" +#include "super.h" +#include "manifest.h" +#include "chunk.h" +#include "block.h" + +static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) +{ + struct scoutfs_ring_block *ring = (void *)bh->b_data; + struct scoutfs_ring_entry *ent = (void *)(ring + 1); + struct scoutfs_ring_manifest_entry *ment; + struct scoutfs_ring_del_manifest *del; + struct scoutfs_ring_bitmap *bm; + int ret = 0; + int i; + + /* XXX verify */ + + for (i = 0; i < le16_to_cpu(ring->nr_entries); i++) { + switch(ent->type) { + case SCOUTFS_RING_ADD_MANIFEST: + ment = (void *)(ent + 1); + ret = scoutfs_add_manifest(sb, ment); + break; + case SCOUTFS_RING_DEL_MANIFEST: + del = (void *)(ent + 1); + scoutfs_delete_manifest(sb, le64_to_cpu(del->blkno)); + break; + case SCOUTFS_RING_BITMAP: + bm = (void *)(ent + 1); + scoutfs_set_chunk_alloc_bits(sb, bm); + break; + default: + /* XXX */ + break; + } + + ent = (void *)(ent + 1) + le16_to_cpu(ent->len); + } + + return ret; +} + +/* + * Read a given logical ring block. + * + * Each ring map block entry maps a chunk's worth of ring blocks. + */ +static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ring_map_block *map; + struct buffer_head *bh; + u64 ring_chunk; + u32 ring_block; + u64 blkno; + u64 div; + u32 rem; + + ring_block = block & SCOUTFS_CHUNK_BLOCK_MASK; + ring_chunk = block >> SCOUTFS_CHUNK_BLOCK_SHIFT; + + div = div_u64_rem(ring_chunk, SCOUTFS_RING_MAP_BLOCKS, &rem); + + bh = scoutfs_read_block(sb, le64_to_cpu(super->ring_map_blkno) + div); + if (!bh) + return NULL; + + /* XXX verify map block */ + + map = (void *)bh->b_data; + blkno = le64_to_cpu(map->blknos[rem]) + ring_block; + brelse(bh); + + return scoutfs_read_block(sb, blkno); +} + +int scoutfs_replay_ring(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buffer_head *bh; + u64 block; + int ret; + int i; + + /* XXX read-ahead map blocks and each set of ring blocks */ + + block = le64_to_cpu(super->ring_first_block); + for (i = 0; i < le64_to_cpu(super->ring_active_blocks); i++) { + bh = read_ring_block(sb, block); + if (!bh) { + ret = -EIO; + break; + } + + ret = replay_ring_block(sb, bh); + brelse(bh); + if (ret) + break; + + if (++block == le64_to_cpu(super->ring_total_blocks)) + block = 0; + } + + return ret; +} diff --git a/kmod/src/ring.h b/kmod/src/ring.h new file mode 100644 index 00000000..b50b67e3 --- /dev/null +++ b/kmod/src/ring.h @@ -0,0 +1,6 @@ +#ifndef _SCOUTFS_RING_H_ +#define _SCOUTFS_RING_H_ + +int scoutfs_replay_ring(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 27e12c52..3f9d001f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -24,6 +24,8 @@ #include "dir.h" #include "msg.h" #include "block.h" +#include "manifest.h" +#include "ring.h" static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, @@ -35,6 +37,7 @@ static int read_supers(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; struct buffer_head *bh = NULL; + unsigned long bytes; int found = -1; int i; @@ -80,6 +83,16 @@ static int read_supers(struct super_block *sb) atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); atomic64_set(&sbi->next_blkno, 2); + /* Initialize all the sb info fields which depends on the supers. */ + + bytes = DIV_ROUND_UP(sbi->super.total_chunks, 64) * sizeof(u64); + sbi->chunk_alloc_bits = vmalloc(bytes); + if (!sbi->chunk_alloc_bits) + return -ENOMEM; + + /* the alloc bits default to all free then ring entries update them */ + memset(sbi->chunk_alloc_bits, 0xff, bytes); + return 0; } @@ -111,6 +124,14 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (ret) return ret; + ret = scoutfs_setup_manifest(sb); + if (ret) + return ret; + + ret = scoutfs_replay_ring(sb); + if (ret) + return ret; + inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -130,6 +151,7 @@ static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, static void scoutfs_kill_sb(struct super_block *sb) { + scoutfs_destroy_manifest(sb); kill_block_super(sb); kfree(sb->s_fs_info); } diff --git a/kmod/src/super.h b/kmod/src/super.h index 538dd773..d7229877 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -4,6 +4,8 @@ #include #include "format.h" +struct scoutfs_manifest; + struct scoutfs_sb_info { struct scoutfs_super_block super; @@ -13,6 +15,10 @@ struct scoutfs_sb_info { spinlock_t item_lock; struct rb_root item_root; struct rb_root dirty_item_root; + + struct scoutfs_manifest *mani; + + __le64 *chunk_alloc_bits; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) From 16abddb46a052ff5095261633143897ff02c6a70 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 28 Feb 2016 17:45:44 -0800 Subject: [PATCH 009/920] scoutfs: add basic segment reading Add the most basic ability to read items from log segment blocks. If an item isn't in the cache then we walk segments in the manifest and check for the item in each one. This is just the core fundamental code. There's still a lot to do: basic corruption validation, multi-block segments, bloom filters and arrays to optimize segment misses, and some day the ability to read file data items directly into page cache pages. The manifest locking is also super broken. But this is enough to let us mount and stat the root inode! Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/item.c | 65 ++++++++++++++++++++-------- kmod/src/item.h | 3 ++ kmod/src/key.h | 21 +++++++++ kmod/src/manifest.c | 86 ++++++++++++++++++++++++++++++++++++ kmod/src/manifest.h | 4 ++ kmod/src/segment.c | 103 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/segment.h | 7 +++ 8 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 kmod/src/segment.c create mode 100644 kmod/src/segment.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f4b293ed..e5712be8 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,4 +1,4 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o scoutfs-y += block.o chunk.o crc.o dir.o inode.o item.o manifest.o msg.o \ - ring.o super.o + ring.o segment.o super.o diff --git a/kmod/src/item.c b/kmod/src/item.c index d5c8f204..c86583ed 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -19,6 +19,7 @@ #include "super.h" #include "key.h" #include "item.h" +#include "segment.h" /* * describe: @@ -159,15 +160,9 @@ static struct scoutfs_item *alloc_item(struct scoutfs_key *key, return item; } -/* - * Create a new item stored at the given key. Return it with a reference. - * return an ERR_PTR with ENOMEM or EEXIST. - * - * The caller is responsible for initializing the item's value. - */ -struct scoutfs_item *scoutfs_item_create(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len) +static struct scoutfs_item *create_item(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len, bool dirty) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_item *existing; @@ -183,8 +178,12 @@ struct scoutfs_item *scoutfs_item_create(struct super_block *sb, existing = find_item(sb, &sbi->item_root, key, 0); if (!existing) { insert_item(sb, &sbi->item_root, item); - insert_item(sb, &sbi->dirty_item_root, item); - atomic_add(2, &item->refcount); + atomic_inc(&item->refcount); + if (dirty) { + insert_item(sb, &sbi->dirty_item_root, item); + atomic_inc(&item->refcount); + } + } spin_unlock_irqrestore(&sbi->item_lock, flags); @@ -198,6 +197,30 @@ struct scoutfs_item *scoutfs_item_create(struct super_block *sb, return item; } +/* + * Create a new item stored at the given key. Return it with a reference. + * return an ERR_PTR with ENOMEM or EEXIST. + * + * The caller is responsible for initializing the item's value. + */ +struct scoutfs_item *scoutfs_item_create(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len) +{ + return create_item(sb, key, val_len, true); +} + +/* + * Allocate a new clean item in the cache for the caller to fill. If the + * item already exists then -EEXIST is returned. + */ +struct scoutfs_item *scoutfs_clean_item(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len) +{ + return create_item(sb, key, val_len, false); +} + /* * The caller is still responsible for unlocking and putting the item. * @@ -230,6 +253,10 @@ void scoutfs_item_delete(struct super_block *sb, struct scoutfs_item *item) spin_unlock_irqrestore(&sbi->item_lock, flags); } +/* + * Find an item in the cache. If it isn't present then we try to read + * it from log segements. + */ static struct scoutfs_item *item_lookup(struct super_block *sb, struct scoutfs_key *key, int np) { @@ -237,15 +264,17 @@ static struct scoutfs_item *item_lookup(struct super_block *sb, struct scoutfs_item *item; unsigned long flags; - spin_lock_irqsave(&sbi->item_lock, flags); + do { + spin_lock_irqsave(&sbi->item_lock, flags); - item = find_item(sb, &sbi->item_root, key, np); - if (item) - atomic_inc(&item->refcount); - else - item = ERR_PTR(-ENOENT); + item = find_item(sb, &sbi->item_root, key, np); + if (item) + atomic_inc(&item->refcount); - spin_unlock_irqrestore(&sbi->item_lock, flags); + spin_unlock_irqrestore(&sbi->item_lock, flags); + if (!item) + item = scoutfs_read_segment_item(sb, key); + } while (item == ERR_PTR(-EEXIST)); return item; } diff --git a/kmod/src/item.h b/kmod/src/item.h index 27c8fe0d..b8225a20 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -19,6 +19,9 @@ struct scoutfs_item { struct scoutfs_item *scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, unsigned int val_len); +struct scoutfs_item *scoutfs_clean_item(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len); struct scoutfs_item *scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key); struct scoutfs_item *scoutfs_item_next(struct super_block *sb, diff --git a/kmod/src/key.h b/kmod/src/key.h index 342a0529..27f7ffc3 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -31,6 +31,27 @@ static inline int scoutfs_key_cmp(struct scoutfs_key *a, struct scoutfs_key *b) le64_cmp(a->offset, b->offset); } +/* + * return -ve, 0, +ve if the key is less than, contained within, or greater + * than the given range of keys. + */ +static inline int scoutfs_key_cmp_range(struct scoutfs_key *key, + struct scoutfs_key *first, + struct scoutfs_key *last) +{ + int cmp; + + WARN_ON_ONCE(scoutfs_key_cmp(first, last) > 0); + + cmp = scoutfs_key_cmp(key, first); + if (cmp > 0) { + cmp = scoutfs_key_cmp(key, last); + if (cmp < 0) + cmp = 0; + } + return cmp; +} + static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, u64 offset) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 48fc7999..dd4e65fe 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -74,6 +74,29 @@ static void insert_mnode(struct rb_root *root, rb_insert_color(&ins->node, root); } +static struct scoutfs_manifest_node *find_mnode(struct rb_root *root, + struct scoutfs_key *key) +{ + struct rb_node *node = root->rb_node; + struct scoutfs_manifest_node *mnode; + int cmp; + + while (node) { + mnode = rb_entry(node, struct scoutfs_manifest_node, node); + + cmp = scoutfs_key_cmp_range(key, &mnode->ment.first, + &mnode->ment.last); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return mnode; + } + + return NULL; +} + static struct scoutfs_manifest_node *delete_mnode(struct scoutfs_manifest *mani, u64 blkno) @@ -151,6 +174,69 @@ int scoutfs_add_manifest(struct super_block *sb, return 0; } +/* + * Fill the caller's ment with the next log segment in the manifest that + * might contain the given key. The ment is initialized to 0 to return + * the first entry. + * + * This can return multiple log segments from level 0 in decreasing age. + * Then it can return at most one log segment in each level that + * intersects with the given key. + */ +bool scoutfs_next_manifest_segment(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_ring_manifest_entry *ment) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnode; + bool found = false; + int i; + + if (ment->level >= SCOUTFS_MAX_LEVEL) + return false; + + spin_lock(&mani->lock); + + if (ment->level == 0) { + if (ment->blkno) { + mnode = radix_tree_lookup(&mani->blkno_radix, + le64_to_cpu(ment->blkno)); + mnode = list_next_entry(mnode, head); + } else { + mnode = list_first_entry(&mani->level_zero, + struct scoutfs_manifest_node, + head); + } + + list_for_each_entry_from(mnode, &mani->level_zero, head) { + if (scoutfs_key_cmp_range(key, &mnode->ment.first, + &mnode->ment.last) == 0) { + *ment = mnode->ment; + found = true; + break; + } + } + } + + if (!found) { + for (i = ment->level + 1; i <= SCOUTFS_MAX_LEVEL; i++) { + mnode = find_mnode(&mani->levels[i].root, key); + if (mnode) { + *ment = mnode->ment; + found = true; + break; + } + } + if (!found) + ment->level = SCOUTFS_MAX_LEVEL; + } + + spin_unlock(&mani->lock); + + return found; +} + int scoutfs_setup_manifest(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index a7685c5d..f22b3709 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -8,4 +8,8 @@ int scoutfs_add_manifest(struct super_block *sb, struct scoutfs_ring_manifest_entry *ment); void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); +bool scoutfs_next_manifest_segment(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_ring_manifest_entry *ment); + #endif diff --git a/kmod/src/segment.c b/kmod/src/segment.c new file mode 100644 index 00000000..4fbc1dd2 --- /dev/null +++ b/kmod/src/segment.c @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "key.h" +#include "item.h" +#include "segment.h" +#include "manifest.h" +#include "block.h" + +static struct scoutfs_item_header *next_ihdr(struct scoutfs_item_header *ihdr) +{ + return (void *)(ihdr + 1) + le16_to_cpu(ihdr->len); +} + +/* + * Use the manifest to search log segments for the most recent version + * of the item with the given key. Return a reference to the item after + * it's been added to the item cache. + */ +struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, + struct scoutfs_key *key) +{ + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_item_header *ihdr; + struct scoutfs_item_block *iblk; + struct scoutfs_item *item; + struct buffer_head *bh; + int cmp; + int err; + int i; + + /* XXX hold manifest */ + + memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); + + item = NULL; + err = -ENOENT; + while (scoutfs_next_manifest_segment(sb, key, &ment)) { + + bh = scoutfs_read_block(sb, le64_to_cpu(ment.blkno)); + if (!bh) { + err = -EIO; + break; + } + + iblk = (void *)bh->b_data; + /* XXX seq corruption */ + + ihdr = (void *)(iblk + 1); + + /* XXX test bloom filter blocks */ + /* XXX binary search of key array */ + /* XXX could populate more from granted range */ + + for (i = 0; i < le32_to_cpu(iblk->nr_items); + i++, ihdr = next_ihdr(ihdr)) { + cmp = scoutfs_key_cmp(key, &ihdr->key); + if (cmp > 0) + continue; + if (cmp < 0) + break; + + item = scoutfs_clean_item(sb, key, + le16_to_cpu(ihdr->len)); + if (IS_ERR(item)) { + err = PTR_ERR(item); + } else { + memcpy(item->val, (void *)(ihdr + 1), + item->val_len); + err = 0; + } + break; + } + + brelse(bh); + if (item) /* also breaks for IS_ERR */ + break; + } + + /* XXX release manifest */ + + if (err) + item = ERR_PTR(err); + + return item; +} diff --git a/kmod/src/segment.h b/kmod/src/segment.h new file mode 100644 index 00000000..6ae33f42 --- /dev/null +++ b/kmod/src/segment.h @@ -0,0 +1,7 @@ +#ifndef _SCOUTFS_SEGMENT_H_ +#define _SCOUTFS_SEGMENT_H_ + +struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, + struct scoutfs_key *key); + +#endif From 4b182c7759639ada620e2dda0c2bc6cb480ead41 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Feb 2016 18:21:54 -0800 Subject: [PATCH 010/920] scoutfs: insert manifest nodes into blkno radix We had forgotten to actually insert manifest nodes in to the blkno radix. This hasn't mattered yet because there's only been one manifest in the level 0 list. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index dd4e65fe..ec1773ac 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -97,7 +97,12 @@ static struct scoutfs_manifest_node *find_mnode(struct rb_root *root, return NULL; } -static struct scoutfs_manifest_node *delete_mnode(struct scoutfs_manifest *mani, +/* + * Find a manifest node at the given block number and return it after + * removing it from either the level 0 list or level rb trees. It's + * left in the blkno radix. + */ +static struct scoutfs_manifest_node *unlink_mnode(struct scoutfs_manifest *mani, u64 blkno) { @@ -129,7 +134,9 @@ void scoutfs_delete_manifest(struct super_block *sb, u64 blkno) struct scoutfs_manifest_node *mnode; spin_lock(&mani->lock); - mnode = delete_mnode(mani, blkno); + mnode = unlink_mnode(mani, blkno); + if (mnode) + radix_tree_delete(&mani->blkno_radix, blkno); spin_unlock(&mani->lock); if (mnode) kfree(mnode); @@ -147,10 +154,13 @@ int scoutfs_add_manifest(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; struct scoutfs_manifest_node *mnode; + u64 blkno = le64_to_cpu(ment->blkno); + bool preloaded = false; + int ret; spin_lock(&mani->lock); - mnode = delete_mnode(mani, le64_to_cpu(ment->blkno)); + mnode = unlink_mnode(mani, blkno); if (!mnode) { spin_unlock(&mani->lock); mnode = kmalloc(sizeof(struct scoutfs_manifest_node), @@ -158,9 +168,18 @@ int scoutfs_add_manifest(struct super_block *sb, if (!mnode) return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ + ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM); + if (ret) { + kfree(mnode); + return ret; + } + preloaded = true; + INIT_LIST_HEAD(&mnode->head); RB_CLEAR_NODE(&mnode->node); spin_lock(&mani->lock); + /* preloading should guarantee this succeeds */ + radix_tree_insert(&mani->blkno_radix, blkno, mnode); } mnode->ment = *ment; @@ -170,6 +189,8 @@ int scoutfs_add_manifest(struct super_block *sb, list_add(&mnode->head, &mani->level_zero); spin_unlock(&mani->lock); + if (preloaded) + radix_tree_preload_end(); return 0; } From c46fb0be7894fc3176a1465f41b8532d7d6337c2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Mar 2016 19:26:47 -0700 Subject: [PATCH 011/920] scoutfs: fix sense of filldir return in readdir The migration from the new iterator interface in upstream to the old readdir interface in rhel7 got the sense of the filldir return code wrong. Any readdir would deadlock livelock as the dot entry was returned at offset 0 without advancing f_pos. Signed-off-by: Zach Brown --- kmod/src/dir.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 744759a9..4ba72db1 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -240,13 +240,13 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) struct inode *parent = dentry->d_parent->d_inode; if (file->f_pos == 0) { - if (!filldir(dirent, ".", 1, 1, scoutfs_ino(inode), DT_DIR)) + if (filldir(dirent, ".", 1, 1, scoutfs_ino(inode), DT_DIR)) return 0; file->f_pos = 1; } if (file->f_pos == 1) { - if (!filldir(dirent, "..", 2, 1, scoutfs_ino(parent), DT_DIR)) + if (filldir(dirent, "..", 2, 1, scoutfs_ino(parent), DT_DIR)) return 0; file->f_pos = 2; } @@ -309,8 +309,8 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (dent->coll_nr < nr) continue; - if (!filldir(dirent, dent->name, dent->name_len, pos, - le64_to_cpu(dent->ino), dent->type)) + if (filldir(dirent, dent->name, dent->name_len, pos, + le64_to_cpu(dent->ino), dent->type)) break; file->f_pos = (pos | dent->coll_nr) + 1; From d2ead58ce48a5e87532f321b618d9e53ad06e596 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 16 Mar 2016 14:04:20 -0700 Subject: [PATCH 012/920] scoutfs: translate d_type in readdir I had forgotten to translate from the scoutfs types in items to the vfs types for filldir() so userspace was seeing garbage d_type values. Signed-off-by: Zach Brown --- kmod/src/dir.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 4ba72db1..66d9006f 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -67,7 +67,6 @@ static unsigned int mode_to_type(umode_t mode) #undef S_SHIFT } -#if 0 static unsigned int dentry_type(unsigned int type) { static unsigned char types[] = { @@ -86,7 +85,6 @@ static unsigned int dentry_type(unsigned int type) return DT_UNKNOWN; } -#endif static int names_equal(const char *name_a, int len_a, const char *name_b, int len_b) @@ -310,7 +308,8 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) continue; if (filldir(dirent, dent->name, dent->name_len, pos, - le64_to_cpu(dent->ino), dent->type)) + le64_to_cpu(dent->ino), + dentry_type(dent->type))) break; file->f_pos = (pos | dent->coll_nr) + 1; From edf3c8a5d43bae05d5c8550c27eb6af144619076 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 17 Mar 2016 17:47:32 -0700 Subject: [PATCH 013/920] scoutfs: add initial item block writing Add a sync_fs method that writes dirty items into level 0 item blocks. Add chunk allocator code to allocate new item blocks in free chunks. As the allocator bitmap is modified it adds bitmap entries to the ring. As new item blocks are allocated we create manifest entries that describe their block location and keys. The entry is added to the in-memory manifest and to entries in the ring. This isn't complete and there's still bugs but this is enough to start building on. Signed-off-by: Zach Brown --- kmod/src/block.c | 51 +++++++++++++++++ kmod/src/block.h | 3 + kmod/src/chunk.c | 51 ++++++++++++++++- kmod/src/chunk.h | 1 + kmod/src/manifest.c | 16 ++++++ kmod/src/manifest.h | 2 + kmod/src/ring.c | 133 ++++++++++++++++++++++++++++++++++++++++++-- kmod/src/ring.h | 3 + kmod/src/segment.c | 102 +++++++++++++++++++++++++++++++++ kmod/src/segment.h | 1 + kmod/src/super.c | 60 +++++++++++++++++++- kmod/src/super.h | 10 ++++ 12 files changed, 426 insertions(+), 7 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index cb403465..9102aaaa 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -58,3 +58,54 @@ struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) brelse(bh); return NULL; } + +/* + * Return a locked dirty buffer with undefined contents. The caller is + * responsible for initializing the entire block. Callers can try and + * read from these dirty blocks so we mark them verified so that they + * don't try to check uninitialized crcs. + */ +struct buffer_head *scoutfs_dirty_bh(struct super_block *sb, u64 blkno) +{ + struct buffer_head *bh; + + bh = sb_getblk(sb, blkno); + if (bh) { + lock_buffer(bh); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + set_buffer_private_verified(bh); + } + + return bh; +} + +/* + * Return a locked dirty buffer with a partially initialized block + * header. The caller has to calculate the header crc before unlocking + * the block. The header will have the sequence number of the dirty super + * by default. + */ +struct buffer_head *scoutfs_dirty_block(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_block_header *hdr; + struct buffer_head *bh; + + bh = scoutfs_dirty_bh(sb, blkno); + if (bh) { + hdr = (void *)bh->b_data; + *hdr = super->hdr; + hdr->blkno = cpu_to_le64(blkno); + } + + return bh; +} + +void scoutfs_calc_hdr_crc(struct buffer_head *bh) +{ + struct scoutfs_block_header *hdr = (void *)bh->b_data; + + hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); +} diff --git a/kmod/src/block.h b/kmod/src/block.h index c87fb6b8..30d79864 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -2,5 +2,8 @@ #define _SCOUTFS_BLOCK_H_ struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno); +struct buffer_head *scoutfs_dirty_bh(struct super_block *sb, u64 blkno); +struct buffer_head *scoutfs_dirty_block(struct super_block *sb, u64 blkno); +void scoutfs_calc_hdr_crc(struct buffer_head *bh); #endif diff --git a/kmod/src/chunk.c b/kmod/src/chunk.c index 6b5758af..4b7a24ec 100644 --- a/kmod/src/chunk.c +++ b/kmod/src/chunk.c @@ -24,16 +24,65 @@ #include "dir.h" #include "msg.h" #include "block.h" +#include "ring.h" void scoutfs_set_chunk_alloc_bits(struct super_block *sb, struct scoutfs_ring_bitmap *bm) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 off = le64_to_cpu(bm->offset); + u64 off = le64_to_cpu(bm->offset) * ARRAY_SIZE(bm->bits); /* XXX check for corruption */ sbi->chunk_alloc_bits[off] = bm->bits[0]; sbi->chunk_alloc_bits[off + 1] = bm->bits[1]; +} +/* + * Return the block number of the first block in a free chunk. + * + * The region around the cleared free bit for the allocation is always + * added to the ring and will generate a ton of overlapping ring + * entries. This is fine for initial testing but won't be good enough + * for real use. We'll have a bitmap of dirtied regions that are only + * logged as the update is written out. + */ +int scoutfs_alloc_chunk(struct super_block *sb, u64 *blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + unsigned long size = le64_to_cpu(super->total_chunks); + struct scoutfs_ring_bitmap bm; + unsigned long off; + unsigned long bit; + int ret; + + spin_lock(&sbi->chunk_alloc_lock); + + bit = find_next_bit_le(sbi->chunk_alloc_bits, size, 0); + if (bit >= size) { + ret = -ENOSPC; + } else { + clear_bit_le(bit, sbi->chunk_alloc_bits); + + off = round_down(bit, sizeof(bm.bits) * 8); + bm.offset = le32_to_cpu(off); + + off *= ARRAY_SIZE(bm.bits); + bm.bits[0] = sbi->chunk_alloc_bits[off]; + bm.bits[1] = sbi->chunk_alloc_bits[off + 1]; + + *blkno = bit << SCOUTFS_CHUNK_BLOCK_SHIFT; + ret = 0; + } + + spin_unlock(&sbi->chunk_alloc_lock); + + if (!ret) { + ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_BITMAP, &bm, + sizeof(bm)); + WARN_ON_ONCE(ret); + } + + return ret; } diff --git a/kmod/src/chunk.h b/kmod/src/chunk.h index b2cb6ff7..eb6615c7 100644 --- a/kmod/src/chunk.h +++ b/kmod/src/chunk.h @@ -3,5 +3,6 @@ void scoutfs_set_chunk_alloc_bits(struct super_block *sb, struct scoutfs_ring_bitmap *bm); +int scoutfs_alloc_chunk(struct super_block *sb, u64 *blkno); #endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index ec1773ac..8666ac49 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -18,6 +18,7 @@ #include "format.h" #include "manifest.h" #include "key.h" +#include "ring.h" /* * The manifest organizes log segment blocks into a tree structure. @@ -195,6 +196,21 @@ int scoutfs_add_manifest(struct super_block *sb, return 0; } +/* + * The caller is writing a new log segment. We add it to the in-memory + * manifest and write it to dirty ring blocks. + * + * XXX we'd also need to add stale manifest entry's to the ring + * XXX In the future we'd send it to the leader + */ +int scoutfs_new_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment) +{ + return scoutfs_add_manifest(sb, ment) ?: + scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, + ment, sizeof(*ment)); +} + /* * Fill the caller's ment with the next log segment in the manifest that * might contain the given key. The ment is initialized to 0 to return diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index f22b3709..407bfa28 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -6,6 +6,8 @@ void scoutfs_destroy_manifest(struct super_block *sb); int scoutfs_add_manifest(struct super_block *sb, struct scoutfs_ring_manifest_entry *ment); +int scoutfs_new_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment); void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); bool scoutfs_next_manifest_segment(struct super_block *sb, diff --git a/kmod/src/ring.c b/kmod/src/ring.c index aeee472b..095c30b2 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -23,6 +23,7 @@ #include "manifest.h" #include "chunk.h" #include "block.h" +#include "ring.h" static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) { @@ -62,11 +63,11 @@ static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) } /* - * Read a given logical ring block. - * - * Each ring map block entry maps a chunk's worth of ring blocks. + * Return the block number of the block that contains the given logical + * block in the ring. We look up ring block chunks in the map blocks + * in the chunk described by the super. */ -static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) +static u64 map_ring_block(struct super_block *sb, u64 block) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; @@ -85,7 +86,7 @@ static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) bh = scoutfs_read_block(sb, le64_to_cpu(super->ring_map_blkno) + div); if (!bh) - return NULL; + return 0; /* XXX verify map block */ @@ -93,9 +94,35 @@ static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) blkno = le64_to_cpu(map->blknos[rem]) + ring_block; brelse(bh); + return blkno; +} + +/* + * Read a given logical ring block. + */ +static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) +{ + u64 blkno = map_ring_block(sb, block); + + if (!blkno) + return NULL; + return scoutfs_read_block(sb, blkno); } +/* + * Return a dirty locked logical ring block. + */ +static struct buffer_head *dirty_ring_block(struct super_block *sb, u64 block) +{ + u64 blkno = map_ring_block(sb, block); + + if (!blkno) + return NULL; + + return scoutfs_dirty_block(sb, blkno); +} + int scoutfs_replay_ring(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -126,3 +153,99 @@ int scoutfs_replay_ring(struct super_block *sb) return ret; } + +/* + * The caller is generating ring entries for manifest and allocator + * bitmap as they write items to blocks. We pin the block that we're + * working on so that it isn't written out until we fill it and + * calculate its checksum. + */ +int scoutfs_dirty_ring_entry(struct super_block *sb, u8 type, void *data, + u16 len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ring_block *ring; + struct scoutfs_ring_entry *ent; + struct buffer_head *bh; + unsigned int avail; + u64 block; + int ret = 0; + + bh = sbi->dirty_ring_bh; + ent = sbi->dirty_ring_ent; + avail = sbi->dirty_ring_ent_avail; + + if (bh && len > avail) { + scoutfs_finish_dirty_ring(sb); + bh = NULL; + } + if (!bh) { + block = le64_to_cpu(super->ring_first_block) + + le64_to_cpu(super->ring_active_blocks); + if (block >= le64_to_cpu(super->ring_total_blocks)) + block -= le64_to_cpu(super->ring_total_blocks); + + bh = dirty_ring_block(sb, block); + if (!bh) { + ret = -ENOMEM; + goto out; + } + + ring = (void *)bh->b_data; + ring->nr_entries = 0; + ent = (void *)(ring + 1); + /* assuming len fits in new empty block */ + } + + ring = (void *)bh->b_data; + + ent->type = type; + ent->len = cpu_to_le16(len); + memcpy(ent + 1, data, len); + le16_add_cpu(&ring->nr_entries, 1); + + ent = (void *)(ent + 1) + le16_to_cpu(ent->len); + avail = SCOUTFS_BLOCK_SIZE - ((char *)(ent + 1) - (char *)ring); +out: + sbi->dirty_ring_bh = bh; + sbi->dirty_ring_ent = ent; + sbi->dirty_ring_ent_avail = avail; + + return ret; +} + +/* + * The super might have a pinned partial dirty ring block. This is + * called as we finish the block or when the commit is done. We + * calculate the checksum and unlock it so it can be written. + * + * XXX This is about to write a partial block. We might as well fill + * that space with more old entries from the manifest and ring before + * we write it. + */ +int scoutfs_finish_dirty_ring(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buffer_head *bh; + + bh = sbi->dirty_ring_bh; + if (!bh) + return 0; + + sbi->dirty_ring_bh = NULL; + + /* + * XXX we're not zeroing the tail of the block here. We will + * when we change the item block format to let us append to + * the block without walking all the items. + */ + scoutfs_calc_hdr_crc(bh); + unlock_buffer(bh); + brelse(bh); + + le64_add_cpu(&super->ring_active_blocks, 1); + + return 0; +} diff --git a/kmod/src/ring.h b/kmod/src/ring.h index b50b67e3..ee929e20 100644 --- a/kmod/src/ring.h +++ b/kmod/src/ring.h @@ -2,5 +2,8 @@ #define _SCOUTFS_RING_H_ int scoutfs_replay_ring(struct super_block *sb); +int scoutfs_dirty_ring_entry(struct super_block *sb, u8 type, void *data, + u16 len); +int scoutfs_finish_dirty_ring(struct super_block *sb); #endif diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 4fbc1dd2..79b40f4f 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -23,6 +23,8 @@ #include "segment.h" #include "manifest.h" #include "block.h" +#include "chunk.h" +#include "ring.h" static struct scoutfs_item_header *next_ihdr(struct scoutfs_item_header *ihdr) { @@ -101,3 +103,103 @@ struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, return item; } + +static int finish_item_block(struct super_block *sb, struct buffer_head *bh, + void *until) +{ + struct scoutfs_item_block *iblk = (void *)bh->b_data; + struct scoutfs_ring_manifest_entry ment; + + memset(until, 0, (void *)bh->b_data + SCOUTFS_BLOCK_SIZE - until); + scoutfs_calc_hdr_crc(bh); + unlock_buffer(bh); + brelse(bh); + + ment.blkno = cpu_to_le64(bh->b_blocknr); + ment.seq = iblk->hdr.seq; + ment.level = 0; + ment.first = iblk->first; + ment.last = iblk->last; + + return scoutfs_new_manifest(sb, &ment); +} + +/* + * Write all the currently dirty items in newly allocated log segments. + * New ring entries are added as the alloc bitmap is modified and as the + * manifest is updated. If we write out all the item and ring blocks then + * we write a new super that references those new blocks. + */ +int scoutfs_write_dirty_items(struct super_block *sb) +{ + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + struct scoutfs_item_header *ihdr; + struct scoutfs_item_block *iblk; + struct scoutfs_item *item; + struct buffer_head *bh; + int val_space; + u64 blkno; + int ret; + + /* XXX wait until transactions are complete */ + + item = NULL; + iblk = NULL; + while ((item = scoutfs_item_next_dirty(sb, item))) { + + if (iblk && (item->val_len > val_space)) { + iblk = NULL; + ret = finish_item_block(sb, bh, ihdr); + if (ret) + break; + } + + if (!iblk) { + /* get the next item block */ + ret = scoutfs_alloc_chunk(sb, &blkno); + if (ret) + break; + + bh = scoutfs_dirty_block(sb, blkno); + if (!bh) { + ret = -ENOMEM; + break; + } + + iblk = (void *)bh->b_data; + iblk->first = item->key; + iblk->nr_items = 0; + ihdr = (void *)(iblk + 1); + /* XXX assuming that val_space is big enough */ + } + + iblk->last = item->key; + ihdr->key = item->key; + ihdr->len = cpu_to_le16(item->val_len); + memcpy((void *)(ihdr + 1), item->val, item->val_len); + le32_add_cpu(&iblk->nr_items, 1); + + /* XXX assuming that the next ihdr fits */ + ihdr = (void *)(ihdr + 1) + le16_to_cpu(ihdr->len); + val_space = (char *)iblk + SCOUTFS_BLOCK_SIZE - + (char *)(ihdr + 1); + } + + scoutfs_item_put(item); /* only if the loop aborted */ + + /* finish writing if we did work and haven't failed */ + if (iblk && !ret) { + ret = finish_item_block(sb, bh, ihdr) ?: + scoutfs_finish_dirty_ring(sb) ?: + filemap_write_and_wait(mapping) ?: + scoutfs_write_dirty_super(sb); + if (!ret) { + scoutfs_advance_dirty_super(sb); + scoutfs_item_all_clean(sb); + } + } + + /* XXX better tear down down in the error case */ + + return ret; +} diff --git a/kmod/src/segment.h b/kmod/src/segment.h index 6ae33f42..4755e5cd 100644 --- a/kmod/src/segment.h +++ b/kmod/src/segment.h @@ -3,5 +3,6 @@ struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, struct scoutfs_key *key); +int scoutfs_write_dirty_items(struct super_block *sb); #endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 3f9d001f..3bdfc743 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -26,12 +26,67 @@ #include "block.h" #include "manifest.h" #include "ring.h" +#include "segment.h" + +static int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + /* XXX always waiting */ + return scoutfs_write_dirty_items(sb); +} static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, + .sync_fs = scoutfs_sync_fs, }; +/* + * The caller advances the block number and sequence number in the super + * every time it wants to dirty it and eventually write it to reference + * dirty data that's been written. + */ +void scoutfs_advance_dirty_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + u64 blkno; + + blkno = le64_to_cpu(super->hdr.blkno) - SCOUTFS_SUPER_BLKNO; + if (++blkno == SCOUTFS_SUPER_NR) + blkno = 0; + super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO + blkno); + + le64_add_cpu(&super->hdr.seq, 1); +} + +/* + * We've been modifying the super copy in the info as we made changes. + * Write the super to finalize. + */ +int scoutfs_write_dirty_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buffer_head *bh; + size_t sz; + int ret; + + bh = scoutfs_dirty_block(sb, le64_to_cpu(super->hdr.blkno)); + if (!bh) + return -ENOMEM; + + sz = sizeof(struct scoutfs_super_block); + memcpy(bh->b_data, super, sz); + memset(bh->b_data + sz, 0, SCOUTFS_BLOCK_SIZE - sz); + scoutfs_calc_hdr_crc(bh); + + unlock_buffer(bh); + ret = sync_dirty_buffer(bh); + brelse(bh); + + return ret; +} + static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -114,6 +169,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->item_lock); sbi->item_root = RB_ROOT; sbi->dirty_item_root = RB_ROOT; + spin_lock_init(&sbi->chunk_alloc_lock); if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { printk(KERN_ERR "couldn't set blocksize\n"); @@ -140,6 +196,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; + scoutfs_advance_dirty_super(sb); + return 0; } @@ -151,8 +209,8 @@ static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, static void scoutfs_kill_sb(struct super_block *sb) { - scoutfs_destroy_manifest(sb); kill_block_super(sb); + scoutfs_destroy_manifest(sb); kfree(sb->s_fs_info); } diff --git a/kmod/src/super.h b/kmod/src/super.h index d7229877..604448ca 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -18,7 +18,14 @@ struct scoutfs_sb_info { struct scoutfs_manifest *mani; + spinlock_t chunk_alloc_lock; __le64 *chunk_alloc_bits; + + /* pinned dirty ring block during commit */ + struct buffer_head *dirty_ring_bh; + struct scoutfs_ring_entry *dirty_ring_ent; + unsigned int dirty_ring_ent_avail; + }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) @@ -26,4 +33,7 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } +void scoutfs_advance_dirty_super(struct super_block *sb); +int scoutfs_write_dirty_super(struct super_block *sb); + #endif From 0c0f2b19d534c5eb3f110cc3a3f3302fd1baf3b0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 17 Mar 2016 19:12:49 -0700 Subject: [PATCH 014/920] scoutfs: update dirty inode items Wire up the code to update dirty inode items as inodes are modified in memory. We had a bit of the code but it wasn't being called. Signed-off-by: Zach Brown --- kmod/src/dir.c | 8 ++++---- kmod/src/inode.c | 11 ++++++----- kmod/src/inode.h | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 66d9006f..f1ab3190 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -418,8 +418,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, inc_nlink(dir); } - mark_inode_dirty(inode); - mark_inode_dirty(dir); + scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(dir); insert_inode_hash(inode); d_instantiate(dentry, inode); @@ -510,8 +510,8 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) drop_nlink(dir); drop_nlink(inode); } - mark_inode_dirty(inode); - mark_inode_dirty(dir); + scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(dir); out: return ret; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 02446332..521a1671 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -192,8 +192,12 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) * Every time we modify the inode in memory we copy it to its inode * item. This lets us write out blocks of items without having to track * down dirty vfs inodes and safely copy them into items before writing. + * + * The caller makes sure that the item is dirty and pinned so they don't + * have to deal with errors and unwinding after they've modified the + * vfs inode and get here. */ -int scoutfs_inode_update(struct inode *inode) +void scoutfs_update_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_item *item; @@ -202,13 +206,10 @@ int scoutfs_inode_update(struct inode *inode) scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); item = scoutfs_item_lookup(sb, &key); - if (IS_ERR(item)) - return PTR_ERR(item); + BUG_ON(IS_ERR(item)); store_inode(item->val, inode); scoutfs_item_put(item); - - return 0; } /* diff --git a/kmod/src/inode.h b/kmod/src/inode.h index bb9a6149..d7009352 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -22,7 +22,7 @@ struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); -int scoutfs_inode_update(struct inode *inode); +void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); From 96b8a6da4618c9147da2a0fd738e1462c3f2bafc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Mar 2016 17:21:12 -0700 Subject: [PATCH 015/920] scoutfs: update created inode times in mknod In mknod the newly created inode's times are set down in the new inode creation path instead of up in the mknod path to match the parent dir's ctime and mtime. This is strictly legal but it's easier to test that all the times have been set in the mknod by having them equal. This stops mkdir-interface test failures when enough time passes between inode creation and parent dir timestamp updates to have them differ. Signed-off-by: Zach Brown --- kmod/src/dir.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index f1ab3190..921fbdef 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -412,6 +412,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; + inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; if (S_ISDIR(mode)) { inc_nlink(inode); From af492a9f27fb5aab668329c8c03087faf874463f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Mar 2016 17:24:12 -0700 Subject: [PATCH 016/920] scoutfs: add scoutfs_inc_key() Add a quick inline function for incrementing a key value across the inode>type>offset sorted key space. Signed-off-by: Zach Brown --- kmod/src/key.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kmod/src/key.h b/kmod/src/key.h index 27f7ffc3..80e60668 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -61,4 +61,13 @@ static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, key->offset = cpu_to_le64(offset); } +static inline void scoutfs_inc_key(struct scoutfs_key *key) +{ + le64_add_cpu(&key->offset, 1); + if (!key->offset) { + if (++key->type == 0) + le64_add_cpu(&key->inode, 1); + } +} + #endif From 12d5d3d216aae9b84dddf43504c3a689b0e2c67b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Mar 2016 17:30:39 -0700 Subject: [PATCH 017/920] scoutfs: add next item reading Add code to walk all the block segments that intersect a key range to find the next item after that key value. It is easier to just return failure from the next item reader and have the caller retry the searches so we change the specific item reading path to use the same convention to keep the caller consistent. This still warns as it falls off the last block but that's fine for now. We're going to be changing all this in the next few commits. Signed-off-by: Zach Brown --- kmod/src/item.c | 17 ++++- kmod/src/segment.c | 162 +++++++++++++++++++++++++++++++++++++++++---- kmod/src/segment.h | 5 +- 3 files changed, 165 insertions(+), 19 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index c86583ed..1f253826 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -263,6 +263,8 @@ static struct scoutfs_item *item_lookup(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_item *item; unsigned long flags; + unsigned retried = 0; + int ret; do { spin_lock_irqsave(&sbi->item_lock, flags); @@ -272,9 +274,18 @@ static struct scoutfs_item *item_lookup(struct super_block *sb, atomic_inc(&item->refcount); spin_unlock_irqrestore(&sbi->item_lock, flags); - if (!item) - item = scoutfs_read_segment_item(sb, key); - } while (item == ERR_PTR(-EEXIST)); + if (!item) { + if (np == FI_NEXT) + ret = scoutfs_read_next_item(sb, key); + else + ret = scoutfs_read_item(sb, key); + if (ret) + item = ERR_PTR(ret); + } + } while (!item && !retried++); + + if (!item) + item = ERR_PTR(-ENOENT); return item; } diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 79b40f4f..0c3d1adb 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -33,32 +33,30 @@ static struct scoutfs_item_header *next_ihdr(struct scoutfs_item_header *ihdr) /* * Use the manifest to search log segments for the most recent version - * of the item with the given key. Return a reference to the item after - * it's been added to the item cache. + * of the item with the given key. This only returns an error if it + * fails to determine if the item exists or not. It's up to the caller + * to retry the lookup after success. */ -struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, - struct scoutfs_key *key) +int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_ring_manifest_entry ment; struct scoutfs_item_header *ihdr; struct scoutfs_item_block *iblk; - struct scoutfs_item *item; + struct scoutfs_item *item = NULL; struct buffer_head *bh; + int ret = 0; int cmp; - int err; int i; /* XXX hold manifest */ memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); - item = NULL; - err = -ENOENT; while (scoutfs_next_manifest_segment(sb, key, &ment)) { bh = scoutfs_read_block(sb, le64_to_cpu(ment.blkno)); if (!bh) { - err = -EIO; + ret = -EIO; break; } @@ -82,11 +80,10 @@ struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, item = scoutfs_clean_item(sb, key, le16_to_cpu(ihdr->len)); if (IS_ERR(item)) { - err = PTR_ERR(item); + ret = PTR_ERR(item); } else { memcpy(item->val, (void *)(ihdr + 1), item->val_len); - err = 0; } break; } @@ -98,10 +95,147 @@ struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, /* XXX release manifest */ - if (err) - item = ERR_PTR(err); + scoutfs_item_put(item); + return ret; +} - return item; +/* + * Reading the next item is more expensive than looking up a specific + * item. We can't use the bloom filters because we don't know what key + * is next. We have to search blocks at all levels because the next + * item could be in any of them. + * + * After having gone to the trouble to establish next item positions in + * all the blocks we take the opportunity to amortize that cost and + * insert multiple items. + * + * This only returns an error if it was unsure if there's a next item + * or not. It will return success if there were no next items. The caller + * is responsible for retrying the lookup after reading. + */ +struct item_block_cursor { + struct list_head list; + + struct buffer_head *bh; + struct scoutfs_item_header *ihdr; + unsigned int i; +}; +int scoutfs_read_next_item(struct super_block *sb, + struct scoutfs_key *first_key) +{ + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_item_header *least; + struct scoutfs_item_header *ihdr; + struct scoutfs_item_block *iblk; + struct item_block_cursor *curs; + struct item_block_cursor *tmp; + struct scoutfs_item *item; + struct scoutfs_key key; + struct buffer_head *bh; + LIST_HEAD(cursors); + int ret = 0; + int pass; + int i; + + /* XXX hold manifest */ + + memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); + + /* find all the log segments that contain our key */ + key = *first_key; + while (scoutfs_next_manifest_segment(sb, &key, &ment)) { + + curs = kmalloc(sizeof(struct item_block_cursor), GFP_NOFS); + if (!curs) { + ret = -ENOMEM; + goto out; + } + + bh = scoutfs_read_block(sb, le64_to_cpu(ment.blkno)); + if (!bh) { + ret = -EIO; + goto out; + } + + /* XXX verify */ + iblk = (void *)bh->b_data; + + curs->bh = bh; + curs->i = 0; + curs->ihdr = (void *)(iblk + 1); + list_add_tail(&curs->list, &cursors); + } + + /* there can be no segments that contain the item */ + if (list_empty(&cursors)) { + ret = 0; + goto out; + } + + /* XXX arbitrary number of next items to insert */ + for (pass = 0; pass < 16; pass++) { + + least = NULL; + list_for_each_entry(curs, &cursors, list) { + iblk = (void *)curs->bh->b_data; + ihdr = curs->ihdr; + i = curs->i; + + /* Find the next item past the search key. */ + for (; i < le32_to_cpu(iblk->nr_items); i++) { + if (scoutfs_key_cmp(&key, &ihdr->key) <= 0) + break; + + ihdr = next_ihdr(ihdr); + } + + /* + * If we fall off a block then we can't know if + * we have the least key without checking the + * next block at that level. It could have an + * item less than the least in our other blocks. + */ + if (WARN_ON_ONCE(i == le32_to_cpu(iblk->nr_items))) { + ret = -EIO; + goto out; + } + + /* + * Remember the newest least key in the blocks that's + * past the search key. + */ + if (!least || + scoutfs_key_cmp(&ihdr->key, &least->key) < 0) + least = ihdr; + + curs->ihdr = ihdr; + curs->i = i; + } + + /* start the next search past the next key */ + key = least->key; + scoutfs_inc_key(&key); + + /* insert the next item (XXX if it's not deleted) */ + item = scoutfs_clean_item(sb, &least->key, + le16_to_cpu(least->len)); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + if (ret == -EEXIST) + continue; + break; + } + + memcpy(item->val, (void *)(least + 1), item->val_len); + scoutfs_item_put(item); + } +out: + list_for_each_entry_safe(curs, tmp, &cursors, list) { + brelse(curs->bh); + list_del_init(&curs->list); + kfree(curs); + } + return ret; } static int finish_item_block(struct super_block *sb, struct buffer_head *bh, diff --git a/kmod/src/segment.h b/kmod/src/segment.h index 4755e5cd..fd0fda69 100644 --- a/kmod/src/segment.h +++ b/kmod/src/segment.h @@ -1,8 +1,9 @@ #ifndef _SCOUTFS_SEGMENT_H_ #define _SCOUTFS_SEGMENT_H_ -struct scoutfs_item *scoutfs_read_segment_item(struct super_block *sb, - struct scoutfs_key *key); +int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key); +int scoutfs_read_next_item(struct super_block *sb, + struct scoutfs_key *first_key); int scoutfs_write_dirty_items(struct super_block *sb); #endif From 1270553f1f3a71f841339c9f4bfe83e26a718086 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Mar 2016 17:40:14 -0700 Subject: [PATCH 018/920] scoutfs: mega item access omnibus commit 9000 Initially items were stored in memory with an rbtree. That let us build up the API above items without worrying about their storage. That gave us dirty items in memory and we could start working on writing them to and reading them from the log segment blocks. Now that we have the code on either side we can get rid of the item cache in between. It had some nice properties but it's fundamentally duplicating the item storage in cached log segment blocks. We'd also have to teach it to differentiate between negative cache entries and missing entries that need to be filled from blocks. And the giant item index becomes a bottleneck. We have to index items in log segments anyway so we rewrite the item APIs to read and write the items in the log segments directly. Creation writes to dirty blocks in memory and reading and iteration walk through the cached blocks in the buffer cache. I've tried to comment the files and functions appropriately so most of the commentary for the new methods is in the body of the commit. The overall theme is making it relatively efficient to operate on individual items in log segments. Previously we could only walk all the items in an existing segment or write all the dirty items to a new segment. Now we have bloom filters and sorted item headers to let us test for the presence of an item's key with progressively more expensive methods. We hold on to a dirty segment and fill it as we create new items. This needs more fleshing out and testing but this is a solid first pass and it passes our existing tests. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/block.c | 80 ++--- kmod/src/block.h | 5 +- kmod/src/bloom.c | 125 +++++++ kmod/src/bloom.h | 16 + kmod/src/crc.c | 1 + kmod/src/dir.c | 201 +++++------- kmod/src/format.h | 48 ++- kmod/src/inode.c | 40 ++- kmod/src/item.c | 463 -------------------------- kmod/src/item.h | 40 --- kmod/src/key.h | 31 +- kmod/src/manifest.c | 35 +- kmod/src/manifest.h | 5 +- kmod/src/ring.c | 8 +- kmod/src/segment.c | 782 ++++++++++++++++++++++++++++++-------------- kmod/src/segment.h | 32 +- kmod/src/skip.c | 325 ++++++++++++++++++ kmod/src/skip.h | 18 + kmod/src/super.c | 38 ++- kmod/src/super.h | 7 +- 21 files changed, 1319 insertions(+), 985 deletions(-) create mode 100644 kmod/src/bloom.c create mode 100644 kmod/src/bloom.h delete mode 100644 kmod/src/item.c delete mode 100644 kmod/src/item.h create mode 100644 kmod/src/skip.c create mode 100644 kmod/src/skip.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index e5712be8..dae6c279 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,4 +1,4 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += block.o chunk.o crc.o dir.o inode.o item.o manifest.o msg.o \ - ring.o segment.o super.o +scoutfs-y += block.o bloom.o chunk.o crc.o dir.o inode.o manifest.o msg.o \ + ring.o segment.o skip.o super.o diff --git a/kmod/src/block.c b/kmod/src/block.c index 9102aaaa..8326382f 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -21,25 +21,13 @@ BUFFER_FNS(Private_Verified, private_verified) - -/* - * A quick metadata read wrapper which knows how to validate the - * block header. - */ -struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) +static void verify_block_header(struct super_block *sb, struct buffer_head *bh) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_block_header *hdr; - struct buffer_head *bh; - u32 crc; - - bh = sb_bread(sb, blkno); - if (!bh || buffer_private_verified(bh)) - return bh; - - hdr = (void *)bh->b_data; - crc = scoutfs_crc_block(hdr); + struct scoutfs_block_header *hdr = (void *)bh->b_data; + u32 crc = scoutfs_crc_block(hdr); + u64 blkno = bh->b_blocknr; if (le32_to_cpu(hdr->crc) != crc) { printk("blkno %llu hdr crc %x != calculated %x\n", blkno, @@ -52,49 +40,67 @@ struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) le64_to_cpu(hdr->blkno)); } else { set_buffer_private_verified(bh); - return bh; } - - brelse(bh); - return NULL; } /* - * Return a locked dirty buffer with undefined contents. The caller is - * responsible for initializing the entire block. Callers can try and - * read from these dirty blocks so we mark them verified so that they - * don't try to check uninitialized crcs. + * Read an existing block from the device and verify its metadata header. */ -struct buffer_head *scoutfs_dirty_bh(struct super_block *sb, u64 blkno) +struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) { struct buffer_head *bh; - bh = sb_getblk(sb, blkno); - if (bh) { - lock_buffer(bh); - set_buffer_uptodate(bh); - mark_buffer_dirty(bh); - set_buffer_private_verified(bh); + bh = sb_bread(sb, blkno); + if (!bh || buffer_private_verified(bh)) + return bh; + + lock_buffer(bh); + if (!buffer_private_verified(bh)) + verify_block_header(sb, bh); + unlock_buffer(bh); + + if (!buffer_private_verified(bh)) { + brelse(bh); + bh = NULL; } return bh; } /* - * Return a locked dirty buffer with a partially initialized block - * header. The caller has to calculate the header crc before unlocking - * the block. The header will have the sequence number of the dirty super - * by default. + * Read the block that contains the given byte offset in the given chunk. */ -struct buffer_head *scoutfs_dirty_block(struct super_block *sb, u64 blkno) +struct buffer_head *scoutfs_read_block_off(struct super_block *sb, u64 blkno, + u32 off) +{ + if (WARN_ON_ONCE(off >= SCOUTFS_CHUNK_SIZE)) + return ERR_PTR(-EINVAL); + + return scoutfs_read_block(sb, blkno + (off >> SCOUTFS_BLOCK_SHIFT)); +} + +/* + * Return a newly allocated metadata block with an updated block header + * to match the current dirty super block. Callers are responsible for + * serializing access to the block and for zeroing unwritten block + * contents. + */ +struct buffer_head *scoutfs_new_block(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_block_header *hdr; struct buffer_head *bh; - bh = scoutfs_dirty_bh(sb, blkno); + bh = sb_getblk(sb, blkno); if (bh) { + if (!buffer_uptodate(bh) || buffer_private_verified(bh)) { + lock_buffer(bh); + set_buffer_uptodate(bh); + set_buffer_private_verified(bh); + unlock_buffer(bh); + } + hdr = (void *)bh->b_data; *hdr = super->hdr; hdr->blkno = cpu_to_le64(blkno); diff --git a/kmod/src/block.h b/kmod/src/block.h index 30d79864..7be8ed6d 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -2,8 +2,9 @@ #define _SCOUTFS_BLOCK_H_ struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno); -struct buffer_head *scoutfs_dirty_bh(struct super_block *sb, u64 blkno); -struct buffer_head *scoutfs_dirty_block(struct super_block *sb, u64 blkno); +struct buffer_head *scoutfs_read_block_off(struct super_block *sb, u64 blkno, + u32 off); +struct buffer_head *scoutfs_new_block(struct super_block *sb, u64 blkno); void scoutfs_calc_hdr_crc(struct buffer_head *bh); #endif diff --git a/kmod/src/bloom.c b/kmod/src/bloom.c new file mode 100644 index 00000000..df528afd --- /dev/null +++ b/kmod/src/bloom.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "block.h" +#include "bloom.h" + +/* + * Each log segment starts with a bloom filters that spans multiple + * blocks. It's used to test for the presence of key in the log segment + * without having to read and search the much larger array of items and + * their keys. + */ + +/* XXX garbage hack until we have siphash */ +static u32 bloom_hash(struct scoutfs_key *key, __le32 salt) +{ + return crc32c(le32_to_cpu(salt), key, sizeof(struct scoutfs_key)); +} + +/* + * Find the bits in the bloom filter for the given key. The caller calculates + * these once and uses them to test all the blocks. + */ +void scoutfs_calc_bloom_bits(struct scoutfs_bloom_bits *bits, + struct scoutfs_key *key, __le32 *salts) +{ + unsigned h_bits = 0; + unsigned int b; + unsigned s = 0; + u64 h = 0; + int i; + + BUILD_BUG_ON(SCOUTFS_BLOOM_BIT_WIDTH > 32); + + for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { + if (h_bits < SCOUTFS_BLOOM_BIT_WIDTH) { + h = (h << 32) | bloom_hash(key, salts[s++]); + h_bits += 32; + } + + b = h & SCOUTFS_BLOOM_BIT_MASK; + h >>= SCOUTFS_BLOOM_BIT_WIDTH; + h_bits -= SCOUTFS_BLOOM_BIT_WIDTH; + + bits->block[i] = (b / SCOUTFS_BLOOM_BITS_PER_BLOCK) % + SCOUTFS_BLOOM_BLOCKS; + bits->bit_off[i] = b % SCOUTFS_BLOOM_BITS_PER_BLOCK; + } +} + +/* + * Set the caller's bit numbers in the bloom filter contained in bloom + * blocks starting at the given block number. The caller has + * initialized the blocks and is responsible for locking and dirtying + * and writeout. + */ +int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_bloom_bits *bits) +{ + struct scoutfs_bloom_block *blm; + struct buffer_head *bh; + int ret = 0; + int i; + + for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { + bh = scoutfs_read_block(sb, blkno + bits->block[i]); + if (!bh) { + ret = -EIO; + break; + } + + blm = (void *)bh->b_data; + set_bit_le(bits->bit_off[i], blm->bits); + + brelse(bh); + } + + return ret; +} + +/* + * Returns zero if the bits' key can't be found in the block, true if it + * might, and -errno if IO fails. + */ +int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_bloom_bits *bits) +{ + struct scoutfs_bloom_block *blm; + struct buffer_head *bh; + int ret; + int i; + + for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { + bh = scoutfs_read_block(sb, blkno + bits->block[i]); + if (!bh) { + ret = -EIO; + break; + } + + blm = (void *)bh->b_data; + ret = !!test_bit_le(bits->bit_off[i], blm->bits); + brelse(bh); + if (!ret) + break; + } + + return ret; +} diff --git a/kmod/src/bloom.h b/kmod/src/bloom.h new file mode 100644 index 00000000..4e843fbe --- /dev/null +++ b/kmod/src/bloom.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_BLOOM_H_ +#define _SCOUTFS_BLOOM_H_ + +struct scoutfs_bloom_bits { + u16 bit_off[SCOUTFS_BLOOM_BITS]; + u8 block[SCOUTFS_BLOOM_BITS]; +}; + +void scoutfs_calc_bloom_bits(struct scoutfs_bloom_bits *bits, + struct scoutfs_key *key, __le32 *salts); +int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_bloom_bits *bits); +int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_bloom_bits *bits); + +#endif diff --git a/kmod/src/crc.c b/kmod/src/crc.c index 9869cbd1..cde9a1ae 100644 --- a/kmod/src/crc.c +++ b/kmod/src/crc.c @@ -10,6 +10,7 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ +#include #include #include "format.h" diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 921fbdef..e0aac04d 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -20,7 +20,7 @@ #include "dir.h" #include "inode.h" #include "key.h" -#include "item.h" +#include "segment.h" #include "super.h" /* @@ -110,26 +110,26 @@ static unsigned int dent_bytes(unsigned int name_len) return sizeof(struct scoutfs_dirent) + name_len; } -static unsigned int dent_val_off(struct scoutfs_item *item, +static unsigned int dent_val_off(struct scoutfs_item_ref *ref, struct scoutfs_dirent *dent) { - return (char *)dent - (char *)item->val; + return (char *)dent - (char *)ref->val; } -static inline struct scoutfs_dirent *next_dent(struct scoutfs_item *item, +static inline struct scoutfs_dirent *next_dent(struct scoutfs_item_ref *ref, struct scoutfs_dirent *dent) { unsigned int next_off; - next_off = dent_val_off(item, dent) + dent_bytes(dent->name_len); - if (next_off == item->val_len) + next_off = dent_val_off(ref, dent) + dent_bytes(dent->name_len); + if (next_off == ref->val_len) return NULL; - return item->val + next_off; + return ref->val + next_off; } -#define for_each_item_dent(item, dent) \ - for (dent = item->val; dent; dent = next_dent(item, dent)) +#define for_each_item_dent(ref, dent) \ + for (dent = (ref)->val; dent; dent = next_dent(ref, dent)) struct dentry_info { /* @@ -175,10 +175,10 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, { struct super_block *sb = dir->i_sb; struct scoutfs_dirent *dent; - struct scoutfs_item *item; struct dentry_info *di; struct scoutfs_key key; struct inode *inode; + DECLARE_SCOUTFS_ITEM_REF(ref); u64 ino = 0; u32 h = 0; u32 nr = 0; @@ -198,14 +198,12 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, h = name_hash(dir, dentry->d_name.name, dentry->d_name.len); scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - item = scoutfs_item_lookup(sb, &key); - if (IS_ERR(item)) { - ret = PTR_ERR(item); + ret = scoutfs_read_item(sb, &key, &ref); + if (ret) goto out; - } ret = -ENOENT; - for_each_item_dent(item, dent) { + for_each_item_dent(&ref, dent) { if (names_equal(dentry->d_name.name, dentry->d_name.len, dent->name, dent->name_len)) { ino = le64_to_cpu(dent->ino); @@ -215,7 +213,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, } } - scoutfs_item_put(item); + scoutfs_put_ref(&ref); out: if (ret == -ENOENT) { inode = NULL; @@ -254,7 +252,7 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) /* * readdir finds the next entry at or past the hash|coll_nr stored in - * the ctx->pos (f_pos). + * the current file position. * * It will need to be careful not to read past the region of the dirent * hash offset keys that it has access to. @@ -263,65 +261,63 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_dirent *dent; - struct scoutfs_key last_key; - struct scoutfs_item *item; - struct scoutfs_key key; - u32 nr; - u32 off; - u64 pos; + struct scoutfs_key first; + struct scoutfs_key last; + LIST_HEAD(iter_list); int ret = 0; + u32 off; + u32 pos; + u32 nr; if (!dir_emit_dots(file, dirent, filldir)) return 0; - scoutfs_set_key(&last_key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + file->f_pos >> SCOUTFS_DIRENT_COLL_BITS); + scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, SCOUTFS_DIRENT_OFF_MASK); - do { - off = file->f_pos >> SCOUTFS_DIRENT_COLL_BITS; - nr = file->f_pos & SCOUTFS_DIRENT_COLL_MASK; - - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - off); - item = scoutfs_item_next(sb, &key); - if (IS_ERR(item)) { - ret = PTR_ERR(item); - if (ret == -ENOENT) - ret = 0; + for(;;) { + scoutfs_put_ref(&ref); + ret = scoutfs_next_item(sb, &first, &last, &iter_list, &ref); + if (ret) break; - } - if (scoutfs_key_cmp(&item->key, &last_key) > 0) { - scoutfs_item_put(item); - break; - } - - /* reset nr to 0 if we found the next item */ - if (scoutfs_key_offset(&item->key) != off) + /* start from first collision if we're in a new item */ + if (scoutfs_key_offset(&first) == scoutfs_key_offset(ref.key)) + nr = file->f_pos & SCOUTFS_DIRENT_COLL_MASK; + else nr = 0; - pos = scoutfs_key_offset(&item->key) - << SCOUTFS_DIRENT_COLL_BITS; - for_each_item_dent(item, dent) { + off = scoutfs_key_offset(ref.key) << SCOUTFS_DIRENT_COLL_BITS; + for_each_item_dent(&ref, dent) { if (dent->coll_nr < nr) continue; + pos = off | dent->coll_nr; + if (filldir(dirent, dent->name, dent->name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) break; - file->f_pos = (pos | dent->coll_nr) + 1; + file->f_pos = pos + 1; } + /* done if filldir broke the loop */ + if (dent) + break; - scoutfs_item_put(item); + first = *ref.key; + scoutfs_inc_key(&first); + } - /* advance to the next hash value if we finished item */ - if (dent == NULL) - file->f_pos = pos + (1 << SCOUTFS_DIRENT_COLL_BITS); + scoutfs_put_ref(&ref); + scoutfs_put_iter_list(&iter_list); - } while (dent == NULL); + if (ret == -ENOENT) + ret = 0; return ret; } @@ -332,12 +328,11 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct super_block *sb = dir->i_sb; struct inode *inode = NULL; struct scoutfs_dirent *dent; - struct scoutfs_item *item; + DECLARE_SCOUTFS_ITEM_REF(ref); struct dentry_info *di; struct scoutfs_key key; int bytes; int ret; - int off; u64 nr; u64 h; @@ -356,60 +351,32 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); bytes = dent_bytes(dentry->d_name.len); - item = scoutfs_item_lookup(sb, &key); - if (item == ERR_PTR(-ENOENT)) { - item = scoutfs_item_create(sb, &key, bytes); - if (!IS_ERR(item)) { - /* mark a newly created item */ - dent = item->val; - dent->name_len = 0; + ret = scoutfs_read_item(sb, &key, &ref); + if (ret != -ENOENT) { + /* XXX implement many hashes, not coll nr */ + if (WARN_ON_ONCE(!ret)) { + scoutfs_put_ref(&ref); + ret = -ENOSPC; } - } - if (IS_ERR(item)) { - ret = PTR_ERR(item); goto out; } - ret = 0; - nr = 0; - for_each_item_dent(item, dent) { - /* the common case of a newly created item */ - if (!dent->name_len) - break; - - /* XXX check for eexist? can't happen? */ - - /* found a free coll nr, insert here */ - if (nr < dent->coll_nr) { - off = dent_val_off(item, dent); - ret = scoutfs_item_expand(item, off, bytes); - if (!ret) - dent = item->val + off; - break; - } - - /* the item's full */ - if (nr++ == SCOUTFS_DIRENT_COLL_MASK) { - ret = -ENOSPC; - break; - } - } - - if (!ret) { - dent->ino = cpu_to_le64(scoutfs_ino(inode)); - dent->type = mode_to_type(inode->i_mode); - dent->coll_nr = nr; - dent->name_len = dentry->d_name.len; - memcpy(dent->name, dentry->d_name.name, dent->name_len); - di->key_offset = h; - di->coll_nr = nr; - } - - scoutfs_item_put(item); - + ret = scoutfs_create_item(sb, &key, bytes, &ref); if (ret) goto out; + dent = ref.val; + nr = 0; + dent->ino = cpu_to_le64(scoutfs_ino(inode)); + dent->type = mode_to_type(inode->i_mode); + dent->coll_nr = nr; + dent->name_len = dentry->d_name.len; + memcpy(dent->name, dentry->d_name.name, dent->name_len); + di->key_offset = h; + di->coll_nr = nr; + + scoutfs_put_ref(&ref); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; @@ -452,8 +419,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_dirent *dent; - struct scoutfs_item *item; + DECLARE_SCOUTFS_ITEM_REF(ref); struct dentry_info *di; struct scoutfs_key key; int ret = 0; @@ -471,33 +437,12 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->key_offset); - item = scoutfs_item_lookup(sb, &key); - if (IS_ERR(item)) { - ret = PTR_ERR(item); + ret = scoutfs_read_item(sb, &key, &ref); + if (ret) goto out; - } - - /* XXX error to not find the coll nr we were looking for? */ - for_each_item_dent(item, dent) { - if (dent->coll_nr != di->coll_nr) - continue; - - /* XXX compare names and eio? */ - - if (item->val_len == dent_bytes(dent->name_len)) { - scoutfs_item_delete(sb, item); - ret = 0; - } else { - ret = scoutfs_item_shrink(item, - dent_val_off(item, dent), - dent_bytes(dent->name_len)); - } - dent = NULL; - break; - } - - scoutfs_item_put(item); + ret = scoutfs_delete_item(sb, &ref); + scoutfs_put_ref(&ref); if (ret) goto out; diff --git a/kmod/src/format.h b/kmod/src/format.h index bafaef80..1310f8d7 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -13,6 +13,7 @@ */ #define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) /* * The allocator works on larger chunks. Smaller metadata structures @@ -34,6 +35,19 @@ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 +/* + * 7 bits in a ~76k bloom filter gives ~1% false positive for our max + * of 64k items. + * + * n = 65,536, p = 0.01 (1 in 100) → m = 628,167 (76.68KB), k = 7 + */ +#define SCOUTFS_BLOOM_BITS 7 +#define SCOUTFS_BLOOM_BIT_WIDTH 20 /* 2^20 > m */ +#define SCOUTFS_BLOOM_BIT_MASK ((1 << SCOUTFS_BLOOM_BIT_WIDTH) - 1) +#define SCOUTFS_BLOOM_BLOCKS ((76 * 1024) / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_BLOOM_SALTS \ + DIV_ROUND_UP(SCOUTFS_BLOOM_BITS * SCOUTFS_BLOOM_BIT_WIDTH, 32) + /* * This header is found at the start of every block so that we can * verify that it's what we were looking for. The crc and padding @@ -64,6 +78,7 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; + __le32 bloom_salts[SCOUTFS_BLOOM_SALTS]; __le64 total_chunks; __le64 ring_map_blkno; __le64 ring_map_seq; @@ -149,22 +164,43 @@ struct scoutfs_ring_bitmap { __le64 bits[2]; } __packed; + +struct scoutfs_bloom_block { + struct scoutfs_block_header hdr; + __le64 bits[0]; +} __packed; + +#define SCOUTFS_BLOOM_BITS_PER_BLOCK \ + (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_block_header)) / 8) * 64) + /* - * To start the log segments are a trivial single item block. We'll - * flesh this out into larger blocks once the rest of the architecture - * is in place. + * Items in log segments are sorted in a skip list by their key. We + * have a rough limit of 64k items. + */ +#define SCOUTFS_SKIP_HEIGHT 16 +struct scoutfs_skip_root { + __le32 next[SCOUTFS_SKIP_HEIGHT]; +} __packed; + +/* + * An item block follows the bloom filter blocks at the start of a log + * segment. Its skip root references the item structs which then + * reference the item values in the rest of the block. The references + * are byte offsets from the start of the chunk. */ struct scoutfs_item_block { struct scoutfs_block_header hdr; struct scoutfs_key first; struct scoutfs_key last; - __le32 nr_items; - /* struct scoutfs_item_header items[0] .. */ + struct scoutfs_skip_root skip_root; } __packed; -struct scoutfs_item_header { +struct scoutfs_item { struct scoutfs_key key; + __le32 offset; __le16 len; + u8 skip_height; + __le32 skip_next[0]; } __packed; struct scoutfs_timespec { diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 521a1671..99187b2c 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -19,7 +19,7 @@ #include "super.h" #include "key.h" #include "inode.h" -#include "item.h" +#include "segment.h" #include "dir.h" /* @@ -110,17 +110,17 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) static int scoutfs_read_locked_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_item *item; + DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; + int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - item = scoutfs_item_lookup(sb, &key); - if (IS_ERR(item)) - return PTR_ERR(item); - - load_inode(inode, item->val); - scoutfs_item_put(item); + ret = scoutfs_read_item(sb, &key, &ref); + if (!ret) { + load_inode(inode, ref.val); + scoutfs_put_ref(&ref); + } return 0; } @@ -200,16 +200,17 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) void scoutfs_update_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_item *item; + DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; + int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - item = scoutfs_item_lookup(sb, &key); - BUG_ON(IS_ERR(item)); + ret = scoutfs_read_item(sb, &key, &ref); + BUG_ON(ret); - store_inode(item->val, inode); - scoutfs_item_put(item); + store_inode(ref.val, inode); + scoutfs_put_ref(&ref); } /* @@ -221,9 +222,10 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_inode_info *ci; - struct scoutfs_item *item; + DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; struct inode *inode; + int ret; inode = new_inode(sb); if (!inode) @@ -242,12 +244,14 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - item = scoutfs_item_create(inode->i_sb, &key, - sizeof(struct scoutfs_inode)); - if (IS_ERR(item)) { + ret = scoutfs_create_item(inode->i_sb, &key, + sizeof(struct scoutfs_inode), &ref); + if (ret) { iput(inode); - inode = ERR_CAST(item); + return ERR_PTR(ret); } + + scoutfs_put_ref(&ref); return inode; } diff --git a/kmod/src/item.c b/kmod/src/item.c deleted file mode 100644 index 1f253826..00000000 --- a/kmod/src/item.c +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Copyright (C) 2015 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "key.h" -#include "item.h" -#include "segment.h" - -/* - * describe: - * - tracks per-item dirty state for writing - * - decouples vfs cache lifetimes from item lifetimes - * - item-granular cache for things vfs doesn't cache (readdir, xattr) - * - * XXX: - * - warnings for invalid keys/lens - * - memory pressure - */ - -enum { - ITW_NEXT = 1, - ITW_PREV, -}; - -static inline struct scoutfs_item *node_item(struct super_block *sb, - struct rb_root *root, - struct rb_node *node) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - unsigned long off; - - if (root == &sbi->item_root) - off = offsetof(struct scoutfs_item, node); - else - off = offsetof(struct scoutfs_item, dirty_node); - - return (void *)((char *)node - off); -} - -static inline struct rb_node *item_node(struct super_block *sb, - struct rb_root *root, - struct scoutfs_item *item) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - unsigned long off; - - if (root == &sbi->item_root) - off = offsetof(struct scoutfs_item, node); - else - off = offsetof(struct scoutfs_item, dirty_node); - - return (void *)((char *)item + off); -} - -/* - * Insert a new item in the tree. The caller must have done a lookup to - * ensure that the key is not already present. - */ -static void insert_item(struct super_block *sb, struct rb_root *root, - struct scoutfs_item *ins) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct scoutfs_item *item; - int cmp; - - while (*node) { - parent = *node; - item = node_item(sb, root, *node); - - cmp = scoutfs_key_cmp(&ins->key, &item->key); - BUG_ON(cmp == 0); - if (cmp < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - rb_link_node(item_node(sb, root, ins), parent, node); - rb_insert_color(item_node(sb, root, ins), root); -} - -enum { - FI_NEXT = 1, - FI_PREV, -}; - -/* - * Walk the tree looking for an item. - * - * If NEXT or PREV are specified then those will be returned - * if the specific item isn't found. - */ -static struct scoutfs_item *find_item(struct super_block *sb, - struct rb_root *root, - struct scoutfs_key *key, int np) -{ - struct rb_node *node = root->rb_node; - struct scoutfs_item *found = NULL; - struct scoutfs_item *item; - int cmp; - - while (node) { - item = node_item(sb, root, node); - - cmp = scoutfs_key_cmp(key, &item->key); - if (cmp < 0) { - if (np == FI_NEXT) - found = item; - node = node->rb_left; - } else if (cmp > 0) { - if (np == FI_PREV) - found = item; - node = node->rb_right; - } else { - found = item; - break; - } - } - - return found; -} - -static struct scoutfs_item *alloc_item(struct scoutfs_key *key, - unsigned int val_len) -{ - struct scoutfs_item *item; - void *val; - - item = kmalloc(sizeof(struct scoutfs_item), GFP_NOFS); - val = kmalloc(val_len, GFP_NOFS); - if (!item || !val) { - kfree(item); - kfree(val); - return ERR_PTR(-ENOMEM); - } - - RB_CLEAR_NODE(&item->node); - RB_CLEAR_NODE(&item->dirty_node); - atomic_set(&item->refcount, 1); - item->key = *key; - item->val_len = val_len; - item->val = val; - - return item; -} - -static struct scoutfs_item *create_item(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len, bool dirty) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *existing; - struct scoutfs_item *item; - unsigned long flags; - - item = alloc_item(key, val_len); - if (IS_ERR(item)) - return item; - - spin_lock_irqsave(&sbi->item_lock, flags); - - existing = find_item(sb, &sbi->item_root, key, 0); - if (!existing) { - insert_item(sb, &sbi->item_root, item); - atomic_inc(&item->refcount); - if (dirty) { - insert_item(sb, &sbi->dirty_item_root, item); - atomic_inc(&item->refcount); - } - - } - spin_unlock_irqrestore(&sbi->item_lock, flags); - - if (existing) { - scoutfs_item_put(item); - item = ERR_PTR(-EEXIST); - } - - trace_printk("item %p key "CKF" val_len %d\n", item, CKA(key), val_len); - - return item; -} - -/* - * Create a new item stored at the given key. Return it with a reference. - * return an ERR_PTR with ENOMEM or EEXIST. - * - * The caller is responsible for initializing the item's value. - */ -struct scoutfs_item *scoutfs_item_create(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len) -{ - return create_item(sb, key, val_len, true); -} - -/* - * Allocate a new clean item in the cache for the caller to fill. If the - * item already exists then -EEXIST is returned. - */ -struct scoutfs_item *scoutfs_clean_item(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len) -{ - return create_item(sb, key, val_len, false); -} - -/* - * The caller is still responsible for unlocking and putting the item. - * - * We don't try and optimize away the lock for items that are already - * removed from the tree. The caller's locking and item behaviour means - * that racing to remove an item is extremely rare. - * - * XXX for now we're just removing it from the rbtree. We'd need to leave - * behind a deletion record for lsm. - */ -void scoutfs_item_delete(struct super_block *sb, struct scoutfs_item *item) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - unsigned long flags; - - spin_lock_irqsave(&sbi->item_lock, flags); - - if (!RB_EMPTY_NODE(&item->dirty_node)) { - rb_erase(&item->dirty_node, &sbi->dirty_item_root); - RB_CLEAR_NODE(&item->dirty_node); - scoutfs_item_put(item); - } - - if (!RB_EMPTY_NODE(&item->node)) { - rb_erase(&item->node, &sbi->item_root); - RB_CLEAR_NODE(&item->node); - scoutfs_item_put(item); - } - - spin_unlock_irqrestore(&sbi->item_lock, flags); -} - -/* - * Find an item in the cache. If it isn't present then we try to read - * it from log segements. - */ -static struct scoutfs_item *item_lookup(struct super_block *sb, - struct scoutfs_key *key, int np) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *item; - unsigned long flags; - unsigned retried = 0; - int ret; - - do { - spin_lock_irqsave(&sbi->item_lock, flags); - - item = find_item(sb, &sbi->item_root, key, np); - if (item) - atomic_inc(&item->refcount); - - spin_unlock_irqrestore(&sbi->item_lock, flags); - if (!item) { - if (np == FI_NEXT) - ret = scoutfs_read_next_item(sb, key); - else - ret = scoutfs_read_item(sb, key); - if (ret) - item = ERR_PTR(ret); - } - } while (!item && !retried++); - - if (!item) - item = ERR_PTR(-ENOENT); - - return item; -} - -struct scoutfs_item *scoutfs_item_lookup(struct super_block *sb, - struct scoutfs_key *key) -{ - return item_lookup(sb, key, 0); -} - -struct scoutfs_item *scoutfs_item_next(struct super_block *sb, - struct scoutfs_key *key) -{ - return item_lookup(sb, key, FI_NEXT); -} - -struct scoutfs_item *scoutfs_item_prev(struct super_block *sb, - struct scoutfs_key *key) -{ - return item_lookup(sb, key, FI_PREV); -} - -/* - * Expand the item's value by inserting bytes at the given offset. The - * new bytes are not initialized. - */ -int scoutfs_item_expand(struct scoutfs_item *item, int off, int bytes) -{ - void *val; - - /* XXX bytes too big */ - if (WARN_ON_ONCE(off < 0 || off > item->val_len)) - return -EINVAL; - - val = kmalloc(item->val_len + bytes, GFP_NOFS); - if (!val) - return -ENOMEM; - - memcpy(val, item->val, off); - memcpy(val + off + bytes, item->val + off, item->val_len - off); - - kfree(item->val); - item->val = val; - item->val_len += bytes; - - return 0; -} - -/* - * Shrink the item's value by remove bytes at the given offset. - */ -int scoutfs_item_shrink(struct scoutfs_item *item, int off, int bytes) -{ - void *val; - - if (WARN_ON_ONCE(off < 0 || off >= item->val_len || - bytes <= 0 || (off + bytes) > item->val_len || - bytes == item->val_len)) - return -EINVAL; - - val = kmalloc(item->val_len - bytes, GFP_NOFS); - if (!val) - return -ENOMEM; - - memcpy(val, item->val, off); - memcpy(val + off, item->val + off + bytes, - item->val_len - (off + bytes)); - - kfree(item->val); - item->val = val; - item->val_len -= bytes; - - return 0; -} - -void scoutfs_item_mark_dirty(struct super_block *sb, struct scoutfs_item *item) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - unsigned long flags; - - spin_lock_irqsave(&sbi->item_lock, flags); - - if (RB_EMPTY_NODE(&item->dirty_node)) { - insert_item(sb, &sbi->dirty_item_root, item); - atomic_inc(&item->refcount); - } - - spin_unlock_irqrestore(&sbi->item_lock, flags); -} - -/* - * Mark all the dirty items clean by emptying the dirty rbtree. The - * caller should be preventing writes from dirtying new items. - * - * We erase leaf nodes with no children to minimize rotation - * overhead during erase. Dirty items must be in the main rbtree if - * they're in the dirty rbtree so the puts here shouldn't free the - * items. - */ -void scoutfs_item_all_clean(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct rb_root *root = &sbi->dirty_item_root; - struct scoutfs_item *item; - struct rb_node *node; - unsigned long flags; - - spin_lock_irqsave(&sbi->item_lock, flags); - - node = sbi->dirty_item_root.rb_node; - while (node) { - if (node->rb_left) - node = node->rb_left; - else if (node->rb_right) - node = node->rb_right; - else { - item = node_item(sb, root, node); - node = rb_parent(node); - - trace_printk("item %p key "CKF"\n", - item, CKA(&item->key)); - rb_erase(&item->dirty_node, root); - RB_CLEAR_NODE(&item->dirty_node); - scoutfs_item_put(item); - } - } - - spin_unlock_irqrestore(&sbi->item_lock, flags); -} - -/* - * If the item is null then the first dirty item is returned. If an - * item is given then the next dirty item is returned. NULL is returned - * if there are no more dirty items. - * - * The caller is given a reference that it has to put. The given item - * will always have its item dropped including if it returns NULL. - */ -struct scoutfs_item *scoutfs_item_next_dirty(struct super_block *sb, - struct scoutfs_item *item) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *next_item; - struct rb_node *node; - unsigned long flags; - - spin_lock_irqsave(&sbi->item_lock, flags); - - if (item) - node = rb_next(&item->dirty_node); - else - node = rb_first(&sbi->dirty_item_root); - - if (node) { - next_item = node_item(sb, &sbi->dirty_item_root, node); - atomic_inc(&next_item->refcount); - } else { - next_item = NULL; - } - - spin_unlock_irqrestore(&sbi->item_lock, flags); - - scoutfs_item_put(item); - - return next_item; -} - -void scoutfs_item_put(struct scoutfs_item *item) -{ - if (!IS_ERR_OR_NULL(item) && atomic_dec_and_test(&item->refcount)) { - WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); - WARN_ON_ONCE(!RB_EMPTY_NODE(&item->dirty_node)); - kfree(item); - } -} diff --git a/kmod/src/item.h b/kmod/src/item.h deleted file mode 100644 index b8225a20..00000000 --- a/kmod/src/item.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef _SCOUTFS_ITEM_H_ -#define _SCOUTFS_ITEM_H_ - -#include "format.h" - -struct scoutfs_item { - struct rb_node node; - struct rb_node dirty_node; - atomic_t refcount; - - /* the key is constant for the life of the item */ - struct scoutfs_key key; - - /* the value can be changed by expansion or shrinking */ - unsigned int val_len; - void *val; -}; - -struct scoutfs_item *scoutfs_item_create(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len); -struct scoutfs_item *scoutfs_clean_item(struct super_block *sb, - struct scoutfs_key *key, - unsigned int val_len); -struct scoutfs_item *scoutfs_item_lookup(struct super_block *sb, - struct scoutfs_key *key); -struct scoutfs_item *scoutfs_item_next(struct super_block *sb, - struct scoutfs_key *key); -struct scoutfs_item *scoutfs_item_prev(struct super_block *sb, - struct scoutfs_key *key); -int scoutfs_item_expand(struct scoutfs_item *item, int off, int bytes); -int scoutfs_item_shrink(struct scoutfs_item *item, int off, int bytes); -void scoutfs_item_delete(struct super_block *sb, struct scoutfs_item *item); -void scoutfs_item_mark_dirty(struct super_block *sb, struct scoutfs_item *item); -struct scoutfs_item *scoutfs_item_next_dirty(struct super_block *sb, - struct scoutfs_item *item); -void scoutfs_item_all_clean(struct super_block *sb); -void scoutfs_item_put(struct scoutfs_item *item); - -#endif diff --git a/kmod/src/key.h b/kmod/src/key.h index 80e60668..c06f898c 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -32,27 +32,28 @@ static inline int scoutfs_key_cmp(struct scoutfs_key *a, struct scoutfs_key *b) } /* - * return -ve, 0, +ve if the key is less than, contained within, or greater - * than the given range of keys. + * return -ve if the first range is completely before the second, +ve for + * completely after, and 0 if they intersect. */ -static inline int scoutfs_key_cmp_range(struct scoutfs_key *key, +static inline int scoutfs_cmp_key_ranges(struct scoutfs_key *a_first, + struct scoutfs_key *a_last, + struct scoutfs_key *b_first, + struct scoutfs_key *b_last) +{ + if (scoutfs_key_cmp(a_last, b_first) < 0) + return -1; + if (scoutfs_key_cmp(a_first, b_last) > 0) + return 1; + return 0; +} + +static inline int scoutfs_cmp_key_range(struct scoutfs_key *key, struct scoutfs_key *first, struct scoutfs_key *last) { - int cmp; - - WARN_ON_ONCE(scoutfs_key_cmp(first, last) > 0); - - cmp = scoutfs_key_cmp(key, first); - if (cmp > 0) { - cmp = scoutfs_key_cmp(key, last); - if (cmp < 0) - cmp = 0; - } - return cmp; + return scoutfs_cmp_key_ranges(key, key, first, last); } - static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, u64 offset) { diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 8666ac49..e1d06e12 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -85,7 +85,7 @@ static struct scoutfs_manifest_node *find_mnode(struct rb_root *root, while (node) { mnode = rb_entry(node, struct scoutfs_manifest_node, node); - cmp = scoutfs_key_cmp_range(key, &mnode->ment.first, + cmp = scoutfs_cmp_key_range(key, &mnode->ment.first, &mnode->ment.last); if (cmp < 0) node = node->rb_left; @@ -213,15 +213,24 @@ int scoutfs_new_manifest(struct super_block *sb, /* * Fill the caller's ment with the next log segment in the manifest that - * might contain the given key. The ment is initialized to 0 to return - * the first entry. + * might contain the given range. The caller initializes the ment to + * zeros to find the first log segment. * * This can return multiple log segments from level 0 in decreasing age. * Then it can return at most one log segment in each level that - * intersects with the given key. + * intersects the given range. + * + * Returns true if an entry was found and is now described in ment, + * false when there are no more segments that contain the range. + * + * XXX could use the l0 seq to walk the list and skipb locks we've + * already seen. I'm not sure that we'll be able to keep manifest + * entries pinned while we're away blocking. We might fail to find the + * last entry's block in the radix when we return. */ -bool scoutfs_next_manifest_segment(struct super_block *sb, - struct scoutfs_key *key, +bool scoutfs_foreach_range_segment(struct super_block *sb, + struct scoutfs_key *first, + struct scoutfs_key *last, struct scoutfs_ring_manifest_entry *ment) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -247,8 +256,9 @@ bool scoutfs_next_manifest_segment(struct super_block *sb, } list_for_each_entry_from(mnode, &mani->level_zero, head) { - if (scoutfs_key_cmp_range(key, &mnode->ment.first, - &mnode->ment.last) == 0) { + if (scoutfs_cmp_key_ranges(first, last, + &mnode->ment.first, + &mnode->ment.last) == 0) { *ment = mnode->ment; found = true; break; @@ -257,8 +267,15 @@ bool scoutfs_next_manifest_segment(struct super_block *sb, } if (!found) { + /* + * The log segments in the each level fully cover the + * key range and don't overlap. So we will always find + * a segment that matches whatever key we look for. We + * look for the start of the range because iterators are + * walk the keyspace sequentially. + */ for (i = ment->level + 1; i <= SCOUTFS_MAX_LEVEL; i++) { - mnode = find_mnode(&mani->levels[i].root, key); + mnode = find_mnode(&mani->levels[i].root, first); if (mnode) { *ment = mnode->ment; found = true; diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 407bfa28..bab32764 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -10,8 +10,9 @@ int scoutfs_new_manifest(struct super_block *sb, struct scoutfs_ring_manifest_entry *ment); void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); -bool scoutfs_next_manifest_segment(struct super_block *sb, - struct scoutfs_key *key, +bool scoutfs_foreach_range_segment(struct super_block *sb, + struct scoutfs_key *first, + struct scoutfs_key *last, struct scoutfs_ring_manifest_entry *ment); #endif diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 095c30b2..19642e21 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -18,7 +18,6 @@ #include "dir.h" #include "inode.h" #include "key.h" -#include "item.h" #include "super.h" #include "manifest.h" #include "chunk.h" @@ -113,14 +112,14 @@ static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) /* * Return a dirty locked logical ring block. */ -static struct buffer_head *dirty_ring_block(struct super_block *sb, u64 block) +static struct buffer_head *new_ring_block(struct super_block *sb, u64 block) { u64 blkno = map_ring_block(sb, block); if (!blkno) return NULL; - return scoutfs_dirty_block(sb, blkno); + return scoutfs_new_block(sb, blkno); } int scoutfs_replay_ring(struct super_block *sb) @@ -186,7 +185,7 @@ int scoutfs_dirty_ring_entry(struct super_block *sb, u8 type, void *data, if (block >= le64_to_cpu(super->ring_total_blocks)) block -= le64_to_cpu(super->ring_total_blocks); - bh = dirty_ring_block(sb, block); + bh = new_ring_block(sb, block); if (!bh) { ret = -ENOMEM; goto out; @@ -242,6 +241,7 @@ int scoutfs_finish_dirty_ring(struct super_block *sb) * the block without walking all the items. */ scoutfs_calc_hdr_crc(bh); + mark_buffer_dirty(bh); unlock_buffer(bh); brelse(bh); diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 0c3d1adb..9ea41977 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -19,321 +19,611 @@ #include "super.h" #include "key.h" -#include "item.h" #include "segment.h" #include "manifest.h" #include "block.h" #include "chunk.h" #include "ring.h" +#include "bloom.h" +#include "skip.h" -static struct scoutfs_item_header *next_ihdr(struct scoutfs_item_header *ihdr) + +/* + * scoutfs log segments are large multi-block structures that contain + * key/value items. This file implements manipulations of the items. + * + * Each log segment starts with a bloom filter to supports quickly + * testing for key values without having to search the whole block for a + * key. + * + * After the bloom filter come the packed structures that describe the + * items that are present in the block. They're sorted in a skip list + * to support reasonably efficient insertion, sorted iteration, and + * deletion. + * + * Finally the item values are stored at the end of the block. This + * supports finding that an item's key isn't present by only reading the + * item structs, not the values. + * + * All told, should we chose to, we can have three large portions of the + * blocks resident for searching. It's likely that we'll keep the bloom + * filters hot but that the items and especially the values may age out + * of the cache. + */ + +void scoutfs_put_ref(struct scoutfs_item_ref *ref) { - return (void *)(ihdr + 1) + le16_to_cpu(ihdr->len); + if (ref->item_bh) + brelse(ref->item_bh); + if (ref->val_bh) + brelse(ref->val_bh); + + memset(ref, 0, sizeof(struct scoutfs_item_ref)); +} + +/* private to here */ +struct scoutfs_item_iter { + struct list_head list; + struct buffer_head *bh; + struct scoutfs_item *item; + u64 blkno; + bool restart_after; +}; + +void scoutfs_put_iter_list(struct list_head *list) +{ + struct scoutfs_item_iter *iter; + struct scoutfs_item_iter *pos; + + list_for_each_entry_safe(iter, pos, list, list) { + list_del_init(&iter->list); + brelse(iter->bh); + kfree(iter); + } } /* - * Use the manifest to search log segments for the most recent version - * of the item with the given key. This only returns an error if it - * fails to determine if the item exists or not. It's up to the caller - * to retry the lookup after success. + * The caller has a pointer to an item and a reference to its block. We + * read the value block and populate the reference. + * + * The item references get their own buffer head references so that the + * caller doesn't have to play funny games. They always have to drop + * their release bh. If this succeeds then they also need to put the + * ref. */ -int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key) +static int populate_ref(struct super_block *sb, u64 blkno, + struct buffer_head *item_bh, struct scoutfs_item *item, + struct scoutfs_item_ref *ref) { - struct scoutfs_ring_manifest_entry ment; - struct scoutfs_item_header *ihdr; - struct scoutfs_item_block *iblk; - struct scoutfs_item *item = NULL; struct buffer_head *bh; - int ret = 0; - int cmp; - int i; + + bh = scoutfs_read_block_off(sb, blkno, le32_to_cpu(item->offset)); + if (!bh) + return -EIO; + + ref->key = &item->key; + ref->val_len = le16_to_cpu(item->len); + ref->val = bh->b_data + (le32_to_cpu(item->offset) & + SCOUTFS_BLOCK_MASK); + get_bh(item_bh); + ref->item_bh = item_bh; + ref->val_bh = bh; + + return 0; +} + +/* + * Return a reference to the item at the given key. We walk the manifest + * to find blocks that might contain the key from most recent to oldest. + * To find the key in each log segment we test it's bloom filter and + * then search through the item keys. The first matching item we find + * is returned. + * + * XXX lock the dirty log segment? + * + * -ENOENT is returned if the item isn't present. The caller needs to put + * the ref if we return success. + */ +int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_item_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_item *item = NULL; + struct scoutfs_bloom_bits bits; + struct buffer_head *bh; + int ret; /* XXX hold manifest */ + scoutfs_calc_bloom_bits(&bits, key, sbi->super.bloom_salts); + + item = NULL; + ret = -ENOENT; memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); + while (scoutfs_foreach_range_segment(sb, key, key, &ment)) { - while (scoutfs_next_manifest_segment(sb, key, &ment)) { + /* XXX read-ahead all bloom blocks */ - bh = scoutfs_read_block(sb, le64_to_cpu(ment.blkno)); - if (!bh) { - ret = -EIO; + ret = scoutfs_test_bloom_bits(sb, le64_to_cpu(ment.blkno), + &bits); + if (ret < 0) break; + if (!ret) { + ret = -ENOENT; + continue; } - iblk = (void *)bh->b_data; - /* XXX seq corruption */ + /* XXX read-ahead all item header blocks */ - ihdr = (void *)(iblk + 1); - - /* XXX test bloom filter blocks */ - /* XXX binary search of key array */ - /* XXX could populate more from granted range */ - - for (i = 0; i < le32_to_cpu(iblk->nr_items); - i++, ihdr = next_ihdr(ihdr)) { - cmp = scoutfs_key_cmp(key, &ihdr->key); - if (cmp > 0) + ret = scoutfs_skip_lookup(sb, le64_to_cpu(ment.blkno), key, + &bh, &item); + if (ret) { + if (ret == -ENOENT) continue; - if (cmp < 0) - break; - - item = scoutfs_clean_item(sb, key, - le16_to_cpu(ihdr->len)); - if (IS_ERR(item)) { - ret = PTR_ERR(item); - } else { - memcpy(item->val, (void *)(ihdr + 1), - item->val_len); - } break; } - - brelse(bh); - if (item) /* also breaks for IS_ERR */ - break; + break; } /* XXX release manifest */ - scoutfs_item_put(item); + /* XXX read-ahead all value blocks? */ + + if (!ret) { + ret = populate_ref(sb, le64_to_cpu(ment.blkno), bh, item, ref); + brelse(bh); + } + return ret; } /* - * Reading the next item is more expensive than looking up a specific - * item. We can't use the bloom filters because we don't know what key - * is next. We have to search blocks at all levels because the next - * item could be in any of them. - * - * After having gone to the trouble to establish next item positions in - * all the blocks we take the opportunity to amortize that cost and - * insert multiple items. - * - * This only returns an error if it was unsure if there's a next item - * or not. It will return success if there were no next items. The caller - * is responsible for retrying the lookup after reading. + * The dirty_item_off points to the byte offset after the last item. + * Advance it past block tails and initial block headers until there's + * room for an item with the given skip list elements height. Then set + * the dirty_item_off past the item offset item we return. */ -struct item_block_cursor { - struct list_head list; - - struct buffer_head *bh; - struct scoutfs_item_header *ihdr; - unsigned int i; -}; -int scoutfs_read_next_item(struct super_block *sb, - struct scoutfs_key *first_key) +static int add_item_off(struct scoutfs_sb_info *sbi, int height) { - struct scoutfs_ring_manifest_entry ment; - struct scoutfs_item_header *least; - struct scoutfs_item_header *ihdr; + int len = offsetof(struct scoutfs_item, skip_next[height]); + int off = sbi->dirty_item_off; + int tail_free; + + /* item's can't cross a block boundary */ + tail_free = SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK); + if (tail_free < len) + off += tail_free + sizeof(struct scoutfs_block_header); + + sbi->dirty_item_off = off + len; + return off; +} + +/* + * The dirty_val_off points to the first byte of the last value that + * was allocated. Subtract the offset to make room for a new item + * of the given length. If that crosses a block boundary or wanders + * into the block header then pull it back into the tail of the previous + * block. + */ +static int sub_val_off(struct scoutfs_sb_info *sbi, int len) +{ + int off = sbi->dirty_val_off - len; + int block_off; + int tail_free; + + /* values can't start in a block header */ + block_off = off & SCOUTFS_BLOCK_MASK; + if (block_off < sizeof(struct scoutfs_block_header)) + off -= (block_off + 1); + + /* values can't cross a block boundary */ + tail_free = SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK); + if (tail_free < len) + off -= len - tail_free; + + sbi->dirty_val_off = off; + return off; +} + +/* + * Initialize the buffers for the next dirty segment. We have to initialize + * the bloom filter bits and the item block header. + * + * XXX we need to really pin the blocks somehow + */ +static int start_dirty_segment(struct super_block *sb, u64 blkno) +{ + struct scoutfs_bloom_block *blm; struct scoutfs_item_block *iblk; - struct item_block_cursor *curs; - struct item_block_cursor *tmp; - struct scoutfs_item *item; - struct scoutfs_key key; struct buffer_head *bh; - LIST_HEAD(cursors); int ret = 0; - int pass; int i; - /* XXX hold manifest */ - - memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); - - /* find all the log segments that contain our key */ - key = *first_key; - while (scoutfs_next_manifest_segment(sb, &key, &ment)) { - - curs = kmalloc(sizeof(struct item_block_cursor), GFP_NOFS); - if (!curs) { - ret = -ENOMEM; - goto out; - } - - bh = scoutfs_read_block(sb, le64_to_cpu(ment.blkno)); + for (i = 0; i < SCOUTFS_BLOCKS_PER_CHUNK; i++) { + bh = scoutfs_new_block(sb, blkno + i); if (!bh) { ret = -EIO; - goto out; + break; } - /* XXX verify */ - iblk = (void *)bh->b_data; + if (i < SCOUTFS_BLOOM_BLOCKS) { + blm = (void *)bh->b_data; + memset(blm->bits, 0, SCOUTFS_BLOCK_SIZE - + offsetof(struct scoutfs_bloom_block, bits)); + } - curs->bh = bh; - curs->i = 0; - curs->ihdr = (void *)(iblk + 1); - list_add_tail(&curs->list, &cursors); + if (i == SCOUTFS_BLOOM_BLOCKS) { + iblk = (void *)bh->b_data; + /* also zero first unused item slot */ + memset(&iblk->skip_root, 0, sizeof(iblk->skip_root) + + sizeof(struct scoutfs_item)); + } + + /* bh is pinned by sbi->dirty_blkno */ } - /* there can be no segments that contain the item */ - if (list_empty(&cursors)) { - ret = 0; + while (ret && i--) { + /* unwind pinned blocks on failure */ + bh = sb_getblk(sb, blkno + i); + if (bh) { + brelse(bh); + brelse(bh); + } + } + + return ret; +} + +/* + * Zero the portion of this block that intersects with the free space in + * the middle of the segment. @start and @end are chunk-relative byte + * offsets of the inclusive start and exclusive end of the free region. + */ +static void zero_unused_block(struct super_block *sb, struct buffer_head *bh, + u32 start, u32 end) +{ + u32 off = bh->b_blocknr << SCOUTFS_BLOCK_SHIFT; + + /* see if the segment range falls outside our block */ + if (start >= off + SCOUTFS_BLOCK_SIZE || end <= off) + return; + + /* convert the chunk offsets to our block offsets */ + start = max(start, off) - off; + end = min(off + SCOUTFS_BLOCK_SIZE, end) - off; + + /* don't zero block headers */ + start = max_t(u32, start, sizeof(struct scoutfs_block_header)); + end = max_t(u32, start, sizeof(struct scoutfs_block_header)); + + if (start < end) + memset(bh->b_data + start, 0, end - start); +} + +/* + * Finish off a dirty segment if we have one. Calculate the checksums of + * all the blocks, mark them dirty, and drop their pinned reference. + */ +int scoutfs_finish_dirty_segment(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + struct buffer_head *bh; + u64 blkno; + int ret = 0; + u64 i; + + /* XXX sync doesn't lock this test? */ + blkno = sbi->dirty_blkno; + if (!blkno) + return 0; + + for (i = 0; i < SCOUTFS_BLOCKS_PER_CHUNK; i++) { + bh = scoutfs_read_block(sb, blkno + i); + /* should have been pinned */ + if (WARN_ON_ONCE(!bh)) { + ret = -EIO; + break; + } + + zero_unused_block(sb, bh, sbi->dirty_item_off, + sbi->dirty_val_off); + + scoutfs_calc_hdr_crc(bh); + mark_buffer_dirty(bh); + brelse(bh); + /* extra release to unpin */ + brelse(bh); + } + + /* + * XXX the manifest entry for this log segment has a key range + * that is much too large. We should shrink it here to reflect + * the real keys. That would reduce the number of blocks involved + * in merging it into level 1. + */ + + /* + * Try to kick off a background write of the finished segment. Callers + * can wait for the buffers in writeback if they need to. + */ + if (!ret) { + filemap_fdatawrite_range(mapping, blkno << SCOUTFS_CHUNK_SHIFT, + ((blkno + 1) << SCOUTFS_CHUNK_SHIFT) - 1); + sbi->dirty_blkno = 0; + } + + return ret; +} + +/* + * Return a reference to a newly allocated and initialized item in a + * block in the currently dirty log segment. + * + * Item creation is purposely kept very simple. Item and value offset + * allocation proceed from either end of the log segment. Once they + * intersect the log segment is full and written out. Deleted dirty + * items don't reclaim their space. The free space will be reclaimed by + * the level 0 -> level 1 merge that happens anyway. Not reclaiming + * free space makes item location more rigid and lets us relax the + * locking requirements of item references. An item reference doesn't + * have to worry about unrelated item modification moving their item + * around to, say, defragment free space. + */ +int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, + unsigned bytes, struct scoutfs_item_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_bloom_bits bits; + struct scoutfs_item *item; + struct buffer_head *bh; + int item_off; + int val_off; + int height; + u64 blkno; + int ret = 0; + + /* XXX how big should items really get? */ + if (WARN_ON_ONCE(bytes == 0 || bytes > 4096)) + return -EINVAL; + + height = scoutfs_skip_random_height(); + + mutex_lock(&sbi->dirty_mutex); + +next_chunk: + if (!sbi->dirty_blkno) { + ret = scoutfs_alloc_chunk(sb, &blkno); + if (ret) + goto out; + + /* XXX free blkno on error? */ + ret = start_dirty_segment(sb, blkno); + if (ret) + goto out; + + /* + * We need a local manifest in memory to find items as + * we insert them in the dirty segment. We don't know + * what keys are going to be used so we cover the whole + * thing. + * + * XXX But we're also adding it to the ring here. We should + * add it as its finalized and its item range is collapsed. + */ + ment.blkno = cpu_to_le64(blkno); + ment.seq = sbi->super.hdr.seq; + ment.level = 0; + memset(&ment.first, 0, sizeof(ment.first)); + memset(&ment.last, ~0, sizeof(ment.last)); + ret = scoutfs_new_manifest(sb, &ment); + if (ret) + goto out; + + sbi->dirty_blkno = blkno; + sbi->dirty_item_off = + (SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + + sizeof(struct scoutfs_item_block); + sbi->dirty_val_off = SCOUTFS_CHUNK_SIZE; + } + + item_off = add_item_off(sbi, height); + val_off = sub_val_off(sbi, bytes); + + if (item_off > val_off) { + ret = scoutfs_finish_dirty_segment(sb); + if (ret) + goto out; + goto next_chunk; + } + + /* XXX fix up this error handling in general */ + + bh = scoutfs_read_block_off(sb, sbi->dirty_blkno, item_off); + if (!bh) { + ret = -EIO; goto out; } - /* XXX arbitrary number of next items to insert */ - for (pass = 0; pass < 16; pass++) { + /* populate iblk first and last? better than in manifest? */ - least = NULL; - list_for_each_entry(curs, &cursors, list) { - iblk = (void *)curs->bh->b_data; - ihdr = curs->ihdr; - i = curs->i; + item = (void *)bh->b_data + (item_off & SCOUTFS_BLOCK_MASK); + item->key = *key; + item->offset = cpu_to_le32(val_off); + item->len = cpu_to_le16(bytes); + item->skip_height = height; - /* Find the next item past the search key. */ - for (; i < le32_to_cpu(iblk->nr_items); i++) { - if (scoutfs_key_cmp(&key, &ihdr->key) <= 0) - break; + ret = scoutfs_skip_insert(sb, sbi->dirty_blkno, item, item_off); + if (ret) + goto out; - ihdr = next_ihdr(ihdr); - } + ret = populate_ref(sb, sbi->dirty_blkno, bh, item, ref); + brelse(bh); + if (ret) + goto out; - /* - * If we fall off a block then we can't know if - * we have the least key without checking the - * next block at that level. It could have an - * item less than the least in our other blocks. - */ - if (WARN_ON_ONCE(i == le32_to_cpu(iblk->nr_items))) { - ret = -EIO; + /* XXX delete skip on failure? */ + + /* set the bloom bits last because we can't unset them */ + scoutfs_calc_bloom_bits(&bits, key, sbi->super.bloom_salts); + ret = scoutfs_set_bloom_bits(sb, sbi->dirty_blkno, &bits); +out: + WARN_ON_ONCE(ret); /* XXX error paths are not robust */ + mutex_unlock(&sbi->dirty_mutex); + return ret; +} + +/* + * This is a really cheesy temporary delete method. It only works on items + * that are stored in dirty blocks. The caller is responsible for dropping + * the ref. XXX be less bad. + */ +int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 blkno; + int ret; + + blkno = round_down(ref->item_bh->b_blocknr, SCOUTFS_BLOCKS_PER_CHUNK); + if (WARN_ON_ONCE(blkno != sbi->dirty_blkno)) + return -EINVAL; + + ret = scoutfs_skip_delete(sb, blkno, ref->key); + WARN_ON_ONCE(ret); + return ret; +} + +/* + * Return a reference to the next item in the inclusive search range. + * The caller should have access to the search key range. + * + * We walk the manifest to find all the log segments that could contain + * the start of the range. We hold cursors on the blocks in the + * segments. Each next item iteration comes from finding the least of + * the next item at all these cursors. + * + * If we exhaust a segment at a given level we may need to search the + * next segment in that level to find the next item. The manifest may + * have changed under us while we walked our old set of segments. So we + * restart the entire search to get another consistent collection of + * segments to search. + * + * We put the segment references and iteration cursors in a list in the + * caller so that they can find many next items by advancing the cursors + * without having to walk the manifest and perform initial binary + * searches in each segment. + * + * The caller is responsible for putting the item ref if we return + * success. -ENOENT is returned if there are no more items in the + * search range. + * + * XXX this is wonky. We don't want to search the manifest for the + * range, just the initial value. Then we record the last key in + * segments we finish and only restart if least is > that or there are + * no least. We have to advance the first key when restarting the + * search. + */ +int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, struct list_head *iter_list, + struct scoutfs_item_ref *ref) +{ + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_item_iter *least; + struct scoutfs_item_iter *iter; + struct scoutfs_item_iter *pos; + int ret; + +restart: + if (list_empty(iter_list)) { + + /* + * Find all the segments that intersect the search range + * and find the next item in the block from the start + * of the range. + */ + memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); + while (scoutfs_foreach_range_segment(sb, first, last, &ment)) { + iter = kzalloc(sizeof(struct scoutfs_item_iter), + GFP_NOFS); + if (!iter) { + ret = -ENOMEM; goto out; } /* - * Remember the newest least key in the blocks that's - * past the search key. + * We will restart the walk of the manifest blocks if + * we iterate over all the items in this block without + * exhausting the search range. */ - if (!least || - scoutfs_key_cmp(&ihdr->key, &least->key) < 0) - least = ihdr; + if (ment.level > 0 && + scoutfs_key_cmp(&ment.last, last) < 0) + iter->restart_after = true; - curs->ihdr = ihdr; - curs->i = i; + iter->blkno = le64_to_cpu(ment.blkno); + list_add_tail(&iter->list, iter_list); + } + if (list_empty(iter_list)) { + ret = -ENOENT; + goto out; + } + } + + least = NULL; + ret = 0; + list_for_each_entry_safe(iter, pos, iter_list, list) { + + /* search towards the first key if we haven't yet */ + if (!iter->item) { + ret = scoutfs_skip_search(sb, iter->blkno, first, + &iter->bh, &iter->item); } - /* start the next search past the next key */ - key = least->key; - scoutfs_inc_key(&key); + /* then iterate until we find or pass the first key */ + while (!ret && scoutfs_key_cmp(&iter->item->key, first) < 0) { + ret = scoutfs_skip_next(sb, iter->blkno, + &iter->bh, &iter->item); + } - /* insert the next item (XXX if it's not deleted) */ - item = scoutfs_clean_item(sb, &least->key, - le16_to_cpu(least->len)); - if (IS_ERR(item)) { - ret = PTR_ERR(item); - if (ret == -EEXIST) + /* we're done with this block if we past the last key */ + while (!ret && scoutfs_key_cmp(&iter->item->key, last) > 0) { + brelse(iter->bh); + iter->bh = NULL; + iter->item = NULL; + ret = -ENOENT; + } + + if (ret == -ENOENT) { + if (iter->restart_after) { + /* need next block at this level */ + scoutfs_put_iter_list(iter_list); + goto restart; + } else { + /* this level is done */ + list_del_init(&iter->list); + brelse(iter->bh); + kfree(iter); continue; - break; - } - - memcpy(item->val, (void *)(least + 1), item->val_len); - scoutfs_item_put(item); - } -out: - list_for_each_entry_safe(curs, tmp, &cursors, list) { - brelse(curs->bh); - list_del_init(&curs->list); - kfree(curs); - } - return ret; -} - -static int finish_item_block(struct super_block *sb, struct buffer_head *bh, - void *until) -{ - struct scoutfs_item_block *iblk = (void *)bh->b_data; - struct scoutfs_ring_manifest_entry ment; - - memset(until, 0, (void *)bh->b_data + SCOUTFS_BLOCK_SIZE - until); - scoutfs_calc_hdr_crc(bh); - unlock_buffer(bh); - brelse(bh); - - ment.blkno = cpu_to_le64(bh->b_blocknr); - ment.seq = iblk->hdr.seq; - ment.level = 0; - ment.first = iblk->first; - ment.last = iblk->last; - - return scoutfs_new_manifest(sb, &ment); -} - -/* - * Write all the currently dirty items in newly allocated log segments. - * New ring entries are added as the alloc bitmap is modified and as the - * manifest is updated. If we write out all the item and ring blocks then - * we write a new super that references those new blocks. - */ -int scoutfs_write_dirty_items(struct super_block *sb) -{ - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - struct scoutfs_item_header *ihdr; - struct scoutfs_item_block *iblk; - struct scoutfs_item *item; - struct buffer_head *bh; - int val_space; - u64 blkno; - int ret; - - /* XXX wait until transactions are complete */ - - item = NULL; - iblk = NULL; - while ((item = scoutfs_item_next_dirty(sb, item))) { - - if (iblk && (item->val_len > val_space)) { - iblk = NULL; - ret = finish_item_block(sb, bh, ihdr); - if (ret) - break; - } - - if (!iblk) { - /* get the next item block */ - ret = scoutfs_alloc_chunk(sb, &blkno); - if (ret) - break; - - bh = scoutfs_dirty_block(sb, blkno); - if (!bh) { - ret = -ENOMEM; - break; } - - iblk = (void *)bh->b_data; - iblk->first = item->key; - iblk->nr_items = 0; - ihdr = (void *)(iblk + 1); - /* XXX assuming that val_space is big enough */ } + if (ret) + goto out; - iblk->last = item->key; - ihdr->key = item->key; - ihdr->len = cpu_to_le16(item->val_len); - memcpy((void *)(ihdr + 1), item->val, item->val_len); - le32_add_cpu(&iblk->nr_items, 1); - - /* XXX assuming that the next ihdr fits */ - ihdr = (void *)(ihdr + 1) + le16_to_cpu(ihdr->len); - val_space = (char *)iblk + SCOUTFS_BLOCK_SIZE - - (char *)(ihdr + 1); + /* remember the most recent smallest key from the first */ + if (!least || + scoutfs_key_cmp(&iter->item->key, &least->item->key) < 0) + least = iter; } - scoutfs_item_put(item); /* only if the loop aborted */ - - /* finish writing if we did work and haven't failed */ - if (iblk && !ret) { - ret = finish_item_block(sb, bh, ihdr) ?: - scoutfs_finish_dirty_ring(sb) ?: - filemap_write_and_wait(mapping) ?: - scoutfs_write_dirty_super(sb); - if (!ret) { - scoutfs_advance_dirty_super(sb); - scoutfs_item_all_clean(sb); - } - } - - /* XXX better tear down down in the error case */ - + if (least) + ret = populate_ref(sb, least->blkno, least->bh, least->item, + ref); + else + ret = -ENOENT; +out: + if (ret) + scoutfs_put_iter_list(iter_list); return ret; + } diff --git a/kmod/src/segment.h b/kmod/src/segment.h index fd0fda69..6b41579b 100644 --- a/kmod/src/segment.h +++ b/kmod/src/segment.h @@ -1,9 +1,33 @@ #ifndef _SCOUTFS_SEGMENT_H_ #define _SCOUTFS_SEGMENT_H_ -int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key); -int scoutfs_read_next_item(struct super_block *sb, - struct scoutfs_key *first_key); -int scoutfs_write_dirty_items(struct super_block *sb); +struct scoutfs_item_ref { + /* usable by callers */ + struct scoutfs_key *key; + unsigned int val_len; + void *val; + + /* private buffer head refs */ + struct buffer_head *item_bh; + struct buffer_head *val_bh; +}; + +#define DECLARE_SCOUTFS_ITEM_REF(name) \ + struct scoutfs_item_ref name = {NULL ,} + +void scoutfs_put_ref(struct scoutfs_item_ref *ref); +void scoutfs_put_iter_list(struct list_head *list); + +int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_item_ref *ref); +int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, + unsigned bytes, struct scoutfs_item_ref *ref); +int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref); +int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, struct list_head *iter_list, + struct scoutfs_item_ref *ref); + +int scoutfs_finish_dirty_segment(struct super_block *sb); + #endif diff --git a/kmod/src/skip.c b/kmod/src/skip.c new file mode 100644 index 00000000..69bae2b8 --- /dev/null +++ b/kmod/src/skip.c @@ -0,0 +1,325 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include + +#include "format.h" +#include "key.h" +#include "block.h" +#include "skip.h" + +/* + * The items in a log segment block are sorted by their keys in a skip + * list. The skip list was chosen because it is so easy to implement + * and could, maybe some day, offer solid concurrent updates and reads. + * It also adds surprisingly little per-item overhead because half of + * the items only have one link. + * + * The list is rooted in the item block which follows the last bloom + * block in the segment. The links in the skip list elements are byte + * offsets of the start of items relative to the start of the log + * segment. + * + * We chose a limit on the height of 16 links. That gives around 64k + * items without going too crazy. That's around the higher end of the + * number of items we expect in log segments. + * + * This isn't quite a generic implementation. It knows that the items + * are rooted in the item block at a given offset in the log segment. + * It knows that the pointers are items and where the skip links are in + * its struct. It knows to compare the items by their key. + * + * The caller is completely responsible for serialization. + * + * The buffer_head reads here won't be as expensive as they might seem. + * The caller holds the blocks pinned so the worst case are block device + * page radix rcu lookups. Repeated reads of the recent blocks will hit + * the per-cpu lru bh reference caches. + */ + +struct skip_path { + struct buffer_head *root_bh; + + /* + * Pointers to the buffer heads which contain the blocks which are + * referenced by the next pointers in the path. + */ + struct buffer_head *bh[SCOUTFS_SKIP_HEIGHT]; + + /* + * Store the location of the index that references the item that + * we found. Insertion will modify the referenced index to add + * an entry before the item and deletion will modify the referenced + * index to remove the item. + */ + __le32 *next[SCOUTFS_SKIP_HEIGHT]; +}; + +#define DECLARE_SKIP_PATH(name) \ + struct skip_path name = {NULL, } + +/* + * Not all byte offsets are possible locations of items. Items have to + * be after the bloom blocks and item block header, can't be in + * the block headers for the rest of the blocks, and can't be a partial + * struct at the end of a block. + * + * This is just a rough check. It doesn't catch items offsets that overlap + * with other items or values. + */ +static int invalid_item_off(u32 off) +{ + return off < ((SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + + sizeof(struct scoutfs_item_block)) || + (off & SCOUTFS_BLOCK_MASK) < + sizeof(struct scoutfs_block_header) || + (off & SCOUTFS_BLOCK_MASK) > + (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_item)); +} + +/* + * Set the caller's item to the item in the segment at the given byte + * offset and set their bh to the block that contains it. + */ +static int skip_read_item(struct super_block *sb, u64 blkno, __le32 off, + struct buffer_head **bh, struct scoutfs_item **item) +{ + if (WARN_ON_ONCE(invalid_item_off(le32_to_cpu(off)))) + return -EINVAL; + + *bh = scoutfs_read_block_off(sb, blkno, le32_to_cpu(off)); + if (!(*bh)) { + *bh = NULL; + *item = NULL; + return -EIO; + } + + *item = (void *)(*bh)->b_data + (le32_to_cpu(off) & SCOUTFS_BLOCK_MASK); + return 0; +} + +/* + * Find the next item in the skiplist with a key greater than or equal + * to the given key. Set the path pointers to the hops before this item + * so that we can modify those pointers to insert an item before it in + * the list or delete it. + * + * The caller is responsible for initializing the path and cleaning it up. + */ +static int skip_search(struct super_block *sb, u64 blkno, + struct skip_path *path, struct scoutfs_key *key, + int *cmp) +{ + struct scoutfs_item_block *iblk; + struct scoutfs_item *item; + struct buffer_head *bh; + __le32 *next; + int ret = 0; + int i; + + /* fake lesser comparison for insertion into an empty list */ + *cmp = -1; + + bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); + if (!bh) + return -EIO; + + /* XXX verify */ + iblk = (void *)bh->b_data; + next = iblk->skip_root.next; + path->root_bh = bh; + + for (i = SCOUTFS_SKIP_HEIGHT - 1; i >= 0; i--) { + while (next[i]) { + ret = skip_read_item(sb, blkno, next[i], &bh, &item); + if (ret) + goto out; + + *cmp = scoutfs_key_cmp(key, &item->key); + if (*cmp <= 0) { + brelse(bh); + break; + } + + next = item->skip_next; + if (path->bh[i]) + brelse(path->bh[i]); + path->bh[i] = bh; + } + + path->next[i] = &next[i]; + } +out: + return ret; +} + +static void skip_release_path(struct skip_path *path) +{ + int i; + + if (path->root_bh) + brelse(path->root_bh); + + for (i = 0; i < SCOUTFS_SKIP_HEIGHT; i++) { + if (path->bh[i]) { + brelse(path->bh[i]); + path->bh[i] = NULL; + } + } +} + +/* + * We want heights with a distribution of 1 / (2^h). Half the items + * have a height of 1, a quarter have 2, an eighth have 3, etc. + * + * Finding the first low set bit in a random number achieves this + * nicely. ffs() even counts the bits from 1 so it matches our height. + * + * But ffs() returns 0 if no bits are set. We prevent a 0 height and + * limit the max height returned by oring in our max height. + */ +u8 scoutfs_skip_random_height(void) +{ + return ffs(get_random_int() | (1 << (SCOUTFS_SKIP_HEIGHT - 1))); +} + +/* + * Insert a new item in the item block's skip list. The caller provides + * an initialized item, particularly it's skip height and key, and + * the byte offset in the log segment of the item struct. + */ +int scoutfs_skip_insert(struct super_block *sb, u64 blkno, + struct scoutfs_item *item, u32 off) +{ + DECLARE_SKIP_PATH(path); + int cmp; + int ret; + int i; + + if (WARN_ON_ONCE(invalid_item_off(off)) || + WARN_ON_ONCE(item->skip_height > SCOUTFS_SKIP_HEIGHT)) + return -EINVAL; + + ret = skip_search(sb, blkno, &path, &item->key, &cmp); + if (ret == 0) { + if (cmp == 0) { + ret = -EEXIST; + } else { + for (i = 0; i < item->skip_height; i++) { + item->skip_next[i] = *path.next[i]; + *path.next[i] = cpu_to_le32(off); + } + } + } + + skip_release_path(&path); + return ret; +} + +static int skip_lookup(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct buffer_head **bh, + struct scoutfs_item **item, bool exact) +{ + DECLARE_SKIP_PATH(path); + int cmp; + int ret; + + ret = skip_search(sb, blkno, &path, key, &cmp); + if (ret == 0) { + if ((exact && cmp) || *path.next[0] == 0) { + ret = -ENOENT; + } else { + ret = skip_read_item(sb, blkno, *path.next[0], + bh, item); + } + } + + skip_release_path(&path); + return ret; +} + +/* + * Find the item at the given key in the skip list. + */ +int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct buffer_head **bh, + struct scoutfs_item **item) +{ + return skip_lookup(sb, blkno, key, bh, item, true); +} + +/* + * Find the next item after the given key in the skip list. + */ +int scoutfs_skip_search(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct buffer_head **bh, + struct scoutfs_item **item) +{ + return skip_lookup(sb, blkno, key, bh, item, false); +} + +int scoutfs_skip_delete(struct super_block *sb, u64 blkno, + struct scoutfs_key *key) +{ + struct scoutfs_item *item; + DECLARE_SKIP_PATH(path); + struct buffer_head *bh; + int cmp; + int ret; + int i; + + ret = skip_search(sb, blkno, &path, key, &cmp); + if (ret == 0) { + if (*path.next[0] && cmp) { + ret = -ENOENT; + } else { + ret = skip_read_item(sb, blkno, *path.next[0], + &bh, &item); + if (!ret) { + for (i = 0; i < item->skip_height; i++) + *path.next[i] = item->skip_next[i]; + brelse(bh); + } + } + } + + skip_release_path(&path); + return ret; +} + +/* + * The caller has found a valid item with search or lookup. We can use + * the lowest level links to advance through the rest of the items. The + * caller has made sure that this is safe. + */ +int scoutfs_skip_next(struct super_block *sb, u64 blkno, + struct buffer_head **bh, struct scoutfs_item **item) +{ + __le32 next; + + if (!(*bh)) + return -ENOENT; + + next = (*item)->skip_next[0]; + brelse(*bh); + + if (!next) { + *bh = NULL; + *item = NULL; + return -ENOENT; + } + + return skip_read_item(sb, blkno, next, bh, item); +} diff --git a/kmod/src/skip.h b/kmod/src/skip.h new file mode 100644 index 00000000..979719cc --- /dev/null +++ b/kmod/src/skip.h @@ -0,0 +1,18 @@ +#ifndef _SCOUTFS_SKIP_H_ +#define _SCOUTFS_SKIP_H_ + +u8 scoutfs_skip_random_height(void); +int scoutfs_skip_insert(struct super_block *sb, u64 blkno, + struct scoutfs_item *item, u32 off); +int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct buffer_head **bh, + struct scoutfs_item **item); +int scoutfs_skip_search(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct buffer_head **bh, + struct scoutfs_item **item); +int scoutfs_skip_delete(struct super_block *sb, u64 blkno, + struct scoutfs_key *key); +int scoutfs_skip_next(struct super_block *sb, u64 blkno, + struct buffer_head **bh, struct scoutfs_item **item); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 3bdfc743..ba200876 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -28,10 +28,23 @@ #include "ring.h" #include "segment.h" +/* + * We've been dirtying log segment blocks and ring blocks as items were + * modified. sync makes sure that they're all persistent and updates + * the super. + * + * XXX need to synchronize with transactions + * XXX is state clean after errors? + */ static int scoutfs_sync_fs(struct super_block *sb, int wait) { - /* XXX always waiting */ - return scoutfs_write_dirty_items(sb); + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + + return scoutfs_finish_dirty_segment(sb) ?: + scoutfs_finish_dirty_ring(sb) ?: + filemap_write_and_wait(mapping) ?: + scoutfs_write_dirty_super(sb) ?: + scoutfs_advance_dirty_super(sb); } static const struct super_operations scoutfs_super_ops = { @@ -45,7 +58,7 @@ static const struct super_operations scoutfs_super_ops = { * every time it wants to dirty it and eventually write it to reference * dirty data that's been written. */ -void scoutfs_advance_dirty_super(struct super_block *sb) +int scoutfs_advance_dirty_super(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; @@ -57,6 +70,8 @@ void scoutfs_advance_dirty_super(struct super_block *sb) super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO + blkno); le64_add_cpu(&super->hdr.seq, 1); + + return 0; } /* @@ -71,16 +86,16 @@ int scoutfs_write_dirty_super(struct super_block *sb) size_t sz; int ret; - bh = scoutfs_dirty_block(sb, le64_to_cpu(super->hdr.blkno)); + bh = scoutfs_new_block(sb, le64_to_cpu(super->hdr.blkno)); if (!bh) return -ENOMEM; sz = sizeof(struct scoutfs_super_block); memcpy(bh->b_data, super, sz); memset(bh->b_data + sz, 0, SCOUTFS_BLOCK_SIZE - sz); - scoutfs_calc_hdr_crc(bh); - unlock_buffer(bh); + scoutfs_calc_hdr_crc(bh); + mark_buffer_dirty(bh); ret = sync_dirty_buffer(bh); brelse(bh); @@ -170,6 +185,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->item_root = RB_ROOT; sbi->dirty_item_root = RB_ROOT; spin_lock_init(&sbi->chunk_alloc_lock); + mutex_init(&sbi->dirty_mutex); if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { printk(KERN_ERR "couldn't set blocksize\n"); @@ -209,9 +225,15 @@ static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, static void scoutfs_kill_sb(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + kill_block_super(sb); - scoutfs_destroy_manifest(sb); - kfree(sb->s_fs_info); + if (sbi) { + /* kill block super should have synced */ + WARN_ON_ONCE(sbi->dirty_blkno); + scoutfs_destroy_manifest(sb); + kfree(sbi); + } } static struct file_system_type scoutfs_fs_type = { diff --git a/kmod/src/super.h b/kmod/src/super.h index 604448ca..1663034d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -26,6 +26,11 @@ struct scoutfs_sb_info { struct scoutfs_ring_entry *dirty_ring_ent; unsigned int dirty_ring_ent_avail; + /* pinned log segment during fs modifications */ + struct mutex dirty_mutex; + u64 dirty_blkno; + int dirty_item_off; + int dirty_val_off; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) @@ -33,7 +38,7 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } -void scoutfs_advance_dirty_super(struct super_block *sb); +int scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); #endif From 3755adddd56d9f26d853697c9f2a46626090dd65 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Mar 2016 20:11:58 -0700 Subject: [PATCH 019/920] scoutfs: store dirents at multiple hash values Previously we dealt with colliding dirent hash values by storing all the dirents that share a hash value in a big item with multiple dirents. This complicated the code and strongly encouraged resizing items as dirents come and go. Resizing items isn't very easy with our simple log segment item creation mechanism. Instead let's deal with collisions by allowing a dirent to be stored at multiple hash values. The code is much simpler. Lookup has to iterate over all possible hash values. We can track the greatest hash iteration stored in the directory inode to limit the overhead of negative lookups in small directories. Signed-off-by: Zach Brown --- kmod/src/dir.c | 203 ++++++++++++++++++++-------------------------- kmod/src/format.h | 29 +++---- kmod/src/inode.c | 2 + kmod/src/inode.h | 1 + 4 files changed, 104 insertions(+), 131 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e0aac04d..50a39290 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -24,30 +24,32 @@ #include "super.h" /* - * Directory entries are stored in items whose offset is determined by - * the hash of the entry's name. This was primarily chosen to minimize - * the amount of data stored for each entry. + * Directory entries are stored in entries with offsets calculated from + * the hash of their entry name. * - * Because we're hashing the name we need to worry about collisions. We - * store all the entries with the same hash value in the item. This was - * done so that create works with one specific item. + * The upside of having a single namespace of items used for both lookup + * and readdir iteration reduces the storage overhead of directories. + * The downside is that dirent operations produce random item access + * patterns. * - * readdir iterates over these items in hash order. The high bits of - * the entry's readdir f_pos come from the item offset while the low - * bits come from a collision number in the entry. + * Hash values are limited to 31 bits to avoid bugs from use of 31 bit + * signed offsets. We also avoid bugs in network protocols limited to + * 32 bit directory positions. * - * The full readdir position, and thus the absolute max number of - * entries in a directory, is limited to 2^31 to avoid the risk of - * breaking legacy environments. Even with a relatively small 27bit - * item offset allowing 16 colliding entries gets well into hundreds of - * millions of entries before an item fills up and we return a premature - * ENOSPC. Hundreds of millions in a single dir ought to be, wait for - * it, good enough for anybody. + * We have to worry about collisions because we're using the hash of the + * name. We simply allow a name to be stored at multiple hash value + * locations. Create iterates until it finds an unused value and lookup + * iterates until it finds an entry at a hash that matches the name. We + * can store the max iteration used during create in the directory to + * limit the number of values we'll check in lookup. With 31bit hash + * values we can get tens of thousands of entries before we use two + * hashes, hundreds for three, millions for four, and so on. The vast + * majority of directories will use one hash value. * - * Each item's contents are protected by the dir inode's i_mutex that - * callers acquire before calling our dir operations. If we wanted more - * fine grained concurrency, and we might, we'd have to be careful to - * manage the shared items. + * This would be a crazy design in systems where dirent lookups perform + * dependent block reads down a radix or btree structure for each hash + * value. scoutfs makes this a lot cheaper by using the bloom filters + * in the log segments to short circuit negative item lookups. */ static unsigned int mode_to_type(umode_t mode) @@ -94,15 +96,17 @@ static int names_equal(const char *name_a, int len_a, const char *name_b, /* * Return the offset portion of a dirent key from the hash of the name. + * The hash can't be 0 or 1 for . and .. and we chose to limit the max + * file->f_pos. * * XXX This crc nonsense is a quick hack. We'll want something a * lot stronger like siphash. */ -static u32 name_hash(struct inode *dir, const char *name, unsigned int len) +static u32 name_hash(const char *name, unsigned int len, u32 salt) { - struct scoutfs_inode_info *ci = SCOUTFS_I(dir); + u32 h = crc32c(salt, name, len) & SCOUTFS_DIRENT_OFF_MASK; - return crc32c(ci->salt, name, len) >> (32 - SCOUTFS_DIRENT_OFF_BITS); + return max_t(u32, 2, min_t(u32, h, SCOUTFS_DIRENT_LAST_POS)); } static unsigned int dent_bytes(unsigned int name_len) @@ -110,35 +114,16 @@ static unsigned int dent_bytes(unsigned int name_len) return sizeof(struct scoutfs_dirent) + name_len; } -static unsigned int dent_val_off(struct scoutfs_item_ref *ref, - struct scoutfs_dirent *dent) +static unsigned int item_name_len(struct scoutfs_item_ref *ref) { - return (char *)dent - (char *)ref->val; + return ref->val_len - sizeof(struct scoutfs_dirent); } - -static inline struct scoutfs_dirent *next_dent(struct scoutfs_item_ref *ref, - struct scoutfs_dirent *dent) -{ - unsigned int next_off; - - next_off = dent_val_off(ref, dent) + dent_bytes(dent->name_len); - if (next_off == ref->val_len) - return NULL; - - return ref->val + next_off; -} - -#define for_each_item_dent(ref, dent) \ - for (dent = (ref)->val; dent; dent = next_dent(ref, dent)) - +/* + * Store the dirent item hash in the dentry so that we don't have to + * calculate and search to remove the item. + */ struct dentry_info { - /* - * The key offset and collision nr are stored so that we don't - * have to either hash the name to find the item or compare - * names to find the dirent in the item. - */ - u32 key_offset; - u8 coll_nr; + u32 hash; }; static struct kmem_cache *scoutfs_dentry_cachep; @@ -173,16 +158,23 @@ static struct dentry_info *alloc_dentry_info(struct dentry *dentry) static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { + struct scoutfs_inode_info *si = SCOUTFS_I(dir); struct super_block *sb = dir->i_sb; + DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_dirent *dent; struct dentry_info *di; struct scoutfs_key key; + unsigned int name_len; struct inode *inode; - DECLARE_SCOUTFS_ITEM_REF(ref); u64 ino = 0; u32 h = 0; - u32 nr = 0; int ret; + int i; + + if (si->max_dirent_hash_nr == 0) { + ret = -ENOENT; + goto out; + } di = alloc_dentry_info(dentry); if (IS_ERR(di)) { @@ -195,21 +187,27 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } - h = name_hash(dir, dentry->d_name.name, dentry->d_name.len); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + h = si->salt; + for (i = 0; i < si->max_dirent_hash_nr; i++) { + h = name_hash(dentry->d_name.name, dentry->d_name.len, h); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - ret = scoutfs_read_item(sb, &key, &ref); - if (ret) - goto out; + scoutfs_put_ref(&ref); + ret = scoutfs_read_item(sb, &key, &ref); + if (ret == -ENOENT) + continue; + if (ret < 0) + break; - ret = -ENOENT; - for_each_item_dent(&ref, dent) { + dent = ref.val; + name_len = item_name_len(&ref); if (names_equal(dentry->d_name.name, dentry->d_name.len, - dent->name, dent->name_len)) { + dent->name, name_len)) { ino = le64_to_cpu(dent->ino); - nr = dent->coll_nr; ret = 0; break; + } else { + ret = -ENOENT; } } @@ -220,8 +218,7 @@ out: } else if (ret) { inode = ERR_PTR(ret); } else { - di->key_offset = h; - di->coll_nr = nr; + di->hash = h; inode = scoutfs_iget(sb, ino); } @@ -265,52 +262,35 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct scoutfs_dirent *dent; struct scoutfs_key first; struct scoutfs_key last; + unsigned int name_len; LIST_HEAD(iter_list); int ret = 0; - u32 off; u32 pos; - u32 nr; if (!dir_emit_dots(file, dirent, filldir)) return 0; - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - file->f_pos >> SCOUTFS_DIRENT_COLL_BITS); scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - SCOUTFS_DIRENT_OFF_MASK); + SCOUTFS_DIRENT_LAST_POS); + + while (file->f_pos <= SCOUTFS_DIRENT_LAST_POS) { + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + file->f_pos); - for(;;) { scoutfs_put_ref(&ref); ret = scoutfs_next_item(sb, &first, &last, &iter_list, &ref); if (ret) break; - /* start from first collision if we're in a new item */ - if (scoutfs_key_offset(&first) == scoutfs_key_offset(ref.key)) - nr = file->f_pos & SCOUTFS_DIRENT_COLL_MASK; - else - nr = 0; + dent = ref.val; + name_len = item_name_len(&ref); + pos = scoutfs_key_offset(ref.key); - off = scoutfs_key_offset(ref.key) << SCOUTFS_DIRENT_COLL_BITS; - for_each_item_dent(&ref, dent) { - if (dent->coll_nr < nr) - continue; - - pos = off | dent->coll_nr; - - if (filldir(dirent, dent->name, dent->name_len, pos, - le64_to_cpu(dent->ino), - dentry_type(dent->type))) - break; - - file->f_pos = pos + 1; - } - /* done if filldir broke the loop */ - if (dent) + if (filldir(dirent, dent->name, name_len, pos, + le64_to_cpu(dent->ino), dentry_type(dent->type))) break; - first = *ref.key; - scoutfs_inc_key(&first); + file->f_pos = pos + 1; } scoutfs_put_ref(&ref); @@ -326,6 +306,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct super_block *sb = dir->i_sb; + struct scoutfs_inode_info *si = SCOUTFS_I(dir); struct inode *inode = NULL; struct scoutfs_dirent *dent; DECLARE_SCOUTFS_ITEM_REF(ref); @@ -333,8 +314,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_key key; int bytes; int ret; - u64 nr; u64 h; + int i; di = alloc_dentry_info(dentry); if (IS_ERR(di)) @@ -347,38 +328,34 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (IS_ERR(inode)) return PTR_ERR(inode); - h = name_hash(dir, dentry->d_name.name, dentry->d_name.len); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); bytes = dent_bytes(dentry->d_name.len); - ret = scoutfs_read_item(sb, &key, &ref); - if (ret != -ENOENT) { - /* XXX implement many hashes, not coll nr */ - if (WARN_ON_ONCE(!ret)) { - scoutfs_put_ref(&ref); + h = si->salt; + for (i = 0; i < SCOUTFS_MAX_DENT_HASH_NR; i++) { + h = name_hash(dentry->d_name.name, dentry->d_name.len, h); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + + ret = scoutfs_create_item(sb, &key, bytes, &ref); + if (ret != -EEXIST) + break; + } + if (ret) { + if (ret == -EEXIST) ret = -ENOSPC; - } goto out; } - ret = scoutfs_create_item(sb, &key, bytes, &ref); - if (ret) - goto out; - dent = ref.val; - nr = 0; dent->ino = cpu_to_le64(scoutfs_ino(inode)); dent->type = mode_to_type(inode->i_mode); - dent->coll_nr = nr; - dent->name_len = dentry->d_name.len; - memcpy(dent->name, dentry->d_name.name, dent->name_len); - di->key_offset = h; - di->coll_nr = nr; + memcpy(dent->name, dentry->d_name.name, dentry->d_name.len); + di->hash = h; scoutfs_put_ref(&ref); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; + si->max_dirent_hash_nr = max_t(int, i + 1, si->max_dirent_hash_nr); inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; if (S_ISDIR(mode)) { @@ -428,14 +405,10 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) return -EINVAL; di = dentry->d_fsdata; - trace_printk("dir size %llu entry k_off nr %u %u\n", - i_size_read(inode), di->key_offset, di->coll_nr); - if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, - di->key_offset); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); ret = scoutfs_read_item(sb, &key, &ref); if (ret) diff --git a/kmod/src/format.h b/kmod/src/format.h index 1310f8d7..989592e3 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -225,6 +225,7 @@ struct scoutfs_inode { __le32 mode; __le32 rdev; __le32 salt; + __u8 max_dirent_hash_nr; struct scoutfs_timespec atime; struct scoutfs_timespec ctime; struct scoutfs_timespec mtime; @@ -238,17 +239,17 @@ struct scoutfs_inode { */ struct scoutfs_dirent { __le64 ino; -#if defined(__LITTLE_ENDIAN_BITFIELD) - __u8 type:4, - coll_nr:4; -#else - __u8 coll_nr:4, - type:4; -#endif - __u8 name_len; + __u8 type; __u8 name[0]; } __packed; +/* + * The max number of dirent hash values determines the overhead of + * lookups in very large directories. With 31bit offsets the number + * of entries stored before enospc tends to plateau around 200 million + * entries around 8 functions. That seems OK for now. + */ +#define SCOUTFS_MAX_DENT_HASH_NR 8 #define SCOUTFS_NAME_LEN 255 /* @@ -257,14 +258,10 @@ struct scoutfs_dirent { * network protocols that have limited readir positions. */ -#define SCOUTFS_DIRENT_OFF_BITS 27 -#define SCOUTFS_DIRENT_OFF_MASK ((1 << SCOUTFS_DIRENT_OFF_BITS) - 1) -#define SCOUTFS_DIRENT_COLL_BITS 4 -#define SCOUTFS_DIRENT_COLL_MASK ((1 << SCOUTFS_DIRENT_COLL_BITS) - 1) - -/* getdents returns the *next* pos with each entry. so we can't return ~0 */ -#define SCOUTFS_DIRENT_MAX_POS \ - (((1 << (SCOUTFS_DIRENT_OFF_BITS + SCOUTFS_DIRENT_COLL_BITS)) - 1) - 1) +#define SCOUTFS_DIRENT_OFF_BITS 31 +#define SCOUTFS_DIRENT_OFF_MASK ((1U << SCOUTFS_DIRENT_OFF_BITS) - 1) +/* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +#define SCOUTFS_DIRENT_LAST_POS (INT_MAX - 1) enum { SCOUTFS_DT_FIFO = 0, diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 99187b2c..ba5eb8c4 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -105,6 +105,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); ci->salt = le32_to_cpu(cinode->salt); + ci->max_dirent_hash_nr = cinode->max_dirent_hash_nr; } static int scoutfs_read_locked_inode(struct inode *inode) @@ -186,6 +187,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); cinode->salt = cpu_to_le32(ci->salt); + cinode->max_dirent_hash_nr = ci->max_dirent_hash_nr; } /* diff --git a/kmod/src/inode.h b/kmod/src/inode.h index d7009352..3da21640 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -4,6 +4,7 @@ struct scoutfs_inode_info { u64 ino; u32 salt; + u8 max_dirent_hash_nr; struct inode inode; }; From fbbfac1b27fa04ac826d6fa54be540fb5ed24428 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Mar 2016 21:44:42 -0700 Subject: [PATCH 020/920] scoutfs: fix sparse errors I was building against a RHEL tree that broke sparse builds. With that fixed I can now see and fix sparse errors. Signed-off-by: Zach Brown --- kmod/src/chunk.c | 5 +++-- kmod/src/msg.c | 2 ++ kmod/src/super.c | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/kmod/src/chunk.c b/kmod/src/chunk.c index 4b7a24ec..3a9080c3 100644 --- a/kmod/src/chunk.c +++ b/kmod/src/chunk.c @@ -25,12 +25,13 @@ #include "msg.h" #include "block.h" #include "ring.h" +#include "chunk.h" void scoutfs_set_chunk_alloc_bits(struct super_block *sb, struct scoutfs_ring_bitmap *bm) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 off = le64_to_cpu(bm->offset) * ARRAY_SIZE(bm->bits); + u64 off = le32_to_cpu(bm->offset) * ARRAY_SIZE(bm->bits); /* XXX check for corruption */ @@ -66,7 +67,7 @@ int scoutfs_alloc_chunk(struct super_block *sb, u64 *blkno) clear_bit_le(bit, sbi->chunk_alloc_bits); off = round_down(bit, sizeof(bm.bits) * 8); - bm.offset = le32_to_cpu(off); + bm.offset = cpu_to_le32(off); off *= ARRAY_SIZE(bm.bits); bm.bits[0] = sbi->chunk_alloc_bits[off]; diff --git a/kmod/src/msg.c b/kmod/src/msg.c index e177af07..98235acf 100644 --- a/kmod/src/msg.c +++ b/kmod/src/msg.c @@ -1,6 +1,8 @@ #include #include +#include "msg.h" + void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...) { diff --git a/kmod/src/super.c b/kmod/src/super.c index ba200876..f8c5ed1b 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -155,7 +155,8 @@ static int read_supers(struct super_block *sb) /* Initialize all the sb info fields which depends on the supers. */ - bytes = DIV_ROUND_UP(sbi->super.total_chunks, 64) * sizeof(u64); + bytes = DIV_ROUND_UP(le64_to_cpu(sbi->super.total_chunks), 64) * + sizeof(u64); sbi->chunk_alloc_bits = vmalloc(bytes); if (!sbi->chunk_alloc_bits) return -ENOMEM; From 3bb00fafdcc1bf91b76ab428fe2b0077d5ba348b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Mar 2016 21:45:08 -0700 Subject: [PATCH 021/920] scoutfs: require sparse builds Now that we know that it's easy to fix sparse build failures against RHEL kernel headers we can require sparse builds when developing. Signed-off-by: Zach Brown --- kmod/Makefile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kmod/Makefile b/kmod/Makefile index 07fbc001..1dbc498c 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -1,4 +1,33 @@ ALL: module +# +# SK_KSRC points to the kernel header build dir to build against. +# On a running machine this could be /lib/modules/$(uname -r)/build with +# the right kernel-headers package installed. I tend to build on other +# hosts so I extract the kernel-headers package for the target machine's +# kernel in a dir somehere. +# +# sparse is critical for avoiding endian mistakes. It should just work +# if the sparse package is installed. +# +# but sometimes kernel-headers are broken. For example, the +# rhel 3.10.0-327.el7.x86_64 kernel needs the following patch. +# We'll try to have a git tree with fixed headers. +# +# +# diff --git a/include/linux/rh_kabi.h b/include/linux/rh_kabi.h +# index 1767770..0a8e5f3 100644 +# --- a/include/linux/rh_kabi.h +# +++ b/include/linux/rh_kabi.h +# @@ -73,7 +73,6 @@ +# struct { \ +# _orig; \ +# } __UNIQUE_ID(rh_kabi_hide); \ +# - __RH_KABI_CHECK_SIZE_ALIGN(_orig, _new); \ +# } +# +# #define _RH_KABI_REPLACE_UNSAFE(_orig, _new) _new + module: make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src + make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src From 434cbb9c78b68b5709706a59f5e2dad047664792 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 25 Mar 2016 10:08:34 -0700 Subject: [PATCH 022/920] scoutfs: create dirty items for inode updates Inode updates weren't persistent because they were being stored in clean segments in memory. This was triggered by the new hashed dirent mechanism returning -ENOENT when the inode still had a 0 max dirent hash nr. We make sure that there is a dirty item in the dirty segment at the start of inode modification so that later updates will store in the dirty segment. Nothing ensures that the dirty segment won't be written out today but that will be added soon. Signed-off-by: Zach Brown --- kmod/src/dir.c | 9 +++++++++ kmod/src/inode.c | 35 +++++++++++++++++++++++++++++++++++ kmod/src/inode.h | 1 + kmod/src/segment.c | 43 +++++++++++++++++++++++++++++++++++++++++++ kmod/src/segment.h | 2 ++ 5 files changed, 90 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 50a39290..5a2902f5 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -324,6 +324,10 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; + ret = scoutfs_dirty_inode_item(dir); + if (ret) + return ret; + inode = scoutfs_new_inode(sb, dir, mode, rdev); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -408,6 +412,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; + ret = scoutfs_dirty_inode_item(dir) ?: + scoutfs_dirty_inode_item(inode); + if (ret) + return ret; + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); ret = scoutfs_read_item(sb, &key, &ref); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index ba5eb8c4..a59cac5d 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -190,6 +190,41 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->max_dirent_hash_nr = ci->max_dirent_hash_nr; } +/* + * Create a pinned dirty inode item so that we can later update the + * inode item without risking failure. We often wouldn't want to have + * to unwind inode modifcations (perhaps by shared vfs code!) if our + * item update failed. This is our chance to return errors for enospc + * for lack of space for new logged dirty inode items. + * + * This dirty inode item will be found by lookups in the interim so we + * have to update it now with the current inode contents. + * + * Callers don't delete these dirty items on errors. They're still + * valid and will be merged with the current item eventually. They can + * be found in the dirty block to avoid future dirtying (say repeated + * creations in a directory). + * + * The caller has to prevent sync between dirtying and updating the + * inodes. + */ +int scoutfs_dirty_inode_item(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_key key; + int ret; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + + ret = scoutfs_dirty_item(sb, &key, sizeof(struct scoutfs_inode), &ref); + if (!ret) { + store_inode(ref.val, inode); + scoutfs_put_ref(&ref); + } + return ret; +} + /* * Every time we modify the inode in memory we copy it to its inode * item. This lets us write out blocks of items without having to track diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 3da21640..650cd05b 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -23,6 +23,7 @@ struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); +int scoutfs_dirty_inode_item(struct inode *inode); void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 9ea41977..4e94760a 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -475,6 +475,49 @@ out: return ret; } +/* + * Ensure that there is a dirty item with the given key in the current + * dirty segment. + * + * The caller locks access to the item and prevents sync and made sure + * that there's enough free space in the segment for their dirty inodes. + * + * This is better than getting -EEXIST from create_item because that + * will leave the allocated item and val dangling in the block when it + * returns the error. + */ +int scoutfs_dirty_item(struct super_block *sb, struct scoutfs_key *key, + unsigned bytes, struct scoutfs_item_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_item *item; + struct buffer_head *bh; + bool create = false; + int ret; + + mutex_lock(&sbi->dirty_mutex); + + if (sbi->dirty_blkno) { + ret = scoutfs_skip_lookup(sb, sbi->dirty_blkno, key, &bh, + &item); + if (ret == -ENOENT) + create = true; + else if (!ret) { + ret = populate_ref(sb, sbi->dirty_blkno, bh, item, + ref); + brelse(bh); + } + } else { + create = true; + } + mutex_unlock(&sbi->dirty_mutex); + + if (create) + ret = scoutfs_create_item(sb, key, bytes, ref); + + return ret; +} + /* * This is a really cheesy temporary delete method. It only works on items * that are stored in dirty blocks. The caller is responsible for dropping diff --git a/kmod/src/segment.h b/kmod/src/segment.h index 6b41579b..a990d422 100644 --- a/kmod/src/segment.h +++ b/kmod/src/segment.h @@ -22,6 +22,8 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_item_ref *ref); int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, unsigned bytes, struct scoutfs_item_ref *ref); +int scoutfs_dirty_item(struct super_block *sb, struct scoutfs_key *key, + unsigned bytes, struct scoutfs_item_ref *ref); int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref); int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct list_head *iter_list, From 6834100251307f036a3972b88e57fdd0211a7580 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 25 Mar 2016 11:08:20 -0700 Subject: [PATCH 023/920] scoutfs: free our dentry info Stop leaking dentry_info allocations by adding a dentry_op with a d_release that frees our dentry info allocation. rmmod tests no longer fail when dmesg screams that we have slab caches that still have allocated objects. Signed-off-by: Zach Brown s --- kmod/src/dir.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 5a2902f5..cec7d878 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -128,6 +128,20 @@ struct dentry_info { static struct kmem_cache *scoutfs_dentry_cachep; +static void scoutfs_d_release(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (di) { + kmem_cache_free(scoutfs_dentry_cachep, di); + dentry->d_fsdata = NULL; + } +} + +static const struct dentry_operations scoutfs_dentry_ops = { + .d_release = scoutfs_d_release, +}; + static struct dentry_info *alloc_dentry_info(struct dentry *dentry) { struct dentry_info *di; @@ -141,8 +155,11 @@ static struct dentry_info *alloc_dentry_info(struct dentry *dentry) return ERR_PTR(-ENOMEM); spin_lock(&dentry->d_lock); - if (!dentry->d_fsdata) + if (!dentry->d_fsdata) { dentry->d_fsdata = di; + d_set_d_op(dentry, &scoutfs_dentry_ops); + } + spin_unlock(&dentry->d_lock); if (di != dentry->d_fsdata) From 867d717d2b25900dca1c6d27c20a04e197dfb28d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 25 Mar 2016 19:28:21 -0700 Subject: [PATCH 024/920] scoutfs: item offsets need to skip block headers The vallue offset allocation knew to skip block headers at the start of each segment block but, weirdly, the item offset allocation didn't. We make item offset calculation skip the header and we add some tracing to help see the problem. Signed-off-by: Zach Brown --- kmod/src/segment.c | 10 +++++++++- kmod/src/skip.c | 16 ++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 4e94760a..85e51c90 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -188,9 +188,15 @@ static int add_item_off(struct scoutfs_sb_info *sbi, int height) { int len = offsetof(struct scoutfs_item, skip_next[height]); int off = sbi->dirty_item_off; + int block_off; int tail_free; - /* item's can't cross a block boundary */ + /* items can't start in a block header */ + block_off = off & SCOUTFS_BLOCK_MASK; + if (block_off < sizeof(struct scoutfs_block_header)) + off += sizeof(struct scoutfs_block_header) - block_off; + + /* items can't cross a block boundary */ tail_free = SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK); if (tail_free < len) off += tail_free + sizeof(struct scoutfs_block_header); @@ -432,6 +438,8 @@ next_chunk: item_off = add_item_off(sbi, height); val_off = sub_val_off(sbi, bytes); + trace_printk("item_off %u val_off %u\n", item_off, val_off); + if (item_off > val_off) { ret = scoutfs_finish_dirty_segment(sb); if (ret) diff --git a/kmod/src/skip.c b/kmod/src/skip.c index 69bae2b8..c855ae49 100644 --- a/kmod/src/skip.c +++ b/kmod/src/skip.c @@ -81,12 +81,16 @@ struct skip_path { */ static int invalid_item_off(u32 off) { - return off < ((SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + - sizeof(struct scoutfs_item_block)) || - (off & SCOUTFS_BLOCK_MASK) < - sizeof(struct scoutfs_block_header) || - (off & SCOUTFS_BLOCK_MASK) > - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_item)); + if (off < ((SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + + sizeof(struct scoutfs_item_block)) || + (off & SCOUTFS_BLOCK_MASK) < sizeof(struct scoutfs_block_header) || + (off & SCOUTFS_BLOCK_MASK) > + (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_item))) { + trace_printk("invalid offset %u\n", off); + return 1; + } + + return 0; } /* From 9cf87ee571f09344ad406541f72a15fbc3d37fd1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 26 Mar 2016 10:58:06 -0700 Subject: [PATCH 025/920] scoutfs: add basic file page cache read and write Add basic file data support by implementing the address space file and page read and write methods. This passis basic read/write tests but is only the seed of a final implementation. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/filerw.c | 218 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/filerw.h | 7 ++ kmod/src/format.h | 17 +++- kmod/src/inode.c | 20 ++++- 5 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 kmod/src/filerw.c create mode 100644 kmod/src/filerw.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index dae6c279..21058481 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,4 +1,4 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -scoutfs-y += block.o bloom.o chunk.o crc.o dir.o inode.o manifest.o msg.o \ - ring.o segment.o skip.o super.o +scoutfs-y += block.o bloom.o chunk.o crc.o dir.o filerw.o inode.o manifest.o \ + msg.o ring.o segment.o skip.o super.o diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c new file mode 100644 index 00000000..2b914d87 --- /dev/null +++ b/kmod/src/filerw.c @@ -0,0 +1,218 @@ +/* +* Copyright (C) 2016 Versity Software, Inc. All rights reserved. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public +* License v2 as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +*/ +#include +#include +#include + +#include "format.h" +#include "segment.h" +#include "inode.h" +#include "key.h" +#include "filerw.h" + +/* + * File data is stored in items just like everything else. This is very + * easy to implement but incurs a copying overhead. We'll see how + * expensive that gets. + * + * By making the max item size a bit less than the block size we can + * still have room for the block header which gets us file data + * checksums. File item key offsets are multiples of this max block + * size though items can be smaller if the data is sparse. This lets us + * do lookups for specific keys and take advantage of the bloom filters. + * + * This is a minimal first pass and will need more work. It'll need to + * worry about enospc in writepage and cluster access for a start. + */ + +/* +* Track the intersection of the logical region of a file with a page +* and file data item. +*/ +struct data_region { + u64 item_key; + unsigned int page_off; + unsigned short len; + unsigned short item_off; +}; + +/* + * Map the file offset to its intersection with the page and item region. + * Returns false if the byte position is outside the page. +*/ +static bool map_data_region(struct data_region *dr, u64 pos, struct page *page) +{ + if (pos >> PAGE_SHIFT != page->index) + return false; + + dr->page_off = pos & ~PAGE_MASK; + + dr->item_off = do_div(pos, SCOUTFS_MAX_ITEM_LEN); + dr->item_key = pos; + + dr->len = min(SCOUTFS_MAX_ITEM_LEN - dr->item_off, + PAGE_SIZE - dr->page_off); + + return true; +} + +#define for_each_data_region(dr, page, pos) \ + for (pos = (u64)page->index << PAGE_SHIFT; \ + map_data_region(dr, pos, page); pos += (dr)->len) + +/* + * Copy the contents of file data items into the page. If we don't + * find an item then we zero that region of the page. + * + * XXX i_size? + * XXX async? + */ +static int scoutfs_readpage(struct file *file, struct page *page) +{ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_key key; + struct data_region dr; + int ret = 0; + void *addr; + u64 pos; + + for_each_data_region(&dr, page, pos) { + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, + dr.item_key); + + ret = scoutfs_read_item(sb, &key, &ref); + if (ret == -ENOENT) { + addr = kmap_atomic(page); + memset(addr + dr.page_off, 0, dr.len); + kunmap_atomic(addr); + continue; + } + if (ret) + break; + + addr = kmap_atomic(page); + memcpy(addr + dr.page_off, ref.val + dr.item_off, dr.len); + kunmap_atomic(addr); + } + + if (!ret) + SetPageUptodate(page); + unlock_page(page); + return ret; +} + +/* + * Copy the contents of the page into file items. Data integrity syncs + * will later write the dirty segment to the device. + * +* XXX zeroing regions of data items? +* XXX wbc counters? +* XXX reserve space so dirty item doesn't get enospc -- our "delalloc"? +*/ +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_key key; + struct data_region dr; + void *addr; + u64 pos; + int ret; + + set_page_writeback(page); + + for_each_data_region(&dr, page, pos) { + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, + dr.item_key); + + ret = scoutfs_dirty_item(sb, &key, SCOUTFS_MAX_ITEM_LEN, &ref); + if (ret) + break; + + addr = kmap_atomic(page); + memcpy(ref.val + dr.item_off, addr + dr.page_off, dr.len); + kunmap_atomic(addr); + + scoutfs_put_ref(&ref); + + } + + scoutfs_put_ref(&ref); + + if (ret) { + SetPageError(page); + mapping_set_error(&inode->i_data, ret); + } + + end_page_writeback(page); + unlock_page(page); + + return ret; +} + +static int scoutfs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + pgoff_t index = pos >> PAGE_CACHE_SHIFT; + struct page *page; + + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) + return -ENOMEM; + + *pagep = page; + return 0; +} + +static int scoutfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = mapping->host; + unsigned off; + + off = pos & (PAGE_CACHE_SIZE - 1); + + /* zero the stale part of the page if we did a short copy */ + if (copied < len) + zero_user_segment(page, off + copied, len); + + if (pos + copied > inode->i_size) + i_size_write(inode, pos + copied); + + if (!PageUptodate(page)) + SetPageUptodate(page); + set_page_dirty(page); + unlock_page(page); + page_cache_release(page); + + return copied; +} + +const struct address_space_operations scoutfs_file_aops = { + .readpage = scoutfs_readpage, + .writepage = scoutfs_writepage, + .write_begin = scoutfs_write_begin, + .write_end = scoutfs_write_end, +}; + +const struct file_operations scoutfs_file_fops = { + .read = do_sync_read, + .write = do_sync_write, + .aio_read = generic_file_aio_read, + .aio_write = generic_file_aio_write, +}; diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h new file mode 100644 index 00000000..2d9d478e --- /dev/null +++ b/kmod/src/filerw.h @@ -0,0 +1,7 @@ +#ifndef _SCOUTFS_FILERW_H_ +#define _SCOUTFS_FILERW_H_ + +extern const struct address_space_operations scoutfs_file_aops; +extern const struct file_operations scoutfs_file_fops; + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 989592e3..8808eb04 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -100,8 +100,14 @@ struct scoutfs_key { #define SCOUTFS_ROOT_INO 1 -#define SCOUTFS_INODE_KEY 128 -#define SCOUTFS_DIRENT_KEY 192 +/* + * Currently we sort keys by the numeric value of the types, but that + * isn't necessary. We could have an arbitrary sort order. So we don't + * have to stress about cleverly allocating the types. + */ +#define SCOUTFS_INODE_KEY 1 +#define SCOUTFS_DIRENT_KEY 2 +#define SCOUTFS_DATA_KEY 3 struct scoutfs_ring_map_block { struct scoutfs_block_header hdr; @@ -203,6 +209,13 @@ struct scoutfs_item { __le32 skip_next[0]; } __packed; +/* + * Item size caps item file data item length so that they fit in checksummed + * 4k blocks with a bit of expansion room. + */ +#define SCOUTFS_MAX_ITEM_LEN \ + (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_block_header) - 32) + struct scoutfs_timespec { __le64 sec; __le32 nsec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index a59cac5d..703ac676 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -21,6 +21,7 @@ #include "inode.h" #include "segment.h" #include "dir.h" +#include "filerw.h" /* * XXX @@ -68,10 +69,25 @@ void scoutfs_destroy_inode(struct inode *inode) static void set_inode_ops(struct inode *inode) { switch (inode->i_mode & S_IFMT) { + /* + * I guess we add a reg.c for regular files? Or pagecache.c? + * I guess that makes more sense. + * + * - page dirtying makes sure there's a dirty item + * - sync writes back page cache pages + * - writepage copies to dirty item + * - crc calculated after copying + * - pages can be pretty large + * - tail items can be partial? + * - tracing all over the place + * - maybe just less than 4k is the answer? + * - so allocation pulls the value back + * - probably leave some overhead for header growth + */ case S_IFREG: -// inode->i_mapping->a_ops = &scoutfs_file_aops; + inode->i_mapping->a_ops = &scoutfs_file_aops; // inode->i_op = &scoutfs_file_iops; -// inode->i_fop = &scoutfs_file_fops; + inode->i_fop = &scoutfs_file_fops; break; case S_IFDIR: inode->i_op = &scoutfs_dir_iops; From 402dd2969fadfd8ed227ebb8052c52ea502269ff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 26 Mar 2016 20:58:31 -0700 Subject: [PATCH 026/920] scoutfs: add tracepoint support with bloom example Add the intrastucture for tracepoints. We include an example user that traces bloom filter hits and misses. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 ++- kmod/src/bloom.c | 7 ++++ kmod/src/bloom.h | 1 + kmod/src/scoutfs_trace.c | 32 +++++++++++++++++ kmod/src/scoutfs_trace.h | 77 ++++++++++++++++++++++++++++++++++++++++ kmod/src/segment.c | 2 +- 6 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 kmod/src/scoutfs_trace.c create mode 100644 kmod/src/scoutfs_trace.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 21058481..31093000 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,4 +1,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o + +CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += block.o bloom.o chunk.o crc.o dir.o filerw.o inode.o manifest.o \ - msg.o ring.o segment.o skip.o super.o + msg.o ring.o scoutfs_trace.o segment.o skip.o super.o diff --git a/kmod/src/bloom.c b/kmod/src/bloom.c index df528afd..d41fa57c 100644 --- a/kmod/src/bloom.c +++ b/kmod/src/bloom.c @@ -20,6 +20,7 @@ #include "format.h" #include "block.h" #include "bloom.h" +#include "scoutfs_trace.h" /* * Each log segment starts with a bloom filters that spans multiple @@ -100,6 +101,7 @@ int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, * might, and -errno if IO fails. */ int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct scoutfs_bloom_bits *bits) { struct scoutfs_bloom_block *blm; @@ -121,5 +123,10 @@ int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, break; } + if (ret) + trace_scoutfs_bloom_hit(key); + else + trace_scoutfs_bloom_miss(key); + return ret; } diff --git a/kmod/src/bloom.h b/kmod/src/bloom.h index 4e843fbe..59739bb1 100644 --- a/kmod/src/bloom.h +++ b/kmod/src/bloom.h @@ -9,6 +9,7 @@ struct scoutfs_bloom_bits { void scoutfs_calc_bloom_bits(struct scoutfs_bloom_bits *bits, struct scoutfs_key *key, __le32 *salts); int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, + struct scoutfs_key *key, struct scoutfs_bloom_bits *bits); int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, struct scoutfs_bloom_bits *bits); diff --git a/kmod/src/scoutfs_trace.c b/kmod/src/scoutfs_trace.c new file mode 100644 index 00000000..38e147dc --- /dev/null +++ b/kmod/src/scoutfs_trace.c @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "inode.h" +#include "dir.h" +#include "msg.h" +#include "block.h" +#include "manifest.h" +#include "ring.h" +#include "segment.h" + +#define CREATE_TRACE_POINTS +#include "scoutfs_trace.h" diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h new file mode 100644 index 00000000..50dde60e --- /dev/null +++ b/kmod/src/scoutfs_trace.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +/* + * This has a crazy name because it's in an external module build at + * the moment. When it's merged upstream it'll move to + * include/trace/events/scoutfs.h + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM scoutfs + +#if !defined(_TRACE_SCOUTFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_SCOUTFS_H + +#include + +TRACE_EVENT(scoutfs_bloom_hit, + TP_PROTO(struct scoutfs_key *key), + + TP_ARGS(key), + + TP_STRUCT__entry( + __field(__u64, inode) + __field(__u8, type) + __field(__u64, offset) + ), + + TP_fast_assign( + __entry->inode = le64_to_cpu(key->inode); + __entry->type = key->type; + __entry->offset = le64_to_cpu(key->offset); + ), + + TP_printk("key %llu.%u.%llu", + __entry->inode, __entry->type, __entry->offset) +); + +TRACE_EVENT(scoutfs_bloom_miss, + TP_PROTO(struct scoutfs_key *key), + + TP_ARGS(key), + + TP_STRUCT__entry( + __field(__u64, inode) + __field(__u8, type) + __field(__u64, offset) + ), + + TP_fast_assign( + __entry->inode = le64_to_cpu(key->inode); + __entry->type = key->type; + __entry->offset = le64_to_cpu(key->offset); + ), + + TP_printk("key %llu.%u.%llu", + __entry->inode, __entry->type, __entry->offset) +); + + +#endif /* _TRACE_SCOUTFS_H */ + +/* This part must be outside protection */ +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE scoutfs_trace +#include diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 85e51c90..8dc7b696 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -146,7 +146,7 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, /* XXX read-ahead all bloom blocks */ ret = scoutfs_test_bloom_bits(sb, le64_to_cpu(ment.blkno), - &bits); + key, &bits); if (ret < 0) break; if (!ret) { From 059212d50e5a5f6621f818317ddb6e6c01ff5175 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 26 Mar 2016 22:24:00 -0700 Subject: [PATCH 027/920] scoutfs: add some basic tracepoints I added these tracepoints to verify that file data isn't reachable after mount because we're not writing out the inode with the current i_size. Signed-off-by: Zach Brown --- kmod/src/filerw.c | 6 ++ kmod/src/inode.c | 3 + kmod/src/scoutfs_trace.h | 120 ++++++++++++++++++++++++++++++++++++++- kmod/src/super.c | 4 ++ 4 files changed, 132 insertions(+), 1 deletion(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 2b914d87..f1febd31 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -19,6 +19,7 @@ #include "inode.h" #include "key.h" #include "filerw.h" +#include "scoutfs_trace.h" /* * File data is stored in items just like everything else. This is very @@ -167,9 +168,12 @@ static int scoutfs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { + struct inode *inode = mapping->host; pgoff_t index = pos >> PAGE_CACHE_SHIFT; struct page *page; + trace_scoutfs_write_begin(scoutfs_ino(inode), pos, len); + page = grab_cache_page_write_begin(mapping, index, flags); if (!page) return -ENOMEM; @@ -185,6 +189,8 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct inode *inode = mapping->host; unsigned off; + trace_scoutfs_write_end(scoutfs_ino(inode), pos, len, copied); + off = pos & (PAGE_CACHE_SIZE - 1); /* zero the stale part of the page if we did a short copy */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 703ac676..01daf394 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -22,6 +22,7 @@ #include "segment.h" #include "dir.h" #include "filerw.h" +#include "scoutfs_trace.h" /* * XXX @@ -237,6 +238,7 @@ int scoutfs_dirty_inode_item(struct inode *inode) if (!ret) { store_inode(ref.val, inode); scoutfs_put_ref(&ref); + trace_scoutfs_dirty_inode(inode); } return ret; } @@ -264,6 +266,7 @@ void scoutfs_update_inode_item(struct inode *inode) store_inode(ref.val, inode); scoutfs_put_ref(&ref); + trace_scoutfs_update_inode(inode); } /* diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 50dde60e..5e95258b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -66,10 +66,128 @@ TRACE_EVENT(scoutfs_bloom_miss, __entry->inode, __entry->type, __entry->offset) ); +TRACE_EVENT(scoutfs_write_begin, + TP_PROTO(u64 ino, loff_t pos, unsigned len), + + TP_ARGS(ino, pos, len), + + TP_STRUCT__entry( + __field(__u64, inode) + __field(__u64, pos) + __field(__u32, len) + ), + + TP_fast_assign( + __entry->inode = ino; + __entry->pos = pos; + __entry->len = len; + ), + + TP_printk("ino %llu pos %llu len %u", + __entry->inode, __entry->pos, __entry->len) +); + +TRACE_EVENT(scoutfs_write_end, + TP_PROTO(u64 ino, loff_t pos, unsigned len, unsigned copied), + + TP_ARGS(ino, pos, len, copied), + + TP_STRUCT__entry( + __field(__u64, inode) + __field(__u64, pos) + __field(__u32, len) + __field(__u32, copied) + ), + + TP_fast_assign( + __entry->inode = ino; + __entry->pos = pos; + __entry->len = len; + __entry->copied = copied; + ), + + TP_printk("ino %llu pos %llu len %u", + __entry->inode, __entry->pos, __entry->len) +); + +TRACE_EVENT(scoutfs_dirty_inode, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(__u64, ino) + __field(__u64, size) + ), + + TP_fast_assign( + __entry->ino = scoutfs_ino(inode); + __entry->size = inode->i_size; + ), + + TP_printk("ino %llu size %llu", + __entry->ino, __entry->size) +); + +TRACE_EVENT(scoutfs_update_inode, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(__u64, ino) + __field(__u64, size) + ), + + TP_fast_assign( + __entry->ino = scoutfs_ino(inode); + __entry->size = inode->i_size; + ), + + TP_printk("ino %llu size %llu", + __entry->ino, __entry->size) +); + +TRACE_EVENT(scoutfs_dirty_super, + TP_PROTO(struct scoutfs_super_block *super), + + TP_ARGS(super), + + TP_STRUCT__entry( + __field(__u64, blkno) + __field(__u64, seq) + ), + + TP_fast_assign( + __entry->blkno = le64_to_cpu(super->hdr.blkno); + __entry->seq = le64_to_cpu(super->hdr.seq); + ), + + TP_printk("blkno %llu seq %llu", + __entry->blkno, __entry->seq) +); + +TRACE_EVENT(scoutfs_write_super, + TP_PROTO(struct scoutfs_super_block *super), + + TP_ARGS(super), + + TP_STRUCT__entry( + __field(__u64, blkno) + __field(__u64, seq) + ), + + TP_fast_assign( + __entry->blkno = le64_to_cpu(super->hdr.blkno); + __entry->seq = le64_to_cpu(super->hdr.seq); + ), + + TP_printk("blkno %llu seq %llu", + __entry->blkno, __entry->seq) +); #endif /* _TRACE_SCOUTFS_H */ -/* This part must be outside protection */ /* This part must be outside protection */ #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH . diff --git a/kmod/src/super.c b/kmod/src/super.c index f8c5ed1b..7482d997 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -27,6 +27,7 @@ #include "manifest.h" #include "ring.h" #include "segment.h" +#include "scoutfs_trace.h" /* * We've been dirtying log segment blocks and ring blocks as items were @@ -71,6 +72,8 @@ int scoutfs_advance_dirty_super(struct super_block *sb) le64_add_cpu(&super->hdr.seq, 1); + trace_scoutfs_dirty_super(super); + return 0; } @@ -96,6 +99,7 @@ int scoutfs_write_dirty_super(struct super_block *sb) scoutfs_calc_hdr_crc(bh); mark_buffer_dirty(bh); + trace_scoutfs_write_super(super); ret = sync_dirty_buffer(bh); brelse(bh); From eff3d78cb1376798fcb708b5a1f050312164dec7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 26 Mar 2016 22:28:45 -0700 Subject: [PATCH 028/920] scoutfs: update inode when write changes i_size Extended file data wasn't persistent because we weren't writing out the inode with the i_size update that covered the newly written data. Signed-off-by: Zach Brown --- kmod/src/filerw.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index f1febd31..fc7a46ec 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -197,8 +197,12 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, if (copied < len) zero_user_segment(page, off + copied, len); - if (pos + copied > inode->i_size) + if (pos + copied > inode->i_size) { i_size_write(inode, pos + copied); + /* XXX need to think about pinning and enospc */ + if (!scoutfs_dirty_inode_item(inode)) + scoutfs_update_inode_item(inode); + } if (!PageUptodate(page)) SetPageUptodate(page); From 9c3918b576c94c8f8548cbbb962e37f63b0d4c6a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 27 Mar 2016 16:19:19 -0700 Subject: [PATCH 029/920] scoutfs: remove accidentally committed notes Some brainstorming notes in a comment accdentally made their way in to a commit. Signed-off-by: Zach Brown --- kmod/src/inode.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 01daf394..a92578c0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -70,21 +70,6 @@ void scoutfs_destroy_inode(struct inode *inode) static void set_inode_ops(struct inode *inode) { switch (inode->i_mode & S_IFMT) { - /* - * I guess we add a reg.c for regular files? Or pagecache.c? - * I guess that makes more sense. - * - * - page dirtying makes sure there's a dirty item - * - sync writes back page cache pages - * - writepage copies to dirty item - * - crc calculated after copying - * - pages can be pretty large - * - tail items can be partial? - * - tracing all over the place - * - maybe just less than 4k is the answer? - * - so allocation pulls the value back - * - probably leave some overhead for header growth - */ case S_IFREG: inode->i_mapping->a_ops = &scoutfs_file_aops; // inode->i_op = &scoutfs_file_iops; From f1b5eb8a804cf93361031ccfa8e9f58e786ae506 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 27 Mar 2016 19:29:38 -0700 Subject: [PATCH 030/920] scoutfs: more dirty segment locking The segment code wasn't always locking around concurrent accesses to the dirty segment. This is mostly a problem for updating all the next elements in skip list modification. But we also want to serialize dirty block writing. Add a little helper function to acquire the dirty mutex when we're reading from the current dirty segment. Bring sync in to segment.c so it's clear that it's intimately related to the dirty segment. The item deletion hack was totally unlocked. Signed-off-by: Zach Brown --- kmod/src/segment.c | 84 +++++++++++++++++++++++++++++++++++++++------- kmod/src/segment.h | 2 +- kmod/src/super.c | 19 ----------- 3 files changed, 73 insertions(+), 32 deletions(-) diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 8dc7b696..082aaa75 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -27,7 +27,6 @@ #include "bloom.h" #include "skip.h" - /* * scoutfs log segments are large multi-block structures that contain * key/value items. This file implements manipulations of the items. @@ -112,6 +111,26 @@ static int populate_ref(struct super_block *sb, u64 blkno, return 0; } +/* + * Segments are immutable once they're written. As they're being + * dirtied we need to lock concurrent access. XXX the dirty blkno test + * is probably racey. We could use reader/writer locks here. And we + * could probably make the skip lists support concurrent access. + */ +static bool try_lock_dirty_mutex(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (blkno == sbi->dirty_blkno) { + mutex_lock(&sbi->dirty_mutex); + if (blkno == sbi->dirty_blkno) + return true; + mutex_unlock(&sbi->dirty_mutex); + } + + return false; +} + /* * Return a reference to the item at the given key. We walk the manifest * to find blocks that might contain the key from most recent to oldest. @@ -132,6 +151,7 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_item *item = NULL; struct scoutfs_bloom_bits bits; struct buffer_head *bh; + bool locked; int ret; /* XXX hold manifest */ @@ -156,8 +176,11 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, /* XXX read-ahead all item header blocks */ + locked = try_lock_dirty_mutex(sb, le64_to_cpu(ment.blkno)); ret = scoutfs_skip_lookup(sb, le64_to_cpu(ment.blkno), key, &bh, &item); + if (locked) + mutex_unlock(&sbi->dirty_mutex); if (ret) { if (ret == -ENOENT) continue; @@ -311,19 +334,16 @@ static void zero_unused_block(struct super_block *sb, struct buffer_head *bh, * Finish off a dirty segment if we have one. Calculate the checksums of * all the blocks, mark them dirty, and drop their pinned reference. */ -int scoutfs_finish_dirty_segment(struct super_block *sb) +static int finish_dirty_segment(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; struct buffer_head *bh; - u64 blkno; + u64 blkno = sbi->dirty_blkno; int ret = 0; u64 i; - /* XXX sync doesn't lock this test? */ - blkno = sbi->dirty_blkno; - if (!blkno) - return 0; + WARN_ON_ONCE(!blkno); for (i = 0; i < SCOUTFS_BLOCKS_PER_CHUNK; i++) { bh = scoutfs_read_block(sb, blkno + i); @@ -363,6 +383,33 @@ int scoutfs_finish_dirty_segment(struct super_block *sb) return ret; } +/* + * We've been dirtying log segment blocks and ring blocks as items were + * modified. sync makes sure that they're all persistent and updates + * the super. + * + * XXX need to synchronize with transactions + * XXX is state clean after errors? + */ +int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret = 0; + + mutex_unlock(&sbi->dirty_mutex); + if (sbi->dirty_blkno) { + ret = finish_dirty_segment(sb) ?: + scoutfs_finish_dirty_ring(sb) ?: + filemap_write_and_wait(mapping) ?: + scoutfs_write_dirty_super(sb) ?: + scoutfs_advance_dirty_super(sb); + } + mutex_unlock(&sbi->dirty_mutex); + return ret; +} + + /* * Return a reference to a newly allocated and initialized item in a * block in the currently dirty log segment. @@ -441,7 +488,7 @@ next_chunk: trace_printk("item_off %u val_off %u\n", item_off, val_off); if (item_off > val_off) { - ret = scoutfs_finish_dirty_segment(sb); + ret = finish_dirty_segment(sb); if (ret) goto out; goto next_chunk; @@ -537,12 +584,18 @@ int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref) u64 blkno; int ret; + mutex_lock(&sbi->dirty_mutex); + blkno = round_down(ref->item_bh->b_blocknr, SCOUTFS_BLOCKS_PER_CHUNK); - if (WARN_ON_ONCE(blkno != sbi->dirty_blkno)) - return -EINVAL; + if (WARN_ON_ONCE(blkno != sbi->dirty_blkno)) { + ret = -EINVAL; + } else { + ret = scoutfs_skip_delete(sb, blkno, ref->key); + WARN_ON_ONCE(ret); + } + + mutex_unlock(&sbi->dirty_mutex); - ret = scoutfs_skip_delete(sb, blkno, ref->key); - WARN_ON_ONCE(ret); return ret; } @@ -580,10 +633,12 @@ int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct list_head *iter_list, struct scoutfs_item_ref *ref) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_ring_manifest_entry ment; struct scoutfs_item_iter *least; struct scoutfs_item_iter *iter; struct scoutfs_item_iter *pos; + bool locked; int ret; restart: @@ -625,6 +680,8 @@ restart: ret = 0; list_for_each_entry_safe(iter, pos, iter_list, list) { + locked = try_lock_dirty_mutex(sb, iter->blkno); + /* search towards the first key if we haven't yet */ if (!iter->item) { ret = scoutfs_skip_search(sb, iter->blkno, first, @@ -637,6 +694,9 @@ restart: &iter->bh, &iter->item); } + if (locked) + mutex_unlock(&sbi->dirty_mutex); + /* we're done with this block if we past the last key */ while (!ret && scoutfs_key_cmp(&iter->item->key, last) > 0) { brelse(iter->bh); diff --git a/kmod/src/segment.h b/kmod/src/segment.h index a990d422..de5b0dd5 100644 --- a/kmod/src/segment.h +++ b/kmod/src/segment.h @@ -29,7 +29,7 @@ int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct list_head *iter_list, struct scoutfs_item_ref *ref); -int scoutfs_finish_dirty_segment(struct super_block *sb); +int scoutfs_sync_fs(struct super_block *sb, int wait); #endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 7482d997..f0c2d6c6 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -29,25 +29,6 @@ #include "segment.h" #include "scoutfs_trace.h" -/* - * We've been dirtying log segment blocks and ring blocks as items were - * modified. sync makes sure that they're all persistent and updates - * the super. - * - * XXX need to synchronize with transactions - * XXX is state clean after errors? - */ -static int scoutfs_sync_fs(struct super_block *sb, int wait) -{ - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - - return scoutfs_finish_dirty_segment(sb) ?: - scoutfs_finish_dirty_ring(sb) ?: - filemap_write_and_wait(mapping) ?: - scoutfs_write_dirty_super(sb) ?: - scoutfs_advance_dirty_super(sb); -} - static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, From c7c8969704d652d9da24bb2607ccce155ffb9d34 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Mar 2016 10:08:37 -0700 Subject: [PATCH 031/920] scoutfs: adjust bloom size for segment item max The bloom filter was much too large for the current typical limit on the number of items that fit in a segment. Having them too large decreases storage efficiency, has us read more data from a cold cache, and bloom tests pin too much data. We can cut it down to 25% for our current segment and item sizes. Signed-off-by: Zach Brown --- kmod/src/format.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 8808eb04..d5e69bcd 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -36,15 +36,18 @@ #define SCOUTFS_SUPER_NR 2 /* - * 7 bits in a ~76k bloom filter gives ~1% false positive for our max - * of 64k items. + * The bloom filters are statically sized. It's a tradeoff between + * storage overhead and false positive rate. At the moment we have + * as few as 1000 and as many as 18000 items in a segment. We can + * get a ~1% false positive rate (triggering header search) rate at + * the high end with a ~20k bloom filter. * - * n = 65,536, p = 0.01 (1 in 100) → m = 628,167 (76.68KB), k = 7 + * n = 18,000, p = 0.01 (1 in 100) → m = 172,532 (21.06KB), k = 7 */ #define SCOUTFS_BLOOM_BITS 7 -#define SCOUTFS_BLOOM_BIT_WIDTH 20 /* 2^20 > m */ +#define SCOUTFS_BLOOM_BIT_WIDTH 18 /* 2^18 > m */ #define SCOUTFS_BLOOM_BIT_MASK ((1 << SCOUTFS_BLOOM_BIT_WIDTH) - 1) -#define SCOUTFS_BLOOM_BLOCKS ((76 * 1024) / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_BLOOM_BLOCKS ((20 * 1024) / SCOUTFS_BLOCK_SIZE) #define SCOUTFS_BLOOM_SALTS \ DIV_ROUND_UP(SCOUTFS_BLOOM_BITS * SCOUTFS_BLOOM_BIT_WIDTH, 32) From 97e6c1e605b82a0e204c37ba82f397998497dce5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Mar 2016 11:25:36 -0700 Subject: [PATCH 032/920] scoutfs: fix final overlapping item/val Item headers are written from the front of the block to the tail. Item values are written from the tail of the block towards the head. The math to detect their overlapping in the center forgot to take the length of the item header into account. We could have final item headers and values overriding each other which causes file data to appear as an item header. Signed-off-by: Zach Brown --- kmod/src/segment.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kmod/src/segment.c b/kmod/src/segment.c index 082aaa75..c17fc90e 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -201,6 +201,12 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, return ret; } +/* return the byte length of the item header including its skip elements */ +static int item_bytes(int height) +{ + return offsetof(struct scoutfs_item, skip_next[height]); +} + /* * The dirty_item_off points to the byte offset after the last item. * Advance it past block tails and initial block headers until there's @@ -209,7 +215,7 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, */ static int add_item_off(struct scoutfs_sb_info *sbi, int height) { - int len = offsetof(struct scoutfs_item, skip_next[height]); + int len = item_bytes(height); int off = sbi->dirty_item_off; int block_off; int tail_free; @@ -487,7 +493,7 @@ next_chunk: trace_printk("item_off %u val_off %u\n", item_off, val_off); - if (item_off > val_off) { + if (item_off + item_bytes(height) > val_off) { ret = finish_dirty_segment(sb); if (ret) goto out; From 52c315942fc05d1d4b9d32947ad607e1a43bd290 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Mar 2016 11:27:27 -0700 Subject: [PATCH 033/920] scoutfs: update item block and manifest item range The manifests for level 0 blocks always claimed that they could contain all keys. That causes a lot of extra bloom filter lookups when in fact the blocks contain a very small range of keys. It's true that we don't know what items a dirty segment is going to contain, don't want to update the manfiest at every insertion, and have to find the items in the segments in regular searching. But when they're finalized we know the items they'll contain and can update the manifest. We do that by initializing the item block range to nonsense and extending it as items are added. When it's finalized we update the manifest in memory and in the ring. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 4 +- kmod/src/segment.c | 98 +++++++++++++++++++++++++++++++++------------ 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index e1d06e12..cc91cf25 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -28,8 +28,8 @@ * the exception whose segments can have key ranges that overlap. * * We also store pointers to the manifest entries in a radix tree - * indexed by their block number so that we can easily find existing - * entries for deletion. + * indexed by their block number so that we can easily update existing + * entries. * * Level 0 segments are stored in the list with the most recent at the * head of the list. Level 0's rb tree will always be empty. diff --git a/kmod/src/segment.c b/kmod/src/segment.c index c17fc90e..a3251eea 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -290,7 +290,8 @@ static int start_dirty_segment(struct super_block *sb, u64 blkno) if (i == SCOUTFS_BLOOM_BLOCKS) { iblk = (void *)bh->b_data; - /* also zero first unused item slot */ + memset(&iblk->first, ~0, sizeof(struct scoutfs_key)); + memset(&iblk->last, 0, sizeof(struct scoutfs_key)); memset(&iblk->skip_root, 0, sizeof(iblk->skip_root) + sizeof(struct scoutfs_item)); } @@ -310,6 +311,52 @@ static int start_dirty_segment(struct super_block *sb, u64 blkno) return ret; } +/* + * As we fill a dirty segment we don't know which keys it's going to + * contain. We add a manifest entry in memory that has it contain all + * items so that reading will know to search the dirty segment. + * + * Once it's finalized we know the specific range of items it contains + * and we update the manifest entry in memory for that range and write + * that to the ring. + */ +static int update_dirty_segment_manifest(struct super_block *sb, u64 blkno, + bool all_items) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_ring_manifest_entry ment; + struct scoutfs_item_block *iblk; + struct buffer_head *bh; + int ret; + + ment.blkno = cpu_to_le64(blkno); + ment.seq = sbi->super.hdr.seq; + ment.level = 0; + + if (all_items) { + memset(&ment.first, 0, sizeof(struct scoutfs_key)); + memset(&ment.last, ~0, sizeof(struct scoutfs_key)); + } else { + bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); + if (!bh) { + ret = -EIO; + goto out; + } + + iblk = (void *)bh->b_data; + ment.first = iblk->first; + ment.last = iblk->last; + brelse(bh); + } + + if (all_items) + ret = scoutfs_add_manifest(sb, &ment); + else + ret = scoutfs_new_manifest(sb, &ment); +out: + return ret; +} + /* * Zero the portion of this block that intersects with the free space in * the middle of the segment. @start and @end are chunk-relative byte @@ -339,6 +386,8 @@ static void zero_unused_block(struct super_block *sb, struct buffer_head *bh, /* * Finish off a dirty segment if we have one. Calculate the checksums of * all the blocks, mark them dirty, and drop their pinned reference. + * + * XXX should do something with empty dirty segments. */ static int finish_dirty_segment(struct super_block *sb) { @@ -369,12 +418,8 @@ static int finish_dirty_segment(struct super_block *sb) brelse(bh); } - /* - * XXX the manifest entry for this log segment has a key range - * that is much too large. We should shrink it here to reflect - * the real keys. That would reduce the number of blocks involved - * in merging it into level 1. - */ + /* update manifest with range of items and add to ring */ + ret = update_dirty_segment_manifest(sb, blkno, false); /* * Try to kick off a background write of the finished segment. Callers @@ -434,9 +479,9 @@ int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, unsigned bytes, struct scoutfs_item_ref *ref) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_ring_manifest_entry ment; struct scoutfs_bloom_bits bits; struct scoutfs_item *item; + struct scoutfs_item_block *iblk; struct buffer_head *bh; int item_off; int val_off; @@ -463,21 +508,8 @@ next_chunk: if (ret) goto out; - /* - * We need a local manifest in memory to find items as - * we insert them in the dirty segment. We don't know - * what keys are going to be used so we cover the whole - * thing. - * - * XXX But we're also adding it to the ring here. We should - * add it as its finalized and its item range is collapsed. - */ - ment.blkno = cpu_to_le64(blkno); - ment.seq = sbi->super.hdr.seq; - ment.level = 0; - memset(&ment.first, 0, sizeof(ment.first)); - memset(&ment.last, ~0, sizeof(ment.last)); - ret = scoutfs_new_manifest(sb, &ment); + /* add initial in-memory manifest entry with all items */ + ret = update_dirty_segment_manifest(sb, blkno, true); if (ret) goto out; @@ -508,8 +540,6 @@ next_chunk: goto out; } - /* populate iblk first and last? better than in manifest? */ - item = (void *)bh->b_data + (item_off & SCOUTFS_BLOCK_MASK); item->key = *key; item->offset = cpu_to_le32(val_off); @@ -525,6 +555,24 @@ next_chunk: if (ret) goto out; + bh = scoutfs_read_block(sb, sbi->dirty_blkno + SCOUTFS_BLOOM_BLOCKS); + if (!bh) { + ret = -EIO; + goto out; + } + + /* + * Update first and last keys as we go. It's ok if future deletions + * make this range larger than the actual keys. That'll almost + * never happen and it'll get fixed up in merging. + */ + iblk = (void *)bh->b_data; + if (scoutfs_key_cmp(key, &iblk->first) < 0) + iblk->first = *key; + if (scoutfs_key_cmp(key, &iblk->last) > 0) + iblk->last = *key; + brelse(bh); + /* XXX delete skip on failure? */ /* set the bloom bits last because we can't unset them */ From 6e209136610f22329885ca02eb9e204d4ebcab3d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Mar 2016 16:15:09 -0700 Subject: [PATCH 034/920] scoutfs: insert new manifests at highest level Manifests for newly written segments can be inserted at the highest level that doesn't have segments they intersect. This avoids ring and merging churn. The change cleans up the code a little bit, which is nice, and adds tracepoints for manifests entering and leaving the in memory structures. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 + kmod/src/manifest.c | 138 +++++++++++++++++++++++++++++---------- kmod/src/manifest.h | 4 +- kmod/src/ring.c | 2 +- kmod/src/scoutfs_trace.h | 74 +++++++++++++++++++++ kmod/src/segment.c | 2 +- 6 files changed, 182 insertions(+), 40 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index d5e69bcd..f1bc61b8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -161,6 +161,8 @@ struct scoutfs_ring_manifest_entry { struct scoutfs_key last; } __packed; +#define SCOUTFS_MANIFESTS_PER_LEVEL 10 + struct scoutfs_ring_del_manifest { __le64 blkno; } __packed; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index cc91cf25..23054330 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -19,6 +19,7 @@ #include "manifest.h" #include "key.h" #include "ring.h" +#include "scoutfs_trace.h" /* * The manifest organizes log segment blocks into a tree structure. @@ -42,6 +43,7 @@ struct scoutfs_manifest { struct scoutfs_level { struct rb_root root; + u64 count; } levels[SCOUTFS_MAX_LEVEL + 1]; }; @@ -111,11 +113,14 @@ static struct scoutfs_manifest_node *unlink_mnode(struct scoutfs_manifest *mani, mnode = radix_tree_lookup(&mani->blkno_radix, blkno); if (mnode) { + trace_scoutfs_delete_manifest(&mnode->ment); + if (!list_empty(&mnode->head)) list_del_init(&mnode->head); if (!RB_EMPTY_NODE(&mnode->node)) { rb_erase(&mnode->node, &mani->levels[mnode->ment.level].root); + mani->levels[mnode->ment.level].count--; RB_CLEAR_NODE(&mnode->node); } } @@ -144,61 +149,114 @@ void scoutfs_delete_manifest(struct super_block *sb, u64 blkno) } /* - * This is called during ring replay to reconstruct the manifest state - * from the ring entries. Moving segments between levels is recorded - * with a single ring entry so we always try to look up the segment in - * the manifest before we add it to the manifest. + * A newly inserted manifest can be inserted at the level + * above the first block that it intersects. */ -int scoutfs_add_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment) +static u8 insertion_level(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; struct scoutfs_manifest_node *mnode; + int i; + + list_for_each_entry(mnode, &mani->level_zero, head) { + if (scoutfs_cmp_key_ranges(&ment->first, &ment->last, + &mnode->ment.first, + &mnode->ment.last) == 0) + return 0; + } + + /* XXX this <= looks fishy :/ */ + for (i = 1; i <= SCOUTFS_MAX_LEVEL; i++) { + mnode = find_mnode(&mani->levels[i].root, &ment->first); + if (mnode) + break; + if (mani->levels[i].count < SCOUTFS_MANIFESTS_PER_LEVEL) + return i; + } + + return i - 1; +} + +/* + * Insert an manifest entry into the blkno radix and either level 0 list + * or greater level rbtrees as appropriate. The new entry will replace + * any existing entry at its blkno, perhaps with different keys and + * level. + * + * The caller can ask that we find the highest level that the entry can + * be inserted into before it intersects with an existing entry. The + * caller's entry is updated with the new level so they can store it in + * the ring. Doing so here avoids extra ring churn of doing it later in + * merging. + */ +static int insert_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment, + bool find_level) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnode; + struct scoutfs_manifest_node *found; u64 blkno = le64_to_cpu(ment->blkno); - bool preloaded = false; - int ret; + int ret = 0; + + /* allocation/preloading should be cheap enough to always try */ + mnode = kmalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); + if (!mnode) + return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ + + ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM); + if (ret) { + kfree(mnode); + return ret; + } + + INIT_LIST_HEAD(&mnode->head); + RB_CLEAR_NODE(&mnode->node); spin_lock(&mani->lock); - mnode = unlink_mnode(mani, blkno); - if (!mnode) { - spin_unlock(&mani->lock); - mnode = kmalloc(sizeof(struct scoutfs_manifest_node), - GFP_NOFS); - if (!mnode) - return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ - - ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM); - if (ret) { - kfree(mnode); - return ret; - } - preloaded = true; - - INIT_LIST_HEAD(&mnode->head); - RB_CLEAR_NODE(&mnode->node); - spin_lock(&mani->lock); - /* preloading should guarantee this succeeds */ + /* reuse found to avoid radix delete/insert churn */ + found = unlink_mnode(mani, blkno); + if (!found) { radix_tree_insert(&mani->blkno_radix, blkno, mnode); + } else { + swap(found, mnode); } + /* careful to find our level after deleting old blkno ment */ + if (find_level) + ment->level = insertion_level(sb, ment); + + trace_scoutfs_insert_manifest(ment); + mnode->ment = *ment; - if (ment->level) + if (ment->level) { insert_mnode(&mani->levels[ment->level].root, mnode); - else + mani->levels[ment->level].count++; + } else { list_add(&mnode->head, &mani->level_zero); + } spin_unlock(&mani->lock); - if (preloaded) - radix_tree_preload_end(); + radix_tree_preload_end(); + kfree(found); return 0; } +/* Index an existing entry */ +int scoutfs_insert_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment) +{ + return insert_manifest(sb, ment, false); +} + /* - * The caller is writing a new log segment. We add it to the in-memory - * manifest and write it to dirty ring blocks. + * Add an entry for a newly written segment to the indexes and record it + * in the ring. The entry can be modified by insertion. * * XXX we'd also need to add stale manifest entry's to the ring * XXX In the future we'd send it to the leader @@ -206,9 +264,17 @@ int scoutfs_add_manifest(struct super_block *sb, int scoutfs_new_manifest(struct super_block *sb, struct scoutfs_ring_manifest_entry *ment) { - return scoutfs_add_manifest(sb, ment) ?: - scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, - ment, sizeof(*ment)); + int ret; + + ret = insert_manifest(sb, ment, true); + if (!ret) { + ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, + ment, sizeof(*ment)); + if (ret) + scoutfs_delete_manifest(sb, le64_to_cpu(ment->blkno)); + } + + return ret; } /* diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index bab32764..ea3eef18 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -4,8 +4,8 @@ int scoutfs_setup_manifest(struct super_block *sb); void scoutfs_destroy_manifest(struct super_block *sb); -int scoutfs_add_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment); +int scoutfs_insert_manifest(struct super_block *sb, + struct scoutfs_ring_manifest_entry *ment); int scoutfs_new_manifest(struct super_block *sb, struct scoutfs_ring_manifest_entry *ment); void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 19642e21..6743fb9f 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -40,7 +40,7 @@ static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) switch(ent->type) { case SCOUTFS_RING_ADD_MANIFEST: ment = (void *)(ent + 1); - ret = scoutfs_add_manifest(sb, ment); + ret = scoutfs_insert_manifest(sb, ment); break; case SCOUTFS_RING_DEL_MANIFEST: del = (void *)(ent + 1); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5e95258b..51da5c23 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -24,6 +24,8 @@ #include +#include "key.h" + TRACE_EVENT(scoutfs_bloom_hit, TP_PROTO(struct scoutfs_key *key), @@ -186,6 +188,78 @@ TRACE_EVENT(scoutfs_write_super, __entry->blkno, __entry->seq) ); +TRACE_EVENT(scoutfs_insert_manifest, + TP_PROTO(struct scoutfs_ring_manifest_entry *ment), + + TP_ARGS(ment), + + TP_STRUCT__entry( + __field(__u64, blkno) + __field(__u64, seq) + __field(__u8, level) + __field(__u64, first_inode) + __field(__u8, first_type) + __field(__u64, first_offset) + __field(__u64, last_inode) + __field(__u8, last_type) + __field(__u64, last_offset) + ), + + TP_fast_assign( + __entry->blkno = le64_to_cpu(ment->blkno); + __entry->seq = le64_to_cpu(ment->seq); + __entry->level = ment->level; + __entry->first_inode = le64_to_cpu(ment->first.inode); + __entry->first_type = ment->first.type; + __entry->first_offset = le64_to_cpu(ment->first.offset); + __entry->last_inode = le64_to_cpu(ment->last.inode); + __entry->last_type = ment->last.type; + __entry->last_offset = le64_to_cpu(ment->last.offset); + ), + + TP_printk("blkno %llu seq %llu level %u first "CKF" last "CKF, + __entry->blkno, __entry->seq, __entry->level, + __entry->first_inode, __entry->first_type, + __entry->first_offset, __entry->last_inode, + __entry->last_type, __entry->last_offset) +); + +TRACE_EVENT(scoutfs_delete_manifest, + TP_PROTO(struct scoutfs_ring_manifest_entry *ment), + + TP_ARGS(ment), + + TP_STRUCT__entry( + __field(__u64, blkno) + __field(__u64, seq) + __field(__u8, level) + __field(__u64, first_inode) + __field(__u8, first_type) + __field(__u64, first_offset) + __field(__u64, last_inode) + __field(__u8, last_type) + __field(__u64, last_offset) + ), + + TP_fast_assign( + __entry->blkno = le64_to_cpu(ment->blkno); + __entry->seq = le64_to_cpu(ment->seq); + __entry->level = ment->level; + __entry->first_inode = le64_to_cpu(ment->first.inode); + __entry->first_type = ment->first.type; + __entry->first_offset = le64_to_cpu(ment->first.offset); + __entry->last_inode = le64_to_cpu(ment->last.inode); + __entry->last_type = ment->last.type; + __entry->last_offset = le64_to_cpu(ment->last.offset); + ), + + TP_printk("blkno %llu seq %llu level %u first "CKF" last "CKF, + __entry->blkno, __entry->seq, __entry->level, + __entry->first_inode, __entry->first_type, + __entry->first_offset, __entry->last_inode, + __entry->last_type, __entry->last_offset) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/segment.c b/kmod/src/segment.c index a3251eea..c5504626 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -350,7 +350,7 @@ static int update_dirty_segment_manifest(struct super_block *sb, u64 blkno, } if (all_items) - ret = scoutfs_add_manifest(sb, &ment); + ret = scoutfs_insert_manifest(sb, &ment); else ret = scoutfs_new_manifest(sb, &ment); out: From 7a565a69df6977eed77d90cf2bc4b588b486f304 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 31 Mar 2016 16:44:37 -0700 Subject: [PATCH 035/920] scoutfs: add percpu coutners with sysfs files Add percpu counters that will let us track all manner of things. To report them we add a sysfs directory full of attribute files in a sysfs dir for each mount: # (cd /sys/fs/scoutfs/loop0/counters && grep . *) skip_delete:0 skip_insert:3218 skip_lookup:8439 skip_next:1190 skip_search:156 The implementation is careful to define each counter in only one place. We don't have to make sure that a bunch of defintions and arrays are in sync. This builds off of Ben's initial patches that added sysfs dirs. Signed-off-by: Zach Brown Signed-off-by: Ben McClelland --- kmod/src/Makefile | 4 +- kmod/src/counters.c | 131 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/counters.h | 47 ++++++++++++++++ kmod/src/skip.c | 9 +++ kmod/src/super.c | 51 ++++++++++++----- kmod/src/super.h | 6 ++ 6 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 kmod/src/counters.c create mode 100644 kmod/src/counters.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 31093000..da32ee19 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o bloom.o chunk.o crc.o dir.o filerw.o inode.o manifest.o \ - msg.o ring.o scoutfs_trace.o segment.o skip.o super.o +scoutfs-y += block.o bloom.o counters.o chunk.o crc.o dir.o filerw.o inode.o \ + manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o super.o diff --git a/kmod/src/counters.c b/kmod/src/counters.c new file mode 100644 index 00000000..fe8b247c --- /dev/null +++ b/kmod/src/counters.c @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "counters.h" + +/* + * Maintain simple percpu counters which are always ticking. sysfs + * makes this a whole lot more noisy than it needs to be. + */ + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) { .name = __stringify(which), .mode = 0644 }, +static struct attribute scoutfs_counter_attrs[] = { + EXPAND_EACH_COUNTER +}; + +/* zero BSS and + 1 makes this null terminated */ +#define NR_ATTRS ARRAY_SIZE(scoutfs_counter_attrs) +static struct attribute *scoutfs_counter_attr_ptrs[NR_ATTRS + 1]; + +static ssize_t scoutfs_counter_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + size_t index; + + /* use the index in the _attrs array to discover the pcpu pointer */ + counters = container_of(kobj, struct scoutfs_counters, kobj); + index = attr - scoutfs_counter_attrs; + pcpu = &counters->FIRST_COUNTER + index; + + return snprintf(buf, PAGE_SIZE, "%lld\n", percpu_counter_sum(pcpu)); +} + +static void scoutfs_counters_kobj_release(struct kobject *kobj) +{ + struct scoutfs_counters *counters; + + counters = container_of(kobj, struct scoutfs_counters, kobj); + + complete(&counters->comp); +} + +static const struct sysfs_ops scoutfs_counter_attr_ops = { + .show = scoutfs_counter_attr_show, +}; + +static struct kobj_type scoutfs_counters_ktype = { + .default_attrs = scoutfs_counter_attr_ptrs, + .sysfs_ops = &scoutfs_counter_attr_ops, + .release = scoutfs_counters_kobj_release, +}; + +int scoutfs_setup_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + int ret; + + counters = kzalloc(sizeof(struct scoutfs_counters), GFP_KERNEL); + if (!counters) + return -ENOMEM; + sbi->counters = counters; + + scoutfs_foreach_counter(sb, pcpu) { + ret = percpu_counter_init(pcpu, 0, GFP_KERNEL); + if (ret) + return ret; + } + + counters->kobj.kset = sbi->kset; + init_completion(&counters->comp); + ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype, + NULL, "counters"); + if (ret) { + /* tear down partial to avoid destroying null kobjs */ + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + kfree(counters); + sbi->counters = NULL; + } + + return ret; +} + +void scoutfs_destroy_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters = sbi->counters; + struct percpu_counter *pcpu; + + /* this only destroys fully initialized counters */ + if (!counters) + return; + + kobject_del(&counters->kobj); + kobject_put(&counters->kobj); + wait_for_completion(&counters->comp); + + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + + kfree(counters); + sbi->counters = NULL; +} + +void __init scoutfs_init_counters(void) +{ + int i; + + /* not ARRAY_SIZE because that would clobber null term */ + for (i = 0; i < NR_ATTRS; i++) + scoutfs_counter_attr_ptrs[i] = &scoutfs_counter_attrs[i]; +} diff --git a/kmod/src/counters.h b/kmod/src/counters.h new file mode 100644 index 00000000..1f6c1843 --- /dev/null +++ b/kmod/src/counters.h @@ -0,0 +1,47 @@ +#ifndef _SCOUTFS_COUNTERS_H_ +#define _SCOUTFS_COUNTERS_H_ + +#include +#include +#include + +#include "super.h" + +/* + * We only have to define each counter here and it'll be enumerated in + * other places by this macro. Don't forget to update LAST_COUNTER. + */ +#define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(skip_lookup) \ + EXPAND_COUNTER(skip_insert) \ + EXPAND_COUNTER(skip_search) \ + EXPAND_COUNTER(skip_delete) \ + EXPAND_COUNTER(skip_next) \ + +#define FIRST_COUNTER skip_lookup +#define LAST_COUNTER skip_next + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) struct percpu_counter which; + +struct scoutfs_counters { + /* $sysfs/fs/scoutfs/$id/counters/ */ + struct kobject kobj; + struct completion comp; + + EXPAND_EACH_COUNTER +}; + +#define scoutfs_foreach_counter(sb, pcpu) \ + for (pcpu = &SCOUTFS_SB(sb)->counters->FIRST_COUNTER; \ + pcpu <= &SCOUTFS_SB(sb)->counters->LAST_COUNTER; \ + pcpu++) + +#define scoutfs_inc_counter(sb, which) \ + percpu_counter_inc(&SCOUTFS_SB(sb)->counters->which) + +void __init scoutfs_init_counters(void); +int scoutfs_setup_counters(struct super_block *sb); +void scoutfs_destroy_counters(struct super_block *sb); + +#endif diff --git a/kmod/src/skip.c b/kmod/src/skip.c index c855ae49..d320e2a7 100644 --- a/kmod/src/skip.c +++ b/kmod/src/skip.c @@ -19,6 +19,7 @@ #include "key.h" #include "block.h" #include "skip.h" +#include "counters.h" /* * The items in a log segment block are sorted by their keys in a skip @@ -216,6 +217,8 @@ int scoutfs_skip_insert(struct super_block *sb, u64 blkno, WARN_ON_ONCE(item->skip_height > SCOUTFS_SKIP_HEIGHT)) return -EINVAL; + scoutfs_inc_counter(sb, skip_insert); + ret = skip_search(sb, blkno, &path, &item->key, &cmp); if (ret == 0) { if (cmp == 0) { @@ -261,6 +264,7 @@ int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, struct scoutfs_key *key, struct buffer_head **bh, struct scoutfs_item **item) { + scoutfs_inc_counter(sb, skip_lookup); return skip_lookup(sb, blkno, key, bh, item, true); } @@ -271,6 +275,7 @@ int scoutfs_skip_search(struct super_block *sb, u64 blkno, struct scoutfs_key *key, struct buffer_head **bh, struct scoutfs_item **item) { + scoutfs_inc_counter(sb, skip_search); return skip_lookup(sb, blkno, key, bh, item, false); } @@ -284,6 +289,8 @@ int scoutfs_skip_delete(struct super_block *sb, u64 blkno, int ret; int i; + scoutfs_inc_counter(sb, skip_delete); + ret = skip_search(sb, blkno, &path, key, &cmp); if (ret == 0) { if (*path.next[0] && cmp) { @@ -316,6 +323,8 @@ int scoutfs_skip_next(struct super_block *sb, u64 blkno, if (!(*bh)) return -ENOENT; + scoutfs_inc_counter(sb, skip_next); + next = (*item)->skip_next[0]; brelse(*bh); diff --git a/kmod/src/super.c b/kmod/src/super.c index f0c2d6c6..95159ca3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -27,8 +27,11 @@ #include "manifest.h" #include "ring.h" #include "segment.h" +#include "counters.h" #include "scoutfs_trace.h" +static struct kset *scoutfs_kset; + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, @@ -178,15 +181,15 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -EINVAL; } - ret = read_supers(sb); - if (ret) - return ret; + /* XXX can have multiple mounts of a device, need mount id */ + sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); + if (!sbi->kset) + return -ENOMEM; - ret = scoutfs_setup_manifest(sb); - if (ret) - return ret; - - ret = scoutfs_replay_ring(sb); + ret = scoutfs_setup_counters(sb) ?: + read_supers(sb) ?: + scoutfs_setup_manifest(sb) ?: + scoutfs_replay_ring(sb); if (ret) return ret; @@ -218,6 +221,9 @@ static void scoutfs_kill_sb(struct super_block *sb) /* kill block super should have synced */ WARN_ON_ONCE(sbi->dirty_blkno); scoutfs_destroy_manifest(sb); + scoutfs_destroy_counters(sb); + if (sbi->kset) + kset_unregister(sbi->kset); kfree(sbi); } } @@ -230,19 +236,38 @@ static struct file_system_type scoutfs_fs_type = { .fs_flags = FS_REQUIRES_DEV, }; +/* safe to call at any failure point in _init */ +static void teardown_module(void) +{ + scoutfs_dir_exit(); + scoutfs_inode_exit(); + if (scoutfs_kset) + kset_unregister(scoutfs_kset); +} + static int __init scoutfs_module_init(void) { - return scoutfs_inode_init() ?: - scoutfs_dir_init() ?: - register_filesystem(&scoutfs_fs_type); + int ret; + + scoutfs_init_counters(); + + scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); + if (!scoutfs_kset) + return -ENOMEM; + + ret = scoutfs_inode_init() ?: + scoutfs_dir_init() ?: + register_filesystem(&scoutfs_fs_type); + if (ret) + teardown_module(); + return ret; } module_init(scoutfs_module_init) static void __exit scoutfs_module_exit(void) { unregister_filesystem(&scoutfs_fs_type); - scoutfs_dir_exit(); - scoutfs_inode_exit(); + teardown_module(); } module_exit(scoutfs_module_exit) diff --git a/kmod/src/super.h b/kmod/src/super.h index 1663034d..0857faf0 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,6 +5,7 @@ #include "format.h" struct scoutfs_manifest; +struct scoutfs_counters; struct scoutfs_sb_info { struct scoutfs_super_block super; @@ -31,6 +32,11 @@ struct scoutfs_sb_info { u64 dirty_blkno; int dirty_item_off; int dirty_val_off; + + /* $sysfs/fs/scoutfs/$id/ */ + struct kset *kset; + + struct scoutfs_counters *counters; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) From d91dc45368e8c66b88cc3058c62d7e804394deab Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 1 Apr 2016 14:51:04 -0700 Subject: [PATCH 036/920] scoutfs: add interval tree Add an interval tree that lets us efficiently discover intervals that overlap a given search region. We're going to need this now to sanely implementing merging and in the future to implement granting access ranges. It's easy to implement an interval tree by using the kernel's augmented rbtree to track the max end value of the subtree of intervals. The tricky bit is that the augmented interface assumes that it can directly compare the augmented value. If we were developing against mainline we'd just patch the interface. But we're developing against distro kernels that development partners deploy so the kernel is frozen in amber. We deploy a giant stinky hack to import a private tweaked version of the interface. It's isolated so we can trivially drop it once we merge with the fixed upstream interface. We also add some build time checks to make sure that we don't accidentally combine rb structures between the private import and the main kernel interface. Signed-off-by: Zach Brown --- kmod/src/Makefile | 3 +- kmod/src/ival.c | 149 +++++++ kmod/src/ival.h | 56 +++ kmod/src/key.h | 6 + kmod/src/rbtree_aug.h | 996 ++++++++++++++++++++++++++++++++++++++++++ kmod/src/super.c | 6 + 6 files changed, 1215 insertions(+), 1 deletion(-) create mode 100644 kmod/src/ival.c create mode 100644 kmod/src/ival.h create mode 100644 kmod/src/rbtree_aug.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index da32ee19..31d960ef 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,4 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += block.o bloom.o counters.o chunk.o crc.o dir.o filerw.o inode.o \ - manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o super.o + ival.o manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o \ + super.o diff --git a/kmod/src/ival.c b/kmod/src/ival.c new file mode 100644 index 00000000..e9e51da5 --- /dev/null +++ b/kmod/src/ival.c @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include "rbtree_aug.h" + +#include "format.h" +#include "key.h" +#include "ival.h" + +/* + * scoutfs wants to store overlapping key ranges and find intersections + * for tracking both segments in level 0 and granting access ranges. + * + * We use a simple augmented rbtree of key intervals that tracks the + * greatest end value of all the intervals in a node's subtree. Wikipedia + * data structures 101. + * + * Unfortunately the augmented rbtree callbacks need a tweak to compare + * our key structs. But we don't want to mess around with updating + * distro kernels. So we backport the augmented rbtree code from + * mainline in a private copy. This'll vanish when we bring scoutfs up + * to mainline. + */ + +static struct scoutfs_key *node_subtree_end(struct rb_node *node) +{ + struct scoutfs_ival *ival; + static struct scoutfs_key static_zero = {0,}; + + if (!node) + return &static_zero; + + ival = container_of(node, struct scoutfs_ival, node); + return &ival->subtree_end; +} + +static struct scoutfs_key compute_subtree_end(struct scoutfs_ival *ival) +{ + return *scoutfs_max_key(node_subtree_end(ival->node.rb_left), + node_subtree_end(ival->node.rb_right)); +} + +RB_DECLARE_CALLBACKS(static, ival_rb_cb, struct scoutfs_ival, node, + struct scoutfs_key, subtree_end, compute_subtree_end) + +void scoutfs_insert_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_ival *ins) +{ + struct rb_node **node = &tree->root.rb_node; + struct rb_node *parent = NULL; + struct scoutfs_ival *ival; + + giant_rbtree_hack_build_bugs(); + + while (*node) { + parent = *node; + ival = container_of(*node, struct scoutfs_ival, node); + + /* extend traversed subtree end to cover inserted end */ + ival->subtree_end = *scoutfs_max_key(&ival->subtree_end, + &ins->end); + + /* XXX <= and >= consistent? */ + if (scoutfs_key_cmp(&ins->start, &ival->start) < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + ins->subtree_end = ins->end; + rb_link_node(&ins->node, parent, node); + rb_insert_augmented(&ins->node, &tree->root, &ival_rb_cb); +} + +void scoutfs_remove_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_ival *ival) +{ + if (!RB_EMPTY_NODE(&ival->node)) { + rb_erase_augmented(&ival->node, &tree->root, &ival_rb_cb); + RB_CLEAR_NODE(&ival->node); + } +} + +/* + * Find the interval in the tree with the lowest start value that + * intersects the search range. + */ +static struct scoutfs_ival *first_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + struct rb_node *node = tree->root.rb_node; + struct scoutfs_ival *ival; + + while (node) { + ival = container_of(node, struct scoutfs_ival, node); + + if (scoutfs_key_cmp(node_subtree_end(ival->node.rb_left), + start) >= 0) + node = node->rb_left; + else if (!scoutfs_cmp_key_ranges(start, end, + &ival->start, &ival->end)) + return ival; + else if (scoutfs_key_cmp(end, &ival->start) < 0) + break; + else + node = node->rb_right; + } + + return NULL; +} + +/* + * Find the next interval sorted by the start value which intersect the + * given search range. ival is null to first return the intersection + * with the lowest start value. The caller must serialize access while + * iterating. + */ +struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct scoutfs_ival *ival) +{ + struct rb_node *node; + + if (!ival) + return first_ival(tree, start, end); + + while ((node = rb_next(&ival->node))) { + ival = container_of(node, struct scoutfs_ival, node); + + if (scoutfs_cmp_key_ranges(start, end, + &ival->start, &ival->end)) + ival = NULL; + break; + } + + return ival; +} diff --git a/kmod/src/ival.h b/kmod/src/ival.h new file mode 100644 index 00000000..4e693e61 --- /dev/null +++ b/kmod/src/ival.h @@ -0,0 +1,56 @@ +#ifndef _SCOUTFS_IVAL_H_ +#define _SCOUTFS_IVAL_H_ + +struct scoutfs_ival_tree { + struct rb_root root; +}; + +struct scoutfs_ival { + struct rb_node node; + struct scoutfs_key start; + struct scoutfs_key end; + struct scoutfs_key subtree_end; +}; + +void scoutfs_insert_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_ival *ins); +void scoutfs_remove_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_ival *ival); +struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct scoutfs_ival *ival); + +// struct rb_node { +// long unsigned int __rb_parent_color; /* 0 8 */ +// struct rb_node * rb_right; /* 8 8 */ +// struct rb_node * rb_left; /* 16 8 */ +// +// /* size: 24, cachelines: 1, members: 3 */ +// /* last cacheline: 24 bytes */ +// }; +// struct rb_root { +// struct rb_node * rb_node; /* 0 8 */ +// +// /* size: 8, cachelines: 1, members: 1 */ +// /* last cacheline: 8 bytes */ +// }; + +/* + * Try to find out if the imported hacked rbtree in ival.c goes out of + * sync with the rbtree in the distro kernel. + */ +static inline void giant_rbtree_hack_build_bugs(void) +{ + size_t sz = sizeof(long); + + BUILD_BUG_ON(offsetof(struct rb_node, __rb_parent_color) != 0); + BUILD_BUG_ON(offsetof(struct rb_node, rb_right) != sz); + BUILD_BUG_ON(offsetof(struct rb_node, rb_left) != (sz * 2)); + BUILD_BUG_ON(sizeof(struct rb_node) != (sz * 3)); + + BUILD_BUG_ON(offsetof(struct rb_root, rb_node) != 0); + BUILD_BUG_ON(sizeof(struct rb_root) != sz); +} + +#endif diff --git a/kmod/src/key.h b/kmod/src/key.h index c06f898c..b61967d0 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -71,4 +71,10 @@ static inline void scoutfs_inc_key(struct scoutfs_key *key) } } +static inline struct scoutfs_key *scoutfs_max_key(struct scoutfs_key *a, + struct scoutfs_key *b) +{ + return scoutfs_key_cmp(a, b) > 0 ? a : b; +} + #endif diff --git a/kmod/src/rbtree_aug.h b/kmod/src/rbtree_aug.h new file mode 100644 index 00000000..97791df0 --- /dev/null +++ b/kmod/src/rbtree_aug.h @@ -0,0 +1,996 @@ +/* + * The upstream augmented rbtree interface currently assumes that it + * can compare the augmented values directly: + * + * if (node->rbaugmented == augmented) + * break; + * + * This doesn't work for our struct key types. The only change needed + * to make this work for us is to turn that into a memcmp. But we're + * developing against distro kernels that sites actually use. For now + * we carry around this giant hack that imports the upstream copy and + * makes the change. It's only used in ival.c. + * + * This is a disgusting hack and also the right thing for this stage of + * the project. We'll fix this up as we submit upstream and trickle + * into distro kernels. + */ +#ifndef _GIANT_RBTREE_HACK_ +#define _GIANT_RBTREE_HACK_ + +/* forbid including kernel rbtree headers by way of includes below */ +#define _LINUX_RBTREE_AUGMENTED_H +#define _LINUX_RBTREE_H + +#include +#include +#include +#include + +#undef EXPORT_SYMBOL +#define EXPORT_SYMBOL(foo) + +/* + * then paste rbtree.h, rbtree_augmented.h, and rbtree.c + */ + +/* --------- rbtree.h ---------- */ + +/* + Red Black Trees + (C) 1999 Andrea Arcangeli + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + linux/include/linux/rbtree.h + + To use rbtrees you'll have to implement your own insert and search cores. + This will avoid us to use callbacks and to drop drammatically performances. + I know it's not the cleaner way, but in C (not in C++) to get + performances and genericity... + + See Documentation/rbtree.txt for documentation and samples. +*/ + + +struct rb_node { + unsigned long __rb_parent_color; + struct rb_node *rb_right; + struct rb_node *rb_left; +} __attribute__((aligned(sizeof(long)))); + /* The alignment might seem pointless, but allegedly CRIS needs it */ + +struct rb_root { + struct rb_node *rb_node; +}; + + +#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) + +#define RB_ROOT (struct rb_root) { NULL, } +#define rb_entry(ptr, type, member) container_of(ptr, type, member) + +#define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL) + +/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */ +#define RB_EMPTY_NODE(node) \ + ((node)->__rb_parent_color == (unsigned long)(node)) +#define RB_CLEAR_NODE(node) \ + ((node)->__rb_parent_color = (unsigned long)(node)) + + +extern void rb_insert_color(struct rb_node *, struct rb_root *); +extern void rb_erase(struct rb_node *, struct rb_root *); + + +/* Find logical next and previous nodes in a tree */ +extern struct rb_node *rb_next(const struct rb_node *); +extern struct rb_node *rb_prev(const struct rb_node *); +extern struct rb_node *rb_first(const struct rb_root *); +extern struct rb_node *rb_last(const struct rb_root *); + +/* Postorder iteration - always visit the parent after its children */ +extern struct rb_node *rb_first_postorder(const struct rb_root *); +extern struct rb_node *rb_next_postorder(const struct rb_node *); + +/* Fast replacement of a single node without remove/rebalance/add/rebalance */ +extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, + struct rb_root *root); + +static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) +{ + node->__rb_parent_color = (unsigned long)parent; + node->rb_left = node->rb_right = NULL; + + *rb_link = node; +} + +static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) +{ + node->__rb_parent_color = (unsigned long)parent; + node->rb_left = node->rb_right = NULL; + + rcu_assign_pointer(*rb_link, node); +} + +#define rb_entry_safe(ptr, type, member) \ + ({ typeof(ptr) ____ptr = (ptr); \ + ____ptr ? rb_entry(____ptr, type, member) : NULL; \ + }) + +/** + * rbtree_postorder_for_each_entry_safe - iterate in post-order over rb_root of + * given type allowing the backing memory of @pos to be invalidated + * + * @pos: the 'type *' to use as a loop cursor. + * @n: another 'type *' to use as temporary storage + * @root: 'rb_root *' of the rbtree. + * @field: the name of the rb_node field within 'type'. + * + * rbtree_postorder_for_each_entry_safe() provides a similar guarantee as + * list_for_each_entry_safe() and allows the iteration to continue independent + * of changes to @pos by the body of the loop. + * + * Note, however, that it cannot handle other modifications that re-order the + * rbtree it is iterating over. This includes calling rb_erase() on @pos, as + * rb_erase() may rebalance the tree, causing us to miss some nodes. + */ +#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \ + for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \ + pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \ + typeof(*pos), field); 1; }); \ + pos = n) + +/* --------- rbtree_augmented.h ---------- */ + +/* + Red Black Trees + (C) 1999 Andrea Arcangeli + (C) 2002 David Woodhouse + (C) 2012 Michel Lespinasse + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + linux/include/linux/rbtree_augmented.h +*/ + + +/* + * Please note - only struct rb_augment_callbacks and the prototypes for + * rb_insert_augmented() and rb_erase_augmented() are intended to be public. + * The rest are implementation details you are not expected to depend on. + * + * See Documentation/rbtree.txt for documentation and samples. + */ + +struct rb_augment_callbacks { + void (*propagate)(struct rb_node *node, struct rb_node *stop); + void (*copy)(struct rb_node *old, struct rb_node *new); + void (*rotate)(struct rb_node *old, struct rb_node *new); +}; + +extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); +/* + * Fixup the rbtree and update the augmented information when rebalancing. + * + * On insertion, the user must update the augmented information on the path + * leading to the inserted node, then call rb_link_node() as usual and + * rb_augment_inserted() instead of the usual rb_insert_color() call. + * If rb_augment_inserted() rebalances the rbtree, it will callback into + * a user provided function to update the augmented information on the + * affected subtrees. + */ +static inline void +rb_insert_augmented(struct rb_node *node, struct rb_root *root, + const struct rb_augment_callbacks *augment) +{ + __rb_insert_augmented(node, root, augment->rotate); +} + +#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \ + rbtype, rbaugmented, rbcompute) \ +static inline void \ +rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \ +{ \ + while (rb != stop) { \ + rbstruct *node = rb_entry(rb, rbstruct, rbfield); \ + rbtype augmented = rbcompute(node); \ + if (!memcmp(&node->rbaugmented, &augmented, \ + sizeof(augmented))) \ + break; \ + node->rbaugmented = augmented; \ + rb = rb_parent(&node->rbfield); \ + } \ +} \ +static inline void \ +rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ +{ \ + rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ + rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ + new->rbaugmented = old->rbaugmented; \ +} \ +static void \ +rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ +{ \ + rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ + rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ + new->rbaugmented = old->rbaugmented; \ + old->rbaugmented = rbcompute(old); \ +} \ +rbstatic const struct rb_augment_callbacks rbname = { \ + rbname ## _propagate, rbname ## _copy, rbname ## _rotate \ +}; + + +#define RB_RED 0 +#define RB_BLACK 1 + +#define __rb_parent(pc) ((struct rb_node *)(pc & ~3)) + +#define __rb_color(pc) ((pc) & 1) +#define __rb_is_black(pc) __rb_color(pc) +#define __rb_is_red(pc) (!__rb_color(pc)) +#define rb_color(rb) __rb_color((rb)->__rb_parent_color) +#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color) +#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color) + +static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) +{ + rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; +} + +static inline void rb_set_parent_color(struct rb_node *rb, + struct rb_node *p, int color) +{ + rb->__rb_parent_color = (unsigned long)p | color; +} + +static inline void +__rb_change_child(struct rb_node *old, struct rb_node *new, + struct rb_node *parent, struct rb_root *root) +{ + if (parent) { + if (parent->rb_left == old) + WRITE_ONCE(parent->rb_left, new); + else + WRITE_ONCE(parent->rb_right, new); + } else + WRITE_ONCE(root->rb_node, new); +} + +extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); + +static __always_inline struct rb_node * +__rb_erase_augmented(struct rb_node *node, struct rb_root *root, + const struct rb_augment_callbacks *augment) +{ + struct rb_node *child = node->rb_right; + struct rb_node *tmp = node->rb_left; + struct rb_node *parent, *rebalance; + unsigned long pc; + + if (!tmp) { + /* + * Case 1: node to erase has no more than 1 child (easy!) + * + * Note that if there is one child it must be red due to 5) + * and node must be black due to 4). We adjust colors locally + * so as to bypass __rb_erase_color() later on. + */ + pc = node->__rb_parent_color; + parent = __rb_parent(pc); + __rb_change_child(node, child, parent, root); + if (child) { + child->__rb_parent_color = pc; + rebalance = NULL; + } else + rebalance = __rb_is_black(pc) ? parent : NULL; + tmp = parent; + } else if (!child) { + /* Still case 1, but this time the child is node->rb_left */ + tmp->__rb_parent_color = pc = node->__rb_parent_color; + parent = __rb_parent(pc); + __rb_change_child(node, tmp, parent, root); + rebalance = NULL; + tmp = parent; + } else { + struct rb_node *successor = child, *child2; + + tmp = child->rb_left; + if (!tmp) { + /* + * Case 2: node's successor is its right child + * + * (n) (s) + * / \ / \ + * (x) (s) -> (x) (c) + * \ + * (c) + */ + parent = successor; + child2 = successor->rb_right; + + augment->copy(node, successor); + } else { + /* + * Case 3: node's successor is leftmost under + * node's right child subtree + * + * (n) (s) + * / \ / \ + * (x) (y) -> (x) (y) + * / / + * (p) (p) + * / / + * (s) (c) + * \ + * (c) + */ + do { + parent = successor; + successor = tmp; + tmp = tmp->rb_left; + } while (tmp); + child2 = successor->rb_right; + WRITE_ONCE(parent->rb_left, child2); + WRITE_ONCE(successor->rb_right, child); + rb_set_parent(child, successor); + + augment->copy(node, successor); + augment->propagate(parent, successor); + } + + tmp = node->rb_left; + WRITE_ONCE(successor->rb_left, tmp); + rb_set_parent(tmp, successor); + + pc = node->__rb_parent_color; + tmp = __rb_parent(pc); + __rb_change_child(node, successor, tmp, root); + + if (child2) { + successor->__rb_parent_color = pc; + rb_set_parent_color(child2, parent, RB_BLACK); + rebalance = NULL; + } else { + unsigned long pc2 = successor->__rb_parent_color; + successor->__rb_parent_color = pc; + rebalance = __rb_is_black(pc2) ? parent : NULL; + } + tmp = successor; + } + + augment->propagate(tmp, NULL); + return rebalance; +} + +static __always_inline void +rb_erase_augmented(struct rb_node *node, struct rb_root *root, + const struct rb_augment_callbacks *augment) +{ + struct rb_node *rebalance = __rb_erase_augmented(node, root, augment); + if (rebalance) + __rb_erase_color(rebalance, root, augment->rotate); +} + +/* --------- rbtree.c ---------- */ + +/* + Red Black Trees + (C) 1999 Andrea Arcangeli + (C) 2002 David Woodhouse + (C) 2012 Michel Lespinasse + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + linux/lib/rbtree.c +*/ + +/* + * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree + * + * 1) A node is either red or black + * 2) The root is black + * 3) All leaves (NULL) are black + * 4) Both children of every red node are black + * 5) Every simple path from root to leaves contains the same number + * of black nodes. + * + * 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two + * consecutive red nodes in a path and every red node is therefore followed by + * a black. So if B is the number of black nodes on every simple path (as per + * 5), then the longest possible path due to 4 is 2B. + * + * We shall indicate color with case, where black nodes are uppercase and red + * nodes will be lowercase. Unknown color nodes shall be drawn as red within + * parentheses and have some accompanying text comment. + */ + +/* + * Notes on lockless lookups: + * + * All stores to the tree structure (rb_left and rb_right) must be done using + * WRITE_ONCE(). And we must not inadvertently cause (temporary) loops in the + * tree structure as seen in program order. + * + * These two requirements will allow lockless iteration of the tree -- not + * correct iteration mind you, tree rotations are not atomic so a lookup might + * miss entire subtrees. + * + * But they do guarantee that any such traversal will only see valid elements + * and that it will indeed complete -- does not get stuck in a loop. + * + * It also guarantees that if the lookup returns an element it is the 'correct' + * one. But not returning an element does _NOT_ mean it's not present. + * + * NOTE: + * + * Stores to __rb_parent_color are not important for simple lookups so those + * are left undone as of now. Nor did I check for loops involving parent + * pointers. + */ + +static inline void rb_set_black(struct rb_node *rb) +{ + rb->__rb_parent_color |= RB_BLACK; +} + +static inline struct rb_node *rb_red_parent(struct rb_node *red) +{ + return (struct rb_node *)red->__rb_parent_color; +} + +/* + * Helper function for rotations: + * - old's parent and color get assigned to new + * - old gets assigned new as a parent and 'color' as a color. + */ +static inline void +__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, + struct rb_root *root, int color) +{ + struct rb_node *parent = rb_parent(old); + new->__rb_parent_color = old->__rb_parent_color; + rb_set_parent_color(old, new, color); + __rb_change_child(old, new, parent, root); +} + +static __always_inline void +__rb_insert(struct rb_node *node, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) +{ + struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; + + while (true) { + /* + * Loop invariant: node is red + * + * If there is a black parent, we are done. + * Otherwise, take some corrective action as we don't + * want a red root or two consecutive red nodes. + */ + if (!parent) { + rb_set_parent_color(node, NULL, RB_BLACK); + break; + } else if (rb_is_black(parent)) + break; + + gparent = rb_red_parent(parent); + + tmp = gparent->rb_right; + if (parent != tmp) { /* parent == gparent->rb_left */ + if (tmp && rb_is_red(tmp)) { + /* + * Case 1 - color flips + * + * G g + * / \ / \ + * p u --> P U + * / / + * n n + * + * However, since g's parent might be red, and + * 4) does not allow this, we need to recurse + * at g. + */ + rb_set_parent_color(tmp, gparent, RB_BLACK); + rb_set_parent_color(parent, gparent, RB_BLACK); + node = gparent; + parent = rb_parent(node); + rb_set_parent_color(node, parent, RB_RED); + continue; + } + + tmp = parent->rb_right; + if (node == tmp) { + /* + * Case 2 - left rotate at parent + * + * G G + * / \ / \ + * p U --> n U + * \ / + * n p + * + * This still leaves us in violation of 4), the + * continuation into Case 3 will fix that. + */ + tmp = node->rb_left; + WRITE_ONCE(parent->rb_right, tmp); + WRITE_ONCE(node->rb_left, parent); + if (tmp) + rb_set_parent_color(tmp, parent, + RB_BLACK); + rb_set_parent_color(parent, node, RB_RED); + augment_rotate(parent, node); + parent = node; + tmp = node->rb_right; + } + + /* + * Case 3 - right rotate at gparent + * + * G P + * / \ / \ + * p U --> n g + * / \ + * n U + */ + WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */ + WRITE_ONCE(parent->rb_right, gparent); + if (tmp) + rb_set_parent_color(tmp, gparent, RB_BLACK); + __rb_rotate_set_parents(gparent, parent, root, RB_RED); + augment_rotate(gparent, parent); + break; + } else { + tmp = gparent->rb_left; + if (tmp && rb_is_red(tmp)) { + /* Case 1 - color flips */ + rb_set_parent_color(tmp, gparent, RB_BLACK); + rb_set_parent_color(parent, gparent, RB_BLACK); + node = gparent; + parent = rb_parent(node); + rb_set_parent_color(node, parent, RB_RED); + continue; + } + + tmp = parent->rb_left; + if (node == tmp) { + /* Case 2 - right rotate at parent */ + tmp = node->rb_right; + WRITE_ONCE(parent->rb_left, tmp); + WRITE_ONCE(node->rb_right, parent); + if (tmp) + rb_set_parent_color(tmp, parent, + RB_BLACK); + rb_set_parent_color(parent, node, RB_RED); + augment_rotate(parent, node); + parent = node; + tmp = node->rb_left; + } + + /* Case 3 - left rotate at gparent */ + WRITE_ONCE(gparent->rb_right, tmp); /* == parent->rb_left */ + WRITE_ONCE(parent->rb_left, gparent); + if (tmp) + rb_set_parent_color(tmp, gparent, RB_BLACK); + __rb_rotate_set_parents(gparent, parent, root, RB_RED); + augment_rotate(gparent, parent); + break; + } + } +} + +/* + * Inline version for rb_erase() use - we want to be able to inline + * and eliminate the dummy_rotate callback there + */ +static __always_inline void +____rb_erase_color(struct rb_node *parent, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) +{ + struct rb_node *node = NULL, *sibling, *tmp1, *tmp2; + + while (true) { + /* + * Loop invariants: + * - node is black (or NULL on first iteration) + * - node is not the root (parent is not NULL) + * - All leaf paths going through parent and node have a + * black node count that is 1 lower than other leaf paths. + */ + sibling = parent->rb_right; + if (node != sibling) { /* node == parent->rb_left */ + if (rb_is_red(sibling)) { + /* + * Case 1 - left rotate at parent + * + * P S + * / \ / \ + * N s --> p Sr + * / \ / \ + * Sl Sr N Sl + */ + tmp1 = sibling->rb_left; + WRITE_ONCE(parent->rb_right, tmp1); + WRITE_ONCE(sibling->rb_left, parent); + rb_set_parent_color(tmp1, parent, RB_BLACK); + __rb_rotate_set_parents(parent, sibling, root, + RB_RED); + augment_rotate(parent, sibling); + sibling = tmp1; + } + tmp1 = sibling->rb_right; + if (!tmp1 || rb_is_black(tmp1)) { + tmp2 = sibling->rb_left; + if (!tmp2 || rb_is_black(tmp2)) { + /* + * Case 2 - sibling color flip + * (p could be either color here) + * + * (p) (p) + * / \ / \ + * N S --> N s + * / \ / \ + * Sl Sr Sl Sr + * + * This leaves us violating 5) which + * can be fixed by flipping p to black + * if it was red, or by recursing at p. + * p is red when coming from Case 1. + */ + rb_set_parent_color(sibling, parent, + RB_RED); + if (rb_is_red(parent)) + rb_set_black(parent); + else { + node = parent; + parent = rb_parent(node); + if (parent) + continue; + } + break; + } + /* + * Case 3 - right rotate at sibling + * (p could be either color here) + * + * (p) (p) + * / \ / \ + * N S --> N Sl + * / \ \ + * sl Sr s + * \ + * Sr + */ + tmp1 = tmp2->rb_right; + WRITE_ONCE(sibling->rb_left, tmp1); + WRITE_ONCE(tmp2->rb_right, sibling); + WRITE_ONCE(parent->rb_right, tmp2); + if (tmp1) + rb_set_parent_color(tmp1, sibling, + RB_BLACK); + augment_rotate(sibling, tmp2); + tmp1 = sibling; + sibling = tmp2; + } + /* + * Case 4 - left rotate at parent + color flips + * (p and sl could be either color here. + * After rotation, p becomes black, s acquires + * p's color, and sl keeps its color) + * + * (p) (s) + * / \ / \ + * N S --> P Sr + * / \ / \ + * (sl) sr N (sl) + */ + tmp2 = sibling->rb_left; + WRITE_ONCE(parent->rb_right, tmp2); + WRITE_ONCE(sibling->rb_left, parent); + rb_set_parent_color(tmp1, sibling, RB_BLACK); + if (tmp2) + rb_set_parent(tmp2, parent); + __rb_rotate_set_parents(parent, sibling, root, + RB_BLACK); + augment_rotate(parent, sibling); + break; + } else { + sibling = parent->rb_left; + if (rb_is_red(sibling)) { + /* Case 1 - right rotate at parent */ + tmp1 = sibling->rb_right; + WRITE_ONCE(parent->rb_left, tmp1); + WRITE_ONCE(sibling->rb_right, parent); + rb_set_parent_color(tmp1, parent, RB_BLACK); + __rb_rotate_set_parents(parent, sibling, root, + RB_RED); + augment_rotate(parent, sibling); + sibling = tmp1; + } + tmp1 = sibling->rb_left; + if (!tmp1 || rb_is_black(tmp1)) { + tmp2 = sibling->rb_right; + if (!tmp2 || rb_is_black(tmp2)) { + /* Case 2 - sibling color flip */ + rb_set_parent_color(sibling, parent, + RB_RED); + if (rb_is_red(parent)) + rb_set_black(parent); + else { + node = parent; + parent = rb_parent(node); + if (parent) + continue; + } + break; + } + /* Case 3 - right rotate at sibling */ + tmp1 = tmp2->rb_left; + WRITE_ONCE(sibling->rb_right, tmp1); + WRITE_ONCE(tmp2->rb_left, sibling); + WRITE_ONCE(parent->rb_left, tmp2); + if (tmp1) + rb_set_parent_color(tmp1, sibling, + RB_BLACK); + augment_rotate(sibling, tmp2); + tmp1 = sibling; + sibling = tmp2; + } + /* Case 4 - left rotate at parent + color flips */ + tmp2 = sibling->rb_right; + WRITE_ONCE(parent->rb_left, tmp2); + WRITE_ONCE(sibling->rb_right, parent); + rb_set_parent_color(tmp1, sibling, RB_BLACK); + if (tmp2) + rb_set_parent(tmp2, parent); + __rb_rotate_set_parents(parent, sibling, root, + RB_BLACK); + augment_rotate(parent, sibling); + break; + } + } +} + +/* Non-inline version for rb_erase_augmented() use */ +void __rb_erase_color(struct rb_node *parent, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) +{ + ____rb_erase_color(parent, root, augment_rotate); +} +EXPORT_SYMBOL(__rb_erase_color); + +/* + * Non-augmented rbtree manipulation functions. + * + * We use dummy augmented callbacks here, and have the compiler optimize them + * out of the rb_insert_color() and rb_erase() function definitions. + */ + +static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {} +static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {} +static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {} + +static const struct rb_augment_callbacks dummy_callbacks = { + dummy_propagate, dummy_copy, dummy_rotate +}; + +void rb_insert_color(struct rb_node *node, struct rb_root *root) +{ + __rb_insert(node, root, dummy_rotate); +} +EXPORT_SYMBOL(rb_insert_color); + +void rb_erase(struct rb_node *node, struct rb_root *root) +{ + struct rb_node *rebalance; + rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); + if (rebalance) + ____rb_erase_color(rebalance, root, dummy_rotate); +} +EXPORT_SYMBOL(rb_erase); + +/* + * Augmented rbtree manipulation functions. + * + * This instantiates the same __always_inline functions as in the non-augmented + * case, but this time with user-defined callbacks. + */ + +void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, + void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) +{ + __rb_insert(node, root, augment_rotate); +} +EXPORT_SYMBOL(__rb_insert_augmented); + +/* + * This function returns the first node (in sort order) of the tree. + */ +struct rb_node *rb_first(const struct rb_root *root) +{ + struct rb_node *n; + + n = root->rb_node; + if (!n) + return NULL; + while (n->rb_left) + n = n->rb_left; + return n; +} +EXPORT_SYMBOL(rb_first); + +struct rb_node *rb_last(const struct rb_root *root) +{ + struct rb_node *n; + + n = root->rb_node; + if (!n) + return NULL; + while (n->rb_right) + n = n->rb_right; + return n; +} +EXPORT_SYMBOL(rb_last); + +struct rb_node *rb_next(const struct rb_node *node) +{ + struct rb_node *parent; + + if (RB_EMPTY_NODE(node)) + return NULL; + + /* + * If we have a right-hand child, go down and then left as far + * as we can. + */ + if (node->rb_right) { + node = node->rb_right; + while (node->rb_left) + node=node->rb_left; + return (struct rb_node *)node; + } + + /* + * No right-hand children. Everything down and left is smaller than us, + * so any 'next' node must be in the general direction of our parent. + * Go up the tree; any time the ancestor is a right-hand child of its + * parent, keep going up. First time it's a left-hand child of its + * parent, said parent is our 'next' node. + */ + while ((parent = rb_parent(node)) && node == parent->rb_right) + node = parent; + + return parent; +} +EXPORT_SYMBOL(rb_next); + +struct rb_node *rb_prev(const struct rb_node *node) +{ + struct rb_node *parent; + + if (RB_EMPTY_NODE(node)) + return NULL; + + /* + * If we have a left-hand child, go down and then right as far + * as we can. + */ + if (node->rb_left) { + node = node->rb_left; + while (node->rb_right) + node=node->rb_right; + return (struct rb_node *)node; + } + + /* + * No left-hand children. Go up till we find an ancestor which + * is a right-hand child of its parent. + */ + while ((parent = rb_parent(node)) && node == parent->rb_left) + node = parent; + + return parent; +} +EXPORT_SYMBOL(rb_prev); + +void rb_replace_node(struct rb_node *victim, struct rb_node *new, + struct rb_root *root) +{ + struct rb_node *parent = rb_parent(victim); + + /* Set the surrounding nodes to point to the replacement */ + __rb_change_child(victim, new, parent, root); + if (victim->rb_left) + rb_set_parent(victim->rb_left, new); + if (victim->rb_right) + rb_set_parent(victim->rb_right, new); + + /* Copy the pointers/colour from the victim to the replacement */ + *new = *victim; +} +EXPORT_SYMBOL(rb_replace_node); + +static struct rb_node *rb_left_deepest_node(const struct rb_node *node) +{ + for (;;) { + if (node->rb_left) + node = node->rb_left; + else if (node->rb_right) + node = node->rb_right; + else + return (struct rb_node *)node; + } +} + +struct rb_node *rb_next_postorder(const struct rb_node *node) +{ + const struct rb_node *parent; + if (!node) + return NULL; + parent = rb_parent(node); + + /* If we're sitting on node, we've already seen our children */ + if (parent && node == parent->rb_left && parent->rb_right) { + /* If we are the parent's left node, go to the parent's right + * node then all the way down to the left */ + return rb_left_deepest_node(parent->rb_right); + } else + /* Otherwise we are the parent's right node, and the parent + * should be next */ + return (struct rb_node *)parent; +} +EXPORT_SYMBOL(rb_next_postorder); + +struct rb_node *rb_first_postorder(const struct rb_root *root) +{ + if (!root->rb_node) + return NULL; + + return rb_left_deepest_node(root->rb_node); +} +EXPORT_SYMBOL(rb_first_postorder); + +#endif /* _GIANT_RBTREE_HACK_ */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 95159ca3..2ace1f05 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -30,6 +30,10 @@ #include "counters.h" #include "scoutfs_trace.h" +/* only for giant rbtree hack */ +#include +#include "ival.h" + static struct kset *scoutfs_kset; static const struct super_operations scoutfs_super_ops = { @@ -249,6 +253,8 @@ static int __init scoutfs_module_init(void) { int ret; + giant_rbtree_hack_build_bugs(); + scoutfs_init_counters(); scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); From eb790a7761a095c4b7c1e33a25f1f33aedf0a71d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Apr 2016 17:16:51 -0700 Subject: [PATCH 037/920] scoutfs: remove nonsense comment I think the range comparisons are correct here. Signed-off-by: Zach Brown --- kmod/src/ival.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/ival.c b/kmod/src/ival.c index e9e51da5..d55e73bd 100644 --- a/kmod/src/ival.c +++ b/kmod/src/ival.c @@ -70,7 +70,6 @@ void scoutfs_insert_ival(struct scoutfs_ival_tree *tree, ival->subtree_end = *scoutfs_max_key(&ival->subtree_end, &ins->end); - /* XXX <= and >= consistent? */ if (scoutfs_key_cmp(&ins->start, &ival->start) < 0) node = &(*node)->rb_left; else From 20cc8c220c17e3f137658d568884c6e882e50d94 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Apr 2016 17:17:19 -0700 Subject: [PATCH 038/920] scoutfs: fix next ival busy loop The next interval interface didn't set the ival to return to null when it finds a null next node. The caller would continuously get the same interval. This is what I get for programming late at night, I guess. Signed-off-by: Zach Brown --- kmod/src/ival.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kmod/src/ival.c b/kmod/src/ival.c index d55e73bd..da111ef8 100644 --- a/kmod/src/ival.c +++ b/kmod/src/ival.c @@ -135,14 +135,13 @@ struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, if (!ival) return first_ival(tree, start, end); - while ((node = rb_next(&ival->node))) { + node = rb_next(&ival->node); + if (node) { ival = container_of(node, struct scoutfs_ival, node); - - if (scoutfs_cmp_key_ranges(start, end, - &ival->start, &ival->end)) - ival = NULL; - break; + if (!scoutfs_cmp_key_ranges(start, end, + &ival->start, &ival->end)) + return ival; } - return ival; + return NULL; } From a07b41fa8b719a843acd1890a9755fbff78aa1d3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Apr 2016 17:27:58 -0700 Subject: [PATCH 039/920] scoutfs: store the manifest in an interval tree Now that we have the interval tree we can use it to store the manifest. Instead of having different indexes for each level we store all the levels in one index. This simplifies the code quite a bit. In particular, we won't have to special case merging between level 0 and 1 quite as much because level 0 is no longer a special list. We have a strong motivation to keep the manifest small. So we get rid of the blkno radix. It wasn't wise to trade off more manifest storage to make the ring a bit smaller. We can store full manifests in the ring instead of just the block numbers. We rework the new_manifest interace that adds a final manifest entry and logs it. The ring entry addition and manifest update are atomic. We're about to implement merging which will permute the manifest. Read methods won't be able to iterate over levels while racing with merging. We change the manifest key search interface to return a full set of all the segments that intersect the key. The next item interface now knows how to restart the search if hits the end of a segment on one level and the next least key is in another segment and greater than the end of that completed segment. There was also a very crazy cut+paste bug where next item was testing that the item is past the last search key with a while instead of an if. It'd spin throwing list_del_init() and brelse() debugging warnings. Signed-off-by: Zach Brown --- kmod/src/format.h | 6 +- kmod/src/ival.h | 15 ++ kmod/src/manifest.c | 505 +++++++++++++++------------------------ kmod/src/manifest.h | 16 +- kmod/src/ring.c | 7 +- kmod/src/scoutfs_trace.h | 5 +- kmod/src/segment.c | 197 ++++++++------- 7 files changed, 333 insertions(+), 418 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index f1bc61b8..c097d0c4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -153,7 +153,7 @@ enum { * isn't unused key space between blocks in a level. We might search * blocks when we didn't need to. */ -struct scoutfs_ring_manifest_entry { +struct scoutfs_manifest_entry { __le64 blkno; __le64 seq; __u8 level; @@ -163,10 +163,6 @@ struct scoutfs_ring_manifest_entry { #define SCOUTFS_MANIFESTS_PER_LEVEL 10 -struct scoutfs_ring_del_manifest { - __le64 blkno; -} __packed; - /* 2^22 * 10^13 > 2^64 */ #define SCOUTFS_MAX_LEVEL 13 diff --git a/kmod/src/ival.h b/kmod/src/ival.h index 4e693e61..6c944e0e 100644 --- a/kmod/src/ival.h +++ b/kmod/src/ival.h @@ -5,6 +5,11 @@ struct scoutfs_ival_tree { struct rb_root root; }; +static inline void scoutfs_init_ival_tree(struct scoutfs_ival_tree *tree) +{ + tree->root = RB_ROOT; +} + struct scoutfs_ival { struct rb_node node; struct scoutfs_key start; @@ -21,6 +26,16 @@ struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, struct scoutfs_key *end, struct scoutfs_ival *ival); +/* + * Walk all the intervals in postorder. This lets us free each ival we + * see without erasing and rebalancing. + */ +#define foreach_postorder_ival_safe(itree, ival, node, tmp) \ + for (node = rb_first_postorder(&(itree)->root); \ + ival = container_of(node, struct scoutfs_ival, node), \ + (node && (tmp = *node, 1)), node; \ + node = rb_next_postorder(&tmp)) + // struct rb_node { // long unsigned int __rb_parent_color; /* 0 8 */ // struct rb_node * rb_right; /* 8 8 */ diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 23054330..236bf911 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -13,366 +13,272 @@ #include #include #include +#include #include "super.h" #include "format.h" #include "manifest.h" #include "key.h" #include "ring.h" +#include "ival.h" #include "scoutfs_trace.h" /* - * The manifest organizes log segment blocks into a tree structure. + * The manifest organizes log segments into levels of item indexes. New + * segments arrive at level 0 which can have many segments with + * overlapping keys. Then segments are merged into progressively larger + * higher levels which do not have segments with overlapping keys. * - * Each level of the tree contains an ordered list of log segments whose - * item keys don't overlap. The first level (level 0) of the tree is - * the exception whose segments can have key ranges that overlap. - * - * We also store pointers to the manifest entries in a radix tree - * indexed by their block number so that we can easily update existing - * entries. - * - * Level 0 segments are stored in the list with the most recent at the - * head of the list. Level 0's rb tree will always be empty. + * All the segments for all the levels are stored in one interval tree. + * This lets reads find all the overlapping segments in all levels with + * one tree walk instead of walks per level. It also lets us move + * segments around the levels by updating their level field rather than + * removing them from one level index and adding them to another. */ struct scoutfs_manifest { spinlock_t lock; - - struct radix_tree_root blkno_radix; - struct list_head level_zero; - - struct scoutfs_level { - struct rb_root root; - u64 count; - } levels[SCOUTFS_MAX_LEVEL + 1]; + struct scoutfs_ival_tree itree; }; +/* + * There's some redundancy between the interval struct and the manifest + * entry struct. If we re-use both we duplicate fields and memory + * pressure is precious here. So we have a native combination of the + * two. + */ struct scoutfs_manifest_node { - struct rb_node node; - struct list_head head; - - struct scoutfs_ring_manifest_entry ment; + struct scoutfs_ival ival; + u64 blkno; + u64 seq; + unsigned char level; }; -static void insert_mnode(struct rb_root *root, - struct scoutfs_manifest_node *ins) -{ - struct rb_node **node = &root->rb_node; - struct scoutfs_manifest_node *mnode; - struct rb_node *parent = NULL; - int cmp; - - while (*node) { - parent = *node; - mnode = rb_entry(*node, struct scoutfs_manifest_node, node); - - cmp = scoutfs_key_cmp(&ins->ment.first, &mnode->ment.first); - if (cmp < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); -} - -static struct scoutfs_manifest_node *find_mnode(struct rb_root *root, - struct scoutfs_key *key) -{ - struct rb_node *node = root->rb_node; - struct scoutfs_manifest_node *mnode; - int cmp; - - while (node) { - mnode = rb_entry(node, struct scoutfs_manifest_node, node); - - cmp = scoutfs_cmp_key_range(key, &mnode->ment.first, - &mnode->ment.last); - if (cmp < 0) - node = node->rb_left; - else if (cmp > 0) - node = node->rb_right; - else - return mnode; - } - - return NULL; -} - /* - * Find a manifest node at the given block number and return it after - * removing it from either the level 0 list or level rb trees. It's - * left in the blkno radix. + * Remove an exact match of the entry from the manifest. It's normal + * for ring replay can try to remove an entry that doesn't exist if ring + * wrapping and manifest deletion combine in just the right way. */ -static struct scoutfs_manifest_node *unlink_mnode(struct scoutfs_manifest *mani, - u64 blkno) - -{ - struct scoutfs_manifest_node *mnode; - - mnode = radix_tree_lookup(&mani->blkno_radix, blkno); - if (mnode) { - trace_scoutfs_delete_manifest(&mnode->ment); - - if (!list_empty(&mnode->head)) - list_del_init(&mnode->head); - if (!RB_EMPTY_NODE(&mnode->node)) { - rb_erase(&mnode->node, - &mani->levels[mnode->ment.level].root); - mani->levels[mnode->ment.level].count--; - RB_CLEAR_NODE(&mnode->node); - } - } - - return mnode; -} - -/* - * This is called during ring replay. Because of the way the ring works - * we can get deletion entries for segments that we don't yet have - * in the replayed ring state. - */ -void scoutfs_delete_manifest(struct super_block *sb, u64 blkno) +static void delete_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *ment) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; struct scoutfs_manifest_node *mnode; + struct scoutfs_ival *ival; + + ival = NULL; + while ((ival = scoutfs_next_ival(&mani->itree, &ment->first, + &ment->last, ival))) { + mnode = container_of(ival, struct scoutfs_manifest_node, ival); + + if (mnode->blkno == le64_to_cpu(ment->blkno) && + mnode->seq == le64_to_cpu(ment->seq) && + !scoutfs_key_cmp(&ment->first, &mnode->ival.start) && + !scoutfs_key_cmp(&ment->last, &mnode->ival.end)) + break; + } + + if (ival) { + trace_scoutfs_delete_manifest(ment); + + scoutfs_remove_ival(&mani->itree, &mnode->ival); + kfree(mnode); + } +} + +void scoutfs_delete_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *ment) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; spin_lock(&mani->lock); - mnode = unlink_mnode(mani, blkno); - if (mnode) - radix_tree_delete(&mani->blkno_radix, blkno); + delete_manifest(sb, ment); spin_unlock(&mani->lock); - if (mnode) - kfree(mnode); } -/* - * A newly inserted manifest can be inserted at the level - * above the first block that it intersects. - */ -static u8 insertion_level(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment) +static void insert_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *ment, + struct scoutfs_manifest_node *mnode) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + + trace_scoutfs_insert_manifest(ment); + + mnode->ival.start = ment->first; + mnode->ival.end = ment->last; + mnode->blkno = le64_to_cpu(ment->blkno); + mnode->seq = le64_to_cpu(ment->seq); + mnode->level = ment->level; + + scoutfs_insert_ival(&mani->itree, &mnode->ival); +} + +int scoutfs_insert_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *ment) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; struct scoutfs_manifest_node *mnode; - int i; - list_for_each_entry(mnode, &mani->level_zero, head) { - if (scoutfs_cmp_key_ranges(&ment->first, &ment->last, - &mnode->ment.first, - &mnode->ment.last) == 0) - return 0; - } - - /* XXX this <= looks fishy :/ */ - for (i = 1; i <= SCOUTFS_MAX_LEVEL; i++) { - mnode = find_mnode(&mani->levels[i].root, &ment->first); - if (mnode) - break; - if (mani->levels[i].count < SCOUTFS_MANIFESTS_PER_LEVEL) - return i; - } - - return i - 1; -} - -/* - * Insert an manifest entry into the blkno radix and either level 0 list - * or greater level rbtrees as appropriate. The new entry will replace - * any existing entry at its blkno, perhaps with different keys and - * level. - * - * The caller can ask that we find the highest level that the entry can - * be inserted into before it intersects with an existing entry. The - * caller's entry is updated with the new level so they can store it in - * the ring. Doing so here avoids extra ring churn of doing it later in - * merging. - */ -static int insert_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment, - bool find_level) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_node *mnode; - struct scoutfs_manifest_node *found; - u64 blkno = le64_to_cpu(ment->blkno); - int ret = 0; - - /* allocation/preloading should be cheap enough to always try */ - mnode = kmalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); + mnode = kzalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); if (!mnode) return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ - ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM); + spin_lock(&mani->lock); + insert_manifest(sb, ment, mnode); + spin_unlock(&mani->lock); + + return 0; +} + +/* + * The caller has inserted a temporary manifest entry while they were + * dirtying a segment. It's done now and they want the final segment + * range stored in the manifest and logged in the ring. + * + * If this returns an error then nothing has changed. + * + * XXX we'd also need to add stale manifest entry's to the ring + * XXX In the future we'd send it to the leader + */ +int scoutfs_finalize_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *existing, + struct scoutfs_manifest_entry *updated) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_node *mnode; + int ret; + + mnode = kzalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); + if (!mnode) + return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ + + ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, + updated, + sizeof(struct scoutfs_manifest_entry)); if (ret) { kfree(mnode); return ret; } - INIT_LIST_HEAD(&mnode->head); - RB_CLEAR_NODE(&mnode->node); - spin_lock(&mani->lock); - - /* reuse found to avoid radix delete/insert churn */ - found = unlink_mnode(mani, blkno); - if (!found) { - radix_tree_insert(&mani->blkno_radix, blkno, mnode); - } else { - swap(found, mnode); - } - - /* careful to find our level after deleting old blkno ment */ - if (find_level) - ment->level = insertion_level(sb, ment); - - trace_scoutfs_insert_manifest(ment); - - mnode->ment = *ment; - if (ment->level) { - insert_mnode(&mani->levels[ment->level].root, mnode); - mani->levels[ment->level].count++; - } else { - list_add(&mnode->head, &mani->level_zero); - } - + delete_manifest(sb, existing); + insert_manifest(sb, updated, mnode); spin_unlock(&mani->lock); - radix_tree_preload_end(); - kfree(found); return 0; } -/* Index an existing entry */ -int scoutfs_insert_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment) +/* sorted by increasing level then decreasing seq */ +static int cmp_ments(const void *A, const void *B) { - return insert_manifest(sb, ment, false); + const struct scoutfs_manifest_entry *a = A; + const struct scoutfs_manifest_entry *b = B; + int cmp; + + cmp = (int)a->level - (int)b->level; + if (cmp) + return cmp; + + if (le64_to_cpu(a->seq) > le64_to_cpu(b->seq)) + return -1; + if (le64_to_cpu(a->seq) < le64_to_cpu(b->seq)) + return 1; + return 0; +} + +static void swap_ments(void *A, void *B, int size) +{ + struct scoutfs_manifest_entry *a = A; + struct scoutfs_manifest_entry *b = B; + + swap(*a, *b); } /* - * Add an entry for a newly written segment to the indexes and record it - * in the ring. The entry can be modified by insertion. + * Give the caller an allocated array of manifest entries that intersect + * their search key. The array is sorted in the order for searching for + * the most recent item: decreasing sequence in level 0 then increasing + * levels. * - * XXX we'd also need to add stale manifest entry's to the ring - * XXX In the future we'd send it to the leader + * The live manifest can change while the caller walks their array but + * the segments will not be reclaimed and the caller has grants that + * protect their items in the segments even if the segments shift over + * time. + * + * The number of elements in the array is returned, or negative errors, + * and the array is not allocated if 0 is returned. + * + * XXX need to actually keep the segments from being reclaimed */ -int scoutfs_new_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment) -{ - int ret; - - ret = insert_manifest(sb, ment, true); - if (!ret) { - ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, - ment, sizeof(*ment)); - if (ret) - scoutfs_delete_manifest(sb, le64_to_cpu(ment->blkno)); - } - - return ret; -} - -/* - * Fill the caller's ment with the next log segment in the manifest that - * might contain the given range. The caller initializes the ment to - * zeros to find the first log segment. - * - * This can return multiple log segments from level 0 in decreasing age. - * Then it can return at most one log segment in each level that - * intersects the given range. - * - * Returns true if an entry was found and is now described in ment, - * false when there are no more segments that contain the range. - * - * XXX could use the l0 seq to walk the list and skipb locks we've - * already seen. I'm not sure that we'll be able to keep manifest - * entries pinned while we're away blocking. We might fail to find the - * last entry's block in the radix when we return. - */ -bool scoutfs_foreach_range_segment(struct super_block *sb, - struct scoutfs_key *first, - struct scoutfs_key *last, - struct scoutfs_ring_manifest_entry *ment) +int scoutfs_manifest_find_key(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_manifest_entry **ments_ret) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; + struct scoutfs_manifest_entry *ments; struct scoutfs_manifest_node *mnode; - bool found = false; + struct scoutfs_ival *ival; + unsigned nr; int i; - if (ment->level >= SCOUTFS_MAX_LEVEL) - return false; + /* make a reasonably large initial guess */ + i = 16; + ments = NULL; + do { + kfree(ments); + nr = i; + ments = kmalloc(nr * sizeof(struct scoutfs_manifest_entry), + GFP_NOFS); + if (!ments) + return -ENOMEM; - spin_lock(&mani->lock); - - if (ment->level == 0) { - if (ment->blkno) { - mnode = radix_tree_lookup(&mani->blkno_radix, - le64_to_cpu(ment->blkno)); - mnode = list_next_entry(mnode, head); - } else { - mnode = list_first_entry(&mani->level_zero, - struct scoutfs_manifest_node, - head); - } - - list_for_each_entry_from(mnode, &mani->level_zero, head) { - if (scoutfs_cmp_key_ranges(first, last, - &mnode->ment.first, - &mnode->ment.last) == 0) { - *ment = mnode->ment; - found = true; - break; + spin_lock(&mani->lock); + i = 0; + ival = NULL; + while ((ival = scoutfs_next_ival(&mani->itree, key, key, + ival))) { + if (i < nr) { + mnode = container_of(ival, + struct scoutfs_manifest_node, ival); + ments[i].blkno = cpu_to_le64(mnode->blkno); + ments[i].seq = cpu_to_le64(mnode->seq); + ments[i].level = mnode->level; + ments[i].first = ival->start; + ments[i].last = ival->end; } + i++; } + spin_unlock(&mani->lock); + + } while (i > nr); + + if (i) { + sort(ments, i, sizeof(struct scoutfs_manifest_entry), + cmp_ments, swap_ments); + } else { + kfree(ments); + ments = NULL; } - if (!found) { - /* - * The log segments in the each level fully cover the - * key range and don't overlap. So we will always find - * a segment that matches whatever key we look for. We - * look for the start of the range because iterators are - * walk the keyspace sequentially. - */ - for (i = ment->level + 1; i <= SCOUTFS_MAX_LEVEL; i++) { - mnode = find_mnode(&mani->levels[i].root, first); - if (mnode) { - *ment = mnode->ment; - found = true; - break; - } - } - if (!found) - ment->level = SCOUTFS_MAX_LEVEL; - } - - spin_unlock(&mani->lock); - - return found; + *ments_ret = ments; + return i; } int scoutfs_setup_manifest(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani; - int i; - mani = kmalloc(sizeof(struct scoutfs_manifest), GFP_KERNEL); + mani = kzalloc(sizeof(struct scoutfs_manifest), GFP_KERNEL); if (!mani) return -ENOMEM; spin_lock_init(&mani->lock); - INIT_RADIX_TREE(&mani->blkno_radix, GFP_NOFS); - INIT_LIST_HEAD(&mani->level_zero); - - for (i = 0; i < ARRAY_SIZE(mani->levels); i++) - mani->levels[i].root = RB_ROOT; + scoutfs_init_ival_tree(&mani->itree); sbi->mani = mani; @@ -380,34 +286,21 @@ int scoutfs_setup_manifest(struct super_block *sb) } /* - * This is called once the manifest will no longer be used. We iterate - * over the blkno radix deleting radix entries and freeing manifest - * nodes. + * This is called once the manifest will no longer be used. */ void scoutfs_destroy_manifest(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_node *mnodes[16]; - unsigned long first_index = 0; - int ret; - int i; + struct scoutfs_ival *ival; + struct rb_node *node; + struct rb_node tmp; - for (;;) { - ret = radix_tree_gang_lookup(&mani->blkno_radix, - (void **)mnodes, first_index, - ARRAY_SIZE(mnodes)); - if (!ret) - break; + if (mani) { + foreach_postorder_ival_safe(&mani->itree, ival, node, tmp) + kfree(ival); - for (i = 0; i < ret; i++) { - first_index = le64_to_cpu(mnodes[i]->ment.blkno); - radix_tree_delete(&mani->blkno_radix, first_index); - kfree(mnodes[i]); - } - first_index++; + kfree(mani); + sbi->mani = NULL; } - - kfree(sbi->mani); - sbi->mani = NULL; } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index ea3eef18..5223f069 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -5,14 +5,14 @@ int scoutfs_setup_manifest(struct super_block *sb); void scoutfs_destroy_manifest(struct super_block *sb); int scoutfs_insert_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment); -int scoutfs_new_manifest(struct super_block *sb, - struct scoutfs_ring_manifest_entry *ment); -void scoutfs_delete_manifest(struct super_block *sb, u64 blkno); + struct scoutfs_manifest_entry *ment); +void scoutfs_delete_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *ment); +int scoutfs_finalize_manifest(struct super_block *sb, + struct scoutfs_manifest_entry *existing, + struct scoutfs_manifest_entry *updated); -bool scoutfs_foreach_range_segment(struct super_block *sb, - struct scoutfs_key *first, - struct scoutfs_key *last, - struct scoutfs_ring_manifest_entry *ment); +int scoutfs_manifest_find_key(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_manifest_entry **ments_ret); #endif diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 6743fb9f..cbb36835 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -28,8 +28,7 @@ static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) { struct scoutfs_ring_block *ring = (void *)bh->b_data; struct scoutfs_ring_entry *ent = (void *)(ring + 1); - struct scoutfs_ring_manifest_entry *ment; - struct scoutfs_ring_del_manifest *del; + struct scoutfs_manifest_entry *ment; struct scoutfs_ring_bitmap *bm; int ret = 0; int i; @@ -43,8 +42,8 @@ static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) ret = scoutfs_insert_manifest(sb, ment); break; case SCOUTFS_RING_DEL_MANIFEST: - del = (void *)(ent + 1); - scoutfs_delete_manifest(sb, le64_to_cpu(del->blkno)); + ment = (void *)(ent + 1); + scoutfs_delete_manifest(sb, ment); break; case SCOUTFS_RING_BITMAP: bm = (void *)(ent + 1); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 51da5c23..44f312c3 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -25,6 +25,7 @@ #include #include "key.h" +#include "format.h" TRACE_EVENT(scoutfs_bloom_hit, TP_PROTO(struct scoutfs_key *key), @@ -189,7 +190,7 @@ TRACE_EVENT(scoutfs_write_super, ); TRACE_EVENT(scoutfs_insert_manifest, - TP_PROTO(struct scoutfs_ring_manifest_entry *ment), + TP_PROTO(struct scoutfs_manifest_entry *ment), TP_ARGS(ment), @@ -225,7 +226,7 @@ TRACE_EVENT(scoutfs_insert_manifest, ); TRACE_EVENT(scoutfs_delete_manifest, - TP_PROTO(struct scoutfs_ring_manifest_entry *ment), + TP_PROTO(struct scoutfs_manifest_entry *ment), TP_ARGS(ment), diff --git a/kmod/src/segment.c b/kmod/src/segment.c index c5504626..0591c7f7 100644 --- a/kmod/src/segment.c +++ b/kmod/src/segment.c @@ -66,7 +66,7 @@ struct scoutfs_item_iter { struct buffer_head *bh; struct scoutfs_item *item; u64 blkno; - bool restart_after; + struct scoutfs_key after_seg; }; void scoutfs_put_iter_list(struct list_head *list) @@ -138,8 +138,6 @@ static bool try_lock_dirty_mutex(struct super_block *sb, u64 blkno) * then search through the item keys. The first matching item we find * is returned. * - * XXX lock the dirty log segment? - * * -ENOENT is returned if the item isn't present. The caller needs to put * the ref if we return success. */ @@ -147,12 +145,15 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_item_ref *ref) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_ring_manifest_entry ment; struct scoutfs_item *item = NULL; struct scoutfs_bloom_bits bits; + struct scoutfs_manifest_entry *ments; struct buffer_head *bh; bool locked; + u64 blkno; int ret; + int nr; + int i; /* XXX hold manifest */ @@ -160,13 +161,19 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, item = NULL; ret = -ENOENT; - memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); - while (scoutfs_foreach_range_segment(sb, key, key, &ment)) { + nr = scoutfs_manifest_find_key(sb, key, &ments); + if (nr < 0) + return nr; + if (nr == 0) + return -ENOENT; + + for (i = 0; i < nr; i++) { /* XXX read-ahead all bloom blocks */ + blkno = le64_to_cpu(ments[i].blkno); + /* XXX verify seqs */ - ret = scoutfs_test_bloom_bits(sb, le64_to_cpu(ment.blkno), - key, &bits); + ret = scoutfs_test_bloom_bits(sb, blkno, key, &bits); if (ret < 0) break; if (!ret) { @@ -176,9 +183,8 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, /* XXX read-ahead all item header blocks */ - locked = try_lock_dirty_mutex(sb, le64_to_cpu(ment.blkno)); - ret = scoutfs_skip_lookup(sb, le64_to_cpu(ment.blkno), key, - &bh, &item); + locked = try_lock_dirty_mutex(sb, blkno); + ret = scoutfs_skip_lookup(sb, blkno, key, &bh, &item); if (locked) mutex_unlock(&sbi->dirty_mutex); if (ret) { @@ -189,12 +195,14 @@ int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, break; } + kfree(ments); + /* XXX release manifest */ /* XXX read-ahead all value blocks? */ if (!ret) { - ret = populate_ref(sb, le64_to_cpu(ment.blkno), bh, item, ref); + ret = populate_ref(sb, blkno, bh, item, ref); brelse(bh); } @@ -312,49 +320,49 @@ static int start_dirty_segment(struct super_block *sb, u64 blkno) } /* - * As we fill a dirty segment we don't know which keys it's going to - * contain. We add a manifest entry in memory that has it contain all - * items so that reading will know to search the dirty segment. + * As we start to fill a dirty segment we don't know which keys it's + * going to contain. We add a manifest entry in memory that has it + * contain all items so that reading will know to search the dirty + * segment. * * Once it's finalized we know the specific range of items it contains * and we update the manifest entry in memory for that range and write * that to the ring. + * + * Inserting the updated segment can fail. If we deleted the segment, + * then insertion failed, then reinserting the original entry could fail. + * Instead we briefly allow two manifest entries for the same segment. */ static int update_dirty_segment_manifest(struct super_block *sb, u64 blkno, bool all_items) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_ring_manifest_entry ment; + struct scoutfs_manifest_entry ment; + struct scoutfs_manifest_entry updated; struct scoutfs_item_block *iblk; struct buffer_head *bh; - int ret; ment.blkno = cpu_to_le64(blkno); ment.seq = sbi->super.hdr.seq; ment.level = 0; - - if (all_items) { - memset(&ment.first, 0, sizeof(struct scoutfs_key)); - memset(&ment.last, ~0, sizeof(struct scoutfs_key)); - } else { - bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); - if (!bh) { - ret = -EIO; - goto out; - } - - iblk = (void *)bh->b_data; - ment.first = iblk->first; - ment.last = iblk->last; - brelse(bh); - } + memset(&ment.first, 0, sizeof(struct scoutfs_key)); + memset(&ment.last, ~0, sizeof(struct scoutfs_key)); if (all_items) - ret = scoutfs_insert_manifest(sb, &ment); - else - ret = scoutfs_new_manifest(sb, &ment); -out: - return ret; + return scoutfs_insert_manifest(sb, &ment); + + bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); + if (!bh) + return -EIO; + + updated = ment; + + iblk = (void *)bh->b_data; + updated.first = iblk->first; + updated.last = iblk->last; + brelse(bh); + + return scoutfs_finalize_manifest(sb, &ment, &updated); } /* @@ -670,41 +678,40 @@ int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref) * * We put the segment references and iteration cursors in a list in the * caller so that they can find many next items by advancing the cursors - * without having to walk the manifest and perform initial binary + * without having to walk the manifest and perform initial skip list * searches in each segment. * * The caller is responsible for putting the item ref if we return * success. -ENOENT is returned if there are no more items in the * search range. - * - * XXX this is wonky. We don't want to search the manifest for the - * range, just the initial value. Then we record the last key in - * segments we finish and only restart if least is > that or there are - * no least. We have to advance the first key when restarting the - * search. */ int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct list_head *iter_list, struct scoutfs_item_ref *ref) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_ring_manifest_entry ment; + struct scoutfs_manifest_entry *ments = NULL; + struct scoutfs_key key = *first; + struct scoutfs_key least_hole; struct scoutfs_item_iter *least; struct scoutfs_item_iter *iter; struct scoutfs_item_iter *pos; bool locked; int ret; + int nr; + int i; restart: if (list_empty(iter_list)) { + /* find all the segments that may contain the key */ + ret = scoutfs_manifest_find_key(sb, &key, &ments); + if (ret == 0) + ret = -ENOENT; + if (ret < 0) + goto out; + nr = ret; - /* - * Find all the segments that intersect the search range - * and find the next item in the block from the start - * of the range. - */ - memset(&ment, 0, sizeof(struct scoutfs_ring_manifest_entry)); - while (scoutfs_foreach_range_segment(sb, first, last, &ment)) { + for (i = 0; i < nr; i++) { iter = kzalloc(sizeof(struct scoutfs_item_iter), GFP_NOFS); if (!iter) { @@ -712,38 +719,32 @@ restart: goto out; } - /* - * We will restart the walk of the manifest blocks if - * we iterate over all the items in this block without - * exhausting the search range. - */ - if (ment.level > 0 && - scoutfs_key_cmp(&ment.last, last) < 0) - iter->restart_after = true; - - iter->blkno = le64_to_cpu(ment.blkno); + iter->blkno = le64_to_cpu(ments[i].blkno); + iter->after_seg = ments[i].last; + scoutfs_inc_key(&iter->after_seg); list_add_tail(&iter->list, iter_list); } - if (list_empty(iter_list)) { - ret = -ENOENT; - goto out; - } + + kfree(ments); + ments = NULL; } + memset(&least_hole, ~0, sizeof(least_hole)); least = NULL; - ret = 0; list_for_each_entry_safe(iter, pos, iter_list, list) { locked = try_lock_dirty_mutex(sb, iter->blkno); - /* search towards the first key if we haven't yet */ + /* search towards the key if we haven't yet */ if (!iter->item) { - ret = scoutfs_skip_search(sb, iter->blkno, first, + ret = scoutfs_skip_search(sb, iter->blkno, &key, &iter->bh, &iter->item); + } else { + ret = 0; } - /* then iterate until we find or pass the first key */ - while (!ret && scoutfs_key_cmp(&iter->item->key, first) < 0) { + /* then iterate until we find or pass the key */ + while (!ret && scoutfs_key_cmp(&iter->item->key, &key) < 0) { ret = scoutfs_skip_next(sb, iter->blkno, &iter->bh, &iter->item); } @@ -751,44 +752,54 @@ restart: if (locked) mutex_unlock(&sbi->dirty_mutex); - /* we're done with this block if we past the last key */ - while (!ret && scoutfs_key_cmp(&iter->item->key, last) > 0) { + /* we're done with this segment if it has an item after last */ + if (!ret && scoutfs_key_cmp(&iter->item->key, last) > 0) { + list_del_init(&iter->list); brelse(iter->bh); - iter->bh = NULL; - iter->item = NULL; - ret = -ENOENT; + kfree(iter); + continue; } + /* + * If we run out of keys in the segment then we don't know + * the state of keys after this segment in this level. If + * the hole after the segment is still inside the search + * range then we might need to search it for the next + * item if the least item of the remaining blocks is + * greater than the hole. + */ if (ret == -ENOENT) { - if (iter->restart_after) { - /* need next block at this level */ - scoutfs_put_iter_list(iter_list); - goto restart; - } else { - /* this level is done */ - list_del_init(&iter->list); - brelse(iter->bh); - kfree(iter); - continue; - } - } - if (ret) - goto out; + if (scoutfs_key_cmp(&iter->after_seg, last) <= 0 && + scoutfs_key_cmp(&iter->after_seg, &least_hole) < 0) + least_hole = iter->after_seg; - /* remember the most recent smallest key from the first */ + list_del_init(&iter->list); + brelse(iter->bh); + kfree(iter); + continue; + } + + /* remember the most recent smallest key */ if (!least || scoutfs_key_cmp(&iter->item->key, &least->item->key) < 0) least = iter; } + /* if we had a gap before the least then we need a new search */ + if (least && scoutfs_key_cmp(&least_hole, &least->item->key) < 0) { + scoutfs_put_iter_list(iter_list); + key = least_hole; + goto restart; + } + if (least) ret = populate_ref(sb, least->blkno, least->bh, least->item, ref); else ret = -ENOENT; out: + kfree(ments); if (ret) scoutfs_put_iter_list(iter_list); return ret; - } From 5369fa1e050099331a6c8d62af440a6938e81e77 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 10 Apr 2016 20:45:29 -0700 Subject: [PATCH 040/920] scoutfs: first step towards multiple btrees Starting to implement LSM merging made me really question if it is the right approach. I'd like to try an experiment to see if we can get our concurrent writes done with much simpler btrees. This commit removes all the functionality that derives from the large LSM segments and distributing the manifest. What's left is a multi-page block layer and the husk of the btree implementation which will give people access to items. Callers that work with items get translated to the btree interface. This gets as far as reading the super block but the format changes and large block size mean that the crc check fails and the mount returns an error. Signed-off-by: Zach Brown --- kmod/src/Makefile | 5 +- kmod/src/block.c | 264 ++++++++--- kmod/src/block.h | 30 +- kmod/src/bloom.c | 132 ------ kmod/src/bloom.h | 17 - kmod/src/btree.h | 58 +++ kmod/src/chunk.c | 89 ---- kmod/src/chunk.h | 8 - kmod/src/counters.h | 11 +- kmod/src/dir.c | 48 +- kmod/src/filerw.c | 24 +- kmod/src/format.h | 150 +----- kmod/src/inode.c | 37 +- kmod/src/ival.c | 147 ------ kmod/src/ival.h | 71 --- kmod/src/manifest.c | 306 ------------ kmod/src/manifest.h | 18 - kmod/src/rbtree_aug.h | 996 --------------------------------------- kmod/src/ring.c | 250 ---------- kmod/src/ring.h | 9 - kmod/src/scoutfs_trace.c | 3 - kmod/src/scoutfs_trace.h | 152 ------ kmod/src/segment.c | 805 ------------------------------- kmod/src/segment.h | 35 -- kmod/src/skip.c | 338 ------------- kmod/src/skip.h | 18 - kmod/src/super.c | 110 +---- kmod/src/super.h | 30 +- 28 files changed, 373 insertions(+), 3788 deletions(-) delete mode 100644 kmod/src/bloom.c delete mode 100644 kmod/src/bloom.h create mode 100644 kmod/src/btree.h delete mode 100644 kmod/src/chunk.c delete mode 100644 kmod/src/chunk.h delete mode 100644 kmod/src/ival.c delete mode 100644 kmod/src/ival.h delete mode 100644 kmod/src/manifest.c delete mode 100644 kmod/src/manifest.h delete mode 100644 kmod/src/rbtree_aug.h delete mode 100644 kmod/src/ring.c delete mode 100644 kmod/src/ring.h delete mode 100644 kmod/src/segment.c delete mode 100644 kmod/src/segment.h delete mode 100644 kmod/src/skip.c delete mode 100644 kmod/src/skip.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 31d960ef..68870aa4 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o bloom.o counters.o chunk.o crc.o dir.o filerw.o inode.o \ - ival.o manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o \ - super.o +scoutfs-y += block.o counters.o crc.o dir.o filerw.o inode.o msg.o \ + scoutfs_trace.o super.o diff --git a/kmod/src/block.c b/kmod/src/block.c index 8326382f..51815f9d 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 Versity Software, Inc. All rights reserved. + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -10,73 +10,206 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ -#include +#include +#include +#include +#include +#include #include "super.h" #include "format.h" #include "block.h" #include "crc.h" +#include "counters.h" -#define BH_Private_Verified BH_PrivateStart +/* + * XXX + * - tie into reclaim + * - per cpu lru of refs? + * - relax locking + * - get, check, and fill slots instead of full radix walks + * - block slab + * - maybe more clever wait functions + */ -BUFFER_FNS(Private_Verified, private_verified) +static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) +{ + struct scoutfs_block *bl; + struct page *page; -static void verify_block_header(struct super_block *sb, struct buffer_head *bh) + /* we'd need to be just a bit more careful */ + BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); + + bl = kzalloc(sizeof(struct scoutfs_block), GFP_NOFS); + if (bl) { + page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); + WARN_ON_ONCE(!page); + if (page) { + init_rwsem(&bl->rwsem); + atomic_set(&bl->refcount, 1); + bl->blkno = blkno; + bl->sb = sb; + bl->page = page; + bl->data = page_address(page); + scoutfs_inc_counter(sb, block_mem_alloc); + } else { + kfree(bl); + bl = NULL; + } + } + + return bl; +} + +void scoutfs_put_block(struct scoutfs_block *bl) +{ + if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { + __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); + kfree(bl); + scoutfs_inc_counter(bl->sb, block_mem_free); + } +} + +static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_block_header *hdr = (void *)bh->b_data; + struct scoutfs_block_header *hdr = bl->data; u32 crc = scoutfs_crc_block(hdr); - u64 blkno = bh->b_blocknr; + int ret = -EIO; if (le32_to_cpu(hdr->crc) != crc) { - printk("blkno %llu hdr crc %x != calculated %x\n", blkno, + printk("blkno %llu hdr crc %x != calculated %x\n", bl->blkno, le32_to_cpu(hdr->crc), crc); } else if (super->hdr.fsid && hdr->fsid != super->hdr.fsid) { - printk("blkno %llu fsid %llx != super fsid %llx\n", blkno, + printk("blkno %llu fsid %llx != super fsid %llx\n", bl->blkno, le64_to_cpu(hdr->fsid), le64_to_cpu(super->hdr.fsid)); - } else if (le64_to_cpu(hdr->blkno) != blkno) { - printk("blkno %llu invalid hdr blkno %llx\n", blkno, + } else if (le64_to_cpu(hdr->blkno) != bl->blkno) { + printk("blkno %llu invalid hdr blkno %llx\n", bl->blkno, le64_to_cpu(hdr->blkno)); } else { - set_buffer_private_verified(bh); + ret = 0; } + + return ret; +} + +static void block_read_end_io(struct bio *bio, int err) +{ + struct scoutfs_block *bl = bio->bi_private; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + + if (!err && !verify_block_header(bl->sb, bl)) + set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits); + else + set_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits); + + /* + * uncontended spin_lock in wake_up and unconditional smp_mb to + * make waitqueue_active safe are about the same cost, so we + * prefer the obviously safe choice. + */ + wake_up(&sbi->block_wq); + + scoutfs_put_block(bl); +} + +static int block_submit_bio(struct scoutfs_block *bl, int rw) +{ + struct super_block *sb = bl->sb; + struct bio *bio; + int ret; + + bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); + if (WARN_ON_ONCE(!bio)) + return -ENOMEM; + + bio->bi_sector = bl->blkno << (SCOUTFS_BLOCK_SHIFT - 9); + bio->bi_bdev = sb->s_bdev; + /* XXX can we do that? */ + ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); + if (rw & WRITE) + ; + else + bio->bi_end_io = block_read_end_io; + bio->bi_private = bl; + atomic_inc(&bl->refcount); + submit_bio(rw, bio); + + return 0; } /* * Read an existing block from the device and verify its metadata header. */ -struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno) +struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) { - struct buffer_head *bh; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *found; + struct scoutfs_block *bl; + int ret; - bh = sb_bread(sb, blkno); - if (!bh || buffer_private_verified(bh)) - return bh; + /* find an existing block, dropping if it's errored */ + spin_lock(&sbi->block_lock); - lock_buffer(bh); - if (!buffer_private_verified(bh)) - verify_block_header(sb, bh); - unlock_buffer(bh); - - if (!buffer_private_verified(bh)) { - brelse(bh); - bh = NULL; + bl = radix_tree_lookup(&sbi->block_radix, blkno); + if (bl && test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + scoutfs_put_block(bl); + bl = NULL; } - return bh; -} + spin_unlock(&sbi->block_lock); + if (bl) + goto wait; -/* - * Read the block that contains the given byte offset in the given chunk. - */ -struct buffer_head *scoutfs_read_block_off(struct super_block *sb, u64 blkno, - u32 off) -{ - if (WARN_ON_ONCE(off >= SCOUTFS_CHUNK_SIZE)) - return ERR_PTR(-EINVAL); + /* allocate a new block and try to insert it */ + bl = alloc_block(sb, blkno); + if (!bl) { + ret = -EIO; + goto out; + } - return scoutfs_read_block(sb, blkno + (off >> SCOUTFS_BLOCK_SHIFT)); + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + spin_lock(&sbi->block_lock); + + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found) { + scoutfs_put_block(bl); + bl = found; + } else { + radix_tree_insert(&sbi->block_radix, blkno, bl); + atomic_inc(&bl->refcount); + } + + spin_unlock(&sbi->block_lock); + radix_tree_preload_end(); + + if (!found) { + ret = block_submit_bio(bl, READ_SYNC | REQ_META); + if (ret) + goto out; + } + +wait: + ret = wait_event_interruptible(sbi->block_wq, + test_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits) || + test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)); + if (test_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits)) + ret = 0; + else if (test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) + ret = -EIO; + +out: + if (ret) { + scoutfs_put_block(bl); + bl = ERR_PTR(ret); + } + + return bl; } /* @@ -85,33 +218,56 @@ struct buffer_head *scoutfs_read_block_off(struct super_block *sb, u64 blkno, * serializing access to the block and for zeroing unwritten block * contents. */ -struct buffer_head *scoutfs_new_block(struct super_block *sb, u64 blkno) +struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct scoutfs_block_header *hdr; - struct buffer_head *bh; + struct scoutfs_block *found; + struct scoutfs_block *bl; + int ret; - bh = sb_getblk(sb, blkno); - if (bh) { - if (!buffer_uptodate(bh) || buffer_private_verified(bh)) { - lock_buffer(bh); - set_buffer_uptodate(bh); - set_buffer_private_verified(bh); - unlock_buffer(bh); - } - - hdr = (void *)bh->b_data; - *hdr = super->hdr; - hdr->blkno = cpu_to_le64(blkno); + /* allocate a new block and try to insert it */ + bl = alloc_block(sb, blkno); + if (!bl) { + ret = -EIO; + goto out; } - return bh; + set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits); + + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + hdr = bl->data; + *hdr = sbi->super.hdr; + hdr->blkno = cpu_to_le64(blkno); + + spin_lock(&sbi->block_lock); + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found) { + radix_tree_delete(&sbi->block_radix, blkno); + scoutfs_put_block(found); + } + + radix_tree_insert(&sbi->block_radix, blkno, bl); + atomic_inc(&bl->refcount); + spin_unlock(&sbi->block_lock); + + radix_tree_preload_end(); + ret = 0; +out: + if (ret) { + scoutfs_put_block(bl); + bl = ERR_PTR(ret); + } + + return bl; } -void scoutfs_calc_hdr_crc(struct buffer_head *bh) +void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) { - struct scoutfs_block_header *hdr = (void *)bh->b_data; + struct scoutfs_block_header *hdr = bl->data; hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } diff --git a/kmod/src/block.h b/kmod/src/block.h index 7be8ed6d..c04586a8 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -1,10 +1,30 @@ #ifndef _SCOUTFS_BLOCK_H_ #define _SCOUTFS_BLOCK_H_ -struct buffer_head *scoutfs_read_block(struct super_block *sb, u64 blkno); -struct buffer_head *scoutfs_read_block_off(struct super_block *sb, u64 blkno, - u32 off); -struct buffer_head *scoutfs_new_block(struct super_block *sb, u64 blkno); -void scoutfs_calc_hdr_crc(struct buffer_head *bh); +#include +#include +#include + +#define SCOUTFS_BLOCK_BIT_UPTODATE (1 << 0) +#define SCOUTFS_BLOCK_BIT_ERROR (1 << 1) + +struct scoutfs_block { + struct rw_semaphore rwsem; + atomic_t refcount; + u64 blkno; + + unsigned long bits; + + struct super_block *sb; + /* only high order page alloc for now */ + struct page *page; + void *data; +}; + +struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno); +void scoutfs_put_block(struct scoutfs_block *bl); + +void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); #endif diff --git a/kmod/src/bloom.c b/kmod/src/bloom.c deleted file mode 100644 index d41fa57c..00000000 --- a/kmod/src/bloom.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "block.h" -#include "bloom.h" -#include "scoutfs_trace.h" - -/* - * Each log segment starts with a bloom filters that spans multiple - * blocks. It's used to test for the presence of key in the log segment - * without having to read and search the much larger array of items and - * their keys. - */ - -/* XXX garbage hack until we have siphash */ -static u32 bloom_hash(struct scoutfs_key *key, __le32 salt) -{ - return crc32c(le32_to_cpu(salt), key, sizeof(struct scoutfs_key)); -} - -/* - * Find the bits in the bloom filter for the given key. The caller calculates - * these once and uses them to test all the blocks. - */ -void scoutfs_calc_bloom_bits(struct scoutfs_bloom_bits *bits, - struct scoutfs_key *key, __le32 *salts) -{ - unsigned h_bits = 0; - unsigned int b; - unsigned s = 0; - u64 h = 0; - int i; - - BUILD_BUG_ON(SCOUTFS_BLOOM_BIT_WIDTH > 32); - - for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { - if (h_bits < SCOUTFS_BLOOM_BIT_WIDTH) { - h = (h << 32) | bloom_hash(key, salts[s++]); - h_bits += 32; - } - - b = h & SCOUTFS_BLOOM_BIT_MASK; - h >>= SCOUTFS_BLOOM_BIT_WIDTH; - h_bits -= SCOUTFS_BLOOM_BIT_WIDTH; - - bits->block[i] = (b / SCOUTFS_BLOOM_BITS_PER_BLOCK) % - SCOUTFS_BLOOM_BLOCKS; - bits->bit_off[i] = b % SCOUTFS_BLOOM_BITS_PER_BLOCK; - } -} - -/* - * Set the caller's bit numbers in the bloom filter contained in bloom - * blocks starting at the given block number. The caller has - * initialized the blocks and is responsible for locking and dirtying - * and writeout. - */ -int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, - struct scoutfs_bloom_bits *bits) -{ - struct scoutfs_bloom_block *blm; - struct buffer_head *bh; - int ret = 0; - int i; - - for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { - bh = scoutfs_read_block(sb, blkno + bits->block[i]); - if (!bh) { - ret = -EIO; - break; - } - - blm = (void *)bh->b_data; - set_bit_le(bits->bit_off[i], blm->bits); - - brelse(bh); - } - - return ret; -} - -/* - * Returns zero if the bits' key can't be found in the block, true if it - * might, and -errno if IO fails. - */ -int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, - struct scoutfs_bloom_bits *bits) -{ - struct scoutfs_bloom_block *blm; - struct buffer_head *bh; - int ret; - int i; - - for (i = 0; i < SCOUTFS_BLOOM_BITS; i++) { - bh = scoutfs_read_block(sb, blkno + bits->block[i]); - if (!bh) { - ret = -EIO; - break; - } - - blm = (void *)bh->b_data; - ret = !!test_bit_le(bits->bit_off[i], blm->bits); - brelse(bh); - if (!ret) - break; - } - - if (ret) - trace_scoutfs_bloom_hit(key); - else - trace_scoutfs_bloom_miss(key); - - return ret; -} diff --git a/kmod/src/bloom.h b/kmod/src/bloom.h deleted file mode 100644 index 59739bb1..00000000 --- a/kmod/src/bloom.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _SCOUTFS_BLOOM_H_ -#define _SCOUTFS_BLOOM_H_ - -struct scoutfs_bloom_bits { - u16 bit_off[SCOUTFS_BLOOM_BITS]; - u8 block[SCOUTFS_BLOOM_BITS]; -}; - -void scoutfs_calc_bloom_bits(struct scoutfs_bloom_bits *bits, - struct scoutfs_key *key, __le32 *salts); -int scoutfs_test_bloom_bits(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, - struct scoutfs_bloom_bits *bits); -int scoutfs_set_bloom_bits(struct super_block *sb, u64 blkno, - struct scoutfs_bloom_bits *bits); - -#endif diff --git a/kmod/src/btree.h b/kmod/src/btree.h new file mode 100644 index 00000000..d7432313 --- /dev/null +++ b/kmod/src/btree.h @@ -0,0 +1,58 @@ +#ifndef _SCOUTFS_BTREE_H_ +#define _SCOUTFS_BTREE_H_ + +struct scoutfs_btree_cursor { + /* for btree.c */ + struct scoutfs_block *bl; + struct scoutfs_btree_item *item; + + /* for callers */ + struct scoutfs_key *key; + unsigned val_len; + void *val; +}; + +static inline int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + +static inline int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_key *key, + unsigned short val_len, + struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + +static inline int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_key *key, + unsigned short val_len, + struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + + +static inline int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + +static inline int scoutfs_btree_next(struct super_block *sb, + struct scoutfs_key *first, + struct scoutfs_key *last, + struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + +static inline int scoutfs_btree_release(struct scoutfs_btree_cursor *curs) +{ + return -ENOSYS; +} + +#endif diff --git a/kmod/src/chunk.c b/kmod/src/chunk.c deleted file mode 100644 index 3a9080c3..00000000 --- a/kmod/src/chunk.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "inode.h" -#include "dir.h" -#include "msg.h" -#include "block.h" -#include "ring.h" -#include "chunk.h" - -void scoutfs_set_chunk_alloc_bits(struct super_block *sb, - struct scoutfs_ring_bitmap *bm) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 off = le32_to_cpu(bm->offset) * ARRAY_SIZE(bm->bits); - - /* XXX check for corruption */ - - sbi->chunk_alloc_bits[off] = bm->bits[0]; - sbi->chunk_alloc_bits[off + 1] = bm->bits[1]; -} - -/* - * Return the block number of the first block in a free chunk. - * - * The region around the cleared free bit for the allocation is always - * added to the ring and will generate a ton of overlapping ring - * entries. This is fine for initial testing but won't be good enough - * for real use. We'll have a bitmap of dirtied regions that are only - * logged as the update is written out. - */ -int scoutfs_alloc_chunk(struct super_block *sb, u64 *blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - unsigned long size = le64_to_cpu(super->total_chunks); - struct scoutfs_ring_bitmap bm; - unsigned long off; - unsigned long bit; - int ret; - - spin_lock(&sbi->chunk_alloc_lock); - - bit = find_next_bit_le(sbi->chunk_alloc_bits, size, 0); - if (bit >= size) { - ret = -ENOSPC; - } else { - clear_bit_le(bit, sbi->chunk_alloc_bits); - - off = round_down(bit, sizeof(bm.bits) * 8); - bm.offset = cpu_to_le32(off); - - off *= ARRAY_SIZE(bm.bits); - bm.bits[0] = sbi->chunk_alloc_bits[off]; - bm.bits[1] = sbi->chunk_alloc_bits[off + 1]; - - *blkno = bit << SCOUTFS_CHUNK_BLOCK_SHIFT; - ret = 0; - } - - spin_unlock(&sbi->chunk_alloc_lock); - - if (!ret) { - ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_BITMAP, &bm, - sizeof(bm)); - WARN_ON_ONCE(ret); - } - - return ret; -} diff --git a/kmod/src/chunk.h b/kmod/src/chunk.h deleted file mode 100644 index eb6615c7..00000000 --- a/kmod/src/chunk.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SCOUTFS_CHUNK_H_ -#define _SCOUTFS_CHUNK_H_ - -void scoutfs_set_chunk_alloc_bits(struct super_block *sb, - struct scoutfs_ring_bitmap *bm); -int scoutfs_alloc_chunk(struct super_block *sb, u64 *blkno); - -#endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 1f6c1843..f6d630c1 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,14 +12,11 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ - EXPAND_COUNTER(skip_lookup) \ - EXPAND_COUNTER(skip_insert) \ - EXPAND_COUNTER(skip_search) \ - EXPAND_COUNTER(skip_delete) \ - EXPAND_COUNTER(skip_next) \ + EXPAND_COUNTER(block_mem_alloc) \ + EXPAND_COUNTER(block_mem_free) -#define FIRST_COUNTER skip_lookup -#define LAST_COUNTER skip_next +#define FIRST_COUNTER block_mem_alloc +#define LAST_COUNTER block_mem_free #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index cec7d878..6a95bfb3 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -20,8 +20,8 @@ #include "dir.h" #include "inode.h" #include "key.h" -#include "segment.h" #include "super.h" +#include "btree.h" /* * Directory entries are stored in entries with offsets calculated from @@ -114,9 +114,9 @@ static unsigned int dent_bytes(unsigned int name_len) return sizeof(struct scoutfs_dirent) + name_len; } -static unsigned int item_name_len(struct scoutfs_item_ref *ref) +static unsigned int item_name_len(struct scoutfs_btree_cursor *curs) { - return ref->val_len - sizeof(struct scoutfs_dirent); + return curs->val_len - sizeof(struct scoutfs_dirent); } /* * Store the dirent item hash in the dentry so that we don't have to @@ -176,8 +176,8 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct scoutfs_inode_info *si = SCOUTFS_I(dir); + struct scoutfs_btree_cursor curs = {NULL,}; struct super_block *sb = dir->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_dirent *dent; struct dentry_info *di; struct scoutfs_key key; @@ -209,15 +209,14 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, h = name_hash(dentry->d_name.name, dentry->d_name.len, h); scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - scoutfs_put_ref(&ref); - ret = scoutfs_read_item(sb, &key, &ref); + ret = scoutfs_btree_lookup(sb, &key, &curs); if (ret == -ENOENT) continue; if (ret < 0) break; - dent = ref.val; - name_len = item_name_len(&ref); + dent = curs.val; + name_len = item_name_len(&curs); if (names_equal(dentry->d_name.name, dentry->d_name.len, dent->name, name_len)) { ino = le64_to_cpu(dent->ino); @@ -228,7 +227,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, } } - scoutfs_put_ref(&ref); + scoutfs_btree_release(&curs); out: if (ret == -ENOENT) { inode = NULL; @@ -275,12 +274,11 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_btree_cursor curs = {NULL,}; struct scoutfs_dirent *dent; struct scoutfs_key first; struct scoutfs_key last; unsigned int name_len; - LIST_HEAD(iter_list); int ret = 0; u32 pos; @@ -294,14 +292,13 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, file->f_pos); - scoutfs_put_ref(&ref); - ret = scoutfs_next_item(sb, &first, &last, &iter_list, &ref); + ret = scoutfs_btree_next(sb, &first, &last, &curs); if (ret) break; - dent = ref.val; - name_len = item_name_len(&ref); - pos = scoutfs_key_offset(ref.key); + dent = curs.val; + name_len = item_name_len(&curs); + pos = scoutfs_key_offset(curs.key); if (filldir(dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) @@ -310,8 +307,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) file->f_pos = pos + 1; } - scoutfs_put_ref(&ref); - scoutfs_put_iter_list(&iter_list); + scoutfs_btree_release(&curs); if (ret == -ENOENT) ret = 0; @@ -324,9 +320,9 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, { struct super_block *sb = dir->i_sb; struct scoutfs_inode_info *si = SCOUTFS_I(dir); + struct scoutfs_btree_cursor curs = {NULL,}; struct inode *inode = NULL; struct scoutfs_dirent *dent; - DECLARE_SCOUTFS_ITEM_REF(ref); struct dentry_info *di; struct scoutfs_key key; int bytes; @@ -356,7 +352,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, h = name_hash(dentry->d_name.name, dentry->d_name.len, h); scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - ret = scoutfs_create_item(sb, &key, bytes, &ref); + ret = scoutfs_btree_insert(sb, &key, bytes, &curs); if (ret != -EEXIST) break; } @@ -366,13 +362,13 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, goto out; } - dent = ref.val; + dent = curs.val; dent->ino = cpu_to_le64(scoutfs_ino(inode)); dent->type = mode_to_type(inode->i_mode); memcpy(dent->name, dentry->d_name.name, dentry->d_name.len); di->hash = h; - scoutfs_put_ref(&ref); + scoutfs_btree_release(&curs); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -417,7 +413,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_btree_cursor curs = {NULL,}; struct dentry_info *di; struct scoutfs_key key; int ret = 0; @@ -436,12 +432,12 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); - ret = scoutfs_read_item(sb, &key, &ref); + ret = scoutfs_btree_lookup(sb, &key, &curs); if (ret) goto out; - ret = scoutfs_delete_item(sb, &ref); - scoutfs_put_ref(&ref); + ret = scoutfs_btree_delete(sb, &curs); + scoutfs_btree_release(&curs); if (ret) goto out; diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index fc7a46ec..074204df 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -15,11 +15,11 @@ #include #include "format.h" -#include "segment.h" #include "inode.h" #include "key.h" #include "filerw.h" #include "scoutfs_trace.h" +#include "btree.h" /* * File data is stored in items just like everything else. This is very @@ -61,8 +61,8 @@ static bool map_data_region(struct data_region *dr, u64 pos, struct page *page) dr->item_off = do_div(pos, SCOUTFS_MAX_ITEM_LEN); dr->item_key = pos; - dr->len = min(SCOUTFS_MAX_ITEM_LEN - dr->item_off, - PAGE_SIZE - dr->page_off); + dr->len = min_t(int, SCOUTFS_MAX_ITEM_LEN - dr->item_off, + PAGE_SIZE - dr->page_off); return true; } @@ -81,8 +81,8 @@ static bool map_data_region(struct data_region *dr, u64 pos, struct page *page) static int scoutfs_readpage(struct file *file, struct page *page) { struct inode *inode = file->f_mapping->host; + struct scoutfs_btree_cursor curs = {NULL,}; struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; struct data_region dr; int ret = 0; @@ -93,7 +93,7 @@ static int scoutfs_readpage(struct file *file, struct page *page) scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, dr.item_key); - ret = scoutfs_read_item(sb, &key, &ref); + ret = scoutfs_btree_lookup(sb, &key, &curs); if (ret == -ENOENT) { addr = kmap_atomic(page); memset(addr + dr.page_off, 0, dr.len); @@ -104,7 +104,7 @@ static int scoutfs_readpage(struct file *file, struct page *page) break; addr = kmap_atomic(page); - memcpy(addr + dr.page_off, ref.val + dr.item_off, dr.len); + memcpy(addr + dr.page_off, curs.val + dr.item_off, dr.len); kunmap_atomic(addr); } @@ -125,8 +125,8 @@ static int scoutfs_readpage(struct file *file, struct page *page) static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) { struct inode *inode = page->mapping->host; + struct scoutfs_btree_cursor curs = {NULL,}; struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; struct data_region dr; void *addr; @@ -139,19 +139,19 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, dr.item_key); - ret = scoutfs_dirty_item(sb, &key, SCOUTFS_MAX_ITEM_LEN, &ref); + /* XXX dirty */ + ret = scoutfs_btree_insert(sb, &key, SCOUTFS_MAX_ITEM_LEN, + &curs); if (ret) break; addr = kmap_atomic(page); - memcpy(ref.val + dr.item_off, addr + dr.page_off, dr.len); + memcpy(curs.val + dr.item_off, addr + dr.page_off, dr.len); kunmap_atomic(addr); - scoutfs_put_ref(&ref); - } - scoutfs_put_ref(&ref); + scoutfs_btree_release(&curs); if (ret) { SetPageError(page); diff --git a/kmod/src/format.h b/kmod/src/format.h index c097d0c4..d35f5613 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -6,27 +6,12 @@ /* super block id */ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ -/* - * Everything is stored in and addressed as 4k fixed size blocks. This - * avoids having to manage contiguous cpu mappings of larger blocks. - * Larger structures are read and written as multiple blocks. - */ -#define SCOUTFS_BLOCK_SHIFT 12 +#define SCOUTFS_BLOCK_SHIFT 14 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) -/* - * The allocator works on larger chunks. Smaller metadata structures - * like the super blocks and the ring are stored in chunks. - * - * A log segment is a collection of smaller blocks (bloom filter, item blocks) - * stored in a chunk. - */ -#define SCOUTFS_CHUNK_SHIFT 22 -#define SCOUTFS_CHUNK_SIZE (1 << SCOUTFS_CHUNK_SHIFT) -#define SCOUTFS_CHUNK_BLOCK_SHIFT (SCOUTFS_CHUNK_SHIFT - SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_CHUNK_BLOCK_MASK ((1 << SCOUTFS_CHUNK_BLOCK_SHIFT) - 1) -#define SCOUTFS_BLOCKS_PER_CHUNK (1 << SCOUTFS_CHUNK_BLOCK_SHIFT) +#define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) /* * The super blocks leave some room at the start of the first block for @@ -35,22 +20,6 @@ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 -/* - * The bloom filters are statically sized. It's a tradeoff between - * storage overhead and false positive rate. At the moment we have - * as few as 1000 and as many as 18000 items in a segment. We can - * get a ~1% false positive rate (triggering header search) rate at - * the high end with a ~20k bloom filter. - * - * n = 18,000, p = 0.01 (1 in 100) → m = 172,532 (21.06KB), k = 7 - */ -#define SCOUTFS_BLOOM_BITS 7 -#define SCOUTFS_BLOOM_BIT_WIDTH 18 /* 2^18 > m */ -#define SCOUTFS_BLOOM_BIT_MASK ((1 << SCOUTFS_BLOOM_BIT_WIDTH) - 1) -#define SCOUTFS_BLOOM_BLOCKS ((20 * 1024) / SCOUTFS_BLOCK_SIZE) -#define SCOUTFS_BLOOM_SALTS \ - DIV_ROUND_UP(SCOUTFS_BLOOM_BITS * SCOUTFS_BLOOM_BIT_WIDTH, 32) - /* * This header is found at the start of every block so that we can * verify that it's what we were looking for. The crc and padding @@ -81,14 +50,6 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; - __le32 bloom_salts[SCOUTFS_BLOOM_SALTS]; - __le64 total_chunks; - __le64 ring_map_blkno; - __le64 ring_map_seq; - __le64 ring_first_block; - __le64 ring_active_blocks; - __le64 ring_total_blocks; - __le64 ring_seq; } __packed; /* @@ -112,110 +73,7 @@ struct scoutfs_key { #define SCOUTFS_DIRENT_KEY 2 #define SCOUTFS_DATA_KEY 3 -struct scoutfs_ring_map_block { - struct scoutfs_block_header hdr; - __le32 nr_chunks; - __le64 blknos[0]; -} __packed; - -#define SCOUTFS_RING_MAP_BLOCKS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_ring_map_block)) / \ - sizeof(__le64)) - -struct scoutfs_ring_entry { - u8 type; - __le16 len; -} __packed; - -/* - * Ring blocks are stored in chunks described by the ring map blocks. - * - * The manifest entries describe the position of a given log segment in - * the manifest. They're keyed by the block number so that we can - * record movement of a log segment in the manifest with one ring entry - * and we can record deletion with just the block number. - */ -struct scoutfs_ring_block { - struct scoutfs_block_header hdr; - __le16 nr_entries; -} __packed; - -enum { - SCOUTFS_RING_ADD_MANIFEST = 0, - SCOUTFS_RING_DEL_MANIFEST, - SCOUTFS_RING_BITMAP, -}; - -/* - * Including both keys might make the manifest too large. It might be - * better to only include one key and infer a block's range from the - * neighbour's key. The downside of that is that we assume that there - * isn't unused key space between blocks in a level. We might search - * blocks when we didn't need to. - */ -struct scoutfs_manifest_entry { - __le64 blkno; - __le64 seq; - __u8 level; - struct scoutfs_key first; - struct scoutfs_key last; -} __packed; - -#define SCOUTFS_MANIFESTS_PER_LEVEL 10 - -/* 2^22 * 10^13 > 2^64 */ -#define SCOUTFS_MAX_LEVEL 13 - -struct scoutfs_ring_bitmap { - __le32 offset; - __le64 bits[2]; -} __packed; - - -struct scoutfs_bloom_block { - struct scoutfs_block_header hdr; - __le64 bits[0]; -} __packed; - -#define SCOUTFS_BLOOM_BITS_PER_BLOCK \ - (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_block_header)) / 8) * 64) - -/* - * Items in log segments are sorted in a skip list by their key. We - * have a rough limit of 64k items. - */ -#define SCOUTFS_SKIP_HEIGHT 16 -struct scoutfs_skip_root { - __le32 next[SCOUTFS_SKIP_HEIGHT]; -} __packed; - -/* - * An item block follows the bloom filter blocks at the start of a log - * segment. Its skip root references the item structs which then - * reference the item values in the rest of the block. The references - * are byte offsets from the start of the chunk. - */ -struct scoutfs_item_block { - struct scoutfs_block_header hdr; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_skip_root skip_root; -} __packed; - -struct scoutfs_item { - struct scoutfs_key key; - __le32 offset; - __le16 len; - u8 skip_height; - __le32 skip_next[0]; -} __packed; - -/* - * Item size caps item file data item length so that they fit in checksummed - * 4k blocks with a bit of expansion room. - */ -#define SCOUTFS_MAX_ITEM_LEN \ - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_block_header) - 32) +#define SCOUTFS_MAX_ITEM_LEN 2048 struct scoutfs_timespec { __le64 sec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index a92578c0..44b4c3de 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -19,7 +19,7 @@ #include "super.h" #include "key.h" #include "inode.h" -#include "segment.h" +#include "btree.h" #include "dir.h" #include "filerw.h" #include "scoutfs_trace.h" @@ -112,17 +112,17 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) static int scoutfs_read_locked_inode(struct inode *inode) { + struct scoutfs_btree_cursor curs = {NULL,}; struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_read_item(sb, &key, &ref); + ret = scoutfs_btree_lookup(sb, &key, &curs); if (!ret) { - load_inode(inode, ref.val); - scoutfs_put_ref(&ref); + load_inode(inode, curs.val); + scoutfs_btree_release(&curs); } return 0; @@ -213,16 +213,17 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) int scoutfs_dirty_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); + struct scoutfs_btree_cursor curs = {NULL,}; struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_dirty_item(sb, &key, sizeof(struct scoutfs_inode), &ref); + ret = scoutfs_btree_dirty(sb, &key, sizeof(struct scoutfs_inode), + &curs); if (!ret) { - store_inode(ref.val, inode); - scoutfs_put_ref(&ref); + store_inode(curs.val, inode); + scoutfs_btree_release(&curs); trace_scoutfs_dirty_inode(inode); } return ret; @@ -239,18 +240,20 @@ int scoutfs_dirty_inode_item(struct inode *inode) */ void scoutfs_update_inode_item(struct inode *inode) { + struct scoutfs_btree_cursor curs = {NULL,}; struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_read_item(sb, &key, &ref); + /* XXX maybe just use dirty again? not sure.. */ + ret = scoutfs_btree_dirty(sb, &key, sizeof(struct scoutfs_inode), + &curs); BUG_ON(ret); - store_inode(ref.val, inode); - scoutfs_put_ref(&ref); + store_inode(curs.val, inode); + scoutfs_btree_release(&curs); trace_scoutfs_update_inode(inode); } @@ -262,8 +265,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_btree_cursor curs = {NULL,}; struct scoutfs_inode_info *ci; - DECLARE_SCOUTFS_ITEM_REF(ref); struct scoutfs_key key; struct inode *inode; int ret; @@ -285,14 +288,14 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_create_item(inode->i_sb, &key, - sizeof(struct scoutfs_inode), &ref); + ret = scoutfs_btree_insert(inode->i_sb, &key, + sizeof(struct scoutfs_inode), &curs); if (ret) { iput(inode); return ERR_PTR(ret); } - scoutfs_put_ref(&ref); + scoutfs_btree_release(&curs); return inode; } diff --git a/kmod/src/ival.c b/kmod/src/ival.c deleted file mode 100644 index da111ef8..00000000 --- a/kmod/src/ival.c +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -#include "rbtree_aug.h" - -#include "format.h" -#include "key.h" -#include "ival.h" - -/* - * scoutfs wants to store overlapping key ranges and find intersections - * for tracking both segments in level 0 and granting access ranges. - * - * We use a simple augmented rbtree of key intervals that tracks the - * greatest end value of all the intervals in a node's subtree. Wikipedia - * data structures 101. - * - * Unfortunately the augmented rbtree callbacks need a tweak to compare - * our key structs. But we don't want to mess around with updating - * distro kernels. So we backport the augmented rbtree code from - * mainline in a private copy. This'll vanish when we bring scoutfs up - * to mainline. - */ - -static struct scoutfs_key *node_subtree_end(struct rb_node *node) -{ - struct scoutfs_ival *ival; - static struct scoutfs_key static_zero = {0,}; - - if (!node) - return &static_zero; - - ival = container_of(node, struct scoutfs_ival, node); - return &ival->subtree_end; -} - -static struct scoutfs_key compute_subtree_end(struct scoutfs_ival *ival) -{ - return *scoutfs_max_key(node_subtree_end(ival->node.rb_left), - node_subtree_end(ival->node.rb_right)); -} - -RB_DECLARE_CALLBACKS(static, ival_rb_cb, struct scoutfs_ival, node, - struct scoutfs_key, subtree_end, compute_subtree_end) - -void scoutfs_insert_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_ival *ins) -{ - struct rb_node **node = &tree->root.rb_node; - struct rb_node *parent = NULL; - struct scoutfs_ival *ival; - - giant_rbtree_hack_build_bugs(); - - while (*node) { - parent = *node; - ival = container_of(*node, struct scoutfs_ival, node); - - /* extend traversed subtree end to cover inserted end */ - ival->subtree_end = *scoutfs_max_key(&ival->subtree_end, - &ins->end); - - if (scoutfs_key_cmp(&ins->start, &ival->start) < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - ins->subtree_end = ins->end; - rb_link_node(&ins->node, parent, node); - rb_insert_augmented(&ins->node, &tree->root, &ival_rb_cb); -} - -void scoutfs_remove_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_ival *ival) -{ - if (!RB_EMPTY_NODE(&ival->node)) { - rb_erase_augmented(&ival->node, &tree->root, &ival_rb_cb); - RB_CLEAR_NODE(&ival->node); - } -} - -/* - * Find the interval in the tree with the lowest start value that - * intersects the search range. - */ -static struct scoutfs_ival *first_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct rb_node *node = tree->root.rb_node; - struct scoutfs_ival *ival; - - while (node) { - ival = container_of(node, struct scoutfs_ival, node); - - if (scoutfs_key_cmp(node_subtree_end(ival->node.rb_left), - start) >= 0) - node = node->rb_left; - else if (!scoutfs_cmp_key_ranges(start, end, - &ival->start, &ival->end)) - return ival; - else if (scoutfs_key_cmp(end, &ival->start) < 0) - break; - else - node = node->rb_right; - } - - return NULL; -} - -/* - * Find the next interval sorted by the start value which intersect the - * given search range. ival is null to first return the intersection - * with the lowest start value. The caller must serialize access while - * iterating. - */ -struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_key *start, - struct scoutfs_key *end, - struct scoutfs_ival *ival) -{ - struct rb_node *node; - - if (!ival) - return first_ival(tree, start, end); - - node = rb_next(&ival->node); - if (node) { - ival = container_of(node, struct scoutfs_ival, node); - if (!scoutfs_cmp_key_ranges(start, end, - &ival->start, &ival->end)) - return ival; - } - - return NULL; -} diff --git a/kmod/src/ival.h b/kmod/src/ival.h deleted file mode 100644 index 6c944e0e..00000000 --- a/kmod/src/ival.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef _SCOUTFS_IVAL_H_ -#define _SCOUTFS_IVAL_H_ - -struct scoutfs_ival_tree { - struct rb_root root; -}; - -static inline void scoutfs_init_ival_tree(struct scoutfs_ival_tree *tree) -{ - tree->root = RB_ROOT; -} - -struct scoutfs_ival { - struct rb_node node; - struct scoutfs_key start; - struct scoutfs_key end; - struct scoutfs_key subtree_end; -}; - -void scoutfs_insert_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_ival *ins); -void scoutfs_remove_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_ival *ival); -struct scoutfs_ival *scoutfs_next_ival(struct scoutfs_ival_tree *tree, - struct scoutfs_key *start, - struct scoutfs_key *end, - struct scoutfs_ival *ival); - -/* - * Walk all the intervals in postorder. This lets us free each ival we - * see without erasing and rebalancing. - */ -#define foreach_postorder_ival_safe(itree, ival, node, tmp) \ - for (node = rb_first_postorder(&(itree)->root); \ - ival = container_of(node, struct scoutfs_ival, node), \ - (node && (tmp = *node, 1)), node; \ - node = rb_next_postorder(&tmp)) - -// struct rb_node { -// long unsigned int __rb_parent_color; /* 0 8 */ -// struct rb_node * rb_right; /* 8 8 */ -// struct rb_node * rb_left; /* 16 8 */ -// -// /* size: 24, cachelines: 1, members: 3 */ -// /* last cacheline: 24 bytes */ -// }; -// struct rb_root { -// struct rb_node * rb_node; /* 0 8 */ -// -// /* size: 8, cachelines: 1, members: 1 */ -// /* last cacheline: 8 bytes */ -// }; - -/* - * Try to find out if the imported hacked rbtree in ival.c goes out of - * sync with the rbtree in the distro kernel. - */ -static inline void giant_rbtree_hack_build_bugs(void) -{ - size_t sz = sizeof(long); - - BUILD_BUG_ON(offsetof(struct rb_node, __rb_parent_color) != 0); - BUILD_BUG_ON(offsetof(struct rb_node, rb_right) != sz); - BUILD_BUG_ON(offsetof(struct rb_node, rb_left) != (sz * 2)); - BUILD_BUG_ON(sizeof(struct rb_node) != (sz * 3)); - - BUILD_BUG_ON(offsetof(struct rb_root, rb_node) != 0); - BUILD_BUG_ON(sizeof(struct rb_root) != sz); -} - -#endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c deleted file mode 100644 index 236bf911..00000000 --- a/kmod/src/manifest.c +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "manifest.h" -#include "key.h" -#include "ring.h" -#include "ival.h" -#include "scoutfs_trace.h" - -/* - * The manifest organizes log segments into levels of item indexes. New - * segments arrive at level 0 which can have many segments with - * overlapping keys. Then segments are merged into progressively larger - * higher levels which do not have segments with overlapping keys. - * - * All the segments for all the levels are stored in one interval tree. - * This lets reads find all the overlapping segments in all levels with - * one tree walk instead of walks per level. It also lets us move - * segments around the levels by updating their level field rather than - * removing them from one level index and adding them to another. - */ -struct scoutfs_manifest { - spinlock_t lock; - struct scoutfs_ival_tree itree; -}; - -/* - * There's some redundancy between the interval struct and the manifest - * entry struct. If we re-use both we duplicate fields and memory - * pressure is precious here. So we have a native combination of the - * two. - */ -struct scoutfs_manifest_node { - struct scoutfs_ival ival; - u64 blkno; - u64 seq; - unsigned char level; -}; - -/* - * Remove an exact match of the entry from the manifest. It's normal - * for ring replay can try to remove an entry that doesn't exist if ring - * wrapping and manifest deletion combine in just the right way. - */ -static void delete_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_node *mnode; - struct scoutfs_ival *ival; - - ival = NULL; - while ((ival = scoutfs_next_ival(&mani->itree, &ment->first, - &ment->last, ival))) { - mnode = container_of(ival, struct scoutfs_manifest_node, ival); - - if (mnode->blkno == le64_to_cpu(ment->blkno) && - mnode->seq == le64_to_cpu(ment->seq) && - !scoutfs_key_cmp(&ment->first, &mnode->ival.start) && - !scoutfs_key_cmp(&ment->last, &mnode->ival.end)) - break; - } - - if (ival) { - trace_scoutfs_delete_manifest(ment); - - scoutfs_remove_ival(&mani->itree, &mnode->ival); - kfree(mnode); - } -} - -void scoutfs_delete_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - - spin_lock(&mani->lock); - delete_manifest(sb, ment); - spin_unlock(&mani->lock); -} - -static void insert_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment, - struct scoutfs_manifest_node *mnode) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - - trace_scoutfs_insert_manifest(ment); - - mnode->ival.start = ment->first; - mnode->ival.end = ment->last; - mnode->blkno = le64_to_cpu(ment->blkno); - mnode->seq = le64_to_cpu(ment->seq); - mnode->level = ment->level; - - scoutfs_insert_ival(&mani->itree, &mnode->ival); -} - -int scoutfs_insert_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_node *mnode; - - mnode = kzalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); - if (!mnode) - return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ - - spin_lock(&mani->lock); - insert_manifest(sb, ment, mnode); - spin_unlock(&mani->lock); - - return 0; -} - -/* - * The caller has inserted a temporary manifest entry while they were - * dirtying a segment. It's done now and they want the final segment - * range stored in the manifest and logged in the ring. - * - * If this returns an error then nothing has changed. - * - * XXX we'd also need to add stale manifest entry's to the ring - * XXX In the future we'd send it to the leader - */ -int scoutfs_finalize_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *existing, - struct scoutfs_manifest_entry *updated) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_node *mnode; - int ret; - - mnode = kzalloc(sizeof(struct scoutfs_manifest_node), GFP_NOFS); - if (!mnode) - return -ENOMEM; /* XXX hmm, fatal? prealloc?*/ - - ret = scoutfs_dirty_ring_entry(sb, SCOUTFS_RING_ADD_MANIFEST, - updated, - sizeof(struct scoutfs_manifest_entry)); - if (ret) { - kfree(mnode); - return ret; - } - - spin_lock(&mani->lock); - delete_manifest(sb, existing); - insert_manifest(sb, updated, mnode); - spin_unlock(&mani->lock); - - return 0; -} - -/* sorted by increasing level then decreasing seq */ -static int cmp_ments(const void *A, const void *B) -{ - const struct scoutfs_manifest_entry *a = A; - const struct scoutfs_manifest_entry *b = B; - int cmp; - - cmp = (int)a->level - (int)b->level; - if (cmp) - return cmp; - - if (le64_to_cpu(a->seq) > le64_to_cpu(b->seq)) - return -1; - if (le64_to_cpu(a->seq) < le64_to_cpu(b->seq)) - return 1; - return 0; -} - -static void swap_ments(void *A, void *B, int size) -{ - struct scoutfs_manifest_entry *a = A; - struct scoutfs_manifest_entry *b = B; - - swap(*a, *b); -} - -/* - * Give the caller an allocated array of manifest entries that intersect - * their search key. The array is sorted in the order for searching for - * the most recent item: decreasing sequence in level 0 then increasing - * levels. - * - * The live manifest can change while the caller walks their array but - * the segments will not be reclaimed and the caller has grants that - * protect their items in the segments even if the segments shift over - * time. - * - * The number of elements in the array is returned, or negative errors, - * and the array is not allocated if 0 is returned. - * - * XXX need to actually keep the segments from being reclaimed - */ -int scoutfs_manifest_find_key(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_manifest_entry **ments_ret) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_manifest_entry *ments; - struct scoutfs_manifest_node *mnode; - struct scoutfs_ival *ival; - unsigned nr; - int i; - - /* make a reasonably large initial guess */ - i = 16; - ments = NULL; - do { - kfree(ments); - nr = i; - ments = kmalloc(nr * sizeof(struct scoutfs_manifest_entry), - GFP_NOFS); - if (!ments) - return -ENOMEM; - - spin_lock(&mani->lock); - i = 0; - ival = NULL; - while ((ival = scoutfs_next_ival(&mani->itree, key, key, - ival))) { - if (i < nr) { - mnode = container_of(ival, - struct scoutfs_manifest_node, ival); - ments[i].blkno = cpu_to_le64(mnode->blkno); - ments[i].seq = cpu_to_le64(mnode->seq); - ments[i].level = mnode->level; - ments[i].first = ival->start; - ments[i].last = ival->end; - } - i++; - } - spin_unlock(&mani->lock); - - } while (i > nr); - - if (i) { - sort(ments, i, sizeof(struct scoutfs_manifest_entry), - cmp_ments, swap_ments); - } else { - kfree(ments); - ments = NULL; - } - - *ments_ret = ments; - return i; -} - -int scoutfs_setup_manifest(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani; - - mani = kzalloc(sizeof(struct scoutfs_manifest), GFP_KERNEL); - if (!mani) - return -ENOMEM; - - spin_lock_init(&mani->lock); - scoutfs_init_ival_tree(&mani->itree); - - sbi->mani = mani; - - return 0; -} - -/* - * This is called once the manifest will no longer be used. - */ -void scoutfs_destroy_manifest(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest *mani = sbi->mani; - struct scoutfs_ival *ival; - struct rb_node *node; - struct rb_node tmp; - - if (mani) { - foreach_postorder_ival_safe(&mani->itree, ival, node, tmp) - kfree(ival); - - kfree(mani); - sbi->mani = NULL; - } -} diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h deleted file mode 100644 index 5223f069..00000000 --- a/kmod/src/manifest.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _SCOUTFS_MANIFEST_H_ -#define _SCOUTFS_MANIFEST_H_ - -int scoutfs_setup_manifest(struct super_block *sb); -void scoutfs_destroy_manifest(struct super_block *sb); - -int scoutfs_insert_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment); -void scoutfs_delete_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *ment); -int scoutfs_finalize_manifest(struct super_block *sb, - struct scoutfs_manifest_entry *existing, - struct scoutfs_manifest_entry *updated); - -int scoutfs_manifest_find_key(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_manifest_entry **ments_ret); - -#endif diff --git a/kmod/src/rbtree_aug.h b/kmod/src/rbtree_aug.h deleted file mode 100644 index 97791df0..00000000 --- a/kmod/src/rbtree_aug.h +++ /dev/null @@ -1,996 +0,0 @@ -/* - * The upstream augmented rbtree interface currently assumes that it - * can compare the augmented values directly: - * - * if (node->rbaugmented == augmented) - * break; - * - * This doesn't work for our struct key types. The only change needed - * to make this work for us is to turn that into a memcmp. But we're - * developing against distro kernels that sites actually use. For now - * we carry around this giant hack that imports the upstream copy and - * makes the change. It's only used in ival.c. - * - * This is a disgusting hack and also the right thing for this stage of - * the project. We'll fix this up as we submit upstream and trickle - * into distro kernels. - */ -#ifndef _GIANT_RBTREE_HACK_ -#define _GIANT_RBTREE_HACK_ - -/* forbid including kernel rbtree headers by way of includes below */ -#define _LINUX_RBTREE_AUGMENTED_H -#define _LINUX_RBTREE_H - -#include -#include -#include -#include - -#undef EXPORT_SYMBOL -#define EXPORT_SYMBOL(foo) - -/* - * then paste rbtree.h, rbtree_augmented.h, and rbtree.c - */ - -/* --------- rbtree.h ---------- */ - -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/include/linux/rbtree.h - - To use rbtrees you'll have to implement your own insert and search cores. - This will avoid us to use callbacks and to drop drammatically performances. - I know it's not the cleaner way, but in C (not in C++) to get - performances and genericity... - - See Documentation/rbtree.txt for documentation and samples. -*/ - - -struct rb_node { - unsigned long __rb_parent_color; - struct rb_node *rb_right; - struct rb_node *rb_left; -} __attribute__((aligned(sizeof(long)))); - /* The alignment might seem pointless, but allegedly CRIS needs it */ - -struct rb_root { - struct rb_node *rb_node; -}; - - -#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) - -#define RB_ROOT (struct rb_root) { NULL, } -#define rb_entry(ptr, type, member) container_of(ptr, type, member) - -#define RB_EMPTY_ROOT(root) (READ_ONCE((root)->rb_node) == NULL) - -/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */ -#define RB_EMPTY_NODE(node) \ - ((node)->__rb_parent_color == (unsigned long)(node)) -#define RB_CLEAR_NODE(node) \ - ((node)->__rb_parent_color = (unsigned long)(node)) - - -extern void rb_insert_color(struct rb_node *, struct rb_root *); -extern void rb_erase(struct rb_node *, struct rb_root *); - - -/* Find logical next and previous nodes in a tree */ -extern struct rb_node *rb_next(const struct rb_node *); -extern struct rb_node *rb_prev(const struct rb_node *); -extern struct rb_node *rb_first(const struct rb_root *); -extern struct rb_node *rb_last(const struct rb_root *); - -/* Postorder iteration - always visit the parent after its children */ -extern struct rb_node *rb_first_postorder(const struct rb_root *); -extern struct rb_node *rb_next_postorder(const struct rb_node *); - -/* Fast replacement of a single node without remove/rebalance/add/rebalance */ -extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, - struct rb_root *root); - -static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, - struct rb_node **rb_link) -{ - node->__rb_parent_color = (unsigned long)parent; - node->rb_left = node->rb_right = NULL; - - *rb_link = node; -} - -static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent, - struct rb_node **rb_link) -{ - node->__rb_parent_color = (unsigned long)parent; - node->rb_left = node->rb_right = NULL; - - rcu_assign_pointer(*rb_link, node); -} - -#define rb_entry_safe(ptr, type, member) \ - ({ typeof(ptr) ____ptr = (ptr); \ - ____ptr ? rb_entry(____ptr, type, member) : NULL; \ - }) - -/** - * rbtree_postorder_for_each_entry_safe - iterate in post-order over rb_root of - * given type allowing the backing memory of @pos to be invalidated - * - * @pos: the 'type *' to use as a loop cursor. - * @n: another 'type *' to use as temporary storage - * @root: 'rb_root *' of the rbtree. - * @field: the name of the rb_node field within 'type'. - * - * rbtree_postorder_for_each_entry_safe() provides a similar guarantee as - * list_for_each_entry_safe() and allows the iteration to continue independent - * of changes to @pos by the body of the loop. - * - * Note, however, that it cannot handle other modifications that re-order the - * rbtree it is iterating over. This includes calling rb_erase() on @pos, as - * rb_erase() may rebalance the tree, causing us to miss some nodes. - */ -#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \ - for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \ - pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \ - typeof(*pos), field); 1; }); \ - pos = n) - -/* --------- rbtree_augmented.h ---------- */ - -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - (C) 2002 David Woodhouse - (C) 2012 Michel Lespinasse - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/include/linux/rbtree_augmented.h -*/ - - -/* - * Please note - only struct rb_augment_callbacks and the prototypes for - * rb_insert_augmented() and rb_erase_augmented() are intended to be public. - * The rest are implementation details you are not expected to depend on. - * - * See Documentation/rbtree.txt for documentation and samples. - */ - -struct rb_augment_callbacks { - void (*propagate)(struct rb_node *node, struct rb_node *stop); - void (*copy)(struct rb_node *old, struct rb_node *new); - void (*rotate)(struct rb_node *old, struct rb_node *new); -}; - -extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); -/* - * Fixup the rbtree and update the augmented information when rebalancing. - * - * On insertion, the user must update the augmented information on the path - * leading to the inserted node, then call rb_link_node() as usual and - * rb_augment_inserted() instead of the usual rb_insert_color() call. - * If rb_augment_inserted() rebalances the rbtree, it will callback into - * a user provided function to update the augmented information on the - * affected subtrees. - */ -static inline void -rb_insert_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - __rb_insert_augmented(node, root, augment->rotate); -} - -#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \ - rbtype, rbaugmented, rbcompute) \ -static inline void \ -rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \ -{ \ - while (rb != stop) { \ - rbstruct *node = rb_entry(rb, rbstruct, rbfield); \ - rbtype augmented = rbcompute(node); \ - if (!memcmp(&node->rbaugmented, &augmented, \ - sizeof(augmented))) \ - break; \ - node->rbaugmented = augmented; \ - rb = rb_parent(&node->rbfield); \ - } \ -} \ -static inline void \ -rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ -{ \ - rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ - rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ - new->rbaugmented = old->rbaugmented; \ -} \ -static void \ -rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ -{ \ - rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ - rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ - new->rbaugmented = old->rbaugmented; \ - old->rbaugmented = rbcompute(old); \ -} \ -rbstatic const struct rb_augment_callbacks rbname = { \ - rbname ## _propagate, rbname ## _copy, rbname ## _rotate \ -}; - - -#define RB_RED 0 -#define RB_BLACK 1 - -#define __rb_parent(pc) ((struct rb_node *)(pc & ~3)) - -#define __rb_color(pc) ((pc) & 1) -#define __rb_is_black(pc) __rb_color(pc) -#define __rb_is_red(pc) (!__rb_color(pc)) -#define rb_color(rb) __rb_color((rb)->__rb_parent_color) -#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color) -#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color) - -static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) -{ - rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; -} - -static inline void rb_set_parent_color(struct rb_node *rb, - struct rb_node *p, int color) -{ - rb->__rb_parent_color = (unsigned long)p | color; -} - -static inline void -__rb_change_child(struct rb_node *old, struct rb_node *new, - struct rb_node *parent, struct rb_root *root) -{ - if (parent) { - if (parent->rb_left == old) - WRITE_ONCE(parent->rb_left, new); - else - WRITE_ONCE(parent->rb_right, new); - } else - WRITE_ONCE(root->rb_node, new); -} - -extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); - -static __always_inline struct rb_node * -__rb_erase_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - struct rb_node *child = node->rb_right; - struct rb_node *tmp = node->rb_left; - struct rb_node *parent, *rebalance; - unsigned long pc; - - if (!tmp) { - /* - * Case 1: node to erase has no more than 1 child (easy!) - * - * Note that if there is one child it must be red due to 5) - * and node must be black due to 4). We adjust colors locally - * so as to bypass __rb_erase_color() later on. - */ - pc = node->__rb_parent_color; - parent = __rb_parent(pc); - __rb_change_child(node, child, parent, root); - if (child) { - child->__rb_parent_color = pc; - rebalance = NULL; - } else - rebalance = __rb_is_black(pc) ? parent : NULL; - tmp = parent; - } else if (!child) { - /* Still case 1, but this time the child is node->rb_left */ - tmp->__rb_parent_color = pc = node->__rb_parent_color; - parent = __rb_parent(pc); - __rb_change_child(node, tmp, parent, root); - rebalance = NULL; - tmp = parent; - } else { - struct rb_node *successor = child, *child2; - - tmp = child->rb_left; - if (!tmp) { - /* - * Case 2: node's successor is its right child - * - * (n) (s) - * / \ / \ - * (x) (s) -> (x) (c) - * \ - * (c) - */ - parent = successor; - child2 = successor->rb_right; - - augment->copy(node, successor); - } else { - /* - * Case 3: node's successor is leftmost under - * node's right child subtree - * - * (n) (s) - * / \ / \ - * (x) (y) -> (x) (y) - * / / - * (p) (p) - * / / - * (s) (c) - * \ - * (c) - */ - do { - parent = successor; - successor = tmp; - tmp = tmp->rb_left; - } while (tmp); - child2 = successor->rb_right; - WRITE_ONCE(parent->rb_left, child2); - WRITE_ONCE(successor->rb_right, child); - rb_set_parent(child, successor); - - augment->copy(node, successor); - augment->propagate(parent, successor); - } - - tmp = node->rb_left; - WRITE_ONCE(successor->rb_left, tmp); - rb_set_parent(tmp, successor); - - pc = node->__rb_parent_color; - tmp = __rb_parent(pc); - __rb_change_child(node, successor, tmp, root); - - if (child2) { - successor->__rb_parent_color = pc; - rb_set_parent_color(child2, parent, RB_BLACK); - rebalance = NULL; - } else { - unsigned long pc2 = successor->__rb_parent_color; - successor->__rb_parent_color = pc; - rebalance = __rb_is_black(pc2) ? parent : NULL; - } - tmp = successor; - } - - augment->propagate(tmp, NULL); - return rebalance; -} - -static __always_inline void -rb_erase_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - struct rb_node *rebalance = __rb_erase_augmented(node, root, augment); - if (rebalance) - __rb_erase_color(rebalance, root, augment->rotate); -} - -/* --------- rbtree.c ---------- */ - -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - (C) 2002 David Woodhouse - (C) 2012 Michel Lespinasse - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/lib/rbtree.c -*/ - -/* - * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree - * - * 1) A node is either red or black - * 2) The root is black - * 3) All leaves (NULL) are black - * 4) Both children of every red node are black - * 5) Every simple path from root to leaves contains the same number - * of black nodes. - * - * 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two - * consecutive red nodes in a path and every red node is therefore followed by - * a black. So if B is the number of black nodes on every simple path (as per - * 5), then the longest possible path due to 4 is 2B. - * - * We shall indicate color with case, where black nodes are uppercase and red - * nodes will be lowercase. Unknown color nodes shall be drawn as red within - * parentheses and have some accompanying text comment. - */ - -/* - * Notes on lockless lookups: - * - * All stores to the tree structure (rb_left and rb_right) must be done using - * WRITE_ONCE(). And we must not inadvertently cause (temporary) loops in the - * tree structure as seen in program order. - * - * These two requirements will allow lockless iteration of the tree -- not - * correct iteration mind you, tree rotations are not atomic so a lookup might - * miss entire subtrees. - * - * But they do guarantee that any such traversal will only see valid elements - * and that it will indeed complete -- does not get stuck in a loop. - * - * It also guarantees that if the lookup returns an element it is the 'correct' - * one. But not returning an element does _NOT_ mean it's not present. - * - * NOTE: - * - * Stores to __rb_parent_color are not important for simple lookups so those - * are left undone as of now. Nor did I check for loops involving parent - * pointers. - */ - -static inline void rb_set_black(struct rb_node *rb) -{ - rb->__rb_parent_color |= RB_BLACK; -} - -static inline struct rb_node *rb_red_parent(struct rb_node *red) -{ - return (struct rb_node *)red->__rb_parent_color; -} - -/* - * Helper function for rotations: - * - old's parent and color get assigned to new - * - old gets assigned new as a parent and 'color' as a color. - */ -static inline void -__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, - struct rb_root *root, int color) -{ - struct rb_node *parent = rb_parent(old); - new->__rb_parent_color = old->__rb_parent_color; - rb_set_parent_color(old, new, color); - __rb_change_child(old, new, parent, root); -} - -static __always_inline void -__rb_insert(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; - - while (true) { - /* - * Loop invariant: node is red - * - * If there is a black parent, we are done. - * Otherwise, take some corrective action as we don't - * want a red root or two consecutive red nodes. - */ - if (!parent) { - rb_set_parent_color(node, NULL, RB_BLACK); - break; - } else if (rb_is_black(parent)) - break; - - gparent = rb_red_parent(parent); - - tmp = gparent->rb_right; - if (parent != tmp) { /* parent == gparent->rb_left */ - if (tmp && rb_is_red(tmp)) { - /* - * Case 1 - color flips - * - * G g - * / \ / \ - * p u --> P U - * / / - * n n - * - * However, since g's parent might be red, and - * 4) does not allow this, we need to recurse - * at g. - */ - rb_set_parent_color(tmp, gparent, RB_BLACK); - rb_set_parent_color(parent, gparent, RB_BLACK); - node = gparent; - parent = rb_parent(node); - rb_set_parent_color(node, parent, RB_RED); - continue; - } - - tmp = parent->rb_right; - if (node == tmp) { - /* - * Case 2 - left rotate at parent - * - * G G - * / \ / \ - * p U --> n U - * \ / - * n p - * - * This still leaves us in violation of 4), the - * continuation into Case 3 will fix that. - */ - tmp = node->rb_left; - WRITE_ONCE(parent->rb_right, tmp); - WRITE_ONCE(node->rb_left, parent); - if (tmp) - rb_set_parent_color(tmp, parent, - RB_BLACK); - rb_set_parent_color(parent, node, RB_RED); - augment_rotate(parent, node); - parent = node; - tmp = node->rb_right; - } - - /* - * Case 3 - right rotate at gparent - * - * G P - * / \ / \ - * p U --> n g - * / \ - * n U - */ - WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */ - WRITE_ONCE(parent->rb_right, gparent); - if (tmp) - rb_set_parent_color(tmp, gparent, RB_BLACK); - __rb_rotate_set_parents(gparent, parent, root, RB_RED); - augment_rotate(gparent, parent); - break; - } else { - tmp = gparent->rb_left; - if (tmp && rb_is_red(tmp)) { - /* Case 1 - color flips */ - rb_set_parent_color(tmp, gparent, RB_BLACK); - rb_set_parent_color(parent, gparent, RB_BLACK); - node = gparent; - parent = rb_parent(node); - rb_set_parent_color(node, parent, RB_RED); - continue; - } - - tmp = parent->rb_left; - if (node == tmp) { - /* Case 2 - right rotate at parent */ - tmp = node->rb_right; - WRITE_ONCE(parent->rb_left, tmp); - WRITE_ONCE(node->rb_right, parent); - if (tmp) - rb_set_parent_color(tmp, parent, - RB_BLACK); - rb_set_parent_color(parent, node, RB_RED); - augment_rotate(parent, node); - parent = node; - tmp = node->rb_left; - } - - /* Case 3 - left rotate at gparent */ - WRITE_ONCE(gparent->rb_right, tmp); /* == parent->rb_left */ - WRITE_ONCE(parent->rb_left, gparent); - if (tmp) - rb_set_parent_color(tmp, gparent, RB_BLACK); - __rb_rotate_set_parents(gparent, parent, root, RB_RED); - augment_rotate(gparent, parent); - break; - } - } -} - -/* - * Inline version for rb_erase() use - we want to be able to inline - * and eliminate the dummy_rotate callback there - */ -static __always_inline void -____rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - struct rb_node *node = NULL, *sibling, *tmp1, *tmp2; - - while (true) { - /* - * Loop invariants: - * - node is black (or NULL on first iteration) - * - node is not the root (parent is not NULL) - * - All leaf paths going through parent and node have a - * black node count that is 1 lower than other leaf paths. - */ - sibling = parent->rb_right; - if (node != sibling) { /* node == parent->rb_left */ - if (rb_is_red(sibling)) { - /* - * Case 1 - left rotate at parent - * - * P S - * / \ / \ - * N s --> p Sr - * / \ / \ - * Sl Sr N Sl - */ - tmp1 = sibling->rb_left; - WRITE_ONCE(parent->rb_right, tmp1); - WRITE_ONCE(sibling->rb_left, parent); - rb_set_parent_color(tmp1, parent, RB_BLACK); - __rb_rotate_set_parents(parent, sibling, root, - RB_RED); - augment_rotate(parent, sibling); - sibling = tmp1; - } - tmp1 = sibling->rb_right; - if (!tmp1 || rb_is_black(tmp1)) { - tmp2 = sibling->rb_left; - if (!tmp2 || rb_is_black(tmp2)) { - /* - * Case 2 - sibling color flip - * (p could be either color here) - * - * (p) (p) - * / \ / \ - * N S --> N s - * / \ / \ - * Sl Sr Sl Sr - * - * This leaves us violating 5) which - * can be fixed by flipping p to black - * if it was red, or by recursing at p. - * p is red when coming from Case 1. - */ - rb_set_parent_color(sibling, parent, - RB_RED); - if (rb_is_red(parent)) - rb_set_black(parent); - else { - node = parent; - parent = rb_parent(node); - if (parent) - continue; - } - break; - } - /* - * Case 3 - right rotate at sibling - * (p could be either color here) - * - * (p) (p) - * / \ / \ - * N S --> N Sl - * / \ \ - * sl Sr s - * \ - * Sr - */ - tmp1 = tmp2->rb_right; - WRITE_ONCE(sibling->rb_left, tmp1); - WRITE_ONCE(tmp2->rb_right, sibling); - WRITE_ONCE(parent->rb_right, tmp2); - if (tmp1) - rb_set_parent_color(tmp1, sibling, - RB_BLACK); - augment_rotate(sibling, tmp2); - tmp1 = sibling; - sibling = tmp2; - } - /* - * Case 4 - left rotate at parent + color flips - * (p and sl could be either color here. - * After rotation, p becomes black, s acquires - * p's color, and sl keeps its color) - * - * (p) (s) - * / \ / \ - * N S --> P Sr - * / \ / \ - * (sl) sr N (sl) - */ - tmp2 = sibling->rb_left; - WRITE_ONCE(parent->rb_right, tmp2); - WRITE_ONCE(sibling->rb_left, parent); - rb_set_parent_color(tmp1, sibling, RB_BLACK); - if (tmp2) - rb_set_parent(tmp2, parent); - __rb_rotate_set_parents(parent, sibling, root, - RB_BLACK); - augment_rotate(parent, sibling); - break; - } else { - sibling = parent->rb_left; - if (rb_is_red(sibling)) { - /* Case 1 - right rotate at parent */ - tmp1 = sibling->rb_right; - WRITE_ONCE(parent->rb_left, tmp1); - WRITE_ONCE(sibling->rb_right, parent); - rb_set_parent_color(tmp1, parent, RB_BLACK); - __rb_rotate_set_parents(parent, sibling, root, - RB_RED); - augment_rotate(parent, sibling); - sibling = tmp1; - } - tmp1 = sibling->rb_left; - if (!tmp1 || rb_is_black(tmp1)) { - tmp2 = sibling->rb_right; - if (!tmp2 || rb_is_black(tmp2)) { - /* Case 2 - sibling color flip */ - rb_set_parent_color(sibling, parent, - RB_RED); - if (rb_is_red(parent)) - rb_set_black(parent); - else { - node = parent; - parent = rb_parent(node); - if (parent) - continue; - } - break; - } - /* Case 3 - right rotate at sibling */ - tmp1 = tmp2->rb_left; - WRITE_ONCE(sibling->rb_right, tmp1); - WRITE_ONCE(tmp2->rb_left, sibling); - WRITE_ONCE(parent->rb_left, tmp2); - if (tmp1) - rb_set_parent_color(tmp1, sibling, - RB_BLACK); - augment_rotate(sibling, tmp2); - tmp1 = sibling; - sibling = tmp2; - } - /* Case 4 - left rotate at parent + color flips */ - tmp2 = sibling->rb_right; - WRITE_ONCE(parent->rb_left, tmp2); - WRITE_ONCE(sibling->rb_right, parent); - rb_set_parent_color(tmp1, sibling, RB_BLACK); - if (tmp2) - rb_set_parent(tmp2, parent); - __rb_rotate_set_parents(parent, sibling, root, - RB_BLACK); - augment_rotate(parent, sibling); - break; - } - } -} - -/* Non-inline version for rb_erase_augmented() use */ -void __rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - ____rb_erase_color(parent, root, augment_rotate); -} -EXPORT_SYMBOL(__rb_erase_color); - -/* - * Non-augmented rbtree manipulation functions. - * - * We use dummy augmented callbacks here, and have the compiler optimize them - * out of the rb_insert_color() and rb_erase() function definitions. - */ - -static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {} -static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {} -static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {} - -static const struct rb_augment_callbacks dummy_callbacks = { - dummy_propagate, dummy_copy, dummy_rotate -}; - -void rb_insert_color(struct rb_node *node, struct rb_root *root) -{ - __rb_insert(node, root, dummy_rotate); -} -EXPORT_SYMBOL(rb_insert_color); - -void rb_erase(struct rb_node *node, struct rb_root *root) -{ - struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); - if (rebalance) - ____rb_erase_color(rebalance, root, dummy_rotate); -} -EXPORT_SYMBOL(rb_erase); - -/* - * Augmented rbtree manipulation functions. - * - * This instantiates the same __always_inline functions as in the non-augmented - * case, but this time with user-defined callbacks. - */ - -void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - __rb_insert(node, root, augment_rotate); -} -EXPORT_SYMBOL(__rb_insert_augmented); - -/* - * This function returns the first node (in sort order) of the tree. - */ -struct rb_node *rb_first(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_left) - n = n->rb_left; - return n; -} -EXPORT_SYMBOL(rb_first); - -struct rb_node *rb_last(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_right) - n = n->rb_right; - return n; -} -EXPORT_SYMBOL(rb_last); - -struct rb_node *rb_next(const struct rb_node *node) -{ - struct rb_node *parent; - - if (RB_EMPTY_NODE(node)) - return NULL; - - /* - * If we have a right-hand child, go down and then left as far - * as we can. - */ - if (node->rb_right) { - node = node->rb_right; - while (node->rb_left) - node=node->rb_left; - return (struct rb_node *)node; - } - - /* - * No right-hand children. Everything down and left is smaller than us, - * so any 'next' node must be in the general direction of our parent. - * Go up the tree; any time the ancestor is a right-hand child of its - * parent, keep going up. First time it's a left-hand child of its - * parent, said parent is our 'next' node. - */ - while ((parent = rb_parent(node)) && node == parent->rb_right) - node = parent; - - return parent; -} -EXPORT_SYMBOL(rb_next); - -struct rb_node *rb_prev(const struct rb_node *node) -{ - struct rb_node *parent; - - if (RB_EMPTY_NODE(node)) - return NULL; - - /* - * If we have a left-hand child, go down and then right as far - * as we can. - */ - if (node->rb_left) { - node = node->rb_left; - while (node->rb_right) - node=node->rb_right; - return (struct rb_node *)node; - } - - /* - * No left-hand children. Go up till we find an ancestor which - * is a right-hand child of its parent. - */ - while ((parent = rb_parent(node)) && node == parent->rb_left) - node = parent; - - return parent; -} -EXPORT_SYMBOL(rb_prev); - -void rb_replace_node(struct rb_node *victim, struct rb_node *new, - struct rb_root *root) -{ - struct rb_node *parent = rb_parent(victim); - - /* Set the surrounding nodes to point to the replacement */ - __rb_change_child(victim, new, parent, root); - if (victim->rb_left) - rb_set_parent(victim->rb_left, new); - if (victim->rb_right) - rb_set_parent(victim->rb_right, new); - - /* Copy the pointers/colour from the victim to the replacement */ - *new = *victim; -} -EXPORT_SYMBOL(rb_replace_node); - -static struct rb_node *rb_left_deepest_node(const struct rb_node *node) -{ - for (;;) { - if (node->rb_left) - node = node->rb_left; - else if (node->rb_right) - node = node->rb_right; - else - return (struct rb_node *)node; - } -} - -struct rb_node *rb_next_postorder(const struct rb_node *node) -{ - const struct rb_node *parent; - if (!node) - return NULL; - parent = rb_parent(node); - - /* If we're sitting on node, we've already seen our children */ - if (parent && node == parent->rb_left && parent->rb_right) { - /* If we are the parent's left node, go to the parent's right - * node then all the way down to the left */ - return rb_left_deepest_node(parent->rb_right); - } else - /* Otherwise we are the parent's right node, and the parent - * should be next */ - return (struct rb_node *)parent; -} -EXPORT_SYMBOL(rb_next_postorder); - -struct rb_node *rb_first_postorder(const struct rb_root *root) -{ - if (!root->rb_node) - return NULL; - - return rb_left_deepest_node(root->rb_node); -} -EXPORT_SYMBOL(rb_first_postorder); - -#endif /* _GIANT_RBTREE_HACK_ */ diff --git a/kmod/src/ring.c b/kmod/src/ring.c deleted file mode 100644 index cbb36835..00000000 --- a/kmod/src/ring.c +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include - -#include "format.h" -#include "dir.h" -#include "inode.h" -#include "key.h" -#include "super.h" -#include "manifest.h" -#include "chunk.h" -#include "block.h" -#include "ring.h" - -static int replay_ring_block(struct super_block *sb, struct buffer_head *bh) -{ - struct scoutfs_ring_block *ring = (void *)bh->b_data; - struct scoutfs_ring_entry *ent = (void *)(ring + 1); - struct scoutfs_manifest_entry *ment; - struct scoutfs_ring_bitmap *bm; - int ret = 0; - int i; - - /* XXX verify */ - - for (i = 0; i < le16_to_cpu(ring->nr_entries); i++) { - switch(ent->type) { - case SCOUTFS_RING_ADD_MANIFEST: - ment = (void *)(ent + 1); - ret = scoutfs_insert_manifest(sb, ment); - break; - case SCOUTFS_RING_DEL_MANIFEST: - ment = (void *)(ent + 1); - scoutfs_delete_manifest(sb, ment); - break; - case SCOUTFS_RING_BITMAP: - bm = (void *)(ent + 1); - scoutfs_set_chunk_alloc_bits(sb, bm); - break; - default: - /* XXX */ - break; - } - - ent = (void *)(ent + 1) + le16_to_cpu(ent->len); - } - - return ret; -} - -/* - * Return the block number of the block that contains the given logical - * block in the ring. We look up ring block chunks in the map blocks - * in the chunk described by the super. - */ -static u64 map_ring_block(struct super_block *sb, u64 block) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_ring_map_block *map; - struct buffer_head *bh; - u64 ring_chunk; - u32 ring_block; - u64 blkno; - u64 div; - u32 rem; - - ring_block = block & SCOUTFS_CHUNK_BLOCK_MASK; - ring_chunk = block >> SCOUTFS_CHUNK_BLOCK_SHIFT; - - div = div_u64_rem(ring_chunk, SCOUTFS_RING_MAP_BLOCKS, &rem); - - bh = scoutfs_read_block(sb, le64_to_cpu(super->ring_map_blkno) + div); - if (!bh) - return 0; - - /* XXX verify map block */ - - map = (void *)bh->b_data; - blkno = le64_to_cpu(map->blknos[rem]) + ring_block; - brelse(bh); - - return blkno; -} - -/* - * Read a given logical ring block. - */ -static struct buffer_head *read_ring_block(struct super_block *sb, u64 block) -{ - u64 blkno = map_ring_block(sb, block); - - if (!blkno) - return NULL; - - return scoutfs_read_block(sb, blkno); -} - -/* - * Return a dirty locked logical ring block. - */ -static struct buffer_head *new_ring_block(struct super_block *sb, u64 block) -{ - u64 blkno = map_ring_block(sb, block); - - if (!blkno) - return NULL; - - return scoutfs_new_block(sb, blkno); -} - -int scoutfs_replay_ring(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buffer_head *bh; - u64 block; - int ret; - int i; - - /* XXX read-ahead map blocks and each set of ring blocks */ - - block = le64_to_cpu(super->ring_first_block); - for (i = 0; i < le64_to_cpu(super->ring_active_blocks); i++) { - bh = read_ring_block(sb, block); - if (!bh) { - ret = -EIO; - break; - } - - ret = replay_ring_block(sb, bh); - brelse(bh); - if (ret) - break; - - if (++block == le64_to_cpu(super->ring_total_blocks)) - block = 0; - } - - return ret; -} - -/* - * The caller is generating ring entries for manifest and allocator - * bitmap as they write items to blocks. We pin the block that we're - * working on so that it isn't written out until we fill it and - * calculate its checksum. - */ -int scoutfs_dirty_ring_entry(struct super_block *sb, u8 type, void *data, - u16 len) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_ring_block *ring; - struct scoutfs_ring_entry *ent; - struct buffer_head *bh; - unsigned int avail; - u64 block; - int ret = 0; - - bh = sbi->dirty_ring_bh; - ent = sbi->dirty_ring_ent; - avail = sbi->dirty_ring_ent_avail; - - if (bh && len > avail) { - scoutfs_finish_dirty_ring(sb); - bh = NULL; - } - if (!bh) { - block = le64_to_cpu(super->ring_first_block) + - le64_to_cpu(super->ring_active_blocks); - if (block >= le64_to_cpu(super->ring_total_blocks)) - block -= le64_to_cpu(super->ring_total_blocks); - - bh = new_ring_block(sb, block); - if (!bh) { - ret = -ENOMEM; - goto out; - } - - ring = (void *)bh->b_data; - ring->nr_entries = 0; - ent = (void *)(ring + 1); - /* assuming len fits in new empty block */ - } - - ring = (void *)bh->b_data; - - ent->type = type; - ent->len = cpu_to_le16(len); - memcpy(ent + 1, data, len); - le16_add_cpu(&ring->nr_entries, 1); - - ent = (void *)(ent + 1) + le16_to_cpu(ent->len); - avail = SCOUTFS_BLOCK_SIZE - ((char *)(ent + 1) - (char *)ring); -out: - sbi->dirty_ring_bh = bh; - sbi->dirty_ring_ent = ent; - sbi->dirty_ring_ent_avail = avail; - - return ret; -} - -/* - * The super might have a pinned partial dirty ring block. This is - * called as we finish the block or when the commit is done. We - * calculate the checksum and unlock it so it can be written. - * - * XXX This is about to write a partial block. We might as well fill - * that space with more old entries from the manifest and ring before - * we write it. - */ -int scoutfs_finish_dirty_ring(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buffer_head *bh; - - bh = sbi->dirty_ring_bh; - if (!bh) - return 0; - - sbi->dirty_ring_bh = NULL; - - /* - * XXX we're not zeroing the tail of the block here. We will - * when we change the item block format to let us append to - * the block without walking all the items. - */ - scoutfs_calc_hdr_crc(bh); - mark_buffer_dirty(bh); - unlock_buffer(bh); - brelse(bh); - - le64_add_cpu(&super->ring_active_blocks, 1); - - return 0; -} diff --git a/kmod/src/ring.h b/kmod/src/ring.h deleted file mode 100644 index ee929e20..00000000 --- a/kmod/src/ring.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef _SCOUTFS_RING_H_ -#define _SCOUTFS_RING_H_ - -int scoutfs_replay_ring(struct super_block *sb); -int scoutfs_dirty_ring_entry(struct super_block *sb, u8 type, void *data, - u16 len); -int scoutfs_finish_dirty_ring(struct super_block *sb); - -#endif diff --git a/kmod/src/scoutfs_trace.c b/kmod/src/scoutfs_trace.c index 38e147dc..038eb228 100644 --- a/kmod/src/scoutfs_trace.c +++ b/kmod/src/scoutfs_trace.c @@ -24,9 +24,6 @@ #include "dir.h" #include "msg.h" #include "block.h" -#include "manifest.h" -#include "ring.h" -#include "segment.h" #define CREATE_TRACE_POINTS #include "scoutfs_trace.h" diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 44f312c3..015a6700 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -27,48 +27,6 @@ #include "key.h" #include "format.h" -TRACE_EVENT(scoutfs_bloom_hit, - TP_PROTO(struct scoutfs_key *key), - - TP_ARGS(key), - - TP_STRUCT__entry( - __field(__u64, inode) - __field(__u8, type) - __field(__u64, offset) - ), - - TP_fast_assign( - __entry->inode = le64_to_cpu(key->inode); - __entry->type = key->type; - __entry->offset = le64_to_cpu(key->offset); - ), - - TP_printk("key %llu.%u.%llu", - __entry->inode, __entry->type, __entry->offset) -); - -TRACE_EVENT(scoutfs_bloom_miss, - TP_PROTO(struct scoutfs_key *key), - - TP_ARGS(key), - - TP_STRUCT__entry( - __field(__u64, inode) - __field(__u8, type) - __field(__u64, offset) - ), - - TP_fast_assign( - __entry->inode = le64_to_cpu(key->inode); - __entry->type = key->type; - __entry->offset = le64_to_cpu(key->offset); - ), - - TP_printk("key %llu.%u.%llu", - __entry->inode, __entry->type, __entry->offset) -); - TRACE_EVENT(scoutfs_write_begin, TP_PROTO(u64 ino, loff_t pos, unsigned len), @@ -151,116 +109,6 @@ TRACE_EVENT(scoutfs_update_inode, __entry->ino, __entry->size) ); -TRACE_EVENT(scoutfs_dirty_super, - TP_PROTO(struct scoutfs_super_block *super), - - TP_ARGS(super), - - TP_STRUCT__entry( - __field(__u64, blkno) - __field(__u64, seq) - ), - - TP_fast_assign( - __entry->blkno = le64_to_cpu(super->hdr.blkno); - __entry->seq = le64_to_cpu(super->hdr.seq); - ), - - TP_printk("blkno %llu seq %llu", - __entry->blkno, __entry->seq) -); - -TRACE_EVENT(scoutfs_write_super, - TP_PROTO(struct scoutfs_super_block *super), - - TP_ARGS(super), - - TP_STRUCT__entry( - __field(__u64, blkno) - __field(__u64, seq) - ), - - TP_fast_assign( - __entry->blkno = le64_to_cpu(super->hdr.blkno); - __entry->seq = le64_to_cpu(super->hdr.seq); - ), - - TP_printk("blkno %llu seq %llu", - __entry->blkno, __entry->seq) -); - -TRACE_EVENT(scoutfs_insert_manifest, - TP_PROTO(struct scoutfs_manifest_entry *ment), - - TP_ARGS(ment), - - TP_STRUCT__entry( - __field(__u64, blkno) - __field(__u64, seq) - __field(__u8, level) - __field(__u64, first_inode) - __field(__u8, first_type) - __field(__u64, first_offset) - __field(__u64, last_inode) - __field(__u8, last_type) - __field(__u64, last_offset) - ), - - TP_fast_assign( - __entry->blkno = le64_to_cpu(ment->blkno); - __entry->seq = le64_to_cpu(ment->seq); - __entry->level = ment->level; - __entry->first_inode = le64_to_cpu(ment->first.inode); - __entry->first_type = ment->first.type; - __entry->first_offset = le64_to_cpu(ment->first.offset); - __entry->last_inode = le64_to_cpu(ment->last.inode); - __entry->last_type = ment->last.type; - __entry->last_offset = le64_to_cpu(ment->last.offset); - ), - - TP_printk("blkno %llu seq %llu level %u first "CKF" last "CKF, - __entry->blkno, __entry->seq, __entry->level, - __entry->first_inode, __entry->first_type, - __entry->first_offset, __entry->last_inode, - __entry->last_type, __entry->last_offset) -); - -TRACE_EVENT(scoutfs_delete_manifest, - TP_PROTO(struct scoutfs_manifest_entry *ment), - - TP_ARGS(ment), - - TP_STRUCT__entry( - __field(__u64, blkno) - __field(__u64, seq) - __field(__u8, level) - __field(__u64, first_inode) - __field(__u8, first_type) - __field(__u64, first_offset) - __field(__u64, last_inode) - __field(__u8, last_type) - __field(__u64, last_offset) - ), - - TP_fast_assign( - __entry->blkno = le64_to_cpu(ment->blkno); - __entry->seq = le64_to_cpu(ment->seq); - __entry->level = ment->level; - __entry->first_inode = le64_to_cpu(ment->first.inode); - __entry->first_type = ment->first.type; - __entry->first_offset = le64_to_cpu(ment->first.offset); - __entry->last_inode = le64_to_cpu(ment->last.inode); - __entry->last_type = ment->last.type; - __entry->last_offset = le64_to_cpu(ment->last.offset); - ), - - TP_printk("blkno %llu seq %llu level %u first "CKF" last "CKF, - __entry->blkno, __entry->seq, __entry->level, - __entry->first_inode, __entry->first_type, - __entry->first_offset, __entry->last_inode, - __entry->last_type, __entry->last_offset) -); - #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/segment.c b/kmod/src/segment.c deleted file mode 100644 index 0591c7f7..00000000 --- a/kmod/src/segment.c +++ /dev/null @@ -1,805 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include - -#include "super.h" -#include "key.h" -#include "segment.h" -#include "manifest.h" -#include "block.h" -#include "chunk.h" -#include "ring.h" -#include "bloom.h" -#include "skip.h" - -/* - * scoutfs log segments are large multi-block structures that contain - * key/value items. This file implements manipulations of the items. - * - * Each log segment starts with a bloom filter to supports quickly - * testing for key values without having to search the whole block for a - * key. - * - * After the bloom filter come the packed structures that describe the - * items that are present in the block. They're sorted in a skip list - * to support reasonably efficient insertion, sorted iteration, and - * deletion. - * - * Finally the item values are stored at the end of the block. This - * supports finding that an item's key isn't present by only reading the - * item structs, not the values. - * - * All told, should we chose to, we can have three large portions of the - * blocks resident for searching. It's likely that we'll keep the bloom - * filters hot but that the items and especially the values may age out - * of the cache. - */ - -void scoutfs_put_ref(struct scoutfs_item_ref *ref) -{ - if (ref->item_bh) - brelse(ref->item_bh); - if (ref->val_bh) - brelse(ref->val_bh); - - memset(ref, 0, sizeof(struct scoutfs_item_ref)); -} - -/* private to here */ -struct scoutfs_item_iter { - struct list_head list; - struct buffer_head *bh; - struct scoutfs_item *item; - u64 blkno; - struct scoutfs_key after_seg; -}; - -void scoutfs_put_iter_list(struct list_head *list) -{ - struct scoutfs_item_iter *iter; - struct scoutfs_item_iter *pos; - - list_for_each_entry_safe(iter, pos, list, list) { - list_del_init(&iter->list); - brelse(iter->bh); - kfree(iter); - } -} - -/* - * The caller has a pointer to an item and a reference to its block. We - * read the value block and populate the reference. - * - * The item references get their own buffer head references so that the - * caller doesn't have to play funny games. They always have to drop - * their release bh. If this succeeds then they also need to put the - * ref. - */ -static int populate_ref(struct super_block *sb, u64 blkno, - struct buffer_head *item_bh, struct scoutfs_item *item, - struct scoutfs_item_ref *ref) -{ - struct buffer_head *bh; - - bh = scoutfs_read_block_off(sb, blkno, le32_to_cpu(item->offset)); - if (!bh) - return -EIO; - - ref->key = &item->key; - ref->val_len = le16_to_cpu(item->len); - ref->val = bh->b_data + (le32_to_cpu(item->offset) & - SCOUTFS_BLOCK_MASK); - get_bh(item_bh); - ref->item_bh = item_bh; - ref->val_bh = bh; - - return 0; -} - -/* - * Segments are immutable once they're written. As they're being - * dirtied we need to lock concurrent access. XXX the dirty blkno test - * is probably racey. We could use reader/writer locks here. And we - * could probably make the skip lists support concurrent access. - */ -static bool try_lock_dirty_mutex(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (blkno == sbi->dirty_blkno) { - mutex_lock(&sbi->dirty_mutex); - if (blkno == sbi->dirty_blkno) - return true; - mutex_unlock(&sbi->dirty_mutex); - } - - return false; -} - -/* - * Return a reference to the item at the given key. We walk the manifest - * to find blocks that might contain the key from most recent to oldest. - * To find the key in each log segment we test it's bloom filter and - * then search through the item keys. The first matching item we find - * is returned. - * - * -ENOENT is returned if the item isn't present. The caller needs to put - * the ref if we return success. - */ -int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_item_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *item = NULL; - struct scoutfs_bloom_bits bits; - struct scoutfs_manifest_entry *ments; - struct buffer_head *bh; - bool locked; - u64 blkno; - int ret; - int nr; - int i; - - /* XXX hold manifest */ - - scoutfs_calc_bloom_bits(&bits, key, sbi->super.bloom_salts); - - item = NULL; - ret = -ENOENT; - - nr = scoutfs_manifest_find_key(sb, key, &ments); - if (nr < 0) - return nr; - if (nr == 0) - return -ENOENT; - - for (i = 0; i < nr; i++) { - /* XXX read-ahead all bloom blocks */ - blkno = le64_to_cpu(ments[i].blkno); - /* XXX verify seqs */ - - ret = scoutfs_test_bloom_bits(sb, blkno, key, &bits); - if (ret < 0) - break; - if (!ret) { - ret = -ENOENT; - continue; - } - - /* XXX read-ahead all item header blocks */ - - locked = try_lock_dirty_mutex(sb, blkno); - ret = scoutfs_skip_lookup(sb, blkno, key, &bh, &item); - if (locked) - mutex_unlock(&sbi->dirty_mutex); - if (ret) { - if (ret == -ENOENT) - continue; - break; - } - break; - } - - kfree(ments); - - /* XXX release manifest */ - - /* XXX read-ahead all value blocks? */ - - if (!ret) { - ret = populate_ref(sb, blkno, bh, item, ref); - brelse(bh); - } - - return ret; -} - -/* return the byte length of the item header including its skip elements */ -static int item_bytes(int height) -{ - return offsetof(struct scoutfs_item, skip_next[height]); -} - -/* - * The dirty_item_off points to the byte offset after the last item. - * Advance it past block tails and initial block headers until there's - * room for an item with the given skip list elements height. Then set - * the dirty_item_off past the item offset item we return. - */ -static int add_item_off(struct scoutfs_sb_info *sbi, int height) -{ - int len = item_bytes(height); - int off = sbi->dirty_item_off; - int block_off; - int tail_free; - - /* items can't start in a block header */ - block_off = off & SCOUTFS_BLOCK_MASK; - if (block_off < sizeof(struct scoutfs_block_header)) - off += sizeof(struct scoutfs_block_header) - block_off; - - /* items can't cross a block boundary */ - tail_free = SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK); - if (tail_free < len) - off += tail_free + sizeof(struct scoutfs_block_header); - - sbi->dirty_item_off = off + len; - return off; -} - -/* - * The dirty_val_off points to the first byte of the last value that - * was allocated. Subtract the offset to make room for a new item - * of the given length. If that crosses a block boundary or wanders - * into the block header then pull it back into the tail of the previous - * block. - */ -static int sub_val_off(struct scoutfs_sb_info *sbi, int len) -{ - int off = sbi->dirty_val_off - len; - int block_off; - int tail_free; - - /* values can't start in a block header */ - block_off = off & SCOUTFS_BLOCK_MASK; - if (block_off < sizeof(struct scoutfs_block_header)) - off -= (block_off + 1); - - /* values can't cross a block boundary */ - tail_free = SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK); - if (tail_free < len) - off -= len - tail_free; - - sbi->dirty_val_off = off; - return off; -} - -/* - * Initialize the buffers for the next dirty segment. We have to initialize - * the bloom filter bits and the item block header. - * - * XXX we need to really pin the blocks somehow - */ -static int start_dirty_segment(struct super_block *sb, u64 blkno) -{ - struct scoutfs_bloom_block *blm; - struct scoutfs_item_block *iblk; - struct buffer_head *bh; - int ret = 0; - int i; - - for (i = 0; i < SCOUTFS_BLOCKS_PER_CHUNK; i++) { - bh = scoutfs_new_block(sb, blkno + i); - if (!bh) { - ret = -EIO; - break; - } - - if (i < SCOUTFS_BLOOM_BLOCKS) { - blm = (void *)bh->b_data; - memset(blm->bits, 0, SCOUTFS_BLOCK_SIZE - - offsetof(struct scoutfs_bloom_block, bits)); - } - - if (i == SCOUTFS_BLOOM_BLOCKS) { - iblk = (void *)bh->b_data; - memset(&iblk->first, ~0, sizeof(struct scoutfs_key)); - memset(&iblk->last, 0, sizeof(struct scoutfs_key)); - memset(&iblk->skip_root, 0, sizeof(iblk->skip_root) + - sizeof(struct scoutfs_item)); - } - - /* bh is pinned by sbi->dirty_blkno */ - } - - while (ret && i--) { - /* unwind pinned blocks on failure */ - bh = sb_getblk(sb, blkno + i); - if (bh) { - brelse(bh); - brelse(bh); - } - } - - return ret; -} - -/* - * As we start to fill a dirty segment we don't know which keys it's - * going to contain. We add a manifest entry in memory that has it - * contain all items so that reading will know to search the dirty - * segment. - * - * Once it's finalized we know the specific range of items it contains - * and we update the manifest entry in memory for that range and write - * that to the ring. - * - * Inserting the updated segment can fail. If we deleted the segment, - * then insertion failed, then reinserting the original entry could fail. - * Instead we briefly allow two manifest entries for the same segment. - */ -static int update_dirty_segment_manifest(struct super_block *sb, u64 blkno, - bool all_items) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest_entry ment; - struct scoutfs_manifest_entry updated; - struct scoutfs_item_block *iblk; - struct buffer_head *bh; - - ment.blkno = cpu_to_le64(blkno); - ment.seq = sbi->super.hdr.seq; - ment.level = 0; - memset(&ment.first, 0, sizeof(struct scoutfs_key)); - memset(&ment.last, ~0, sizeof(struct scoutfs_key)); - - if (all_items) - return scoutfs_insert_manifest(sb, &ment); - - bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); - if (!bh) - return -EIO; - - updated = ment; - - iblk = (void *)bh->b_data; - updated.first = iblk->first; - updated.last = iblk->last; - brelse(bh); - - return scoutfs_finalize_manifest(sb, &ment, &updated); -} - -/* - * Zero the portion of this block that intersects with the free space in - * the middle of the segment. @start and @end are chunk-relative byte - * offsets of the inclusive start and exclusive end of the free region. - */ -static void zero_unused_block(struct super_block *sb, struct buffer_head *bh, - u32 start, u32 end) -{ - u32 off = bh->b_blocknr << SCOUTFS_BLOCK_SHIFT; - - /* see if the segment range falls outside our block */ - if (start >= off + SCOUTFS_BLOCK_SIZE || end <= off) - return; - - /* convert the chunk offsets to our block offsets */ - start = max(start, off) - off; - end = min(off + SCOUTFS_BLOCK_SIZE, end) - off; - - /* don't zero block headers */ - start = max_t(u32, start, sizeof(struct scoutfs_block_header)); - end = max_t(u32, start, sizeof(struct scoutfs_block_header)); - - if (start < end) - memset(bh->b_data + start, 0, end - start); -} - -/* - * Finish off a dirty segment if we have one. Calculate the checksums of - * all the blocks, mark them dirty, and drop their pinned reference. - * - * XXX should do something with empty dirty segments. - */ -static int finish_dirty_segment(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - struct buffer_head *bh; - u64 blkno = sbi->dirty_blkno; - int ret = 0; - u64 i; - - WARN_ON_ONCE(!blkno); - - for (i = 0; i < SCOUTFS_BLOCKS_PER_CHUNK; i++) { - bh = scoutfs_read_block(sb, blkno + i); - /* should have been pinned */ - if (WARN_ON_ONCE(!bh)) { - ret = -EIO; - break; - } - - zero_unused_block(sb, bh, sbi->dirty_item_off, - sbi->dirty_val_off); - - scoutfs_calc_hdr_crc(bh); - mark_buffer_dirty(bh); - brelse(bh); - /* extra release to unpin */ - brelse(bh); - } - - /* update manifest with range of items and add to ring */ - ret = update_dirty_segment_manifest(sb, blkno, false); - - /* - * Try to kick off a background write of the finished segment. Callers - * can wait for the buffers in writeback if they need to. - */ - if (!ret) { - filemap_fdatawrite_range(mapping, blkno << SCOUTFS_CHUNK_SHIFT, - ((blkno + 1) << SCOUTFS_CHUNK_SHIFT) - 1); - sbi->dirty_blkno = 0; - } - - return ret; -} - -/* - * We've been dirtying log segment blocks and ring blocks as items were - * modified. sync makes sure that they're all persistent and updates - * the super. - * - * XXX need to synchronize with transactions - * XXX is state clean after errors? - */ -int scoutfs_sync_fs(struct super_block *sb, int wait) -{ - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret = 0; - - mutex_unlock(&sbi->dirty_mutex); - if (sbi->dirty_blkno) { - ret = finish_dirty_segment(sb) ?: - scoutfs_finish_dirty_ring(sb) ?: - filemap_write_and_wait(mapping) ?: - scoutfs_write_dirty_super(sb) ?: - scoutfs_advance_dirty_super(sb); - } - mutex_unlock(&sbi->dirty_mutex); - return ret; -} - - -/* - * Return a reference to a newly allocated and initialized item in a - * block in the currently dirty log segment. - * - * Item creation is purposely kept very simple. Item and value offset - * allocation proceed from either end of the log segment. Once they - * intersect the log segment is full and written out. Deleted dirty - * items don't reclaim their space. The free space will be reclaimed by - * the level 0 -> level 1 merge that happens anyway. Not reclaiming - * free space makes item location more rigid and lets us relax the - * locking requirements of item references. An item reference doesn't - * have to worry about unrelated item modification moving their item - * around to, say, defragment free space. - */ -int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, - unsigned bytes, struct scoutfs_item_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_bloom_bits bits; - struct scoutfs_item *item; - struct scoutfs_item_block *iblk; - struct buffer_head *bh; - int item_off; - int val_off; - int height; - u64 blkno; - int ret = 0; - - /* XXX how big should items really get? */ - if (WARN_ON_ONCE(bytes == 0 || bytes > 4096)) - return -EINVAL; - - height = scoutfs_skip_random_height(); - - mutex_lock(&sbi->dirty_mutex); - -next_chunk: - if (!sbi->dirty_blkno) { - ret = scoutfs_alloc_chunk(sb, &blkno); - if (ret) - goto out; - - /* XXX free blkno on error? */ - ret = start_dirty_segment(sb, blkno); - if (ret) - goto out; - - /* add initial in-memory manifest entry with all items */ - ret = update_dirty_segment_manifest(sb, blkno, true); - if (ret) - goto out; - - sbi->dirty_blkno = blkno; - sbi->dirty_item_off = - (SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + - sizeof(struct scoutfs_item_block); - sbi->dirty_val_off = SCOUTFS_CHUNK_SIZE; - } - - item_off = add_item_off(sbi, height); - val_off = sub_val_off(sbi, bytes); - - trace_printk("item_off %u val_off %u\n", item_off, val_off); - - if (item_off + item_bytes(height) > val_off) { - ret = finish_dirty_segment(sb); - if (ret) - goto out; - goto next_chunk; - } - - /* XXX fix up this error handling in general */ - - bh = scoutfs_read_block_off(sb, sbi->dirty_blkno, item_off); - if (!bh) { - ret = -EIO; - goto out; - } - - item = (void *)bh->b_data + (item_off & SCOUTFS_BLOCK_MASK); - item->key = *key; - item->offset = cpu_to_le32(val_off); - item->len = cpu_to_le16(bytes); - item->skip_height = height; - - ret = scoutfs_skip_insert(sb, sbi->dirty_blkno, item, item_off); - if (ret) - goto out; - - ret = populate_ref(sb, sbi->dirty_blkno, bh, item, ref); - brelse(bh); - if (ret) - goto out; - - bh = scoutfs_read_block(sb, sbi->dirty_blkno + SCOUTFS_BLOOM_BLOCKS); - if (!bh) { - ret = -EIO; - goto out; - } - - /* - * Update first and last keys as we go. It's ok if future deletions - * make this range larger than the actual keys. That'll almost - * never happen and it'll get fixed up in merging. - */ - iblk = (void *)bh->b_data; - if (scoutfs_key_cmp(key, &iblk->first) < 0) - iblk->first = *key; - if (scoutfs_key_cmp(key, &iblk->last) > 0) - iblk->last = *key; - brelse(bh); - - /* XXX delete skip on failure? */ - - /* set the bloom bits last because we can't unset them */ - scoutfs_calc_bloom_bits(&bits, key, sbi->super.bloom_salts); - ret = scoutfs_set_bloom_bits(sb, sbi->dirty_blkno, &bits); -out: - WARN_ON_ONCE(ret); /* XXX error paths are not robust */ - mutex_unlock(&sbi->dirty_mutex); - return ret; -} - -/* - * Ensure that there is a dirty item with the given key in the current - * dirty segment. - * - * The caller locks access to the item and prevents sync and made sure - * that there's enough free space in the segment for their dirty inodes. - * - * This is better than getting -EEXIST from create_item because that - * will leave the allocated item and val dangling in the block when it - * returns the error. - */ -int scoutfs_dirty_item(struct super_block *sb, struct scoutfs_key *key, - unsigned bytes, struct scoutfs_item_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_item *item; - struct buffer_head *bh; - bool create = false; - int ret; - - mutex_lock(&sbi->dirty_mutex); - - if (sbi->dirty_blkno) { - ret = scoutfs_skip_lookup(sb, sbi->dirty_blkno, key, &bh, - &item); - if (ret == -ENOENT) - create = true; - else if (!ret) { - ret = populate_ref(sb, sbi->dirty_blkno, bh, item, - ref); - brelse(bh); - } - } else { - create = true; - } - mutex_unlock(&sbi->dirty_mutex); - - if (create) - ret = scoutfs_create_item(sb, key, bytes, ref); - - return ret; -} - -/* - * This is a really cheesy temporary delete method. It only works on items - * that are stored in dirty blocks. The caller is responsible for dropping - * the ref. XXX be less bad. - */ -int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 blkno; - int ret; - - mutex_lock(&sbi->dirty_mutex); - - blkno = round_down(ref->item_bh->b_blocknr, SCOUTFS_BLOCKS_PER_CHUNK); - if (WARN_ON_ONCE(blkno != sbi->dirty_blkno)) { - ret = -EINVAL; - } else { - ret = scoutfs_skip_delete(sb, blkno, ref->key); - WARN_ON_ONCE(ret); - } - - mutex_unlock(&sbi->dirty_mutex); - - return ret; -} - -/* - * Return a reference to the next item in the inclusive search range. - * The caller should have access to the search key range. - * - * We walk the manifest to find all the log segments that could contain - * the start of the range. We hold cursors on the blocks in the - * segments. Each next item iteration comes from finding the least of - * the next item at all these cursors. - * - * If we exhaust a segment at a given level we may need to search the - * next segment in that level to find the next item. The manifest may - * have changed under us while we walked our old set of segments. So we - * restart the entire search to get another consistent collection of - * segments to search. - * - * We put the segment references and iteration cursors in a list in the - * caller so that they can find many next items by advancing the cursors - * without having to walk the manifest and perform initial skip list - * searches in each segment. - * - * The caller is responsible for putting the item ref if we return - * success. -ENOENT is returned if there are no more items in the - * search range. - */ -int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, struct list_head *iter_list, - struct scoutfs_item_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_manifest_entry *ments = NULL; - struct scoutfs_key key = *first; - struct scoutfs_key least_hole; - struct scoutfs_item_iter *least; - struct scoutfs_item_iter *iter; - struct scoutfs_item_iter *pos; - bool locked; - int ret; - int nr; - int i; - -restart: - if (list_empty(iter_list)) { - /* find all the segments that may contain the key */ - ret = scoutfs_manifest_find_key(sb, &key, &ments); - if (ret == 0) - ret = -ENOENT; - if (ret < 0) - goto out; - nr = ret; - - for (i = 0; i < nr; i++) { - iter = kzalloc(sizeof(struct scoutfs_item_iter), - GFP_NOFS); - if (!iter) { - ret = -ENOMEM; - goto out; - } - - iter->blkno = le64_to_cpu(ments[i].blkno); - iter->after_seg = ments[i].last; - scoutfs_inc_key(&iter->after_seg); - list_add_tail(&iter->list, iter_list); - } - - kfree(ments); - ments = NULL; - } - - memset(&least_hole, ~0, sizeof(least_hole)); - least = NULL; - list_for_each_entry_safe(iter, pos, iter_list, list) { - - locked = try_lock_dirty_mutex(sb, iter->blkno); - - /* search towards the key if we haven't yet */ - if (!iter->item) { - ret = scoutfs_skip_search(sb, iter->blkno, &key, - &iter->bh, &iter->item); - } else { - ret = 0; - } - - /* then iterate until we find or pass the key */ - while (!ret && scoutfs_key_cmp(&iter->item->key, &key) < 0) { - ret = scoutfs_skip_next(sb, iter->blkno, - &iter->bh, &iter->item); - } - - if (locked) - mutex_unlock(&sbi->dirty_mutex); - - /* we're done with this segment if it has an item after last */ - if (!ret && scoutfs_key_cmp(&iter->item->key, last) > 0) { - list_del_init(&iter->list); - brelse(iter->bh); - kfree(iter); - continue; - } - - /* - * If we run out of keys in the segment then we don't know - * the state of keys after this segment in this level. If - * the hole after the segment is still inside the search - * range then we might need to search it for the next - * item if the least item of the remaining blocks is - * greater than the hole. - */ - if (ret == -ENOENT) { - if (scoutfs_key_cmp(&iter->after_seg, last) <= 0 && - scoutfs_key_cmp(&iter->after_seg, &least_hole) < 0) - least_hole = iter->after_seg; - - list_del_init(&iter->list); - brelse(iter->bh); - kfree(iter); - continue; - } - - /* remember the most recent smallest key */ - if (!least || - scoutfs_key_cmp(&iter->item->key, &least->item->key) < 0) - least = iter; - } - - /* if we had a gap before the least then we need a new search */ - if (least && scoutfs_key_cmp(&least_hole, &least->item->key) < 0) { - scoutfs_put_iter_list(iter_list); - key = least_hole; - goto restart; - } - - if (least) - ret = populate_ref(sb, least->blkno, least->bh, least->item, - ref); - else - ret = -ENOENT; -out: - kfree(ments); - if (ret) - scoutfs_put_iter_list(iter_list); - return ret; -} diff --git a/kmod/src/segment.h b/kmod/src/segment.h deleted file mode 100644 index de5b0dd5..00000000 --- a/kmod/src/segment.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _SCOUTFS_SEGMENT_H_ -#define _SCOUTFS_SEGMENT_H_ - -struct scoutfs_item_ref { - /* usable by callers */ - struct scoutfs_key *key; - unsigned int val_len; - void *val; - - /* private buffer head refs */ - struct buffer_head *item_bh; - struct buffer_head *val_bh; -}; - -#define DECLARE_SCOUTFS_ITEM_REF(name) \ - struct scoutfs_item_ref name = {NULL ,} - -void scoutfs_put_ref(struct scoutfs_item_ref *ref); -void scoutfs_put_iter_list(struct list_head *list); - -int scoutfs_read_item(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_item_ref *ref); -int scoutfs_create_item(struct super_block *sb, struct scoutfs_key *key, - unsigned bytes, struct scoutfs_item_ref *ref); -int scoutfs_dirty_item(struct super_block *sb, struct scoutfs_key *key, - unsigned bytes, struct scoutfs_item_ref *ref); -int scoutfs_delete_item(struct super_block *sb, struct scoutfs_item_ref *ref); -int scoutfs_next_item(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, struct list_head *iter_list, - struct scoutfs_item_ref *ref); - -int scoutfs_sync_fs(struct super_block *sb, int wait); - - -#endif diff --git a/kmod/src/skip.c b/kmod/src/skip.c deleted file mode 100644 index d320e2a7..00000000 --- a/kmod/src/skip.c +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -#include -#include -#include - -#include "format.h" -#include "key.h" -#include "block.h" -#include "skip.h" -#include "counters.h" - -/* - * The items in a log segment block are sorted by their keys in a skip - * list. The skip list was chosen because it is so easy to implement - * and could, maybe some day, offer solid concurrent updates and reads. - * It also adds surprisingly little per-item overhead because half of - * the items only have one link. - * - * The list is rooted in the item block which follows the last bloom - * block in the segment. The links in the skip list elements are byte - * offsets of the start of items relative to the start of the log - * segment. - * - * We chose a limit on the height of 16 links. That gives around 64k - * items without going too crazy. That's around the higher end of the - * number of items we expect in log segments. - * - * This isn't quite a generic implementation. It knows that the items - * are rooted in the item block at a given offset in the log segment. - * It knows that the pointers are items and where the skip links are in - * its struct. It knows to compare the items by their key. - * - * The caller is completely responsible for serialization. - * - * The buffer_head reads here won't be as expensive as they might seem. - * The caller holds the blocks pinned so the worst case are block device - * page radix rcu lookups. Repeated reads of the recent blocks will hit - * the per-cpu lru bh reference caches. - */ - -struct skip_path { - struct buffer_head *root_bh; - - /* - * Pointers to the buffer heads which contain the blocks which are - * referenced by the next pointers in the path. - */ - struct buffer_head *bh[SCOUTFS_SKIP_HEIGHT]; - - /* - * Store the location of the index that references the item that - * we found. Insertion will modify the referenced index to add - * an entry before the item and deletion will modify the referenced - * index to remove the item. - */ - __le32 *next[SCOUTFS_SKIP_HEIGHT]; -}; - -#define DECLARE_SKIP_PATH(name) \ - struct skip_path name = {NULL, } - -/* - * Not all byte offsets are possible locations of items. Items have to - * be after the bloom blocks and item block header, can't be in - * the block headers for the rest of the blocks, and can't be a partial - * struct at the end of a block. - * - * This is just a rough check. It doesn't catch items offsets that overlap - * with other items or values. - */ -static int invalid_item_off(u32 off) -{ - if (off < ((SCOUTFS_BLOCK_SIZE * SCOUTFS_BLOOM_BLOCKS) + - sizeof(struct scoutfs_item_block)) || - (off & SCOUTFS_BLOCK_MASK) < sizeof(struct scoutfs_block_header) || - (off & SCOUTFS_BLOCK_MASK) > - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_item))) { - trace_printk("invalid offset %u\n", off); - return 1; - } - - return 0; -} - -/* - * Set the caller's item to the item in the segment at the given byte - * offset and set their bh to the block that contains it. - */ -static int skip_read_item(struct super_block *sb, u64 blkno, __le32 off, - struct buffer_head **bh, struct scoutfs_item **item) -{ - if (WARN_ON_ONCE(invalid_item_off(le32_to_cpu(off)))) - return -EINVAL; - - *bh = scoutfs_read_block_off(sb, blkno, le32_to_cpu(off)); - if (!(*bh)) { - *bh = NULL; - *item = NULL; - return -EIO; - } - - *item = (void *)(*bh)->b_data + (le32_to_cpu(off) & SCOUTFS_BLOCK_MASK); - return 0; -} - -/* - * Find the next item in the skiplist with a key greater than or equal - * to the given key. Set the path pointers to the hops before this item - * so that we can modify those pointers to insert an item before it in - * the list or delete it. - * - * The caller is responsible for initializing the path and cleaning it up. - */ -static int skip_search(struct super_block *sb, u64 blkno, - struct skip_path *path, struct scoutfs_key *key, - int *cmp) -{ - struct scoutfs_item_block *iblk; - struct scoutfs_item *item; - struct buffer_head *bh; - __le32 *next; - int ret = 0; - int i; - - /* fake lesser comparison for insertion into an empty list */ - *cmp = -1; - - bh = scoutfs_read_block(sb, blkno + SCOUTFS_BLOOM_BLOCKS); - if (!bh) - return -EIO; - - /* XXX verify */ - iblk = (void *)bh->b_data; - next = iblk->skip_root.next; - path->root_bh = bh; - - for (i = SCOUTFS_SKIP_HEIGHT - 1; i >= 0; i--) { - while (next[i]) { - ret = skip_read_item(sb, blkno, next[i], &bh, &item); - if (ret) - goto out; - - *cmp = scoutfs_key_cmp(key, &item->key); - if (*cmp <= 0) { - brelse(bh); - break; - } - - next = item->skip_next; - if (path->bh[i]) - brelse(path->bh[i]); - path->bh[i] = bh; - } - - path->next[i] = &next[i]; - } -out: - return ret; -} - -static void skip_release_path(struct skip_path *path) -{ - int i; - - if (path->root_bh) - brelse(path->root_bh); - - for (i = 0; i < SCOUTFS_SKIP_HEIGHT; i++) { - if (path->bh[i]) { - brelse(path->bh[i]); - path->bh[i] = NULL; - } - } -} - -/* - * We want heights with a distribution of 1 / (2^h). Half the items - * have a height of 1, a quarter have 2, an eighth have 3, etc. - * - * Finding the first low set bit in a random number achieves this - * nicely. ffs() even counts the bits from 1 so it matches our height. - * - * But ffs() returns 0 if no bits are set. We prevent a 0 height and - * limit the max height returned by oring in our max height. - */ -u8 scoutfs_skip_random_height(void) -{ - return ffs(get_random_int() | (1 << (SCOUTFS_SKIP_HEIGHT - 1))); -} - -/* - * Insert a new item in the item block's skip list. The caller provides - * an initialized item, particularly it's skip height and key, and - * the byte offset in the log segment of the item struct. - */ -int scoutfs_skip_insert(struct super_block *sb, u64 blkno, - struct scoutfs_item *item, u32 off) -{ - DECLARE_SKIP_PATH(path); - int cmp; - int ret; - int i; - - if (WARN_ON_ONCE(invalid_item_off(off)) || - WARN_ON_ONCE(item->skip_height > SCOUTFS_SKIP_HEIGHT)) - return -EINVAL; - - scoutfs_inc_counter(sb, skip_insert); - - ret = skip_search(sb, blkno, &path, &item->key, &cmp); - if (ret == 0) { - if (cmp == 0) { - ret = -EEXIST; - } else { - for (i = 0; i < item->skip_height; i++) { - item->skip_next[i] = *path.next[i]; - *path.next[i] = cpu_to_le32(off); - } - } - } - - skip_release_path(&path); - return ret; -} - -static int skip_lookup(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, struct buffer_head **bh, - struct scoutfs_item **item, bool exact) -{ - DECLARE_SKIP_PATH(path); - int cmp; - int ret; - - ret = skip_search(sb, blkno, &path, key, &cmp); - if (ret == 0) { - if ((exact && cmp) || *path.next[0] == 0) { - ret = -ENOENT; - } else { - ret = skip_read_item(sb, blkno, *path.next[0], - bh, item); - } - } - - skip_release_path(&path); - return ret; -} - -/* - * Find the item at the given key in the skip list. - */ -int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, struct buffer_head **bh, - struct scoutfs_item **item) -{ - scoutfs_inc_counter(sb, skip_lookup); - return skip_lookup(sb, blkno, key, bh, item, true); -} - -/* - * Find the next item after the given key in the skip list. - */ -int scoutfs_skip_search(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, struct buffer_head **bh, - struct scoutfs_item **item) -{ - scoutfs_inc_counter(sb, skip_search); - return skip_lookup(sb, blkno, key, bh, item, false); -} - -int scoutfs_skip_delete(struct super_block *sb, u64 blkno, - struct scoutfs_key *key) -{ - struct scoutfs_item *item; - DECLARE_SKIP_PATH(path); - struct buffer_head *bh; - int cmp; - int ret; - int i; - - scoutfs_inc_counter(sb, skip_delete); - - ret = skip_search(sb, blkno, &path, key, &cmp); - if (ret == 0) { - if (*path.next[0] && cmp) { - ret = -ENOENT; - } else { - ret = skip_read_item(sb, blkno, *path.next[0], - &bh, &item); - if (!ret) { - for (i = 0; i < item->skip_height; i++) - *path.next[i] = item->skip_next[i]; - brelse(bh); - } - } - } - - skip_release_path(&path); - return ret; -} - -/* - * The caller has found a valid item with search or lookup. We can use - * the lowest level links to advance through the rest of the items. The - * caller has made sure that this is safe. - */ -int scoutfs_skip_next(struct super_block *sb, u64 blkno, - struct buffer_head **bh, struct scoutfs_item **item) -{ - __le32 next; - - if (!(*bh)) - return -ENOENT; - - scoutfs_inc_counter(sb, skip_next); - - next = (*item)->skip_next[0]; - brelse(*bh); - - if (!next) { - *bh = NULL; - *item = NULL; - return -ENOENT; - } - - return skip_read_item(sb, blkno, next, bh, item); -} diff --git a/kmod/src/skip.h b/kmod/src/skip.h deleted file mode 100644 index 979719cc..00000000 --- a/kmod/src/skip.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _SCOUTFS_SKIP_H_ -#define _SCOUTFS_SKIP_H_ - -u8 scoutfs_skip_random_height(void); -int scoutfs_skip_insert(struct super_block *sb, u64 blkno, - struct scoutfs_item *item, u32 off); -int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, struct buffer_head **bh, - struct scoutfs_item **item); -int scoutfs_skip_search(struct super_block *sb, u64 blkno, - struct scoutfs_key *key, struct buffer_head **bh, - struct scoutfs_item **item); -int scoutfs_skip_delete(struct super_block *sb, u64 blkno, - struct scoutfs_key *key); -int scoutfs_skip_next(struct super_block *sb, u64 blkno, - struct buffer_head **bh, struct scoutfs_item **item); - -#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 2ace1f05..0e8a7d60 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -24,95 +24,33 @@ #include "dir.h" #include "msg.h" #include "block.h" -#include "manifest.h" -#include "ring.h" -#include "segment.h" #include "counters.h" #include "scoutfs_trace.h" -/* only for giant rbtree hack */ -#include -#include "ival.h" - static struct kset *scoutfs_kset; static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, - .sync_fs = scoutfs_sync_fs, }; -/* - * The caller advances the block number and sequence number in the super - * every time it wants to dirty it and eventually write it to reference - * dirty data that's been written. - */ -int scoutfs_advance_dirty_super(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - u64 blkno; - - blkno = le64_to_cpu(super->hdr.blkno) - SCOUTFS_SUPER_BLKNO; - if (++blkno == SCOUTFS_SUPER_NR) - blkno = 0; - super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO + blkno); - - le64_add_cpu(&super->hdr.seq, 1); - - trace_scoutfs_dirty_super(super); - - return 0; -} - -/* - * We've been modifying the super copy in the info as we made changes. - * Write the super to finalize. - */ -int scoutfs_write_dirty_super(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buffer_head *bh; - size_t sz; - int ret; - - bh = scoutfs_new_block(sb, le64_to_cpu(super->hdr.blkno)); - if (!bh) - return -ENOMEM; - - sz = sizeof(struct scoutfs_super_block); - memcpy(bh->b_data, super, sz); - memset(bh->b_data + sz, 0, SCOUTFS_BLOCK_SIZE - sz); - - scoutfs_calc_hdr_crc(bh); - mark_buffer_dirty(bh); - trace_scoutfs_write_super(super); - ret = sync_dirty_buffer(bh); - brelse(bh); - - return ret; -} - static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; - struct buffer_head *bh = NULL; - unsigned long bytes; + struct scoutfs_block *bl = NULL; int found = -1; int i; for (i = 0; i < SCOUTFS_SUPER_NR; i++) { - if (bh) - brelse(bh); - bh = scoutfs_read_block(sb, SCOUTFS_SUPER_BLKNO + i); - if (!bh) { + scoutfs_put_block(bl); + bl = scoutfs_read_block(sb, SCOUTFS_SUPER_BLKNO + i); + if (IS_ERR(bl)) { scoutfs_warn(sb, "couldn't read super block %u", i); continue; } - super = (void *)bh->b_data; + super = bl->data; if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { scoutfs_warn(sb, "super block %u has invalid id %llx", @@ -128,8 +66,7 @@ static int read_supers(struct super_block *sb) } } - if (bh) - brelse(bh); + scoutfs_put_block(bl); if (found < 0) { scoutfs_err(sb, "unable to read valid super block"); @@ -145,17 +82,6 @@ static int read_supers(struct super_block *sb) atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); atomic64_set(&sbi->next_blkno, 2); - /* Initialize all the sb info fields which depends on the supers. */ - - bytes = DIV_ROUND_UP(le64_to_cpu(sbi->super.total_chunks), 64) * - sizeof(u64); - sbi->chunk_alloc_bits = vmalloc(bytes); - if (!sbi->chunk_alloc_bits) - return -ENOMEM; - - /* the alloc bits default to all free then ring entries update them */ - memset(sbi->chunk_alloc_bits, 0xff, bytes); - return 0; } @@ -174,16 +100,9 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; - spin_lock_init(&sbi->item_lock); - sbi->item_root = RB_ROOT; - sbi->dirty_item_root = RB_ROOT; - spin_lock_init(&sbi->chunk_alloc_lock); - mutex_init(&sbi->dirty_mutex); - - if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { - printk(KERN_ERR "couldn't set blocksize\n"); - return -EINVAL; - } + spin_lock_init(&sbi->block_lock); + INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); + init_waitqueue_head(&sbi->block_wq); /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); @@ -191,9 +110,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -ENOMEM; ret = scoutfs_setup_counters(sb) ?: - read_supers(sb) ?: - scoutfs_setup_manifest(sb) ?: - scoutfs_replay_ring(sb); + read_supers(sb); if (ret) return ret; @@ -205,8 +122,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; - scoutfs_advance_dirty_super(sb); - return 0; } @@ -222,9 +137,6 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { - /* kill block super should have synced */ - WARN_ON_ONCE(sbi->dirty_blkno); - scoutfs_destroy_manifest(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); @@ -253,8 +165,6 @@ static int __init scoutfs_module_init(void) { int ret; - giant_rbtree_hack_build_bugs(); - scoutfs_init_counters(); scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); diff --git a/kmod/src/super.h b/kmod/src/super.h index 0857faf0..1d5170f4 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -1,38 +1,23 @@ #ifndef _SCOUTFS_SUPER_H_ #define _SCOUTFS_SUPER_H_ +#include #include + #include "format.h" -struct scoutfs_manifest; struct scoutfs_counters; struct scoutfs_sb_info { struct scoutfs_super_block super; + spinlock_t block_lock; + struct radix_tree_root block_radix; + wait_queue_head_t block_wq; + atomic64_t next_ino; atomic64_t next_blkno; - spinlock_t item_lock; - struct rb_root item_root; - struct rb_root dirty_item_root; - - struct scoutfs_manifest *mani; - - spinlock_t chunk_alloc_lock; - __le64 *chunk_alloc_bits; - - /* pinned dirty ring block during commit */ - struct buffer_head *dirty_ring_bh; - struct scoutfs_ring_entry *dirty_ring_ent; - unsigned int dirty_ring_ent_avail; - - /* pinned log segment during fs modifications */ - struct mutex dirty_mutex; - u64 dirty_blkno; - int dirty_item_off; - int dirty_val_off; - /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; @@ -44,7 +29,4 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } -int scoutfs_advance_dirty_super(struct super_block *sb); -int scoutfs_write_dirty_super(struct super_block *sb); - #endif From 5651d48c18028bab9a76dcd310850e698f1dd2d9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 Apr 2016 19:33:09 -0700 Subject: [PATCH 041/920] scoutfs: add core btree functionality Previously we had stubbed out the btree item API with static inlines. Those are replaced with real functions in a reasonably functional btree implementation. The btree implementation itself is pretty straight forward. Operations are performed top-down and we dirty, lock, and split/merge blocks as we go. Callers are given a cursor to give them full access to the item. Items in the btree blocks are stored in a treap. There are a lot of comments in the code to help make things clear. We add the notion of block references and some block functions for reading and dirtying blocks by reference. This passes tests up to the point where unmount tries to write out data and the world catches fire. That's far enough to commit what we have and iterate from there. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/block.c | 139 ++++++- kmod/src/block.h | 7 + kmod/src/btree.c | 950 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/btree.h | 58 +-- kmod/src/dir.c | 35 +- kmod/src/format.h | 88 ++++- kmod/src/inode.c | 24 +- kmod/src/super.c | 1 + kmod/src/super.h | 3 + kmod/src/treap.c | 364 ++++++++++++++++++ kmod/src/treap.h | 38 ++ 12 files changed, 1603 insertions(+), 108 deletions(-) create mode 100644 kmod/src/btree.c create mode 100644 kmod/src/treap.c create mode 100644 kmod/src/treap.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 68870aa4..3ceb5af4 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o counters.o crc.o dir.o filerw.o inode.o msg.o \ - scoutfs_trace.o super.o +scoutfs-y += block.o btree.o counters.o crc.o dir.o filerw.o inode.o msg.o \ + scoutfs_trace.o super.o treap.o diff --git a/kmod/src/block.c b/kmod/src/block.c index 51815f9d..fc232e38 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -64,6 +64,7 @@ static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) void scoutfs_put_block(struct scoutfs_block *bl) { if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { + trace_printk("freeing bl %p\n", bl); __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); kfree(bl); scoutfs_inc_counter(bl->sb, block_mem_free); @@ -153,10 +154,14 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) spin_lock(&sbi->block_lock); bl = radix_tree_lookup(&sbi->block_radix, blkno); - if (bl && test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - scoutfs_put_block(bl); - bl = NULL; + if (bl) { + if (test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + scoutfs_put_block(bl); + bl = NULL; + } else { + atomic_inc(&bl->refcount); + } } spin_unlock(&sbi->block_lock); @@ -180,6 +185,7 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) if (found) { scoutfs_put_block(bl); bl = found; + atomic_inc(&bl->refcount); } else { radix_tree_insert(&sbi->block_radix, blkno, bl); atomic_inc(&bl->refcount); @@ -212,9 +218,116 @@ out: return bl; } +/* + * Return the block pointed to by the caller's reference. + * + * If the reference sequence numbers don't match then we could be racing + * with another writer. We back off and try again. If it happens too + * many times the caller assumes that we've hit persistent corruption + * and returns an error. + * + * XXX how does this race with + * - reads that span transactions? + * - writers creating a new dirty block? + */ +struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block_header *hdr; + struct scoutfs_block *bl; + struct scoutfs_block *found; + + bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR(bl)) { + hdr = bl->data; + + if (WARN_ON_ONCE(hdr->seq != ref->seq)) { + /* XXX hack, make this a function */ + spin_lock(&sbi->block_lock); + found = radix_tree_lookup(&sbi->block_radix, + bl->blkno); + if (found == bl) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + scoutfs_put_block(bl); + } + spin_unlock(&sbi->block_lock); + + scoutfs_put_block(bl); + bl = ERR_PTR(-EAGAIN); + } + } + + return bl; +} + +/* + * Give the caller a dirty block that they can safely modify. If the + * reference refers to a stable clean block then we allocate a new block + * and update the reference. + * + * Blocks are dirtied and modified within a transaction that has a given + * sequence number which we use to determine if the block is currently + * dirty or not. + * + * For now we're using the dirty super block in the sb_info to track + * the dirty seq. That'll be different when we have multiple btrees. + * + * Callers are working in structures that have sufficient locking to + * protect references to the source block. If we've come to dirty it then + * there won't be concurrent users and we can just move it in the cache. + */ +struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block_header *hdr; + struct scoutfs_block *found; + struct scoutfs_block *bl; + u64 blkno; + int ret; + + bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) + return bl; + + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + scoutfs_put_block(bl); + return ERR_PTR(ret); + } + + /* XXX cheesy */ + blkno = atomic64_inc_return(&sbi->next_blkno); + hdr = bl->data; + + spin_lock(&sbi->block_lock); + + /* XXX don't really like this */ + found = radix_tree_lookup(&sbi->block_radix, bl->blkno); + if (found == bl) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + atomic_dec(&bl->refcount); + } + + bl->blkno = blkno; + hdr->blkno = cpu_to_le64(blkno); + hdr->seq = sbi->super.hdr.seq; + radix_tree_insert(&sbi->block_radix, blkno, bl); + atomic_inc(&bl->refcount); + + spin_unlock(&sbi->block_lock); + radix_tree_preload_end(); + + ref->blkno = hdr->blkno; + ref->seq = hdr->seq; + + return bl; +} + /* * Return a newly allocated metadata block with an updated block header - * to match the current dirty super block. Callers are responsible for + * to match the current dirty seq. Callers are responsible for * serializing access to the block and for zeroing unwritten block * contents. */ @@ -242,6 +355,7 @@ struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) hdr = bl->data; *hdr = sbi->super.hdr; hdr->blkno = cpu_to_le64(blkno); + hdr->seq = sbi->super.hdr.seq; spin_lock(&sbi->block_lock); found = radix_tree_lookup(&sbi->block_radix, blkno); @@ -265,6 +379,21 @@ out: return bl; } +/* + * Allocate a new dirty writable block. The caller must be in a + * transaction so that we can assign the dirty seq. + */ +struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 blkno; + + /* XXX cheesy */ + blkno = atomic64_inc_return(&sbi->next_blkno); + + return scoutfs_new_block(sb, blkno); +} + void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) { struct scoutfs_block_header *hdr = bl->data; diff --git a/kmod/src/block.h b/kmod/src/block.h index c04586a8..dac2b4e8 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -23,6 +23,13 @@ struct scoutfs_block { struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno); struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb); + +struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, + struct scoutfs_block_ref *ref); +struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref); + void scoutfs_put_block(struct scoutfs_block *bl); void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); diff --git a/kmod/src/btree.c b/kmod/src/btree.c new file mode 100644 index 00000000..cf3e700e --- /dev/null +++ b/kmod/src/btree.c @@ -0,0 +1,950 @@ +/* + * Copyright (C) 2016 Zach Brown. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "block.h" +#include "key.h" +#include "treap.h" +#include "btree.h" + +/* + * scoutfs stores file system metadata in btrees whose items have fixed + * sized keys and variable length values. + * + * Items are stored as a small header with the key followed by the + * value. New items are appended to the end of the block. Free space + * is not indexed. Deleted items can be reclaimed by walking all the + * items from the front of the block and moving later live items onto + * earlier deleted items. + * + * The items are kept in a treap sorted by their keys. Using a dynamic + * structure keeps the modification costs low. Modifying persistent + * structures avoids translation to and from run-time structures around + * read and write. The treap was chosen because it's very simple to + * implement and has some cool merging and splitting functions that we + * could make use of. The treap has parent pointers so that we can + * perform operations relative to a node without having to keep a record + * of the path down the tree. + * + * Parent blocks in the btree have the same format as leaf blocks. + * There's one key for every child reference instead of having separator + * keys between child references. The key in a child reference contains + * the largest key that may be found in the child subtree. The right + * spine of the tree has maximal keys so that they don't have to be + * updated if we insert an item with a key greater than everything in + * the tree. + * + * Operations are performed in one pass down the tree. This lets us + * cascade locks from the root down to the leaves and avoids having to + * maintain a record of the path down the tree. Splits and merges are + * performed as we descend. + * + * XXX + * - actually free blknos + * - do we want a level in the btree header? seems like we would? + * - validate structures on read? + */ + +/* size of the item with a value of the given length */ +static inline unsigned int val_bytes(unsigned int val_len) +{ + return sizeof(struct scoutfs_btree_item) + val_len; +} + +static inline unsigned int item_bytes(struct scoutfs_btree_item *item) +{ + return val_bytes(le16_to_cpu(item->val_len)); +} + +static inline unsigned int used_total(struct scoutfs_btree_block *bt) +{ + return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - + le16_to_cpu(bt->total_free); +} + +static int cmp_tnode_items(struct scoutfs_treap_node *A, + struct scoutfs_treap_node *B) +{ + struct scoutfs_btree_item *a; + struct scoutfs_btree_item *b; + + a = container_of(A, struct scoutfs_btree_item, tnode); + b = container_of(B, struct scoutfs_btree_item, tnode); + + return scoutfs_key_cmp(&a->key, &b->key); +} + +/* A bunch of wrappers for navigating items through treap nodes. */ + +#define BT_TREAP_KEY_WRAPPER(which) \ +static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt, \ + struct scoutfs_key *key) \ +{ \ + struct scoutfs_btree_item dummy = { .key = *key }; \ + struct scoutfs_treap_node *node; \ + \ + node = scoutfs_treap_##which(&bt->treap, cmp_tnode_items, \ + &dummy.tnode); \ + if (!node) \ + return NULL; \ + \ + return container_of(node, struct scoutfs_btree_item, tnode); \ +} + +BT_TREAP_KEY_WRAPPER(lookup) +/* BT_TREAP_KEY_WRAPPER(before) */ +BT_TREAP_KEY_WRAPPER(after) + +#define BT_TREAP_ROOT_WRAPPER(which) \ +static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt) \ +{ \ + struct scoutfs_treap_node *node; \ + \ + node = scoutfs_treap_##which(&bt->treap); \ + if (!node) \ + return NULL; \ + \ + return container_of(node, struct scoutfs_btree_item, tnode); \ +} + +BT_TREAP_ROOT_WRAPPER(first) +BT_TREAP_ROOT_WRAPPER(last) + +#define BT_TREAP_NODE_WRAPPER(which) \ +static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt, \ + struct scoutfs_btree_item *item)\ +{ \ + struct scoutfs_treap_node *node; \ + \ + node = scoutfs_treap_##which(&bt->treap, &item->tnode); \ + if (!node) \ + return NULL; \ + \ + return container_of(node, struct scoutfs_btree_item, tnode); \ +} + +BT_TREAP_NODE_WRAPPER(next) +BT_TREAP_NODE_WRAPPER(prev) + +static inline struct scoutfs_key *least_key(struct scoutfs_btree_block *bt) +{ + return &bt_first(bt)->key; +} + +static inline struct scoutfs_key *greatest_key(struct scoutfs_btree_block *bt) +{ + return &bt_last(bt)->key; +} + +/* + * Allocate and insert a new item into the block. + * + * The caller has made sure that there's room for everything. + * + * The caller is responsible for initializing the value. + */ +static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, + unsigned int val_len) +{ + unsigned int bytes = val_bytes(val_len); + struct scoutfs_btree_item *item; + + item = (void *)((char *)bt + SCOUTFS_BLOCK_SIZE - + le16_to_cpu(bt->tail_free)); + le16_add_cpu(&bt->tail_free, -bytes); + le16_add_cpu(&bt->total_free, -bytes); + le16_add_cpu(&bt->nr_items, 1); + + item->key = *key; + item->val_len = cpu_to_le16(val_len); + + scoutfs_treap_insert(&bt->treap, cmp_tnode_items, &item->tnode); + + return item; +} + +#define MAGIC_DELETED_PARENT cpu_to_le16(1) + +/* + * Delete an item from a btree block. We set the deleted item's parent + * treap offset to a magic value for compaction. + */ +static void delete_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + scoutfs_treap_delete(&bt->treap, &item->tnode); + item->tnode.parent = MAGIC_DELETED_PARENT; + + le16_add_cpu(&bt->total_free, item_bytes(item)); + le16_add_cpu(&bt->nr_items, -1); +} + +/* + * Move items from a source block to a destination block. The caller + * tells us if we're moving from the tail of the source block right to + * the head of the destination block, or vice versa. We stop moving + * once we've moved enough bytes of items. + * + * XXX This could use fancy treap splitting and merging. We don't need + * to go there yet. + */ +static void move_items(struct scoutfs_btree_block *dst, + struct scoutfs_btree_block *src, bool move_right, + int to_move) +{ + struct scoutfs_btree_item *from; + struct scoutfs_btree_item *to; + unsigned int val_len; + + if (move_right) + from = bt_last(src); + else + from = bt_first(src); + + while (from && to_move > 0) { + val_len = le16_to_cpu(from->val_len); + + to = create_item(dst, &from->key, val_len); + memcpy(to->val, from->val, val_len); + + delete_item(src, from); + + if (move_right) + from = bt_prev(src, from); + else + from = bt_next(src, from); + to_move -= item_bytes(to); + } +} + +/* + * As items are deleted they create fragmented free space. Even if we + * indexed free space in the block it could still get sufficiently + * fragmented to force a split on insertion even though the two + * resulting blocks would have less than the minimum space consumed by + * items. + * + * We don't bother implementing free space indexing and addressing that + * corner case. Instead we track the number of total free bytes in the + * block. If free space needed is available in the block but is not + * available at the end of the block then we reclaim the fragmented free + * space by compacting the items. + * + * We move the free space to the tail of the block by walk forward + * through the items in allocated order moving live items back in to + * free space. + * + * Compaction is only attempted during descent as we find a block that + * needs more or less free space. The caller has the parent locked for + * writing and there are no references to the items at this point so + * it's safe to scramble the block contents. + */ +static void compact_items(struct scoutfs_btree_block *bt) +{ + struct scoutfs_btree_item *from = (void *)(bt + 1); + struct scoutfs_btree_item *to = from; + unsigned int bytes; + unsigned int i; + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + bytes = item_bytes(from); + + if (from->tnode.parent != MAGIC_DELETED_PARENT) { + if (from != to) { + memmove(to, from, bytes); + scoutfs_treap_move(&bt->treap, + &from->tnode, + &to->tnode); + } + to = (void *)to + bytes; + } else { + i--; + } + + from = (void *)from + bytes; + } + + bytes = SCOUTFS_BLOCK_SIZE - ((char *)to - (char *)bt); + bt->tail_free = cpu_to_le16(bytes); +} + +/* + * Allocate and initialize a new tree block. The caller adds references + * to it. + */ +static struct scoutfs_block *alloc_tree_block(struct super_block *sb) +{ + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + + bl = scoutfs_alloc_block(sb); + if (!IS_ERR(bl)) { + bt = bl->data; + + bt->total_free = cpu_to_le16(SCOUTFS_BLOCK_SIZE - + sizeof(struct scoutfs_btree_block)); + bt->tail_free = bt->total_free; + bt->nr_items = 0; + } + + return bl; +} + +/* + * Allocate a new tree block and point the root at it. The caller + * is responsible for the items in the new root block. + */ +static struct scoutfs_block *grow_tree(struct super_block *sb, + struct scoutfs_btree_root *root) +{ + struct scoutfs_block_header *hdr; + struct scoutfs_block *bl; + + bl = alloc_tree_block(sb); + if (!IS_ERR(bl)) { + hdr = bl->data; + + root->height++; + root->ref.blkno = hdr->blkno; + root->ref.seq = hdr->seq; + } + + return bl; +} + +/* + * Create a new item in the parent which references the child. The caller + * specifies the key in the item that describes the items in the child. + */ +static void create_parent_item(struct scoutfs_btree_block *parent, + struct scoutfs_btree_block *child, + struct scoutfs_key *key) +{ + struct scoutfs_btree_item *item; + struct scoutfs_block_ref ref = { + .blkno = child->hdr.blkno, + .seq = child->hdr.seq, + }; + + item = create_item(parent, key, sizeof(ref)); + memcpy(&item->val, &ref, sizeof(ref)); +} + +/* + * See if we need to split this block while descending for insertion so + * that we have enough space to insert. + * + * Parent blocks need enough space for a new item and child ref if a + * child block splits. Leaf blocks need enough space to insert the new + * item with its value. + * + * We split to the left so that the greatest key in the existing block + * doesn't change so we don't have to update the key in its parent item. + * + * If the search key falls in the new split block then we return it + * to the caller to walk through. + * + * The locking in the case where we add the first parent is a little wonky. + * We're creating a parent block that the walk doesn't know about. It + * holds the tree mutex while we add the parent ref and then will lock + * the child that we return. It's skipping locking the new parent as it + * descends but that's fine. + */ +static struct scoutfs_block *try_split(struct super_block *sb, + struct scoutfs_btree_root *root, + int level, struct scoutfs_key *key, + unsigned int val_len, + struct scoutfs_btree_block *parent, + struct scoutfs_btree_item *par_item, + struct scoutfs_block *right_bl) +{ + struct scoutfs_btree_block *right = right_bl->data; + struct scoutfs_btree_block *left; + struct scoutfs_block *left_bl; + struct scoutfs_block *par_bl = NULL; + unsigned int bytes; + + if (level) + val_len = sizeof(struct scoutfs_block_ref); + bytes = val_bytes(val_len); + + if (le16_to_cpu(right->tail_free) >= bytes) + return right_bl; + + if (le16_to_cpu(right->total_free) >= bytes) { + compact_items(right); + return right_bl; + } + + if (!parent) { + par_bl = grow_tree(sb, root); + if (IS_ERR(par_bl)) { + scoutfs_put_block(right_bl); + return par_bl; + } + + parent = par_bl->data; + } + + left_bl = alloc_tree_block(sb); + if (IS_ERR(left_bl)) { + /* XXX free parent block? */ + scoutfs_put_block(par_bl); + scoutfs_put_block(right_bl); + return left_bl; + } + left = left_bl->data; + + /* only grow the tree once we have the split neighbour */ + if (par_bl) { + struct scoutfs_key ones; + memset(&ones, 0xff, sizeof(ones)); + create_parent_item(parent, right, &ones); + } + + move_items(left, right, false, used_total(right) / 2); + create_parent_item(parent, left, greatest_key(left)); + + if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { + scoutfs_put_block(right_bl); + right_bl = left_bl; + } else { + scoutfs_put_block(left_bl); + } + + scoutfs_put_block(par_bl); + + return right_bl; +} + +/* + * This is called during descent for deletion when we have a parent and + * might need to merge items from a sibling block if this block has too + * much free space. Eventually we'll be able to fit all of the + * sibling's items in our free space which lets us delete the sibling + * block. + * + * The error handling here is a little weird. We're returning an + * ERR_PTR buffer to match splitting so that the walk can handle errors + * from both easily. We have to unlock and release our buffer to return + * an error. + * + * The caller only has the parent locked. They'll lock whichever + * block we return. + * + * XXX this could more cleverly chose a merge candidate sibling + */ +static struct scoutfs_block *try_merge(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_btree_block *parent, + struct scoutfs_btree_item *par_item, + struct scoutfs_block *bl) +{ + struct scoutfs_btree_block *bt = bl->data; + struct scoutfs_btree_block *sib_bt; + struct scoutfs_block *sib_bl; + struct scoutfs_btree_item *sib_item; + int to_move; + bool move_right; + + if (le16_to_cpu(bt->total_free) <= SCOUTFS_BTREE_FREE_LIMIT) + return bl; + + /* move items right into our block if we have a left sibling */ + sib_item = bt_prev(parent, par_item); + if (sib_item) { + move_right = false; + } else { + sib_item = bt_next(parent, par_item); + move_right = true; + } + + sib_bl = scoutfs_dirty_ref(sb, (void *)sib_item->val); + if (IS_ERR(sib_bl)) { + /* XXX do we need to unlock this? don't think so */ + scoutfs_put_block(bl); + return sib_bl; + } + sib_bt = sib_bl->data; + + if (used_total(sib_bt) <= le16_to_cpu(bt->total_free)) + to_move = used_total(sib_bt); + else + to_move = le16_to_cpu(bt->total_free) - + SCOUTFS_BTREE_FREE_LIMIT; + + if (le16_to_cpu(bt->tail_free) < to_move) + compact_items(bt); + + move_items(bt, sib_bt, move_right, to_move); + + /* update our parent's ref if we changed our greatest key */ + if (!move_right) + par_item->key = *greatest_key(bt); + + /* delete an empty sib or update if we changed its greatest key */ + if (sib_bt->nr_items == 0) { + delete_item(parent, sib_item); + /* XXX free sib block */ + } else if (move_right) { + sib_item->key = *greatest_key(sib_bt); + } + + /* and finally shrink the tree if our parent is the root with 1 */ + if (le16_to_cpu(parent->nr_items) == 1) { + root->height--; + root->ref.blkno = bt->hdr.blkno; + root->ref.seq = bt->hdr.seq; + /* XXX free block */ + } + + return bl; +} + +enum { + WALK_INSERT = 1, + WALK_DELETE, + WALK_NEXT, + WALK_PREV, + WALK_DIRTY, +}; + +/* + * Usually we descend to a leaf that contains the key. But if we're + * searching for a next or previous item then we might hit a leaf block + * that contains the key but no items in the direction of the search. + * We might need to ascend and continue the search in the next block. + * + * The caller has just descended to a leaf and is asking us to discover + * this case. We set the parent item and return true to tell the caller + * to read the new parent item's referenced block. + * + * XXX I don't think this can see bts with 0 items? would need to verify? + */ +static bool next_leaf(struct scoutfs_btree_block *parent, + struct scoutfs_btree_item **par_item, + struct scoutfs_btree_block *bt, int op, int level, + struct scoutfs_key *key) +{ + struct scoutfs_btree_item *nei; + + if (level > 0 || !parent || le16_to_cpu(parent->nr_items) < 2) + return false; + + if (op == WALK_NEXT) { + nei = bt_next(parent, *par_item); + if (nei && (scoutfs_key_cmp(key, greatest_key(bt)) > 0)) { + *par_item = nei; + return true; + } + } + + if (op == WALK_PREV) { + nei = bt_prev(parent, *par_item); + if (nei && (scoutfs_key_cmp(key, least_key(bt)) < 0)) { + *par_item = nei; + return true; + } + } + + return false; +} + +/* + * As we descend we lock parent blocks (or the root), then lock the child, + * then unlock the parent. + */ +static void lock_block(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, + bool dirty) +{ + struct rw_semaphore *rwsem; + + if (bl == NULL) + rwsem = &sbi->btree_rwsem; + else + rwsem = &bl->rwsem; + + if (dirty) + down_write(rwsem); + else + down_read(rwsem); +} + +static void unlock_block(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, + bool dirty) +{ + struct rw_semaphore *rwsem; + + if (bl == NULL) + rwsem = &sbi->btree_rwsem; + else + rwsem = &bl->rwsem; + + if (dirty) + up_write(rwsem); + else + up_read(rwsem); +} + + +/* + * Return the leaf block that should contain the given key. The caller + * is responsible for searching the leaf block and performing their + * operation. The block is returned locked for either reading or writing + * depending on the operation. + */ +static struct scoutfs_block *btree_walk(struct super_block *sb, + struct scoutfs_key *key, + unsigned int val_len, int op) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_btree_block *parent = NULL; + struct scoutfs_btree_block *bt; + struct scoutfs_btree_root *root; + struct scoutfs_block *par_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_btree_item *item = NULL; + struct scoutfs_block_ref *ref; + unsigned int level; + const bool dirty = op == WALK_INSERT || op == WALK_DELETE || + op == WALK_DIRTY; + + lock_block(sbi, par_bl, dirty); + + /* XXX one for now */ + root = &sbi->super.btree_root; + ref = &root->ref; + level = root->height; + + if (!root->height) { + if (op == WALK_INSERT) { + bl = ERR_PTR(-ENOENT); + } else { + bl = grow_tree(sb, root); + if (!IS_ERR(bl)) + lock_block(sbi, bl, dirty); + } + unlock_block(sbi, par_bl, dirty); + return bl; + } + + while (level--) { + /* XXX hmm, need to think about retry */ + if (dirty) { + bl = scoutfs_dirty_ref(sb, ref); + } else { + bl = scoutfs_read_ref(sb, ref); + } + if (IS_ERR(bl)) + break; + bt = bl->data; + + /* see if a search needs to move to the next parent ref */ + if (next_leaf(parent, &item, bt, op, level, key)) { + ref = (void *)item->val; + level++; + scoutfs_put_block(bl); + continue; + } + + if (op == WALK_INSERT) + bl = try_split(sb, root, level, key, val_len, parent, + item, bl); + if ((op == WALK_DELETE) && parent) + bl = try_merge(sb, root, parent, item, bl); + if (IS_ERR(bl)) + break; + + lock_block(sbi, bl, dirty); + + if (!level) + break; + + /* unlock parent before searching so others can use it */ + unlock_block(sbi, par_bl, dirty); + scoutfs_put_block(par_bl); + par_bl = bl; + parent = bt; + + /* there should always be a parent item */ + item = bt_after(parent, key); + if (!item) { + /* current block dropped as parent below */ + bl = ERR_PTR(-EIO); + break; + } + + /* XXX verify sane length */ + ref = (void *)item->val; + } + + unlock_block(sbi, par_bl, dirty); + scoutfs_put_block(par_bl); + + return bl; +} + +static void set_cursor(struct scoutfs_btree_cursor *curs, + struct scoutfs_block *bl, + struct scoutfs_btree_item *item, bool write) +{ + curs->bl = bl; + curs->item = item; + curs->key = &item->key; + curs->val = item->val; + curs->val_len = le16_to_cpu(item->val_len); + curs->write = !!write; +} + +/* + * Point the caller's cursor at the item if it's found. It can't be + * modified. -ENOENT is returned if the key isn't found in the tree. + */ +int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs) +{ + struct scoutfs_btree_item *item; + struct scoutfs_block *bl; + int ret; + + BUG_ON(curs->bl); + + bl = btree_walk(sb, key, 0, 0); + if (IS_ERR(bl)) + return PTR_ERR(bl); + + item = bt_lookup(bl->data, key); + if (item) { + set_cursor(curs, bl, item, false); + ret = 0; + } else { + up_read(&bl->rwsem); + scoutfs_put_block(bl); + ret = -ENOENT; + } + + return ret; +} + +/* + * Insert a new item in the tree and point the caller's cursor at it. + * The caller is responsible for setting the value. + * + * -EEXIST is returned if the key is already present in the tree. + * + * XXX this walks the treap twice, which isn't great + */ +int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, + unsigned int val_len, + struct scoutfs_btree_cursor *curs) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + BUG_ON(curs->bl); + + bl = btree_walk(sb, key, val_len, WALK_INSERT); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = bl->data; + + /* XXX should this return -eexist? */ + item = bt_lookup(bt, key); + if (!item) { + item = create_item(bt, key, val_len); + set_cursor(curs, bl, item, true); + ret = 0; + } else { + up_write(&bl->rwsem); + scoutfs_put_block(bl); + ret = -ENOENT; + } + + return ret; +} + +/* + * Delete an item from the tree. -ENOENT is returned if the key isn't + * found. + */ +int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int ret; + + bl = btree_walk(sb, key, 0, WALK_DELETE); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = bl->data; + + item = bt_lookup(bt, key); + if (item) { + delete_item(bt, item); + ret = 0; + + /* XXX this locking is broken.. hold root rwsem? */ + + /* delete the final block in the tree */ + if (bt->nr_items == 0) { + memset(&sbi->super.btree_root, 0, + sizeof(struct scoutfs_btree_root)); + /* XXX free block */ + } + } else { + ret = -ENOENT; + } + + up_write(&bl->rwsem); + scoutfs_put_block(bl); + + return ret; +} + +/* + * The caller initializes the cursor and first and last keys and then + * gets the cursor set to each item within those keys. + * + * The btree walk takes care of advancing past interior leaves that + * don't contain items past the key. Our job is to find the next item + * after the key. If that next item's key is past the caller's last key + * then the iteration is done. + * + * returns 0 if no next, > 0 when curs contains next, < 0 on error + */ +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, + struct scoutfs_btree_cursor *curs) +{ + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + struct scoutfs_key key = *first; + int ret; + + trace_printk("first "CKF" last "CKF" %d curs "CKF"\n", + CKA(first), CKA(last), + !!curs->bl, CKA(curs->bl ? curs->key : &key)); + + /* find the next item after the cursor, releasing if we're done */ + if (curs->bl) { + key = curs->item->key; + scoutfs_inc_key(&key); + + curs->item = bt_next(curs->bl->data, curs->item); + trace_printk("next %p\n", curs->item); + if (curs->item) + set_cursor(curs, curs->bl, curs->item, curs->write); + else + scoutfs_btree_release(curs); + } + + /* walk the tree to find the key, can be first or later */ + if (!curs->bl) { + bl = btree_walk(sb, &key, 0, WALK_NEXT); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = bl->data; + + curs->item = bt_after(bl->data, &key); + trace_printk("after %p\n", curs->item); + if (curs->item) { + set_cursor(curs, bl, curs->item, false); + } else { + up_read(&bl->rwsem); + scoutfs_put_block(bl); + } + } + + /* only return the next item if it's within last */ + if (curs->item && scoutfs_key_cmp(curs->key, last) <= 0) { + ret = 1; + } else { + scoutfs_btree_release(curs); + ret = 0; + } + + trace_printk("ret %d\n", ret); + return ret; +} + +/* + * Ensure that the blocks that lead to the item with the given key are + * dirty. caller can hold a transaction to pin the dirty blocks and + * guarantee that later updates of the item will succeed. + * + * <0 is returned on error, including -ENOENT if the key isn't present. + */ +int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) +{ + struct scoutfs_btree_item *item; + struct scoutfs_block *bl; + int ret; + + bl = btree_walk(sb, key, 0, WALK_DIRTY); + if (IS_ERR(bl)) + return PTR_ERR(bl); + + item = bt_lookup(bl->data, key); + if (item) { + ret = 0; + } else { + ret = -ENOENT; + } + + up_write(&bl->rwsem); + scoutfs_put_block(bl); + + return ret; +} + +/* + * For this to be safe the caller has to have pinned the dirty blocks + * for the item in their transaction. + */ +void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs) +{ + struct scoutfs_btree_item *item; + struct scoutfs_block *bl; + + BUG_ON(curs->bl); + + bl = btree_walk(sb, key, 0, WALK_DIRTY); + BUG_ON(IS_ERR(bl)); + + item = bt_lookup(bl->data, key); + BUG_ON(!item); + + set_cursor(curs, bl, item, true); +} + +void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) +{ + if (curs->bl) { + if (curs->write) + up_write(&curs->bl->rwsem); + else + up_read(&curs->bl->rwsem); + scoutfs_put_block(curs->bl); + } + curs->bl = NULL; +} diff --git a/kmod/src/btree.h b/kmod/src/btree.h index d7432313..e902a6bd 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -8,51 +8,27 @@ struct scoutfs_btree_cursor { /* for callers */ struct scoutfs_key *key; - unsigned val_len; void *val; + u16 val_len; + u16 write:1; }; -static inline int scoutfs_btree_lookup(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} +#define DECLARE_SCOUTFS_BTREE_CURSOR(name) \ + struct scoutfs_btree_cursor name = {NULL,} -static inline int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_key *key, - unsigned short val_len, - struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} +int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs); +int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, + unsigned int val_len, + struct scoutfs_btree_cursor *curs); +int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key); +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, + struct scoutfs_btree_cursor *curs); +int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key); +void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs); -static inline int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_key *key, - unsigned short val_len, - struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} - - -static inline int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} - -static inline int scoutfs_btree_next(struct super_block *sb, - struct scoutfs_key *first, - struct scoutfs_key *last, - struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} - -static inline int scoutfs_btree_release(struct scoutfs_btree_cursor *curs) -{ - return -ENOSYS; -} +void scoutfs_btree_release(struct scoutfs_btree_cursor *curs); #endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 6a95bfb3..140ae023 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -176,7 +176,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct scoutfs_inode_info *si = SCOUTFS_I(dir); - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = dir->i_sb; struct scoutfs_dirent *dent; struct dentry_info *di; @@ -209,6 +209,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, h = name_hash(dentry->d_name.name, dentry->d_name.len, h); scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + scoutfs_btree_release(&curs); ret = scoutfs_btree_lookup(sb, &key, &curs); if (ret == -ENOENT) continue; @@ -274,44 +275,38 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_dirent *dent; struct scoutfs_key first; struct scoutfs_key last; unsigned int name_len; - int ret = 0; + int ret; u32 pos; if (!dir_emit_dots(file, dirent, filldir)) return 0; + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + file->f_pos); scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, SCOUTFS_DIRENT_LAST_POS); - while (file->f_pos <= SCOUTFS_DIRENT_LAST_POS) { - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - file->f_pos); - - ret = scoutfs_btree_next(sb, &first, &last, &curs); - if (ret) - break; - + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { dent = curs.val; name_len = item_name_len(&curs); pos = scoutfs_key_offset(curs.key); if (filldir(dirent, dent->name, name_len, pos, - le64_to_cpu(dent->ino), dentry_type(dent->type))) + le64_to_cpu(dent->ino), dentry_type(dent->type))) { + ret = 0; break; + } file->f_pos = pos + 1; } scoutfs_btree_release(&curs); - if (ret == -ENOENT) - ret = 0; - return ret; } @@ -320,7 +315,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, { struct super_block *sb = dir->i_sb; struct scoutfs_inode_info *si = SCOUTFS_I(dir); - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct inode *inode = NULL; struct scoutfs_dirent *dent; struct dentry_info *di; @@ -413,7 +408,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_btree_cursor curs = {NULL,}; struct dentry_info *di; struct scoutfs_key key; int ret = 0; @@ -432,12 +426,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); - ret = scoutfs_btree_lookup(sb, &key, &curs); - if (ret) - goto out; - - ret = scoutfs_btree_delete(sb, &curs); - scoutfs_btree_release(&curs); + ret = scoutfs_btree_delete(sb, &key); if (ret) goto out; diff --git a/kmod/src/format.h b/kmod/src/format.h index d35f5613..a80e95b0 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -34,6 +34,72 @@ struct scoutfs_block_header { __le64 blkno; } __packed; +/* + * We should be able to make the offset smaller if neither dirents nor + * data items use the full 64 bits. + */ +struct scoutfs_key { + __le64 inode; + u8 type; + __le64 offset; +} __packed; + +/* + * Currently we sort keys by the numeric value of the types, but that + * isn't necessary. We could have an arbitrary sort order. So we don't + * have to stress about cleverly allocating the types. + */ +#define SCOUTFS_INODE_KEY 1 +#define SCOUTFS_DIRENT_KEY 2 +#define SCOUTFS_DATA_KEY 3 + +#define SCOUTFS_MAX_ITEM_LEN 2048 + +/* + * Block references include the sequence number so that we can detect + * readers racing with writers and so that we can tell that we don't + * need to follow a reference when traversing based on seqs. + */ +struct scoutfs_block_ref { + __le64 blkno; + __le64 seq; +} __packed; + +struct scoutfs_treap_root { + __le16 off; +} __packed; + +struct scoutfs_treap_node { + __le16 parent; + __le16 left; + __le16 right; + __le32 prio; +} __packed; + +struct scoutfs_btree_root { + u8 height; + struct scoutfs_block_ref ref; +} __packed; + +struct scoutfs_btree_block { + struct scoutfs_block_header hdr; + struct scoutfs_treap_root treap; + __le16 total_free; + __le16 tail_free; + __le16 nr_items; +} __packed; + +struct scoutfs_btree_item { + struct scoutfs_key key; + struct scoutfs_treap_node tnode; + __le16 val_len; + char val[0]; +} __packed; + +/* Blocks are no more than half free. */ +#define SCOUTFS_BTREE_FREE_LIMIT \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) + #define SCOUTFS_UUID_BYTES 16 /* @@ -50,31 +116,11 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; -} __packed; - -/* - * We should be able to make the offset smaller if neither dirents nor - * data items use the full 64 bits. - */ -struct scoutfs_key { - __le64 inode; - u8 type; - __le64 offset; + struct scoutfs_btree_root btree_root; } __packed; #define SCOUTFS_ROOT_INO 1 -/* - * Currently we sort keys by the numeric value of the types, but that - * isn't necessary. We could have an arbitrary sort order. So we don't - * have to stress about cleverly allocating the types. - */ -#define SCOUTFS_INODE_KEY 1 -#define SCOUTFS_DIRENT_KEY 2 -#define SCOUTFS_DATA_KEY 3 - -#define SCOUTFS_MAX_ITEM_LEN 2048 - struct scoutfs_timespec { __le64 sec; __le32 nsec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 44b4c3de..8ce039b3 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -112,7 +112,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) static int scoutfs_read_locked_inode(struct inode *inode) { - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_key key; int ret; @@ -209,23 +209,20 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) * * The caller has to prevent sync between dirtying and updating the * inodes. + * + * XXX this will have to do something about variable length inodes */ int scoutfs_dirty_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_btree_cursor curs = {NULL,}; struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_dirty(sb, &key, sizeof(struct scoutfs_inode), - &curs); - if (!ret) { - store_inode(curs.val, inode); - scoutfs_btree_release(&curs); + ret = scoutfs_btree_dirty(sb, &key); + if (!ret) trace_scoutfs_dirty_inode(inode); - } return ret; } @@ -240,18 +237,13 @@ int scoutfs_dirty_inode_item(struct inode *inode) */ void scoutfs_update_inode_item(struct inode *inode) { - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_key key; - int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - /* XXX maybe just use dirty again? not sure.. */ - ret = scoutfs_btree_dirty(sb, &key, sizeof(struct scoutfs_inode), - &curs); - BUG_ON(ret); - + scoutfs_btree_update(sb, &key, &curs); store_inode(curs.val, inode); scoutfs_btree_release(&curs); trace_scoutfs_update_inode(inode); @@ -265,7 +257,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; struct scoutfs_key key; struct inode *inode; diff --git a/kmod/src/super.c b/kmod/src/super.c index 0e8a7d60..0921e313 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -103,6 +103,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->block_lock); INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); init_waitqueue_head(&sbi->block_wq); + init_rwsem(&sbi->btree_rwsem); /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); diff --git a/kmod/src/super.h b/kmod/src/super.h index 1d5170f4..c7659844 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -18,6 +18,9 @@ struct scoutfs_sb_info { atomic64_t next_ino; atomic64_t next_blkno; + /* XXX there will be a lot more of these :) */ + struct rw_semaphore btree_rwsem; + /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; diff --git a/kmod/src/treap.c b/kmod/src/treap.c new file mode 100644 index 00000000..229e4daf --- /dev/null +++ b/kmod/src/treap.c @@ -0,0 +1,364 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "format.h" +#include "treap.h" + +/* + * Implement a simple treap in memory. The caller is responsible for + * allocating and freeing roots and nodes. This only performs the tree + * operations on them. + * + * Node references are stored as byte offsets from the root to the node. + * As long as we have the root the byte offsets or node pointers are + * interchangeable. The code tries to prefer to use pointers to be + * slightly easier to read. + * + * The caller is responsible for locking access to the tree. + */ + +static struct scoutfs_treap_node *off_node(struct scoutfs_treap_root *root, + __le16 off) +{ + if (!off) + return NULL; + + return (void *)root + le16_to_cpu(off); +} + +static __le16 node_off(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node) +{ + if (!node) + return 0; + + return cpu_to_le16((char *)node - (char *)root); +} + +/* + * Walk the tree looking for a node that matches a node in the tree. + * Return the found node or the last node traversed. Set the caller's + * cmp to the comparison between the key and the returned node. The + * caller can ask that we set their pointers to the most recently + * traversed node before or after the returned node. + */ +static struct scoutfs_treap_node *descend(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key, + int *cmp, + struct scoutfs_treap_node **before, + struct scoutfs_treap_node **after) +{ + struct scoutfs_treap_node *node = NULL; + __le16 off = root->off; + + *cmp = -1; + if (before) + *before = NULL; + if (after) + *after = NULL; + + while (off) { + node = off_node(root, off); + *cmp = cmp_func(key, node); + if (*cmp < 0) { + if (after) + *after = node; + off = node->left; + } else if (*cmp > 0) { + if (before) + *before = node; + off = node->right; + } else { + break; + } + } + + return node; +} + +/* + * Link the two nodes together by setting their child and parent pointers + * as needed. Both parent and child can be null. + */ +static void set_links(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *parent, bool left, + struct scoutfs_treap_node *child) +{ + if (!parent) + root->off = node_off(root, child); + else if (left) + parent->left = node_off(root, child); + else + parent->right = node_off(root, child); + + if (child) + child->parent = node_off(root, parent); +} + +/* + * Perform a tree rotation. The node pointer names describe their + * relationships before the rotation. We use the relationship between + * the node and its child to determine the direction of the rotation. + * After the rotation the child will be higher than the node. Only the + * node and child must exist. + * + * Here's a right rotation: + * + * parent parent + * | | + * node child + * / \ / \ + * child a b node + * / \ / \ + * b gr_chi gr_chi a + * + */ +static void rotation(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node, + struct scoutfs_treap_node *child) +{ + struct scoutfs_treap_node *parent = off_node(root, node->parent); + struct scoutfs_treap_node *grand_child; + bool right; + + if (node->left == node_off(root, child)) { + right = true; + grand_child = off_node(root, child->right); + } else { + right = false; + grand_child = off_node(root, child->left); + } + + set_links(root, parent, + parent && (parent->left == node_off(root, node)), child); + set_links(root, node, right, grand_child); + set_links(root, child, !right, node); +} + +/* + * Insertion links a node in at a leaf and then rotates it up the + * tree until its parent has a higher priority. + */ +int scoutfs_treap_insert(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *ins) +{ + struct scoutfs_treap_node *parent; + int cmp; + + ins->prio = cpu_to_le32(get_random_int()); + ins->parent = 0; + ins->left = 0; + ins->right = 0; + + parent = descend(root, cmp_func, ins, &cmp, NULL, NULL); + if (cmp == 0) + return -EEXIST; + + set_links(root, parent, cmp < 0, ins); + + while (ins->parent) { + parent = off_node(root, ins->parent); + if (le32_to_cpu(ins->prio) < le32_to_cpu(parent->prio)) + break; + + rotation(root, parent, ins); + } + + return 0; +} + +/* + * Deletion rotates the node down the tree until it doesn't have two + * children so that it can be unlinked by pointing its parent at its + * child, if it has one. + */ +void scoutfs_treap_delete(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node) +{ + struct scoutfs_treap_node *left; + struct scoutfs_treap_node *right; + struct scoutfs_treap_node *child; + struct scoutfs_treap_node *parent; + + while (node->left && node->right) { + left = off_node(root, node->left); + right = off_node(root, node->right); + + if (le32_to_cpu(left->prio) > le32_to_cpu(right->prio)) + rotation(root, node, left); + else + rotation(root, node, right); + } + + parent = off_node(root, node->parent); + + if (node->left) + child = off_node(root, node->left); + else + child = off_node(root, node->right); + + set_links(root, parent, + parent && parent->left == node_off(root, node), child); +} + +struct scoutfs_treap_node *scoutfs_treap_lookup(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key) +{ + struct scoutfs_treap_node *node; + int cmp; + + node = descend(root, cmp_func, key, &cmp, NULL, NULL); + if (cmp != 0) + return NULL; + + return node; +} + +/* return the first node in the tree */ +struct scoutfs_treap_node *scoutfs_treap_first(struct scoutfs_treap_root *root) +{ + struct scoutfs_treap_node *node = off_node(root, root->off); + + while (node && node->left) + node = off_node(root, node->left); + + return node; +} + +/* return the last node in the tree */ +struct scoutfs_treap_node *scoutfs_treap_last(struct scoutfs_treap_root *root) +{ + struct scoutfs_treap_node *node = off_node(root, root->off); + + while (node && node->right) + node = off_node(root, node->right); + + return node; +} + +/* return the last node whose key is less than or equal to the key */ +struct scoutfs_treap_node *scoutfs_treap_before(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key) +{ + struct scoutfs_treap_node *before; + struct scoutfs_treap_node *node; + int cmp; + + node = descend(root, cmp_func, key, &cmp, &before, NULL); + if (cmp == 0) + return node; + + return before; +} + +/* return the first node whose key is greater than or equal to the key */ +struct scoutfs_treap_node *scoutfs_treap_after(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key) +{ + struct scoutfs_treap_node *after; + struct scoutfs_treap_node *node; + int cmp; + + node = descend(root, cmp_func, key, &cmp, NULL, &after); + if (cmp == 0) + return node; + + return after; +} + +/* + * The usual BST iteration: either the least descendant or the first + * ancestor in the direction of the iteration. + */ +struct scoutfs_treap_node *scoutfs_treap_next(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node) +{ + struct scoutfs_treap_node *parent; + + if (node->right) { + node = off_node(root, node->right); + while (node->left) + node = off_node(root, node->left); + return node; + } + + while ((parent = off_node(root, node->parent)) && + parent->right == node_off(root, node)) { + node = parent; + } + + return parent; +} + +struct scoutfs_treap_node *scoutfs_treap_prev(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node) +{ + struct scoutfs_treap_node *parent; + + if (node->left) { + node = off_node(root, node->left); + while (node->right) + node = off_node(root, node->right); + return node; + } + + while ((parent = off_node(root, node->parent)) && + parent->left == node_off(root, node)) { + node = parent; + } + + return parent; +} + +static void update_relative(struct scoutfs_treap_root *root, __le16 node_off, + __le16 from_off, __le16 to_off) +{ + struct scoutfs_treap_node *node = off_node(root, node_off); + + if (node) { + if (node->parent == from_off) + node->parent = to_off; + else if (node->left == from_off) + node->left = to_off; + else if (node->right == from_off) + node->right = to_off; + } +} + +/* + * A node has moved from one storage location to another. Update the + * nodes that refer to it. The from pointer can only be used to + * determine the old offset. Its contents are undefined. + */ +void scoutfs_treap_move(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *from, + struct scoutfs_treap_node *to) +{ + __le16 from_off = node_off(root, from); + __le16 to_off = node_off(root, to); + + if (root->off == from_off) + root->off = to_off; + else + update_relative(root, to->parent, from_off, to_off); + + update_relative(root, to->left, from_off, to_off); + update_relative(root, to->right, from_off, to_off); +} diff --git a/kmod/src/treap.h b/kmod/src/treap.h new file mode 100644 index 00000000..d02f406c --- /dev/null +++ b/kmod/src/treap.h @@ -0,0 +1,38 @@ +#ifndef _SCOUTFS_TREAP_H_ +#define _SCOUTFS_TREAP_H_ + +#include "format.h" + +typedef int (*scoutfs_treap_cmp_t)(struct scoutfs_treap_node *a, + struct scoutfs_treap_node *b); + +static inline void scoutfs_treap_init(struct scoutfs_treap_root *root) +{ + root->off = 0; +} + +int scoutfs_treap_insert(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *ins); +void scoutfs_treap_delete(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node); +struct scoutfs_treap_node *scoutfs_treap_lookup(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key); +struct scoutfs_treap_node *scoutfs_treap_first(struct scoutfs_treap_root *root); +struct scoutfs_treap_node *scoutfs_treap_last(struct scoutfs_treap_root *root); +struct scoutfs_treap_node *scoutfs_treap_before(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key); +struct scoutfs_treap_node *scoutfs_treap_after(struct scoutfs_treap_root *root, + scoutfs_treap_cmp_t cmp_func, + struct scoutfs_treap_node *key); +struct scoutfs_treap_node *scoutfs_treap_next(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node); +struct scoutfs_treap_node *scoutfs_treap_prev(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *node); +void scoutfs_treap_move(struct scoutfs_treap_root *root, + struct scoutfs_treap_node *from, + struct scoutfs_treap_node *to); + +#endif From affee9da7c32abba2b4c3b6e3d9a66376510e38d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 Apr 2016 19:37:31 -0700 Subject: [PATCH 042/920] scoutfs: add cscope noise to .gitignore Signed-off-by: Zach Brown --- kmod/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/.gitignore b/kmod/.gitignore index 9d66c4e8..50873cec 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -5,3 +5,4 @@ src/*.cmd src/.tmp_versions/ src/Module.symvers src/modules.order +cscope.* From 0234abf098edc1bbef8c9d73af1d65242c0275b1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 13 Apr 2016 09:49:13 -0700 Subject: [PATCH 043/920] scoutfs: update filerw cursor use The conversion of the filerw item callers of the btree cursor wasn't updated to consistently release the cursors. This was causing block refcounting problems that could scribble on freed and realloced memory. Signed-off-by: Zach Brown --- kmod/src/filerw.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 074204df..7b7cb532 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -81,7 +81,7 @@ static bool map_data_region(struct data_region *dr, u64 pos, struct page *page) static int scoutfs_readpage(struct file *file, struct page *page) { struct inode *inode = file->f_mapping->host; - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_key key; struct data_region dr; @@ -93,6 +93,7 @@ static int scoutfs_readpage(struct file *file, struct page *page) scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, dr.item_key); + scoutfs_btree_release(&curs); ret = scoutfs_btree_lookup(sb, &key, &curs); if (ret == -ENOENT) { addr = kmap_atomic(page); @@ -108,6 +109,8 @@ static int scoutfs_readpage(struct file *file, struct page *page) kunmap_atomic(addr); } + scoutfs_btree_release(&curs); + if (!ret) SetPageUptodate(page); unlock_page(page); @@ -125,7 +128,7 @@ static int scoutfs_readpage(struct file *file, struct page *page) static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) { struct inode *inode = page->mapping->host; - struct scoutfs_btree_cursor curs = {NULL,}; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_key key; struct data_region dr; @@ -140,6 +143,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) dr.item_key); /* XXX dirty */ + scoutfs_btree_release(&curs); ret = scoutfs_btree_insert(sb, &key, SCOUTFS_MAX_ITEM_LEN, &curs); if (ret) From 5d77fa4f1881852f5101024885599b809839869a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 Apr 2016 13:08:56 -0700 Subject: [PATCH 044/920] scoutfs: fix serious but small btree bugs Not surprisingly, testing the btree code shook out a few bugs - the treap root wasn't initialized - existing split source block wasn't compacted - item movement used item treap fields after deletion All of these had the consequence of feeding the treap code bad offsets so its node/u16 casts could lead it to scribble over memory. Signed-off-by: Zach Brown --- kmod/src/btree.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index cf3e700e..60f63622 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -208,6 +208,7 @@ static void move_items(struct scoutfs_btree_block *dst, int to_move) { struct scoutfs_btree_item *from; + struct scoutfs_btree_item *del; struct scoutfs_btree_item *to; unsigned int val_len; @@ -222,12 +223,13 @@ static void move_items(struct scoutfs_btree_block *dst, to = create_item(dst, &from->key, val_len); memcpy(to->val, from->val, val_len); - delete_item(src, from); - + del = from; if (move_right) from = bt_prev(src, from); else from = bt_next(src, from); + + delete_item(src, del); to_move -= item_bytes(to); } } @@ -267,8 +269,7 @@ static void compact_items(struct scoutfs_btree_block *bt) if (from->tnode.parent != MAGIC_DELETED_PARENT) { if (from != to) { memmove(to, from, bytes); - scoutfs_treap_move(&bt->treap, - &from->tnode, + scoutfs_treap_move(&bt->treap, &from->tnode, &to->tnode); } to = (void *)to + bytes; @@ -296,6 +297,7 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) if (!IS_ERR(bl)) { bt = bl->data; + bt->treap.off = 0; bt->total_free = cpu_to_le16(SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)); bt->tail_free = bt->total_free; @@ -421,10 +423,15 @@ static struct scoutfs_block *try_split(struct super_block *sb, create_parent_item(parent, left, greatest_key(left)); if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { + /* insertion will go to the new left block */ scoutfs_put_block(right_bl); right_bl = left_bl; } else { + /* insertion will still go through us, might need to compact */ scoutfs_put_block(left_bl); + + if (le16_to_cpu(right->tail_free) < bytes) + compact_items(right); } scoutfs_put_block(par_bl); From 3e5eeaa80c6ef5568ed54ab11c8319a50d39bf3f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 Apr 2016 13:13:18 -0700 Subject: [PATCH 045/920] scoutfs: initialize block alloc past mkfs blocks The format doesn't yet record free blocks. We've been relying on the scary initialization of the block allocator past the blocks that are written by mkfs. And it was wrong. This garbage will be replaced with an allocator in a few commits. Signed-off-by: Zach Brown --- kmod/src/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 0921e313..393afc46 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -80,7 +80,7 @@ static int read_supers(struct super_block *sb) * XXX These don't exist in the super yet. They should soon. */ atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); - atomic64_set(&sbi->next_blkno, 2); + atomic64_set(&sbi->next_blkno, 6); return 0; } From 1c284af8543ff30dea56409cb213a3c1bb14cf32 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 Apr 2016 13:15:12 -0700 Subject: [PATCH 046/920] scoutfs: add assertions for bad treap offsets The wild casting in the treap code can cause memory corruption if it's fed bad offsets. Add some assertions so that we can see when this is happening. Signed-off-by: Zach Brown --- kmod/src/treap.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 229e4daf..39ab4a3b 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -29,22 +29,44 @@ * The caller is responsible for locking access to the tree. */ +/* + * treap nodes are embedded in btree items. Their offset is relative to + * the treap root which is embedded in the btree block header. Their + * offset can't have the item overlap the btree block header, nor can + * the item fall off the end of the block. + */ +static void bug_on_bad_node_off(u16 off) +{ + BUG_ON(off < (sizeof(struct scoutfs_btree_block) - + offsetof(struct scoutfs_btree_block, treap) + + offsetof(struct scoutfs_btree_item, tnode))); + BUG_ON(off > (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_item) + + offsetof(struct scoutfs_btree_item, tnode))); +} + static struct scoutfs_treap_node *off_node(struct scoutfs_treap_root *root, __le16 off) { if (!off) return NULL; + bug_on_bad_node_off(le16_to_cpu(off)); + return (void *)root + le16_to_cpu(off); } static __le16 node_off(struct scoutfs_treap_root *root, struct scoutfs_treap_node *node) { + u16 off; + if (!node) return 0; - return cpu_to_le16((char *)node - (char *)root); + off = (char *)node - (char *)root; + bug_on_bad_node_off(off); + + return cpu_to_le16(off); } /* From a2f55f02a18aa0b106c7623e6a636e8ede3c75cf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 Apr 2016 14:32:21 -0700 Subject: [PATCH 047/920] scoutfs: avoid stale btree block pointer The btree walk was storing a pointer to the current btree block that it was working on. It later used this when the walk continues and the block becomes a parent. But it didn't update this pointer if splitting changed the block to traverse. By removing this pointer and using the block data pointers directly we remove the risk of the pointer going stale. Signed-off-by: Zach Brown --- kmod/src/btree.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 60f63622..069a50bf 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -621,7 +621,6 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_block *parent = NULL; - struct scoutfs_btree_block *bt; struct scoutfs_btree_root *root; struct scoutfs_block *par_bl = NULL; struct scoutfs_block *bl = NULL; @@ -659,10 +658,9 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, } if (IS_ERR(bl)) break; - bt = bl->data; /* see if a search needs to move to the next parent ref */ - if (next_leaf(parent, &item, bt, op, level, key)) { + if (next_leaf(parent, &item, bl->data, op, level, key)) { ref = (void *)item->val; level++; scoutfs_put_block(bl); @@ -686,7 +684,7 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, unlock_block(sbi, par_bl, dirty); scoutfs_put_block(par_bl); par_bl = bl; - parent = bt; + parent = par_bl->data; /* there should always be a parent item */ item = bt_after(parent, key); From e3b308c0d07572a61c29ebd0fe0acbe00f22a1a5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 Apr 2016 14:35:32 -0700 Subject: [PATCH 048/920] scoutfs: add transactions and metadata writing Add the transaction machinery that writes out dirty metadata blocks as atomic transactions. The block radix tracks dirty blocks with a dirty radix tag. Blocks are written with bios whose completion marks them clean and propogates errors through the super info. The blocks are left tagged during writeout so that they won't be (someday) mistaken for clean by eviction. Since we're modifying the radix from io completion we change all block lock acquisitions to be interrupt safe. All the operations that modify blocks hold and release the transaction while they're doing their work. sync kicks off work that waits for the transaction to be released so that it can write out all the dirty blocks and then the new supers that reference them. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/block.c | 175 ++++++++++++++++++++++++++++++++++++++++---- kmod/src/block.h | 4 ++ kmod/src/dir.c | 21 ++++-- kmod/src/filerw.c | 25 +++++-- kmod/src/super.c | 59 ++++++++++++++- kmod/src/super.h | 17 +++++ kmod/src/trans.c | 180 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/trans.h | 13 ++++ 9 files changed, 471 insertions(+), 25 deletions(-) create mode 100644 kmod/src/trans.c create mode 100644 kmod/src/trans.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 3ceb5af4..95f35039 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,4 +3,4 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += block.o btree.o counters.o crc.o dir.o filerw.o inode.o msg.o \ - scoutfs_trace.o super.o treap.o + scoutfs_trace.o super.o trans.o treap.o diff --git a/kmod/src/block.c b/kmod/src/block.c index fc232e38..efaff13c 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -22,6 +22,8 @@ #include "crc.h" #include "counters.h" +#define DIRTY_RADIX_TAG 0 + /* * XXX * - tie into reclaim @@ -113,6 +115,38 @@ static void block_read_end_io(struct bio *bio, int err) wake_up(&sbi->block_wq); scoutfs_put_block(bl); + bio_put(bio); +} + +/* + * Once a transaction block is persistent it's fine to drop the dirty + * tag. It's been checksummed so it can be read in again. It's seq + * will be in the current transaction so it'll simply be dirtied and + * checksummed and written out again. + */ +static void block_write_end_io(struct bio *bio, int err) +{ + struct scoutfs_block *bl = bio->bi_private; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + unsigned long flags; + + if (!err) { + spin_lock_irqsave(&sbi->block_lock, flags); + radix_tree_tag_clear(&sbi->block_radix, + bl->blkno, DIRTY_RADIX_TAG); + spin_unlock_irqrestore(&sbi->block_lock, flags); + } + + /* not too worried about racing ints */ + if (err && !sbi->block_write_err) + sbi->block_write_err = err; + + if (atomic_dec_and_test(&sbi->block_writes)) + wake_up(&sbi->block_wq); + + scoutfs_put_block(bl); + bio_put(bio); + } static int block_submit_bio(struct scoutfs_block *bl, int rw) @@ -121,19 +155,30 @@ static int block_submit_bio(struct scoutfs_block *bl, int rw) struct bio *bio; int ret; + if (WARN_ON_ONCE(bl->blkno >= + i_size_read(sb->s_bdev->bd_inode) >> SCOUTFS_BLOCK_SHIFT)) { + printk("trying to read bad blkno %llu\n", bl->blkno); + } + + bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); if (WARN_ON_ONCE(!bio)) return -ENOMEM; bio->bi_sector = bl->blkno << (SCOUTFS_BLOCK_SHIFT - 9); bio->bi_bdev = sb->s_bdev; - /* XXX can we do that? */ - ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); - if (rw & WRITE) - ; - else + if (rw & WRITE) { + bio->bi_end_io = block_write_end_io; + } else bio->bi_end_io = block_read_end_io; bio->bi_private = bl; + + ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); + if (WARN_ON_ONCE(ret != SCOUTFS_BLOCK_SIZE)) { + bio_put(bio); + return -ENOMEM; + } + atomic_inc(&bl->refcount); submit_bio(rw, bio); @@ -148,10 +193,11 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block *found; struct scoutfs_block *bl; + unsigned long flags; int ret; /* find an existing block, dropping if it's errored */ - spin_lock(&sbi->block_lock); + spin_lock_irqsave(&sbi->block_lock, flags); bl = radix_tree_lookup(&sbi->block_radix, blkno); if (bl) { @@ -164,7 +210,7 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) } } - spin_unlock(&sbi->block_lock); + spin_unlock_irqrestore(&sbi->block_lock, flags); if (bl) goto wait; @@ -179,7 +225,7 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) if (ret) goto out; - spin_lock(&sbi->block_lock); + spin_lock_irqsave(&sbi->block_lock, flags); found = radix_tree_lookup(&sbi->block_radix, blkno); if (found) { @@ -191,7 +237,7 @@ struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) atomic_inc(&bl->refcount); } - spin_unlock(&sbi->block_lock); + spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); if (!found) { @@ -237,6 +283,7 @@ struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, struct scoutfs_block_header *hdr; struct scoutfs_block *bl; struct scoutfs_block *found; + unsigned long flags; bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); if (!IS_ERR(bl)) { @@ -244,14 +291,14 @@ struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, if (WARN_ON_ONCE(hdr->seq != ref->seq)) { /* XXX hack, make this a function */ - spin_lock(&sbi->block_lock); + spin_lock_irqsave(&sbi->block_lock, flags); found = radix_tree_lookup(&sbi->block_radix, bl->blkno); if (found == bl) { radix_tree_delete(&sbi->block_radix, bl->blkno); scoutfs_put_block(bl); } - spin_unlock(&sbi->block_lock); + spin_unlock_irqrestore(&sbi->block_lock, flags); scoutfs_put_block(bl); bl = ERR_PTR(-EAGAIN); @@ -261,6 +308,100 @@ struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, return bl; } +/* + * XXX This is a gross hack for writing the super. It doesn't have + * per-block write completion indication, it just knows that it's the + * only thing that will be writing. + */ +int scoutfs_write_block(struct scoutfs_block *bl) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + int ret; + + BUG_ON(atomic_read(&sbi->block_writes) != 0); + + atomic_inc(&sbi->block_writes); + ret = block_submit_bio(bl, WRITE); + if (ret) + atomic_dec(&sbi->block_writes); + else + wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); + + return ret ?: sbi->block_write_err; +} + +/* + * A quick cheap test so that write dirty blocks only has to return + * success or error, not also the lack of dirty blocks. + */ +int scoutfs_has_dirty_blocks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return radix_tree_tagged(&sbi->block_radix, DIRTY_RADIX_TAG); +} + +/* + * Write out all the currently dirty blocks. The caller has waited + * for all the dirty blocks to be consistent and has prevented further + * writes while we're working. + * + * The blocks are kept dirty so that they won't be evicted by reclaim + * while they're in flight. Reads can traverse the blocks while they're + * in flight. + */ +int scoutfs_write_dirty_blocks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *blocks[16]; + struct scoutfs_block *bl; + unsigned long flags; + unsigned long blkno; + int ret; + int nr; + int i; + + blkno = 0; + sbi->block_write_err = 0; + ret = 0; + atomic_inc(&sbi->block_writes); + + do { + /* get refs to a bunch of dirty blocks */ + spin_lock_irqsave(&sbi->block_lock, flags); + nr = radix_tree_gang_lookup_tag(&sbi->block_radix, + (void **)blocks, blkno, + ARRAY_SIZE(blocks), + DIRTY_RADIX_TAG); + if (nr > 0) + blkno = blocks[nr - 1]->blkno + 1; + for (i = 0; i < nr; i++) + atomic_inc(&blocks[i]->refcount); + spin_unlock_irqrestore(&sbi->block_lock, flags); + + /* submit them in order, being careful to put all on err */ + for (i = 0; i < nr; i++) { + bl = blocks[i]; + + if (ret == 0) { + /* XXX crc could be farmed out */ + scoutfs_calc_hdr_crc(bl); + atomic_inc(&sbi->block_writes); + ret = block_submit_bio(bl, WRITE); + if (ret) + atomic_dec(&sbi->block_writes); + } + scoutfs_put_block(bl); + } + } while (nr && !ret); + + /* wait for all io to drain */ + atomic_dec(&sbi->block_writes); + wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); + + return ret ?: sbi->block_write_err; +} + /* * Give the caller a dirty block that they can safely modify. If the * reference refers to a stable clean block then we allocate a new block @@ -284,6 +425,7 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, struct scoutfs_block_header *hdr; struct scoutfs_block *found; struct scoutfs_block *bl; + unsigned long flags; u64 blkno; int ret; @@ -301,7 +443,7 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, blkno = atomic64_inc_return(&sbi->next_blkno); hdr = bl->data; - spin_lock(&sbi->block_lock); + spin_lock_irqsave(&sbi->block_lock, flags); /* XXX don't really like this */ found = radix_tree_lookup(&sbi->block_radix, bl->blkno); @@ -314,9 +456,10 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; radix_tree_insert(&sbi->block_radix, blkno, bl); + radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); atomic_inc(&bl->refcount); - spin_unlock(&sbi->block_lock); + spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); ref->blkno = hdr->blkno; @@ -337,6 +480,7 @@ struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) struct scoutfs_block_header *hdr; struct scoutfs_block *found; struct scoutfs_block *bl; + unsigned long flags; int ret; /* allocate a new block and try to insert it */ @@ -357,7 +501,7 @@ struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; - spin_lock(&sbi->block_lock); + spin_lock_irqsave(&sbi->block_lock, flags); found = radix_tree_lookup(&sbi->block_radix, blkno); if (found) { radix_tree_delete(&sbi->block_radix, blkno); @@ -365,8 +509,9 @@ struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) } radix_tree_insert(&sbi->block_radix, blkno, bl); + radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); atomic_inc(&bl->refcount); - spin_unlock(&sbi->block_lock); + spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); ret = 0; diff --git a/kmod/src/block.h b/kmod/src/block.h index dac2b4e8..724f4aca 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -30,6 +30,10 @@ struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, struct scoutfs_block_ref *ref); +int scoutfs_has_dirty_blocks(struct super_block *sb); +int scoutfs_write_block(struct scoutfs_block *bl); +int scoutfs_write_dirty_blocks(struct super_block *sb); + void scoutfs_put_block(struct scoutfs_block *bl); void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 140ae023..73c8e938 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -22,6 +22,7 @@ #include "key.h" #include "super.h" #include "btree.h" +#include "trans.h" /* * Directory entries are stored in entries with offsets calculated from @@ -332,13 +333,19 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_dirty_inode_item(dir); + ret = scoutfs_hold_trans(sb); if (ret) return ret; + ret = scoutfs_dirty_inode_item(dir); + if (ret) + goto out; + inode = scoutfs_new_inode(sb, dir, mode, rdev); - if (IS_ERR(inode)) - return PTR_ERR(inode); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } bytes = dent_bytes(dentry->d_name.len); @@ -384,6 +391,7 @@ out: /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); + scoutfs_release_trans(sb); return ret; } @@ -419,10 +427,14 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + ret = scoutfs_dirty_inode_item(dir) ?: scoutfs_dirty_inode_item(inode); if (ret) - return ret; + goto out; scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); @@ -444,6 +456,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(dir); out: + scoutfs_release_trans(sb); return ret; } diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 7b7cb532..a6e75fc7 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -18,6 +18,7 @@ #include "inode.h" #include "key.h" #include "filerw.h" +#include "trans.h" #include "scoutfs_trace.h" #include "btree.h" @@ -138,6 +139,10 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) set_page_writeback(page); + ret = scoutfs_hold_trans(sb); + if (ret) + goto out; + for_each_data_region(&dr, page, pos) { scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, dr.item_key); @@ -156,7 +161,8 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) } scoutfs_btree_release(&curs); - + scoutfs_release_trans(sb); +out: if (ret) { SetPageError(page); mapping_set_error(&inode->i_data, ret); @@ -191,6 +197,7 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct page *page, void *fsdata) { struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; unsigned off; trace_scoutfs_write_end(scoutfs_ino(inode), pos, len, copied); @@ -203,9 +210,19 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, if (pos + copied > inode->i_size) { i_size_write(inode, pos + copied); - /* XXX need to think about pinning and enospc */ - if (!scoutfs_dirty_inode_item(inode)) - scoutfs_update_inode_item(inode); + + /* + * XXX This is a crazy hack that will go away when the + * file data paths are more robust. We're barely + * holding them together with duct tape while building + * up the robust metadata support that's needed to do a + * good job with the data pats. + */ + if (!scoutfs_hold_trans(sb)) { + if (!scoutfs_dirty_inode_item(inode)) + scoutfs_update_inode_item(inode); + scoutfs_release_trans(sb); + } } if (!PageUptodate(page)) diff --git a/kmod/src/super.c b/kmod/src/super.c index 393afc46..b79c6b8f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -25,6 +25,7 @@ #include "msg.h" #include "block.h" #include "counters.h" +#include "trans.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -32,8 +33,53 @@ static struct kset *scoutfs_kset; static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, + .sync_fs = scoutfs_sync_fs, }; +/* + * The caller advances the block number and sequence number in the super + * every time it wants to dirty it and eventually write it to reference + * dirty data that's been written. + */ +void scoutfs_advance_dirty_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + le64_add_cpu(&super->hdr.blkno, 1); + if (le64_to_cpu(super->hdr.blkno) == (SCOUTFS_SUPER_BLKNO + + SCOUTFS_SUPER_NR)) + super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO); + + le64_add_cpu(&super->hdr.seq, 1); +} + +/* + * The caller is responsible for setting the super header's blkno + * and seq to something reasonable. + */ +int scoutfs_write_dirty_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + size_t sz = sizeof(struct scoutfs_super_block); + u64 blkno = le64_to_cpu(sbi->super.hdr.blkno); + struct scoutfs_block *bl; + int ret; + + /* XXX prealloc? */ + bl = scoutfs_new_block(sb, blkno); + if (WARN_ON_ONCE(IS_ERR(bl))) + return PTR_ERR(bl); + + memcpy(bl->data, &sbi->super, sz); + memset(bl->data + sz, 0, SCOUTFS_BLOCK_SIZE - sz); + scoutfs_calc_hdr_crc(bl); + ret = scoutfs_write_block(bl); + + scoutfs_put_block(bl); + return ret; +} + static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -97,13 +143,20 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi = kzalloc(sizeof(struct scoutfs_sb_info), GFP_KERNEL); sb->s_fs_info = sbi; + sbi->sb = sb; if (!sbi) return -ENOMEM; spin_lock_init(&sbi->block_lock); INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); init_waitqueue_head(&sbi->block_wq); + atomic_set(&sbi->block_writes, 0); init_rwsem(&sbi->btree_rwsem); + atomic_set(&sbi->trans_holds, 0); + init_waitqueue_head(&sbi->trans_hold_wq); + spin_lock_init(&sbi->trans_write_lock); + INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); + init_waitqueue_head(&sbi->trans_write_wq); /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); @@ -111,10 +164,13 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -ENOMEM; ret = scoutfs_setup_counters(sb) ?: - read_supers(sb); + read_supers(sb) ?: + scoutfs_setup_trans(sb); if (ret) return ret; + scoutfs_advance_dirty_super(sb); + inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -138,6 +194,7 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_shutdown_trans(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); diff --git a/kmod/src/super.h b/kmod/src/super.h index c7659844..df2dd30f 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -9,11 +9,15 @@ struct scoutfs_counters; struct scoutfs_sb_info { + struct super_block *sb; + struct scoutfs_super_block super; spinlock_t block_lock; struct radix_tree_root block_radix; wait_queue_head_t block_wq; + atomic_t block_writes; + int block_write_err; atomic64_t next_ino; atomic64_t next_blkno; @@ -21,6 +25,16 @@ struct scoutfs_sb_info { /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; + atomic_t trans_holds; + wait_queue_head_t trans_hold_wq; + + spinlock_t trans_write_lock; + u64 trans_write_count; + int trans_write_ret; + struct work_struct trans_write_work; + wait_queue_head_t trans_write_wq; + struct workqueue_struct *trans_write_workq; + /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; @@ -32,4 +46,7 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } +void scoutfs_advance_dirty_super(struct super_block *sb); +int scoutfs_write_dirty_super(struct super_block *sb); + #endif diff --git a/kmod/src/trans.c b/kmod/src/trans.c new file mode 100644 index 00000000..7194e9af --- /dev/null +++ b/kmod/src/trans.c @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "block.h" +#include "trans.h" +#include "scoutfs_trace.h" + +/* + * scoutfs metadata blocks are written in atomic transactions. + * + * Writers hold transactions to dirty blocks. The transaction can't be + * written until these active writers release the transaction. We don't + * track the relationships between dirty blocks so there's only ever one + * transaction being built. + * + * The copy of the on-disk super block in the fs sb info has its header + * sequence advanced so that new dirty blocks inherit this dirty + * sequence number. It's only advanced once all those dirty blocks are + * reachable after having first written them all out and then the new + * super with that seq. It's first incremented at mount. + * + * Unfortunately writers can nest. We don't bother trying to special + * case holding a transaction that you're already holding because that + * requires per-task storage. We just let anyone hold transactions + * regardless of waiters waiting to write, which risks waiters waiting a + * very long time. + */ + +/* + * It's critical that this not try to perform IO if there's nothing + * dirty. The sync at unmount can have this work scheduled after sync + * returns and the unmount path starts to tear down supers and block + * devices. We have to safely detect that there's nothing to do using + * nothing in the vfs. + */ +void scoutfs_trans_write_func(struct work_struct *work) +{ + struct scoutfs_sb_info *sbi = container_of(work, struct scoutfs_sb_info, + trans_write_work); + struct super_block *sb = sbi->sb; + bool advance = false; + int ret = 0; + + wait_event(sbi->trans_hold_wq, + atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); + + /* XXX probably want to write out dirty pages in inodes */ + + if (scoutfs_has_dirty_blocks(sb)) { + ret = scoutfs_write_dirty_blocks(sb) ?: + scoutfs_write_dirty_super(sb); + if (!ret) + advance = 1; + } + + + spin_lock(&sbi->trans_write_lock); + if (advance) + scoutfs_advance_dirty_super(sb); + sbi->trans_write_count++; + sbi->trans_write_ret = ret; + spin_unlock(&sbi->trans_write_lock); + wake_up(&sbi->trans_write_wq); + + atomic_set(&sbi->trans_holds, 0); + wake_up(&sbi->trans_hold_wq); +} + +struct write_attempt { + u64 seq; + u64 count; + int ret; +}; + +/* this is called as a wait_event() condition so it can't change task state */ +static int write_attempted(struct scoutfs_sb_info *sbi, + struct write_attempt *attempt) +{ + int done = 1; + + spin_lock(&sbi->trans_write_lock); + if (le64_to_cpu(sbi->super.hdr.seq) > attempt->seq) + attempt->ret = 0; + else if (sbi->trans_write_count > attempt->count) + attempt->ret = sbi->trans_write_ret; + else + done = 0; + spin_unlock(&sbi->trans_write_lock); + + return done; +} + +/* + * sync records the current dirty seq and write count and waits for + * either to change. If there's nothing to write or the write returned + * an error then only the write count advances and sets the appropriate + * return code. + */ +int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct write_attempt attempt; + int ret; + + if (!wait) { + schedule_work(&sbi->trans_write_work); + return 0; + } + + spin_lock(&sbi->trans_write_lock); + attempt.seq = le64_to_cpu(sbi->super.hdr.seq); + attempt.count = sbi->trans_write_count; + spin_unlock(&sbi->trans_write_lock); + + schedule_work(&sbi->trans_write_work); + + ret = wait_event_interruptible(sbi->trans_write_wq, + write_attempted(sbi, &attempt)); + if (ret == 0) + ret = attempt.ret; + + return ret; +} + +int scoutfs_hold_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return wait_event_interruptible(sbi->trans_hold_wq, + atomic_add_unless(&sbi->trans_holds, 1, -1)); +} + +void scoutfs_release_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (atomic_sub_return(1, &sbi->trans_holds) == 0) + wake_up(&sbi->trans_hold_wq); +} + +int scoutfs_setup_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", 0, 1); + if (!sbi->trans_write_workq) + return -ENOMEM; + + return 0; +} + +/* + * kill_sb calls sync before getting here so we know that dirty data + * should be in flight. We just have to wait for it to quiesce. + */ +void scoutfs_shutdown_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (sbi->trans_write_workq) { + flush_work(&sbi->trans_write_work); + destroy_workqueue(sbi->trans_write_workq); + } +} diff --git a/kmod/src/trans.h b/kmod/src/trans.h new file mode 100644 index 00000000..9cd0753a --- /dev/null +++ b/kmod/src/trans.h @@ -0,0 +1,13 @@ +#ifndef _SCOUTFS_TRANS_H_ +#define _SCOUTFS_TRANS_H_ + +void scoutfs_trans_write_func(struct work_struct *work); +int scoutfs_sync_fs(struct super_block *sb, int wait); + +int scoutfs_hold_trans(struct super_block *sb); +void scoutfs_release_trans(struct super_block *sb); + +int scoutfs_setup_trans(struct super_block *sb); +void scoutfs_shutdown_trans(struct super_block *sb); + +#endif From 979a36e175fffb5060803d196e51c686e077a071 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 30 Apr 2016 12:20:18 -0700 Subject: [PATCH 049/920] scoutfs: add buddy block allocator Add the block allocator. Logically we use a buddy allocator that's built from bitmaps for allocators of each order up to the largest allocator that fits in the device. This ends up using two bits per block. On disk we log modified regions of these bitmaps in chunks in blocks in a preallocated ring. We carefully coordinate logging the chunks and the ring size so that we can always write to the tail of the ring. There's one allocator and it's only read on mount today. We'll eventually have multiple of these allocators covering the device and nodes will coordinate exclusive access to them. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/block.c | 53 +++- kmod/src/block.h | 1 + kmod/src/buddy.c | 670 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/buddy.h | 11 + kmod/src/format.h | 38 +++ kmod/src/super.c | 6 +- kmod/src/super.h | 6 +- kmod/src/trans.c | 8 +- 9 files changed, 778 insertions(+), 19 deletions(-) create mode 100644 kmod/src/buddy.c create mode 100644 kmod/src/buddy.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 95f35039..6749f25b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o btree.o counters.o crc.o dir.o filerw.o inode.o msg.o \ - scoutfs_trace.o super.o trans.o treap.o +scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ + inode.o msg.o scoutfs_trace.o super.o trans.o treap.o diff --git a/kmod/src/block.c b/kmod/src/block.c index efaff13c..70ec7a57 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -21,6 +21,7 @@ #include "block.h" #include "crc.h" #include "counters.h" +#include "buddy.h" #define DIRTY_RADIX_TAG 0 @@ -428,20 +429,19 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, unsigned long flags; u64 blkno; int ret; + int err; bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) return bl; - ret = radix_tree_preload(GFP_NOFS); - if (ret) { - scoutfs_put_block(bl); - return ERR_PTR(ret); - } + ret = scoutfs_buddy_alloc(sb, &blkno, 0); + if (ret < 0) + goto out; - /* XXX cheesy */ - blkno = atomic64_inc_return(&sbi->next_blkno); - hdr = bl->data; + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; spin_lock_irqsave(&sbi->block_lock, flags); @@ -453,8 +453,10 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, } bl->blkno = blkno; + hdr = bl->data; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; + radix_tree_insert(&sbi->block_radix, blkno, bl); radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); atomic_inc(&bl->refcount); @@ -464,6 +466,16 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, ref->blkno = hdr->blkno; ref->seq = hdr->seq; + ret = 0; +out: + if (ret) { + if (blkno) { + err = scoutfs_buddy_free(sb, blkno, 0); + WARN_ON_ONCE(err); /* XXX hmm */ + } + scoutfs_put_block(bl); + bl = ERR_PTR(ret); + } return bl; } @@ -530,13 +542,21 @@ out: */ struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *bl; u64 blkno; + int ret; + int err; - /* XXX cheesy */ - blkno = atomic64_inc_return(&sbi->next_blkno); + ret = scoutfs_buddy_alloc(sb, &blkno, 0); + if (ret < 0) + return ERR_PTR(ret); - return scoutfs_new_block(sb, blkno); + bl = scoutfs_new_block(sb, blkno); + if (IS_ERR(bl)) { + err = scoutfs_buddy_free(sb, blkno, 0); + WARN_ON_ONCE(err); /* XXX hmm */ + } + return bl; } void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) @@ -545,3 +565,12 @@ void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } + +void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off) +{ + if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) + return; + + if (off < SCOUTFS_BLOCK_SIZE) + memset(bl->data + off, 0, SCOUTFS_BLOCK_SIZE - off); +} diff --git a/kmod/src/block.h b/kmod/src/block.h index 724f4aca..91f55f81 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -37,5 +37,6 @@ int scoutfs_write_dirty_blocks(struct super_block *sb); void scoutfs_put_block(struct scoutfs_block *bl); void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); +void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off); #endif diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c new file mode 100644 index 00000000..47d89504 --- /dev/null +++ b/kmod/src/buddy.c @@ -0,0 +1,670 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "super.h" +#include "format.h" +#include "block.h" +#include "buddy.h" +#include "msg.h" + +/* + * scoutfs uses buddy bitmaps to allocate block regions. It has a nice + * and simple implementation and reasonably small storage and memory + * overhead, particularly in the pathological fragmented case, but + * results in more rigid allocation constraints and fragmentation. + * + * The buddy allocator is build from a hierarchy of bitmaps for each + * power of two order of blocks that we can allocate. If a high order + * buddy bit is set then all the lower order bits that it covers are + * clear. + * + * At runtime all the bitmaps for all the orders are stored in a single + * packed bitmap in memory. We construct an array of pointers into the + * big bitmap for each individual order bitmap. This lets us easily + * track modifications of all the order bitmaps with a second bitmap + * which tracks fixed size chunks of the main bitmap. + * + * As a transaction is written the modified chunks of the main bitmap + * are written to the tail of a preallocated ring of buddy blocks. + * This turns noisy scattered bit modification operations into one large + * contiguous block IO. + * + * We always write to the tail of the ring so we need to ensure that the + * blocks at the tail don't contain live data. As we mark each chunk of + * the bitmap modified during a transaction we also sweep through the + * bitmap finding another chunk that has never been modified by the + * current sweep. Eventually enough chunks are modified by transactions + * to advance the sweep through the whole bitmap. At this point we're + * sure that all the blocks written to the tail during the sweep have to + * contain the full bitmap. By sizing the ring to 4x the bitmap size we + * ensure that we'll finish the sweep in each half, ensuring that the + * tail is always far enough behind the head to not overwrite live + * chunks. + * + * The entire ring is read the first time the allocator is needed. + * Today that's on mount for the entire system. As we layer on + * functionality we'll have multiple allocators and they'll be passed + * around the cluster as mounts are given access. As mounts get access + * they only need to read the newly written blocks in the ring to bring + * their stale allocator up to date with recent modifications written to + * the tail. The ring indices are full 64bits so that readers can + * recognize when they need to read the whole ring. + * + * The allocator only covers the blocks after the ring blocks to the end + * of the device. When we move to multiple allocators each will cover a + * fixed set of blocks excluding their ring blocks. Resizing will + * change the number of allocators needed to cover the device and will + * modify the bits in a final allocator. The bitmap modifications for + * resizing would be written to ring blocks as usual. Care will be + * taken to recognize device sizes whose final blocks land in the ring + * blocks. + */ + +struct buddy_alloc { + + /* + * addr: pointer to le64 that contains the start of the bitmap + * addr_bit: full bit nr of lsb at addr + * addr_off: bit offset from addr to first order bit + * addr_size: bit count from addr of the order's bits + * first_set: first logical order bit offset that might be set + */ + struct buddy_order { + __le64 *addr; + long addr_bit; + long addr_off; + long addr_size; + long first_set; + } orders[64]; + + int max_order; + + u64 orig_tail; + long *modified; + long modified_size; + + long reserved_chunks; + + __le64 *bitmap; +}; + +/* return the first device blkno covered by the allocator */ +static u64 first_blkno(struct scoutfs_super_block *super) +{ + return SCOUTFS_BUDDY_BLKNO + le32_to_cpu(super->buddy_blocks); +} + +/* return the number of blocks addressible by the allocator. */ +static u64 covered_blocks(struct scoutfs_super_block *super) +{ + return le64_to_cpu(super->total_blocks) - first_blkno(super); +} + +/* return the device block number of a ring index */ +static u64 ring_blkno(struct scoutfs_super_block *super, u64 index) +{ + return SCOUTFS_BUDDY_BLKNO + + do_div(index, le32_to_cpu(super->buddy_blocks)); +} + +/* + * Find and mark the next chunk in the bitmap that has never been + * written to the current half of the block ring. + * + * If we finish the sweep through the bitmap then we know that the most + * current half of the ring contain the full bitmap and reading at the + * head no longer has to start from the previous half. + */ +static bool modify_sweep_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud) +{ + bool did_set; + long bit; + + bit = le32_to_cpu(super->buddy_sweep_bit); + if (bit >= bud->modified_size) + return false; + + bit = find_next_zero_bit(bud->modified, bud->modified_size, bit); + if (bit < bud->modified_size) { + set_bit(bit, bud->modified); + bud->reserved_chunks--; + bit++; + did_set = true; + } else { + bit = bud->modified_size; + did_set = false; + } + + super->buddy_sweep_bit = cpu_to_le32(bit); + + /* advance head once we finish the sweep */ + if (bit == bud->modified_size) { + u64 head = le64_to_cpu(super->buddy_head); + u64 tail = le64_to_cpu(super->buddy_tail); + u32 half = le32_to_cpu(super->buddy_blocks) / 2; + + if ((tail - head) > half) + le64_add_cpu(&super->buddy_head, half); + } + + return did_set; +} + +/* + * The caller has modified the given bit in the full buddy bitmap. We + * try to mark its chunk modified and advance the sweep through older + * chunks. + */ +static void modified_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + bit = (ord->addr_bit + ord->addr_off + bit) / SCOUTFS_BUDDY_CHUNK_BITS; + + if (!test_and_set_bit(bit, bud->modified)) { + bud->reserved_chunks--; + modify_sweep_bit(super, bud); + } +} + +static int test_buddy_bit(struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + return !!test_bit_le(ord->addr_off + bit, ord->addr); +} + +static void set_buddy_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + set_bit_le(ord->addr_off + bit, ord->addr); + ord->first_set = min(bit, ord->first_set); + + modified_bit(super, bud, order, bit); +} + +static void clear_buddy_bit(struct scoutfs_super_block *super, + struct buddy_alloc *bud, int order, long bit) +{ + struct buddy_order *ord = &bud->orders[order]; + + clear_bit_le(ord->addr_off + bit, ord->addr); + if (ord->first_set == bit) + ord->first_set++; + + modified_bit(super, bud, order, bit); +} + +/* returns LONG_MAX when there are no bits set */ +static long find_first_buddy_bit(struct buddy_alloc *bud, int order) +{ + struct buddy_order *ord = &bud->orders[order]; + long ret; + + ret = find_next_bit_le(ord->addr, ord->addr_size, + ord->addr_off + ord->first_set); + if (ret >= ord->addr_size) { + ret = LONG_MAX; + ord->first_set = ord->addr_size - ord->addr_off; + } else { + ret -= ord->addr_off; + ord->first_set = ret; + } + + return ret; +} + +/* test if the index is at the first block in either half of the ring */ +static bool start_of_half(struct scoutfs_super_block *super, u64 index) +{ + u32 half = le32_to_cpu(super->buddy_blocks) / 2; + + return do_div(index, half) == 0; +} + +/* + * A buddy operation can modify bits at every order in the worst case. + * (This is a bit overly conservative because high orders will + * eventually share a chunk.) We'll also try to mark old chunks + * modified for each new chunk we modify. + * + * Before we modify the buddy bits we pin dirty blocks to make sure that + * we have enough chunks to store the modified chunks. + * + * As we advance the tail to store new blocks we might wander into the + * next half of the ring. When that happens we reset the sweep bit so + * that we'll start migrating chunks into this new half of the ring. + * + * This is called with the buddy mutex held. It's the only thing that + * does blocking work under the mutex so we could be more clever and + * make the allocation fast path locking more efficient. + */ +static int reserve_block_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + struct scoutfs_block *bl; + u64 blkno; + + if (bud->reserved_chunks >= (bud->max_order * 2)) + return 0; + + blkno = ring_blkno(super, le64_to_cpu(super->buddy_tail)); + bl = scoutfs_new_block(sb, blkno); + if (IS_ERR(bl)) + return PTR_ERR(bl); + + scoutfs_put_block(bl); + bud->reserved_chunks += SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; + le64_add_cpu(&super->buddy_tail, 1); + if (start_of_half(super, le64_to_cpu(super->buddy_tail))) + super->buddy_sweep_bit = 0; + + return 0; +} + +/* + * Return the block number of an allocation of at least the requested + * order. If an allocation at the given order isn't free then first try + * to satisfy the allocation with a part of a larger order, then return + * a smaller allocation. + * + * The order of the allocation is returned. + */ +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + int found; + long bit; + int ret; + int i; + + if (WARN_ON_ONCE(order < 0 || order > bud->max_order)) + return -EINVAL; + + mutex_lock(&sbi->buddy_mutex); + + ret = reserve_block_chunks(sb); + if (ret) + goto out; + + /* search for larger and smaller orders */ + i = order; + while (i >= 0) { + bit = find_first_buddy_bit(bud, i); + if (bit < LONG_MAX) + break; + + if (i >= order && i < bud->max_order) + i++; + else if (i == bud->max_order) + i = order - 1; + else + i--; + } + if (i < 0) { + ret = -ENOSPC; + goto out; + } + found = i; + + /* we'll succeed from this point on, use bit before mangling it */ + *blkno = first_blkno(super) + ((u64)bit << found); + ret = min(found, order); + + /* always clear the found order */ + clear_buddy_bit(super, bud, found, bit); + + /* free right buddies if we're breaking up a larger order */ + for (bit <<= 1, i = found - 1; i >= order; i--, bit <<= 1) + set_buddy_bit(super, bud, i, bit | 1); + +out: + mutex_unlock(&sbi->buddy_mutex); + if (WARN_ON_ONCE(ret < 0)) + *blkno = 0; + return ret; +} + +/* + * Free the aligned allocation of the given order at the given blkno to + * the allocator. We merge it into adjoining free space by looking for + * free buddies as we increase the order. + */ +int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + long bit; + int ret; + int i; + + if (WARN_ON_ONCE(order < 0 || order > bud->max_order) || + WARN_ON_ONCE(((blkno + 1) << order) >= covered_blocks(super))) + return -EINVAL; + + mutex_lock(&sbi->buddy_mutex); + + ret = reserve_block_chunks(sb); + if (ret) + goto out; + + bit = (blkno - first_blkno(super)) >> order; + for (i = order; i <= bud->max_order; i++) { + + /* set bit free and finish when buddy isn't free */ + if (!test_buddy_bit(bud, i, bit ^ 1)) { + set_buddy_bit(super, bud, i, bit); + break; + } + + /* otherwise clear buddy and try to set higher parent */ + clear_buddy_bit(super, bud, i, bit ^ 1); + bit >>= 1; + } + +out: + mutex_unlock(&sbi->buddy_mutex); + return ret; +} + +/* + * We're writing a transaction. The buddy allocator records chunks of + * the main bitmap which have been modified during the transaction. We + * copy them to the pinned dirty blocks which will be written as part of + * the transaction. The bitmap of modified chunks and the old ring tail + * are only reset when the transaction is successfully written. + */ +int scoutfs_dirty_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + struct scoutfs_buddy_chunk *chunk; + struct scoutfs_buddy_block *bb; + struct scoutfs_block *bl; + long bit; + long ind; + u64 tail; + int i; + + /* short circuit a transaction with no modified chunks */ + if (bud->orig_tail == le64_to_cpu(super->buddy_tail)) + return 0; + + while (bud->reserved_chunks && modify_sweep_bit(super, bud)) + ; + + for (tail = bud->orig_tail, bit = 0; + tail < le64_to_cpu(super->buddy_tail) && bit < bud->modified_size; + tail++) { + + bl = scoutfs_read_block(sb, ring_blkno(super, tail)); + if (WARN_ON_ONCE(IS_ERR(bl))) + return PTR_ERR(bl); + + bb = bl->data; + bb->hdr.seq = cpu_to_le64(tail); + bb->nr_chunks = 0; + + for (i = 0; i < SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; i++) { + bit = find_next_bit(bud->modified, bud->modified_size, + bit); + if (bit >= bud->modified_size) + break; + + chunk = &bb->chunks[i]; + chunk->pos = cpu_to_le32(bit); + ind = bit * SCOUTFS_BUDDY_CHUNK_LE64S; + memcpy(chunk->bits, &bud->bitmap[ind], + SCOUTFS_BUDDY_CHUNK_BYTES); + bit++; + } + + bb->nr_chunks = i; + scoutfs_zero_block_tail(bl, offsetof(struct scoutfs_buddy_block, + chunks[bb->nr_chunks])); + scoutfs_put_block(bl); + } + + /* + * Chunk reservation should have ensured that there's always room + * in the tail blocks for the modified chunks. + */ + if (WARN_ON_ONCE(bit < bud->modified_size)) + return -EIO; + + return 0; +} + +void scoutfs_reset_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_alloc *bud = sbi->bud; + + bud->orig_tail = le64_to_cpu(super->buddy_tail); + memset(bud->modified, 0, DIV_ROUND_UP(bud->modified_size, 8)); +} + +static int check_buddy_fields(struct super_block *sb, + struct scoutfs_super_block *super) +{ + u32 blocks = le32_to_cpu(super->buddy_blocks); + u32 half = blocks / 2; + u64 head = le64_to_cpu(super->buddy_head); + u64 tail = le64_to_cpu(super->buddy_tail); + u64 buddy_bits; + u64 chunk_bits; + + /* have to at least have two halves */ + if (blocks < 2) { + scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); + return -EIO; + } + + /* + * insist that blocks be a multiple of two so that we don't have + * scary fencepost off by ones around the half calculations. + */ + if (blocks & 1) { + scoutfs_info(sb, "buddy_blocks %lu isn't even", blocks); + return -EIO; + } + + /* shouldn't fill the device with buddy blocks */ + if (first_blkno(super) >= le64_to_cpu(super->total_blocks)) { + scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); + return -EIO; + } + + /* can only reference a 32bit long's worth of buddy bits */ + buddy_bits = covered_blocks(super) * 2; + if (buddy_bits >= INT_MAX) { + scoutfs_info(sb, "device needs %llu > INT_MAX buddy bits", + buddy_bits); + return -EIO; + } + + /* need enough ring blocks for 4 full buddy copies */ + chunk_bits = blocks * SCOUTFS_BUDDY_CHUNKS_PER_BLOCK * + SCOUTFS_BUDDY_CHUNK_BITS; + if (buddy_bits * 4 > chunk_bits) { + scoutfs_info(sb, "only room for %llu bits in chunks, need %llu", + chunk_bits, buddy_bits * 4); + return -EIO; + } + + if (head > tail) { + scoutfs_info(sb, "buddy_head %llu > buddy_tail %llu", + head, tail); + return -EIO; + } + + /* tail can't wrap around into head */ + if ((tail - head) >= blocks) { + scoutfs_info(sb, "buddy_tail %llu overlaps buddy_head %llu", + tail, head); + return -EIO; + } + + /* head always has to start one of the halves */ + if (!start_of_half(super, head)) { + scoutfs_info(sb, "buddy_head %llu isn't multiple of half %u", + head, half); + return -EIO; + } + + return 0; +} + +/* + * Reconstruct the entire buddy bitmap by replaying the chunks that are + * contained in the buddy block ring. + * + * The allocator doesn't cover the super blocks and ring blocks and is + * initialized with all the device blocks marked free so that mkfs + * doesn't have to write any chunks to initialize free space. + * + * We go a little nuts with variables to make it easier to read. + */ +int scoutfs_read_buddy_chunks(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_buddy_chunk *chunk; + struct scoutfs_buddy_block *bb; + struct scoutfs_block *bl; + struct buddy_alloc *bud; + struct buddy_order *ord; + u64 buddy_bits; + u64 dev_blocks; + u64 chunks; + u64 head; + u64 tail; + long bits; + long bit; + long ind; + int ret; + int i; + + ret = check_buddy_fields(sb, super); + if (ret) + return ret; + + dev_blocks = covered_blocks(super); + buddy_bits = dev_blocks * 2; + chunks = DIV_ROUND_UP(buddy_bits, SCOUTFS_BUDDY_CHUNK_BITS); + + bud = kzalloc(sizeof(struct buddy_alloc), GFP_KERNEL); + if (bud) { + bud->bitmap = vzalloc(round_up(buddy_bits, 64) / 8); + bud->modified = vzalloc(round_up(chunks, BITS_PER_LONG) / 8); + } + if (!bud || !bud->bitmap || !bud->modified) { + ret = -ENOMEM; + goto out; + } + sbi->bud = bud; + + bud->modified_size = chunks; + + /* + * Updating first_set across the orders would be tricky so we + * initialize it to 0 and suffer an initial expensive find_first + * call. + */ + bit = 0; + bits = dev_blocks; + for (i = 0; i < ARRAY_SIZE(bud->orders); i++) { + ord = &bud->orders[i]; + + ord->addr = &bud->bitmap[bit / 64]; + ord->addr_bit = bit & ~63ULL; + ord->addr_off = bit & 63; + ord->addr_size = ord->addr_off + bits; + ord->first_set = 0; + + bit += bits; + bits >>= 1; + if (!bits) + break; + } + bud->max_order = i; + + /* + * Initialize the allocator with the all the blocks covered by + * the fewest number of greatest order free allocations. Ring + * replay will overwrite this. + */ + bit = 0; + for (i = bud->max_order; i >= 0; i--) { + ord = &bud->orders[i]; + + if (ord->addr_off + bit == ord->addr_size) + break; + + set_bit_le(ord->addr_off + bit, ord->addr); + bit = (bit + 1) << 1; + } + + head = le64_to_cpu(super->buddy_head); + tail = le64_to_cpu(super->buddy_tail); + while (head < tail) { + bl = scoutfs_read_block(sb, ring_blkno(super, head)); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + + bb = bl->data; + if (le64_to_cpu(bb->hdr.seq) != head) { + /* XXX corruption */ + ret = -EIO; + scoutfs_put_block(bl); + goto out; + } + + for (i = 0; i < bb->nr_chunks; i++) { + chunk = &bb->chunks[i]; + + /* XXX check */ + ind = le32_to_cpu(chunk->pos) * + SCOUTFS_BUDDY_CHUNK_LE64S; + + memcpy(&bud->bitmap[ind], chunk->bits, + SCOUTFS_BUDDY_CHUNK_BYTES); + } + scoutfs_put_block(bl); + head++; + } + ret = 0; +out: + if (ret) { + if (bud) { + vfree(bud->bitmap); + vfree(bud->modified); + } + } + return ret; +} diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h new file mode 100644 index 00000000..8a411178 --- /dev/null +++ b/kmod/src/buddy.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_BUDDY_H_ +#define _SCOUTFS_BUDDY_H_ + +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); +int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); + +int scoutfs_read_buddy_chunks(struct super_block *sb); +void scoutfs_reset_buddy_chunks(struct super_block *sb); +int scoutfs_dirty_buddy_chunks(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index a80e95b0..6f80059e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -19,6 +19,7 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 +#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) /* * This header is found at the start of every block so that we can @@ -102,6 +103,38 @@ struct scoutfs_btree_item { #define SCOUTFS_UUID_BYTES 16 +/* + * Arbitrarily choose a reasonably fine grained 64byte chunk. This is a + * balance between write amplification of writing chunks with a single + * modified bit, storage overhead of partial blocks losing a chunk to + * make room for the block header and having a pos field per chunk, and + * runtime memory overhead of a bit per chunk. + */ +#define SCOUTFS_BUDDY_CHUNK_LE64S 8 +#define SCOUTFS_BUDDY_CHUNK_BYTES (SCOUTFS_BUDDY_CHUNK_LE64S * 8) +#define SCOUTFS_BUDDY_CHUNK_BITS (SCOUTFS_BUDDY_CHUNK_BYTES * 8) + +/* + * After the pair of super blocks are a preallocated ring of blocks + * which record modified regions of the buddy bitmap allocator. + * + * The seq's header needs to match the unwrapped ring index of the + * block. + */ +struct scoutfs_buddy_block { + struct scoutfs_block_header hdr; + u8 nr_chunks; + struct scoutfs_buddy_chunk { + __le32 pos; + __le64 bits[SCOUTFS_BUDDY_CHUNK_LE64S]; + } __packed chunks[0]; +} __packed; + +#define SCOUTFS_BUDDY_CHUNKS_PER_BLOCK \ + ((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_buddy_block, chunks)) /\ + SCOUTFS_BUDDY_CHUNK_BYTES) + + /* * The super is stored in a pair of blocks in the first chunk on the * device. @@ -116,6 +149,11 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 total_blocks; + __le32 buddy_blocks; + __le32 buddy_sweep_bit; + __le64 buddy_head; + __le64 buddy_tail; struct scoutfs_btree_root btree_root; } __packed; diff --git a/kmod/src/super.c b/kmod/src/super.c index b79c6b8f..e8354d5e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -126,7 +126,6 @@ static int read_supers(struct super_block *sb) * XXX These don't exist in the super yet. They should soon. */ atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); - atomic64_set(&sbi->next_blkno, 6); return 0; } @@ -151,6 +150,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); + mutex_init(&sbi->buddy_mutex); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); @@ -165,11 +165,13 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: - scoutfs_setup_trans(sb); + scoutfs_setup_trans(sb) ?: + scoutfs_read_buddy_chunks(sb); if (ret) return ret; scoutfs_advance_dirty_super(sb); + scoutfs_reset_buddy_chunks(sb); inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) diff --git a/kmod/src/super.h b/kmod/src/super.h index df2dd30f..18c9be86 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,8 +5,10 @@ #include #include "format.h" +#include "buddy.h" struct scoutfs_counters; +struct buddy_alloc; struct scoutfs_sb_info { struct super_block *sb; @@ -20,7 +22,9 @@ struct scoutfs_sb_info { int block_write_err; atomic64_t next_ino; - atomic64_t next_blkno; + + struct mutex buddy_mutex; + struct buddy_alloc *bud; /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 7194e9af..4dcfdb99 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -19,6 +19,7 @@ #include "super.h" #include "block.h" #include "trans.h" +#include "buddy.h" #include "scoutfs_trace.h" /* @@ -63,7 +64,8 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX probably want to write out dirty pages in inodes */ if (scoutfs_has_dirty_blocks(sb)) { - ret = scoutfs_write_dirty_blocks(sb) ?: + ret = scoutfs_dirty_buddy_chunks(sb) ?: + scoutfs_write_dirty_blocks(sb) ?: scoutfs_write_dirty_super(sb); if (!ret) advance = 1; @@ -71,8 +73,10 @@ void scoutfs_trans_write_func(struct work_struct *work) spin_lock(&sbi->trans_write_lock); - if (advance) + if (advance) { scoutfs_advance_dirty_super(sb); + scoutfs_reset_buddy_chunks(sb); + } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock); From e0f38231b36b45847c0acfd479a6cea079179909 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 1 May 2016 09:16:40 -0700 Subject: [PATCH 050/920] scoutfs: store next allocated inode in super The next inode number to be allocated has been stored only in the in-memory super block and hasn't survived across mounts. This sometimes accidentally worked if the tests removed the initial inodes but often would cause failures when inode allocation returned existing inodes. This tracks the next inode to allocate in the super block and maintains it across mounts. Tests now consistently pass as inode allocations consistently return free inode numbers. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/inode.c | 31 ++++++++++++++++++++++++++++--- kmod/src/super.c | 6 +----- kmod/src/super.h | 4 ++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 6f80059e..e7bbcb59 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -149,6 +149,7 @@ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; + __le64 next_ino; __le64 total_blocks; __le32 buddy_blocks; __le32 buddy_sweep_bit; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 8ce039b3..1b37381f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -249,6 +249,27 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } +static int alloc_ino(struct super_block *sb, u64 *ino) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + int ret; + + spin_lock(&sbi->next_ino_lock); + + if (super->next_ino == 0) { + ret = -ENOSPC; + } else { + *ino = le64_to_cpu(super->next_ino); + le64_add_cpu(&super->next_ino, 1); + ret = 0; + } + + spin_unlock(&sbi->next_ino_lock); + + return ret; +} + /* * Allocate and initialize a new inode. The caller is responsible for * creating links to it and updating it. @dir can be null. @@ -256,22 +277,26 @@ void scoutfs_update_inode_item(struct inode *inode) struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; struct scoutfs_key key; struct inode *inode; + u64 ino; int ret; + ret = alloc_ino(sb, &ino); + if (ret) + return ERR_PTR(ret); + inode = new_inode(sb); if (!inode) return ERR_PTR(-ENOMEM); ci = SCOUTFS_I(inode); - ci->ino = atomic64_inc_return(&sbi->next_ino); + ci->ino = ino; get_random_bytes(&ci->salt, sizeof(ci->salt)); - inode->i_ino = ci->ino; + inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); inode_set_bytes(inode, 0); inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; diff --git a/kmod/src/super.c b/kmod/src/super.c index e8354d5e..1867d1f8 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -122,11 +122,6 @@ static int read_supers(struct super_block *sb) scoutfs_info(sb, "using super %u with seq %llu", found, le64_to_cpu(sbi->super.hdr.seq)); - /* - * XXX These don't exist in the super yet. They should soon. - */ - atomic64_set(&sbi->next_ino, SCOUTFS_ROOT_INO + 1); - return 0; } @@ -146,6 +141,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; + spin_lock_init(&sbi->next_ino_lock); spin_lock_init(&sbi->block_lock); INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); init_waitqueue_head(&sbi->block_wq); diff --git a/kmod/src/super.h b/kmod/src/super.h index 18c9be86..03226baf 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -15,14 +15,14 @@ struct scoutfs_sb_info { struct scoutfs_super_block super; + spinlock_t next_ino_lock; + spinlock_t block_lock; struct radix_tree_root block_radix; wait_queue_head_t block_wq; atomic_t block_writes; int block_write_err; - atomic64_t next_ino; - struct mutex buddy_mutex; struct buddy_alloc *bud; From 4163236fc129cf60cd9abc8c03a505fa979de12c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 May 2016 21:55:39 -0700 Subject: [PATCH 051/920] scoutfs: dirent hashes use linear probing The current mechanism for dealing with dirent name hash collisions is to use multiple hash functions. This won't work great with the btree where it's expensive to search multiple distant items for a given entry. Instead of having multiple full precision functions we linearly probe a given number of hash values after the initial name hash. Now the slow colliding path walks adjacent items in the tree instead of bouncing around the tree. Signed-off-by: Zach Brown --- kmod/src/dir.c | 129 ++++++++++++++++++++++------------------------ kmod/src/format.h | 18 ++++--- kmod/src/inode.c | 2 - kmod/src/inode.h | 1 - 4 files changed, 74 insertions(+), 76 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 73c8e938..730cc7b2 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -28,29 +28,24 @@ * Directory entries are stored in entries with offsets calculated from * the hash of their entry name. * - * The upside of having a single namespace of items used for both lookup - * and readdir iteration reduces the storage overhead of directories. - * The downside is that dirent operations produce random item access - * patterns. + * Having a single index of items used for both lookup and readdir + * iteration reduces the storage overhead of directories. It also + * avoids having to manage the allocation of readdir positions as + * directories age and the aggregate create count inches towards the + * small 31 bit position limit. The downside is that dirent name + * operations produce random item access patterns. * - * Hash values are limited to 31 bits to avoid bugs from use of 31 bit - * signed offsets. We also avoid bugs in network protocols limited to - * 32 bit directory positions. + * Hash values are limited to 31 bits primarily to support older + * deployed protocols that only support 31 bits of file entry offsets, + * but also to avoid unlikely bugs in programs that store offsets in + * signed ints. * - * We have to worry about collisions because we're using the hash of the - * name. We simply allow a name to be stored at multiple hash value - * locations. Create iterates until it finds an unused value and lookup - * iterates until it finds an entry at a hash that matches the name. We - * can store the max iteration used during create in the directory to - * limit the number of values we'll check in lookup. With 31bit hash - * values we can get tens of thousands of entries before we use two - * hashes, hundreds for three, millions for four, and so on. The vast - * majority of directories will use one hash value. - * - * This would be a crazy design in systems where dirent lookups perform - * dependent block reads down a radix or btree structure for each hash - * value. scoutfs makes this a lot cheaper by using the bloom filters - * in the log segments to short circuit negative item lookups. + * We have to worry about hash collisions. We linearly probe a fixed + * number of hash values past the natural value. In a typical small + * directory this search will terminate immediately because adjacent + * items will have distant offset values. It's only as the directory + * gets very large that hash values will start to be this dense and + * sweeping over items in a btree leaf is reasonably efficient. */ static unsigned int mode_to_type(umode_t mode) @@ -96,10 +91,6 @@ static int names_equal(const char *name_a, int len_a, const char *name_b, } /* - * Return the offset portion of a dirent key from the hash of the name. - * The hash can't be 0 or 1 for . and .. and we chose to limit the max - * file->f_pos. - * * XXX This crc nonsense is a quick hack. We'll want something a * lot stronger like siphash. */ @@ -169,6 +160,12 @@ static struct dentry_info *alloc_dentry_info(struct dentry *dentry) return dentry->d_fsdata; } +static u64 last_dirent_key_offset(u32 h) +{ + return min_t(u64, (u64)h + SCOUTFS_DIRENT_COLL_NR - 1, + SCOUTFS_DIRENT_LAST_POS); +} + /* * Lookup searches for an entry for the given name amongst the entries * stored in the item at the name's hash. @@ -181,18 +178,13 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, struct super_block *sb = dir->i_sb; struct scoutfs_dirent *dent; struct dentry_info *di; - struct scoutfs_key key; + struct scoutfs_key first; + struct scoutfs_key last; unsigned int name_len; struct inode *inode; u64 ino = 0; u32 h = 0; int ret; - int i; - - if (si->max_dirent_hash_nr == 0) { - ret = -ENOENT; - goto out; - } di = alloc_dentry_info(dentry); if (IS_ERR(di)) { @@ -205,40 +197,35 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } - h = si->salt; - for (i = 0; i < si->max_dirent_hash_nr; i++) { - h = name_hash(dentry->d_name.name, dentry->d_name.len, h); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); - scoutfs_btree_release(&curs); - ret = scoutfs_btree_lookup(sb, &key, &curs); - if (ret == -ENOENT) - continue; - if (ret < 0) - break; + scoutfs_set_key(&first, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, + last_dirent_key_offset(h)); + + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + + /* XXX verify */ dent = curs.val; name_len = item_name_len(&curs); if (names_equal(dentry->d_name.name, dentry->d_name.len, dent->name, name_len)) { ino = le64_to_cpu(dent->ino); - ret = 0; + di->hash = scoutfs_key_offset(curs.key); break; - } else { - ret = -ENOENT; } } scoutfs_btree_release(&curs); + out: - if (ret == -ENOENT) { - inode = NULL; - } else if (ret) { + if (ret < 0) inode = ERR_PTR(ret); - } else { - di->hash = h; + else if (ino == 0) + inode = NULL; + else inode = scoutfs_iget(sb, ino); - } return d_splice_alias(inode, dentry); } @@ -266,8 +253,8 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) } /* - * readdir finds the next entry at or past the hash|coll_nr stored in - * the current file position. + * readdir simply iterates over the dirent items for the dir inode and + * uses their offset as the readdir position. * * It will need to be careful not to read past the region of the dirent * hash offset keys that it has access to. @@ -320,11 +307,12 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct inode *inode = NULL; struct scoutfs_dirent *dent; struct dentry_info *di; + struct scoutfs_key first; + struct scoutfs_key last; struct scoutfs_key key; int bytes; int ret; u64 h; - int i; di = alloc_dentry_info(dentry); if (IS_ERR(di)) @@ -348,33 +336,40 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, } bytes = dent_bytes(dentry->d_name.len); + h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); + scoutfs_set_key(&first, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, + last_dirent_key_offset(h)); - h = si->salt; - for (i = 0; i < SCOUTFS_MAX_DENT_HASH_NR; i++) { - h = name_hash(dentry->d_name.name, dentry->d_name.len, h); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - - ret = scoutfs_btree_insert(sb, &key, bytes, &curs); - if (ret != -EEXIST) - break; + /* find the first unoccupied key offset after the hashed name */ + key = first; + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + key = *curs.key; + scoutfs_inc_key(&key); } - if (ret) { - if (ret == -EEXIST) - ret = -ENOSPC; + scoutfs_btree_release(&curs); + if (ret < 0) + goto out; + + if (scoutfs_key_cmp(&key, &last) > 0) { + ret = -ENOSPC; goto out; } + ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + if (ret) + goto out; + dent = curs.val; dent->ino = cpu_to_le64(scoutfs_ino(inode)); dent->type = mode_to_type(inode->i_mode); memcpy(dent->name, dentry->d_name.name, dentry->d_name.len); - di->hash = h; + di->hash = scoutfs_key_offset(&key); scoutfs_btree_release(&curs); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; - si->max_dirent_hash_nr = max_t(int, i + 1, si->max_dirent_hash_nr); inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; if (S_ISDIR(mode)) { diff --git a/kmod/src/format.h b/kmod/src/format.h index e7bbcb59..5deca747 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -182,7 +182,6 @@ struct scoutfs_inode { __le32 mode; __le32 rdev; __le32 salt; - __u8 max_dirent_hash_nr; struct scoutfs_timespec atime; struct scoutfs_timespec ctime; struct scoutfs_timespec mtime; @@ -201,12 +200,19 @@ struct scoutfs_dirent { } __packed; /* - * The max number of dirent hash values determines the overhead of - * lookups in very large directories. With 31bit offsets the number - * of entries stored before enospc tends to plateau around 200 million - * entries around 8 functions. That seems OK for now. + * Dirent items are stored at keys with the offset set to the hash of + * the name. Creation can find that hash values collide and will + * attempt to linearly probe this many following hash values looking for + * an unused value. + * + * In small directories this doesn't really matter because hash values + * will so very rarely collide. At around 50k items we start to see our + * first collisions. 16 slots is still pretty quick to scan in the + * btree and it gets us up into the hundreds of millions of entries + * before enospc is returned as we run out of hash values. */ -#define SCOUTFS_MAX_DENT_HASH_NR 8 +#define SCOUTFS_DIRENT_COLL_NR 16 + #define SCOUTFS_NAME_LEN 255 /* diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 1b37381f..0543c991 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -107,7 +107,6 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); ci->salt = le32_to_cpu(cinode->salt); - ci->max_dirent_hash_nr = cinode->max_dirent_hash_nr; } static int scoutfs_read_locked_inode(struct inode *inode) @@ -189,7 +188,6 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); cinode->salt = cpu_to_le32(ci->salt); - cinode->max_dirent_hash_nr = ci->max_dirent_hash_nr; } /* diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 650cd05b..f2846201 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -4,7 +4,6 @@ struct scoutfs_inode_info { u64 ino; u32 salt; - u8 max_dirent_hash_nr; struct inode inode; }; From 0820a7b5bd77cc073fee60104f3f94e883a297d0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 23 May 2016 17:25:06 -0700 Subject: [PATCH 052/920] scoutfs: introduce write locking Introduce the concept of acquiring write locks around write operations. The core idea is that reads are unlocked and that write lock contention between nodes should be rare. This first pass simply broadcasts write lock requests to all the mounts in the volume. It achieves a reasonable degree of fairness and doesn't require centralizing state in a lock server. We have to flesh out a bit of initial infrastructure to support the write locking protocol. The roster manages cluster membership and messaging and only understands mounts in the same kernel for now. Creation needs to know which inodes to try and lock so we see the start of per-mount free inode reservations. The transformation of users is straight forward: they aquire the write lock on the inodes they're working with instead of holding a transaction. The write lock machinery now manages transactions. This passes single mount testing but that isn't saying much. The next step is to run multi-mount tests. Signed-off-by: Zach Brown --- kmod/src/Makefile | 3 +- kmod/src/dir.c | 20 +- kmod/src/filerw.c | 12 +- kmod/src/format.h | 2 + kmod/src/inode.c | 72 +++- kmod/src/inode.h | 4 +- kmod/src/roster.c | 159 ++++++++ kmod/src/roster.h | 14 + kmod/src/super.c | 7 + kmod/src/super.h | 8 + kmod/src/wire.h | 36 ++ kmod/src/wrlock.c | 964 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/wrlock.h | 30 ++ 13 files changed, 1303 insertions(+), 28 deletions(-) create mode 100644 kmod/src/roster.c create mode 100644 kmod/src/roster.h create mode 100644 kmod/src/wire.h create mode 100644 kmod/src/wrlock.c create mode 100644 kmod/src/wrlock.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 6749f25b..c60bf9f9 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,4 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o msg.o scoutfs_trace.o super.o trans.o treap.o + inode.o msg.o roster.o scoutfs_trace.o super.o trans.o treap.o \ + wrlock.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 730cc7b2..c275d7f7 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -22,7 +22,7 @@ #include "key.h" #include "super.h" #include "btree.h" -#include "trans.h" +#include "wrlock.h" /* * Directory entries are stored in entries with offsets calculated from @@ -310,7 +310,9 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; + DECLARE_SCOUTFS_WRLOCK_HELD(held); int bytes; + u64 ino; int ret; u64 h; @@ -321,7 +323,11 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_hold_trans(sb); + ret = scoutfs_alloc_ino(sb, &ino); + if (ret) + return ret; + + ret = scoutfs_wrlock_lock(sb, &held, 2, scoutfs_ino(dir), ino); if (ret) return ret; @@ -329,7 +335,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out; - inode = scoutfs_new_inode(sb, dir, mode, rdev); + inode = scoutfs_new_inode(sb, dir, ino, mode, rdev); if (IS_ERR(inode)) { ret = PTR_ERR(inode); goto out; @@ -386,7 +392,7 @@ out: /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); - scoutfs_release_trans(sb); + scoutfs_wrlock_unlock(sb, &held); return ret; } @@ -411,6 +417,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); + DECLARE_SCOUTFS_WRLOCK_HELD(held); struct dentry_info *di; struct scoutfs_key key; int ret = 0; @@ -422,7 +429,8 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - ret = scoutfs_hold_trans(sb); + ret = scoutfs_wrlock_lock(sb, &held, 2, scoutfs_ino(dir), + scoutfs_ino(inode)); if (ret) return ret; @@ -451,7 +459,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(dir); out: - scoutfs_release_trans(sb); + scoutfs_wrlock_unlock(sb, &held); return ret; } diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index a6e75fc7..2082d17e 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -18,7 +18,7 @@ #include "inode.h" #include "key.h" #include "filerw.h" -#include "trans.h" +#include "wrlock.h" #include "scoutfs_trace.h" #include "btree.h" @@ -130,6 +130,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) { struct inode *inode = page->mapping->host; DECLARE_SCOUTFS_BTREE_CURSOR(curs); + DECLARE_SCOUTFS_WRLOCK_HELD(held); struct super_block *sb = inode->i_sb; struct scoutfs_key key; struct data_region dr; @@ -139,7 +140,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) set_page_writeback(page); - ret = scoutfs_hold_trans(sb); + ret = scoutfs_wrlock_lock(sb, &held, 1, scoutfs_ino(inode)); if (ret) goto out; @@ -161,7 +162,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) } scoutfs_btree_release(&curs); - scoutfs_release_trans(sb); + scoutfs_wrlock_unlock(sb, &held); out: if (ret) { SetPageError(page); @@ -198,6 +199,7 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_WRLOCK_HELD(held); unsigned off; trace_scoutfs_write_end(scoutfs_ino(inode), pos, len, copied); @@ -218,10 +220,10 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, * up the robust metadata support that's needed to do a * good job with the data pats. */ - if (!scoutfs_hold_trans(sb)) { + if (!scoutfs_wrlock_lock(sb, &held, 1, scoutfs_ino(inode))) { if (!scoutfs_dirty_inode_item(inode)) scoutfs_update_inode_item(inode); - scoutfs_release_trans(sb); + scoutfs_wrlock_unlock(sb, &held); } } diff --git a/kmod/src/format.h b/kmod/src/format.h index 5deca747..5c8ecb13 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -159,6 +159,8 @@ struct scoutfs_super_block { } __packed; #define SCOUTFS_ROOT_INO 1 +#define SCOUTFS_INO_BATCH_SHIFT 20 +#define SCOUTFS_INO_BATCH (1 << SCOUTFS_INO_BATCH_SHIFT) struct scoutfs_timespec { __le64 sec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 0543c991..a9bc7010 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -22,6 +22,7 @@ #include "btree.h" #include "dir.h" #include "filerw.h" +#include "wrlock.h" #include "scoutfs_trace.h" /* @@ -247,24 +248,69 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } -static int alloc_ino(struct super_block *sb, u64 *ino) +/* + * This will need to try and find a mostly idle shard. For now we only + * have one :). + */ +static int get_next_ino_batch(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + DECLARE_SCOUTFS_WRLOCK_HELD(held); int ret; + ret = scoutfs_wrlock_lock(sb, &held, 1, 1); + if (ret) + return ret; + spin_lock(&sbi->next_ino_lock); - - if (super->next_ino == 0) { - ret = -ENOSPC; - } else { - *ino = le64_to_cpu(super->next_ino); - le64_add_cpu(&super->next_ino, 1); - ret = 0; + if (!sbi->next_ino_count) { + sbi->next_ino = le64_to_cpu(sbi->super.next_ino); + if (sbi->next_ino + SCOUTFS_INO_BATCH < sbi->next_ino) { + ret = -ENOSPC; + } else { + le64_add_cpu(&sbi->super.next_ino, SCOUTFS_INO_BATCH); + sbi->next_ino_count = SCOUTFS_INO_BATCH; + ret = 0; + } } - spin_unlock(&sbi->next_ino_lock); + scoutfs_wrlock_unlock(sb, &held); + + return ret; +} + +/* + * Inode allocation is at the core of supporting parallel creation. + * Each mount needs to allocate from a pool of free inode numbers which + * map to a shard that it has locked. + */ +int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret; + + do { + /* don't really care if this is racey */ + if (!sbi->next_ino_count) { + ret = get_next_ino_batch(sb); + if (ret) + break; + } + + spin_lock(&sbi->next_ino_lock); + + if (sbi->next_ino_count) { + *ino = sbi->next_ino++; + sbi->next_ino_count--; + ret = 0; + } else { + ret = -EAGAIN; + } + spin_unlock(&sbi->next_ino_lock); + + } while (ret == -EAGAIN); + return ret; } @@ -273,18 +319,14 @@ static int alloc_ino(struct super_block *sb, u64 *ino) * creating links to it and updating it. @dir can be null. */ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev) + u64 ino, umode_t mode, dev_t rdev) { DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; struct scoutfs_key key; struct inode *inode; - u64 ino; int ret; - ret = alloc_ino(sb, &ino); - if (ret) - return ERR_PTR(ret); inode = new_inode(sb); if (!inode) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f2846201..5dfc6b1a 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -18,6 +18,8 @@ static inline u64 scoutfs_ino(struct inode *inode) return SCOUTFS_I(inode)->ino; } +int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); + struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); @@ -25,7 +27,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev); + u64 ino, umode_t mode, dev_t rdev); void scoutfs_inode_exit(void); int scoutfs_inode_init(void); diff --git a/kmod/src/roster.c b/kmod/src/roster.c new file mode 100644 index 00000000..e05b7160 --- /dev/null +++ b/kmod/src/roster.c @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "super.h" +#include "wire.h" +#include "wrlock.h" +#include "roster.h" + +/* + * The roster tracks all the mounts on nodes that are working with a + * scoutfs volume. + * + * This trivial first pass lets us test multiple mounts on the same + * node. It'll get a lot more involved as all the nodes manage a roster + * in the shared device. + */ +static DEFINE_MUTEX(roster_mutex); +static u64 roster_next_id = 1; +static LIST_HEAD(roster_list); + +/* + * A new mount is adding itself to the roster. It gets a new increasing + * id assigned and all the other mounts are told that it's now a member. + */ +int scoutfs_roster_add(struct super_block *sb) +{ + struct scoutfs_sb_info *us = SCOUTFS_SB(sb); + struct scoutfs_sb_info *them; + + mutex_lock(&roster_mutex); + list_add_tail(&us->roster_head, &roster_list); + us->roster_id = roster_next_id++; + + list_for_each_entry(them, &roster_list, roster_head) { + if (us->roster_id != them->roster_id) { + scoutfs_wrlock_roster_update(them->sb, us->roster_id, + true); + } + } + + mutex_unlock(&roster_mutex); + + return 0; +} + +/* + * A mount is removing itself to the roster. All the other remaining + * mounts are told that it has gone away. + * + * This is safe to call without having called _add. + */ +void scoutfs_roster_remove(struct super_block *sb) +{ + struct scoutfs_sb_info *us = SCOUTFS_SB(sb); + struct scoutfs_sb_info *them; + + mutex_lock(&roster_mutex); + + if (!list_empty(&us->roster_head)) { + list_del_init(&us->roster_head); + + list_for_each_entry(them, &roster_list, roster_head) + scoutfs_wrlock_roster_update(them->sb, us->roster_id, + false); + } + + mutex_unlock(&roster_mutex); +} + +static int process_message(struct super_block *sb, u64 peer_id, + struct scoutfs_message *msg) +{ + int ret = 0; + + switch (msg->cmd) { + case SCOUTFS_MSG_WRLOCK_REQUEST: + ret = scoutfs_wrlock_process_request(sb, peer_id, + &msg->request); + break; + case SCOUTFS_MSG_WRLOCK_GRANT: + scoutfs_wrlock_process_grant(sb, &msg->grant); + ret = 0; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +/* + * Send a message to a specific member of the roster identified by its + * id. + * + * We don't actually send anything, we call directly into the receivers + * message processing path with the caller's message. + */ +void scoutfs_roster_send(struct super_block *sb, u64 peer_id, + struct scoutfs_message *msg) +{ + struct scoutfs_sb_info *us = SCOUTFS_SB(sb); + struct scoutfs_sb_info *them; + int ret; + + mutex_lock(&roster_mutex); + + list_for_each_entry(them, &roster_list, roster_head) { + if (them->roster_id == peer_id) { + ret = process_message(them->sb, us->roster_id, msg); + break; + } + } + + /* XXX errors? */ + + mutex_unlock(&roster_mutex); +} + +/* + * Send a message to all of the current members which have an id greater + * than the caller's specified id. + * + * We don't actually send anything, we call directly into the receivers + * message processing path with the caller's message. + */ +void scoutfs_roster_broadcast(struct super_block *sb, u64 since_id, + struct scoutfs_message *msg) +{ + struct scoutfs_sb_info *us = SCOUTFS_SB(sb); + struct scoutfs_sb_info *them; + int ret; + + mutex_lock(&roster_mutex); + + list_for_each_entry(them, &roster_list, roster_head) { + if (us->roster_id != them->roster_id && + them->roster_id > since_id) { + ret = process_message(them->sb, us->roster_id, msg); + if (ret) + break; + } + } + + /* XXX errors? */ + + mutex_unlock(&roster_mutex); +} diff --git a/kmod/src/roster.h b/kmod/src/roster.h new file mode 100644 index 00000000..e78a7cc9 --- /dev/null +++ b/kmod/src/roster.h @@ -0,0 +1,14 @@ +#ifndef _SCOUTFS_ROSTER_H_ +#define _SCOUTFS_ROSTER_H_ + +struct scoutfs_message; + +int scoutfs_roster_add(struct super_block *sb); +void scoutfs_roster_remove(struct super_block *sb); + +void scoutfs_roster_send(struct super_block *sb, u64 peer_id, + struct scoutfs_message *msg); +void scoutfs_roster_broadcast(struct super_block *sb, u64 since_id, + struct scoutfs_message *msg); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 1867d1f8..f6cab079 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -26,6 +26,8 @@ #include "block.h" #include "counters.h" #include "trans.h" +#include "roster.h" +#include "wrlock.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -153,6 +155,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); + INIT_LIST_HEAD(&sbi->roster_head); /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); @@ -162,6 +165,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: scoutfs_setup_trans(sb) ?: + scoutfs_wrlock_setup(sb) ?: + scoutfs_roster_add(sb) ?: scoutfs_read_buddy_chunks(sb); if (ret) return ret; @@ -192,6 +197,8 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_roster_remove(sb); + scoutfs_wrlock_teardown(sb); scoutfs_shutdown_trans(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index 03226baf..2d0392ac 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -9,6 +9,7 @@ struct scoutfs_counters; struct buddy_alloc; +struct wrlock_context; struct scoutfs_sb_info { struct super_block *sb; @@ -16,6 +17,8 @@ struct scoutfs_sb_info { struct scoutfs_super_block super; spinlock_t next_ino_lock; + u64 next_ino; + u64 next_ino_count; spinlock_t block_lock; struct radix_tree_root block_radix; @@ -43,6 +46,11 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; + + struct list_head roster_head; + u64 roster_id; + + struct wrlock_context *wrlock_context; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) diff --git a/kmod/src/wire.h b/kmod/src/wire.h new file mode 100644 index 00000000..474703bf --- /dev/null +++ b/kmod/src/wire.h @@ -0,0 +1,36 @@ +#ifndef _SCOUTFS_WIRE_H_ +#define _SCOUTFS_WIRE_H_ + +/* an arbitrarily small number to keep things reasonable */ +#define SCOUTFS_WRLOCK_MAX_SHARDS 5 + +enum { + SCOUTFS_MSG_WRLOCK_REQUEST = 1, + SCOUTFS_MSG_WRLOCK_GRANT = 2, +}; + +struct scoutfs_wrlock_id { + __le64 counter; + __le32 jitter; +} __packed; + +struct scoutfs_wrlock_request { + struct scoutfs_wrlock_id wid; + u8 nr_shards; + __le32 shards[SCOUTFS_WRLOCK_MAX_SHARDS]; +} __packed; + +struct scoutfs_wrlock_grant { + struct scoutfs_wrlock_id wid; +} __packed; + +struct scoutfs_message { + u8 cmd; + u8 len; + union { + struct scoutfs_wrlock_grant grant; + struct scoutfs_wrlock_request request; + } __packed; +} __packed; + +#endif diff --git a/kmod/src/wrlock.c b/kmod/src/wrlock.c new file mode 100644 index 00000000..62721923 --- /dev/null +++ b/kmod/src/wrlock.c @@ -0,0 +1,964 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "wire.h" +#include "wrlock.h" +#include "trans.h" +#include "roster.h" + +/* + * The persistent structures in each shard in a scoutfs volume can only + * have one writer at a time. Mounts send messages around to request + * and grant locks on each shard. (Reads are fully unlocked and have + * enough metadata to detect and retry reads that raced and were + * inconsistent.) + * + * When a local task needs to lock some shards it sends a request to all + * the other mounts listing all the shards. If the receiving mounts + * don't have any of the shards locked they send a grant reply. + * + * Each mount has a granted lock and a tree of blocked lock entries for + * every shard. Local lock attempts and remote requests are always + * inserted into the tree. The first entry in the tree can be unblocked + * if the granted lock in the shard doesn't block it. When local + * entries are granted the locking task is allowed to start modifying + * the shard. While they're modifying the shard their granted locks + * block remote locks from being sent replies. Once the writers under + * the lock are done the grant can be removed and the remote entry is + * sent a reply and freed. + * + * Processes can try to lock multiple shards so entries can be present + * in the blocking tree and granted pointer on multiple shards. They're + * only unblocked when they're the first entry in all their shards' + * blocking trees. + * + * The entries have to be very carefully ordered in the trees on all the + * mounts to avoid locking cycle deadlocks. We can't have two mounts + * race to lock the same shard and both have their local ungranted entry + * blocking the remote's request entry. To address this entries aren't + * sorted in the tree by time. They're sorted by an id. This ensures + * that all of the entries will have the same blocking tree on all the + * mounts and so will always be processed in the same order. + * + * Entries are created on a mount when a task tries to lock some shards. + * The id is constructed from a counter, a random number, and the + * mount's unique id. The counter is one greater than the greatest + * counter ever seen in received lock requests. This ensures that lock + * attempts that don't race are granted in order. But attempts can race + * so entries can have the same counter. Next they're sorted by a + * random number to ensure a kind of fairness. Then if the mounts are + * unlucky enough to chose the same number we fall back to sorting by + * the unique mount id. + * + * The roster determines the set of mounts that are participating in the + * locking protocol. We have to carefully manage the entries as mounts + * join and leave the cluster. When mounts join we send them all our + * blocking locks and if they leave we remove their entries and resend + * all our blocked entries to everyone because we don't track which + * mounts had send grants to which local blocked entries, or not. + * + * XXX + * - sync if we revoke a local grant before we send a reply + */ + +/* + * Every mount tracks their write locking state for all the shards in + * the volume. + */ +struct wrlock_context { + struct super_block *sb; + wait_queue_head_t waitq; + spinlock_t lock; + + struct rb_root id_root; + struct list_head mark_list; + struct list_head send_list; + struct workqueue_struct *send_workq; + struct work_struct send_work; + + /* private copies of roster state used under the lock */ + long grants_needed; + u64 last_peer_id; + + u64 next_id_counter; + + /* XXX redundant in the super? only one for now ;) */ + u32 nr_shards; + struct wrlock_context_shard { + struct list_head mark_head; + struct rb_root blocked_root; + struct wrlock_entry *granted; + } shards[0]; +}; + +/* a native version of the wire wrlock_id that includes the roster id */ +struct wrlock_id { + u64 counter; + u32 jitter; + u64 roster_id; +}; + +/* + * Entries represent an attempt to lock multiple shards. + * + * Local entries exist on the context that initiated the request. They + * count the number of grant replies and then count the number of + * writers actively modifying the shards under the lock. + * + * Remote entries only exist while other entries are before them in the + * blocked trees in any of their shards. Once they're first in all the + * blocked trees a grant message is sent and they're freed. + */ +struct wrlock_entry { + struct rb_node id_node; + struct list_head send_head; + + /* local lock tasks wait for the entry to be granted */ + struct task_struct *waiter; + struct scoutfs_wrlock_held *held; + long grants; + long writers; + + /* tells roster broadcast who to send to */ + u64 last_peer_id; + struct wrlock_id id; + + u8 nr_shards; + struct wrlock_entry_shard { + struct rb_node blocked_node; + u32 shd; + u8 index; + } shards[SCOUTFS_WRLOCK_MAX_SHARDS]; +}; + +#define ENTF "ent %p %llu.%u.%llu gr %ld wr %ld lpi %llu nr %u" +#define ENTA(ent) ent, ent->id.counter, ent->id.jitter, ent->id.roster_id, \ + ent->grants, ent->writers, ent->last_peer_id, ent->nr_shards + +static struct wrlock_entry *ent_from_blocked_node(struct rb_node *node) +{ + struct wrlock_entry_shard *shard; + + shard = container_of(node, struct wrlock_entry_shard, + blocked_node); + return container_of(shard, struct wrlock_entry, + shards[shard->index]); +} + +/* Return the first blocked entry */ +static struct wrlock_entry *blocked_ent(struct wrlock_context_shard *shard) +{ + struct rb_node *node = rb_first(&shard->blocked_root); + + return node ? ent_from_blocked_node(node) : NULL; +} + +static int cmp_u64s(u64 a, u64 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +static int cmp_u32s(u32 a, u32 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +static int cmp_ids(struct wrlock_id *a, struct wrlock_id *b) +{ + return cmp_u64s(a->counter, b->counter) ?: + cmp_u32s(a->jitter, b->jitter) ?: + cmp_u64s(a->roster_id, b->roster_id); +} + +static void insert_ent_shard(struct rb_root *root, struct wrlock_entry *ins, + struct rb_node *ins_node) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct wrlock_entry *ent; + + while (*node) { + parent = *node; + ent = ent_from_blocked_node(*node); + + if (cmp_ids(&ins->id, &ent->id) < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(ins_node, parent, node); + rb_insert_color(ins_node, root); +} + +/* Insert the entry into the blocked tree for each of its shards. */ +/* + * Insert an entry in to all of its trees. All entries have to be on + * the blocked tree for all of its shards. + * + * But the id tree is a little lazy. It's only used to look up local + * entries when grants are received. It could be a hash table instead of + * a tree and remote entries don't need to be in it. But this re-use + * of the tree code is easy and isn't that expensive compared to all + * the rest of the processing. + */ +static void insert_ent(struct wrlock_context *ctx, struct wrlock_entry *ins) +{ + int i; + + insert_ent_shard(&ctx->id_root, ins, &ins->id_node); + + for (i = 0; i < ins->nr_shards; i++) + insert_ent_shard(&ctx->shards[ins->shards[i].shd].blocked_root, + ins, &ins->shards[i].blocked_node); +} + +static struct wrlock_entry *lookup_ent(struct wrlock_context *ctx, + struct wrlock_id *id) +{ + struct rb_node *node = ctx->id_root.rb_node; + struct wrlock_entry *ent; + int cmp; + + while (node) { + ent = ent_from_blocked_node(node); + + cmp = cmp_ids(id, &ent->id); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return ent; + } + + return NULL; +} + +static void erase_and_clear(struct rb_node *node, struct rb_root *root) +{ + if (!RB_EMPTY_NODE(node)) { + rb_erase(node, root); + RB_CLEAR_NODE(node); + } +} + +/* remove all of the entry's rb nodes from the context's trees */ +static void erase_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) +{ + int i; + + erase_and_clear(&ent->id_node, &ctx->id_root); + + for (i = 0; i < ent->nr_shards; i++) + erase_and_clear(&ent->shards[i].blocked_node, + &ctx->shards[ent->shards[i].shd].blocked_root); +} + +static struct wrlock_entry *alloc_ent(void) +{ + struct wrlock_entry *ent; + int i; + + ent = kzalloc(sizeof(*ent), GFP_NOFS); + if (!ent) + return ERR_PTR(-ENOMEM); + + RB_CLEAR_NODE(&ent->id_node); + INIT_LIST_HEAD(&ent->send_head); + + /* for container_of to find the ent while walking shard nodes */ + for (i = 0; i < ARRAY_SIZE(ent->shards); i++) { + RB_CLEAR_NODE(&ent->shards[i].blocked_node); + ent->shards[i].index = i; + } + + return ent; +} + +/* + * Callers try to free the ent every time they remove a reference to it + * from the context and are done with it. We only free it if there are + * no more references to it in the context. + */ +static void try_free_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) +{ + int i; + + if (!RB_EMPTY_NODE(&ent->id_node) || !list_empty(&ent->send_head)) + return; + + for (i = 0; i < ent->nr_shards; i++) { + if (!RB_EMPTY_NODE(&ent->shards[i].blocked_node) || + ctx->shards[ent->shards[i].shd].granted == ent) + return; + } + + trace_printk("ent "ENTF"\n", ENTA(ent)); + WARN_ON_ONCE(ent->writers); + + kfree(ent); +} + +static bool is_local(struct wrlock_context *ctx, struct wrlock_entry *ent) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(ctx->sb); + + return ent->id.roster_id == sbi->roster_id; +} + +/* + * An entry in the blocked tree can be blocked for a few reasons: + * + * - a local entry hasn't received its grant replies yet + * - there are blocked entries before it on any of its shards + * - a remote entry is waiting for local writers to drain + * + * Note in particular that a local entry isn't blocked by granted writers + * because it'll join them and that remote entries aren't blocked by local + * grants with no writers because it revokes them to send a grant reply. + */ +static bool is_blocked(struct wrlock_context *ctx, struct wrlock_entry *ent) +{ + struct wrlock_context_shard *shard; + int i; + + if (is_local(ctx, ent) && ent->grants < ctx->grants_needed) + return true; + + for (i = 0; i < ent->nr_shards; i++) { + if (rb_prev(&ent->shards[i].blocked_node)) + return true; + + if (!is_local(ctx, ent)) { + shard = &ctx->shards[ent->shards[i].shd]; + if (shard->granted && shard->granted->writers) + return true; + } + } + + return false; +} + +/* mark a given shard for later processing to see if entries aren't blocked */ +static void mark_context_shard(struct wrlock_context *ctx, u32 shard) +{ + struct list_head *head = &ctx->shards[shard].mark_head; + + if (list_empty(head)) + list_add_tail(head, &ctx->mark_list); +} + +static void mark_ent_shards(struct wrlock_context *ctx, + struct wrlock_entry *ent) +{ + int i; + + for (i = 0; i < ent->nr_shards; i++) + mark_context_shard(ctx, ent->shards[i].shd); +} + +static void queue_send(struct wrlock_context *ctx, struct wrlock_entry *ent) +{ + if (list_empty(&ent->send_head)) { + list_add_tail(&ent->send_head, &ctx->send_list); + queue_work(ctx->send_workq, &ctx->send_work); + } +} + +/* + * Try to unblock entries in the shard. We're done when the first entry + * in the shard is still blocked. + * + * If we unblock a remote entry then we have to send its grant message. + * If there is a granted local entry but it has no writers then we + * remove it so that future writers will have to request a new lock from + * the remote peer whose request we granted. + * + * If we unblock a local entry then we move it to the granted pointers + * for each of its shards. There are two tricky cases here. + * + * The first is a local entry being granted which covers more shards + * than the current granted entry on some of its shards. We don't want + * the larger unblocked entry to wait for the smaller granted entry's + * writers to drain. Instead we set the granted pointers to the new + * unblocked large entry after giving it the smaller granted entry's + * writer counters. Unlocking will drop the write counters on whatever + * entry is currently granted on its shards. + * + * The second is making sure that a waiting locking task gets a chance + * to work with a newly granted local entry before the next blocking + * remote entry revokes it. We increment the writers count the moment a + * local entry is granted. It will stay that way until the task drops + * the writer count. We just have to be careful to address all the + * races with the task sleeping, waking, and interrupting. + */ +static void unblock_shard(struct wrlock_context *ctx, + struct wrlock_context_shard *shard) +{ + struct wrlock_entry *ent; + int i; + + ent = blocked_ent(shard); + if (!ent) + return; + + if (is_blocked(ctx, ent)) { + /* send initial requests for local blocked entries */ + if (ent->last_peer_id < ctx->last_peer_id) + queue_send(ctx, ent); + return; + } + + trace_printk("ent "ENTF"\n", ENTA(ent)); + + erase_ent(ctx, ent); + mark_ent_shards(ctx, ent); + + /* unblocked remote entries remove local grants and send replies */ + if (!is_local(ctx, ent)) { + for (i = 0; i < ent->nr_shards; i++) { + shard = &ctx->shards[ent->shards[i].shd]; + if (shard->granted) { + WARN_ON_ONCE(shard->granted->writers); + try_free_ent(ctx, shard->granted); + shard->granted = NULL; + } + } + + queue_send(ctx, ent); + return; + } + + /* grant the entry on all its shards */ + for (i = 0; i < ent->nr_shards; i++) { + shard = &ctx->shards[ent->shards[i].shd]; + + /* the ent couldn't have been granted if it was blocked */ + WARN_ON_ONCE(shard->granted == ent); + + if (shard->granted) { + ent->writers += shard->granted->writers; + try_free_ent(ctx, shard->granted); + } + + shard->granted = ent; + if (ent->waiter) + ent->writers++; + } + + if (ent->waiter) { + /* the task is responsible for the writer count if nr is set */ + ent->held->nr_shards = ent->nr_shards; + smp_mb(); /* wait_event condition isn't locked */ + wake_up_process(ent->waiter); + ent->waiter = NULL; + ent->held = NULL; + } +} + +/* + * Walk all the shards that have been marked and see if their blocked + * entry is still blocked. As we unblock entries we mark all their + * shards and keep going until the blocked entries in the shards + * stabilize. + */ +static void unblock_marked_shards(struct wrlock_context *ctx) +{ + struct wrlock_context_shard *shard; + + while ((shard = list_first_entry_or_null(&ctx->mark_list, + struct wrlock_context_shard, + mark_head))) { + trace_printk("ctx %p shard %p\n", ctx, shard); + list_del_init(&shard->mark_head); + unblock_shard(ctx, shard); + } +} + +/* + * Statically round robin every 1M inodes to each shard. + * + * XXX this will almost certainly need to be more clever. We'll want + * to size the batching more carefully and we'll need to cope with growing + * and shrinking the number of shards. + */ +static u32 ino_shd(struct wrlock_context *ctx, u64 ino) +{ + return (u32)(ino >> SCOUTFS_INO_BATCH_SHIFT) % ctx->nr_shards; +} + +/* + * Shards in entries are sorted and unique to make receive verification + * easier. Entries will only have a small handful of shards. + */ +static void add_ent_shd(struct wrlock_entry *ent, u32 shd) +{ + int i; + + trace_printk("shd %u nr %u\n", shd, ent->nr_shards); + + for (i = 0; i < ent->nr_shards; i++) { + if (shd < ent->shards[i].shd) + swap(shd, ent->shards[i].shd); + else if (shd == ent->shards[i].shd) + return; + } + + ent->shards[i].shd = shd; + ent->nr_shards++; +} + +/* + * Get write locks on the shards that contain the given inodes. + * + * We always insert a new entry so that local attempts are inserted in + * the blocking tree after blocked remote entries. This way local lock + * matching doesn't stave remote lock attempts. + * + * In the fast path the inserted entry will be first and all its shards + * will be granted so we'll increase entry writer counts and return. In + * the slow path we send lock requests and sleep until we get grant + * replies. + * + * The writer counts are set when our entry is granted while we're still + * waiting for it so that we're guaranteed to get to work with our + * granted lock before a remote request has a chance to revoke it. + */ +int scoutfs_wrlock_lock(struct super_block *sb, + struct scoutfs_wrlock_held *held, int nr_inos, ...) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct wrlock_entry *ent; + va_list args; + int ret; + int i; + + if (WARN_ON_ONCE(nr_inos <= 0 || nr_inos > SCOUTFS_WRLOCK_MAX_SHARDS) || + WARN_ON_ONCE(held->nr_shards)) + return -EINVAL; + + ent = alloc_ent(); + if (!ent) + return -ENOMEM; + + va_start(args, nr_inos); + while (nr_inos--) { + /* XXX verify inodes? */ + add_ent_shd(ent, ino_shd(ctx, va_arg(args, u64))); + } + va_end(args); + + /* held's nr_shards is set when the ent is granted and writers inced */ + for (i = 0; i < ent->nr_shards; i++) + held->shards[i] = ent->shards[i].shd; + + ent->waiter = current; + ent->held = held; + ent->id.jitter = get_random_int(); /* XXX how expensive? */ + ent->id.roster_id = sbi->roster_id; + + /* the context owns and can free the entry after we unlock */ + spin_lock(&ctx->lock); + + ent->id.counter = ctx->next_id_counter++; + + trace_printk("ent "ENTF"\n", ENTA(ent)); + + insert_ent(ctx, ent); + mark_ent_shards(ctx, ent); + unblock_marked_shards(ctx); + + spin_unlock(&ctx->lock); + + ret = wait_event_interruptible(ctx->waitq, held->nr_shards); + if (ret == 0) + ret = scoutfs_hold_trans(sb); + + /* unlock on error locks the context before using held.nr_shards */ + if (ret) + scoutfs_wrlock_unlock(sb, held); + + return ret; +} + +/* + * The held shards must have had granted entries for us to increment the + * write counts. The increased write counts should have pinned entries + * to the shards so they must still be around for us to decrease the + * counts. + * + * If we're the last writer of an entry then we'll check to see if any + * of its shards have blocked remote entries that can now make progress. + * + * XXX we'd need to sync dirty blocks before sending the grant. + */ +void scoutfs_wrlock_unlock(struct super_block *sb, + struct scoutfs_wrlock_held *held) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct wrlock_context_shard *shard; + u32 shd; + int i; + + scoutfs_release_trans(sb); + + spin_lock(&ctx->lock); + + for (i = 0; i < held->nr_shards; i++) { + shd = held->shards[i]; + shard = &ctx->shards[shd]; + + /* XXX this would imply unlocked writing, very bad indeed */ + if (WARN_ON_ONCE(!shard->granted) || + WARN_ON_ONCE(shard->granted->writers <= 0)) + continue; + + if (--shard->granted->writers == 0) + mark_context_shard(ctx, shd); + } + + unblock_marked_shards(ctx); + + spin_unlock(&ctx->lock); + +} + +/* + * Process an incoming request message. We allocate and insert an entry + * for the request. When it's not blocked by previous entries or a + * granted entry on all its shards then we send a reply and free the + * entry. + * + * Shard numbers in the incoming request must be unique and sorted. + */ +int scoutfs_wrlock_process_request(struct super_block *sb, u64 peer_id, + struct scoutfs_wrlock_request *req) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct wrlock_entry *ent; + int ret = 0; + u32 shd; + u32 prev; + int i; + + ent = alloc_ent(); + if (!ent) + return -ENOMEM; + + if (req->nr_shards > SCOUTFS_WRLOCK_MAX_SHARDS) { + ret = -EINVAL; + goto out; + } + + for (i = 0, prev = 0; i < req->nr_shards; prev = shd, i++) { + shd = le32_to_cpu(req->shards[i]); + + if (shd >= ctx->nr_shards || (prev && shd <= prev)) { + ret = -EINVAL; + goto out; + } + + add_ent_shd(ent, shd); + } + + ent->id.counter = le64_to_cpu(req->wid.counter); + ent->id.jitter = le32_to_cpu(req->wid.jitter); + ent->id.roster_id = peer_id; + + spin_lock(&ctx->lock); + + ctx->next_id_counter = max(ent->id.counter + 1, ctx->next_id_counter); + + insert_ent(ctx, ent); + mark_ent_shards(ctx, ent); + unblock_marked_shards(ctx); + + spin_unlock(&ctx->lock); + +out: + if (ret) + kfree(ent); + return ret; +} + +/* + * Process an incoming grant message. The sending peer is telling us + * that they don't have any entries blocking our lock. We increment its + * count and wake the locker on the last grant. + * + * An entry won't be found at the id if the process attempting the lock + * exited and removed the entry before all the grants arrived. + * + * XXX freak out if grants is greater than grants_needed? That'd imply + * that we could have prematurely given a locker access to its shards. + */ +void scoutfs_wrlock_process_grant(struct super_block *sb, + struct scoutfs_wrlock_grant *grant) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct wrlock_entry *ent; + struct wrlock_id id = { + .counter = le64_to_cpu(grant->wid.counter), + .jitter = le32_to_cpu(grant->wid.jitter), + .roster_id = sbi->roster_id, + }; + + spin_lock(&ctx->lock); + + ent = lookup_ent(ctx, &id); + if (ent && ++ent->grants == ctx->grants_needed) { + mark_ent_shards(ctx, ent); + unblock_marked_shards(ctx); + } + + spin_unlock(&ctx->lock); +} + +/* + * Send wrlock messages to peers. Entries are put on the send queue + * when we need to either broadcast requests to new peers or send a + * grant reply to a specific requesting peer. + * + * XXX As currently imagined any send failures trigger reconnection and + * recovery. We need a bit more clarity on what the roster + * implementation before worrying too much about the details of recovery + * in here. + */ +static void send_work_func(struct work_struct *work) +{ + struct wrlock_context *ctx = container_of(work, struct wrlock_context, + send_work); + struct super_block *sb = ctx->sb; + struct scoutfs_message msg; + struct wrlock_entry *ent; + struct wrlock_entry *tmp; + u64 peer_id; + int i; + + spin_lock(&ctx->lock); + + list_for_each_entry_safe(ent, tmp, &ctx->send_list, send_head) { + + if (is_local(ctx, ent)) { + msg.cmd = SCOUTFS_MSG_WRLOCK_REQUEST; + msg.request.wid.counter = cpu_to_le64(ent->id.counter); + msg.request.wid.jitter = cpu_to_le32(ent->id.jitter); + msg.request.nr_shards = ent->nr_shards; + for (i = 0; i < ent->nr_shards; i++) { + msg.request.shards[i] = + cpu_to_le32(ent->shards[i].shd); + } + + msg.len = offsetof(struct scoutfs_wrlock_request, + shards[ent->nr_shards]); + peer_id = ent->last_peer_id; + ent->last_peer_id = ctx->last_peer_id; + } else { + msg.cmd = SCOUTFS_MSG_WRLOCK_GRANT; + msg.grant.wid.counter = cpu_to_le64(ent->id.counter); + msg.grant.wid.jitter = cpu_to_le32(ent->id.jitter); + + msg.len = sizeof(msg.grant); + peer_id = ent->id.roster_id; + } + + list_del_init(&ent->send_head); + try_free_ent(ctx, ent); + + spin_unlock(&ctx->lock); + + if (msg.cmd == SCOUTFS_MSG_WRLOCK_GRANT) + scoutfs_roster_send(sb, peer_id, &msg); + else + scoutfs_roster_broadcast(sb, peer_id, &msg); + + spin_lock(&ctx->lock); + } + + spin_unlock(&ctx->lock); +} + +/* + * The roster tells us when mounts join or leave the cluster. + * + * Our job is easy if a peer is joining because they don't have any + * entries yet. They could start sending requests immediately and their + * entries could be inserted behind our blocked local entries. We send + * them all our blocked entries so that they can grant them and make + * forward progress in that case. + * + * If a peer is leaving then we have two problems. + * + * First they might have already granted some entries but we can't tell + * which. We don't track grant replies per peer. We can't adjust the + * entry grant counts to match a smaller number of needed grants. So we + * reset all the blocked local entries and resend them to everyone. We + * reset the id so that we're not confused by grants in flight. It's + * not great but it's simple and rare. + * + * XXX Worse, they might have held locks. We'd need to wait a grace + * period or fence them so that we're sure that they are no longer + * writing to shards. + */ +void scoutfs_wrlock_roster_update(struct super_block *sb, u64 peer_id, + bool join) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct rb_node *node; + struct wrlock_entry *ent; + LIST_HEAD(list); + int i; + + spin_lock(&ctx->lock); + + /* take the peer change into account before walking entries */ + if (join) { + ctx->grants_needed++; + ctx->last_peer_id = peer_id; + } else { + ctx->grants_needed--; + } + + /* + * Walk all the blocked entries on all the shards. Entries can + * be on multiple shards so we're careful to only modify them on + * the first visit. + */ + for (i = 0; i < ctx->nr_shards; i++) { + node = rb_first(&ctx->shards[i].blocked_root); + while (node) { + ent = ent_from_blocked_node(node); + node = rb_next(node); + + /* drop remote blocked entries from a leaving peer */ + if (!join && ent->id.roster_id == peer_id) { + erase_ent(ctx, ent); + mark_ent_shards(ctx, ent); + try_free_ent(ctx, ent); + } + + /* send blocked local locks just to the new peer */ + if (join && is_local(ctx, ent)) + queue_send(ctx, ent); + + /* reset and resend local entries when leaving */ + if (!join && is_local(ctx, ent) && ent->last_peer_id) { + ent->grants = 0; + ent->last_peer_id = 0; + ent->id.counter = ctx->next_id_counter++; + ent->id.jitter = get_random_int(); + + erase_ent(ctx, ent); + insert_ent(ctx, ent); + mark_ent_shards(ctx, ent); + queue_send(ctx, ent); + } + } + } + + unblock_marked_shards(ctx); + + spin_unlock(&ctx->lock); +} + +int scoutfs_wrlock_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx; + u32 nr = 1; /* XXX */ + int i; + + ctx = vmalloc(offsetof(struct wrlock_context, shards[nr])); + if (!ctx) + return -ENOMEM; + + /* XXX need some kind of mount id */ + ctx->send_workq = alloc_ordered_workqueue("scoutfs-%s-%u:%u-send", 0, + sb->s_id, + MAJOR(sb->s_bdev->bd_dev), + MINOR(sb->s_bdev->bd_dev)); + if (!ctx->send_workq) { + vfree(ctx); + return -ENOMEM; + } + + ctx->sb = sb; + init_waitqueue_head(&ctx->waitq); + spin_lock_init(&ctx->lock); + ctx->id_root = RB_ROOT; + INIT_LIST_HEAD(&ctx->mark_list); + INIT_LIST_HEAD(&ctx->send_list); + INIT_WORK(&ctx->send_work, send_work_func); + ctx->nr_shards = nr; + + for (i = 0; i < nr; i++) { + INIT_LIST_HEAD(&ctx->shards[i].mark_head); + ctx->shards[i].blocked_root = RB_ROOT; + } + + sbi->wrlock_context = ctx; + + return 0; +} + +/* + * Destroy the messaging work and free the wrlock entries. There should + * be no more active lockers at this point. + */ +void scoutfs_wrlock_teardown(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct wrlock_context *ctx = sbi->wrlock_context; + struct wrlock_context_shard *shard; + struct wrlock_entry *ent; + struct wrlock_entry *tmp; + int i; + + if (!ctx) + return; + + trace_printk("ctx %p\n", ctx); + + destroy_workqueue(ctx->send_workq); + + for (i = 0; i < ctx->nr_shards; i++) { + shard = &ctx->shards[i]; + + try_free_ent(ctx, shard->granted); + shard->granted = NULL; + + list_for_each_entry_safe(ent, tmp, &ctx->send_list, send_head) { + list_del_init(&ent->send_head); + try_free_ent(ctx, ent); + } + + while ((ent = blocked_ent(shard))) { + erase_ent(ctx, ent); + try_free_ent(ctx, ent); + } + } + + vfree(ctx); +} diff --git a/kmod/src/wrlock.h b/kmod/src/wrlock.h new file mode 100644 index 00000000..2a88b62d --- /dev/null +++ b/kmod/src/wrlock.h @@ -0,0 +1,30 @@ +#ifndef _SCOUTFS_WRLOCK_H_ +#define _SCOUTFS_WRLOCK_H_ + +#include "wire.h" + +struct scoutfs_wrlock_held { + bool held_trans; + u8 nr_shards; + u32 shards[SCOUTFS_WRLOCK_MAX_SHARDS]; +}; + +#define DECLARE_SCOUTFS_WRLOCK_HELD(held) \ + struct scoutfs_wrlock_held held = {0, } + +int scoutfs_wrlock_lock(struct super_block *sb, + struct scoutfs_wrlock_held *held, int nr_inos, ...); +void scoutfs_wrlock_unlock(struct super_block *sb, + struct scoutfs_wrlock_held *held); + +void scoutfs_wrlock_roster_update(struct super_block *sb, u64 peer_id, + bool join); +int scoutfs_wrlock_process_request(struct super_block *sb, u64 peer_id, + struct scoutfs_wrlock_request *req); +void scoutfs_wrlock_process_grant(struct super_block *sb, + struct scoutfs_wrlock_grant *grant); + +int scoutfs_wrlock_setup(struct super_block *sb); +void scoutfs_wrlock_teardown(struct super_block *sb); + +#endif From 7d6dd91a24c26f5d52b93082f091c2193f3ad520 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 May 2016 14:14:12 -0700 Subject: [PATCH 053/920] scoutfs: add tracing messages This adds tracing functionality that's cheap and easy to use. By constantly gathering traces we'll always have rich history to analyze when something goes wrong. Signed-off-by: Zach Brown --- kmod/src/Makefile | 14 +- kmod/src/first.c | 4 + kmod/src/ioctl.c | 46 ++++++ kmod/src/ioctl.h | 32 +++++ kmod/src/last.c | 4 + kmod/src/super.c | 3 + kmod/src/trace.c | 360 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/trace.h | 118 +++++++++++++++ 8 files changed, 579 insertions(+), 2 deletions(-) create mode 100644 kmod/src/first.c create mode 100644 kmod/src/ioctl.c create mode 100644 kmod/src/ioctl.h create mode 100644 kmod/src/last.c create mode 100644 kmod/src/trace.c create mode 100644 kmod/src/trace.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index c60bf9f9..01a96cdf 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,16 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include +# +# these first and last objects are a super lame hack to put boundary +# symbols around trace printf formats that are put in an elf section. +# That should be done in a linker script, of course, but I'll be honest: +# I didn't go digging to see if modules can have linker scripts today. +# +scoutfs-y += first.o + scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o msg.o roster.o scoutfs_trace.o super.o trans.o treap.o \ - wrlock.o + inode.o ioctl.o msg.o roster.o scoutfs_trace.o super.o trace.o \ + trans.o treap.o wrlock.o + +scoutfs-y += last.o diff --git a/kmod/src/first.c b/kmod/src/first.c new file mode 100644 index 00000000..568ff207 --- /dev/null +++ b/kmod/src/first.c @@ -0,0 +1,4 @@ + +#include "trace.h" + +char __scoutfs_trace_section scoutfs_trace_first_format[] = ""; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c new file mode 100644 index 00000000..fc9766fb --- /dev/null +++ b/kmod/src/ioctl.c @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "ioctl.h" +#include "trace.h" + +int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg) +{ + struct scoutfs_ioctl_buf __user *user_ibuf = (void __user *)arg; + struct scoutfs_ioctl_buf ibuf; + + if (copy_from_user(&ibuf, user_ibuf, sizeof(ibuf))) + return -EFAULT; + + /* limit lengths to an int for some helpers that take int len args */ + if (ibuf.len < 0) + return -EINVAL; + + iov->iov_base = (void __user *)(long)ibuf.ptr; + iov->iov_len = ibuf.len; + + /* + * This is not meant to protect the rest of the code from + * faults, it can't. It's meant to return early for iovecs that + * are completely garbage. + */ + if (!access_ok(VERIFY_READ, iov->iov_base, iov->iov_len)) + return -EFAULT; + + return 0; +} diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h new file mode 100644 index 00000000..40372ae0 --- /dev/null +++ b/kmod/src/ioctl.h @@ -0,0 +1,32 @@ +#ifndef _SCOUTFS_IOCTL_H_ +#define _SCOUTFS_IOCTL_H_ + +/* XXX I have no idea how these are chosen. */ +#define SCOUTFS_IOCTL_MAGIC 's' + +struct scoutfs_ioctl_buf { + __u64 ptr; + __s32 len; +} __packed; + +/* + * Fills the buffer with a packed array of format strings. Trace records + * refer to the format strings in the buffer by their byte offset. + */ +#define SCOUTFS_IOC_GET_TRACE_FORMATS _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ + struct scoutfs_ioctl_buf) + +struct scoutfs_trace_record { + __u16 format_off; + __u8 nr; + __u8 data[0]; +} __packed; +/* + * Fills the buffer with trace records. + */ +#define SCOUTFS_IOC_GET_TRACE_RECORDS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_buf) + +int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg); + +#endif diff --git a/kmod/src/last.c b/kmod/src/last.c new file mode 100644 index 00000000..44e6d7dc --- /dev/null +++ b/kmod/src/last.c @@ -0,0 +1,4 @@ + +#include "trace.h" + +char __scoutfs_trace_section scoutfs_trace_last_format[] = ""; diff --git a/kmod/src/super.c b/kmod/src/super.c index f6cab079..27630b6a 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -28,6 +28,7 @@ #include "trans.h" #include "roster.h" #include "wrlock.h" +#include "trace.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -222,12 +223,14 @@ static void teardown_module(void) scoutfs_inode_exit(); if (scoutfs_kset) kset_unregister(scoutfs_kset); + scoutfs_trace_exit(); } static int __init scoutfs_module_init(void) { int ret; + scoutfs_trace_init(); scoutfs_init_counters(); scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); diff --git a/kmod/src/trace.c b/kmod/src/trace.c new file mode 100644 index 00000000..ba0d97bf --- /dev/null +++ b/kmod/src/trace.c @@ -0,0 +1,360 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "super.h" +#include "ioctl.h" + +/* + * This tracing gives us: + * + * - Always on. We get history leading up to an event without having + * had to predict the event. + * + * - Cheap. Recording the format pointer index and packed arguments is + * cheap enough that we don't mind always doing it at a reasonable + * frequency. + * + * - Trivial to add. We want to err on the side of too much logging. + * We don't want there to be so much garbage associated with adding a + * single logging message that people are discouraged from doing it. + * + * - Easy to extract from crash dumps. The more the computer can tell + * us about what happened when the world went sideways, the better. + * + * The implementation is reasonably straight forward. + * + * Log statements are simple printf format strings and arguments. The + * first trick bit is that we only support u64 arguments. This lets us + * use macro hacks to walk the arguments without having to parse the + * format string. This actually isn't a great hardship because often + * the things we might want to print as strings -- process names, + * xattrs, directory entries -- could in fact be sensitive user data + * that we don't want to see. + * + * Each log statement is packed into a variable byte size record. The + * records are packed into long term per-page pages. We only support + * logging from task context so that we don't have to fool around with + * serializing between contexts on a cpu. Writers record the number of + * record bytes stored in each page in page->private. + * + * Userspace reads the format strings and trace records from trivial + * ioctls that copy the entire data set in one go. This avoids all the + * nonsense of trying to translate the changing set of records into a + * seekable byte stream of formatted output. Readers of each page of + * records samples page->private to discover when they race with writers + * and retry. + */ + +/* + * This tries to strike a balance between having enough logging on a cpu + * and not allocating an enormous amount of memory on systems with many + * cpus. + */ +#define TRACE_PAGES_PER_CPU DIV_ROUND_UP(256 * 1024, PAGE_SIZE) + +struct trace_percpu { + int cur_page; + struct page *pages[TRACE_PAGES_PER_CPU]; +}; + +static DEFINE_PER_CPU(struct trace_percpu, scoutfs_trace_percpu); + +static int rec_bytes(int bytes) +{ + return offsetof(struct scoutfs_trace_record, data[bytes]); +} + +/* + * We compact the record of the trace format by referencing it with a + * small offset into a section that contains all the format strings. + * This shrinks the per-record format reference from an 8 byte pointer + * to a 2 byte offset. 6 bytes is a lot when records are 15 bytes. + */ +static char *trace_format(u16 off) +{ + return &scoutfs_trace_first_format[1 + off]; +} + +static u16 trace_format_off(char *fmt) +{ + return fmt - scoutfs_trace_first_format - 1; +} + +static int trace_format_bytes(void) +{ + return trace_format_off(scoutfs_trace_last_format); +} + +static int valid_trace_format(char *fmt) +{ + return fmt > scoutfs_trace_first_format && + fmt < scoutfs_trace_last_format; +} + +/* + * We only support trace messages with integer arguments. Most of them + * are small: counters, pids, sizes, cpus, etc. It's worth spending a + * few cycles to remove the leading bytes full of zeros. + * + * VLQ is very simple and does reasonably well. I'd happily consider + * alternatives with similar complexity but better space efficiency. + * + * This is the most boring conservative iterative implementation. A + * much cooler implementation would efficiently transform all the bits, + * store the whole little endian value, and return the number of bytes + * with bits set. + */ +static unsigned char encode_u64_bytes(u8 *data, u64 val) +{ + unsigned char bytes = 0; + + do { + *data = val & 127; + val >>= 7; + *(data++) += (!!val) << 7; + bytes++; + } while (val); + + return bytes; +} + +/* + * Write a trace record to a percpu page. We only write from task + * context so one writer is racing with many readers. Readers sample + * the count of total written bytes in the page at page private and + * retry the copy if the count changes. It's a poor man's seqlock. + * + * The calling trace wrapper has pinned our task to the cpu. + */ +void scoutfs_trace_write(char *fmt, int nr, ...) +{ + struct trace_percpu *pcpu = this_cpu_ptr(&scoutfs_trace_percpu); + struct scoutfs_trace_record *rec; + struct page *page; + unsigned long page_bytes; + int encoded; + va_list args; + int i; + + if (WARN_ON_ONCE(in_interrupt() || in_softirq() || in_irq()) || + WARN_ON_ONCE(!valid_trace_format(fmt)) || + WARN_ON_ONCE(trace_format_bytes() > U16_MAX)) + return; + +next_page: + page = pcpu->pages[pcpu->cur_page]; + page_bytes = page->private & ~PAGE_MASK; + rec = page_address(page) + page_bytes; + + encoded = 0; + va_start(args, nr); + for (i = 0; i < nr; i++) { + if (page_bytes + rec_bytes(encoded + 9) >= PAGE_SIZE) { + if (++pcpu->cur_page == TRACE_PAGES_PER_CPU) + pcpu->cur_page = 0; + + page = pcpu->pages[pcpu->cur_page]; + /* XXX barriers? */ + page->private = round_up(page->private, PAGE_SIZE); + va_end(args); + goto next_page; + } + + encoded += encode_u64_bytes(&rec->data[encoded], + va_arg(args, u64)); + } + va_end(args); + + rec->format_off = trace_format_off(fmt); + rec->nr = nr; + /* XXX barriers? */ + page->private += rec_bytes(encoded); +} + +/* + * Give userspace all of the format strings. They're packed and null + * terminated. + * + * We return the number of bytes copied. A return size smaller than the + * buffer len indicates a partial copy and the user can retry with a + * larger buffer. + */ +static int scoutfs_ioc_get_trace_formats(void __user *buf, int len) +{ + int bytes= trace_format_bytes(); + + if (bytes <= len) { + if (copy_to_user(buf, trace_format(0), bytes)) + return -EFAULT; + } + + return bytes; +} + +/* + * Copy all the trace records on all the cpus' pages to the user buffer. + * Each page's records will be copied atomically so records won't be + * scrambled. But writers can cycle through the pages as we copy so the + * entire set of records returned is not an atomic snapshot of all the + * pages. + * + * We return the number of bytes copied. A return size smaller than the + * buffer len indicates a partial copy and the user can retry with a + * larger buffer. + */ +static int scoutfs_ioc_get_trace_records(void __user *buf, int len) +{ + struct trace_percpu *pcpu; + unsigned long before; + unsigned long after; + struct page *page; + int total = 0; + int bytes; + int ret; + int cpu; + int i; + + if (len < 0) + return -EINVAL; + + /* quickly give the caller the largest possible buffer size */ + ret = num_online_cpus() * TRACE_PAGES_PER_CPU; + if (ret > len) + return ret; + + for_each_online_cpu(cpu) { + pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); + + for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { + page = pcpu->pages[i]; + + do { + before = ACCESS_ONCE(page->private); + bytes = before & ~PAGE_MASK; + + /* ret still nr * pages */ + if (total + bytes > len) + goto out; + + if (copy_to_user(buf + total, + page_address(page), bytes)) { + ret = -EFAULT; + goto out; + } + after = ACCESS_ONCE(page->private); + } while (after != before); + + total += bytes; + } + } + + ret = total; +out: + return ret; +} + +static long scoutfs_trace_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct iovec iov; + + switch (cmd) { + case SCOUTFS_IOC_GET_TRACE_FORMATS: + return scoutfs_copy_ibuf(&iov, arg) ?: + scoutfs_ioc_get_trace_formats(iov.iov_base, iov.iov_len); + + case SCOUTFS_IOC_GET_TRACE_RECORDS: + return scoutfs_copy_ibuf(&iov, arg) ?: + scoutfs_ioc_get_trace_records(iov.iov_base, iov.iov_len); + } + + return -ENOTTY; +} + +static const struct file_operations scoutfs_trace_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = scoutfs_trace_ioctl, +}; + +static struct dentry *scoutfs_debugfs_dir; +static struct dentry *scoutfs_trace_dentry; + +int __init scoutfs_trace_init(void) +{ + struct trace_percpu *pcpu; + int cpu; + int i; + + if (WARN_ON_ONCE(&scoutfs_trace_first_format >= + &scoutfs_trace_last_format) || + WARN_ON_ONCE(trace_format_bytes() > U16_MAX)) + return -EINVAL; + + /* XXX possible instead of online? yikes? */ + for_each_possible_cpu(cpu) { + pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); + for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { + pcpu->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pcpu->pages[i]) + return -ENOMEM; + pcpu->pages[i]->private = 0; + } + } + + scoutfs_debugfs_dir = debugfs_create_dir("scoutfs", NULL); + if (!scoutfs_debugfs_dir) + return -ENOMEM; + + scoutfs_trace_dentry = debugfs_create_file("trace", 0600, + scoutfs_debugfs_dir, NULL, + &scoutfs_trace_fops); + if (!scoutfs_trace_dentry) + return -ENOMEM; + + return 0; +} + +void __exit scoutfs_trace_exit(void) +{ + struct trace_percpu *pcpu; + int cpu; + int i; + + if (scoutfs_trace_dentry) { + debugfs_remove(scoutfs_trace_dentry); + scoutfs_trace_dentry = NULL; + } + + if (scoutfs_debugfs_dir) { + debugfs_remove(scoutfs_debugfs_dir); + scoutfs_debugfs_dir = NULL; + } + + /* XXX possible instead of online? yikes? */ + for_each_possible_cpu(cpu) { + pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); + for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { + if (pcpu->pages[i]) { + __free_page(pcpu->pages[i]); + pcpu->pages[i] = NULL; + } + } + } +} diff --git a/kmod/src/trace.h b/kmod/src/trace.h new file mode 100644 index 00000000..96e0cd77 --- /dev/null +++ b/kmod/src/trace.h @@ -0,0 +1,118 @@ +#ifndef _SCOUTFS_TRACE_H_ +#define _SCOUTFS_TRACE_H_ + +#include +#include + +#define __scoutfs_trace_section __attribute__((section("__scoutfs_trace_fmt"))) + +extern char scoutfs_trace_first_format[]; +extern char scoutfs_trace_last_format[]; + +/* + * What a beautifully baffling construct! First our arguments are added + * to a reverse sequence of numbers. Then all the arguments are handed + * to a macro that only returns its 64th argument. The presence of our + * arguments before the sequence means that the 64th argument will be + * the number in the reverse sequence that matches the number of our + * initial arguments. + * + * h/t to: + * https://groups.google.com/forum/#!topic/comp.std.c/d-6Mj5Lko_s + */ +#define NR_VA_ARGS(...) \ + _ONLY_64TH(__VA_ARGS__, _reverse_sequence()) +#define _ONLY_64TH(...) \ + __ONLY_64TH(__VA_ARGS__) +#define __ONLY_64TH( \ + _1, _2, _3, _4, _5, _6, _7, _8, _9,_10, \ + _11,_12,_13,_14,_15,_16,_17,_18,_19,_20, \ + _21,_22,_23,_24,_25,_26,_27,_28,_29,_30, \ + _31,_32,_33,_34,_35,_36,_37,_38,_39,_40, \ + _41,_42,_43,_44,_45,_46,_47,_48,_49,_50, \ + _51,_52,_53,_54,_55,_56,_57,_58,_59,_60, \ + _61,_62,_63,N,...) N +#define _reverse_sequence() \ + 63,62,61,60, \ + 59,58,57,56,55,54,53,52,51,50, \ + 49,48,47,46,45,44,43,42,41,40, \ + 39,38,37,36,35,34,33,32,31,30, \ + 29,28,27,26,25,24,23,22,21,20, \ + 19,18,17,16,15,14,13,12,11,10, \ + 9,8,7,6,5,4,3,2,1,0 + + +/* + * surround each arg with (u64)( .. ), + * + * A 'called object not a function' error can mean there's too many args. + * + * XXX doesn't yet work with no args + */ +#define CAST_ARGS_U64(...) \ + EXPAND_MACRO(__VA_ARGS__,CU_16,CU_15,CU_14,CU_13,CU_12,\ + CU_11,CU_10,CU_9,CU_8,CU_7,CU_6,CU_5,CU_4,\ + CU_3,CU_2,CU_1)(__VA_ARGS__) +#define EXPAND_MACRO(_1,_2,_3,_4,_5,_6,_7,_8,\ + _9,_10,_11,_12,_13,_14,_15,_16,NAME,...) NAME +#define CU_1(X) (u64)(X) +#define CU_2(X, ...) (u64)(X),CU_1(__VA_ARGS__) +#define CU_3(X, ...) (u64)(X),CU_2(__VA_ARGS__) +#define CU_4(X, ...) (u64)(X),CU_3(__VA_ARGS__) +#define CU_5(X, ...) (u64)(X),CU_4(__VA_ARGS__) +#define CU_6(X, ...) (u64)(X),CU_5(__VA_ARGS__) +#define CU_7(X, ...) (u64)(X),CU_6(__VA_ARGS__) +#define CU_8(X, ...) (u64)(X),CU_7(__VA_ARGS__) +#define CU_9(X, ...) (u64)(X),CU_8(__VA_ARGS__) +#define CU_10(X, ...) (u64)(X),CU_9(__VA_ARGS__) +#define CU_11(X, ...) (u64)(X),CU_10(__VA_ARGS__) +#define CU_12(X, ...) (u64)(X),CU_11(__VA_ARGS__) +#define CU_13(X, ...) (u64)(X),CU_12(__VA_ARGS__) +#define CU_14(X, ...) (u64)(X),CU_13(__VA_ARGS__) +#define CU_15(X, ...) (u64)(X),CU_14(__VA_ARGS__) +#define CU_16(X, ...) (u64)(X),CU_15(__VA_ARGS__) + +struct super_block; +void scoutfs_trace_write(char *fmt, int nr, ...); + +__attribute__((format(printf, 1, 2))) +static inline void only_check_format(const char *fmt, ...) +{ +} + +#define __trace_write(fmtp, args...) \ + scoutfs_trace_write(fmtp, NR_VA_ARGS(args), ##args) + +/* + * Record an unstructured trace message for debugging. + * + * The arguments can only be scalar integers and will be cast to u64 so + * only %llu formats can be used. + * + * This can only be called from task context. + * + * The super block is only used to indicate which mount initiated the + * trace and it can be null for trace messages not associated with + * mounts. + */ +#define scoutfs_trace(sb, fmt, ...) \ +do { \ + static char __scoutfs_trace_section __fmt[] = \ + "ns %llu sb %llx pid %llu cpu %llu "fmt; \ + \ + BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ + \ + /* check the caller's format before we prepend things to it */ \ + only_check_format(fmt, CAST_ARGS_U64(__VA_ARGS__)); \ + \ + __trace_write(__fmt, \ + CAST_ARGS_U64(sched_clock(), (long)(sb), \ + current->pid, get_cpu(), \ + __VA_ARGS__)); \ + put_cpu(); \ +} while (0) + +int __init scoutfs_trace_init(void); +void __exit scoutfs_trace_exit(void); + +#endif From ad5a58c348365a167010f77ac0176ae4b553831d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Jun 2016 20:34:31 -0700 Subject: [PATCH 054/920] scoutfs: make trace format a little nicer The first trace format was pretty noisy. Now the time is printed in a gettimeofday timeval so that it can be correlated with other time stamps. The super block gets a counter instead of a pointer. The pid and cpu are printed without a lavel and we add the line number so that it's easy to grep the source to find a trace caller. Signed-off-by: Zach Brown --- kmod/src/super.c | 8 ++++++++ kmod/src/super.h | 2 ++ kmod/src/trace.h | 14 +++++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 27630b6a..5e409db0 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -128,6 +128,12 @@ static int read_supers(struct super_block *sb) return 0; } +/* + * Only used for tracing output, it's a convenient way to cheaply differentiate + * messages from different super blocks. + */ +static atomic64_t scoutfs_sb_ctr = ATOMIC64_INIT(0); + static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; @@ -158,6 +164,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) init_waitqueue_head(&sbi->trans_write_wq); INIT_LIST_HEAD(&sbi->roster_head); + sbi->ctr = atomic64_inc_return(&scoutfs_sb_ctr); + /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index 2d0392ac..c99a77d7 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,8 @@ struct wrlock_context; struct scoutfs_sb_info { struct super_block *sb; + u64 ctr; + struct scoutfs_super_block super; spinlock_t next_ino_lock; diff --git a/kmod/src/trace.h b/kmod/src/trace.h index 96e0cd77..0e9e3735 100644 --- a/kmod/src/trace.h +++ b/kmod/src/trace.h @@ -97,18 +97,22 @@ static inline void only_check_format(const char *fmt, ...) */ #define scoutfs_trace(sb, fmt, ...) \ do { \ + struct scoutfs_sb_info *__sbi = SCOUTFS_SB(sb); \ static char __scoutfs_trace_section __fmt[] = \ - "ns %llu sb %llx pid %llu cpu %llu "fmt; \ + "[%llu.%llu] %llu %llu %llu " __stringify(__LINE__) ": "\ + fmt; \ + struct timeval __tv; \ \ BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ \ /* check the caller's format before we prepend things to it */ \ only_check_format(fmt, CAST_ARGS_U64(__VA_ARGS__)); \ \ - __trace_write(__fmt, \ - CAST_ARGS_U64(sched_clock(), (long)(sb), \ - current->pid, get_cpu(), \ - __VA_ARGS__)); \ + do_gettimeofday(&__tv); \ + \ + __trace_write(__fmt, CAST_ARGS_U64(__tv.tv_sec, __tv.tv_usec, \ + __sbi->ctr, current->pid, get_cpu(), \ + __VA_ARGS__)); \ put_cpu(); \ } while (0) From c9caebc117d40fbc72d5171d706aae345dd7820d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Jun 2016 21:07:05 -0700 Subject: [PATCH 055/920] scoutfs: remove unused held_trans The held lock struct had an unused 'held_trans' field from a previous version of the code that specifically tried to track if a held lock had the trans open. Signed-off-by: Zach Brown --- kmod/src/wrlock.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/wrlock.h b/kmod/src/wrlock.h index 2a88b62d..d8304498 100644 --- a/kmod/src/wrlock.h +++ b/kmod/src/wrlock.h @@ -4,7 +4,6 @@ #include "wire.h" struct scoutfs_wrlock_held { - bool held_trans; u8 nr_shards; u32 shards[SCOUTFS_WRLOCK_MAX_SHARDS]; }; From 171aea62cd67bd3b6acddde2b69b666ea5d34563 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Jun 2016 21:10:16 -0700 Subject: [PATCH 056/920] scoutfs: add some wrlock tracing Add a bunch of tracing to the wrlock code paths. Signed-off-by: Zach Brown --- kmod/src/wrlock.c | 54 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/kmod/src/wrlock.c b/kmod/src/wrlock.c index 62721923..21c3f07e 100644 --- a/kmod/src/wrlock.c +++ b/kmod/src/wrlock.c @@ -21,6 +21,7 @@ #include "wrlock.h" #include "trans.h" #include "roster.h" +#include "trace.h" /* * The persistent structures in each shard in a scoutfs volume can only @@ -147,9 +148,8 @@ struct wrlock_entry { } shards[SCOUTFS_WRLOCK_MAX_SHARDS]; }; -#define ENTF "ent %p %llu.%u.%llu gr %ld wr %ld lpi %llu nr %u" -#define ENTA(ent) ent, ent->id.counter, ent->id.jitter, ent->id.roster_id, \ - ent->grants, ent->writers, ent->last_peer_id, ent->nr_shards +#define ENTF "ent id %llu.%llu.%llu" +#define ENTA(ent) ent->id.counter, ent->id.jitter, ent->id.roster_id static struct wrlock_entry *ent_from_blocked_node(struct rb_node *node) { @@ -227,6 +227,8 @@ static void insert_ent(struct wrlock_context *ctx, struct wrlock_entry *ins) for (i = 0; i < ins->nr_shards; i++) insert_ent_shard(&ctx->shards[ins->shards[i].shd].blocked_root, ins, &ins->shards[i].blocked_node); + + scoutfs_trace(ctx->sb, "inserted "ENTF, ENTA(ins)); } static struct wrlock_entry *lookup_ent(struct wrlock_context *ctx, @@ -269,6 +271,8 @@ static void erase_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) for (i = 0; i < ent->nr_shards; i++) erase_and_clear(&ent->shards[i].blocked_node, &ctx->shards[ent->shards[i].shd].blocked_root); + + scoutfs_trace(ctx->sb, "erased "ENTF, ENTA(ent)); } static struct wrlock_entry *alloc_ent(void) @@ -310,8 +314,8 @@ static void try_free_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) return; } - trace_printk("ent "ENTF"\n", ENTA(ent)); WARN_ON_ONCE(ent->writers); + scoutfs_trace(ctx->sb, "freed "ENTF, ENTA(ent)); kfree(ent); } @@ -379,6 +383,7 @@ static void queue_send(struct wrlock_context *ctx, struct wrlock_entry *ent) if (list_empty(&ent->send_head)) { list_add_tail(&ent->send_head, &ctx->send_list); queue_work(ctx->send_workq, &ctx->send_work); + scoutfs_trace(ctx->sb, "queued "ENTF, ENTA(ent)); } } @@ -426,7 +431,7 @@ static void unblock_shard(struct wrlock_context *ctx, return; } - trace_printk("ent "ENTF"\n", ENTA(ent)); + scoutfs_trace(ctx->sb, "unblocked "ENTF, ENTA(ent)); erase_ent(ctx, ent); mark_ent_shards(ctx, ent); @@ -461,6 +466,9 @@ static void unblock_shard(struct wrlock_context *ctx, shard->granted = ent; if (ent->waiter) ent->writers++; + + scoutfs_trace(ctx->sb, "granted ctx 0x%llx shd %llu wr %llu", + ctx, ent->shards[i].shd, ent->writers); } if (ent->waiter) { @@ -486,7 +494,6 @@ static void unblock_marked_shards(struct wrlock_context *ctx) while ((shard = list_first_entry_or_null(&ctx->mark_list, struct wrlock_context_shard, mark_head))) { - trace_printk("ctx %p shard %p\n", ctx, shard); list_del_init(&shard->mark_head); unblock_shard(ctx, shard); } @@ -512,8 +519,6 @@ static void add_ent_shd(struct wrlock_entry *ent, u32 shd) { int i; - trace_printk("shd %u nr %u\n", shd, ent->nr_shards); - for (i = 0; i < ent->nr_shards; i++) { if (shd < ent->shards[i].shd) swap(shd, ent->shards[i].shd); @@ -580,8 +585,6 @@ int scoutfs_wrlock_lock(struct super_block *sb, ent->id.counter = ctx->next_id_counter++; - trace_printk("ent "ENTF"\n", ENTA(ent)); - insert_ent(ctx, ent); mark_ent_shards(ctx, ent); unblock_marked_shards(ctx); @@ -592,6 +595,8 @@ int scoutfs_wrlock_lock(struct super_block *sb, if (ret == 0) ret = scoutfs_hold_trans(sb); + scoutfs_trace(sb, "lock nr %llu ret %lld", held->nr_shards, ret); + /* unlock on error locks the context before using held.nr_shards */ if (ret) scoutfs_wrlock_unlock(sb, held); @@ -632,7 +637,12 @@ void scoutfs_wrlock_unlock(struct super_block *sb, WARN_ON_ONCE(shard->granted->writers <= 0)) continue; - if (--shard->granted->writers == 0) + shard->granted->writers--; + + scoutfs_trace(sb, "unlock ctx 0x%llx shd %llu wr %llu", + ctx, shd, shard->granted->writers); + + if (shard->granted->writers == 0) mark_context_shard(ctx, shd); } @@ -727,9 +737,16 @@ void scoutfs_wrlock_process_grant(struct super_block *sb, spin_lock(&ctx->lock); ent = lookup_ent(ctx, &id); - if (ent && ++ent->grants == ctx->grants_needed) { - mark_ent_shards(ctx, ent); - unblock_marked_shards(ctx); + if (ent) { + ent->grants++; + + scoutfs_trace(sb, "grant rx "ENTF" grants %llu needed %llu", + ENTA(ent), ent->grants, ctx->grants_needed); + + if (ent->grants == ctx->grants_needed) { + mark_ent_shards(ctx, ent); + unblock_marked_shards(ctx); + } } spin_unlock(&ctx->lock); @@ -783,6 +800,8 @@ static void send_work_func(struct work_struct *work) peer_id = ent->id.roster_id; } + scoutfs_trace(sb, "send "ENTF" cmd %llu", ENTA(ent), msg.cmd); + list_del_init(&ent->send_head); try_free_ent(ctx, ent); @@ -841,6 +860,9 @@ void scoutfs_wrlock_roster_update(struct super_block *sb, u64 peer_id, ctx->grants_needed--; } + scoutfs_trace(sb, "update ctx 0x%llx peer_id %llu join %llu gr %llu", + ctx, peer_id, join, ctx->grants_needed); + /* * Walk all the blocked entries on all the shards. Entries can * be on multiple shards so we're careful to only modify them on @@ -920,6 +942,8 @@ int scoutfs_wrlock_setup(struct super_block *sb) sbi->wrlock_context = ctx; + scoutfs_trace(sb, "setup ctx 0x%llx", ctx); + return 0; } @@ -939,7 +963,7 @@ void scoutfs_wrlock_teardown(struct super_block *sb) if (!ctx) return; - trace_printk("ctx %p\n", ctx); + scoutfs_trace(sb, "teardown ctx 0x%llx", ctx); destroy_workqueue(ctx->send_workq); From 4689bf08811bce7cf433edb6e9dd7918328715a1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Jun 2016 21:17:30 -0700 Subject: [PATCH 057/920] scoutfs: free once granted wrlock entries The entry free routine only frees entries that don't have any references from its context. Callers are supposed to try to free entries after removing references to them. Callers that were removing entries from a shard's granted pointer were trying to free the entry before removing the pointer to the entry. Entries that were last removed from shard granted pointers were never freed. Signed-off-by: Zach Brown --- kmod/src/wrlock.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/kmod/src/wrlock.c b/kmod/src/wrlock.c index 21c3f07e..9bb7b06c 100644 --- a/kmod/src/wrlock.c +++ b/kmod/src/wrlock.c @@ -299,7 +299,9 @@ static struct wrlock_entry *alloc_ent(void) /* * Callers try to free the ent every time they remove a reference to it * from the context and are done with it. We only free it if there are - * no more references to it in the context. + * no more references to it in the context. This is called with the + * context lock held so it's not racing with other tasks that are + * removing references and trying to free. */ static void try_free_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) { @@ -418,6 +420,7 @@ static void unblock_shard(struct wrlock_context *ctx, struct wrlock_context_shard *shard) { struct wrlock_entry *ent; + struct wrlock_entry *gr; int i; ent = blocked_ent(shard); @@ -440,10 +443,13 @@ static void unblock_shard(struct wrlock_context *ctx, if (!is_local(ctx, ent)) { for (i = 0; i < ent->nr_shards; i++) { shard = &ctx->shards[ent->shards[i].shd]; + if (shard->granted) { - WARN_ON_ONCE(shard->granted->writers); - try_free_ent(ctx, shard->granted); + gr = shard->granted; + WARN_ON_ONCE(gr->writers); + shard->granted = NULL; + try_free_ent(ctx, gr); } } @@ -459,8 +465,11 @@ static void unblock_shard(struct wrlock_context *ctx, WARN_ON_ONCE(shard->granted == ent); if (shard->granted) { - ent->writers += shard->granted->writers; - try_free_ent(ctx, shard->granted); + gr = shard->granted; + ent->writers += gr->writers; + + shard->granted = NULL; + try_free_ent(ctx, gr); } shard->granted = ent; @@ -970,8 +979,11 @@ void scoutfs_wrlock_teardown(struct super_block *sb) for (i = 0; i < ctx->nr_shards; i++) { shard = &ctx->shards[i]; - try_free_ent(ctx, shard->granted); - shard->granted = NULL; + if (shard->granted) { + ent = shard->granted; + shard->granted = NULL; + try_free_ent(ctx, ent); + } list_for_each_entry_safe(ent, tmp, &ctx->send_list, send_head) { list_del_init(&ent->send_head); From 5c7ba5ed395f8953bef81c5e8468fe066d3785d4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 1 Jul 2016 21:03:40 -0700 Subject: [PATCH 058/920] scoutfs: remove wrlock and roster These were interesting experiments in how to manage locks across the cluster but we'll be going in a more flexible direction. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/dir.c | 20 +- kmod/src/filerw.c | 12 +- kmod/src/format.h | 2 - kmod/src/inode.c | 72 +--- kmod/src/inode.h | 4 +- kmod/src/roster.c | 159 ------- kmod/src/roster.h | 14 - kmod/src/super.c | 7 - kmod/src/super.h | 8 - kmod/src/wire.h | 36 -- kmod/src/wrlock.c | 1000 --------------------------------------------- kmod/src/wrlock.h | 29 -- 13 files changed, 29 insertions(+), 1338 deletions(-) delete mode 100644 kmod/src/roster.c delete mode 100644 kmod/src/roster.h delete mode 100644 kmod/src/wire.h delete mode 100644 kmod/src/wrlock.c delete mode 100644 kmod/src/wrlock.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 01a96cdf..62aacf23 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -11,7 +11,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += first.o scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o ioctl.o msg.o roster.o scoutfs_trace.o super.o trace.o \ - trans.o treap.o wrlock.o + inode.o ioctl.o msg.o scoutfs_trace.o super.o trace.o trans.o \ + treap.o scoutfs-y += last.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c index c275d7f7..730cc7b2 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -22,7 +22,7 @@ #include "key.h" #include "super.h" #include "btree.h" -#include "wrlock.h" +#include "trans.h" /* * Directory entries are stored in entries with offsets calculated from @@ -310,9 +310,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; - DECLARE_SCOUTFS_WRLOCK_HELD(held); int bytes; - u64 ino; int ret; u64 h; @@ -323,11 +321,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_alloc_ino(sb, &ino); - if (ret) - return ret; - - ret = scoutfs_wrlock_lock(sb, &held, 2, scoutfs_ino(dir), ino); + ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -335,7 +329,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out; - inode = scoutfs_new_inode(sb, dir, ino, mode, rdev); + inode = scoutfs_new_inode(sb, dir, mode, rdev); if (IS_ERR(inode)) { ret = PTR_ERR(inode); goto out; @@ -392,7 +386,7 @@ out: /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); - scoutfs_wrlock_unlock(sb, &held); + scoutfs_release_trans(sb); return ret; } @@ -417,7 +411,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - DECLARE_SCOUTFS_WRLOCK_HELD(held); struct dentry_info *di; struct scoutfs_key key; int ret = 0; @@ -429,8 +422,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - ret = scoutfs_wrlock_lock(sb, &held, 2, scoutfs_ino(dir), - scoutfs_ino(inode)); + ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -459,7 +451,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(dir); out: - scoutfs_wrlock_unlock(sb, &held); + scoutfs_release_trans(sb); return ret; } diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 2082d17e..a6e75fc7 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -18,7 +18,7 @@ #include "inode.h" #include "key.h" #include "filerw.h" -#include "wrlock.h" +#include "trans.h" #include "scoutfs_trace.h" #include "btree.h" @@ -130,7 +130,6 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) { struct inode *inode = page->mapping->host; DECLARE_SCOUTFS_BTREE_CURSOR(curs); - DECLARE_SCOUTFS_WRLOCK_HELD(held); struct super_block *sb = inode->i_sb; struct scoutfs_key key; struct data_region dr; @@ -140,7 +139,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) set_page_writeback(page); - ret = scoutfs_wrlock_lock(sb, &held, 1, scoutfs_ino(inode)); + ret = scoutfs_hold_trans(sb); if (ret) goto out; @@ -162,7 +161,7 @@ static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) } scoutfs_btree_release(&curs); - scoutfs_wrlock_unlock(sb, &held); + scoutfs_release_trans(sb); out: if (ret) { SetPageError(page); @@ -199,7 +198,6 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_WRLOCK_HELD(held); unsigned off; trace_scoutfs_write_end(scoutfs_ino(inode), pos, len, copied); @@ -220,10 +218,10 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, * up the robust metadata support that's needed to do a * good job with the data pats. */ - if (!scoutfs_wrlock_lock(sb, &held, 1, scoutfs_ino(inode))) { + if (!scoutfs_hold_trans(sb)) { if (!scoutfs_dirty_inode_item(inode)) scoutfs_update_inode_item(inode); - scoutfs_wrlock_unlock(sb, &held); + scoutfs_release_trans(sb); } } diff --git a/kmod/src/format.h b/kmod/src/format.h index 5c8ecb13..5deca747 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -159,8 +159,6 @@ struct scoutfs_super_block { } __packed; #define SCOUTFS_ROOT_INO 1 -#define SCOUTFS_INO_BATCH_SHIFT 20 -#define SCOUTFS_INO_BATCH (1 << SCOUTFS_INO_BATCH_SHIFT) struct scoutfs_timespec { __le64 sec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index a9bc7010..0543c991 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -22,7 +22,6 @@ #include "btree.h" #include "dir.h" #include "filerw.h" -#include "wrlock.h" #include "scoutfs_trace.h" /* @@ -248,69 +247,24 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } -/* - * This will need to try and find a mostly idle shard. For now we only - * have one :). - */ -static int get_next_ino_batch(struct super_block *sb) +static int alloc_ino(struct super_block *sb, u64 *ino) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_SCOUTFS_WRLOCK_HELD(held); + struct scoutfs_super_block *super = &sbi->super; int ret; - ret = scoutfs_wrlock_lock(sb, &held, 1, 1); - if (ret) - return ret; - spin_lock(&sbi->next_ino_lock); - if (!sbi->next_ino_count) { - sbi->next_ino = le64_to_cpu(sbi->super.next_ino); - if (sbi->next_ino + SCOUTFS_INO_BATCH < sbi->next_ino) { - ret = -ENOSPC; - } else { - le64_add_cpu(&sbi->super.next_ino, SCOUTFS_INO_BATCH); - sbi->next_ino_count = SCOUTFS_INO_BATCH; - ret = 0; - } + + if (super->next_ino == 0) { + ret = -ENOSPC; + } else { + *ino = le64_to_cpu(super->next_ino); + le64_add_cpu(&super->next_ino, 1); + ret = 0; } + spin_unlock(&sbi->next_ino_lock); - scoutfs_wrlock_unlock(sb, &held); - - return ret; -} - -/* - * Inode allocation is at the core of supporting parallel creation. - * Each mount needs to allocate from a pool of free inode numbers which - * map to a shard that it has locked. - */ -int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - do { - /* don't really care if this is racey */ - if (!sbi->next_ino_count) { - ret = get_next_ino_batch(sb); - if (ret) - break; - } - - spin_lock(&sbi->next_ino_lock); - - if (sbi->next_ino_count) { - *ino = sbi->next_ino++; - sbi->next_ino_count--; - ret = 0; - } else { - ret = -EAGAIN; - } - spin_unlock(&sbi->next_ino_lock); - - } while (ret == -EAGAIN); - return ret; } @@ -319,14 +273,18 @@ int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) * creating links to it and updating it. @dir can be null. */ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - u64 ino, umode_t mode, dev_t rdev) + umode_t mode, dev_t rdev) { DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; struct scoutfs_key key; struct inode *inode; + u64 ino; int ret; + ret = alloc_ino(sb, &ino); + if (ret) + return ERR_PTR(ret); inode = new_inode(sb); if (!inode) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 5dfc6b1a..f2846201 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -18,8 +18,6 @@ static inline u64 scoutfs_ino(struct inode *inode) return SCOUTFS_I(inode)->ino; } -int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); - struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); @@ -27,7 +25,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - u64 ino, umode_t mode, dev_t rdev); + umode_t mode, dev_t rdev); void scoutfs_inode_exit(void); int scoutfs_inode_init(void); diff --git a/kmod/src/roster.c b/kmod/src/roster.c deleted file mode 100644 index e05b7160..00000000 --- a/kmod/src/roster.c +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include - -#include "super.h" -#include "wire.h" -#include "wrlock.h" -#include "roster.h" - -/* - * The roster tracks all the mounts on nodes that are working with a - * scoutfs volume. - * - * This trivial first pass lets us test multiple mounts on the same - * node. It'll get a lot more involved as all the nodes manage a roster - * in the shared device. - */ -static DEFINE_MUTEX(roster_mutex); -static u64 roster_next_id = 1; -static LIST_HEAD(roster_list); - -/* - * A new mount is adding itself to the roster. It gets a new increasing - * id assigned and all the other mounts are told that it's now a member. - */ -int scoutfs_roster_add(struct super_block *sb) -{ - struct scoutfs_sb_info *us = SCOUTFS_SB(sb); - struct scoutfs_sb_info *them; - - mutex_lock(&roster_mutex); - list_add_tail(&us->roster_head, &roster_list); - us->roster_id = roster_next_id++; - - list_for_each_entry(them, &roster_list, roster_head) { - if (us->roster_id != them->roster_id) { - scoutfs_wrlock_roster_update(them->sb, us->roster_id, - true); - } - } - - mutex_unlock(&roster_mutex); - - return 0; -} - -/* - * A mount is removing itself to the roster. All the other remaining - * mounts are told that it has gone away. - * - * This is safe to call without having called _add. - */ -void scoutfs_roster_remove(struct super_block *sb) -{ - struct scoutfs_sb_info *us = SCOUTFS_SB(sb); - struct scoutfs_sb_info *them; - - mutex_lock(&roster_mutex); - - if (!list_empty(&us->roster_head)) { - list_del_init(&us->roster_head); - - list_for_each_entry(them, &roster_list, roster_head) - scoutfs_wrlock_roster_update(them->sb, us->roster_id, - false); - } - - mutex_unlock(&roster_mutex); -} - -static int process_message(struct super_block *sb, u64 peer_id, - struct scoutfs_message *msg) -{ - int ret = 0; - - switch (msg->cmd) { - case SCOUTFS_MSG_WRLOCK_REQUEST: - ret = scoutfs_wrlock_process_request(sb, peer_id, - &msg->request); - break; - case SCOUTFS_MSG_WRLOCK_GRANT: - scoutfs_wrlock_process_grant(sb, &msg->grant); - ret = 0; - break; - default: - ret = -EINVAL; - } - - return ret; -} - -/* - * Send a message to a specific member of the roster identified by its - * id. - * - * We don't actually send anything, we call directly into the receivers - * message processing path with the caller's message. - */ -void scoutfs_roster_send(struct super_block *sb, u64 peer_id, - struct scoutfs_message *msg) -{ - struct scoutfs_sb_info *us = SCOUTFS_SB(sb); - struct scoutfs_sb_info *them; - int ret; - - mutex_lock(&roster_mutex); - - list_for_each_entry(them, &roster_list, roster_head) { - if (them->roster_id == peer_id) { - ret = process_message(them->sb, us->roster_id, msg); - break; - } - } - - /* XXX errors? */ - - mutex_unlock(&roster_mutex); -} - -/* - * Send a message to all of the current members which have an id greater - * than the caller's specified id. - * - * We don't actually send anything, we call directly into the receivers - * message processing path with the caller's message. - */ -void scoutfs_roster_broadcast(struct super_block *sb, u64 since_id, - struct scoutfs_message *msg) -{ - struct scoutfs_sb_info *us = SCOUTFS_SB(sb); - struct scoutfs_sb_info *them; - int ret; - - mutex_lock(&roster_mutex); - - list_for_each_entry(them, &roster_list, roster_head) { - if (us->roster_id != them->roster_id && - them->roster_id > since_id) { - ret = process_message(them->sb, us->roster_id, msg); - if (ret) - break; - } - } - - /* XXX errors? */ - - mutex_unlock(&roster_mutex); -} diff --git a/kmod/src/roster.h b/kmod/src/roster.h deleted file mode 100644 index e78a7cc9..00000000 --- a/kmod/src/roster.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _SCOUTFS_ROSTER_H_ -#define _SCOUTFS_ROSTER_H_ - -struct scoutfs_message; - -int scoutfs_roster_add(struct super_block *sb); -void scoutfs_roster_remove(struct super_block *sb); - -void scoutfs_roster_send(struct super_block *sb, u64 peer_id, - struct scoutfs_message *msg); -void scoutfs_roster_broadcast(struct super_block *sb, u64 since_id, - struct scoutfs_message *msg); - -#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 5e409db0..a90caf75 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -26,8 +26,6 @@ #include "block.h" #include "counters.h" #include "trans.h" -#include "roster.h" -#include "wrlock.h" #include "trace.h" #include "scoutfs_trace.h" @@ -162,7 +160,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - INIT_LIST_HEAD(&sbi->roster_head); sbi->ctr = atomic64_inc_return(&scoutfs_sb_ctr); @@ -174,8 +171,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: scoutfs_setup_trans(sb) ?: - scoutfs_wrlock_setup(sb) ?: - scoutfs_roster_add(sb) ?: scoutfs_read_buddy_chunks(sb); if (ret) return ret; @@ -206,8 +201,6 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { - scoutfs_roster_remove(sb); - scoutfs_wrlock_teardown(sb); scoutfs_shutdown_trans(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index c99a77d7..d588f36a 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -9,7 +9,6 @@ struct scoutfs_counters; struct buddy_alloc; -struct wrlock_context; struct scoutfs_sb_info { struct super_block *sb; @@ -19,8 +18,6 @@ struct scoutfs_sb_info { struct scoutfs_super_block super; spinlock_t next_ino_lock; - u64 next_ino; - u64 next_ino_count; spinlock_t block_lock; struct radix_tree_root block_radix; @@ -48,11 +45,6 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; - - struct list_head roster_head; - u64 roster_id; - - struct wrlock_context *wrlock_context; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) diff --git a/kmod/src/wire.h b/kmod/src/wire.h deleted file mode 100644 index 474703bf..00000000 --- a/kmod/src/wire.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef _SCOUTFS_WIRE_H_ -#define _SCOUTFS_WIRE_H_ - -/* an arbitrarily small number to keep things reasonable */ -#define SCOUTFS_WRLOCK_MAX_SHARDS 5 - -enum { - SCOUTFS_MSG_WRLOCK_REQUEST = 1, - SCOUTFS_MSG_WRLOCK_GRANT = 2, -}; - -struct scoutfs_wrlock_id { - __le64 counter; - __le32 jitter; -} __packed; - -struct scoutfs_wrlock_request { - struct scoutfs_wrlock_id wid; - u8 nr_shards; - __le32 shards[SCOUTFS_WRLOCK_MAX_SHARDS]; -} __packed; - -struct scoutfs_wrlock_grant { - struct scoutfs_wrlock_id wid; -} __packed; - -struct scoutfs_message { - u8 cmd; - u8 len; - union { - struct scoutfs_wrlock_grant grant; - struct scoutfs_wrlock_request request; - } __packed; -} __packed; - -#endif diff --git a/kmod/src/wrlock.c b/kmod/src/wrlock.c deleted file mode 100644 index 9bb7b06c..00000000 --- a/kmod/src/wrlock.c +++ /dev/null @@ -1,1000 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "wire.h" -#include "wrlock.h" -#include "trans.h" -#include "roster.h" -#include "trace.h" - -/* - * The persistent structures in each shard in a scoutfs volume can only - * have one writer at a time. Mounts send messages around to request - * and grant locks on each shard. (Reads are fully unlocked and have - * enough metadata to detect and retry reads that raced and were - * inconsistent.) - * - * When a local task needs to lock some shards it sends a request to all - * the other mounts listing all the shards. If the receiving mounts - * don't have any of the shards locked they send a grant reply. - * - * Each mount has a granted lock and a tree of blocked lock entries for - * every shard. Local lock attempts and remote requests are always - * inserted into the tree. The first entry in the tree can be unblocked - * if the granted lock in the shard doesn't block it. When local - * entries are granted the locking task is allowed to start modifying - * the shard. While they're modifying the shard their granted locks - * block remote locks from being sent replies. Once the writers under - * the lock are done the grant can be removed and the remote entry is - * sent a reply and freed. - * - * Processes can try to lock multiple shards so entries can be present - * in the blocking tree and granted pointer on multiple shards. They're - * only unblocked when they're the first entry in all their shards' - * blocking trees. - * - * The entries have to be very carefully ordered in the trees on all the - * mounts to avoid locking cycle deadlocks. We can't have two mounts - * race to lock the same shard and both have their local ungranted entry - * blocking the remote's request entry. To address this entries aren't - * sorted in the tree by time. They're sorted by an id. This ensures - * that all of the entries will have the same blocking tree on all the - * mounts and so will always be processed in the same order. - * - * Entries are created on a mount when a task tries to lock some shards. - * The id is constructed from a counter, a random number, and the - * mount's unique id. The counter is one greater than the greatest - * counter ever seen in received lock requests. This ensures that lock - * attempts that don't race are granted in order. But attempts can race - * so entries can have the same counter. Next they're sorted by a - * random number to ensure a kind of fairness. Then if the mounts are - * unlucky enough to chose the same number we fall back to sorting by - * the unique mount id. - * - * The roster determines the set of mounts that are participating in the - * locking protocol. We have to carefully manage the entries as mounts - * join and leave the cluster. When mounts join we send them all our - * blocking locks and if they leave we remove their entries and resend - * all our blocked entries to everyone because we don't track which - * mounts had send grants to which local blocked entries, or not. - * - * XXX - * - sync if we revoke a local grant before we send a reply - */ - -/* - * Every mount tracks their write locking state for all the shards in - * the volume. - */ -struct wrlock_context { - struct super_block *sb; - wait_queue_head_t waitq; - spinlock_t lock; - - struct rb_root id_root; - struct list_head mark_list; - struct list_head send_list; - struct workqueue_struct *send_workq; - struct work_struct send_work; - - /* private copies of roster state used under the lock */ - long grants_needed; - u64 last_peer_id; - - u64 next_id_counter; - - /* XXX redundant in the super? only one for now ;) */ - u32 nr_shards; - struct wrlock_context_shard { - struct list_head mark_head; - struct rb_root blocked_root; - struct wrlock_entry *granted; - } shards[0]; -}; - -/* a native version of the wire wrlock_id that includes the roster id */ -struct wrlock_id { - u64 counter; - u32 jitter; - u64 roster_id; -}; - -/* - * Entries represent an attempt to lock multiple shards. - * - * Local entries exist on the context that initiated the request. They - * count the number of grant replies and then count the number of - * writers actively modifying the shards under the lock. - * - * Remote entries only exist while other entries are before them in the - * blocked trees in any of their shards. Once they're first in all the - * blocked trees a grant message is sent and they're freed. - */ -struct wrlock_entry { - struct rb_node id_node; - struct list_head send_head; - - /* local lock tasks wait for the entry to be granted */ - struct task_struct *waiter; - struct scoutfs_wrlock_held *held; - long grants; - long writers; - - /* tells roster broadcast who to send to */ - u64 last_peer_id; - struct wrlock_id id; - - u8 nr_shards; - struct wrlock_entry_shard { - struct rb_node blocked_node; - u32 shd; - u8 index; - } shards[SCOUTFS_WRLOCK_MAX_SHARDS]; -}; - -#define ENTF "ent id %llu.%llu.%llu" -#define ENTA(ent) ent->id.counter, ent->id.jitter, ent->id.roster_id - -static struct wrlock_entry *ent_from_blocked_node(struct rb_node *node) -{ - struct wrlock_entry_shard *shard; - - shard = container_of(node, struct wrlock_entry_shard, - blocked_node); - return container_of(shard, struct wrlock_entry, - shards[shard->index]); -} - -/* Return the first blocked entry */ -static struct wrlock_entry *blocked_ent(struct wrlock_context_shard *shard) -{ - struct rb_node *node = rb_first(&shard->blocked_root); - - return node ? ent_from_blocked_node(node) : NULL; -} - -static int cmp_u64s(u64 a, u64 b) -{ - return a < b ? -1 : a > b ? 1 : 0; -} - -static int cmp_u32s(u32 a, u32 b) -{ - return a < b ? -1 : a > b ? 1 : 0; -} - -static int cmp_ids(struct wrlock_id *a, struct wrlock_id *b) -{ - return cmp_u64s(a->counter, b->counter) ?: - cmp_u32s(a->jitter, b->jitter) ?: - cmp_u64s(a->roster_id, b->roster_id); -} - -static void insert_ent_shard(struct rb_root *root, struct wrlock_entry *ins, - struct rb_node *ins_node) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct wrlock_entry *ent; - - while (*node) { - parent = *node; - ent = ent_from_blocked_node(*node); - - if (cmp_ids(&ins->id, &ent->id) < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - rb_link_node(ins_node, parent, node); - rb_insert_color(ins_node, root); -} - -/* Insert the entry into the blocked tree for each of its shards. */ -/* - * Insert an entry in to all of its trees. All entries have to be on - * the blocked tree for all of its shards. - * - * But the id tree is a little lazy. It's only used to look up local - * entries when grants are received. It could be a hash table instead of - * a tree and remote entries don't need to be in it. But this re-use - * of the tree code is easy and isn't that expensive compared to all - * the rest of the processing. - */ -static void insert_ent(struct wrlock_context *ctx, struct wrlock_entry *ins) -{ - int i; - - insert_ent_shard(&ctx->id_root, ins, &ins->id_node); - - for (i = 0; i < ins->nr_shards; i++) - insert_ent_shard(&ctx->shards[ins->shards[i].shd].blocked_root, - ins, &ins->shards[i].blocked_node); - - scoutfs_trace(ctx->sb, "inserted "ENTF, ENTA(ins)); -} - -static struct wrlock_entry *lookup_ent(struct wrlock_context *ctx, - struct wrlock_id *id) -{ - struct rb_node *node = ctx->id_root.rb_node; - struct wrlock_entry *ent; - int cmp; - - while (node) { - ent = ent_from_blocked_node(node); - - cmp = cmp_ids(id, &ent->id); - if (cmp < 0) - node = node->rb_left; - else if (cmp > 0) - node = node->rb_right; - else - return ent; - } - - return NULL; -} - -static void erase_and_clear(struct rb_node *node, struct rb_root *root) -{ - if (!RB_EMPTY_NODE(node)) { - rb_erase(node, root); - RB_CLEAR_NODE(node); - } -} - -/* remove all of the entry's rb nodes from the context's trees */ -static void erase_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) -{ - int i; - - erase_and_clear(&ent->id_node, &ctx->id_root); - - for (i = 0; i < ent->nr_shards; i++) - erase_and_clear(&ent->shards[i].blocked_node, - &ctx->shards[ent->shards[i].shd].blocked_root); - - scoutfs_trace(ctx->sb, "erased "ENTF, ENTA(ent)); -} - -static struct wrlock_entry *alloc_ent(void) -{ - struct wrlock_entry *ent; - int i; - - ent = kzalloc(sizeof(*ent), GFP_NOFS); - if (!ent) - return ERR_PTR(-ENOMEM); - - RB_CLEAR_NODE(&ent->id_node); - INIT_LIST_HEAD(&ent->send_head); - - /* for container_of to find the ent while walking shard nodes */ - for (i = 0; i < ARRAY_SIZE(ent->shards); i++) { - RB_CLEAR_NODE(&ent->shards[i].blocked_node); - ent->shards[i].index = i; - } - - return ent; -} - -/* - * Callers try to free the ent every time they remove a reference to it - * from the context and are done with it. We only free it if there are - * no more references to it in the context. This is called with the - * context lock held so it's not racing with other tasks that are - * removing references and trying to free. - */ -static void try_free_ent(struct wrlock_context *ctx, struct wrlock_entry *ent) -{ - int i; - - if (!RB_EMPTY_NODE(&ent->id_node) || !list_empty(&ent->send_head)) - return; - - for (i = 0; i < ent->nr_shards; i++) { - if (!RB_EMPTY_NODE(&ent->shards[i].blocked_node) || - ctx->shards[ent->shards[i].shd].granted == ent) - return; - } - - WARN_ON_ONCE(ent->writers); - scoutfs_trace(ctx->sb, "freed "ENTF, ENTA(ent)); - - kfree(ent); -} - -static bool is_local(struct wrlock_context *ctx, struct wrlock_entry *ent) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(ctx->sb); - - return ent->id.roster_id == sbi->roster_id; -} - -/* - * An entry in the blocked tree can be blocked for a few reasons: - * - * - a local entry hasn't received its grant replies yet - * - there are blocked entries before it on any of its shards - * - a remote entry is waiting for local writers to drain - * - * Note in particular that a local entry isn't blocked by granted writers - * because it'll join them and that remote entries aren't blocked by local - * grants with no writers because it revokes them to send a grant reply. - */ -static bool is_blocked(struct wrlock_context *ctx, struct wrlock_entry *ent) -{ - struct wrlock_context_shard *shard; - int i; - - if (is_local(ctx, ent) && ent->grants < ctx->grants_needed) - return true; - - for (i = 0; i < ent->nr_shards; i++) { - if (rb_prev(&ent->shards[i].blocked_node)) - return true; - - if (!is_local(ctx, ent)) { - shard = &ctx->shards[ent->shards[i].shd]; - if (shard->granted && shard->granted->writers) - return true; - } - } - - return false; -} - -/* mark a given shard for later processing to see if entries aren't blocked */ -static void mark_context_shard(struct wrlock_context *ctx, u32 shard) -{ - struct list_head *head = &ctx->shards[shard].mark_head; - - if (list_empty(head)) - list_add_tail(head, &ctx->mark_list); -} - -static void mark_ent_shards(struct wrlock_context *ctx, - struct wrlock_entry *ent) -{ - int i; - - for (i = 0; i < ent->nr_shards; i++) - mark_context_shard(ctx, ent->shards[i].shd); -} - -static void queue_send(struct wrlock_context *ctx, struct wrlock_entry *ent) -{ - if (list_empty(&ent->send_head)) { - list_add_tail(&ent->send_head, &ctx->send_list); - queue_work(ctx->send_workq, &ctx->send_work); - scoutfs_trace(ctx->sb, "queued "ENTF, ENTA(ent)); - } -} - -/* - * Try to unblock entries in the shard. We're done when the first entry - * in the shard is still blocked. - * - * If we unblock a remote entry then we have to send its grant message. - * If there is a granted local entry but it has no writers then we - * remove it so that future writers will have to request a new lock from - * the remote peer whose request we granted. - * - * If we unblock a local entry then we move it to the granted pointers - * for each of its shards. There are two tricky cases here. - * - * The first is a local entry being granted which covers more shards - * than the current granted entry on some of its shards. We don't want - * the larger unblocked entry to wait for the smaller granted entry's - * writers to drain. Instead we set the granted pointers to the new - * unblocked large entry after giving it the smaller granted entry's - * writer counters. Unlocking will drop the write counters on whatever - * entry is currently granted on its shards. - * - * The second is making sure that a waiting locking task gets a chance - * to work with a newly granted local entry before the next blocking - * remote entry revokes it. We increment the writers count the moment a - * local entry is granted. It will stay that way until the task drops - * the writer count. We just have to be careful to address all the - * races with the task sleeping, waking, and interrupting. - */ -static void unblock_shard(struct wrlock_context *ctx, - struct wrlock_context_shard *shard) -{ - struct wrlock_entry *ent; - struct wrlock_entry *gr; - int i; - - ent = blocked_ent(shard); - if (!ent) - return; - - if (is_blocked(ctx, ent)) { - /* send initial requests for local blocked entries */ - if (ent->last_peer_id < ctx->last_peer_id) - queue_send(ctx, ent); - return; - } - - scoutfs_trace(ctx->sb, "unblocked "ENTF, ENTA(ent)); - - erase_ent(ctx, ent); - mark_ent_shards(ctx, ent); - - /* unblocked remote entries remove local grants and send replies */ - if (!is_local(ctx, ent)) { - for (i = 0; i < ent->nr_shards; i++) { - shard = &ctx->shards[ent->shards[i].shd]; - - if (shard->granted) { - gr = shard->granted; - WARN_ON_ONCE(gr->writers); - - shard->granted = NULL; - try_free_ent(ctx, gr); - } - } - - queue_send(ctx, ent); - return; - } - - /* grant the entry on all its shards */ - for (i = 0; i < ent->nr_shards; i++) { - shard = &ctx->shards[ent->shards[i].shd]; - - /* the ent couldn't have been granted if it was blocked */ - WARN_ON_ONCE(shard->granted == ent); - - if (shard->granted) { - gr = shard->granted; - ent->writers += gr->writers; - - shard->granted = NULL; - try_free_ent(ctx, gr); - } - - shard->granted = ent; - if (ent->waiter) - ent->writers++; - - scoutfs_trace(ctx->sb, "granted ctx 0x%llx shd %llu wr %llu", - ctx, ent->shards[i].shd, ent->writers); - } - - if (ent->waiter) { - /* the task is responsible for the writer count if nr is set */ - ent->held->nr_shards = ent->nr_shards; - smp_mb(); /* wait_event condition isn't locked */ - wake_up_process(ent->waiter); - ent->waiter = NULL; - ent->held = NULL; - } -} - -/* - * Walk all the shards that have been marked and see if their blocked - * entry is still blocked. As we unblock entries we mark all their - * shards and keep going until the blocked entries in the shards - * stabilize. - */ -static void unblock_marked_shards(struct wrlock_context *ctx) -{ - struct wrlock_context_shard *shard; - - while ((shard = list_first_entry_or_null(&ctx->mark_list, - struct wrlock_context_shard, - mark_head))) { - list_del_init(&shard->mark_head); - unblock_shard(ctx, shard); - } -} - -/* - * Statically round robin every 1M inodes to each shard. - * - * XXX this will almost certainly need to be more clever. We'll want - * to size the batching more carefully and we'll need to cope with growing - * and shrinking the number of shards. - */ -static u32 ino_shd(struct wrlock_context *ctx, u64 ino) -{ - return (u32)(ino >> SCOUTFS_INO_BATCH_SHIFT) % ctx->nr_shards; -} - -/* - * Shards in entries are sorted and unique to make receive verification - * easier. Entries will only have a small handful of shards. - */ -static void add_ent_shd(struct wrlock_entry *ent, u32 shd) -{ - int i; - - for (i = 0; i < ent->nr_shards; i++) { - if (shd < ent->shards[i].shd) - swap(shd, ent->shards[i].shd); - else if (shd == ent->shards[i].shd) - return; - } - - ent->shards[i].shd = shd; - ent->nr_shards++; -} - -/* - * Get write locks on the shards that contain the given inodes. - * - * We always insert a new entry so that local attempts are inserted in - * the blocking tree after blocked remote entries. This way local lock - * matching doesn't stave remote lock attempts. - * - * In the fast path the inserted entry will be first and all its shards - * will be granted so we'll increase entry writer counts and return. In - * the slow path we send lock requests and sleep until we get grant - * replies. - * - * The writer counts are set when our entry is granted while we're still - * waiting for it so that we're guaranteed to get to work with our - * granted lock before a remote request has a chance to revoke it. - */ -int scoutfs_wrlock_lock(struct super_block *sb, - struct scoutfs_wrlock_held *held, int nr_inos, ...) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct wrlock_entry *ent; - va_list args; - int ret; - int i; - - if (WARN_ON_ONCE(nr_inos <= 0 || nr_inos > SCOUTFS_WRLOCK_MAX_SHARDS) || - WARN_ON_ONCE(held->nr_shards)) - return -EINVAL; - - ent = alloc_ent(); - if (!ent) - return -ENOMEM; - - va_start(args, nr_inos); - while (nr_inos--) { - /* XXX verify inodes? */ - add_ent_shd(ent, ino_shd(ctx, va_arg(args, u64))); - } - va_end(args); - - /* held's nr_shards is set when the ent is granted and writers inced */ - for (i = 0; i < ent->nr_shards; i++) - held->shards[i] = ent->shards[i].shd; - - ent->waiter = current; - ent->held = held; - ent->id.jitter = get_random_int(); /* XXX how expensive? */ - ent->id.roster_id = sbi->roster_id; - - /* the context owns and can free the entry after we unlock */ - spin_lock(&ctx->lock); - - ent->id.counter = ctx->next_id_counter++; - - insert_ent(ctx, ent); - mark_ent_shards(ctx, ent); - unblock_marked_shards(ctx); - - spin_unlock(&ctx->lock); - - ret = wait_event_interruptible(ctx->waitq, held->nr_shards); - if (ret == 0) - ret = scoutfs_hold_trans(sb); - - scoutfs_trace(sb, "lock nr %llu ret %lld", held->nr_shards, ret); - - /* unlock on error locks the context before using held.nr_shards */ - if (ret) - scoutfs_wrlock_unlock(sb, held); - - return ret; -} - -/* - * The held shards must have had granted entries for us to increment the - * write counts. The increased write counts should have pinned entries - * to the shards so they must still be around for us to decrease the - * counts. - * - * If we're the last writer of an entry then we'll check to see if any - * of its shards have blocked remote entries that can now make progress. - * - * XXX we'd need to sync dirty blocks before sending the grant. - */ -void scoutfs_wrlock_unlock(struct super_block *sb, - struct scoutfs_wrlock_held *held) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct wrlock_context_shard *shard; - u32 shd; - int i; - - scoutfs_release_trans(sb); - - spin_lock(&ctx->lock); - - for (i = 0; i < held->nr_shards; i++) { - shd = held->shards[i]; - shard = &ctx->shards[shd]; - - /* XXX this would imply unlocked writing, very bad indeed */ - if (WARN_ON_ONCE(!shard->granted) || - WARN_ON_ONCE(shard->granted->writers <= 0)) - continue; - - shard->granted->writers--; - - scoutfs_trace(sb, "unlock ctx 0x%llx shd %llu wr %llu", - ctx, shd, shard->granted->writers); - - if (shard->granted->writers == 0) - mark_context_shard(ctx, shd); - } - - unblock_marked_shards(ctx); - - spin_unlock(&ctx->lock); - -} - -/* - * Process an incoming request message. We allocate and insert an entry - * for the request. When it's not blocked by previous entries or a - * granted entry on all its shards then we send a reply and free the - * entry. - * - * Shard numbers in the incoming request must be unique and sorted. - */ -int scoutfs_wrlock_process_request(struct super_block *sb, u64 peer_id, - struct scoutfs_wrlock_request *req) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct wrlock_entry *ent; - int ret = 0; - u32 shd; - u32 prev; - int i; - - ent = alloc_ent(); - if (!ent) - return -ENOMEM; - - if (req->nr_shards > SCOUTFS_WRLOCK_MAX_SHARDS) { - ret = -EINVAL; - goto out; - } - - for (i = 0, prev = 0; i < req->nr_shards; prev = shd, i++) { - shd = le32_to_cpu(req->shards[i]); - - if (shd >= ctx->nr_shards || (prev && shd <= prev)) { - ret = -EINVAL; - goto out; - } - - add_ent_shd(ent, shd); - } - - ent->id.counter = le64_to_cpu(req->wid.counter); - ent->id.jitter = le32_to_cpu(req->wid.jitter); - ent->id.roster_id = peer_id; - - spin_lock(&ctx->lock); - - ctx->next_id_counter = max(ent->id.counter + 1, ctx->next_id_counter); - - insert_ent(ctx, ent); - mark_ent_shards(ctx, ent); - unblock_marked_shards(ctx); - - spin_unlock(&ctx->lock); - -out: - if (ret) - kfree(ent); - return ret; -} - -/* - * Process an incoming grant message. The sending peer is telling us - * that they don't have any entries blocking our lock. We increment its - * count and wake the locker on the last grant. - * - * An entry won't be found at the id if the process attempting the lock - * exited and removed the entry before all the grants arrived. - * - * XXX freak out if grants is greater than grants_needed? That'd imply - * that we could have prematurely given a locker access to its shards. - */ -void scoutfs_wrlock_process_grant(struct super_block *sb, - struct scoutfs_wrlock_grant *grant) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct wrlock_entry *ent; - struct wrlock_id id = { - .counter = le64_to_cpu(grant->wid.counter), - .jitter = le32_to_cpu(grant->wid.jitter), - .roster_id = sbi->roster_id, - }; - - spin_lock(&ctx->lock); - - ent = lookup_ent(ctx, &id); - if (ent) { - ent->grants++; - - scoutfs_trace(sb, "grant rx "ENTF" grants %llu needed %llu", - ENTA(ent), ent->grants, ctx->grants_needed); - - if (ent->grants == ctx->grants_needed) { - mark_ent_shards(ctx, ent); - unblock_marked_shards(ctx); - } - } - - spin_unlock(&ctx->lock); -} - -/* - * Send wrlock messages to peers. Entries are put on the send queue - * when we need to either broadcast requests to new peers or send a - * grant reply to a specific requesting peer. - * - * XXX As currently imagined any send failures trigger reconnection and - * recovery. We need a bit more clarity on what the roster - * implementation before worrying too much about the details of recovery - * in here. - */ -static void send_work_func(struct work_struct *work) -{ - struct wrlock_context *ctx = container_of(work, struct wrlock_context, - send_work); - struct super_block *sb = ctx->sb; - struct scoutfs_message msg; - struct wrlock_entry *ent; - struct wrlock_entry *tmp; - u64 peer_id; - int i; - - spin_lock(&ctx->lock); - - list_for_each_entry_safe(ent, tmp, &ctx->send_list, send_head) { - - if (is_local(ctx, ent)) { - msg.cmd = SCOUTFS_MSG_WRLOCK_REQUEST; - msg.request.wid.counter = cpu_to_le64(ent->id.counter); - msg.request.wid.jitter = cpu_to_le32(ent->id.jitter); - msg.request.nr_shards = ent->nr_shards; - for (i = 0; i < ent->nr_shards; i++) { - msg.request.shards[i] = - cpu_to_le32(ent->shards[i].shd); - } - - msg.len = offsetof(struct scoutfs_wrlock_request, - shards[ent->nr_shards]); - peer_id = ent->last_peer_id; - ent->last_peer_id = ctx->last_peer_id; - } else { - msg.cmd = SCOUTFS_MSG_WRLOCK_GRANT; - msg.grant.wid.counter = cpu_to_le64(ent->id.counter); - msg.grant.wid.jitter = cpu_to_le32(ent->id.jitter); - - msg.len = sizeof(msg.grant); - peer_id = ent->id.roster_id; - } - - scoutfs_trace(sb, "send "ENTF" cmd %llu", ENTA(ent), msg.cmd); - - list_del_init(&ent->send_head); - try_free_ent(ctx, ent); - - spin_unlock(&ctx->lock); - - if (msg.cmd == SCOUTFS_MSG_WRLOCK_GRANT) - scoutfs_roster_send(sb, peer_id, &msg); - else - scoutfs_roster_broadcast(sb, peer_id, &msg); - - spin_lock(&ctx->lock); - } - - spin_unlock(&ctx->lock); -} - -/* - * The roster tells us when mounts join or leave the cluster. - * - * Our job is easy if a peer is joining because they don't have any - * entries yet. They could start sending requests immediately and their - * entries could be inserted behind our blocked local entries. We send - * them all our blocked entries so that they can grant them and make - * forward progress in that case. - * - * If a peer is leaving then we have two problems. - * - * First they might have already granted some entries but we can't tell - * which. We don't track grant replies per peer. We can't adjust the - * entry grant counts to match a smaller number of needed grants. So we - * reset all the blocked local entries and resend them to everyone. We - * reset the id so that we're not confused by grants in flight. It's - * not great but it's simple and rare. - * - * XXX Worse, they might have held locks. We'd need to wait a grace - * period or fence them so that we're sure that they are no longer - * writing to shards. - */ -void scoutfs_wrlock_roster_update(struct super_block *sb, u64 peer_id, - bool join) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct rb_node *node; - struct wrlock_entry *ent; - LIST_HEAD(list); - int i; - - spin_lock(&ctx->lock); - - /* take the peer change into account before walking entries */ - if (join) { - ctx->grants_needed++; - ctx->last_peer_id = peer_id; - } else { - ctx->grants_needed--; - } - - scoutfs_trace(sb, "update ctx 0x%llx peer_id %llu join %llu gr %llu", - ctx, peer_id, join, ctx->grants_needed); - - /* - * Walk all the blocked entries on all the shards. Entries can - * be on multiple shards so we're careful to only modify them on - * the first visit. - */ - for (i = 0; i < ctx->nr_shards; i++) { - node = rb_first(&ctx->shards[i].blocked_root); - while (node) { - ent = ent_from_blocked_node(node); - node = rb_next(node); - - /* drop remote blocked entries from a leaving peer */ - if (!join && ent->id.roster_id == peer_id) { - erase_ent(ctx, ent); - mark_ent_shards(ctx, ent); - try_free_ent(ctx, ent); - } - - /* send blocked local locks just to the new peer */ - if (join && is_local(ctx, ent)) - queue_send(ctx, ent); - - /* reset and resend local entries when leaving */ - if (!join && is_local(ctx, ent) && ent->last_peer_id) { - ent->grants = 0; - ent->last_peer_id = 0; - ent->id.counter = ctx->next_id_counter++; - ent->id.jitter = get_random_int(); - - erase_ent(ctx, ent); - insert_ent(ctx, ent); - mark_ent_shards(ctx, ent); - queue_send(ctx, ent); - } - } - } - - unblock_marked_shards(ctx); - - spin_unlock(&ctx->lock); -} - -int scoutfs_wrlock_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx; - u32 nr = 1; /* XXX */ - int i; - - ctx = vmalloc(offsetof(struct wrlock_context, shards[nr])); - if (!ctx) - return -ENOMEM; - - /* XXX need some kind of mount id */ - ctx->send_workq = alloc_ordered_workqueue("scoutfs-%s-%u:%u-send", 0, - sb->s_id, - MAJOR(sb->s_bdev->bd_dev), - MINOR(sb->s_bdev->bd_dev)); - if (!ctx->send_workq) { - vfree(ctx); - return -ENOMEM; - } - - ctx->sb = sb; - init_waitqueue_head(&ctx->waitq); - spin_lock_init(&ctx->lock); - ctx->id_root = RB_ROOT; - INIT_LIST_HEAD(&ctx->mark_list); - INIT_LIST_HEAD(&ctx->send_list); - INIT_WORK(&ctx->send_work, send_work_func); - ctx->nr_shards = nr; - - for (i = 0; i < nr; i++) { - INIT_LIST_HEAD(&ctx->shards[i].mark_head); - ctx->shards[i].blocked_root = RB_ROOT; - } - - sbi->wrlock_context = ctx; - - scoutfs_trace(sb, "setup ctx 0x%llx", ctx); - - return 0; -} - -/* - * Destroy the messaging work and free the wrlock entries. There should - * be no more active lockers at this point. - */ -void scoutfs_wrlock_teardown(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct wrlock_context *ctx = sbi->wrlock_context; - struct wrlock_context_shard *shard; - struct wrlock_entry *ent; - struct wrlock_entry *tmp; - int i; - - if (!ctx) - return; - - scoutfs_trace(sb, "teardown ctx 0x%llx", ctx); - - destroy_workqueue(ctx->send_workq); - - for (i = 0; i < ctx->nr_shards; i++) { - shard = &ctx->shards[i]; - - if (shard->granted) { - ent = shard->granted; - shard->granted = NULL; - try_free_ent(ctx, ent); - } - - list_for_each_entry_safe(ent, tmp, &ctx->send_list, send_head) { - list_del_init(&ent->send_head); - try_free_ent(ctx, ent); - } - - while ((ent = blocked_ent(shard))) { - erase_ent(ctx, ent); - try_free_ent(ctx, ent); - } - } - - vfree(ctx); -} diff --git a/kmod/src/wrlock.h b/kmod/src/wrlock.h deleted file mode 100644 index d8304498..00000000 --- a/kmod/src/wrlock.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef _SCOUTFS_WRLOCK_H_ -#define _SCOUTFS_WRLOCK_H_ - -#include "wire.h" - -struct scoutfs_wrlock_held { - u8 nr_shards; - u32 shards[SCOUTFS_WRLOCK_MAX_SHARDS]; -}; - -#define DECLARE_SCOUTFS_WRLOCK_HELD(held) \ - struct scoutfs_wrlock_held held = {0, } - -int scoutfs_wrlock_lock(struct super_block *sb, - struct scoutfs_wrlock_held *held, int nr_inos, ...); -void scoutfs_wrlock_unlock(struct super_block *sb, - struct scoutfs_wrlock_held *held); - -void scoutfs_wrlock_roster_update(struct super_block *sb, u64 peer_id, - bool join); -int scoutfs_wrlock_process_request(struct super_block *sb, u64 peer_id, - struct scoutfs_wrlock_request *req); -void scoutfs_wrlock_process_grant(struct super_block *sb, - struct scoutfs_wrlock_grant *grant); - -int scoutfs_wrlock_setup(struct super_block *sb); -void scoutfs_wrlock_teardown(struct super_block *sb); - -#endif From a64ca8018ae6a6e65c91ca8da55517017ab1904c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 4 Jul 2016 10:45:17 -0700 Subject: [PATCH 059/920] scoutfs: add scoutfs_btree_hole() for finding keys Directory entries found a hole in the key range between the first and last possible hash value for a new entry's key. The xattrs want to do the same thing so let's extract this into a proper function. Signed-off-by: Zach Brown --- kmod/src/btree.c | 35 +++++++++++++++++++++++++++++++++++ kmod/src/btree.h | 2 ++ kmod/src/dir.c | 15 ++------------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 069a50bf..23878451 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -953,3 +953,38 @@ void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) } curs->bl = NULL; } + +/* + * Find the first missing key between the caller's keys, inclusive. Set + * the caller's hole key and return 0 if we find a missing key. Return + * -ENOSPC if all the keys in the range were present or -errno on errors. + * + * The caller ensures that it's safe for us to be walking this region + * of the tree. + */ +int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, struct scoutfs_key *hole) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + int ret; + + *hole = *first; + while ((ret = scoutfs_btree_next(sb, first, last, &curs)) > 0) { + /* return our expected hole if we skipped it */ + if (scoutfs_key_cmp(hole, curs.key) < 0) + break; + + *hole = *curs.key; + scoutfs_inc_key(hole); + } + scoutfs_btree_release(&curs); + + if (ret >= 0) { + if (scoutfs_key_cmp(hole, last) <= 0) + ret = 0; + else + ret = -ENOSPC; + } + + return ret; +} diff --git a/kmod/src/btree.h b/kmod/src/btree.h index e902a6bd..5804cbc3 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -28,6 +28,8 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key); void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_cursor *curs); +int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, struct scoutfs_key *hole); void scoutfs_btree_release(struct scoutfs_btree_cursor *curs); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 730cc7b2..39745fb7 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -341,21 +341,10 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, last_dirent_key_offset(h)); - /* find the first unoccupied key offset after the hashed name */ - key = first; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { - key = *curs.key; - scoutfs_inc_key(&key); - } - scoutfs_btree_release(&curs); - if (ret < 0) + ret = scoutfs_btree_hole(sb, &first, &last, &key); + if (ret) goto out; - if (scoutfs_key_cmp(&key, &last) > 0) { - ret = -ENOSPC; - goto out; - } - ret = scoutfs_btree_insert(sb, &key, bytes, &curs); if (ret) goto out; From cedeacacb8dd5d73dc14f3f2c664107c6c450812 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 4 Jul 2016 10:49:41 -0700 Subject: [PATCH 060/920] scoutfs: add file with simple name functions Directory entries and extended attributes similarly hash and compare strings so we'll give them some shared functions for doing so. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 ++-- kmod/src/dir.c | 10 +++------- kmod/src/name.c | 35 +++++++++++++++++++++++++++++++++++ kmod/src/name.h | 8 ++++++++ 4 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 kmod/src/name.c create mode 100644 kmod/src/name.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 62aacf23..f6c32636 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -11,7 +11,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += first.o scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o ioctl.o msg.o scoutfs_trace.o super.o trace.o trans.o \ - treap.o + inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trace.o \ + trans.o treap.o scoutfs-y += last.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 39745fb7..db741f62 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -23,6 +23,7 @@ #include "super.h" #include "btree.h" #include "trans.h" +#include "name.h" /* * Directory entries are stored in entries with offsets calculated from @@ -84,11 +85,6 @@ static unsigned int dentry_type(unsigned int type) return DT_UNKNOWN; } -static int names_equal(const char *name_a, int len_a, const char *name_b, - int len_b) -{ - return (len_a == len_b) && !memcmp(name_a, name_b, len_a); -} /* * XXX This crc nonsense is a quick hack. We'll want something a @@ -209,8 +205,8 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, dent = curs.val; name_len = item_name_len(&curs); - if (names_equal(dentry->d_name.name, dentry->d_name.len, - dent->name, name_len)) { + if (scoutfs_names_equal(dentry->d_name.name, dentry->d_name.len, + dent->name, name_len)) { ino = le64_to_cpu(dent->ino); di->hash = scoutfs_key_offset(curs.key); break; diff --git a/kmod/src/name.c b/kmod/src/name.c new file mode 100644 index 00000000..e14f52bd --- /dev/null +++ b/kmod/src/name.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "name.h" + +/* + * XXX This crc nonsense is a quick hack. We'll want something a + * lot stronger like siphash. + */ +u64 scoutfs_name_hash(const char *name, unsigned int len) +{ + unsigned int half = (len + 1) / 2; + + return crc32c(~0, name, half) | + ((u64)crc32c(~0, name + len - half, half) << 32); +} + +int scoutfs_names_equal(const char *name_a, int len_a, + const char *name_b, int len_b) +{ + return (len_a == len_b) && !memcmp(name_a, name_b, len_a); +} diff --git a/kmod/src/name.h b/kmod/src/name.h new file mode 100644 index 00000000..020ecb0f --- /dev/null +++ b/kmod/src/name.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_NAME_H_ +#define _SCOUTFS_NAME_H_ + +u64 scoutfs_name_hash(const char *data, unsigned int len); +int scoutfs_names_equal(const char *name_a, int len_a, + const char *name_b, int len_b); + +#endif From 59b1f62df894c5a8ed82c7aa74b8c3962d8dd920 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 4 Jul 2016 10:59:43 -0700 Subject: [PATCH 061/920] scoutfs: add basic xattr support Add basic support for extended attributes. The next steps are to add support for more prefixes, including ACLs, and to properly delete them on unlink. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/dir.c | 6 + kmod/src/format.h | 15 ++- kmod/src/inode.c | 13 +- kmod/src/inode.h | 2 + kmod/src/super.c | 1 + kmod/src/xattr.c | 327 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/xattr.h | 11 ++ 8 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 kmod/src/xattr.c create mode 100644 kmod/src/xattr.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f6c32636..05d0d590 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -12,6 +12,6 @@ scoutfs-y += first.o scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trace.o \ - trans.o treap.o + trans.o treap.o xattr.o scoutfs-y += last.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c index db741f62..e6cdb5d0 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "format.h" #include "dir.h" @@ -24,6 +25,7 @@ #include "btree.h" #include "trans.h" #include "name.h" +#include "xattr.h" /* * Directory entries are stored in entries with offsets calculated from @@ -451,6 +453,10 @@ const struct inode_operations scoutfs_dir_iops = { .mkdir = scoutfs_mkdir, .unlink = scoutfs_unlink, .rmdir = scoutfs_unlink, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, }; void scoutfs_dir_exit(void) diff --git a/kmod/src/format.h b/kmod/src/format.h index 5deca747..5210e7cb 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -51,8 +51,9 @@ struct scoutfs_key { * have to stress about cleverly allocating the types. */ #define SCOUTFS_INODE_KEY 1 -#define SCOUTFS_DIRENT_KEY 2 -#define SCOUTFS_DATA_KEY 3 +#define SCOUTFS_XATTR_KEY 2 +#define SCOUTFS_DIRENT_KEY 3 +#define SCOUTFS_DATA_KEY 4 #define SCOUTFS_MAX_ITEM_LEN 2048 @@ -237,4 +238,14 @@ enum { SCOUTFS_DT_WHT, }; +#define SCOUTFS_MAX_XATTR_NAME_LEN 255 +#define SCOUTFS_MAX_XATTR_VALUE_LEN 255 +#define SCOUTFS_XATTR_HASH_MASK 7ULL + +struct scoutfs_xattr { + __u8 name_len; + __u8 value_len; + __u8 name[0]; +} __packed; + #endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 0543c991..6caaeb06 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -23,6 +24,7 @@ #include "dir.h" #include "filerw.h" #include "scoutfs_trace.h" +#include "xattr.h" /* * XXX @@ -36,6 +38,8 @@ static void scoutfs_inode_ctor(void *obj) { struct scoutfs_inode_info *ci = obj; + init_rwsem(&ci->xattr_rwsem); + inode_init_once(&ci->inode); } @@ -63,6 +67,13 @@ void scoutfs_destroy_inode(struct inode *inode) call_rcu(&inode->i_rcu, scoutfs_i_callback); } +static const struct inode_operations scoutfs_file_iops = { + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, +}; + /* * Called once new inode allocation or inode reading has initialized * enough of the inode for us to set the ops based on the mode. @@ -72,7 +83,7 @@ static void set_inode_ops(struct inode *inode) switch (inode->i_mode & S_IFMT) { case S_IFREG: inode->i_mapping->a_ops = &scoutfs_file_aops; -// inode->i_op = &scoutfs_file_iops; + inode->i_op = &scoutfs_file_iops; inode->i_fop = &scoutfs_file_fops; break; case S_IFDIR: diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f2846201..3f68f4e5 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -5,6 +5,8 @@ struct scoutfs_inode_info { u64 ino; u32 salt; + struct rw_semaphore xattr_rwsem; + struct inode inode; }; diff --git a/kmod/src/super.c b/kmod/src/super.c index a90caf75..052311de 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -22,6 +22,7 @@ #include "format.h" #include "inode.h" #include "dir.h" +#include "xattr.h" #include "msg.h" #include "block.h" #include "counters.h" diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c new file mode 100644 index 00000000..57159e6d --- /dev/null +++ b/kmod/src/xattr.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "format.h" +#include "inode.h" +#include "key.h" +#include "super.h" +#include "btree.h" +#include "trans.h" +#include "name.h" +#include "xattr.h" +#include "trace.h" + +/* + * xattrs are stored in items with offsets set to the hash of their + * name. The item's value contains the xattr name and value. + * + * We reserve a few low bits of the key offset for hash collisions. + * Lookup walks collisions looking for an xattr with its name and create + * looks for a hole in the colliding key space for the new xattr. + * + * Usually btree block locking would protect the atomicity of xattr + * value updates. Lookups would have to wait for modification to + * finish. But the collision items are updated with multiple btree + * operations. And we insert new items before deleting the old so that + * we can always unwind on errors. This means that there can be + * multiple versions of an xattr in the btree. So we add an inode rw + * semaphore around xattr operations. + * + * XXX + * - add acl support and call generic xattr->handlers for SYSTEM + * - remove all xattrs on unlink + */ + +/* the value immediately follows the name and there is no null termination */ +static char *xat_value(struct scoutfs_xattr *xat) +{ + return &xat->name[xat->name_len]; +} + +static unsigned int xat_bytes(unsigned int name_len, unsigned int value_len) +{ + return offsetof(struct scoutfs_xattr, name[name_len + value_len]); +} + +/* + * The caller provides an initialized cursor. + * + * If we return > 0 then the cursor points to an xattr with the given + * name and the caller must clean up the cursor. + * + * Returns 0 when no matching xattr is found or -errno on error. + */ +static int lookup_xattr(struct inode *inode, const char *name, + unsigned int name_len, + struct scoutfs_btree_cursor *curs) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key first; + struct scoutfs_key last; + struct scoutfs_xattr *xat; + int ret; + u64 h; + + if (name_len > SCOUTFS_MAX_XATTR_NAME_LEN) + return -EINVAL; + + /* XXX could be a lookup helper? */ + h = scoutfs_name_hash(name, name_len) & ~SCOUTFS_XATTR_HASH_MASK; + + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); + scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, + h | SCOUTFS_XATTR_HASH_MASK); + + while ((ret = scoutfs_btree_next(sb, &first, &last, curs)) > 0) { + xat = curs->val; + + if (scoutfs_names_equal(name, name_len, xat->name, + xat->name_len)) + break; + } + + if (ret <= 0) + scoutfs_btree_release(curs); + + return ret; +} + +/* + * Insert a new xattr and set the caller's key to the key that we used. + * The caller is responsible for managing transactions and locking. + */ +static int insert_xattr(struct inode *inode, const char *name, + unsigned int name_len, const void *value, size_t size, + struct scoutfs_key *key) +{ + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_xattr *xat; + struct scoutfs_key first; + struct scoutfs_key last; + int ret; + u64 h; + + if (name_len > SCOUTFS_MAX_XATTR_NAME_LEN || + size > SCOUTFS_MAX_XATTR_NAME_LEN) + return -EINVAL; + + /* XXX could be a lookup helper? */ + h = scoutfs_name_hash(name, name_len) & ~SCOUTFS_XATTR_HASH_MASK; + + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); + scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, + h | SCOUTFS_XATTR_HASH_MASK); + + /* find the first unoccupied key offset after the hashed name */ + ret = scoutfs_btree_hole(sb, &first, &last, key); + if (ret) + return ret; + + ret = scoutfs_btree_insert(sb, key, xat_bytes(name_len, size), &curs); + if (!ret) { + xat = curs.val; + xat->name_len = name_len; + xat->value_len = size; + memcpy(xat->name, name, name_len); + memcpy(xat_value(xat), value, size); + + scoutfs_btree_release(&curs); + } + + return ret; +} + +/* + * This will grow to have all the supported prefixes (then will turn + * into xattr_handlers with prefixes upstream). + */ +static int unknown_prefix(const char *name) +{ + return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); +} + +ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, + size_t size) +{ + struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + size_t name_len = strlen(name); + struct scoutfs_xattr *xat; + int ret; + + if (unknown_prefix(name)) + return -EOPNOTSUPP; + + down_read(&si->xattr_rwsem); + + ret = lookup_xattr(inode, name, name_len, &curs); + if (ret == 0) { + ret = -ENODATA; + } else if (ret > 0) { + xat = curs.val; + + ret = xat->value_len; + if (buffer) { + if (ret <= size) + memcpy(buffer, xat_value(xat), ret); + else + ret = -ERANGE; + } + scoutfs_btree_release(&curs); + } + + up_read(&si->xattr_rwsem); + + return ret; +} + +/* + * Set the xattr with the given name to the given value. The value can + * have a size of 0. A null value pointer indicates that we should + * delete the xattr. + */ +static int scoutfs_xattr_set(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) + +{ + struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + size_t name_len = strlen(name); + struct scoutfs_key old_key; + struct scoutfs_key new_key; + bool old; + int ret; + + scoutfs_trace(sb, "name %llx value %llx size %llu flags %lld", + name, value, size, flags); + + if (unknown_prefix(name)) + return -EOPNOTSUPP; + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + ret = scoutfs_dirty_inode_item(inode); + if (ret) + goto out; + + down_write(&si->xattr_rwsem); + + ret = lookup_xattr(inode, name, name_len, &curs); + if (ret > 0) { + old = true; + old_key = *curs.key; + scoutfs_btree_release(&curs); + } else if (ret == 0) { + old = false; + } else { + goto out; + } + + if (old && (flags & XATTR_CREATE)) { + ret = -EEXIST; + goto out; + } + if (!old && (flags & XATTR_REPLACE)) { + ret = -ENODATA; + goto out; + } + + if (value) { + ret = insert_xattr(inode, name, name_len, value, size, + &new_key); + if (ret) + goto out; + } + + if (old) { + ret = scoutfs_btree_delete(sb, &old_key); + if (ret) { + scoutfs_btree_delete(sb, &new_key); + goto out; + } + } + + inode_inc_iversion(inode); + inode->i_ctime = CURRENT_TIME; + scoutfs_update_inode_item(inode); + ret = 0; +out: + up_write(&si->xattr_rwsem); + scoutfs_release_trans(sb); + return ret; +} + +int scoutfs_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + if (size == 0) + value = ""; /* set empty value */ + + return scoutfs_xattr_set(dentry, name, value, size, 0); +} + +int scoutfs_removexattr(struct dentry *dentry, const char *name) +{ + return scoutfs_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); +} + +ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = dentry->d_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_xattr *xat; + struct scoutfs_key first; + struct scoutfs_key last; + ssize_t total; + int ret; + + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, 0); + scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, ~0ULL); + + down_read(&si->xattr_rwsem); + + total = 0; + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + xat = curs.val; + + total += xat->name_len + 1; + if (!size) + continue; + if (!buffer || total > size) { + ret = -ERANGE; + break; + } + + memcpy(buffer, xat->name, xat->name_len); + buffer += xat->name_len; + *(buffer++) = '\0'; + } + + scoutfs_btree_release(&curs); + + up_read(&si->xattr_rwsem); + + return ret < 0 ? ret : total; +} diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h new file mode 100644 index 00000000..7abb00c3 --- /dev/null +++ b/kmod/src/xattr.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_XATTR_H_ +#define _SCOUTFS_XATTR_H_ + +ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, + size_t size); +int scoutfs_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags); +int scoutfs_removexattr(struct dentry *dentry, const char *name); +ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); + +#endif From ae748f0ebcf814709c419eddee050614c1aaf85f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Jul 2016 14:20:19 -0700 Subject: [PATCH 062/920] scoutfs: allow tracing with a null sb The sb counter field isn't necessary, allow a null sb pointer arg which then results in a counter output of 0. Signed-off-by: Zach Brown --- kmod/src/trace.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/trace.h b/kmod/src/trace.h index 0e9e3735..7f890843 100644 --- a/kmod/src/trace.h +++ b/kmod/src/trace.h @@ -97,7 +97,8 @@ static inline void only_check_format(const char *fmt, ...) */ #define scoutfs_trace(sb, fmt, ...) \ do { \ - struct scoutfs_sb_info *__sbi = SCOUTFS_SB(sb); \ + struct super_block *__sb = (sb); \ + u64 __sbi_ctr = __sb ? SCOUTFS_SB(__sb)->ctr : 0; \ static char __scoutfs_trace_section __fmt[] = \ "[%llu.%llu] %llu %llu %llu " __stringify(__LINE__) ": "\ fmt; \ @@ -111,7 +112,7 @@ do { \ do_gettimeofday(&__tv); \ \ __trace_write(__fmt, CAST_ARGS_U64(__tv.tv_sec, __tv.tv_usec, \ - __sbi->ctr, current->pid, get_cpu(), \ + __sbi_ctr, current->pid, get_cpu(), \ __VA_ARGS__)); \ put_cpu(); \ } while (0) From 3efec0c094274ea4c7ea989d652893ee79a9c1d5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Jul 2016 14:24:10 -0700 Subject: [PATCH 063/920] scoutfs: add scoutfs_set_max_key() It's nice to have a helper that sets the max possible key instead of messing around with memset or ~0 manually. Signed-off-by: Zach Brown --- kmod/src/btree.c | 6 +++--- kmod/src/key.h | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 23878451..4cfe1feb 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -414,9 +414,9 @@ static struct scoutfs_block *try_split(struct super_block *sb, /* only grow the tree once we have the split neighbour */ if (par_bl) { - struct scoutfs_key ones; - memset(&ones, 0xff, sizeof(ones)); - create_parent_item(parent, right, &ones); + struct scoutfs_key maximal; + scoutfs_set_max_key(&maximal); + create_parent_item(parent, right, &maximal); } move_items(left, right, false, used_total(right) / 2); diff --git a/kmod/src/key.h b/kmod/src/key.h index b61967d0..11a7c721 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -62,6 +62,11 @@ static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, key->offset = cpu_to_le64(offset); } +static inline void scoutfs_set_max_key(struct scoutfs_key *key) +{ + scoutfs_set_key(key, ~0ULL, ~0, ~0ULL); +} + static inline void scoutfs_inc_key(struct scoutfs_key *key) { le64_add_cpu(&key->offset, 1); From b51511466ad93096dd573fb29af628f46b7c4d67 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Jul 2016 14:46:20 -0700 Subject: [PATCH 064/920] scoutfs: add inodes_since ioctl Add the ioctl that let's us find out about inodes that have changed since a given sequence number. A sequence number is added to the btree items so that we can track the tree update that it last changed in. We update this as we modify items and maintain it across item copying for splits and merges. The big change is using the parent item ref and item sequence numbers to guide iteration over items in the tree. The easier change is to have the current iteration skip over items whose sequence number is too old. The more subtle change has to do with how iteration is terminated. The current termination could stop when it doesn't find an item because that could only happen at the final leaf. When we're ignoring items with old seqs this can happen at the end of any leaf. So we change iteration to keep advancing through leaf blocks until it crosses the last key value. We add an argument to btree walking which communicates the next key that can be used to continue iterating from the next leaf block. This works for the normal walk case as well as the seq walking case where walking terminates prematurely in an interior node full of parent items with old seqs. Now that we're more robustly advancing iteration with btree walk calls and the next key we can get rid fo the 'next_leaf' hack which was trying to do the same thing inside the btree walk code. It wasn't right for the seq walking case and was pretty fiddly. The next_key increment could wrap the maximal key at the right spine of the tree so we have _inc saturate instead of wrap. And finally, we want these inode scans to not have to skip over all the other items associated with each inode as it walks looking for inodes with the given sequence number. We change the item sort order to first sort by type instead of by inode. We've wanted this more generally to isolate item types that have different access patterns. Signed-off-by: Zach Brown --- kmod/src/btree.c | 255 +++++++++++++++++++++++++++++++--------------- kmod/src/btree.h | 4 + kmod/src/filerw.c | 2 + kmod/src/format.h | 5 + kmod/src/ioctl.c | 85 ++++++++++++++++ kmod/src/ioctl.h | 22 +++- kmod/src/key.h | 24 ++++- 7 files changed, 311 insertions(+), 86 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 4cfe1feb..7a6ed014 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -20,6 +20,7 @@ #include "key.h" #include "treap.h" #include "btree.h" +#include "trace.h" /* * scoutfs stores file system metadata in btrees whose items have fixed @@ -48,6 +49,11 @@ * updated if we insert an item with a key greater than everything in * the tree. * + * btree blocks, block references, and items all have sequence numbers + * that are set to the current dirty btree sequence number when they're + * modified. This lets us efficiently search a range of keys for items + * that are newer than a given sequence number. + * * Operations are performed in one pass down the tree. This lets us * cascade locks from the root down to the leaves and avoids having to * maintain a record of the path down the tree. Splits and merges are @@ -171,6 +177,7 @@ static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, le16_add_cpu(&bt->nr_items, 1); item->key = *key; + item->seq = bt->hdr.seq; item->val_len = cpu_to_le16(val_len); scoutfs_treap_insert(&bt->treap, cmp_tnode_items, &item->tnode); @@ -222,6 +229,7 @@ static void move_items(struct scoutfs_btree_block *dst, to = create_item(dst, &from->key, val_len); memcpy(to->val, from->val, val_len); + to->seq = from->seq; del = from; if (move_right) @@ -527,51 +535,10 @@ enum { WALK_INSERT = 1, WALK_DELETE, WALK_NEXT, - WALK_PREV, + WALK_NEXT_SEQ, WALK_DIRTY, }; -/* - * Usually we descend to a leaf that contains the key. But if we're - * searching for a next or previous item then we might hit a leaf block - * that contains the key but no items in the direction of the search. - * We might need to ascend and continue the search in the next block. - * - * The caller has just descended to a leaf and is asking us to discover - * this case. We set the parent item and return true to tell the caller - * to read the new parent item's referenced block. - * - * XXX I don't think this can see bts with 0 items? would need to verify? - */ -static bool next_leaf(struct scoutfs_btree_block *parent, - struct scoutfs_btree_item **par_item, - struct scoutfs_btree_block *bt, int op, int level, - struct scoutfs_key *key) -{ - struct scoutfs_btree_item *nei; - - if (level > 0 || !parent || le16_to_cpu(parent->nr_items) < 2) - return false; - - if (op == WALK_NEXT) { - nei = bt_next(parent, *par_item); - if (nei && (scoutfs_key_cmp(key, greatest_key(bt)) > 0)) { - *par_item = nei; - return true; - } - } - - if (op == WALK_PREV) { - nei = bt_prev(parent, *par_item); - if (nei && (scoutfs_key_cmp(key, least_key(bt)) < 0)) { - *par_item = nei; - return true; - } - } - - return false; -} - /* * As we descend we lock parent blocks (or the root), then lock the child, * then unlock the parent. @@ -608,16 +575,72 @@ static void unlock_block(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, up_read(rwsem); } +static u64 item_block_ref_seq(struct scoutfs_btree_item *item) +{ + struct scoutfs_block_ref *ref = (void *)item->val; + + return le64_to_cpu(ref->seq); +} + +/* + * Return true if we should skip this item while iterating by sequence + * number. If it's a parent then we test the block ref's seq, if it's a + * leaf item then we check the item's seq. + */ +static int item_skip_seq(struct scoutfs_btree_item *item, + int level, u64 seq, int op) +{ + return op == WALK_NEXT_SEQ && item && + ((level > 0 && item_block_ref_seq(item) < seq) || + (level == 0 && le64_to_cpu(item->seq) < seq)); +} + +/* + * Return the next item, possibly skipping those with sequence numbers + * less than the desired sequence number. + */ +static struct scoutfs_btree_item * +item_next_seq(struct scoutfs_btree_block *bt, struct scoutfs_btree_item *item, + int level, u64 seq, int op) +{ + do { + item = bt_next(bt, item); + } while (item_skip_seq(item, level, seq, op)); + + return item; +} + +/* + * Return the first item after the given key, possibly skipping those + * with sequence numbers less than the desired sequence number. + */ +static struct scoutfs_btree_item * +item_after_seq(struct scoutfs_btree_block *bt, struct scoutfs_key *key, + int level, u64 seq, int op) +{ + struct scoutfs_btree_item *item; + + item = bt_after(bt, key); + if (item_skip_seq(item, level, seq, op)) + item = item_next_seq(bt, item, level, seq, op); + + return item; +} /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their - * operation. The block is returned locked for either reading or writing - * depending on the operation. + * operation. The block is returned locked for either reading or + * writing depending on the operation. + * + * As we descend through parent items we set next_key to the first key + * in the next sibling's block. This is used by iteration to advance to + * the next block when they're done with the block this returns. */ static struct scoutfs_block *btree_walk(struct super_block *sb, struct scoutfs_key *key, - unsigned int val_len, int op) + struct scoutfs_key *next_key, + unsigned int val_len, u64 seq, int op) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_block *parent = NULL; @@ -630,6 +653,13 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, const bool dirty = op == WALK_INSERT || op == WALK_DELETE || op == WALK_DIRTY; + scoutfs_trace(sb, "key "CKF" level %llu seq %llu op %llu", + CKA(key), val_len, seq, op); + + /* no sibling blocks if we don't have parent blocks */ + if (next_key) + scoutfs_set_max_key(next_key); + lock_block(sbi, par_bl, dirty); /* XXX one for now */ @@ -649,6 +679,13 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, return bl; } + + /* skip the whole tree if the root ref's seq is old */ + if (op == WALK_NEXT_SEQ && le64_to_cpu(ref->seq) < seq) { + unlock_block(sbi, par_bl, dirty); + return ERR_PTR(-ENOENT); + } + while (level--) { /* XXX hmm, need to think about retry */ if (dirty) { @@ -659,12 +696,14 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, if (IS_ERR(bl)) break; - /* see if a search needs to move to the next parent ref */ - if (next_leaf(parent, &item, bl->data, op, level, key)) { - ref = (void *)item->val; - level++; - scoutfs_put_block(bl); - continue; + /* + * Update the next key an iterator should read from. + * Keep in mind that iteration is read only so the + * parent item won't be changed splitting or merging. + */ + if (parent && next_key) { + *next_key = item->key; + scoutfs_inc_key(next_key); } if (op == WALK_INSERT) @@ -686,12 +725,20 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, par_bl = bl; parent = par_bl->data; - /* there should always be a parent item */ - item = bt_after(parent, key); + /* + * Find the parent item that references the next child + * block to search. If we're skipping items with old + * seqs then we might not have any child items to + * search. + */ + item = item_after_seq(parent, key, level, seq, op); if (!item) { /* current block dropped as parent below */ - bl = ERR_PTR(-EIO); - break; + if (op == WALK_NEXT_SEQ) { + bl = ERR_PTR(-ENOENT); + } else { + bl = ERR_PTR(-EIO); + } break; } /* XXX verify sane length */ @@ -711,6 +758,7 @@ static void set_cursor(struct scoutfs_btree_cursor *curs, curs->bl = bl; curs->item = item; curs->key = &item->key; + curs->seq = le64_to_cpu(item->seq); curs->val = item->val; curs->val_len = le16_to_cpu(item->val_len); curs->write = !!write; @@ -729,7 +777,7 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, BUG_ON(curs->bl); - bl = btree_walk(sb, key, 0, 0); + bl = btree_walk(sb, key, NULL, 0, 0, 0); if (IS_ERR(bl)) return PTR_ERR(bl); @@ -765,7 +813,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, BUG_ON(curs->bl); - bl = btree_walk(sb, key, val_len, WALK_INSERT); + bl = btree_walk(sb, key, NULL, val_len, 0, WALK_INSERT); if (IS_ERR(bl)) return PTR_ERR(bl); bt = bl->data; @@ -797,7 +845,7 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) struct scoutfs_block *bl; int ret; - bl = btree_walk(sb, key, 0, WALK_DELETE); + bl = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); if (IS_ERR(bl)) return PTR_ERR(bl); bt = bl->data; @@ -826,57 +874,84 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) } /* - * The caller initializes the cursor and first and last keys and then - * gets the cursor set to each item within those keys. + * Iterate over items in the tree starting with first and ending with + * last. We point the cursor at each item and return to the caller. + * The caller continues the search with the cursor. * - * The btree walk takes care of advancing past interior leaves that - * don't contain items past the key. Our job is to find the next item - * after the key. If that next item's key is past the caller's last key - * then the iteration is done. + * The caller can limit results to items with a sequence number greater + * than or equal to their sequence number. * - * returns 0 if no next, > 0 when curs contains next, < 0 on error + * When there isn't an item in the cursor then we walk the btree to the + * leaf that should contain the key and look for items from there. When + * we exhaust leaves we search the tree again from the next key that was + * increased past the leaf's parent's item. + * + * Returns > 0 when the cursor has an item, 0 when done, and -errno on error. */ -int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, - struct scoutfs_btree_cursor *curs) +static int btree_next(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, u64 seq, int op, + struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_block *bt; struct scoutfs_block *bl; struct scoutfs_key key = *first; + struct scoutfs_key next_key; int ret; - trace_printk("first "CKF" last "CKF" %d curs "CKF"\n", - CKA(first), CKA(last), - !!curs->bl, CKA(curs->bl ? curs->key : &key)); + scoutfs_trace(sb, "first "CKF" last "CKF" seq %llu op %llu curs "CKF, + CKA(first), CKA(last), seq, op, + CKA(curs->bl ? curs->key : first)); + + if (scoutfs_key_cmp(first, last) > 0) + return 0; /* find the next item after the cursor, releasing if we're done */ if (curs->bl) { key = curs->item->key; scoutfs_inc_key(&key); - curs->item = bt_next(curs->bl->data, curs->item); - trace_printk("next %p\n", curs->item); + curs->item = item_next_seq(curs->bl->data, curs->item, + 0, seq, op); if (curs->item) set_cursor(curs, curs->bl, curs->item, curs->write); else scoutfs_btree_release(curs); } - /* walk the tree to find the key, can be first or later */ - if (!curs->bl) { - bl = btree_walk(sb, &key, 0, WALK_NEXT); - if (IS_ERR(bl)) + /* find the leaf that contains the next item after the key */ + while (!curs->bl && scoutfs_key_cmp(&key, last) <= 0) { + + bl = btree_walk(sb, &key, &next_key, 0, seq, op); + + /* next seq walks can terminate in parents with old seqs */ + if (op == WALK_NEXT_SEQ && bl == ERR_PTR(-ENOENT)) { + key = next_key; + continue; + } + + if (IS_ERR(bl)) { + if (bl == ERR_PTR(-ENOENT)) + break; return PTR_ERR(bl); + } bt = bl->data; - curs->item = bt_after(bl->data, &key); - trace_printk("after %p\n", curs->item); + /* keep trying leaves until next_key passes last */ + curs->item = item_after_seq(bl->data, &key, 0, seq, op); + if (!curs->item) { + key = next_key; + up_read(&bl->rwsem); + scoutfs_put_block(bl); + continue; + } + if (curs->item) { set_cursor(curs, bl, curs->item, false); } else { up_read(&bl->rwsem); scoutfs_put_block(bl); } + break; } /* only return the next item if it's within last */ @@ -887,10 +962,23 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, ret = 0; } - trace_printk("ret %d\n", ret); return ret; } +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, + struct scoutfs_btree_cursor *curs) +{ + return btree_next(sb, first, last, 0, WALK_NEXT, curs); +} + +int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, u64 seq, + struct scoutfs_btree_cursor *curs) +{ + return btree_next(sb, first, last, seq, WALK_NEXT_SEQ, curs); +} + /* * Ensure that the blocks that lead to the item with the given key are * dirty. caller can hold a transaction to pin the dirty blocks and @@ -904,7 +992,7 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) struct scoutfs_block *bl; int ret; - bl = btree_walk(sb, key, 0, WALK_DIRTY); + bl = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bl)) return PTR_ERR(bl); @@ -929,16 +1017,19 @@ void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; struct scoutfs_block *bl; BUG_ON(curs->bl); - bl = btree_walk(sb, key, 0, WALK_DIRTY); + bl = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); BUG_ON(IS_ERR(bl)); item = bt_lookup(bl->data, key); BUG_ON(!item); + bt = bl->data; + item->seq = bt->hdr.seq; set_cursor(curs, bl, item, true); } diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 5804cbc3..fb9b1716 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -8,6 +8,7 @@ struct scoutfs_btree_cursor { /* for callers */ struct scoutfs_key *key; + u64 seq; void *val; u16 val_len; u16 write:1; @@ -30,6 +31,9 @@ void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_cursor *curs); int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole); +int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last, u64 seq, + struct scoutfs_btree_cursor *curs); void scoutfs_btree_release(struct scoutfs_btree_cursor *curs); diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index a6e75fc7..df946ee3 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -21,6 +21,7 @@ #include "trans.h" #include "scoutfs_trace.h" #include "btree.h" +#include "ioctl.h" /* * File data is stored in items just like everything else. This is very @@ -246,4 +247,5 @@ const struct file_operations scoutfs_file_fops = { .write = do_sync_write, .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, + .unlocked_ioctl = scoutfs_ioctl, }; diff --git a/kmod/src/format.h b/kmod/src/format.h index 5210e7cb..e3112b3e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -91,9 +91,14 @@ struct scoutfs_btree_block { __le16 nr_items; } __packed; +/* + * The item sequence number is set to the dirty block's sequence number + * when the item is modified. It is not changed by splits or merges. + */ struct scoutfs_btree_item { struct scoutfs_key key; struct scoutfs_treap_node tnode; + __le64 seq; __le16 val_len; char val[0]; } __packed; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index fc9766fb..e0d78ed5 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -16,6 +16,9 @@ #include #include +#include "format.h" +#include "btree.h" +#include "key.h" #include "ioctl.h" #include "trace.h" @@ -44,3 +47,85 @@ int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg) return 0; } + +/* + * Find all the inodes in the given inode range that have changed since + * the given tree update sequence number. + * + * The inodes are returned in inode order, not sequence order. + */ +static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_inodes_since __user *uargs = (void __user *)arg; + struct scoutfs_ioctl_inodes_since args; + struct scoutfs_ioctl_ino_seq __user *uiseq; + struct scoutfs_ioctl_ino_seq iseq; + struct scoutfs_key first; + struct scoutfs_key last; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + long bytes; + int ret; + + if (copy_from_user(&args, uargs, sizeof(args))) + return -EFAULT; + + uiseq = (void __user *)(unsigned long)args.results.ptr; + if (args.results.len < sizeof(iseq)) + return -EINVAL; + + scoutfs_set_key(&first, args.first_ino, SCOUTFS_INODE_KEY, 0); + scoutfs_set_key(&last, args.last_ino, SCOUTFS_INODE_KEY, 0); + + bytes = 0; + while ((ret = scoutfs_btree_since(sb, &first, &last, + args.seq, &curs)) > 0) { + + iseq.ino = scoutfs_key_inode(curs.key); + iseq.seq = curs.seq; + + /* + * We can't copy to userspace with our locks held + * because faults could try to use tree blocks that we + * have locked. If a non-faulting copy fails we release + * the cursor and try a blocking copy and pick up where + * we left off. + */ + pagefault_disable(); + ret = __copy_to_user_inatomic(uiseq, &iseq, sizeof(iseq)); + pagefault_enable(); + if (ret) { + first = *curs.key; + scoutfs_inc_key(&first); + scoutfs_btree_release(&curs); + if (copy_to_user(uiseq, &iseq, sizeof(iseq))) { + ret = -EFAULT; + break; + } + } + + uiseq++; + bytes += sizeof(iseq); + if (bytes + sizeof(iseq) > args.results.len) { + ret = 0; + break; + } + } + + scoutfs_btree_release(&curs); + + if (bytes) + ret = bytes; + + return ret; +} + +long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case SCOUTFS_IOC_INODES_SINCE: + return scoutfs_ioc_inodes_since(file, arg); + } + + return -ENOTTY; +} diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 40372ae0..88c62cae 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -1,6 +1,9 @@ #ifndef _SCOUTFS_IOCTL_H_ #define _SCOUTFS_IOCTL_H_ +int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg); +long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + /* XXX I have no idea how these are chosen. */ #define SCOUTFS_IOCTL_MAGIC 's' @@ -27,6 +30,23 @@ struct scoutfs_trace_record { #define SCOUTFS_IOC_GET_TRACE_RECORDS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ struct scoutfs_ioctl_buf) -int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg); +struct scoutfs_ioctl_ino_seq { + __u64 ino; + __u64 seq; +} __packed; + +struct scoutfs_ioctl_inodes_since { + __u64 first_ino; + __u64 last_ino; + __u64 seq; + struct scoutfs_ioctl_buf results; +} __packed; + +/* + * Adds entries to the user's buffer for each inode whose sequence + * number is greater than or equal to the given seq. + */ +#define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ + struct scoutfs_ioctl_inodes_since) #endif diff --git a/kmod/src/key.h b/kmod/src/key.h index 11a7c721..34ac5be6 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -4,7 +4,7 @@ #include #include "format.h" -#define CKF "%llu.%u.%llu" +#define CKF "%llu.%llu.%llu" #define CKA(key) \ le64_to_cpu((key)->inode), (key)->type, le64_to_cpu((key)->offset) @@ -24,10 +24,17 @@ static inline int le64_cmp(__le64 a, __le64 b) le64_to_cpu(a) > le64_to_cpu(b) ? 1 : 0; } +/* + * Items are sorted by type and then by inode to reflect the relative + * frequency of use. Inodes and xattrs are hot, then dirents, then file + * data extents. We want each use class to be hot and dense, we don't + * want a scan of the inodes to have to skip over each inode's extent + * items. + */ static inline int scoutfs_key_cmp(struct scoutfs_key *a, struct scoutfs_key *b) { - return le64_cmp(a->inode, b->inode) ?: - ((short)a->type - (short)b->type) ?: + return ((short)a->type - (short)b->type) ?: + le64_cmp(a->inode, b->inode) ?: le64_cmp(a->offset, b->offset); } @@ -67,8 +74,19 @@ static inline void scoutfs_set_max_key(struct scoutfs_key *key) scoutfs_set_key(key, ~0ULL, ~0, ~0ULL); } +/* + * This saturates at (~0,~0,~0) instead of wrapping. This will never be + * an issue for real item keys but parent item keys along the right + * spine of the tree have maximal key values that could wrap if + * incremented. + */ static inline void scoutfs_inc_key(struct scoutfs_key *key) { + if (key->inode == cpu_to_le64(~0ULL) && + key->type == (u8)~0 && + key->offset == cpu_to_le64(~0ULL)) + return; + le64_add_cpu(&key->offset, 1); if (!key->offset) { if (++key->type == 0) From 90a73506c124dfda04e1a9300eeb68b7527e8aa7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 20 Jul 2016 12:08:12 -0700 Subject: [PATCH 065/920] scoutfs: remove homebrew tracing Oh, thank goodness. It turns out that there's a crash extension for working with tracepoints in crash dumps. Let's use standard tracepoints and pretend this tracing hack never happened. Signed-off-by: Zach Brown --- kmod/src/Makefile | 14 +- kmod/src/btree.c | 8 -- kmod/src/first.c | 4 - kmod/src/ioctl.c | 33 +---- kmod/src/ioctl.h | 29 +--- kmod/src/last.c | 4 - kmod/src/super.c | 11 -- kmod/src/super.h | 2 - kmod/src/trace.c | 360 ---------------------------------------------- kmod/src/trace.h | 123 ---------------- kmod/src/xattr.c | 4 - 11 files changed, 8 insertions(+), 584 deletions(-) delete mode 100644 kmod/src/first.c delete mode 100644 kmod/src/last.c delete mode 100644 kmod/src/trace.c delete mode 100644 kmod/src/trace.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 05d0d590..2e484055 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,16 +2,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -# -# these first and last objects are a super lame hack to put boundary -# symbols around trace printf formats that are put in an elf section. -# That should be done in a linker script, of course, but I'll be honest: -# I didn't go digging to see if modules can have linker scripts today. -# -scoutfs-y += first.o - scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trace.o \ - trans.o treap.o xattr.o - -scoutfs-y += last.o + inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trans.o \ + treap.o xattr.o diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 7a6ed014..cb5e081b 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -20,7 +20,6 @@ #include "key.h" #include "treap.h" #include "btree.h" -#include "trace.h" /* * scoutfs stores file system metadata in btrees whose items have fixed @@ -653,9 +652,6 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, const bool dirty = op == WALK_INSERT || op == WALK_DELETE || op == WALK_DIRTY; - scoutfs_trace(sb, "key "CKF" level %llu seq %llu op %llu", - CKA(key), val_len, seq, op); - /* no sibling blocks if we don't have parent blocks */ if (next_key) scoutfs_set_max_key(next_key); @@ -898,10 +894,6 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key next_key; int ret; - scoutfs_trace(sb, "first "CKF" last "CKF" seq %llu op %llu curs "CKF, - CKA(first), CKA(last), seq, op, - CKA(curs->bl ? curs->key : first)); - if (scoutfs_key_cmp(first, last) > 0) return 0; diff --git a/kmod/src/first.c b/kmod/src/first.c deleted file mode 100644 index 568ff207..00000000 --- a/kmod/src/first.c +++ /dev/null @@ -1,4 +0,0 @@ - -#include "trace.h" - -char __scoutfs_trace_section scoutfs_trace_first_format[] = ""; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index e0d78ed5..d0311da7 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -20,33 +20,6 @@ #include "btree.h" #include "key.h" #include "ioctl.h" -#include "trace.h" - -int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg) -{ - struct scoutfs_ioctl_buf __user *user_ibuf = (void __user *)arg; - struct scoutfs_ioctl_buf ibuf; - - if (copy_from_user(&ibuf, user_ibuf, sizeof(ibuf))) - return -EFAULT; - - /* limit lengths to an int for some helpers that take int len args */ - if (ibuf.len < 0) - return -EINVAL; - - iov->iov_base = (void __user *)(long)ibuf.ptr; - iov->iov_len = ibuf.len; - - /* - * This is not meant to protect the rest of the code from - * faults, it can't. It's meant to return early for iovecs that - * are completely garbage. - */ - if (!access_ok(VERIFY_READ, iov->iov_base, iov->iov_len)) - return -EFAULT; - - return 0; -} /* * Find all the inodes in the given inode range that have changed since @@ -70,8 +43,8 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) if (copy_from_user(&args, uargs, sizeof(args))) return -EFAULT; - uiseq = (void __user *)(unsigned long)args.results.ptr; - if (args.results.len < sizeof(iseq)) + uiseq = (void __user *)(unsigned long)args.buf_ptr; + if (args.buf_len < sizeof(iseq) || args.buf_len > INT_MAX) return -EINVAL; scoutfs_set_key(&first, args.first_ino, SCOUTFS_INODE_KEY, 0); @@ -106,7 +79,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) uiseq++; bytes += sizeof(iseq); - if (bytes + sizeof(iseq) > args.results.len) { + if (bytes + sizeof(iseq) > args.buf_len) { ret = 0; break; } diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 88c62cae..259478e9 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -1,35 +1,11 @@ #ifndef _SCOUTFS_IOCTL_H_ #define _SCOUTFS_IOCTL_H_ -int scoutfs_copy_ibuf(struct iovec *iov, unsigned long arg); long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); /* XXX I have no idea how these are chosen. */ #define SCOUTFS_IOCTL_MAGIC 's' -struct scoutfs_ioctl_buf { - __u64 ptr; - __s32 len; -} __packed; - -/* - * Fills the buffer with a packed array of format strings. Trace records - * refer to the format strings in the buffer by their byte offset. - */ -#define SCOUTFS_IOC_GET_TRACE_FORMATS _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ - struct scoutfs_ioctl_buf) - -struct scoutfs_trace_record { - __u16 format_off; - __u8 nr; - __u8 data[0]; -} __packed; -/* - * Fills the buffer with trace records. - */ -#define SCOUTFS_IOC_GET_TRACE_RECORDS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ - struct scoutfs_ioctl_buf) - struct scoutfs_ioctl_ino_seq { __u64 ino; __u64 seq; @@ -39,14 +15,15 @@ struct scoutfs_ioctl_inodes_since { __u64 first_ino; __u64 last_ino; __u64 seq; - struct scoutfs_ioctl_buf results; + __u64 buf_ptr; + __u32 buf_len; } __packed; /* * Adds entries to the user's buffer for each inode whose sequence * number is greater than or equal to the given seq. */ -#define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ +#define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ struct scoutfs_ioctl_inodes_since) #endif diff --git a/kmod/src/last.c b/kmod/src/last.c deleted file mode 100644 index 44e6d7dc..00000000 --- a/kmod/src/last.c +++ /dev/null @@ -1,4 +0,0 @@ - -#include "trace.h" - -char __scoutfs_trace_section scoutfs_trace_last_format[] = ""; diff --git a/kmod/src/super.c b/kmod/src/super.c index 052311de..2a0d1614 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -27,7 +27,6 @@ #include "block.h" #include "counters.h" #include "trans.h" -#include "trace.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -127,12 +126,6 @@ static int read_supers(struct super_block *sb) return 0; } -/* - * Only used for tracing output, it's a convenient way to cheaply differentiate - * messages from different super blocks. - */ -static atomic64_t scoutfs_sb_ctr = ATOMIC64_INIT(0); - static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; @@ -162,8 +155,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - sbi->ctr = atomic64_inc_return(&scoutfs_sb_ctr); - /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) @@ -225,14 +216,12 @@ static void teardown_module(void) scoutfs_inode_exit(); if (scoutfs_kset) kset_unregister(scoutfs_kset); - scoutfs_trace_exit(); } static int __init scoutfs_module_init(void) { int ret; - scoutfs_trace_init(); scoutfs_init_counters(); scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); diff --git a/kmod/src/super.h b/kmod/src/super.h index d588f36a..03226baf 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -13,8 +13,6 @@ struct buddy_alloc; struct scoutfs_sb_info { struct super_block *sb; - u64 ctr; - struct scoutfs_super_block super; spinlock_t next_ino_lock; diff --git a/kmod/src/trace.c b/kmod/src/trace.c deleted file mode 100644 index ba0d97bf..00000000 --- a/kmod/src/trace.c +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include - -#include "trace.h" -#include "super.h" -#include "ioctl.h" - -/* - * This tracing gives us: - * - * - Always on. We get history leading up to an event without having - * had to predict the event. - * - * - Cheap. Recording the format pointer index and packed arguments is - * cheap enough that we don't mind always doing it at a reasonable - * frequency. - * - * - Trivial to add. We want to err on the side of too much logging. - * We don't want there to be so much garbage associated with adding a - * single logging message that people are discouraged from doing it. - * - * - Easy to extract from crash dumps. The more the computer can tell - * us about what happened when the world went sideways, the better. - * - * The implementation is reasonably straight forward. - * - * Log statements are simple printf format strings and arguments. The - * first trick bit is that we only support u64 arguments. This lets us - * use macro hacks to walk the arguments without having to parse the - * format string. This actually isn't a great hardship because often - * the things we might want to print as strings -- process names, - * xattrs, directory entries -- could in fact be sensitive user data - * that we don't want to see. - * - * Each log statement is packed into a variable byte size record. The - * records are packed into long term per-page pages. We only support - * logging from task context so that we don't have to fool around with - * serializing between contexts on a cpu. Writers record the number of - * record bytes stored in each page in page->private. - * - * Userspace reads the format strings and trace records from trivial - * ioctls that copy the entire data set in one go. This avoids all the - * nonsense of trying to translate the changing set of records into a - * seekable byte stream of formatted output. Readers of each page of - * records samples page->private to discover when they race with writers - * and retry. - */ - -/* - * This tries to strike a balance between having enough logging on a cpu - * and not allocating an enormous amount of memory on systems with many - * cpus. - */ -#define TRACE_PAGES_PER_CPU DIV_ROUND_UP(256 * 1024, PAGE_SIZE) - -struct trace_percpu { - int cur_page; - struct page *pages[TRACE_PAGES_PER_CPU]; -}; - -static DEFINE_PER_CPU(struct trace_percpu, scoutfs_trace_percpu); - -static int rec_bytes(int bytes) -{ - return offsetof(struct scoutfs_trace_record, data[bytes]); -} - -/* - * We compact the record of the trace format by referencing it with a - * small offset into a section that contains all the format strings. - * This shrinks the per-record format reference from an 8 byte pointer - * to a 2 byte offset. 6 bytes is a lot when records are 15 bytes. - */ -static char *trace_format(u16 off) -{ - return &scoutfs_trace_first_format[1 + off]; -} - -static u16 trace_format_off(char *fmt) -{ - return fmt - scoutfs_trace_first_format - 1; -} - -static int trace_format_bytes(void) -{ - return trace_format_off(scoutfs_trace_last_format); -} - -static int valid_trace_format(char *fmt) -{ - return fmt > scoutfs_trace_first_format && - fmt < scoutfs_trace_last_format; -} - -/* - * We only support trace messages with integer arguments. Most of them - * are small: counters, pids, sizes, cpus, etc. It's worth spending a - * few cycles to remove the leading bytes full of zeros. - * - * VLQ is very simple and does reasonably well. I'd happily consider - * alternatives with similar complexity but better space efficiency. - * - * This is the most boring conservative iterative implementation. A - * much cooler implementation would efficiently transform all the bits, - * store the whole little endian value, and return the number of bytes - * with bits set. - */ -static unsigned char encode_u64_bytes(u8 *data, u64 val) -{ - unsigned char bytes = 0; - - do { - *data = val & 127; - val >>= 7; - *(data++) += (!!val) << 7; - bytes++; - } while (val); - - return bytes; -} - -/* - * Write a trace record to a percpu page. We only write from task - * context so one writer is racing with many readers. Readers sample - * the count of total written bytes in the page at page private and - * retry the copy if the count changes. It's a poor man's seqlock. - * - * The calling trace wrapper has pinned our task to the cpu. - */ -void scoutfs_trace_write(char *fmt, int nr, ...) -{ - struct trace_percpu *pcpu = this_cpu_ptr(&scoutfs_trace_percpu); - struct scoutfs_trace_record *rec; - struct page *page; - unsigned long page_bytes; - int encoded; - va_list args; - int i; - - if (WARN_ON_ONCE(in_interrupt() || in_softirq() || in_irq()) || - WARN_ON_ONCE(!valid_trace_format(fmt)) || - WARN_ON_ONCE(trace_format_bytes() > U16_MAX)) - return; - -next_page: - page = pcpu->pages[pcpu->cur_page]; - page_bytes = page->private & ~PAGE_MASK; - rec = page_address(page) + page_bytes; - - encoded = 0; - va_start(args, nr); - for (i = 0; i < nr; i++) { - if (page_bytes + rec_bytes(encoded + 9) >= PAGE_SIZE) { - if (++pcpu->cur_page == TRACE_PAGES_PER_CPU) - pcpu->cur_page = 0; - - page = pcpu->pages[pcpu->cur_page]; - /* XXX barriers? */ - page->private = round_up(page->private, PAGE_SIZE); - va_end(args); - goto next_page; - } - - encoded += encode_u64_bytes(&rec->data[encoded], - va_arg(args, u64)); - } - va_end(args); - - rec->format_off = trace_format_off(fmt); - rec->nr = nr; - /* XXX barriers? */ - page->private += rec_bytes(encoded); -} - -/* - * Give userspace all of the format strings. They're packed and null - * terminated. - * - * We return the number of bytes copied. A return size smaller than the - * buffer len indicates a partial copy and the user can retry with a - * larger buffer. - */ -static int scoutfs_ioc_get_trace_formats(void __user *buf, int len) -{ - int bytes= trace_format_bytes(); - - if (bytes <= len) { - if (copy_to_user(buf, trace_format(0), bytes)) - return -EFAULT; - } - - return bytes; -} - -/* - * Copy all the trace records on all the cpus' pages to the user buffer. - * Each page's records will be copied atomically so records won't be - * scrambled. But writers can cycle through the pages as we copy so the - * entire set of records returned is not an atomic snapshot of all the - * pages. - * - * We return the number of bytes copied. A return size smaller than the - * buffer len indicates a partial copy and the user can retry with a - * larger buffer. - */ -static int scoutfs_ioc_get_trace_records(void __user *buf, int len) -{ - struct trace_percpu *pcpu; - unsigned long before; - unsigned long after; - struct page *page; - int total = 0; - int bytes; - int ret; - int cpu; - int i; - - if (len < 0) - return -EINVAL; - - /* quickly give the caller the largest possible buffer size */ - ret = num_online_cpus() * TRACE_PAGES_PER_CPU; - if (ret > len) - return ret; - - for_each_online_cpu(cpu) { - pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); - - for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { - page = pcpu->pages[i]; - - do { - before = ACCESS_ONCE(page->private); - bytes = before & ~PAGE_MASK; - - /* ret still nr * pages */ - if (total + bytes > len) - goto out; - - if (copy_to_user(buf + total, - page_address(page), bytes)) { - ret = -EFAULT; - goto out; - } - after = ACCESS_ONCE(page->private); - } while (after != before); - - total += bytes; - } - } - - ret = total; -out: - return ret; -} - -static long scoutfs_trace_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct iovec iov; - - switch (cmd) { - case SCOUTFS_IOC_GET_TRACE_FORMATS: - return scoutfs_copy_ibuf(&iov, arg) ?: - scoutfs_ioc_get_trace_formats(iov.iov_base, iov.iov_len); - - case SCOUTFS_IOC_GET_TRACE_RECORDS: - return scoutfs_copy_ibuf(&iov, arg) ?: - scoutfs_ioc_get_trace_records(iov.iov_base, iov.iov_len); - } - - return -ENOTTY; -} - -static const struct file_operations scoutfs_trace_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = scoutfs_trace_ioctl, -}; - -static struct dentry *scoutfs_debugfs_dir; -static struct dentry *scoutfs_trace_dentry; - -int __init scoutfs_trace_init(void) -{ - struct trace_percpu *pcpu; - int cpu; - int i; - - if (WARN_ON_ONCE(&scoutfs_trace_first_format >= - &scoutfs_trace_last_format) || - WARN_ON_ONCE(trace_format_bytes() > U16_MAX)) - return -EINVAL; - - /* XXX possible instead of online? yikes? */ - for_each_possible_cpu(cpu) { - pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); - for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { - pcpu->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!pcpu->pages[i]) - return -ENOMEM; - pcpu->pages[i]->private = 0; - } - } - - scoutfs_debugfs_dir = debugfs_create_dir("scoutfs", NULL); - if (!scoutfs_debugfs_dir) - return -ENOMEM; - - scoutfs_trace_dentry = debugfs_create_file("trace", 0600, - scoutfs_debugfs_dir, NULL, - &scoutfs_trace_fops); - if (!scoutfs_trace_dentry) - return -ENOMEM; - - return 0; -} - -void __exit scoutfs_trace_exit(void) -{ - struct trace_percpu *pcpu; - int cpu; - int i; - - if (scoutfs_trace_dentry) { - debugfs_remove(scoutfs_trace_dentry); - scoutfs_trace_dentry = NULL; - } - - if (scoutfs_debugfs_dir) { - debugfs_remove(scoutfs_debugfs_dir); - scoutfs_debugfs_dir = NULL; - } - - /* XXX possible instead of online? yikes? */ - for_each_possible_cpu(cpu) { - pcpu = per_cpu_ptr(&scoutfs_trace_percpu, cpu); - for (i = 0; i < TRACE_PAGES_PER_CPU; i++) { - if (pcpu->pages[i]) { - __free_page(pcpu->pages[i]); - pcpu->pages[i] = NULL; - } - } - } -} diff --git a/kmod/src/trace.h b/kmod/src/trace.h deleted file mode 100644 index 7f890843..00000000 --- a/kmod/src/trace.h +++ /dev/null @@ -1,123 +0,0 @@ -#ifndef _SCOUTFS_TRACE_H_ -#define _SCOUTFS_TRACE_H_ - -#include -#include - -#define __scoutfs_trace_section __attribute__((section("__scoutfs_trace_fmt"))) - -extern char scoutfs_trace_first_format[]; -extern char scoutfs_trace_last_format[]; - -/* - * What a beautifully baffling construct! First our arguments are added - * to a reverse sequence of numbers. Then all the arguments are handed - * to a macro that only returns its 64th argument. The presence of our - * arguments before the sequence means that the 64th argument will be - * the number in the reverse sequence that matches the number of our - * initial arguments. - * - * h/t to: - * https://groups.google.com/forum/#!topic/comp.std.c/d-6Mj5Lko_s - */ -#define NR_VA_ARGS(...) \ - _ONLY_64TH(__VA_ARGS__, _reverse_sequence()) -#define _ONLY_64TH(...) \ - __ONLY_64TH(__VA_ARGS__) -#define __ONLY_64TH( \ - _1, _2, _3, _4, _5, _6, _7, _8, _9,_10, \ - _11,_12,_13,_14,_15,_16,_17,_18,_19,_20, \ - _21,_22,_23,_24,_25,_26,_27,_28,_29,_30, \ - _31,_32,_33,_34,_35,_36,_37,_38,_39,_40, \ - _41,_42,_43,_44,_45,_46,_47,_48,_49,_50, \ - _51,_52,_53,_54,_55,_56,_57,_58,_59,_60, \ - _61,_62,_63,N,...) N -#define _reverse_sequence() \ - 63,62,61,60, \ - 59,58,57,56,55,54,53,52,51,50, \ - 49,48,47,46,45,44,43,42,41,40, \ - 39,38,37,36,35,34,33,32,31,30, \ - 29,28,27,26,25,24,23,22,21,20, \ - 19,18,17,16,15,14,13,12,11,10, \ - 9,8,7,6,5,4,3,2,1,0 - - -/* - * surround each arg with (u64)( .. ), - * - * A 'called object not a function' error can mean there's too many args. - * - * XXX doesn't yet work with no args - */ -#define CAST_ARGS_U64(...) \ - EXPAND_MACRO(__VA_ARGS__,CU_16,CU_15,CU_14,CU_13,CU_12,\ - CU_11,CU_10,CU_9,CU_8,CU_7,CU_6,CU_5,CU_4,\ - CU_3,CU_2,CU_1)(__VA_ARGS__) -#define EXPAND_MACRO(_1,_2,_3,_4,_5,_6,_7,_8,\ - _9,_10,_11,_12,_13,_14,_15,_16,NAME,...) NAME -#define CU_1(X) (u64)(X) -#define CU_2(X, ...) (u64)(X),CU_1(__VA_ARGS__) -#define CU_3(X, ...) (u64)(X),CU_2(__VA_ARGS__) -#define CU_4(X, ...) (u64)(X),CU_3(__VA_ARGS__) -#define CU_5(X, ...) (u64)(X),CU_4(__VA_ARGS__) -#define CU_6(X, ...) (u64)(X),CU_5(__VA_ARGS__) -#define CU_7(X, ...) (u64)(X),CU_6(__VA_ARGS__) -#define CU_8(X, ...) (u64)(X),CU_7(__VA_ARGS__) -#define CU_9(X, ...) (u64)(X),CU_8(__VA_ARGS__) -#define CU_10(X, ...) (u64)(X),CU_9(__VA_ARGS__) -#define CU_11(X, ...) (u64)(X),CU_10(__VA_ARGS__) -#define CU_12(X, ...) (u64)(X),CU_11(__VA_ARGS__) -#define CU_13(X, ...) (u64)(X),CU_12(__VA_ARGS__) -#define CU_14(X, ...) (u64)(X),CU_13(__VA_ARGS__) -#define CU_15(X, ...) (u64)(X),CU_14(__VA_ARGS__) -#define CU_16(X, ...) (u64)(X),CU_15(__VA_ARGS__) - -struct super_block; -void scoutfs_trace_write(char *fmt, int nr, ...); - -__attribute__((format(printf, 1, 2))) -static inline void only_check_format(const char *fmt, ...) -{ -} - -#define __trace_write(fmtp, args...) \ - scoutfs_trace_write(fmtp, NR_VA_ARGS(args), ##args) - -/* - * Record an unstructured trace message for debugging. - * - * The arguments can only be scalar integers and will be cast to u64 so - * only %llu formats can be used. - * - * This can only be called from task context. - * - * The super block is only used to indicate which mount initiated the - * trace and it can be null for trace messages not associated with - * mounts. - */ -#define scoutfs_trace(sb, fmt, ...) \ -do { \ - struct super_block *__sb = (sb); \ - u64 __sbi_ctr = __sb ? SCOUTFS_SB(__sb)->ctr : 0; \ - static char __scoutfs_trace_section __fmt[] = \ - "[%llu.%llu] %llu %llu %llu " __stringify(__LINE__) ": "\ - fmt; \ - struct timeval __tv; \ - \ - BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ - \ - /* check the caller's format before we prepend things to it */ \ - only_check_format(fmt, CAST_ARGS_U64(__VA_ARGS__)); \ - \ - do_gettimeofday(&__tv); \ - \ - __trace_write(__fmt, CAST_ARGS_U64(__tv.tv_sec, __tv.tv_usec, \ - __sbi_ctr, current->pid, get_cpu(), \ - __VA_ARGS__)); \ - put_cpu(); \ -} while (0) - -int __init scoutfs_trace_init(void); -void __exit scoutfs_trace_exit(void); - -#endif diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 57159e6d..c45578da 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -23,7 +23,6 @@ #include "trans.h" #include "name.h" #include "xattr.h" -#include "trace.h" /* * xattrs are stored in items with offsets set to the hash of their @@ -210,9 +209,6 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, bool old; int ret; - scoutfs_trace(sb, "name %llx value %llx size %llu flags %lld", - name, value, size, flags); - if (unknown_prefix(name)) return -EOPNOTSUPP; From e22692717463fb3e69e18a1fde78f167a7e9b898 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Jul 2016 11:25:04 -0700 Subject: [PATCH 066/920] scoutfs: add support for cowing blocks The current block interface for allocating a dirty copy of a given stable block didn't cow. It moved the existing stable block into its new dirty location. This is fine for the btree which will never reference old stable blocks. It's not optimal for the allocator which is going to want to combine the previous stable allocator blocks with the current dirty allocator blocks to determine which free regions can satisfy allocations. If we invalidate the old stable cached copy it'll immediately read it back in. And it turns out that it was a little buggy in how it moved the stable block to its new dirty location. It didn't remove any old blocks at the new blkno. So we offer two high level interfaces for either moving or copying the contents of the dirty block. And we're sure to always invalidate old cached blocks at the new dirty blkno location. Signed-off-by: Zach Brown --- kmod/src/block.c | 70 +++++++++++++++++++++++++++++++++++++++--------- kmod/src/block.h | 6 +++-- kmod/src/btree.c | 4 +-- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 70ec7a57..4923832f 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -412,18 +412,25 @@ int scoutfs_write_dirty_blocks(struct super_block *sb) * sequence number which we use to determine if the block is currently * dirty or not. * - * For now we're using the dirty super block in the sb_info to track - * the dirty seq. That'll be different when we have multiple btrees. + * For now we're using the dirty super block in the sb_info to track the + * dirty seq. That'll be different when we have multiple btrees. * * Callers are working in structures that have sufficient locking to - * protect references to the source block. If we've come to dirty it then - * there won't be concurrent users and we can just move it in the cache. + * protect references to the source block. If we've come to dirty it + * then there won't be concurrent users and we can just move it in the + * cache. + * + * The caller can ask that we either move the existing cached block to + * its new dirty blkno in the cache or copy its contents to a newly + * allocated dirty block. The caller knows if they'll ever reference + * the old clean block again (buddy does, btree doesn't.) */ -struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) +static struct scoutfs_block *dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref, bool cow) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; + struct scoutfs_block *copy_bl = NULL; struct scoutfs_block *found; struct scoutfs_block *bl; unsigned long flags; @@ -439,24 +446,51 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, if (ret < 0) goto out; + if (cow) { + copy_bl = alloc_block(sb, blkno); + if (IS_ERR(copy_bl)) { + ret = PTR_ERR(copy_bl); + goto out; + } + set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, ©_bl->bits); + } + ret = radix_tree_preload(GFP_NOFS); if (ret) goto out; spin_lock_irqsave(&sbi->block_lock, flags); - /* XXX don't really like this */ - found = radix_tree_lookup(&sbi->block_radix, bl->blkno); - if (found == bl) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - atomic_dec(&bl->refcount); + /* delete anything at the new blkno */ + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found) { + radix_tree_delete(&sbi->block_radix, blkno); + scoutfs_put_block(found); + } + + if (cow) { + /* copy contents to the new block, hdr updated below */ + memcpy(copy_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); + scoutfs_put_block(bl); + bl = copy_bl; + copy_bl = NULL; + } else { + /* move the existing block to its new dirty blkno */ + found = radix_tree_lookup(&sbi->block_radix, bl->blkno); + if (found == bl) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + atomic_dec(&bl->refcount); + } } bl->blkno = blkno; hdr = bl->data; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; + ref->blkno = hdr->blkno; + ref->seq = hdr->seq; + /* insert the dirty block at its new blkno */ radix_tree_insert(&sbi->block_radix, blkno, bl); radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); atomic_inc(&bl->refcount); @@ -464,10 +498,9 @@ struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); - ref->blkno = hdr->blkno; - ref->seq = hdr->seq; ret = 0; out: + scoutfs_put_block(copy_bl); if (ret) { if (blkno) { err = scoutfs_buddy_free(sb, blkno, 0); @@ -480,6 +513,17 @@ out: return bl; } +struct scoutfs_block *scoutfs_block_cow_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) +{ + return dirty_ref(sb, ref, true); +} +struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) +{ + return dirty_ref(sb, ref, false); +} + /* * Return a newly allocated metadata block with an updated block header * to match the current dirty seq. Callers are responsible for diff --git a/kmod/src/block.h b/kmod/src/block.h index 91f55f81..b464f0c3 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -27,8 +27,10 @@ struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb); struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, struct scoutfs_block_ref *ref); -struct scoutfs_block *scoutfs_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref); +struct scoutfs_block *scoutfs_block_cow_ref(struct super_block *sb, + struct scoutfs_block_ref *ref); +struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref); int scoutfs_has_dirty_blocks(struct super_block *sb); int scoutfs_write_block(struct scoutfs_block *bl); diff --git a/kmod/src/btree.c b/kmod/src/btree.c index cb5e081b..5ffeb9f5 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -488,7 +488,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, move_right = true; } - sib_bl = scoutfs_dirty_ref(sb, (void *)sib_item->val); + sib_bl = scoutfs_block_dirty_ref(sb, (void *)sib_item->val); if (IS_ERR(sib_bl)) { /* XXX do we need to unlock this? don't think so */ scoutfs_put_block(bl); @@ -685,7 +685,7 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, while (level--) { /* XXX hmm, need to think about retry */ if (dirty) { - bl = scoutfs_dirty_ref(sb, ref); + bl = scoutfs_block_dirty_ref(sb, ref); } else { bl = scoutfs_read_ref(sb, ref); } From dcef9c0ada5d8c52dc1039b9202a3314283fbfe2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 22 Jul 2016 13:51:50 -0700 Subject: [PATCH 067/920] scoutfs: store the buddy allocator in a radix The current implementation of the allocator was built for for a world where blocks were much, much, larger. It could get away with keeping the entire bitmap resident and having to read it its entirety before being able to use it for the first time. That will not work in the current architecture that's built around a smaller metadata block size. The raw size of the allocator gets large enough that all of those behaviours become problematic at scale. This shifts the buddy allocator to be stored in a radix of blocks instead of in a ring log. This brings it more in line with the structure of the btree item indexes. It can be initially read, cached, and invalidated at block granularity. In addition, it cleverly uses the cow block structures to solve the unreferenced space allocation constraint that the previous allocator hadn't. It can compare the dirty and stable blocks to discover free blocks that aren't referenced by the old stable state. The old allocator would have grown a bunch of extra special complexity to address this. There's still work to be done but this is a solid start. Signed-off-by: Zach Brown --- kmod/src/block.c | 4 +- kmod/src/buddy.c | 1146 ++++++++++++++++++++------------------ kmod/src/buddy.h | 6 +- kmod/src/format.h | 106 ++-- kmod/src/scoutfs_trace.h | 47 ++ kmod/src/super.c | 11 +- kmod/src/super.h | 2 +- kmod/src/trans.c | 7 +- 8 files changed, 702 insertions(+), 627 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 4923832f..874d04ab 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -434,7 +434,7 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb, struct scoutfs_block *found; struct scoutfs_block *bl; unsigned long flags; - u64 blkno; + u64 blkno = 0; int ret; int err; @@ -442,7 +442,7 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb, if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) return bl; - ret = scoutfs_buddy_alloc(sb, &blkno, 0); + ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 47d89504..e274e3d2 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -11,660 +11,698 @@ * General Public License for more details. */ #include -#include #include "super.h" #include "format.h" #include "block.h" #include "buddy.h" -#include "msg.h" +#include "scoutfs_trace.h" /* - * scoutfs uses buddy bitmaps to allocate block regions. It has a nice - * and simple implementation and reasonably small storage and memory - * overhead, particularly in the pathological fragmented case, but - * results in more rigid allocation constraints and fragmentation. + * scoutfs uses buddy bitmaps to allocate block regions. The buddy + * allocator is nice because it uses one index for allocating by size + * and freeing and merging by location. The index is dense and has a + * predictable worst case size that we can preallocate. As described + * below, it also makes it easy to find unions of free regions between + * two indexes. * - * The buddy allocator is build from a hierarchy of bitmaps for each + * The buddy allocator is built from a hierarchy of bitmaps for each * power of two order of blocks that we can allocate. If a high order * buddy bit is set then all the lower order bits that it covers are - * clear. + * clear. The bits are stored in blocks that are stored in a fixed + * depth radix with a single parent indirect block. The super block + * references the indirect block. The block references in the indirect + * block also include a bitmap of orders that are free in the referenced + * block. * - * At runtime all the bitmaps for all the orders are stored in a single - * packed bitmap in memory. We construct an array of pointers into the - * big bitmap for each individual order bitmap. This lets us easily - * track modifications of all the order bitmaps with a second bitmap - * which tracks fixed size chunks of the main bitmap. + * The blknos for the buddy blocks themselves are allocated out of a + * single bitmap block that is referenced by the super. * - * As a transaction is written the modified chunks of the main bitmap - * are written to the tail of a preallocated ring of buddy blocks. - * This turns noisy scattered bit modification operations into one large - * contiguous block IO. + * All the blocks are read and cowed with the usual block layer routines + * so that we reuse the same code to evict and retry stale cached + * blocks, cow, etc. The allocator in the block code gives us the + * source blkno for a cow operation so we can use the correct allocator + * (none for bitmap blocks, bitmap for buddy blocks, buddy for btree + * blocks and extents). * - * We always write to the tail of the ring so we need to ensure that the - * blocks at the tail don't contain live data. As we mark each chunk of - * the bitmap modified during a transaction we also sweep through the - * bitmap finding another chunk that has never been modified by the - * current sweep. Eventually enough chunks are modified by transactions - * to advance the sweep through the whole bitmap. At this point we're - * sure that all the blocks written to the tail during the sweep have to - * contain the full bitmap. By sizing the ring to 4x the bitmap size we - * ensure that we'll finish the sweep in each half, ensuring that the - * tail is always far enough behind the head to not overwrite live - * chunks. + * The trickiest part of the allocator is due to the cow nature of our + * consistent updates. We can't satisfy an allocation with a region + * that's been freed in this transaction and is still referenced by the + * old stable transaction. We solve this by only returning regions that + * are free in both the stable and currently dirty allocator structures. * - * The entire ring is read the first time the allocator is needed. - * Today that's on mount for the entire system. As we layer on - * functionality we'll have multiple allocators and they'll be passed - * around the cluster as mounts are given access. As mounts get access - * they only need to read the newly written blocks in the ring to bring - * their stale allocator up to date with recent modifications written to - * the tail. The ring indices are full 64bits so that readers can - * recognize when they need to read the whole ring. + * The single indirect block in the radix limits the number of blocks + * that can be described by the radix to just under a TB. The device + * will be managed by multiple radix trees some day. * - * The allocator only covers the blocks after the ring blocks to the end - * of the device. When we move to multiple allocators each will cover a - * fixed set of blocks excluding their ring blocks. Resizing will - * change the number of allocators needed to cover the device and will - * modify the bits in a final allocator. The bitmap modifications for - * resizing would be written to ring blocks as usual. Care will be - * taken to recognize device sizes whose final blocks land in the ring - * blocks. + * XXX: + * - verify blocks on read? + * - more rigorously test valid blkno/order inputs + * - detect corruption/errors when trying to free free extents + * - mkfs should initialize all the slots + * - shrink and grow + * - metadata and data regions + * - worry about testing for free buddies outside device during free? + * - scoutfs_dirty_ref should call us to free old stable + * - btree should free blocks on merge and some failure + * - might want to add a alloc predirty call to avoid error unwind failure + * - we could track the first set in order bitmaps, dunno if it'd be worth it */ -struct buddy_alloc { - - /* - * addr: pointer to le64 that contains the start of the bitmap - * addr_bit: full bit nr of lsb at addr - * addr_off: bit offset from addr to first order bit - * addr_size: bit count from addr of the order's bits - * first_set: first logical order bit offset that might be set - */ - struct buddy_order { - __le64 *addr; - long addr_bit; - long addr_off; - long addr_size; - long first_set; - } orders[64]; - - int max_order; - - u64 orig_tail; - long *modified; - long modified_size; - - long reserved_chunks; - - __le64 *bitmap; +enum { + REGION_PAIR, /* two bitmap blocks at known blknos */ + REGION_BM, /* buddy blocks in the bitmap block off the super */ + REGION_BUDDY, /* btree blocks and extents in the buddy bitmaps */ }; -/* return the first device blkno covered by the allocator */ +static int blkno_region(struct scoutfs_super_block *super, u64 blkno) +{ + u64 end; + + end = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR; + if (blkno < end) + return REGION_PAIR; + + end += le32_to_cpu(super->buddy_blocks); + if (blkno < end) + return REGION_BM; + + return REGION_BUDDY; +} + +/* the first device blkno covered by the buddy allocator */ static u64 first_blkno(struct scoutfs_super_block *super) { - return SCOUTFS_BUDDY_BLKNO + le32_to_cpu(super->buddy_blocks); + return SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + + le32_to_cpu(super->buddy_blocks); } -/* return the number of blocks addressible by the allocator. */ -static u64 covered_blocks(struct scoutfs_super_block *super) +/* the slot in the indirect block of a given blkno */ +static int indirect_slot(struct scoutfs_super_block *super, u64 blkno) { - return le64_to_cpu(super->total_blocks) - first_blkno(super); + return (u32)(blkno - first_blkno(super)) / SCOUTFS_BUDDY_ORDER0_BITS; } -/* return the device block number of a ring index */ -static u64 ring_blkno(struct scoutfs_super_block *super, u64 index) +/* device blkno of order bit in slot */ +static u64 slot_buddy_blkno(struct scoutfs_super_block *super, int sl, + int order, int nr) { - return SCOUTFS_BUDDY_BLKNO + - do_div(index, le32_to_cpu(super->buddy_blocks)); + return first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS) + + ((u64)nr << order); +} + +/* number of blocks managed by the buddy block referenced by the given slot */ +static int slot_count(struct scoutfs_super_block *super, int sl) +{ + u64 first = first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS); + + return min_t(int, le64_to_cpu(super->total_blocks) - first, + SCOUTFS_BUDDY_ORDER0_BITS); +} + +/* the order 0 bit offset of blkno */ +static int buddy_bit(struct scoutfs_super_block *super, u64 blkno) +{ + return (u32)(blkno - first_blkno(super)) % SCOUTFS_BUDDY_ORDER0_BITS; +} + +/* true if the blkno could be the start of an allocation of the order */ +static bool valid_order(struct scoutfs_super_block *super, u64 blkno, int order) +{ + return (buddy_bit(super, blkno) & ((1 << order) - 1)) == 0; +} + +/* the starting bit offset in the block bitmap of an order's bitmap */ +static int order_off(int order) +{ + if (order == 0) + return 0; + + return (2 * SCOUTFS_BUDDY_ORDER0_BITS) - + (SCOUTFS_BUDDY_ORDER0_BITS / (1 << (order - 1))); +} + +/* the bit offset in the block bitmap of an order's bit */ +static int order_nr(int order, int nr) +{ + return order_off(order) + nr; +} + +static int test_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) +{ + return !!test_bit_le(order_nr(order, nr), bud->bits); +} + +static int test_buddy_bit_or_higher(struct scoutfs_buddy_block *bud, int order, + int nr) +{ + int i; + + for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { + if (test_buddy_bit(bud, i, nr)) + return true; + nr >>= 1; + } + + return false; +} + +static void set_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) +{ + if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) + le32_add_cpu(&bud->order_counts[order], 1); +} + +static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) +{ + if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) + le32_add_cpu(&bud->order_counts[order], -1); +} + +/* returns INT_MAX when there are no bits set */ +static int find_next_buddy_bit(struct scoutfs_buddy_block *bud, int order, + int nr) +{ + int size = order_off(order + 1); + + nr = find_next_bit_le(bud->bits, size, order_nr(order, nr)); + if (nr >= size) + return INT_MAX; + + return nr - order_off(order); +} + +static void update_free_orders(struct scoutfs_buddy_slot *slot, + struct scoutfs_buddy_block *bud) +{ + u8 free = 0; + int i; + + for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) + free |= (!!bud->order_counts[i]) << i; + + slot->free_orders = free; } /* - * Find and mark the next chunk in the bitmap that has never been - * written to the current half of the block ring. - * - * If we finish the sweep through the bitmap then we know that the most - * current half of the ring contain the full bitmap and reading at the - * head no longer has to start from the previous half. + * Allocate a buddy block blkno from the super's dirty bitmap block. + * Stable buddy blocks are freed as they're cowed so we have to make + * sure that we only return blknos that were free in the previous stable + * bitmap block. */ -static bool modify_sweep_bit(struct scoutfs_super_block *super, - struct buddy_alloc *bud) +static int bitmap_alloc(struct super_block *sb, u64 *blkno) { - bool did_set; - long bit; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_bitmap_block *st_bm; + struct scoutfs_bitmap_block *bm; + struct scoutfs_block *st_bl; + struct scoutfs_block *bm_bl; + int size; + int ret; + int d; + int s; - bit = le32_to_cpu(super->buddy_sweep_bit); - if (bit >= bud->modified_size) - return false; + /* mkfs should have ensured that there's bitmap blocks */ + /* XXX corruption */ + if (sbi->super.buddy_bm_ref.blkno == 0 || + sbi->stable_super.buddy_bm_ref.blkno == 0) + return -EIO; - bit = find_next_zero_bit(bud->modified, bud->modified_size, bit); - if (bit < bud->modified_size) { - set_bit(bit, bud->modified); - bud->reserved_chunks--; - bit++; - did_set = true; - } else { - bit = bud->modified_size; - did_set = false; + /* dirty the bitmap block */ + bm_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); + if (IS_ERR(bm_bl)) + return PTR_ERR(bm_bl); + bm = bm_bl->data; + + /* read the stable bitmap block */ + st_bl = scoutfs_read_ref(sb, &sbi->stable_super.buddy_bm_ref); + if (IS_ERR(st_bl)) { + ret = PTR_ERR(st_bl); + goto out; } - - super->buddy_sweep_bit = cpu_to_le32(bit); - - /* advance head once we finish the sweep */ - if (bit == bud->modified_size) { - u64 head = le64_to_cpu(super->buddy_head); - u64 tail = le64_to_cpu(super->buddy_tail); - u32 half = le32_to_cpu(super->buddy_blocks) / 2; - - if ((tail - head) > half) - le64_add_cpu(&super->buddy_head, half); - } - - return did_set; -} - -/* - * The caller has modified the given bit in the full buddy bitmap. We - * try to mark its chunk modified and advance the sweep through older - * chunks. - */ -static void modified_bit(struct scoutfs_super_block *super, - struct buddy_alloc *bud, int order, long bit) -{ - struct buddy_order *ord = &bud->orders[order]; - - bit = (ord->addr_bit + ord->addr_off + bit) / SCOUTFS_BUDDY_CHUNK_BITS; - - if (!test_and_set_bit(bit, bud->modified)) { - bud->reserved_chunks--; - modify_sweep_bit(super, bud); - } -} - -static int test_buddy_bit(struct buddy_alloc *bud, int order, long bit) -{ - struct buddy_order *ord = &bud->orders[order]; - - return !!test_bit_le(ord->addr_off + bit, ord->addr); -} - -static void set_buddy_bit(struct scoutfs_super_block *super, - struct buddy_alloc *bud, int order, long bit) -{ - struct buddy_order *ord = &bud->orders[order]; - - set_bit_le(ord->addr_off + bit, ord->addr); - ord->first_set = min(bit, ord->first_set); - - modified_bit(super, bud, order, bit); -} - -static void clear_buddy_bit(struct scoutfs_super_block *super, - struct buddy_alloc *bud, int order, long bit) -{ - struct buddy_order *ord = &bud->orders[order]; - - clear_bit_le(ord->addr_off + bit, ord->addr); - if (ord->first_set == bit) - ord->first_set++; - - modified_bit(super, bud, order, bit); -} - -/* returns LONG_MAX when there are no bits set */ -static long find_first_buddy_bit(struct buddy_alloc *bud, int order) -{ - struct buddy_order *ord = &bud->orders[order]; - long ret; - - ret = find_next_bit_le(ord->addr, ord->addr_size, - ord->addr_off + ord->first_set); - if (ret >= ord->addr_size) { - ret = LONG_MAX; - ord->first_set = ord->addr_size - ord->addr_off; - } else { - ret -= ord->addr_off; - ord->first_set = ret; + st_bm = st_bl->data; + + /* find the first bit that is set in both dirty and stable bitmaps */ + size = le32_to_cpu(sbi->super.buddy_blocks); + s = 0; + do { + d = find_next_bit_le(bm->bits, size, s); + s = find_next_bit_le(st_bm->bits, size, d); + } while (d != s); + if (d >= size) { + ret = -ENOSPC; + goto out; } + *blkno = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + d; + clear_bit_le(d, &bm->bits); + ret = 0; +out: + scoutfs_put_block(st_bl); + scoutfs_put_block(bm_bl); return ret; } -/* test if the index is at the first block in either half of the ring */ -static bool start_of_half(struct scoutfs_super_block *super, u64 index) -{ - u32 half = le32_to_cpu(super->buddy_blocks) / 2; - - return do_div(index, half) == 0; -} - -/* - * A buddy operation can modify bits at every order in the worst case. - * (This is a bit overly conservative because high orders will - * eventually share a chunk.) We'll also try to mark old chunks - * modified for each new chunk we modify. - * - * Before we modify the buddy bits we pin dirty blocks to make sure that - * we have enough chunks to store the modified chunks. - * - * As we advance the tail to store new blocks we might wander into the - * next half of the ring. When that happens we reset the sweep bit so - * that we'll start migrating chunks into this new half of the ring. - * - * This is called with the buddy mutex held. It's the only thing that - * does blocking work under the mutex so we could be more clever and - * make the allocation fast path locking more efficient. - */ -static int reserve_block_chunks(struct super_block *sb) +/* Free a buddy block blkno in the super's bitmap block. */ +static int bitmap_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_alloc *bud = sbi->bud; + struct scoutfs_bitmap_block *bm; struct scoutfs_block *bl; - u64 blkno; + int nr; - if (bud->reserved_chunks >= (bud->max_order * 2)) - return 0; + /* mkfs should have ensured that there's bitmap blocks */ + /* XXX corruption */ + if (sbi->super.buddy_bm_ref.blkno == 0) + return -EIO; - blkno = ring_blkno(super, le64_to_cpu(super->buddy_tail)); - bl = scoutfs_new_block(sb, blkno); + bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); if (IS_ERR(bl)) return PTR_ERR(bl); + bm = bl->data; + nr = blkno - (SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR); + set_bit_le(nr, bm->bits); scoutfs_put_block(bl); - bud->reserved_chunks += SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; - le64_add_cpu(&super->buddy_tail, 1); - if (start_of_half(super, le64_to_cpu(super->buddy_tail))) - super->buddy_sweep_bit = 0; return 0; } /* - * Return the block number of an allocation of at least the requested - * order. If an allocation at the given order isn't free then first try - * to satisfy the allocation with a part of a larger order, then return - * a smaller allocation. - * - * The order of the allocation is returned. + * Give the caller a dirty buddy block. If the slot hasn't been used + * yet then we need to allocate and initialize a new block. */ -int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) +static struct scoutfs_block *dirty_buddy_block(struct super_block *sb, int sl, + struct scoutfs_buddy_slot *slot) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct buddy_alloc *bud = sbi->bud; + struct scoutfs_buddy_block *bud; + struct scoutfs_block *bl; + u64 blkno; + int count; + int order; + int size; + int ret; + int nr; + + /* the fast path is to dirty an existing block */ + if (slot->ref.blkno) + return scoutfs_block_cow_ref(sb, &slot->ref); + + ret = bitmap_alloc(sb, &blkno); + if (ret) + return ERR_PTR(ret); + + bl = scoutfs_new_block(sb, blkno); + if (IS_ERR(bl)) { + bitmap_free(sb, blkno); + return bl; + } + bud = bl->data; + scoutfs_zero_block_tail(bl, sizeof(bud->hdr)); + + /* mark the initial run of highest orders free */ + count = slot_count(super, sl); + order = SCOUTFS_BUDDY_ORDERS - 1; + size = 1 << order; + nr = 0; + while (count > size) { + set_buddy_bit(bud, order, nr); + nr++; + count -= size; + } + + /* set order bits for each of the bits set in the remaining count */ + do { + if (count & (1 << order)) { + set_buddy_bit(bud, order, nr); + nr = (nr + 1) << 1; + } else { + nr <<= 1; + } + } while (order--); + + slot->ref.blkno = bud->hdr.blkno; + slot->ref.seq = bud->hdr.seq; + + update_free_orders(slot, bud); + + return bl; +} + +/* + * Return the order bitmap offset and order of the first allocation + * that fits the desired order. + * + * Returns INT_MAX if there are no free orders. + */ +static int find_first_fit(struct scoutfs_super_block *super, int sl, + struct scoutfs_buddy_block *bud, + struct scoutfs_buddy_block *st_bud, + int order, int *order_ret) +{ + int nrs[SCOUTFS_BUDDY_ORDERS] = {0,}; + u64 blkno = U64_MAX; + bool made_progress; + int ret = INT_MAX; + u64 bno; + int nr; + int i; + + do { + made_progress = false; + for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { + /* find the next bit in each order */ + nr = find_next_buddy_bit(bud, i, nrs[i]); + nrs[i] = nr; + if (nr == INT_MAX) { + continue; + } + made_progress = true; + + /* advance to next bit if it's not free in stable */ + if (!st_bud || + !test_buddy_bit_or_higher(st_bud, i, nr)) { + nrs[i] = nr + 1; + continue; + } + + /* use the first lowest order blkno */ + bno = slot_buddy_blkno(super, sl, i, nr); + if (bno < blkno) { + blkno = bno; + *order_ret = i; + ret = nr; + } + } + + } while (ret == INT_MAX && made_progress); + + return ret; +} + +/* + * Find the first free region that satisfies the given order that is + * also free in the stable buddy bitmaps. This can return an allocation + * that breaks up a larger order. Higher level callers iterate over + * smaller orders to provide partial allocations. + */ +static int alloc_slot(struct super_block *sb, int sl, + struct scoutfs_buddy_slot *slot, + struct scoutfs_block_ref *stable_ref, + u64 *blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_buddy_block *bud; + struct scoutfs_buddy_block *st_bud; + struct scoutfs_block *st_bl; + struct scoutfs_block *bl; int found; - long bit; + int ret; + int nr; + int i; + + /* initialize or dirty the slot's buddy block */ + bl = dirty_buddy_block(sb, sl, slot); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bud = bl->data; + + /* read stable slots's buddy block if there is one */ + if (stable_ref->blkno) { + st_bl = scoutfs_read_ref(sb, stable_ref); + if (IS_ERR(st_bl)) { + ret = PTR_ERR(st_bl); + goto out; + } + st_bud = st_bl->data; + } else { + st_bl = NULL; + st_bud = NULL; + } + + nr = find_first_fit(super, sl, bud, st_bud, order, &found); + if (nr == INT_MAX) { + ret = -ENOSPC; + goto out; + } + + /* we'll succeed from this point on, use nr before mangling it */ + *blkno = slot_buddy_blkno(super, sl, found, nr); + + /* always clear the found order */ + clear_buddy_bit(bud, found, nr); + + /* free right buddies if we're breaking up a larger order */ + for (nr <<= 1, i = found - 1; i >= order; i--, nr <<= 1) + set_buddy_bit(bud, i, nr | 1); + + update_free_orders(slot, bud); + ret = 0; +out: + scoutfs_put_block(st_bl); + scoutfs_put_block(bl); + return ret; +} + +/* + * Try and find a free block extent of the given order. We can fail to + * find a free order when none of the slots have free orders as the + * volume fills or gets fragmented. + * + * We also have to be careful to only return free extents that were free + * in the old stable buddy allocator so that we don't allocate and write + * over referenced data. This can cause us to skip otherwise available + * extents but it should be rare. There can only be a transaction's + * worth of difference between the dirty allocator and the stable + * allocator. This is one of the motivations to cap the size of + * transactions. + */ +static int alloc_order(struct super_block *sb, u64 *blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_buddy_indirect *st_ind; + struct scoutfs_buddy_indirect *ind; + struct scoutfs_block *st_bl = NULL; + struct scoutfs_block *bl = NULL; + u8 mask; int ret; int i; - if (WARN_ON_ONCE(order < 0 || order > bud->max_order)) + /* mkfs should have ensured that there's indirect blocks */ + if (sbi->super.buddy_ind_ref.blkno == 0 || + sbi->stable_super.buddy_ind_ref.blkno == 0) { + ret = -EIO; + goto out; + } + + /* get the dirty indirect block */ + bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + ind = bl->data; + + /* get the stable indirect block */ + st_bl = scoutfs_read_ref(sb, &sbi->stable_super.buddy_ind_ref); + if (IS_ERR(st_bl)) { + ret = PTR_ERR(st_bl); + goto out; + } + st_ind = st_bl->data; + + mask = ~0U << order; + + /* + * try to alloc from each slot that has at least the order free + * in both the dirty and stable buddy blocks. + */ + for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) { + if (!((mask & ind->slots[i].free_orders) && + (mask & st_ind->slots[i].free_orders))) { + ret = -ENOSPC; + continue; + } + + ret = alloc_slot(sb, i, &ind->slots[i], &st_ind->slots[i].ref, + blkno, order); + if (ret != -ENOSPC) + break; + } + +out: + scoutfs_put_block(st_bl); + scoutfs_put_block(bl); + + return ret; +} + +/* + * The buddy allocator keeps trying smaller orders until it finds an + * allocation. + * + * The order of the allocation is returned. + */ +static int buddy_alloc(struct super_block *sb, u64 *blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret; + + if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS)) return -EINVAL; mutex_lock(&sbi->buddy_mutex); - ret = reserve_block_chunks(sb); - if (ret) - goto out; + do { + ret = alloc_order(sb, blkno, order); + } while (ret == -ENOSPC && order--); - /* search for larger and smaller orders */ - i = order; - while (i >= 0) { - bit = find_first_buddy_bit(bud, i); - if (bit < LONG_MAX) - break; - - if (i >= order && i < bud->max_order) - i++; - else if (i == bud->max_order) - i = order - 1; - else - i--; - } - if (i < 0) { - ret = -ENOSPC; - goto out; - } - found = i; - - /* we'll succeed from this point on, use bit before mangling it */ - *blkno = first_blkno(super) + ((u64)bit << found); - ret = min(found, order); - - /* always clear the found order */ - clear_buddy_bit(super, bud, found, bit); - - /* free right buddies if we're breaking up a larger order */ - for (bit <<= 1, i = found - 1; i >= order; i--, bit <<= 1) - set_buddy_bit(super, bud, i, bit | 1); - -out: mutex_unlock(&sbi->buddy_mutex); - if (WARN_ON_ONCE(ret < 0)) - *blkno = 0; + + return ret ?: order; +} + +/* + * Allocate a block from the given region. The caller has the buddy + * mutex if we're called for either of the pair or bitmap internal + * regions. + */ +static int alloc_region(struct super_block *sb, u64 *blkno, int order, + u64 existing, int region) +{ + int ret; + + switch(region) { + case REGION_PAIR: + *blkno = existing ^ 1; + ret = 0; + break; + case REGION_BM: + ret = bitmap_alloc(sb, blkno); + break; + case REGION_BUDDY: + ret = buddy_alloc(sb, blkno, order); + break; + } + + trace_scoutfs_buddy_alloc(*blkno, order, region, ret); return ret; } +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) +{ + return alloc_region(sb, blkno, order, 0, REGION_BUDDY); +} + +/* + * The block layer allocates from the same region as an existing blkno + * when it's allocating for cow. + */ +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, + u64 existing) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + return alloc_region(sb, blkno, order, existing, + blkno_region(super, existing)); +} + /* * Free the aligned allocation of the given order at the given blkno to * the allocator. We merge it into adjoining free space by looking for * free buddies as we increase the order. */ -int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +static int buddy_free(struct super_block *sb, u64 blkno, int order) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct buddy_alloc *bud = sbi->bud; - long bit; + struct scoutfs_buddy_indirect *ind; + struct scoutfs_buddy_block *bud; + struct scoutfs_block *ind_bl = NULL; + struct scoutfs_block *bl = NULL; int ret; + int sl; + int nr; int i; - if (WARN_ON_ONCE(order < 0 || order > bud->max_order) || - WARN_ON_ONCE(((blkno + 1) << order) >= covered_blocks(super))) + if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS) || + WARN_ON_ONCE(!valid_order(super, blkno, order))) return -EINVAL; mutex_lock(&sbi->buddy_mutex); - ret = reserve_block_chunks(sb); - if (ret) - goto out; - - bit = (blkno - first_blkno(super)) >> order; - for (i = order; i <= bud->max_order; i++) { - - /* set bit free and finish when buddy isn't free */ - if (!test_buddy_bit(bud, i, bit ^ 1)) { - set_buddy_bit(super, bud, i, bit); - break; - } - - /* otherwise clear buddy and try to set higher parent */ - clear_buddy_bit(super, bud, i, bit ^ 1); - bit >>= 1; - } - -out: - mutex_unlock(&sbi->buddy_mutex); - return ret; -} - -/* - * We're writing a transaction. The buddy allocator records chunks of - * the main bitmap which have been modified during the transaction. We - * copy them to the pinned dirty blocks which will be written as part of - * the transaction. The bitmap of modified chunks and the old ring tail - * are only reset when the transaction is successfully written. - */ -int scoutfs_dirty_buddy_chunks(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_alloc *bud = sbi->bud; - struct scoutfs_buddy_chunk *chunk; - struct scoutfs_buddy_block *bb; - struct scoutfs_block *bl; - long bit; - long ind; - u64 tail; - int i; - - /* short circuit a transaction with no modified chunks */ - if (bud->orig_tail == le64_to_cpu(super->buddy_tail)) - return 0; - - while (bud->reserved_chunks && modify_sweep_bit(super, bud)) - ; - - for (tail = bud->orig_tail, bit = 0; - tail < le64_to_cpu(super->buddy_tail) && bit < bud->modified_size; - tail++) { - - bl = scoutfs_read_block(sb, ring_blkno(super, tail)); - if (WARN_ON_ONCE(IS_ERR(bl))) - return PTR_ERR(bl); - - bb = bl->data; - bb->hdr.seq = cpu_to_le64(tail); - bb->nr_chunks = 0; - - for (i = 0; i < SCOUTFS_BUDDY_CHUNKS_PER_BLOCK; i++) { - bit = find_next_bit(bud->modified, bud->modified_size, - bit); - if (bit >= bud->modified_size) - break; - - chunk = &bb->chunks[i]; - chunk->pos = cpu_to_le32(bit); - ind = bit * SCOUTFS_BUDDY_CHUNK_LE64S; - memcpy(chunk->bits, &bud->bitmap[ind], - SCOUTFS_BUDDY_CHUNK_BYTES); - bit++; - } - - bb->nr_chunks = i; - scoutfs_zero_block_tail(bl, offsetof(struct scoutfs_buddy_block, - chunks[bb->nr_chunks])); - scoutfs_put_block(bl); - } - - /* - * Chunk reservation should have ensured that there's always room - * in the tail blocks for the modified chunks. - */ - if (WARN_ON_ONCE(bit < bud->modified_size)) - return -EIO; - - return 0; -} - -void scoutfs_reset_buddy_chunks(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_alloc *bud = sbi->bud; - - bud->orig_tail = le64_to_cpu(super->buddy_tail); - memset(bud->modified, 0, DIV_ROUND_UP(bud->modified_size, 8)); -} - -static int check_buddy_fields(struct super_block *sb, - struct scoutfs_super_block *super) -{ - u32 blocks = le32_to_cpu(super->buddy_blocks); - u32 half = blocks / 2; - u64 head = le64_to_cpu(super->buddy_head); - u64 tail = le64_to_cpu(super->buddy_tail); - u64 buddy_bits; - u64 chunk_bits; - - /* have to at least have two halves */ - if (blocks < 2) { - scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); - return -EIO; - } - - /* - * insist that blocks be a multiple of two so that we don't have - * scary fencepost off by ones around the half calculations. - */ - if (blocks & 1) { - scoutfs_info(sb, "buddy_blocks %lu isn't even", blocks); - return -EIO; - } - - /* shouldn't fill the device with buddy blocks */ - if (first_blkno(super) >= le64_to_cpu(super->total_blocks)) { - scoutfs_info(sb, "buddy_blocks %lu must be at least 2", blocks); - return -EIO; - } - - /* can only reference a 32bit long's worth of buddy bits */ - buddy_bits = covered_blocks(super) * 2; - if (buddy_bits >= INT_MAX) { - scoutfs_info(sb, "device needs %llu > INT_MAX buddy bits", - buddy_bits); - return -EIO; - } - - /* need enough ring blocks for 4 full buddy copies */ - chunk_bits = blocks * SCOUTFS_BUDDY_CHUNKS_PER_BLOCK * - SCOUTFS_BUDDY_CHUNK_BITS; - if (buddy_bits * 4 > chunk_bits) { - scoutfs_info(sb, "only room for %llu bits in chunks, need %llu", - chunk_bits, buddy_bits * 4); - return -EIO; - } - - if (head > tail) { - scoutfs_info(sb, "buddy_head %llu > buddy_tail %llu", - head, tail); - return -EIO; - } - - /* tail can't wrap around into head */ - if ((tail - head) >= blocks) { - scoutfs_info(sb, "buddy_tail %llu overlaps buddy_head %llu", - tail, head); - return -EIO; - } - - /* head always has to start one of the halves */ - if (!start_of_half(super, head)) { - scoutfs_info(sb, "buddy_head %llu isn't multiple of half %u", - head, half); - return -EIO; - } - - return 0; -} - -/* - * Reconstruct the entire buddy bitmap by replaying the chunks that are - * contained in the buddy block ring. - * - * The allocator doesn't cover the super blocks and ring blocks and is - * initialized with all the device blocks marked free so that mkfs - * doesn't have to write any chunks to initialize free space. - * - * We go a little nuts with variables to make it easier to read. - */ -int scoutfs_read_buddy_chunks(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_chunk *chunk; - struct scoutfs_buddy_block *bb; - struct scoutfs_block *bl; - struct buddy_alloc *bud; - struct buddy_order *ord; - u64 buddy_bits; - u64 dev_blocks; - u64 chunks; - u64 head; - u64 tail; - long bits; - long bit; - long ind; - int ret; - int i; - - ret = check_buddy_fields(sb, super); - if (ret) - return ret; - - dev_blocks = covered_blocks(super); - buddy_bits = dev_blocks * 2; - chunks = DIV_ROUND_UP(buddy_bits, SCOUTFS_BUDDY_CHUNK_BITS); - - bud = kzalloc(sizeof(struct buddy_alloc), GFP_KERNEL); - if (bud) { - bud->bitmap = vzalloc(round_up(buddy_bits, 64) / 8); - bud->modified = vzalloc(round_up(chunks, BITS_PER_LONG) / 8); - } - if (!bud || !bud->bitmap || !bud->modified) { - ret = -ENOMEM; + /* mkfs should have ensured that there's indirect blocks */ + if (sbi->super.buddy_ind_ref.blkno == 0) { + ret = -EIO; goto out; } - sbi->bud = bud; - bud->modified_size = chunks; + ind_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(ind_bl)) { + ret = PTR_ERR(ind_bl); + goto out; + } + ind = ind_bl->data; + + sl = indirect_slot(super, blkno); + bl = scoutfs_block_cow_ref(sb, &ind->slots[sl].ref); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + bud = bl->data; /* - * Updating first_set across the orders would be tricky so we - * initialize it to 0 and suffer an initial expensive find_first - * call. + * Merge our region with its free buddy and then try to merge + * that higher order region with its buddy, and so on, until the + * highest order. The highest order doesn't have buddies. */ - bit = 0; - bits = dev_blocks; - for (i = 0; i < ARRAY_SIZE(bud->orders); i++) { - ord = &bud->orders[i]; + nr = buddy_bit(super, blkno) >> order; + for (i = order; i < SCOUTFS_BUDDY_ORDERS - 1; i++) { - ord->addr = &bud->bitmap[bit / 64]; - ord->addr_bit = bit & ~63ULL; - ord->addr_off = bit & 63; - ord->addr_size = ord->addr_off + bits; - ord->first_set = 0; - - bit += bits; - bits >>= 1; - if (!bits) - break; - } - bud->max_order = i; - - /* - * Initialize the allocator with the all the blocks covered by - * the fewest number of greatest order free allocations. Ring - * replay will overwrite this. - */ - bit = 0; - for (i = bud->max_order; i >= 0; i--) { - ord = &bud->orders[i]; - - if (ord->addr_off + bit == ord->addr_size) + if (!test_buddy_bit(bud, i, nr ^ 1)) break; - set_bit_le(ord->addr_off + bit, ord->addr); - bit = (bit + 1) << 1; + clear_buddy_bit(bud, i, nr ^ 1); + nr >>= 1; } - head = le64_to_cpu(super->buddy_head); - tail = le64_to_cpu(super->buddy_tail); - while (head < tail) { - bl = scoutfs_read_block(sb, ring_blkno(super, head)); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - goto out; - } + set_buddy_bit(bud, i, nr); - bb = bl->data; - if (le64_to_cpu(bb->hdr.seq) != head) { - /* XXX corruption */ - ret = -EIO; - scoutfs_put_block(bl); - goto out; - } - - for (i = 0; i < bb->nr_chunks; i++) { - chunk = &bb->chunks[i]; - - /* XXX check */ - ind = le32_to_cpu(chunk->pos) * - SCOUTFS_BUDDY_CHUNK_LE64S; - - memcpy(&bud->bitmap[ind], chunk->bits, - SCOUTFS_BUDDY_CHUNK_BYTES); - } - scoutfs_put_block(bl); - head++; - } + update_free_orders(&ind->slots[sl], bud); + scoutfs_put_block(bl); ret = 0; out: - if (ret) { - if (bud) { - vfree(bud->bitmap); - vfree(bud->modified); - } + mutex_unlock(&sbi->buddy_mutex); + scoutfs_put_block(ind_bl); + + return ret; +} + +int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + int region; + int ret; + + region = blkno_region(super, blkno); + switch(blkno_region(super, blkno)) { + case REGION_PAIR: + ret = 0; + break; + case REGION_BM: + ret = bitmap_free(sb, blkno); + break; + case REGION_BUDDY: + ret = buddy_free(sb, blkno, order); + break; } + + trace_scoutfs_buddy_free(blkno, order, region, ret); return ret; } diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 8a411178..b4e5b2f6 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -2,10 +2,8 @@ #define _SCOUTFS_BUDDY_H_ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, + u64 existing); int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); -int scoutfs_read_buddy_chunks(struct super_block *sb); -void scoutfs_reset_buddy_chunks(struct super_block *sb); -int scoutfs_dirty_buddy_chunks(struct super_block *sb); - #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index e3112b3e..4c23d6e4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -19,7 +19,8 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 -#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) +#define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) +#define SCOUTFS_BUDDY_BM_NR 2 /* * This header is found at the start of every block so that we can @@ -35,6 +36,52 @@ struct scoutfs_block_header { __le64 blkno; } __packed; +/* + * Block references include the sequence number so that we can detect + * readers racing with writers and so that we can tell that we don't + * need to follow a reference when traversing based on seqs. + */ +struct scoutfs_block_ref { + __le64 blkno; + __le64 seq; +} __packed; + +struct scoutfs_bitmap_block { + struct scoutfs_block_header hdr; + __le64 bits[0]; +} __packed; + +/* + * Track allocations from BLOCK_SIZE to (BLOCK_SIZE << ..._ORDERS). + */ +#define SCOUTFS_BUDDY_ORDERS 8 + +struct scoutfs_buddy_block { + struct scoutfs_block_header hdr; + __le32 order_counts[SCOUTFS_BUDDY_ORDERS]; + __le64 bits[0]; +} __packed; + +/* + * If we had log2(raw bits) orders we'd fully use all of the raw bits in + * the block. We're close enough that the amount of space wasted at the + * end (~1/256th of the block, ~64 bytes) isn't worth worrying about. + */ +#define SCOUTFS_BUDDY_ORDER0_BITS \ + (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2) + +struct scoutfs_buddy_indirect { + struct scoutfs_block_header hdr; + struct scoutfs_buddy_slot { + __u8 free_orders; + struct scoutfs_block_ref ref; + } slots[0]; +} __packed; + +#define SCOUTFS_BUDDY_SLOTS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) / \ + sizeof(struct scoutfs_buddy_slot)) + /* * We should be able to make the offset smaller if neither dirents nor * data items use the full 64 bits. @@ -57,16 +104,6 @@ struct scoutfs_key { #define SCOUTFS_MAX_ITEM_LEN 2048 -/* - * Block references include the sequence number so that we can detect - * readers racing with writers and so that we can tell that we don't - * need to follow a reference when traversing based on seqs. - */ -struct scoutfs_block_ref { - __le64 blkno; - __le64 seq; -} __packed; - struct scoutfs_treap_root { __le16 off; } __packed; @@ -109,48 +146,6 @@ struct scoutfs_btree_item { #define SCOUTFS_UUID_BYTES 16 -/* - * Arbitrarily choose a reasonably fine grained 64byte chunk. This is a - * balance between write amplification of writing chunks with a single - * modified bit, storage overhead of partial blocks losing a chunk to - * make room for the block header and having a pos field per chunk, and - * runtime memory overhead of a bit per chunk. - */ -#define SCOUTFS_BUDDY_CHUNK_LE64S 8 -#define SCOUTFS_BUDDY_CHUNK_BYTES (SCOUTFS_BUDDY_CHUNK_LE64S * 8) -#define SCOUTFS_BUDDY_CHUNK_BITS (SCOUTFS_BUDDY_CHUNK_BYTES * 8) - -/* - * After the pair of super blocks are a preallocated ring of blocks - * which record modified regions of the buddy bitmap allocator. - * - * The seq's header needs to match the unwrapped ring index of the - * block. - */ -struct scoutfs_buddy_block { - struct scoutfs_block_header hdr; - u8 nr_chunks; - struct scoutfs_buddy_chunk { - __le32 pos; - __le64 bits[SCOUTFS_BUDDY_CHUNK_LE64S]; - } __packed chunks[0]; -} __packed; - -#define SCOUTFS_BUDDY_CHUNKS_PER_BLOCK \ - ((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_buddy_block, chunks)) /\ - SCOUTFS_BUDDY_CHUNK_BYTES) - - -/* - * The super is stored in a pair of blocks in the first chunk on the - * device. - * - * The ring map blocks describe the chunks that make up the ring. - * - * The rest of the ring fields describe the state of the ring blocks - * that are stored in their chunks. The active portion of the ring - * describes the current state of the system and is replayed on mount. - */ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -158,10 +153,9 @@ struct scoutfs_super_block { __le64 next_ino; __le64 total_blocks; __le32 buddy_blocks; - __le32 buddy_sweep_bit; - __le64 buddy_head; - __le64 buddy_tail; struct scoutfs_btree_root btree_root; + struct scoutfs_block_ref buddy_ind_ref; + struct scoutfs_block_ref buddy_bm_ref; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 015a6700..fb96658d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -109,6 +109,53 @@ TRACE_EVENT(scoutfs_update_inode, __entry->ino, __entry->size) ); +TRACE_EVENT(scoutfs_buddy_alloc, + TP_PROTO(u64 blkno, int order, int region, int ret), + + TP_ARGS(blkno, order, region, ret), + + TP_STRUCT__entry( + __field(u64, blkno) + __field(int, order) + __field(int, region) + __field(int, ret) + ), + + TP_fast_assign( + __entry->blkno = blkno; + __entry->order = order; + __entry->region = region; + __entry->ret = ret; + ), + + TP_printk("blkno %llu order %d region %d ret %d", + __entry->blkno, __entry->order, __entry->region, __entry->ret) +); + + +TRACE_EVENT(scoutfs_buddy_free, + TP_PROTO(u64 blkno, int order, int region, int ret), + + TP_ARGS(blkno, order, region, ret), + + TP_STRUCT__entry( + __field(u64, blkno) + __field(int, order) + __field(int, region) + __field(int, ret) + ), + + TP_fast_assign( + __entry->blkno = blkno; + __entry->order = order; + __entry->region = region; + __entry->ret = ret; + ), + + TP_printk("blkno %llu order %d region %d ret %d", + __entry->blkno, __entry->order, __entry->region, __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 2a0d1614..8d7137b4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -47,6 +47,8 @@ void scoutfs_advance_dirty_super(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + sbi->stable_super = sbi->super; + le64_add_cpu(&super->hdr.blkno, 1); if (le64_to_cpu(super->hdr.blkno) == (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)) @@ -107,8 +109,7 @@ static int read_supers(struct super_block *sb) if (found < 0 || (le64_to_cpu(super->hdr.seq) > le64_to_cpu(sbi->super.hdr.seq))) { - memcpy(&sbi->super, super, - sizeof(struct scoutfs_super_block)); + sbi->super = *super; found = i; } } @@ -123,6 +124,8 @@ static int read_supers(struct super_block *sb) scoutfs_info(sb, "using super %u with seq %llu", found, le64_to_cpu(sbi->super.hdr.seq)); + sbi->stable_super = sbi->super; + return 0; } @@ -162,13 +165,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: - scoutfs_setup_trans(sb) ?: - scoutfs_read_buddy_chunks(sb); + scoutfs_setup_trans(sb); if (ret) return ret; scoutfs_advance_dirty_super(sb); - scoutfs_reset_buddy_chunks(sb); inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) diff --git a/kmod/src/super.h b/kmod/src/super.h index 03226baf..f0a2e8cf 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,7 @@ struct scoutfs_sb_info { struct super_block *sb; struct scoutfs_super_block super; + struct scoutfs_super_block stable_super; spinlock_t next_ino_lock; @@ -24,7 +25,6 @@ struct scoutfs_sb_info { int block_write_err; struct mutex buddy_mutex; - struct buddy_alloc *bud; /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 4dcfdb99..e1376546 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -64,8 +64,7 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX probably want to write out dirty pages in inodes */ if (scoutfs_has_dirty_blocks(sb)) { - ret = scoutfs_dirty_buddy_chunks(sb) ?: - scoutfs_write_dirty_blocks(sb) ?: + ret = scoutfs_write_dirty_blocks(sb) ?: scoutfs_write_dirty_super(sb); if (!ret) advance = 1; @@ -73,10 +72,8 @@ void scoutfs_trans_write_func(struct work_struct *work) spin_lock(&sbi->trans_write_lock); - if (advance) { + if (advance) scoutfs_advance_dirty_super(sb); - scoutfs_reset_buddy_chunks(sb); - } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock); From ad34f40744da0ea56c24948dafc7fe6620c2a78a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Jul 2016 16:13:37 -0700 Subject: [PATCH 068/920] scoutfs: free source blkno after cow As we update references to point to newly allocated dirty blocks in a transaction we need to free the old referenced blknos. By using a two-phase dirty/free interface we can avoid freeing failing after we've made it through stages of the cow processing which can't be easily undone. Signed-off-by: Zach Brown --- kmod/src/block.c | 10 ++++++ kmod/src/buddy.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/buddy.h | 1 + 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 874d04ab..7808ab5b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -434,6 +434,7 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb, struct scoutfs_block *found; struct scoutfs_block *bl; unsigned long flags; + u64 clean_blkno; u64 blkno = 0; int ret; int err; @@ -442,6 +443,12 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb, if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) return bl; + clean_blkno = bl->blkno; + + ret = scoutfs_buddy_dirty(sb, clean_blkno, 0); + if (ret < 0) + goto out; + ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; @@ -498,6 +505,9 @@ static struct scoutfs_block *dirty_ref(struct super_block *sb, spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); + /* free clean blkno after preload end enables preemption */ + err = scoutfs_buddy_free(sb, clean_blkno, 0); + WARN_ON(err); /* XXX corruption (dirtying should prevent) */ ret = 0; out: scoutfs_put_block(copy_bl); diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index e274e3d2..4021ba3b 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -63,9 +63,7 @@ * - shrink and grow * - metadata and data regions * - worry about testing for free buddies outside device during free? - * - scoutfs_dirty_ref should call us to free old stable * - btree should free blocks on merge and some failure - * - might want to add a alloc predirty call to avoid error unwind failure * - we could track the first set in order bitmaps, dunno if it'd be worth it */ @@ -597,6 +595,89 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) return alloc_region(sb, blkno, order, 0, REGION_BUDDY); } +static int bitmap_dirty(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *bl; + + /* mkfs should have ensured that there's bitmap blocks */ + /* XXX corruption */ + if (sbi->super.buddy_bm_ref.blkno == 0) + return -EIO; + + /* dirty the bitmap block */ + bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); + if (IS_ERR(bl)) + return PTR_ERR(bl); + + scoutfs_put_block(bl); + return 0; +} + +static int buddy_dirty(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_buddy_indirect *ind; + struct scoutfs_block *ind_bl = NULL; + struct scoutfs_block *bl = NULL; + int ret; + int sl; + + mutex_lock(&sbi->buddy_mutex); + + /* mkfs should have ensured that there's indirect blocks */ + if (sbi->super.buddy_ind_ref.blkno == 0) { + ret = -EIO; + goto out; + } + + /* get the dirty indirect block */ + ind_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(ind_bl)) { + ret = PTR_ERR(ind_bl); + goto out; + } + ind = ind_bl->data; + + sl = indirect_slot(super, blkno); + bl = dirty_buddy_block(sb, sl, &ind->slots[sl]); + if (IS_ERR(bl)) + ret = PTR_ERR(bl); + else + ret = 0; +out: + mutex_unlock(&sbi->buddy_mutex); + scoutfs_put_block(ind_bl); + scoutfs_put_block(bl); + + return ret; +} + + +/* + * Create dirty cow copies of the bitmap, indirect, and buddy blocks + * so that a free of the given extent in the current transaction is + * guaranteed to succeed. + * + * This is only meant for buddy allocators who are complicated enough + * to need help avoiding error conditions. + */ +int scoutfs_buddy_dirty(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + switch(blkno_region(super, blkno)) { + case REGION_BM: + return bitmap_dirty(sb, blkno); + case REGION_BUDDY: + return buddy_dirty(sb, blkno, order); + } + + return 0; +} + /* * The block layer allocates from the same region as an existing blkno * when it's allocating for cow. diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index b4e5b2f6..8397aa12 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -4,6 +4,7 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, u64 existing); +int scoutfs_buddy_dirty(struct super_block *sb, u64 blkno, int order); int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); #endif From 0e017ff0dc654174a7c878675f17af83ef1f9b15 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Jul 2016 16:40:22 -0700 Subject: [PATCH 069/920] scoutfs: free btree unused btree blocks The btree code wasn't freeing blocks either when it had removed references to them or when an operation fails after having allocated a new block. Now that the allocator is more capable we can add in these free calls. Signed-off-by: Zach Brown --- kmod/src/btree.c | 46 +++++++++++++++++++++++++++++++--------------- kmod/src/buddy.c | 1 - 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 5ffeb9f5..ceecbee1 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -59,7 +59,6 @@ * performed as we descend. * * XXX - * - actually free blknos * - do we want a level in the btree header? seems like we would? * - validate structures on read? */ @@ -314,6 +313,13 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) return bl; } +/* the caller has ensured that the free must succeed */ +static void free_tree_block(struct super_block *sb, __le64 blkno) +{ + int err = scoutfs_buddy_free(sb, le64_to_cpu(blkno), 0); + WARN_ON_ONCE(err); +} + /* * Allocate a new tree block and point the root at it. The caller * is responsible for the items in the new root block. @@ -400,9 +406,19 @@ static struct scoutfs_block *try_split(struct super_block *sb, return right_bl; } + /* alloc split neighbour first to avoid unwinding tree growth */ + left_bl = alloc_tree_block(sb); + if (IS_ERR(left_bl)) { + scoutfs_put_block(right_bl); + return left_bl; + } + left = left_bl->data; + if (!parent) { par_bl = grow_tree(sb, root); if (IS_ERR(par_bl)) { + free_tree_block(sb, left->hdr.blkno); + scoutfs_put_block(left_bl); scoutfs_put_block(right_bl); return par_bl; } @@ -410,15 +426,6 @@ static struct scoutfs_block *try_split(struct super_block *sb, parent = par_bl->data; } - left_bl = alloc_tree_block(sb); - if (IS_ERR(left_bl)) { - /* XXX free parent block? */ - scoutfs_put_block(par_bl); - scoutfs_put_block(right_bl); - return left_bl; - } - left = left_bl->data; - /* only grow the tree once we have the split neighbour */ if (par_bl) { struct scoutfs_key maximal; @@ -461,6 +468,10 @@ static struct scoutfs_block *try_split(struct super_block *sb, * The caller only has the parent locked. They'll lock whichever * block we return. * + * We free sibling or parent btree block blknos if we drain them of items. + * They're dirtied either by descent or before we start migrating items + * so freeing their blkno must succeed. + * * XXX this could more cleverly chose a merge candidate sibling */ static struct scoutfs_block *try_merge(struct super_block *sb, @@ -514,7 +525,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, /* delete an empty sib or update if we changed its greatest key */ if (sib_bt->nr_items == 0) { delete_item(parent, sib_item); - /* XXX free sib block */ + free_tree_block(sb, sib_bt->hdr.blkno); } else if (move_right) { sib_item->key = *greatest_key(sib_bt); } @@ -524,7 +535,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; - /* XXX free block */ + free_tree_block(sb, parent->hdr.blkno); } return bl; @@ -836,6 +847,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_btree_root *root; struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; @@ -855,9 +867,13 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) /* delete the final block in the tree */ if (bt->nr_items == 0) { - memset(&sbi->super.btree_root, 0, - sizeof(struct scoutfs_btree_root)); - /* XXX free block */ + root = &sbi->super.btree_root; + + root->height = 0; + root->ref.blkno = 0; + root->ref.seq = 0; + + free_tree_block(sb, bt->hdr.blkno); } } else { ret = -ENOENT; diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 4021ba3b..c6e5974b 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -63,7 +63,6 @@ * - shrink and grow * - metadata and data regions * - worry about testing for free buddies outside device during free? - * - btree should free blocks on merge and some failure * - we could track the first set in order bitmaps, dunno if it'd be worth it */ From f024c70802408f8d91ae7d6fadcb71f3883a92aa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Jul 2016 14:12:25 -0700 Subject: [PATCH 070/920] scoutfs: decrease block size File data extent tracking can get very complicated if we have to worry about page sized writes that are less than the block size. We can avoid all that complexity if we define the block size to be the smallest possible page size. Signed-off-by: Zach Brown --- kmod/src/format.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 4c23d6e4..4e9eeaad 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -6,7 +6,7 @@ /* super block id */ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ -#define SCOUTFS_BLOCK_SHIFT 14 +#define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) @@ -102,7 +102,7 @@ struct scoutfs_key { #define SCOUTFS_DIRENT_KEY 3 #define SCOUTFS_DATA_KEY 4 -#define SCOUTFS_MAX_ITEM_LEN 2048 +#define SCOUTFS_MAX_ITEM_LEN 512 struct scoutfs_treap_root { __le16 off; From 7b18bce2e2fb6a30755d6c877eefa08538bac36e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 Aug 2016 11:30:36 -0700 Subject: [PATCH 071/920] scoutfs: use buffer heads Now that we have a fixed small block size we don't need our own code for tracking contiguous memory for blocks that are larger than the page size. We can use buffer heads which support block sizes smaller than the page size. Our block API remains to enforce transactions, cheksumming, cow, and eventually invalidating and retrying reads of stale bloks. We set the logical blocksize of the bdev buffer cache to our fixed block size. We use a private bh state bit to indicate that the contents of a block have had their checksum verified. We use a small structure stored at b_private to track dirty blocks so that we can control when they're written. The btree block traversal code uses the buffer_head lock to serialize access to btree block contents now that the block rwsem has gone away. This isn't great but works for now. Not being able to relocate blocks in the buffer cache (really fragments of pages in the bdev page cache.. blkno determines memory location) means that the cow path always has to copy. Callers are easily translated: use struct buffer_head instead of scoutfs_block and use a little helper instead of dereferencing ->data directly. I took the opportunity to clean up some of the inconsistent block function names. Now more of the functions follow the scoutfs_block_*() pattern. Signed-off-by: Zach Brown --- kmod/src/block.c | 684 +++++++++++++++++++---------------------------- kmod/src/block.h | 56 ++-- kmod/src/btree.c | 333 +++++++++++------------ kmod/src/btree.h | 2 +- kmod/src/buddy.c | 160 +++++------ kmod/src/super.c | 49 ++-- kmod/src/super.h | 2 +- kmod/src/trans.c | 7 +- 8 files changed, 578 insertions(+), 715 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 7808ab5b..e6ddf0a8 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -11,10 +11,9 @@ * General Public License for more details. */ #include +#include +#include #include -#include -#include -#include #include "super.h" #include "format.h" @@ -23,74 +22,51 @@ #include "counters.h" #include "buddy.h" -#define DIRTY_RADIX_TAG 0 - /* + * scoutfs has a fixed 4k small block size for metadata blocks. This + * lets us consistently use buffer heads without worrying about having a + * block size greater than the page size. + * + * This block interface does the work to cow dirty blocks, track dirty + * blocks, generate checksums as they're written, only write them in + * transactions, verify checksums on read, and invalidate and retry + * reads of stale cached blocks. (That last bit only has a hint of an + * implementation.) + * * XXX - * - tie into reclaim - * - per cpu lru of refs? - * - relax locking - * - get, check, and fill slots instead of full radix walks - * - block slab - * - maybe more clever wait functions + * - tear down dirty blocks left by write errors on unmount + * - should invalidate dirty blocks if freed */ -static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) +struct block_bh_private { + struct super_block *sb; + struct buffer_head *bh; + struct rb_node node; +}; + +enum { + BH_ScoutfsVerified = BH_PrivateStart, +}; +BUFFER_FNS(ScoutfsVerified, scoutfs_verified) + +static int verify_block_header(struct scoutfs_sb_info *sbi, + struct buffer_head *bh) { - struct scoutfs_block *bl; - struct page *page; - - /* we'd need to be just a bit more careful */ - BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); - - bl = kzalloc(sizeof(struct scoutfs_block), GFP_NOFS); - if (bl) { - page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); - WARN_ON_ONCE(!page); - if (page) { - init_rwsem(&bl->rwsem); - atomic_set(&bl->refcount, 1); - bl->blkno = blkno; - bl->sb = sb; - bl->page = page; - bl->data = page_address(page); - scoutfs_inc_counter(sb, block_mem_alloc); - } else { - kfree(bl); - bl = NULL; - } - } - - return bl; -} - -void scoutfs_put_block(struct scoutfs_block *bl) -{ - if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { - trace_printk("freeing bl %p\n", bl); - __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); - kfree(bl); - scoutfs_inc_counter(bl->sb, block_mem_free); - } -} - -static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_block_header *hdr = bl->data; + struct scoutfs_block_header *hdr = (void *)bh->b_data; u32 crc = scoutfs_crc_block(hdr); int ret = -EIO; if (le32_to_cpu(hdr->crc) != crc) { - printk("blkno %llu hdr crc %x != calculated %x\n", bl->blkno, - le32_to_cpu(hdr->crc), crc); + printk("blkno %llu hdr crc %x != calculated %x\n", + (u64)bh->b_blocknr, le32_to_cpu(hdr->crc), crc); } else if (super->hdr.fsid && hdr->fsid != super->hdr.fsid) { - printk("blkno %llu fsid %llx != super fsid %llx\n", bl->blkno, - le64_to_cpu(hdr->fsid), le64_to_cpu(super->hdr.fsid)); - } else if (le64_to_cpu(hdr->blkno) != bl->blkno) { - printk("blkno %llu invalid hdr blkno %llx\n", bl->blkno, - le64_to_cpu(hdr->blkno)); + printk("blkno %llu fsid %llx != super fsid %llx\n", + (u64)bh->b_blocknr, le64_to_cpu(hdr->fsid), + le64_to_cpu(super->hdr.fsid)); + } else if (le64_to_cpu(hdr->blkno) != bh->b_blocknr) { + printk("blkno %llu invalid hdr blkno %llx\n", + (u64)bh->b_blocknr, le64_to_cpu(hdr->blkno)); } else { ret = 0; } @@ -98,175 +74,145 @@ static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) return ret; } -static void block_read_end_io(struct bio *bio, int err) +static struct buffer_head *bh_from_bhp_node(struct rb_node *node) { - struct scoutfs_block *bl = bio->bi_private; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + struct block_bh_private *bhp; - if (!err && !verify_block_header(bl->sb, bl)) - set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits); - else - set_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits); + bhp = container_of(node, struct block_bh_private, node); + return bhp->bh; +} - /* - * uncontended spin_lock in wake_up and unconditional smp_mb to - * make waitqueue_active safe are about the same cost, so we - * prefer the obviously safe choice. - */ - wake_up(&sbi->block_wq); +static struct scoutfs_sb_info *sbi_from_bh(struct buffer_head *bh) +{ + struct block_bh_private *bhp = bh->b_private; - scoutfs_put_block(bl); - bio_put(bio); + return SCOUTFS_SB(bhp->sb); +} + +static void insert_bhp_rb(struct rb_root *root, struct buffer_head *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct block_bh_private *bhp; + struct buffer_head *bh; + + while (*node) { + parent = *node; + bh = bh_from_bhp_node(*node); + + if (ins->b_blocknr < bh->b_blocknr) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + bhp = ins->b_private; + rb_link_node(&bhp->node, parent, node); + rb_insert_color(&bhp->node, root); } /* - * Once a transaction block is persistent it's fine to drop the dirty - * tag. It's been checksummed so it can be read in again. It's seq - * will be in the current transaction so it'll simply be dirtied and - * checksummed and written out again. + * Track a dirty block by allocating private data and inserting it into + * the dirty rbtree in the super block. + * + * Callers are in transactions that prevent metadata writeback so blocks + * won't be written and cleaned while we're trying to dirty them. We + * serialize racing to add dirty tracking to the same block in case the + * caller didn't. + * + * Presence in the dirty tree holds a bh ref. */ -static void block_write_end_io(struct bio *bio, int err) +static int insert_bhp(struct super_block *sb, struct buffer_head *bh) { - struct scoutfs_block *bl = bio->bi_private; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct block_bh_private *bhp; unsigned long flags; + int ret = 0; - if (!err) { - spin_lock_irqsave(&sbi->block_lock, flags); - radix_tree_tag_clear(&sbi->block_radix, - bl->blkno, DIRTY_RADIX_TAG); - spin_unlock_irqrestore(&sbi->block_lock, flags); + if (bh->b_private) + return 0; + + lock_buffer(bh); + if (bh->b_private) + goto out; + + bhp = kmalloc(sizeof(*bhp), GFP_NOFS); + if (!bhp) { + ret = -ENOMEM; + goto out; } - /* not too worried about racing ints */ - if (err && !sbi->block_write_err) - sbi->block_write_err = err; + bhp->sb = sb; + bhp->bh = bh; + get_bh(bh); + bh->b_private = bhp; - if (atomic_dec_and_test(&sbi->block_writes)) - wake_up(&sbi->block_wq); - - scoutfs_put_block(bl); - bio_put(bio); + spin_lock_irqsave(&sbi->block_lock, flags); + insert_bhp_rb(&sbi->block_dirty_tree, bh); + spin_unlock_irqrestore(&sbi->block_lock, flags); + trace_printk("blkno %llu bh %p\n", (u64)bh->b_blocknr, bh); +out: + unlock_buffer(bh); + return ret; } -static int block_submit_bio(struct scoutfs_block *bl, int rw) +static void erase_bhp(struct buffer_head *bh) { - struct super_block *sb = bl->sb; - struct bio *bio; - int ret; + struct block_bh_private *bhp = bh->b_private; + struct scoutfs_sb_info *sbi = sbi_from_bh(bh); + unsigned long flags; - if (WARN_ON_ONCE(bl->blkno >= - i_size_read(sb->s_bdev->bd_inode) >> SCOUTFS_BLOCK_SHIFT)) { - printk("trying to read bad blkno %llu\n", bl->blkno); - } + spin_lock_irqsave(&sbi->block_lock, flags); + rb_erase(&bhp->node, &sbi->block_dirty_tree); + spin_unlock_irqrestore(&sbi->block_lock, flags); + put_bh(bh); + kfree(bhp); + bh->b_private = NULL; - bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); - if (WARN_ON_ONCE(!bio)) - return -ENOMEM; - - bio->bi_sector = bl->blkno << (SCOUTFS_BLOCK_SHIFT - 9); - bio->bi_bdev = sb->s_bdev; - if (rw & WRITE) { - bio->bi_end_io = block_write_end_io; - } else - bio->bi_end_io = block_read_end_io; - bio->bi_private = bl; - - ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); - if (WARN_ON_ONCE(ret != SCOUTFS_BLOCK_SIZE)) { - bio_put(bio); - return -ENOMEM; - } - - atomic_inc(&bl->refcount); - submit_bio(rw, bio); - - return 0; + trace_printk("blkno %llu bh %p\n", (u64)bh->b_blocknr, bh); } /* * Read an existing block from the device and verify its metadata header. + * The buffer head is returned unlocked and uptodate. */ -struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno) +struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *found; - struct scoutfs_block *bl; - unsigned long flags; + struct buffer_head *bh; int ret; - /* find an existing block, dropping if it's errored */ - spin_lock_irqsave(&sbi->block_lock, flags); + bh = sb_bread(sb, blkno); + if (!bh) { + bh = ERR_PTR(-EIO); + goto out; + } - bl = radix_tree_lookup(&sbi->block_radix, blkno); - if (bl) { - if (test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - scoutfs_put_block(bl); - bl = NULL; - } else { - atomic_inc(&bl->refcount); + if (!buffer_scoutfs_verified(bh)) { + lock_buffer(bh); + if (!buffer_scoutfs_verified(bh)) { + ret = verify_block_header(sbi, bh); + if (ret < 0) { + scoutfs_block_put(bh); + bh = ERR_PTR(ret); + } else { + set_buffer_scoutfs_verified(bh); + } } + unlock_buffer(bh); } - spin_unlock_irqrestore(&sbi->block_lock, flags); - if (bl) - goto wait; - - /* allocate a new block and try to insert it */ - bl = alloc_block(sb, blkno); - if (!bl) { - ret = -EIO; - goto out; - } - - ret = radix_tree_preload(GFP_NOFS); - if (ret) - goto out; - - spin_lock_irqsave(&sbi->block_lock, flags); - - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) { - scoutfs_put_block(bl); - bl = found; - atomic_inc(&bl->refcount); - } else { - radix_tree_insert(&sbi->block_radix, blkno, bl); - atomic_inc(&bl->refcount); - } - - spin_unlock_irqrestore(&sbi->block_lock, flags); - radix_tree_preload_end(); - - if (!found) { - ret = block_submit_bio(bl, READ_SYNC | REQ_META); - if (ret) - goto out; - } - -wait: - ret = wait_event_interruptible(sbi->block_wq, - test_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits) || - test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)); - if (test_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits)) - ret = 0; - else if (test_bit(SCOUTFS_BLOCK_BIT_ERROR, &bl->bits)) - ret = -EIO; - out: - if (ret) { - scoutfs_put_block(bl); - bl = ERR_PTR(ret); - } - - return bl; + trace_printk("blkno %llu bh %p (ret %ld)\n", + blkno, bh, IS_ERR(bh) ? PTR_ERR(bh) : 0); + return bh; } /* - * Return the block pointed to by the caller's reference. + * Read an existing block from the device described by the caller's + * reference. * * If the reference sequence numbers don't match then we could be racing * with another writer. We back off and try again. If it happens too @@ -277,130 +223,122 @@ out: * - reads that span transactions? * - writers creating a new dirty block? */ -struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) +struct buffer_head *scoutfs_block_read_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; - struct scoutfs_block *bl; - struct scoutfs_block *found; - unsigned long flags; - - bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); - if (!IS_ERR(bl)) { - hdr = bl->data; + struct buffer_head *bh; + bh = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR(bh)) { + hdr = bh_data(bh); if (WARN_ON_ONCE(hdr->seq != ref->seq)) { - /* XXX hack, make this a function */ - spin_lock_irqsave(&sbi->block_lock, flags); - found = radix_tree_lookup(&sbi->block_radix, - bl->blkno); - if (found == bl) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - scoutfs_put_block(bl); - } - spin_unlock_irqrestore(&sbi->block_lock, flags); - - scoutfs_put_block(bl); - bl = ERR_PTR(-EAGAIN); + clear_buffer_uptodate(bh); + brelse(bh); + bh = ERR_PTR(-EAGAIN); } } - return bl; + return bh; } /* - * XXX This is a gross hack for writing the super. It doesn't have - * per-block write completion indication, it just knows that it's the - * only thing that will be writing. + * We stop tracking dirty metadata blocks when their IO succeeds. This + * happens in the context of transaction commit which excludes other + * metadata dirtying paths. */ -int scoutfs_write_block(struct scoutfs_block *bl) +static void block_write_end_io(struct buffer_head *bh, int uptodate) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - int ret; + struct scoutfs_sb_info *sbi = sbi_from_bh(bh); - BUG_ON(atomic_read(&sbi->block_writes) != 0); + trace_printk("bh %p uptdate %d\n", bh, uptodate); - atomic_inc(&sbi->block_writes); - ret = block_submit_bio(bl, WRITE); - if (ret) - atomic_dec(&sbi->block_writes); - else - wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); + /* XXX */ + unlock_buffer(bh); - return ret ?: sbi->block_write_err; + if (uptodate) { + erase_bhp(bh); + } else { + /* don't care if this is racey? */ + if (!sbi->block_write_err) + sbi->block_write_err = -EIO; + } + + if (atomic_dec_and_test(&sbi->block_writes)) + wake_up(&sbi->block_wq); } /* - * A quick cheap test so that write dirty blocks only has to return - * success or error, not also the lack of dirty blocks. - */ -int scoutfs_has_dirty_blocks(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - return radix_tree_tagged(&sbi->block_radix, DIRTY_RADIX_TAG); -} - -/* - * Write out all the currently dirty blocks. The caller has waited - * for all the dirty blocks to be consistent and has prevented further - * writes while we're working. + * Submit writes for all the buffer heads in the dirty block tree. The + * write transaction machinery ensures that the dirty blocks form a + * consistent image and excludes future dirtying while we're working. * - * The blocks are kept dirty so that they won't be evicted by reclaim - * while they're in flight. Reads can traverse the blocks while they're - * in flight. + * Presence in the dirty tree holds a reference. Blocks are only + * removed from the tree which drops the ref when IO completes. + * + * Blocks that see write errors remain in the dirty tree and will try to + * be written again in the next transaction commit. + * + * Reads can traverse the blocks while they're in flight. + * + * The number of blocks written is returned, or -errno on error. */ -int scoutfs_write_dirty_blocks(struct super_block *sb) +int scoutfs_block_write_dirty(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *blocks[16]; - struct scoutfs_block *bl; + struct buffer_head *bh; + struct rb_node *node; + struct blk_plug plug; unsigned long flags; - unsigned long blkno; - int ret; - int nr; - int i; + int count; + int err; - blkno = 0; + atomic_set(&sbi->block_writes, 1); sbi->block_write_err = 0; - ret = 0; - atomic_inc(&sbi->block_writes); + count = 0; + err = 0; - do { - /* get refs to a bunch of dirty blocks */ - spin_lock_irqsave(&sbi->block_lock, flags); - nr = radix_tree_gang_lookup_tag(&sbi->block_radix, - (void **)blocks, blkno, - ARRAY_SIZE(blocks), - DIRTY_RADIX_TAG); - if (nr > 0) - blkno = blocks[nr - 1]->blkno + 1; - for (i = 0; i < nr; i++) - atomic_inc(&blocks[i]->refcount); + blk_start_plug(&plug); + + spin_lock_irqsave(&sbi->block_lock, flags); + node = rb_first(&sbi->block_dirty_tree); + while(node) { + bh = bh_from_bhp_node(node); + node = rb_next(node); spin_unlock_irqrestore(&sbi->block_lock, flags); - /* submit them in order, being careful to put all on err */ - for (i = 0; i < nr; i++) { - bl = blocks[i]; + atomic_inc(&sbi->block_writes); + count++; + scoutfs_block_set_crc(bh); - if (ret == 0) { - /* XXX crc could be farmed out */ - scoutfs_calc_hdr_crc(bl); - atomic_inc(&sbi->block_writes); - ret = block_submit_bio(bl, WRITE); - if (ret) - atomic_dec(&sbi->block_writes); - } - scoutfs_put_block(bl); - } - } while (nr && !ret); + /* + * XXX submit_bh() forces us to lock the block while IO is + * in flight. This is unfortunate because we use the buffer + * head lock to serialize access to btree block contents. + * We should fix that and only use the buffer head lock + * when the APIs force us to. + */ + lock_buffer(bh); + + bh->b_end_io = block_write_end_io; + err = submit_bh(WRITE, bh); /* doesn't actually fail? */ + + spin_lock_irqsave(&sbi->block_lock, flags); + if (err) + break; + } + spin_unlock_irqrestore(&sbi->block_lock, flags); + + blk_finish_plug(&plug); /* wait for all io to drain */ atomic_dec(&sbi->block_writes); wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); - return ret ?: sbi->block_write_err; + trace_printk("err %d sbi err %d count %d\n", + err, sbi->block_write_err, count); + + return err ?: sbi->block_write_err ?: count; } /* @@ -415,188 +353,108 @@ int scoutfs_write_dirty_blocks(struct super_block *sb) * For now we're using the dirty super block in the sb_info to track the * dirty seq. That'll be different when we have multiple btrees. * - * Callers are working in structures that have sufficient locking to - * protect references to the source block. If we've come to dirty it - * then there won't be concurrent users and we can just move it in the - * cache. - * - * The caller can ask that we either move the existing cached block to - * its new dirty blkno in the cache or copy its contents to a newly - * allocated dirty block. The caller knows if they'll ever reference - * the old clean block again (buddy does, btree doesn't.) + * Callers are responsible for serializing modification to the reference + * which is probably embedded in some other dirty persistent structure. */ -static struct scoutfs_block *dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref, bool cow) +struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; - struct scoutfs_block *copy_bl = NULL; - struct scoutfs_block *found; - struct scoutfs_block *bl; - unsigned long flags; - u64 clean_blkno; + struct buffer_head *copy_bh = NULL; + struct buffer_head *bh; u64 blkno = 0; int ret; int err; - bl = scoutfs_read_block(sb, le64_to_cpu(ref->blkno)); - if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) - return bl; - - clean_blkno = bl->blkno; - - ret = scoutfs_buddy_dirty(sb, clean_blkno, 0); - if (ret < 0) - goto out; + bh = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bh) || ref->seq == sbi->super.hdr.seq) + return bh; ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; - if (cow) { - copy_bl = alloc_block(sb, blkno); - if (IS_ERR(copy_bl)) { - ret = PTR_ERR(copy_bl); - goto out; - } - set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, ©_bl->bits); + copy_bh = scoutfs_block_dirty(sb, blkno); + if (IS_ERR(copy_bh)) { + ret = PTR_ERR(copy_bh); + goto out; } - ret = radix_tree_preload(GFP_NOFS); + ret = scoutfs_buddy_free(sb, bh->b_blocknr, 0); if (ret) goto out; - spin_lock_irqsave(&sbi->block_lock, flags); + memcpy(copy_bh->b_data, bh->b_data, SCOUTFS_BLOCK_SIZE); - /* delete anything at the new blkno */ - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) { - radix_tree_delete(&sbi->block_radix, blkno); - scoutfs_put_block(found); - } - - if (cow) { - /* copy contents to the new block, hdr updated below */ - memcpy(copy_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); - scoutfs_put_block(bl); - bl = copy_bl; - copy_bl = NULL; - } else { - /* move the existing block to its new dirty blkno */ - found = radix_tree_lookup(&sbi->block_radix, bl->blkno); - if (found == bl) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - atomic_dec(&bl->refcount); - } - } - - bl->blkno = blkno; - hdr = bl->data; + hdr = bh_data(copy_bh); hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; ref->blkno = hdr->blkno; ref->seq = hdr->seq; - /* insert the dirty block at its new blkno */ - radix_tree_insert(&sbi->block_radix, blkno, bl); - radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); - atomic_inc(&bl->refcount); - - spin_unlock_irqrestore(&sbi->block_lock, flags); - radix_tree_preload_end(); - - /* free clean blkno after preload end enables preemption */ - err = scoutfs_buddy_free(sb, clean_blkno, 0); - WARN_ON(err); /* XXX corruption (dirtying should prevent) */ ret = 0; out: - scoutfs_put_block(copy_bl); + scoutfs_block_put(bh); if (ret) { - if (blkno) { - err = scoutfs_buddy_free(sb, blkno, 0); - WARN_ON_ONCE(err); /* XXX hmm */ + if (!IS_ERR_OR_NULL(copy_bh)) { + err = scoutfs_buddy_free(sb, copy_bh->b_blocknr, 0); + WARN_ON_ONCE(err); /* freeing dirty must work */ } - scoutfs_put_block(bl); - bl = ERR_PTR(ret); + scoutfs_block_put(copy_bh); + copy_bh = ERR_PTR(ret); } - return bl; -} - -struct scoutfs_block *scoutfs_block_cow_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) -{ - return dirty_ref(sb, ref, true); -} -struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) -{ - return dirty_ref(sb, ref, false); + return copy_bh; } /* - * Return a newly allocated metadata block with an updated block header - * to match the current dirty seq. Callers are responsible for - * serializing access to the block and for zeroing unwritten block - * contents. + * Return a dirty metadata block with an updated block header to match + * the current dirty seq. Callers are responsible for serializing + * access to the block and for zeroing unwritten block contents. */ -struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno) +struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; - struct scoutfs_block *found; - struct scoutfs_block *bl; - unsigned long flags; + struct buffer_head *bh; int ret; /* allocate a new block and try to insert it */ - bl = alloc_block(sb, blkno); - if (!bl) { - ret = -EIO; + bh = sb_getblk(sb, blkno); + if (!bh) { + bh = ERR_PTR(-ENOMEM); goto out; } - set_bit(SCOUTFS_BLOCK_BIT_UPTODATE, &bl->bits); - - ret = radix_tree_preload(GFP_NOFS); - if (ret) + ret = insert_bhp(sb, bh); + if (ret < 0) { + scoutfs_block_put(bh); + bh = ERR_PTR(ret); goto out; + } - hdr = bl->data; + hdr = bh_data(bh); *hdr = sbi->super.hdr; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; - spin_lock_irqsave(&sbi->block_lock, flags); - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) { - radix_tree_delete(&sbi->block_radix, blkno); - scoutfs_put_block(found); - } - - radix_tree_insert(&sbi->block_radix, blkno, bl); - radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); - atomic_inc(&bl->refcount); - spin_unlock_irqrestore(&sbi->block_lock, flags); - - radix_tree_preload_end(); - ret = 0; + set_buffer_uptodate(bh); + set_buffer_scoutfs_verified(bh); out: - if (ret) { - scoutfs_put_block(bl); - bl = ERR_PTR(ret); - } + trace_printk("blkno %llu bh %p (ret %ld)\n", + blkno, bh, IS_ERR(bh) ? PTR_ERR(bh) : 0); - return bl; + return bh; } /* * Allocate a new dirty writable block. The caller must be in a * transaction so that we can assign the dirty seq. */ -struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb) +struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) { - struct scoutfs_block *bl; + struct buffer_head *bh; u64 blkno; int ret; int err; @@ -605,26 +463,26 @@ struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb) if (ret < 0) return ERR_PTR(ret); - bl = scoutfs_new_block(sb, blkno); - if (IS_ERR(bl)) { + bh = scoutfs_block_dirty(sb, blkno); + if (IS_ERR(bh)) { err = scoutfs_buddy_free(sb, blkno, 0); - WARN_ON_ONCE(err); /* XXX hmm */ + WARN_ON_ONCE(err); /* freeing dirty must work */ } - return bl; + return bh; } -void scoutfs_calc_hdr_crc(struct scoutfs_block *bl) +void scoutfs_block_set_crc(struct buffer_head *bh) { - struct scoutfs_block_header *hdr = bl->data; + struct scoutfs_block_header *hdr = bh_data(bh); hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } -void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off) +void scoutfs_block_zero(struct buffer_head *bh, size_t off) { if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) return; if (off < SCOUTFS_BLOCK_SIZE) - memset(bl->data + off, 0, SCOUTFS_BLOCK_SIZE - off); + memset((char *)bh->b_data + off, 0, SCOUTFS_BLOCK_SIZE - off); } diff --git a/kmod/src/block.h b/kmod/src/block.h index b464f0c3..8bb2a136 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -2,43 +2,35 @@ #define _SCOUTFS_BLOCK_H_ #include -#include -#include +#include -#define SCOUTFS_BLOCK_BIT_UPTODATE (1 << 0) -#define SCOUTFS_BLOCK_BIT_ERROR (1 << 1) +struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno); +struct buffer_head *scoutfs_block_read_ref(struct super_block *sb, + struct scoutfs_block_ref *ref); -struct scoutfs_block { - struct rw_semaphore rwsem; - atomic_t refcount; - u64 blkno; - - unsigned long bits; - - struct super_block *sb; - /* only high order page alloc for now */ - struct page *page; - void *data; -}; - -struct scoutfs_block *scoutfs_read_block(struct super_block *sb, u64 blkno); -struct scoutfs_block *scoutfs_new_block(struct super_block *sb, u64 blkno); -struct scoutfs_block *scoutfs_alloc_block(struct super_block *sb); - -struct scoutfs_block *scoutfs_read_ref(struct super_block *sb, - struct scoutfs_block_ref *ref); -struct scoutfs_block *scoutfs_block_cow_ref(struct super_block *sb, +struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno); +struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb); +struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, struct scoutfs_block_ref *ref); -struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref); -int scoutfs_has_dirty_blocks(struct super_block *sb); -int scoutfs_write_block(struct scoutfs_block *bl); -int scoutfs_write_dirty_blocks(struct super_block *sb); +int scoutfs_block_write_dirty(struct super_block *sb); -void scoutfs_put_block(struct scoutfs_block *bl); +void scoutfs_block_set_crc(struct buffer_head *bh); +void scoutfs_block_zero(struct buffer_head *bh, size_t off); -void scoutfs_calc_hdr_crc(struct scoutfs_block *bl); -void scoutfs_zero_block_tail(struct scoutfs_block *bl, size_t off); +/* XXX seems like this should be upstream :) */ +static inline void *bh_data(struct buffer_head *bh) +{ + return (void *)bh->b_data; +} + +static inline void scoutfs_block_put(struct buffer_head *bh) +{ + if (!IS_ERR_OR_NULL(bh)) { + trace_printk("putting bh %p count %d\n", + bh, atomic_read(&bh->b_count)); + brelse(bh); + } +} #endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c index ceecbee1..df0b75e3 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -294,14 +294,14 @@ static void compact_items(struct scoutfs_btree_block *bt) * Allocate and initialize a new tree block. The caller adds references * to it. */ -static struct scoutfs_block *alloc_tree_block(struct super_block *sb) +static struct buffer_head *alloc_tree_block(struct super_block *sb) { struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; + struct buffer_head *bh; - bl = scoutfs_alloc_block(sb); - if (!IS_ERR(bl)) { - bt = bl->data; + bh = scoutfs_block_dirty_alloc(sb); + if (!IS_ERR(bh)) { + bt = bh_data(bh); bt->treap.off = 0; bt->total_free = cpu_to_le16(SCOUTFS_BLOCK_SIZE - @@ -310,7 +310,7 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) bt->nr_items = 0; } - return bl; + return bh; } /* the caller has ensured that the free must succeed */ @@ -324,22 +324,22 @@ static void free_tree_block(struct super_block *sb, __le64 blkno) * Allocate a new tree block and point the root at it. The caller * is responsible for the items in the new root block. */ -static struct scoutfs_block *grow_tree(struct super_block *sb, +static struct buffer_head *grow_tree(struct super_block *sb, struct scoutfs_btree_root *root) { struct scoutfs_block_header *hdr; - struct scoutfs_block *bl; + struct buffer_head *bh; - bl = alloc_tree_block(sb); - if (!IS_ERR(bl)) { - hdr = bl->data; + bh = alloc_tree_block(sb); + if (!IS_ERR(bh)) { + hdr = bh_data(bh); root->height++; root->ref.blkno = hdr->blkno; root->ref.seq = hdr->seq; } - return bl; + return bh; } /* @@ -380,18 +380,18 @@ static void create_parent_item(struct scoutfs_btree_block *parent, * the child that we return. It's skipping locking the new parent as it * descends but that's fine. */ -static struct scoutfs_block *try_split(struct super_block *sb, +static struct buffer_head *try_split(struct super_block *sb, struct scoutfs_btree_root *root, int level, struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_block *parent, struct scoutfs_btree_item *par_item, - struct scoutfs_block *right_bl) + struct buffer_head *right_bh) { - struct scoutfs_btree_block *right = right_bl->data; + struct scoutfs_btree_block *right = bh_data(right_bh); struct scoutfs_btree_block *left; - struct scoutfs_block *left_bl; - struct scoutfs_block *par_bl = NULL; + struct buffer_head *left_bh; + struct buffer_head *par_bh = NULL; unsigned int bytes; if (level) @@ -399,35 +399,35 @@ static struct scoutfs_block *try_split(struct super_block *sb, bytes = val_bytes(val_len); if (le16_to_cpu(right->tail_free) >= bytes) - return right_bl; + return right_bh; if (le16_to_cpu(right->total_free) >= bytes) { compact_items(right); - return right_bl; + return right_bh; } /* alloc split neighbour first to avoid unwinding tree growth */ - left_bl = alloc_tree_block(sb); - if (IS_ERR(left_bl)) { - scoutfs_put_block(right_bl); - return left_bl; + left_bh = alloc_tree_block(sb); + if (IS_ERR(left_bh)) { + scoutfs_block_put(right_bh); + return left_bh; } - left = left_bl->data; + left = bh_data(left_bh); if (!parent) { - par_bl = grow_tree(sb, root); - if (IS_ERR(par_bl)) { + par_bh = grow_tree(sb, root); + if (IS_ERR(par_bh)) { free_tree_block(sb, left->hdr.blkno); - scoutfs_put_block(left_bl); - scoutfs_put_block(right_bl); - return par_bl; + scoutfs_block_put(left_bh); + scoutfs_block_put(right_bh); + return par_bh; } - parent = par_bl->data; + parent = bh_data(par_bh); } /* only grow the tree once we have the split neighbour */ - if (par_bl) { + if (par_bh) { struct scoutfs_key maximal; scoutfs_set_max_key(&maximal); create_parent_item(parent, right, &maximal); @@ -438,19 +438,19 @@ static struct scoutfs_block *try_split(struct super_block *sb, if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { /* insertion will go to the new left block */ - scoutfs_put_block(right_bl); - right_bl = left_bl; + scoutfs_block_put(right_bh); + right_bh = left_bh; } else { /* insertion will still go through us, might need to compact */ - scoutfs_put_block(left_bl); + scoutfs_block_put(left_bh); if (le16_to_cpu(right->tail_free) < bytes) compact_items(right); } - scoutfs_put_block(par_bl); + scoutfs_block_put(par_bh); - return right_bl; + return right_bh; } /* @@ -474,21 +474,21 @@ static struct scoutfs_block *try_split(struct super_block *sb, * * XXX this could more cleverly chose a merge candidate sibling */ -static struct scoutfs_block *try_merge(struct super_block *sb, +static struct buffer_head *try_merge(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, struct scoutfs_btree_item *par_item, - struct scoutfs_block *bl) + struct buffer_head *bh) { - struct scoutfs_btree_block *bt = bl->data; + struct scoutfs_btree_block *bt = bh_data(bh); struct scoutfs_btree_block *sib_bt; - struct scoutfs_block *sib_bl; + struct buffer_head *sib_bh; struct scoutfs_btree_item *sib_item; int to_move; bool move_right; if (le16_to_cpu(bt->total_free) <= SCOUTFS_BTREE_FREE_LIMIT) - return bl; + return bh; /* move items right into our block if we have a left sibling */ sib_item = bt_prev(parent, par_item); @@ -499,13 +499,13 @@ static struct scoutfs_block *try_merge(struct super_block *sb, move_right = true; } - sib_bl = scoutfs_block_dirty_ref(sb, (void *)sib_item->val); - if (IS_ERR(sib_bl)) { + sib_bh = scoutfs_block_dirty_ref(sb, (void *)sib_item->val); + if (IS_ERR(sib_bh)) { /* XXX do we need to unlock this? don't think so */ - scoutfs_put_block(bl); - return sib_bl; + scoutfs_block_put(bh); + return sib_bh; } - sib_bt = sib_bl->data; + sib_bt = bh_data(sib_bh); if (used_total(sib_bt) <= le16_to_cpu(bt->total_free)) to_move = used_total(sib_bt); @@ -538,7 +538,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, free_tree_block(sb, parent->hdr.blkno); } - return bl; + return bh; } enum { @@ -549,40 +549,42 @@ enum { WALK_DIRTY, }; +static inline void lock_root(struct scoutfs_sb_info *sbi, bool dirty) +{ + if (dirty) + down_write(&sbi->btree_rwsem); + else + down_read(&sbi->btree_rwsem); +} + +static inline void unlock_root(struct scoutfs_sb_info *sbi, bool dirty) +{ + if (dirty) + up_write(&sbi->btree_rwsem); + else + up_read(&sbi->btree_rwsem); +} + /* * As we descend we lock parent blocks (or the root), then lock the child, * then unlock the parent. */ -static void lock_block(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, - bool dirty) +static inline void lock_block(struct scoutfs_sb_info *sbi, + struct buffer_head *bh, bool dirty) { - struct rw_semaphore *rwsem; - - if (bl == NULL) - rwsem = &sbi->btree_rwsem; + if (bh == NULL) + lock_root(sbi, dirty); else - rwsem = &bl->rwsem; - - if (dirty) - down_write(rwsem); - else - down_read(rwsem); + lock_buffer(bh); } -static void unlock_block(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, - bool dirty) +static inline void unlock_block(struct scoutfs_sb_info *sbi, + struct buffer_head *bh, bool dirty) { - struct rw_semaphore *rwsem; - - if (bl == NULL) - rwsem = &sbi->btree_rwsem; + if (bh == NULL) + unlock_root(sbi, dirty); else - rwsem = &bl->rwsem; - - if (dirty) - up_write(rwsem); - else - up_read(rwsem); + unlock_buffer(bh); } static u64 item_block_ref_seq(struct scoutfs_btree_item *item) @@ -647,7 +649,7 @@ item_after_seq(struct scoutfs_btree_block *bt, struct scoutfs_key *key, * in the next sibling's block. This is used by iteration to advance to * the next block when they're done with the block this returns. */ -static struct scoutfs_block *btree_walk(struct super_block *sb, +static struct buffer_head *btree_walk(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next_key, unsigned int val_len, u64 seq, int op) @@ -655,8 +657,8 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_block *parent = NULL; struct scoutfs_btree_root *root; - struct scoutfs_block *par_bl = NULL; - struct scoutfs_block *bl = NULL; + struct buffer_head *par_bh = NULL; + struct buffer_head *bh = NULL; struct scoutfs_btree_item *item = NULL; struct scoutfs_block_ref *ref; unsigned int level; @@ -667,7 +669,7 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, if (next_key) scoutfs_set_max_key(next_key); - lock_block(sbi, par_bl, dirty); + lock_block(sbi, par_bh, dirty); /* XXX one for now */ root = &sbi->super.btree_root; @@ -676,31 +678,31 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, if (!root->height) { if (op == WALK_INSERT) { - bl = ERR_PTR(-ENOENT); + bh = ERR_PTR(-ENOENT); } else { - bl = grow_tree(sb, root); - if (!IS_ERR(bl)) - lock_block(sbi, bl, dirty); + bh = grow_tree(sb, root); + if (!IS_ERR(bh)) + lock_block(sbi, bh, dirty); } - unlock_block(sbi, par_bl, dirty); - return bl; + unlock_block(sbi, par_bh, dirty); + return bh; } /* skip the whole tree if the root ref's seq is old */ if (op == WALK_NEXT_SEQ && le64_to_cpu(ref->seq) < seq) { - unlock_block(sbi, par_bl, dirty); + unlock_block(sbi, par_bh, dirty); return ERR_PTR(-ENOENT); } while (level--) { /* XXX hmm, need to think about retry */ if (dirty) { - bl = scoutfs_block_dirty_ref(sb, ref); + bh = scoutfs_block_dirty_ref(sb, ref); } else { - bl = scoutfs_read_ref(sb, ref); + bh = scoutfs_block_read_ref(sb, ref); } - if (IS_ERR(bl)) + if (IS_ERR(bh)) break; /* @@ -714,23 +716,23 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, } if (op == WALK_INSERT) - bl = try_split(sb, root, level, key, val_len, parent, - item, bl); + bh = try_split(sb, root, level, key, val_len, parent, + item, bh); if ((op == WALK_DELETE) && parent) - bl = try_merge(sb, root, parent, item, bl); - if (IS_ERR(bl)) + bh = try_merge(sb, root, parent, item, bh); + if (IS_ERR(bh)) break; - lock_block(sbi, bl, dirty); + lock_block(sbi, bh, dirty); if (!level) break; /* unlock parent before searching so others can use it */ - unlock_block(sbi, par_bl, dirty); - scoutfs_put_block(par_bl); - par_bl = bl; - parent = par_bl->data; + unlock_block(sbi, par_bh, dirty); + scoutfs_block_put(par_bh); + par_bh = bh; + parent = bh_data(par_bh); /* * Find the parent item that references the next child @@ -742,9 +744,9 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, if (!item) { /* current block dropped as parent below */ if (op == WALK_NEXT_SEQ) { - bl = ERR_PTR(-ENOENT); + bh = ERR_PTR(-ENOENT); } else { - bl = ERR_PTR(-EIO); + bh = ERR_PTR(-EIO); } break; } @@ -752,17 +754,17 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, ref = (void *)item->val; } - unlock_block(sbi, par_bl, dirty); - scoutfs_put_block(par_bl); + unlock_block(sbi, par_bh, dirty); + scoutfs_block_put(par_bh); - return bl; + return bh; } static void set_cursor(struct scoutfs_btree_cursor *curs, - struct scoutfs_block *bl, + struct buffer_head *bh, struct scoutfs_btree_item *item, bool write) { - curs->bl = bl; + curs->bh = bh; curs->item = item; curs->key = &item->key; curs->seq = le64_to_cpu(item->seq); @@ -779,22 +781,24 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_item *item; - struct scoutfs_block *bl; + struct scoutfs_btree_block *bt; + struct buffer_head *bh; int ret; - BUG_ON(curs->bl); + BUG_ON(curs->bh); - bl = btree_walk(sb, key, NULL, 0, 0, 0); - if (IS_ERR(bl)) - return PTR_ERR(bl); + bh = btree_walk(sb, key, NULL, 0, 0, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); - item = bt_lookup(bl->data, key); + bt = bh_data(bh); + item = bt_lookup(bt, key); if (item) { - set_cursor(curs, bl, item, false); + set_cursor(curs, bh, item, false); ret = 0; } else { - up_read(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, false); + scoutfs_block_put(bh); ret = -ENOENT; } @@ -815,25 +819,25 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; + struct buffer_head *bh; int ret; - BUG_ON(curs->bl); + BUG_ON(curs->bh); - bl = btree_walk(sb, key, NULL, val_len, 0, WALK_INSERT); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = bl->data; + bh = btree_walk(sb, key, NULL, val_len, 0, WALK_INSERT); + if (IS_ERR(bh)) + return PTR_ERR(bh); + bt = bh_data(bh); /* XXX should this return -eexist? */ item = bt_lookup(bt, key); if (!item) { item = create_item(bt, key, val_len); - set_cursor(curs, bl, item, true); + set_cursor(curs, bh, item, true); ret = 0; } else { - up_write(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); ret = -ENOENT; } @@ -850,13 +854,13 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) struct scoutfs_btree_root *root; struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; + struct buffer_head *bh; int ret; - bl = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = bl->data; + bh = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); + if (IS_ERR(bh)) + return PTR_ERR(bh); + bt = bh_data(bh); item = bt_lookup(bt, key); if (item) { @@ -879,8 +883,8 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) ret = -ENOENT; } - up_write(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); return ret; } @@ -905,7 +909,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; + struct buffer_head *bh; struct scoutfs_key key = *first; struct scoutfs_key next_key; int ret; @@ -914,50 +918,50 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, return 0; /* find the next item after the cursor, releasing if we're done */ - if (curs->bl) { + if (curs->bh) { + bt = bh_data(curs->bh); key = curs->item->key; scoutfs_inc_key(&key); - curs->item = item_next_seq(curs->bl->data, curs->item, - 0, seq, op); + curs->item = item_next_seq(bt, curs->item, 0, seq, op); if (curs->item) - set_cursor(curs, curs->bl, curs->item, curs->write); + set_cursor(curs, curs->bh, curs->item, curs->write); else scoutfs_btree_release(curs); } /* find the leaf that contains the next item after the key */ - while (!curs->bl && scoutfs_key_cmp(&key, last) <= 0) { + while (!curs->bh && scoutfs_key_cmp(&key, last) <= 0) { - bl = btree_walk(sb, &key, &next_key, 0, seq, op); + bh = btree_walk(sb, &key, &next_key, 0, seq, op); /* next seq walks can terminate in parents with old seqs */ - if (op == WALK_NEXT_SEQ && bl == ERR_PTR(-ENOENT)) { + if (op == WALK_NEXT_SEQ && bh == ERR_PTR(-ENOENT)) { key = next_key; continue; } - if (IS_ERR(bl)) { - if (bl == ERR_PTR(-ENOENT)) + if (IS_ERR(bh)) { + if (bh == ERR_PTR(-ENOENT)) break; - return PTR_ERR(bl); + return PTR_ERR(bh); } - bt = bl->data; + bt = bh_data(bh); /* keep trying leaves until next_key passes last */ - curs->item = item_after_seq(bl->data, &key, 0, seq, op); + curs->item = item_after_seq(bt, &key, 0, seq, op); if (!curs->item) { key = next_key; - up_read(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, false); + scoutfs_block_put(bh); continue; } if (curs->item) { - set_cursor(curs, bl, curs->item, false); + set_cursor(curs, bh, curs->item, false); } else { - up_read(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, false); + scoutfs_block_put(bh); } break; } @@ -997,22 +1001,24 @@ int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_btree_item *item; - struct scoutfs_block *bl; + struct scoutfs_btree_block *bt; + struct buffer_head *bh; int ret; - bl = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); - if (IS_ERR(bl)) - return PTR_ERR(bl); + bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); + if (IS_ERR(bh)) + return PTR_ERR(bh); - item = bt_lookup(bl->data, key); + bt = bh_data(bh); + item = bt_lookup(bt, key); if (item) { ret = 0; } else { ret = -ENOENT; } - up_write(&bl->rwsem); - scoutfs_put_block(bl); + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); return ret; } @@ -1026,31 +1032,28 @@ void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; + struct buffer_head *bh; - BUG_ON(curs->bl); + BUG_ON(curs->bh); - bl = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); - BUG_ON(IS_ERR(bl)); + bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); + BUG_ON(IS_ERR(bh)); + bt = bh_data(bh); - item = bt_lookup(bl->data, key); + item = bt_lookup(bt, key); BUG_ON(!item); - bt = bl->data; item->seq = bt->hdr.seq; - set_cursor(curs, bl, item, true); + set_cursor(curs, bh, item, true); } void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) { - if (curs->bl) { - if (curs->write) - up_write(&curs->bl->rwsem); - else - up_read(&curs->bl->rwsem); - scoutfs_put_block(curs->bl); + if (curs->bh) { + unlock_block(NULL, curs->bh, curs->write); + scoutfs_block_put(curs->bh); } - curs->bl = NULL; + curs->bh = NULL; } /* diff --git a/kmod/src/btree.h b/kmod/src/btree.h index fb9b1716..2dc05bb5 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -3,7 +3,7 @@ struct scoutfs_btree_cursor { /* for btree.c */ - struct scoutfs_block *bl; + struct buffer_head *bh; struct scoutfs_btree_item *item; /* for callers */ diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index c6e5974b..52ff738f 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -212,8 +212,8 @@ static int bitmap_alloc(struct super_block *sb, u64 *blkno) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_bitmap_block *st_bm; struct scoutfs_bitmap_block *bm; - struct scoutfs_block *st_bl; - struct scoutfs_block *bm_bl; + struct buffer_head *st_bh; + struct buffer_head *bm_bh; int size; int ret; int d; @@ -226,18 +226,18 @@ static int bitmap_alloc(struct super_block *sb, u64 *blkno) return -EIO; /* dirty the bitmap block */ - bm_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bm_bl)) - return PTR_ERR(bm_bl); - bm = bm_bl->data; + bm_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); + if (IS_ERR(bm_bh)) + return PTR_ERR(bm_bh); + bm = bh_data(bm_bh); /* read the stable bitmap block */ - st_bl = scoutfs_read_ref(sb, &sbi->stable_super.buddy_bm_ref); - if (IS_ERR(st_bl)) { - ret = PTR_ERR(st_bl); + st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_bm_ref); + if (IS_ERR(st_bh)) { + ret = PTR_ERR(st_bh); goto out; } - st_bm = st_bl->data; + st_bm = bh_data(st_bh); /* find the first bit that is set in both dirty and stable bitmaps */ size = le32_to_cpu(sbi->super.buddy_blocks); @@ -255,8 +255,8 @@ static int bitmap_alloc(struct super_block *sb, u64 *blkno) clear_bit_le(d, &bm->bits); ret = 0; out: - scoutfs_put_block(st_bl); - scoutfs_put_block(bm_bl); + scoutfs_block_put(st_bh); + scoutfs_block_put(bm_bh); return ret; } @@ -265,7 +265,7 @@ static int bitmap_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_bitmap_block *bm; - struct scoutfs_block *bl; + struct buffer_head *bh; int nr; /* mkfs should have ensured that there's bitmap blocks */ @@ -273,14 +273,14 @@ static int bitmap_free(struct super_block *sb, u64 blkno) if (sbi->super.buddy_bm_ref.blkno == 0) return -EIO; - bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bm = bl->data; + bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); + if (IS_ERR(bh)) + return PTR_ERR(bh); + bm = bh_data(bh); nr = blkno - (SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR); set_bit_le(nr, bm->bits); - scoutfs_put_block(bl); + scoutfs_block_put(bh); return 0; } @@ -289,13 +289,13 @@ static int bitmap_free(struct super_block *sb, u64 blkno) * Give the caller a dirty buddy block. If the slot hasn't been used * yet then we need to allocate and initialize a new block. */ -static struct scoutfs_block *dirty_buddy_block(struct super_block *sb, int sl, +static struct buffer_head *dirty_buddy_block(struct super_block *sb, int sl, struct scoutfs_buddy_slot *slot) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_buddy_block *bud; - struct scoutfs_block *bl; + struct buffer_head *bh; u64 blkno; int count; int order; @@ -305,19 +305,19 @@ static struct scoutfs_block *dirty_buddy_block(struct super_block *sb, int sl, /* the fast path is to dirty an existing block */ if (slot->ref.blkno) - return scoutfs_block_cow_ref(sb, &slot->ref); + return scoutfs_block_dirty_ref(sb, &slot->ref); ret = bitmap_alloc(sb, &blkno); if (ret) return ERR_PTR(ret); - bl = scoutfs_new_block(sb, blkno); - if (IS_ERR(bl)) { + bh = scoutfs_block_dirty(sb, blkno); + if (IS_ERR(bh)) { bitmap_free(sb, blkno); - return bl; + return bh; } - bud = bl->data; - scoutfs_zero_block_tail(bl, sizeof(bud->hdr)); + bud = bh_data(bh); + scoutfs_block_zero(bh, sizeof(bud->hdr)); /* mark the initial run of highest orders free */ count = slot_count(super, sl); @@ -345,7 +345,7 @@ static struct scoutfs_block *dirty_buddy_block(struct super_block *sb, int sl, update_free_orders(slot, bud); - return bl; + return bh; } /* @@ -414,29 +414,29 @@ static int alloc_slot(struct super_block *sb, int sl, struct scoutfs_super_block *super = &sbi->super; struct scoutfs_buddy_block *bud; struct scoutfs_buddy_block *st_bud; - struct scoutfs_block *st_bl; - struct scoutfs_block *bl; + struct buffer_head *st_bh; + struct buffer_head *bh; int found; int ret; int nr; int i; /* initialize or dirty the slot's buddy block */ - bl = dirty_buddy_block(sb, sl, slot); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bud = bl->data; + bh = dirty_buddy_block(sb, sl, slot); + if (IS_ERR(bh)) + return PTR_ERR(bh); + bud = bh_data(bh); /* read stable slots's buddy block if there is one */ if (stable_ref->blkno) { - st_bl = scoutfs_read_ref(sb, stable_ref); - if (IS_ERR(st_bl)) { - ret = PTR_ERR(st_bl); + st_bh = scoutfs_block_read_ref(sb, stable_ref); + if (IS_ERR(st_bh)) { + ret = PTR_ERR(st_bh); goto out; } - st_bud = st_bl->data; + st_bud = bh_data(st_bh); } else { - st_bl = NULL; + st_bh = NULL; st_bud = NULL; } @@ -459,8 +459,8 @@ static int alloc_slot(struct super_block *sb, int sl, update_free_orders(slot, bud); ret = 0; out: - scoutfs_put_block(st_bl); - scoutfs_put_block(bl); + scoutfs_block_put(st_bh); + scoutfs_block_put(bh); return ret; } @@ -482,8 +482,8 @@ static int alloc_order(struct super_block *sb, u64 *blkno, int order) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_buddy_indirect *st_ind; struct scoutfs_buddy_indirect *ind; - struct scoutfs_block *st_bl = NULL; - struct scoutfs_block *bl = NULL; + struct buffer_head *st_bh = NULL; + struct buffer_head *bh = NULL; u8 mask; int ret; int i; @@ -496,20 +496,20 @@ static int alloc_order(struct super_block *sb, u64 *blkno, int order) } /* get the dirty indirect block */ - bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); + bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); goto out; } - ind = bl->data; + ind = bh_data(bh); /* get the stable indirect block */ - st_bl = scoutfs_read_ref(sb, &sbi->stable_super.buddy_ind_ref); - if (IS_ERR(st_bl)) { - ret = PTR_ERR(st_bl); + st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_ind_ref); + if (IS_ERR(st_bh)) { + ret = PTR_ERR(st_bh); goto out; } - st_ind = st_bl->data; + st_ind = bh_data(st_bh); mask = ~0U << order; @@ -531,8 +531,8 @@ static int alloc_order(struct super_block *sb, u64 *blkno, int order) } out: - scoutfs_put_block(st_bl); - scoutfs_put_block(bl); + scoutfs_block_put(st_bh); + scoutfs_block_put(bh); return ret; } @@ -597,7 +597,7 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) static int bitmap_dirty(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *bl; + struct buffer_head *bh; /* mkfs should have ensured that there's bitmap blocks */ /* XXX corruption */ @@ -605,11 +605,11 @@ static int bitmap_dirty(struct super_block *sb, u64 blkno) return -EIO; /* dirty the bitmap block */ - bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bl)) - return PTR_ERR(bl); + bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); + if (IS_ERR(bh)) + return PTR_ERR(bh); - scoutfs_put_block(bl); + scoutfs_block_put(bh); return 0; } @@ -618,8 +618,8 @@ static int buddy_dirty(struct super_block *sb, u64 blkno, int order) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_buddy_indirect *ind; - struct scoutfs_block *ind_bl = NULL; - struct scoutfs_block *bl = NULL; + struct buffer_head *ind_bh = NULL; + struct buffer_head *bh = NULL; int ret; int sl; @@ -632,23 +632,23 @@ static int buddy_dirty(struct super_block *sb, u64 blkno, int order) } /* get the dirty indirect block */ - ind_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(ind_bl)) { - ret = PTR_ERR(ind_bl); + ind_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(ind_bh)) { + ret = PTR_ERR(ind_bh); goto out; } - ind = ind_bl->data; + ind = bh_data(ind_bh); sl = indirect_slot(super, blkno); - bl = dirty_buddy_block(sb, sl, &ind->slots[sl]); - if (IS_ERR(bl)) - ret = PTR_ERR(bl); + bh = dirty_buddy_block(sb, sl, &ind->slots[sl]); + if (IS_ERR(bh)) + ret = PTR_ERR(bh); else ret = 0; out: mutex_unlock(&sbi->buddy_mutex); - scoutfs_put_block(ind_bl); - scoutfs_put_block(bl); + scoutfs_block_put(ind_bh); + scoutfs_block_put(bh); return ret; } @@ -702,8 +702,8 @@ static int buddy_free(struct super_block *sb, u64 blkno, int order) struct scoutfs_super_block *super = &sbi->super; struct scoutfs_buddy_indirect *ind; struct scoutfs_buddy_block *bud; - struct scoutfs_block *ind_bl = NULL; - struct scoutfs_block *bl = NULL; + struct buffer_head *ind_bh = NULL; + struct buffer_head *bh = NULL; int ret; int sl; int nr; @@ -721,20 +721,20 @@ static int buddy_free(struct super_block *sb, u64 blkno, int order) goto out; } - ind_bl = scoutfs_block_cow_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(ind_bl)) { - ret = PTR_ERR(ind_bl); + ind_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); + if (IS_ERR(ind_bh)) { + ret = PTR_ERR(ind_bh); goto out; } - ind = ind_bl->data; + ind = bh_data(ind_bh); sl = indirect_slot(super, blkno); - bl = scoutfs_block_cow_ref(sb, &ind->slots[sl].ref); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); + bh = scoutfs_block_dirty_ref(sb, &ind->slots[sl].ref); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); goto out; } - bud = bl->data; + bud = bh_data(bh); /* * Merge our region with its free buddy and then try to merge @@ -754,11 +754,11 @@ static int buddy_free(struct super_block *sb, u64 blkno, int order) set_buddy_bit(bud, i, nr); update_free_orders(&ind->slots[sl], bud); - scoutfs_put_block(bl); + scoutfs_block_put(bh); ret = 0; out: mutex_unlock(&sbi->buddy_mutex); - scoutfs_put_block(ind_bl); + scoutfs_block_put(ind_bh); return ret; } diff --git a/kmod/src/super.c b/kmod/src/super.c index 8d7137b4..baa384d0 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -55,6 +55,8 @@ void scoutfs_advance_dirty_super(struct super_block *sb) super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO); le64_add_cpu(&super->hdr.seq, 1); + + trace_printk("super seq now %llu\n", le64_to_cpu(super->hdr.seq)); } /* @@ -64,22 +66,24 @@ void scoutfs_advance_dirty_super(struct super_block *sb) int scoutfs_write_dirty_super(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - size_t sz = sizeof(struct scoutfs_super_block); - u64 blkno = le64_to_cpu(sbi->super.hdr.blkno); - struct scoutfs_block *bl; + struct scoutfs_super_block *super; + struct buffer_head *bh; int ret; /* XXX prealloc? */ - bl = scoutfs_new_block(sb, blkno); - if (WARN_ON_ONCE(IS_ERR(bl))) - return PTR_ERR(bl); + bh = sb_getblk(sb, le64_to_cpu(sbi->super.hdr.blkno)); + if (!bh) + return -ENOMEM; + super = bh_data(bh); - memcpy(bl->data, &sbi->super, sz); - memset(bl->data + sz, 0, SCOUTFS_BLOCK_SIZE - sz); - scoutfs_calc_hdr_crc(bl); - ret = scoutfs_write_block(bl); + *super = sbi->super; + scoutfs_block_zero(bh, sizeof(struct scoutfs_super_block)); + scoutfs_block_set_crc(bh); - scoutfs_put_block(bl); + mark_buffer_dirty(bh); + ret = sync_dirty_buffer(bh); + + scoutfs_block_put(bh); return ret; } @@ -87,19 +91,18 @@ static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; - struct scoutfs_block *bl = NULL; + struct buffer_head *bh = NULL; int found = -1; int i; for (i = 0; i < SCOUTFS_SUPER_NR; i++) { - scoutfs_put_block(bl); - bl = scoutfs_read_block(sb, SCOUTFS_SUPER_BLKNO + i); - if (IS_ERR(bl)) { + scoutfs_block_put(bh); + bh = scoutfs_block_read(sb, SCOUTFS_SUPER_BLKNO + i); + if (IS_ERR(bh)) { scoutfs_warn(sb, "couldn't read super block %u", i); continue; } - - super = bl->data; + super = bh_data(bh); if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { scoutfs_warn(sb, "super block %u has invalid id %llx", @@ -114,7 +117,7 @@ static int read_supers(struct super_block *sb) } } - scoutfs_put_block(bl); + scoutfs_block_put(bh); if (found < 0) { scoutfs_err(sb, "unable to read valid super block"); @@ -147,7 +150,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->next_ino_lock); spin_lock_init(&sbi->block_lock); - INIT_RADIX_TREE(&sbi->block_radix, GFP_NOFS); + sbi->block_dirty_tree = RB_ROOT; init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); mutex_init(&sbi->buddy_mutex); @@ -158,6 +161,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); + if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { + printk(KERN_ERR "couldn't set blocksize\n"); + return -EINVAL; + } + /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) @@ -198,6 +206,9 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); + + /* XXX write errors can leave dirty blocks */ + WARN_ON_ONCE(!RB_EMPTY_ROOT(&sbi->block_dirty_tree)); kfree(sbi); } } diff --git a/kmod/src/super.h b/kmod/src/super.h index f0a2e8cf..7c3617b4 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -19,7 +19,7 @@ struct scoutfs_sb_info { spinlock_t next_ino_lock; spinlock_t block_lock; - struct radix_tree_root block_radix; + struct rb_root block_dirty_tree; wait_queue_head_t block_wq; atomic_t block_writes; int block_write_err; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e1376546..3291b83c 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -63,14 +63,13 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX probably want to write out dirty pages in inodes */ - if (scoutfs_has_dirty_blocks(sb)) { - ret = scoutfs_write_dirty_blocks(sb) ?: - scoutfs_write_dirty_super(sb); + ret = scoutfs_block_write_dirty(sb); + if (ret > 0) { + ret = scoutfs_write_dirty_super(sb); if (!ret) advance = 1; } - spin_lock(&sbi->trans_write_lock); if (advance) scoutfs_advance_dirty_super(sb); From 8bc2b15e3dacfa4aed57a1855ade42b86aa65110 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 Aug 2016 11:31:13 -0700 Subject: [PATCH 072/920] scoutfs: remove scoutfs_buddy_dirty The buffer head rewrite got rid of the only caller who needed to ensure that a free couldn't fail. Let's get rid of this. We can always bring it back if it's needed again. Signed-off-by: Zach Brown --- kmod/src/buddy.c | 83 ------------------------------------------------ kmod/src/buddy.h | 1 - 2 files changed, 84 deletions(-) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 52ff738f..f2edb18b 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -594,89 +594,6 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) return alloc_region(sb, blkno, order, 0, REGION_BUDDY); } -static int bitmap_dirty(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buffer_head *bh; - - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0) - return -EIO; - - /* dirty the bitmap block */ - bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bh)) - return PTR_ERR(bh); - - scoutfs_block_put(bh); - return 0; -} - -static int buddy_dirty(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_indirect *ind; - struct buffer_head *ind_bh = NULL; - struct buffer_head *bh = NULL; - int ret; - int sl; - - mutex_lock(&sbi->buddy_mutex); - - /* mkfs should have ensured that there's indirect blocks */ - if (sbi->super.buddy_ind_ref.blkno == 0) { - ret = -EIO; - goto out; - } - - /* get the dirty indirect block */ - ind_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(ind_bh)) { - ret = PTR_ERR(ind_bh); - goto out; - } - ind = bh_data(ind_bh); - - sl = indirect_slot(super, blkno); - bh = dirty_buddy_block(sb, sl, &ind->slots[sl]); - if (IS_ERR(bh)) - ret = PTR_ERR(bh); - else - ret = 0; -out: - mutex_unlock(&sbi->buddy_mutex); - scoutfs_block_put(ind_bh); - scoutfs_block_put(bh); - - return ret; -} - - -/* - * Create dirty cow copies of the bitmap, indirect, and buddy blocks - * so that a free of the given extent in the current transaction is - * guaranteed to succeed. - * - * This is only meant for buddy allocators who are complicated enough - * to need help avoiding error conditions. - */ -int scoutfs_buddy_dirty(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - switch(blkno_region(super, blkno)) { - case REGION_BM: - return bitmap_dirty(sb, blkno); - case REGION_BUDDY: - return buddy_dirty(sb, blkno, order); - } - - return 0; -} - /* * The block layer allocates from the same region as an existing blkno * when it's allocating for cow. diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 8397aa12..b4e5b2f6 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -4,7 +4,6 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, u64 existing); -int scoutfs_buddy_dirty(struct super_block *sb, u64 blkno, int order); int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); #endif From 1fde47170ba1fe69143d1af40fe2fbf0e3159d7b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 2 Aug 2016 13:26:52 -0700 Subject: [PATCH 073/920] scoutfs: simplify btree block format Now that we are using fixed smaller blocks we can make the btree format significantly simpler. The fixed small block size limits the number of items that will be stored in each block. We can use a simple sorted array of item offsets to maintain the item sort order instead of the treap. Getting rid of the treap not only removes a bunch of code, it makes tasks like verifying or repairing a btree block a lot simpler. The main impact on the code is that now an item doesn't record its position in the sort order. Users of sorted item position now need to track an items sorted position instead of just the item. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/btree.c | 625 ++++++++++++++++++++++++++-------------------- kmod/src/btree.h | 4 +- kmod/src/format.h | 29 +-- kmod/src/treap.c | 386 ---------------------------- kmod/src/treap.h | 38 --- 6 files changed, 375 insertions(+), 709 deletions(-) delete mode 100644 kmod/src/treap.c delete mode 100644 kmod/src/treap.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 2e484055..143929b7 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -4,4 +4,4 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trans.o \ - treap.o xattr.o + xattr.o diff --git a/kmod/src/btree.c b/kmod/src/btree.c index df0b75e3..57e9ffe9 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -13,12 +13,12 @@ #include #include #include +#include #include "super.h" #include "format.h" #include "block.h" #include "key.h" -#include "treap.h" #include "btree.h" /* @@ -26,19 +26,14 @@ * sized keys and variable length values. * * Items are stored as a small header with the key followed by the - * value. New items are appended to the end of the block. Free space - * is not indexed. Deleted items can be reclaimed by walking all the - * items from the front of the block and moving later live items onto - * earlier deleted items. + * value. New items are allocated from the back of the block towards + * the front. Deleted items can be reclaimed by packing items towards + * the back of the block by walking them in reverse offset order. * - * The items are kept in a treap sorted by their keys. Using a dynamic - * structure keeps the modification costs low. Modifying persistent - * structures avoids translation to and from run-time structures around - * read and write. The treap was chosen because it's very simple to - * implement and has some cool merging and splitting functions that we - * could make use of. The treap has parent pointers so that we can - * perform operations relative to a node without having to keep a record - * of the path down the tree. + * A dense array of item offsets after the btree block header header + * maintains the sorted order of the items by their keys. The array is + * small enough that the memmoves to keep it dense involves a few cache + * lines at most. * * Parent blocks in the btree have the same format as leaf blocks. * There's one key for every child reference instead of having separator @@ -61,142 +56,161 @@ * XXX * - do we want a level in the btree header? seems like we would? * - validate structures on read? + * - internal bh/pos/cmp interface is clumsy.. could use cursor */ -/* size of the item with a value of the given length */ +/* number of contiguous bytes used by the item header and val of given len */ static inline unsigned int val_bytes(unsigned int val_len) { return sizeof(struct scoutfs_btree_item) + val_len; } +/* number of contiguous bytes used by the item header its current value */ static inline unsigned int item_bytes(struct scoutfs_btree_item *item) { return val_bytes(le16_to_cpu(item->val_len)); } +/* total bytes consumed by an item with given val len: offset, header, value */ +static inline unsigned int all_val_bytes(unsigned int val_len) +{ + return sizeof(((struct scoutfs_btree_block *)NULL)->item_offs[0]) + + val_bytes(val_len); +} + +/* total bytes consumed by an item with its current value */ +static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) +{ + return all_val_bytes(le16_to_cpu(item->val_len)); +} + +/* number of contig free bytes between item offset and first item */ +static inline unsigned int contig_free(struct scoutfs_btree_block *bt) +{ + return le16_to_cpu(bt->free_end) - + offsetof(struct scoutfs_btree_block, item_offs[bt->nr_items]); +} + +/* number of contig bytes free after reclaiming free amongst items */ +static inline unsigned int reclaimable_free(struct scoutfs_btree_block *bt) +{ + return contig_free(bt) + le16_to_cpu(bt->free_reclaim); +} + +/* all bytes used by item offsets, headers, and values */ static inline unsigned int used_total(struct scoutfs_btree_block *bt) { return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - - le16_to_cpu(bt->total_free); + reclaimable_free(bt); } -static int cmp_tnode_items(struct scoutfs_treap_node *A, - struct scoutfs_treap_node *B) +static inline struct scoutfs_btree_item * +off_item(struct scoutfs_btree_block *bt, __le16 off) { - struct scoutfs_btree_item *a; - struct scoutfs_btree_item *b; - - a = container_of(A, struct scoutfs_btree_item, tnode); - b = container_of(B, struct scoutfs_btree_item, tnode); - - return scoutfs_key_cmp(&a->key, &b->key); + return (void *)bt + le16_to_cpu(off); } -/* A bunch of wrappers for navigating items through treap nodes. */ - -#define BT_TREAP_KEY_WRAPPER(which) \ -static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt, \ - struct scoutfs_key *key) \ -{ \ - struct scoutfs_btree_item dummy = { .key = *key }; \ - struct scoutfs_treap_node *node; \ - \ - node = scoutfs_treap_##which(&bt->treap, cmp_tnode_items, \ - &dummy.tnode); \ - if (!node) \ - return NULL; \ - \ - return container_of(node, struct scoutfs_btree_item, tnode); \ -} - -BT_TREAP_KEY_WRAPPER(lookup) -/* BT_TREAP_KEY_WRAPPER(before) */ -BT_TREAP_KEY_WRAPPER(after) - -#define BT_TREAP_ROOT_WRAPPER(which) \ -static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt) \ -{ \ - struct scoutfs_treap_node *node; \ - \ - node = scoutfs_treap_##which(&bt->treap); \ - if (!node) \ - return NULL; \ - \ - return container_of(node, struct scoutfs_btree_item, tnode); \ -} - -BT_TREAP_ROOT_WRAPPER(first) -BT_TREAP_ROOT_WRAPPER(last) - -#define BT_TREAP_NODE_WRAPPER(which) \ -static struct scoutfs_btree_item *bt_##which(struct scoutfs_btree_block *bt, \ - struct scoutfs_btree_item *item)\ -{ \ - struct scoutfs_treap_node *node; \ - \ - node = scoutfs_treap_##which(&bt->treap, &item->tnode); \ - if (!node) \ - return NULL; \ - \ - return container_of(node, struct scoutfs_btree_item, tnode); \ -} - -BT_TREAP_NODE_WRAPPER(next) -BT_TREAP_NODE_WRAPPER(prev) - -static inline struct scoutfs_key *least_key(struct scoutfs_btree_block *bt) +static inline struct scoutfs_btree_item * +pos_item(struct scoutfs_btree_block *bt, unsigned int pos) { - return &bt_first(bt)->key; + return off_item(bt, bt->item_offs[pos]); } static inline struct scoutfs_key *greatest_key(struct scoutfs_btree_block *bt) { - return &bt_last(bt)->key; + return &pos_item(bt, bt->nr_items - 1)->key; } /* - * Allocate and insert a new item into the block. + * Returns the sorted item position that an item with the given key + * should occupy. * - * The caller has made sure that there's room for everything. + * It sets *cmp to the final comparison of the given key and the + * position's item key. * - * The caller is responsible for initializing the value. + * If the given key is greater then all items' keys then the number of + * items can be returned. Callers need to be careful to test for this + * invalid index. + */ +static int find_pos(struct scoutfs_btree_block *bt, struct scoutfs_key *key, + int *cmp) +{ + unsigned int start = 0; + unsigned int end = bt->nr_items; + unsigned int pos = 0; + + *cmp = -1; + + while (start < end) { + pos = start + (end - start) / 2; + + *cmp = scoutfs_key_cmp(key, &pos_item(bt, pos)->key); + if (*cmp < 0) { + end = pos; + } else if (*cmp > 0) { + start = ++pos; + *cmp = -1; + } else { + break; + } + } + + return pos; +} + +/* move a number of contigous elements from the src index to the dst index */ +#define memmove_arr(arr, dst, src, nr) \ + memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) + +/* + * Allocate and insert a new item into the block. The caller has made + * sure that there's room for everything. The caller is responsible for + * initializing the value. */ static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, + unsigned int pos, struct scoutfs_key *key, unsigned int val_len) { - unsigned int bytes = val_bytes(val_len); struct scoutfs_btree_item *item; - item = (void *)((char *)bt + SCOUTFS_BLOCK_SIZE - - le16_to_cpu(bt->tail_free)); - le16_add_cpu(&bt->tail_free, -bytes); - le16_add_cpu(&bt->total_free, -bytes); - le16_add_cpu(&bt->nr_items, 1); + if (pos < bt->nr_items) + memmove_arr(bt->item_offs, pos + 1, pos, bt->nr_items - pos); + le16_add_cpu(&bt->free_end, -val_bytes(val_len)); + bt->item_offs[pos] = bt->free_end; + bt->nr_items++; + + item = pos_item(bt, pos); item->key = *key; item->seq = bt->hdr.seq; item->val_len = cpu_to_le16(val_len); - scoutfs_treap_insert(&bt->treap, cmp_tnode_items, &item->tnode); + trace_printk("pos %u off %u\n", pos, le16_to_cpu(bt->item_offs[pos])); return item; } -#define MAGIC_DELETED_PARENT cpu_to_le16(1) - /* - * Delete an item from a btree block. We set the deleted item's parent - * treap offset to a magic value for compaction. + * Delete an item from a btree block. We record the amount of space it + * frees to later decide if we can satisfy an insertion by compaction + * instead of splitting. */ -static void delete_item(struct scoutfs_btree_block *bt, - struct scoutfs_btree_item *item) +static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) { - scoutfs_treap_delete(&bt->treap, &item->tnode); - item->tnode.parent = MAGIC_DELETED_PARENT; + struct scoutfs_btree_item *item = pos_item(bt, pos); - le16_add_cpu(&bt->total_free, item_bytes(item)); - le16_add_cpu(&bt->nr_items, -1); + trace_printk("pos %u off %u\n", pos, le16_to_cpu(bt->item_offs[pos])); + + if (pos < (bt->nr_items - 1)) + memmove_arr(bt->item_offs, pos, pos + 1, + bt->nr_items - 1 - pos); + + le16_add_cpu(&bt->free_reclaim, item_bytes(item)); + bt->nr_items--; + + /* wipe deleted items to avoid leaking data */ + memset(item, 0, item_bytes(item)); } /* @@ -204,40 +218,71 @@ static void delete_item(struct scoutfs_btree_block *bt, * tells us if we're moving from the tail of the source block right to * the head of the destination block, or vice versa. We stop moving * once we've moved enough bytes of items. - * - * XXX This could use fancy treap splitting and merging. We don't need - * to go there yet. */ static void move_items(struct scoutfs_btree_block *dst, struct scoutfs_btree_block *src, bool move_right, int to_move) { struct scoutfs_btree_item *from; - struct scoutfs_btree_item *del; struct scoutfs_btree_item *to; - unsigned int val_len; + unsigned int t; + unsigned int f; - if (move_right) - from = bt_last(src); - else - from = bt_first(src); - - while (from && to_move > 0) { - val_len = le16_to_cpu(from->val_len); - - to = create_item(dst, &from->key, val_len); - memcpy(to->val, from->val, val_len); - to->seq = from->seq; - - del = from; - if (move_right) - from = bt_prev(src, from); - else - from = bt_next(src, from); - - delete_item(src, del); - to_move -= item_bytes(to); + if (move_right) { + f = src->nr_items - 1; + t = 0; + } else { + f = 0; + t = dst->nr_items; } + + while (f < src->nr_items && to_move > 0) { + from = pos_item(src, f); + + to = create_item(dst, t, &from->key, + le16_to_cpu(from->val_len)); + + memcpy(to, from, item_bytes(from)); + to_move -= all_item_bytes(from); + + delete_item(src, f); + if (move_right) + f--; + else + t++; + } +} + +static struct scoutfs_btree_block *aligned_bt(const void *ptr) +{ + unsigned long addr = (unsigned long)ptr; + + return (void *)(addr & ~((unsigned long)SCOUTFS_BLOCK_MASK)); +} + +static int sort_key_cmp(const void *A, const void *B) +{ + struct scoutfs_btree_block *bt = aligned_bt(A); + const __le16 * __packed a = A; + const __le16 * __packed b = B; + + return scoutfs_key_cmp(&off_item(bt, *a)->key, &off_item(bt, *b)->key); +} + +static int sort_off_cmp(const void *A, const void *B) +{ + const __le16 * __packed a = A; + const __le16 * __packed b = B; + + return (int)le16_to_cpu(*a) - (int)le16_to_cpu(*b); +} + +static void sort_off_swap(void *A, void *B, int size) +{ + __le16 * __packed a = A; + __le16 * __packed b = B; + + swap(*a, *b); } /* @@ -248,14 +293,20 @@ static void move_items(struct scoutfs_btree_block *dst, * items. * * We don't bother implementing free space indexing and addressing that - * corner case. Instead we track the number of total free bytes in the - * block. If free space needed is available in the block but is not - * available at the end of the block then we reclaim the fragmented free - * space by compacting the items. + * corner case. Instead we track the number of bytes that could be + * reclaimed if we compacted the item space after the free_end offset. + * block. If this additional free space would satisfy an insertion then + * we compact the items instead of splitting the block. * - * We move the free space to the tail of the block by walk forward - * through the items in allocated order moving live items back in to - * free space. + * We move the free space to the center of the block by walking + * backwards through the items in offset order, moving items into free + * space between items towards the end of the block. + * + * We don't have specific metadata to either walk the items in offset + * order or to update the item offsets as we move items. We sort the + * item offset array to achieve both ends. First we sort it by offset + * so we can walk in reverse order. As we move items we update their + * position and then sort by keys once we're done. * * Compaction is only attempted during descent as we find a block that * needs more or less free space. The caller has the parent locked for @@ -264,30 +315,49 @@ static void move_items(struct scoutfs_btree_block *dst, */ static void compact_items(struct scoutfs_btree_block *bt) { - struct scoutfs_btree_item *from = (void *)(bt + 1); - struct scoutfs_btree_item *to = from; + struct scoutfs_btree_item *from; + struct scoutfs_btree_item *to; unsigned int bytes; - unsigned int i; + __le16 end; + int i; + + trace_printk("free_reclaim %u\n", le16_to_cpu(bt->free_reclaim)); + + sort(bt->item_offs, bt->nr_items, sizeof(bt->item_offs[0]), + sort_off_cmp, sort_off_swap); + + end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); + + for (i = bt->nr_items - 1; i >= 0; i--) { + from = pos_item(bt, i); - for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { bytes = item_bytes(from); + le16_add_cpu(&end, -bytes); + to = off_item(bt, end); + bt->item_offs[i] = end; - if (from->tnode.parent != MAGIC_DELETED_PARENT) { - if (from != to) { - memmove(to, from, bytes); - scoutfs_treap_move(&bt->treap, &from->tnode, - &to->tnode); - } - to = (void *)to + bytes; - } else { - i--; - } - - from = (void *)from + bytes; + if (from != to) + memmove(to, from, bytes); } - bytes = SCOUTFS_BLOCK_SIZE - ((char *)to - (char *)bt); - bt->tail_free = cpu_to_le16(bytes); + bt->free_end = end; + bt->free_reclaim = 0; + + sort(bt->item_offs, bt->nr_items, sizeof(bt->item_offs[0]), + sort_key_cmp, sort_off_swap); +} + +/* sorting relies on masking pointers to find the containing block */ +static inline struct buffer_head *check_bh_alignment(struct buffer_head *bh) +{ + struct scoutfs_btree_block *bt = bh_data(bh); + + if (!IS_ERR_OR_NULL(bh) && WARN_ON_ONCE(aligned_bt(bt) != bt)) { + scoutfs_block_put(bh); + return ERR_PTR(-EIO); + } + + return bh; } /* @@ -303,14 +373,12 @@ static struct buffer_head *alloc_tree_block(struct super_block *sb) if (!IS_ERR(bh)) { bt = bh_data(bh); - bt->treap.off = 0; - bt->total_free = cpu_to_le16(SCOUTFS_BLOCK_SIZE - - sizeof(struct scoutfs_btree_block)); - bt->tail_free = bt->total_free; + bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); + bt->free_reclaim = 0; bt->nr_items = 0; } - return bh; + return check_bh_alignment(bh); } /* the caller has ensured that the free must succeed */ @@ -342,11 +410,26 @@ static struct buffer_head *grow_tree(struct super_block *sb, return bh; } +static struct buffer_head *get_block_ref(struct super_block *sb, + struct scoutfs_block_ref *ref, + bool dirty) +{ + struct buffer_head *bh; + + if (dirty) + bh = scoutfs_block_dirty_ref(sb, ref); + else + bh = scoutfs_block_read_ref(sb, ref); + + return check_bh_alignment(bh); +} + /* * Create a new item in the parent which references the child. The caller * specifies the key in the item that describes the items in the child. */ static void create_parent_item(struct scoutfs_btree_block *parent, + unsigned int pos, struct scoutfs_btree_block *child, struct scoutfs_key *key) { @@ -356,7 +439,7 @@ static void create_parent_item(struct scoutfs_btree_block *parent, .seq = child->hdr.seq, }; - item = create_item(parent, key, sizeof(ref)); + item = create_item(parent, pos, key, sizeof(ref)); memcpy(&item->val, &ref, sizeof(ref)); } @@ -385,23 +468,24 @@ static struct buffer_head *try_split(struct super_block *sb, int level, struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_block *parent, - struct scoutfs_btree_item *par_item, + unsigned int parent_pos, struct buffer_head *right_bh) { struct scoutfs_btree_block *right = bh_data(right_bh); struct scoutfs_btree_block *left; struct buffer_head *left_bh; struct buffer_head *par_bh = NULL; - unsigned int bytes; + struct scoutfs_key maximal; + unsigned int all_bytes; if (level) val_len = sizeof(struct scoutfs_block_ref); - bytes = val_bytes(val_len); + all_bytes = all_val_bytes(val_len); - if (le16_to_cpu(right->tail_free) >= bytes) + if (contig_free(right) >= all_bytes) return right_bh; - if (le16_to_cpu(right->total_free) >= bytes) { + if (reclaimable_free(right) >= all_bytes) { compact_items(right); return right_bh; } @@ -424,27 +508,25 @@ static struct buffer_head *try_split(struct super_block *sb, } parent = bh_data(par_bh); - } + parent_pos = 0; - /* only grow the tree once we have the split neighbour */ - if (par_bh) { - struct scoutfs_key maximal; scoutfs_set_max_key(&maximal); - create_parent_item(parent, right, &maximal); + create_parent_item(parent, parent_pos, right, &maximal); } move_items(left, right, false, used_total(right) / 2); - create_parent_item(parent, left, greatest_key(left)); + create_parent_item(parent, parent_pos, left, greatest_key(left)); + parent_pos++; /* not that anything uses it again :P */ if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { /* insertion will go to the new left block */ scoutfs_block_put(right_bh); right_bh = left_bh; } else { - /* insertion will still go through us, might need to compact */ scoutfs_block_put(left_bh); - if (le16_to_cpu(right->tail_free) < bytes) + /* insertion will still go through us, might need to compact */ + if (contig_free(right) < all_bytes) compact_items(right); } @@ -477,29 +559,31 @@ static struct buffer_head *try_split(struct super_block *sb, static struct buffer_head *try_merge(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, - struct scoutfs_btree_item *par_item, + unsigned int pos, struct buffer_head *bh) { struct scoutfs_btree_block *bt = bh_data(bh); + struct scoutfs_btree_item *sib_item; struct scoutfs_btree_block *sib_bt; struct buffer_head *sib_bh; - struct scoutfs_btree_item *sib_item; - int to_move; + unsigned int sib_pos; bool move_right; + int to_move; - if (le16_to_cpu(bt->total_free) <= SCOUTFS_BTREE_FREE_LIMIT) + if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) return bh; /* move items right into our block if we have a left sibling */ - sib_item = bt_prev(parent, par_item); - if (sib_item) { - move_right = false; - } else { - sib_item = bt_next(parent, par_item); + if (pos) { + sib_pos = pos - 1; move_right = true; + } else { + sib_pos = pos + 1; + move_right = false; } + sib_item = pos_item(parent, sib_pos); - sib_bh = scoutfs_block_dirty_ref(sb, (void *)sib_item->val); + sib_bh = get_block_ref(sb, (void *)sib_item->val, true); if (IS_ERR(sib_bh)) { /* XXX do we need to unlock this? don't think so */ scoutfs_block_put(bh); @@ -507,31 +591,33 @@ static struct buffer_head *try_merge(struct super_block *sb, } sib_bt = bh_data(sib_bh); - if (used_total(sib_bt) <= le16_to_cpu(bt->total_free)) + if (used_total(sib_bt) <= reclaimable_free(bt)) to_move = used_total(sib_bt); else - to_move = le16_to_cpu(bt->total_free) - - SCOUTFS_BTREE_FREE_LIMIT; + to_move = reclaimable_free(bt) - SCOUTFS_BTREE_FREE_LIMIT; - if (le16_to_cpu(bt->tail_free) < to_move) + if (contig_free(bt) < to_move) compact_items(bt); + trace_printk("sib_pos %d move_right %u to_move %u\n", + sib_pos, move_right, to_move); + move_items(bt, sib_bt, move_right, to_move); /* update our parent's ref if we changed our greatest key */ if (!move_right) - par_item->key = *greatest_key(bt); + pos_item(parent, pos)->key = *greatest_key(bt); /* delete an empty sib or update if we changed its greatest key */ if (sib_bt->nr_items == 0) { - delete_item(parent, sib_item); + delete_item(parent, sib_pos); free_tree_block(sb, sib_bt->hdr.blkno); } else if (move_right) { sib_item->key = *greatest_key(sib_bt); } /* and finally shrink the tree if our parent is the root with 1 */ - if (le16_to_cpu(parent->nr_items) == 1) { + if (parent->nr_items == 1) { root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; @@ -599,44 +685,50 @@ static u64 item_block_ref_seq(struct scoutfs_btree_item *item) * number. If it's a parent then we test the block ref's seq, if it's a * leaf item then we check the item's seq. */ -static int item_skip_seq(struct scoutfs_btree_item *item, +static bool skip_pos_seq(struct scoutfs_btree_block *bt, unsigned int pos, int level, u64 seq, int op) { - return op == WALK_NEXT_SEQ && item && - ((level > 0 && item_block_ref_seq(item) < seq) || + struct scoutfs_btree_item *item; + + if (op != WALK_NEXT_SEQ || pos >= bt->nr_items) + return false; + + item = pos_item(bt, pos); + + return ((level > 0 && item_block_ref_seq(item) < seq) || (level == 0 && le64_to_cpu(item->seq) < seq)); } /* - * Return the next item, possibly skipping those with sequence numbers - * less than the desired sequence number. + * Return the next sorted item position, possibly skipping those with + * sequence numbers less than the desired sequence number. */ -static struct scoutfs_btree_item * -item_next_seq(struct scoutfs_btree_block *bt, struct scoutfs_btree_item *item, - int level, u64 seq, int op) +static unsigned int next_pos_seq(struct scoutfs_btree_block *bt, + unsigned int pos, int level, u64 seq, int op) { do { - item = bt_next(bt, item); - } while (item_skip_seq(item, level, seq, op)); + pos++; + } while (skip_pos_seq(bt, pos, level, seq, op)); - return item; + return pos; } /* * Return the first item after the given key, possibly skipping those * with sequence numbers less than the desired sequence number. */ -static struct scoutfs_btree_item * -item_after_seq(struct scoutfs_btree_block *bt, struct scoutfs_key *key, - int level, u64 seq, int op) +static unsigned int find_pos_after_seq(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, int level, + u64 seq, int op) { - struct scoutfs_btree_item *item; + unsigned int pos; + int cmp; - item = bt_after(bt, key); - if (item_skip_seq(item, level, seq, op)) - item = item_next_seq(bt, item, level, seq, op); + pos = find_pos(bt, key, &cmp); + if (skip_pos_seq(bt, pos, level, seq, op)) + pos = next_pos_seq(bt, pos, level, seq, op); - return item; + return pos; } /* @@ -662,6 +754,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, struct scoutfs_btree_item *item = NULL; struct scoutfs_block_ref *ref; unsigned int level; + unsigned int pos = 0; const bool dirty = op == WALK_INSERT || op == WALK_DELETE || op == WALK_DIRTY; @@ -697,29 +790,15 @@ static struct buffer_head *btree_walk(struct super_block *sb, while (level--) { /* XXX hmm, need to think about retry */ - if (dirty) { - bh = scoutfs_block_dirty_ref(sb, ref); - } else { - bh = scoutfs_block_read_ref(sb, ref); - } + bh = get_block_ref(sb, ref, dirty); if (IS_ERR(bh)) break; - /* - * Update the next key an iterator should read from. - * Keep in mind that iteration is read only so the - * parent item won't be changed splitting or merging. - */ - if (parent && next_key) { - *next_key = item->key; - scoutfs_inc_key(next_key); - } - if (op == WALK_INSERT) bh = try_split(sb, root, level, key, val_len, parent, - item, bh); + pos, bh); if ((op == WALK_DELETE) && parent) - bh = try_merge(sb, root, parent, item, bh); + bh = try_merge(sb, root, parent, pos, bh); if (IS_ERR(bh)) break; @@ -740,18 +819,29 @@ static struct buffer_head *btree_walk(struct super_block *sb, * seqs then we might not have any child items to * search. */ - item = item_after_seq(parent, key, level, seq, op); - if (!item) { + pos = find_pos_after_seq(parent, key, level, seq, op); + if (pos >= parent->nr_items) { /* current block dropped as parent below */ - if (op == WALK_NEXT_SEQ) { + if (op == WALK_NEXT_SEQ) bh = ERR_PTR(-ENOENT); - } else { + else bh = ERR_PTR(-EIO); - } break; + break; } /* XXX verify sane length */ + item = pos_item(parent, pos); ref = (void *)item->val; + + /* + * Update the next key an iterator should read from. + * Keep in mind that iteration is read only so the + * parent item won't be changed splitting or merging. + */ + if (next_key) { + *next_key = item->key; + scoutfs_inc_key(next_key); + } } unlock_block(sbi, par_bh, dirty); @@ -761,16 +851,19 @@ static struct buffer_head *btree_walk(struct super_block *sb, } static void set_cursor(struct scoutfs_btree_cursor *curs, - struct buffer_head *bh, - struct scoutfs_btree_item *item, bool write) + struct buffer_head *bh, unsigned int pos, bool write) { + struct scoutfs_btree_block *bt = bh_data(bh); + struct scoutfs_btree_item *item = pos_item(bt, pos); + curs->bh = bh; - curs->item = item; + curs->pos = pos; + curs->write = write; + curs->key = &item->key; curs->seq = le64_to_cpu(item->seq); curs->val = item->val; curs->val_len = le16_to_cpu(item->val_len); - curs->write = !!write; } /* @@ -780,9 +873,10 @@ static void set_cursor(struct scoutfs_btree_cursor *curs, int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_cursor *curs) { - struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + unsigned int pos; + int cmp; int ret; BUG_ON(curs->bh); @@ -790,11 +884,11 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, bh = btree_walk(sb, key, NULL, 0, 0, 0); if (IS_ERR(bh)) return PTR_ERR(bh); - bt = bh_data(bh); - item = bt_lookup(bt, key); - if (item) { - set_cursor(curs, bh, item, false); + + pos = find_pos(bt, key, &cmp); + if (cmp == 0) { + set_cursor(curs, bh, pos, false); ret = 0; } else { unlock_block(NULL, bh, false); @@ -817,9 +911,10 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_cursor *curs) { - struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + int pos; + int cmp; int ret; BUG_ON(curs->bh); @@ -829,16 +924,15 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, return PTR_ERR(bh); bt = bh_data(bh); - /* XXX should this return -eexist? */ - item = bt_lookup(bt, key); - if (!item) { - item = create_item(bt, key, val_len); - set_cursor(curs, bh, item, true); + pos = find_pos(bt, key, &cmp); + if (cmp) { + create_item(bt, pos, key, val_len); + set_cursor(curs, bh, pos, true); ret = 0; } else { unlock_block(NULL, bh, true); scoutfs_block_put(bh); - ret = -ENOENT; + ret = -EEXIST; } return ret; @@ -852,9 +946,10 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_root *root; - struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + int pos; + int cmp; int ret; bh = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); @@ -862,9 +957,9 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) return PTR_ERR(bh); bt = bh_data(bh); - item = bt_lookup(bt, key); - if (item) { - delete_item(bt, item); + pos = find_pos(bt, key, &cmp); + if (cmp == 0) { + delete_item(bt, pos); ret = 0; /* XXX this locking is broken.. hold root rwsem? */ @@ -920,12 +1015,12 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, /* find the next item after the cursor, releasing if we're done */ if (curs->bh) { bt = bh_data(curs->bh); - key = curs->item->key; + key = *curs->key; scoutfs_inc_key(&key); - curs->item = item_next_seq(bt, curs->item, 0, seq, op); - if (curs->item) - set_cursor(curs, curs->bh, curs->item, curs->write); + curs->pos = next_pos_seq(bt, curs->pos, 0, seq, op); + if (curs->pos < bt->nr_items) + set_cursor(curs, curs->bh, curs->pos, curs->write); else scoutfs_btree_release(curs); } @@ -949,25 +1044,20 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, bt = bh_data(bh); /* keep trying leaves until next_key passes last */ - curs->item = item_after_seq(bt, &key, 0, seq, op); - if (!curs->item) { + curs->pos = find_pos_after_seq(bt, &key, 0, seq, op); + if (curs->pos >= bt->nr_items) { key = next_key; unlock_block(NULL, bh, false); scoutfs_block_put(bh); continue; } - if (curs->item) { - set_cursor(curs, bh, curs->item, false); - } else { - unlock_block(NULL, bh, false); - scoutfs_block_put(bh); - } + set_cursor(curs, bh, curs->pos, false); break; } /* only return the next item if it's within last */ - if (curs->item && scoutfs_key_cmp(curs->key, last) <= 0) { + if (curs->bh && scoutfs_key_cmp(curs->key, last) <= 0) { ret = 1; } else { scoutfs_btree_release(curs); @@ -1000,18 +1090,18 @@ int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, */ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) { - struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + int cmp; int ret; bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); - bt = bh_data(bh); - item = bt_lookup(bt, key); - if (item) { + + find_pos(bt, key, &cmp); + if (cmp == 0) { ret = 0; } else { ret = -ENOENT; @@ -1033,6 +1123,8 @@ void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + int pos; + int cmp; BUG_ON(curs->bh); @@ -1040,11 +1132,12 @@ void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, BUG_ON(IS_ERR(bh)); bt = bh_data(bh); - item = bt_lookup(bt, key); - BUG_ON(!item); + pos = find_pos(bt, key, &cmp); + BUG_ON(cmp); + item = pos_item(bt, pos); item->seq = bt->hdr.seq; - set_cursor(curs, bh, item, true); + set_cursor(curs, bh, pos, true); } void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 2dc05bb5..a7faca87 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -4,14 +4,14 @@ struct scoutfs_btree_cursor { /* for btree.c */ struct buffer_head *bh; - struct scoutfs_btree_item *item; + unsigned int pos; + bool write; /* for callers */ struct scoutfs_key *key; u64 seq; void *val; u16 val_len; - u16 write:1; }; #define DECLARE_SCOUTFS_BTREE_CURSOR(name) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 4e9eeaad..2a3605c4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -104,28 +104,26 @@ struct scoutfs_key { #define SCOUTFS_MAX_ITEM_LEN 512 -struct scoutfs_treap_root { - __le16 off; -} __packed; - -struct scoutfs_treap_node { - __le16 parent; - __le16 left; - __le16 right; - __le32 prio; -} __packed; - struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; } __packed; +/* + * @free_end: records the byte offset of the first byte after the free + * space in the block between the header and the first item. New items + * are allocated by subtracting the space they need. + * + * @free_reclaim: records the number of bytes of free space amongst the + * items after free_end. If a block is compacted then this much new + * free space would be reclaimed. + */ struct scoutfs_btree_block { struct scoutfs_block_header hdr; - struct scoutfs_treap_root treap; - __le16 total_free; - __le16 tail_free; - __le16 nr_items; + __le16 free_end; + __le16 free_reclaim; + __u8 nr_items; + __le16 item_offs[0]; } __packed; /* @@ -134,7 +132,6 @@ struct scoutfs_btree_block { */ struct scoutfs_btree_item { struct scoutfs_key key; - struct scoutfs_treap_node tnode; __le64 seq; __le16 val_len; char val[0]; diff --git a/kmod/src/treap.c b/kmod/src/treap.c deleted file mode 100644 index 39ab4a3b..00000000 --- a/kmod/src/treap.c +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include - -#include "format.h" -#include "treap.h" - -/* - * Implement a simple treap in memory. The caller is responsible for - * allocating and freeing roots and nodes. This only performs the tree - * operations on them. - * - * Node references are stored as byte offsets from the root to the node. - * As long as we have the root the byte offsets or node pointers are - * interchangeable. The code tries to prefer to use pointers to be - * slightly easier to read. - * - * The caller is responsible for locking access to the tree. - */ - -/* - * treap nodes are embedded in btree items. Their offset is relative to - * the treap root which is embedded in the btree block header. Their - * offset can't have the item overlap the btree block header, nor can - * the item fall off the end of the block. - */ -static void bug_on_bad_node_off(u16 off) -{ - BUG_ON(off < (sizeof(struct scoutfs_btree_block) - - offsetof(struct scoutfs_btree_block, treap) + - offsetof(struct scoutfs_btree_item, tnode))); - BUG_ON(off > (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_item) + - offsetof(struct scoutfs_btree_item, tnode))); -} - -static struct scoutfs_treap_node *off_node(struct scoutfs_treap_root *root, - __le16 off) -{ - if (!off) - return NULL; - - bug_on_bad_node_off(le16_to_cpu(off)); - - return (void *)root + le16_to_cpu(off); -} - -static __le16 node_off(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node) -{ - u16 off; - - if (!node) - return 0; - - off = (char *)node - (char *)root; - bug_on_bad_node_off(off); - - return cpu_to_le16(off); -} - -/* - * Walk the tree looking for a node that matches a node in the tree. - * Return the found node or the last node traversed. Set the caller's - * cmp to the comparison between the key and the returned node. The - * caller can ask that we set their pointers to the most recently - * traversed node before or after the returned node. - */ -static struct scoutfs_treap_node *descend(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key, - int *cmp, - struct scoutfs_treap_node **before, - struct scoutfs_treap_node **after) -{ - struct scoutfs_treap_node *node = NULL; - __le16 off = root->off; - - *cmp = -1; - if (before) - *before = NULL; - if (after) - *after = NULL; - - while (off) { - node = off_node(root, off); - *cmp = cmp_func(key, node); - if (*cmp < 0) { - if (after) - *after = node; - off = node->left; - } else if (*cmp > 0) { - if (before) - *before = node; - off = node->right; - } else { - break; - } - } - - return node; -} - -/* - * Link the two nodes together by setting their child and parent pointers - * as needed. Both parent and child can be null. - */ -static void set_links(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *parent, bool left, - struct scoutfs_treap_node *child) -{ - if (!parent) - root->off = node_off(root, child); - else if (left) - parent->left = node_off(root, child); - else - parent->right = node_off(root, child); - - if (child) - child->parent = node_off(root, parent); -} - -/* - * Perform a tree rotation. The node pointer names describe their - * relationships before the rotation. We use the relationship between - * the node and its child to determine the direction of the rotation. - * After the rotation the child will be higher than the node. Only the - * node and child must exist. - * - * Here's a right rotation: - * - * parent parent - * | | - * node child - * / \ / \ - * child a b node - * / \ / \ - * b gr_chi gr_chi a - * - */ -static void rotation(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node, - struct scoutfs_treap_node *child) -{ - struct scoutfs_treap_node *parent = off_node(root, node->parent); - struct scoutfs_treap_node *grand_child; - bool right; - - if (node->left == node_off(root, child)) { - right = true; - grand_child = off_node(root, child->right); - } else { - right = false; - grand_child = off_node(root, child->left); - } - - set_links(root, parent, - parent && (parent->left == node_off(root, node)), child); - set_links(root, node, right, grand_child); - set_links(root, child, !right, node); -} - -/* - * Insertion links a node in at a leaf and then rotates it up the - * tree until its parent has a higher priority. - */ -int scoutfs_treap_insert(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *ins) -{ - struct scoutfs_treap_node *parent; - int cmp; - - ins->prio = cpu_to_le32(get_random_int()); - ins->parent = 0; - ins->left = 0; - ins->right = 0; - - parent = descend(root, cmp_func, ins, &cmp, NULL, NULL); - if (cmp == 0) - return -EEXIST; - - set_links(root, parent, cmp < 0, ins); - - while (ins->parent) { - parent = off_node(root, ins->parent); - if (le32_to_cpu(ins->prio) < le32_to_cpu(parent->prio)) - break; - - rotation(root, parent, ins); - } - - return 0; -} - -/* - * Deletion rotates the node down the tree until it doesn't have two - * children so that it can be unlinked by pointing its parent at its - * child, if it has one. - */ -void scoutfs_treap_delete(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node) -{ - struct scoutfs_treap_node *left; - struct scoutfs_treap_node *right; - struct scoutfs_treap_node *child; - struct scoutfs_treap_node *parent; - - while (node->left && node->right) { - left = off_node(root, node->left); - right = off_node(root, node->right); - - if (le32_to_cpu(left->prio) > le32_to_cpu(right->prio)) - rotation(root, node, left); - else - rotation(root, node, right); - } - - parent = off_node(root, node->parent); - - if (node->left) - child = off_node(root, node->left); - else - child = off_node(root, node->right); - - set_links(root, parent, - parent && parent->left == node_off(root, node), child); -} - -struct scoutfs_treap_node *scoutfs_treap_lookup(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key) -{ - struct scoutfs_treap_node *node; - int cmp; - - node = descend(root, cmp_func, key, &cmp, NULL, NULL); - if (cmp != 0) - return NULL; - - return node; -} - -/* return the first node in the tree */ -struct scoutfs_treap_node *scoutfs_treap_first(struct scoutfs_treap_root *root) -{ - struct scoutfs_treap_node *node = off_node(root, root->off); - - while (node && node->left) - node = off_node(root, node->left); - - return node; -} - -/* return the last node in the tree */ -struct scoutfs_treap_node *scoutfs_treap_last(struct scoutfs_treap_root *root) -{ - struct scoutfs_treap_node *node = off_node(root, root->off); - - while (node && node->right) - node = off_node(root, node->right); - - return node; -} - -/* return the last node whose key is less than or equal to the key */ -struct scoutfs_treap_node *scoutfs_treap_before(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key) -{ - struct scoutfs_treap_node *before; - struct scoutfs_treap_node *node; - int cmp; - - node = descend(root, cmp_func, key, &cmp, &before, NULL); - if (cmp == 0) - return node; - - return before; -} - -/* return the first node whose key is greater than or equal to the key */ -struct scoutfs_treap_node *scoutfs_treap_after(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key) -{ - struct scoutfs_treap_node *after; - struct scoutfs_treap_node *node; - int cmp; - - node = descend(root, cmp_func, key, &cmp, NULL, &after); - if (cmp == 0) - return node; - - return after; -} - -/* - * The usual BST iteration: either the least descendant or the first - * ancestor in the direction of the iteration. - */ -struct scoutfs_treap_node *scoutfs_treap_next(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node) -{ - struct scoutfs_treap_node *parent; - - if (node->right) { - node = off_node(root, node->right); - while (node->left) - node = off_node(root, node->left); - return node; - } - - while ((parent = off_node(root, node->parent)) && - parent->right == node_off(root, node)) { - node = parent; - } - - return parent; -} - -struct scoutfs_treap_node *scoutfs_treap_prev(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node) -{ - struct scoutfs_treap_node *parent; - - if (node->left) { - node = off_node(root, node->left); - while (node->right) - node = off_node(root, node->right); - return node; - } - - while ((parent = off_node(root, node->parent)) && - parent->left == node_off(root, node)) { - node = parent; - } - - return parent; -} - -static void update_relative(struct scoutfs_treap_root *root, __le16 node_off, - __le16 from_off, __le16 to_off) -{ - struct scoutfs_treap_node *node = off_node(root, node_off); - - if (node) { - if (node->parent == from_off) - node->parent = to_off; - else if (node->left == from_off) - node->left = to_off; - else if (node->right == from_off) - node->right = to_off; - } -} - -/* - * A node has moved from one storage location to another. Update the - * nodes that refer to it. The from pointer can only be used to - * determine the old offset. Its contents are undefined. - */ -void scoutfs_treap_move(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *from, - struct scoutfs_treap_node *to) -{ - __le16 from_off = node_off(root, from); - __le16 to_off = node_off(root, to); - - if (root->off == from_off) - root->off = to_off; - else - update_relative(root, to->parent, from_off, to_off); - - update_relative(root, to->left, from_off, to_off); - update_relative(root, to->right, from_off, to_off); -} diff --git a/kmod/src/treap.h b/kmod/src/treap.h deleted file mode 100644 index d02f406c..00000000 --- a/kmod/src/treap.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef _SCOUTFS_TREAP_H_ -#define _SCOUTFS_TREAP_H_ - -#include "format.h" - -typedef int (*scoutfs_treap_cmp_t)(struct scoutfs_treap_node *a, - struct scoutfs_treap_node *b); - -static inline void scoutfs_treap_init(struct scoutfs_treap_root *root) -{ - root->off = 0; -} - -int scoutfs_treap_insert(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *ins); -void scoutfs_treap_delete(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node); -struct scoutfs_treap_node *scoutfs_treap_lookup(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key); -struct scoutfs_treap_node *scoutfs_treap_first(struct scoutfs_treap_root *root); -struct scoutfs_treap_node *scoutfs_treap_last(struct scoutfs_treap_root *root); -struct scoutfs_treap_node *scoutfs_treap_before(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key); -struct scoutfs_treap_node *scoutfs_treap_after(struct scoutfs_treap_root *root, - scoutfs_treap_cmp_t cmp_func, - struct scoutfs_treap_node *key); -struct scoutfs_treap_node *scoutfs_treap_next(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node); -struct scoutfs_treap_node *scoutfs_treap_prev(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *node); -void scoutfs_treap_move(struct scoutfs_treap_root *root, - struct scoutfs_treap_node *from, - struct scoutfs_treap_node *to); - -#endif From 8a6715ff02a7f1e8cb6d6e4ad4543e8611f33b5f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 9 Aug 2016 16:56:27 -0700 Subject: [PATCH 074/920] scoutfs: add buddy was_free and free_extent Add helpers to discover if a given allocation was free and to free all the buddy order allocations that make up an abritrary block extent. These are going to be used by the file data block mapping code. Signed-off-by: Zach Brown --- kmod/src/buddy.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/buddy.h | 3 ++ 2 files changed, 103 insertions(+) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index f2edb18b..82f02da5 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -703,3 +703,103 @@ int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) trace_scoutfs_buddy_free(blkno, order, region, ret); return ret; } + +/* XXX this should be generic */ +#define min3_t(t, a, b, c) min3((t)(a), (t)(b), (t)(c)) + +/* + * Free all the order allocations that make up the given unaligned block + * extent. Think of it as figuring out the largest aligned allocation + * that starts at the blkno and then clamping it by the count. + * + * For now this is only used by callers who have pinned the blocks that + * provided the allocation that they're now freeing from. It can't + * fail. If it could we would ensure that we re-alloc partial frees + * before returning an error. + */ +void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; + int order; + int size; + int ret; + + while (count) { + /* both blkno and count have to have bits set */ + order = min3_t(int, __ffs64(buddy_bit(super, blkno)), + fls64(count) - 1, + SCOUTFS_BUDDY_ORDERS - 1); + size = 1 << order; + + ret = scoutfs_buddy_free(sb, blkno, order); + BUG_ON(ret); + + blkno += size; + count -= size; + } +} + +/* + * Return > 1 if the given order allocation was free in the old stable + * transaction, 0 if it wasn't, and -errno if errors prevented us from + * finding out. + * + * XXX I bet we could get away without using the buddy mutex + */ +int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; + struct buffer_head *ind_bh = NULL; + struct buffer_head *bh = NULL; + struct scoutfs_buddy_indirect *ind; + struct scoutfs_buddy_block *bud; + struct scoutfs_block_ref *ref; + int ret; + int nr; + int sl; + + /* mkfs should have ensured that there's bitmap blocks */ + /* XXX corruption */ + if (sbi->super.buddy_bm_ref.blkno == 0 || + sbi->stable_super.buddy_bm_ref.blkno == 0) + return -EIO; + + mutex_lock(&sbi->buddy_mutex); + + /* get the stable indirect block */ + ind_bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); + if (IS_ERR(ind_bh)) { + ret = PTR_ERR(ind_bh); + goto out; + } + ind = bh_data(ind_bh); + + /* allocation was free if it's slot wasn't populated */ + sl = indirect_slot(super, blkno); + ref = &ind->slots[sl].ref; + if (!ref->blkno) { + ret = 1; + goto out; + } + + /* check the allocation bit in the old stable bitmap block */ + bh = scoutfs_block_read_ref(sb, ref); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + goto out; + } + bud = bh_data(bh); + + nr = buddy_bit(super, blkno) >> order; + ret = !!test_buddy_bit_or_higher(bud, order, nr); + +out: + mutex_unlock(&sbi->buddy_mutex); + scoutfs_block_put(ind_bh); + scoutfs_block_put(bh); + + trace_printk("blkno %llu order %d ret %d\n", blkno, order, ret); + return ret; +} diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index b4e5b2f6..7f7df982 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -5,5 +5,8 @@ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, u64 existing); int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); +void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); + +int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); #endif From 198ec2ed5b2036e4cb9c9799c1f2b72ebf9b66ba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 9 Aug 2016 17:01:00 -0700 Subject: [PATCH 075/920] scoutfs: have btree_update return errors We can certainly have btree update callers that haven't yet dirtied the blocks but who can deal with errors. So make it return errors and have its only current caller freak out if it fails. This will let the file data block mapping code attempt to get a dirty item without first dirtying. Signed-off-by: Zach Brown --- kmod/src/btree.c | 27 ++++++++++++++++++--------- kmod/src/btree.h | 4 ++-- kmod/src/inode.c | 5 ++++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 57e9ffe9..d3a1c9d0 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1114,30 +1114,39 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) } /* - * For this to be safe the caller has to have pinned the dirty blocks - * for the item in their transaction. + * This is guaranteed not to fail if the caller has already dirtied the + * block that contains the item in the current transaction. */ -void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs) +int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; int pos; int cmp; + int ret; BUG_ON(curs->bh); bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); - BUG_ON(IS_ERR(bh)); + if (IS_ERR(bh)) + return PTR_ERR(bh); bt = bh_data(bh); pos = find_pos(bt, key, &cmp); - BUG_ON(cmp); + if (cmp == 0) { + item = pos_item(bt, pos); + item->seq = bt->hdr.seq; + set_cursor(curs, bh, pos, true); + ret = 0; + } else { + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); + ret = -ENOENT; + } - item = pos_item(bt, pos); - item->seq = bt->hdr.seq; - set_cursor(curs, bh, pos, true); + return ret; } void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) diff --git a/kmod/src/btree.h b/kmod/src/btree.h index a7faca87..60d4a0b5 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -27,8 +27,8 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_btree_cursor *curs); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key); -void scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs); +int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs); int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole); int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 6caaeb06..fcdbd455 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -249,10 +249,13 @@ void scoutfs_update_inode_item(struct inode *inode) DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_key key; + int err; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - scoutfs_btree_update(sb, &key, &curs); + err = scoutfs_btree_update(sb, &key, &curs); + BUG_ON(err); + store_inode(curs.val, inode); scoutfs_btree_release(&curs); trace_scoutfs_update_inode(inode); From 77e0ffb9816d9e6f498b3b4a8fbb8d72236725c4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 10 Aug 2016 15:18:45 -0700 Subject: [PATCH 076/920] scoutfs: track data blocks in bmap items Up to this point we'd been storing file data in large fixed size items. This obviously needed to change to get decent large file IO patterns. This wires the file IO into the usual page cache and buffer head paths so that we write data blocks into allocations referenced by btree items. We're aggressively trying to find the highest ratio of performance to implementation complexity. Writing dirty metadata blocks during transaction commit changes a bit. We need to discover if we have dirty blocks before trying to sync the inodes. We add our _block_has_dirty() function back and use it to avoid write attempts during transaction commit. Signed-off-by: Zach Brown --- kmod/src/block.c | 27 +- kmod/src/block.h | 1 + kmod/src/filerw.c | 648 ++++++++++++++++++++++++++++++++++------------ kmod/src/filerw.h | 2 + kmod/src/format.h | 20 +- kmod/src/super.h | 5 + kmod/src/trans.c | 46 +++- 7 files changed, 561 insertions(+), 188 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index e6ddf0a8..4f82eae9 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -280,8 +280,6 @@ static void block_write_end_io(struct buffer_head *bh, int uptodate) * be written again in the next transaction commit. * * Reads can traverse the blocks while they're in flight. - * - * The number of blocks written is returned, or -errno on error. */ int scoutfs_block_write_dirty(struct super_block *sb) { @@ -290,13 +288,11 @@ int scoutfs_block_write_dirty(struct super_block *sb) struct rb_node *node; struct blk_plug plug; unsigned long flags; - int count; - int err; + int ret; atomic_set(&sbi->block_writes, 1); sbi->block_write_err = 0; - count = 0; - err = 0; + ret = 0; blk_start_plug(&plug); @@ -308,7 +304,6 @@ int scoutfs_block_write_dirty(struct super_block *sb) spin_unlock_irqrestore(&sbi->block_lock, flags); atomic_inc(&sbi->block_writes); - count++; scoutfs_block_set_crc(bh); /* @@ -321,10 +316,10 @@ int scoutfs_block_write_dirty(struct super_block *sb) lock_buffer(bh); bh->b_end_io = block_write_end_io; - err = submit_bh(WRITE, bh); /* doesn't actually fail? */ + ret = submit_bh(WRITE, bh); /* doesn't actually fail? */ spin_lock_irqsave(&sbi->block_lock, flags); - if (err) + if (ret) break; } spin_unlock_irqrestore(&sbi->block_lock, flags); @@ -335,10 +330,18 @@ int scoutfs_block_write_dirty(struct super_block *sb) atomic_dec(&sbi->block_writes); wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); - trace_printk("err %d sbi err %d count %d\n", - err, sbi->block_write_err, count); + trace_printk("ret %d\n", ret); + return ret; +} - return err ?: sbi->block_write_err ?: count; +/* + * The caller knows that it's not racing with writers. + */ +int scoutfs_block_has_dirty(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return !RB_EMPTY_ROOT(&sbi->block_dirty_tree); } /* diff --git a/kmod/src/block.h b/kmod/src/block.h index 8bb2a136..ee38b677 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -13,6 +13,7 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb); struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, struct scoutfs_block_ref *ref); +int scoutfs_block_has_dirty(struct super_block *sb); int scoutfs_block_write_dirty(struct super_block *sb); void scoutfs_block_set_crc(struct buffer_head *bh); diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index df946ee3..7ef5ea2a 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -13,8 +13,11 @@ #include #include #include +#include +#include #include "format.h" +#include "super.h" #include "inode.h" #include "key.h" #include "filerw.h" @@ -24,173 +27,512 @@ #include "ioctl.h" /* - * File data is stored in items just like everything else. This is very - * easy to implement but incurs a copying overhead. We'll see how - * expensive that gets. + * scoutfs uses simple fixed size block mapping items to map aligned + * groups of logical file data blocks to physical block locations. * - * By making the max item size a bit less than the block size we can - * still have room for the block header which gets us file data - * checksums. File item key offsets are multiples of this max block - * size though items can be smaller if the data is sparse. This lets us - * do lookups for specific keys and take advantage of the bloom filters. + * The small block size is set to the smallest supported page size. + * This means that our file IO code never has to worry about the + * situation where a page write is smaller than the block size. We + * never have to perform RMW of blocks larger than pages, nor do we have + * to punch a whole and worry about block tracking items that could be + * sharing references to a block on either side of a smaller dirty page. + * We can simply use the kernel's buffer head code, loathed though it + * is, and have a 1:1 relationship between block writes and block + * mapping item entries. * - * This is a minimal first pass and will need more work. It'll need to - * worry about enospc in writepage and cluster access for a start. + * Dirty blocks are only written to free space. The first time a block + * hits write_page in a transaction it gets a newly allocated block. We + * get decent contiguous allocations by having per-task preallocation + * streams. These are trimmed back as the transaction is committed. We + * don't bother worrying about small transactions. + * + * Because we only write to allocated space we can't naively use the + * buffer head get_blocks support functions. They assume that they can + * write dirty buffers to existing clean mappings which is absolutely + * not true for us. We clear mappings for clean pages before we call + * block_write_begin() so that it won't write to blocks that were caned + * from previous reads. We make sure that the page is uptodate ourself + * so that it won't use readpage to read the existing block and then + * turn around and write to it. + * + * Data blocks aren't pinned for the duration of the transaction. They + * can be written out and read back in and redirtied during the lifetime + * of a transaction. As we map dirty pages we see if its current allocation + * is newly allocated in the transaction and can reuse it. + * + * XXX + * - need to wire up dirty inode? + * - enforce writing to free blknos + * - per-task allocation regions + * - tear down dirty blocks left by write errors on unmount + * - should invalidate dirty blocks if freed + * - data block checksumming (stable pages) + * - mmap creating dirty unmapped pages at writepage + * - pack small tails into inline items + * - direct IO */ -/* -* Track the intersection of the logical region of a file with a page -* and file data item. -*/ -struct data_region { - u64 item_key; - unsigned int page_off; - unsigned short len; - unsigned short item_off; -}; /* - * Map the file offset to its intersection with the page and item region. - * Returns false if the byte position is outside the page. -*/ -static bool map_data_region(struct data_region *dr, u64 pos, struct page *page) -{ - if (pos >> PAGE_SHIFT != page->index) - return false; - - dr->page_off = pos & ~PAGE_MASK; - - dr->item_off = do_div(pos, SCOUTFS_MAX_ITEM_LEN); - dr->item_key = pos; - - dr->len = min_t(int, SCOUTFS_MAX_ITEM_LEN - dr->item_off, - PAGE_SIZE - dr->page_off); - - return true; -} - -#define for_each_data_region(dr, page, pos) \ - for (pos = (u64)page->index << PAGE_SHIFT; \ - map_data_region(dr, pos, page); pos += (dr)->len) - -/* - * Copy the contents of file data items into the page. If we don't - * find an item then we zero that region of the page. + * trace_printk() doesn't support %c? * - * XXX i_size? - * XXX async? + * 1 - 1ocked + * a - uptodAte + * d - Dirty + * b - writeBack + * e - Error */ -static int scoutfs_readpage(struct file *file, struct page *page) +#define page_hexflag(page, name, val, shift) \ + (Page##name(page) ? (val << (shift * 4)) : 0) + +#define page_hexflags(page) \ + (page_hexflag(page, Locked, 0x1, 4) | \ + page_hexflag(page, Uptodate, 0xa, 3) | \ + page_hexflag(page, Dirty, 0xd, 2) | \ + page_hexflag(page, Writeback, 0xb, 1) | \ + page_hexflag(page, Error, 0xe, 0)) + +#define PGF "page %p [index %lu flags %x]" +#define PGA(page) \ + (page), (page)->index, page_hexflags(page) \ + +#define BHF "bh %p [blocknr %llu size %zu state %lx]" +#define BHA(bh) \ + (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ + +/* + * For now this is super cheesy. We just have one allocation on the + * super that is consumed as buffered writes make their way through unmapped + * buffer heads and alloc in get_block. + */ +static int alloc_file_block(struct super_block *sb, u64 *blkno) { - struct inode *inode = file->f_mapping->host; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct super_block *sb = inode->i_sb; - struct scoutfs_key key; - struct data_region dr; - int ret = 0; - void *addr; - u64 pos; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 alloc_blkno; + int order = 0; + int ret; - for_each_data_region(&dr, page, pos) { - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, - dr.item_key); + *blkno = 0; - scoutfs_btree_release(&curs); - ret = scoutfs_btree_lookup(sb, &key, &curs); - if (ret == -ENOENT) { - addr = kmap_atomic(page); - memset(addr + dr.page_off, 0, dr.len); - kunmap_atomic(addr); - continue; + spin_lock(&sbi->file_alloc_lock); + + if (sbi->file_alloc_count == 0) { + spin_unlock(&sbi->file_alloc_lock); + + order = scoutfs_buddy_alloc(sb, &alloc_blkno, + SCOUTFS_BUDDY_ORDERS - 1); + if (order < 0) { + ret = order; + goto out; } - if (ret) - break; - addr = kmap_atomic(page); - memcpy(addr + dr.page_off, curs.val + dr.item_off, dr.len); - kunmap_atomic(addr); + spin_lock(&sbi->file_alloc_lock); + + if (sbi->file_alloc_count == 0) { + sbi->file_alloc_blkno = alloc_blkno; + sbi->file_alloc_count = 1 << order; + order = -1; + } } - scoutfs_btree_release(&curs); + if (sbi->file_alloc_count) { + *blkno = sbi->file_alloc_blkno; + sbi->file_alloc_blkno++; + sbi->file_alloc_count--; + ret = 0; + } else { + ret = -ENOSPC; + } - if (!ret) - SetPageUptodate(page); - unlock_page(page); + spin_unlock(&sbi->file_alloc_lock); + + if (order > 0) + scoutfs_buddy_free(sb, alloc_blkno, order); + +out: + trace_printk("allocated blkno %llu ret %d\n", *blkno, ret); return ret; } /* - * Copy the contents of the page into file items. Data integrity syncs - * will later write the dirty segment to the device. - * -* XXX zeroing regions of data items? -* XXX wbc counters? -* XXX reserve space so dirty item doesn't get enospc -- our "delalloc"? -*/ -static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) + * The caller didn't need an allocated file block after all. We return + * it to the pool. This has to succeed because it's called after we've + * done things that would be annoying to revert. + */ +static void return_file_block(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + spin_lock(&sbi->file_alloc_lock); + + BUG_ON(sbi->file_alloc_count && + sbi->file_alloc_blkno != (blkno + 1)); + + if (sbi->file_alloc_count == 0) + sbi->file_alloc_blkno = blkno + 1; + + sbi->file_alloc_blkno--; + sbi->file_alloc_count++; + + spin_unlock(&sbi->file_alloc_lock); +} + +/* + * The caller ensures that this is serialized against all other callers + * and writers. + */ +void scoutfs_filerw_free_alloc(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + trace_printk("blkno %llu count %llu\n", sbi->file_alloc_blkno, + sbi->file_alloc_count); + + if (sbi->file_alloc_count) + scoutfs_buddy_free_extent(sb, sbi->file_alloc_blkno, + sbi->file_alloc_count); + + sbi->file_alloc_blkno = 0; + sbi->file_alloc_count = 0; +} + +static void set_bmap_key(struct scoutfs_key *key, struct inode *inode, + u64 iblock) +{ + scoutfs_set_key(key, scoutfs_ino(inode), SCOUTFS_BMAP_KEY, + iblock >> SCOUTFS_BLOCK_MAP_SHIFT); +} + +/* + * Return the number of contiguously mapped blocks starting from the + * given logical block in the inode. We only return the number + * contained in one block map item. We walk through more items if it + * makes a difference. + */ +static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) { - struct inode *inode = page->mapping->host; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_block_map *bmap; struct scoutfs_key key; - struct data_region dr; - void *addr; - u64 pos; + int ret; + int i; + + *blkno = 0; + + set_bmap_key(&key, inode, iblock); + ret = scoutfs_btree_lookup(sb, &key, &curs); + if (!ret) { + bmap = curs.val; + + i = iblock & SCOUTFS_BLOCK_MAP_MASK; + *blkno = le64_to_cpu(bmap->blkno[i]); + + while (i < SCOUTFS_BLOCK_MAP_COUNT && bmap->blkno[i]) { + ret++; + i++; + } + scoutfs_btree_release(&curs); + } else if (ret == -ENOENT) { + ret = 0; + } + + trace_printk("ino %llu iblock %llu blkno %llu ret %d\n", + scoutfs_ino(inode), iblock, *blkno, ret); + + return ret; +} + +/* + * Make sure that the mapped block at the given logical block number is + * writable in this transaction. If it's not we allocate and reference + * a new block. If there was a previous stable block we free it. We + * give the caller the writable block number. + * + * Writeback is allowed during a transaction so we can get here with + * buffer heads that are newly allocated and being written to but for + * blocks that were allocated in the current transacation. In that + * case we re-use the existing mapping. None of it will be stable until + * there's a sync that writes all the referencing metadata. + */ +static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) +{ + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_block_map *bmap; + struct scoutfs_key key; + bool inserted = false; + u64 old_blkno = 0; + u64 new_blkno = 0; + int ret; + int err; + int i; + + set_bmap_key(&key, inode, iblock); + + /* we always need a writable block map item */ + ret = scoutfs_btree_update(sb, &key, &curs); + if (ret < 0 && ret != -ENOENT) + goto out; + + /* might need to create a new item and delete it after errors */ + if (ret == -ENOENT) { + ret = scoutfs_btree_insert(sb, &key, sizeof(*bmap), &curs); + if (ret < 0) + goto out; + memset(curs.val, 0, sizeof(*bmap)); + inserted = true; + } + + bmap = curs.val; + i = iblock & SCOUTFS_BLOCK_MAP_MASK; + old_blkno = le64_to_cpu(bmap->blkno[i]); + + /* + * If the existing block was free in stable then its dirty in + * this trans and we can use it. + */ + if (old_blkno) { + ret = scoutfs_buddy_was_free(sb, old_blkno, 0); + if (ret < 0) + goto out; + if (ret > 0) { + *blkno_ret = old_blkno; + ret = 0; + goto out; + } + } + + ret = alloc_file_block(sb, &new_blkno); + if (ret < 0) + goto out; + + if (old_blkno) { + ret = scoutfs_buddy_free(sb, old_blkno, 0); + if (ret) + goto out; + } + + bmap->blkno[i] = cpu_to_le64(new_blkno); + *blkno_ret = new_blkno; + new_blkno = 0; + ret = 0; +out: + scoutfs_btree_release(&curs); + if (ret) { + if (new_blkno) + return_file_block(sb, new_blkno); + if (inserted) { + err = scoutfs_btree_delete(sb, &key); + BUG_ON(err); /* always succeeds */ + } + } + + return ret; +} + +static int scoutfs_readpage_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + u64 blkno; int ret; - set_page_writeback(page); + if (WARN_ON_ONCE(create)) + return -EINVAL; + + ret = contig_mapped_blocks(inode, iblock, &blkno); + if (ret > 0) { + map_bh(bh, inode->i_sb, blkno); + bh->b_size = min_t(int, bh->b_size, ret << inode->i_blkbits); + ret = 0; + } + + trace_printk("ino %llu iblock %llu create %d "BHF"\n", + scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); + + return ret; +} + +static int scoutfs_readpage(struct file *file, struct page *page) +{ + trace_printk(PGF"\n", PGA(page)); + + return mpage_readpage(page, scoutfs_readpage_get_block); +} + +static int scoutfs_readpages(struct file *file, struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) +{ + return mpage_readpages(mapping, pages, nr_pages, + scoutfs_readpage_get_block); +} + +/* + * For now we don't know what to do if unmapped blocks make it to + * writepage (mmap?). + */ +static int scoutfs_writepage_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + trace_printk("ino %llu iblock %llu create %d "BHF"\n", + scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); + + return WARN_ON_ONCE(-EINVAL); +} + +/* + * Dirty file blocks can be written to their newly allocated free blocks + * at any time. They won't be referenced by metadata until the current + * transaction is committed. They can be re-read and re-dirtied at + * their free block number in this transaction. + */ +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + trace_printk(PGF"\n", PGA(page)); + + return block_write_full_page(page, scoutfs_writepage_get_block, wbc); +} + +static int scoutfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + trace_printk("mapping %p\n", mapping); + + return mpage_writepages(mapping, wbc, scoutfs_writepage_get_block); +} + +/* + * Block allocation during buffered writes needs to make sure that the + * dirty block will be written to free space. + */ +static int scoutfs_write_begin_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + u64 blkno = 0; + int ret; + + if (WARN_ON_ONCE(!create)) + return -EINVAL; + + ret = map_writable_block(inode, iblock, &blkno); + if (ret == 0) { + map_bh(bh, inode->i_sb, blkno); + bh->b_size = SCOUTFS_BLOCK_SIZE; + ret = 0; + } + + trace_printk("ino %llu iblock %llu create %d ret %d "BHF"\n", + scoutfs_ino(inode), (u64)iblock, create, ret, BHA(bh)); + return ret; +} + +/* XXX could make a for_each wrapper if we get a few of these */ +static inline void clear_mapped_page_buffers(struct page *page) +{ + struct buffer_head *head; + struct buffer_head *bh; + + if (!page_has_buffers(page)) + return; + + head = page_buffers(page); + bh = head; + do { + if (buffer_mapped(bh)) { + trace_printk(BHF"\n", BHA(bh)); + clear_buffer_mapped(bh); + } + + bh = bh->b_this_page; + } while (bh != head); +} + +/* + * Dirty blocks have to be mapped to be written out to free space so + * that we don't overwrite live data. We're relying on + * block_write_begin() to call get_block(). There are two problems with + * this. + * + * First, if it's going to be trying to read a partial block before writing + * then we can't give it the location to read. It'll just mark the + * block dirty and write to that same location. We use readpage to make + * the page uptodate if it's going to be satisfying a partial overwrite. + * + * Second, we can't let it use mappings that were used by readpage to + * read the current stable data. We need to have get_block be called + * for existing clean uptodate pages so that we can reallocate them to + * free space. We do this by clearing the buffer mappings for every buffer + * on the page for every call. This is probably unnecessarily expensive + * because we don't need to do it for clean buffers. That optimization + * would need to be done very carefully. + */ +static int scoutfs_write_begin(struct file *file, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; + pgoff_t index = pos >> PAGE_SHIFT; + struct page *page; + int ret; + +retry: + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) + return -ENOMEM; + + trace_printk(PGF" pos %llu len %u\n", PGA(page), (u64)pos, len); + + /* + * read in the page if we're going to be dirtying part of the + * page. readpage catches when this is a read past i_size or + * from a hole and zeros the buffer. + */ + if (!PageUptodate(page) && !IS_ALIGNED(pos | len, SCOUTFS_BLOCK_SIZE)) { + ClearPageError(page); + ret = scoutfs_readpage(NULL, page); + if (ret) { + page_cache_release(page); + goto out; + } + + wait_on_page_locked(page); + if (!PageUptodate(page)) { + page_cache_release(page); + ret = -EIO; + goto out; + } + + /* let grabbing deal with weird page states */ + page_cache_release(page); + goto retry; + } ret = scoutfs_hold_trans(sb); if (ret) goto out; - for_each_data_region(&dr, page, pos) { - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DATA_KEY, - dr.item_key); + /* can't re-enter fs, have trans */ + flags |= AOP_FLAG_NOFS; - /* XXX dirty */ - scoutfs_btree_release(&curs); - ret = scoutfs_btree_insert(sb, &key, SCOUTFS_MAX_ITEM_LEN, - &curs); - if (ret) - break; + /* make sure our get_block gets a chance to alloc */ + clear_mapped_page_buffers(page); - addr = kmap_atomic(page); - memcpy(curs.val + dr.item_off, addr + dr.page_off, dr.len); - kunmap_atomic(addr); - - } - - scoutfs_btree_release(&curs); - scoutfs_release_trans(sb); + ret = __block_write_begin(page, pos, len, + scoutfs_write_begin_get_block); out: - if (ret) { - SetPageError(page); - mapping_set_error(&inode->i_data, ret); - } + trace_printk(PGF" pos %llu len %u ret %d\n", + PGA(page), (u64)pos, len, ret); + if (ret < 0) { + /* XXX handle truncating? */ + unlock_page(page); + put_page(page); + page = NULL; + } - end_page_writeback(page); - unlock_page(page); - - return ret; -} - -static int scoutfs_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned flags, - struct page **pagep, void **fsdata) -{ - struct inode *inode = mapping->host; - pgoff_t index = pos >> PAGE_CACHE_SHIFT; - struct page *page; - - trace_scoutfs_write_begin(scoutfs_ino(inode), pos, len); - - page = grab_cache_page_write_begin(mapping, index, flags); - if (!page) - return -ENOMEM; - - *pagep = page; - return 0; + *pagep = page; + return ret; } static int scoutfs_write_end(struct file *file, struct address_space *mapping, @@ -199,45 +541,21 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; - unsigned off; + int ret; - trace_scoutfs_write_end(scoutfs_ino(inode), pos, len, copied); + trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", + scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); - off = pos & (PAGE_CACHE_SIZE - 1); - - /* zero the stale part of the page if we did a short copy */ - if (copied < len) - zero_user_segment(page, off + copied, len); - - if (pos + copied > inode->i_size) { - i_size_write(inode, pos + copied); - - /* - * XXX This is a crazy hack that will go away when the - * file data paths are more robust. We're barely - * holding them together with duct tape while building - * up the robust metadata support that's needed to do a - * good job with the data pats. - */ - if (!scoutfs_hold_trans(sb)) { - if (!scoutfs_dirty_inode_item(inode)) - scoutfs_update_inode_item(inode); - scoutfs_release_trans(sb); - } - } - - if (!PageUptodate(page)) - SetPageUptodate(page); - set_page_dirty(page); - unlock_page(page); - page_cache_release(page); - - return copied; + ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); + scoutfs_release_trans(sb); + return ret; } const struct address_space_operations scoutfs_file_aops = { .readpage = scoutfs_readpage, + .readpages = scoutfs_readpages, .writepage = scoutfs_writepage, + .writepages = scoutfs_writepages, .write_begin = scoutfs_write_begin, .write_end = scoutfs_write_end, }; diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h index 2d9d478e..c182349d 100644 --- a/kmod/src/filerw.h +++ b/kmod/src/filerw.h @@ -4,4 +4,6 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; +void scoutfs_filerw_free_alloc(struct super_block *sb); + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 2a3605c4..fbd09ddd 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -100,7 +100,7 @@ struct scoutfs_key { #define SCOUTFS_INODE_KEY 1 #define SCOUTFS_XATTR_KEY 2 #define SCOUTFS_DIRENT_KEY 3 -#define SCOUTFS_DATA_KEY 4 +#define SCOUTFS_BMAP_KEY 4 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -244,4 +244,22 @@ struct scoutfs_xattr { __u8 name[0]; } __packed; +/* + * We use simple block map items to map a aligned fixed group of logical + * block offsets to physical blocks. We make them a decent size to + * reduce the item storage overhead per block referenced, but we don't + * want them so large that they start to take up an extraordinary amount + * of space for small files. 8 block items ranges from around 3% to .3% + * overhead for files that use only one or all of the blocks in the + * mapping item. + */ +#define SCOUTFS_BLOCK_MAP_SHIFT 3 +#define SCOUTFS_BLOCK_MAP_COUNT (1 << SCOUTFS_BLOCK_MAP_SHIFT) +#define SCOUTFS_BLOCK_MAP_MASK (SCOUTFS_BLOCK_MAP_COUNT - 1) + +struct scoutfs_block_map { + __le32 crc[SCOUTFS_BLOCK_MAP_COUNT]; + __le64 blkno[SCOUTFS_BLOCK_MAP_COUNT]; +}; + #endif diff --git a/kmod/src/super.h b/kmod/src/super.h index 7c3617b4..455f397e 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -43,6 +43,11 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; + + /* XXX we'd like this to be per task, not per super */ + spinlock_t file_alloc_lock; + u64 file_alloc_blkno; + u64 file_alloc_count; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 3291b83c..9be98ede 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -15,15 +15,17 @@ #include #include #include +#include #include "super.h" #include "block.h" #include "trans.h" #include "buddy.h" +#include "filerw.h" #include "scoutfs_trace.h" /* - * scoutfs metadata blocks are written in atomic transactions. + * scoutfs blocks are written in atomic transactions. * * Writers hold transactions to dirty blocks. The transaction can't be * written until these active writers release the transaction. We don't @@ -44,11 +46,24 @@ */ /* - * It's critical that this not try to perform IO if there's nothing - * dirty. The sync at unmount can have this work scheduled after sync - * returns and the unmount path starts to tear down supers and block - * devices. We have to safely detect that there's nothing to do using - * nothing in the vfs. + * This work func is responsible for writing out all the dirty blocks + * that make up the current dirty transaction. It prevents writers from + * holding a transaction so it doesn't have to worry about blocks being + * dirtied while it is working. + * + * Any dirty block had to have allocated a new blkno which would have + * created dirty allocator metadata blocks. We can avoid writing + * entirely if we don't have any dirty metadata blocks. This is + * important because we don't try to serialize this work during + * unmount.. we can execute as the vfs is shutting down.. we need to + * decide that nothing is dirty without calling the vfs at all. + * + * We first try to sync the dirty inodes and write their dirty data blocks, + * then we write all our dirty metadata blocks, and only when those succeed + * do we write the new super that references all of these newly written blocks. + * + * If there are write errors then blocks are kept dirty in memory and will + * be written again at the next sync. */ void scoutfs_trans_write_func(struct work_struct *work) { @@ -57,15 +72,26 @@ void scoutfs_trans_write_func(struct work_struct *work) struct super_block *sb = sbi->sb; bool advance = false; int ret = 0; + bool have_umount; wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); - /* XXX probably want to write out dirty pages in inodes */ + if (scoutfs_block_has_dirty(sb)) { + /* XXX need writeback errors from inode address spaces? */ - ret = scoutfs_block_write_dirty(sb); - if (ret > 0) { - ret = scoutfs_write_dirty_super(sb); + /* XXX definitely don't understand this */ + have_umount = down_read_trylock(&sb->s_umount); + + sync_inodes_sb(sb); + + if (have_umount) + up_read(&sb->s_umount); + + scoutfs_filerw_free_alloc(sb); + + ret = scoutfs_block_write_dirty(sb) ?: + scoutfs_write_dirty_super(sb); if (!ret) advance = 1; } From 0991622a21a265f0ca69c2efe93885308a01a2b5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 11 Aug 2016 16:46:18 -0700 Subject: [PATCH 077/920] scoutfs: add inode_paths ioctl This adds the ioctl that returns all the paths from the root to a given inode. The implementation only traverses btree items to keep it isolated from the vfs object locking and life cycles, but that could be a performance problem. This is another motivation to accelerate the btree code. Signed-off-by: Zach Brown --- kmod/src/dir.c | 268 ++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/dir.h | 12 +++ kmod/src/format.h | 21 +++- kmod/src/inode.c | 3 + kmod/src/inode.h | 1 + kmod/src/ioctl.c | 109 +++++++++++++++++++ kmod/src/ioctl.h | 13 +++ 7 files changed, 417 insertions(+), 10 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e6cdb5d0..bde01b89 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -49,6 +49,13 @@ * items will have distant offset values. It's only as the directory * gets very large that hash values will start to be this dense and * sweeping over items in a btree leaf is reasonably efficient. + * + * For each directory entry item stored in a directory inode there is a + * corresponding link backref item stored at the target inode. This + * lets us find all the paths that refer to a given inode. The link + * backref offset comes from an advancing counter in the inode and the + * item value contains the dir inode and dirent offset of the referring + * link. */ static unsigned int mode_to_type(umode_t mode) @@ -108,11 +115,14 @@ static unsigned int item_name_len(struct scoutfs_btree_cursor *curs) { return curs->val_len - sizeof(struct scoutfs_dirent); } + /* - * Store the dirent item hash in the dentry so that we don't have to - * calculate and search to remove the item. + * Each dirent stores the values that are needed to build the keys of + * the items that are removed on unlink so that we don't to search through + * items on unlink. */ struct dentry_info { + u64 lref_counter; u32 hash; }; @@ -158,6 +168,13 @@ static struct dentry_info *alloc_dentry_info(struct dentry *dentry) return dentry->d_fsdata; } +static void update_dentry_info(struct dentry_info *di, struct scoutfs_key *key, + struct scoutfs_dirent *dent) +{ + di->lref_counter = le64_to_cpu(dent->counter); + di->hash = scoutfs_key_offset(key); +} + static u64 last_dirent_key_offset(u32 h) { return min_t(u64, (u64)h + SCOUTFS_DIRENT_COLL_NR - 1, @@ -210,7 +227,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, if (scoutfs_names_equal(dentry->d_name.name, dentry->d_name.len, dent->name, name_len)) { ino = le64_to_cpu(dent->ino); - di->hash = scoutfs_key_offset(curs.key); + update_dentry_info(di, curs.key, dent); break; } } @@ -296,6 +313,34 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) return ret; } +static void set_lref_key(struct scoutfs_key *key, u64 ino, u64 ctr) +{ + scoutfs_set_key(key, ino, SCOUTFS_LINK_BACKREF_KEY, ctr); +} + +static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, + u64 dir_ino, u64 dir_off, bool update) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_link_backref *lref; + int ret; + + if (update) + ret = scoutfs_btree_update(sb, key, &curs); + else + ret = scoutfs_btree_insert(sb, key, sizeof(*lref), &curs); + + /* XXX verify size */ + if (ret == 0) { + lref = curs.val; + lref->ino = cpu_to_le64(dir_ino); + lref->offset = cpu_to_le64(dir_off); + scoutfs_btree_release(&curs); + } + + return ret; +} + static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { @@ -308,6 +353,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; + struct scoutfs_key lref_key; int bytes; int ret; u64 h; @@ -343,15 +389,25 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out; - ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + set_lref_key(&lref_key, scoutfs_ino(inode), + atomic64_inc_return(&SCOUTFS_I(inode)->link_counter)); + ret = update_lref_item(sb, &lref_key, scoutfs_ino(dir), + scoutfs_key_offset(&key), false); if (ret) goto out; + ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + if (ret) { + scoutfs_btree_delete(sb, &lref_key); + goto out; + } + dent = curs.val; dent->ino = cpu_to_le64(scoutfs_ino(inode)); + dent->counter = lref_key.offset; dent->type = mode_to_type(inode->i_mode); memcpy(dent->name, dentry->d_name.name, dentry->d_name.len); - di->hash = scoutfs_key_offset(&key); + update_dentry_info(di, &key, dent); scoutfs_btree_release(&curs); @@ -400,6 +456,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct timespec ts = current_kernel_time(); struct dentry_info *di; struct scoutfs_key key; + struct scoutfs_key lref_key; int ret = 0; if (WARN_ON_ONCE(!dentry->d_fsdata)) @@ -413,8 +470,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (ret) return ret; + set_lref_key(&lref_key, scoutfs_ino(inode), di->lref_counter); + ret = scoutfs_dirty_inode_item(dir) ?: - scoutfs_dirty_inode_item(inode); + scoutfs_dirty_inode_item(inode) ?: + scoutfs_btree_dirty(sb, &lref_key); if (ret) goto out; @@ -424,6 +484,8 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (ret) goto out; + scoutfs_btree_delete(sb, &lref_key); + dir->i_ctime = ts; dir->i_mtime = ts; i_size_write(dir, i_size_read(dir) - dentry->d_name.len); @@ -442,6 +504,200 @@ out: return ret; } +/* + * Add an allocated path component to the callers list which links to + * the target inode at a counter past the given counter. + * + * This is implemented by searching for link backrefs on the inode + * starting from the given counter. Those contain references to the + * parent directory and dirent key offset that contain the link to the + * inode. + * + * The caller holds no locks that protect components in the path. We + * search the link backref to find the parent dir then acquire it's + * i_mutex to make sure that its entries and backrefs are stable. If + * the next backref points to a different dir after we acquire the lock + * we bounce off and retry. + * + * Backref counters are never reused and rename only modifies the + * existing backref counter under the dir's mutex. + */ +static int add_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, + u64 *ctr, struct list_head *list) +{ + struct scoutfs_path_component *comp; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_link_backref *lref; + struct scoutfs_dirent *dent; + struct inode *inode = NULL; + struct scoutfs_key first; + struct scoutfs_key last; + struct scoutfs_key key; + u64 retried = 0; + u64 off; + int len; + int ret; + + comp = kmalloc(sizeof(struct scoutfs_path_component), GFP_KERNEL); + if (!comp) + return -ENOMEM; + +retry: + scoutfs_set_key(&first, ino, SCOUTFS_LINK_BACKREF_KEY, *ctr); + scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); + + ret = scoutfs_btree_next(sb, &first, &last, &curs); + if (ret <= 0) + goto out; + + lref = curs.val; + *dir_ino = le64_to_cpu(lref->ino), + off = le64_to_cpu(lref->offset); + *ctr = scoutfs_key_offset(curs.key); + + trace_printk("ino %llu ctr %llu dir_ino %llu off %llu\n", + ino, *ctr, *dir_ino, off); + + scoutfs_btree_release(&curs); + + /* XXX corruption, should never be key == U64_MAX */ + if (*ctr == U64_MAX) { + ret = -EIO; + goto out; + } + + /* XXX should verify ino and offset, too */ + + if (inode && scoutfs_ino(inode) != *dir_ino) { + mutex_unlock(&inode->i_mutex); + iput(inode); + inode = NULL; + } + + if (!inode) { + inode = scoutfs_iget(sb, *dir_ino); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + inode = NULL; + if (ret == -ENOENT && retried != *dir_ino) { + retried = *dir_ino; + goto retry; + } + goto out; + } + + mutex_lock(&inode->i_mutex); + goto retry; + } + + scoutfs_set_key(&key, *dir_ino, SCOUTFS_DIRENT_KEY, off); + + ret = scoutfs_btree_lookup(sb, &key, &curs); + if (ret < 0) { + /* XXX corruption, should always have dirent for backref */ + if (ret == -ENOENT) + ret = -EIO; + goto out; + } + + dent = curs.val; + len = item_name_len(&curs); + + trace_printk("dent ino %llu len %d\n", le64_to_cpu(dent->ino), len); + + /* XXX corruption */ + if (len < 1 || len > SCOUTFS_NAME_LEN) { + ret = -EIO; + goto out; + } + + /* XXX corruption, dirents should always match link backref */ + if (le64_to_cpu(dent->ino) != ino) { + ret = -EIO; + goto out; + } + + (*ctr)++; + comp->len = len; + memcpy(comp->name, dent->name, len); + list_add(&comp->head, list); + comp = NULL; /* won't be freed */ + + scoutfs_btree_release(&curs); + ret = 1; +out: + if (inode) { + mutex_unlock(&inode->i_mutex); + iput(inode); + } + + kfree(comp); + return ret; +} + +void scoutfs_dir_free_path(struct list_head *list) +{ + struct scoutfs_path_component *comp; + struct scoutfs_path_component *tmp; + + list_for_each_entry_safe(comp, tmp, list, head) { + list_del_init(&comp->head); + kfree(comp); + } +} + +/* + * Fill the list with the allocated path components that link the root + * to the target inode. The caller's ctr gives the link counter to + * start from. + * + * This is racing with modification of components in the path. We can + * traverse a partial path only to find that it's been blown away + * entirely. If we see a component go missing we retry. The removal of + * the final link to the inode should prevent repeatedly traversing + * paths that no longer exist. + * + * Returns > 0 and *ctr is updated if an allocated name was added to the + * list, 0 if no name past *ctr was found, or -errno on errors. + */ +int scoutfs_dir_next_path(struct super_block *sb, u64 ino, u64 *ctr, + struct list_head *list) +{ + u64 our_ctr; + u64 par_ctr; + u64 par_ino; + int ret; + + if (*ctr == U64_MAX) + return 0; + +retry: + our_ctr = *ctr; + /* get the next link name to the given inode */ + ret = add_linkref_name(sb, &par_ino, ino, &our_ctr, list); + if (ret <= 0) + goto out; + + /* then get the names of all the parent dirs */ + while (par_ino != SCOUTFS_ROOT_INO) { + par_ctr = 0; + ret = add_linkref_name(sb, &par_ino, par_ino, &par_ctr, list); + if (ret < 0) + goto out; + + /* restart if there was no parent component */ + if (ret == 0) { + scoutfs_dir_free_path(list); + goto retry; + } + } + +out: + if (ret > 0) + *ctr = our_ctr; + return ret; +} + const struct file_operations scoutfs_dir_fops = { .readdir = scoutfs_readdir, }; diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 3ee15f0f..44c715c5 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -1,10 +1,22 @@ #ifndef _SCOUTFS_DIR_H_ #define _SCOUTFS_DIR_H_ +#include "format.h" + extern const struct file_operations scoutfs_dir_fops; extern const struct inode_operations scoutfs_dir_iops; int scoutfs_dir_init(void); void scoutfs_dir_exit(void); +struct scoutfs_path_component { + struct list_head head; + unsigned int len; + char name[SCOUTFS_NAME_LEN]; +}; + +int scoutfs_dir_next_path(struct super_block *sb, u64 ino, u64 *ctr, + struct list_head *list); +void scoutfs_dir_free_path(struct list_head *list); + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index fbd09ddd..a2137ca9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -97,10 +97,11 @@ struct scoutfs_key { * isn't necessary. We could have an arbitrary sort order. So we don't * have to stress about cleverly allocating the types. */ -#define SCOUTFS_INODE_KEY 1 -#define SCOUTFS_XATTR_KEY 2 -#define SCOUTFS_DIRENT_KEY 3 -#define SCOUTFS_BMAP_KEY 4 +#define SCOUTFS_INODE_KEY 1 +#define SCOUTFS_XATTR_KEY 2 +#define SCOUTFS_DIRENT_KEY 3 +#define SCOUTFS_LINK_BACKREF_KEY 4 +#define SCOUTFS_BMAP_KEY 5 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -173,6 +174,7 @@ struct scoutfs_timespec { struct scoutfs_inode { __le64 size; __le64 blocks; + __le64 link_counter; __le32 nlink; __le32 uid; __le32 gid; @@ -192,6 +194,7 @@ struct scoutfs_inode { */ struct scoutfs_dirent { __le64 ino; + __le64 counter; __u8 type; __u8 name[0]; } __packed; @@ -262,4 +265,14 @@ struct scoutfs_block_map { __le64 blkno[SCOUTFS_BLOCK_MAP_COUNT]; }; +/* + * link backrefs give us a way to find all the hard links that refer + * to a target inode. They're stored at an offset determined by an + * advancing counter in their inode. + */ +struct scoutfs_link_backref { + __le64 ino; + __le64 offset; +} __packed; + #endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index fcdbd455..9750024f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -118,6 +118,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); ci->salt = le32_to_cpu(cinode->salt); + atomic64_set(&ci->link_counter, le64_to_cpu(cinode->link_counter)); } static int scoutfs_read_locked_inode(struct inode *inode) @@ -199,6 +200,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); cinode->salt = cpu_to_le32(ci->salt); + cinode->link_counter = cpu_to_le64(atomic64_read(&ci->link_counter)); } /* @@ -307,6 +309,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; get_random_bytes(&ci->salt, sizeof(ci->salt)); + atomic64_set(&ci->link_counter, 0); inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 3f68f4e5..563d08f5 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -5,6 +5,7 @@ struct scoutfs_inode_info { u64 ino; u32 salt; + atomic64_t link_counter; struct rw_semaphore xattr_rwsem; struct inode inode; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index d0311da7..99a4e273 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -15,10 +15,12 @@ #include #include #include +#include #include "format.h" #include "btree.h" #include "key.h" +#include "dir.h" #include "ioctl.h" /* @@ -93,11 +95,118 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) return ret; } +static int copy_to_ptr(char __user **to, const void *from, + unsigned long n, int space) +{ + if (n > space) + return -EOVERFLOW; + + if (copy_to_user(*to, from, n)) + return -EFAULT; + + *to += n; + return space - n; +} + +/* + * Fill the caller's buffer with all the paths from the on-disk root + * directory to the target inode. It will provide as many full paths as + * there are final links to the target inode. + * + * The null terminated paths are stored consecutively in the buffer. A + * final zero length null terminated string follows the last path. + * + * This only walks back through full hard links. None of the returned + * paths will reflect symlinks to components in the path. + * + * This doesn't ensure that the caller has permissions to traverse the + * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which + * bypasses permissions checking. + * + * If the provided buffer isn't large enough EOVERFLOW will be returned. + * The buffer can be approximately sized by multiplying the inode's + * nlink by PATH_MAX. + * + * This call is not serialized with any modification (create, rename, + * unlink) of the path components. It will return all the paths that + * were stable both before and after the call. It may or may not return + * paths which are created or unlinked during the call. + * + * This will return failure if it fails to read any path. An empty + * buffer is returned if the target inode doesn't exist or is + * disconnected from the root. + * + * XXX + * - we may want to support partial failure + * - can dir renaming trick us into returning garbage paths? seems likely. + */ +static long scoutfs_ioc_inode_paths(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_inode_paths __user *uargs = (void __user *)arg; + struct scoutfs_ioctl_inode_paths args; + struct scoutfs_path_component *comp; + struct scoutfs_path_component *tmp; + static char slash = '/'; + static char null = '\0'; + char __user *ptr; + LIST_HEAD(list); + u64 ctr; + int ret; + int len; + + if (!capable(CAP_DAC_READ_SEARCH)) + return -EPERM; + + if (copy_from_user(&args, uargs, sizeof(args))) + return -EFAULT; + + if (args.buf_len > INT_MAX) + return -EINVAL; + + ptr = (void __user *)(unsigned long)args.buf_ptr; + len = args.buf_len; + + ctr = 0; + while ((ret = scoutfs_dir_next_path(sb, args.ino, &ctr, &list)) > 0) { + ret = 0; + + /* copy the components out as a path */ + list_for_each_entry_safe(comp, tmp, &list, head) { + len = copy_to_ptr(&ptr, comp->name, comp->len, len); + if (len < 0) + goto out; + + list_del_init(&comp->head); + kfree(comp); + + if (!list_empty(&list)) { + len = copy_to_ptr(&ptr, &slash, 1, len); + if (len < 0) + goto out; + } + } + len = copy_to_ptr(&ptr, &null, 1, len); + if (len < 0) + goto out; + } + + len = copy_to_ptr(&ptr, &null, 1, len); +out: + scoutfs_dir_free_path(&list); + + if (ret == 0 && len < 0) + ret = len; + return ret; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { case SCOUTFS_IOC_INODES_SINCE: return scoutfs_ioc_inodes_since(file, arg); + case SCOUTFS_IOC_INODE_PATHS: + return scoutfs_ioc_inode_paths(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 259478e9..0d88b9fe 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -26,4 +26,17 @@ struct scoutfs_ioctl_inodes_since { #define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ struct scoutfs_ioctl_inodes_since) +struct scoutfs_ioctl_inode_paths { + __u64 ino; + __u64 buf_ptr; + __u32 buf_len; +} __packed; + +/* + * Fills the callers buffer with all the paths from the root to the + * target inode. + */ +#define SCOUTFS_IOC_INODE_PATHS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_inode_paths) + #endif From 6c12e7c38b10892f65fb9b8f07b0290162517cc6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 17 Aug 2016 16:22:00 -0700 Subject: [PATCH 078/920] scoutfs: add hard link support Now that we have the link backrefs let's add support for hard links so we can verify that an inode can have multiple backrefs. (It can.) It's a straight forward refactoring of mknod to let callers either allocate or use existing inodes. We push all the btree item specific work into a function called by mknod and link. The only surprising bit is the small max link count. It's limiting the worst case buffer size for the inode_paths ioctl. Signed-off-by: Zach Brown --- kmod/src/dir.c | 91 ++++++++++++++++++++++++++++++++++++++--------- kmod/src/format.h | 9 +++++ 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index bde01b89..79c0310c 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -341,15 +341,14 @@ static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, return ret; } -static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, - dev_t rdev) +static int add_entry_items(struct inode *dir, struct dentry *dentry, + struct inode *inode) { + struct dentry_info *di = dentry->d_fsdata; struct super_block *sb = dir->i_sb; struct scoutfs_inode_info *si = SCOUTFS_I(dir); DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct inode *inode = NULL; struct scoutfs_dirent *dent; - struct dentry_info *di; struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; @@ -358,27 +357,17 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, int ret; u64 h; - di = alloc_dentry_info(dentry); - if (IS_ERR(di)) - return PTR_ERR(di); + /* caller should have allocated the dentry info */ + if (WARN_ON_ONCE(di == NULL)) + return -EINVAL; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_hold_trans(sb); - if (ret) - return ret; - ret = scoutfs_dirty_inode_item(dir); if (ret) goto out; - inode = scoutfs_new_inode(sb, dir, mode, rdev); - if (IS_ERR(inode)) { - ret = PTR_ERR(inode); - goto out; - } - bytes = dent_bytes(dentry->d_name.len); h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); scoutfs_set_key(&first, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); @@ -410,6 +399,35 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, update_dentry_info(di, &key, dent); scoutfs_btree_release(&curs); +out: + return ret; +} + +static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t rdev) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct dentry_info *di; + int ret; + + di = alloc_dentry_info(dentry); + if (IS_ERR(di)) + return PTR_ERR(di); + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + inode = scoutfs_new_inode(sb, dir, mode, rdev); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + ret = add_entry_items(dir, dentry, inode); + if (ret) + goto out; i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -445,6 +463,44 @@ static int scoutfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) return scoutfs_mknod(dir, dentry, mode | S_IFDIR, 0); } +static int scoutfs_link(struct dentry *old_dentry, + struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = old_dentry->d_inode; + struct super_block *sb = dir->i_sb; + struct dentry_info *di; + int ret; + + if (inode->i_nlink >= SCOUTFS_LINK_MAX) + return -EMLINK; + + di = alloc_dentry_info(dentry); + if (IS_ERR(di)) + return PTR_ERR(di); + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + ret = add_entry_items(dir, dentry, inode); + if (ret) + goto out; + + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + inode->i_ctime = dir->i_mtime; + inc_nlink(inode); + + scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(dir); + + atomic_inc(&inode->i_count); + d_instantiate(dentry, inode); +out: + scoutfs_release_trans(sb); + return ret; +} + /* * Unlink removes the entry from its item and removes the item if ours * was the only remaining entry. @@ -707,6 +763,7 @@ const struct inode_operations scoutfs_dir_iops = { .mknod = scoutfs_mknod, .create = scoutfs_create, .mkdir = scoutfs_mkdir, + .link = scoutfs_link, .unlink = scoutfs_unlink, .rmdir = scoutfs_unlink, .setxattr = scoutfs_setxattr, diff --git a/kmod/src/format.h b/kmod/src/format.h index a2137ca9..69365083 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -215,6 +215,15 @@ struct scoutfs_dirent { #define SCOUTFS_NAME_LEN 255 +/* + * This is arbitrarily limiting the max size of the single buffer + * that's needed in the inode_paths ioctl to return all the paths + * that link to an inode. The structures could easily support much + * more than this but then we'd need to grow a more thorough interface + * for iterating over referring paths. That sounds horrible. + */ +#define SCOUTFS_LINK_MAX 255 + /* * We only use 31 bits for readdir positions so that we don't confuse * old signed 32bit f_pos applications or those on the other side of From 634114f364603bbecd5772146538879306e9ec82 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Aug 2016 12:05:59 -0700 Subject: [PATCH 079/920] scoutfs: update CKF key format The previous %llu for the key type came from the weird tracing functions that cast all the arguments to long long. Those have since been removed. Signed-off-by: Zach Brown --- kmod/src/key.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/key.h b/kmod/src/key.h index 34ac5be6..55b557f4 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -4,7 +4,7 @@ #include #include "format.h" -#define CKF "%llu.%llu.%llu" +#define CKF "%llu.%u.%llu" #define CKA(key) \ le64_to_cpu((key)->inode), (key)->type, le64_to_cpu((key)->offset) From c90710d26b2d18da1242eb54eb387b8ce0a82e2e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Aug 2016 12:14:55 -0700 Subject: [PATCH 080/920] scoutfs: add find xattr ioctls Add ioctls that return the inode numbers that probably contain the given xattr name or value. To support these we add items that index inodes by the presence of xattr items whose names or values hash to a give hash value. Signed-off-by: Zach Brown --- kmod/src/format.h | 13 +- kmod/src/ioctl.c | 100 +++++++++++++++ kmod/src/ioctl.h | 15 +++ kmod/src/xattr.c | 314 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 362 insertions(+), 80 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 69365083..fbe60040 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -99,9 +99,11 @@ struct scoutfs_key { */ #define SCOUTFS_INODE_KEY 1 #define SCOUTFS_XATTR_KEY 2 -#define SCOUTFS_DIRENT_KEY 3 -#define SCOUTFS_LINK_BACKREF_KEY 4 -#define SCOUTFS_BMAP_KEY 5 +#define SCOUTFS_XATTR_NAME_HASH_KEY 3 +#define SCOUTFS_XATTR_VAL_HASH_KEY 4 +#define SCOUTFS_DIRENT_KEY 5 +#define SCOUTFS_LINK_BACKREF_KEY 6 +#define SCOUTFS_BMAP_KEY 7 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -246,9 +248,8 @@ enum { SCOUTFS_DT_WHT, }; -#define SCOUTFS_MAX_XATTR_NAME_LEN 255 -#define SCOUTFS_MAX_XATTR_VALUE_LEN 255 -#define SCOUTFS_XATTR_HASH_MASK 7ULL +#define SCOUTFS_MAX_XATTR_LEN 255 +#define SCOUTFS_XATTR_NAME_HASH_MASK 7ULL struct scoutfs_xattr { __u8 name_len; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 99a4e273..752c665f 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -21,6 +21,7 @@ #include "btree.h" #include "key.h" #include "dir.h" +#include "name.h" #include "ioctl.h" /* @@ -200,6 +201,101 @@ out: return ret; } +/* + * Find inodes that might contain a given xattr name or value. + * + * The inodes are filled in sorted order from the first to the last + * inode. The number of found inodes is returned. If an error is hit + * it can return the number of inodes found before the error. + * + * The search can be continued from the next inode after the last + * returned. + */ +static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, + bool find_name) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_find_xattr args; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_key first; + struct scoutfs_key last; + char __user *ustr; + u64 __user *uino; + u64 inos[32]; + char *str; + int nr_inos = 0; + int copied = 0; + int ret; + u8 type; + u64 h; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + if (args.str_len > SCOUTFS_MAX_XATTR_LEN || args.ino_count > INT_MAX) + return -EINVAL; + + if (args.ino_count == 0) + return 0; + + ustr = (void __user *)(unsigned long)args.str_ptr; + uino = (void __user *)(unsigned long)args.ino_ptr; + + str = kmalloc(args.str_len, GFP_KERNEL); + if (!str) + return -ENOMEM; + + if (copy_from_user(str, ustr, args.str_len)) { + ret = -EFAULT; + goto out; + } + + h = scoutfs_name_hash(str, args.str_len); + + if (find_name) { + h &= ~SCOUTFS_XATTR_NAME_HASH_MASK; + type = SCOUTFS_XATTR_NAME_HASH_KEY; + } else { + type = SCOUTFS_XATTR_VAL_HASH_KEY; + } + + scoutfs_set_key(&first, h, type, args.first_ino); + scoutfs_set_key(&last, h, type, args.last_ino); + + while (copied < args.ino_count) { + + while ((ret = scoutfs_btree_next(sb, &first, &last, + &curs)) > 0) { + inos[nr_inos++] = scoutfs_key_offset(curs.key); + + first = *curs.key; + scoutfs_inc_key(&first); + + if (nr_inos == ARRAY_SIZE(inos) || + (nr_inos + copied) == args.ino_count) { + scoutfs_btree_release(&curs); + ret = 0; + break; + } + } + if (ret < 0 || nr_inos == 0) + break; + + if (copy_to_user(uino, inos, nr_inos * sizeof(u64))) { + ret = -EFAULT; + break; + } + + uino += nr_inos; + copied += nr_inos; + nr_inos = 0; + } + +out: + kfree(str); + return copied ?: ret; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -207,6 +303,10 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_inodes_since(file, arg); case SCOUTFS_IOC_INODE_PATHS: return scoutfs_ioc_inode_paths(file, arg); + case SCOUTFS_IOC_FIND_XATTR_NAME: + return scoutfs_ioc_find_xattr(file, arg, true); + case SCOUTFS_IOC_FIND_XATTR_VAL: + return scoutfs_ioc_find_xattr(file, arg, false); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 0d88b9fe..1abf6597 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -39,4 +39,19 @@ struct scoutfs_ioctl_inode_paths { #define SCOUTFS_IOC_INODE_PATHS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ struct scoutfs_ioctl_inode_paths) +/* XXX might as well include a seq? 0 for current behaviour? */ +struct scoutfs_ioctl_find_xattr { + __u64 first_ino; + __u64 last_ino; + __u64 str_ptr; + __u32 str_len; + __u64 ino_ptr; + __u32 ino_count; +} __packed; + +#define SCOUTFS_IOC_FIND_XATTR_NAME _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ + struct scoutfs_ioctl_find_xattr) +#define SCOUTFS_IOC_FIND_XATTR_VAL _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ + struct scoutfs_ioctl_find_xattr) + #endif diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index c45578da..95def5ae 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -40,6 +40,14 @@ * multiple versions of an xattr in the btree. So we add an inode rw * semaphore around xattr operations. * + * We support ioctls which find inodes that may contain xattrs with + * either a given name or value. A name hash item is created for a + * given hash value with no collision bits as long as there are any + * names at that hash value. A value hash item is created but it + * contains a refcount in its value to track the number of values with + * that hash value because we can't use the xattr keys to determine if + * there are matching values or not. + * * XXX * - add acl support and call generic xattr->handlers for SYSTEM * - remove all xattrs on unlink @@ -56,92 +64,221 @@ static unsigned int xat_bytes(unsigned int name_len, unsigned int value_len) return offsetof(struct scoutfs_xattr, name[name_len + value_len]); } +static void set_xattr_keys(struct inode *inode, struct scoutfs_key *first, + struct scoutfs_key *last, const char *name, + unsigned int name_len) +{ + u64 h = scoutfs_name_hash(name, name_len) & + ~SCOUTFS_XATTR_NAME_HASH_MASK; + + scoutfs_set_key(first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); + scoutfs_set_key(last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, + h | SCOUTFS_XATTR_NAME_HASH_MASK); +} + +static void set_name_val_keys(struct scoutfs_key *name_key, + struct scoutfs_key *val_key, + struct scoutfs_key *key, u64 val_hash) +{ + u64 h = scoutfs_key_offset(key) & ~SCOUTFS_XATTR_NAME_HASH_MASK; + + scoutfs_set_key(name_key, h, SCOUTFS_XATTR_NAME_HASH_KEY, + scoutfs_key_inode(key)); + + scoutfs_set_key(val_key, val_hash, SCOUTFS_XATTR_VAL_HASH_KEY, + scoutfs_key_inode(key)); +} + /* - * The caller provides an initialized cursor. + * Before insertion we perform a pretty through search of the xattr + * items whose offset collides with the name to be inserted. * - * If we return > 0 then the cursor points to an xattr with the given - * name and the caller must clean up the cursor. - * - * Returns 0 when no matching xattr is found or -errno on error. + * We try to find the item with the matching item so it can be removed. + * We notice if there are other colliding names so that the caller can + * correctly maintain the name hash items. We calculate the value hash + * of the existing item so that the caller can maintain the value hash + * items. And we notice if there are any free colliding items that are + * available for new item insertion. */ -static int lookup_xattr(struct inode *inode, const char *name, - unsigned int name_len, - struct scoutfs_btree_cursor *curs) +struct xattr_search_results { + bool found; + bool other_coll; + struct scoutfs_key key; + u64 val_hash; + bool found_hole; + struct scoutfs_key hole_key; +}; + +static int search_xattr_items(struct inode *inode, const char *name, + unsigned int name_len, + struct xattr_search_results *res) { struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_xattr *xat; int ret; - u64 h; - if (name_len > SCOUTFS_MAX_XATTR_NAME_LEN) - return -EINVAL; + set_xattr_keys(inode, &first, &last, name, name_len); - /* XXX could be a lookup helper? */ - h = scoutfs_name_hash(name, name_len) & ~SCOUTFS_XATTR_HASH_MASK; + res->found = false; + res->other_coll = false; + res->found_hole = false; + res->hole_key = first; - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); - scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, - h | SCOUTFS_XATTR_HASH_MASK); + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + xat = curs.val; - while ((ret = scoutfs_btree_next(sb, &first, &last, curs)) > 0) { - xat = curs->val; + /* found a hole when we skip past next expected key */ + if (!res->found_hole && + scoutfs_key_cmp(&res->hole_key, curs.key) < 0) + res->found_hole = true; - if (scoutfs_names_equal(name, name_len, xat->name, - xat->name_len)) + /* keep searching for a hole past this cursor key */ + if (!res->found_hole) { + res->hole_key = *curs.key; + scoutfs_inc_key(&res->hole_key); + } + + /* only compare the names until we find our given name */ + if (!res->found && + scoutfs_names_equal(name, name_len, xat->name, + xat->name_len)) { + res->found = true; + res->key = *curs.key; + res->val_hash = scoutfs_name_hash(xat_value(xat), + xat->value_len); + } else { + res->other_coll = true; + } + + /* finished once we have all the caller needs */ + if (res->found && res->other_coll && res->found_hole) { + ret = 0; + scoutfs_btree_release(&curs); break; + } } - if (ret <= 0) - scoutfs_btree_release(curs); - return ret; } /* - * Insert a new xattr and set the caller's key to the key that we used. - * The caller is responsible for managing transactions and locking. + * Inset a new xattr item, updating the name and value hash items as + * needed. The caller is responsible for managing transactions and + * locking. If this returns an error then no changes will have been + * made. */ static int insert_xattr(struct inode *inode, const char *name, unsigned int name_len, const void *value, size_t size, - struct scoutfs_key *key) + struct scoutfs_key *key, bool other_coll, + u64 val_hash) { struct super_block *sb = inode->i_sb; DECLARE_SCOUTFS_BTREE_CURSOR(curs); + bool inserted_name_hash_item = false; + __le64 * __packed refcount; + struct scoutfs_key name_key; + struct scoutfs_key val_key; struct scoutfs_xattr *xat; - struct scoutfs_key first; - struct scoutfs_key last; int ret; - u64 h; - if (name_len > SCOUTFS_MAX_XATTR_NAME_LEN || - size > SCOUTFS_MAX_XATTR_NAME_LEN) - return -EINVAL; + set_name_val_keys(&name_key, &val_key, key, val_hash); - /* XXX could be a lookup helper? */ - h = scoutfs_name_hash(name, name_len) & ~SCOUTFS_XATTR_HASH_MASK; - - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); - scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, - h | SCOUTFS_XATTR_HASH_MASK); - - /* find the first unoccupied key offset after the hashed name */ - ret = scoutfs_btree_hole(sb, &first, &last, key); + ret = scoutfs_btree_insert(sb, key, xat_bytes(name_len, size), &curs); if (ret) return ret; - ret = scoutfs_btree_insert(sb, key, xat_bytes(name_len, size), &curs); - if (!ret) { - xat = curs.val; - xat->name_len = name_len; - xat->value_len = size; - memcpy(xat->name, name, name_len); - memcpy(xat_value(xat), value, size); + xat = curs.val; + xat->name_len = name_len; + xat->value_len = size; + memcpy(xat->name, name, name_len); + memcpy(xat_value(xat), value, size); + scoutfs_btree_release(&curs); + + /* insert the name hash item for find_xattr if we're first */ + if (!other_coll) { + ret = scoutfs_btree_insert(sb, &name_key, 0, &curs); + /* XXX eexist would be corruption */ + if (ret) + goto out; + scoutfs_btree_release(&curs); + inserted_name_hash_item = true; + } + + /* increment the val hash item for find_xattr, inserting if first */ + ret = scoutfs_btree_update(sb, &val_key, &curs); + if (ret == -ENOENT) { + ret = scoutfs_btree_insert(sb, &val_key, sizeof(*refcount), + &curs); + if (ret == 0) { + /* XXX test sane item size */ + refcount = curs.val; + *refcount = 0; + } + } + if (ret == 0) { + refcount = curs.val; + le64_add_cpu(refcount, 1); scoutfs_btree_release(&curs); } +out: + if (ret) { + scoutfs_btree_delete(sb, key); + if (inserted_name_hash_item) + scoutfs_btree_delete(sb, &name_key); + } + return ret; +} + +/* + * Remove an xattr. Remove the name hash item if there are no more xattrs + * in the inode that hash to the name's hash value. Remove the value hash + * item if there are no more xattr values in the inode with this value + * hash. + */ +static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, + bool other_coll, u64 val_hash) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_key name_key; + struct scoutfs_key val_key; + __le64 * __packed refcount; + bool del_val = false; + int ret; + + set_name_val_keys(&name_key, &val_key, key, val_hash); + + if (!other_coll) { + ret = scoutfs_btree_dirty(sb, &name_key); + if (ret) + goto out; + } + ret = scoutfs_btree_dirty(sb, &val_key); + if (ret) + goto out; + + ret = scoutfs_btree_delete(sb, key); + if (ret) + goto out; + + if (!other_coll) + scoutfs_btree_delete(sb, &name_key); + + scoutfs_btree_update(sb, &val_key, &curs); + refcount = curs.val; + le64_add_cpu(refcount, -1ULL); + if (*refcount == 0) + del_val = true; + scoutfs_btree_release(&curs); + + if (del_val) + scoutfs_btree_delete(sb, &val_key); + ret = 0; +out: return ret; } @@ -158,23 +295,32 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, size_t size) { struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; struct scoutfs_inode_info *si = SCOUTFS_I(inode); DECLARE_SCOUTFS_BTREE_CURSOR(curs); size_t name_len = strlen(name); struct scoutfs_xattr *xat; + struct scoutfs_key first; + struct scoutfs_key last; int ret; if (unknown_prefix(name)) return -EOPNOTSUPP; + set_xattr_keys(inode, &first, &last, name, name_len); + down_read(&si->xattr_rwsem); - ret = lookup_xattr(inode, name, name_len, &curs); - if (ret == 0) { - ret = -ENODATA; - } else if (ret > 0) { + ret = -ENODATA; + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { xat = curs.val; + if (!scoutfs_names_equal(name, name_len, xat->name, + xat->name_len)) { + ret = -ENODATA; + continue; + } + ret = xat->value_len; if (buffer) { if (ret <= size) @@ -183,6 +329,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = -ERANGE; } scoutfs_btree_release(&curs); + break; } up_read(&si->xattr_rwsem); @@ -191,9 +338,16 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, } /* - * Set the xattr with the given name to the given value. The value can - * have a size of 0. A null value pointer indicates that we should - * delete the xattr. + * The confusing swiss army knife of creating, modifying, and deleting + * xattrs. + * + * If the value pointer is non-null then we always create a new item. The + * value can have a size of 0. We create a new item before possibly + * deleting an old item. + * + * We always delete the old xattr item. If we have a null value then we're + * deleting the xattr. If there's a value then we're effectively updating + * the xattr by deleting old and creating new. */ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) @@ -202,13 +356,15 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct xattr_search_results old = {0,}; size_t name_len = strlen(name); - struct scoutfs_key old_key; - struct scoutfs_key new_key; - bool old; + u64 new_val_hash = 0; int ret; + if (name_len > SCOUTFS_MAX_XATTR_LEN || + (value && size > SCOUTFS_MAX_XATTR_LEN)) + return -EINVAL; + if (unknown_prefix(name)) return -EOPNOTSUPP; @@ -220,39 +376,48 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (ret) goto out; + /* might as well do this outside locking */ + if (value) + new_val_hash = scoutfs_name_hash(value, size); + down_write(&si->xattr_rwsem); - ret = lookup_xattr(inode, name, name_len, &curs); - if (ret > 0) { - old = true; - old_key = *curs.key; - scoutfs_btree_release(&curs); - } else if (ret == 0) { - old = false; - } else { + /* + * The presence of other colliding names is a little tricky. + * Searching will set it if there are other non-matching names. + * It will be false if we only found the old matching name. That + * old match is also considered a collision for later insertion. + * Then *that* insertion is considered a collision for deletion + * of the existing old matching name. + */ + ret = search_xattr_items(inode, name, name_len, &old); + if (ret) goto out; - } - if (old && (flags & XATTR_CREATE)) { + if (old.found && (flags & XATTR_CREATE)) { ret = -EEXIST; goto out; } - if (!old && (flags & XATTR_REPLACE)) { + if (!old.found && (flags & XATTR_REPLACE)) { ret = -ENODATA; goto out; } if (value) { ret = insert_xattr(inode, name, name_len, value, size, - &new_key); + &old.hole_key, old.other_coll || old.found, + new_val_hash); if (ret) goto out; } - if (old) { - ret = scoutfs_btree_delete(sb, &old_key); + if (old.found) { + ret = delete_xattr(sb, &old.key, old.other_coll || value, + old.val_hash); if (ret) { - scoutfs_btree_delete(sb, &new_key); + if (value) + delete_xattr(sb, &old.hole_key, true, + new_val_hash); goto out; } } @@ -264,6 +429,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, out: up_write(&si->xattr_rwsem); scoutfs_release_trans(sb); + return ret; } From cb318982c9def9c0efb968f4b7e4c653993cc15c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 24 Aug 2016 15:52:54 -0700 Subject: [PATCH 081/920] scoutfs: add support for statfs To do a credible job of this we need to track the number of free blocks. We add counters of order allocations free to the indirect blocks so that we can quickly scan them. We also need a bit of help to count inodes. Finally I noticed that we were miscalculating the number of slots in the indirect blocks because we were using the size of the buddy block header, not the size of the indirect block header. Signed-off-by: Zach Brown --- kmod/src/buddy.c | 77 +++++++++++++++++++++++++++++++++++++---------- kmod/src/buddy.h | 1 + kmod/src/format.h | 3 +- kmod/src/inode.c | 16 ++++++++++ kmod/src/inode.h | 2 ++ kmod/src/super.c | 42 ++++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 17 deletions(-) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 82f02da5..48b85fca 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -11,6 +11,7 @@ * General Public License for more details. */ #include +#include #include "super.h" #include "format.h" @@ -164,16 +165,22 @@ static int test_buddy_bit_or_higher(struct scoutfs_buddy_block *bud, int order, return false; } -static void set_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) +static void set_buddy_bit(struct scoutfs_buddy_indirect *ind, + struct scoutfs_buddy_block *bud, int order, int nr) { - if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) + if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) { + le64_add_cpu(&ind->order_totals[order], 1); le32_add_cpu(&bud->order_counts[order], 1); + } } -static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) +static void clear_buddy_bit(struct scoutfs_buddy_indirect *ind, + struct scoutfs_buddy_block *bud, int order, int nr) { - if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) + if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) { + le64_add_cpu(&ind->order_totals[order], -1); le32_add_cpu(&bud->order_counts[order], -1); + } } /* returns INT_MAX when there are no bits set */ @@ -289,8 +296,10 @@ static int bitmap_free(struct super_block *sb, u64 blkno) * Give the caller a dirty buddy block. If the slot hasn't been used * yet then we need to allocate and initialize a new block. */ -static struct buffer_head *dirty_buddy_block(struct super_block *sb, int sl, - struct scoutfs_buddy_slot *slot) +static struct buffer_head *dirty_buddy_block(struct super_block *sb, + struct scoutfs_buddy_indirect *ind, + int sl, + struct scoutfs_buddy_slot *slot) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; @@ -325,7 +334,7 @@ static struct buffer_head *dirty_buddy_block(struct super_block *sb, int sl, size = 1 << order; nr = 0; while (count > size) { - set_buddy_bit(bud, order, nr); + set_buddy_bit(ind, bud, order, nr); nr++; count -= size; } @@ -333,7 +342,7 @@ static struct buffer_head *dirty_buddy_block(struct super_block *sb, int sl, /* set order bits for each of the bits set in the remaining count */ do { if (count & (1 << order)) { - set_buddy_bit(bud, order, nr); + set_buddy_bit(ind, bud, order, nr); nr = (nr + 1) << 1; } else { nr <<= 1; @@ -405,7 +414,8 @@ static int find_first_fit(struct scoutfs_super_block *super, int sl, * that breaks up a larger order. Higher level callers iterate over * smaller orders to provide partial allocations. */ -static int alloc_slot(struct super_block *sb, int sl, +static int alloc_slot(struct super_block *sb, + struct scoutfs_buddy_indirect *ind, int sl, struct scoutfs_buddy_slot *slot, struct scoutfs_block_ref *stable_ref, u64 *blkno, int order) @@ -422,7 +432,7 @@ static int alloc_slot(struct super_block *sb, int sl, int i; /* initialize or dirty the slot's buddy block */ - bh = dirty_buddy_block(sb, sl, slot); + bh = dirty_buddy_block(sb, ind, sl, slot); if (IS_ERR(bh)) return PTR_ERR(bh); bud = bh_data(bh); @@ -450,11 +460,11 @@ static int alloc_slot(struct super_block *sb, int sl, *blkno = slot_buddy_blkno(super, sl, found, nr); /* always clear the found order */ - clear_buddy_bit(bud, found, nr); + clear_buddy_bit(ind, bud, found, nr); /* free right buddies if we're breaking up a larger order */ for (nr <<= 1, i = found - 1; i >= order; i--, nr <<= 1) - set_buddy_bit(bud, i, nr | 1); + set_buddy_bit(ind, bud, i, nr | 1); update_free_orders(slot, bud); ret = 0; @@ -524,8 +534,8 @@ static int alloc_order(struct super_block *sb, u64 *blkno, int order) continue; } - ret = alloc_slot(sb, i, &ind->slots[i], &st_ind->slots[i].ref, - blkno, order); + ret = alloc_slot(sb, ind, i, &ind->slots[i], + &st_ind->slots[i].ref, blkno, order); if (ret != -ENOSPC) break; } @@ -664,11 +674,11 @@ static int buddy_free(struct super_block *sb, u64 blkno, int order) if (!test_buddy_bit(bud, i, nr ^ 1)) break; - clear_buddy_bit(bud, i, nr ^ 1); + clear_buddy_bit(ind, bud, i, nr ^ 1); nr >>= 1; } - set_buddy_bit(bud, i, nr); + set_buddy_bit(ind, bud, i, nr); update_free_orders(&ind->slots[sl], bud); scoutfs_block_put(bh); @@ -803,3 +813,38 @@ out: trace_printk("blkno %llu order %d ret %d\n", blkno, order, ret); return ret; } + +/* + * For now we only have one indirect block off the super. When we grow + * multiple commit block pairs that reference root and indirect blocks + * then we'll need to iterate over those. These results will only ever + * be approximate so we can simply use racey valid ref reads to be able + * to sample while others are writing. + */ +int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_buddy_indirect *ind; + struct buffer_head *bh; + int ret; + int i; + + *bfree = 0; + + bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + goto out; + } + ind = bh_data(bh); + + for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) + *bfree += le64_to_cpu(ind->order_totals[i]) << i; + + scoutfs_block_put(bh); + ret = 0; +out: + return ret; + +} diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 7f7df982..5c6c1d85 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -8,5 +8,6 @@ int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); +int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree); #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index fbe60040..82fa436b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -72,6 +72,7 @@ struct scoutfs_buddy_block { struct scoutfs_buddy_indirect { struct scoutfs_block_header hdr; + __le64 order_totals[SCOUTFS_BUDDY_ORDERS]; struct scoutfs_buddy_slot { __u8 free_orders; struct scoutfs_block_ref ref; @@ -79,7 +80,7 @@ struct scoutfs_buddy_indirect { } __packed; #define SCOUTFS_BUDDY_SLOTS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) / \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_indirect)) / \ sizeof(struct scoutfs_buddy_slot)) /* diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 9750024f..66681210 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -263,6 +263,22 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } +/* + * A quick atomic sample of the last inode number that's been allocated. + */ +u64 scoutfs_last_ino(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + u64 last; + + spin_lock(&sbi->next_ino_lock); + last = le64_to_cpu(super->next_ino); + spin_unlock(&sbi->next_ino_lock); + + return last; +} + static int alloc_ino(struct super_block *sb, u64 *ino) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 563d08f5..fab38b3f 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -30,6 +30,8 @@ void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); +u64 scoutfs_last_ino(struct super_block *sb); + void scoutfs_inode_exit(void); int scoutfs_inode_init(void); diff --git a/kmod/src/super.c b/kmod/src/super.c index baa384d0..252a27ca 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -27,14 +28,55 @@ #include "block.h" #include "counters.h" #include "trans.h" +#include "buddy.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; +/* + * We fake the number of free inodes value by assuming that we can fill + * free blocks with a certain number of inodes. We then the number of + * current inodes to that free count to determine the total possible + * inodes. + * + * The fsid that we report is constructed from the xor of the first two + * and second two little endian u32s that make up the uuid bytes. + */ +static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) +{ + struct super_block *sb = dentry->d_inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + __le32 * __packed uuid = (void *)super->uuid; + int ret; + + ret = scoutfs_buddy_bfree(sb, &kst->f_bfree); + if (ret) + return ret; + + kst->f_type = SCOUTFS_SUPER_MAGIC; + kst->f_bsize = SCOUTFS_BLOCK_SIZE; + kst->f_blocks = le64_to_cpu(super->total_blocks); + kst->f_bavail = kst->f_bfree; + + kst->f_ffree = kst->f_bfree * 17; + kst->f_files = kst->f_ffree + scoutfs_last_ino(sb); + + /* this fsid is constant.. the uuid is different */ + kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); + kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); + kst->f_namelen = SCOUTFS_NAME_LEN; + kst->f_frsize = SCOUTFS_BLOCK_SIZE; + /* the vfs fills f_flags */ + + return 0; +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, .sync_fs = scoutfs_sync_fs, + .statfs = scoutfs_statfs, }; /* From df930739718562130e74444e82103542bf04ad92 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Aug 2016 16:51:47 -0700 Subject: [PATCH 082/920] scoutfs: don't unlock err bh after validation If block validation failed then we'd end up trying to unlock an IS_ERR buffer_head pointer. Fix it so that we drop the ref and set the pointer after unlocking. Signed-off-by: Zach Brown --- kmod/src/block.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 4f82eae9..faf9ae33 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -194,14 +194,16 @@ struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno) lock_buffer(bh); if (!buffer_scoutfs_verified(bh)) { ret = verify_block_header(sbi, bh); - if (ret < 0) { - scoutfs_block_put(bh); - bh = ERR_PTR(ret); - } else { + if (!ret) set_buffer_scoutfs_verified(bh); - } + } else { + ret = 0; } unlock_buffer(bh); + if (ret < 0) { + scoutfs_block_put(bh); + bh = ERR_PTR(ret); + } } out: From 64b82e1ac3bd859a8222610fc0a944fce4b1a3dd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Aug 2016 10:21:27 -0700 Subject: [PATCH 083/920] scoutfs: add symlink support Symlinks are easily implemented by storing the target path in btree items. Signed-off-by: Zach Brown --- kmod/src/dir.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/dir.h | 1 + kmod/src/format.h | 6 +- kmod/src/inode.c | 2 +- 4 files changed, 172 insertions(+), 2 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 79c0310c..02b02f4f 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -560,6 +560,170 @@ out: return ret; } +/* + * Full a buffer with the null terminated symlink, point nd at it, and + * return it so put_link can free it once the vfs is done. + * + * We chose to pay the runtime cost of per-call allocation and copy + * overhead instead of wiring up symlinks to the page cache, storing + * each small link in a full page, and later having to reclaim them. + */ +static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + loff_t size = i_size_read(inode); + struct scoutfs_key first; + struct scoutfs_key last; + char *path; + int off; + int ret; + int k; + + /* XXX corruption */ + if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) + return ERR_PTR(-EIO); + + /* unlikely, but possible I suppose */ + if (size > PATH_MAX) + return ERR_PTR(-ENAMETOOLONG); + + path = kmalloc(size, GFP_NOFS); + if (!path) + return ERR_PTR(-ENOMEM); + + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, 0); + scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, ~0ULL); + + off = 0; + k = 0; + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + if (scoutfs_key_offset(curs.key) != k || + off + curs.val_len > size) { + /* XXX corruption */ + scoutfs_btree_release(&curs); + ret = -EIO; + break; + } + + memcpy(path + off, curs.val, curs.val_len); + + off += curs.val_len; + k++; + } + + /* XXX corruption */ + if (ret == 0 && (off != size || path[off - 1] != '\0')) + ret = -EIO; + + if (ret) { + kfree(path); + path = ERR_PTR(ret); + } + + return path; +} + +static void scoutfs_put_link(struct dentry *dentry, struct nameidata *nd, + void *cookie) +{ + if (!IS_ERR_OR_NULL(cookie)) + kfree(cookie); +} + +const struct inode_operations scoutfs_symlink_iops = { + .readlink = generic_readlink, + .follow_link = scoutfs_follow_link, + .put_link = scoutfs_put_link, + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, +}; + +/* + * Symlink target paths can be annoyingly huge. We don't want large + * items gumming up the btree so we store relatively rare large paths in + * multiple items. + */ +static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, + const char *symname) +{ + struct super_block *sb = dir->i_sb; + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct inode *inode = NULL; + struct scoutfs_key key; + struct dentry_info *di; + const int name_len = strlen(symname) + 1; + int off; + int bytes; + int ret; + int k = 0; + + /* path_max includes null as does our value for nd_set_link */ + if (name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) + return -ENAMETOOLONG; + + di = alloc_dentry_info(dentry); + if (IS_ERR(di)) + return PTR_ERR(di); + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + for (k = 0, off = 0; off < name_len; off += bytes, k++) { + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, + k); + bytes = min(name_len, SCOUTFS_MAX_ITEM_LEN); + + ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + if (ret) + goto out; + + memcpy(curs.val, symname + off, bytes); + scoutfs_btree_release(&curs); + } + + ret = add_entry_items(dir, dentry, inode); + if (ret) + goto out; + + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + + inode->i_ctime = dir->i_mtime; + i_size_write(inode, name_len); + + scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(dir); + + insert_inode_hash(inode); + /* XXX need to set i_op/fop before here for sec callbacks */ + d_instantiate(dentry, inode); +out: + if (ret < 0) { + if (!IS_ERR_OR_NULL(inode)) + iput(inode); + + while (k--) { + scoutfs_set_key(&key, scoutfs_ino(inode), + SCOUTFS_SYMLINK_KEY, k); + scoutfs_btree_delete(sb, &key); + } + } + + scoutfs_release_trans(sb); + return ret; +} + /* * Add an allocated path component to the callers list which links to * the target inode at a counter past the given counter. @@ -770,6 +934,7 @@ const struct inode_operations scoutfs_dir_iops = { .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, .removexattr = scoutfs_removexattr, + .symlink = scoutfs_symlink, }; void scoutfs_dir_exit(void) diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 44c715c5..c550ed00 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -5,6 +5,7 @@ extern const struct file_operations scoutfs_dir_fops; extern const struct inode_operations scoutfs_dir_iops; +extern const struct inode_operations scoutfs_symlink_iops; int scoutfs_dir_init(void); void scoutfs_dir_exit(void); diff --git a/kmod/src/format.h b/kmod/src/format.h index 82fa436b..d2e542d9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -104,7 +104,8 @@ struct scoutfs_key { #define SCOUTFS_XATTR_VAL_HASH_KEY 4 #define SCOUTFS_DIRENT_KEY 5 #define SCOUTFS_LINK_BACKREF_KEY 6 -#define SCOUTFS_BMAP_KEY 7 +#define SCOUTFS_SYMLINK_KEY 7 +#define SCOUTFS_BMAP_KEY 8 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -191,6 +192,9 @@ struct scoutfs_inode { #define SCOUTFS_ROOT_INO 1 +/* like the block size, a reasonable min PATH_MAX across platforms */ +#define SCOUTFS_SYMLINK_MAX_SIZE 4096 + /* * Dirents are stored in items with an offset of the hash of their name. * Colliding names are packed into the value. diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 66681210..75060fc9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -91,7 +91,7 @@ static void set_inode_ops(struct inode *inode) inode->i_fop = &scoutfs_dir_fops; break; case S_IFLNK: -// inode->i_op = &scoutfs_symlink_iops; + inode->i_op = &scoutfs_symlink_iops; break; default: // inode->i_op = &scoutfs_special_iops; From 06c718e16abe0c743f78516d5206393519e2826c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 31 Aug 2016 09:31:23 -0700 Subject: [PATCH 084/920] scoutfs: remove unlinked inode items Wire up the inode callbacks that let us remove all the persistent items associated with an unlinked inode as its final reference is dropped. This is the first part of full truncate and orphan inode support. Signed-off-by: Zach Brown --- kmod/src/dir.c | 29 +++++++++++++++ kmod/src/dir.h | 2 ++ kmod/src/filerw.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/filerw.h | 1 + kmod/src/inode.c | 79 +++++++++++++++++++++++++++++++++++++++++ kmod/src/inode.h | 2 ++ kmod/src/super.c | 2 ++ kmod/src/xattr.c | 67 +++++++++++++++++++++++++++++++++++ kmod/src/xattr.h | 2 ++ 9 files changed, 273 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 02b02f4f..a605fcfa 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -724,6 +724,35 @@ out: return ret; } +/* + * Delete all the symlink items. There should only ever be a handful of + * these that contain the target path of the symlink. + */ +int scoutfs_symlink_drop(struct super_block *sb, u64 ino) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_key first; + struct scoutfs_key last; + struct scoutfs_key key; + int ret; + + scoutfs_set_key(&first, ino, SCOUTFS_SYMLINK_KEY, 0); + scoutfs_set_key(&last, ino, SCOUTFS_SYMLINK_KEY, ~0ULL); + + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + key = *curs.key; + first = *curs.key; + scoutfs_inc_key(&first); + scoutfs_btree_release(&curs); + + ret = scoutfs_btree_delete(sb, &key); + if (ret) + break; + } + + return ret; +} + /* * Add an allocated path component to the callers list which links to * the target inode at a counter past the given counter. diff --git a/kmod/src/dir.h b/kmod/src/dir.h index c550ed00..07edc195 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -20,4 +20,6 @@ int scoutfs_dir_next_path(struct super_block *sb, u64 ino, u64 *ctr, struct list_head *list); void scoutfs_dir_free_path(struct list_head *list); +int scoutfs_symlink_drop(struct super_block *sb, u64 ino); + #endif diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 7ef5ea2a..e42a485f 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -177,6 +177,95 @@ static void return_file_block(struct super_block *sb, u64 blkno) spin_unlock(&sbi->file_alloc_lock); } +static bool bmap_has_blocks(struct scoutfs_block_map *bmap) +{ + int i; + + for (i = 0; i < SCOUTFS_BLOCK_MAP_COUNT; i++) { + if (bmap->blkno[i]) + return true; + } + + return false; +} + +/* + * Free mapped blocks whose entire contents are past the new specified + * size. The caller holds a transaction. If we truncate all the blocks + * in a mapping item then we remove the item. + * + * This is the low level block allocation and bmap item manipulation. + * Callers manage higher order truncation and orphan cleanup. + * + * XXX what to do about leaving items past i_size? + * XXX probably should be a range + */ +int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_block_map *bmap; + struct scoutfs_key first; + struct scoutfs_key last; + struct scoutfs_key key; + bool delete; + u64 iblock; + u64 blkno; + int ret; + int i; + + iblock = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); + i = iblock & SCOUTFS_BLOCK_MAP_MASK; + + scoutfs_set_key(&first, ino, SCOUTFS_BMAP_KEY, + iblock & ~(u64)SCOUTFS_BLOCK_MAP_MASK); + scoutfs_set_key(&last, ino, SCOUTFS_BMAP_KEY, ~0ULL); + + trace_printk("iblock %llu i %d\n", iblock, i); + + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + key = *curs.key; + first = *curs.key; + scoutfs_inc_key(&first); + scoutfs_btree_release(&curs); + + ret = scoutfs_btree_update(sb, &key, &curs); + if (ret) + break; + + /* XXX check sanity */ + bmap = curs.val; + + for (; i < SCOUTFS_BLOCK_MAP_COUNT; i++) { + blkno = le64_to_cpu(bmap->blkno[i]); + if (blkno == 0) + continue; + + ret = scoutfs_buddy_free(sb, blkno, 0); + if (ret) + break; + + bmap->blkno[i] = 0; + } + delete = !bmap_has_blocks(bmap); + + scoutfs_btree_release(&curs); + if (ret) + break; + + i = 0; + + if (delete) { + ret = scoutfs_btree_delete(sb, &key); + if (ret) + break; + } + + /* XXX sync transaction if it's enormous */ + } + + return ret; +} + /* * The caller ensures that this is serialized against all other callers * and writers. diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h index c182349d..ba2bb81f 100644 --- a/kmod/src/filerw.h +++ b/kmod/src/filerw.h @@ -5,5 +5,6 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; void scoutfs_filerw_free_alloc(struct super_block *sb); +int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size); #endif diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 75060fc9..93d68202 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -25,6 +26,7 @@ #include "filerw.h" #include "scoutfs_trace.h" #include "xattr.h" +#include "trans.h" /* * XXX @@ -347,6 +349,83 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, return inode; } +/* + * Remove all the items associated with a given inode. + */ +static void drop_inode_items(struct super_block *sb, u64 ino) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_inode *sinode; + struct scoutfs_key key; + bool release = false; + umode_t mode; + int ret; + + /* sample the inode mode */ + scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); + ret = scoutfs_btree_lookup(sb, &key, &curs); + if (ret) + goto out; + + sinode = curs.val; + mode = le32_to_cpu(sinode->mode); + scoutfs_btree_release(&curs); + + ret = scoutfs_hold_trans(sb); + if (ret) + goto out; + release = true; + + ret = scoutfs_xattr_drop(sb, ino); + if (ret) + goto out; + + if (S_ISLNK(mode)) + ret = scoutfs_symlink_drop(sb, ino); + else if (S_ISREG(mode)) + ret = scoutfs_truncate_block_items(sb, ino, 0); + if (ret) + goto out; + + ret = scoutfs_btree_delete(sb, &key); +out: + if (ret) + trace_printk("drop items failed ret %d ino %llu\n", ret, ino); + if (release) + scoutfs_release_trans(sb); +} + +/* + * iput_final has already written out the dirty pages to the inode + * before we get here. We're left with a clean inode that we have to + * tear down. If there are no more links to the inode then we also + * remove all its persistent structures. + */ +void scoutfs_evict_inode(struct inode *inode) +{ + trace_printk("ino %llu nlink %d bad %d\n", + scoutfs_ino(inode), inode->i_nlink, is_bad_inode(inode)); + + if (is_bad_inode(inode)) + goto clear; + + truncate_inode_pages_final(&inode->i_data); + + if (inode->i_nlink == 0) + drop_inode_items(inode->i_sb, scoutfs_ino(inode)); +clear: + clear_inode(inode); +} + +int scoutfs_drop_inode(struct inode *inode) +{ + int ret = generic_drop_inode(inode); + + trace_printk("ret %d nlink %d unhashed %d\n", + ret, inode->i_nlink, inode_unhashed(inode)); + return ret; +} + void scoutfs_inode_exit(void) { if (scoutfs_inode_cachep) { diff --git a/kmod/src/inode.h b/kmod/src/inode.h index fab38b3f..e02acf27 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -23,6 +23,8 @@ static inline u64 scoutfs_ino(struct inode *inode) struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); +int scoutfs_drop_inode(struct inode *inode); +void scoutfs_evict_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); diff --git a/kmod/src/super.c b/kmod/src/super.c index 252a27ca..605b5ef0 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -74,6 +74,8 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, + .drop_inode = scoutfs_drop_inode, + .evict_inode = scoutfs_evict_inode, .destroy_inode = scoutfs_destroy_inode, .sync_fs = scoutfs_sync_fs, .statfs = scoutfs_statfs, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 95def5ae..82d92961 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -487,3 +487,70 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) return ret < 0 ? ret : total; } + +/* + * Delete all the xattr items associted with this inode. The caller + * holds a transaction. + * + * The name and value hashes are sorted by the hash value instead of the + * inode so we have to use the inode's xattr items to find them. We + * only remove the xattr item once the hash items are removed. + * + * Hash items can be shared amongst xattrs whose names or values hash to + * the same hash value. We don't bother trying to remove the hash items + * as the last xattr is removed. We remove it the first chance we get, + * try to avoid obviously removing the same hash item next, and allow + * failure when we try to remove a hash item that wasn't found. + */ +int scoutfs_xattr_drop(struct super_block *sb, u64 ino) +{ + DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_xattr *xat; + struct scoutfs_key first; + struct scoutfs_key last; + struct scoutfs_key key; + struct scoutfs_key name_key; + struct scoutfs_key val_key; + __le64 last_name; + __le64 last_val; + u64 val_hash; + bool have_last; + int ret; + + scoutfs_set_key(&first, ino, SCOUTFS_XATTR_KEY, 0); + scoutfs_set_key(&last, ino, SCOUTFS_XATTR_KEY, ~0ULL); + + have_last = false; + while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + xat = curs.val; + key = *curs.key; + val_hash = scoutfs_name_hash(xat_value(xat), xat->value_len); + set_name_val_keys(&name_key, &val_key, &key, val_hash); + + first = *curs.key; + scoutfs_inc_key(&first); + scoutfs_btree_release(&curs); + + if (!have_last || last_name != name_key.inode) { + ret = scoutfs_btree_delete(sb, &name_key); + if (ret && ret != -ENOENT) + break; + last_name = name_key.inode; + } + + if (!have_last || last_val != val_key.inode) { + ret = scoutfs_btree_delete(sb, &val_key); + if (ret && ret != -ENOENT) + break; + last_val = val_key.inode; + } + + have_last = true; + + ret = scoutfs_btree_delete(sb, &key); + if (ret && ret != -ENOENT) + break; + } + + return ret; +} diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index 7abb00c3..e0fadf32 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -8,4 +8,6 @@ int scoutfs_setxattr(struct dentry *dentry, const char *name, int scoutfs_removexattr(struct dentry *dentry, const char *name); ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +int scoutfs_xattr_drop(struct super_block *sb, u64 ino); + #endif From b2e12a9f279f372f67ace8d64739c979b3f3638b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Sep 2016 15:13:21 -0700 Subject: [PATCH 085/920] scoutfs: sync large transactions as released We don't want very large transactions to build up and create huge commit latencies. All blocks are written to free space so we use a count of allocations to count dirty blocks. We arbitrarily limit the transaction to 128MB and try to kick off commits when we release transactions that have gotten that big. Signed-off-by: Zach Brown --- kmod/src/buddy.c | 28 ++++++++++++++++++++++++++++ kmod/src/buddy.h | 3 +++ kmod/src/format.h | 2 ++ kmod/src/super.c | 1 + kmod/src/super.h | 1 + kmod/src/trans.c | 17 +++++++++++++++-- 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 48b85fca..bc4f843c 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -580,6 +580,7 @@ static int buddy_alloc(struct super_block *sb, u64 *blkno, int order) static int alloc_region(struct super_block *sb, u64 *blkno, int order, u64 existing, int region) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); int ret; switch(region) { @@ -593,8 +594,15 @@ static int alloc_region(struct super_block *sb, u64 *blkno, int order, case REGION_BUDDY: ret = buddy_alloc(sb, blkno, order); break; + default: + WARN_ON_ONCE(1); + ret = -EINVAL; } + /* this misses other direct calls to bitmap_alloc, but that's minor */ + if (ret >= 0) + atomic_add(1 << ret, &sbi->buddy_count); + trace_scoutfs_buddy_alloc(*blkno, order, region, ret); return ret; } @@ -848,3 +856,23 @@ out: return ret; } + +/* + * Return the number of block allocations since the last time the + * counter was reset. This count doesn't include some internal bitmap + * block allocations but that should be a small fraction of the main + * allocations. + */ +unsigned int scoutfs_buddy_alloc_count(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return atomic_read(&sbi->buddy_count); +} + +void scoutfs_buddy_reset_count(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return atomic_set(&sbi->buddy_count, 0); +} diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 5c6c1d85..059d586a 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -10,4 +10,7 @@ void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree); +unsigned int scoutfs_buddy_alloc_count(struct super_block *sb); +void scoutfs_buddy_reset_count(struct super_block *sb); + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index d2e542d9..007e0d31 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -22,6 +22,8 @@ #define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) #define SCOUTFS_BUDDY_BM_NR 2 +#define SCOUTFS_MAX_TRANS_BLOCKS (128 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) + /* * This header is found at the start of every block so that we can * verify that it's what we were looking for. The crc and padding diff --git a/kmod/src/super.c b/kmod/src/super.c index 605b5ef0..b0f9e218 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -198,6 +198,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); mutex_init(&sbi->buddy_mutex); + atomic_set(&sbi->buddy_count, 0); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); diff --git a/kmod/src/super.h b/kmod/src/super.h index 455f397e..4e7d8723 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -25,6 +25,7 @@ struct scoutfs_sb_info { int block_write_err; struct mutex buddy_mutex; + atomic_t buddy_count; /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 9be98ede..e2679dd7 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -97,8 +97,10 @@ void scoutfs_trans_write_func(struct work_struct *work) } spin_lock(&sbi->trans_write_lock); - if (advance) + if (advance) { scoutfs_advance_dirty_super(sb); + scoutfs_buddy_reset_count(sb); + } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock); @@ -172,12 +174,23 @@ int scoutfs_hold_trans(struct super_block *sb) atomic_add_unless(&sbi->trans_holds, 1, -1)); } +/* + * As we release we ask the allocator how many blocks have been + * allocated since the last transaction was successfully committed. If + * it's large enough we kick off a write. This is mostly to reduce the + * commit latency. We also don't want to let the IO pipeline sit idle. + * Once we have enough blocks to write efficiently we should do so. + */ void scoutfs_release_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - if (atomic_sub_return(1, &sbi->trans_holds) == 0) + if (atomic_sub_return(1, &sbi->trans_holds) == 0) { + if (scoutfs_buddy_alloc_count(sb) >= SCOUTFS_MAX_TRANS_BLOCKS) + scoutfs_sync_fs(sb, 0); + wake_up(&sbi->trans_hold_wq); + } } int scoutfs_setup_trans(struct super_block *sb) From 04e0df4f3658aff0de4ed4faa3e64497bc049cfe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 13:38:58 -0700 Subject: [PATCH 086/920] scoutfs: forgot to initialize file alloc lock Thank goodness for lockdep! Signed-off-by: Zach Brown --- kmod/src/super.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index b0f9e218..6c4a4d51 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -205,6 +205,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); + spin_lock_init(&sbi->file_alloc_lock); if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { printk(KERN_ERR "couldn't set blocksize\n"); From 5375ed5f388f5d93a124ebfe9e1545094aec557d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 13:47:19 -0700 Subject: [PATCH 087/920] scoutfs: fill nameidata with symlink path Our follow_link method forgot to fill the nameidata with the target path of the symlink. The uninitialized nameidata tripped up the generic readlink code in a debugging kernel. Signed-off-by: Zach Brown --- kmod/src/dir.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index a605fcfa..0e0b1ab4 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "format.h" #include "dir.h" @@ -620,6 +621,8 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (ret) { kfree(path); path = ERR_PTR(ret); + } else { + nd_set_link(nd, path); } return path; From 164bcb5d99c1021e5f27c81c20cbbc2cdb79e2eb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 14:28:22 -0700 Subject: [PATCH 088/920] scoutfs: bug if btree item creation corrupts Add a BUG_ON() assertion for the case where we create an item that starts in the item offset array. This happens if the callers free space calculations are incorrect. It shouldn't be triggerable by corrupt blocks if we're verifying the blocks as we read them in. Signed-off-by: Zach Brown --- kmod/src/btree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index d3a1c9d0..db9bfe18 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -181,6 +181,9 @@ static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, bt->item_offs[pos] = bt->free_end; bt->nr_items++; + BUG_ON(le16_to_cpu(bt->free_end) < + offsetof(struct scoutfs_btree_block, item_offs[bt->nr_items])); + item = pos_item(bt, pos); item->key = *key; item->seq = bt->hdr.seq; From b55da5ecb7a623c7727fd29b888be644f147a9f6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 14:36:35 -0700 Subject: [PATCH 089/920] scoutfs: compact btree more carefully when merging The btree block merging code knew to try and compact the destination block if it was going to move more bytes worth of items than there was contiguous free space in the destination block. But it missed the case where item movement moves more than the hint because the last item it moves was big. In the worst case this creates an item which overlaps the item offsets and ends up looking like corrupt items. Signed-off-by: Zach Brown --- kmod/src/btree.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index db9bfe18..babe6b57 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -599,7 +599,14 @@ static struct buffer_head *try_merge(struct super_block *sb, else to_move = reclaimable_free(bt) - SCOUTFS_BTREE_FREE_LIMIT; - if (contig_free(bt) < to_move) + /* + * Make sure there's room to move a max size item if it's the + * next in line when we only have one byte left to try and move. + * + * XXX This is getting awfully fiddly. Should we be refactoring + * item insertion/deletion to do this for us? + */ + if (contig_free(bt) < (to_move + (SCOUTFS_MAX_ITEM_LEN - 1))) compact_items(bt); trace_printk("sib_pos %d move_right %u to_move %u\n", From f44306757c62b3bf4eedb7df8ceb4a1066af761b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 14:38:45 -0700 Subject: [PATCH 090/920] scoutfs: add btree deletion trace message Add a simple trace message with the result of item deletion calls. Signed-off-by: Zach Brown --- kmod/src/btree.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index babe6b57..618e88e7 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -963,8 +963,10 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) int ret; bh = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); - if (IS_ERR(bh)) - return PTR_ERR(bh); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + goto out; + } bt = bh_data(bh); pos = find_pos(bt, key, &cmp); @@ -991,6 +993,8 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) unlock_block(NULL, bh, true); scoutfs_block_put(bh); +out: + trace_printk("key "CKF" ret %d\n", CKA(key), ret); return ret; } From 49c3d5ed34dc49bf39e90a0bf663a2657c8d7767 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 14:49:37 -0700 Subject: [PATCH 091/920] scoutfs: add btree block verification Add a function to verify that a btree block is valid. It's disabled for now because it's expensive. Signed-off-by: Zach Brown --- kmod/src/btree.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/format.h | 8 ++++ 2 files changed, 112 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 618e88e7..cf58f5f2 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -741,6 +741,92 @@ static unsigned int find_pos_after_seq(struct scoutfs_btree_block *bt, return pos; } +/* + * Verify that the btree block isn't corrupt. This is way too expensive + * to do for each block access though that's very helpful for debugging + * btree block corruption. + * + * It should be done the first time we read blocks and it doing it for + * every block access should be hidden behind runtime options. + * + * XXX + * - make sure items don't overlap + * - make sure offs point to live items + * - do things with level + * - see if item keys make sense + */ +static int verify_btree_block(struct scoutfs_btree_block *bt, int level, + struct scoutfs_key *small, + struct scoutfs_key *large) +{ + struct scoutfs_btree_item *item; + struct scoutfs_key *prev; + unsigned int bytes = 0; + unsigned int after_offs = sizeof(struct scoutfs_btree_block); + unsigned int first_off; + unsigned int off; + unsigned int nr; + unsigned int i = 0; + int bad = 1; + + nr = bt->nr_items; + if (nr == 0) + goto out; + + if (nr > SCOUTFS_BTREE_MAX_ITEMS) { + nr = SCOUTFS_BTREE_MAX_ITEMS; + goto out; + } + + after_offs = offsetof(struct scoutfs_btree_block, item_offs[nr]); + first_off = SCOUTFS_BLOCK_SIZE; + + for (i = 0; i < nr; i++) { + + off = le16_to_cpu(bt->item_offs[i]); + if (off >= SCOUTFS_BLOCK_SIZE || off < after_offs) + goto out; + + first_off = min(first_off, off); + + item = pos_item(bt, i); + bytes += item_bytes(item); + + if ((i == 0 && scoutfs_key_cmp(&item->key, small) < 0) || + (i > 0 && scoutfs_key_cmp(&item->key, prev) <= 0) || + (i == (nr - 1) && scoutfs_key_cmp(&item->key, large) > 0)) + goto out; + + prev = &item->key; + } + + if (first_off < le16_to_cpu(bt->free_end)) + goto out; + + if ((le16_to_cpu(bt->free_end) + bytes + + le16_to_cpu(bt->free_reclaim)) != SCOUTFS_BLOCK_SIZE) + goto out; + + bad = 0; +out: + if (bad) { + printk("bt %p small "CKF" large "CKF" end %u reclaim %u nr %u (max %lu after %u bytes %u)\n", + bt, CKA(small), CKA(large), le16_to_cpu(bt->free_end), + le16_to_cpu(bt->free_reclaim), bt->nr_items, + SCOUTFS_BTREE_MAX_ITEMS, after_offs, bytes); + for (i = 0; i < nr; i++) { + item = pos_item(bt, i); + off = le16_to_cpu(bt->item_offs[i]); + printk(" [%u] off %u key "CKF" len %u\n", + i, off, CKA(&item->key), + le16_to_cpu(item->val_len)); + } + BUG_ON(bad); + } + + return 0; +} + /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their @@ -763,10 +849,13 @@ static struct buffer_head *btree_walk(struct super_block *sb, struct buffer_head *bh = NULL; struct scoutfs_btree_item *item = NULL; struct scoutfs_block_ref *ref; + struct scoutfs_key small; + struct scoutfs_key large; unsigned int level; unsigned int pos = 0; const bool dirty = op == WALK_INSERT || op == WALK_DELETE || op == WALK_DIRTY; + int ret; /* no sibling blocks if we don't have parent blocks */ if (next_key) @@ -798,12 +887,23 @@ static struct buffer_head *btree_walk(struct super_block *sb, return ERR_PTR(-ENOENT); } + scoutfs_set_key(&small, 0, 0, 0); + scoutfs_set_key(&large, ~0ULL, ~0, ~0ULL); + while (level--) { /* XXX hmm, need to think about retry */ bh = get_block_ref(sb, ref, dirty); if (IS_ERR(bh)) break; + /* XXX enable this */ + ret = 0 && verify_btree_block(bh_data(bh), level, &small, &large); + if (ret) { + scoutfs_block_put(bh); + bh = ERR_PTR(ret); + break; + } + if (op == WALK_INSERT) bh = try_split(sb, root, level, key, val_len, parent, pos, bh); @@ -852,6 +952,10 @@ static struct buffer_head *btree_walk(struct super_block *sb, *next_key = item->key; scoutfs_inc_key(next_key); } + + if (pos) + small = pos_item(parent, pos - 1)->key; + large = item->key; } unlock_block(sbi, par_bh, dirty); diff --git a/kmod/src/format.h b/kmod/src/format.h index 007e0d31..932a9d2b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -148,6 +148,14 @@ struct scoutfs_btree_item { #define SCOUTFS_BTREE_FREE_LIMIT \ ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) +/* XXX does this exist upstream somewhere? */ +#define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) + +#define SCOUTFS_BTREE_MAX_ITEMS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / \ + (member_sizeof(struct scoutfs_btree_block, item_offs[0]) + \ + sizeof(struct scoutfs_btree_item))) + #define SCOUTFS_UUID_BYTES 16 struct scoutfs_super_block { From 1dd4a14d04e8187e421489216639e22680d8b743 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 15:36:25 -0700 Subject: [PATCH 092/920] scoutfs: don't dereference IS_ERR buffer_head The check for aligned buffer head data pointers was trying to dereference a bad IS_ERR pointer when allocation of a new block failed with ENOSPC. Signed-off-by: Zach Brown --- kmod/src/btree.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index cf58f5f2..e29070da 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -353,11 +353,13 @@ static void compact_items(struct scoutfs_btree_block *bt) /* sorting relies on masking pointers to find the containing block */ static inline struct buffer_head *check_bh_alignment(struct buffer_head *bh) { - struct scoutfs_btree_block *bt = bh_data(bh); + if (!IS_ERR_OR_NULL(bh)) { + struct scoutfs_btree_block *bt = bh_data(bh); - if (!IS_ERR_OR_NULL(bh) && WARN_ON_ONCE(aligned_bt(bt) != bt)) { - scoutfs_block_put(bh); - return ERR_PTR(-EIO); + if (WARN_ON_ONCE(aligned_bt(bt) != bt)) { + scoutfs_block_put(bh); + return ERR_PTR(-EIO); + } } return bh; From 3bb0c8068629b456eb8d9f979dbd85be1bddcfe2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Sep 2016 16:52:45 -0700 Subject: [PATCH 093/920] scoutfs: fix buddy stable bit test The buddy allocator had the test for non-existant stable bitmap blocks backwards. An uninitialized block implies that all the bits are marked free and we don't need to test that the specific bits are free. Signed-off-by: Zach Brown --- kmod/src/buddy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index bc4f843c..96f3048f 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -388,7 +388,7 @@ static int find_first_fit(struct scoutfs_super_block *super, int sl, made_progress = true; /* advance to next bit if it's not free in stable */ - if (!st_bud || + if (st_bud && !test_buddy_bit_or_higher(st_bud, i, nr)) { nrs[i] = nr + 1; continue; From d2a696f4bd8edbf547181e9826258bbb4195e526 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 14 Sep 2016 14:33:38 -0700 Subject: [PATCH 094/920] scoutfs: add zero key set and test functions Add some quick functions to set a key to all zeros and to test if a key is all zeros. Signed-off-by: Zach Brown --- kmod/src/key.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kmod/src/key.h b/kmod/src/key.h index 55b557f4..dac0871b 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -100,4 +100,16 @@ static inline struct scoutfs_key *scoutfs_max_key(struct scoutfs_key *a, return scoutfs_key_cmp(a, b) > 0 ? a : b; } +static inline bool scoutfs_key_is_zero(struct scoutfs_key *key) +{ + return key->inode == 0 && key->type == 0 && key->offset == 0; +} + +static inline void scoutfs_key_set_zero(struct scoutfs_key *key) +{ + key->inode = 0; + key->type = 0; + key->offset = 0; +} + #endif From 2bed78c269c56ca4f74acc5988d22f5fdc231b10 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 14 Sep 2016 15:14:19 -0700 Subject: [PATCH 095/920] scoutfs: specify btree root The btree functions currently don't take a specific root argument. They assume, deep down in btree_walk, that there's only one btree in the system. We're going to be adding a few more to support richer allocation. To prepare for this we have the btree functions take an explicit btree argument. This should make no functional difference. Signed-off-by: Zach Brown --- kmod/src/btree.c | 67 +++++++++++++++++++++++++---------------------- kmod/src/btree.h | 37 +++++++++++++++++--------- kmod/src/dir.c | 43 ++++++++++++++++++------------ kmod/src/filerw.c | 18 ++++++++----- kmod/src/inode.c | 17 +++++++----- kmod/src/ioctl.c | 7 +++-- kmod/src/super.h | 6 +++++ kmod/src/xattr.c | 47 +++++++++++++++++++-------------- 8 files changed, 146 insertions(+), 96 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index e29070da..90493967 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -840,13 +840,13 @@ out: * the next block when they're done with the block this returns. */ static struct buffer_head *btree_walk(struct super_block *sb, + struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_key *next_key, unsigned int val_len, u64 seq, int op) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_block *parent = NULL; - struct scoutfs_btree_root *root; struct buffer_head *par_bh = NULL; struct buffer_head *bh = NULL; struct scoutfs_btree_item *item = NULL; @@ -865,8 +865,6 @@ static struct buffer_head *btree_walk(struct super_block *sb, lock_block(sbi, par_bh, dirty); - /* XXX one for now */ - root = &sbi->super.btree_root; ref = &root->ref; level = root->height; @@ -986,7 +984,9 @@ static void set_cursor(struct scoutfs_btree_cursor *curs, * Point the caller's cursor at the item if it's found. It can't be * modified. -ENOENT is returned if the key isn't found in the tree. */ -int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_block *bt; @@ -997,7 +997,7 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, BUG_ON(curs->bh); - bh = btree_walk(sb, key, NULL, 0, 0, 0); + bh = btree_walk(sb, root, key, NULL, 0, 0, 0); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1023,8 +1023,9 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, * * XXX this walks the treap twice, which isn't great */ -int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, - unsigned int val_len, +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_block *bt; @@ -1035,7 +1036,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, BUG_ON(curs->bh); - bh = btree_walk(sb, key, NULL, val_len, 0, WALK_INSERT); + bh = btree_walk(sb, root, key, NULL, val_len, 0, WALK_INSERT); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1058,17 +1059,17 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, * Delete an item from the tree. -ENOENT is returned if the key isn't * found. */ -int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_btree_root *root; struct scoutfs_btree_block *bt; struct buffer_head *bh; int pos; int cmp; int ret; - bh = btree_walk(sb, key, NULL, 0, 0, WALK_DELETE); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DELETE); if (IS_ERR(bh)) { ret = PTR_ERR(bh); goto out; @@ -1084,8 +1085,6 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) /* delete the final block in the tree */ if (bt->nr_items == 0) { - root = &sbi->super.btree_root; - root->height = 0; root->ref.blkno = 0; root->ref.seq = 0; @@ -1119,9 +1118,9 @@ out: * * Returns > 0 when the cursor has an item, 0 when done, and -errno on error. */ -static int btree_next(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, u64 seq, int op, - struct scoutfs_btree_cursor *curs) +static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, + u64 seq, int op, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_block *bt; struct buffer_head *bh; @@ -1148,7 +1147,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, /* find the leaf that contains the next item after the key */ while (!curs->bh && scoutfs_key_cmp(&key, last) <= 0) { - bh = btree_walk(sb, &key, &next_key, 0, seq, op); + bh = btree_walk(sb, root, &key, &next_key, 0, seq, op); /* next seq walks can terminate in parents with old seqs */ if (op == WALK_NEXT_SEQ && bh == ERR_PTR(-ENOENT)) { @@ -1187,18 +1186,19 @@ static int btree_next(struct super_block *sb, struct scoutfs_key *first, return ret; } -int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_btree_cursor *curs) { - return btree_next(sb, first, last, 0, WALK_NEXT, curs); + return btree_next(sb, root, first, last, 0, WALK_NEXT, curs); } -int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, u64 seq, - struct scoutfs_btree_cursor *curs) +int scoutfs_btree_since(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, + u64 seq, struct scoutfs_btree_cursor *curs) { - return btree_next(sb, first, last, seq, WALK_NEXT_SEQ, curs); + return btree_next(sb, root, first, last, seq, WALK_NEXT_SEQ, curs); } /* @@ -1208,14 +1208,16 @@ int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, * * <0 is returned on error, including -ENOENT if the key isn't present. */ -int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key) { struct scoutfs_btree_block *bt; struct buffer_head *bh; int cmp; int ret; - bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1237,7 +1239,9 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) * This is guaranteed not to fail if the caller has already dirtied the * block that contains the item in the current transaction. */ -int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_cursor *curs) { struct scoutfs_btree_item *item; @@ -1249,7 +1253,7 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, BUG_ON(curs->bh); - bh = btree_walk(sb, key, NULL, 0, 0, WALK_DIRTY); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1286,14 +1290,15 @@ void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) * The caller ensures that it's safe for us to be walking this region * of the tree. */ -int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, +int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole) { DECLARE_SCOUTFS_BTREE_CURSOR(curs); int ret; *hole = *first; - while ((ret = scoutfs_btree_next(sb, first, last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, root, first, last, &curs)) > 0) { /* return our expected hole if we skipped it */ if (scoutfs_key_cmp(hole, curs.key) < 0) break; diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 60d4a0b5..a7a200fd 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -17,23 +17,34 @@ struct scoutfs_btree_cursor { #define DECLARE_SCOUTFS_BTREE_CURSOR(name) \ struct scoutfs_btree_cursor name = {NULL,} -int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_cursor *curs); -int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, - unsigned int val_len, +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_cursor *curs); -int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key); -int scoutfs_btree_next(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key); +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_btree_cursor *curs); -int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key); -int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs); -int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_key *first, +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key); +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_btree_cursor *curs); +int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole); -int scoutfs_btree_since(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last, u64 seq, - struct scoutfs_btree_cursor *curs); +int scoutfs_btree_since(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, + u64 seq, struct scoutfs_btree_cursor *curs); void scoutfs_btree_release(struct scoutfs_btree_cursor *curs); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0e0b1ab4..2b70dbee 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -192,6 +192,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, struct scoutfs_inode_info *si = SCOUTFS_I(dir); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = dir->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_dirent *dent; struct dentry_info *di; struct scoutfs_key first; @@ -219,7 +220,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, last_dirent_key_offset(h)); - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { /* XXX verify */ @@ -279,6 +280,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_dirent *dent; struct scoutfs_key first; @@ -295,7 +297,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, SCOUTFS_DIRENT_LAST_POS); - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { dent = curs.val; name_len = item_name_len(&curs); pos = scoutfs_key_offset(curs.key); @@ -322,14 +324,15 @@ static void set_lref_key(struct scoutfs_key *key, u64 ino, u64 ctr) static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, u64 dir_ino, u64 dir_off, bool update) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_link_backref *lref; int ret; if (update) - ret = scoutfs_btree_update(sb, key, &curs); + ret = scoutfs_btree_update(sb, meta, key, &curs); else - ret = scoutfs_btree_insert(sb, key, sizeof(*lref), &curs); + ret = scoutfs_btree_insert(sb, meta, key, sizeof(*lref), &curs); /* XXX verify size */ if (ret == 0) { @@ -347,6 +350,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, { struct dentry_info *di = dentry->d_fsdata; struct super_block *sb = dir->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *si = SCOUTFS_I(dir); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_dirent *dent; @@ -375,7 +379,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, last_dirent_key_offset(h)); - ret = scoutfs_btree_hole(sb, &first, &last, &key); + ret = scoutfs_btree_hole(sb, meta, &first, &last, &key); if (ret) goto out; @@ -386,9 +390,9 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) goto out; - ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + ret = scoutfs_btree_insert(sb, meta, &key, bytes, &curs); if (ret) { - scoutfs_btree_delete(sb, &lref_key); + scoutfs_btree_delete(sb, meta, &lref_key); goto out; } @@ -509,6 +513,7 @@ out: static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) { struct super_block *sb = dir->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); struct dentry_info *di; @@ -531,17 +536,17 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) ret = scoutfs_dirty_inode_item(dir) ?: scoutfs_dirty_inode_item(inode) ?: - scoutfs_btree_dirty(sb, &lref_key); + scoutfs_btree_dirty(sb, meta, &lref_key); if (ret) goto out; scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); - ret = scoutfs_btree_delete(sb, &key); + ret = scoutfs_btree_delete(sb, meta, &key); if (ret) goto out; - scoutfs_btree_delete(sb, &lref_key); + scoutfs_btree_delete(sb, meta, &lref_key); dir->i_ctime = ts; dir->i_mtime = ts; @@ -573,6 +578,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); loff_t size = i_size_read(inode); struct scoutfs_key first; @@ -599,7 +605,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) off = 0; k = 0; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { if (scoutfs_key_offset(curs.key) != k || off + curs.val_len > size) { /* XXX corruption */ @@ -654,6 +660,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) { struct super_block *sb = dir->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct inode *inode = NULL; struct scoutfs_key key; @@ -687,7 +694,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, k); bytes = min(name_len, SCOUTFS_MAX_ITEM_LEN); - ret = scoutfs_btree_insert(sb, &key, bytes, &curs); + ret = scoutfs_btree_insert(sb, meta, &key, bytes, &curs); if (ret) goto out; @@ -719,7 +726,7 @@ out: while (k--) { scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, k); - scoutfs_btree_delete(sb, &key); + scoutfs_btree_delete(sb, meta, &key); } } @@ -733,6 +740,7 @@ out: */ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_key first; struct scoutfs_key last; @@ -742,13 +750,13 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) scoutfs_set_key(&first, ino, SCOUTFS_SYMLINK_KEY, 0); scoutfs_set_key(&last, ino, SCOUTFS_SYMLINK_KEY, ~0ULL); - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { key = *curs.key; first = *curs.key; scoutfs_inc_key(&first); scoutfs_btree_release(&curs); - ret = scoutfs_btree_delete(sb, &key); + ret = scoutfs_btree_delete(sb, meta, &key); if (ret) break; } @@ -777,6 +785,7 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) static int add_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, u64 *ctr, struct list_head *list) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_path_component *comp; DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_link_backref *lref; @@ -798,7 +807,7 @@ retry: scoutfs_set_key(&first, ino, SCOUTFS_LINK_BACKREF_KEY, *ctr); scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); - ret = scoutfs_btree_next(sb, &first, &last, &curs); + ret = scoutfs_btree_next(sb, meta, &first, &last, &curs); if (ret <= 0) goto out; @@ -844,7 +853,7 @@ retry: scoutfs_set_key(&key, *dir_ino, SCOUTFS_DIRENT_KEY, off); - ret = scoutfs_btree_lookup(sb, &key, &curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &curs); if (ret < 0) { /* XXX corruption, should always have dirent for backref */ if (ret == -ENOENT) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index e42a485f..75faacc0 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -202,6 +202,7 @@ static bool bmap_has_blocks(struct scoutfs_block_map *bmap) */ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_block_map *bmap; struct scoutfs_key first; @@ -222,13 +223,13 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) trace_printk("iblock %llu i %d\n", iblock, i); - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { key = *curs.key; first = *curs.key; scoutfs_inc_key(&first); scoutfs_btree_release(&curs); - ret = scoutfs_btree_update(sb, &key, &curs); + ret = scoutfs_btree_update(sb, meta, &key, &curs); if (ret) break; @@ -255,7 +256,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) i = 0; if (delete) { - ret = scoutfs_btree_delete(sb, &key); + ret = scoutfs_btree_delete(sb, meta, &key); if (ret) break; } @@ -301,6 +302,7 @@ static void set_bmap_key(struct scoutfs_key *key, struct inode *inode, static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) { struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_block_map *bmap; struct scoutfs_key key; @@ -310,7 +312,7 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) *blkno = 0; set_bmap_key(&key, inode, iblock); - ret = scoutfs_btree_lookup(sb, &key, &curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &curs); if (!ret) { bmap = curs.val; @@ -347,6 +349,7 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) { struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_block_map *bmap; struct scoutfs_key key; @@ -360,13 +363,14 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) set_bmap_key(&key, inode, iblock); /* we always need a writable block map item */ - ret = scoutfs_btree_update(sb, &key, &curs); + ret = scoutfs_btree_update(sb, meta, &key, &curs); if (ret < 0 && ret != -ENOENT) goto out; /* might need to create a new item and delete it after errors */ if (ret == -ENOENT) { - ret = scoutfs_btree_insert(sb, &key, sizeof(*bmap), &curs); + ret = scoutfs_btree_insert(sb, meta, &key, sizeof(*bmap), + &curs); if (ret < 0) goto out; memset(curs.val, 0, sizeof(*bmap)); @@ -412,7 +416,7 @@ out: if (new_blkno) return_file_block(sb, new_blkno); if (inserted) { - err = scoutfs_btree_delete(sb, &key); + err = scoutfs_btree_delete(sb, meta, &key); BUG_ON(err); /* always succeeds */ } } diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 93d68202..779dee62 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -127,12 +127,13 @@ static int scoutfs_read_locked_inode(struct inode *inode) { DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_lookup(sb, &key, &curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &curs); if (!ret) { load_inode(inode, curs.val); scoutfs_btree_release(&curs); @@ -228,12 +229,13 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) int scoutfs_dirty_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_dirty(sb, &key); + ret = scoutfs_btree_dirty(sb, meta, &key); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -252,12 +254,13 @@ void scoutfs_update_inode_item(struct inode *inode) { DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_key key; int err; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - err = scoutfs_btree_update(sb, &key, &curs); + err = scoutfs_btree_update(sb, meta, &key, &curs); BUG_ON(err); store_inode(curs.val, inode); @@ -309,6 +312,7 @@ static int alloc_ino(struct super_block *sb, u64 *ino) struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; struct scoutfs_key key; @@ -338,7 +342,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_insert(inode->i_sb, &key, + ret = scoutfs_btree_insert(inode->i_sb, meta, &key, sizeof(struct scoutfs_inode), &curs); if (ret) { iput(inode); @@ -354,6 +358,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, */ static void drop_inode_items(struct super_block *sb, u64 ino) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode *sinode; struct scoutfs_key key; @@ -363,7 +368,7 @@ static void drop_inode_items(struct super_block *sb, u64 ino) /* sample the inode mode */ scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_lookup(sb, &key, &curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &curs); if (ret) goto out; @@ -387,7 +392,7 @@ static void drop_inode_items(struct super_block *sb, u64 ino) if (ret) goto out; - ret = scoutfs_btree_delete(sb, &key); + ret = scoutfs_btree_delete(sb, meta, &key); out: if (ret) trace_printk("drop items failed ret %d ino %llu\n", ret, ino); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 752c665f..4f5c20e7 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -23,6 +23,7 @@ #include "dir.h" #include "name.h" #include "ioctl.h" +#include "super.h" /* * Find all the inodes in the given inode range that have changed since @@ -33,6 +34,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_ioctl_inodes_since __user *uargs = (void __user *)arg; struct scoutfs_ioctl_inodes_since args; struct scoutfs_ioctl_ino_seq __user *uiseq; @@ -54,7 +56,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) scoutfs_set_key(&last, args.last_ino, SCOUTFS_INODE_KEY, 0); bytes = 0; - while ((ret = scoutfs_btree_since(sb, &first, &last, + while ((ret = scoutfs_btree_since(sb, meta, &first, &last, args.seq, &curs)) > 0) { iseq.ino = scoutfs_key_inode(curs.key); @@ -215,6 +217,7 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, bool find_name) { struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_ioctl_find_xattr args; DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_key first; @@ -264,7 +267,7 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, while (copied < args.ino_count) { - while ((ret = scoutfs_btree_next(sb, &first, &last, + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { inos[nr_inos++] = scoutfs_key_offset(curs.key); diff --git a/kmod/src/super.h b/kmod/src/super.h index 4e7d8723..79c2168f 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -56,6 +56,12 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } +/* The root of the metadata btree */ +static inline struct scoutfs_btree_root *SCOUTFS_META(struct super_block *sb) +{ + return &SCOUTFS_SB(sb)->super.btree_root; +} + void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 82d92961..97ad19d7 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -114,6 +114,7 @@ static int search_xattr_items(struct inode *inode, const char *name, struct xattr_search_results *res) { struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_key first; struct scoutfs_key last; @@ -127,7 +128,7 @@ static int search_xattr_items(struct inode *inode, const char *name, res->found_hole = false; res->hole_key = first; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { xat = curs.val; /* found a hole when we skip past next expected key */ @@ -176,6 +177,7 @@ static int insert_xattr(struct inode *inode, const char *name, u64 val_hash) { struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); bool inserted_name_hash_item = false; __le64 * __packed refcount; @@ -186,7 +188,8 @@ static int insert_xattr(struct inode *inode, const char *name, set_name_val_keys(&name_key, &val_key, key, val_hash); - ret = scoutfs_btree_insert(sb, key, xat_bytes(name_len, size), &curs); + ret = scoutfs_btree_insert(sb, meta, key, + xat_bytes(name_len, size), &curs); if (ret) return ret; @@ -200,7 +203,7 @@ static int insert_xattr(struct inode *inode, const char *name, /* insert the name hash item for find_xattr if we're first */ if (!other_coll) { - ret = scoutfs_btree_insert(sb, &name_key, 0, &curs); + ret = scoutfs_btree_insert(sb, meta, &name_key, 0, &curs); /* XXX eexist would be corruption */ if (ret) goto out; @@ -209,10 +212,10 @@ static int insert_xattr(struct inode *inode, const char *name, } /* increment the val hash item for find_xattr, inserting if first */ - ret = scoutfs_btree_update(sb, &val_key, &curs); + ret = scoutfs_btree_update(sb, meta, &val_key, &curs); if (ret == -ENOENT) { - ret = scoutfs_btree_insert(sb, &val_key, sizeof(*refcount), - &curs); + ret = scoutfs_btree_insert(sb, meta, &val_key, + sizeof(*refcount), &curs); if (ret == 0) { /* XXX test sane item size */ refcount = curs.val; @@ -227,9 +230,9 @@ static int insert_xattr(struct inode *inode, const char *name, out: if (ret) { - scoutfs_btree_delete(sb, key); + scoutfs_btree_delete(sb, meta, key); if (inserted_name_hash_item) - scoutfs_btree_delete(sb, &name_key); + scoutfs_btree_delete(sb, meta, &name_key); } return ret; } @@ -243,6 +246,7 @@ out: static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, bool other_coll, u64 val_hash) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_key name_key; struct scoutfs_key val_key; @@ -253,22 +257,22 @@ static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, set_name_val_keys(&name_key, &val_key, key, val_hash); if (!other_coll) { - ret = scoutfs_btree_dirty(sb, &name_key); + ret = scoutfs_btree_dirty(sb, meta, &name_key); if (ret) goto out; } - ret = scoutfs_btree_dirty(sb, &val_key); + ret = scoutfs_btree_dirty(sb, meta, &val_key); if (ret) goto out; - ret = scoutfs_btree_delete(sb, key); + ret = scoutfs_btree_delete(sb, meta, key); if (ret) goto out; if (!other_coll) - scoutfs_btree_delete(sb, &name_key); + scoutfs_btree_delete(sb, meta, &name_key); - scoutfs_btree_update(sb, &val_key, &curs); + scoutfs_btree_update(sb, meta, &val_key, &curs); refcount = curs.val; le64_add_cpu(refcount, -1ULL); if (*refcount == 0) @@ -276,7 +280,7 @@ static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, scoutfs_btree_release(&curs); if (del_val) - scoutfs_btree_delete(sb, &val_key); + scoutfs_btree_delete(sb, meta, &val_key); ret = 0; out: return ret; @@ -296,6 +300,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *si = SCOUTFS_I(inode); DECLARE_SCOUTFS_BTREE_CURSOR(curs); size_t name_len = strlen(name); @@ -312,7 +317,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, down_read(&si->xattr_rwsem); ret = -ENODATA; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { xat = curs.val; if (!scoutfs_names_equal(name, name_len, xat->name, @@ -452,6 +457,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_xattr *xat; struct scoutfs_key first; @@ -465,7 +471,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) down_read(&si->xattr_rwsem); total = 0; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { xat = curs.val; total += xat->name_len + 1; @@ -504,6 +510,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_xattr *xat; struct scoutfs_key first; @@ -521,7 +528,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) scoutfs_set_key(&last, ino, SCOUTFS_XATTR_KEY, ~0ULL); have_last = false; - while ((ret = scoutfs_btree_next(sb, &first, &last, &curs)) > 0) { + while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { xat = curs.val; key = *curs.key; val_hash = scoutfs_name_hash(xat_value(xat), xat->value_len); @@ -532,14 +539,14 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) scoutfs_btree_release(&curs); if (!have_last || last_name != name_key.inode) { - ret = scoutfs_btree_delete(sb, &name_key); + ret = scoutfs_btree_delete(sb, meta, &name_key); if (ret && ret != -ENOENT) break; last_name = name_key.inode; } if (!have_last || last_val != val_key.inode) { - ret = scoutfs_btree_delete(sb, &val_key); + ret = scoutfs_btree_delete(sb, meta, &val_key); if (ret && ret != -ENOENT) break; last_val = val_key.inode; @@ -547,7 +554,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) have_last = true; - ret = scoutfs_btree_delete(sb, &key); + ret = scoutfs_btree_delete(sb, meta, &key); if (ret && ret != -ENOENT) break; } From 161063c8d65d0758055e04d99172cbe3c3feab0f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Sep 2016 14:22:50 -0700 Subject: [PATCH 096/920] scoutfs: remove very noisy bh ref tracing This wasn't adding much value and was exceptionally noisy. Signed-off-by: Zach Brown --- kmod/src/block.c | 5 ----- kmod/src/block.h | 5 +---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index faf9ae33..7c049a9a 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -207,8 +207,6 @@ struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno) } out: - trace_printk("blkno %llu bh %p (ret %ld)\n", - blkno, bh, IS_ERR(bh) ? PTR_ERR(bh) : 0); return bh; } @@ -447,9 +445,6 @@ struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno) set_buffer_uptodate(bh); set_buffer_scoutfs_verified(bh); out: - trace_printk("blkno %llu bh %p (ret %ld)\n", - blkno, bh, IS_ERR(bh) ? PTR_ERR(bh) : 0); - return bh; } diff --git a/kmod/src/block.h b/kmod/src/block.h index ee38b677..d431d90e 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -27,11 +27,8 @@ static inline void *bh_data(struct buffer_head *bh) static inline void scoutfs_block_put(struct buffer_head *bh) { - if (!IS_ERR_OR_NULL(bh)) { - trace_printk("putting bh %p count %d\n", - bh, atomic_read(&bh->b_count)); + if (!IS_ERR_OR_NULL(bh)) brelse(bh); - } } #endif From 10a42724a90414594caa01d0c7972a6651980050 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Sep 2016 15:32:04 -0700 Subject: [PATCH 097/920] scoutfs: add scoutfs_dec_key() This is analagous to scoutfs_inc_key(). It decreases the next highest order key value each time its decrement wraps. Signed-off-by: Zach Brown --- kmod/src/key.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kmod/src/key.h b/kmod/src/key.h index dac0871b..cb8460d6 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -94,6 +94,15 @@ static inline void scoutfs_inc_key(struct scoutfs_key *key) } } +static inline void scoutfs_dec_key(struct scoutfs_key *key) +{ + le64_add_cpu(&key->offset, -1ULL); + if (key->offset == cpu_to_le64(~0ULL)) { + if (key->type-- == 0) + le64_add_cpu(&key->inode, -1ULL); + } +} + static inline struct scoutfs_key *scoutfs_max_key(struct scoutfs_key *a, struct scoutfs_key *b) { From a9afa9248287f942108d8dd10f29787be9229125 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Sep 2016 15:34:25 -0700 Subject: [PATCH 098/920] scoutfs: correctly set the last symlink item The final symlink item insertion was taking the min of the entire path and the max symlink item size, not the min of the remaining length of the path after having created all the previous items. For paths larger than the max item size this could use too much space. Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 2b70dbee..13f51a7d 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -692,7 +692,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, for (k = 0, off = 0; off < name_len; off += bytes, k++) { scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, k); - bytes = min(name_len, SCOUTFS_MAX_ITEM_LEN); + bytes = min(name_len - off, SCOUTFS_MAX_ITEM_LEN); ret = scoutfs_btree_insert(sb, meta, &key, bytes, &curs); if (ret) From 84f23296fd8d457dc5a5bd2dc712822b8f0beda8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Sep 2016 15:38:01 -0700 Subject: [PATCH 099/920] scoutfs: remove btree cursor The btree cursor was built to address two problems. First it accelerates iteration by avoiding full descents down the tree by holding on to leaf blocks. Second it lets callers reference item value contents directly to avoid copies. But it also has serious complexity costs. It pushes refcounting and locking out to the caller. There have already been a few bugs where callers did things while holding the cursor without realizing that they're holding a btree lock and can't perform certain btree operations or even copies to user space. Future changes to the allocator to use the btree motivates cleaning up the tree locking which is complicated by the cursor being a stand alone lock reference. Instead of continuing to layer complexity onto this construct let's remove it. The iteration acceleration will be addressed the same way we're going to accelerate the other btree operations: with per-cpu cached leaf block references. Unlike the cursor this doesn't push interface changes out to callers who want repeated btree calls to perform well. We'll leave the value copying for now. If it becomes an issue we can add variants that call a function to operate on the value. Let's hope we don't have to go there. This change replaces the cursor with a vector to memory that the value should be copied to and from. The vector has a fixed number of elements and is wrapped in a struct for easy declaration and initialization. This change to the interface looks noisy but each caller's change is pretty mechanical. They tend to involve: - replace the cursor with the value struct and initialization - allocate some memory to copy the value in to - reading functions return the number of value bytes copied - verify copied bytes makes sense for item being read - getting rid of confusing ((ret = _next())) looping - _next now returns -ENOENT instead of 0 for no next item - _next iterators now need to increase the key themselves - make sure to free allocated mem Sometimes the order of operations changes significantly. Now that we can't modify in place we need to read, modify, write. This looks like changing a modification of the item through the cursor to a lookup/update pattern. The symlink item iterators didn't need to use next because they walk a contiguous set of keys. They're changed to use simple insert or lookup. Signed-off-by: Zach Brown --- kmod/src/btree.c | 324 +++++++++++++++++++++++++++++----------------- kmod/src/btree.h | 60 ++++++--- kmod/src/dir.c | 256 +++++++++++++++++++++--------------- kmod/src/filerw.c | 120 ++++++++++------- kmod/src/inode.c | 56 ++++---- kmod/src/ioctl.c | 85 +++++------- kmod/src/xattr.c | 297 +++++++++++++++++++++++++++--------------- 7 files changed, 729 insertions(+), 469 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 90493967..d17a1b76 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -56,7 +56,7 @@ * XXX * - do we want a level in the btree header? seems like we would? * - validate structures on read? - * - internal bh/pos/cmp interface is clumsy.. could use cursor + * - internal bh/pos/cmp interface is clumsy.. */ /* number of contiguous bytes used by the item header and val of given len */ @@ -121,6 +121,73 @@ static inline struct scoutfs_key *greatest_key(struct scoutfs_btree_block *bt) return &pos_item(bt, bt->nr_items - 1)->key; } +/* + * Copy as much of the item as fits in the value vector. The min of the + * value vec length and the item length is returned, including possibly + * 0. + */ +static int copy_to_val(struct scoutfs_btree_val *val, + struct scoutfs_btree_item *item) +{ + size_t val_len = le16_to_cpu(item->val_len); + char *val_ptr = item->val; + struct kvec *kv; + size_t bytes; + size_t off; + int i; + + for (i = 0, off = 0; val_len > 0 && i < ARRAY_SIZE(val->vec); i++) { + kv = &val->vec[i]; + + if (WARN_ON_ONCE(kv->iov_len && !kv->iov_base)) + return -EINVAL; + + bytes = min(val_len, kv->iov_len); + if (bytes) + memcpy(kv->iov_base, val_ptr + off, bytes); + + val_len -= bytes; + off += bytes; + } + + return off; +} + +/* + * Copy the caller's value vector into the item in the tree block. This + * is only called when the item should exactly match the value vector. + * + * -EINVAL is returned if the lengths don't match. + */ +static int copy_to_item(struct scoutfs_btree_item *item, + struct scoutfs_btree_val *val) +{ + size_t val_len = le16_to_cpu(item->val_len); + char *val_ptr = item->val; + struct kvec *kv; + size_t bytes; + int i; + + if (val_len != scoutfs_btree_val_length(val)) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(val->vec); i++) { + kv = &val->vec[i]; + + if (WARN_ON_ONCE(kv->iov_len && !kv->iov_base)) + return -EINVAL; + + bytes = min(val_len, kv->iov_len); + if (bytes) + memcpy(val_ptr, kv->iov_base, bytes); + + val_len -= bytes; + val_ptr += bytes; + } + + return 0; +} + /* * Returns the sorted item position that an item with the given key * should occupy. @@ -964,38 +1031,25 @@ static struct buffer_head *btree_walk(struct super_block *sb, return bh; } -static void set_cursor(struct scoutfs_btree_cursor *curs, - struct buffer_head *bh, unsigned int pos, bool write) -{ - struct scoutfs_btree_block *bt = bh_data(bh); - struct scoutfs_btree_item *item = pos_item(bt, pos); - - curs->bh = bh; - curs->pos = pos; - curs->write = write; - - curs->key = &item->key; - curs->seq = le64_to_cpu(item->seq); - curs->val = item->val; - curs->val_len = le16_to_cpu(item->val_len); -} - /* - * Point the caller's cursor at the item if it's found. It can't be - * modified. -ENOENT is returned if the key isn't found in the tree. + * Copy the given value identified by the given key into the caller's + * buffer. The number of bytes copied is returned, -ENOENT if the key + * wasn't found, or -errno on errors. */ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs) + struct scoutfs_btree_val *val) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; unsigned int pos; int cmp; int ret; - BUG_ON(curs->bh); + trace_printk("key "CKF" val_len %d\n", + CKA(key), scoutfs_btree_val_length(val)); bh = btree_walk(sb, root, key, NULL, 0, 0, 0); if (IS_ERR(bh)) @@ -1004,37 +1058,49 @@ int scoutfs_btree_lookup(struct super_block *sb, pos = find_pos(bt, key, &cmp); if (cmp == 0) { - set_cursor(curs, bh, pos, false); - ret = 0; + item = pos_item(bt, pos); + ret = copy_to_val(val, item); } else { - unlock_block(NULL, bh, false); - scoutfs_block_put(bh); ret = -ENOENT; } + unlock_block(NULL, bh, false); + scoutfs_block_put(bh); + + trace_printk("key "CKF" ret %d\n", CKA(key), ret); + return ret; } /* - * Insert a new item in the tree and point the caller's cursor at it. - * The caller is responsible for setting the value. + * Insert a new item in the tree. * - * -EEXIST is returned if the key is already present in the tree. + * 0 is returned on success. -EEXIST is returned if the key is already + * present in the tree. * - * XXX this walks the treap twice, which isn't great + * If no value pointer is given then the item is created with a zero + * length value. */ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *key, unsigned int val_len, - struct scoutfs_btree_cursor *curs) + struct scoutfs_key *key, + struct scoutfs_btree_val *val) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct buffer_head *bh; + unsigned int val_len; int pos; int cmp; int ret; - BUG_ON(curs->bh); + if (val) + val_len = scoutfs_btree_val_length(val); + else + val_len = 0; + + if (WARN_ON_ONCE(val_len > SCOUTFS_MAX_ITEM_LEN)) + return -EINVAL; bh = btree_walk(sb, root, key, NULL, val_len, 0, WALK_INSERT); if (IS_ERR(bh)) @@ -1043,15 +1109,18 @@ int scoutfs_btree_insert(struct super_block *sb, pos = find_pos(bt, key, &cmp); if (cmp) { - create_item(bt, pos, key, val_len); - set_cursor(curs, bh, pos, true); - ret = 0; + item = create_item(bt, pos, key, val_len); + if (val) + ret = copy_to_item(item, val); + else + ret = 0; } else { - unlock_block(NULL, bh, true); - scoutfs_block_put(bh); ret = -EEXIST; } + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); + return ret; } @@ -1104,48 +1173,46 @@ out: } /* - * Iterate over items in the tree starting with first and ending with - * last. We point the cursor at each item and return to the caller. - * The caller continues the search with the cursor. + * Find the next key in the tree starting from 'first', and ending at + * 'last'. 'found', 'found_seq', and 'val' are set to the discovered + * item if they're provided. * * The caller can limit results to items with a sequence number greater * than or equal to their sequence number. * - * When there isn't an item in the cursor then we walk the btree to the - * leaf that should contain the key and look for items from there. When - * we exhaust leaves we search the tree again from the next key that was - * increased past the leaf's parent's item. + * The only tricky bit is that they key we're searching for might not + * exist in the tree. We can get to the leaf and find that there are no + * greater items in the leaf. We have to search again from the keys + * greater than the parent item's keys which the walk gives us. We also + * star the search over from this next key if walking while filtering + * based on seqs terminates early. * - * Returns > 0 when the cursor has an item, 0 when done, and -errno on error. + * Returns the bytes copied into the value (0 if not provided), -ENOENT + * if there is no item past first until last, or -errno on errors. + * + * It's a common pattern to use the same key for first and found so we're + * careful to copy first before we modify found. */ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, int op, struct scoutfs_btree_cursor *curs) + u64 seq, int op, struct scoutfs_key *found, + u64 *found_seq, struct scoutfs_btree_val *val) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_key start = *first; struct scoutfs_key key = *first; struct scoutfs_key next_key; + struct buffer_head *bh; + int pos; int ret; - if (scoutfs_key_cmp(first, last) > 0) - return 0; - - /* find the next item after the cursor, releasing if we're done */ - if (curs->bh) { - bt = bh_data(curs->bh); - key = *curs->key; - scoutfs_inc_key(&key); - - curs->pos = next_pos_seq(bt, curs->pos, 0, seq, op); - if (curs->pos < bt->nr_items) - set_cursor(curs, curs->bh, curs->pos, curs->write); - else - scoutfs_btree_release(curs); - } + trace_printk("finding next first "CKF" last "CKF"\n", + CKA(&start), CKA(last)); /* find the leaf that contains the next item after the key */ - while (!curs->bh && scoutfs_key_cmp(&key, last) <= 0) { + ret = -ENOENT; + while (scoutfs_key_cmp(&key, last) <= 0) { bh = btree_walk(sb, root, &key, &next_key, 0, seq, op); @@ -1156,49 +1223,60 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, } if (IS_ERR(bh)) { - if (bh == ERR_PTR(-ENOENT)) - break; - return PTR_ERR(bh); + ret = PTR_ERR(bh); + break; } bt = bh_data(bh); /* keep trying leaves until next_key passes last */ - curs->pos = find_pos_after_seq(bt, &key, 0, seq, op); - if (curs->pos >= bt->nr_items) { + pos = find_pos_after_seq(bt, &key, 0, seq, op); + if (pos >= bt->nr_items) { key = next_key; unlock_block(NULL, bh, false); scoutfs_block_put(bh); continue; } - set_cursor(curs, bh, curs->pos, false); + item = pos_item(bt, pos); + if (scoutfs_key_cmp(&item->key, last) <= 0) { + *found = item->key; + if (found_seq) + *found_seq = le64_to_cpu(item->seq); + if (val) + ret = copy_to_val(val, item); + else + ret = 0; + } else { + ret = -ENOENT; + } + + unlock_block(NULL, bh, false); + scoutfs_block_put(bh); break; } - /* only return the next item if it's within last */ - if (curs->bh && scoutfs_key_cmp(curs->key, last) <= 0) { - ret = 1; - } else { - scoutfs_btree_release(curs); - ret = 0; - } - + trace_printk("next first "CKF" last "CKF" found "CKF" ret %d\n", + CKA(&start), CKA(last), CKA(found), ret); return ret; } int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_btree_cursor *curs) + struct scoutfs_key *found, + struct scoutfs_btree_val *val) { - return btree_next(sb, root, first, last, 0, WALK_NEXT, curs); + return btree_next(sb, root, first, last, 0, WALK_NEXT, + found, NULL, val); } int scoutfs_btree_since(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, struct scoutfs_btree_cursor *curs) + u64 seq, struct scoutfs_key *found, u64 *found_seq, + struct scoutfs_btree_val *val) { - return btree_next(sb, root, first, last, seq, WALK_NEXT_SEQ, curs); + return btree_next(sb, root, first, last, seq, WALK_NEXT_SEQ, + found, found_seq, val); } /* @@ -1217,6 +1295,8 @@ int scoutfs_btree_dirty(struct super_block *sb, int cmp; int ret; + trace_printk("key "CKF"\n", CKA(key)); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); @@ -1232,17 +1312,22 @@ int scoutfs_btree_dirty(struct super_block *sb, unlock_block(NULL, bh, true); scoutfs_block_put(bh); + trace_printk("key "CKF" ret %d\n", CKA(key), ret); + return ret; } /* * This is guaranteed not to fail if the caller has already dirtied the * block that contains the item in the current transaction. + * + * 0 is returned on success. -EINVAL is returned if the caller's value + * length doesn't match the existing item's value length. */ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs) + struct scoutfs_btree_val *val) { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; @@ -1251,8 +1336,6 @@ int scoutfs_btree_update(struct super_block *sb, int cmp; int ret; - BUG_ON(curs->bh); - bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); @@ -1261,59 +1344,64 @@ int scoutfs_btree_update(struct super_block *sb, pos = find_pos(bt, key, &cmp); if (cmp == 0) { item = pos_item(bt, pos); - item->seq = bt->hdr.seq; - set_cursor(curs, bh, pos, true); - ret = 0; + ret = copy_to_item(item, val); + if (ret == 0) + item->seq = bt->hdr.seq; } else { - unlock_block(NULL, bh, true); - scoutfs_block_put(bh); ret = -ENOENT; } + unlock_block(NULL, bh, true); + scoutfs_block_put(bh); + return ret; } -void scoutfs_btree_release(struct scoutfs_btree_cursor *curs) -{ - if (curs->bh) { - unlock_block(NULL, curs->bh, curs->write); - scoutfs_block_put(curs->bh); - } - curs->bh = NULL; -} - /* - * Find the first missing key between the caller's keys, inclusive. Set - * the caller's hole key and return 0 if we find a missing key. Return - * -ENOSPC if all the keys in the range were present or -errno on errors. + * Set hole to a missing key in the caller's range. * - * The caller ensures that it's safe for us to be walking this region - * of the tree. + * 0 is returned if we find a missing key, -ENOSPC is returned if all + * the keys in the range are present in the tree, and -errno is returned + * if we saw an error. + * + * We try to find the first key in the range. If the next key is past + * the first key then we return the key before the found key. This will + * tend to let us find the hole with one btree search. + * + * We keep searching as long as we keep finding the first key and will + * return -ENOSPC if we fall off the end of the range doing so. */ int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole) { - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_key key = *first; + struct scoutfs_key found; int ret; - *hole = *first; - while ((ret = scoutfs_btree_next(sb, root, first, last, &curs)) > 0) { - /* return our expected hole if we skipped it */ - if (scoutfs_key_cmp(hole, curs.key) < 0) - break; - - *hole = *curs.key; - scoutfs_inc_key(hole); + if (WARN_ON_ONCE(scoutfs_key_cmp(first, last) > 0)) { + scoutfs_key_set_zero(hole); + return -EINVAL; } - scoutfs_btree_release(&curs); - if (ret >= 0) { - if (scoutfs_key_cmp(hole, last) <= 0) - ret = 0; - else - ret = -ENOSPC; + /* search as long as we keep finding our first key */ + do { + ret = scoutfs_btree_next(sb, root, &key, last, &found, NULL); + } while (ret == 0 && + scoutfs_key_cmp(&found, &key) == 0 && + (scoutfs_inc_key(&key), ret = -ENOSPC, + scoutfs_key_cmp(&key, last) <= 0)); + + if (ret == 0) { + *hole = found; + scoutfs_dec_key(hole); + } else if (ret == -ENOENT) { + *hole = *last; + ret = 0; } + trace_printk("first "CKF" last "CKF" hole "CKF" ret %d\n", + CKA(first), CKA(last), CKA(hole), ret); + return ret; } diff --git a/kmod/src/btree.h b/kmod/src/btree.h index a7a200fd..dc22b4c6 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -1,51 +1,71 @@ #ifndef _SCOUTFS_BTREE_H_ #define _SCOUTFS_BTREE_H_ -struct scoutfs_btree_cursor { - /* for btree.c */ - struct buffer_head *bh; - unsigned int pos; - bool write; +#include - /* for callers */ - struct scoutfs_key *key; - u64 seq; - void *val; - u16 val_len; +struct scoutfs_btree_val { + struct kvec vec[3]; }; -#define DECLARE_SCOUTFS_BTREE_CURSOR(name) \ - struct scoutfs_btree_cursor name = {NULL,} +static inline void __scoutfs_btree_init_val(struct scoutfs_btree_val *val, + void *ptr0, unsigned int len0, + void *ptr1, unsigned int len1, + void *ptr2, unsigned int len2) +{ + *val = (struct scoutfs_btree_val) { + { { ptr0, len0 }, { ptr1, len1 }, { ptr2, len2 } } + }; +} + +#define _scoutfs_btree_init_val(v, p0, l0, p1, l1, p2, l2, ...) \ + __scoutfs_btree_init_val(v, p0, l0, p1, l1, p2, l2) + +/* + * Provide a nice variadic initialization function without having to + * iterate over the callers arg types. We play some macro games to pad + * out the callers ptr/len pairs to the full possible number. This will + * produce confusing errors if an odd number of arguments is given and + * the padded ptr/length types aren't compatible with the fixed + * arguments in the static inline. + */ +#define scoutfs_btree_init_val(val, ...) \ + _scoutfs_btree_init_val(val, __VA_ARGS__, NULL, 0, NULL, 0, NULL, 0) + +static inline int scoutfs_btree_val_length(struct scoutfs_btree_val *val) +{ + + return iov_length((struct iovec *)val->vec, ARRAY_SIZE(val->vec)); +} int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs); + struct scoutfs_btree_val *val); int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *key, unsigned int val_len, - struct scoutfs_btree_cursor *curs); + struct scoutfs_key *key, + struct scoutfs_btree_val *val); int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key); int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_btree_cursor *curs); + struct scoutfs_key *found, + struct scoutfs_btree_val *val); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key); int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, - struct scoutfs_btree_cursor *curs); + struct scoutfs_btree_val *val); int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *hole); int scoutfs_btree_since(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, struct scoutfs_btree_cursor *curs); - -void scoutfs_btree_release(struct scoutfs_btree_cursor *curs); + u64 seq, struct scoutfs_key *found, u64 *found_seq, + struct scoutfs_btree_val *val); #endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 13f51a7d..e1c90f31 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -112,11 +112,6 @@ static unsigned int dent_bytes(unsigned int name_len) return sizeof(struct scoutfs_dirent) + name_len; } -static unsigned int item_name_len(struct scoutfs_btree_cursor *curs) -{ - return curs->val_len - sizeof(struct scoutfs_dirent); -} - /* * Each dirent stores the values that are needed to build the keys of * the items that are removed on unlink so that we don't to search through @@ -190,13 +185,14 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct scoutfs_inode_info *si = SCOUTFS_I(dir); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = dir->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_dirent *dent; + struct scoutfs_dirent *dent = NULL; + struct scoutfs_btree_val val; struct dentry_info *di; - struct scoutfs_key first; struct scoutfs_key last; + struct scoutfs_key key; + unsigned int item_len; unsigned int name_len; struct inode *inode; u64 ino = 0; @@ -214,29 +210,52 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } + item_len = offsetof(struct scoutfs_dirent, name[dentry->d_name.len]); + dent = kmalloc(item_len, GFP_KERNEL); + if (!dent) { + ret = -ENOMEM; + goto out; + } + h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); - scoutfs_set_key(&first, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); + scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, last_dirent_key_offset(h)); - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { + scoutfs_btree_init_val(&val, dent, item_len); - /* XXX verify */ + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } - dent = curs.val; - name_len = item_name_len(&curs); + /* XXX more verification */ + /* XXX corruption */ + if (ret <= sizeof(struct scoutfs_dirent)) { + ret = -EIO; + break; + } + + + name_len = ret - sizeof(struct scoutfs_dirent); if (scoutfs_names_equal(dentry->d_name.name, dentry->d_name.len, dent->name, name_len)) { ino = le64_to_cpu(dent->ino); - update_dentry_info(di, curs.key, dent); + update_dentry_info(di, &key, dent); + ret = 0; break; } + + scoutfs_inc_key(&key); } - scoutfs_btree_release(&curs); - out: + kfree(dent); + if (ret < 0) inode = ERR_PTR(ret); else if (ino == 0) @@ -281,26 +300,46 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_btree_val val; struct scoutfs_dirent *dent; - struct scoutfs_key first; + struct scoutfs_key key; struct scoutfs_key last; + unsigned int item_len; unsigned int name_len; - int ret; u32 pos; + int ret; if (!dir_emit_dots(file, dirent, filldir)) return 0; - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, + item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); + dent = kmalloc(item_len, GFP_KERNEL); + if (!dent) + return -ENOMEM; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, file->f_pos); scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, SCOUTFS_DIRENT_LAST_POS); - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - dent = curs.val; - name_len = item_name_len(&curs); - pos = scoutfs_key_offset(curs.key); + scoutfs_btree_init_val(&val, dent, item_len); + + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* XXX corruption */ + if (ret <= sizeof(dent)) { + ret = -EIO; + break; + } + + name_len = ret - sizeof(struct scoutfs_dirent); + pos = scoutfs_key_offset(&key); if (filldir(dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) { @@ -309,10 +348,10 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } file->f_pos = pos + 1; + scoutfs_inc_key(&key); } - scoutfs_btree_release(&curs); - + kfree(dent); return ret; } @@ -325,22 +364,19 @@ static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, u64 dir_ino, u64 dir_off, bool update) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_link_backref *lref; + struct scoutfs_link_backref lref; + struct scoutfs_btree_val val; int ret; - if (update) - ret = scoutfs_btree_update(sb, meta, key, &curs); - else - ret = scoutfs_btree_insert(sb, meta, key, sizeof(*lref), &curs); + lref.ino = cpu_to_le64(dir_ino); + lref.offset = cpu_to_le64(dir_off); - /* XXX verify size */ - if (ret == 0) { - lref = curs.val; - lref->ino = cpu_to_le64(dir_ino); - lref->offset = cpu_to_le64(dir_off); - scoutfs_btree_release(&curs); - } + scoutfs_btree_init_val(&val, &lref, sizeof(lref)); + + if (update) + ret = scoutfs_btree_update(sb, meta, key, &val); + else + ret = scoutfs_btree_insert(sb, meta, key, &val); return ret; } @@ -352,8 +388,8 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, struct super_block *sb = dir->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *si = SCOUTFS_I(dir); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_dirent *dent; + struct scoutfs_btree_val val; + struct scoutfs_dirent dent; struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; @@ -390,20 +426,19 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) goto out; - ret = scoutfs_btree_insert(sb, meta, &key, bytes, &curs); - if (ret) { + dent.ino = cpu_to_le64(scoutfs_ino(inode)); + dent.counter = lref_key.offset; + dent.type = mode_to_type(inode->i_mode); + + scoutfs_btree_init_val(&val, &dent, sizeof(dent), + (void *)dentry->d_name.name, + dentry->d_name.len); + + ret = scoutfs_btree_insert(sb, meta, &key, &val); + if (ret) scoutfs_btree_delete(sb, meta, &lref_key); - goto out; - } - - dent = curs.val; - dent->ino = cpu_to_le64(scoutfs_ino(inode)); - dent->counter = lref_key.offset; - dent->type = mode_to_type(inode->i_mode); - memcpy(dent->name, dentry->d_name.name, dentry->d_name.len); - update_dentry_info(di, &key, dent); - - scoutfs_btree_release(&curs); + else + update_dentry_info(di, &key, &dent); out: return ret; } @@ -579,11 +614,11 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); loff_t size = i_size_read(inode); - struct scoutfs_key first; - struct scoutfs_key last; + struct scoutfs_btree_val val; + struct scoutfs_key key; char *path; + int bytes; int off; int ret; int k; @@ -600,24 +635,28 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (!path) return ERR_PTR(-ENOMEM); - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, 0); - scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, ~0ULL); + for (off = 0, k = 0; off < size ; k++) { + scoutfs_set_key(&key, scoutfs_ino(inode), + SCOUTFS_SYMLINK_KEY, k); + bytes = min_t(int, size - off, SCOUTFS_MAX_ITEM_LEN); + scoutfs_btree_init_val(&val, path + off, bytes); - off = 0; - k = 0; - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - if (scoutfs_key_offset(curs.key) != k || - off + curs.val_len > size) { + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret < 0) { /* XXX corruption */ - scoutfs_btree_release(&curs); + if (ret == -ENOENT) + ret = -EIO; + break; + } + + /* XXX corruption */ + if (ret != bytes) { ret = -EIO; break; } - memcpy(path + off, curs.val, curs.val_len); - - off += curs.val_len; - k++; + off += bytes; + ret = 0; } /* XXX corruption */ @@ -661,7 +700,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, { struct super_block *sb = dir->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_btree_val val; struct inode *inode = NULL; struct scoutfs_key key; struct dentry_info *di; @@ -694,12 +733,11 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, k); bytes = min(name_len - off, SCOUTFS_MAX_ITEM_LEN); - ret = scoutfs_btree_insert(sb, meta, &key, bytes, &curs); + scoutfs_btree_init_val(&val, (char *)symname + off, bytes); + + ret = scoutfs_btree_insert(sb, meta, &key, &val); if (ret) goto out; - - memcpy(curs.val, symname + off, bytes); - scoutfs_btree_release(&curs); } ret = add_entry_items(dir, dentry, inode); @@ -741,24 +779,22 @@ out: int scoutfs_symlink_drop(struct super_block *sb, u64 ino) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_key first; - struct scoutfs_key last; struct scoutfs_key key; int ret; + int nr; + int k; - scoutfs_set_key(&first, ino, SCOUTFS_SYMLINK_KEY, 0); - scoutfs_set_key(&last, ino, SCOUTFS_SYMLINK_KEY, ~0ULL); + nr = DIV_ROUND_UP(SCOUTFS_SYMLINK_MAX_SIZE, SCOUTFS_MAX_ITEM_LEN); - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - key = *curs.key; - first = *curs.key; - scoutfs_inc_key(&first); - scoutfs_btree_release(&curs); + for (k = 0; k < nr; k++) { + scoutfs_set_key(&key, ino, SCOUTFS_SYMLINK_KEY, k); ret = scoutfs_btree_delete(sb, meta, &key); - if (ret) + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; break; + } } return ret; @@ -787,9 +823,9 @@ static int add_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_path_component *comp; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_link_backref *lref; - struct scoutfs_dirent *dent; + struct scoutfs_link_backref lref; + struct scoutfs_btree_val val; + struct scoutfs_dirent dent; struct inode *inode = NULL; struct scoutfs_key first; struct scoutfs_key last; @@ -807,20 +843,28 @@ retry: scoutfs_set_key(&first, ino, SCOUTFS_LINK_BACKREF_KEY, *ctr); scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); - ret = scoutfs_btree_next(sb, meta, &first, &last, &curs); - if (ret <= 0) - goto out; + scoutfs_btree_init_val(&val, &lref, sizeof(lref)); - lref = curs.val; - *dir_ino = le64_to_cpu(lref->ino), - off = le64_to_cpu(lref->offset); - *ctr = scoutfs_key_offset(curs.key); + ret = scoutfs_btree_next(sb, meta, &first, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + /* XXX corruption */ + if (ret != sizeof(lref)) { + ret = -EIO; + goto out; + } + + *dir_ino = le64_to_cpu(lref.ino), + off = le64_to_cpu(lref.offset); + *ctr = scoutfs_key_offset(&key); trace_printk("ino %llu ctr %llu dir_ino %llu off %llu\n", ino, *ctr, *dir_ino, off); - scoutfs_btree_release(&curs); - /* XXX corruption, should never be key == U64_MAX */ if (*ctr == U64_MAX) { ret = -EIO; @@ -852,8 +896,10 @@ retry: } scoutfs_set_key(&key, *dir_ino, SCOUTFS_DIRENT_KEY, off); + scoutfs_btree_init_val(&val, &dent, sizeof(dent), + comp->name, SCOUTFS_NAME_LEN); - ret = scoutfs_btree_lookup(sb, meta, &key, &curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0) { /* XXX corruption, should always have dirent for backref */ if (ret == -ENOENT) @@ -861,10 +907,14 @@ retry: goto out; } - dent = curs.val; - len = item_name_len(&curs); + /* XXX corruption */ + if (ret < sizeof(dent)) { + ret = -EIO; + goto out; + } - trace_printk("dent ino %llu len %d\n", le64_to_cpu(dent->ino), len); + len = ret - sizeof(dent); + trace_printk("dent ino %llu len %d\n", le64_to_cpu(dent.ino), len); /* XXX corruption */ if (len < 1 || len > SCOUTFS_NAME_LEN) { @@ -873,18 +923,16 @@ retry: } /* XXX corruption, dirents should always match link backref */ - if (le64_to_cpu(dent->ino) != ino) { + if (le64_to_cpu(dent.ino) != ino) { ret = -EIO; goto out; } (*ctr)++; comp->len = len; - memcpy(comp->name, dent->name, len); list_add(&comp->head, list); comp = NULL; /* won't be freed */ - scoutfs_btree_release(&curs); ret = 1; out: if (inode) { diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 75faacc0..4cbac7b3 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -203,12 +203,11 @@ static bool bmap_has_blocks(struct scoutfs_block_map *bmap) int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_block_map *bmap; - struct scoutfs_key first; + struct scoutfs_block_map bmap; + struct scoutfs_btree_val val; struct scoutfs_key last; struct scoutfs_key key; - bool delete; + bool modified; u64 iblock; u64 blkno; int ret; @@ -217,27 +216,38 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) iblock = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); i = iblock & SCOUTFS_BLOCK_MAP_MASK; - scoutfs_set_key(&first, ino, SCOUTFS_BMAP_KEY, + scoutfs_set_key(&key, ino, SCOUTFS_BMAP_KEY, iblock & ~(u64)SCOUTFS_BLOCK_MAP_MASK); scoutfs_set_key(&last, ino, SCOUTFS_BMAP_KEY, ~0ULL); trace_printk("iblock %llu i %d\n", iblock, i); - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - key = *curs.key; - first = *curs.key; - scoutfs_inc_key(&first); - scoutfs_btree_release(&curs); + scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); - ret = scoutfs_btree_update(sb, meta, &key, &curs); + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* XXX corruption */ + if (ret != sizeof(bmap)) { + ret = -EIO; + break; + } + + /* XXX check bmap sanity */ + + /* make sure we can update bmap after freeing */ + ret = scoutfs_btree_dirty(sb, meta, &key); if (ret) break; - /* XXX check sanity */ - bmap = curs.val; - + modified = false; for (; i < SCOUTFS_BLOCK_MAP_COUNT; i++) { - blkno = le64_to_cpu(bmap->blkno[i]); + blkno = le64_to_cpu(bmap.blkno[i]); if (blkno == 0) continue; @@ -245,23 +255,22 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) if (ret) break; - bmap->blkno[i] = 0; + bmap.blkno[i] = 0; + modified = true; } - delete = !bmap_has_blocks(bmap); + i = 0; + + /* dirtying should have prevented these from failing */ + if (!bmap_has_blocks(&bmap)) + scoutfs_btree_delete(sb, meta, &key); + else if (modified) + scoutfs_btree_update(sb, meta, &key, &val); - scoutfs_btree_release(&curs); if (ret) break; - i = 0; - - if (delete) { - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret) - break; - } - /* XXX sync transaction if it's enormous */ + scoutfs_inc_key(&key); } return ret; @@ -303,8 +312,8 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) { struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_block_map *bmap; + struct scoutfs_btree_val val; + struct scoutfs_block_map bmap; struct scoutfs_key key; int ret; int i; @@ -312,18 +321,21 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) *blkno = 0; set_bmap_key(&key, inode, iblock); - ret = scoutfs_btree_lookup(sb, meta, &key, &curs); - if (!ret) { - bmap = curs.val; + scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret == sizeof(bmap)) { i = iblock & SCOUTFS_BLOCK_MAP_MASK; - *blkno = le64_to_cpu(bmap->blkno[i]); + *blkno = le64_to_cpu(bmap.blkno[i]); - while (i < SCOUTFS_BLOCK_MAP_COUNT && bmap->blkno[i]) { + ret = 0; + while (i < SCOUTFS_BLOCK_MAP_COUNT && bmap.blkno[i]) { ret++; i++; } - scoutfs_btree_release(&curs); + } else if (ret >= 0) { + /* XXX corruption */ + ret = -EIO; } else if (ret == -ENOENT) { ret = 0; } @@ -350,8 +362,8 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) { struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_block_map *bmap; + struct scoutfs_block_map bmap; + struct scoutfs_btree_val val; struct scoutfs_key key; bool inserted = false; u64 old_blkno = 0; @@ -361,25 +373,35 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) int i; set_bmap_key(&key, inode, iblock); + scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); - /* we always need a writable block map item */ - ret = scoutfs_btree_update(sb, meta, &key, &curs); + /* see if there's an existing mapping */ + ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0 && ret != -ENOENT) goto out; - /* might need to create a new item and delete it after errors */ + /* make sure that updating the bmap item won't fail */ if (ret == -ENOENT) { - ret = scoutfs_btree_insert(sb, meta, &key, sizeof(*bmap), - &curs); - if (ret < 0) + memset(&bmap, 0, sizeof(bmap)); + ret = scoutfs_btree_insert(sb, meta, &key, &val); + if (ret) goto out; - memset(curs.val, 0, sizeof(*bmap)); inserted = true; + + } else { + /* XXX corruption */ + if (ret != sizeof(bmap)) { + ret = -EIO; + goto out; + } + + ret = scoutfs_btree_dirty(sb, meta, &key); + if (ret) + goto out; } - bmap = curs.val; i = iblock & SCOUTFS_BLOCK_MAP_MASK; - old_blkno = le64_to_cpu(bmap->blkno[i]); + old_blkno = le64_to_cpu(bmap.blkno[i]); /* * If the existing block was free in stable then its dirty in @@ -406,12 +428,16 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) goto out; } - bmap->blkno[i] = cpu_to_le64(new_blkno); + bmap.blkno[i] = cpu_to_le64(new_blkno); + + /* dirtying guarantees success */ + err = scoutfs_btree_update(sb, meta, &key, &val); + BUG_ON(err); + *blkno_ret = new_blkno; new_blkno = 0; ret = 0; out: - scoutfs_btree_release(&curs); if (ret) { if (new_blkno) return_file_block(sb, new_blkno); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 779dee62..e60d92c6 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -125,21 +125,25 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) static int scoutfs_read_locked_inode(struct inode *inode) { - DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; struct scoutfs_key key; int ret; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); - ret = scoutfs_btree_lookup(sb, meta, &key, &curs); - if (!ret) { - load_inode(inode, curs.val); - scoutfs_btree_release(&curs); + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret == sizeof(sinode)) { + load_inode(inode, &sinode); + ret = 0; + } else if (ret >= 0) { + ret = -EIO; } - return 0; + return ret; } static int scoutfs_iget_test(struct inode *inode, void *arg) @@ -252,19 +256,20 @@ int scoutfs_dirty_inode_item(struct inode *inode) */ void scoutfs_update_inode_item(struct inode *inode) { - DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; struct scoutfs_key key; int err; scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + store_inode(&sinode, inode); - err = scoutfs_btree_update(sb, meta, &key, &curs); + err = scoutfs_btree_update(sb, meta, &key, &val); BUG_ON(err); - store_inode(curs.val, inode); - scoutfs_btree_release(&curs); trace_scoutfs_update_inode(inode); } @@ -313,8 +318,9 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); struct scoutfs_inode_info *ci; + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; struct scoutfs_key key; struct inode *inode; u64 ino; @@ -341,15 +347,15 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, set_inode_ops(inode); scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + store_inode(&sinode, inode); - ret = scoutfs_btree_insert(inode->i_sb, meta, &key, - sizeof(struct scoutfs_inode), &curs); + ret = scoutfs_btree_insert(inode->i_sb, meta, &key, &val); if (ret) { iput(inode); return ERR_PTR(ret); } - scoutfs_btree_release(&curs); return inode; } @@ -359,22 +365,28 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, static void drop_inode_items(struct super_block *sb, u64 ino) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_inode *sinode; + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; struct scoutfs_key key; bool release = false; umode_t mode; int ret; - /* sample the inode mode */ + /* sample the inode mode, XXX don't need to copy whole thing here */ scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); - ret = scoutfs_btree_lookup(sb, meta, &key, &curs); - if (ret) + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret < 0) goto out; - sinode = curs.val; - mode = le32_to_cpu(sinode->mode); - scoutfs_btree_release(&curs); + /* XXX corruption */ + if (ret != sizeof(sinode)) { + ret = -EIO; + goto out; + } + + mode = le32_to_cpu(sinode.mode); ret = scoutfs_hold_trans(sb); if (ret) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 4f5c20e7..ff7036ae 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -39,9 +39,9 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) struct scoutfs_ioctl_inodes_since args; struct scoutfs_ioctl_ino_seq __user *uiseq; struct scoutfs_ioctl_ino_seq iseq; - struct scoutfs_key first; + struct scoutfs_key key; struct scoutfs_key last; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + u64 seq; long bytes; int ret; @@ -52,34 +52,25 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) if (args.buf_len < sizeof(iseq) || args.buf_len > INT_MAX) return -EINVAL; - scoutfs_set_key(&first, args.first_ino, SCOUTFS_INODE_KEY, 0); + scoutfs_set_key(&key, args.first_ino, SCOUTFS_INODE_KEY, 0); scoutfs_set_key(&last, args.last_ino, SCOUTFS_INODE_KEY, 0); bytes = 0; - while ((ret = scoutfs_btree_since(sb, meta, &first, &last, - args.seq, &curs)) > 0) { + for (;;) { + ret = scoutfs_btree_since(sb, meta, &key, &last, args.seq, + &key, &seq, NULL); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } - iseq.ino = scoutfs_key_inode(curs.key); - iseq.seq = curs.seq; + iseq.ino = scoutfs_key_inode(&key); + iseq.seq = seq; - /* - * We can't copy to userspace with our locks held - * because faults could try to use tree blocks that we - * have locked. If a non-faulting copy fails we release - * the cursor and try a blocking copy and pick up where - * we left off. - */ - pagefault_disable(); - ret = __copy_to_user_inatomic(uiseq, &iseq, sizeof(iseq)); - pagefault_enable(); - if (ret) { - first = *curs.key; - scoutfs_inc_key(&first); - scoutfs_btree_release(&curs); - if (copy_to_user(uiseq, &iseq, sizeof(iseq))) { - ret = -EFAULT; - break; - } + if (copy_to_user(uiseq, &iseq, sizeof(iseq))) { + ret = -EFAULT; + break; } uiseq++; @@ -88,9 +79,9 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) ret = 0; break; } - } - scoutfs_btree_release(&curs); + scoutfs_inc_key(&key); + } if (bytes) ret = bytes; @@ -219,16 +210,14 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, struct super_block *sb = file_inode(file)->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_ioctl_find_xattr args; - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_key first; + struct scoutfs_key key; struct scoutfs_key last; char __user *ustr; u64 __user *uino; - u64 inos[32]; char *str; - int nr_inos = 0; int copied = 0; - int ret; + int ret = 0; + u64 ino; u8 type; u64 h; @@ -238,6 +227,9 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, if (args.str_len > SCOUTFS_MAX_XATTR_LEN || args.ino_count > INT_MAX) return -EINVAL; + if (args.first_ino > args.last_ino) + return -EINVAL; + if (args.ino_count == 0) return 0; @@ -262,36 +254,27 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, type = SCOUTFS_XATTR_VAL_HASH_KEY; } - scoutfs_set_key(&first, h, type, args.first_ino); + scoutfs_set_key(&key, h, type, args.first_ino); scoutfs_set_key(&last, h, type, args.last_ino); while (copied < args.ino_count) { - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, - &curs)) > 0) { - inos[nr_inos++] = scoutfs_key_offset(curs.key); - - first = *curs.key; - scoutfs_inc_key(&first); - - if (nr_inos == ARRAY_SIZE(inos) || - (nr_inos + copied) == args.ino_count) { - scoutfs_btree_release(&curs); + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, NULL); + if (ret < 0) { + if (ret == -ENOENT) ret = 0; - break; - } - } - if (ret < 0 || nr_inos == 0) break; + } - if (copy_to_user(uino, inos, nr_inos * sizeof(u64))) { + ino = scoutfs_key_offset(&key); + if (put_user(ino, uino)) { ret = -EFAULT; break; } - uino += nr_inos; - copied += nr_inos; - nr_inos = 0; + uino++; + copied++; + scoutfs_inc_key(&key); } out: diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 97ad19d7..abfc6a8c 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -115,30 +115,49 @@ static int search_xattr_items(struct inode *inode, const char *name, { struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); - struct scoutfs_key first; - struct scoutfs_key last; + struct scoutfs_btree_val val; struct scoutfs_xattr *xat; + struct scoutfs_key last; + struct scoutfs_key key; + unsigned int max_len; int ret; - set_xattr_keys(inode, &first, &last, name, name_len); + max_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, SCOUTFS_MAX_XATTR_LEN), + xat = kmalloc(max_len, GFP_KERNEL); + if (!xat) + return -ENOMEM; + + set_xattr_keys(inode, &key, &last, name, name_len); + scoutfs_btree_init_val(&val, xat, max_len); res->found = false; res->other_coll = false; res->found_hole = false; - res->hole_key = first; + res->hole_key = key; - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - xat = curs.val; + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* XXX corruption */ + if (ret < sizeof(struct scoutfs_xattr) || + ret != xat_bytes(xat->name_len, xat->value_len)) { + ret = -EIO; + break; + } /* found a hole when we skip past next expected key */ if (!res->found_hole && - scoutfs_key_cmp(&res->hole_key, curs.key) < 0) + scoutfs_key_cmp(&res->hole_key, &key) < 0) res->found_hole = true; - /* keep searching for a hole past this cursor key */ + /* keep searching for a hole past this key */ if (!res->found_hole) { - res->hole_key = *curs.key; + res->hole_key = key; scoutfs_inc_key(&res->hole_key); } @@ -147,7 +166,7 @@ static int search_xattr_items(struct inode *inode, const char *name, scoutfs_names_equal(name, name_len, xat->name, xat->name_len)) { res->found = true; - res->key = *curs.key; + res->key = key; res->val_hash = scoutfs_name_hash(xat_value(xat), xat->value_len); } else { @@ -157,11 +176,13 @@ static int search_xattr_items(struct inode *inode, const char *name, /* finished once we have all the caller needs */ if (res->found && res->other_coll && res->found_hole) { ret = 0; - scoutfs_btree_release(&curs); break; } + + scoutfs_inc_key(&key); } + kfree(xat); return ret; } @@ -178,56 +199,55 @@ static int insert_xattr(struct inode *inode, const char *name, { struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); bool inserted_name_hash_item = false; - __le64 * __packed refcount; + struct scoutfs_btree_val val; + __le64 refcount; struct scoutfs_key name_key; struct scoutfs_key val_key; - struct scoutfs_xattr *xat; + struct scoutfs_xattr xat; int ret; + /* insert the main xattr item */ set_name_val_keys(&name_key, &val_key, key, val_hash); + scoutfs_btree_init_val(&val, &xat, sizeof(xat), (void *)name, name_len, + (void *)value, size); - ret = scoutfs_btree_insert(sb, meta, key, - xat_bytes(name_len, size), &curs); + xat.name_len = name_len; + xat.value_len = size; + + ret = scoutfs_btree_insert(sb, meta, key, &val); if (ret) return ret; - xat = curs.val; - xat->name_len = name_len; - xat->value_len = size; - memcpy(xat->name, name, name_len); - memcpy(xat_value(xat), value, size); - - scoutfs_btree_release(&curs); - /* insert the name hash item for find_xattr if we're first */ if (!other_coll) { - ret = scoutfs_btree_insert(sb, meta, &name_key, 0, &curs); + ret = scoutfs_btree_insert(sb, meta, &name_key, NULL); /* XXX eexist would be corruption */ if (ret) goto out; - scoutfs_btree_release(&curs); inserted_name_hash_item = true; } /* increment the val hash item for find_xattr, inserting if first */ - ret = scoutfs_btree_update(sb, meta, &val_key, &curs); - if (ret == -ENOENT) { - ret = scoutfs_btree_insert(sb, meta, &val_key, - sizeof(*refcount), &curs); - if (ret == 0) { - /* XXX test sane item size */ - refcount = curs.val; - *refcount = 0; - } - } - if (ret == 0) { - refcount = curs.val; - le64_add_cpu(refcount, 1); - scoutfs_btree_release(&curs); - } + scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); + ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == -ENOENT) { + refcount = cpu_to_le64(1); + ret = scoutfs_btree_insert(sb, meta, &val_key, &val); + } else { + /* XXX corruption */ + if (ret != sizeof(refcount)) { + ret = -EIO; + goto out; + } + + le64_add_cpu(&refcount, 1); + ret = scoutfs_btree_update(sb, meta, &val_key, &val); + } out: if (ret) { scoutfs_btree_delete(sb, meta, key); @@ -247,15 +267,29 @@ static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, bool other_coll, u64 val_hash) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_btree_val val; struct scoutfs_key name_key; struct scoutfs_key val_key; - __le64 * __packed refcount; - bool del_val = false; + __le64 refcount; int ret; set_name_val_keys(&name_key, &val_key, key, val_hash); + /* update the val_hash refcount, making sure it's not nonsense */ + scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); + ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); + if (ret < 0) + goto out; + + /* XXX corruption */ + if (ret != sizeof(refcount)) { + ret = -EIO; + goto out; + } + + le64_add_cpu(&refcount, -1ULL); + + /* ensure that we can update and delete name_ and val_ keys */ if (!other_coll) { ret = scoutfs_btree_dirty(sb, meta, &name_key); if (ret) @@ -272,14 +306,9 @@ static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, if (!other_coll) scoutfs_btree_delete(sb, meta, &name_key); - scoutfs_btree_update(sb, meta, &val_key, &curs); - refcount = curs.val; - le64_add_cpu(refcount, -1ULL); - if (*refcount == 0) - del_val = true; - scoutfs_btree_release(&curs); - - if (del_val) + if (refcount) + scoutfs_btree_update(sb, meta, &val_key, &val); + else scoutfs_btree_delete(sb, meta, &val_key); ret = 0; out: @@ -295,6 +324,11 @@ static int unknown_prefix(const char *name) return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); } +/* + * Look up an xattr matching the given name. We walk our xattr items stored + * at the hashed name. We'll only be able to copy out a value that fits + * in the callers buffer. + */ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, size_t size) { @@ -302,27 +336,49 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *si = SCOUTFS_I(inode); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); size_t name_len = strlen(name); + struct scoutfs_btree_val val; struct scoutfs_xattr *xat; - struct scoutfs_key first; + struct scoutfs_key key; struct scoutfs_key last; + unsigned int item_len; int ret; if (unknown_prefix(name)) return -EOPNOTSUPP; - set_xattr_keys(inode, &first, &last, name, name_len); + /* make sure we don't allocate an enormous item */ + if (name_len > SCOUTFS_MAX_XATTR_LEN) + return -ENODATA; + size = min_t(size_t, size, SCOUTFS_MAX_XATTR_LEN); + + item_len = xat_bytes(name_len, size); + xat = kmalloc(item_len, GFP_KERNEL); + if (!xat) + return -ENOMEM; + + set_xattr_keys(inode, &key, &last, name, name_len); + scoutfs_btree_init_val(&val, xat, item_len); down_read(&si->xattr_rwsem); - ret = -ENODATA; - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - xat = curs.val; + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENODATA; + break; + } + + /* XXX corruption */ + if (ret < sizeof(struct scoutfs_xattr)) { + ret = -EIO; + break; + } if (!scoutfs_names_equal(name, name_len, xat->name, xat->name_len)) { - ret = -ENODATA; + scoutfs_inc_key(&key); continue; } @@ -333,12 +389,12 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, else ret = -ERANGE; } - scoutfs_btree_release(&curs); break; } up_read(&si->xattr_rwsem); + kfree(xat); return ret; } @@ -458,39 +514,60 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_btree_val val; struct scoutfs_xattr *xat; - struct scoutfs_key first; + struct scoutfs_key key; struct scoutfs_key last; + unsigned int item_len; ssize_t total; int ret; - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, 0); + item_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, 0); + xat = kmalloc(item_len, GFP_KERNEL); + if (!xat) + return -ENOMEM; + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, 0); scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, ~0ULL); + scoutfs_btree_init_val(&val, xat, item_len); down_read(&si->xattr_rwsem); total = 0; - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - xat = curs.val; - - total += xat->name_len + 1; - if (!size) - continue; - if (!buffer || total > size) { - ret = -ERANGE; + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; break; } - memcpy(buffer, xat->name, xat->name_len); - buffer += xat->name_len; - *(buffer++) = '\0'; + /* XXX corruption */ + if (ret < sizeof(struct scoutfs_xattr)) { + ret = -EIO; + break; + } + + total += xat->name_len + 1; + + if (size) { + if (!buffer || total > size) { + ret = -ERANGE; + break; + } + + memcpy(buffer, xat->name, xat->name_len); + buffer += xat->name_len; + *(buffer++) = '\0'; + } + + scoutfs_inc_key(&key); } - scoutfs_btree_release(&curs); - up_read(&si->xattr_rwsem); + kfree(xat); + return ret < 0 ? ret : total; } @@ -504,60 +581,66 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) * * Hash items can be shared amongst xattrs whose names or values hash to * the same hash value. We don't bother trying to remove the hash items - * as the last xattr is removed. We remove it the first chance we get, - * try to avoid obviously removing the same hash item next, and allow + * as the last xattr is removed. We always try to remove them and allow * failure when we try to remove a hash item that wasn't found. */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - DECLARE_SCOUTFS_BTREE_CURSOR(curs); + struct scoutfs_btree_val val; struct scoutfs_xattr *xat; - struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; struct scoutfs_key name_key; struct scoutfs_key val_key; - __le64 last_name; - __le64 last_val; + unsigned int item_len; u64 val_hash; - bool have_last; int ret; - scoutfs_set_key(&first, ino, SCOUTFS_XATTR_KEY, 0); + scoutfs_set_key(&key, ino, SCOUTFS_XATTR_KEY, 0); scoutfs_set_key(&last, ino, SCOUTFS_XATTR_KEY, ~0ULL); - have_last = false; - while ((ret = scoutfs_btree_next(sb, meta, &first, &last, &curs)) > 0) { - xat = curs.val; - key = *curs.key; + item_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, SCOUTFS_MAX_XATTR_LEN), + xat = kmalloc(item_len, GFP_KERNEL); + if (!xat) + return -ENOMEM; + + scoutfs_btree_init_val(&val, xat, item_len); + + for (;;) { + ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* XXX corruption */ + if (ret < sizeof(struct scoutfs_xattr) || + ret != xat_bytes(xat->name_len, xat->value_len)) { + ret = -EIO; + break; + } + val_hash = scoutfs_name_hash(xat_value(xat), xat->value_len); set_name_val_keys(&name_key, &val_key, &key, val_hash); - first = *curs.key; - scoutfs_inc_key(&first); - scoutfs_btree_release(&curs); + ret = scoutfs_btree_delete(sb, meta, &name_key); + if (ret && ret != -ENOENT) + break; - if (!have_last || last_name != name_key.inode) { - ret = scoutfs_btree_delete(sb, meta, &name_key); - if (ret && ret != -ENOENT) - break; - last_name = name_key.inode; - } - - if (!have_last || last_val != val_key.inode) { - ret = scoutfs_btree_delete(sb, meta, &val_key); - if (ret && ret != -ENOENT) - break; - last_val = val_key.inode; - } - - have_last = true; + ret = scoutfs_btree_delete(sb, meta, &val_key); + if (ret && ret != -ENOENT) + break; ret = scoutfs_btree_delete(sb, meta, &key); if (ret && ret != -ENOENT) break; + + scoutfs_inc_key(&key); } + kfree(xat); + return ret; } From bb3a5742f420488c106a50987d7c526067f1189d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Sep 2016 09:51:42 -0700 Subject: [PATCH 100/920] scoutfs: drop sib bh ref in split We forgot to drop the sibling bh reference while splitting. Oopsie! Signed-off-by: Zach Brown --- kmod/src/btree.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index d17a1b76..b3498858 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -703,6 +703,8 @@ static struct buffer_head *try_merge(struct super_block *sb, free_tree_block(sb, parent->hdr.blkno); } + scoutfs_block_put(sib_bh); + return bh; } From cf0199da00db6902a19d6c22d29c39436da599f0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Sep 2016 09:59:53 -0700 Subject: [PATCH 101/920] scoutfs: allow more concurrent btree locking The btree locking so far was a quick interim measure to get the rest of the system going. We want to clean it up both for correctness and performance but also to make way for using the btree for block allocation. We were unconditionally using the buffer head lock for tree block locking. This is bad for at least four reasons: it's invisible to lockdep, it doesn't allow concurrent reads, it doesn't allow reading while a block is being written during the transaction, and it's not necessary at all when the for stable read-only blocks. Instead we add a rwsem to the buffer head private which we use to lock the block when it's writable. We clean up the locking functions to make it clearer that btree_walk holds one lock at a time and either returns it to the caller with the buffer head or unlocks the parent if its returning an error. We also add the missing sibling block locking during splits and merges. Locking the parent prevented walks from descending down our path but it didn't protect against previous walks that were already down at our sibling's level. Getting all this working with lockdep adds a bit more class/subclass plumbing calls but nothing too ornerous. Signed-off-by: Zach Brown --- kmod/src/block.c | 51 ++++++++-- kmod/src/block.h | 5 + kmod/src/btree.c | 246 ++++++++++++++++++++++++++++++++++------------ kmod/src/format.h | 15 +++ kmod/src/super.h | 1 - 5 files changed, 245 insertions(+), 73 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 7c049a9a..0f6269af 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -42,6 +42,8 @@ struct block_bh_private { struct super_block *sb; struct buffer_head *bh; struct rb_node node; + struct rw_semaphore rwsem; + bool rwsem_class; }; enum { @@ -146,6 +148,9 @@ static int insert_bhp(struct super_block *sb, struct buffer_head *bh) bhp->bh = bh; get_bh(bh); bh->b_private = bhp; + /* lockdep class can be set by callers that use the lock */ + init_rwsem(&bhp->rwsem); + bhp->rwsem_class = false; spin_lock_irqsave(&sbi->block_lock, flags); insert_bhp_rb(&sbi->block_dirty_tree, bh); @@ -306,13 +311,6 @@ int scoutfs_block_write_dirty(struct super_block *sb) atomic_inc(&sbi->block_writes); scoutfs_block_set_crc(bh); - /* - * XXX submit_bh() forces us to lock the block while IO is - * in flight. This is unfortunate because we use the buffer - * head lock to serialize access to btree block contents. - * We should fix that and only use the buffer head lock - * when the APIs force us to. - */ lock_buffer(bh); bh->b_end_io = block_write_end_io; @@ -486,3 +484,42 @@ void scoutfs_block_zero(struct buffer_head *bh, size_t off) if (off < SCOUTFS_BLOCK_SIZE) memset((char *)bh->b_data + off, 0, SCOUTFS_BLOCK_SIZE - off); } + +void scoutfs_block_set_lock_class(struct buffer_head *bh, + struct lock_class_key *class) +{ + struct block_bh_private *bhp = bh->b_private; + + if (bhp && !bhp->rwsem_class) { + lockdep_set_class(&bhp->rwsem, class); + bhp->rwsem_class = true; + } +} + +void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass) +{ + struct block_bh_private *bhp = bh->b_private; + + trace_printk("lock write %d bhp %p\n", write, bhp); + + if (bhp) { + if (write) + down_write_nested(&bhp->rwsem, subclass); + else + down_read_nested(&bhp->rwsem, subclass); + } +} + +void scoutfs_block_unlock(struct buffer_head *bh, bool write) +{ + struct block_bh_private *bhp = bh->b_private; + + trace_printk("unlock write %d bhp %p\n", write, bhp); + + if (bhp) { + if (write) + up_write(&bhp->rwsem); + else + up_read(&bhp->rwsem); + } +} diff --git a/kmod/src/block.h b/kmod/src/block.h index d431d90e..32ea0f38 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -19,6 +19,11 @@ int scoutfs_block_write_dirty(struct super_block *sb); void scoutfs_block_set_crc(struct buffer_head *bh); void scoutfs_block_zero(struct buffer_head *bh, size_t off); +void scoutfs_block_set_lock_class(struct buffer_head *bh, + struct lock_class_key *class); +void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass); +void scoutfs_block_unlock(struct buffer_head *bh, bool write); + /* XXX seems like this should be upstream :) */ static inline void *bh_data(struct buffer_head *bh) { diff --git a/kmod/src/btree.c b/kmod/src/btree.c index b3498858..fbd40f7c 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -417,6 +417,141 @@ static void compact_items(struct scoutfs_btree_block *bt) sort_key_cmp, sort_off_swap); } + +/* + * Let's talk about btree locking. + * + * The main metadata btree has lots of callers who want concurrency. + * They have their own locks that protect multi item consistency -- say + * an inode's i_mutex protecting the items related to a given inode. + * But it's our responsibility to lock the btree itself. + * + * Our btree operations are implemented with a single walk down the + * tree. This gives us the opportunity to cascade block locks down the + * tree. We first lock the root. Then we lock the first block and + * unlock the root. Then lock the next block and unlock the first + * block. And so on down the tree. Except for that brief transition + * function the btree walk always holds a single lock on either the root + * or a block at a level. After contention on the root and first block + * we have lots of concurrency down paths of the tree to the leaves. + * + * As we walk down the tree we have to split or merge. While we do this + * we hold the parent block lock. We have to also lock the sibling + * blocks. We always acquire them left to right to avoid deadlocks. + * + * The cow tree updates let us skip block locking entirely for stable + * blocks because they're read only. The block layer only has to worry + * about locking blocks that could be written to. While they're + * writable they have a buffer_head private that pins them in the + * transaction and we store the block lock there. The block layer + * ignores our locking attempts for read-only blocks. + * + * lockdep has to not be freaked out by all of this. The cascading + * block locks really make it angry without annotation so we add classes + * for each level and use nested subclasses for the locking of siblings + * during split and merge. + * + * We also use the btree API for the block allocator. This introduces + * nesting btree allocator calls inside main fs metadata btree calls. + * The locking would be safe as the blocks will never be in both trees + * but lockdep would think they're the same class and get raise + * warnings. We'd need to have tree level classes for all the trees. + * It turns out that the allocator has to maintain multi-item + * consistency across its entire tree so it has a tree-wide lock. We + * don't have to lock the btree at all when we're working on the + * allocator roots. They're the only non-metadata roots so far so we + * invert the test and only lock the btree when we're working on the + * main metadata btree root. + */ + +static void set_block_lock_class(struct buffer_head *bh, int level) +{ +#ifdef CONFIG_LOCKDEP + static struct lock_class_key tree_depth_classes[SCOUTFS_BTREE_MAX_DEPTH]; + + scoutfs_block_set_lock_class(bh, &tree_depth_classes[level]); +#endif +} + +static void lock_root(struct super_block *sb, struct scoutfs_btree_root *root, + bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (root == &sbi->super.btree_root) { + if (write) + down_write(&sbi->btree_rwsem); + else + down_read(&sbi->btree_rwsem); + } +} + +static void unlock_root(struct super_block *sb, bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (write) + up_write(&sbi->btree_rwsem); + else + up_read(&sbi->btree_rwsem); +} + +static void unlock_level(struct super_block *sb, + struct scoutfs_btree_root *root, + struct buffer_head *bh, bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (root == &sbi->super.btree_root) { + if (bh) + scoutfs_block_unlock(bh, write); + else + unlock_root(sb, write); + } +} + +static void lock_next_level(struct super_block *sb, + struct scoutfs_btree_root *root, + struct buffer_head *par_bh, + struct buffer_head *bh, bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (root == &sbi->super.btree_root) { + scoutfs_block_lock(bh, write, 0); + + unlock_level(sb, root, par_bh, write); + } +} + +static void lock_siblings(struct super_block *sb, + struct scoutfs_btree_root *root, + struct buffer_head *left, struct buffer_head *right, + bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (root == &sbi->super.btree_root) { + scoutfs_block_lock(left, write, 0); + scoutfs_block_lock(right, write, 1); + } +} + +static void unlock_siblings(struct super_block *sb, + struct scoutfs_btree_root *root, + struct buffer_head *left, struct buffer_head *right, + bool write) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (root == &sbi->super.btree_root) { + scoutfs_block_unlock(left, write); + scoutfs_block_unlock(right, write); + } +} + + + /* sorting relies on masking pointers to find the containing block */ static inline struct buffer_head *check_bh_alignment(struct buffer_head *bh) { @@ -477,12 +612,14 @@ static struct buffer_head *grow_tree(struct super_block *sb, root->height++; root->ref.blkno = hdr->blkno; root->ref.seq = hdr->seq; + + set_block_lock_class(bh, root->height - 1); } return bh; } -static struct buffer_head *get_block_ref(struct super_block *sb, +static struct buffer_head *get_block_ref(struct super_block *sb, int level, struct scoutfs_block_ref *ref, bool dirty) { @@ -493,6 +630,9 @@ static struct buffer_head *get_block_ref(struct super_block *sb, else bh = scoutfs_block_read_ref(sb, ref); + if (!IS_ERR(bh)) + set_block_lock_class(bh, level); + return check_bh_alignment(bh); } @@ -549,6 +689,7 @@ static struct buffer_head *try_split(struct super_block *sb, struct buffer_head *par_bh = NULL; struct scoutfs_key maximal; unsigned int all_bytes; + bool swap_return = false; if (level) val_len = sizeof(struct scoutfs_block_ref); @@ -586,23 +727,28 @@ static struct buffer_head *try_split(struct super_block *sb, create_parent_item(parent, parent_pos, right, &maximal); } + lock_siblings(sb, root, left_bh, right_bh, true); + move_items(left, right, false, used_total(right) / 2); create_parent_item(parent, parent_pos, left, greatest_key(left)); parent_pos++; /* not that anything uses it again :P */ if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { /* insertion will go to the new left block */ - scoutfs_block_put(right_bh); - right_bh = left_bh; + swap_return = true; } else { - scoutfs_block_put(left_bh); - /* insertion will still go through us, might need to compact */ if (contig_free(right) < all_bytes) compact_items(right); } + unlock_siblings(sb, root, left_bh, right_bh, true); + + if (swap_return) + swap(right_bh, left_bh); + scoutfs_block_put(par_bh); + scoutfs_block_put(left_bh); return right_bh; } @@ -631,7 +777,7 @@ static struct buffer_head *try_split(struct super_block *sb, static struct buffer_head *try_merge(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, - unsigned int pos, + int level, unsigned int pos, struct buffer_head *bh) { struct scoutfs_btree_block *bt = bh_data(bh); @@ -655,7 +801,7 @@ static struct buffer_head *try_merge(struct super_block *sb, } sib_item = pos_item(parent, sib_pos); - sib_bh = get_block_ref(sb, (void *)sib_item->val, true); + sib_bh = get_block_ref(sb, level, (void *)sib_item->val, true); if (IS_ERR(sib_bh)) { /* XXX do we need to unlock this? don't think so */ scoutfs_block_put(bh); @@ -663,6 +809,11 @@ static struct buffer_head *try_merge(struct super_block *sb, } sib_bt = bh_data(sib_bh); + if (move_right) + lock_siblings(sb, root, sib_bh, bh, true); + else + lock_siblings(sb, root, bh, sib_bh, true); + if (used_total(sib_bt) <= reclaimable_free(bt)) to_move = used_total(sib_bt); else @@ -703,6 +854,11 @@ static struct buffer_head *try_merge(struct super_block *sb, free_tree_block(sb, parent->hdr.blkno); } + if (move_right) + unlock_siblings(sb, root, sib_bh, bh, true); + else + unlock_siblings(sb, root, bh, sib_bh, true); + scoutfs_block_put(sib_bh); return bh; @@ -716,44 +872,6 @@ enum { WALK_DIRTY, }; -static inline void lock_root(struct scoutfs_sb_info *sbi, bool dirty) -{ - if (dirty) - down_write(&sbi->btree_rwsem); - else - down_read(&sbi->btree_rwsem); -} - -static inline void unlock_root(struct scoutfs_sb_info *sbi, bool dirty) -{ - if (dirty) - up_write(&sbi->btree_rwsem); - else - up_read(&sbi->btree_rwsem); -} - -/* - * As we descend we lock parent blocks (or the root), then lock the child, - * then unlock the parent. - */ -static inline void lock_block(struct scoutfs_sb_info *sbi, - struct buffer_head *bh, bool dirty) -{ - if (bh == NULL) - lock_root(sbi, dirty); - else - lock_buffer(bh); -} - -static inline void unlock_block(struct scoutfs_sb_info *sbi, - struct buffer_head *bh, bool dirty) -{ - if (bh == NULL) - unlock_root(sbi, dirty); - else - unlock_buffer(bh); -} - static u64 item_block_ref_seq(struct scoutfs_btree_item *item) { struct scoutfs_block_ref *ref = (void *)item->val; @@ -914,7 +1032,6 @@ static struct buffer_head *btree_walk(struct super_block *sb, struct scoutfs_key *next_key, unsigned int val_len, u64 seq, int op) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_btree_block *parent = NULL; struct buffer_head *par_bh = NULL; struct buffer_head *bh = NULL; @@ -932,7 +1049,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, if (next_key) scoutfs_set_max_key(next_key); - lock_block(sbi, par_bh, dirty); + lock_root(sb, root, dirty); ref = &root->ref; level = root->height; @@ -943,17 +1060,16 @@ static struct buffer_head *btree_walk(struct super_block *sb, } else { bh = grow_tree(sb, root); if (!IS_ERR(bh)) - lock_block(sbi, bh, dirty); + lock_next_level(sb, root, NULL, bh, dirty); } - unlock_block(sbi, par_bh, dirty); - return bh; + goto out; } /* skip the whole tree if the root ref's seq is old */ if (op == WALK_NEXT_SEQ && le64_to_cpu(ref->seq) < seq) { - unlock_block(sbi, par_bh, dirty); - return ERR_PTR(-ENOENT); + bh = ERR_PTR(-ENOENT); + goto out; } scoutfs_set_key(&small, 0, 0, 0); @@ -961,7 +1077,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, while (level--) { /* XXX hmm, need to think about retry */ - bh = get_block_ref(sb, ref, dirty); + bh = get_block_ref(sb, level, ref, dirty); if (IS_ERR(bh)) break; @@ -977,17 +1093,15 @@ static struct buffer_head *btree_walk(struct super_block *sb, bh = try_split(sb, root, level, key, val_len, parent, pos, bh); if ((op == WALK_DELETE) && parent) - bh = try_merge(sb, root, parent, pos, bh); + bh = try_merge(sb, root, parent, level, pos, bh); if (IS_ERR(bh)) break; - lock_block(sbi, bh, dirty); + lock_next_level(sb, root, par_bh, bh, dirty); if (!level) break; - /* unlock parent before searching so others can use it */ - unlock_block(sbi, par_bh, dirty); scoutfs_block_put(par_bh); par_bh = bh; parent = bh_data(par_bh); @@ -1027,7 +1141,9 @@ static struct buffer_head *btree_walk(struct super_block *sb, large = item->key; } - unlock_block(sbi, par_bh, dirty); +out: + if (IS_ERR(bh)) + unlock_level(sb, root, par_bh, dirty); scoutfs_block_put(par_bh); return bh; @@ -1066,7 +1182,7 @@ int scoutfs_btree_lookup(struct super_block *sb, ret = -ENOENT; } - unlock_block(NULL, bh, false); + unlock_level(sb, root, bh, false); scoutfs_block_put(bh); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1120,7 +1236,7 @@ int scoutfs_btree_insert(struct super_block *sb, ret = -EEXIST; } - unlock_block(NULL, bh, true); + unlock_level(sb, root, bh, true); scoutfs_block_put(bh); return ret; @@ -1166,7 +1282,7 @@ int scoutfs_btree_delete(struct super_block *sb, ret = -ENOENT; } - unlock_block(NULL, bh, true); + unlock_level(sb, root, bh, true); scoutfs_block_put(bh); out: @@ -1234,7 +1350,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, pos = find_pos_after_seq(bt, &key, 0, seq, op); if (pos >= bt->nr_items) { key = next_key; - unlock_block(NULL, bh, false); + unlock_level(sb, root, bh, false); scoutfs_block_put(bh); continue; } @@ -1252,7 +1368,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; } - unlock_block(NULL, bh, false); + unlock_level(sb, root, bh, false); scoutfs_block_put(bh); break; } @@ -1311,7 +1427,7 @@ int scoutfs_btree_dirty(struct super_block *sb, ret = -ENOENT; } - unlock_block(NULL, bh, true); + unlock_level(sb, root, bh, true); scoutfs_block_put(bh); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1353,7 +1469,7 @@ int scoutfs_btree_update(struct super_block *sb, ret = -ENOENT; } - unlock_block(NULL, bh, true); + unlock_level(sb, root, bh, true); scoutfs_block_put(bh); return ret; diff --git a/kmod/src/format.h b/kmod/src/format.h index 932a9d2b..fd0cbd62 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -156,6 +156,21 @@ struct scoutfs_btree_item { (member_sizeof(struct scoutfs_btree_block, item_offs[0]) + \ sizeof(struct scoutfs_btree_item))) +/* + * We can calculate the max tree depth by calculating how many leaf + * blocks the tree could reference. The block device can only reference + * 2^64 bytes. The tallest parent tree has half full parent blocks. + * + * So we have the relation: + * + * ceil(max_items / 2) ^ (max_depth - 1) >= 2^64 / block_size + * + * and solve for depth: + * + * max_depth = log(ceil(max_items / 2), 2^64 / block_size) + 1 + */ +#define SCOUTFS_BTREE_MAX_DEPTH 10 + #define SCOUTFS_UUID_BYTES 16 struct scoutfs_super_block { diff --git a/kmod/src/super.h b/kmod/src/super.h index 79c2168f..6f550d0c 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -27,7 +27,6 @@ struct scoutfs_sb_info { struct mutex buddy_mutex; atomic_t buddy_count; - /* XXX there will be a lot more of these :) */ struct rw_semaphore btree_rwsem; atomic_t trans_holds; From f7f7a2e53fa15d27eefb52ffdf1979da41d6cf05 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Sep 2016 13:42:27 -0700 Subject: [PATCH 102/920] scoutfs: add scoutfs_block_zero_from() We already have a function that zeros the end of a block starting at a given offset. Some callers have a pointer to the byte to zero from so let's add a convenience function that calculates the offset from the pointer. Signed-off-by: Zach Brown --- kmod/src/block.c | 11 +++++++++++ kmod/src/block.h | 1 + 2 files changed, 12 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 0f6269af..2b1037e3 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -476,6 +476,9 @@ void scoutfs_block_set_crc(struct buffer_head *bh) hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } +/* + * Zero the block from the given byte to the end of the block. + */ void scoutfs_block_zero(struct buffer_head *bh, size_t off) { if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) @@ -485,6 +488,14 @@ void scoutfs_block_zero(struct buffer_head *bh, size_t off) memset((char *)bh->b_data + off, 0, SCOUTFS_BLOCK_SIZE - off); } +/* + * Zero the block from the given byte to the end of the block. + */ +void scoutfs_block_zero_from(struct buffer_head *bh, void *ptr) +{ + return scoutfs_block_zero(bh, (char *)ptr - (char *)bh->b_data); +} + void scoutfs_block_set_lock_class(struct buffer_head *bh, struct lock_class_key *class) { diff --git a/kmod/src/block.h b/kmod/src/block.h index 32ea0f38..2209f7fb 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -18,6 +18,7 @@ int scoutfs_block_write_dirty(struct super_block *sb); void scoutfs_block_set_crc(struct buffer_head *bh); void scoutfs_block_zero(struct buffer_head *bh, size_t off); +void scoutfs_block_zero_from(struct buffer_head *bh, void *ptr); void scoutfs_block_set_lock_class(struct buffer_head *bh, struct lock_class_key *class); From 9d08b34791f37ad58efef5c7156440177f4b3bba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Sep 2016 13:44:31 -0700 Subject: [PATCH 103/920] scoutfs: remove excessive block locking tracing I accidentally left some lock tracing in the btree locking commit that is very noisy and not particularly useful. Let's remove it. Signed-off-by: Zach Brown --- kmod/src/block.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 2b1037e3..7c3fcf7d 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -511,8 +511,6 @@ void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass) { struct block_bh_private *bhp = bh->b_private; - trace_printk("lock write %d bhp %p\n", write, bhp); - if (bhp) { if (write) down_write_nested(&bhp->rwsem, subclass); @@ -525,8 +523,6 @@ void scoutfs_block_unlock(struct buffer_head *bh, bool write) { struct block_bh_private *bhp = bh->b_private; - trace_printk("unlock write %d bhp %p\n", write, bhp); - if (bhp) { if (write) up_write(&bhp->rwsem); From 5601f8cef54dd8c4a9edc4e3a554fceb319b310d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Sep 2016 13:46:18 -0700 Subject: [PATCH 104/920] scoutfs: add scoutfs_block_forget() The upcoming allocator changes have a need to forget dirty blocks so they're not written. It proabably won't be the only one. Signed-off-by: Zach Brown --- kmod/src/block.c | 23 +++++++++++++++++++++++ kmod/src/block.h | 2 ++ 2 files changed, 25 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 7c3fcf7d..b8b3ccb8 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -469,6 +469,29 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) return bh; } +/* + * Make sure that we don't have a dirty block at the given blkno. If we + * do we remove it from our tree of dirty blocks and clear the buffer + * dirty bit. + * + * XXX for now callers have only needed to forget blknos, maybe they'll + * have the bh some day. + */ +void scoutfs_block_forget(struct super_block *sb, u64 blkno) +{ + struct block_bh_private *bhp; + struct buffer_head *bh; + + bh = sb_find_get_block(sb, blkno); + if (bh) { + bhp = bh->b_private; + if (bhp) { + erase_bhp(bh); + bforget(bh); + } + } +} + void scoutfs_block_set_crc(struct buffer_head *bh) { struct scoutfs_block_header *hdr = bh_data(bh); diff --git a/kmod/src/block.h b/kmod/src/block.h index 2209f7fb..cef25815 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -25,6 +25,8 @@ void scoutfs_block_set_lock_class(struct buffer_head *bh, void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass); void scoutfs_block_unlock(struct buffer_head *bh, bool write); +void scoutfs_block_forget(struct super_block *sb, u64 blkno); + /* XXX seems like this should be upstream :) */ static inline void *bh_data(struct buffer_head *bh) { From 31d182e2dbd16042fe1209ec9c56206344edb9d9 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 13 Oct 2016 11:41:29 -0700 Subject: [PATCH 105/920] Add 'make clean' target Signed-off-by: Mark Fasheh Signed-off-by: Zach Brown --- kmod/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/Makefile b/kmod/Makefile index 1dbc498c..6a3508c2 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -31,3 +31,6 @@ ALL: module module: make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src + +clean: + make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src clean From 5b7f9ddbe2490a014d981e1696bd6624a3e4e1d4 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 12 Oct 2016 14:10:05 -0700 Subject: [PATCH 106/920] Trace scoutfs btree functions We make an event class for the two most common btree op patterns, and reuse that to make our tracepoints for each function. This covers all the entry points listed in btree.h. We don't get every single parameter of every function but this is enough that we can see which keys are being queried / inserted. Signed-off-by: Mark Fasheh Signed-off-by: Zach Brown --- kmod/src/btree.c | 20 +++++- kmod/src/scoutfs_trace.h | 127 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index fbd40f7c..441fb36b 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -21,6 +21,8 @@ #include "key.h" #include "btree.h" +#include "scoutfs_trace.h" + /* * scoutfs stores file system metadata in btrees whose items have fixed * sized keys and variable length values. @@ -1166,8 +1168,7 @@ int scoutfs_btree_lookup(struct super_block *sb, int cmp; int ret; - trace_printk("key "CKF" val_len %d\n", - CKA(key), scoutfs_btree_val_length(val)); + trace_scoutfs_btree_lookup(sb, key, scoutfs_btree_val_length(val)); bh = btree_walk(sb, root, key, NULL, 0, 0, 0); if (IS_ERR(bh)) @@ -1217,6 +1218,8 @@ int scoutfs_btree_insert(struct super_block *sb, else val_len = 0; + trace_scoutfs_btree_insert(sb, key, val_len); + if (WARN_ON_ONCE(val_len > SCOUTFS_MAX_ITEM_LEN)) return -EINVAL; @@ -1256,6 +1259,8 @@ int scoutfs_btree_delete(struct super_block *sb, int cmp; int ret; + trace_scoutfs_btree_delete(sb, key, 0); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DELETE); if (IS_ERR(bh)) { ret = PTR_ERR(bh); @@ -1383,6 +1388,8 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *found, struct scoutfs_btree_val *val) { + trace_scoutfs_btree_next(sb, first, last); + return btree_next(sb, root, first, last, 0, WALK_NEXT, found, NULL, val); } @@ -1393,6 +1400,8 @@ int scoutfs_btree_since(struct super_block *sb, u64 seq, struct scoutfs_key *found, u64 *found_seq, struct scoutfs_btree_val *val) { + trace_scoutfs_btree_since(sb, first, last); + return btree_next(sb, root, first, last, seq, WALK_NEXT_SEQ, found, found_seq, val); } @@ -1413,7 +1422,7 @@ int scoutfs_btree_dirty(struct super_block *sb, int cmp; int ret; - trace_printk("key "CKF"\n", CKA(key)); + trace_scoutfs_btree_dirty(sb, key, 0); bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) @@ -1454,6 +1463,9 @@ int scoutfs_btree_update(struct super_block *sb, int cmp; int ret; + trace_scoutfs_btree_update(sb, key, + val ? scoutfs_btree_val_length(val) : 0); + bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); @@ -1497,6 +1509,8 @@ int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key found; int ret; + trace_scoutfs_btree_hole(sb, first, last); + if (WARN_ON_ONCE(scoutfs_key_cmp(first, last) > 0)) { scoutfs_key_set_zero(hole); return -EINVAL; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index fb96658d..81cef012 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -23,10 +23,26 @@ #define _TRACE_SCOUTFS_H #include +#include #include "key.h" #include "format.h" +struct scoutfs_sb_info; + +#define show_key_type(type) \ + __print_symbolic(type, \ + { SCOUTFS_INODE_KEY, "INODE" }, \ + { SCOUTFS_XATTR_KEY, "XATTR" }, \ + { SCOUTFS_XATTR_NAME_HASH_KEY, "XATTR_NAME_HASH"}, \ + { SCOUTFS_XATTR_VAL_HASH_KEY, "XATTR_VAL_HASH" }, \ + { SCOUTFS_DIRENT_KEY, "DIRENT" }, \ + { SCOUTFS_LINK_BACKREF_KEY, "LINK_BACKREF"}, \ + { SCOUTFS_SYMLINK_KEY, "SYMLINK" }, \ + { SCOUTFS_BMAP_KEY, "BMAP" }) + +#define TRACE_KEYF "%llu.%s.%llu" + TRACE_EVENT(scoutfs_write_begin, TP_PROTO(u64 ino, loff_t pos, unsigned len), @@ -156,6 +172,117 @@ TRACE_EVENT(scoutfs_buddy_free, __entry->blkno, __entry->order, __entry->region, __entry->ret) ); +DECLARE_EVENT_CLASS(scoutfs_btree_op, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, key_ino ) + __field( u64, key_off ) + __field( u8, key_type ) + __field( int, val_len ) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->key_ino = le64_to_cpu(key->inode); + __entry->key_off = le64_to_cpu(key->offset); + __entry->key_type = key->type; + __entry->val_len = len; + ), + + TP_printk("dev %d,%d key "TRACE_KEYF" size %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->key_ino, show_key_type(__entry->key_type), + __entry->key_off, __entry->val_len) +); + +DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_lookup, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len) +); + +DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_insert, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len) +); + +DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_delete, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len) +); + +DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_dirty, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len) +); + +DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_update, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), + + TP_ARGS(sb, key, len) +); + +DECLARE_EVENT_CLASS(scoutfs_btree_ranged_op, + TP_PROTO(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last), + + TP_ARGS(sb, first, last), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, first_ino ) + __field( u64, first_off ) + __field( u8, first_type ) + __field( u64, last_ino ) + __field( u64, last_off ) + __field( u8, last_type ) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->first_ino = le64_to_cpu(first->inode); + __entry->first_off = le64_to_cpu(first->offset); + __entry->first_type = first->type; + __entry->last_ino = le64_to_cpu(last->inode); + __entry->last_off = le64_to_cpu(last->offset); + __entry->last_type = last->type; + ), + + TP_printk("dev %d,%d first key "TRACE_KEYF" last key "TRACE_KEYF, + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->first_ino, + show_key_type(__entry->first_type), __entry->first_off, + __entry->last_ino, show_key_type(__entry->last_type), + __entry->last_off) +); + +DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_hole, + TP_PROTO(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last), + + TP_ARGS(sb, first, last) +); + +DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_next, + TP_PROTO(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last), + + TP_ARGS(sb, first, last) +); + +DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_since, + TP_PROTO(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last), + + TP_ARGS(sb, first, last) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 16e94f6b7c7cf04d5b9f094984521a4aba2230c0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 19 Oct 2016 15:05:04 -0700 Subject: [PATCH 107/920] Search for file data that has changed We don't overwrite existing data. Every file data write has to allocate new blocks and update block mapping items. We can search for inodes whose data has changed by filtering block mapping item walks by the sequence number. We do this by using the exact same code for finding changed inodes but using the block mapping key type. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 28 ++++++++++++++++++++-------- kmod/src/ioctl.h | 2 ++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index ff7036ae..10d519df 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -26,12 +26,22 @@ #include "super.h" /* - * Find all the inodes in the given inode range that have changed since - * the given tree update sequence number. + * Find all the inodes that have had keys of a given type modified since + * a given sequence number. The user's arg struct specifies the inode + * range to search within and the sequence value to return results from. + * Different ioctls call this for different key types. * - * The inodes are returned in inode order, not sequence order. + * When this is used for file data items the user is trying to find + * inodes whose data has changed since a given time in the past. + * + * XXX We'll need to improve the walk and search to notice when file + * data items have been truncated away. + * + * Inodes and their sequence numbers are copied out to userspace in + * inode order, not sequence order. */ -static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) +static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, + u8 type) { struct super_block *sb = file_inode(file)->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); @@ -52,8 +62,8 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) if (args.buf_len < sizeof(iseq) || args.buf_len > INT_MAX) return -EINVAL; - scoutfs_set_key(&key, args.first_ino, SCOUTFS_INODE_KEY, 0); - scoutfs_set_key(&last, args.last_ino, SCOUTFS_INODE_KEY, 0); + scoutfs_set_key(&key, args.first_ino, type, 0); + scoutfs_set_key(&last, args.last_ino, type, 0); bytes = 0; for (;;) { @@ -80,7 +90,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg) break; } - scoutfs_inc_key(&key); + key.inode = cpu_to_le64(iseq.ino + 1); } if (bytes) @@ -286,13 +296,15 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { case SCOUTFS_IOC_INODES_SINCE: - return scoutfs_ioc_inodes_since(file, arg); + return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_INODE_KEY); case SCOUTFS_IOC_INODE_PATHS: return scoutfs_ioc_inode_paths(file, arg); case SCOUTFS_IOC_FIND_XATTR_NAME: return scoutfs_ioc_find_xattr(file, arg, true); case SCOUTFS_IOC_FIND_XATTR_VAL: return scoutfs_ioc_find_xattr(file, arg, false); + case SCOUTFS_IOC_INODE_DATA_SINCE: + return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_BMAP_KEY); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 1abf6597..3ea55a03 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -54,4 +54,6 @@ struct scoutfs_ioctl_find_xattr { #define SCOUTFS_IOC_FIND_XATTR_VAL _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ struct scoutfs_ioctl_find_xattr) +#define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ + struct scoutfs_ioctl_inodes_since) #endif From ad2f5b33eef281ae5d7986960c57100dfae54afe Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Thu, 20 Oct 2016 13:58:29 -0600 Subject: [PATCH 108/920] Use make variable CURDIR instead of PWD When running make in a limited shell or in docker, there is no PWD from shell. By using CURDIR we avoid worrying about the environment and let make take care of this for us. Signed-off-by: Nic Henke Signed-off-by: Zach Brown --- kmod/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/Makefile b/kmod/Makefile index 6a3508c2..31cee8f2 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -29,8 +29,8 @@ ALL: module # #define _RH_KABI_REPLACE_UNSAFE(_orig, _new) _new module: - make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src - make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src + make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src + make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src clean: - make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(PWD)/src clean + make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src clean From d4355dd587f0816aec9e55ae3c2a50e8cbc1cf4b Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Thu, 20 Oct 2016 13:56:44 -0600 Subject: [PATCH 109/920] Add all target for make Adding in an 'all' target allows us to use canned build scripts for any of the scoutfs related repositories. Signed-off-by: Nic Henke Signed-off-by: Zach Brown --- kmod/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/Makefile b/kmod/Makefile index 31cee8f2..d0abca9b 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -28,6 +28,8 @@ ALL: module # # #define _RH_KABI_REPLACE_UNSAFE(_orig, _new) _new +all: module + module: make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src From ebbb2e842e40c3612f9a2a8bf3d745d7119cd335 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 18 Oct 2016 11:59:18 -0700 Subject: [PATCH 110/920] scoutfs: implement inode orphaning This is pretty straight forward - we define a new item type, SCOUTFS_ORPHAN_KEY. We don't need to store any value with this, the inode and type fields are enough for us to find what inode has been orphaned. Otherwise this works as one would expect. Unlink sets the item, and ->evict_inode removes it. On mount, we scan for orphan items and remove any corresponding inodes. Signed-off-by: Mark Fasheh Signed-off-by: Zach Brown --- kmod/src/dir.c | 30 +++++-- kmod/src/format.h | 1 + kmod/src/inode.c | 170 +++++++++++++++++++++++++++++++++------ kmod/src/inode.h | 3 + kmod/src/scoutfs_trace.h | 56 +++++++++++++ kmod/src/super.c | 2 + 6 files changed, 229 insertions(+), 33 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e1c90f31..d45fd05e 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -568,19 +568,33 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) return ret; set_lref_key(&lref_key, scoutfs_ino(inode), di->lref_counter); - - ret = scoutfs_dirty_inode_item(dir) ?: - scoutfs_dirty_inode_item(inode) ?: - scoutfs_btree_dirty(sb, meta, &lref_key); - if (ret) - goto out; - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); - ret = scoutfs_btree_delete(sb, meta, &key); + /* + * Dirty most of the metadata up front so that later btree + * operations can't fail. + */ + ret = scoutfs_dirty_inode_item(dir) ?: + scoutfs_dirty_inode_item(inode) ?: + scoutfs_btree_dirty(sb, meta, &lref_key) ?: + scoutfs_btree_dirty(sb, meta, &key); if (ret) goto out; + if ((inode->i_nlink == 1) || + (S_ISDIR(inode->i_mode) && inode->i_nlink == 2)) { + /* + * Insert the orphan item before we modify any inode + * metadata so we can gracefully exit should it + * fail. + */ + ret = scoutfs_orphan_inode(inode); + if (ret) + goto out; + } + + /* XXX: In thoery this can't fail but we should trap errors anyway */ + scoutfs_btree_delete(sb, meta, &key); scoutfs_btree_delete(sb, meta, &lref_key); dir->i_ctime = ts; diff --git a/kmod/src/format.h b/kmod/src/format.h index fd0cbd62..7a02c26a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -108,6 +108,7 @@ struct scoutfs_key { #define SCOUTFS_LINK_BACKREF_KEY 6 #define SCOUTFS_SYMLINK_KEY 7 #define SCOUTFS_BMAP_KEY 8 +#define SCOUTFS_ORPHAN_KEY 9 #define SCOUTFS_MAX_ITEM_LEN 512 diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e60d92c6..f8e6b6f8 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -27,6 +27,8 @@ #include "scoutfs_trace.h" #include "xattr.h" #include "trans.h" +#include "btree.h" +#include "msg.h" /* * XXX @@ -359,34 +361,29 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, return inode; } -/* - * Remove all the items associated with a given inode. - */ -static void drop_inode_items(struct super_block *sb, u64 ino) +static int remove_orphan_item(struct super_block *sb, u64 ino) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_inode sinode; struct scoutfs_key key; - bool release = false; - umode_t mode; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); int ret; - /* sample the inode mode, XXX don't need to copy whole thing here */ - scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + scoutfs_set_key(&key, ino, SCOUTFS_ORPHAN_KEY, 0); - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret < 0) - goto out; + ret = scoutfs_btree_delete(sb, meta, &key); + if (ret == -ENOENT) + ret = 0; - /* XXX corruption */ - if (ret != sizeof(sinode)) { - ret = -EIO; - goto out; - } + return ret; +} - mode = le32_to_cpu(sinode.mode); +static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, + u64 ino, umode_t mode) +{ + int ret; + bool release = false; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + + trace_delete_inode(sb, ino, mode); ret = scoutfs_hold_trans(sb); if (ret) @@ -404,12 +401,48 @@ static void drop_inode_items(struct super_block *sb, u64 ino) if (ret) goto out; - ret = scoutfs_btree_delete(sb, meta, &key); + ret = scoutfs_btree_delete(sb, meta, key); + if (ret) + goto out; + + ret = remove_orphan_item(sb, ino); +out: + if (release) + scoutfs_release_trans(sb); + return ret; +} + +/* + * Remove all the items associated with a given inode. + */ +static void delete_inode(struct super_block *sb, u64 ino) +{ + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; + struct scoutfs_key key; + umode_t mode; + int ret; + + /* sample the inode mode, XXX don't need to copy whole thing here */ + scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret < 0) + goto out; + + /* XXX corruption */ + if (ret != sizeof(sinode)) { + ret = -EIO; + goto out; + } + mode = le32_to_cpu(sinode.mode); + + ret = __delete_inode(sb, &key, ino, mode); out: if (ret) trace_printk("drop items failed ret %d ino %llu\n", ret, ino); - if (release) - scoutfs_release_trans(sb); } /* @@ -429,7 +462,7 @@ void scoutfs_evict_inode(struct inode *inode) truncate_inode_pages_final(&inode->i_data); if (inode->i_nlink == 0) - drop_inode_items(inode->i_sb, scoutfs_ino(inode)); + delete_inode(inode->i_sb, scoutfs_ino(inode)); clear: clear_inode(inode); } @@ -443,6 +476,93 @@ int scoutfs_drop_inode(struct inode *inode) return ret; } +static int process_orphaned_inode(struct super_block *sb, u64 ino) +{ + int ret; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_val val; + struct scoutfs_inode sinode; + struct scoutfs_key key; + + scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); + scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + + ret = scoutfs_btree_lookup(sb, meta, &key, &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + return ret; + } + + /* XXX corruption */ + if (ret != sizeof(sinode)) { + ret = -EIO; + goto out; + } + + if (le32_to_cpu(sinode.nlink) == 0) + __delete_inode(sb, &key, ino, le32_to_cpu(sinode.mode)); + else + scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); + +out: + return ret; +} + +/* + * Scan the metadata tree for orphan items and process each one. + * + * Runtime of this will be bounded by the number of orphans, which could + * theoretically be very large. If that becomes a problem we might want to push + * this work off to a thread. + */ +int scoutfs_scan_orphans(struct super_block *sb) +{ + int ret, err = 0; + struct scoutfs_key first, last, found; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + + trace_scoutfs_scan_orphans(sb); + + scoutfs_set_key(&first, 0, SCOUTFS_ORPHAN_KEY, 0); + scoutfs_set_key(&last, ~0ULL, SCOUTFS_ORPHAN_KEY, 0); + + while (1) { + ret = scoutfs_btree_next(sb, meta, &first, &last, &found, NULL); + if (ret == -ENOENT) /* No more orphan items */ + break; + if (ret < 0) + goto out; + + ret = process_orphaned_inode(sb, le64_to_cpu(found.inode)); + if (ret && ret != -ENOENT && !err) + err = ret; + + first = found; + scoutfs_inc_key(&first); + } + + ret = 0; +out: + return err ? err : ret; +} + +int scoutfs_orphan_inode(struct inode *inode) +{ + int ret; + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + + trace_scoutfs_orphan_inode(sb, inode); + + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_ORPHAN_KEY, 0); + + ret = scoutfs_btree_insert(sb, meta, &key, NULL); + + return ret; +} + void scoutfs_inode_exit(void) { if (scoutfs_inode_cachep) { diff --git a/kmod/src/inode.h b/kmod/src/inode.h index e02acf27..f303ba97 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -25,6 +25,7 @@ struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); int scoutfs_drop_inode(struct inode *inode); void scoutfs_evict_inode(struct inode *inode); +int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); @@ -32,6 +33,8 @@ void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); +int scoutfs_scan_orphans(struct super_block *sb); + u64 scoutfs_last_ino(struct super_block *sb); void scoutfs_inode_exit(void); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 81cef012..1157c26a 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -125,6 +125,62 @@ TRACE_EVENT(scoutfs_update_inode, __entry->ino, __entry->size) ); +TRACE_EVENT(scoutfs_orphan_inode, + TP_PROTO(struct super_block *sb, struct inode *inode), + + TP_ARGS(sb, inode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u64, ino) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->ino = scoutfs_ino(inode); + ), + + TP_printk("dev %d,%d ino %llu", MAJOR(__entry->dev), + MINOR(__entry->dev), __entry->ino) +); + +TRACE_EVENT(delete_inode, + TP_PROTO(struct super_block *sb, u64 ino, umode_t mode), + + TP_ARGS(sb, ino, mode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u64, ino) + __field(umode_t, mode) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->ino = ino; + __entry->mode = mode; + ), + + TP_printk("dev %d,%d ino %llu, mode 0x%x", MAJOR(__entry->dev), + MINOR(__entry->dev), __entry->ino, __entry->mode) +); + +TRACE_EVENT(scoutfs_scan_orphans, + TP_PROTO(struct super_block *sb), + + TP_ARGS(sb), + + TP_STRUCT__entry( + __field(dev_t, dev) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + ), + + TP_printk("dev %d,%d", MAJOR(__entry->dev), MINOR(__entry->dev)) +); + TRACE_EVENT(scoutfs_buddy_alloc, TP_PROTO(u64 blkno, int order, int region, int ret), diff --git a/kmod/src/super.c b/kmod/src/super.c index 6c4a4d51..f7f0c81f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -233,6 +233,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; + scoutfs_scan_orphans(sb); + return 0; } From 2fc1b99698abb0765386ec44aa17966ced65909a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 27 Oct 2016 14:24:40 -0500 Subject: [PATCH 111/920] scoutfs: replace some open coded corruption checks We can trivially do the simple check of value length against what the caller expects in btree.c. Signed-off-by: Mark Fasheh Signed-off-by: Zach Brown --- kmod/src/btree.c | 10 ++++++++++ kmod/src/btree.h | 1 + kmod/src/dir.c | 14 ++------------ kmod/src/filerw.c | 14 ++------------ kmod/src/inode.c | 14 ++------------ kmod/src/xattr.c | 14 ++------------ 6 files changed, 19 insertions(+), 48 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 441fb36b..0ea2d590 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -138,6 +138,16 @@ static int copy_to_val(struct scoutfs_btree_val *val, size_t off; int i; + /* + * Corruption check, right now we just return -EIO if the + * caller wants this. In the future we can grow this to do + * different things (go readonly, ignore, return error) based + * on the severity of the problem. + */ + /* XXX corruption */ + if (val->check_size_eq && val_len != scoutfs_btree_val_length(val)) + return -EIO; + for (i = 0, off = 0; val_len > 0 && i < ARRAY_SIZE(val->vec); i++) { kv = &val->vec[i]; diff --git a/kmod/src/btree.h b/kmod/src/btree.h index dc22b4c6..792ba83d 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -5,6 +5,7 @@ struct scoutfs_btree_val { struct kvec vec[3]; + unsigned int check_size_eq:1; }; static inline void __scoutfs_btree_init_val(struct scoutfs_btree_val *val, diff --git a/kmod/src/dir.c b/kmod/src/dir.c index d45fd05e..8a03f986 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -654,6 +654,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) SCOUTFS_SYMLINK_KEY, k); bytes = min_t(int, size - off, SCOUTFS_MAX_ITEM_LEN); scoutfs_btree_init_val(&val, path + off, bytes); + val.check_size_eq = 1; ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0) { @@ -663,12 +664,6 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) break; } - /* XXX corruption */ - if (ret != bytes) { - ret = -EIO; - break; - } - off += bytes; ret = 0; } @@ -858,6 +853,7 @@ retry: scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); scoutfs_btree_init_val(&val, &lref, sizeof(lref)); + val.check_size_eq = 1; ret = scoutfs_btree_next(sb, meta, &first, &last, &key, &val); if (ret < 0) { @@ -866,12 +862,6 @@ retry: goto out; } - /* XXX corruption */ - if (ret != sizeof(lref)) { - ret = -EIO; - goto out; - } - *dir_ino = le64_to_cpu(lref.ino), off = le64_to_cpu(lref.offset); *ctr = scoutfs_key_offset(&key); diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 4cbac7b3..40429532 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -223,6 +223,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) trace_printk("iblock %llu i %d\n", iblock, i); scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + val.check_size_eq = 1; for (;;) { ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); @@ -232,12 +233,6 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) break; } - /* XXX corruption */ - if (ret != sizeof(bmap)) { - ret = -EIO; - break; - } - /* XXX check bmap sanity */ /* make sure we can update bmap after freeing */ @@ -374,6 +369,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) set_bmap_key(&key, inode, iblock); scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + val.check_size_eq = 1; /* see if there's an existing mapping */ ret = scoutfs_btree_lookup(sb, meta, &key, &val); @@ -389,12 +385,6 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) inserted = true; } else { - /* XXX corruption */ - if (ret != sizeof(bmap)) { - ret = -EIO; - goto out; - } - ret = scoutfs_btree_dirty(sb, meta, &key); if (ret) goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index f8e6b6f8..da3e150f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -427,16 +427,12 @@ static void delete_inode(struct super_block *sb, u64 ino) /* sample the inode mode, XXX don't need to copy whole thing here */ scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + val.check_size_eq = 1; ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0) goto out; - /* XXX corruption */ - if (ret != sizeof(sinode)) { - ret = -EIO; - goto out; - } mode = le32_to_cpu(sinode.mode); ret = __delete_inode(sb, &key, ino, mode); @@ -486,6 +482,7 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + val.check_size_eq = 1; ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0) { @@ -494,18 +491,11 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) return ret; } - /* XXX corruption */ - if (ret != sizeof(sinode)) { - ret = -EIO; - goto out; - } - if (le32_to_cpu(sinode.nlink) == 0) __delete_inode(sb, &key, ino, le32_to_cpu(sinode.mode)); else scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); -out: return ret; } diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index abfc6a8c..4a0a2a7c 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -230,6 +230,7 @@ static int insert_xattr(struct inode *inode, const char *name, /* increment the val hash item for find_xattr, inserting if first */ scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); + val.check_size_eq = 1; ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); if (ret < 0 && ret != -ENOENT) @@ -239,12 +240,6 @@ static int insert_xattr(struct inode *inode, const char *name, refcount = cpu_to_le64(1); ret = scoutfs_btree_insert(sb, meta, &val_key, &val); } else { - /* XXX corruption */ - if (ret != sizeof(refcount)) { - ret = -EIO; - goto out; - } - le64_add_cpu(&refcount, 1); ret = scoutfs_btree_update(sb, meta, &val_key, &val); } @@ -277,16 +272,11 @@ static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, /* update the val_hash refcount, making sure it's not nonsense */ scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); + val.check_size_eq = 1; ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); if (ret < 0) goto out; - /* XXX corruption */ - if (ret != sizeof(refcount)) { - ret = -EIO; - goto out; - } - le64_add_cpu(&refcount, -1ULL); /* ensure that we can update and delete name_ and val_ keys */ From 165d833c46af05d2a45c67914d456d3774d7342b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Oct 2016 15:34:40 -0700 Subject: [PATCH 112/920] Walk stable trees in _since ioctls The _since ioctls walk btrees and return items that are newer than a given sequence number. The intended behaviour is that items will appear in a greater sequence number if they change after appearing in the queries. This promise doesn't hold for items that are being modified in the current transaction. The caller would have to always ask for seq X + 1 after seeing seq X to make sure it got all the changes that happened in seq X while it was the current dirty transaction. This is fixed by having the interfaces walk the stable btrees from the previous transaction. The results will always be a little stale but userspace already has to deal with stale results because it can't lock out change, and it can use sync (and a commit age tunable we'll add) to limit how stale the results can be. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 4 ++-- kmod/src/super.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 10d519df..92346254 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -44,7 +44,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, u8 type) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_root *meta = SCOUTFS_STABLE_META(sb); struct scoutfs_ioctl_inodes_since __user *uargs = (void __user *)arg; struct scoutfs_ioctl_inodes_since args; struct scoutfs_ioctl_ino_seq __user *uiseq; @@ -218,7 +218,7 @@ static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, bool find_name) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_btree_root *meta = SCOUTFS_STABLE_META(sb); struct scoutfs_ioctl_find_xattr args; struct scoutfs_key key; struct scoutfs_key last; diff --git a/kmod/src/super.h b/kmod/src/super.h index 6f550d0c..604e0231 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -61,6 +61,11 @@ static inline struct scoutfs_btree_root *SCOUTFS_META(struct super_block *sb) return &SCOUTFS_SB(sb)->super.btree_root; } +static inline struct scoutfs_btree_root *SCOUTFS_STABLE_META(struct super_block *sb) +{ + return &SCOUTFS_SB(sb)->stable_super.btree_root; +} + void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); From 1cbd84eecebf619c8e72aeeb1329398d6d0806cb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Oct 2016 18:06:08 -0700 Subject: [PATCH 113/920] scoutfs: wire up sop->dirty_inode We're using the generic block buffer_head write_begin and write_end functions. They call sop->dirty_inode() to update the inode i_size. We didn't have that method wired up so updates to the inode in the write path wasn't dirtying the inode item. Lost i_size updates would trivially lose data but we first noticed this when looking at inode item sequence numbers while overwriting. Signed-off-by: Zach Brown --- kmod/src/filerw.c | 5 +++++ kmod/src/inode.c | 28 ++++++++++++++++++++++++++++ kmod/src/inode.h | 1 + kmod/src/super.c | 1 + 4 files changed, 35 insertions(+) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 40429532..6f9a322d 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -625,6 +625,11 @@ retry: /* can't re-enter fs, have trans */ flags |= AOP_FLAG_NOFS; + /* generic write_end updates i_size and calls dirty_inode */ + ret = scoutfs_dirty_inode_item(inode); + if (ret) + goto out; + /* make sure our get_block gets a chance to alloc */ clear_mapped_page_buffers(page); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index da3e150f..98a005ca 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -275,6 +275,34 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } +/* + * sop->dirty_inode() can't return failure. Our use of it has to be + * careful to pin the inode during a transaction. The generic write + * paths pin the inode in write_begin and get called to update the inode + * in write_end. + * + * The caller should have a trans but it's cheap for us to grab it + * ourselves to make sure. + * + * This will holler at us if a caller didn't pin the inode and we + * couldn't dirty the inode ourselves. + */ +void scoutfs_dirty_inode(struct inode *inode, int flags) +{ + struct super_block *sb = inode->i_sb; + int ret; + + ret = scoutfs_hold_trans(sb); + if (ret == 0) { + ret = scoutfs_dirty_inode_item(inode); + if (ret == 0) + scoutfs_update_inode_item(inode); + scoutfs_release_trans(sb); + } + + WARN_ON_ONCE(ret); +} + /* * A quick atomic sample of the last inode number that's been allocated. */ diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f303ba97..52ad52a6 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -29,6 +29,7 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); +void scoutfs_dirty_inode(struct inode *inode, int flags); void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); diff --git a/kmod/src/super.c b/kmod/src/super.c index f7f0c81f..a668f8a2 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -74,6 +74,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, + .dirty_inode = scoutfs_dirty_inode, .drop_inode = scoutfs_drop_inode, .evict_inode = scoutfs_evict_inode, .destroy_inode = scoutfs_destroy_inode, From f32365321dd866c5c1934bc663430fc978b998c5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Oct 2016 13:03:06 -0700 Subject: [PATCH 114/920] Remove unused btree internal WALK_NEXT In the past the WALK_NEXT enum was used to tell the walking core that the caller was iterating and that they'd need to advance to sibling blocks if their key landed off the end of a leaf. In the current code that's now handled by giving the walk caller a next_key which will continue the search from the next leaf. WALK_NEXT is unused and we can remove it. Signed-off-by: Zach Brown --- kmod/src/btree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 0ea2d590..b527a264 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -879,7 +879,6 @@ static struct buffer_head *try_merge(struct super_block *sb, enum { WALK_INSERT = 1, WALK_DELETE, - WALK_NEXT, WALK_NEXT_SEQ, WALK_DIRTY, }; @@ -1400,8 +1399,7 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, { trace_scoutfs_btree_next(sb, first, last); - return btree_next(sb, root, first, last, 0, WALK_NEXT, - found, NULL, val); + return btree_next(sb, root, first, last, 0, 0, found, NULL, val); } int scoutfs_btree_since(struct super_block *sb, From a77f88386cb8751d625042ddff186edfc696c50b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Oct 2016 13:32:44 -0700 Subject: [PATCH 115/920] Add scoutfs_btree_prev() We haven't yet had a pressing need for a search for the previous item before a given key value. File extent items offer the first strong candidate. We'd like to naturally store the start of the extent in the key so to find an extent that overlaps a block we'd like to find the previous key before the search block offset. The _prev search is easy enough to implement. We have to update tree walking to update the prev key and update leaf block processing to find the correct item position after the binary search. Signed-off-by: Zach Brown --- kmod/src/btree.c | 107 ++++++++++++++++++++++++++++++++++----- kmod/src/btree.h | 4 ++ kmod/src/scoutfs_trace.h | 7 +++ 3 files changed, 105 insertions(+), 13 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index b527a264..ffdd1697 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1033,13 +1033,16 @@ out: * operation. The block is returned locked for either reading or * writing depending on the operation. * - * As we descend through parent items we set next_key to the first key - * in the next sibling's block. This is used by iteration to advance to - * the next block when they're done with the block this returns. + * As we descend through parent items we set prev_key or next_key to the + * last key in the previous sibling's block or to the first key in the + * next sibling's block, respectively. This is used by iteration to + * keep searching sibling blocks if their search key falls at the end of + * a leaf in their search direction. */ static struct buffer_head *btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, + struct scoutfs_key *prev_key, struct scoutfs_key *next_key, unsigned int val_len, u64 seq, int op) { @@ -1059,6 +1062,8 @@ static struct buffer_head *btree_walk(struct super_block *sb, /* no sibling blocks if we don't have parent blocks */ if (next_key) scoutfs_set_max_key(next_key); + if (prev_key) + scoutfs_key_set_zero(prev_key); lock_root(sb, root, dirty); @@ -1138,17 +1143,21 @@ static struct buffer_head *btree_walk(struct super_block *sb, ref = (void *)item->val; /* - * Update the next key an iterator should read from. - * Keep in mind that iteration is read only so the - * parent item won't be changed splitting or merging. + * Update the keys that iterators should continue + * searching from. Keep in mind that iteration is read + * only so the parent item won't be changed splitting or + * merging. */ if (next_key) { *next_key = item->key; scoutfs_inc_key(next_key); } - if (pos) + if (pos) { small = pos_item(parent, pos - 1)->key; + if (prev_key) + *prev_key = small; + } large = item->key; } @@ -1179,7 +1188,7 @@ int scoutfs_btree_lookup(struct super_block *sb, trace_scoutfs_btree_lookup(sb, key, scoutfs_btree_val_length(val)); - bh = btree_walk(sb, root, key, NULL, 0, 0, 0); + bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, 0); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1232,7 +1241,7 @@ int scoutfs_btree_insert(struct super_block *sb, if (WARN_ON_ONCE(val_len > SCOUTFS_MAX_ITEM_LEN)) return -EINVAL; - bh = btree_walk(sb, root, key, NULL, val_len, 0, WALK_INSERT); + bh = btree_walk(sb, root, key, NULL, NULL, val_len, 0, WALK_INSERT); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1270,7 +1279,7 @@ int scoutfs_btree_delete(struct super_block *sb, trace_scoutfs_btree_delete(sb, key, 0); - bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DELETE); + bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DELETE); if (IS_ERR(bh)) { ret = PTR_ERR(bh); goto out; @@ -1346,7 +1355,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; while (scoutfs_key_cmp(&key, last) <= 0) { - bh = btree_walk(sb, root, &key, &next_key, 0, seq, op); + bh = btree_walk(sb, root, &key, NULL, &next_key, 0, seq, op); /* next seq walks can terminate in parents with old seqs */ if (op == WALK_NEXT_SEQ && bh == ERR_PTR(-ENOENT)) { @@ -1414,6 +1423,78 @@ int scoutfs_btree_since(struct super_block *sb, found, found_seq, val); } +/* + * Find the greatest key that is >= first and <= last, starting at last. + * For each search cursor key we descend to the leaf and find its + * position in the items. The item binary search returns the position + * that the key would be inserted into, so if we didn't find the key + * specifically we go to the previous position. The btree walk gives us + * the previous key to search from if we fall off the front of the + * block. + * + * This doesn't support filtering the tree traversal by seqs. + */ +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, + struct scoutfs_key *found, + struct scoutfs_btree_val *val) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_key key = *last; + struct scoutfs_key prev_key; + struct buffer_head *bh; + int pos; + int cmp; + int ret; + + trace_scoutfs_btree_prev(sb, first, last); + + /* find the leaf that contains the next item after the key */ + ret = -ENOENT; + while (scoutfs_key_cmp(&key, first) >= 0) { + + bh = btree_walk(sb, root, &key, NULL, &prev_key, 0, 0, 0); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + break; + } + bt = bh_data(bh); + + pos = find_pos(bt, &key, &cmp); + + /* walk to the prev leaf if we hit the front of this leaf */ + if (pos == 0 && cmp != 0) { + unlock_level(sb, root, bh, false); + scoutfs_block_put(bh); + if (scoutfs_key_is_zero(&key)) + break; + key = prev_key; + continue; + } + + /* we want the item before a non-matching position */ + if (pos && cmp) + pos--; + + /* return the item if it's still within our first bound */ + item = pos_item(bt, pos); + if (cmp == 0 || scoutfs_key_cmp(&item->key, first) >= 0) { + *found = item->key; + if (val) + ret = copy_to_val(val, item); + else + ret = 0; + } + + unlock_level(sb, root, bh, false); + scoutfs_block_put(bh); + break; + } + + return ret; +} + /* * Ensure that the blocks that lead to the item with the given key are * dirty. caller can hold a transaction to pin the dirty blocks and @@ -1432,7 +1513,7 @@ int scoutfs_btree_dirty(struct super_block *sb, trace_scoutfs_btree_dirty(sb, key, 0); - bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); + bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); @@ -1474,7 +1555,7 @@ int scoutfs_btree_update(struct super_block *sb, trace_scoutfs_btree_update(sb, key, val ? scoutfs_btree_val_length(val) : 0); - bh = btree_walk(sb, root, key, NULL, 0, 0, WALK_DIRTY); + bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); if (IS_ERR(bh)) return PTR_ERR(bh); bt = bh_data(bh); diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 792ba83d..7615a733 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -53,6 +53,10 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, struct scoutfs_key *found, struct scoutfs_btree_val *val); +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *first, struct scoutfs_key *last, + struct scoutfs_key *found, + struct scoutfs_btree_val *val); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 1157c26a..a1ca45d8 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -332,6 +332,13 @@ DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_next, TP_ARGS(sb, first, last) ); +DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_prev, + TP_PROTO(struct super_block *sb, struct scoutfs_key *first, + struct scoutfs_key *last), + + TP_ARGS(sb, first, last) +); + DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_since, TP_PROTO(struct super_block *sb, struct scoutfs_key *first, struct scoutfs_key *last), From 44588c1d8b5109575a9236517107976de6497acb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 3 Nov 2016 14:23:04 -0700 Subject: [PATCH 116/920] Lock btree merges The btree locking wasn't covering the merge candidate block before the siblings were locked. In that unlocked code it could compact the block corrupting it for whatever other tree walk might only have the merge candidate locked after having unlocked the parent. This extends locking coverage to merge and split attempts by acquiring the block lock immediately after we read it. Split doesn't have to lock its destination block but it does have to know to unlock the block on errors. Merge has to more carefully lock both of its existing blocks in a consistent order. To clearly implement this we simplify the locking helpers to just unlock and lock a given block, falling back to the btree rwsem if there isn't a block. I started down this road while chasing allocator bugs that manifested as tree corruption. Signed-off-by: Zach Brown --- kmod/src/btree.c | 203 ++++++++++++++++++----------------------------- 1 file changed, 78 insertions(+), 125 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index ffdd1697..ae3d4714 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -442,14 +442,13 @@ static void compact_items(struct scoutfs_btree_block *bt) * tree. This gives us the opportunity to cascade block locks down the * tree. We first lock the root. Then we lock the first block and * unlock the root. Then lock the next block and unlock the first - * block. And so on down the tree. Except for that brief transition - * function the btree walk always holds a single lock on either the root - * or a block at a level. After contention on the root and first block - * we have lots of concurrency down paths of the tree to the leaves. + * block. And so on down the tree. After contention on the root and + * first block we have lots of concurrency down paths of the tree to the + * leaves. * - * As we walk down the tree we have to split or merge. While we do this - * we hold the parent block lock. We have to also lock the sibling - * blocks. We always acquire them left to right to avoid deadlocks. + * Merging during descent has to lock the sibling block that it's + * pulling items from. It has to acquire these nested locks in + * consistent tree order. * * The cow tree updates let us skip block locking entirely for stable * blocks because they're read only. The block layer only has to worry @@ -458,22 +457,13 @@ static void compact_items(struct scoutfs_btree_block *bt) * transaction and we store the block lock there. The block layer * ignores our locking attempts for read-only blocks. * + * And all of the blocks referenced by the stable super will be stable + * so we only try to lock at all when working with the dirty super. + * * lockdep has to not be freaked out by all of this. The cascading * block locks really make it angry without annotation so we add classes * for each level and use nested subclasses for the locking of siblings - * during split and merge. - * - * We also use the btree API for the block allocator. This introduces - * nesting btree allocator calls inside main fs metadata btree calls. - * The locking would be safe as the blocks will never be in both trees - * but lockdep would think they're the same class and get raise - * warnings. We'd need to have tree level classes for all the trees. - * It turns out that the allocator has to maintain multi-item - * consistency across its entire tree so it has a tree-wide lock. We - * don't have to lock the btree at all when we're working on the - * allocator roots. They're the only non-metadata roots so far so we - * invert the test and only lock the btree when we're working on the - * main metadata btree root. + * during merge. */ static void set_block_lock_class(struct buffer_head *bh, int level) @@ -485,85 +475,42 @@ static void set_block_lock_class(struct buffer_head *bh, int level) #endif } -static void lock_root(struct super_block *sb, struct scoutfs_btree_root *root, - bool write) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (root == &sbi->super.btree_root) { - if (write) - down_write(&sbi->btree_rwsem); - else - down_read(&sbi->btree_rwsem); - } -} - -static void unlock_root(struct super_block *sb, bool write) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (write) - up_write(&sbi->btree_rwsem); - else - up_read(&sbi->btree_rwsem); -} - -static void unlock_level(struct super_block *sb, - struct scoutfs_btree_root *root, - struct buffer_head *bh, bool write) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (root == &sbi->super.btree_root) { - if (bh) - scoutfs_block_unlock(bh, write); - else - unlock_root(sb, write); - } -} - -static void lock_next_level(struct super_block *sb, +static void lock_tree_block(struct super_block *sb, struct scoutfs_btree_root *root, - struct buffer_head *par_bh, - struct buffer_head *bh, bool write) + struct buffer_head *bh, bool write, int subclass) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); if (root == &sbi->super.btree_root) { - scoutfs_block_lock(bh, write, 0); - - unlock_level(sb, root, par_bh, write); + if (bh) { + scoutfs_block_lock(bh, write, subclass); + } else { + if (write) + down_write(&sbi->btree_rwsem); + else + down_read(&sbi->btree_rwsem); + } } } -static void lock_siblings(struct super_block *sb, - struct scoutfs_btree_root *root, - struct buffer_head *left, struct buffer_head *right, - bool write) +static void unlock_tree_block(struct super_block *sb, + struct scoutfs_btree_root *root, + struct buffer_head *bh, bool write) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); if (root == &sbi->super.btree_root) { - scoutfs_block_lock(left, write, 0); - scoutfs_block_lock(right, write, 1); + if (bh) { + scoutfs_block_unlock(bh, write); + } else { + if (write) + up_write(&sbi->btree_rwsem); + else + up_read(&sbi->btree_rwsem); + } } } -static void unlock_siblings(struct super_block *sb, - struct scoutfs_btree_root *root, - struct buffer_head *left, struct buffer_head *right, - bool write) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (root == &sbi->super.btree_root) { - scoutfs_block_unlock(left, write); - scoutfs_block_unlock(right, write); - } -} - - - /* sorting relies on masking pointers to find the containing block */ static inline struct buffer_head *check_bh_alignment(struct buffer_head *bh) { @@ -678,14 +625,13 @@ static void create_parent_item(struct scoutfs_btree_block *parent, * We split to the left so that the greatest key in the existing block * doesn't change so we don't have to update the key in its parent item. * - * If the search key falls in the new split block then we return it - * to the caller to walk through. + * If the search key falls in the new split block then we return it to + * the caller to walk through. * - * The locking in the case where we add the first parent is a little wonky. - * We're creating a parent block that the walk doesn't know about. It - * holds the tree mutex while we add the parent ref and then will lock - * the child that we return. It's skipping locking the new parent as it - * descends but that's fine. + * The caller has the parent (or root) and our block locked. We don't + * have to lock the blocks we allocate while we have the references to + * them locked. We only need to lock the new sibling if we return it + * instead of our given block for the caller to continue descent. */ static struct buffer_head *try_split(struct super_block *sb, struct scoutfs_btree_root *root, @@ -701,7 +647,6 @@ static struct buffer_head *try_split(struct super_block *sb, struct buffer_head *par_bh = NULL; struct scoutfs_key maximal; unsigned int all_bytes; - bool swap_return = false; if (level) val_len = sizeof(struct scoutfs_block_ref); @@ -718,6 +663,7 @@ static struct buffer_head *try_split(struct super_block *sb, /* alloc split neighbour first to avoid unwinding tree growth */ left_bh = alloc_tree_block(sb); if (IS_ERR(left_bh)) { + unlock_tree_block(sb, root, right_bh, true); scoutfs_block_put(right_bh); return left_bh; } @@ -728,6 +674,7 @@ static struct buffer_head *try_split(struct super_block *sb, if (IS_ERR(par_bh)) { free_tree_block(sb, left->hdr.blkno); scoutfs_block_put(left_bh); + unlock_tree_block(sb, root, right_bh, true); scoutfs_block_put(right_bh); return par_bh; } @@ -739,26 +686,21 @@ static struct buffer_head *try_split(struct super_block *sb, create_parent_item(parent, parent_pos, right, &maximal); } - lock_siblings(sb, root, left_bh, right_bh, true); - move_items(left, right, false, used_total(right) / 2); create_parent_item(parent, parent_pos, left, greatest_key(left)); parent_pos++; /* not that anything uses it again :P */ if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { /* insertion will go to the new left block */ - swap_return = true; + unlock_tree_block(sb, root, right_bh, true); + lock_tree_block(sb, root, left_bh, true, 0); + swap(right_bh, left_bh); } else { /* insertion will still go through us, might need to compact */ if (contig_free(right) < all_bytes) compact_items(right); } - unlock_siblings(sb, root, left_bh, right_bh, true); - - if (swap_return) - swap(right_bh, left_bh); - scoutfs_block_put(par_bh); scoutfs_block_put(left_bh); @@ -777,8 +719,10 @@ static struct buffer_head *try_split(struct super_block *sb, * from both easily. We have to unlock and release our buffer to return * an error. * - * The caller only has the parent locked. They'll lock whichever - * block we return. + * The caller locks the parent and our given block. We need to + * lock sibling blocks in consistent tree order. Our common case + * has us pulling from our left sibling so we prefer to lock blocks + * from right to left. Splitting doesn't hold both sibling locks. * * We free sibling or parent btree block blknos if we drain them of items. * They're dirtied either by descent or before we start migrating items @@ -821,10 +765,19 @@ static struct buffer_head *try_merge(struct super_block *sb, } sib_bt = bh_data(sib_bh); - if (move_right) - lock_siblings(sb, root, sib_bh, bh, true); - else - lock_siblings(sb, root, bh, sib_bh, true); + if (!move_right) { + unlock_tree_block(sb, root, bh, true); + lock_tree_block(sb, root, sib_bh, true, 0); + lock_tree_block(sb, root, bh, true, 1); + + if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) { + unlock_tree_block(sb, root, sib_bh, true); + scoutfs_block_put(sib_bh); + return bh; + } + } else { + lock_tree_block(sb, root, sib_bh, true, 1); + } if (used_total(sib_bt) <= reclaimable_free(bt)) to_move = used_total(sib_bt); @@ -866,11 +819,7 @@ static struct buffer_head *try_merge(struct super_block *sb, free_tree_block(sb, parent->hdr.blkno); } - if (move_right) - unlock_siblings(sb, root, sib_bh, bh, true); - else - unlock_siblings(sb, root, bh, sib_bh, true); - + unlock_tree_block(sb, root, sib_bh, true); scoutfs_block_put(sib_bh); return bh; @@ -1065,7 +1014,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, if (prev_key) scoutfs_key_set_zero(prev_key); - lock_root(sb, root, dirty); + lock_tree_block(sb, root, NULL, dirty, 0); ref = &root->ref; level = root->height; @@ -1075,8 +1024,10 @@ static struct buffer_head *btree_walk(struct super_block *sb, bh = ERR_PTR(-ENOENT); } else { bh = grow_tree(sb, root); - if (!IS_ERR(bh)) - lock_next_level(sb, root, NULL, bh, dirty); + if (!IS_ERR(bh)) { + lock_tree_block(sb, root, bh, dirty, 0); + unlock_tree_block(sb, root, NULL, dirty); + } } goto out; } @@ -1105,6 +1056,8 @@ static struct buffer_head *btree_walk(struct super_block *sb, break; } + lock_tree_block(sb, root, bh, dirty, 0); + if (op == WALK_INSERT) bh = try_split(sb, root, level, key, val_len, parent, pos, bh); @@ -1113,7 +1066,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, if (IS_ERR(bh)) break; - lock_next_level(sb, root, par_bh, bh, dirty); + unlock_tree_block(sb, root, par_bh, dirty); if (!level) break; @@ -1163,7 +1116,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, out: if (IS_ERR(bh)) - unlock_level(sb, root, par_bh, dirty); + unlock_tree_block(sb, root, par_bh, dirty); scoutfs_block_put(par_bh); return bh; @@ -1201,7 +1154,7 @@ int scoutfs_btree_lookup(struct super_block *sb, ret = -ENOENT; } - unlock_level(sb, root, bh, false); + unlock_tree_block(sb, root, bh, false); scoutfs_block_put(bh); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1257,7 +1210,7 @@ int scoutfs_btree_insert(struct super_block *sb, ret = -EEXIST; } - unlock_level(sb, root, bh, true); + unlock_tree_block(sb, root, bh, true); scoutfs_block_put(bh); return ret; @@ -1305,7 +1258,7 @@ int scoutfs_btree_delete(struct super_block *sb, ret = -ENOENT; } - unlock_level(sb, root, bh, true); + unlock_tree_block(sb, root, bh, true); scoutfs_block_put(bh); out: @@ -1373,7 +1326,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, pos = find_pos_after_seq(bt, &key, 0, seq, op); if (pos >= bt->nr_items) { key = next_key; - unlock_level(sb, root, bh, false); + unlock_tree_block(sb, root, bh, false); scoutfs_block_put(bh); continue; } @@ -1391,7 +1344,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; } - unlock_level(sb, root, bh, false); + unlock_tree_block(sb, root, bh, false); scoutfs_block_put(bh); break; } @@ -1465,7 +1418,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, /* walk to the prev leaf if we hit the front of this leaf */ if (pos == 0 && cmp != 0) { - unlock_level(sb, root, bh, false); + unlock_tree_block(sb, root, bh, false); scoutfs_block_put(bh); if (scoutfs_key_is_zero(&key)) break; @@ -1487,7 +1440,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, ret = 0; } - unlock_level(sb, root, bh, false); + unlock_tree_block(sb, root, bh, false); scoutfs_block_put(bh); break; } @@ -1525,7 +1478,7 @@ int scoutfs_btree_dirty(struct super_block *sb, ret = -ENOENT; } - unlock_level(sb, root, bh, true); + unlock_tree_block(sb, root, bh, true); scoutfs_block_put(bh); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1570,7 +1523,7 @@ int scoutfs_btree_update(struct super_block *sb, ret = -ENOENT; } - unlock_level(sb, root, bh, true); + unlock_tree_block(sb, root, bh, true); scoutfs_block_put(bh); return ret; From c8d1703196d8b699f811d5240163bd9ca30780c6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 3 Nov 2016 15:35:16 -0700 Subject: [PATCH 117/920] Add blkno and level to bad btree printk Add the blkno and level to the output for a btree block that fails verification. Signed-off-by: Zach Brown --- kmod/src/btree.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index ae3d4714..cf45346e 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -959,8 +959,9 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level, bad = 0; out: if (bad) { - printk("bt %p small "CKF" large "CKF" end %u reclaim %u nr %u (max %lu after %u bytes %u)\n", - bt, CKA(small), CKA(large), le16_to_cpu(bt->free_end), + printk("bt %p blkno %llu level %d small "CKF" large "CKF" end %u reclaim %u nr %u (max %lu after %u bytes %u)\n", + bt, le64_to_cpu(bt->hdr.blkno), level, + CKA(small), CKA(large), le16_to_cpu(bt->free_end), le16_to_cpu(bt->free_reclaim), bt->nr_items, SCOUTFS_BTREE_MAX_ITEMS, after_offs, bytes); for (i = 0; i < nr; i++) { From 17ec4a1480ecb32ce6d048b3b14856a43d2c150c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Oct 2016 18:06:58 -0700 Subject: [PATCH 118/920] Add seq field to block map item The file block mapping code needs to know if an existing block mapping is dirty in the current transaction or not. It was doing that by calling in to the allocator. Instead of calling in to the allocator we can instead store the seq of the block in the mapping item. We also probably want to know the seq of data blocks to make it possible to discover regions of files that have changed since a previous seq. This does increase the size of the block mapping item but they're not long for this world. We're going to replace them with proper extent items in the near future. Signed-off-by: Zach Brown --- kmod/src/filerw.c | 20 +++++++------------- kmod/src/format.h | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 6f9a322d..597ce144 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -251,6 +251,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) break; bmap.blkno[i] = 0; + bmap.seq[i] = 0; modified = true; } i = 0; @@ -393,19 +394,11 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) i = iblock & SCOUTFS_BLOCK_MAP_MASK; old_blkno = le64_to_cpu(bmap.blkno[i]); - /* - * If the existing block was free in stable then its dirty in - * this trans and we can use it. - */ - if (old_blkno) { - ret = scoutfs_buddy_was_free(sb, old_blkno, 0); - if (ret < 0) - goto out; - if (ret > 0) { - *blkno_ret = old_blkno; - ret = 0; - goto out; - } + /* If the existing block is dirty then we can use it */ + if (old_blkno && (bmap.seq[i] == super->hdr.seq)) { + *blkno_ret = old_blkno; + ret = 0; + goto out; } ret = alloc_file_block(sb, &new_blkno); @@ -419,6 +412,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) } bmap.blkno[i] = cpu_to_le64(new_blkno); + bmap.seq[i] = super->hdr.seq; /* dirtying guarantees success */ err = scoutfs_btree_update(sb, meta, &key, &val); diff --git a/kmod/src/format.h b/kmod/src/format.h index 7a02c26a..58fea85f 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -302,8 +302,8 @@ struct scoutfs_xattr { #define SCOUTFS_BLOCK_MAP_MASK (SCOUTFS_BLOCK_MAP_COUNT - 1) struct scoutfs_block_map { - __le32 crc[SCOUTFS_BLOCK_MAP_COUNT]; __le64 blkno[SCOUTFS_BLOCK_MAP_COUNT]; + __le64 seq[SCOUTFS_BLOCK_MAP_COUNT]; }; /* From c65b70f2aac731364623b59c63652b9cb20a9376 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 27 Oct 2016 15:11:49 -0700 Subject: [PATCH 119/920] Use full radix for buddy and record first set The first pass of the buddy allocator had a fixed indirect block so it couldn't address large devices. It didn't index set bits or slots for each order so we spent a lot of cpu searching for free space. And it didn't precisely account for stable free space so it could spend a lot of cpu time discovering that free space can't be used because it wasn't stable. This fixes these initial critical flaws in the buddy allocator. Before it could only address a few hundred megs and now it can address 2^64 blocks. Before it limited bulk inode creation searching for slots and leaf bits and now other components are much higher in the profiles with greater create rates. First we remove the special case single indirect block. The root now references a block that can be at any height. The root records the height and each block records its level. We descend until we hit the leaf. We add a stack of the blocks traversed so that we can ascend and fix up parent indexing after we modify a leaf. Now that we can have quite a lot of parent indirect blocks we can no longer have a static bitmap for allocating buddy blocks. We instead precisely preallocate two blocks for every buddy block that will be used to address all the device blocks. The blkno offset of these pairs of buddy blocks can be calculated for a given position in the tree. Allocating a blkno xors the low bit of the blkno and freeing is a nop. This happily gets rid of the specific allocation of buddy blocks with its regions and worrying about stable free blocks itself. Then we index the first set index in a block for each order. In parent blocks this tells you the slot you can traverse to find a free region of that order. In leaf blocks it tells you the specific block offset of the first free extent. This is kept up to date as we set and clear buddy bits in leaves and free_order bits in parent slots. Allocation now is a simple matter of block reads and array dereferencing. And we now precisely account for frees that should not satisfy allocation until after a transaction commit. We record frees of stable data in extent nodes in an rbtree after their buddy blocks have been dirtied. Because their blocks are dirtied we can free them as the transaction commits without errors. Similarly, we can also revert them if the transaction commit fails so that they don't satisfy allocation. This prevents us from having to hang or go read-only if a transaction commit fails. The two changes visible to callers are easy argument changes: scoutfs_buddy_free() now takes a seq to specify when the allocation was first allocated, and scoutfs_buddy_alloc_same() has its arguments match that it only makes sense for single block allocations. Unfortunately all these changes are interrelated so the resulting patch amounts to a rewrite. The core buddy bitmap helper functions and loops are the same but the surrounding block container code changes significnatly. Signed-off-by: Zach Brown --- kmod/src/block.c | 11 +- kmod/src/btree.c | 5 +- kmod/src/buddy.c | 1523 +++++++++++++++++++++++++-------------------- kmod/src/buddy.h | 14 +- kmod/src/filerw.c | 8 +- kmod/src/format.h | 60 +- kmod/src/super.c | 10 +- kmod/src/super.h | 5 +- kmod/src/trans.c | 13 +- 9 files changed, 924 insertions(+), 725 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index b8b3ccb8..b3f99f09 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -372,7 +372,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, if (IS_ERR(bh) || ref->seq == sbi->super.hdr.seq) return bh; - ret = scoutfs_buddy_alloc_same(sb, &blkno, 0, le64_to_cpu(ref->blkno)); + ret = scoutfs_buddy_alloc_same(sb, &blkno, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; @@ -382,7 +382,7 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, goto out; } - ret = scoutfs_buddy_free(sb, bh->b_blocknr, 0); + ret = scoutfs_buddy_free(sb, ref->seq, bh->b_blocknr, 0); if (ret) goto out; @@ -399,7 +399,8 @@ out: scoutfs_block_put(bh); if (ret) { if (!IS_ERR_OR_NULL(copy_bh)) { - err = scoutfs_buddy_free(sb, copy_bh->b_blocknr, 0); + err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, + copy_bh->b_blocknr, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } scoutfs_block_put(copy_bh); @@ -452,6 +453,8 @@ out: */ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; struct buffer_head *bh; u64 blkno; int ret; @@ -463,7 +466,7 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) bh = scoutfs_block_dirty(sb, blkno); if (IS_ERR(bh)) { - err = scoutfs_buddy_free(sb, blkno, 0); + err = scoutfs_buddy_free(sb, super->hdr.seq, blkno, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } return bh; diff --git a/kmod/src/btree.c b/kmod/src/btree.c index cf45346e..9f6a0ed4 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -550,7 +550,10 @@ static struct buffer_head *alloc_tree_block(struct super_block *sb) /* the caller has ensured that the free must succeed */ static void free_tree_block(struct super_block *sb, __le64 blkno) { - int err = scoutfs_buddy_free(sb, le64_to_cpu(blkno), 0); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + int err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, + le64_to_cpu(blkno), 0); WARN_ON_ONCE(err); } diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 96f3048f..40dc2e65 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -12,6 +12,7 @@ */ #include #include +#include #include "super.h" #include "format.h" @@ -20,117 +21,139 @@ #include "scoutfs_trace.h" /* - * scoutfs uses buddy bitmaps to allocate block regions. The buddy - * allocator is nice because it uses one index for allocating by size - * and freeing and merging by location. The index is dense and has a - * predictable worst case size that we can preallocate. As described - * below, it also makes it easy to find unions of free regions between - * two indexes. + * scoutfs uses buddy bitmaps in an augmented radix to index free space. * - * The buddy allocator is built from a hierarchy of bitmaps for each - * power of two order of blocks that we can allocate. If a high order - * buddy bit is set then all the lower order bits that it covers are - * clear. The bits are stored in blocks that are stored in a fixed - * depth radix with a single parent indirect block. The super block - * references the indirect block. The block references in the indirect - * block also include a bitmap of orders that are free in the referenced - * block. + * At the heart of the allocator are the buddy bitmaps in the radix + * leaves. For a given region of blocks there are bitmaps for each + * power of two order of blocks that can be allocated. N bits record + * whether each order 0 size block region is allocated or freed, then + * N/2 bits describe order 1 regions that span pairs of order 0 blocks, + * and so on. This ends up using two bits in the bitmaps for each + * device block that's managed. * - * The blknos for the buddy blocks themselves are allocated out of a - * single bitmap block that is referenced by the super. + * An order bit is set when it is free. All of its lower order bits + * will be clear. To allocate we clear a bit. A partial allocation + * clears the higher order bit and each buddy for each lower order until + * the allocated order. Freeing sets an order bit. Then if it's buddy + * order is also set we clear both and set their higher order bit. This + * proceeds to the highest order. * - * All the blocks are read and cowed with the usual block layer routines - * so that we reuse the same code to evict and retry stale cached - * blocks, cow, etc. The allocator in the block code gives us the - * source blkno for a cow operation so we can use the correct allocator - * (none for bitmap blocks, bitmap for buddy blocks, buddy for btree - * blocks and extents). + * Each buddy block records the first set bit in each order bitmap. As + * bits are set they update these first set records if they're before + * the previous value. As bits are cleared we find the next set if it + * was the first. * - * The trickiest part of the allocator is due to the cow nature of our - * consistent updates. We can't satisfy an allocation with a region - * that's been freed in this transaction and is still referenced by the - * old stable transaction. We solve this by only returning regions that - * are free in both the stable and currently dirty allocator structures. + * These buddy bitmap blocks that each fully describe a region of blocks + * are assembled into a radix tree. Each reference to a leaf block in + * parent blocks have a bitmap of the orders that are free in its leaf + * block. The parent blocks then also record the first slot that has + * each order bit set in its child references. This indexing holds all + * the way to the root. This lets us quickly determine an order that + * will satisfy an allocation and descend to the leaf that contains the + * first free region of that order. * - * The single indirect block in the radix limits the number of blocks - * that can be described by the radix to just under a TB. The device - * will be managed by multiple radix trees some day. + * These buddy blocks themselves are located in preallocated space. Each + * logical position in the tree occupies two blocks on the device. In + * each transaction we use the currently referenced block to cow into + * its partner. Since the block positions are calculated the block + * references only need a bit to specify which of the pair is being + * referenced. The number of blocks needed is precisely calculated by + * taking the number of leaf blocks needed to track the device blocks + * and dividing by the radix fanout until we have a single root block. * - * XXX: - * - verify blocks on read? - * - more rigorously test valid blkno/order inputs - * - detect corruption/errors when trying to free free extents - * - mkfs should initialize all the slots - * - shrink and grow - * - metadata and data regions - * - worry about testing for free buddies outside device during free? - * - we could track the first set in order bitmaps, dunno if it'd be worth it + * Each aligned block allocation order is stored in a path down the + * radix to a leaf that's a function of the block offset. This lets us + * ensure that we can allocate or free a given allocation order by + * dirtying those blocks. If we've allocated an order in a transaction + * it can always be freed (or re-allocated) while the transaction holds + * the dirty buddy blocks. + * + * We use that property to ensure that frees of stable data don't + * satisfy allocation until the next transaction. When we free stable + * data we dirty the path to its position in the radix and record the + * free in an rbtree. We can then apply these frees as we commit the + * transaction. If the transaction fails we can undo the frees and let + * the file system carry on. We'll try to reapply the frees before the + * next transaction commits. The allocator never introduces + * unrecoverable errors. + * + * The radix isn't fully populated when it's created. mkfs only + * initializes the two paths down the tree that have partially + * initialized parent slots and leaf bitmaps. The path down the left + * spine has the initial file system blocks allocated. The path down + * the right spine can have partial parent slots and bits set in the + * leaf when device sizes aren't multiples of the leaf block bit count + * and radix fanout. The kernel then only has to initialize the rest of + * the buddy blocks blocks which have fully populated parent slots and + * leaf bitmaps. + * + * XXX + * - resize is going to be a thing. figure out that thing. */ -enum { - REGION_PAIR, /* two bitmap blocks at known blknos */ - REGION_BM, /* buddy blocks in the bitmap block off the super */ - REGION_BUDDY, /* btree blocks and extents in the buddy bitmaps */ +struct buddy_info { + struct mutex mutex; + + atomic_t alloc_count; + struct rb_root pending_frees; + + /* max height given total blocks */ + u8 max_height; + /* the device blkno of the first block of a given level */ + u64 level_blkno[SCOUTFS_BUDDY_MAX_HEIGHT]; + /* blk divisor to find slot index at each level */ + u64 level_div[SCOUTFS_BUDDY_MAX_HEIGHT]; + + struct buddy_stack { + struct buffer_head *bh[SCOUTFS_BUDDY_MAX_HEIGHT]; + u16 sl[SCOUTFS_BUDDY_MAX_HEIGHT]; + int nr; + } stack; }; -static int blkno_region(struct scoutfs_super_block *super, u64 blkno) -{ - u64 end; - - end = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR; - if (blkno < end) - return REGION_PAIR; - - end += le32_to_cpu(super->buddy_blocks); - if (blkno < end) - return REGION_BM; - - return REGION_BUDDY; -} - /* the first device blkno covered by the buddy allocator */ static u64 first_blkno(struct scoutfs_super_block *super) { - return SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + - le32_to_cpu(super->buddy_blocks); + return SCOUTFS_BUDDY_BLKNO + le64_to_cpu(super->buddy_blocks); } -/* the slot in the indirect block of a given blkno */ -static int indirect_slot(struct scoutfs_super_block *super, u64 blkno) +/* the last device blkno covered by the buddy allocator */ +static u64 last_blkno(struct scoutfs_super_block *super) { - return (u32)(blkno - first_blkno(super)) / SCOUTFS_BUDDY_ORDER0_BITS; + return le64_to_cpu(super->total_blocks) - 1; } -/* device blkno of order bit in slot */ -static u64 slot_buddy_blkno(struct scoutfs_super_block *super, int sl, - int order, int nr) +/* the last relative blkno covered by the buddy allocator */ +static u64 last_blk(struct scoutfs_super_block *super) { - return first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS) + - ((u64)nr << order); + return last_blkno(super) - first_blkno(super); } -/* number of blocks managed by the buddy block referenced by the given slot */ -static int slot_count(struct scoutfs_super_block *super, int sl) +/* true when the device blkno is covered by the allocator */ +static bool device_blkno(struct scoutfs_super_block *super, u64 blkno) { - u64 first = first_blkno(super) + ((u64)sl * SCOUTFS_BUDDY_ORDER0_BITS); - - return min_t(int, le64_to_cpu(super->total_blocks) - first, - SCOUTFS_BUDDY_ORDER0_BITS); + return blkno >= first_blkno(super) && blkno <= last_blkno(super); } -/* the order 0 bit offset of blkno */ -static int buddy_bit(struct scoutfs_super_block *super, u64 blkno) +/* true when the device blkno is used for buddy blocks */ +static bool buddy_blkno(struct scoutfs_super_block *super, u64 blkno) { - return (u32)(blkno - first_blkno(super)) % SCOUTFS_BUDDY_ORDER0_BITS; + return blkno < first_blkno(super); } -/* true if the blkno could be the start of an allocation of the order */ -static bool valid_order(struct scoutfs_super_block *super, u64 blkno, int order) +/* the order 0 bit offset in a buddy block of a given relative blk */ +static int buddy_bit(u64 blk) { - return (buddy_bit(super, blkno) & ((1 << order) - 1)) == 0; + return do_div(blk, SCOUTFS_BUDDY_ORDER0_BITS); } -/* the starting bit offset in the block bitmap of an order's bitmap */ +/* true if the rel blk could be the start of an allocation of the order */ +static bool valid_order(u64 blk, int order) +{ + return (buddy_bit(blk) & ((1 << order) - 1)) == 0; +} + +/* the block bit offset of the first bit of the given order's bitmap */ static int order_off(int order) { if (order == 0) @@ -146,733 +169,893 @@ static int order_nr(int order, int nr) return order_off(order) + nr; } +static void stack_push(struct buddy_stack *sta, struct buffer_head *bh, u16 sl) +{ + sta->bh[sta->nr] = bh; + sta->sl[sta->nr++] = sl; +} + +/* sl isn't returned because callers peek the leaf where sl is meaningless */ +static struct buffer_head *stack_peek(struct buddy_stack *sta) +{ + if (sta->nr) + return sta->bh[sta->nr - 1]; + + return NULL; +} + +static struct buffer_head *stack_pop(struct buddy_stack *sta, u16 *sl) +{ + if (sta->nr) { + *sl = sta->sl[--sta->nr]; + return sta->bh[sta->nr]; + } + + return NULL; +} + +/* update first_set if the caller set an earlier nr for the given order */ +static void set_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) +{ + u16 first = le16_to_cpu(bud->first_set[order]); + + trace_printk("set level %u order %d nr %u first %u\n", + bud->level, order, nr, first); + + if (nr <= first) + bud->first_set[order] = cpu_to_le16(nr); +} + +/* find the next first set if the caller just cleared the current first_set */ +static void clear_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) +{ + u16 first = le16_to_cpu(bud->first_set[order]); + int size; + int i; + + trace_printk("cleared level %u order %d nr %u first %u\n", + bud->level, order, nr, first); + + if (nr != first) + return; + + if (bud->level) { + for (i = nr + 1; i < SCOUTFS_BUDDY_SLOTS; i++) { + if (le16_to_cpu(bud->slots[i].free_orders) & + (1 << order)) + break; + } + if (i == SCOUTFS_BUDDY_SLOTS) + i = U16_MAX; + + } else { + size = order_off(order + 1); + i = find_next_bit_le(bud->bits, size, + order_nr(order, first) + 1); + if (i >= size) + i = U16_MAX; + else + i -= order_off(order); + } + + bud->first_set[order] = cpu_to_le16(i); + +} + +#define for_each_changed_bit(nr, bit, old, new, tmp) \ + for (tmp = old ^ new; \ + tmp && (nr = ffs(tmp) - 1, bit = 1 << nr, 1); \ + tmp ^= bit) + +/* + * Set a slot's free_orders value and update first_set for each order + * that it changes. Returns true of the slot's free_orders was changed. + */ +static bool set_slot_free_orders(struct scoutfs_buddy_block *bud, u16 sl, + u16 free_orders) +{ + u16 old = le16_to_cpu(bud->slots[sl].free_orders); + int order; + int tmp; + int bit; + + if (old == free_orders) + return false; + + for_each_changed_bit(order, bit, old, free_orders, tmp) { + if (old & bit) + clear_order_nr(bud, order, sl); + else + set_order_nr(bud, order, sl); + } + + bud->slots[sl].free_orders = cpu_to_le16(free_orders); + return true; +} + +/* + * The block at the top of the stack has changed its bits or slots and + * updated its first set. We propagate those changes up through + * free_orders in parents slots and their first_set up through the tree + * to free_orders in the root. We can stop when a block's first_set + * values don't change free_orders in their parent's slot. + */ +static void stack_cleanup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; + struct scoutfs_buddy_root *root = &sbi->super.buddy_root; + struct scoutfs_buddy_block *bud; + struct buffer_head *bh; + u16 free_orders = 0; + bool parent; + u16 sl; + int i; + + parent = false; + while ((bh = stack_pop(sta, &sl))) { + + bud = bh_data(bh); + if (parent && !set_slot_free_orders(bud, sl, free_orders)) + break; + + free_orders = 0; + for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) { + if (bud->first_set[i] != cpu_to_le16(U16_MAX)) + free_orders |= 1 << i; + } + + scoutfs_block_put(bh); + parent = true; + } + + /* set root if we got that far */ + if (bh == NULL) + root->slot.free_orders = cpu_to_le16(free_orders); + + /* put any remaining blocks */ + while ((bh = stack_pop(sta, &sl))) + scoutfs_block_put(bh); + +} + static int test_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { return !!test_bit_le(order_nr(order, nr), bud->bits); } -static int test_buddy_bit_or_higher(struct scoutfs_buddy_block *bud, int order, - int nr) +static void set_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { - int i; - - for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { - if (test_buddy_bit(bud, i, nr)) - return true; - nr >>= 1; - } - - return false; + if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) + set_order_nr(bud, order, nr); } -static void set_buddy_bit(struct scoutfs_buddy_indirect *ind, - struct scoutfs_buddy_block *bud, int order, int nr) +static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) { - if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) { - le64_add_cpu(&ind->order_totals[order], 1); - le32_add_cpu(&bud->order_counts[order], 1); - } -} - -static void clear_buddy_bit(struct scoutfs_buddy_indirect *ind, - struct scoutfs_buddy_block *bud, int order, int nr) -{ - if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) { - le64_add_cpu(&ind->order_totals[order], -1); - le32_add_cpu(&bud->order_counts[order], -1); - } -} - -/* returns INT_MAX when there are no bits set */ -static int find_next_buddy_bit(struct scoutfs_buddy_block *bud, int order, - int nr) -{ - int size = order_off(order + 1); - - nr = find_next_bit_le(bud->bits, size, order_nr(order, nr)); - if (nr >= size) - return INT_MAX; - - return nr - order_off(order); -} - -static void update_free_orders(struct scoutfs_buddy_slot *slot, - struct scoutfs_buddy_block *bud) -{ - u8 free = 0; - int i; - - for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) - free |= (!!bud->order_counts[i]) << i; - - slot->free_orders = free; + if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) + clear_order_nr(bud, order, nr); } /* - * Allocate a buddy block blkno from the super's dirty bitmap block. - * Stable buddy blocks are freed as they're cowed so we have to make - * sure that we only return blknos that were free in the previous stable - * bitmap block. + * mkfs always writes the paths down the sides of the radix that have + * partially populated blocks. We only have to initialize full blocks + * in the middle of the tree. */ -static int bitmap_alloc(struct super_block *sb, u64 *blkno) +static void init_buddy_block(struct buddy_info *binf, + struct scoutfs_super_block *super, + struct buffer_head *bh, int level) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_bitmap_block *st_bm; - struct scoutfs_bitmap_block *bm; - struct buffer_head *st_bh; - struct buffer_head *bm_bh; - int size; - int ret; - int d; - int s; - - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0 || - sbi->stable_super.buddy_bm_ref.blkno == 0) - return -EIO; - - /* dirty the bitmap block */ - bm_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bm_bh)) - return PTR_ERR(bm_bh); - bm = bh_data(bm_bh); - - /* read the stable bitmap block */ - st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_bm_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); - goto out; - } - st_bm = bh_data(st_bh); - - /* find the first bit that is set in both dirty and stable bitmaps */ - size = le32_to_cpu(sbi->super.buddy_blocks); - s = 0; - do { - d = find_next_bit_le(bm->bits, size, s); - s = find_next_bit_le(st_bm->bits, size, d); - } while (d != s); - if (d >= size) { - ret = -ENOSPC; - goto out; - } - - *blkno = SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR + d; - clear_bit_le(d, &bm->bits); - ret = 0; -out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bm_bh); - return ret; -} - -/* Free a buddy block blkno in the super's bitmap block. */ -static int bitmap_free(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_bitmap_block *bm; - struct buffer_head *bh; + struct scoutfs_buddy_block *bud = bh_data(bh); + u16 count; int nr; + int i; - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0) - return -EIO; + scoutfs_block_zero(bh, sizeof(bud->hdr)); - bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_bm_ref); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bm = bh_data(bh); + for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) + bud->first_set[i] = cpu_to_le16(U16_MAX); - nr = blkno - (SCOUTFS_BUDDY_BM_BLKNO + SCOUTFS_BUDDY_BM_NR); - set_bit_le(nr, bm->bits); - scoutfs_block_put(bh); + bud->level = level; - return 0; + if (level) { + for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) + set_slot_free_orders(bud, i, SCOUTFS_BUDDY_ORDER0_BITS); + } else { + /* ensure that there aren't multiple highest orders */ + BUILD_BUG_ON((SCOUTFS_BUDDY_ORDER0_BITS / + (1 << (SCOUTFS_BUDDY_ORDERS - 1))) > 1); + + count = SCOUTFS_BUDDY_ORDER0_BITS; + nr = 0; + for (i = SCOUTFS_BUDDY_ORDERS - 1; i >= 0; i--) { + if (count & (1 << i)) { + set_buddy_bit(bud, i, nr); + nr = (nr + 1) << 1; + } else { + nr <<= 1; + } + } + } } /* - * Give the caller a dirty buddy block. If the slot hasn't been used - * yet then we need to allocate and initialize a new block. + * Give the caller the block referenced by the given slot. They've + * calculated the blkno of the pair of blocks while walking the tree. + * The slot describes which of the pair its referencing. The caller is + * always going to modify the block so we always try and cow it. We + * construct a fake ref so we can re-use the block ref cow code. When + * we initialize the first use of a block we use the first of the pair. */ -static struct buffer_head *dirty_buddy_block(struct super_block *sb, - struct scoutfs_buddy_indirect *ind, - int sl, - struct scoutfs_buddy_slot *slot) +static struct buffer_head *get_buddy_block(struct super_block *sb, + struct scoutfs_buddy_slot *slot, + u64 blkno, int level) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; struct scoutfs_buddy_block *bud; + struct scoutfs_block_ref ref; struct buffer_head *bh; - u64 blkno; - int count; - int order; - int size; - int ret; - int nr; - /* the fast path is to dirty an existing block */ - if (slot->ref.blkno) - return scoutfs_block_dirty_ref(sb, &slot->ref); + trace_printk("getting block level %d blkno %llu slot seq %llu off %u\n", + level, blkno, le64_to_cpu(slot->seq), slot->blkno_off); - ret = bitmap_alloc(sb, &blkno); - if (ret) - return ERR_PTR(ret); - - bh = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(bh)) { - bitmap_free(sb, blkno); - return bh; - } - bud = bh_data(bh); - scoutfs_block_zero(bh, sizeof(bud->hdr)); - - /* mark the initial run of highest orders free */ - count = slot_count(super, sl); - order = SCOUTFS_BUDDY_ORDERS - 1; - size = 1 << order; - nr = 0; - while (count > size) { - set_buddy_bit(ind, bud, order, nr); - nr++; - count -= size; + /* init a new block for an unused slot */ + if (slot->seq == 0) { + bh = scoutfs_block_dirty(sb, blkno); + if (!IS_ERR(bh)) + init_buddy_block(binf, super, bh, level); + } else { + /* construct block ref from tree walk blkno and slot ref */ + ref.blkno = cpu_to_le64(blkno + slot->blkno_off); + ref.seq = slot->seq; + bh = scoutfs_block_dirty_ref(sb, &ref); } - /* set order bits for each of the bits set in the remaining count */ - do { - if (count & (1 << order)) { - set_buddy_bit(ind, bud, order, nr); - nr = (nr + 1) << 1; - } else { - nr <<= 1; + if (!IS_ERR(bh)) { + bud = bh_data(bh); + + trace_printk("got blkno %llu\n", (u64)bh->b_blocknr); + + /* rebuild slot ref to blkno */ + if (slot->seq != bud->hdr.seq) { + slot->blkno_off = le64_to_cpu(bud->hdr.blkno) - blkno; + /* alloc_same only xors low bit */ + BUG_ON(slot->blkno_off > 1); + slot->seq = bud->hdr.seq; } - } while (order--); - - slot->ref.blkno = bud->hdr.blkno; - slot->ref.seq = bud->hdr.seq; - - update_free_orders(slot, bud); + } return bh; } /* - * Return the order bitmap offset and order of the first allocation - * that fits the desired order. + * Walk the buddy block radix to the leaf that contains either the given + * relative blk or the first free given order. The radix is of a fixed + * depth and we initialize new blocks as we descend through + * uninitialized refs. * - * Returns INT_MAX if there are no free orders. + * If order is -1 then we search for the blk. + * + * As we descend we calculate the base blk offset of the path we're + * taking down the tree. This is used to find the blkno of the next + * block relative to the blkno of the given level. It's then used by + * the caller to calculate the total blk offset by adding the bit they + * find in the block. + * + * The path through the tree is recorded in the stack in the buddy info. + * The caller is responsible for cleaning up the stack and must do so + * even if we return an error. */ -static int find_first_fit(struct scoutfs_super_block *super, int sl, - struct scoutfs_buddy_block *bud, - struct scoutfs_buddy_block *st_bud, - int order, int *order_ret) +static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) { - int nrs[SCOUTFS_BUDDY_ORDERS] = {0,}; - u64 blkno = U64_MAX; - bool made_progress; - int ret = INT_MAX; - u64 bno; - int nr; - int i; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; + struct scoutfs_buddy_root *root = &sbi->super.buddy_root; + struct scoutfs_buddy_block *bud; + struct scoutfs_buddy_slot *slot; + struct buffer_head *bh; + u64 blkno; + int level; + int ret = 0; + int sl = 0; - do { - made_progress = false; - for (i = order; i < SCOUTFS_BUDDY_ORDERS; i++) { - /* find the next bit in each order */ - nr = find_next_buddy_bit(bud, i, nrs[i]); - nrs[i] = nr; - if (nr == INT_MAX) { - continue; - } - made_progress = true; + /* XXX corruption? */ + if (blk > last_blk(super) || root->height == 0 || + root->height > SCOUTFS_BUDDY_MAX_HEIGHT) + return -EIO; - /* advance to next bit if it's not free in stable */ - if (st_bud && - !test_buddy_bit_or_higher(st_bud, i, nr)) { - nrs[i] = nr + 1; - continue; - } + slot = &root->slot; + level = root->height; + blkno = SCOUTFS_BUDDY_BLKNO; + *base = 0; - /* use the first lowest order blkno */ - bno = slot_buddy_blkno(super, sl, i, nr); - if (bno < blkno) { - blkno = bno; - *order_ret = i; - ret = nr; - } + while (level--) { + /* XXX do base and level make sense here? */ + bh = get_buddy_block(sb, slot, blkno, level); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + break; } - } while (ret == INT_MAX && made_progress); + trace_printk("before blk %llu order %d level %d blkno %llu base %llu sl %d\n", + blk, order, level, blkno, *base, sl); + + bud = bh_data(bh); + + if (level) { + if (order >= 0) { + /* find first slot with order free */ + sl = le16_to_cpu(bud->first_set[order]); + /* XXX corruption */ + if (sl == U16_MAX) { + ret = -EIO; + break; + } + } else { + /* find slot based on blk */ + sl = div64_u64_rem(blk, binf->level_div[level], + &blk); + } + + /* shouldn't be sl * 2, right? */ + *base = (*base * SCOUTFS_BUDDY_SLOTS) + sl; + /* this is the only place we * 2 */ + blkno = binf->level_blkno[level - 1] + (*base * 2); + slot = &bud->slots[sl]; + } else { + *base *= SCOUTFS_BUDDY_ORDER0_BITS; + /* sl in stack is 0 for final leaf block */ + sl = 0; + } + + trace_printk("after blk %llu order %d level %d blkno %llu base %llu sl %d\n", + blk, order, level, blkno, *base, sl); + + + stack_push(sta, bh, sl); + } + + trace_printk("walking ret %d\n", ret); return ret; } /* - * Find the first free region that satisfies the given order that is - * also free in the stable buddy bitmaps. This can return an allocation - * that breaks up a larger order. Higher level callers iterate over - * smaller orders to provide partial allocations. + * Find the order to search for to allocate a requested order. We try + * to use the smallest greater or equal order and then the largest + * smaller order. */ -static int alloc_slot(struct super_block *sb, - struct scoutfs_buddy_indirect *ind, int sl, - struct scoutfs_buddy_slot *slot, - struct scoutfs_block_ref *stable_ref, - u64 *blkno, int order) +static int find_free_order(struct scoutfs_buddy_root *root, int order) +{ + u16 free = le16_to_cpu(root->slot.free_orders); + u16 smaller_mask = (1 << order) - 1; + u16 larger = free & ~smaller_mask; + u16 smaller = free & smaller_mask; + + if (larger) + return ffs(larger) - 1; + if (smaller) + return fls(smaller) - 1; + + return -ENOSPC; +} + +/* + * Walk to the leaf that contains the found order and allocate a region + * of the given order, returning the relative blk to the caller. + */ +static int buddy_alloc(struct super_block *sb, u64 *blk, int order, int found) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct scoutfs_buddy_block *st_bud; - struct buffer_head *st_bh; struct buffer_head *bh; - int found; + u64 base; int ret; int nr; int i; - /* initialize or dirty the slot's buddy block */ - bh = dirty_buddy_block(sb, ind, sl, slot); - if (IS_ERR(bh)) - return PTR_ERR(bh); + trace_printk("alloc order %d found %d\n", order, found); + + if (WARN_ON_ONCE(found >= 0 && order > found)) + return -EINVAL; + + ret = buddy_walk(sb, *blk, found, &base); + if (ret) + goto out; + + bh = stack_peek(sta); bud = bh_data(bh); - /* read stable slots's buddy block if there is one */ - if (stable_ref->blkno) { - st_bh = scoutfs_block_read_ref(sb, stable_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); + if (found >= 0) { + nr = le16_to_cpu(bud->first_set[found]); + /* XXX corruption */ + if (nr == U16_MAX) { + ret = -EIO; goto out; } - st_bud = bh_data(st_bh); + + /* give caller the found blk for the order */ + *blk = base + (nr << found); } else { - st_bh = NULL; - st_bud = NULL; + nr = buddy_bit(*blk) >> found; } - nr = find_first_fit(super, sl, bud, st_bud, order, &found); - if (nr == INT_MAX) { - ret = -ENOSPC; - goto out; + /* always allocate the higher or equal found order */ + clear_buddy_bit(bud, found, nr); + + /* and maybe free our buddies between smaller order and larger found */ + nr = buddy_bit(*blk) >> order; + for (i = order; i < found; i++) { + set_buddy_bit(bud, i, nr ^ 1); + nr >>= 1; } - /* we'll succeed from this point on, use nr before mangling it */ - *blkno = slot_buddy_blkno(super, sl, found, nr); - - /* always clear the found order */ - clear_buddy_bit(ind, bud, found, nr); - - /* free right buddies if we're breaking up a larger order */ - for (nr <<= 1, i = found - 1; i >= order; i--, nr <<= 1) - set_buddy_bit(ind, bud, i, nr | 1); - - update_free_orders(slot, bud); ret = 0; out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bh); + trace_printk("alloc order %d found %d blk %llu ret %d\n", + order, found, *blk, ret); + stack_cleanup(sb); return ret; } /* - * Try and find a free block extent of the given order. We can fail to - * find a free order when none of the slots have free orders as the - * volume fills or gets fragmented. + * Free a given order by setting its order bit. If the order's buddy + * isn't set then it isn't free and we can't merge so we set our order + * and are done. If the buddy is free then we can clear it and ascend + * up to try and set the next higher order. That performs the same + * buddy merging test. Eventually we make it to the highest order which + * doesn't have a buddy so we can always set it. * - * We also have to be careful to only return free extents that were free - * in the old stable buddy allocator so that we don't allocate and write - * over referenced data. This can cause us to skip otherwise available - * extents but it should be rare. There can only be a transaction's - * worth of difference between the dirty allocator and the stable - * allocator. This is one of the motivations to cap the size of - * transactions. + * As we're freeing orders in the final buddy bitmap that only partially + * covers the end of the device we might try to test buddies which are + * past the end of the device. The test will still fall within the leaf + * block bitmap and those bits past the device will never be set so we + * will fail the merge and correctly set the orders free. */ -static int alloc_order(struct super_block *sb, u64 *blkno, int order) +static int buddy_free(struct super_block *sb, u64 blk, int order) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_buddy_indirect *st_ind; - struct scoutfs_buddy_indirect *ind; - struct buffer_head *st_bh = NULL; - struct buffer_head *bh = NULL; - u8 mask; - int ret; - int i; - - /* mkfs should have ensured that there's indirect blocks */ - if (sbi->super.buddy_ind_ref.blkno == 0 || - sbi->stable_super.buddy_ind_ref.blkno == 0) { - ret = -EIO; - goto out; - } - - /* get the dirty indirect block */ - bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - ind = bh_data(bh); - - /* get the stable indirect block */ - st_bh = scoutfs_block_read_ref(sb, &sbi->stable_super.buddy_ind_ref); - if (IS_ERR(st_bh)) { - ret = PTR_ERR(st_bh); - goto out; - } - st_ind = bh_data(st_bh); - - mask = ~0U << order; - - /* - * try to alloc from each slot that has at least the order free - * in both the dirty and stable buddy blocks. - */ - for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) { - if (!((mask & ind->slots[i].free_orders) && - (mask & st_ind->slots[i].free_orders))) { - ret = -ENOSPC; - continue; - } - - ret = alloc_slot(sb, ind, i, &ind->slots[i], - &st_ind->slots[i].ref, blkno, order); - if (ret != -ENOSPC) - break; - } - -out: - scoutfs_block_put(st_bh); - scoutfs_block_put(bh); - - return ret; -} - -/* - * The buddy allocator keeps trying smaller orders until it finds an - * allocation. - * - * The order of the allocation is returned. - */ -static int buddy_alloc(struct super_block *sb, u64 *blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS)) - return -EINVAL; - - mutex_lock(&sbi->buddy_mutex); - - do { - ret = alloc_order(sb, blkno, order); - } while (ret == -ENOSPC && order--); - - mutex_unlock(&sbi->buddy_mutex); - - return ret ?: order; -} - -/* - * Allocate a block from the given region. The caller has the buddy - * mutex if we're called for either of the pair or bitmap internal - * regions. - */ -static int alloc_region(struct super_block *sb, u64 *blkno, int order, - u64 existing, int region) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - switch(region) { - case REGION_PAIR: - *blkno = existing ^ 1; - ret = 0; - break; - case REGION_BM: - ret = bitmap_alloc(sb, blkno); - break; - case REGION_BUDDY: - ret = buddy_alloc(sb, blkno, order); - break; - default: - WARN_ON_ONCE(1); - ret = -EINVAL; - } - - /* this misses other direct calls to bitmap_alloc, but that's minor */ - if (ret >= 0) - atomic_add(1 << ret, &sbi->buddy_count); - - trace_scoutfs_buddy_alloc(*blkno, order, region, ret); - return ret; -} - -int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) -{ - return alloc_region(sb, blkno, order, 0, REGION_BUDDY); -} - -/* - * The block layer allocates from the same region as an existing blkno - * when it's allocating for cow. - */ -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, - u64 existing) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - return alloc_region(sb, blkno, order, existing, - blkno_region(super, existing)); -} - -/* - * Free the aligned allocation of the given order at the given blkno to - * the allocator. We merge it into adjoining free space by looking for - * free buddies as we increase the order. - */ -static int buddy_free(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_indirect *ind; + struct buddy_info *binf = sbi->buddy_info; + struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct buffer_head *ind_bh = NULL; - struct buffer_head *bh = NULL; + struct buffer_head *bh; + u64 unused; int ret; - int sl; int nr; int i; - if (WARN_ON_ONCE(order < 0 || order >= SCOUTFS_BUDDY_ORDERS) || - WARN_ON_ONCE(!valid_order(super, blkno, order))) - return -EINVAL; - - mutex_lock(&sbi->buddy_mutex); - - /* mkfs should have ensured that there's indirect blocks */ - if (sbi->super.buddy_ind_ref.blkno == 0) { - ret = -EIO; + ret = buddy_walk(sb, blk, -1, &unused); + if (ret) goto out; - } - ind_bh = scoutfs_block_dirty_ref(sb, &sbi->super.buddy_ind_ref); - if (IS_ERR(ind_bh)) { - ret = PTR_ERR(ind_bh); - goto out; - } - ind = bh_data(ind_bh); - - sl = indirect_slot(super, blkno); - bh = scoutfs_block_dirty_ref(sb, &ind->slots[sl].ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } + bh = stack_peek(sta); bud = bh_data(bh); - /* - * Merge our region with its free buddy and then try to merge - * that higher order region with its buddy, and so on, until the - * highest order. The highest order doesn't have buddies. - */ - nr = buddy_bit(super, blkno) >> order; - for (i = order; i < SCOUTFS_BUDDY_ORDERS - 1; i++) { + nr = buddy_bit(blk) >> order; + for (i = order; i < SCOUTFS_BUDDY_ORDERS - 2; i++) { if (!test_buddy_bit(bud, i, nr ^ 1)) break; - clear_buddy_bit(ind, bud, i, nr ^ 1); + clear_buddy_bit(bud, i, nr ^ 1); nr >>= 1; } - set_buddy_bit(ind, bud, i, nr); + set_buddy_bit(bud, i, nr); - update_free_orders(&ind->slots[sl], bud); - scoutfs_block_put(bh); ret = 0; out: - mutex_unlock(&sbi->buddy_mutex); - scoutfs_block_put(ind_bh); - + stack_cleanup(sb); return ret; } -int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order) +/* + * Try to allocate an extent with the size number of blocks. blkno is + * set to the start of the extent and the order of the block count is + * returned. + */ +int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - int region; + struct buddy_info *binf = sbi->buddy_info; + int found; + u64 blk; int ret; - region = blkno_region(super, blkno); - switch(blkno_region(super, blkno)) { - case REGION_PAIR: - ret = 0; - break; - case REGION_BM: - ret = bitmap_free(sb, blkno); - break; - case REGION_BUDDY: - ret = buddy_free(sb, blkno, order); - break; + trace_printk("order %d\n", order); + + mutex_lock(&binf->mutex); + + found = find_free_order(&super->buddy_root, order); + if (found < 0) { + ret = found; + goto out; } - trace_scoutfs_buddy_free(blkno, order, region, ret); + if (found < order) + order = found; + + blk = 0; + ret = buddy_alloc(sb, &blk, order, found); + if (ret) + goto out; + + *blkno = first_blkno(super) + blk; + le64_add_cpu(&super->free_blocks, -(1ULL << order)); + atomic_add((1ULL << order), &binf->alloc_count); + ret = order; + +out: + trace_printk("blkno %llu order %d ret %d\n", *blkno, order, ret); + mutex_unlock(&binf->mutex); return ret; } +/* + * We use the block _ref() routines to dirty existing blocks to reuse + * all the block verification and cow machinery. During cow this is + * called to allocate a new blkno to cow an existing buddy block. We + * use the existing blkno to see if we have to return the other mirrored + * buddy blkno or do a real allocation for every other kind of block + * being cowed. + */ +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + if (buddy_blkno(super, existing)) { + *blkno = existing ^ 1; + trace_printk("existing %llu ret blkno %llu\n", + existing, *blkno); + return 0; + } + + return scoutfs_buddy_alloc(sb, blkno, 0); +} + +struct extent_node { + struct rb_node node; + u64 start; + u64 len; +}; + +static int add_enode_extent(struct rb_root *root, u64 start, u64 len) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct extent_node *left = NULL; + struct extent_node *right = NULL; + struct extent_node *enode; + + trace_printk("adding enode [%llu,%llu]\n", start, len); + + while (*node && !(left && right)) { + parent = *node; + enode = container_of(*node, struct extent_node, node); + + if (start < enode->start) { + if (!right && start + len == enode->start) + right = enode; + node = &(*node)->rb_left; + } else { + if (!left && enode->start + enode->len == start) + left = enode; + node = &(*node)->rb_right; + } + } + + if (right) { + right->start = start; + right->len += len; + trace_printk("right now [%llu, %llu]\n", + right->start, right->len); + } + + if (left) { + if (right) { + left->len += right->len; + rb_erase(&right->node, root); + kfree(right); + } else { + left->len += len; + } + trace_printk("left now [%llu, %llu]\n", left->start, left->len); + } + + if (left || right) + return 0; + + enode = kmalloc(sizeof(struct extent_node), GFP_NOFS); + if (!enode) + return -ENOMEM; + + enode->start = start; + enode->len = len; + + trace_printk("inserted new [%llu, %llu]\n", enode->start, enode->len); + + rb_link_node(&enode->node, parent, node); + rb_insert_color(&enode->node, root); + + return 0; +} + +static void destroy_pending_frees(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct extent_node *enode; + struct rb_node *node; + + for (node = rb_first(&binf->pending_frees); node;) { + enode = rb_entry(node, struct extent_node, node); + node = rb_next(node); + + rb_erase(&enode->node, &binf->pending_frees); + kfree(enode); + } +} + /* XXX this should be generic */ #define min3_t(t, a, b, c) min3((t)(a), (t)(b), (t)(c)) /* - * Free all the order allocations that make up the given unaligned block - * extent. Think of it as figuring out the largest aligned allocation - * that starts at the blkno and then clamping it by the count. + * Allocate or free all the orders that make up a given arbitrary block + * extent. Today this is used by callers who know that the blocks for + * the extent have already been pinned so we BUG on error. + */ +static void apply_extent(struct super_block *sb, bool alloc, u64 blk, u64 len) +{ + unsigned int blk_order; + unsigned int blk_bit; + unsigned int size; + int order; + int ret; + + trace_printk("applying extent blk %llu len %llu\n", blk, len); + + while (len) { + /* buddy bit might be 0, len always has a bit set */ + blk_bit = buddy_bit(blk); + blk_order = blk_bit ? ffs(blk_bit) - 1 : 0; + order = min3_t(int, blk_order, fls64(len) - 1, + SCOUTFS_BUDDY_ORDERS - 1); + size = 1 << order; + + trace_printk("applying blk %llu order %d\n", blk, order); + + if (alloc) + ret = buddy_alloc(sb, &blk, order, -1); + else + ret = buddy_free(sb, blk, order); + BUG_ON(ret); + + blk += size; + len -= size; + } +} + +/* + * The pending rbtree has recorded frees of stable data that we had to + * wait until transaction commit to record. Once these are tracked in + * the allocator we can't use the allocator until the commit succeeds. + * This is called by transaction commit to get these pending frees into + * the current commit. If it fails they pull them back out. + */ +int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct extent_node *enode; + struct rb_node *node; + + for (node = rb_first(&binf->pending_frees); node;) { + enode = rb_entry(node, struct extent_node, node); + node = rb_next(node); + + apply_extent(sb, alloc, enode->start, enode->len); + } + + return 0; +} + +/* + * Free a given allocated extent. The seq tells us which transaction + * first allocated the extent. If it was allocated in this transaction + * then we can return it to the free buddy and that must succeed. * - * For now this is only used by callers who have pinned the blocks that - * provided the allocation that they're now freeing from. It can't - * fail. If it could we would ensure that we re-alloc partial frees - * before returning an error. + * If it was allocated in a previous transaction then we dirty the + * blocks it will take to free it then record it in an rbtree. The + * rbtree entries are replayed into the dirty blocks as the transaction + * commits. + * + * Buddy block numbers are preallocated and calculated from the radix + * tree structure so we can ignore the block layer's calls to free buddy + * blocks during cow. + */ +int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, int order) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + u64 unused; + u64 blk; + int ret; + + trace_printk("seq %llu blkno %llu order %d rsv %u\n", + le64_to_cpu(seq), blkno, order, buddy_blkno(super, blkno)); + + /* no specific free tracking for buddy blocks */ + if (buddy_blkno(super, blkno)) + return 0; + + /* XXX corruption? */ + if (!device_blkno(super, blkno)) + return -EINVAL; + + blk = blkno - first_blkno(super); + + if (!valid_order(blk, order)) + return -EINVAL; + + mutex_lock(&binf->mutex); + + if (seq == super->hdr.seq) { + ret = buddy_free(sb, blk, order); + /* + * If this order was allocated in this transaction then its + * blocks should be pinned and we should always be able + * to free it. + */ + BUG_ON(ret); + } else { + ret = buddy_walk(sb, blk, -1, &unused) ?: + add_enode_extent(&binf->pending_frees, blk, 1 << order); + if (ret == 0) + trace_printk("added blk %llu order %d\n", blk, order); + stack_cleanup(sb); + } + + if (ret == 0) + le64_add_cpu(&super->free_blocks, 1ULL << order); + + mutex_unlock(&binf->mutex); + + return ret; +} + +/* + * This is current only used to return partial extents from larger + * allocations in this transaction. */ void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; struct scoutfs_super_block *super = &sbi->stable_super; - int order; - int size; - int ret; + u64 blk; - while (count) { - /* both blkno and count have to have bits set */ - order = min3_t(int, __ffs64(buddy_bit(super, blkno)), - fls64(count) - 1, - SCOUTFS_BUDDY_ORDERS - 1); - size = 1 << order; + BUG_ON(!device_blkno(super, blkno)); - ret = scoutfs_buddy_free(sb, blkno, order); - BUG_ON(ret); + blk = blkno - first_blkno(super); - blkno += size; - count -= size; - } -} + mutex_lock(&binf->mutex); -/* - * Return > 1 if the given order allocation was free in the old stable - * transaction, 0 if it wasn't, and -errno if errors prevented us from - * finding out. - * - * XXX I bet we could get away without using the buddy mutex - */ -int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->stable_super; - struct buffer_head *ind_bh = NULL; - struct buffer_head *bh = NULL; - struct scoutfs_buddy_indirect *ind; - struct scoutfs_buddy_block *bud; - struct scoutfs_block_ref *ref; - int ret; - int nr; - int sl; - - /* mkfs should have ensured that there's bitmap blocks */ - /* XXX corruption */ - if (sbi->super.buddy_bm_ref.blkno == 0 || - sbi->stable_super.buddy_bm_ref.blkno == 0) - return -EIO; - - mutex_lock(&sbi->buddy_mutex); - - /* get the stable indirect block */ - ind_bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); - if (IS_ERR(ind_bh)) { - ret = PTR_ERR(ind_bh); - goto out; - } - ind = bh_data(ind_bh); - - /* allocation was free if it's slot wasn't populated */ - sl = indirect_slot(super, blkno); - ref = &ind->slots[sl].ref; - if (!ref->blkno) { - ret = 1; - goto out; - } - - /* check the allocation bit in the old stable bitmap block */ - bh = scoutfs_block_read_ref(sb, ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - bud = bh_data(bh); - - nr = buddy_bit(super, blkno) >> order; - ret = !!test_buddy_bit_or_higher(bud, order, nr); - -out: - mutex_unlock(&sbi->buddy_mutex); - scoutfs_block_put(ind_bh); - scoutfs_block_put(bh); - - trace_printk("blkno %llu order %d ret %d\n", blkno, order, ret); - return ret; -} - -/* - * For now we only have one indirect block off the super. When we grow - * multiple commit block pairs that reference root and indirect blocks - * then we'll need to iterate over those. These results will only ever - * be approximate so we can simply use racey valid ref reads to be able - * to sample while others are writing. - */ -int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_buddy_indirect *ind; - struct buffer_head *bh; - int ret; - int i; - - *bfree = 0; - - bh = scoutfs_block_read_ref(sb, &super->buddy_ind_ref); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); - goto out; - } - ind = bh_data(bh); - - for (i = 0; i < SCOUTFS_BUDDY_ORDERS; i++) - *bfree += le64_to_cpu(ind->order_totals[i]) << i; - - scoutfs_block_put(bh); - ret = 0; -out: - return ret; + apply_extent(sb, false, blkno - first_blkno(super), count); + le64_add_cpu(&super->free_blocks, count); + mutex_unlock(&binf->mutex); } /* * Return the number of block allocations since the last time the - * counter was reset. This count doesn't include some internal bitmap - * block allocations but that should be a small fraction of the main - * allocations. + * counter was reset. This count doesn't include dirty buddy blocks. */ unsigned int scoutfs_buddy_alloc_count(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; - return atomic_read(&sbi->buddy_count); + return atomic_read(&binf->alloc_count); } -void scoutfs_buddy_reset_count(struct super_block *sb) +u64 scoutfs_buddy_bfree(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + struct scoutfs_super_block *super = &sbi->super; + u64 ret; - return atomic_set(&sbi->buddy_count, 0); + mutex_lock(&binf->mutex); + ret = le64_to_cpu(super->free_blocks); + mutex_unlock(&binf->mutex); + + return ret; } + +void scoutfs_buddy_committed(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + + atomic_set(&binf->alloc_count, 0); + destroy_pending_frees(sb); +} + +int scoutfs_buddy_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct buddy_info *binf = sbi->buddy_info; + u64 level_blocks[SCOUTFS_BUDDY_MAX_HEIGHT]; + u64 blocks; + int i; + + /* first bit offsets in blocks are __le16 */ + BUILD_BUG_ON(SCOUTFS_BUDDY_ORDER0_BITS >= U16_MAX); + + /* bits need to be naturally aligned to long for _le bitops */ + BUILD_BUG_ON(offsetof(struct scoutfs_buddy_block, bits) & + (sizeof(long) - 1)); + + binf = kzalloc(sizeof(struct buddy_info), GFP_KERNEL); + if (!binf) + return -ENOMEM; + sbi->buddy_info = binf; + + mutex_init(&binf->mutex); + atomic_set(&binf->alloc_count, 0); + binf->pending_frees = RB_ROOT; + + /* calculate blocks at each level */ + blocks = DIV_ROUND_UP_ULL(last_blk(super) + 1, + SCOUTFS_BUDDY_ORDER0_BITS); + for (i = 0; i < SCOUTFS_BUDDY_MAX_HEIGHT; i++) { + level_blocks[i] = (blocks * 2); + if (blocks == 1) { + binf->max_height = i + 1; + break; + } + blocks = DIV_ROUND_UP_ULL(blocks, SCOUTFS_BUDDY_SLOTS); + } + + /* calculate device blkno of first block in each level */ + binf->level_blkno[binf->max_height - 1] = SCOUTFS_BUDDY_BLKNO; + for (i = (binf->max_height - 2); i >= 0; i--) { + binf->level_blkno[i] = binf->level_blkno[i + 1] + + level_blocks[i + 1]; + } + + /* calculate blk divisor to find slot at a given level */ + binf->level_div[1] = SCOUTFS_BUDDY_ORDER0_BITS; + for (i = 2; i < binf->max_height; i++) { + binf->level_div[i] = binf->level_div[i - 1] * + SCOUTFS_BUDDY_SLOTS; + } + + for (i = 0; i < binf->max_height; i++) + trace_printk("level %d div %llu blkno %llu blocks %llu\n", + i, binf->level_div[i], binf->level_blkno[i], + level_blocks[i]); + + return 0; +} + +void scoutfs_buddy_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct buddy_info *binf = sbi->buddy_info; + + if (binf) + WARN_ON_ONCE(!RB_EMPTY_ROOT(&binf->pending_frees)); + kfree(binf); +} + diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h index 059d586a..24c0ed0c 100644 --- a/kmod/src/buddy.h +++ b/kmod/src/buddy.h @@ -2,15 +2,19 @@ #define _SCOUTFS_BUDDY_H_ int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, int order, - u64 existing); -int scoutfs_buddy_free(struct super_block *sb, u64 blkno, int order); +int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing); +int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, + int order); void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); -int scoutfs_buddy_bfree(struct super_block *sb, u64 *bfree); +u64 scoutfs_buddy_bfree(struct super_block *sb); unsigned int scoutfs_buddy_alloc_count(struct super_block *sb); -void scoutfs_buddy_reset_count(struct super_block *sb); +int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc); +void scoutfs_buddy_committed(struct super_block *sb); + +int scoutfs_buddy_setup(struct super_block *sb); +void scoutfs_buddy_destroy(struct super_block *sb); #endif diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 597ce144..abd754a7 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -147,7 +147,7 @@ static int alloc_file_block(struct super_block *sb, u64 *blkno) spin_unlock(&sbi->file_alloc_lock); if (order > 0) - scoutfs_buddy_free(sb, alloc_blkno, order); + scoutfs_buddy_free(sb, sbi->super.hdr.seq, alloc_blkno, order); out: trace_printk("allocated blkno %llu ret %d\n", *blkno, ret); @@ -246,7 +246,7 @@ int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) if (blkno == 0) continue; - ret = scoutfs_buddy_free(sb, blkno, 0); + ret = scoutfs_buddy_free(sb, bmap.seq[i], blkno, 0); if (ret) break; @@ -357,6 +357,8 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) { struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->stable_super; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_block_map bmap; struct scoutfs_btree_val val; @@ -406,7 +408,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) goto out; if (old_blkno) { - ret = scoutfs_buddy_free(sb, old_blkno, 0); + ret = scoutfs_buddy_free(sb, bmap.seq[i], old_blkno, 0); if (ret) goto out; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 58fea85f..fe83a346 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -19,8 +19,7 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 -#define SCOUTFS_BUDDY_BM_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) -#define SCOUTFS_BUDDY_BM_NR 2 +#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) #define SCOUTFS_MAX_TRANS_BLOCKS (128 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) @@ -48,42 +47,49 @@ struct scoutfs_block_ref { __le64 seq; } __packed; -struct scoutfs_bitmap_block { - struct scoutfs_block_header hdr; - __le64 bits[0]; -} __packed; - /* - * Track allocations from BLOCK_SIZE to (BLOCK_SIZE << ..._ORDERS). + * If the block was full of bits the largest possible order would be + * the block size shift + 3 (BITS_PER_BYTE). But the header uses + * up some space and then the buddy bits mean two bits per block. + * Then +1 for this being the number, not the greatest order. */ -#define SCOUTFS_BUDDY_ORDERS 8 +#define SCOUTFS_BUDDY_ORDERS (SCOUTFS_BLOCK_SHIFT + 3 - 2 + 1) struct scoutfs_buddy_block { struct scoutfs_block_header hdr; - __le32 order_counts[SCOUTFS_BUDDY_ORDERS]; - __le64 bits[0]; + __le16 first_set[SCOUTFS_BUDDY_ORDERS]; + __u8 level; + __u8 __pad[3]; /* naturally align bits */ + union { + struct scoutfs_buddy_slot { + __le64 seq; + __le16 free_orders; + /* XXX seems like we could hide a bit somewhere */ + __u8 blkno_off; + } __packed slots[0]; + __le64 bits[0]; + } __packed; } __packed; /* - * If we had log2(raw bits) orders we'd fully use all of the raw bits in - * the block. We're close enough that the amount of space wasted at the - * end (~1/256th of the block, ~64 bytes) isn't worth worrying about. + * Each buddy leaf block references order 0 blocks with half of its + * bitmap. The other half of the bits are used for the higher order + * bits. */ #define SCOUTFS_BUDDY_ORDER0_BITS \ (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2) -struct scoutfs_buddy_indirect { - struct scoutfs_block_header hdr; - __le64 order_totals[SCOUTFS_BUDDY_ORDERS]; - struct scoutfs_buddy_slot { - __u8 free_orders; - struct scoutfs_block_ref ref; - } slots[0]; +#define SCOUTFS_BUDDY_SLOTS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) / \ + sizeof(struct scoutfs_buddy_slot)) + +struct scoutfs_buddy_root { + struct scoutfs_buddy_slot slot; + __u8 height; } __packed; -#define SCOUTFS_BUDDY_SLOTS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_indirect)) / \ - sizeof(struct scoutfs_buddy_slot)) +/* ((SCOUTFS_BUDDY_SLOTS^5) * SCOUTFS_BUDDY_ORDER0_BITS) > 2^52 */ +#define SCOUTFS_BUDDY_MAX_HEIGHT 6 /* * We should be able to make the offset smaller if neither dirents nor @@ -180,10 +186,10 @@ struct scoutfs_super_block { __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 total_blocks; - __le32 buddy_blocks; + __le64 free_blocks; + __le64 buddy_blocks; + struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; - struct scoutfs_block_ref buddy_ind_ref; - struct scoutfs_block_ref buddy_bm_ref; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/super.c b/kmod/src/super.c index a668f8a2..b6d94a42 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -48,12 +48,8 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; __le32 * __packed uuid = (void *)super->uuid; - int ret; - - ret = scoutfs_buddy_bfree(sb, &kst->f_bfree); - if (ret) - return ret; + kst->f_bfree = scoutfs_buddy_bfree(sb); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; kst->f_blocks = le64_to_cpu(super->total_blocks); @@ -198,8 +194,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->block_dirty_tree = RB_ROOT; init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); - mutex_init(&sbi->buddy_mutex); - atomic_set(&sbi->buddy_count, 0); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); @@ -220,6 +214,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: + scoutfs_buddy_setup(sb) ?: scoutfs_setup_trans(sb); if (ret) return ret; @@ -252,6 +247,7 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { scoutfs_shutdown_trans(sb); + scoutfs_buddy_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); diff --git a/kmod/src/super.h b/kmod/src/super.h index 604e0231..9247f917 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -8,7 +8,7 @@ #include "buddy.h" struct scoutfs_counters; -struct buddy_alloc; +struct buddy_info; struct scoutfs_sb_info { struct super_block *sb; @@ -24,8 +24,7 @@ struct scoutfs_sb_info { atomic_t block_writes; int block_write_err; - struct mutex buddy_mutex; - atomic_t buddy_count; + struct buddy_info *buddy_info; struct rw_semaphore btree_rwsem; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e2679dd7..d089a1d4 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -90,17 +90,20 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); - ret = scoutfs_block_write_dirty(sb) ?: + ret = scoutfs_buddy_apply_pending(sb, false) ?: + scoutfs_block_write_dirty(sb) ?: scoutfs_write_dirty_super(sb); - if (!ret) + if (ret) { + scoutfs_buddy_apply_pending(sb, true); + } else { + scoutfs_buddy_committed(sb); advance = 1; + } } spin_lock(&sbi->trans_write_lock); - if (advance) { + if (advance) scoutfs_advance_dirty_super(sb); - scoutfs_buddy_reset_count(sb); - } sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock); From 3d66a4b3ddb5d137e47476bf6bf74ef14a4d8947 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 11:44:33 -0800 Subject: [PATCH 120/920] Block API offers scoutfs_block instead of bh Our thin block wrappers exposed buffer heads to all the callers. We're about to revert back to the block interface that uses its own scoutfs_block struct instead of buffer heads. Let's reduce the churn of that patch by first having the block API give callers an opaque struct scoutfs_block. Internally it's still buffer heads but the callers don't know that. scoutfs_write_dirty_super() is the exception who has magical knowledge of buffer heads. That's fixed once the new block API offers a function for writing a single block. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 126 +++++++++++------- kmod/src/block.h | 40 +++--- kmod/src/btree.c | 337 +++++++++++++++++++++++------------------------ kmod/src/buddy.c | 83 ++++++------ kmod/src/super.c | 24 ++-- 5 files changed, 312 insertions(+), 298 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index b3f99f09..af90b5ef 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -38,6 +38,8 @@ * - should invalidate dirty blocks if freed */ +struct scoutfs_block; + struct block_bh_private { struct super_block *sb; struct buffer_head *bh; @@ -183,7 +185,7 @@ static void erase_bhp(struct buffer_head *bh) * Read an existing block from the device and verify its metadata header. * The buffer head is returned unlocked and uptodate. */ -struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno) +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct buffer_head *bh; @@ -206,13 +208,13 @@ struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno) } unlock_buffer(bh); if (ret < 0) { - scoutfs_block_put(bh); + scoutfs_block_put((void *)bh); bh = ERR_PTR(ret); } } out: - return bh; + return (void *)bh; } /* @@ -228,23 +230,23 @@ out: * - reads that span transactions? * - writers creating a new dirty block? */ -struct buffer_head *scoutfs_block_read_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) +struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) { struct scoutfs_block_header *hdr; - struct buffer_head *bh; + struct scoutfs_block *bl; - bh = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); - if (!IS_ERR(bh)) { - hdr = bh_data(bh); + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR(bl)) { + hdr = scoutfs_block_data(bl); if (WARN_ON_ONCE(hdr->seq != ref->seq)) { - clear_buffer_uptodate(bh); - brelse(bh); - bh = ERR_PTR(-EAGAIN); + clear_buffer_uptodate(bl); + scoutfs_block_put(bl); + bl = ERR_PTR(-EAGAIN); } } - return bh; + return bl; } /* @@ -309,7 +311,7 @@ int scoutfs_block_write_dirty(struct super_block *sb) spin_unlock_irqrestore(&sbi->block_lock, flags); atomic_inc(&sbi->block_writes); - scoutfs_block_set_crc(bh); + scoutfs_block_set_crc((void *)bh); lock_buffer(bh); @@ -357,38 +359,40 @@ int scoutfs_block_has_dirty(struct super_block *sb) * Callers are responsible for serializing modification to the reference * which is probably embedded in some other dirty persistent structure. */ -struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) +struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, + struct scoutfs_block_ref *ref) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; - struct buffer_head *copy_bh = NULL; - struct buffer_head *bh; + struct scoutfs_block *copy_bl = NULL; + struct scoutfs_block *bl; u64 blkno = 0; int ret; int err; - bh = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); - if (IS_ERR(bh) || ref->seq == sbi->super.hdr.seq) - return bh; + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) + return bl; ret = scoutfs_buddy_alloc_same(sb, &blkno, le64_to_cpu(ref->blkno)); if (ret < 0) goto out; - copy_bh = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(copy_bh)) { - ret = PTR_ERR(copy_bh); + copy_bl = scoutfs_block_dirty(sb, blkno); + if (IS_ERR(copy_bl)) { + ret = PTR_ERR(copy_bl); goto out; } - ret = scoutfs_buddy_free(sb, ref->seq, bh->b_blocknr, 0); + hdr = scoutfs_block_data(bl); + ret = scoutfs_buddy_free(sb, hdr->seq, le64_to_cpu(hdr->blkno), 0); if (ret) goto out; - memcpy(copy_bh->b_data, bh->b_data, SCOUTFS_BLOCK_SIZE); + memcpy(scoutfs_block_data(copy_bl), scoutfs_block_data(bl), + SCOUTFS_BLOCK_SIZE); - hdr = bh_data(copy_bh); + hdr = scoutfs_block_data(copy_bl); hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; ref->blkno = hdr->blkno; @@ -396,18 +400,18 @@ struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, ret = 0; out: - scoutfs_block_put(bh); + scoutfs_block_put(bl); if (ret) { - if (!IS_ERR_OR_NULL(copy_bh)) { + if (!IS_ERR_OR_NULL(copy_bl)) { err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, - copy_bh->b_blocknr, 0); + blkno, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } - scoutfs_block_put(copy_bh); - copy_bh = ERR_PTR(ret); + scoutfs_block_put(copy_bl); + copy_bl = ERR_PTR(ret); } - return copy_bh; + return copy_bl; } /* @@ -415,7 +419,7 @@ out: * the current dirty seq. Callers are responsible for serializing * access to the block and for zeroing unwritten block contents. */ -struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno) +struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; @@ -431,12 +435,12 @@ struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno) ret = insert_bhp(sb, bh); if (ret < 0) { - scoutfs_block_put(bh); + scoutfs_block_put((void *)bh); bh = ERR_PTR(ret); goto out; } - hdr = bh_data(bh); + hdr = scoutfs_block_data((void *)bh); *hdr = sbi->super.hdr; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; @@ -444,18 +448,18 @@ struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno) set_buffer_uptodate(bh); set_buffer_scoutfs_verified(bh); out: - return bh; + return (void *)bh; } /* * Allocate a new dirty writable block. The caller must be in a * transaction so that we can assign the dirty seq. */ -struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) +struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->stable_super; - struct buffer_head *bh; + struct scoutfs_block *bl; u64 blkno; int ret; int err; @@ -464,12 +468,12 @@ struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb) if (ret < 0) return ERR_PTR(ret); - bh = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(bh)) { + bl = scoutfs_block_dirty(sb, blkno); + if (IS_ERR(bl)) { err = scoutfs_buddy_free(sb, super->hdr.seq, blkno, 0); WARN_ON_ONCE(err); /* freeing dirty must work */ } - return bh; + return bl; } /* @@ -495,9 +499,9 @@ void scoutfs_block_forget(struct super_block *sb, u64 blkno) } } -void scoutfs_block_set_crc(struct buffer_head *bh) +void scoutfs_block_set_crc(struct scoutfs_block *bl) { - struct scoutfs_block_header *hdr = bh_data(bh); + struct scoutfs_block_header *hdr = scoutfs_block_data(bl); hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); } @@ -505,26 +509,29 @@ void scoutfs_block_set_crc(struct buffer_head *bh) /* * Zero the block from the given byte to the end of the block. */ -void scoutfs_block_zero(struct buffer_head *bh, size_t off) +void scoutfs_block_zero(struct scoutfs_block *bl, size_t off) { if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) return; if (off < SCOUTFS_BLOCK_SIZE) - memset((char *)bh->b_data + off, 0, SCOUTFS_BLOCK_SIZE - off); + memset(scoutfs_block_data(bl) + off, 0, + SCOUTFS_BLOCK_SIZE - off); } /* * Zero the block from the given byte to the end of the block. */ -void scoutfs_block_zero_from(struct buffer_head *bh, void *ptr) +void scoutfs_block_zero_from(struct scoutfs_block *bl, void *ptr) { - return scoutfs_block_zero(bh, (char *)ptr - (char *)bh->b_data); + return scoutfs_block_zero(bl, (char *)ptr - + (char *)scoutfs_block_data(bl)); } -void scoutfs_block_set_lock_class(struct buffer_head *bh, +void scoutfs_block_set_lock_class(struct scoutfs_block *bl, struct lock_class_key *class) { + struct buffer_head *bh = (void *)bl; struct block_bh_private *bhp = bh->b_private; if (bhp && !bhp->rwsem_class) { @@ -533,8 +540,9 @@ void scoutfs_block_set_lock_class(struct buffer_head *bh, } } -void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass) +void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass) { + struct buffer_head *bh = (void *)bl; struct block_bh_private *bhp = bh->b_private; if (bhp) { @@ -545,8 +553,9 @@ void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass) } } -void scoutfs_block_unlock(struct buffer_head *bh, bool write) +void scoutfs_block_unlock(struct scoutfs_block *bl, bool write) { + struct buffer_head *bh = (void *)bl; struct block_bh_private *bhp = bh->b_private; if (bhp) { @@ -556,3 +565,18 @@ void scoutfs_block_unlock(struct buffer_head *bh, bool write) up_read(&bhp->rwsem); } } + +void *scoutfs_block_data(struct scoutfs_block *bl) +{ + struct buffer_head *bh = (void *)bl; + + return (void *)bh->b_data; +} + +void scoutfs_block_put(struct scoutfs_block *bl) +{ + struct buffer_head *bh = (void *)bl; + + if (!IS_ERR_OR_NULL(bh)) + brelse(bh); +} diff --git a/kmod/src/block.h b/kmod/src/block.h index cef25815..e33a22f4 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -1,42 +1,34 @@ #ifndef _SCOUTFS_BLOCK_H_ #define _SCOUTFS_BLOCK_H_ -#include -#include +struct scoutfs_block; -struct buffer_head *scoutfs_block_read(struct super_block *sb, u64 blkno); -struct buffer_head *scoutfs_block_read_ref(struct super_block *sb, +#include + +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, struct scoutfs_block_ref *ref); -struct buffer_head *scoutfs_block_dirty(struct super_block *sb, u64 blkno); -struct buffer_head *scoutfs_block_dirty_alloc(struct super_block *sb); -struct buffer_head *scoutfs_block_dirty_ref(struct super_block *sb, +struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb); +struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, struct scoutfs_block_ref *ref); int scoutfs_block_has_dirty(struct super_block *sb); int scoutfs_block_write_dirty(struct super_block *sb); -void scoutfs_block_set_crc(struct buffer_head *bh); -void scoutfs_block_zero(struct buffer_head *bh, size_t off); -void scoutfs_block_zero_from(struct buffer_head *bh, void *ptr); +void scoutfs_block_set_crc(struct scoutfs_block *bl); +void scoutfs_block_zero(struct scoutfs_block *bl, size_t off); +void scoutfs_block_zero_from(struct scoutfs_block *bl, void *ptr); -void scoutfs_block_set_lock_class(struct buffer_head *bh, +void scoutfs_block_set_lock_class(struct scoutfs_block *bl, struct lock_class_key *class); -void scoutfs_block_lock(struct buffer_head *bh, bool write, int subclass); -void scoutfs_block_unlock(struct buffer_head *bh, bool write); +void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass); +void scoutfs_block_unlock(struct scoutfs_block *bl, bool write); void scoutfs_block_forget(struct super_block *sb, u64 blkno); -/* XXX seems like this should be upstream :) */ -static inline void *bh_data(struct buffer_head *bh) -{ - return (void *)bh->b_data; -} - -static inline void scoutfs_block_put(struct buffer_head *bh) -{ - if (!IS_ERR_OR_NULL(bh)) - brelse(bh); -} +void *scoutfs_block_data(struct scoutfs_block *bl); +void scoutfs_block_put(struct scoutfs_block *bl); #endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 9f6a0ed4..83005661 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -58,7 +58,7 @@ * XXX * - do we want a level in the btree header? seems like we would? * - validate structures on read? - * - internal bh/pos/cmp interface is clumsy.. + * - internal bl/pos/cmp interface is clumsy.. */ /* number of contiguous bytes used by the item header and val of given len */ @@ -451,14 +451,10 @@ static void compact_items(struct scoutfs_btree_block *bt) * consistent tree order. * * The cow tree updates let us skip block locking entirely for stable - * blocks because they're read only. The block layer only has to worry - * about locking blocks that could be written to. While they're - * writable they have a buffer_head private that pins them in the - * transaction and we store the block lock there. The block layer - * ignores our locking attempts for read-only blocks. - * - * And all of the blocks referenced by the stable super will be stable - * so we only try to lock at all when working with the dirty super. + * blocks because they're read only. All the blocks in the stable + * super tree are stable so we don't have to lock that tree at all. + * We let the block layer use the header's seq to avoid locking + * stable blocks. * * lockdep has to not be freaked out by all of this. The cascading * block locks really make it angry without annotation so we add classes @@ -466,24 +462,24 @@ static void compact_items(struct scoutfs_btree_block *bt) * during merge. */ -static void set_block_lock_class(struct buffer_head *bh, int level) +static void set_block_lock_class(struct scoutfs_block *bl, int level) { #ifdef CONFIG_LOCKDEP static struct lock_class_key tree_depth_classes[SCOUTFS_BTREE_MAX_DEPTH]; - scoutfs_block_set_lock_class(bh, &tree_depth_classes[level]); + scoutfs_block_set_lock_class(bl, &tree_depth_classes[level]); #endif } static void lock_tree_block(struct super_block *sb, struct scoutfs_btree_root *root, - struct buffer_head *bh, bool write, int subclass) + struct scoutfs_block *bl, bool write, int subclass) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); if (root == &sbi->super.btree_root) { - if (bh) { - scoutfs_block_lock(bh, write, subclass); + if (bl) { + scoutfs_block_lock(bl, write, subclass); } else { if (write) down_write(&sbi->btree_rwsem); @@ -495,13 +491,13 @@ static void lock_tree_block(struct super_block *sb, static void unlock_tree_block(struct super_block *sb, struct scoutfs_btree_root *root, - struct buffer_head *bh, bool write) + struct scoutfs_block *bl, bool write) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); if (root == &sbi->super.btree_root) { - if (bh) { - scoutfs_block_unlock(bh, write); + if (bl) { + scoutfs_block_unlock(bl, write); } else { if (write) up_write(&sbi->btree_rwsem); @@ -512,39 +508,39 @@ static void unlock_tree_block(struct super_block *sb, } /* sorting relies on masking pointers to find the containing block */ -static inline struct buffer_head *check_bh_alignment(struct buffer_head *bh) +static inline struct scoutfs_block *check_bl_alignment(struct scoutfs_block *bl) { - if (!IS_ERR_OR_NULL(bh)) { - struct scoutfs_btree_block *bt = bh_data(bh); + if (!IS_ERR_OR_NULL(bl)) { + struct scoutfs_btree_block *bt = scoutfs_block_data(bl); if (WARN_ON_ONCE(aligned_bt(bt) != bt)) { - scoutfs_block_put(bh); + scoutfs_block_put(bl); return ERR_PTR(-EIO); } } - return bh; + return bl; } /* * Allocate and initialize a new tree block. The caller adds references * to it. */ -static struct buffer_head *alloc_tree_block(struct super_block *sb) +static struct scoutfs_block *alloc_tree_block(struct super_block *sb) { struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; - bh = scoutfs_block_dirty_alloc(sb); - if (!IS_ERR(bh)) { - bt = bh_data(bh); + bl = scoutfs_block_dirty_alloc(sb); + if (!IS_ERR(bl)) { + bt = scoutfs_block_data(bl); bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); bt->free_reclaim = 0; bt->nr_items = 0; } - return check_bh_alignment(bh); + return check_bl_alignment(bl); } /* the caller has ensured that the free must succeed */ @@ -561,41 +557,41 @@ static void free_tree_block(struct super_block *sb, __le64 blkno) * Allocate a new tree block and point the root at it. The caller * is responsible for the items in the new root block. */ -static struct buffer_head *grow_tree(struct super_block *sb, +static struct scoutfs_block *grow_tree(struct super_block *sb, struct scoutfs_btree_root *root) { struct scoutfs_block_header *hdr; - struct buffer_head *bh; + struct scoutfs_block *bl; - bh = alloc_tree_block(sb); - if (!IS_ERR(bh)) { - hdr = bh_data(bh); + bl = alloc_tree_block(sb); + if (!IS_ERR(bl)) { + hdr = scoutfs_block_data(bl); root->height++; root->ref.blkno = hdr->blkno; root->ref.seq = hdr->seq; - set_block_lock_class(bh, root->height - 1); + set_block_lock_class(bl, root->height - 1); } - return bh; + return bl; } -static struct buffer_head *get_block_ref(struct super_block *sb, int level, +static struct scoutfs_block *get_block_ref(struct super_block *sb, int level, struct scoutfs_block_ref *ref, bool dirty) { - struct buffer_head *bh; + struct scoutfs_block *bl; if (dirty) - bh = scoutfs_block_dirty_ref(sb, ref); + bl = scoutfs_block_dirty_ref(sb, ref); else - bh = scoutfs_block_read_ref(sb, ref); + bl = scoutfs_block_read_ref(sb, ref); - if (!IS_ERR(bh)) - set_block_lock_class(bh, level); + if (!IS_ERR(bl)) + set_block_lock_class(bl, level); - return check_bh_alignment(bh); + return check_bl_alignment(bl); } /* @@ -636,18 +632,18 @@ static void create_parent_item(struct scoutfs_btree_block *parent, * them locked. We only need to lock the new sibling if we return it * instead of our given block for the caller to continue descent. */ -static struct buffer_head *try_split(struct super_block *sb, +static struct scoutfs_block *try_split(struct super_block *sb, struct scoutfs_btree_root *root, int level, struct scoutfs_key *key, unsigned int val_len, struct scoutfs_btree_block *parent, unsigned int parent_pos, - struct buffer_head *right_bh) + struct scoutfs_block *right_bl) { - struct scoutfs_btree_block *right = bh_data(right_bh); + struct scoutfs_btree_block *right = scoutfs_block_data(right_bl); struct scoutfs_btree_block *left; - struct buffer_head *left_bh; - struct buffer_head *par_bh = NULL; + struct scoutfs_block *left_bl; + struct scoutfs_block *par_bl = NULL; struct scoutfs_key maximal; unsigned int all_bytes; @@ -656,33 +652,33 @@ static struct buffer_head *try_split(struct super_block *sb, all_bytes = all_val_bytes(val_len); if (contig_free(right) >= all_bytes) - return right_bh; + return right_bl; if (reclaimable_free(right) >= all_bytes) { compact_items(right); - return right_bh; + return right_bl; } /* alloc split neighbour first to avoid unwinding tree growth */ - left_bh = alloc_tree_block(sb); - if (IS_ERR(left_bh)) { - unlock_tree_block(sb, root, right_bh, true); - scoutfs_block_put(right_bh); - return left_bh; + left_bl = alloc_tree_block(sb); + if (IS_ERR(left_bl)) { + unlock_tree_block(sb, root, right_bl, true); + scoutfs_block_put(right_bl); + return left_bl; } - left = bh_data(left_bh); + left = scoutfs_block_data(left_bl); if (!parent) { - par_bh = grow_tree(sb, root); - if (IS_ERR(par_bh)) { + par_bl = grow_tree(sb, root); + if (IS_ERR(par_bl)) { free_tree_block(sb, left->hdr.blkno); - scoutfs_block_put(left_bh); - unlock_tree_block(sb, root, right_bh, true); - scoutfs_block_put(right_bh); - return par_bh; + scoutfs_block_put(left_bl); + unlock_tree_block(sb, root, right_bl, true); + scoutfs_block_put(right_bl); + return par_bl; } - parent = bh_data(par_bh); + parent = scoutfs_block_data(par_bl); parent_pos = 0; scoutfs_set_max_key(&maximal); @@ -695,19 +691,19 @@ static struct buffer_head *try_split(struct super_block *sb, if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { /* insertion will go to the new left block */ - unlock_tree_block(sb, root, right_bh, true); - lock_tree_block(sb, root, left_bh, true, 0); - swap(right_bh, left_bh); + unlock_tree_block(sb, root, right_bl, true); + lock_tree_block(sb, root, left_bl, true, 0); + swap(right_bl, left_bl); } else { /* insertion will still go through us, might need to compact */ if (contig_free(right) < all_bytes) compact_items(right); } - scoutfs_block_put(par_bh); - scoutfs_block_put(left_bh); + scoutfs_block_put(par_bl); + scoutfs_block_put(left_bl); - return right_bh; + return right_bl; } /* @@ -733,22 +729,22 @@ static struct buffer_head *try_split(struct super_block *sb, * * XXX this could more cleverly chose a merge candidate sibling */ -static struct buffer_head *try_merge(struct super_block *sb, +static struct scoutfs_block *try_merge(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, int level, unsigned int pos, - struct buffer_head *bh) + struct scoutfs_block *bl) { - struct scoutfs_btree_block *bt = bh_data(bh); + struct scoutfs_btree_block *bt = scoutfs_block_data(bl); struct scoutfs_btree_item *sib_item; struct scoutfs_btree_block *sib_bt; - struct buffer_head *sib_bh; + struct scoutfs_block *sib_bl; unsigned int sib_pos; bool move_right; int to_move; if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) - return bh; + return bl; /* move items right into our block if we have a left sibling */ if (pos) { @@ -760,26 +756,26 @@ static struct buffer_head *try_merge(struct super_block *sb, } sib_item = pos_item(parent, sib_pos); - sib_bh = get_block_ref(sb, level, (void *)sib_item->val, true); - if (IS_ERR(sib_bh)) { + sib_bl = get_block_ref(sb, level, (void *)sib_item->val, true); + if (IS_ERR(sib_bl)) { /* XXX do we need to unlock this? don't think so */ - scoutfs_block_put(bh); - return sib_bh; + scoutfs_block_put(bl); + return sib_bl; } - sib_bt = bh_data(sib_bh); + sib_bt = scoutfs_block_data(sib_bl); if (!move_right) { - unlock_tree_block(sb, root, bh, true); - lock_tree_block(sb, root, sib_bh, true, 0); - lock_tree_block(sb, root, bh, true, 1); + unlock_tree_block(sb, root, bl, true); + lock_tree_block(sb, root, sib_bl, true, 0); + lock_tree_block(sb, root, bl, true, 1); if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) { - unlock_tree_block(sb, root, sib_bh, true); - scoutfs_block_put(sib_bh); - return bh; + unlock_tree_block(sb, root, sib_bl, true); + scoutfs_block_put(sib_bl); + return bl; } } else { - lock_tree_block(sb, root, sib_bh, true, 1); + lock_tree_block(sb, root, sib_bl, true, 1); } if (used_total(sib_bt) <= reclaimable_free(bt)) @@ -822,10 +818,10 @@ static struct buffer_head *try_merge(struct super_block *sb, free_tree_block(sb, parent->hdr.blkno); } - unlock_tree_block(sb, root, sib_bh, true); - scoutfs_block_put(sib_bh); + unlock_tree_block(sb, root, sib_bl, true); + scoutfs_block_put(sib_bl); - return bh; + return bl; } enum { @@ -992,7 +988,7 @@ out: * keep searching sibling blocks if their search key falls at the end of * a leaf in their search direction. */ -static struct buffer_head *btree_walk(struct super_block *sb, +static struct scoutfs_block *btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_key *prev_key, @@ -1000,8 +996,8 @@ static struct buffer_head *btree_walk(struct super_block *sb, unsigned int val_len, u64 seq, int op) { struct scoutfs_btree_block *parent = NULL; - struct buffer_head *par_bh = NULL; - struct buffer_head *bh = NULL; + struct scoutfs_block *par_bl = NULL; + struct scoutfs_block *bl = NULL; struct scoutfs_btree_item *item = NULL; struct scoutfs_block_ref *ref; struct scoutfs_key small; @@ -1025,11 +1021,11 @@ static struct buffer_head *btree_walk(struct super_block *sb, if (!root->height) { if (op == WALK_INSERT) { - bh = ERR_PTR(-ENOENT); + bl = ERR_PTR(-ENOENT); } else { - bh = grow_tree(sb, root); - if (!IS_ERR(bh)) { - lock_tree_block(sb, root, bh, dirty, 0); + bl = grow_tree(sb, root); + if (!IS_ERR(bl)) { + lock_tree_block(sb, root, bl, dirty, 0); unlock_tree_block(sb, root, NULL, dirty); } } @@ -1039,7 +1035,7 @@ static struct buffer_head *btree_walk(struct super_block *sb, /* skip the whole tree if the root ref's seq is old */ if (op == WALK_NEXT_SEQ && le64_to_cpu(ref->seq) < seq) { - bh = ERR_PTR(-ENOENT); + bl = ERR_PTR(-ENOENT); goto out; } @@ -1048,36 +1044,37 @@ static struct buffer_head *btree_walk(struct super_block *sb, while (level--) { /* XXX hmm, need to think about retry */ - bh = get_block_ref(sb, level, ref, dirty); - if (IS_ERR(bh)) + bl = get_block_ref(sb, level, ref, dirty); + if (IS_ERR(bl)) break; /* XXX enable this */ - ret = 0 && verify_btree_block(bh_data(bh), level, &small, &large); + ret = 0 && verify_btree_block(scoutfs_block_data(bl), level, + &small, &large); if (ret) { - scoutfs_block_put(bh); - bh = ERR_PTR(ret); + scoutfs_block_put(bl); + bl = ERR_PTR(ret); break; } - lock_tree_block(sb, root, bh, dirty, 0); + lock_tree_block(sb, root, bl, dirty, 0); if (op == WALK_INSERT) - bh = try_split(sb, root, level, key, val_len, parent, - pos, bh); + bl = try_split(sb, root, level, key, val_len, parent, + pos, bl); if ((op == WALK_DELETE) && parent) - bh = try_merge(sb, root, parent, level, pos, bh); - if (IS_ERR(bh)) + bl = try_merge(sb, root, parent, level, pos, bl); + if (IS_ERR(bl)) break; - unlock_tree_block(sb, root, par_bh, dirty); + unlock_tree_block(sb, root, par_bl, dirty); if (!level) break; - scoutfs_block_put(par_bh); - par_bh = bh; - parent = bh_data(par_bh); + scoutfs_block_put(par_bl); + par_bl = bl; + parent = scoutfs_block_data(par_bl); /* * Find the parent item that references the next child @@ -1089,9 +1086,9 @@ static struct buffer_head *btree_walk(struct super_block *sb, if (pos >= parent->nr_items) { /* current block dropped as parent below */ if (op == WALK_NEXT_SEQ) - bh = ERR_PTR(-ENOENT); + bl = ERR_PTR(-ENOENT); else - bh = ERR_PTR(-EIO); + bl = ERR_PTR(-EIO); break; } @@ -1119,11 +1116,11 @@ static struct buffer_head *btree_walk(struct super_block *sb, } out: - if (IS_ERR(bh)) - unlock_tree_block(sb, root, par_bh, dirty); - scoutfs_block_put(par_bh); + if (IS_ERR(bl)) + unlock_tree_block(sb, root, par_bl, dirty); + scoutfs_block_put(par_bl); - return bh; + return bl; } /* @@ -1138,17 +1135,17 @@ int scoutfs_btree_lookup(struct super_block *sb, { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; unsigned int pos; int cmp; int ret; trace_scoutfs_btree_lookup(sb, key, scoutfs_btree_val_length(val)); - bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, 0); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bt = bh_data(bh); + bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, 0); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = scoutfs_block_data(bl); pos = find_pos(bt, key, &cmp); if (cmp == 0) { @@ -1158,8 +1155,8 @@ int scoutfs_btree_lookup(struct super_block *sb, ret = -ENOENT; } - unlock_tree_block(sb, root, bh, false); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, false); + scoutfs_block_put(bl); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1182,7 +1179,7 @@ int scoutfs_btree_insert(struct super_block *sb, { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; unsigned int val_len; int pos; int cmp; @@ -1198,10 +1195,10 @@ int scoutfs_btree_insert(struct super_block *sb, if (WARN_ON_ONCE(val_len > SCOUTFS_MAX_ITEM_LEN)) return -EINVAL; - bh = btree_walk(sb, root, key, NULL, NULL, val_len, 0, WALK_INSERT); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bt = bh_data(bh); + bl = btree_walk(sb, root, key, NULL, NULL, val_len, 0, WALK_INSERT); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = scoutfs_block_data(bl); pos = find_pos(bt, key, &cmp); if (cmp) { @@ -1214,8 +1211,8 @@ int scoutfs_btree_insert(struct super_block *sb, ret = -EEXIST; } - unlock_tree_block(sb, root, bh, true); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, true); + scoutfs_block_put(bl); return ret; } @@ -1229,19 +1226,19 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; int pos; int cmp; int ret; trace_scoutfs_btree_delete(sb, key, 0); - bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DELETE); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); + bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DELETE); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); goto out; } - bt = bh_data(bh); + bt = scoutfs_block_data(bl); pos = find_pos(bt, key, &cmp); if (cmp == 0) { @@ -1262,8 +1259,8 @@ int scoutfs_btree_delete(struct super_block *sb, ret = -ENOENT; } - unlock_tree_block(sb, root, bh, true); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, true); + scoutfs_block_put(bl); out: trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1301,7 +1298,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key start = *first; struct scoutfs_key key = *first; struct scoutfs_key next_key; - struct buffer_head *bh; + struct scoutfs_block *bl; int pos; int ret; @@ -1312,26 +1309,26 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; while (scoutfs_key_cmp(&key, last) <= 0) { - bh = btree_walk(sb, root, &key, NULL, &next_key, 0, seq, op); + bl = btree_walk(sb, root, &key, NULL, &next_key, 0, seq, op); /* next seq walks can terminate in parents with old seqs */ - if (op == WALK_NEXT_SEQ && bh == ERR_PTR(-ENOENT)) { + if (op == WALK_NEXT_SEQ && bl == ERR_PTR(-ENOENT)) { key = next_key; continue; } - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); break; } - bt = bh_data(bh); + bt = scoutfs_block_data(bl); /* keep trying leaves until next_key passes last */ pos = find_pos_after_seq(bt, &key, 0, seq, op); if (pos >= bt->nr_items) { key = next_key; - unlock_tree_block(sb, root, bh, false); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, false); + scoutfs_block_put(bl); continue; } @@ -1348,8 +1345,8 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; } - unlock_tree_block(sb, root, bh, false); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, false); + scoutfs_block_put(bl); break; } @@ -1400,7 +1397,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *bt; struct scoutfs_key key = *last; struct scoutfs_key prev_key; - struct buffer_head *bh; + struct scoutfs_block *bl; int pos; int cmp; int ret; @@ -1411,19 +1408,19 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; while (scoutfs_key_cmp(&key, first) >= 0) { - bh = btree_walk(sb, root, &key, NULL, &prev_key, 0, 0, 0); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); + bl = btree_walk(sb, root, &key, NULL, &prev_key, 0, 0, 0); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); break; } - bt = bh_data(bh); + bt = scoutfs_block_data(bl); pos = find_pos(bt, &key, &cmp); /* walk to the prev leaf if we hit the front of this leaf */ if (pos == 0 && cmp != 0) { - unlock_tree_block(sb, root, bh, false); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, false); + scoutfs_block_put(bl); if (scoutfs_key_is_zero(&key)) break; key = prev_key; @@ -1444,8 +1441,8 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, ret = 0; } - unlock_tree_block(sb, root, bh, false); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, false); + scoutfs_block_put(bl); break; } @@ -1464,16 +1461,16 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_key *key) { struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; int cmp; int ret; trace_scoutfs_btree_dirty(sb, key, 0); - bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bt = bh_data(bh); + bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = scoutfs_block_data(bl); find_pos(bt, key, &cmp); if (cmp == 0) { @@ -1482,8 +1479,8 @@ int scoutfs_btree_dirty(struct super_block *sb, ret = -ENOENT; } - unlock_tree_block(sb, root, bh, true); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, true); + scoutfs_block_put(bl); trace_printk("key "CKF" ret %d\n", CKA(key), ret); @@ -1504,7 +1501,7 @@ int scoutfs_btree_update(struct super_block *sb, { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct buffer_head *bh; + struct scoutfs_block *bl; int pos; int cmp; int ret; @@ -1512,10 +1509,10 @@ int scoutfs_btree_update(struct super_block *sb, trace_scoutfs_btree_update(sb, key, val ? scoutfs_btree_val_length(val) : 0); - bh = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); - if (IS_ERR(bh)) - return PTR_ERR(bh); - bt = bh_data(bh); + bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); + if (IS_ERR(bl)) + return PTR_ERR(bl); + bt = scoutfs_block_data(bl); pos = find_pos(bt, key, &cmp); if (cmp == 0) { @@ -1527,8 +1524,8 @@ int scoutfs_btree_update(struct super_block *sb, ret = -ENOENT; } - unlock_tree_block(sb, root, bh, true); - scoutfs_block_put(bh); + unlock_tree_block(sb, root, bl, true); + scoutfs_block_put(bl); return ret; } diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 40dc2e65..204dbb88 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -105,7 +105,7 @@ struct buddy_info { u64 level_div[SCOUTFS_BUDDY_MAX_HEIGHT]; struct buddy_stack { - struct buffer_head *bh[SCOUTFS_BUDDY_MAX_HEIGHT]; + struct scoutfs_block *bl[SCOUTFS_BUDDY_MAX_HEIGHT]; u16 sl[SCOUTFS_BUDDY_MAX_HEIGHT]; int nr; } stack; @@ -169,26 +169,27 @@ static int order_nr(int order, int nr) return order_off(order) + nr; } -static void stack_push(struct buddy_stack *sta, struct buffer_head *bh, u16 sl) +static void stack_push(struct buddy_stack *sta, struct scoutfs_block *bl, + u16 sl) { - sta->bh[sta->nr] = bh; + sta->bl[sta->nr] = bl; sta->sl[sta->nr++] = sl; } /* sl isn't returned because callers peek the leaf where sl is meaningless */ -static struct buffer_head *stack_peek(struct buddy_stack *sta) +static struct scoutfs_block *stack_peek(struct buddy_stack *sta) { if (sta->nr) - return sta->bh[sta->nr - 1]; + return sta->bl[sta->nr - 1]; return NULL; } -static struct buffer_head *stack_pop(struct buddy_stack *sta, u16 *sl) +static struct scoutfs_block *stack_pop(struct buddy_stack *sta, u16 *sl) { if (sta->nr) { *sl = sta->sl[--sta->nr]; - return sta->bh[sta->nr]; + return sta->bl[sta->nr]; } return NULL; @@ -287,16 +288,16 @@ static void stack_cleanup(struct super_block *sb) struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_root *root = &sbi->super.buddy_root; struct scoutfs_buddy_block *bud; - struct buffer_head *bh; + struct scoutfs_block *bl; u16 free_orders = 0; bool parent; u16 sl; int i; parent = false; - while ((bh = stack_pop(sta, &sl))) { + while ((bl = stack_pop(sta, &sl))) { - bud = bh_data(bh); + bud = scoutfs_block_data(bl); if (parent && !set_slot_free_orders(bud, sl, free_orders)) break; @@ -306,17 +307,17 @@ static void stack_cleanup(struct super_block *sb) free_orders |= 1 << i; } - scoutfs_block_put(bh); + scoutfs_block_put(bl); parent = true; } /* set root if we got that far */ - if (bh == NULL) + if (bl == NULL) root->slot.free_orders = cpu_to_le16(free_orders); /* put any remaining blocks */ - while ((bh = stack_pop(sta, &sl))) - scoutfs_block_put(bh); + while ((bl = stack_pop(sta, &sl))) + scoutfs_block_put(bl); } @@ -344,14 +345,14 @@ static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) */ static void init_buddy_block(struct buddy_info *binf, struct scoutfs_super_block *super, - struct buffer_head *bh, int level) + struct scoutfs_block *bl, int level) { - struct scoutfs_buddy_block *bud = bh_data(bh); + struct scoutfs_buddy_block *bud = scoutfs_block_data(bl); u16 count; int nr; int i; - scoutfs_block_zero(bh, sizeof(bud->hdr)); + scoutfs_block_zero(bl, sizeof(bud->hdr)); for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) bud->first_set[i] = cpu_to_le16(U16_MAX); @@ -387,36 +388,34 @@ static void init_buddy_block(struct buddy_info *binf, * construct a fake ref so we can re-use the block ref cow code. When * we initialize the first use of a block we use the first of the pair. */ -static struct buffer_head *get_buddy_block(struct super_block *sb, - struct scoutfs_buddy_slot *slot, - u64 blkno, int level) +static struct scoutfs_block *get_buddy_block(struct super_block *sb, + struct scoutfs_buddy_slot *slot, + u64 blkno, int level) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct buddy_info *binf = sbi->buddy_info; struct scoutfs_buddy_block *bud; struct scoutfs_block_ref ref; - struct buffer_head *bh; + struct scoutfs_block *bl; trace_printk("getting block level %d blkno %llu slot seq %llu off %u\n", level, blkno, le64_to_cpu(slot->seq), slot->blkno_off); /* init a new block for an unused slot */ if (slot->seq == 0) { - bh = scoutfs_block_dirty(sb, blkno); - if (!IS_ERR(bh)) - init_buddy_block(binf, super, bh, level); + bl = scoutfs_block_dirty(sb, blkno); + if (!IS_ERR(bl)) + init_buddy_block(binf, super, bl, level); } else { /* construct block ref from tree walk blkno and slot ref */ ref.blkno = cpu_to_le64(blkno + slot->blkno_off); ref.seq = slot->seq; - bh = scoutfs_block_dirty_ref(sb, &ref); + bl = scoutfs_block_dirty_ref(sb, &ref); } - if (!IS_ERR(bh)) { - bud = bh_data(bh); - - trace_printk("got blkno %llu\n", (u64)bh->b_blocknr); + if (!IS_ERR(bl)) { + bud = scoutfs_block_data(bl); /* rebuild slot ref to blkno */ if (slot->seq != bud->hdr.seq) { @@ -427,7 +426,7 @@ static struct buffer_head *get_buddy_block(struct super_block *sb, } } - return bh; + return bl; } /* @@ -457,7 +456,7 @@ static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) struct scoutfs_buddy_root *root = &sbi->super.buddy_root; struct scoutfs_buddy_block *bud; struct scoutfs_buddy_slot *slot; - struct buffer_head *bh; + struct scoutfs_block *bl; u64 blkno; int level; int ret = 0; @@ -475,16 +474,16 @@ static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) while (level--) { /* XXX do base and level make sense here? */ - bh = get_buddy_block(sb, slot, blkno, level); - if (IS_ERR(bh)) { - ret = PTR_ERR(bh); + bl = get_buddy_block(sb, slot, blkno, level); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); break; } trace_printk("before blk %llu order %d level %d blkno %llu base %llu sl %d\n", blk, order, level, blkno, *base, sl); - bud = bh_data(bh); + bud = scoutfs_block_data(bl); if (level) { if (order >= 0) { @@ -516,7 +515,7 @@ static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) blk, order, level, blkno, *base, sl); - stack_push(sta, bh, sl); + stack_push(sta, bl, sl); } trace_printk("walking ret %d\n", ret); @@ -554,7 +553,7 @@ static int buddy_alloc(struct super_block *sb, u64 *blk, int order, int found) struct buddy_info *binf = sbi->buddy_info; struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct buffer_head *bh; + struct scoutfs_block *bl; u64 base; int ret; int nr; @@ -569,8 +568,8 @@ static int buddy_alloc(struct super_block *sb, u64 *blk, int order, int found) if (ret) goto out; - bh = stack_peek(sta); - bud = bh_data(bh); + bl = stack_peek(sta); + bud = scoutfs_block_data(bl); if (found >= 0) { nr = le16_to_cpu(bud->first_set[found]); @@ -624,7 +623,7 @@ static int buddy_free(struct super_block *sb, u64 blk, int order) struct buddy_info *binf = sbi->buddy_info; struct buddy_stack *sta = &binf->stack; struct scoutfs_buddy_block *bud; - struct buffer_head *bh; + struct scoutfs_block *bl; u64 unused; int ret; int nr; @@ -634,8 +633,8 @@ static int buddy_free(struct super_block *sb, u64 blk, int order) if (ret) goto out; - bh = stack_peek(sta); - bud = bh_data(bh); + bl = stack_peek(sta); + bud = scoutfs_block_data(bl); nr = buddy_bit(blk) >> order; for (i = order; i < SCOUTFS_BUDDY_ORDERS - 2; i++) { diff --git a/kmod/src/super.c b/kmod/src/super.c index b6d94a42..471bffef 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -108,23 +108,25 @@ int scoutfs_write_dirty_super(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; + struct scoutfs_block *bl; struct buffer_head *bh; int ret; - /* XXX prealloc? */ + /* XXX hack is immediately repaired in the coming patches */ bh = sb_getblk(sb, le64_to_cpu(sbi->super.hdr.blkno)); if (!bh) return -ENOMEM; - super = bh_data(bh); + bl = (void *)bh; + super = scoutfs_block_data(bl); *super = sbi->super; - scoutfs_block_zero(bh, sizeof(struct scoutfs_super_block)); - scoutfs_block_set_crc(bh); + scoutfs_block_zero(bl, sizeof(struct scoutfs_super_block)); + scoutfs_block_set_crc(bl); mark_buffer_dirty(bh); ret = sync_dirty_buffer(bh); - scoutfs_block_put(bh); + scoutfs_block_put(bl); return ret; } @@ -132,18 +134,18 @@ static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; - struct buffer_head *bh = NULL; + struct scoutfs_block *bl = NULL; int found = -1; int i; for (i = 0; i < SCOUTFS_SUPER_NR; i++) { - scoutfs_block_put(bh); - bh = scoutfs_block_read(sb, SCOUTFS_SUPER_BLKNO + i); - if (IS_ERR(bh)) { + scoutfs_block_put(bl); + bl = scoutfs_block_read(sb, SCOUTFS_SUPER_BLKNO + i); + if (IS_ERR(bl)) { scoutfs_warn(sb, "couldn't read super block %u", i); continue; } - super = bh_data(bh); + super = scoutfs_block_data(bl); if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { scoutfs_warn(sb, "super block %u has invalid id %llx", @@ -158,7 +160,7 @@ static int read_supers(struct super_block *sb) } } - scoutfs_block_put(bh); + scoutfs_block_put(bl); if (found < 0) { scoutfs_err(sb, "unable to read valid super block"); From 03787f23d3202ca3f68d4c59b9e33091b883bb43 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 11:53:03 -0800 Subject: [PATCH 121/920] Add scoutfs_block_data_from_contents() The btree code needs to get a pointer to a whole block from just pointers to elements that it's sorting. It had some manual code that assumed details of the blocks. Let's give it a real block interface to do what it wants and make it the block API's problem to figure out how to do it. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 7 +++++++ kmod/src/block.h | 1 + kmod/src/btree.c | 28 +++------------------------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index af90b5ef..cee3b681 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -573,6 +573,13 @@ void *scoutfs_block_data(struct scoutfs_block *bl) return (void *)bh->b_data; } +void *scoutfs_block_data_from_contents(const void *ptr) +{ + unsigned long addr = (unsigned long)ptr; + + return (void *)(addr & ~((unsigned long)SCOUTFS_BLOCK_MASK)); +} + void scoutfs_block_put(struct scoutfs_block *bl) { struct buffer_head *bh = (void *)bl; diff --git a/kmod/src/block.h b/kmod/src/block.h index e33a22f4..58a0f952 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -29,6 +29,7 @@ void scoutfs_block_unlock(struct scoutfs_block *bl, bool write); void scoutfs_block_forget(struct super_block *sb, u64 blkno); void *scoutfs_block_data(struct scoutfs_block *bl); +void *scoutfs_block_data_from_contents(const void *ptr); void scoutfs_block_put(struct scoutfs_block *bl); #endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 83005661..5beb9c24 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -335,16 +335,9 @@ static void move_items(struct scoutfs_btree_block *dst, } } -static struct scoutfs_btree_block *aligned_bt(const void *ptr) -{ - unsigned long addr = (unsigned long)ptr; - - return (void *)(addr & ~((unsigned long)SCOUTFS_BLOCK_MASK)); -} - static int sort_key_cmp(const void *A, const void *B) { - struct scoutfs_btree_block *bt = aligned_bt(A); + struct scoutfs_btree_block *bt = scoutfs_block_data_from_contents(A); const __le16 * __packed a = A; const __le16 * __packed b = B; @@ -507,21 +500,6 @@ static void unlock_tree_block(struct super_block *sb, } } -/* sorting relies on masking pointers to find the containing block */ -static inline struct scoutfs_block *check_bl_alignment(struct scoutfs_block *bl) -{ - if (!IS_ERR_OR_NULL(bl)) { - struct scoutfs_btree_block *bt = scoutfs_block_data(bl); - - if (WARN_ON_ONCE(aligned_bt(bt) != bt)) { - scoutfs_block_put(bl); - return ERR_PTR(-EIO); - } - } - - return bl; -} - /* * Allocate and initialize a new tree block. The caller adds references * to it. @@ -540,7 +518,7 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) bt->nr_items = 0; } - return check_bl_alignment(bl); + return bl; } /* the caller has ensured that the free must succeed */ @@ -591,7 +569,7 @@ static struct scoutfs_block *get_block_ref(struct super_block *sb, int level, if (!IS_ERR(bl)) set_block_lock_class(bl, level); - return check_bl_alignment(bl); + return bl; } /* From 4042927519133386549da7732880e35754934dd6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 11:58:44 -0800 Subject: [PATCH 122/920] Make btree nr_items le16 If we increase the block size the btree is going to need to be able to store more than 255 items in a btree block. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 54 +++++++++++++++++++++++++++-------------------- kmod/src/format.h | 2 +- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 5beb9c24..829dca63 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -89,8 +89,10 @@ static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) /* number of contig free bytes between item offset and first item */ static inline unsigned int contig_free(struct scoutfs_btree_block *bt) { + unsigned int nr = le16_to_cpu(bt->nr_items); + return le16_to_cpu(bt->free_end) - - offsetof(struct scoutfs_btree_block, item_offs[bt->nr_items]); + offsetof(struct scoutfs_btree_block, item_offs[nr]); } /* number of contig bytes free after reclaiming free amongst items */ @@ -120,7 +122,9 @@ pos_item(struct scoutfs_btree_block *bt, unsigned int pos) static inline struct scoutfs_key *greatest_key(struct scoutfs_btree_block *bt) { - return &pos_item(bt, bt->nr_items - 1)->key; + unsigned int nr = le16_to_cpu(bt->nr_items); + + return &pos_item(bt, nr - 1)->key; } /* @@ -215,7 +219,7 @@ static int find_pos(struct scoutfs_btree_block *bt, struct scoutfs_key *key, int *cmp) { unsigned int start = 0; - unsigned int end = bt->nr_items; + unsigned int end = le16_to_cpu(bt->nr_items); unsigned int pos = 0; *cmp = -1; @@ -251,17 +255,19 @@ static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, struct scoutfs_key *key, unsigned int val_len) { + unsigned int nr = le16_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; - if (pos < bt->nr_items) - memmove_arr(bt->item_offs, pos + 1, pos, bt->nr_items - pos); + if (pos < nr) + memmove_arr(bt->item_offs, pos + 1, pos, nr - pos); le16_add_cpu(&bt->free_end, -val_bytes(val_len)); bt->item_offs[pos] = bt->free_end; - bt->nr_items++; + nr++; + bt->nr_items = cpu_to_le16(nr); BUG_ON(le16_to_cpu(bt->free_end) < - offsetof(struct scoutfs_btree_block, item_offs[bt->nr_items])); + offsetof(struct scoutfs_btree_block, item_offs[nr])); item = pos_item(bt, pos); item->key = *key; @@ -281,15 +287,16 @@ static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) { struct scoutfs_btree_item *item = pos_item(bt, pos); + unsigned int nr = le16_to_cpu(bt->nr_items); trace_printk("pos %u off %u\n", pos, le16_to_cpu(bt->item_offs[pos])); - if (pos < (bt->nr_items - 1)) - memmove_arr(bt->item_offs, pos, pos + 1, - bt->nr_items - 1 - pos); + if (pos < (nr - 1)) + memmove_arr(bt->item_offs, pos, pos + 1, nr - 1 - pos); le16_add_cpu(&bt->free_reclaim, item_bytes(item)); - bt->nr_items--; + nr--; + bt->nr_items = cpu_to_le16(nr); /* wipe deleted items to avoid leaking data */ memset(item, 0, item_bytes(item)); @@ -311,14 +318,14 @@ static void move_items(struct scoutfs_btree_block *dst, unsigned int f; if (move_right) { - f = src->nr_items - 1; + f = le16_to_cpu(src->nr_items) - 1; t = 0; } else { f = 0; - t = dst->nr_items; + t = le16_to_cpu(dst->nr_items); } - while (f < src->nr_items && to_move > 0) { + while (f < le16_to_cpu(src->nr_items) && to_move > 0) { from = pos_item(src, f); to = create_item(dst, t, &from->key, @@ -390,6 +397,7 @@ static void sort_off_swap(void *A, void *B, int size) */ static void compact_items(struct scoutfs_btree_block *bt) { + unsigned int nr = le16_to_cpu(bt->nr_items); struct scoutfs_btree_item *from; struct scoutfs_btree_item *to; unsigned int bytes; @@ -398,12 +406,12 @@ static void compact_items(struct scoutfs_btree_block *bt) trace_printk("free_reclaim %u\n", le16_to_cpu(bt->free_reclaim)); - sort(bt->item_offs, bt->nr_items, sizeof(bt->item_offs[0]), + sort(bt->item_offs, nr, sizeof(bt->item_offs[0]), sort_off_cmp, sort_off_swap); end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); - for (i = bt->nr_items - 1; i >= 0; i--) { + for (i = nr - 1; i >= 0; i--) { from = pos_item(bt, i); bytes = item_bytes(from); @@ -418,7 +426,7 @@ static void compact_items(struct scoutfs_btree_block *bt) bt->free_end = end; bt->free_reclaim = 0; - sort(bt->item_offs, bt->nr_items, sizeof(bt->item_offs[0]), + sort(bt->item_offs, nr, sizeof(bt->item_offs[0]), sort_key_cmp, sort_off_swap); } @@ -781,7 +789,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, pos_item(parent, pos)->key = *greatest_key(bt); /* delete an empty sib or update if we changed its greatest key */ - if (sib_bt->nr_items == 0) { + if (le16_to_cpu(sib_bt->nr_items) == 0) { delete_item(parent, sib_pos); free_tree_block(sb, sib_bt->hdr.blkno); } else if (move_right) { @@ -789,7 +797,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, } /* and finally shrink the tree if our parent is the root with 1 */ - if (parent->nr_items == 1) { + if (le16_to_cpu(parent->nr_items) == 1) { root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; @@ -826,7 +834,7 @@ static bool skip_pos_seq(struct scoutfs_btree_block *bt, unsigned int pos, { struct scoutfs_btree_item *item; - if (op != WALK_NEXT_SEQ || pos >= bt->nr_items) + if (op != WALK_NEXT_SEQ || pos >= le16_to_cpu(bt->nr_items)) return false; item = pos_item(bt, pos); @@ -895,7 +903,7 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level, unsigned int i = 0; int bad = 1; - nr = bt->nr_items; + nr = le16_to_cpu(bt->nr_items); if (nr == 0) goto out; @@ -1061,7 +1069,7 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, * search. */ pos = find_pos_after_seq(parent, key, level, seq, op); - if (pos >= parent->nr_items) { + if (pos >= le16_to_cpu(parent->nr_items)) { /* current block dropped as parent below */ if (op == WALK_NEXT_SEQ) bl = ERR_PTR(-ENOENT); @@ -1303,7 +1311,7 @@ static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, /* keep trying leaves until next_key passes last */ pos = find_pos_after_seq(bt, &key, 0, seq, op); - if (pos >= bt->nr_items) { + if (pos >= le16_to_cpu(bt->nr_items)) { key = next_key; unlock_tree_block(sb, root, bl, false); scoutfs_block_put(bl); diff --git a/kmod/src/format.h b/kmod/src/format.h index fe83a346..685f1d53 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -136,7 +136,7 @@ struct scoutfs_btree_block { struct scoutfs_block_header hdr; __le16 free_end; __le16 free_reclaim; - __u8 nr_items; + __le16 nr_items; __le16 item_offs[0]; } __packed; From f57c07381a1ccd9b9c598935212402d4f5aff6f3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 14:39:42 -0800 Subject: [PATCH 123/920] Go back to having our own scoutfs_block cache We used to have 16k blocks in our own radix_tree cache. When we introduced the simple file block mapping code it preferred to have block size == page size. That let us remove a bunch of code and reuse all the kernel's buffer head code. But it turns out that the buffer heads are just a bit too inflexible. We'd like to have blocks larger than page size, obviously, but it turns out there's real functional differences. Resolving the problem of unlocked readers and allocating writers working with the same blkno is the most powerful example of this. It's trivial to fix by always inserting new allocated cached blocks in the cache. But solving it with buffer heads requires expensive and risky locking around the buffer head cache which can only support a single physical instance of a given blkno because there can be multiple blocks per page. So this restores the simple block cache that was removed back in commit 'c8e76e2 scoutfs: use buffer heads'. There's still work to do to get this fully functional but it's worth it. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 566 +++++++++++++++++++++++++++-------------------- kmod/src/block.h | 5 +- kmod/src/super.c | 31 +-- kmod/src/super.h | 2 +- 4 files changed, 340 insertions(+), 264 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index cee3b681..08b37570 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -11,7 +11,6 @@ * General Public License for more details. */ #include -#include #include #include @@ -23,54 +22,105 @@ #include "buddy.h" /* - * scoutfs has a fixed 4k small block size for metadata blocks. This - * lets us consistently use buffer heads without worrying about having a - * block size greater than the page size. + * scoutfs maintains a cache of metadata blocks in a radix tree. This + * gives us blocks bigger than page size and avoids fixing the location + * of a logical cached block in one possible position in a larger block + * device page cache page. * - * This block interface does the work to cow dirty blocks, track dirty - * blocks, generate checksums as they're written, only write them in - * transactions, verify checksums on read, and invalidate and retry - * reads of stale cached blocks. (That last bit only has a hint of an - * implementation.) + * This does the work to cow dirty blocks, track dirty blocks, generate + * checksums as they're written, only write them in transactions, verify + * checksums on read, and invalidate and retry reads of stale cached + * blocks. (That last bit only has a hint of an implementation.) * * XXX * - tear down dirty blocks left by write errors on unmount - * - should invalidate dirty blocks if freed + * - multiple smaller page allocs + * - vmalloc? vm_map_ram? + * - blocks allocated from per-cpu pages when page size > block size + * - cmwq crc calcs if that makes sense + * - slab of block structs + * - don't verify checksums in end_io context? + * - fall back to multiple single bios per block io if bio alloc fails? + * - fail mount if total_blocks is greater than long radix blkno */ -struct scoutfs_block; - -struct block_bh_private { - struct super_block *sb; - struct buffer_head *bh; - struct rb_node node; +struct scoutfs_block { struct rw_semaphore rwsem; - bool rwsem_class; + atomic_t refcount; + u64 blkno; + + unsigned long bits; + + struct super_block *sb; + struct page *page; + void *data; }; +#define DIRTY_RADIX_TAG 0 + enum { - BH_ScoutfsVerified = BH_PrivateStart, + BLOCK_BIT_UPTODATE = 0, + BLOCK_BIT_ERROR, + BLOCK_BIT_CLASS_SET, }; -BUFFER_FNS(ScoutfsVerified, scoutfs_verified) -static int verify_block_header(struct scoutfs_sb_info *sbi, - struct buffer_head *bh) +static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) { + struct scoutfs_block *bl; + struct page *page; + + /* we'd need to be just a bit more careful */ + BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); + + bl = kzalloc(sizeof(struct scoutfs_block), GFP_NOFS); + if (bl) { + /* change _from_contents if allocs not aligned */ + page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); + WARN_ON_ONCE(!page); + if (page) { + init_rwsem(&bl->rwsem); + atomic_set(&bl->refcount, 1); + bl->blkno = blkno; + bl->sb = sb; + bl->page = page; + bl->data = page_address(page); + trace_printk("allocated bl %p\n", bl); + } else { + kfree(bl); + bl = NULL; + } + } + + return bl; +} + +void scoutfs_block_put(struct scoutfs_block *bl) +{ + if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { + trace_printk("freeing bl %p\n", bl); + __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); + kfree(bl); + scoutfs_inc_counter(bl->sb, block_mem_free); + } +} + +static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_block_header *hdr = (void *)bh->b_data; + struct scoutfs_block_header *hdr = bl->data; u32 crc = scoutfs_crc_block(hdr); int ret = -EIO; if (le32_to_cpu(hdr->crc) != crc) { - printk("blkno %llu hdr crc %x != calculated %x\n", - (u64)bh->b_blocknr, le32_to_cpu(hdr->crc), crc); + printk("blkno %llu hdr crc %x != calculated %x\n", bl->blkno, + le32_to_cpu(hdr->crc), crc); } else if (super->hdr.fsid && hdr->fsid != super->hdr.fsid) { - printk("blkno %llu fsid %llx != super fsid %llx\n", - (u64)bh->b_blocknr, le64_to_cpu(hdr->fsid), - le64_to_cpu(super->hdr.fsid)); - } else if (le64_to_cpu(hdr->blkno) != bh->b_blocknr) { - printk("blkno %llu invalid hdr blkno %llx\n", - (u64)bh->b_blocknr, le64_to_cpu(hdr->blkno)); + printk("blkno %llu fsid %llx != super fsid %llx\n", bl->blkno, + le64_to_cpu(hdr->fsid), le64_to_cpu(super->hdr.fsid)); + } else if (le64_to_cpu(hdr->blkno) != bl->blkno) { + printk("blkno %llu invalid hdr blkno %llx\n", bl->blkno, + le64_to_cpu(hdr->blkno)); } else { ret = 0; } @@ -78,143 +128,161 @@ static int verify_block_header(struct scoutfs_sb_info *sbi, return ret; } -static struct buffer_head *bh_from_bhp_node(struct rb_node *node) +static void block_read_end_io(struct bio *bio, int err) { - struct block_bh_private *bhp; + struct scoutfs_block *bl = bio->bi_private; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - bhp = container_of(node, struct block_bh_private, node); - return bhp->bh; -} + if (!err && !verify_block_header(bl->sb, bl)) + set_bit(BLOCK_BIT_UPTODATE, &bl->bits); + else + set_bit(BLOCK_BIT_ERROR, &bl->bits); -static struct scoutfs_sb_info *sbi_from_bh(struct buffer_head *bh) -{ - struct block_bh_private *bhp = bh->b_private; + /* + * uncontended spin_lock in wake_up and unconditional smp_mb to + * make waitqueue_active safe are about the same cost, so we + * prefer the obviously safe choice. + */ + wake_up(&sbi->block_wq); - return SCOUTFS_SB(bhp->sb); -} - -static void insert_bhp_rb(struct rb_root *root, struct buffer_head *ins) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct block_bh_private *bhp; - struct buffer_head *bh; - - while (*node) { - parent = *node; - bh = bh_from_bhp_node(*node); - - if (ins->b_blocknr < bh->b_blocknr) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - bhp = ins->b_private; - rb_link_node(&bhp->node, parent, node); - rb_insert_color(&bhp->node, root); + scoutfs_block_put(bl); + bio_put(bio); } /* - * Track a dirty block by allocating private data and inserting it into - * the dirty rbtree in the super block. - * - * Callers are in transactions that prevent metadata writeback so blocks - * won't be written and cleaned while we're trying to dirty them. We - * serialize racing to add dirty tracking to the same block in case the - * caller didn't. - * - * Presence in the dirty tree holds a bh ref. + * Once a transaction block is persistent it's fine to drop the dirty + * tag. It's been checksummed so it can be read in again. It's seq + * will be in the current transaction so it'll simply be dirtied and + * checksummed and written out again. */ -static int insert_bhp(struct super_block *sb, struct buffer_head *bh) +static void block_write_end_io(struct bio *bio, int err) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct block_bh_private *bhp; + struct scoutfs_block *bl = bio->bi_private; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); unsigned long flags; - int ret = 0; - if (bh->b_private) - return 0; - - lock_buffer(bh); - if (bh->b_private) - goto out; - - bhp = kmalloc(sizeof(*bhp), GFP_NOFS); - if (!bhp) { - ret = -ENOMEM; - goto out; + if (!err) { + spin_lock_irqsave(&sbi->block_lock, flags); + radix_tree_tag_clear(&sbi->block_radix, + bl->blkno, DIRTY_RADIX_TAG); + spin_unlock_irqrestore(&sbi->block_lock, flags); } - bhp->sb = sb; - bhp->bh = bh; - get_bh(bh); - bh->b_private = bhp; - /* lockdep class can be set by callers that use the lock */ - init_rwsem(&bhp->rwsem); - bhp->rwsem_class = false; + /* not too worried about racing ints */ + if (err && !sbi->block_write_err) + sbi->block_write_err = err; - spin_lock_irqsave(&sbi->block_lock, flags); - insert_bhp_rb(&sbi->block_dirty_tree, bh); - spin_unlock_irqrestore(&sbi->block_lock, flags); + if (atomic_dec_and_test(&sbi->block_writes)) + wake_up(&sbi->block_wq); + + scoutfs_block_put(bl); + bio_put(bio); - trace_printk("blkno %llu bh %p\n", (u64)bh->b_blocknr, bh); -out: - unlock_buffer(bh); - return ret; } -static void erase_bhp(struct buffer_head *bh) +static int block_submit_bio(struct scoutfs_block *bl, int rw) { - struct block_bh_private *bhp = bh->b_private; - struct scoutfs_sb_info *sbi = sbi_from_bh(bh); - unsigned long flags; + struct super_block *sb = bl->sb; + struct bio *bio; + int ret; - spin_lock_irqsave(&sbi->block_lock, flags); - rb_erase(&bhp->node, &sbi->block_dirty_tree); - spin_unlock_irqrestore(&sbi->block_lock, flags); + bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); + if (WARN_ON_ONCE(!bio)) + return -ENOMEM; - put_bh(bh); - kfree(bhp); - bh->b_private = NULL; + bio->bi_sector = bl->blkno << (SCOUTFS_BLOCK_SHIFT - 9); + bio->bi_bdev = sb->s_bdev; + if (rw & WRITE) { + bio->bi_end_io = block_write_end_io; + } else + bio->bi_end_io = block_read_end_io; + bio->bi_private = bl; - trace_printk("blkno %llu bh %p\n", (u64)bh->b_blocknr, bh); + ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); + if (WARN_ON_ONCE(ret != SCOUTFS_BLOCK_SIZE)) { + bio_put(bio); + return -ENOMEM; + } + + atomic_inc(&bl->refcount); + submit_bio(rw, bio); + + return 0; } /* * Read an existing block from the device and verify its metadata header. - * The buffer head is returned unlocked and uptodate. */ struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buffer_head *bh; + struct scoutfs_block *found; + struct scoutfs_block *bl; + unsigned long flags; int ret; - bh = sb_bread(sb, blkno); - if (!bh) { - bh = ERR_PTR(-EIO); + /* find an existing block, dropping if it's errored */ + spin_lock_irqsave(&sbi->block_lock, flags); + + bl = radix_tree_lookup(&sbi->block_radix, blkno); + if (bl) { + if (test_bit(BLOCK_BIT_ERROR, &bl->bits)) { + radix_tree_delete(&sbi->block_radix, bl->blkno); + scoutfs_block_put(bl); + bl = NULL; + } else { + atomic_inc(&bl->refcount); + } + } + spin_unlock_irqrestore(&sbi->block_lock, flags); + if (bl) + goto wait; + + /* allocate a new block and try to insert it */ + bl = alloc_block(sb, blkno); + if (!bl) { + ret = -EIO; goto out; } - if (!buffer_scoutfs_verified(bh)) { - lock_buffer(bh); - if (!buffer_scoutfs_verified(bh)) { - ret = verify_block_header(sbi, bh); - if (!ret) - set_buffer_scoutfs_verified(bh); - } else { - ret = 0; - } - unlock_buffer(bh); - if (ret < 0) { - scoutfs_block_put((void *)bh); - bh = ERR_PTR(ret); - } + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + spin_lock_irqsave(&sbi->block_lock, flags); + + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found) { + scoutfs_block_put(bl); + bl = found; + atomic_inc(&bl->refcount); + } else { + radix_tree_insert(&sbi->block_radix, blkno, bl); + atomic_inc(&bl->refcount); } + spin_unlock_irqrestore(&sbi->block_lock, flags); + radix_tree_preload_end(); + + if (!found) { + ret = block_submit_bio(bl, READ_SYNC | REQ_META); + if (ret) + goto out; + } + +wait: + ret = wait_event_interruptible(sbi->block_wq, + test_bit(BLOCK_BIT_UPTODATE, &bl->bits) || + test_bit(BLOCK_BIT_ERROR, &bl->bits)); + if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bl->bits)) + ret = -EIO; out: - return (void *)bh; + if (ret) { + scoutfs_block_put(bl); + bl = ERR_PTR(ret); + } + + return bl; } /* @@ -226,7 +294,8 @@ out: * many times the caller assumes that we've hit persistent corruption * and returns an error. * - * XXX how does this race with + * XXX: + * - actually implement this * - reads that span transactions? * - writers creating a new dirty block? */ @@ -240,7 +309,6 @@ struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, if (!IS_ERR(bl)) { hdr = scoutfs_block_data(bl); if (WARN_ON_ONCE(hdr->seq != ref->seq)) { - clear_buffer_uptodate(bl); scoutfs_block_put(bl); bl = ERR_PTR(-EAGAIN); } @@ -250,35 +318,19 @@ struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, } /* - * We stop tracking dirty metadata blocks when their IO succeeds. This - * happens in the context of transaction commit which excludes other - * metadata dirtying paths. + * The caller knows that it's not racing with writers. */ -static void block_write_end_io(struct buffer_head *bh, int uptodate) +int scoutfs_block_has_dirty(struct super_block *sb) { - struct scoutfs_sb_info *sbi = sbi_from_bh(bh); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - trace_printk("bh %p uptdate %d\n", bh, uptodate); - - /* XXX */ - unlock_buffer(bh); - - if (uptodate) { - erase_bhp(bh); - } else { - /* don't care if this is racey? */ - if (!sbi->block_write_err) - sbi->block_write_err = -EIO; - } - - if (atomic_dec_and_test(&sbi->block_writes)) - wake_up(&sbi->block_wq); + return radix_tree_tagged(&sbi->block_radix, DIRTY_RADIX_TAG); } /* - * Submit writes for all the buffer heads in the dirty block tree. The - * write transaction machinery ensures that the dirty blocks form a - * consistent image and excludes future dirtying while we're working. + * Submit writes for all the blocks in the radix with their dirty tag + * set. The transaction machinery ensures that the dirty blocks form a + * consistent image and excludes future dirtying while IO is in flight. * * Presence in the dirty tree holds a reference. Blocks are only * removed from the tree which drops the ref when IO completes. @@ -291,38 +343,49 @@ static void block_write_end_io(struct buffer_head *bh, int uptodate) int scoutfs_block_write_dirty(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buffer_head *bh; - struct rb_node *node; + struct scoutfs_block *blocks[16]; + struct scoutfs_block *bl; struct blk_plug plug; unsigned long flags; + u64 blkno; int ret; + int nr; + int i; atomic_set(&sbi->block_writes, 1); sbi->block_write_err = 0; + blkno = 0; ret = 0; blk_start_plug(&plug); - spin_lock_irqsave(&sbi->block_lock, flags); - node = rb_first(&sbi->block_dirty_tree); - while(node) { - bh = bh_from_bhp_node(node); - node = rb_next(node); + do { + /* get refs to a bunch of dirty blocks */ + spin_lock_irqsave(&sbi->block_lock, flags); + nr = radix_tree_gang_lookup_tag(&sbi->block_radix, + (void **)blocks, blkno, + ARRAY_SIZE(blocks), + DIRTY_RADIX_TAG); + if (nr > 0) + blkno = blocks[nr - 1]->blkno + 1; + for (i = 0; i < nr; i++) + atomic_inc(&blocks[i]->refcount); spin_unlock_irqrestore(&sbi->block_lock, flags); - atomic_inc(&sbi->block_writes); - scoutfs_block_set_crc((void *)bh); + /* submit them in order, being careful to put all on err */ + for (i = 0; i < nr; i++) { + bl = blocks[i]; - lock_buffer(bh); - - bh->b_end_io = block_write_end_io; - ret = submit_bh(WRITE, bh); /* doesn't actually fail? */ - - spin_lock_irqsave(&sbi->block_lock, flags); - if (ret) - break; - } - spin_unlock_irqrestore(&sbi->block_lock, flags); + if (ret == 0) { + scoutfs_block_set_crc(bl); + atomic_inc(&sbi->block_writes); + ret = block_submit_bio(bl, WRITE); + if (ret) + atomic_dec(&sbi->block_writes); + } + scoutfs_block_put(bl); + } + } while (nr && !ret); blk_finish_plug(&plug); @@ -330,18 +393,29 @@ int scoutfs_block_write_dirty(struct super_block *sb) atomic_dec(&sbi->block_writes); wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); - trace_printk("ret %d\n", ret); - return ret; + return ret ?: sbi->block_write_err; } /* - * The caller knows that it's not racing with writers. + * XXX This is a gross hack for writing the super. It doesn't have + * per-block write completion indication. It knows that it's the only + * thing that will be writing. */ -int scoutfs_block_has_dirty(struct super_block *sb) +int scoutfs_block_write_sync(struct scoutfs_block *bl) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + int ret; - return !RB_EMPTY_ROOT(&sbi->block_dirty_tree); + BUG_ON(atomic_read(&sbi->block_writes) != 0); + + atomic_inc(&sbi->block_writes); + ret = block_submit_bio(bl, WRITE); + if (ret) + atomic_dec(&sbi->block_writes); + else + wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); + + return ret ?: sbi->block_write_err; } /* @@ -418,37 +492,64 @@ out: * Return a dirty metadata block with an updated block header to match * the current dirty seq. Callers are responsible for serializing * access to the block and for zeroing unwritten block contents. + * + * Always allocating a new block and replacing any old cached block + * serves a very specific purpose. We can have an unlocked reader + * traversing stable structures actively using a clean block while a + * writer gets that same blkno from the allocator and starts modifying + * it. By always allocating a new block we let the reader continue + * safely using their old immutable block while the writer works on the + * newly allocated block. The old stable block will be freed once the + * reader drops their reference. */ struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_block_header *hdr; - struct buffer_head *bh; + struct scoutfs_block *found; + struct scoutfs_block *bl; + unsigned long flags; int ret; /* allocate a new block and try to insert it */ - bh = sb_getblk(sb, blkno); - if (!bh) { - bh = ERR_PTR(-ENOMEM); + bl = alloc_block(sb, blkno); + if (!bl) { + ret = -EIO; goto out; } - ret = insert_bhp(sb, bh); - if (ret < 0) { - scoutfs_block_put((void *)bh); - bh = ERR_PTR(ret); - goto out; - } + set_bit(BLOCK_BIT_UPTODATE, &bl->bits); - hdr = scoutfs_block_data((void *)bh); + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + hdr = bl->data; *hdr = sbi->super.hdr; hdr->blkno = cpu_to_le64(blkno); hdr->seq = sbi->super.hdr.seq; - set_buffer_uptodate(bh); - set_buffer_scoutfs_verified(bh); + spin_lock_irqsave(&sbi->block_lock, flags); + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found) { + radix_tree_delete(&sbi->block_radix, blkno); + scoutfs_block_put(found); + } + + radix_tree_insert(&sbi->block_radix, blkno, bl); + radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); + atomic_inc(&bl->refcount); + spin_unlock_irqrestore(&sbi->block_lock, flags); + + radix_tree_preload_end(); + ret = 0; out: - return (void *)bh; + if (ret) { + scoutfs_block_put(bl); + bl = ERR_PTR(ret); + } + + return bl; } /* @@ -476,29 +577,6 @@ struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb) return bl; } -/* - * Make sure that we don't have a dirty block at the given blkno. If we - * do we remove it from our tree of dirty blocks and clear the buffer - * dirty bit. - * - * XXX for now callers have only needed to forget blknos, maybe they'll - * have the bh some day. - */ -void scoutfs_block_forget(struct super_block *sb, u64 blkno) -{ - struct block_bh_private *bhp; - struct buffer_head *bh; - - bh = sb_find_get_block(sb, blkno); - if (bh) { - bhp = bh->b_private; - if (bhp) { - erase_bhp(bh); - bforget(bh); - } - } -} - void scoutfs_block_set_crc(struct scoutfs_block *bl) { struct scoutfs_block_header *hdr = scoutfs_block_data(bl); @@ -531,46 +609,41 @@ void scoutfs_block_zero_from(struct scoutfs_block *bl, void *ptr) void scoutfs_block_set_lock_class(struct scoutfs_block *bl, struct lock_class_key *class) { - struct buffer_head *bh = (void *)bl; - struct block_bh_private *bhp = bh->b_private; - - if (bhp && !bhp->rwsem_class) { - lockdep_set_class(&bhp->rwsem, class); - bhp->rwsem_class = true; + if (!test_bit(BLOCK_BIT_CLASS_SET, &bl->bits)) { + lockdep_set_class(&bl->rwsem, class); + set_bit(BLOCK_BIT_CLASS_SET, &bl->bits); } } void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass) { - struct buffer_head *bh = (void *)bl; - struct block_bh_private *bhp = bh->b_private; + struct scoutfs_block_header *hdr = scoutfs_block_data(bl); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - if (bhp) { + if (hdr->seq == sbi->super.hdr.seq) { if (write) - down_write_nested(&bhp->rwsem, subclass); + down_write_nested(&bl->rwsem, subclass); else - down_read_nested(&bhp->rwsem, subclass); + down_read_nested(&bl->rwsem, subclass); } } void scoutfs_block_unlock(struct scoutfs_block *bl, bool write) { - struct buffer_head *bh = (void *)bl; - struct block_bh_private *bhp = bh->b_private; + struct scoutfs_block_header *hdr = scoutfs_block_data(bl); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - if (bhp) { + if (hdr->seq == sbi->super.hdr.seq) { if (write) - up_write(&bhp->rwsem); + up_write(&bl->rwsem); else - up_read(&bhp->rwsem); + up_read(&bl->rwsem); } } void *scoutfs_block_data(struct scoutfs_block *bl) { - struct buffer_head *bh = (void *)bl; - - return (void *)bh->b_data; + return bl->data; } void *scoutfs_block_data_from_contents(const void *ptr) @@ -580,10 +653,23 @@ void *scoutfs_block_data_from_contents(const void *ptr) return (void *)(addr & ~((unsigned long)SCOUTFS_BLOCK_MASK)); } -void scoutfs_block_put(struct scoutfs_block *bl) +void scoutfs_block_destroy(struct super_block *sb) { - struct buffer_head *bh = (void *)bl; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_block *blocks[16]; + struct scoutfs_block *bl; + unsigned long blkno = 0; + int nr; + int i; - if (!IS_ERR_OR_NULL(bh)) - brelse(bh); + do { + nr = radix_tree_gang_lookup(&sbi->block_radix, (void **)blocks, + blkno, ARRAY_SIZE(blocks)); + for (i = 0; i < nr; i++) { + bl = blocks[i]; + radix_tree_delete(&sbi->block_radix, bl->blkno); + blkno = bl->blkno + 1; + scoutfs_block_put(bl); + } + } while (nr); } diff --git a/kmod/src/block.h b/kmod/src/block.h index 58a0f952..8981fd9d 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -16,6 +16,7 @@ struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, int scoutfs_block_has_dirty(struct super_block *sb); int scoutfs_block_write_dirty(struct super_block *sb); +int scoutfs_block_write_sync(struct scoutfs_block *bl); void scoutfs_block_set_crc(struct scoutfs_block *bl); void scoutfs_block_zero(struct scoutfs_block *bl, size_t off); @@ -26,10 +27,10 @@ void scoutfs_block_set_lock_class(struct scoutfs_block *bl, void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass); void scoutfs_block_unlock(struct scoutfs_block *bl, bool write); -void scoutfs_block_forget(struct super_block *sb, u64 blkno); - void *scoutfs_block_data(struct scoutfs_block *bl); void *scoutfs_block_data_from_contents(const void *ptr); void scoutfs_block_put(struct scoutfs_block *bl); +void scoutfs_block_destroy(struct super_block *sb); + #endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 471bffef..aa7c2bcc 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -109,23 +108,19 @@ int scoutfs_write_dirty_super(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; struct scoutfs_block *bl; - struct buffer_head *bh; int ret; - /* XXX hack is immediately repaired in the coming patches */ - bh = sb_getblk(sb, le64_to_cpu(sbi->super.hdr.blkno)); - if (!bh) - return -ENOMEM; - bl = (void *)bh; + /* XXX prealloc? */ + bl = scoutfs_block_dirty(sb, le64_to_cpu(sbi->super.hdr.blkno)); + if (WARN_ON_ONCE(IS_ERR(bl))) + return PTR_ERR(bl); super = scoutfs_block_data(bl); - *super = sbi->super; - scoutfs_block_zero(bl, sizeof(struct scoutfs_super_block)); + memcpy(super, &sbi->super, sizeof(*super)); + scoutfs_block_zero(bl, sizeof(*super)); scoutfs_block_set_crc(bl); - mark_buffer_dirty(bh); - ret = sync_dirty_buffer(bh); - + ret = scoutfs_block_write_sync(bl); scoutfs_block_put(bl); return ret; } @@ -193,7 +188,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->next_ino_lock); spin_lock_init(&sbi->block_lock); - sbi->block_dirty_tree = RB_ROOT; + /* radix only inserted with NOFS _preload */ + INIT_RADIX_TREE(&sbi->block_radix, GFP_ATOMIC); init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); init_rwsem(&sbi->btree_rwsem); @@ -204,11 +200,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) init_waitqueue_head(&sbi->trans_write_wq); spin_lock_init(&sbi->file_alloc_lock); - if (!sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE)) { - printk(KERN_ERR "couldn't set blocksize\n"); - return -EINVAL; - } - /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) @@ -250,12 +241,10 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sbi) { scoutfs_shutdown_trans(sb); scoutfs_buddy_destroy(sb); + scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); - - /* XXX write errors can leave dirty blocks */ - WARN_ON_ONCE(!RB_EMPTY_ROOT(&sbi->block_dirty_tree)); kfree(sbi); } } diff --git a/kmod/src/super.h b/kmod/src/super.h index 9247f917..141a606d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -19,7 +19,7 @@ struct scoutfs_sb_info { spinlock_t next_ino_lock; spinlock_t block_lock; - struct rb_root block_dirty_tree; + struct radix_tree_root block_radix; wait_queue_head_t block_wq; atomic_t block_writes; int block_write_err; From d4571b6db32d025b59bb4a685ba7e3b3a4dd188d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 17:47:12 -0800 Subject: [PATCH 124/920] Add scoutfs_block_forget() Add scoutfs_block_forget() which ensures that a block won't satisfy future lookups and will not be written out. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 24 ++++++++++++++++++++++++ kmod/src/block.h | 1 + 2 files changed, 25 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 08b37570..faabaf1f 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -577,6 +577,30 @@ struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb) return bl; } +/* + * Forget the given block by removing it from the radix and clearing its + * dirty tag. It will not be found by future lookups and will not be + * written out. The caller can still use it until it drops its + * reference. + */ +void scoutfs_block_forget(struct scoutfs_block *bl) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); + struct scoutfs_block *found; + unsigned long flags; + u64 blkno = bl->blkno; + + spin_lock_irqsave(&sbi->block_lock, flags); + found = radix_tree_lookup(&sbi->block_radix, blkno); + if (found == bl) { + radix_tree_delete(&sbi->block_radix, blkno); + radix_tree_tag_clear(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); + scoutfs_block_put(found); + } + + spin_unlock_irqrestore(&sbi->block_lock, flags); +} + void scoutfs_block_set_crc(struct scoutfs_block *bl) { struct scoutfs_block_header *hdr = scoutfs_block_data(bl); diff --git a/kmod/src/block.h b/kmod/src/block.h index 8981fd9d..5d760f52 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -29,6 +29,7 @@ void scoutfs_block_unlock(struct scoutfs_block *bl, bool write); void *scoutfs_block_data(struct scoutfs_block *bl); void *scoutfs_block_data_from_contents(const void *ptr); +void scoutfs_block_forget(struct scoutfs_block *bl); void scoutfs_block_put(struct scoutfs_block *bl); void scoutfs_block_destroy(struct super_block *sb); From 0c67dd51efcf68721cb3f19cadffd40ee8b4c60b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Nov 2016 17:56:52 -0800 Subject: [PATCH 125/920] Forget freed btree blocks When the btree stops referencing a block and frees it we can also forget it so that it isn't uselessly written to disk. Callers who forget are careful to only unlock and release the block ref after freeing it. They won't be confused if something allocates the block and starts using it. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 829dca63..2e0427b1 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -530,13 +530,19 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) } /* the caller has ensured that the free must succeed */ -static void free_tree_block(struct super_block *sb, __le64 blkno) +static void free_tree_block(struct super_block *sb, struct scoutfs_block *bl) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_btree_block *bt = scoutfs_block_data(bl); + int err; - int err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, - le64_to_cpu(blkno), 0); - WARN_ON_ONCE(err); + BUG_ON(bt->hdr.seq != super->hdr.seq); + + scoutfs_block_forget(bl); + err = scoutfs_buddy_free(sb, bt->hdr.seq, + le64_to_cpu(bt->hdr.blkno), 0); + BUG_ON(err); } /* @@ -657,7 +663,7 @@ static struct scoutfs_block *try_split(struct super_block *sb, if (!parent) { par_bl = grow_tree(sb, root); if (IS_ERR(par_bl)) { - free_tree_block(sb, left->hdr.blkno); + free_tree_block(sb, left_bl); scoutfs_block_put(left_bl); unlock_tree_block(sb, root, right_bl, true); scoutfs_block_put(right_bl); @@ -717,10 +723,11 @@ static struct scoutfs_block *try_split(struct super_block *sb, */ static struct scoutfs_block *try_merge(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_btree_block *parent, + struct scoutfs_block *par_bl, int level, unsigned int pos, struct scoutfs_block *bl) { + struct scoutfs_btree_block *parent = scoutfs_block_data(par_bl); struct scoutfs_btree_block *bt = scoutfs_block_data(bl); struct scoutfs_btree_item *sib_item; struct scoutfs_btree_block *sib_bt; @@ -791,7 +798,7 @@ static struct scoutfs_block *try_merge(struct super_block *sb, /* delete an empty sib or update if we changed its greatest key */ if (le16_to_cpu(sib_bt->nr_items) == 0) { delete_item(parent, sib_pos); - free_tree_block(sb, sib_bt->hdr.blkno); + free_tree_block(sb, sib_bl); } else if (move_right) { sib_item->key = *greatest_key(sib_bt); } @@ -801,7 +808,8 @@ static struct scoutfs_block *try_merge(struct super_block *sb, root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; - free_tree_block(sb, parent->hdr.blkno); + free_tree_block(sb, par_bl); + /* caller just unlocks and drops parent */ } unlock_tree_block(sb, root, sib_bl, true); @@ -1049,7 +1057,7 @@ static struct scoutfs_block *btree_walk(struct super_block *sb, bl = try_split(sb, root, level, key, val_len, parent, pos, bl); if ((op == WALK_DELETE) && parent) - bl = try_merge(sb, root, parent, level, pos, bl); + bl = try_merge(sb, root, par_bl, level, pos, bl); if (IS_ERR(bl)) break; @@ -1239,7 +1247,7 @@ int scoutfs_btree_delete(struct super_block *sb, root->ref.blkno = 0; root->ref.seq = 0; - free_tree_block(sb, bt->hdr.blkno); + free_tree_block(sb, bl); } } else { ret = -ENOENT; From d71f7a24ecb324088dbf4d44fb994cfc22c321da Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Nov 2016 13:46:39 -0800 Subject: [PATCH 126/920] Don't check meta seq before locking The block lock functions were trying to compare the block header seq and the super seq to decide if the block is stable and if it should lock, or not. Readers trying to lock races with transaction commits. Transaction commit can update the super after the reader locks and before it unlocks. The unlock will then fail the test and fail to unlock. fsstress triggered this in xfstests generic/013. Instead we can always acquire the read lock on stable blocks. We'll be bouncing the rwsem cacheline around like the refcount cacheline. If this is a problem we can carefully maintain bits in the block to safely indicate if it should be locked or unlocked but let's not go there if we don't have to. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index faabaf1f..3413bb3c 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -641,28 +641,18 @@ void scoutfs_block_set_lock_class(struct scoutfs_block *bl, void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass) { - struct scoutfs_block_header *hdr = scoutfs_block_data(bl); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - - if (hdr->seq == sbi->super.hdr.seq) { - if (write) - down_write_nested(&bl->rwsem, subclass); - else - down_read_nested(&bl->rwsem, subclass); - } + if (write) + down_write_nested(&bl->rwsem, subclass); + else + down_read_nested(&bl->rwsem, subclass); } void scoutfs_block_unlock(struct scoutfs_block *bl, bool write) { - struct scoutfs_block_header *hdr = scoutfs_block_data(bl); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - - if (hdr->seq == sbi->super.hdr.seq) { - if (write) - up_write(&bl->rwsem); - else - up_read(&bl->rwsem); - } + if (write) + up_write(&bl->rwsem); + else + up_read(&bl->rwsem); } void *scoutfs_block_data(struct scoutfs_block *bl) From b612438abcf12bf0d3cbaa5a2ac90035ea5278e7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 8 Nov 2016 17:42:03 -0800 Subject: [PATCH 127/920] Buddy forgot to put blocks in a few places The buddy code missed putting the block in a few error cases. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/buddy.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c index 204dbb88..9f4a16fd 100644 --- a/kmod/src/buddy.c +++ b/kmod/src/buddy.c @@ -298,8 +298,10 @@ static void stack_cleanup(struct super_block *sb) while ((bl = stack_pop(sta, &sl))) { bud = scoutfs_block_data(bl); - if (parent && !set_slot_free_orders(bud, sl, free_orders)) + if (parent && !set_slot_free_orders(bud, sl, free_orders)) { + scoutfs_block_put(bl); break; + } free_orders = 0; for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) { @@ -491,6 +493,7 @@ static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) sl = le16_to_cpu(bud->first_set[order]); /* XXX corruption */ if (sl == U16_MAX) { + scoutfs_block_put(bl); ret = -EIO; break; } From 6fd5396fbe438035f0cb4726a49d5b67832098f3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Nov 2016 10:05:50 -0800 Subject: [PATCH 128/920] Add block cache shrinker Now that we have our own allocated block cache struct we need to add a shrinker so that it's reclaimed under memory pressure. We keep clean blocks in a simple lru list that the shrinker walks to free the oldest blocks. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/block.c | 133 ++++++++++++++++++++++++++++++++++++++++------- kmod/src/block.h | 1 + kmod/src/super.c | 7 +++ kmod/src/super.h | 4 ++ 4 files changed, 127 insertions(+), 18 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 3413bb3c..3ecf6a7b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -47,6 +47,7 @@ struct scoutfs_block { struct rw_semaphore rwsem; atomic_t refcount; + struct list_head lru_entry; u64 blkno; unsigned long bits; @@ -80,6 +81,7 @@ static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) if (page) { init_rwsem(&bl->rwsem); atomic_set(&bl->refcount, 1); + INIT_LIST_HEAD(&bl->lru_entry); bl->blkno = blkno; bl->sb = sb; bl->page = page; @@ -98,12 +100,60 @@ void scoutfs_block_put(struct scoutfs_block *bl) { if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { trace_printk("freeing bl %p\n", bl); + WARN_ON_ONCE(!list_empty(&bl->lru_entry)); __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); kfree(bl); scoutfs_inc_counter(bl->sb, block_mem_free); } } +static void lru_add(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) +{ + if (list_empty(&bl->lru_entry)) { + list_add_tail(&bl->lru_entry, &sbi->block_lru_list); + sbi->block_lru_nr++; + } +} + +static void lru_del(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) +{ + if (!list_empty(&bl->lru_entry)) { + list_del_init(&bl->lru_entry); + sbi->block_lru_nr--; + } +} + +/* + * The caller is referencing a block but doesn't know if its in the LRU + * or not. If it is move it to the tail so it's last to be dropped by + * the shrinker. + */ +static void lru_move(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) +{ + if (!list_empty(&bl->lru_entry)) + list_move_tail(&bl->lru_entry, &sbi->block_lru_list); +} + +static void radix_insert(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, + bool dirty) +{ + radix_tree_insert(&sbi->block_radix, bl->blkno, bl); + if (dirty) + radix_tree_tag_set(&sbi->block_radix, bl->blkno, + DIRTY_RADIX_TAG); + else + lru_add(sbi, bl); + atomic_inc(&bl->refcount); +} + +/* deleting the blkno from the radix also clears the dirty tag if it was set */ +static void radix_delete(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) +{ + lru_del(sbi, bl); + radix_tree_delete(&sbi->block_radix, bl->blkno); + scoutfs_block_put(bl); +} + static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -165,6 +215,7 @@ static void block_write_end_io(struct bio *bio, int err) spin_lock_irqsave(&sbi->block_lock, flags); radix_tree_tag_clear(&sbi->block_radix, bl->blkno, DIRTY_RADIX_TAG); + lru_add(sbi, bl); spin_unlock_irqrestore(&sbi->block_lock, flags); } @@ -227,10 +278,10 @@ struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) bl = radix_tree_lookup(&sbi->block_radix, blkno); if (bl) { if (test_bit(BLOCK_BIT_ERROR, &bl->bits)) { - radix_tree_delete(&sbi->block_radix, bl->blkno); - scoutfs_block_put(bl); + radix_delete(sbi, bl); bl = NULL; } else { + lru_move(sbi, bl); atomic_inc(&bl->refcount); } } @@ -255,10 +306,10 @@ struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) if (found) { scoutfs_block_put(bl); bl = found; + lru_move(sbi, bl); atomic_inc(&bl->refcount); } else { - radix_tree_insert(&sbi->block_radix, blkno, bl); - atomic_inc(&bl->refcount); + radix_insert(sbi, bl, false); } spin_unlock_irqrestore(&sbi->block_lock, flags); @@ -531,14 +582,9 @@ struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno) spin_lock_irqsave(&sbi->block_lock, flags); found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) { - radix_tree_delete(&sbi->block_radix, blkno); - scoutfs_block_put(found); - } - - radix_tree_insert(&sbi->block_radix, blkno, bl); - radix_tree_tag_set(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); - atomic_inc(&bl->refcount); + if (found) + radix_delete(sbi, found); + radix_insert(sbi, bl, true); spin_unlock_irqrestore(&sbi->block_lock, flags); radix_tree_preload_end(); @@ -592,13 +638,65 @@ void scoutfs_block_forget(struct scoutfs_block *bl) spin_lock_irqsave(&sbi->block_lock, flags); found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found == bl) { - radix_tree_delete(&sbi->block_radix, blkno); - radix_tree_tag_clear(&sbi->block_radix, blkno, DIRTY_RADIX_TAG); - scoutfs_block_put(found); + if (found == bl) + radix_delete(sbi, bl); + spin_unlock_irqrestore(&sbi->block_lock, flags); +} + +/* + * We maintain an LRU of blocks so that the shrinker can free the oldest + * under memory pressure. We can't reclaim dirty blocks so only clean + * blocks are kept in the LRU. Blocks are only in the LRU while their + * presence in the radix holds a reference. We don't care if a reader + * has an active ref on a clean block that gets reclaimed. All we're + * doing is removing from the radix. The caller can still work with the + * block and it will be freed once they drop their ref. + * + * If this is called with nr_to_scan == 0 then it only returns the nr. + * We avoid acquiring the lock in that case. + * + * Lookup code only moves blocks around in the LRU while they're in the + * radix. Once we remove the block from the radix we're able to use the + * lru_entry to drop all the blocks outside the lock. + * + * XXX: + * - are sc->nr_to_scan and our return meant to be in units of pages? + * - should we sync a transaction here? + */ +int scoutfs_block_shrink(struct shrinker *shrink, struct shrink_control *sc) +{ + struct scoutfs_sb_info *sbi = container_of(shrink, + struct scoutfs_sb_info, + block_shrinker); + struct scoutfs_block *tmp; + struct scoutfs_block *bl; + unsigned long flags; + unsigned long nr; + LIST_HEAD(list); + + nr = sc->nr_to_scan; + if (!nr) + goto out; + + spin_lock_irqsave(&sbi->block_lock, flags); + + list_for_each_entry_safe(bl, tmp, &sbi->block_lru_list, lru_entry) { + if (nr-- == 0) + break; + atomic_inc(&bl->refcount); + radix_delete(sbi, bl); + list_add(&bl->lru_entry, &list); } spin_unlock_irqrestore(&sbi->block_lock, flags); + + list_for_each_entry_safe(bl, tmp, &list, lru_entry) { + list_del_init(&bl->lru_entry); + scoutfs_block_put(bl); + } + +out: + return min_t(unsigned long, sbi->block_lru_nr, INT_MAX); } void scoutfs_block_set_crc(struct scoutfs_block *bl) @@ -681,9 +779,8 @@ void scoutfs_block_destroy(struct super_block *sb) blkno, ARRAY_SIZE(blocks)); for (i = 0; i < nr; i++) { bl = blocks[i]; - radix_tree_delete(&sbi->block_radix, bl->blkno); blkno = bl->blkno + 1; - scoutfs_block_put(bl); + radix_delete(sbi, bl); } } while (nr); } diff --git a/kmod/src/block.h b/kmod/src/block.h index 5d760f52..0eb86837 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -32,6 +32,7 @@ void *scoutfs_block_data_from_contents(const void *ptr); void scoutfs_block_forget(struct scoutfs_block *bl); void scoutfs_block_put(struct scoutfs_block *bl); +int scoutfs_block_shrink(struct shrinker *shrink, struct shrink_control *sc); void scoutfs_block_destroy(struct super_block *sb); #endif diff --git a/kmod/src/super.c b/kmod/src/super.c index aa7c2bcc..ca085815 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -192,6 +192,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_RADIX_TREE(&sbi->block_radix, GFP_ATOMIC); init_waitqueue_head(&sbi->block_wq); atomic_set(&sbi->block_writes, 0); + INIT_LIST_HEAD(&sbi->block_lru_list); init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); @@ -200,6 +201,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) init_waitqueue_head(&sbi->trans_write_wq); spin_lock_init(&sbi->file_alloc_lock); + sbi->block_shrinker.shrink = scoutfs_block_shrink; + sbi->block_shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&sbi->block_shrinker); + /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) @@ -241,6 +246,8 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sbi) { scoutfs_shutdown_trans(sb); scoutfs_buddy_destroy(sb); + if (sbi->block_shrinker.shrink == scoutfs_block_shrink) + unregister_shrinker(&sbi->block_shrinker); scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index 141a606d..db453cb2 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -23,6 +23,10 @@ struct scoutfs_sb_info { wait_queue_head_t block_wq; atomic_t block_writes; int block_write_err; + /* block cache lru */ + struct shrinker block_shrinker; + struct list_head block_lru_list; + unsigned long block_lru_nr; struct buddy_info *buddy_info; From f54f61f064d07dea8783e0ddf0db3d483e8d2ba4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 4 Nov 2016 14:32:22 -0700 Subject: [PATCH 129/920] Always initialize btree lockdep class We have to set the lock class to the btree level to keep lockdep from building long depdencency chains. We initialized allocated blocks for tree growth but not for splitting. We fix this by moving the init up into allocation instead of in tree growth. Now all the places we get blocks from the block calls are set. This silences a lockdep warning during merge during rm -rf which is the first place where multiple blocks in a level are locked. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 2e0427b1..db857103 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -512,7 +512,7 @@ static void unlock_tree_block(struct super_block *sb, * Allocate and initialize a new tree block. The caller adds references * to it. */ -static struct scoutfs_block *alloc_tree_block(struct super_block *sb) +static struct scoutfs_block *alloc_tree_block(struct super_block *sb, int level) { struct scoutfs_btree_block *bt; struct scoutfs_block *bl; @@ -524,6 +524,8 @@ static struct scoutfs_block *alloc_tree_block(struct super_block *sb) bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); bt->free_reclaim = 0; bt->nr_items = 0; + + set_block_lock_class(bl, level); } return bl; @@ -555,7 +557,7 @@ static struct scoutfs_block *grow_tree(struct super_block *sb, struct scoutfs_block_header *hdr; struct scoutfs_block *bl; - bl = alloc_tree_block(sb); + bl = alloc_tree_block(sb, root->height); if (!IS_ERR(bl)) { hdr = scoutfs_block_data(bl); @@ -652,7 +654,7 @@ static struct scoutfs_block *try_split(struct super_block *sb, } /* alloc split neighbour first to avoid unwinding tree growth */ - left_bl = alloc_tree_block(sb); + left_bl = alloc_tree_block(sb, level); if (IS_ERR(left_bl)) { unlock_tree_block(sb, root, right_bl, true); scoutfs_block_put(right_bl); From 256166db32d3bbc9d9d73905cbe07cc15118dc25 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Nov 2016 12:40:54 -0800 Subject: [PATCH 130/920] Fix write trans/page lock inversions scoutfs_write_begin() was riddled with lock ordering and cleanup bugs: - blocked holding the trans with the page lock held - dirtied the inode with the page lock held - didn't release the trans on error - tried to double unlock and release pages on readpage error We fix all this up by reordering things so we hold the trans, dirty the inode, then work pages all while more carefully cleaning up. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/filerw.c | 67 ++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index abd754a7..5c9196d9 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -582,41 +582,9 @@ static int scoutfs_write_begin(struct file *file, struct page *page; int ret; -retry: - page = grab_cache_page_write_begin(mapping, index, flags); - if (!page) - return -ENOMEM; - - trace_printk(PGF" pos %llu len %u\n", PGA(page), (u64)pos, len); - - /* - * read in the page if we're going to be dirtying part of the - * page. readpage catches when this is a read past i_size or - * from a hole and zeros the buffer. - */ - if (!PageUptodate(page) && !IS_ALIGNED(pos | len, SCOUTFS_BLOCK_SIZE)) { - ClearPageError(page); - ret = scoutfs_readpage(NULL, page); - if (ret) { - page_cache_release(page); - goto out; - } - - wait_on_page_locked(page); - if (!PageUptodate(page)) { - page_cache_release(page); - ret = -EIO; - goto out; - } - - /* let grabbing deal with weird page states */ - page_cache_release(page); - goto retry; - } - ret = scoutfs_hold_trans(sb); if (ret) - goto out; + return ret; /* can't re-enter fs, have trans */ flags |= AOP_FLAG_NOFS; @@ -626,14 +594,38 @@ retry: if (ret) goto out; +retry: + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) { + ret = -ENOMEM; + goto out; + } + + /* + * read in the page if we're going to be dirtying part of the + * page. readpage catches when this is a read past i_size or + * from a hole and zeros the buffer. We try to grab the page + * again to let it deal with locking and races. + */ + if (!PageUptodate(page) && !IS_ALIGNED(pos | len, SCOUTFS_BLOCK_SIZE)) { + ClearPageError(page); + ret = scoutfs_readpage(file, page); + if (!ret) { + wait_on_page_locked(page); + if (!PageUptodate(page)) + ret = -EIO; + } + page_cache_release(page); + if (ret) + goto out; + goto retry; + } + /* make sure our get_block gets a chance to alloc */ clear_mapped_page_buffers(page); ret = __block_write_begin(page, pos, len, scoutfs_write_begin_get_block); -out: - trace_printk(PGF" pos %llu len %u ret %d\n", - PGA(page), (u64)pos, len, ret); if (ret < 0) { /* XXX handle truncating? */ unlock_page(page); @@ -642,6 +634,9 @@ out: } *pagep = page; +out: + if (ret) + scoutfs_release_trans(sb); return ret; } From 1d0cd95b5557fadba1042658c242ff60c2c0abc3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Nov 2016 13:02:12 -0800 Subject: [PATCH 131/920] Let the commit task hold transactions The commit work sets trans_holds so that all hold attempts block while it's doing its work. Now that it's calling in to generic vfs functions to write out dirty file data it can end up in generic write functions that try to hold the trans and can deadlock. This adds tracking of the commit task so that holds know to let it proceed without deadlocking. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/super.h | 1 + kmod/src/trans.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/kmod/src/super.h b/kmod/src/super.h index db453cb2..25e6eec1 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -34,6 +34,7 @@ struct scoutfs_sb_info { atomic_t trans_holds; wait_queue_head_t trans_hold_wq; + struct task_struct *trans_task; spinlock_t trans_write_lock; u64 trans_write_count; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index d089a1d4..0b6b7be6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -51,6 +51,10 @@ * holding a transaction so it doesn't have to worry about blocks being * dirtied while it is working. * + * In the course of doing its work this task might need to use write + * functions that would try to hold the transaction. We record the task + * whose committing the transaction so that holding won't deadlock. + * * Any dirty block had to have allocated a new blkno which would have * created dirty allocator metadata blocks. We can avoid writing * entirely if we don't have any dirty metadata blocks. This is @@ -74,6 +78,8 @@ void scoutfs_trans_write_func(struct work_struct *work) int ret = 0; bool have_umount; + sbi->trans_task = current; + wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); @@ -111,6 +117,8 @@ void scoutfs_trans_write_func(struct work_struct *work) atomic_set(&sbi->trans_holds, 0); wake_up(&sbi->trans_hold_wq); + + sbi->trans_task = NULL; } struct write_attempt { @@ -173,6 +181,9 @@ int scoutfs_hold_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + if (current == sbi->trans_task) + return 0; + return wait_event_interruptible(sbi->trans_hold_wq, atomic_add_unless(&sbi->trans_holds, 1, -1)); } @@ -188,6 +199,9 @@ void scoutfs_release_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + if (current == sbi->trans_task) + return; + if (atomic_sub_return(1, &sbi->trans_holds) == 0) { if (scoutfs_buddy_alloc_count(sb) >= SCOUTFS_MAX_TRANS_BLOCKS) scoutfs_sync_fs(sb, 0); From 243a36e40501ffddf3f9d43e340399182473e93f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 10:30:05 -0800 Subject: [PATCH 132/920] Add fsync file operation method Add a scoutfs_file_fsync() which synchronously commits the current transaction and call it to fsync files and directories. This fixes a number of generic xfstests in the quick group which were failing because fsync wasn't supported. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/dir.c | 1 + kmod/src/filerw.c | 1 + kmod/src/trans.c | 6 ++++++ kmod/src/trans.h | 2 ++ 4 files changed, 10 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 8a03f986..59d2aabd 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1013,6 +1013,7 @@ out: const struct file_operations scoutfs_dir_fops = { .readdir = scoutfs_readdir, + .fsync = scoutfs_file_fsync, }; const struct inode_operations scoutfs_dir_iops = { diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 5c9196d9..091f2333 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -671,4 +671,5 @@ const struct file_operations scoutfs_file_fops = { .aio_read = generic_file_aio_read, .aio_write = generic_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, + .fsync = scoutfs_file_fsync, }; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 0b6b7be6..b9108500 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -177,6 +177,12 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) return ret; } +int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + return scoutfs_sync_fs(file->f_inode->i_sb, 1); +} + int scoutfs_hold_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 9cd0753a..22c5755a 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -3,6 +3,8 @@ void scoutfs_trans_write_func(struct work_struct *work); int scoutfs_sync_fs(struct super_block *sb, int wait); +int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync); int scoutfs_hold_trans(struct super_block *sb); void scoutfs_release_trans(struct super_block *sb); From 37bc86b558cb5686430f28e848b4d13acb0e6b35 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 14:20:37 -0800 Subject: [PATCH 133/920] Add check_size_lte Add a _lte val boolean so that -EOVERFLOW is returned if the item is greater than the value vector. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 2 ++ kmod/src/btree.h | 1 + 2 files changed, 3 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index db857103..ab2387bb 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -151,6 +151,8 @@ static int copy_to_val(struct scoutfs_btree_val *val, /* XXX corruption */ if (val->check_size_eq && val_len != scoutfs_btree_val_length(val)) return -EIO; + if (val->check_size_lte && val_len > scoutfs_btree_val_length(val)) + return -EOVERFLOW; for (i = 0, off = 0; val_len > 0 && i < ARRAY_SIZE(val->vec); i++) { kv = &val->vec[i]; diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 7615a733..6df548d1 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -6,6 +6,7 @@ struct scoutfs_btree_val { struct kvec vec[3]; unsigned int check_size_eq:1; + unsigned int check_size_lte:1; }; static inline void __scoutfs_btree_init_val(struct scoutfs_btree_val *val, From ae6cc83d0185373ab771882992f9a2e5964fb6ba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 13:10:39 -0800 Subject: [PATCH 134/920] Raise the nlink limit A few xfstests tests were failing because they tried to create a decent number of hard links to a file. We had a small nlink limit because the inode-paths ioctl copied all the paths for all the hard links to a userspace buffer which could be enormous if there was a larger nlink limit. The hard link backref disk format already has a natural counter that could be used as a cursor to iterate over all the hard links that point to a given inode. This refactors the inode_paths ioctl into a ino_path ioctl that returns a single path for the given counter and returns the counter for the next path that links to the inode. Happily this lets us get rid of all the weird path component lists and allocations. Now there's just the kernel path buffer that gets null terminated path components and the userspace buffer that those are copied to. We don't fully relax the nlink limit. stat(2) returns the link count as a u32. We go a step further and limit it to S32_MAX so that apps might avoid sign bugs. That still gives us a more generous limit than ext4 and btrfs which are around U16_MAX. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/dir.c | 107 ++++++++++++++++++++------------------------ kmod/src/dir.h | 5 +-- kmod/src/format.h | 10 +---- kmod/src/ioctl.c | 110 +++++++++++++++++++--------------------------- kmod/src/ioctl.h | 17 ++++--- 5 files changed, 105 insertions(+), 144 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 59d2aabd..88a78610 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -810,8 +810,8 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) } /* - * Add an allocated path component to the callers list which links to - * the target inode at a counter past the given counter. + * Store the null terminated path component that links to the inode at + * the given counter in the callers buffer. * * This is implemented by searching for link backrefs on the inode * starting from the given counter. Those contain references to the @@ -827,11 +827,10 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) * Backref counters are never reused and rename only modifies the * existing backref counter under the dir's mutex. */ -static int add_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, - u64 *ctr, struct list_head *list) +static int append_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, + u64 *ctr, char *path, unsigned int bytes) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_path_component *comp; struct scoutfs_link_backref lref; struct scoutfs_btree_val val; struct scoutfs_dirent dent; @@ -844,10 +843,6 @@ static int add_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, int len; int ret; - comp = kmalloc(sizeof(struct scoutfs_path_component), GFP_KERNEL); - if (!comp) - return -ENOMEM; - retry: scoutfs_set_key(&first, ino, SCOUTFS_LINK_BACKREF_KEY, *ctr); scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); @@ -900,69 +895,52 @@ retry: } scoutfs_set_key(&key, *dir_ino, SCOUTFS_DIRENT_KEY, off); - scoutfs_btree_init_val(&val, &dent, sizeof(dent), - comp->name, SCOUTFS_NAME_LEN); + scoutfs_btree_init_val(&val, &dent, sizeof(dent), path, bytes - 1); + val.check_size_lte = 1; ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret < 0) { /* XXX corruption, should always have dirent for backref */ if (ret == -ENOENT) ret = -EIO; + else if (ret == -EOVERFLOW) + ret = -ENAMETOOLONG; goto out; } /* XXX corruption */ - if (ret < sizeof(dent)) { + if (ret <= sizeof(dent)) { + ret = -EIO; + goto out; + } + + len = ret - sizeof(dent); /* just name len, no null term */ + + /* XXX corruption */ + if (len > SCOUTFS_NAME_LEN || le64_to_cpu(dent.ino) != ino) { ret = -EIO; goto out; } - len = ret - sizeof(dent); trace_printk("dent ino %llu len %d\n", le64_to_cpu(dent.ino), len); - /* XXX corruption */ - if (len < 1 || len > SCOUTFS_NAME_LEN) { - ret = -EIO; - goto out; - } - - /* XXX corruption, dirents should always match link backref */ - if (le64_to_cpu(dent.ino) != ino) { - ret = -EIO; - goto out; - } - (*ctr)++; - comp->len = len; - list_add(&comp->head, list); - comp = NULL; /* won't be freed */ - - ret = 1; + path[len] = '\0'; + ret = len + 1; out: if (inode) { mutex_unlock(&inode->i_mutex); iput(inode); } - kfree(comp); return ret; } -void scoutfs_dir_free_path(struct list_head *list) -{ - struct scoutfs_path_component *comp; - struct scoutfs_path_component *tmp; - - list_for_each_entry_safe(comp, tmp, list, head) { - list_del_init(&comp->head); - kfree(comp); - } -} - /* - * Fill the list with the allocated path components that link the root - * to the target inode. The caller's ctr gives the link counter to - * start from. + * Fill the caller's buffer with the null terminated path components + * from the target inode to the root. These will be in the opposite + * order of a typical slash delimited path. The caller's ctr gives the + * specific link to start from. * * This is racing with modification of components in the path. We can * traverse a partial path only to find that it's been blown away @@ -970,44 +948,53 @@ void scoutfs_dir_free_path(struct list_head *list) * the final link to the inode should prevent repeatedly traversing * paths that no longer exist. * - * Returns > 0 and *ctr is updated if an allocated name was added to the - * list, 0 if no name past *ctr was found, or -errno on errors. + * Returns > 0 and *ctr is updated if a full path from the link to the + * root dir was filled, 0 if no name past *ctr was found, or -errno on + * errors. */ -int scoutfs_dir_next_path(struct super_block *sb, u64 ino, u64 *ctr, - struct list_head *list) +int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, + char *path, unsigned int bytes) { - u64 our_ctr; + u64 final_ctr; u64 par_ctr; u64 par_ino; int ret; + int nr; if (*ctr == U64_MAX) return 0; retry: - our_ctr = *ctr; + final_ctr = *ctr; + ret = 0; + /* get the next link name to the given inode */ - ret = add_linkref_name(sb, &par_ino, ino, &our_ctr, list); - if (ret <= 0) + nr = append_linkref_name(sb, &par_ino, ino, &final_ctr, path, bytes); + if (nr <= 0) { + ret = nr; goto out; + } + ret += nr; /* then get the names of all the parent dirs */ while (par_ino != SCOUTFS_ROOT_INO) { par_ctr = 0; - ret = add_linkref_name(sb, &par_ino, par_ino, &par_ctr, list); - if (ret < 0) + nr = append_linkref_name(sb, &par_ino, par_ino, &par_ctr, + path + ret, bytes - ret); + if (nr < 0) { + ret = nr; goto out; + } /* restart if there was no parent component */ - if (ret == 0) { - scoutfs_dir_free_path(list); + if (nr == 0) goto retry; - } + + ret += nr; } out: - if (ret > 0) - *ctr = our_ctr; + *ctr = final_ctr; return ret; } diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 07edc195..4953af9e 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -16,9 +16,8 @@ struct scoutfs_path_component { char name[SCOUTFS_NAME_LEN]; }; -int scoutfs_dir_next_path(struct super_block *sb, u64 ino, u64 *ctr, - struct list_head *list); -void scoutfs_dir_free_path(struct list_head *list); +int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, + char *path, unsigned int bytes); int scoutfs_symlink_drop(struct super_block *sb, u64 ino); diff --git a/kmod/src/format.h b/kmod/src/format.h index 685f1d53..fafe802a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -254,14 +254,8 @@ struct scoutfs_dirent { #define SCOUTFS_NAME_LEN 255 -/* - * This is arbitrarily limiting the max size of the single buffer - * that's needed in the inode_paths ioctl to return all the paths - * that link to an inode. The structures could easily support much - * more than this but then we'd need to grow a more thorough interface - * for iterating over referring paths. That sounds horrible. - */ -#define SCOUTFS_LINK_MAX 255 +/* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ +#define SCOUTFS_LINK_MAX S32_MAX /* * We only use 31 bits for readdir positions so that we don't confuse diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 92346254..14c54448 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -99,26 +99,13 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, return ret; } -static int copy_to_ptr(char __user **to, const void *from, - unsigned long n, int space) -{ - if (n > space) - return -EOVERFLOW; - - if (copy_to_user(*to, from, n)) - return -EFAULT; - - *to += n; - return space - n; -} - /* - * Fill the caller's buffer with all the paths from the on-disk root - * directory to the target inode. It will provide as many full paths as - * there are final links to the target inode. + * Fill the caller's buffer with one of the paths from the on-disk root + * directory to the target inode. * - * The null terminated paths are stored consecutively in the buffer. A - * final zero length null terminated string follows the last path. + * Userspace provides a u64 counter used to chose which path to return. + * It should be initialized to zero to start iterating. After each path + * it is set to the next counter to search from. * * This only walks back through full hard links. None of the returned * paths will reflect symlinks to components in the path. @@ -127,35 +114,32 @@ static int copy_to_ptr(char __user **to, const void *from, * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which * bypasses permissions checking. * - * If the provided buffer isn't large enough EOVERFLOW will be returned. - * The buffer can be approximately sized by multiplying the inode's - * nlink by PATH_MAX. + * ENAMETOOLONG is returned when the next path from the given counter + * doesn't fit in the buffer. Providing a buffer of PATH_MAX should + * succeed. * * This call is not serialized with any modification (create, rename, * unlink) of the path components. It will return all the paths that * were stable both before and after the call. It may or may not return * paths which are created or unlinked during the call. * - * This will return failure if it fails to read any path. An empty - * buffer is returned if the target inode doesn't exist or is - * disconnected from the root. + * The number of bytes in the path, including the null terminator, are + * returned when a path is found. 0 is returned when there are no more + * paths to the link from the given counter. -errno is returned on + * errors. * * XXX - * - we may want to support partial failure * - can dir renaming trick us into returning garbage paths? seems likely. */ -static long scoutfs_ioc_inode_paths(struct file *file, unsigned long arg) +static long scoutfs_ioc_ino_path(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_inode_paths __user *uargs = (void __user *)arg; - struct scoutfs_ioctl_inode_paths args; - struct scoutfs_path_component *comp; - struct scoutfs_path_component *tmp; - static char slash = '/'; - static char null = '\0'; - char __user *ptr; - LIST_HEAD(list); - u64 ctr; + struct scoutfs_ioctl_ino_path __user *uargs = (void __user *)arg; + struct scoutfs_ioctl_ino_path args; + unsigned int bytes; + char __user *upath; + char *comp; + char *path; int ret; int len; @@ -165,42 +149,40 @@ static long scoutfs_ioc_inode_paths(struct file *file, unsigned long arg) if (copy_from_user(&args, uargs, sizeof(args))) return -EFAULT; - if (args.buf_len > INT_MAX) + if (args.path_bytes <= 1) return -EINVAL; - ptr = (void __user *)(unsigned long)args.buf_ptr; - len = args.buf_len; + bytes = min_t(unsigned int, args.path_bytes, PATH_MAX); + path = kmalloc(bytes, GFP_KERNEL); + if (path == NULL) + return -ENOMEM; - ctr = 0; - while ((ret = scoutfs_dir_next_path(sb, args.ino, &ctr, &list)) > 0) { - ret = 0; + /* positive ret is len of all components including null terminators */ + ret = scoutfs_dir_get_ino_path(sb, args.ino, &args.ctr, path, bytes); + if (ret <= 0) + goto out; - /* copy the components out as a path */ - list_for_each_entry_safe(comp, tmp, &list, head) { - len = copy_to_ptr(&ptr, comp->name, comp->len, len); - if (len < 0) - goto out; + /* reverse the components from backref order to path/ order */ + comp = path; + upath = (void __user *)((unsigned long)args.path_ptr + ret); + while (comp < (path + ret)) { + len = strlen(comp); + if (comp != path) + comp[len] = '/'; + len++; - list_del_init(&comp->head); - kfree(comp); - - if (!list_empty(&list)) { - len = copy_to_ptr(&ptr, &slash, 1, len); - if (len < 0) - goto out; - } + upath -= len; + if (copy_to_user(upath, comp, len)) { + ret = -EFAULT; + break; } - len = copy_to_ptr(&ptr, &null, 1, len); - if (len < 0) - goto out; + comp += len; } - len = copy_to_ptr(&ptr, &null, 1, len); + if (ret > 0 && put_user(args.ctr, &uargs->ctr)) + ret = -EFAULT; out: - scoutfs_dir_free_path(&list); - - if (ret == 0 && len < 0) - ret = len; + kfree(path); return ret; } @@ -297,8 +279,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case SCOUTFS_IOC_INODES_SINCE: return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_INODE_KEY); - case SCOUTFS_IOC_INODE_PATHS: - return scoutfs_ioc_inode_paths(file, arg); + case SCOUTFS_IOC_INO_PATH: + return scoutfs_ioc_ino_path(file, arg); case SCOUTFS_IOC_FIND_XATTR_NAME: return scoutfs_ioc_find_xattr(file, arg, true); case SCOUTFS_IOC_FIND_XATTR_VAL: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 3ea55a03..d0592d2f 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -26,18 +26,17 @@ struct scoutfs_ioctl_inodes_since { #define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ struct scoutfs_ioctl_inodes_since) -struct scoutfs_ioctl_inode_paths { +/* returns bytes of path buffer set starting at _off, including null */ +struct scoutfs_ioctl_ino_path { __u64 ino; - __u64 buf_ptr; - __u32 buf_len; + __u64 ctr; /* init to 0, set to next */ + __u64 path_ptr; + __u16 path_bytes; /* total buffer space, including null term */ } __packed; -/* - * Fills the callers buffer with all the paths from the root to the - * target inode. - */ -#define SCOUTFS_IOC_INODE_PATHS _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ - struct scoutfs_ioctl_inode_paths) +/* Get a single path from the root to the given inode number */ +#define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_ino_path) /* XXX might as well include a seq? 0 for current behaviour? */ struct scoutfs_ioctl_find_xattr { From af5955e95ac32ba221cb67880f7d19b38c01d141 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 15 Nov 2016 12:39:32 -0800 Subject: [PATCH 135/920] Add found_seq argument to scoutfs_btree_prev Add a *found_seq argument to _prev so that it can give the caller the seq of the item that's returned. The extent code is going to use this to find seqs of extents. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 4 +++- kmod/src/btree.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index ab2387bb..9342b209 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1388,7 +1388,7 @@ int scoutfs_btree_since(struct super_block *sb, */ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, + struct scoutfs_key *found, u64 *found_seq, struct scoutfs_btree_val *val) { struct scoutfs_btree_item *item; @@ -1433,6 +1433,8 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, item = pos_item(bt, pos); if (cmp == 0 || scoutfs_key_cmp(&item->key, first) >= 0) { *found = item->key; + if (found_seq) + *found_seq = le64_to_cpu(item->seq); if (val) ret = copy_to_val(val, item); else diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 6df548d1..dec2310c 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -56,7 +56,7 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_val *val); int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, + struct scoutfs_key *found, u64 *found_seq, struct scoutfs_btree_val *val); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, From f1b29c8372def30a7528f339e1eda95dd23e39c8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 16 Nov 2016 11:30:31 -0800 Subject: [PATCH 136/920] scoutfs_btree_prev() searches prev block, not next Oops, scoutfs_btree_prev() asked btree_walk() for the key for the next block, not the previous block to search when it's walk lands in the space before all the items in the leaf block. I saw it when truncate's check_size_eq constraint failed on items outside the range which stopped the truncate and left inodes, extents, and the orphan item around after rm -rf. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/btree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 9342b209..a1410134 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1406,7 +1406,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, ret = -ENOENT; while (scoutfs_key_cmp(&key, first) >= 0) { - bl = btree_walk(sb, root, &key, NULL, &prev_key, 0, 0, 0); + bl = btree_walk(sb, root, &key, &prev_key, NULL, 0, 0, 0); if (IS_ERR(bl)) { ret = PTR_ERR(bl); break; From 467801de73dd1847c5e7ca8739b7e97aca88500a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 27 Oct 2016 14:03:41 -0500 Subject: [PATCH 137/920] scoutfs: use extents for file data We're very basic here at this stage and simply put a single-block extent item where we would have previously had a multi-block bmap item. Multi-block extents will come in future patches. Signed-off-by: Mark Fasheh Signed-off-by: Zach Brown --- kmod/src/filerw.c | 165 +++++++++++++++------------------------ kmod/src/filerw.h | 2 +- kmod/src/format.h | 24 ++---- kmod/src/inode.c | 2 +- kmod/src/ioctl.c | 2 +- kmod/src/scoutfs_trace.h | 2 +- 6 files changed, 74 insertions(+), 123 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 091f2333..841b5a01 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -27,8 +27,8 @@ #include "ioctl.h" /* - * scoutfs uses simple fixed size block mapping items to map aligned - * groups of logical file data blocks to physical block locations. + * scoutfs uses an extent item to map logical file data blocks to + * physical block locations. * * The small block size is set to the smallest supported page size. * This means that our file IO code never has to worry about the @@ -40,7 +40,7 @@ * is, and have a 1:1 relationship between block writes and block * mapping item entries. * - * Dirty blocks are only written to free space. The first time a block + * Dirty extents are only written to free space. The first time a block * hits write_page in a transaction it gets a newly allocated block. We * get decent contiguous allocations by having per-task preallocation * streams. These are trimmed back as the transaction is committed. We @@ -64,7 +64,7 @@ * - need to wire up dirty inode? * - enforce writing to free blknos * - per-task allocation regions - * - tear down dirty blocks left by write errors on unmount + * - tear down dirty extents left by write errors on unmount * - should invalidate dirty blocks if freed * - data block checksumming (stable pages) * - mmap creating dirty unmapped pages at writepage @@ -177,96 +177,72 @@ static void return_file_block(struct super_block *sb, u64 blkno) spin_unlock(&sbi->file_alloc_lock); } -static bool bmap_has_blocks(struct scoutfs_block_map *bmap) -{ - int i; - - for (i = 0; i < SCOUTFS_BLOCK_MAP_COUNT; i++) { - if (bmap->blkno[i]) - return true; - } - - return false; -} - /* - * Free mapped blocks whose entire contents are past the new specified - * size. The caller holds a transaction. If we truncate all the blocks - * in a mapping item then we remove the item. + * Free mapped extents whose entire contents are past the new + * specified size. The caller holds a transaction. * - * This is the low level block allocation and bmap item manipulation. + * This is the low level extent item truncate code. * Callers manage higher order truncation and orphan cleanup. * - * XXX what to do about leaving items past i_size? * XXX probably should be a range */ -int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size) +int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 size) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_block_map bmap; + struct scoutfs_extent extent; struct scoutfs_btree_val val; - struct scoutfs_key last; struct scoutfs_key key; - bool modified; + struct scoutfs_key first; u64 iblock; - u64 blkno; + u64 len; + u64 loff; + u64 seq; int ret; - int i; iblock = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); - i = iblock & SCOUTFS_BLOCK_MAP_MASK; - scoutfs_set_key(&key, ino, SCOUTFS_BMAP_KEY, - iblock & ~(u64)SCOUTFS_BLOCK_MAP_MASK); - scoutfs_set_key(&last, ino, SCOUTFS_BMAP_KEY, ~0ULL); + scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, 0); + scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, ~0ULL); - trace_printk("iblock %llu i %d\n", iblock, i); + trace_printk("iblock %llu\n", iblock); - scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + scoutfs_btree_init_val(&val, &extent, sizeof(extent)); val.check_size_eq = 1; for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, + &val); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - /* XXX check bmap sanity */ + loff = le64_to_cpu(key.offset); + len = le64_to_cpu(extent.len); - /* make sure we can update bmap after freeing */ + if (WARN_ON_ONCE(len != 1)) { + ret = -EIO; + break; + } + + if ((loff + len) <= iblock) + break; + + /* make sure we can delete the extent after freeing */ ret = scoutfs_btree_dirty(sb, meta, &key); if (ret) break; - modified = false; - for (; i < SCOUTFS_BLOCK_MAP_COUNT; i++) { - blkno = le64_to_cpu(bmap.blkno[i]); - if (blkno == 0) - continue; - - ret = scoutfs_buddy_free(sb, bmap.seq[i], blkno, 0); - if (ret) - break; - - bmap.blkno[i] = 0; - bmap.seq[i] = 0; - modified = true; - } - i = 0; - - /* dirtying should have prevented these from failing */ - if (!bmap_has_blocks(&bmap)) - scoutfs_btree_delete(sb, meta, &key); - else if (modified) - scoutfs_btree_update(sb, meta, &key, &val); - + ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), + le64_to_cpu(extent.blkno), 0); if (ret) break; + scoutfs_btree_delete(sb, meta, &key); + /* XXX sync transaction if it's enormous */ - scoutfs_inc_key(&key); + scoutfs_dec_key(&key); } return ret; @@ -291,44 +267,27 @@ void scoutfs_filerw_free_alloc(struct super_block *sb) sbi->file_alloc_count = 0; } -static void set_bmap_key(struct scoutfs_key *key, struct inode *inode, - u64 iblock) -{ - scoutfs_set_key(key, scoutfs_ino(inode), SCOUTFS_BMAP_KEY, - iblock >> SCOUTFS_BLOCK_MAP_SHIFT); -} - /* * Return the number of contiguously mapped blocks starting from the - * given logical block in the inode. We only return the number - * contained in one block map item. We walk through more items if it - * makes a difference. + * given logical block in the inode. */ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) { struct super_block *sb = inode->i_sb; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_btree_val val; - struct scoutfs_block_map bmap; + struct scoutfs_extent extent; struct scoutfs_key key; int ret; - int i; *blkno = 0; - - set_bmap_key(&key, inode, iblock); - scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); + scoutfs_btree_init_val(&val, &extent, sizeof(extent)); ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret == sizeof(bmap)) { - i = iblock & SCOUTFS_BLOCK_MAP_MASK; - *blkno = le64_to_cpu(bmap.blkno[i]); - - ret = 0; - while (i < SCOUTFS_BLOCK_MAP_COUNT && bmap.blkno[i]) { - ret++; - i++; - } + if (ret == sizeof(extent)) { + *blkno = le64_to_cpu(extent.blkno); + ret = min_t(u64, le64_to_cpu(extent.len), INT_MAX); } else if (ret >= 0) { /* XXX corruption */ ret = -EIO; @@ -360,44 +319,47 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->stable_super; struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_block_map bmap; + struct scoutfs_extent extent; struct scoutfs_btree_val val; + struct scoutfs_key first; struct scoutfs_key key; bool inserted = false; u64 old_blkno = 0; u64 new_blkno = 0; + u64 seq; int ret; int err; - int i; - set_bmap_key(&key, inode, iblock); - scoutfs_btree_init_val(&val, &bmap, sizeof(bmap)); + scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, 0); + scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); + scoutfs_btree_init_val(&val, &extent, sizeof(extent)); val.check_size_eq = 1; /* see if there's an existing mapping */ - ret = scoutfs_btree_lookup(sb, meta, &key, &val); + ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, &val); + if (ret == 0 && ((le64_to_cpu(key.offset) + + le64_to_cpu(extent.len)) <= iblock)) + ret = -ENOENT; if (ret < 0 && ret != -ENOENT) goto out; - /* make sure that updating the bmap item won't fail */ + /* make sure that updating the extent item won't fail */ if (ret == -ENOENT) { - memset(&bmap, 0, sizeof(bmap)); + memset(&extent, 0, sizeof(extent)); ret = scoutfs_btree_insert(sb, meta, &key, &val); if (ret) goto out; inserted = true; - } else { ret = scoutfs_btree_dirty(sb, meta, &key); if (ret) goto out; } - i = iblock & SCOUTFS_BLOCK_MAP_MASK; - old_blkno = le64_to_cpu(bmap.blkno[i]); + old_blkno = le64_to_cpu(extent.blkno); /* If the existing block is dirty then we can use it */ - if (old_blkno && (bmap.seq[i] == super->hdr.seq)) { + if (old_blkno && cpu_to_le64(seq) == super->hdr.seq) { *blkno_ret = old_blkno; ret = 0; goto out; @@ -408,13 +370,13 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) goto out; if (old_blkno) { - ret = scoutfs_buddy_free(sb, bmap.seq[i], old_blkno, 0); + ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), old_blkno, 0); if (ret) goto out; } - bmap.blkno[i] = cpu_to_le64(new_blkno); - bmap.seq[i] = super->hdr.seq; + extent.blkno = cpu_to_le64(new_blkno); + extent.len = cpu_to_le64(1); /* dirtying guarantees success */ err = scoutfs_btree_update(sb, meta, &key, &val); @@ -448,7 +410,8 @@ static int scoutfs_readpage_get_block(struct inode *inode, sector_t iblock, ret = contig_mapped_blocks(inode, iblock, &blkno); if (ret > 0) { map_bh(bh, inode->i_sb, blkno); - bh->b_size = min_t(int, bh->b_size, ret << inode->i_blkbits); + bh->b_size = min_t(u64, bh->b_size, + (u64)ret << inode->i_blkbits); ret = 0; } @@ -486,7 +449,7 @@ static int scoutfs_writepage_get_block(struct inode *inode, sector_t iblock, } /* - * Dirty file blocks can be written to their newly allocated free blocks + * Dirty file pages can be written to their newly allocated free extents * at any time. They won't be referenced by metadata until the current * transaction is committed. They can be re-read and re-dirtied at * their free block number in this transaction. @@ -507,8 +470,8 @@ static int scoutfs_writepages(struct address_space *mapping, } /* - * Block allocation during buffered writes needs to make sure that the - * dirty block will be written to free space. + * Extent allocation during buffered writes needs to make sure that the + * dirty blocks will be written to free space. */ static int scoutfs_write_begin_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h index ba2bb81f..de4972d3 100644 --- a/kmod/src/filerw.h +++ b/kmod/src/filerw.h @@ -5,6 +5,6 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; void scoutfs_filerw_free_alloc(struct super_block *sb); -int scoutfs_truncate_block_items(struct super_block *sb, u64 ino, u64 size); +int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 size); #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index fafe802a..2b2b31b0 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -113,7 +113,7 @@ struct scoutfs_key { #define SCOUTFS_DIRENT_KEY 5 #define SCOUTFS_LINK_BACKREF_KEY 6 #define SCOUTFS_SYMLINK_KEY 7 -#define SCOUTFS_BMAP_KEY 8 +#define SCOUTFS_EXTENT_KEY 8 #define SCOUTFS_ORPHAN_KEY 9 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -288,23 +288,11 @@ struct scoutfs_xattr { __u8 name[0]; } __packed; -/* - * We use simple block map items to map a aligned fixed group of logical - * block offsets to physical blocks. We make them a decent size to - * reduce the item storage overhead per block referenced, but we don't - * want them so large that they start to take up an extraordinary amount - * of space for small files. 8 block items ranges from around 3% to .3% - * overhead for files that use only one or all of the blocks in the - * mapping item. - */ -#define SCOUTFS_BLOCK_MAP_SHIFT 3 -#define SCOUTFS_BLOCK_MAP_COUNT (1 << SCOUTFS_BLOCK_MAP_SHIFT) -#define SCOUTFS_BLOCK_MAP_MASK (SCOUTFS_BLOCK_MAP_COUNT - 1) - -struct scoutfs_block_map { - __le64 blkno[SCOUTFS_BLOCK_MAP_COUNT]; - __le64 seq[SCOUTFS_BLOCK_MAP_COUNT]; -}; +struct scoutfs_extent { + __le64 blkno; + __le64 len; + __u8 flags; +} __packed; /* * link backrefs give us a way to find all the hard links that refer diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 98a005ca..81a18a85 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -425,7 +425,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, if (S_ISLNK(mode)) ret = scoutfs_symlink_drop(sb, ino); else if (S_ISREG(mode)) - ret = scoutfs_truncate_block_items(sb, ino, 0); + ret = scoutfs_truncate_extent_items(sb, ino, 0); if (ret) goto out; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 14c54448..47082869 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -286,7 +286,7 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case SCOUTFS_IOC_FIND_XATTR_VAL: return scoutfs_ioc_find_xattr(file, arg, false); case SCOUTFS_IOC_INODE_DATA_SINCE: - return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_BMAP_KEY); + return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_EXTENT_KEY); } return -ENOTTY; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a1ca45d8..a9a18118 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -39,7 +39,7 @@ struct scoutfs_sb_info; { SCOUTFS_DIRENT_KEY, "DIRENT" }, \ { SCOUTFS_LINK_BACKREF_KEY, "LINK_BACKREF"}, \ { SCOUTFS_SYMLINK_KEY, "SYMLINK" }, \ - { SCOUTFS_BMAP_KEY, "BMAP" }) + { SCOUTFS_EXTENT_KEY, "EXTENT" }) #define TRACE_KEYF "%llu.%s.%llu" From f86fab116246a25dcf77f6a22881ccb87e44cc28 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 14:52:28 -0800 Subject: [PATCH 138/920] Add an inode data_version field The data_version field is changed every time the contents of the file could have changed. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/filerw.c | 5 +++++ kmod/src/format.h | 5 +++++ kmod/src/inode.c | 31 ++++++++++++++++++++++++++++++- kmod/src/inode.h | 5 +++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 841b5a01..daa975df 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -615,6 +615,11 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); + if (ret > 0) { + scoutfs_inode_inc_data_version(inode); + /* XXX kind of a big hammer, inode life cycle needs work */ + scoutfs_update_inode_item(inode); + } scoutfs_release_trans(sb); return ret; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 2b2b31b0..528b2e5d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -200,6 +200,10 @@ struct scoutfs_timespec { } __packed; /* + * @data_version: incremented every time the contents of a file could + * have changed. It is exposed via an ioctl and is then provided as an + * argument to data functions to protect racing modification. + * * XXX * - otime? * - compat flags? @@ -211,6 +215,7 @@ struct scoutfs_inode { __le64 size; __le64 blocks; __le64 link_counter; + __le64 data_version; __le32 nlink; __le32 uid; __le32 gid; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 81a18a85..aabb0910 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -120,9 +120,10 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_mtime.tv_nsec = le32_to_cpu(cinode->mtime.nsec); inode->i_ctime.tv_sec = le64_to_cpu(cinode->ctime.sec); inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); - + ci->salt = le32_to_cpu(cinode->salt); atomic64_set(&ci->link_counter, le64_to_cpu(cinode->link_counter)); + ci->data_version = le64_to_cpu(cinode->data_version); } static int scoutfs_read_locked_inode(struct inode *inode) @@ -148,6 +149,31 @@ static int scoutfs_read_locked_inode(struct inode *inode) return ret; } +void scoutfs_inode_inc_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version++; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + +u64 scoutfs_inode_get_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + unsigned int seq; + u64 vers; + + do { + seq = read_seqcount_begin(&si->seqcount); + vers = si->data_version; + } while (read_seqcount_retry(&si->seqcount, seq)); + + return vers; +} + static int scoutfs_iget_test(struct inode *inode, void *arg) { struct scoutfs_inode_info *ci = SCOUTFS_I(inode); @@ -210,6 +236,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->salt = cpu_to_le32(ci->salt); cinode->link_counter = cpu_to_le64(atomic64_read(&ci->link_counter)); + cinode->data_version = cpu_to_le64(ci->data_version); } /* @@ -366,6 +393,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; + seqcount_init(&ci->seqcount); + ci->data_version = 0; get_random_bytes(&ci->salt, sizeof(ci->salt)); atomic64_set(&ci->link_counter, 0); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 52ad52a6..1becde1d 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -5,6 +5,9 @@ struct scoutfs_inode_info { u64 ino; u32 salt; + seqcount_t seqcount; + u64 data_version; + atomic64_t link_counter; struct rw_semaphore xattr_rwsem; @@ -33,6 +36,8 @@ void scoutfs_dirty_inode(struct inode *inode, int flags); void scoutfs_update_inode_item(struct inode *inode); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); +void scoutfs_inode_inc_data_version(struct inode *inode); +u64 scoutfs_inode_get_data_version(struct inode *inode); int scoutfs_scan_orphans(struct super_block *sb); From 5d874189250afdbbd13027823cfaa352f894334a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 15:12:30 -0800 Subject: [PATCH 139/920] Add ioctl for sampling inode data version Add an ioctl that samples the inode's data_version. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/ioctl.c | 18 ++++++++++++++++++ kmod/src/ioctl.h | 3 +++ 2 files changed, 21 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 47082869..a4fbd013 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -24,6 +24,7 @@ #include "name.h" #include "ioctl.h" #include "super.h" +#include "inode.h" /* * Find all the inodes that have had keys of a given type modified since @@ -274,6 +275,21 @@ out: return copied ?: ret; } +/* + * Sample the inode's data_version. It is not strictly serialized with + * writes that are in flight. + */ +static long scoutfs_ioc_data_version(struct file *file, unsigned long arg) +{ + u64 __user *uvers = (void __user *)arg; + u64 vers = scoutfs_inode_get_data_version(file_inode(file)); + + if (put_user(vers, uvers)) + return -EFAULT; + + return 0; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -287,6 +303,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_find_xattr(file, arg, false); case SCOUTFS_IOC_INODE_DATA_SINCE: return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_EXTENT_KEY); + case SCOUTFS_IOC_DATA_VERSION: + return scoutfs_ioc_data_version(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index d0592d2f..964ccd92 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -55,4 +55,7 @@ struct scoutfs_ioctl_find_xattr { #define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ struct scoutfs_ioctl_inodes_since) + +#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 6, u64) + #endif From df561bbd19179e91453c150b56c8834321f2d729 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Nov 2016 15:43:12 -0800 Subject: [PATCH 140/920] Add offline extent flag and release ioctl Add the _OFFLINE flag to indicate offline extents. The release ioctl frees extents within the release range and sets their _OFFLINE flag if the data_version still matches. We tweak the existing truncate item function just a bit to support making extents offline. We make it take an explicit range of blocks to remove instead of just giving it the size and it learns to mark extents offline and update them instead of always deleting them. Reads from offline extents return zeros like reading from a sparse region (later it will trigger demand staging) and writing to offline extents clears the offline flag (later only staging can do that). Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/filerw.c | 69 +++++++++++++++++++++++--------------- kmod/src/filerw.h | 3 +- kmod/src/format.h | 2 ++ kmod/src/inode.c | 2 +- kmod/src/ioctl.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 9 +++++ 6 files changed, 142 insertions(+), 28 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index daa975df..727737cd 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -178,31 +178,28 @@ static void return_file_block(struct super_block *sb, u64 blkno) } /* - * Free mapped extents whose entire contents are past the new - * specified size. The caller holds a transaction. + * Free extents whose blocks fall inside the specified blocks. The + * caller holds a transaction. * - * This is the low level extent item truncate code. - * Callers manage higher order truncation and orphan cleanup. + * If 'release' is given then blocks are freed inside i_size but the + * extent items are left behind and their _OFFLINE flag is set. * - * XXX probably should be a range + * This is the low level extent item truncate code. Callers manage + * higher order truncation and orphan cleanup. */ -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 size) +int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline) { struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_extent extent; struct scoutfs_btree_val val; struct scoutfs_key key; struct scoutfs_key first; - u64 iblock; - u64 len; - u64 loff; u64 seq; int ret; - iblock = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); - - scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, 0); - scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, ~0ULL); + scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); + scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); trace_printk("iblock %llu\n", iblock); @@ -218,28 +215,43 @@ int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 size) break; } - loff = le64_to_cpu(key.offset); len = le64_to_cpu(extent.len); - if (WARN_ON_ONCE(len != 1)) { ret = -EIO; break; } - if ((loff + len) <= iblock) + /* XXX corruption: offline and allocation are exclusive */ + if (!!extent.blkno == + !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { + ret = -EIO; break; + } + + if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) + continue; /* make sure we can delete the extent after freeing */ - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - break; + if (extent.blkno) { + ret = scoutfs_btree_dirty(sb, meta, &key); + if (ret) + break; - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), - le64_to_cpu(extent.blkno), 0); - if (ret) - break; + ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), + le64_to_cpu(extent.blkno), 0); + if (ret) + break; + } - scoutfs_btree_delete(sb, meta, &key); + if (offline) { + extent.blkno = 0; + extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; + scoutfs_btree_update(sb, meta, &key, &val); + } else { + ret = scoutfs_btree_delete(sb, meta, &key); + if (ret) + break; + } /* XXX sync transaction if it's enormous */ scoutfs_dec_key(&key); @@ -286,8 +298,12 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) ret = scoutfs_btree_lookup(sb, meta, &key, &val); if (ret == sizeof(extent)) { - *blkno = le64_to_cpu(extent.blkno); - ret = min_t(u64, le64_to_cpu(extent.len), INT_MAX); + if (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) { + ret = 0; + } else { + *blkno = le64_to_cpu(extent.blkno); + ret = min_t(u64, le64_to_cpu(extent.len), INT_MAX); + } } else if (ret >= 0) { /* XXX corruption */ ret = -EIO; @@ -377,6 +393,7 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) extent.blkno = cpu_to_le64(new_blkno); extent.len = cpu_to_le64(1); + extent.flags &= ~SCOUTFS_EXTENT_FLAG_OFFLINE; /* dirtying guarantees success */ err = scoutfs_btree_update(sb, meta, &key, &val); diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h index de4972d3..f5924d71 100644 --- a/kmod/src/filerw.h +++ b/kmod/src/filerw.h @@ -5,6 +5,7 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; void scoutfs_filerw_free_alloc(struct super_block *sb); -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 size); +int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline); #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 528b2e5d..8c7bb7a1 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -299,6 +299,8 @@ struct scoutfs_extent { __u8 flags; } __packed; +#define SCOUTFS_EXTENT_FLAG_OFFLINE (1 << 0) + /* * link backrefs give us a way to find all the hard links that refer * to a target inode. They're stored at an offset determined by an diff --git a/kmod/src/inode.c b/kmod/src/inode.c index aabb0910..0d98d8bf 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -454,7 +454,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, if (S_ISLNK(mode)) ret = scoutfs_symlink_drop(sb, ino); else if (S_ISREG(mode)) - ret = scoutfs_truncate_extent_items(sb, ino, 0); + ret = scoutfs_truncate_extent_items(sb, ino, 0, ~0ULL, false); if (ret) goto out; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index a4fbd013..3144f940 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include "format.h" #include "btree.h" @@ -25,6 +27,8 @@ #include "ioctl.h" #include "super.h" #include "inode.h" +#include "trans.h" +#include "filerw.h" /* * Find all the inodes that have had keys of a given type modified since @@ -290,6 +294,85 @@ static long scoutfs_ioc_data_version(struct file *file, unsigned long arg) return 0; } +/* + * The caller has a version of the data available in the given byte + * range in an external archive. As long as the data version still + * matches we free the blocks fully contained in the range and mark them + * offline. Attempts to use the blocks in the future will trigger + * recall from the archive. + * + * XXX permissions? + * XXX a lot of this could be generic file write prep + */ +static long scoutfs_ioc_release(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_ioctl_release args; + loff_t start; + loff_t end_inc; + u64 iblock; + u64 end_block; + u64 len; + int ret; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + if (args.count == 0) + return 0; + if ((args.offset + args.count) < args.offset) + return -EINVAL; + + start = round_up(args.offset, SCOUTFS_BLOCK_SIZE); + end_inc = round_down(args.offset + args.count, SCOUTFS_BLOCK_SIZE) - 1; + if (end_inc > start) + return 0; + + iblock = start >> SCOUTFS_BLOCK_SHIFT; + end_block = end_inc >> SCOUTFS_BLOCK_SHIFT; + len = end_block - iblock + 1; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + mutex_lock(&inode->i_mutex); + + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + goto out; + } + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EINVAL; + goto out; + } + + if (scoutfs_inode_get_data_version(inode) != args.data_version) { + ret = -ESTALE; + goto out; + } + + inode_dio_wait(inode); + + /* drop all clean and dirty cached blocks in the range */ + truncate_inode_pages_range(&inode->i_data, start, end_inc); + + ret = scoutfs_hold_trans(sb); + if (ret) + goto out; + + ret = scoutfs_truncate_extent_items(sb, scoutfs_ino(inode), + iblock, len, true); + scoutfs_release_trans(sb); +out: + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); + + return ret; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -305,6 +388,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_EXTENT_KEY); case SCOUTFS_IOC_DATA_VERSION: return scoutfs_ioc_data_version(file, arg); + case SCOUTFS_IOC_RELEASE: + return scoutfs_ioc_release(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 964ccd92..1be28b9b 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -58,4 +58,13 @@ struct scoutfs_ioctl_find_xattr { #define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 6, u64) +struct scoutfs_ioctl_release { + __u64 offset; + __u64 count; + __u64 data_version; +} __packed; + +#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ + struct scoutfs_ioctl_release) + #endif From c6b688c2bf13aaf17c4ab1f433578aabd4dab9b1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 15 Nov 2016 15:45:02 -0800 Subject: [PATCH 141/920] Add staging ioctl This adds the ioctl for writing archived file contents back into the file if the data_version still matches. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/filerw.c | 7 ++++ kmod/src/inode.c | 13 +++--- kmod/src/inode.h | 3 ++ kmod/src/ioctl.c | 101 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 10 +++++ 5 files changed, 129 insertions(+), 5 deletions(-) diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c index 727737cd..5cdce097 100644 --- a/kmod/src/filerw.c +++ b/kmod/src/filerw.c @@ -331,6 +331,7 @@ static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) */ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->stable_super; @@ -367,6 +368,12 @@ static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) goto out; inserted = true; } else { + if ((extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) && + !si->staging) { + ret = -EINVAL; + goto out; + } + ret = scoutfs_btree_dirty(sb, meta, &key); if (ret) goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 0d98d8bf..f990dcf9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -153,11 +153,13 @@ void scoutfs_inode_inc_data_version(struct inode *inode) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); - preempt_disable(); - write_seqcount_begin(&si->seqcount); - si->data_version++; - write_seqcount_end(&si->seqcount); - preempt_enable(); + if (!si->staging) { + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version++; + write_seqcount_end(&si->seqcount); + preempt_enable(); + } } u64 scoutfs_inode_get_data_version(struct inode *inode) @@ -395,6 +397,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->ino = ino; seqcount_init(&ci->seqcount); ci->data_version = 0; + ci->staging = false; get_random_bytes(&ci->salt, sizeof(ci->salt)); atomic64_set(&ci->link_counter, 0); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 1becde1d..f0f74024 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -8,6 +8,9 @@ struct scoutfs_inode_info { seqcount_t seqcount; u64 data_version; + /* holder of i_mutex is staging */ + bool staging; + atomic64_t link_counter; struct rw_semaphore xattr_rwsem; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 3144f940..d40cfae1 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include "format.h" #include "btree.h" @@ -373,6 +375,103 @@ out: return ret; } +/* + * Write the archived contents of the file back if the data_version + * still matches. + * + * This is a data plane operation only. We don't want the write to + * change any fields in the inode. It only changes the file contents. + * + * Keep in mind that the staging writes can easily span transactions and + * can crash partway through. If we called the normal write path and + * restored the inode afterwards the modified inode could be commited + * partway through by a transaction and then left that way by a crash + * before the write finishes and we restore the fields. It also + * wouldn't be great if the temporarily updated inode was visible to + * paths that don't serialize with write. + * + * We're implementing the buffered write path down to the start of + * generic_file_buffered_writes() without all the stuff that would + * change the inode: file_remove_suid(), file_update_time(). The + * easiest way to do that is to call generic_file_buffered_write(). + * We're careful to only allow staging writes inside i_size. + * + * We set a bool on the inode which tells our code to update the + * offline extents and to not update the data_version counter. + * + * This doesn't support any fancy write modes or side-effects: aio, + * direct, append, sync, breaking suid, sending rlimit signals. + */ +static long scoutfs_ioc_stage(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct scoutfs_ioctl_stage args; + struct kiocb kiocb; + struct iovec iov; + size_t written; + loff_t pos; + int ret; + + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + + if (args.count < 0 || (args.offset + args.count < args.offset)) + return -EINVAL; + if (args.count == 0) + return 0; + + /* the iocb is really only used for the file pointer :P */ + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = args.offset; + kiocb.ki_left = args.count; + kiocb.ki_nbytes = args.count; + iov.iov_base = (void __user *)(unsigned long)args.buf_ptr; + iov.iov_len = args.count; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + mutex_lock(&inode->i_mutex); + + if (!S_ISREG(inode->i_mode) || + !(file->f_mode & FMODE_WRITE) || + (file->f_flags & (O_APPEND | O_DIRECT | O_DSYNC)) || + IS_SYNC(file->f_mapping->host) || + (args.offset + args.count > i_size_read(inode))) { + ret = -EINVAL; + goto out; + } + + if (scoutfs_inode_get_data_version(inode) != args.data_version) { + ret = -ESTALE; + goto out; + } + + si->staging = true; + current->backing_dev_info = mapping->backing_dev_info; + + pos = args.offset; + written = 0; + do { + ret = generic_file_buffered_write(&kiocb, &iov, 1, pos, &pos, + args.count, written); + BUG_ON(ret == -EIOCBQUEUED); + if (ret > 0) + written += ret; + } while (ret > 0 && written < args.count); + + si->staging = false; + current->backing_dev_info = NULL; +out: + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); + + return ret; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -390,6 +489,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_data_version(file, arg); case SCOUTFS_IOC_RELEASE: return scoutfs_ioc_release(file, arg); + case SCOUTFS_IOC_STAGE: + return scoutfs_ioc_stage(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 1be28b9b..d39c6272 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -67,4 +67,14 @@ struct scoutfs_ioctl_release { #define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_release) +struct scoutfs_ioctl_stage { + __u64 data_version; + __u64 buf_ptr; + __u64 offset; + __s32 count; +} __packed; + +#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ + struct scoutfs_ioctl_stage) + #endif From 43d0d44e48ad9923e6666db96b78842914348334 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 2 Dec 2016 20:37:44 -0800 Subject: [PATCH 142/920] Add initial LSM implementation Add the initial core components of the LSM implementation to be able to read the root inode: - bio.c: read big block regions - seg.c: cache logical segments - ring.c: read the manifest from storage - manifest.c: organize segments into an LSM - kvec.c: work with arbitrary memory vectors - item.c: cache fs metadata items read from segments Signed-off-by: Zach Brown --- kmod/src/Makefile | 6 +- kmod/src/bio.c | 169 +++++++++++++++++ kmod/src/bio.h | 23 +++ kmod/src/format.h | 90 +++++++++ kmod/src/inode.c | 27 +-- kmod/src/inode.h | 3 + kmod/src/item.c | 217 +++++++++++++++++++++ kmod/src/item.h | 16 ++ kmod/src/kvec.c | 141 ++++++++++++++ kmod/src/kvec.h | 67 +++++++ kmod/src/manifest.c | 449 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/manifest.h | 11 ++ kmod/src/ring.c | 263 ++++++++++++++++++++++++++ kmod/src/ring.h | 8 + kmod/src/seg.c | 399 +++++++++++++++++++++++++++++++++++++++ kmod/src/seg.h | 20 ++ kmod/src/super.c | 15 +- kmod/src/super.h | 7 + 18 files changed, 1915 insertions(+), 16 deletions(-) create mode 100644 kmod/src/bio.c create mode 100644 kmod/src/bio.h create mode 100644 kmod/src/item.c create mode 100644 kmod/src/item.h create mode 100644 kmod/src/kvec.c create mode 100644 kmod/src/kvec.h create mode 100644 kmod/src/manifest.c create mode 100644 kmod/src/manifest.h create mode 100644 kmod/src/ring.c create mode 100644 kmod/src/ring.h create mode 100644 kmod/src/seg.c create mode 100644 kmod/src/seg.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 143929b7..d808143b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - inode.o ioctl.o msg.o name.o scoutfs_trace.o super.o trans.o \ - xattr.o +scoutfs-y += bio.o block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ + kvec.o inode.o ioctl.o item.o manifest.o msg.o name.o ring.o \ + seg.o scoutfs_trace.o super.o trans.o xattr.o diff --git a/kmod/src/bio.c b/kmod/src/bio.c new file mode 100644 index 00000000..d58eebe9 --- /dev/null +++ b/kmod/src/bio.c @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "bio.h" + +struct bio_end_io_args { + struct super_block *sb; + atomic_t bytes_in_flight; + int err; + scoutfs_bio_end_io_t end_io; + void *data; +}; + +static void dec_end_io(struct bio_end_io_args *args, size_t bytes, int err) +{ + if (err && !args->err) + args->err = err; + + if (atomic_sub_return(bytes, &args->bytes_in_flight) == 0) { + args->end_io(args->sb, args->data, args->err); + kfree(args); + } +} + +static void bio_end_io(struct bio *bio, int err) +{ + struct bio_end_io_args *args = bio->bi_private; + + dec_end_io(args, bio->bi_size, err); + bio_put(bio); +} + +/* + * Read or write the given number of 4k blocks from the front of the + * pages provided by the caller. We translate the block count into a + * page count and fill bios a page at a time. + * + * The caller is responsible for ensuring that the pages aren't freed + * while bios are in flight. + * + * The end_io function is always called once with the error result of + * the IO. It can be called before _submit returns. + */ +void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, + u64 blkno, unsigned int nr_blocks, + scoutfs_bio_end_io_t end_io, void *data) +{ + unsigned int nr_pages = DIV_ROUND_UP(nr_blocks, + SCOUTFS_BLOCKS_PER_PAGE); + struct bio_end_io_args *args; + struct blk_plug plug; + unsigned int bytes; + struct page *page; + struct bio *bio = NULL; + int ret = 0; + int i; + + args = kmalloc(sizeof(struct bio_end_io_args), GFP_NOFS); + if (!args) { + end_io(sb, data, -ENOMEM); + return; + } + + args->sb = sb; + atomic_set(&args->bytes_in_flight, 1); + args->err = 0; + args->end_io = end_io; + args->data = data; + + blk_start_plug(&plug); + + for (i = 0; i < nr_pages; i++) { + page = pages[i]; + + if (!bio) { + bio = bio_alloc(GFP_NOFS, nr_pages - i); + if (!bio) + bio = bio_alloc(GFP_NOFS, 1); + if (!bio) { + ret = -ENOMEM; + break; + } + + bio->bi_sector = blkno << (SCOUTFS_BLOCK_SHIFT - 9); + bio->bi_bdev = sb->s_bdev; + bio->bi_end_io = bio_end_io; + bio->bi_private = args; + } + + bytes = min_t(int, nr_blocks << SCOUTFS_BLOCK_SHIFT, PAGE_SIZE); + + if (bio_add_page(bio, page, bytes, 0) != bytes) { + /* submit the full bio and retry this page */ + atomic_add(bio->bi_size, &args->bytes_in_flight); + submit_bio(rw, bio); + bio = NULL; + i--; + continue; + } + + blkno += SCOUTFS_BLOCKS_PER_PAGE; + nr_blocks -= SCOUTFS_BLOCKS_PER_PAGE; + } + + if (bio) { + atomic_add(bio->bi_size, &args->bytes_in_flight); + submit_bio(rw, bio); + } + + blk_finish_plug(&plug); + dec_end_io(args, 1, ret); +} + +struct end_io_completion { + struct completion comp; + int err; +}; + +static void end_io_complete(struct super_block *sb, void *data, int err) +{ + struct end_io_completion *comp = data; + + comp->err = err; + complete(&comp->comp); +} + +/* + * A synchronous read of the given blocks. + * + * XXX we could make this interruptible. + */ +int scoutfs_bio_read(struct super_block *sb, struct page **pages, + u64 blkno, unsigned int nr_blocks) +{ + struct end_io_completion comp = { + .comp = COMPLETION_INITIALIZER(comp.comp), + }; + + scoutfs_bio_submit(sb, READ, pages, blkno, nr_blocks, + end_io_complete, &comp); + wait_for_completion(&comp.comp); + return comp.err; +} + +/* return pointer to the blk 4k block offset amongst the pages */ +void *scoutfs_page_block_address(struct page **pages, unsigned int blk) +{ + unsigned int i = blk / SCOUTFS_BLOCKS_PER_PAGE; + unsigned int off = (blk % SCOUTFS_BLOCKS_PER_PAGE) << + SCOUTFS_BLOCK_SHIFT; + + return page_address(pages[i]) + off; +} diff --git a/kmod/src/bio.h b/kmod/src/bio.h new file mode 100644 index 00000000..094f6038 --- /dev/null +++ b/kmod/src/bio.h @@ -0,0 +1,23 @@ +#ifndef _SCOUTFS_BIO_H_ +#define _SCOUTFS_BIO_H_ + +/* + * Our little block IO wrapper is just a convenience wrapper that takes + * our block size units and handles tracks multiple bios per larger io. + * + * If bios could hold an unlimited number of pages instead of + * BIO_MAX_PAGES then this would just use a single bio directly. + */ + +typedef void (*scoutfs_bio_end_io_t)(struct super_block *sb, void *data, + int err); + +void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, + u64 blkno, unsigned int nr_blocks, + scoutfs_bio_end_io_t end_io, void *data); +int scoutfs_bio_read(struct super_block *sb, struct page **pages, + u64 blkno, unsigned int nr_blocks); + +void *scoutfs_page_block_address(struct page **pages, unsigned int blk); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 8c7bb7a1..d0efbae4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -6,9 +6,23 @@ /* super block id */ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ +/* + * The super block and ring blocks are fixed 4k. + */ #define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) +#define SCOUTFS_BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) + +/* + * FS data is stored in segments, for now they're fixed size. They'll + * be dynamic. + */ +#define SCOUTFS_SEGMENT_SHIFT 20 +#define SCOUTFS_SEGMENT_SIZE (1 << SCOUTFS_SEGMENT_SHIFT) +#define SCOUTFS_SEGMENT_MASK (SCOUTFS_SEGMENT_SIZE - 1) +#define SCOUTFS_SEGMENT_PAGES (SCOUTFS_SEGMENT_SIZE / PAGE_SIZE) +#define SCOUTFS_SEGMENT_BLOCKS (SCOUTFS_SEGMENT_SIZE / BLOCK_SIZE) #define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) #define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) @@ -37,6 +51,67 @@ struct scoutfs_block_header { __le64 blkno; } __packed; +struct scoutfs_ring_entry_header { + __u8 type; + __le16 len; +} __packed; + +#define SCOUTFS_RING_ADD_MANIFEST 1 + +struct scoutfs_ring_add_manifest { + struct scoutfs_ring_entry_header eh; + __le64 segno; + __le64 seq; + __le16 first_key_len; + __le16 last_key_len; + __u8 level; + /* first and last key bytes */ +} __packed; + +/* + * This is absurdly huge. If there was only ever 1 item per segment and + * 2^64 items the tree could get this deep. + */ +#define SCOUTFS_MANIFEST_MAX_LEVEL 20 + +struct scoutfs_ring_block { + struct scoutfs_block_header hdr; + __le32 nr_entries; + struct scoutfs_ring_entry_header entries[0]; +} __packed; + +struct scoutfs_segment_item { + __le64 seq; + __le32 key_off; + __le32 val_off; + __le16 key_len; + __le16 val_len; +} __packed; + +/* + * Each large segment starts with a segment block that describes the + * rest of the blocks that make up the segment. + */ +struct scoutfs_segment_block { + __le32 crc; + __le32 _padding; + __le64 segno; + __le64 max_seq; + __le32 nr_items; + /* item array with gaps so they don't cross 4k blocks */ + /* packed keys */ + /* packed vals */ +} __packed; + +/* the first block in the segment has the header and items */ +#define SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_segment_block)) / \ + sizeof(struct scoutfs_segment_item)) + +/* the rest of the header blocks are full of items */ +#define SCOUTFS_SEGMENT_ITEMS_PER_BLOCK \ + (SCOUTFS_BLOCK_SIZE / sizeof(struct scoutfs_segment_item)) + /* * Block references include the sequence number so that we can detect * readers racing with writers and so that we can tell that we don't @@ -118,6 +193,11 @@ struct scoutfs_key { #define SCOUTFS_MAX_ITEM_LEN 512 +struct scoutfs_inode_key { + __u8 type; + __be64 ino; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; @@ -180,6 +260,11 @@ struct scoutfs_btree_item { #define SCOUTFS_UUID_BYTES 16 +/* + * The ring fields describe the statically allocated ring log. The + * head and tail indexes are logical 4k blocks offsets inside the ring. + * The head block should contain the seq. + */ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -187,6 +272,11 @@ struct scoutfs_super_block { __le64 next_ino; __le64 total_blocks; __le64 free_blocks; + __le64 ring_blkno; + __le64 ring_blocks; + __le64 ring_head_index; + __le64 ring_tail_index; + __le64 ring_head_seq; __le64 buddy_blocks; struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index f990dcf9..0ccfe006 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -29,6 +29,8 @@ #include "trans.h" #include "btree.h" #include "msg.h" +#include "kvec.h" +#include "item.h" /* * XXX @@ -126,25 +128,28 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->data_version = le64_to_cpu(cinode->data_version); } +static void set_inode_key(struct scoutfs_inode_key *ikey, u64 ino) +{ + ikey->type = SCOUTFS_INODE_KEY; + ikey->ino = cpu_to_be64(ino); +} + static int scoutfs_read_locked_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; + struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - struct scoutfs_key key; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); int ret; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); + set_inode_key(&ikey, scoutfs_ino(inode)); + scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret == sizeof(sinode)) { + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(sinode)); + if (ret == 0) load_inode(inode, &sinode); - ret = 0; - } else if (ret >= 0) { - ret = -EIO; - } return ret; } diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f0f74024..0d48f158 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -49,4 +49,7 @@ u64 scoutfs_last_ino(struct super_block *sb); void scoutfs_inode_exit(void); int scoutfs_inode_init(void); +int scoutfs_item_setup(struct super_block *sb); +void scoutfs_item_destroy(struct super_block *sb); + #endif diff --git a/kmod/src/item.c b/kmod/src/item.c new file mode 100644 index 00000000..73e9665b --- /dev/null +++ b/kmod/src/item.c @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "kvec.h" +#include "manifest.h" +#include "item.h" + +struct item_cache { + spinlock_t lock; + struct rb_root root; +}; + +struct cached_item { + struct rb_node node; + + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); +}; + +static struct cached_item *find_item(struct rb_root *root, struct kvec *key) +{ + struct rb_node *node = root->rb_node; + struct rb_node *parent = NULL; + struct cached_item *item; + int cmp; + + while (node) { + parent = node; + item = container_of(node, struct cached_item, node); + + cmp = scoutfs_kvec_memcmp(key, item->key); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return item; + } + + return NULL; +} + +static struct cached_item *insert_item(struct rb_root *root, + struct cached_item *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct cached_item *found = NULL; + struct cached_item *item; + int cmp; + + while (*node) { + parent = *node; + item = container_of(*node, struct cached_item, node); + + cmp = scoutfs_kvec_memcmp(ins->key, item->key); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + rb_replace_node(&item->node, &ins->node, root); + found = item; + break; + } + } + + if (!found) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + } + + return found; +} + +/* + * Find an item with the given key and copy its value into the caller's + * value vector. The amount of bytes copied is returned which can be + * 0 or truncated if the caller's buffer isn't big enough. + */ +int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, + struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + do { + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(&cac->root, key); + if (item) + ret = scoutfs_kvec_memcpy(val, item->val); + else + ret = -ENOENT; + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + + return ret; +} + +/* + * This requires that the item at the specified key has a value of the + * same length as the specified value. Callers are asserting that + * mismatched size are corruption so it returns -EIO if the sizes don't + * match. This isn't the fast path so we don't mind the copying + * overhead that comes from only detecting the size mismatch after the + * copy by reusing the more permissive _lookup(). + */ +int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, + struct kvec *val, int size) +{ + int ret; + + ret = scoutfs_item_lookup(sb, key, val); + if (ret >= 0 && ret != size) + ret = -EIO; + + return ret; +} + +static void free_item(struct cached_item *item) +{ + if (!IS_ERR_OR_NULL(item)) { + scoutfs_kvec_kfree(item->val); + scoutfs_kvec_kfree(item->key); + kfree(item); + } +} + +/* + * Add an item with the key and value to the item cache. The new item + * is clean. Any existing item at the key will be removed and freed. + */ +int scoutfs_item_insert(struct super_block *sb, struct kvec *key, + struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *found; + struct cached_item *item; + unsigned long flags; + int ret; + + item = kmalloc(sizeof(struct cached_item), GFP_NOFS); + if (!item) + return -ENOMEM; + + ret = scoutfs_kvec_dup_flatten(item->key, key) ?: + scoutfs_kvec_dup_flatten(item->val, val); + if (ret) { + free_item(item); + return ret; + } + + spin_lock_irqsave(&cac->lock, flags); + found = insert_item(&cac->root, item); + spin_unlock_irqrestore(&cac->lock, flags); + free_item(found); + + return 0; +} + +int scoutfs_item_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac; + + cac = kzalloc(sizeof(struct item_cache), GFP_KERNEL); + if (!cac) + return -ENOMEM; + sbi->item_cache = cac; + + spin_lock_init(&cac->lock); + cac->root = RB_ROOT; + + return 0; +} + +void scoutfs_item_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct rb_node *node; + struct cached_item *item; + + if (cac) { + for (node = rb_first(&cac->root); node; ) { + item = container_of(node, struct cached_item, node); + node = rb_next(node); + free_item(item); + } + + kfree(cac); + } + +} diff --git a/kmod/src/item.h b/kmod/src/item.h new file mode 100644 index 00000000..bfaae9db --- /dev/null +++ b/kmod/src/item.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_ITEM_H_ +#define _SCOUTFS_ITEM_H_ + +#include + +int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, + struct kvec *val); +int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, + struct kvec *val, int size); +int scoutfs_item_insert(struct super_block *sb, struct kvec *key, + struct kvec *val); + +int scoutfs_item_setup(struct super_block *sb); +void scoutfs_item_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c new file mode 100644 index 00000000..e2b26061 --- /dev/null +++ b/kmod/src/kvec.c @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "inode.h" +#include "dir.h" +#include "xattr.h" +#include "msg.h" +#include "block.h" +#include "counters.h" +#include "trans.h" +#include "buddy.h" +#include "kvec.h" +#include "scoutfs_trace.h" + +/* + * Return the result of memcmp between the min of the two total lengths. + * If their shorter lengths are equal than the shorter length is considered + * smaller than the longer. + */ +int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) +{ + int b_off = 0; + int a_off = 0; + int len; + int ret; + + while (a->iov_base && b->iov_base) { + len = min(a->iov_len - a_off, b->iov_len - b_off); + ret = memcmp(a->iov_base + a_off, b->iov_base + b_off, len); + if (ret) + return ret; + + b_off += len; + if (b_off == b->iov_len) + b++; + a_off += len; + if (a_off == a->iov_len) + a++; + } + + return a->iov_base ? 1 : b->iov_base ? -1 : 0; +} + +/* + * Returns 0 if [a,b] overlaps with [c,d]. Returns -1 if a < c and + * 1 if b > d. + */ +int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, + struct kvec *c, struct kvec *d) +{ + return scoutfs_kvec_memcmp(a, c) < 0 ? -1 : + scoutfs_kvec_memcmp(b, d) > 0 ? 1 : 0; +} + +/* + * Set just the pointers and length fields in the dst vector to point to + * the source vector. + */ +void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src) +{ + int i; + + for (i = 0; i < SCOUTFS_KVEC_NR; i++) + *(dst++) = *(src++); +} + +/* + * Copy as much of src as fits in dst. Null base pointers termintae the + * copy. The number of bytes copied is returned. Only the buffers + * pointed to by dst are changed, the kvec elements are not changed. + */ +int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src) +{ + int src_off = 0; + int dst_off = 0; + int copied = 0; + int len; + + while (dst->iov_base && src->iov_base) { + len = min(dst->iov_len - dst_off, src->iov_len - src_off); + memcpy(dst->iov_base + dst_off, src->iov_base + src_off, len); + + copied += len; + + src_off += len; + if (src_off == src->iov_len) + src++; + dst_off += len; + if (dst_off == dst->iov_len) + dst++; + } + + return copied; +} + +/* + * Copy the src key vector into one new allocation in the dst. The existing + * dst is clobbered. The source isn't changed. + */ +int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src) +{ + void *ptr; + size_t len = scoutfs_kvec_length(src); + + ptr = kmalloc(len, GFP_NOFS); + if (!ptr) + return -ENOMEM; + + scoutfs_kvec_init(dst, ptr, len); + scoutfs_kvec_memcpy(dst, src); + return 0; +} + +/* + * Free all the set pointers in the kvec. The pointer values aren't modified + * if they're freed. + */ +void scoutfs_kvec_kfree(struct kvec *kvec) +{ + while (kvec->iov_base) + kfree((kvec++)->iov_base); +} diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h new file mode 100644 index 00000000..600055e9 --- /dev/null +++ b/kmod/src/kvec.h @@ -0,0 +1,67 @@ +#ifndef _SCOUTFS_KVEC_H_ +#define _SCOUTFS_KVEC_H_ + +#include + +/* + * The item APIs use kvecs to represent variable size item keys and + * values. + */ + +/* + * This ends up defining the max item size as nr - 1 * page _size. + */ +#define SCOUTFS_KVEC_NR 4 + +#define SCOUTFS_DECLARE_KVEC(name) \ + struct kvec name[SCOUTFS_KVEC_NR] + +static inline void scoutfs_kvec_init_all(struct kvec *kvec, + void *ptr0, size_t len0, + void *ptr1, size_t len1, + void *ptr2, size_t len2, + void *ptr3, size_t len3, ...) +{ + BUG_ON(ptr3 != NULL); + + kvec[0].iov_base = ptr0; + kvec[0].iov_len = len0; + kvec[1].iov_base = ptr1; + kvec[1].iov_len = len1; + kvec[2].iov_base = ptr2; + kvec[2].iov_len = len2; + kvec[3].iov_base = ptr3; + kvec[3].iov_len = len3; +} + +/* + * Provide a nice variadic initialization function without having to + * iterate over the callers arg types. We play some macro games to pad + * out the callers ptr/len pairs to the full possible number. This will + * produce confusing errors if an odd number of arguments is given and + * the padded ptr/length types aren't compatible with the fixed + * arguments in the static inline. + */ +#define scoutfs_kvec_init(val, ...) \ + scoutfs_kvec_init_all(val, __VA_ARGS__, NULL, 0, NULL, 0, NULL, 0) + +static inline int scoutfs_kvec_length(struct kvec *kvec) +{ + BUILD_BUG_ON(sizeof(struct kvec) != sizeof(struct iovec)); + BUILD_BUG_ON(offsetof(struct kvec, iov_len) != + offsetof(struct iovec, iov_len)); + BUILD_BUG_ON(member_sizeof(struct kvec, iov_len) != + member_sizeof(struct iovec, iov_len)); + + return iov_length((struct iovec *)kvec, SCOUTFS_KVEC_NR); +} + +void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src); +int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b); +int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, + struct kvec *c, struct kvec *d); +int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src); +int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src); +void scoutfs_kvec_kfree(struct kvec *kvec); + +#endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c new file mode 100644 index 00000000..a7ea9dc0 --- /dev/null +++ b/kmod/src/manifest.c @@ -0,0 +1,449 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "kvec.h" +#include "seg.h" +#include "item.h" +#include "manifest.h" + +struct manifest { + spinlock_t lock; + + struct list_head level0_list; + unsigned int level0_nr; + + u8 last_level; + struct rb_root level_roots[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; +}; + +#define DECLARE_MANIFEST(sb, name) \ + struct manifest *name = SCOUTFS_SB(sb)->manifest + +struct manifest_entry { + union { + struct list_head level0_entry; + struct rb_node node; + }; + + struct kvec *first; + struct kvec *last; + u64 segno; + u64 seq; + u8 level; +}; + +/* + * A path tracks all the segments from level 0 to the last level that + * overlap with the search key. + */ +struct manifest_ref { + u64 segno; + u64 seq; + struct scoutfs_segment *seg; + int pos; + u8 level; +}; + +static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) +{ + struct rb_node *node = root->rb_node; + struct manifest_entry *ment; + int cmp; + + while (node) { + ment = container_of(node, struct manifest_entry, node); + + cmp = scoutfs_kvec_cmp_overlap(key, key, + ment->first, ment->last); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return ment; + } + + return NULL; +} + +/* + * Insert a new entry into one of the L1+ trees. There should never be + * entries that overlap. + */ +static int insert_ment(struct rb_root *root, struct manifest_entry *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct manifest_entry *ment; + int cmp; + + while (*node) { + parent = *node; + ment = container_of(*node, struct manifest_entry, node); + + cmp = scoutfs_kvec_cmp_overlap(ins->first, ins->last, + ment->first, ment->last); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + return -EEXIST; + } + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + + return 0; +} + +static void free_ment(struct manifest_entry *ment) +{ + if (!IS_ERR_OR_NULL(ment)) { + scoutfs_kvec_kfree(ment->first); + scoutfs_kvec_kfree(ment->last); + kfree(ment); + } +} + +static int add_ment(struct manifest *mani, struct manifest_entry *ment) +{ + int ret; + + if (ment->level) { + ret = insert_ment(&mani->level_roots[ment->level], ment); + if (!ret) + mani->last_level = max(mani->last_level, ment->level); + } else { + list_add_tail(&ment->level0_entry, &mani->level0_list); + mani->level0_nr++; + ret = 0; + } + + return ret; +} + +static void update_last_level(struct manifest *mani) +{ + int i; + + for (i = mani->last_level; + i > 0 && RB_EMPTY_ROOT(&mani->level_roots[i]); i--) + ; + + mani->last_level = i; +} + +static void remove_ment(struct manifest *mani, struct manifest_entry *ment) +{ + if (ment->level) { + rb_erase(&ment->node, &mani->level_roots[ment->level]); + update_last_level(mani); + } else { + list_del_init(&ment->level0_entry); + mani->level0_nr--; + } +} + +int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, + struct kvec *last, u64 segno, u64 seq, u8 level) +{ + DECLARE_MANIFEST(sb, mani); + struct manifest_entry *ment; + unsigned long flags; + int ret; + + ment = kmalloc(sizeof(struct manifest_entry), GFP_NOFS); + if (!ment) + return -ENOMEM; + + ret = scoutfs_kvec_dup_flatten(ment->first, first) ?: + scoutfs_kvec_dup_flatten(ment->first, last); + if (ret) { + free_ment(ment); + return -ENOMEM; + } + + ment->segno = segno; + ment->seq = seq; + ment->level = level; + + /* XXX think about where to insert level 0 */ + spin_lock_irqsave(&mani->lock, flags); + ret = add_ment(mani, ment); + spin_unlock_irqrestore(&mani->lock, flags); + if (WARN_ON_ONCE(ret)) /* XXX can this happen? ring corruption? */ + free_ment(ment); + + return ret; +} + +static void set_ref(struct manifest_ref *ref, struct manifest_entry *mani) +{ + ref->segno = mani->segno; + ref->seq = mani->seq; + ref->level = mani->level; +} + +/* + * Returns refs if intersecting segments are found, NULL if none intersect, + * and PTR_ERR on failure. + */ +static struct manifest_ref *get_key_refs(struct manifest *mani, + struct kvec *key, + unsigned int *nr_ret) +{ + struct manifest_ref *refs = NULL; + struct manifest_entry *ment; + struct rb_root *root; + unsigned long flags; + unsigned int total; + unsigned int nr; + int i; + + spin_lock_irqsave(&mani->lock, flags); + + total = mani->level0_nr + mani->last_level; + while (nr != total) { + nr = total; + spin_unlock_irqrestore(&mani->lock, flags); + + kfree(refs); + refs = kcalloc(total, sizeof(struct manifest_ref), GFP_NOFS); + if (!refs) + return ERR_PTR(-ENOMEM); + + spin_lock_irqsave(&mani->lock, flags); + } + + nr = 0; + + list_for_each_entry(ment, &mani->level0_list, level0_entry) { + if (scoutfs_kvec_cmp_overlap(key, key, + ment->first, ment->last)) + continue; + + set_ref(&refs[nr++], ment); + } + + for (i = 1; i <= mani->last_level; i++) { + root = &mani->level_roots[i]; + if (RB_EMPTY_ROOT(root)) + continue; + + ment = find_ment(root, key); + if (ment) + set_ref(&refs[nr++], ment); + } + + spin_unlock_irqrestore(&mani->lock, flags); + + *nr_ret = nr; + if (!nr) { + kfree(refs); + refs = NULL; + } + + return refs; +} + +/* + * The caller didn't find an item for the given key in the item cache + * and wants us to search for it in the lsm segments. We search the + * manifest for all the segments that contain the key. We then read the + * segments and iterate over their items looking for ours. We insert it + * and some number of other surrounding items to amortize the relatively + * expensive multi-segment searches. + * + * This is asking the seg code to read each entire segment. The seg + * code could give it it helpers to submit and wait on blocks within the + * segment so that we don't have wild bandwidth amplification in the + * cold random read case. + * + * The segments are immutable at this point so we can use their contents + * as long as we hold refs. + */ +int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) +{ + DECLARE_MANIFEST(sb, mani); + SCOUTFS_DECLARE_KVEC(item_key); + SCOUTFS_DECLARE_KVEC(item_val); + SCOUTFS_DECLARE_KVEC(found_key); + SCOUTFS_DECLARE_KVEC(found_val); + struct scoutfs_segment *seg; + struct manifest_ref *refs; + unsigned long had_found; + bool found; + int ret = 0; + int err; + int nr_refs; + int cmp; + int i; + int n; + + refs = get_key_refs(mani, key, &nr_refs); + if (IS_ERR(refs)) + return PTR_ERR(refs); + if (!refs) + return -ENOENT; + + /* submit reads for all the segments */ + for (i = 0; i < nr_refs; i++) { + seg = scoutfs_seg_submit_read(sb, refs[i].segno); + if (IS_ERR(seg)) { + ret = PTR_ERR(seg); + break; + } + + refs[i].seg = seg; + } + + /* wait for submitted segments and search if we haven't seen failure */ + for (n = 0; n < i; n++) { + seg = refs[i].seg; + + err = scoutfs_seg_wait(sb, seg); + if (err && !ret) + ret = err; + + if (!ret) + refs[i].pos = scoutfs_seg_find_pos(seg, key); + } + + /* done if we saw errors */ + if (ret) + goto out; + + /* walk sorted items, resolving across segments, and insert */ + for (n = 0; n < 16; n++) { + + found = false; + + /* find the most recent least key */ + for (i = 0; i < nr_refs; i++) { + seg = refs[i].seg; + if (!seg) + continue; + + /* get kvecs, removing if we ran out of items */ + ret = scoutfs_seg_item_kvecs(seg, refs[i].pos, + item_key, item_val); + if (ret < 0) { + scoutfs_seg_put(seg); + refs[i].seg = NULL; + continue; + } + + if (found) { + cmp = scoutfs_kvec_memcmp(item_key, found_key); + if (cmp >= 0) { + if (cmp == 0) + set_bit(i, &had_found); + continue; + } + } + + /* remember new least key */ + scoutfs_kvec_clone(found_key, key); + scoutfs_kvec_clone(found_val, item_val); + found = true; + had_found = 0; + set_bit(i, &had_found); + } + + /* return -ENOENT if we didn't find any or the callers item */ + if (n == 0 && + (!found || scoutfs_kvec_memcmp(key, found_key))) { + ret = -ENOENT; + break; + } + + if (!found) { + ret = 0; + break; + } + + ret = scoutfs_item_insert(sb, item_key, item_val); + if (ret) + break; + + /* advance all the positions past the found key */ + for_each_set_bit(i, &had_found, BITS_PER_LONG) + refs[i].pos++; + } + +out: + for (i = 0; i < nr_refs; i++) + scoutfs_seg_put(refs[i].seg); + + kfree(refs); + return ret; +} + +int scoutfs_manifest_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct manifest *mani; + int i; + + mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); + if (!mani) + return -ENOMEM; + sbi->manifest = mani; + + spin_lock_init(&mani->lock); + INIT_LIST_HEAD(&mani->level0_list); + for (i = 0; i < ARRAY_SIZE(mani->level_roots); i++) + mani->level_roots[i] = RB_ROOT; + + return 0; +} + +void scoutfs_manifest_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct manifest *mani = sbi->manifest; + struct manifest_entry *ment; + struct manifest_entry *tmp; + struct rb_node *node; + struct rb_root *root; + int i; + + if (!mani) + return; + + for (i = 1; i <= mani->last_level; i++) { + root = &mani->level_roots[i]; + + for (node = rb_first(root); node; ) { + ment = container_of(node, struct manifest_entry, node); + node = rb_next(node); + remove_ment(mani, ment); + free_ment(ment); + } + } + + list_for_each_entry_safe(ment, tmp, &mani->level0_list, level0_entry) { + remove_ment(mani, ment); + free_ment(ment); + } + + kfree(mani); +} diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h new file mode 100644 index 00000000..c1ea0160 --- /dev/null +++ b/kmod/src/manifest.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_MANIFEST_H_ +#define _SCOUTFS_MANIFEST_H_ + +int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, + struct kvec *last, u64 segno, u64 seq, u8 level); +int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key); + +int scoutfs_manifest_setup(struct super_block *sb); +void scoutfs_manifest_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/ring.c b/kmod/src/ring.c new file mode 100644 index 00000000..865071aa --- /dev/null +++ b/kmod/src/ring.c @@ -0,0 +1,263 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "kvec.h" +#include "bio.h" +#include "manifest.h" +#include "ring.h" + +/* + * OK, log: + * - big preallocated ring of variable length entries + * - entries are rounded to 4k blocks + * - entire thing is read and indexed in rbtree + * - static allocated page is kept around to record and write entries + * - indexes have cursor that points to next node to migrate + * - any time an entry is written an entry is migrated + * - allocate room for 4x (maybe including worst case rounding) + * - mount does binary search looking for newest entry + * - newest entry describes block where we started migrating + * - replay then walks from oldest to newest replaying + * - entries are marked with migration so we know where to set cursor after + * + * XXX + * - verify blocks + * - could compress + */ + +/* read in a meg at a time */ +#define NR_PAGES DIV_ROUND_UP(1024 * 1024, PAGE_SIZE) +#define NR_BLOCKS (NR_PAGES * SCOUTFS_BLOCKS_PER_PAGE) + +#if 0 +#define BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) +static void read_page_end_io(struct bio *bio, int err) +{ + struct bio_vec *bvec; + struct page *page; + unsigned long i; + + for_each_bio_segment(bio, bvec, i) { + page = bvec->bv_page; + + if (err) + SetPageError(page); + else + SetPageUptodate(page); + unlock_page(page); + } + + bio_put(bio); +} + +/* + * Read the given number of 4k blocks into the pages provided by the + * caller. We translate the block count into a page count and fill + * bios a page at a time. + */ +static int read_blocks(struct super_block *sb, struct page **pages, + u64 blkno, unsigned int nr_blocks) +{ + unsigned int nr_pages = DIV_ROUND_UP(nr_blocks, PAGES_PER_BLOCK); + unsigned int bytes; + struct bio *bio; + int ret = 0; + + for (i = 0; i < nr_pages; i++) { + page = pages[i]; + + if (!bio) { + bio = bio_alloc(GFP_NOFS, nr_pages - i); + if (!bio) + bio = bio_alloc(GFP_NOFS, 1); + if (!bio) { + ret = -ENOMEM; + break; + } + + bio->bi_sector = blkno << (SCOUTFS_BLOCK_SHIFT - 9); + bio->bi_bdev = sb->s_bdev; + bio->bi_end_io = read_pages_end_io; + } + + lock_page(page); + ClearPageError(page); + ClearPageUptodate(page); + + bytes = min(nr_blocks << SCOUTFS_BLOCK_SHIFT, PAGE_SIZE); + + if (bio_add_page(bio, page, bytes, 0) != bytes) { + /* submit the full bio and retry this page */ + submit_bio(READ, bio); + bio = NULL; + unlock_page(page); + i--; + continue; + } + + blkno += BLOCKS_PER_PAGE; + nr_blocks -= BLOCKS_PER_PAGE; + } + + if (bio) + submit_bio(READ, bio); + + for (i = 0; i < nr_pages; i++) { + page = pages[i]; + + wait_on_page_locked(page); + if (!ret && (!PageUptodate(page) || PageError(page))) + ret = -EIO; + } + + return ret; +} +#endif + + +static int read_one_entry(struct super_block *sb, + struct scoutfs_ring_entry_header *eh) +{ + struct scoutfs_ring_add_manifest *am; + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); + int ret; + + switch(eh->type) { + case SCOUTFS_RING_ADD_MANIFEST: + am = container_of(eh, struct scoutfs_ring_add_manifest, eh); + + scoutfs_kvec_init(first, am + 1, + le16_to_cpu(am->first_key_len)); + scoutfs_kvec_init(last, + first[0].iov_base + first[0].iov_len, + le16_to_cpu(am->last_key_len)); + + ret = scoutfs_manifest_add(sb, first, last, + le64_to_cpu(am->segno), + le64_to_cpu(am->seq), am->level); + break; + + default: + ret = -EINVAL; + } + + return ret; +} + +static int read_entries(struct super_block *sb, + struct scoutfs_ring_block *ring) +{ + struct scoutfs_ring_entry_header *eh; + int ret = 0; + int i; + + eh = ring->entries; + + for (i = 0; i < le32_to_cpu(ring->nr_entries); i++) { + ret = read_one_entry(sb, eh); + if (ret) + break; + + eh = (void *)eh + le16_to_cpu(eh->len); + } + + return ret; +} + +#if 0 +/* return pointer to the blk 4k block offset amongst the pages */ +static void *page_block_address(struct page **pages, unsigned int blk) +{ + unsigned int i = blk / BLOCKS_PER_PAGE; + unsigned int off = (blk % BLOCKS_PER_PAGE) << SCOUTFS_BLOCK_SHIFT; + + return page_address(pages[i]) + off; +} +#endif + +int scoutfs_ring_read(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ring_block *ring; + struct page **pages; + struct page *page; + u64 index; + u64 blkno; + u64 tail; + u64 seq; + int ret; + int nr; + int i; + + /* nr_blocks/pages calc doesn't handle multiple pages per block */ + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SIZE); + + pages = kcalloc(NR_PAGES, sizeof(struct page *), GFP_NOFS); + if (!pages) + return -ENOMEM; + + for (i = 0; i < NR_PAGES; i++) { + page = alloc_page(GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + + pages[i] = page; + } + + index = le64_to_cpu(super->ring_head_index); + tail = le64_to_cpu(super->ring_tail_index); + seq = le64_to_cpu(super->ring_head_seq); + + do { + blkno = le64_to_cpu(super->ring_blkno) + index; + + if (index <= tail) + nr = tail - index + 1; + else + nr = le64_to_cpu(super->ring_blocks) - index; + nr = min_t(int, nr, NR_BLOCKS); + + ret = scoutfs_bio_read(sb, pages, index, nr); + if (ret) + goto out; + + /* XXX verify block header */ + + for (i = 0; i < nr; i++) { + ring = scoutfs_page_block_address(pages, i); + ret = read_entries(sb, ring); + if (ret) + goto out; + } + + index += nr; + if (index == le64_to_cpu(super->ring_blocks)) + index = 0; + } while (index != tail); + +out: + for (i = 0; i < NR_PAGES && pages && pages[i]; i++) + __free_page(pages[i]); + kfree(pages); + + return ret; +} diff --git a/kmod/src/ring.h b/kmod/src/ring.h new file mode 100644 index 00000000..4f6930c9 --- /dev/null +++ b/kmod/src/ring.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_RING_H_ +#define _SCOUTFS_RING_H_ + +#include + +int scoutfs_ring_read(struct super_block *sb); + +#endif diff --git a/kmod/src/seg.c b/kmod/src/seg.c new file mode 100644 index 00000000..9f884845 --- /dev/null +++ b/kmod/src/seg.c @@ -0,0 +1,399 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "seg.h" +#include "bio.h" +#include "kvec.h" + +/* + * seg.c should just be about the cache and io, and maybe + * iteration and stuff. + * + * XXX: + * - lru and shrinker + * - verify csum + * - make sure item headers don't cross page boundaries + * - just wait on pages instead of weird flags? + */ + +struct segment_cache { + spinlock_t lock; + struct rb_root root; + wait_queue_head_t waitq; +}; + +struct scoutfs_segment { + struct rb_node node; + atomic_t refcount; + u64 segno; + unsigned long flags; + int err; + struct page *pages[SCOUTFS_SEGMENT_PAGES]; +}; + +enum { + SF_END_IO = 0, +}; + +static struct scoutfs_segment *alloc_seg(u64 segno) +{ + struct scoutfs_segment *seg; + struct page *page; + int i; + + /* don't waste the tail of pages */ + BUILD_BUG_ON(SCOUTFS_SEGMENT_SIZE % PAGE_SIZE); + + seg = kzalloc(sizeof(struct scoutfs_segment), GFP_NOFS); + if (!seg) + return seg; + + RB_CLEAR_NODE(&seg->node); + atomic_set(&seg->refcount, 1); + seg->segno = segno; + + for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) { + page = alloc_page(GFP_NOFS); + if (!page) { + scoutfs_seg_put(seg); + return ERR_PTR(-ENOMEM); + } + + seg->pages[i] = page; + } + + return seg; +} + +void scoutfs_seg_put(struct scoutfs_segment *seg) +{ + int i; + + if (!IS_ERR_OR_NULL(seg) && atomic_dec_and_test(&seg->refcount)) { + WARN_ON_ONCE(!RB_EMPTY_NODE(&seg->node)); + for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) + if (seg->pages[i]) + __free_page(seg->pages[i]); + kfree(seg); + } +} + +static int cmp_u64s(u64 a, u64 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) +{ + struct rb_node *node = root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_segment *seg; + int cmp; + + while (node) { + parent = node; + seg = container_of(node, struct scoutfs_segment, node); + + cmp = cmp_u64s(segno, seg->segno); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return seg; + } + + return NULL; +} + +/* + * This always inserts the segment into the rbtree. If there's already + * a segment at the given seg then it is removed and returned. The caller + * doesn't have to erase it from the tree if it's returned. + */ +static struct scoutfs_segment *replace_seg(struct rb_root *root, + struct scoutfs_segment *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_segment *seg; + struct scoutfs_segment *found = NULL; + int cmp; + + while (*node) { + parent = *node; + seg = container_of(*node, struct scoutfs_segment, node); + + cmp = cmp_u64s(ins->segno, seg->segno); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + rb_replace_node(&seg->node, &ins->node, root); + found = seg; + break; + } + } + + if (!found) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + } + + return found; +} + +static bool erase_seg(struct rb_root *root, struct scoutfs_segment *seg) +{ + if (!RB_EMPTY_NODE(&seg->node)) { + rb_erase(&seg->node, root); + RB_CLEAR_NODE(&seg->node); + return true; + } + + return false; +} + +static void seg_end_io(struct super_block *sb, void *data, int err) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac = sbi->segment_cache; + struct scoutfs_segment *seg = data; + unsigned long flags; + bool erased; + + if (err) { + seg->err = err; + + spin_lock_irqsave(&cac->lock, flags); + erased = erase_seg(&cac->root, seg); + spin_unlock_irqrestore(&cac->lock, flags); + if (erased) + scoutfs_seg_put(seg); + } + + set_bit(SF_END_IO, &seg->flags); + smp_mb__after_atomic(); + if (waitqueue_active(&cac->waitq)) + wake_up(&cac->waitq); + + scoutfs_seg_put(seg); +} + +static u64 segno_to_blkno(u64 blkno) +{ + return blkno << (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT); +} + +/* + * The bios submitted by this don't have page references themselves. If + * this succeeds then the caller must call _wait before putting their + * seg ref. + */ +struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, + u64 segno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac = sbi->segment_cache; + struct scoutfs_segment *existing; + struct scoutfs_segment *seg; + unsigned long flags; + + spin_lock_irqsave(&cac->lock, flags); + seg = find_seg(&cac->root, segno); + if (seg) + atomic_inc(&seg->refcount); + spin_unlock_irqrestore(&cac->lock, flags); + if (seg) + return seg; + + seg = alloc_seg(segno); + if (IS_ERR(seg)) + return seg; + + /* always drop existing segs, could compare seqs */ + spin_lock_irqsave(&cac->lock, flags); + atomic_inc(&seg->refcount); + existing = replace_seg(&cac->root, seg); + spin_unlock_irqrestore(&cac->lock, flags); + if (existing) + scoutfs_seg_put(existing); + + atomic_inc(&seg->refcount); + scoutfs_bio_submit(sb, READ, seg->pages, segno_to_blkno(seg->segno), + SCOUTFS_SEGMENT_BLOCKS, seg_end_io, seg); + + return seg; +} + +int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac = sbi->segment_cache; + int ret; + + ret = wait_event_interruptible(cac->waitq, + test_bit(SF_END_IO, &seg->flags)); + if (!ret) + ret = seg->err; + + return ret; +} + +static void *off_ptr(struct scoutfs_segment *seg, u32 off) +{ + unsigned int pg = off >> PAGE_SHIFT; + unsigned int pg_off = off & ~PAGE_MASK; + + return page_address(seg->pages[pg]) + pg_off; +} + +/* + * Return a pointer to the item in the array at the given position. + * + * The item structs fill the first block in the segment after the + * initial segment block struct. Item structs don't cross block + * boundaries so the final bytes that would make up a partial item + * struct are skipped. + */ +static struct scoutfs_segment_item *pos_item(struct scoutfs_segment *seg, + int pos) +{ + u32 off; + + if (pos < SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS) { + off = sizeof(struct scoutfs_segment_block); + } else { + pos -= SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS; + off = (1 + (pos / SCOUTFS_SEGMENT_ITEMS_PER_BLOCK)) * + SCOUTFS_BLOCK_SIZE; + pos %= SCOUTFS_SEGMENT_ITEMS_PER_BLOCK; + } + + return off_ptr(seg, off + (pos * sizeof(struct scoutfs_segment_item))); +} + +static void kvec_from_pages(struct scoutfs_segment *seg, + struct kvec *kvec, u32 off, u16 len) +{ + u32 first; + + first = min_t(int, len, PAGE_SIZE - (off & ~PAGE_MASK)); + + if (first == len) + scoutfs_kvec_init(kvec, off_ptr(seg, off), len); + else + scoutfs_kvec_init(kvec, off_ptr(seg, off), first, + off_ptr(seg, off + first), len - first); +} + +int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, + struct kvec *key, struct kvec *val) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct scoutfs_segment_item *item; + + if (pos < 0 || pos >= le32_to_cpu(sblk->nr_items)) + return -ENOENT; + + item = pos_item(seg, pos); + + if (key) + kvec_from_pages(seg, key, le32_to_cpu(item->key_off), + le16_to_cpu(item->key_len)); + if (val) + kvec_from_pages(seg, val, le32_to_cpu(item->val_off), + le16_to_cpu(item->val_len)); + + return 0; +} + +/* + * Find the first item array position whose key is >= the search key. + * This can return the number of positions if the key is greater than + * all the keys. + */ +static int find_key_pos(struct scoutfs_segment *seg, struct kvec *search) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + SCOUTFS_DECLARE_KVEC(key); + unsigned int start = 0; + unsigned int end = le32_to_cpu(sblk->nr_items); + unsigned int pos = 0; + int cmp; + + while (start < end) { + pos = start + (end - start) / 2; + scoutfs_seg_item_kvecs(seg, pos, key, NULL); + + cmp = scoutfs_kvec_memcmp(search, key); + if (cmp < 0) + end = pos; + else if (cmp > 0) + start = ++pos; + else + break; + } + + return pos; +} + +int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key) +{ + return find_key_pos(seg, key); +} + +int scoutfs_seg_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac; + + cac = kzalloc(sizeof(struct segment_cache), GFP_KERNEL); + if (!cac) + return -ENOMEM; + sbi->segment_cache = cac; + + spin_lock_init(&cac->lock); + cac->root = RB_ROOT; + init_waitqueue_head(&cac->waitq); + + return 0; +} + +void scoutfs_seg_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac = sbi->segment_cache; + struct scoutfs_segment *seg; + struct rb_node *node; + + if (cac) { + for (node = rb_first(&cac->root); node; ) { + seg = container_of(node, struct scoutfs_segment, node); + node = rb_next(node); + erase_seg(&cac->root, seg); + scoutfs_seg_put(seg); + } + + kfree(cac); + } +} + diff --git a/kmod/src/seg.h b/kmod/src/seg.h new file mode 100644 index 00000000..1957a308 --- /dev/null +++ b/kmod/src/seg.h @@ -0,0 +1,20 @@ +#ifndef _SCOUTFS_SEG_H_ +#define _SCOUTFS_SEG_H_ + +struct scoutfs_segment; +struct kvec; + +struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, + u64 segno); +int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); + +int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key); +int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, + struct kvec *key, struct kvec *val); + +void scoutfs_seg_put(struct scoutfs_segment *seg); + +int scoutfs_seg_setup(struct super_block *sb); +void scoutfs_seg_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index ca085815..4fdcb1f5 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -28,6 +28,10 @@ #include "counters.h" #include "trans.h" #include "buddy.h" +#include "ring.h" +#include "item.h" +#include "manifest.h" +#include "seg.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -212,7 +216,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: read_supers(sb) ?: - scoutfs_buddy_setup(sb) ?: + scoutfs_seg_setup(sb) ?: + scoutfs_manifest_setup(sb) ?: + scoutfs_item_setup(sb) ?: + scoutfs_ring_read(sb) ?: +// scoutfs_buddy_setup(sb) ?: scoutfs_setup_trans(sb); if (ret) return ret; @@ -227,7 +235,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; - scoutfs_scan_orphans(sb); +// scoutfs_scan_orphans(sb); return 0; } @@ -248,6 +256,9 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_buddy_destroy(sb); if (sbi->block_shrinker.shrink == scoutfs_block_shrink) unregister_shrinker(&sbi->block_shrinker); + scoutfs_item_destroy(sb); + scoutfs_manifest_destroy(sb); + scoutfs_seg_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index 25e6eec1..b1b20e97 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -9,6 +9,9 @@ struct scoutfs_counters; struct buddy_info; +struct item_cache; +struct manifest; +struct segment_cache; struct scoutfs_sb_info { struct super_block *sb; @@ -28,6 +31,10 @@ struct scoutfs_sb_info { struct list_head block_lru_list; unsigned long block_lru_nr; + struct manifest *manifest; + struct item_cache *item_cache; + struct segment_cache *segment_cache; + struct buddy_info *buddy_info; struct rw_semaphore btree_rwsem; From f7f840a3423d477e8d109f303fbd1e5623e4318a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 18:08:43 -0800 Subject: [PATCH 143/920] Fix bio read completion init Signed-off-by: Zach Brown --- kmod/src/bio.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index d58eebe9..d1ed293d 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -148,10 +148,9 @@ static void end_io_complete(struct super_block *sb, void *data, int err) int scoutfs_bio_read(struct super_block *sb, struct page **pages, u64 blkno, unsigned int nr_blocks) { - struct end_io_completion comp = { - .comp = COMPLETION_INITIALIZER(comp.comp), - }; + struct end_io_completion comp; + init_completion(&comp.comp); scoutfs_bio_submit(sb, READ, pages, blkno, nr_blocks, end_io_complete, &comp); wait_for_completion(&comp.comp); From a201cff5adba7dcf05d61a0ddfee3cd3c3ce6318 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 18:09:06 -0800 Subject: [PATCH 144/920] Read supers with bios instead of bl blocks Signed-off-by: Zach Brown --- kmod/src/super.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 4fdcb1f5..7866185a 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -32,6 +32,7 @@ #include "item.h" #include "manifest.h" #include "seg.h" +#include "bio.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -133,18 +134,24 @@ static int read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; - struct scoutfs_block *bl = NULL; + struct page *page; int found = -1; + int ret; int i; + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + for (i = 0; i < SCOUTFS_SUPER_NR; i++) { - scoutfs_block_put(bl); - bl = scoutfs_block_read(sb, SCOUTFS_SUPER_BLKNO + i); - if (IS_ERR(bl)) { + + ret = scoutfs_bio_read(sb, &page, SCOUTFS_SUPER_BLKNO + i, 1); + if (ret) { scoutfs_warn(sb, "couldn't read super block %u", i); continue; } - super = scoutfs_block_data(bl); + + super = scoutfs_page_block_address(&page, 0); if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { scoutfs_warn(sb, "super block %u has invalid id %llx", @@ -159,7 +166,7 @@ static int read_supers(struct super_block *sb) } } - scoutfs_block_put(bl); + __free_page(page); if (found < 0) { scoutfs_err(sb, "unable to read valid super block"); From 3f27de0b2cb4cb0a0138322f2434befc5079fc4d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:16:37 -0800 Subject: [PATCH 145/920] Fix hilarious BLOCK_SIZE typo Turns out BLOCK_SIZE is a thing and confused scoutfs into thinking it had many more blocks per segment then it did. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index d0efbae4..ff1b69dd 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -22,7 +22,7 @@ #define SCOUTFS_SEGMENT_SIZE (1 << SCOUTFS_SEGMENT_SHIFT) #define SCOUTFS_SEGMENT_MASK (SCOUTFS_SEGMENT_SIZE - 1) #define SCOUTFS_SEGMENT_PAGES (SCOUTFS_SEGMENT_SIZE / PAGE_SIZE) -#define SCOUTFS_SEGMENT_BLOCKS (SCOUTFS_SEGMENT_SIZE / BLOCK_SIZE) +#define SCOUTFS_SEGMENT_BLOCKS (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) #define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) #define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) From 6957c73aba658ab5d9b1547a0609f8a0d5eae1f7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:17:20 -0800 Subject: [PATCH 146/920] Have _lookup_exact return 0 scoutfs_item_lookup_exact() exists to only return one size. Have it just return 0 on success so callers don't have to remember that it returns > 0. Signed-off-by: Zach Brown --- kmod/src/item.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 73e9665b..7525a6ba 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -126,6 +126,8 @@ int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, * match. This isn't the fast path so we don't mind the copying * overhead that comes from only detecting the size mismatch after the * copy by reusing the more permissive _lookup(). + * + * Returns 0 or -errno. */ int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, struct kvec *val, int size) @@ -133,7 +135,9 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, int ret; ret = scoutfs_item_lookup(sb, key, val); - if (ret >= 0 && ret != size) + if (ret == size) + ret = 0; + else if (ret >= 0 && ret != size) ret = -EIO; return ret; From f3288f27c6df0d6b6eef44e5e063f0ddf3974c33 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:18:06 -0800 Subject: [PATCH 147/920] Declare full kvecs in manifest The manifest had silly single kvecs instead of the macros that define our maximal kvecs. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index a7ea9dc0..555b1c3b 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -41,8 +41,8 @@ struct manifest_entry { struct rb_node node; }; - struct kvec *first; - struct kvec *last; + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); u64 segno; u64 seq; u8 level; From d1f36e2165b00fd896c1240e6a767d19cae7f4f0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:20:10 -0800 Subject: [PATCH 148/920] Correctly store last manifest key A copy+paste error led us to overwrite the first key in the manifest with the last, leaving the last uninitialized. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 555b1c3b..2542476c 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -175,7 +175,7 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, return -ENOMEM; ret = scoutfs_kvec_dup_flatten(ment->first, first) ?: - scoutfs_kvec_dup_flatten(ment->first, last); + scoutfs_kvec_dup_flatten(ment->last, last); if (ret) { free_ment(ment); return -ENOMEM; From 21d313e0f66c512e490852b75d94e84f00e7ec1d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:21:20 -0800 Subject: [PATCH 149/920] Corretly wait on submitted segment reads The segment waiting loop was rewritten to use n to iterate up to i, but the body of the loop still had i. Take that as a signal to Always Iterate With 'i' and store the last i and then iterate towards it with i again. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2542476c..b927a142 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -295,6 +295,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) int err; int nr_refs; int cmp; + int last; int i; int n; @@ -314,9 +315,10 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) refs[i].seg = seg; } + last = i; /* wait for submitted segments and search if we haven't seen failure */ - for (n = 0; n < i; n++) { + for (i = 0; i < last; i++) { seg = refs[i].seg; err = scoutfs_seg_wait(sb, seg); From 641aae50ed844488a59630222583d7fea3b6a649 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:23:40 -0800 Subject: [PATCH 150/920] Fix ring block replay The ring block replay walk messed up the blkno it read from and its exit condition. It needed to test for having just replayed the tail before moving on. Signed-off-by: Zach Brown --- kmod/src/ring.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 865071aa..04729eaa 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -227,7 +227,7 @@ int scoutfs_ring_read(struct super_block *sb) tail = le64_to_cpu(super->ring_tail_index); seq = le64_to_cpu(super->ring_head_seq); - do { + for(;;) { blkno = le64_to_cpu(super->ring_blkno) + index; if (index <= tail) @@ -236,7 +236,7 @@ int scoutfs_ring_read(struct super_block *sb) nr = le64_to_cpu(super->ring_blocks) - index; nr = min_t(int, nr, NR_BLOCKS); - ret = scoutfs_bio_read(sb, pages, index, nr); + ret = scoutfs_bio_read(sb, pages, blkno, nr); if (ret) goto out; @@ -249,10 +249,13 @@ int scoutfs_ring_read(struct super_block *sb) goto out; } + if (index == tail) + break; + index += nr; if (index == le64_to_cpu(super->ring_blocks)) index = 0; - } while (index != tail); + } out: for (i = 0; i < NR_PAGES && pages && pages[i]; i++) From b45ec8824b960900780825ffb436c05ce1bc967d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 3 Dec 2016 19:24:29 -0800 Subject: [PATCH 151/920] Add a bunch of trace_printk()s There's nothing particularly coherent about these, they're what I added while debugging. Signed-off-by: Zach Brown --- kmod/src/bio.c | 4 ++++ kmod/src/item.c | 2 ++ kmod/src/manifest.c | 11 +++++++++++ kmod/src/ring.c | 10 ++++++++++ kmod/src/seg.c | 4 ++++ 5 files changed, 31 insertions(+) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index d1ed293d..fe41a689 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -42,6 +42,8 @@ static void bio_end_io(struct bio *bio, int err) { struct bio_end_io_args *args = bio->bi_private; + trace_printk("bio %p end io\n", bio); + dec_end_io(args, bio->bi_size, err); bio_put(bio); } @@ -114,6 +116,8 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, continue; } + trace_printk("added page %p to bio %p\n", page, bio); + blkno += SCOUTFS_BLOCKS_PER_PAGE; nr_blocks -= SCOUTFS_BLOCKS_PER_PAGE; } diff --git a/kmod/src/item.c b/kmod/src/item.c index 7525a6ba..f65388ec 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -116,6 +116,8 @@ int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + trace_printk("ret %d\n", ret); + return ret; } diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index b927a142..c6b1a33f 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -127,6 +127,8 @@ static int add_ment(struct manifest *mani, struct manifest_entry *ment) { int ret; + trace_printk("adding ment %p level %u\n", ment, ment->level); + if (ment->level) { ret = insert_ment(&mani->level_roots[ment->level], ment); if (!ret) @@ -218,6 +220,8 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, unsigned int nr; int i; + trace_printk("getting refs\n"); + spin_lock_irqsave(&mani->lock, flags); total = mani->level0_nr + mani->last_level; @@ -227,6 +231,7 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, kfree(refs); refs = kcalloc(total, sizeof(struct manifest_ref), GFP_NOFS); + trace_printk("alloc refs %p total %u\n", refs, total); if (!refs) return ERR_PTR(-ENOMEM); @@ -236,6 +241,7 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, nr = 0; list_for_each_entry(ment, &mani->level0_list, level0_entry) { + trace_printk("trying l0 ment %p\n", ment); if (scoutfs_kvec_cmp_overlap(key, key, ment->first, ment->last)) continue; @@ -261,6 +267,9 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, refs = NULL; } + trace_printk("refs %p (err %ld)\n", + refs, IS_ERR(refs) ? PTR_ERR(refs) : 0); + return refs; } @@ -299,6 +308,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) int i; int n; + trace_printk("reading items\n"); + refs = get_key_refs(mani, key, &nr_refs); if (IS_ERR(refs)) return PTR_ERR(refs); diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 04729eaa..867acd3b 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -138,10 +138,16 @@ static int read_one_entry(struct super_block *sb, SCOUTFS_DECLARE_KVEC(last); int ret; + trace_printk("type %u len %u\n", eh->type, le16_to_cpu(eh->len)); + switch(eh->type) { case SCOUTFS_RING_ADD_MANIFEST: am = container_of(eh, struct scoutfs_ring_add_manifest, eh); + trace_printk("lens %u %u\n", + le16_to_cpu(am->first_key_len), + le16_to_cpu(am->last_key_len)); + scoutfs_kvec_init(first, am + 1, le16_to_cpu(am->first_key_len)); scoutfs_kvec_init(last, @@ -167,6 +173,8 @@ static int read_entries(struct super_block *sb, int ret = 0; int i; + trace_printk("reading %u entries\n", le32_to_cpu(ring->nr_entries)); + eh = ring->entries; for (i = 0; i < le32_to_cpu(ring->nr_entries); i++) { @@ -236,6 +244,8 @@ int scoutfs_ring_read(struct super_block *sb) nr = le64_to_cpu(super->ring_blocks) - index; nr = min_t(int, nr, NR_BLOCKS); + trace_printk("index %llu tail %llu nr %u\n", index, tail, nr); + ret = scoutfs_bio_read(sb, pages, blkno, nr); if (ret) goto out; diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 9f884845..4537c50c 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -71,6 +71,8 @@ static struct scoutfs_segment *alloc_seg(u64 segno) for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) { page = alloc_page(GFP_NOFS); + trace_printk("seg %p segno %llu page %u %p\n", + seg, segno, i, page); if (!page) { scoutfs_seg_put(seg); return ERR_PTR(-ENOMEM); @@ -217,6 +219,8 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, struct scoutfs_segment *seg; unsigned long flags; + trace_printk("segno %llu\n", segno); + spin_lock_irqsave(&cac->lock, flags); seg = find_seg(&cac->root, segno); if (seg) From c4954eb6f460f63443bead1617e7e13aacc70526 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Dec 2016 09:38:31 -0800 Subject: [PATCH 152/920] Add initial LSM write implementation Add all the core strutural components to be able to modify metadata. We modify items in fs write operations, track dirty items in the cache, allocate free segment block reagions, stream dirty items into segments, write out the segments, update the manifest to reference the written segments, and write out a new ring that has the new manifest. Signed-off-by: Zach Brown --- kmod/src/Makefile | 6 +- kmod/src/alloc.c | 334 ++++++++++++++++++++++++++++++ kmod/src/alloc.h | 16 ++ kmod/src/bio.c | 51 +++-- kmod/src/bio.h | 18 ++ kmod/src/dir.c | 372 ++++++++++------------------------ kmod/src/dir.h | 3 - kmod/src/format.h | 80 ++++++-- kmod/src/inode.c | 44 ++-- kmod/src/item.c | 482 ++++++++++++++++++++++++++++++++++++++++++-- kmod/src/item.h | 15 ++ kmod/src/kvec.c | 36 ++++ kmod/src/kvec.h | 3 + kmod/src/manifest.c | 146 ++++++++++---- kmod/src/manifest.h | 6 +- kmod/src/ring.c | 245 +++++++++++++--------- kmod/src/ring.h | 10 + kmod/src/seg.c | 232 ++++++++++++++++++--- kmod/src/seg.h | 15 ++ kmod/src/super.c | 7 +- kmod/src/super.h | 3 + kmod/src/trans.c | 77 ++++--- 22 files changed, 1669 insertions(+), 532 deletions(-) create mode 100644 kmod/src/alloc.c create mode 100644 kmod/src/alloc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index d808143b..bffe9ec6 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += bio.o block.o btree.o buddy.o counters.o crc.o dir.o filerw.o \ - kvec.o inode.o ioctl.o item.o manifest.o msg.o name.o ring.o \ - seg.o scoutfs_trace.o super.o trans.o xattr.o +scoutfs-y += alloc.o bio.o block.o btree.o buddy.o counters.o crc.o dir.o \ + filerw.o kvec.o inode.o ioctl.o item.o manifest.o msg.o name.o \ + ring.o seg.o scoutfs_trace.o super.o trans.o xattr.o diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c new file mode 100644 index 00000000..dbacf288 --- /dev/null +++ b/kmod/src/alloc.c @@ -0,0 +1,334 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "ring.h" +#include "alloc.h" + +/* + * scoutfs allocates segments by storing regions of a bitmap in a radix. + * As the regions are modified their index in the radix is marked dirty + * for writeout. + * + * Frees are tracked in a separate radix. They're only applied to the + * free regions as a transaction is written. The frees can't satisfy + * allocation until they're committed so that we don't overwrite stable + * referenced data. + * + * The allocated segments are large enough to be effectively + * independent. We allocate by sweeping a cursor through the volume. + * This gives racing unlocked readers more time to try to sample a stale + * freed segment, when its safe to do so, before it is reallocated and + * rewritten and they're forced to retry their racey read. + * + * XXX + * - make sure seg fits in long index + * - frees can delete region, leave non-NULL nul behind for logging + */ + +struct seg_alloc { + spinlock_t lock; + struct radix_tree_root regs; + struct radix_tree_root pending; + u64 next_segno; +}; + +#define DECLARE_SEG_ALLOC(sb, name) \ + struct seg_alloc *name = SCOUTFS_SB(sb)->seg_alloc + +enum { + DIRTY_RADIX_TAG = 0, +}; + +int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ring_alloc_region *reg; + DECLARE_SEG_ALLOC(sb, sal); + unsigned long flags; + unsigned long ind; + int ret; + int nr; + + spin_lock_irqsave(&sal->lock, flags); + + /* start by sweeping through the device for the first time */ + if (sal->next_segno == le64_to_cpu(super->alloc_uninit)) { + le64_add_cpu(&super->alloc_uninit, 1); + *segno = sal->next_segno++; + if (sal->next_segno == le64_to_cpu(super->total_segs)) + sal->next_segno = 0; + ret = 0; + goto out; + } + + /* then fall back to the allocator */ + ind = sal->next_segno >> SCOUTFS_ALLOC_REGION_SHIFT; + nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; + + do { + ret = radix_tree_gang_lookup(&sal->regs, (void **)®, ind, 1); + } while (ret == 0 && ind && (ind = 0, nr = 0, 1)); + + if (ret == 0) { + ret = -ENOSPC; + goto out; + } + + nr = find_next_bit_le(reg->bits, SCOUTFS_ALLOC_REGION_BITS, nr); + if (nr >= SCOUTFS_ALLOC_REGION_BITS) { + /* XXX corruption? shouldn't find empty regions */ + ret = -EIO; + goto out; + } + + clear_bit_le(nr, reg->bits); + radix_tree_tag_set(&sal->regs, ind, DIRTY_RADIX_TAG); + + *segno = (ind << SCOUTFS_ALLOC_REGION_SHIFT) + nr; + + /* once this wraps it will never equal alloc_uninit */ + sal->next_segno = *segno + 1; + if (sal->next_segno == le64_to_cpu(super->total_segs)) + sal->next_segno = 0; + + ret = 0; +out: + spin_unlock_irqrestore(&sal->lock, flags); + + trace_printk("segno %llu ret %d\n", *segno, ret); + return ret; +} + +/* + * Record newly freed sgements in pending regions. These can't be + * applied to the main allocator regions until the next commit so that + * they're not still referenced by the stable tree in event of a crash. + * + * The pending regions are merged into dirty regions for the next commit. + */ +int scoutfs_alloc_free(struct super_block *sb, u64 segno) +{ + struct scoutfs_ring_alloc_region *reg; + struct scoutfs_ring_alloc_region *ins; + DECLARE_SEG_ALLOC(sb, sal); + unsigned long flags; + unsigned long ind; + int ret; + int nr; + + ind = segno >> SCOUTFS_ALLOC_REGION_SHIFT; + nr = segno & SCOUTFS_ALLOC_REGION_MASK; + + ins = kzalloc(sizeof(struct scoutfs_ring_alloc_region), GFP_NOFS); + if (!ins) { + ret = -ENOMEM; + goto out; + } + + ins->eh.type = SCOUTFS_RING_ADD_ALLOC; + ins->eh.len = cpu_to_le16(sizeof(struct scoutfs_ring_alloc_region)); + ins->index = cpu_to_le64(ind); + + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + goto out; + } + + spin_lock_irqsave(&sal->lock, flags); + + reg = radix_tree_lookup(&sal->pending, ind); + if (!reg) { + reg = ins; + ins = NULL; + radix_tree_insert(&sal->pending, ind, reg); + } + + set_bit_le(nr, reg->bits); + + spin_unlock_irqrestore(&sal->lock, flags); + radix_tree_preload_end(); +out: + kfree(ins); + trace_printk("freeing segno %llu ind %lu nr %d ret %d\n", + segno, ind, nr, ret); + return ret; +} + +/* + * Add a new clean region from the ring. It can be replacing existing + * clean stale entries during replay as we make our way through the + * ring. + */ +int scoutfs_alloc_add(struct super_block *sb, + struct scoutfs_ring_alloc_region *ins) +{ + struct scoutfs_ring_alloc_region *existing; + struct scoutfs_ring_alloc_region *reg; + DECLARE_SEG_ALLOC(sb, sal); + unsigned long flags; + int ret; + + reg = kmalloc(sizeof(struct scoutfs_ring_alloc_region), GFP_NOFS); + if (!reg) { + ret = -ENOMEM; + goto out; + } + + memcpy(reg, ins, sizeof(struct scoutfs_ring_alloc_region)); + + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + kfree(reg); + goto out; + } + + spin_lock_irqsave(&sal->lock, flags); + + existing = radix_tree_lookup(&sal->regs, le64_to_cpu(reg->index)); + if (existing) + radix_tree_delete(&sal->regs, le64_to_cpu(reg->index)); + radix_tree_insert(&sal->regs, le64_to_cpu(reg->index), reg); + + spin_unlock_irqrestore(&sal->lock, flags); + radix_tree_preload_end(); + + if (existing) + kfree(existing); + + ret = 0; +out: + trace_printk("inserted reg ind %llu ret %d\n", + le64_to_cpu(ins->index), ret); + return ret; +} + +/* + * Append all the dirty alloc regions to the end of the ring. First we + * apply the pending frees to create the final set of dirty regions. + * + * This can't fail and always returns 0. + */ +int scoutfs_alloc_dirty_ring(struct super_block *sb) +{ + struct scoutfs_ring_alloc_region *regs[16]; + struct scoutfs_ring_alloc_region *reg; + DECLARE_SEG_ALLOC(sb, sal); + unsigned long start; + unsigned long ind; + int nr; + int i; + int b; + + /* + * Merge pending free regions into dirty regions. If the dirty + * region doesn't exist we can just move the pending region over. + * If it does we or the pending bits in the region. + */ + start = 0; + do { + nr = radix_tree_gang_lookup(&sal->pending, (void **)regs, + start, ARRAY_SIZE(regs)); + for (i = 0; i < nr; i++) { + ind = le64_to_cpu(regs[i]->index); + + reg = radix_tree_lookup(&sal->regs, ind); + if (!reg) { + radix_tree_insert(&sal->regs, ind, regs[i]); + } else { + for (b = 0; b < ARRAY_SIZE(reg->bits); b++) + reg->bits[i] |= regs[i]->bits[i]; + kfree(regs[i]); + } + + radix_tree_delete(&sal->pending, ind); + radix_tree_tag_set(&sal->regs, ind, DIRTY_RADIX_TAG); + start = ind + 1; + } + } while (nr); + + /* and append all the dirty regions to the ring */ + start = 0; + do { + nr = radix_tree_gang_lookup_tag(&sal->regs, (void **)regs, + start, ARRAY_SIZE(regs), + DIRTY_RADIX_TAG); + for (i = 0; i < nr; i++) { + reg = regs[i]; + ind = le64_to_cpu(reg->index); + + scoutfs_ring_append(sb, ®->eh); + radix_tree_tag_clear(&sal->regs, ind, DIRTY_RADIX_TAG); + start = ind + 1; + } + } while (nr); + + return 0; +} + +int scoutfs_alloc_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct seg_alloc *sal; + + /* bits need to be aligned so hosts can use native bitops */ + BUILD_BUG_ON(offsetof(struct scoutfs_ring_alloc_region, bits) & + (sizeof(long) - 1)); + + sal = kzalloc(sizeof(struct seg_alloc), GFP_KERNEL); + if (!sal) + return -ENOMEM; + sbi->seg_alloc = sal; + + spin_lock_init(&sal->lock); + /* inserts preload with _NOFS */ + INIT_RADIX_TREE(&sal->pending, GFP_ATOMIC); + INIT_RADIX_TREE(&sal->regs, GFP_ATOMIC); + /* XXX read next_segno from super? */ + + return 0; +} + +static void destroy_radix_regs(struct radix_tree_root *radix) +{ + struct scoutfs_ring_alloc_region *regs[16]; + int nr; + int i; + + + do { + nr = radix_tree_gang_lookup(radix, (void **)regs, + 0, ARRAY_SIZE(regs)); + for (i = 0; i < nr; i++) { + radix_tree_delete(radix, le64_to_cpu(regs[i]->index)); + kfree(regs[i]); + } + } while (nr); +} + +void scoutfs_alloc_destroy(struct super_block *sb) +{ + DECLARE_SEG_ALLOC(sb, sal); + + if (sal) { + destroy_radix_regs(&sal->pending); + destroy_radix_regs(&sal->regs); + kfree(sal); + } +} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h new file mode 100644 index 00000000..4d3d398b --- /dev/null +++ b/kmod/src/alloc.h @@ -0,0 +1,16 @@ +#ifndef _SCOUTFS_ALLOC_H_ +#define _SCOUTFS_ALLOC_H_ + +struct scoutfs_alloc_region; + +int scoutfs_alloc_segno(struct super_block *sb, u64 *segno); +int scoutfs_alloc_free(struct super_block *sb, u64 segno); + +int scoutfs_alloc_add(struct super_block *sb, + struct scoutfs_ring_alloc_region *ins); +int scoutfs_alloc_dirty_ring(struct super_block *sb); + +int scoutfs_alloc_setup(struct super_block *sb); +void scoutfs_alloc_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/bio.c b/kmod/src/bio.c index fe41a689..119cd13e 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -131,17 +131,40 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, dec_end_io(args, 1, ret); } -struct end_io_completion { - struct completion comp; - int err; -}; - -static void end_io_complete(struct super_block *sb, void *data, int err) +void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp) { - struct end_io_completion *comp = data; + /* this initial pending is dropped by wait */ + atomic_set(&comp->pending, 1); + init_completion(&comp->comp); + comp->err = 0; +} - comp->err = err; - complete(&comp->comp); +static void comp_end_io(struct super_block *sb, void *data, int err) +{ + struct scoutfs_bio_completion *comp = data; + + if (err && !comp->err) + comp->err = err; + + if (atomic_dec_and_test(&comp->pending)) + complete(&comp->comp); +} + +void scoutfs_bio_submit_comp(struct super_block *sb, int rw, + struct page **pages, u64 blkno, + unsigned int nr_blocks, + struct scoutfs_bio_completion *comp) +{ + atomic_inc(&comp->pending); + scoutfs_bio_submit(sb, rw, pages, blkno, nr_blocks, comp_end_io, comp); +} + +int scoutfs_bio_wait_comp(struct super_block *sb, + struct scoutfs_bio_completion *comp) +{ + comp_end_io(sb, comp, 0); + wait_for_completion(&comp->comp); + return comp->err; } /* @@ -152,13 +175,11 @@ static void end_io_complete(struct super_block *sb, void *data, int err) int scoutfs_bio_read(struct super_block *sb, struct page **pages, u64 blkno, unsigned int nr_blocks) { - struct end_io_completion comp; + struct scoutfs_bio_completion comp; - init_completion(&comp.comp); - scoutfs_bio_submit(sb, READ, pages, blkno, nr_blocks, - end_io_complete, &comp); - wait_for_completion(&comp.comp); - return comp.err; + scoutfs_bio_init_comp(&comp); + scoutfs_bio_submit_comp(sb, READ, pages, blkno, nr_blocks, &comp); + return scoutfs_bio_wait_comp(sb, &comp); } /* return pointer to the blk 4k block offset amongst the pages */ diff --git a/kmod/src/bio.h b/kmod/src/bio.h index 094f6038..d2e3390a 100644 --- a/kmod/src/bio.h +++ b/kmod/src/bio.h @@ -9,12 +9,30 @@ * BIO_MAX_PAGES then this would just use a single bio directly. */ +/* + * Track aggregate IO completion for multiple multi-bio submissions. + */ +struct scoutfs_bio_completion { + atomic_t pending; + struct completion comp; + long err; +}; + typedef void (*scoutfs_bio_end_io_t)(struct super_block *sb, void *data, int err); void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, u64 blkno, unsigned int nr_blocks, scoutfs_bio_end_io_t end_io, void *data); + +void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp); +void scoutfs_bio_submit_comp(struct super_block *sb, int rw, + struct page **pages, u64 blkno, + unsigned int nr_blocks, + struct scoutfs_bio_completion *comp); +int scoutfs_bio_wait_comp(struct super_block *sb, + struct scoutfs_bio_completion *comp); + int scoutfs_bio_read(struct super_block *sb, struct page **pages, u64 blkno, unsigned int nr_blocks); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 88a78610..f979fec6 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -27,6 +27,8 @@ #include "trans.h" #include "name.h" #include "xattr.h" +#include "kvec.h" +#include "item.h" /* * Directory entries are stored in entries with offsets calculated from @@ -95,167 +97,39 @@ static unsigned int dentry_type(unsigned int type) return DT_UNKNOWN; } - -/* - * XXX This crc nonsense is a quick hack. We'll want something a - * lot stronger like siphash. - */ -static u32 name_hash(const char *name, unsigned int len, u32 salt) -{ - u32 h = crc32c(salt, name, len) & SCOUTFS_DIRENT_OFF_MASK; - - return max_t(u32, 2, min_t(u32, h, SCOUTFS_DIRENT_LAST_POS)); -} - -static unsigned int dent_bytes(unsigned int name_len) -{ - return sizeof(struct scoutfs_dirent) + name_len; -} - -/* - * Each dirent stores the values that are needed to build the keys of - * the items that are removed on unlink so that we don't to search through - * items on unlink. - */ -struct dentry_info { - u64 lref_counter; - u32 hash; -}; - -static struct kmem_cache *scoutfs_dentry_cachep; - -static void scoutfs_d_release(struct dentry *dentry) -{ - struct dentry_info *di = dentry->d_fsdata; - - if (di) { - kmem_cache_free(scoutfs_dentry_cachep, di); - dentry->d_fsdata = NULL; - } -} - -static const struct dentry_operations scoutfs_dentry_ops = { - .d_release = scoutfs_d_release, -}; - -static struct dentry_info *alloc_dentry_info(struct dentry *dentry) -{ - struct dentry_info *di; - - /* XXX read mb? */ - if (dentry->d_fsdata) - return dentry->d_fsdata; - - di = kmem_cache_zalloc(scoutfs_dentry_cachep, GFP_NOFS); - if (!di) - return ERR_PTR(-ENOMEM); - - spin_lock(&dentry->d_lock); - if (!dentry->d_fsdata) { - dentry->d_fsdata = di; - d_set_d_op(dentry, &scoutfs_dentry_ops); - } - - spin_unlock(&dentry->d_lock); - - if (di != dentry->d_fsdata) - kmem_cache_free(scoutfs_dentry_cachep, di); - - return dentry->d_fsdata; -} - -static void update_dentry_info(struct dentry_info *di, struct scoutfs_key *key, - struct scoutfs_dirent *dent) -{ - di->lref_counter = le64_to_cpu(dent->counter); - di->hash = scoutfs_key_offset(key); -} - -static u64 last_dirent_key_offset(u32 h) -{ - return min_t(u64, (u64)h + SCOUTFS_DIRENT_COLL_NR - 1, - SCOUTFS_DIRENT_LAST_POS); -} - -/* - * Lookup searches for an entry for the given name amongst the entries - * stored in the item at the name's hash. - */ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { - struct scoutfs_inode_info *si = SCOUTFS_I(dir); struct super_block *sb = dir->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_dirent *dent = NULL; - struct scoutfs_btree_val val; - struct dentry_info *di; - struct scoutfs_key last; - struct scoutfs_key key; - unsigned int item_len; - unsigned int name_len; + struct scoutfs_dirent_key dkey; + struct scoutfs_dirent dent; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); struct inode *inode; u64 ino = 0; - u32 h = 0; int ret; - di = alloc_dentry_info(dentry); - if (IS_ERR(di)) { - ret = PTR_ERR(di); - goto out; - } - if (dentry->d_name.len > SCOUTFS_NAME_LEN) { ret = -ENAMETOOLONG; goto out; } - item_len = offsetof(struct scoutfs_dirent, name[dentry->d_name.len]); - dent = kmalloc(item_len, GFP_KERNEL); - if (!dent) { - ret = -ENOMEM; - goto out; - } + dkey.type = SCOUTFS_DIRENT_KEY; + dkey.ino = cpu_to_be64(scoutfs_ino(dir)); + scoutfs_kvec_init(key, &dkey, sizeof(dkey), + (void *)dentry->d_name.name, dentry->d_name.len); - h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); + scoutfs_kvec_init(val, &dent, sizeof(dent)); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, - last_dirent_key_offset(h)); - - scoutfs_btree_init_val(&val, dent, item_len); - - for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - /* XXX more verification */ - /* XXX corruption */ - if (ret <= sizeof(struct scoutfs_dirent)) { - ret = -EIO; - break; - } - - - name_len = ret - sizeof(struct scoutfs_dirent); - if (scoutfs_names_equal(dentry->d_name.name, dentry->d_name.len, - dent->name, name_len)) { - ino = le64_to_cpu(dent->ino); - update_dentry_info(di, &key, dent); - ret = 0; - break; - } - - scoutfs_inc_key(&key); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent)); + if (ret == -ENOENT) { + ino = 0; + ret = 0; + } else if (ret == 0) { + ino = le64_to_cpu(dent.ino); } out: - kfree(dent); - if (ret < 0) inode = ERR_PTR(ret); else if (ino == 0) @@ -299,47 +173,48 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; struct scoutfs_dirent *dent; - struct scoutfs_key key; - struct scoutfs_key last; + struct scoutfs_readdir_key rkey; + struct scoutfs_readdir_key last_rkey; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(last_key); + SCOUTFS_DECLARE_KVEC(val); unsigned int item_len; unsigned int name_len; - u32 pos; + u64 pos; int ret; if (!dir_emit_dots(file, dirent, filldir)) return 0; + rkey.type = SCOUTFS_READDIR_KEY; + rkey.ino = cpu_to_be64(scoutfs_ino(inode)); + /* pos set in each loop */ + scoutfs_kvec_init(key, &rkey, sizeof(rkey)); + + last_rkey.type = SCOUTFS_READDIR_KEY; + last_rkey.ino = cpu_to_be64(scoutfs_ino(inode)); + last_rkey.pos = cpu_to_be64(SCOUTFS_DIRENT_LAST_POS); + scoutfs_kvec_init(last_key, &last_rkey, sizeof(last_rkey)); + item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); dent = kmalloc(item_len, GFP_KERNEL); if (!dent) return -ENOMEM; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - file->f_pos); - scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_DIRENT_KEY, - SCOUTFS_DIRENT_LAST_POS); - - scoutfs_btree_init_val(&val, dent, item_len); - for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + rkey.pos = cpu_to_be64(file->f_pos); + scoutfs_kvec_init(val, dent, item_len); + ret = scoutfs_item_next_same_min(sb, key, last_key, val, + offsetof(struct scoutfs_dirent, name[1])); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - /* XXX corruption */ - if (ret <= sizeof(dent)) { - ret = -EIO; - break; - } - name_len = ret - sizeof(struct scoutfs_dirent); - pos = scoutfs_key_offset(&key); + pos = be64_to_cpu(rkey.pos); if (filldir(dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) { @@ -348,13 +223,13 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } file->f_pos = pos + 1; - scoutfs_inc_key(&key); } kfree(dent); return ret; } +#if 0 static void set_lref_key(struct scoutfs_key *key, u64 ino, u64 ctr) { scoutfs_set_key(key, ino, SCOUTFS_LINK_BACKREF_KEY, ctr); @@ -380,66 +255,74 @@ static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, return ret; } +#endif static int add_entry_items(struct inode *dir, struct dentry *dentry, struct inode *inode) { - struct dentry_info *di = dentry->d_fsdata; struct super_block *sb = dir->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_inode_info *si = SCOUTFS_I(dir); - struct scoutfs_btree_val val; + struct scoutfs_dirent_key dkey; struct scoutfs_dirent dent; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_key key; - struct scoutfs_key lref_key; - int bytes; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); int ret; - u64 h; - - /* caller should have allocated the dentry info */ - if (WARN_ON_ONCE(di == NULL)) - return -EINVAL; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; ret = scoutfs_dirty_inode_item(dir); if (ret) - goto out; + return ret; - bytes = dent_bytes(dentry->d_name.len); - h = name_hash(dentry->d_name.name, dentry->d_name.len, si->salt); - scoutfs_set_key(&first, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, h); - scoutfs_set_key(&last, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, - last_dirent_key_offset(h)); - - ret = scoutfs_btree_hole(sb, meta, &first, &last, &key); - if (ret) - goto out; - - set_lref_key(&lref_key, scoutfs_ino(inode), - atomic64_inc_return(&SCOUTFS_I(inode)->link_counter)); - ret = update_lref_item(sb, &lref_key, scoutfs_ino(dir), - scoutfs_key_offset(&key), false); - if (ret) - goto out; + /* dirent item for lookup */ + dkey.type = SCOUTFS_DIRENT_KEY; + dkey.ino = cpu_to_be64(scoutfs_ino(dir)); + scoutfs_kvec_init(key, &dkey, sizeof(dkey), + (void *)dentry->d_name.name, dentry->d_name.len); dent.ino = cpu_to_le64(scoutfs_ino(inode)); - dent.counter = lref_key.offset; dent.type = mode_to_type(inode->i_mode); + scoutfs_kvec_init(val, &dent, sizeof(dent)); - scoutfs_btree_init_val(&val, &dent, sizeof(dent), - (void *)dentry->d_name.name, - dentry->d_name.len); - - ret = scoutfs_btree_insert(sb, meta, &key, &val); + ret = scoutfs_item_create(sb, key, val); if (ret) - scoutfs_btree_delete(sb, meta, &lref_key); - else - update_dentry_info(di, &key, &dent); -out: + return ret; + +#if 0 + struct scoutfs_inode_info *si = SCOUTFS_I(dir); + + /* readdir item for .. readdir */ + si->readdir_pos++; + rkey.type = SCOUTFS_READDIR_KEY; + rkey.ino = cpu_to_le64(scoutfs_ino(dir)); + rkey.pos = cpu_to_le64(si->readdir_pos); + scoutfs_kvec_init(key, &rkey, sizeof(rkey)); + + scoutfs_kvec_init(val, &dent, sizeof(dent), + dentry->d_name.name, dentry->d_name.len); + + ret = scoutfs_item_create(sb, key, val); + if (ret) + goto out_dent; + + /* backref item for inode to path resolution */ + lrkey.type = SCOUTFS_LINK_BACKREF_KEY; + lrey.ino = cpu_to_le64(scoutfs_ino(inode)); + lrey.dir = cpu_to_le64(scoutfs_ino(dir)); + scoutfs_kvec_init(key, &lrkey, sizeof(lrkey), + dentry->d_name.name, dentry->d_name.len); + + ret = scoutfs_item_create(sb, key, NULL); + if (ret) { + scoutfs_kvec_init(key, &rkey, sizeof(rkey)); + scoutfs_item_delete(sb, key); +out_dent: + scoutfs_kvec_init(key, &dkey, sizeof(dkey), + dentry->d_name.name, dentry->d_name.len); + scoutfs_item_delete(sb, key); + } +#endif + return ret; } @@ -448,13 +331,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, { struct super_block *sb = dir->i_sb; struct inode *inode; - struct dentry_info *di; int ret; - di = alloc_dentry_info(dentry); - if (IS_ERR(di)) - return PTR_ERR(di); - ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -508,16 +386,11 @@ static int scoutfs_link(struct dentry *old_dentry, { struct inode *inode = old_dentry->d_inode; struct super_block *sb = dir->i_sb; - struct dentry_info *di; int ret; if (inode->i_nlink >= SCOUTFS_LINK_MAX) return -EMLINK; - di = alloc_dentry_info(dentry); - if (IS_ERR(di)) - return PTR_ERR(di); - ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -548,17 +421,14 @@ out: static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) { struct super_block *sb = dir->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct dentry_info *di; - struct scoutfs_key key; - struct scoutfs_key lref_key; + struct scoutfs_dirent_key dkey; + SCOUTFS_DECLARE_KVEC(key); int ret = 0; - if (WARN_ON_ONCE(!dentry->d_fsdata)) - return -EINVAL; - di = dentry->d_fsdata; + /* will need to add deletion items */ + return -EINVAL; if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; @@ -567,17 +437,18 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (ret) return ret; - set_lref_key(&lref_key, scoutfs_ino(inode), di->lref_counter); - scoutfs_set_key(&key, scoutfs_ino(dir), SCOUTFS_DIRENT_KEY, di->hash); - - /* - * Dirty most of the metadata up front so that later btree - * operations can't fail. - */ ret = scoutfs_dirty_inode_item(dir) ?: - scoutfs_dirty_inode_item(inode) ?: - scoutfs_btree_dirty(sb, meta, &lref_key) ?: - scoutfs_btree_dirty(sb, meta, &key); + scoutfs_dirty_inode_item(inode); + if (ret) + goto out; + + /* XXX same items as add_entry_items */ + dkey.type = SCOUTFS_DIRENT_KEY; + dkey.ino = cpu_to_be64(scoutfs_ino(dir)); + scoutfs_kvec_init(key, &dkey, sizeof(dkey), + (void *)dentry->d_name.name, dentry->d_name.len); + + ret = scoutfs_item_delete(sb, key); if (ret) goto out; @@ -593,10 +464,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto out; } - /* XXX: In thoery this can't fail but we should trap errors anyway */ - scoutfs_btree_delete(sb, meta, &key); - scoutfs_btree_delete(sb, meta, &lref_key); - dir->i_ctime = ts; dir->i_mtime = ts; i_size_write(dir, i_size_read(dir) - dentry->d_name.len); @@ -637,6 +504,9 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) int ret; int k; + /* update for kvec items */ + return ERR_PTR(-EINVAL); + /* XXX corruption */ if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) return ERR_PTR(-EIO); @@ -712,21 +582,19 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct scoutfs_btree_val val; struct inode *inode = NULL; struct scoutfs_key key; - struct dentry_info *di; const int name_len = strlen(symname) + 1; int off; int bytes; int ret; int k = 0; + /* update for kvec items */ + return -EINVAL; + /* path_max includes null as does our value for nd_set_link */ if (name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) return -ENAMETOOLONG; - di = alloc_dentry_info(dentry); - if (IS_ERR(di)) - return PTR_ERR(di); - ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -961,6 +829,9 @@ int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, int ret; int nr; + /* update for kvec items */ + return -EINVAL; + if (*ctr == U64_MAX) return 0; @@ -1017,22 +888,3 @@ const struct inode_operations scoutfs_dir_iops = { .removexattr = scoutfs_removexattr, .symlink = scoutfs_symlink, }; - -void scoutfs_dir_exit(void) -{ - if (scoutfs_dentry_cachep) { - kmem_cache_destroy(scoutfs_dentry_cachep); - scoutfs_dentry_cachep = NULL; - } -} - -int scoutfs_dir_init(void) -{ - scoutfs_dentry_cachep = kmem_cache_create("scoutfs_dentry_info", - sizeof(struct dentry_info), 0, - SLAB_RECLAIM_ACCOUNT, NULL); - if (!scoutfs_dentry_cachep) - return -ENOMEM; - - return 0; -} diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 4953af9e..2327518b 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -7,9 +7,6 @@ extern const struct file_operations scoutfs_dir_fops; extern const struct inode_operations scoutfs_dir_iops; extern const struct inode_operations scoutfs_symlink_iops; -int scoutfs_dir_init(void); -void scoutfs_dir_exit(void); - struct scoutfs_path_component { struct list_head head; unsigned int len; diff --git a/kmod/src/format.h b/kmod/src/format.h index ff1b69dd..2f126610 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -56,7 +56,8 @@ struct scoutfs_ring_entry_header { __le16 len; } __packed; -#define SCOUTFS_RING_ADD_MANIFEST 1 +#define SCOUTFS_RING_ADD_MANIFEST 1 +#define SCOUTFS_RING_ADD_ALLOC 2 struct scoutfs_ring_add_manifest { struct scoutfs_ring_entry_header eh; @@ -68,26 +69,55 @@ struct scoutfs_ring_add_manifest { /* first and last key bytes */ } __packed; +#define SCOUTFS_ALLOC_REGION_SHIFT 8 +#define SCOUTFS_ALLOC_REGION_BITS (1 << SCOUTFS_ALLOC_REGION_SHIFT) +#define SCOUTFS_ALLOC_REGION_MASK (SCOUTFS_ALLOC_REGION_BITS - 1) + +/* + * The bits need to be aligned so that the host can use native long + * bitops on the bits in memory. + */ +struct scoutfs_ring_alloc_region { + struct scoutfs_ring_entry_header eh; + __le64 index; + __u8 pad[5]; + __le64 bits[SCOUTFS_ALLOC_REGION_BITS / 64]; +} __packed; + /* * This is absurdly huge. If there was only ever 1 item per segment and * 2^64 items the tree could get this deep. */ #define SCOUTFS_MANIFEST_MAX_LEVEL 20 +/* + * The packed entries in the block are terminated by a header with a 0 length. + */ struct scoutfs_ring_block { struct scoutfs_block_header hdr; - __le32 nr_entries; struct scoutfs_ring_entry_header entries[0]; } __packed; +/* + * We really want these to be a power of two size so that they're naturally + * aligned. This ensures that they won't cross page boundaries and we + * can use pointers to them in the page vecs that make up segments without + * funny business. + * + * We limit segment sizes to 8 megs (23 bits) and value lengths to 512 bytes + * (9 bits). The item offsets and lengths then take up 64 bits. + * + * We then operate on the items in on-stack nice native structs. + */ struct scoutfs_segment_item { __le64 seq; - __le32 key_off; - __le32 val_off; - __le16 key_len; - __le16 val_len; + __le32 key_off_len; + __le32 val_off_len; } __packed; +#define SCOUTFS_SEGMENT_ITEM_OFF_SHIFT 9 +#define SCOUTFS_SEGMENT_ITEM_LEN_MASK ((1 << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT)-1) + /* * Each large segment starts with a segment block that describes the * rest of the blocks that make up the segment. @@ -98,20 +128,12 @@ struct scoutfs_segment_block { __le64 segno; __le64 max_seq; __le32 nr_items; - /* item array with gaps so they don't cross 4k blocks */ + __le32 _moar_pads; + struct scoutfs_segment_item items[0]; /* packed keys */ /* packed vals */ } __packed; -/* the first block in the segment has the header and items */ -#define SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_segment_block)) / \ - sizeof(struct scoutfs_segment_item)) - -/* the rest of the header blocks are full of items */ -#define SCOUTFS_SEGMENT_ITEMS_PER_BLOCK \ - (SCOUTFS_BLOCK_SIZE / sizeof(struct scoutfs_segment_item)) - /* * Block references include the sequence number so that we can detect * readers racing with writers and so that we can tell that we don't @@ -186,18 +208,34 @@ struct scoutfs_key { #define SCOUTFS_XATTR_NAME_HASH_KEY 3 #define SCOUTFS_XATTR_VAL_HASH_KEY 4 #define SCOUTFS_DIRENT_KEY 5 -#define SCOUTFS_LINK_BACKREF_KEY 6 -#define SCOUTFS_SYMLINK_KEY 7 -#define SCOUTFS_EXTENT_KEY 8 -#define SCOUTFS_ORPHAN_KEY 9 +#define SCOUTFS_READDIR_KEY 6 +#define SCOUTFS_LINK_BACKREF_KEY 7 +#define SCOUTFS_SYMLINK_KEY 8 +#define SCOUTFS_EXTENT_KEY 9 +#define SCOUTFS_ORPHAN_KEY 10 #define SCOUTFS_MAX_ITEM_LEN 512 +/* value is struct scoutfs_inode */ struct scoutfs_inode_key { __u8 type; __be64 ino; } __packed; +/* value is struct scoutfs_dirent without the name */ +struct scoutfs_dirent_key { + __u8 type; + __be64 ino; + __u8 name[0]; +} __packed; + +/* value is struct scoutfs_dirent with the name */ +struct scoutfs_readdir_key { + __u8 type; + __be64 ino; + __be64 pos; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; @@ -270,6 +308,8 @@ struct scoutfs_super_block { __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; + __le64 alloc_uninit; + __le64 total_segs; __le64 total_blocks; __le64 free_blocks; __le64 ring_blkno; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 0ccfe006..c34babf7 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -27,7 +27,6 @@ #include "scoutfs_trace.h" #include "xattr.h" #include "trans.h" -#include "btree.h" #include "msg.h" #include "kvec.h" #include "item.h" @@ -269,13 +268,17 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) int scoutfs_dirty_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_key key; + struct scoutfs_inode_key ikey; + struct scoutfs_inode sinode; + SCOUTFS_DECLARE_KVEC(key); int ret; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); + store_inode(&sinode, inode); - ret = scoutfs_btree_dirty(sb, meta, &key); + set_inode_key(&ikey, scoutfs_ino(inode)); + scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + + ret = scoutfs_item_dirty(sb, key); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -283,8 +286,8 @@ int scoutfs_dirty_inode_item(struct inode *inode) /* * Every time we modify the inode in memory we copy it to its inode - * item. This lets us write out blocks of items without having to track - * down dirty vfs inodes and safely copy them into items before writing. + * item. This lets us write out items without having to track down + * dirty vfs inodes. * * The caller makes sure that the item is dirty and pinned so they don't * have to deal with errors and unwinding after they've modified the @@ -293,17 +296,19 @@ int scoutfs_dirty_inode_item(struct inode *inode) void scoutfs_update_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; + struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - struct scoutfs_key key; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); int err; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); store_inode(&sinode, inode); - err = scoutfs_btree_update(sb, meta, &key, &val); + set_inode_key(&ikey, scoutfs_ino(inode)); + scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + + err = scoutfs_item_update(sb, key, val); BUG_ON(err); trace_scoutfs_update_inode(inode); @@ -381,11 +386,11 @@ static int alloc_ino(struct super_block *sb, u64 *ino) struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *ci; - struct scoutfs_btree_val val; + struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - struct scoutfs_key key; + SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(val); struct inode *inode; u64 ino; int ret; @@ -413,11 +418,12 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, inode->i_rdev = rdev; set_inode_ops(inode); - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); store_inode(&sinode, inode); + set_inode_key(&ikey, scoutfs_ino(inode)); + scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_btree_insert(inode->i_sb, meta, &key, &val); + ret = scoutfs_item_create(sb, key, val); if (ret) { iput(inode); return ERR_PTR(ret); diff --git a/kmod/src/item.c b/kmod/src/item.c index f65388ec..df2d187a 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -14,20 +14,31 @@ #include #include #include +#include #include "super.h" #include "format.h" #include "kvec.h" #include "manifest.h" #include "item.h" +#include "seg.h" struct item_cache { spinlock_t lock; struct rb_root root; + + unsigned long nr_dirty_items; + unsigned long dirty_key_bytes; + unsigned long dirty_val_bytes; }; +/* + * The dirty bits track if the given item is dirty and if its child + * subtrees contain any dirty items. + */ struct cached_item { struct rb_node node; + long dirty; SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); @@ -56,12 +67,53 @@ static struct cached_item *find_item(struct rb_root *root, struct kvec *key) return NULL; } +/* + * We store the dirty bits in a single value so that the simple + * augmented rbtree implementation gets a single scalar value to compare + * and store. + */ +#define ITEM_DIRTY 0x1 +#define LEFT_DIRTY 0x2 +#define RIGHT_DIRTY 0x4 + +/* + * Return the given dirty bit if the item with the given node is dirty + * or has dirty children. + */ +static long node_dirty_bit(struct rb_node *node, long dirty) +{ + struct cached_item *item; + + if (node) { + item = container_of(node, struct cached_item, node); + if (item->dirty) + return dirty; + } + + return 0; +} + +static long compute_item_dirty(struct cached_item *item) +{ + return (item->dirty & ITEM_DIRTY) | + node_dirty_bit(item->node.rb_left, LEFT_DIRTY) | + node_dirty_bit(item->node.rb_right, RIGHT_DIRTY); +} + +RB_DECLARE_CALLBACKS(static, scoutfs_item_rb_cb, struct cached_item, node, + long, dirty, compute_item_dirty); + +/* + * Always insert the given item. If there's an existing item it is + * returned. This can briefly leave duplicate items in the tree until + * the caller removes the existing item. + */ static struct cached_item *insert_item(struct rb_root *root, struct cached_item *ins) { struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; - struct cached_item *found = NULL; + struct cached_item *existing = NULL; struct cached_item *item; int cmp; @@ -71,22 +123,23 @@ static struct cached_item *insert_item(struct rb_root *root, cmp = scoutfs_kvec_memcmp(ins->key, item->key); if (cmp < 0) { + if (ins->dirty) + item->dirty |= LEFT_DIRTY; node = &(*node)->rb_left; } else if (cmp > 0) { + if (ins->dirty) + item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; } else { - rb_replace_node(&item->node, &ins->node, root); - found = item; + existing = item; break; } } - if (!found) { - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); - } + rb_link_node(&ins->node, parent, node); + rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); - return found; + return existing; } /* @@ -139,12 +192,96 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, ret = scoutfs_item_lookup(sb, key, val); if (ret == size) ret = 0; - else if (ret >= 0 && ret != size) + else if (ret >= 0) ret = -EIO; return ret; } +/* + * Return the next cached item starting with the given key. + * + * -ENOENT is returned if there are no cached items past the given key. + * If the last key is specified then -ENOENT is returned if there are no + * cached items up until that last key, inclusive. + * + * The found key is copied to the caller's key. -ENOBUFS is returned if + * the found key didn't fit in the caller's key. + * + * The found value is copied into the callers value. The number of + * value bytes copied is returned. The copied value can be truncated by + * the caller's value buffer length. + */ +int scoutfs_item_next(struct super_block *sb, struct kvec *key, + struct kvec *last, struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + /* + * This partial copy and paste of lookup is stubbed out for now. + * we'll want the negative caching fixes to be able to iterate + * without constantly searching the manifest between cached + * items. + */ + return -EINVAL; + + do { + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(&cac->root, key); + if (!item) { + ret = -ENOENT; + } else if (scoutfs_kvec_length(item->key) > + scoutfs_kvec_length(key)) { + ret = -ENOBUFS; + } else { + scoutfs_kvec_memcpy_truncate(key, item->key); + if (val) + ret = scoutfs_kvec_memcpy(val, item->val); + else + ret = 0; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + + trace_printk("ret %d\n", ret); + + return ret; +} + +/* + * Like _next but requires that the found keys be the same length as the + * search key and that values be of at least a minimum size. It treats + * size mismatches as a sign of corruption. A found key larger than the + * found key buffer gives -ENOBUFS and is a sign of corruption. + */ +int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, + struct kvec *last, struct kvec *val, int len) +{ + int key_len = scoutfs_kvec_length(key); + int ret; + + trace_printk("key len %u min val len %d\n", key_len, len); + + if (WARN_ON_ONCE(!val || scoutfs_kvec_length(val) < len)) + return -EINVAL; + + ret = scoutfs_item_next(sb, key, last, val); + if (ret == -ENOBUFS || + (ret >= 0 && (scoutfs_kvec_length(key) != key_len || ret < len))) + ret = -EIO; + + trace_printk("ret %d\n", ret); + + return ret; +} + static void free_item(struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { @@ -154,21 +291,77 @@ static void free_item(struct cached_item *item) } } +/* + * The caller might have modified the item's dirty flags. Ascend + * through parents updating their dirty flags until there's no change. + */ +static void update_dirty_parents(struct cached_item *item) +{ + struct cached_item *parent; + struct rb_node *node; + long dirty; + + while ((node = rb_parent(&item->node))) { + parent = container_of(node, struct cached_item, node); + dirty = compute_item_dirty(parent); + + if (parent->dirty == dirty) + break; + + parent->dirty = dirty; + item = parent; + } +} + +static void mark_item_dirty(struct item_cache *cac, + struct cached_item *item) +{ + if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) + return; + + if (item->dirty & ITEM_DIRTY) + return; + + item->dirty |= ITEM_DIRTY; + cac->nr_dirty_items++; + cac->dirty_key_bytes += scoutfs_kvec_length(item->key); + cac->dirty_val_bytes += scoutfs_kvec_length(item->val); + + update_dirty_parents(item); +} + +static void clear_item_dirty(struct item_cache *cac, + struct cached_item *item) +{ + if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) + return; + + if (!(item->dirty & ITEM_DIRTY)) + return; + + item->dirty &= ~ITEM_DIRTY; + cac->nr_dirty_items--; + cac->dirty_key_bytes -= scoutfs_kvec_length(item->key); + cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); + + update_dirty_parents(item); +} + /* * Add an item with the key and value to the item cache. The new item * is clean. Any existing item at the key will be removed and freed. */ -int scoutfs_item_insert(struct super_block *sb, struct kvec *key, - struct kvec *val) +static int add_item(struct super_block *sb, struct kvec *key, struct kvec *val, + bool dirty) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct cached_item *found; + struct cached_item *existing; struct cached_item *item; unsigned long flags; int ret; - item = kmalloc(sizeof(struct cached_item), GFP_NOFS); + item = kzalloc(sizeof(struct cached_item), GFP_NOFS); if (!item) return -ENOMEM; @@ -180,9 +373,265 @@ int scoutfs_item_insert(struct super_block *sb, struct kvec *key, } spin_lock_irqsave(&cac->lock, flags); - found = insert_item(&cac->root, item); + existing = insert_item(&cac->root, item); + if (existing) { + clear_item_dirty(cac, existing); + rb_erase_augmented(&item->node, &cac->root, + &scoutfs_item_rb_cb); + } + mark_item_dirty(cac, item); spin_unlock_irqrestore(&cac->lock, flags); - free_item(found); + free_item(existing); + + return 0; +} + +/* + * Add a clean item to the cache. This is used to populate items while + * reading segments. + */ +int scoutfs_item_insert(struct super_block *sb, struct kvec *key, + struct kvec *val) +{ + return add_item(sb, key, val, false); +} + +/* + * Create a new dirty item in the cache. + */ +int scoutfs_item_create(struct super_block *sb, struct kvec *key, + struct kvec *val) +{ + return add_item(sb, key, val, true); +} + +/* + * If the item with the key exists make sure it's cached and dirty. -ENOENT + * will be returned if it doesn't exist. + */ +int scoutfs_item_dirty(struct super_block *sb, struct kvec *key) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + do { + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(&cac->root, key); + if (item) { + mark_item_dirty(cac, item); + ret = 0; + } else { + ret = -ENOENT; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + + trace_printk("ret %d\n", ret); + + return ret; +} + +/* + * Set the value of an existing item in the tree. The item is marked dirty + * and the previous value is freed. The provided value may be null. + * + * Returns -ENOENT if the item doesn't exist. + */ +int scoutfs_item_update(struct super_block *sb, struct kvec *key, + struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + SCOUTFS_DECLARE_KVEC(up_val); + struct cached_item *item; + unsigned long flags; + int ret; + + if (val) { + ret = scoutfs_kvec_dup_flatten(up_val, val); + if (ret) + return -ENOMEM; + } else { + scoutfs_kvec_init_null(up_val); + } + + spin_lock_irqsave(&cac->lock, flags); + + /* XXX update seq */ + item = find_item(&cac->root, key); + if (item) { + scoutfs_kvec_swap(up_val, item->val); + mark_item_dirty(cac, item); + } else { + ret = -ENOENT; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + scoutfs_kvec_kfree(up_val); + + trace_printk("ret %d\n", ret); + + return ret; +} + +/* + * XXX how nice, it'd just creates a cached deletion item. It doesn't + * have to read. + */ +int scoutfs_item_delete(struct super_block *sb, struct kvec *key) +{ + return WARN_ON_ONCE(-EINVAL); +} + +/* + * Return the first dirty node in the subtree starting at the given node. + */ +static struct cached_item *first_dirty(struct rb_node *node) +{ + struct cached_item *ret = NULL; + struct cached_item *item; + + while (node) { + item = container_of(node, struct cached_item, node); + + if (item->dirty & LEFT_DIRTY) { + node = item->node.rb_left; + } else if (item->dirty & ITEM_DIRTY) { + ret = item; + break; + } else if (item->dirty & RIGHT_DIRTY) { + node = item->node.rb_right; + } + } + + return ret; +} + +/* + * Find the next dirty item after a given item. First we see if we have + * a dirty item in our right subtree. If not we ascend through parents + * skipping those that are less than us. If we find a parent that's + * greater than us then we see if it's dirty, if not we start the search + * all over again by checking its right subtree then ascending. + */ +static struct cached_item *next_dirty(struct cached_item *item) +{ + struct rb_node *parent; + struct rb_node *node; + + while (item) { + if (item->dirty & RIGHT_DIRTY) + return first_dirty(item->node.rb_right); + + /* find next greatest parent */ + node = &item->node; + while ((parent = rb_parent(node)) && parent->rb_right == node) + node = parent; + if (!parent) + break; + + /* done if our next greatest parent itself is dirty */ + item = container_of(parent, struct cached_item, node); + if (item->dirty & ITEM_DIRTY) + return item; + + /* continue to check right subtree */ + } + + return NULL; +} + +/* + * The total number of bytes that will be stored in segments if we were + * to write out all the currently dirty items. + * + * XXX this isn't strictly correct because item's aren't of a uniform + * size. We might need more segments when large items leave gaps at the + * tail of each segment as it is filled with sorted items. It's close + * enough for now. + */ +long scoutfs_item_dirty_bytes(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + unsigned long flags; + long bytes; + + spin_lock_irqsave(&cac->lock, flags); + + bytes = (cac->nr_dirty_items * sizeof(struct scoutfs_segment_item)) + + cac->dirty_key_bytes + cac->dirty_val_bytes; + + spin_unlock_irqrestore(&cac->lock, flags); + + bytes += DIV_ROUND_UP(bytes, sizeof(struct scoutfs_segment_block)) * + sizeof(struct scoutfs_segment_block); + + return bytes; +} + +/* + * Find the initial sorted dirty items that will fit in a segment. Give + * the caller the number of items and the total bytes of their keys. + */ +static void count_seg_items(struct item_cache *cac, u32 *nr_items, + u32 *key_bytes) +{ + struct cached_item *item; + u32 total; + + *nr_items = 0; + *key_bytes = 0; + total = sizeof(struct scoutfs_segment_block); + + for (item = first_dirty(cac->root.rb_node); item; + item = next_dirty(item)) { + + total += sizeof(struct scoutfs_segment_item) + + scoutfs_kvec_length(item->key) + + scoutfs_kvec_length(item->val); + + if (total > SCOUTFS_SEGMENT_SIZE) + break; + + (*nr_items)++; + (*key_bytes) += scoutfs_kvec_length(item->key); + } +} + +/* + * Fill the given segment with sorted dirty items. + * + * The caller is responsible for the consistency of the dirty items once + * they're in its seg. We can consider them clean once we store them. + */ +int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + u32 key_bytes; + u32 nr_items; + + count_seg_items(cac, &nr_items, &key_bytes); + if (nr_items) { + item = first_dirty(cac->root.rb_node); + scoutfs_seg_first_item(sb, seg, item->key, item->val, + nr_items, key_bytes); + clear_item_dirty(cac, item); + + while ((item = next_dirty(item))) { + scoutfs_seg_append_item(sb, seg, item->key, item->val); + clear_item_dirty(cac, item); + } + } return 0; } @@ -207,8 +656,8 @@ void scoutfs_item_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct rb_node *node; struct cached_item *item; + struct rb_node *node; if (cac) { for (node = rb_first(&cac->root); node; ) { @@ -219,5 +668,4 @@ void scoutfs_item_destroy(struct super_block *sb) kfree(cac); } - } diff --git a/kmod/src/item.h b/kmod/src/item.h index bfaae9db..62d93815 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -3,12 +3,27 @@ #include +struct scoutfs_segment; + int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, struct kvec *val); int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, struct kvec *val, int size); +int scoutfs_item_next(struct super_block *sb, struct kvec *key, + struct kvec *last, struct kvec *val); +int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, + struct kvec *last, struct kvec *val, int len); int scoutfs_item_insert(struct super_block *sb, struct kvec *key, struct kvec *val); +int scoutfs_item_create(struct super_block *sb, struct kvec *key, + struct kvec *val); +int scoutfs_item_dirty(struct super_block *sb, struct kvec *key); +int scoutfs_item_update(struct super_block *sb, struct kvec *key, + struct kvec *val); +int scoutfs_item_delete(struct super_block *sb, struct kvec *key); + +long scoutfs_item_dirty_bytes(struct super_block *sb); +int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_item_setup(struct super_block *sb); void scoutfs_item_destroy(struct super_block *sb); diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index e2b26061..6cddb073 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -112,6 +112,28 @@ int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src) return copied; } +/* + * Copy bytes in src into dst, stopping if dst is full. The number of copied + * bytes is returned and the lengths of dst are updated if the size changes. + * The pointers in dst are not changed. + */ +int scoutfs_kvec_memcpy_truncate(struct kvec *dst, struct kvec *src) +{ + int copied = scoutfs_kvec_memcpy(dst, src); + size_t bytes; + int i; + + if (copied < scoutfs_kvec_length(dst)) { + bytes = copied; + for (i = 0; i < SCOUTFS_KVEC_NR; i++) { + dst[i].iov_len = min(dst[i].iov_len, bytes); + bytes -= dst[i].iov_len; + } + } + + return copied; +} + /* * Copy the src key vector into one new allocation in the dst. The existing * dst is clobbered. The source isn't changed. @@ -139,3 +161,17 @@ void scoutfs_kvec_kfree(struct kvec *kvec) while (kvec->iov_base) kfree((kvec++)->iov_base); } + +void scoutfs_kvec_init_null(struct kvec *kvec) +{ + memset(kvec, 0, SCOUTFS_KVEC_NR * sizeof(kvec[0])); +} + +void scoutfs_kvec_swap(struct kvec *a, struct kvec *b) +{ + SCOUTFS_DECLARE_KVEC(tmp); + + memcpy(tmp, a, sizeof(tmp)); + memcpy(a, b, sizeof(tmp)); + memcpy(b, tmp, sizeof(tmp)); +} diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index 600055e9..49d51ae9 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -61,7 +61,10 @@ int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b); int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, struct kvec *c, struct kvec *d); int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src); +int scoutfs_kvec_memcpy_truncate(struct kvec *dst, struct kvec *src); int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src); void scoutfs_kvec_kfree(struct kvec *kvec); +void scoutfs_kvec_init_null(struct kvec *kvec); +void scoutfs_kvec_swap(struct kvec *a, struct kvec *b); #endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index c6b1a33f..15c22fd5 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -20,6 +20,7 @@ #include "kvec.h" #include "seg.h" #include "item.h" +#include "ring.h" #include "manifest.h" struct manifest { @@ -30,6 +31,8 @@ struct manifest { u8 last_level; struct rb_root level_roots[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + + struct list_head dirty_list; }; #define DECLARE_MANIFEST(sb, name) \ @@ -40,12 +43,11 @@ struct manifest_entry { struct list_head level0_entry; struct rb_node node; }; + struct list_head dirty_entry; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); - u64 segno; - u64 seq; - u8 level; + struct scoutfs_ring_add_manifest am; + /* u8 key_bytes[am.first_key_len]; */ + /* u8 val_bytes[am.last_key_len]; */ }; /* @@ -60,6 +62,32 @@ struct manifest_ref { u8 level; }; +static void init_ment_keys(struct manifest_entry *ment, struct kvec *first, + struct kvec *last) +{ + scoutfs_kvec_init(first, &ment->am + 1, + le16_to_cpu(ment->am.first_key_len)); + scoutfs_kvec_init(last, &ment->am + 1 + + le16_to_cpu(ment->am.first_key_len), + le16_to_cpu(ment->am.last_key_len)); +} + +/* + * returns: + * < 0 : key < ment->first_key + * > 0 : key > ment->first_key + * == 0 : ment->first_key <= key <= ment->last_key + */ +static bool cmp_key_ment(struct kvec *key, struct manifest_entry *ment) +{ + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); + + init_ment_keys(ment, first, last); + + return scoutfs_kvec_cmp_overlap(key, key, first, last); +} + static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) { struct rb_node *node = root->rb_node; @@ -69,8 +97,7 @@ static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) while (node) { ment = container_of(node, struct manifest_entry, node); - cmp = scoutfs_kvec_cmp_overlap(key, key, - ment->first, ment->last); + cmp = cmp_key_ment(key, ment); if (cmp < 0) node = node->rb_left; else if (cmp > 0) @@ -91,14 +118,17 @@ static int insert_ment(struct rb_root *root, struct manifest_entry *ins) struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; struct manifest_entry *ment; + SCOUTFS_DECLARE_KVEC(key); int cmp; + /* either first or last works */ + init_ment_keys(ins, key, key); + while (*node) { parent = *node; ment = container_of(*node, struct manifest_entry, node); - cmp = scoutfs_kvec_cmp_overlap(ins->first, ins->last, - ment->first, ment->last); + cmp = cmp_key_ment(key, ment); if (cmp < 0) { node = &(*node)->rb_left; } else if (cmp > 0) { @@ -116,29 +146,32 @@ static int insert_ment(struct rb_root *root, struct manifest_entry *ins) static void free_ment(struct manifest_entry *ment) { - if (!IS_ERR_OR_NULL(ment)) { - scoutfs_kvec_kfree(ment->first); - scoutfs_kvec_kfree(ment->last); + if (!IS_ERR_OR_NULL(ment)) kfree(ment); - } } -static int add_ment(struct manifest *mani, struct manifest_entry *ment) +static int add_ment(struct manifest *mani, struct manifest_entry *ment, + bool dirty) { + u8 level = ment->am.level; int ret; - trace_printk("adding ment %p level %u\n", ment, ment->level); - if (ment->level) { - ret = insert_ment(&mani->level_roots[ment->level], ment); + trace_printk("adding ment %p level %u\n", ment, level); + + if (level) { + ret = insert_ment(&mani->level_roots[level], ment); if (!ret) - mani->last_level = max(mani->last_level, ment->level); + mani->last_level = max(mani->last_level, level); } else { list_add_tail(&ment->level0_entry, &mani->level0_list); mani->level0_nr++; ret = 0; } + if (dirty) + list_add_tail(&ment->dirty_entry, &mani->dirty_list); + return ret; } @@ -155,41 +188,52 @@ static void update_last_level(struct manifest *mani) static void remove_ment(struct manifest *mani, struct manifest_entry *ment) { - if (ment->level) { - rb_erase(&ment->node, &mani->level_roots[ment->level]); + u8 level = ment->am.level; + + if (level) { + rb_erase(&ment->node, &mani->level_roots[level]); update_last_level(mani); } else { list_del_init(&ment->level0_entry); mani->level0_nr--; } + + /* XXX more carefully remove dirty ments.. should be exceptional */ + if (!list_empty(&ment->dirty_entry)) + list_del_init(&ment->dirty_entry); } int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level) + struct kvec *last, u64 segno, u64 seq, u8 level, + bool dirty) { DECLARE_MANIFEST(sb, mani); struct manifest_entry *ment; unsigned long flags; + int bytes; int ret; - ment = kmalloc(sizeof(struct manifest_entry), GFP_NOFS); + bytes = sizeof(struct manifest_entry) + scoutfs_kvec_length(first), + scoutfs_kvec_length(last); + ment = kmalloc(bytes, GFP_NOFS); if (!ment) return -ENOMEM; - ret = scoutfs_kvec_dup_flatten(ment->first, first) ?: - scoutfs_kvec_dup_flatten(ment->last, last); - if (ret) { - free_ment(ment); - return -ENOMEM; - } + if (level) + RB_CLEAR_NODE(&ment->node); + else + INIT_LIST_HEAD(&ment->level0_entry); + INIT_LIST_HEAD(&ment->dirty_entry); - ment->segno = segno; - ment->seq = seq; - ment->level = level; + ment->am.eh.type = SCOUTFS_RING_ADD_MANIFEST; + ment->am.eh.len = cpu_to_le16(bytes); + ment->am.segno = cpu_to_le64(segno); + ment->am.seq = cpu_to_le64(seq); + ment->am.level = level; /* XXX think about where to insert level 0 */ spin_lock_irqsave(&mani->lock, flags); - ret = add_ment(mani, ment); + ret = add_ment(mani, ment, dirty); spin_unlock_irqrestore(&mani->lock, flags); if (WARN_ON_ONCE(ret)) /* XXX can this happen? ring corruption? */ free_ment(ment); @@ -197,11 +241,11 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, return ret; } -static void set_ref(struct manifest_ref *ref, struct manifest_entry *mani) +static void set_ref(struct manifest_ref *ref, struct manifest_entry *ment) { - ref->segno = mani->segno; - ref->seq = mani->seq; - ref->level = mani->level; + ref->segno = le64_to_cpu(ment->am.segno); + ref->seq = le64_to_cpu(ment->am.seq); + ref->level = ment->am.level; } /* @@ -242,8 +286,7 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, list_for_each_entry(ment, &mani->level0_list, level0_entry) { trace_printk("trying l0 ment %p\n", ment); - if (scoutfs_kvec_cmp_overlap(key, key, - ment->first, ment->last)) + if (cmp_key_ment(key, ment)) continue; set_ref(&refs[nr++], ment); @@ -410,6 +453,32 @@ out: return ret; } +int scoutfs_manifest_has_dirty(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + + return !list_empty_careful(&mani->dirty_list); +} + +/* + * Append the dirty manifest entries to the end of the ring. + * + * This returns 0 but can't fail. + */ +int scoutfs_manifest_dirty_ring(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + struct manifest_entry *ment; + struct manifest_entry *tmp; + + list_for_each_entry_safe(ment, tmp, &mani->dirty_list, dirty_entry) { + scoutfs_ring_append(sb, &ment->am.eh); + list_del_init(&ment->dirty_entry); + } + + return 0; +} + int scoutfs_manifest_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -423,6 +492,7 @@ int scoutfs_manifest_setup(struct super_block *sb) spin_lock_init(&mani->lock); INIT_LIST_HEAD(&mani->level0_list); + INIT_LIST_HEAD(&mani->dirty_list); for (i = 0; i < ARRAY_SIZE(mani->level_roots); i++) mani->level_roots[i] = RB_ROOT; diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index c1ea0160..f3bea21a 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -2,7 +2,11 @@ #define _SCOUTFS_MANIFEST_H_ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level); + struct kvec *last, u64 segno, u64 seq, u8 level, + bool dirty); +int scoutfs_manifest_has_dirty(struct super_block *sb); +int scoutfs_manifest_dirty_ring(struct super_block *sb); + int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key); int scoutfs_manifest_setup(struct super_block *sb); diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 867acd3b..3cda13b5 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -13,126 +13,140 @@ #include #include #include +#include #include "super.h" #include "format.h" #include "kvec.h" #include "bio.h" #include "manifest.h" +#include "alloc.h" #include "ring.h" +#include "crc.h" + /* - * OK, log: - * - big preallocated ring of variable length entries - * - entries are rounded to 4k blocks - * - entire thing is read and indexed in rbtree - * - static allocated page is kept around to record and write entries - * - indexes have cursor that points to next node to migrate - * - any time an entry is written an entry is migrated - * - allocate room for 4x (maybe including worst case rounding) - * - mount does binary search looking for newest entry - * - newest entry describes block where we started migrating - * - replay then walks from oldest to newest replaying - * - entries are marked with migration so we know where to set cursor after + * Right now we're only writing a segment a time. The entries needed to + * write a segment will always be smaller than a segment itself. * + * XXX This'll get more clever as we can write multiple segments and build + * up dirty entries while processing compaction results. + */ +struct ring_info { + struct page *pages[SCOUTFS_SEGMENT_PAGES]; + struct scoutfs_ring_block *ring; + struct scoutfs_ring_entry_header *next_eh; + unsigned int nr_blocks; + unsigned int space; +}; + +#define DECLARE_RING_INFO(sb, name) \ + struct ring_info *name = SCOUTFS_SB(sb)->ring_info + +/* * XXX * - verify blocks * - could compress + * - have all entry sources dirty at cursors before dirtying + * - advancing cursor updates head as cursor wraps */ -/* read in a meg at a time */ -#define NR_PAGES DIV_ROUND_UP(1024 * 1024, PAGE_SIZE) -#define NR_BLOCKS (NR_PAGES * SCOUTFS_BLOCKS_PER_PAGE) - -#if 0 -#define BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) -static void read_page_end_io(struct bio *bio, int err) +/* + * The space calculation when starting a block included a final empty + * entry header. That is zeroed here. + */ +static void finish_block(struct scoutfs_ring_block *ring, unsigned int tail) { - struct bio_vec *bvec; - struct page *page; - unsigned long i; + memset((char *)ring + tail, 0, SCOUTFS_BLOCK_SIZE - tail); + scoutfs_crc_block(&ring->hdr); +} - for_each_bio_segment(bio, bvec, i) { - page = bvec->bv_page; +void scoutfs_ring_append(struct super_block *sb, + struct scoutfs_ring_entry_header *eh) +{ + DECLARE_RING_INFO(sb, rinf); + struct scoutfs_ring_block *ring = rinf->ring; + unsigned int len = le16_to_cpu(eh->len); - if (err) - SetPageError(page); - else - SetPageUptodate(page); - unlock_page(page); + if (rinf->space < len) { + if (ring) + finish_block(ring, rinf->space); + ring = scoutfs_page_block_address(rinf->pages, rinf->nr_blocks); + rinf->ring = ring; + + memset(ring, 0, sizeof(struct scoutfs_ring_block)); + + rinf->nr_blocks++; + rinf->next_eh = ring->entries; + rinf->space = SCOUTFS_BLOCK_SIZE - + offsetof(struct scoutfs_ring_block, entries) - + sizeof(struct scoutfs_ring_entry_header); } - bio_put(bio); + memcpy(rinf->next_eh, eh, len); + rinf->next_eh = (void *)((char *)eh + len); + rinf->space -= len; } /* - * Read the given number of 4k blocks into the pages provided by the - * caller. We translate the block count into a page count and fill - * bios a page at a time. + * Kick off the writes to update the ring. Update the dirty super to + * reference the written ring. */ -static int read_blocks(struct super_block *sb, struct page **pages, - u64 blkno, unsigned int nr_blocks) +int scoutfs_ring_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp) { - unsigned int nr_pages = DIV_ROUND_UP(nr_blocks, PAGES_PER_BLOCK); - unsigned int bytes; - struct bio *bio; - int ret = 0; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_RING_INFO(sb, rinf); + u64 head_blocks; + u64 blocks; + u64 blkno; + u64 ind; - for (i = 0; i < nr_pages; i++) { - page = pages[i]; + if (!rinf->nr_blocks) + return 0; - if (!bio) { - bio = bio_alloc(GFP_NOFS, nr_pages - i); - if (!bio) - bio = bio_alloc(GFP_NOFS, 1); - if (!bio) { - ret = -ENOMEM; - break; - } + if (rinf->space) + finish_block(rinf->ring, rinf->space); - bio->bi_sector = blkno << (SCOUTFS_BLOCK_SHIFT - 9); - bio->bi_bdev = sb->s_bdev; - bio->bi_end_io = read_pages_end_io; - } + ind = le64_to_cpu(super->ring_tail_index) + 1; + blocks = rinf->nr_blocks; + blkno = le64_to_cpu(super->ring_blkno) + ind; - lock_page(page); - ClearPageError(page); - ClearPageUptodate(page); + /* + * If the log wrapped then we have to write two fragments to the + * tail and head of the ring. We submit the head fragment + * first. + * + * The head fragment starts at some block offset in the + * preallocated pages. This hacky page math only works when our + * 4k blocks size == page_size. To fix it we'd add a offset + * block to the bio submit loop which could add an initial + * partial page vec to the bios. + */ + BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); - bytes = min(nr_blocks << SCOUTFS_BLOCK_SHIFT, PAGE_SIZE); - - if (bio_add_page(bio, page, bytes, 0) != bytes) { - /* submit the full bio and retry this page */ - submit_bio(READ, bio); - bio = NULL; - unlock_page(page); - i--; - continue; - } - - blkno += BLOCKS_PER_PAGE; - nr_blocks -= BLOCKS_PER_PAGE; + if (ind + blocks > le64_to_cpu(super->ring_blocks)) { + head_blocks = (ind + blocks) - le64_to_cpu(super->ring_blocks); + blocks -= head_blocks; + scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + blocks, + le64_to_cpu(super->ring_blkno), + head_blocks, comp); } - if (bio) - submit_bio(READ, bio); + scoutfs_bio_submit_comp(sb, WRITE, rinf->pages, blkno, blocks, comp); - for (i = 0; i < nr_pages; i++) { - page = pages[i]; + ind += blocks; + if (ind == le64_to_cpu(super->ring_blocks)) + ind = 0; + super->ring_tail_index = cpu_to_le64(ind); - wait_on_page_locked(page); - if (!ret && (!PageUptodate(page) || PageError(page))) - ret = -EIO; - } - - return ret; + return 0; } -#endif - static int read_one_entry(struct super_block *sb, struct scoutfs_ring_entry_header *eh) { + struct scoutfs_ring_alloc_region *reg; struct scoutfs_ring_add_manifest *am; SCOUTFS_DECLARE_KVEC(first); SCOUTFS_DECLARE_KVEC(last); @@ -156,7 +170,13 @@ static int read_one_entry(struct super_block *sb, ret = scoutfs_manifest_add(sb, first, last, le64_to_cpu(am->segno), - le64_to_cpu(am->seq), am->level); + le64_to_cpu(am->seq), am->level, + false); + break; + + case SCOUTFS_RING_ADD_ALLOC: + reg = container_of(eh, struct scoutfs_ring_alloc_region, eh); + ret = scoutfs_alloc_add(sb, reg); break; default: @@ -171,33 +191,22 @@ static int read_entries(struct super_block *sb, { struct scoutfs_ring_entry_header *eh; int ret = 0; - int i; - trace_printk("reading %u entries\n", le32_to_cpu(ring->nr_entries)); + for (eh = ring->entries; eh->len; + eh = (void *)eh + le16_to_cpu(eh->len)) { - eh = ring->entries; - - for (i = 0; i < le32_to_cpu(ring->nr_entries); i++) { ret = read_one_entry(sb, eh); if (ret) break; - - eh = (void *)eh + le16_to_cpu(eh->len); } return ret; } -#if 0 -/* return pointer to the blk 4k block offset amongst the pages */ -static void *page_block_address(struct page **pages, unsigned int blk) -{ - unsigned int i = blk / BLOCKS_PER_PAGE; - unsigned int off = (blk % BLOCKS_PER_PAGE) << SCOUTFS_BLOCK_SHIFT; - return page_address(pages[i]) + off; -} -#endif +/* read in a meg at a time */ +#define NR_PAGES DIV_ROUND_UP(1024 * 1024, PAGE_SIZE) +#define NR_BLOCKS (NR_PAGES * SCOUTFS_BLOCKS_PER_PAGE) int scoutfs_ring_read(struct super_block *sb) { @@ -274,3 +283,43 @@ out: return ret; } + +int scoutfs_ring_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct ring_info *rinf; + struct page *page; + int i; + + rinf = kzalloc(sizeof(struct ring_info), GFP_KERNEL); + if (!rinf) + return -ENOMEM; + sbi->ring_info = rinf; + + for (i = 0; i < ARRAY_SIZE(rinf->pages); i++) { + page = alloc_page(GFP_KERNEL); + if (!page) { + while (--i >= 0) + __free_page(rinf->pages[i]); + return -ENOMEM; + } + + rinf->pages[i] = page; + } + + return 0; +} + +void scoutfs_ring_destroy(struct super_block *sb) +{ + DECLARE_RING_INFO(sb, rinf); + int i; + + if (rinf) { + for (i = 0; i < ARRAY_SIZE(rinf->pages); i++) + __free_page(rinf->pages[i]); + + kfree(rinf); + } +} + diff --git a/kmod/src/ring.h b/kmod/src/ring.h index 4f6930c9..94eb84c3 100644 --- a/kmod/src/ring.h +++ b/kmod/src/ring.h @@ -3,6 +3,16 @@ #include +struct scoutfs_bio_completion; + int scoutfs_ring_read(struct super_block *sb); +void scoutfs_ring_append(struct super_block *sb, + struct scoutfs_ring_entry_header *eh); + +int scoutfs_ring_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp); + +int scoutfs_ring_setup(struct super_block *sb); +void scoutfs_ring_destroy(struct super_block *sb); #endif diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 4537c50c..e86d595d 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -21,6 +21,8 @@ #include "seg.h" #include "bio.h" #include "kvec.h" +#include "manifest.h" +#include "alloc.h" /* * seg.c should just be about the cache and io, and maybe @@ -127,8 +129,9 @@ static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) /* * This always inserts the segment into the rbtree. If there's already - * a segment at the given seg then it is removed and returned. The caller - * doesn't have to erase it from the tree if it's returned. + * a segment at the given seg then it is removed and returned. The + * caller doesn't have to erase it from the tree if it's returned but it + * does have to put the reference that it's given. */ static struct scoutfs_segment *replace_seg(struct rb_root *root, struct scoutfs_segment *ins) @@ -205,6 +208,45 @@ static u64 segno_to_blkno(u64 blkno) return blkno << (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT); } +int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct segment_cache *cac = sbi->segment_cache; + struct scoutfs_segment *existing; + struct scoutfs_segment *seg; + unsigned long flags; + u64 segno; + int ret; + + *seg_ret = NULL; + + ret = scoutfs_alloc_segno(sb, &segno); + if (ret) + goto out; + + seg = alloc_seg(segno); + if (!seg) { + ret = scoutfs_alloc_free(sb, segno); + BUG_ON(ret); /* XXX could make pending when allocating */ + ret = -ENOMEM; + goto out; + } + + /* XXX always remove existing segs, is that necessary? */ + spin_lock_irqsave(&cac->lock, flags); + atomic_inc(&seg->refcount); + existing = replace_seg(&cac->root, seg); + spin_unlock_irqrestore(&cac->lock, flags); + if (existing) + scoutfs_seg_put(existing); + + *seg_ret = seg; + ret = 0; +out: + return ret; + +} + /* * The bios submitted by this don't have page references themselves. If * this succeeds then the caller must call _wait before putting their @@ -248,6 +290,19 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, return seg; } +int scoutfs_seg_submit_write(struct super_block *sb, + struct scoutfs_segment *seg, + struct scoutfs_bio_completion *comp) +{ + trace_printk("submitting segno %llu\n", seg->segno); + + scoutfs_bio_submit_comp(sb, WRITE, seg->pages, + segno_to_blkno(seg->segno), + SCOUTFS_SEGMENT_BLOCKS, comp); + + return 0; +} + int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -270,29 +325,67 @@ static void *off_ptr(struct scoutfs_segment *seg, u32 off) return page_address(seg->pages[pg]) + pg_off; } -/* - * Return a pointer to the item in the array at the given position. - * - * The item structs fill the first block in the segment after the - * initial segment block struct. Item structs don't cross block - * boundaries so the final bytes that would make up a partial item - * struct are skipped. - */ -static struct scoutfs_segment_item *pos_item(struct scoutfs_segment *seg, - int pos) +static u32 pos_off(struct scoutfs_segment *seg, u32 pos) { - u32 off; + /* items need of be a power of two */ + BUILD_BUG_ON(!is_power_of_2(sizeof(struct scoutfs_segment_item))); + /* and the first item has to be naturally aligned */ + BUILD_BUG_ON(offsetof(struct scoutfs_segment_block, items) & + sizeof(struct scoutfs_segment_item)); - if (pos < SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS) { - off = sizeof(struct scoutfs_segment_block); - } else { - pos -= SCOUTFS_SEGMENT_FIRST_BLOCK_ITEMS; - off = (1 + (pos / SCOUTFS_SEGMENT_ITEMS_PER_BLOCK)) * - SCOUTFS_BLOCK_SIZE; - pos %= SCOUTFS_SEGMENT_ITEMS_PER_BLOCK; - } + return offsetof(struct scoutfs_segment_block, items[pos]); +} - return off_ptr(seg, off + (pos * sizeof(struct scoutfs_segment_item))); +static void *pos_ptr(struct scoutfs_segment *seg, u32 pos) +{ + return off_ptr(seg, pos_off(seg, pos)); +} + +/* + * The persistent item fields that are stored in the segment are packed + * with funny precision. We translate those to and from a much more + * natural native representation of the fields. + */ +struct native_item { + u64 seq; + u32 key_off; + u32 val_off; + u16 key_len; + u16 val_len; +}; + +static void load_item(struct scoutfs_segment *seg, u32 pos, + struct native_item *item) +{ + struct scoutfs_segment_item *sitem = pos_ptr(seg, pos); + u32 packed; + + item->seq = le64_to_cpu(sitem->seq); + + packed = le32_to_cpu(sitem->key_off_len); + item->key_off = packed >> SCOUTFS_SEGMENT_ITEM_OFF_SHIFT; + item->key_len = packed & SCOUTFS_SEGMENT_ITEM_LEN_MASK; + + packed = le32_to_cpu(sitem->val_off_len); + item->val_off = packed >> SCOUTFS_SEGMENT_ITEM_OFF_SHIFT; + item->val_len = packed & SCOUTFS_SEGMENT_ITEM_LEN_MASK; +} + +static void store_item(struct scoutfs_segment *seg, u32 pos, + struct native_item *item) +{ + struct scoutfs_segment_item *sitem = pos_ptr(seg, pos); + u32 packed; + + sitem->seq = cpu_to_le64(item->seq); + + packed = (item->key_off << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT) | + (item->key_len & SCOUTFS_SEGMENT_ITEM_LEN_MASK); + sitem->key_off_len = cpu_to_le32(packed); + + packed = (item->val_off << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT) | + (item->val_len & SCOUTFS_SEGMENT_ITEM_LEN_MASK); + sitem->val_off_len = cpu_to_le32(packed); } static void kvec_from_pages(struct scoutfs_segment *seg, @@ -313,19 +406,17 @@ int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, struct kvec *key, struct kvec *val) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; + struct native_item item; if (pos < 0 || pos >= le32_to_cpu(sblk->nr_items)) return -ENOENT; - item = pos_item(seg, pos); + load_item(seg, pos, &item); if (key) - kvec_from_pages(seg, key, le32_to_cpu(item->key_off), - le16_to_cpu(item->key_len)); + kvec_from_pages(seg, key, item.key_off, item.key_len); if (val) - kvec_from_pages(seg, val, le32_to_cpu(item->val_off), - le16_to_cpu(item->val_len)); + kvec_from_pages(seg, val, item.val_off, item.val_len); return 0; } @@ -365,6 +456,90 @@ int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key) return find_key_pos(seg, key); } +/* + * Store the first item in the segment. The caller knows the number + * of items and bytes of keys that determine where the keys and values + * start. Future items are appended by looking at the last item. + * + * This should never fail because any item must always fit in a segment. + */ +void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, + struct kvec *key, struct kvec *val, + unsigned int nr_items, unsigned int key_bytes) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct native_item item; + SCOUTFS_DECLARE_KVEC(item_key); + SCOUTFS_DECLARE_KVEC(item_val); + u32 key_off; + u32 val_off; + + key_off = pos_off(seg, nr_items); + val_off = key_off + key_bytes; + + sblk->nr_items = cpu_to_le32(1); + + item.seq = 1; + item.key_off = key_off; + item.val_off = val_off; + item.key_len = scoutfs_kvec_length(key); + item.val_len = scoutfs_kvec_length(val); + store_item(seg, 0, &item); + + scoutfs_seg_item_kvecs(seg, 0, key, val); + scoutfs_kvec_memcpy(item_key, key); + scoutfs_kvec_memcpy(item_val, val); +} + +void scoutfs_seg_append_item(struct super_block *sb, + struct scoutfs_segment *seg, + struct kvec *key, struct kvec *val) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct native_item item; + struct native_item prev; + SCOUTFS_DECLARE_KVEC(item_key); + SCOUTFS_DECLARE_KVEC(item_val); + u32 nr; + + nr = le32_to_cpu(sblk->nr_items); + sblk->nr_items = cpu_to_le32(nr + 1); + + load_item(seg, nr - 1, &prev); + + item.seq = 1; + item.key_off = prev.key_off + prev.key_len; + item.key_len = scoutfs_kvec_length(key); + item.val_off = prev.val_off + prev.val_len; + item.val_len = scoutfs_kvec_length(val); + store_item(seg, 0, &item); + + scoutfs_seg_item_kvecs(seg, nr, key, val); + scoutfs_kvec_memcpy(item_key, key); + scoutfs_kvec_memcpy(item_val, val); +} + +/* + * Add a dirty manifest entry for the given segment at the given level. + */ +int scoutfs_seg_add_ment(struct super_block *sb, struct scoutfs_segment *seg, + u8 level) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct native_item item; + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); + + load_item(seg, 0, &item); + kvec_from_pages(seg, first, item.key_off, item.key_len); + + load_item(seg, le32_to_cpu(sblk->nr_items) - 1, &item); + kvec_from_pages(seg, last, item.key_off, item.key_len); + + return scoutfs_manifest_add(sb, first, last, le64_to_cpu(sblk->segno), + le64_to_cpu(sblk->max_seq), level, true); +} + int scoutfs_seg_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -400,4 +575,3 @@ void scoutfs_seg_destroy(struct super_block *sb) kfree(cac); } } - diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 1957a308..c5ae81d4 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -1,6 +1,7 @@ #ifndef _SCOUTFS_SEG_H_ #define _SCOUTFS_SEG_H_ +struct scoutfs_bio_completion; struct scoutfs_segment; struct kvec; @@ -14,6 +15,20 @@ int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, void scoutfs_seg_put(struct scoutfs_segment *seg); +int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); +void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, + struct kvec *key, struct kvec *val, + unsigned int nr_items, unsigned int key_bytes); +void scoutfs_seg_append_item(struct super_block *sb, + struct scoutfs_segment *seg, + struct kvec *key, struct kvec *val); +int scoutfs_seg_add_ment(struct super_block *sb, struct scoutfs_segment *seg, + u8 level); + +int scoutfs_seg_submit_write(struct super_block *sb, + struct scoutfs_segment *seg, + struct scoutfs_bio_completion *comp); + int scoutfs_seg_setup(struct super_block *sb); void scoutfs_seg_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index 7866185a..5ce1a2b7 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -33,6 +33,7 @@ #include "manifest.h" #include "seg.h" #include "bio.h" +#include "alloc.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -226,6 +227,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_seg_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: + scoutfs_alloc_setup(sb) ?: + scoutfs_ring_setup(sb) ?: scoutfs_ring_read(sb) ?: // scoutfs_buddy_setup(sb) ?: scoutfs_setup_trans(sb); @@ -264,8 +267,10 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sbi->block_shrinker.shrink == scoutfs_block_shrink) unregister_shrinker(&sbi->block_shrinker); scoutfs_item_destroy(sb); + scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); scoutfs_seg_destroy(sb); + scoutfs_ring_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) @@ -285,7 +290,6 @@ static struct file_system_type scoutfs_fs_type = { /* safe to call at any failure point in _init */ static void teardown_module(void) { - scoutfs_dir_exit(); scoutfs_inode_exit(); if (scoutfs_kset) kset_unregister(scoutfs_kset); @@ -302,7 +306,6 @@ static int __init scoutfs_module_init(void) return -ENOMEM; ret = scoutfs_inode_init() ?: - scoutfs_dir_init() ?: register_filesystem(&scoutfs_fs_type); if (ret) teardown_module(); diff --git a/kmod/src/super.h b/kmod/src/super.h index b1b20e97..bb803105 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -12,6 +12,7 @@ struct buddy_info; struct item_cache; struct manifest; struct segment_cache; +struct ring_info; struct scoutfs_sb_info { struct super_block *sb; @@ -34,6 +35,8 @@ struct scoutfs_sb_info { struct manifest *manifest; struct item_cache *item_cache; struct segment_cache *segment_cache; + struct seg_alloc *seg_alloc; + struct ring_info *ring_info; struct buddy_info *buddy_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index b9108500..217ec29f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -22,6 +22,12 @@ #include "trans.h" #include "buddy.h" #include "filerw.h" +#include "bio.h" +#include "item.h" +#include "manifest.h" +#include "seg.h" +#include "alloc.h" +#include "ring.h" #include "scoutfs_trace.h" /* @@ -74,37 +80,43 @@ void scoutfs_trans_write_func(struct work_struct *work) struct scoutfs_sb_info *sbi = container_of(work, struct scoutfs_sb_info, trans_write_work); struct super_block *sb = sbi->sb; + struct scoutfs_bio_completion comp; + struct scoutfs_segment *seg; bool advance = false; int ret = 0; - bool have_umount; - sbi->trans_task = current; + scoutfs_bio_init_comp(&comp); + sbi->trans_task = NULL; wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); - if (scoutfs_block_has_dirty(sb)) { - /* XXX need writeback errors from inode address spaces? */ + /* XXX file data needs to be updated to the new item api */ +#if 0 + scoutfs_filerw_free_alloc(sb); +#endif - /* XXX definitely don't understand this */ - have_umount = down_read_trylock(&sb->s_umount); + /* + * We only have to check if there are dirty items or manifest + * entries. You can't have dirty alloc regions without having + * changed references to the allocated segments which produces + * dirty manfiest entries. + */ + if (scoutfs_item_dirty_bytes(sb) || scoutfs_manifest_has_dirty(sb)) { - sync_inodes_sb(sb); - - if (have_umount) - up_read(&sb->s_umount); - - scoutfs_filerw_free_alloc(sb); - - ret = scoutfs_buddy_apply_pending(sb, false) ?: - scoutfs_block_write_dirty(sb) ?: + ret = scoutfs_seg_alloc(sb, &seg) ?: + scoutfs_item_dirty_seg(sb, seg); + scoutfs_seg_add_ment(sb, seg, 0) ?: + scoutfs_manifest_dirty_ring(sb) ?: + scoutfs_alloc_dirty_ring(sb) ?: + scoutfs_ring_submit_write(sb, &comp) ?: + scoutfs_seg_submit_write(sb, seg, &comp) ?: + scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_write_dirty_super(sb); - if (ret) { - scoutfs_buddy_apply_pending(sb, true); - } else { - scoutfs_buddy_committed(sb); - advance = 1; - } + BUG_ON(ret); + + scoutfs_seg_put(seg); + advance = true; } spin_lock(&sbi->trans_write_lock); @@ -183,6 +195,10 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, return scoutfs_sync_fs(file->f_inode->i_sb, 1); } +/* + * The first holders race to try and allocate the segment that will be + * written by the next commit. + */ int scoutfs_hold_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -195,21 +211,28 @@ int scoutfs_hold_trans(struct super_block *sb) } /* - * As we release we ask the allocator how many blocks have been - * allocated since the last transaction was successfully committed. If - * it's large enough we kick off a write. This is mostly to reduce the - * commit latency. We also don't want to let the IO pipeline sit idle. - * Once we have enough blocks to write efficiently we should do so. + * As we release we kick off a commit if we have a segment's worth of + * dirty items. + * + * Right now it's conservatively kicking off writes at ~95% full blocks. + * This leaves a lot of slop for the largest item bytes created by a + * holder and overrun by concurrent holders (who aren't accounted + * today). + * + * It should more precisely know the worst case item byte consumption of + * holders and only kick off a write when someone tries to hold who + * might fill the segment. */ void scoutfs_release_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + unsigned int target = (SCOUTFS_SEGMENT_SIZE * 95 / 100); if (current == sbi->trans_task) return; if (atomic_sub_return(1, &sbi->trans_holds) == 0) { - if (scoutfs_buddy_alloc_count(sb) >= SCOUTFS_MAX_TRANS_BLOCKS) + if (scoutfs_item_dirty_bytes(sb) >= target) scoutfs_sync_fs(sb, 0); wake_up(&sbi->trans_hold_wq); From be98c4dfd889eff1f30ba3a902dc2276501d67f6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Dec 2016 10:33:05 -0800 Subject: [PATCH 153/920] Fix up manifest key use The manifest entries were changed to be a single contiguous allocation. The calculation of the vec that points to the last key vec was adding the key length in units of the add manifest struct. Adding the manifest wasn't setting the key lengths nor copying the keys into their position in the entry alloc. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 15c22fd5..2901c724 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -67,7 +67,7 @@ static void init_ment_keys(struct manifest_entry *ment, struct kvec *first, { scoutfs_kvec_init(first, &ment->am + 1, le16_to_cpu(ment->am.first_key_len)); - scoutfs_kvec_init(last, &ment->am + 1 + + scoutfs_kvec_init(last, (void *)(&ment->am + 1) + le16_to_cpu(ment->am.first_key_len), le16_to_cpu(ment->am.last_key_len)); } @@ -209,6 +209,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, { DECLARE_MANIFEST(sb, mani); struct manifest_entry *ment; + SCOUTFS_DECLARE_KVEC(ment_first); + SCOUTFS_DECLARE_KVEC(ment_last); unsigned long flags; int bytes; int ret; @@ -229,8 +231,14 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, ment->am.eh.len = cpu_to_le16(bytes); ment->am.segno = cpu_to_le64(segno); ment->am.seq = cpu_to_le64(seq); + ment->am.first_key_len = cpu_to_le16(scoutfs_kvec_length(first)); + ment->am.last_key_len = cpu_to_le16(scoutfs_kvec_length(last)); ment->am.level = level; + init_ment_keys(ment, ment_first, ment_last); + scoutfs_kvec_memcpy(ment_first, first); + scoutfs_kvec_memcpy(ment_last, last); + /* XXX think about where to insert level 0 */ spin_lock_irqsave(&mani->lock, flags); ret = add_ment(mani, ment, dirty); From 471405f8cdf363432373c1429f8a4e23931d8986 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 08:42:46 -0800 Subject: [PATCH 154/920] Fix kvec iterators A few of the kvec iterators that work with byte offsets forgot to reset the offsets as they advanced to the next vec. These should probably be refactored into a set of iterator helpers. Signed-off-by: Zach Brown --- kmod/src/kvec.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 6cddb073..0e0b1f84 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -50,11 +50,15 @@ int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) return ret; b_off += len; - if (b_off == b->iov_len) + if (b_off == b->iov_len) { b++; + b_off = 0; + } a_off += len; - if (a_off == a->iov_len) + if (a_off == a->iov_len) { a++; + a_off = 0; + } } return a->iov_base ? 1 : b->iov_base ? -1 : 0; @@ -102,11 +106,15 @@ int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src) copied += len; src_off += len; - if (src_off == src->iov_len) + if (src_off == src->iov_len) { src++; + src_off = 0; + } dst_off += len; - if (dst_off == dst->iov_len) + if (dst_off == dst->iov_len) { dst++; + dst_off = 0; + } } return copied; From 9e02573e062072b60b879304097505dfefba7fe9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 08:45:24 -0800 Subject: [PATCH 155/920] Rename scoutfs_seg_manfest_add Rename scoutfs_seg_add_ment to _manifest_add as that makes it a lot more clear that it's a wrapper around scoutfs_manifest_add() that gets its arguments from the segment. Signed-off-by: Zach Brown --- kmod/src/seg.c | 4 ++-- kmod/src/seg.h | 4 ++-- kmod/src/trans.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index e86d595d..743575ed 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -522,8 +522,8 @@ void scoutfs_seg_append_item(struct super_block *sb, /* * Add a dirty manifest entry for the given segment at the given level. */ -int scoutfs_seg_add_ment(struct super_block *sb, struct scoutfs_segment *seg, - u8 level) +int scoutfs_seg_manifest_add(struct super_block *sb, + struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; diff --git a/kmod/src/seg.h b/kmod/src/seg.h index c5ae81d4..683d9a3e 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -22,8 +22,8 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, struct kvec *key, struct kvec *val); -int scoutfs_seg_add_ment(struct super_block *sb, struct scoutfs_segment *seg, - u8 level); +int scoutfs_seg_manifest_add(struct super_block *sb, + struct scoutfs_segment *seg, u8 level); int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 217ec29f..ab3da53d 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -106,7 +106,7 @@ void scoutfs_trans_write_func(struct work_struct *work) ret = scoutfs_seg_alloc(sb, &seg) ?: scoutfs_item_dirty_seg(sb, seg); - scoutfs_seg_add_ment(sb, seg, 0) ?: + scoutfs_seg_manifest_add(sb, seg, 0) ?: scoutfs_manifest_dirty_ring(sb) ?: scoutfs_alloc_dirty_ring(sb) ?: scoutfs_ring_submit_write(sb, &comp) ?: From 48f9be8455ba46a9f4e053575e1ab5d3f4827db7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 08:51:45 -0800 Subject: [PATCH 156/920] Free key and value in the right order! Always key then value! *twitch* Signed-off-by: Zach Brown --- kmod/src/item.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index df2d187a..605bae54 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -285,8 +285,8 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, static void free_item(struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { - scoutfs_kvec_kfree(item->val); scoutfs_kvec_kfree(item->key); + scoutfs_kvec_kfree(item->val); kfree(item); } } From 07ba01f6b0257d760f2d7df0eec1f1184f92d1db Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 08:53:56 -0800 Subject: [PATCH 157/920] Iniitialize segment header when writing item Initialize the segment header as the items are written. This isn't a great place to do it. Signed-off-by: Zach Brown --- kmod/src/seg.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 743575ed..a4a85de4 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -474,6 +474,10 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, u32 key_off; u32 val_off; + /* XXX the segment block header is a mess, be better */ + sblk->segno = cpu_to_le64(seg->segno); + sblk->max_seq = cpu_to_le64(1); + key_off = pos_off(seg, nr_items); val_off = key_off + key_bytes; From 5eb388ae6e4bf641099c50ec35c699e343db6036 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 08:57:59 -0800 Subject: [PATCH 158/920] Fix seg item filling The two functions that added to items had little bugs. They initialized the item vectors incorrectly and didn't actually store the keys and values. Appending was always overwriting the first segment. Have it call 'nr' 'pos' like the rest of the code to make it more clear. Signed-off-by: Zach Brown --- kmod/src/seg.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index a4a85de4..4c125f1c 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -490,7 +490,7 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, item.val_len = scoutfs_kvec_length(val); store_item(seg, 0, &item); - scoutfs_seg_item_kvecs(seg, 0, key, val); + scoutfs_seg_item_kvecs(seg, 0, item_key, item_val); scoutfs_kvec_memcpy(item_key, key); scoutfs_kvec_memcpy(item_val, val); } @@ -504,21 +504,21 @@ void scoutfs_seg_append_item(struct super_block *sb, struct native_item prev; SCOUTFS_DECLARE_KVEC(item_key); SCOUTFS_DECLARE_KVEC(item_val); - u32 nr; + u32 pos; - nr = le32_to_cpu(sblk->nr_items); - sblk->nr_items = cpu_to_le32(nr + 1); + pos = le32_to_cpu(sblk->nr_items); + sblk->nr_items = cpu_to_le32(pos + 1); - load_item(seg, nr - 1, &prev); + load_item(seg, pos - 1, &prev); item.seq = 1; item.key_off = prev.key_off + prev.key_len; item.key_len = scoutfs_kvec_length(key); item.val_off = prev.val_off + prev.val_len; item.val_len = scoutfs_kvec_length(val); - store_item(seg, 0, &item); + store_item(seg, pos, &item); - scoutfs_seg_item_kvecs(seg, nr, key, val); + scoutfs_seg_item_kvecs(seg, pos, item_key, item_val); scoutfs_kvec_memcpy(item_key, key); scoutfs_kvec_memcpy(item_val, val); } From fbd12b4ddaa0bc1fbdf403074414431395e88af9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:10:55 -0800 Subject: [PATCH 159/920] Fix existing item insertion Inserting an item over an existing key was super broken. Now that we're not replacing we can't stop descent if we find an existing item. We need to keep descending and then insert. And the caller needs to, you know, actually remove the existing item when it's found -- not the item it just inserted :P. Signed-off-by: Zach Brown --- kmod/src/item.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 605bae54..89c14e76 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -122,17 +122,19 @@ static struct cached_item *insert_item(struct rb_root *root, item = container_of(*node, struct cached_item, node); cmp = scoutfs_kvec_memcmp(ins->key, item->key); + if (cmp == 0) { + BUG_ON(existing); + existing = item; + } + if (cmp < 0) { if (ins->dirty) item->dirty |= LEFT_DIRTY; node = &(*node)->rb_left; - } else if (cmp > 0) { + } else { if (ins->dirty) item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; - } else { - existing = item; - break; } } @@ -376,7 +378,7 @@ static int add_item(struct super_block *sb, struct kvec *key, struct kvec *val, existing = insert_item(&cac->root, item); if (existing) { clear_item_dirty(cac, existing); - rb_erase_augmented(&item->node, &cac->root, + rb_erase_augmented(&existing->node, &cac->root, &scoutfs_item_rb_cb); } mark_item_dirty(cac, item); From a8a6d3697b43149dd87e91bd6213325877909396 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:14:01 -0800 Subject: [PATCH 160/920] Fix dirty item counter calculations Unfortunately the generic augmented callbacks don't work for our augmented node bits which specifically reflect the left and right nodes. We need our own rotation callback and then we have boilerplate for the other two copy and propagate callbacks. Once we have to provide .propagate we can call it instead of our own update_dirty_parents() equivalent. In addition some callers messed up marking and clearing dirty. We only want to mark dirty item insertions, not all inserted items. And if we update an item's keys and values we need to clear and mark it to keep the counters consistent. Signed-off-by: Zach Brown --- kmod/src/item.c | 99 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 89c14e76..7f0cc3d7 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -27,9 +27,9 @@ struct item_cache { spinlock_t lock; struct rb_root root; - unsigned long nr_dirty_items; - unsigned long dirty_key_bytes; - unsigned long dirty_val_bytes; + long nr_dirty_items; + long dirty_key_bytes; + long dirty_val_bytes; }; /* @@ -100,8 +100,63 @@ static long compute_item_dirty(struct cached_item *item) node_dirty_bit(item->node.rb_right, RIGHT_DIRTY); } -RB_DECLARE_CALLBACKS(static, scoutfs_item_rb_cb, struct cached_item, node, - long, dirty, compute_item_dirty); +static void scoutfs_item_rb_propagate(struct rb_node *node, + struct rb_node *stop) +{ + struct cached_item *item; + long dirty; + + while (node != stop) { + item = container_of(node, struct cached_item, node); + dirty = compute_item_dirty(item); + + if (item->dirty == dirty) + break; + + item->dirty = dirty; + node = rb_parent(&item->node); + } +} + +static void scoutfs_item_rb_copy(struct rb_node *old, struct rb_node *new) +{ + struct cached_item *o = container_of(old, struct cached_item, node); + struct cached_item *n = container_of(new, struct cached_item, node); + + n->dirty = o->dirty; +} + +/* calculate the new parent last as it depends on the old parent */ +static void scoutfs_item_rb_rotate(struct rb_node *old, struct rb_node *new) +{ + struct cached_item *o = container_of(old, struct cached_item, node); + struct cached_item *n = container_of(new, struct cached_item, node); + + BUG_ON(rb_parent(old) != new); + + o->dirty = compute_item_dirty(o); + n->dirty = compute_item_dirty(n); +} + +/* + * The generic RB_DECLARE_CALLBACKS() helpers are built for augmented + * values that are simple commutative function of the left and right + * children's augmented values. During rotation the new parent just + * gets the old parent's augmented value and then the old parent's value + * is calculated. + * + * Our dirty bits don't work that way. They are not just an or of the + * child's bits, the bits depend on the left and right children + * specifically. During rotation both parents need to be specifically + * recalculated. (They could be masked and asigned based on the + * direction of the rotation but that's annoying, let's just + * recalculate.) + */ +static const struct rb_augment_callbacks scoutfs_item_rb_cb = { + .propagate = scoutfs_item_rb_propagate, + .copy = scoutfs_item_rb_copy, + .rotate = scoutfs_item_rb_rotate, +}; /* * Always insert the given item. If there's an existing item it is @@ -293,28 +348,6 @@ static void free_item(struct cached_item *item) } } -/* - * The caller might have modified the item's dirty flags. Ascend - * through parents updating their dirty flags until there's no change. - */ -static void update_dirty_parents(struct cached_item *item) -{ - struct cached_item *parent; - struct rb_node *node; - long dirty; - - while ((node = rb_parent(&item->node))) { - parent = container_of(node, struct cached_item, node); - dirty = compute_item_dirty(parent); - - if (parent->dirty == dirty) - break; - - parent->dirty = dirty; - item = parent; - } -} - static void mark_item_dirty(struct item_cache *cac, struct cached_item *item) { @@ -329,7 +362,7 @@ static void mark_item_dirty(struct item_cache *cac, cac->dirty_key_bytes += scoutfs_kvec_length(item->key); cac->dirty_val_bytes += scoutfs_kvec_length(item->val); - update_dirty_parents(item); + scoutfs_item_rb_propagate(&item->node, NULL); } static void clear_item_dirty(struct item_cache *cac, @@ -346,7 +379,10 @@ static void clear_item_dirty(struct item_cache *cac, cac->dirty_key_bytes -= scoutfs_kvec_length(item->key); cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); - update_dirty_parents(item); + WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || + cac->dirty_val_bytes < 0); + + scoutfs_item_rb_propagate(&item->node, NULL); } /* @@ -381,7 +417,8 @@ static int add_item(struct super_block *sb, struct kvec *key, struct kvec *val, rb_erase_augmented(&existing->node, &cac->root, &scoutfs_item_rb_cb); } - mark_item_dirty(cac, item); + if (dirty) + mark_item_dirty(cac, item); spin_unlock_irqrestore(&cac->lock, flags); free_item(existing); @@ -468,6 +505,8 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, /* XXX update seq */ item = find_item(&cac->root, key); if (item) { + /* keep dirty counters in sync */ + clear_item_dirty(cac, item); scoutfs_kvec_swap(up_val, item->val); mark_item_dirty(cac, item); } else { From 454767e9927b3ac2c88af207a7c051ba84359f1d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:17:06 -0800 Subject: [PATCH 161/920] Stop first dirty search looping first_dirty() forgot to stop if the tree had no dirty items at all. It'd just spin forever. Signed-off-by: Zach Brown --- kmod/src/item.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 7f0cc3d7..379a5dc4 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -549,6 +549,8 @@ static struct cached_item *first_dirty(struct rb_node *node) break; } else if (item->dirty & RIGHT_DIRTY) { node = item->node.rb_right; + } else { + break; } } From e418629beaf63e5502795244c0a7e1b5f454bae3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:17:30 -0800 Subject: [PATCH 162/920] Remove items from trees before freeing Signed-off-by: Zach Brown --- kmod/src/item.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 379a5dc4..700945aa 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -706,6 +706,7 @@ void scoutfs_item_destroy(struct super_block *sb) for (node = rb_first(&cac->root); node; ) { item = container_of(node, struct cached_item, node); node = rb_next(node); + rb_erase(&item->node, &cac->root); free_item(item); } From fd7b09b4e47a1c4346d9aa30252a33e3c63d98b2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:19:38 -0800 Subject: [PATCH 163/920] Fix written manifest entry length Manifest entries were being written with the size of their in-memory nodes, not the smaller persistent add_manifest structure size. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2901c724..9858a168 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -212,12 +212,11 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, SCOUTFS_DECLARE_KVEC(ment_first); SCOUTFS_DECLARE_KVEC(ment_last); unsigned long flags; - int bytes; + int key_bytes; int ret; - bytes = sizeof(struct manifest_entry) + scoutfs_kvec_length(first), - scoutfs_kvec_length(last); - ment = kmalloc(bytes, GFP_NOFS); + key_bytes = scoutfs_kvec_length(first) + scoutfs_kvec_length(last); + ment = kmalloc(sizeof(struct manifest_entry) + key_bytes, GFP_NOFS); if (!ment) return -ENOMEM; @@ -228,7 +227,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, INIT_LIST_HEAD(&ment->dirty_entry); ment->am.eh.type = SCOUTFS_RING_ADD_MANIFEST; - ment->am.eh.len = cpu_to_le16(bytes); + ment->am.eh.len = cpu_to_le16(sizeof(struct scoutfs_ring_add_manifest) + + key_bytes); ment->am.segno = cpu_to_le64(segno); ment->am.seq = cpu_to_le64(seq); ment->am.first_key_len = cpu_to_le16(scoutfs_kvec_length(first)); From b8ede1f6ee4d1cf7dd15994664f7041e044cd019 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:39:46 -0800 Subject: [PATCH 164/920] Fix ring block tail zeroing The ring block tail zeroing memset treated the value as the offset to zero from, not the number of bytes at the tail to zero. Signed-off-by: Zach Brown --- kmod/src/ring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 3cda13b5..ca72d15b 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -57,7 +57,7 @@ struct ring_info { */ static void finish_block(struct scoutfs_ring_block *ring, unsigned int tail) { - memset((char *)ring + tail, 0, SCOUTFS_BLOCK_SIZE - tail); + memset((char *)ring + SCOUTFS_BLOCK_SIZE - tail, 0, tail); scoutfs_crc_block(&ring->hdr); } From b598bf855daa8bc992c63790b993497a3997b2fe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 09:41:22 -0800 Subject: [PATCH 165/920] Fix ring appending and writing The ring appending next entry header cursor assignment was pointing past the caller's src header, not at the next header to write to in the block. The writing block index and blkno calculations were just bad. Pretend they never happened. And finally we need to point the dirty super at the ring index for the commit and we need to reset the append state for the next commit. Signed-off-by: Zach Brown --- kmod/src/ring.c | 77 +++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index ca72d15b..a3b46bdd 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -69,7 +69,7 @@ void scoutfs_ring_append(struct super_block *sb, unsigned int len = le16_to_cpu(eh->len); if (rinf->space < len) { - if (ring) + if (rinf->space) finish_block(ring, rinf->space); ring = scoutfs_page_block_address(rinf->pages, rinf->nr_blocks); rinf->ring = ring; @@ -84,23 +84,42 @@ void scoutfs_ring_append(struct super_block *sb, } memcpy(rinf->next_eh, eh, len); - rinf->next_eh = (void *)((char *)eh + len); + rinf->next_eh = (void *)rinf->next_eh + len; rinf->space -= len; } +static u64 ring_ind_wrap(struct scoutfs_super_block *super, u64 ind) +{ + u64 ring_blocks = le64_to_cpu(super->ring_blocks); + + while (ind >= ring_blocks) + ind -= ring_blocks; + + return ind; +} + /* - * Kick off the writes to update the ring. Update the dirty super to - * reference the written ring. + * Submit writes for all the dirty ring blocks that accumulated as dirty + * entries were appended. The dirty ring blocks are contiguous in the + * page array but can wrap in the block ring on disk. + * + * If it wraps then we submit the earlier fragment at the head of the + * ring first. + * + * The wrapped fragment starts at some block offset in the page array. + * The hacky page array math only works when our fixed 4k block size == + * page_size. To fix it we'd add a offset block to the bio submit loop + * which could add an initial partial page vec to the bios. */ int scoutfs_ring_submit_write(struct super_block *sb, struct scoutfs_bio_completion *comp) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_RING_INFO(sb, rinf); + u64 first_blocks; u64 head_blocks; - u64 blocks; - u64 blkno; - u64 ind; + u64 first; + u64 last; if (!rinf->nr_blocks) return 0; @@ -108,37 +127,33 @@ int scoutfs_ring_submit_write(struct super_block *sb, if (rinf->space) finish_block(rinf->ring, rinf->space); - ind = le64_to_cpu(super->ring_tail_index) + 1; - blocks = rinf->nr_blocks; - blkno = le64_to_cpu(super->ring_blkno) + ind; + /* first and last ring block indexes that will be written */ + first = ring_ind_wrap(super, le64_to_cpu(super->ring_tail_index) + 1); + last = ring_ind_wrap(super, first + rinf->nr_blocks - 1); - /* - * If the log wrapped then we have to write two fragments to the - * tail and head of the ring. We submit the head fragment - * first. - * - * The head fragment starts at some block offset in the - * preallocated pages. This hacky page math only works when our - * 4k blocks size == page_size. To fix it we'd add a offset - * block to the bio submit loop which could add an initial - * partial page vec to the bios. - */ - BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); + /* number of blocks to write from first index and from head of ring */ + first_blocks = min(last - first + 1, + le64_to_cpu(super->ring_blocks) - first); + if (last < first) + head_blocks = last + 1; + else + head_blocks = 0; - if (ind + blocks > le64_to_cpu(super->ring_blocks)) { - head_blocks = (ind + blocks) - le64_to_cpu(super->ring_blocks); - blocks -= head_blocks; - scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + blocks, + if (head_blocks) { + BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); + scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + first_blocks, le64_to_cpu(super->ring_blkno), head_blocks, comp); } - scoutfs_bio_submit_comp(sb, WRITE, rinf->pages, blkno, blocks, comp); + scoutfs_bio_submit_comp(sb, WRITE, rinf->pages, + le64_to_cpu(super->ring_blkno) + first, + first_blocks, comp); - ind += blocks; - if (ind == le64_to_cpu(super->ring_blocks)) - ind = 0; - super->ring_tail_index = cpu_to_le64(ind); + /* record new tail index in super and reset for next trans */ + super->ring_tail_index = cpu_to_le64(last); + rinf->nr_blocks = 0; + rinf->space = 0; return 0; } From f9ca1885f945cbba03292abf24ddb5f865145148 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 11:19:43 -0800 Subject: [PATCH 166/920] Specify ring blocks with index,nr Specifying the ring blocks with a head and tail index lead to pretty confusing code to figure out how many blocks to read and if we had passed the tail. Instead specify the ring with a starting index and number of blocks. The code to read and write the ring blocks naturally falls out and is a lot more clear. Signed-off-by: Zach Brown --- kmod/src/format.h | 6 ++--- kmod/src/ring.c | 65 +++++++++++++++++++---------------------------- 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 2f126610..2124d970 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -314,9 +314,9 @@ struct scoutfs_super_block { __le64 free_blocks; __le64 ring_blkno; __le64 ring_blocks; - __le64 ring_head_index; - __le64 ring_tail_index; - __le64 ring_head_seq; + __le64 ring_index; + __le64 ring_nr; + __le64 ring_seq; __le64 buddy_blocks; struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; diff --git a/kmod/src/ring.c b/kmod/src/ring.c index a3b46bdd..7b9bc6c4 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -116,10 +116,9 @@ int scoutfs_ring_submit_write(struct super_block *sb, { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_RING_INFO(sb, rinf); - u64 first_blocks; - u64 head_blocks; - u64 first; - u64 last; + u64 wrapped_blocks; + u64 index_blocks; + u64 index; if (!rinf->nr_blocks) return 0; @@ -128,30 +127,25 @@ int scoutfs_ring_submit_write(struct super_block *sb, finish_block(rinf->ring, rinf->space); /* first and last ring block indexes that will be written */ - first = ring_ind_wrap(super, le64_to_cpu(super->ring_tail_index) + 1); - last = ring_ind_wrap(super, first + rinf->nr_blocks - 1); + index = ring_ind_wrap(super, le64_to_cpu(super->ring_index) + + le64_to_cpu(super->ring_nr)); + index_blocks = min_t(u64, rinf->nr_blocks, + le64_to_cpu(super->ring_blocks) - index); + wrapped_blocks = rinf->nr_blocks - index_blocks; - /* number of blocks to write from first index and from head of ring */ - first_blocks = min(last - first + 1, - le64_to_cpu(super->ring_blocks) - first); - if (last < first) - head_blocks = last + 1; - else - head_blocks = 0; - - if (head_blocks) { + if (wrapped_blocks) { BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); - scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + first_blocks, + scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + index_blocks, le64_to_cpu(super->ring_blkno), - head_blocks, comp); + wrapped_blocks, comp); } scoutfs_bio_submit_comp(sb, WRITE, rinf->pages, - le64_to_cpu(super->ring_blkno) + first, - first_blocks, comp); + le64_to_cpu(super->ring_blkno) + index, + index_blocks, comp); /* record new tail index in super and reset for next trans */ - super->ring_tail_index = cpu_to_le64(last); + le64_add_cpu(&super->ring_nr, rinf->nr_blocks); rinf->nr_blocks = 0; rinf->space = 0; @@ -232,10 +226,10 @@ int scoutfs_ring_read(struct super_block *sb) struct page *page; u64 index; u64 blkno; - u64 tail; + u64 part; u64 seq; + u64 nr; int ret; - int nr; int i; /* nr_blocks/pages calc doesn't handle multiple pages per block */ @@ -255,20 +249,17 @@ int scoutfs_ring_read(struct super_block *sb) pages[i] = page; } - index = le64_to_cpu(super->ring_head_index); - tail = le64_to_cpu(super->ring_tail_index); - seq = le64_to_cpu(super->ring_head_seq); + index = le64_to_cpu(super->ring_index); + nr = le64_to_cpu(super->ring_nr); + seq = le64_to_cpu(super->ring_seq); - for(;;) { + while (nr) { blkno = le64_to_cpu(super->ring_blkno) + index; + /* XXX min3_t should be a thing */ + part = min3(nr, (u64)NR_BLOCKS, + le64_to_cpu(super->ring_blocks) - index); - if (index <= tail) - nr = tail - index + 1; - else - nr = le64_to_cpu(super->ring_blocks) - index; - nr = min_t(int, nr, NR_BLOCKS); - - trace_printk("index %llu tail %llu nr %u\n", index, tail, nr); + trace_printk("index %llu nr %llu\n", index, nr); ret = scoutfs_bio_read(sb, pages, blkno, nr); if (ret) @@ -283,12 +274,8 @@ int scoutfs_ring_read(struct super_block *sb) goto out; } - if (index == tail) - break; - - index += nr; - if (index == le64_to_cpu(super->ring_blocks)) - index = 0; + index = ring_ind_wrap(super, index + part); + nr -= part; } out: From a5cac107a16115582b5916f6dbd6a523ff9643bf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 13:14:45 -0800 Subject: [PATCH 167/920] Set END_IO on allocated segs A reader that hits an allocated segment would wait on IO forever. Setting the end_io bit lets readers use written segments. Signed-off-by: Zach Brown --- kmod/src/seg.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 4c125f1c..d4eeb32f 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -232,6 +232,9 @@ int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret) goto out; } + /* reads shouldn't wait for this */ + set_bit(SF_END_IO, &seg->flags); + /* XXX always remove existing segs, is that necessary? */ spin_lock_irqsave(&cac->lock, flags); atomic_inc(&seg->refcount); From b7b43de8c7ffc07ad830aa8a36dc29b277ac5b8c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:30:51 -0800 Subject: [PATCH 168/920] Queue trans work in our work queue We went to the trouble of allocating a work queue with one work in flight but then didn't use it. We could have concurrent trans write func execution. Signed-off-by: Zach Brown --- kmod/src/trans.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index ab3da53d..1e23295f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -157,6 +157,11 @@ static int write_attempted(struct scoutfs_sb_info *sbi, return done; } +static void queue_trans_work(struct scoutfs_sb_info *sbi) +{ + queue_work(sbi->trans_write_workq, &sbi->trans_write_work); +} + /* * sync records the current dirty seq and write count and waits for * either to change. If there's nothing to write or the write returned @@ -170,7 +175,7 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) int ret; if (!wait) { - schedule_work(&sbi->trans_write_work); + queue_trans_work(sbi); return 0; } @@ -179,7 +184,7 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) attempt.count = sbi->trans_write_count; spin_unlock(&sbi->trans_write_lock); - schedule_work(&sbi->trans_write_work); + queue_trans_work(sbi); ret = wait_event_interruptible(sbi->trans_write_wq, write_attempted(sbi, &attempt)); From b251f918428cb94a5c7d84e932202d43b6eaa0e3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:33:05 -0800 Subject: [PATCH 169/920] Fix updating parent dirty bits We were trying to propagate dirty bits from a node itself when its dirty bit is set. But it's bits are consistent so it stops immediately. We need to propagate from the parent of the node that changed. Signed-off-by: Zach Brown --- kmod/src/item.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 700945aa..eb453755 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -348,6 +348,17 @@ static void free_item(struct cached_item *item) } } +/* + * The caller has changed an item's dirty bit. Its child dirty bits are + * still consistent. But its parent's bits might need to be updated. + * Its bits are consistent so we don't propagate from the node itself + * because it would immediately terminate. + */ +static void update_dirty_parents(struct cached_item *item) +{ + scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); +} + static void mark_item_dirty(struct item_cache *cac, struct cached_item *item) { @@ -362,7 +373,7 @@ static void mark_item_dirty(struct item_cache *cac, cac->dirty_key_bytes += scoutfs_kvec_length(item->key); cac->dirty_val_bytes += scoutfs_kvec_length(item->val); - scoutfs_item_rb_propagate(&item->node, NULL); + update_dirty_parents(item); } static void clear_item_dirty(struct item_cache *cac, @@ -382,7 +393,7 @@ static void clear_item_dirty(struct item_cache *cac, WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || cac->dirty_val_bytes < 0); - scoutfs_item_rb_propagate(&item->node, NULL); + update_dirty_parents(item); } /* From 967e90e5ef728aa9cd5d680ef8f14f1e1d0be2ba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:35:52 -0800 Subject: [PATCH 170/920] Fix kvec overlapping comparison The comparisons were a bit wrong when comparing overlaping kvec endpoints. We want to compare the starts and ends with the ends and starts, respectively. Signed-off-by: Zach Brown --- kmod/src/kvec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 0e0b1f84..0db41c87 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -71,8 +71,8 @@ int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, struct kvec *c, struct kvec *d) { - return scoutfs_kvec_memcmp(a, c) < 0 ? -1 : - scoutfs_kvec_memcmp(b, d) > 0 ? 1 : 0; + return scoutfs_kvec_memcmp(b, c) < 0 ? -1 : + scoutfs_kvec_memcmp(a, d) > 0 ? 1 : 0; } /* From acee97ba2a6aacc0acf4e5dc35dca9f16c45e25a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:37:31 -0800 Subject: [PATCH 171/920] Fix tail ring entry zeroing The space calculation didn't include the terminating zero entry. That ensured that the space for the netry would never be consumed. But the remaining space was used to zero the end of the block so the final entry wasn't being zeroed. So have the space remaining include the terminating entry and factor that into the space consumption of each entry being appended. Signed-off-by: Zach Brown --- kmod/src/ring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 7b9bc6c4..bc5e7db7 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -68,7 +68,7 @@ void scoutfs_ring_append(struct super_block *sb, struct scoutfs_ring_block *ring = rinf->ring; unsigned int len = le16_to_cpu(eh->len); - if (rinf->space < len) { + if (rinf->space < (len + sizeof(struct scoutfs_ring_entry_header))) { if (rinf->space) finish_block(ring, rinf->space); ring = scoutfs_page_block_address(rinf->pages, rinf->nr_blocks); @@ -79,8 +79,7 @@ void scoutfs_ring_append(struct super_block *sb, rinf->nr_blocks++; rinf->next_eh = ring->entries; rinf->space = SCOUTFS_BLOCK_SIZE - - offsetof(struct scoutfs_ring_block, entries) - - sizeof(struct scoutfs_ring_entry_header); + offsetof(struct scoutfs_ring_block, entries); } memcpy(rinf->next_eh, eh, len); From 51a84447dd972d0223254e9ef39778837c0ef9ab Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:39:41 -0800 Subject: [PATCH 172/920] Fix calculation of number of dirty segments The estimate for the number of dirty segment bytes was wildly over calculating the number of segment headers by confusing the length of the segment header with the length of segments. Signed-off-by: Zach Brown --- kmod/src/item.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index eb453755..ca9a555d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -625,8 +625,8 @@ long scoutfs_item_dirty_bytes(struct super_block *sb) spin_unlock_irqrestore(&cac->lock, flags); - bytes += DIV_ROUND_UP(bytes, sizeof(struct scoutfs_segment_block)) * - sizeof(struct scoutfs_segment_block); + bytes += DIV_ROUND_UP(bytes, SCOUTFS_SEGMENT_SIZE) * + sizeof(struct scoutfs_segment_block); return bytes; } From c74787a848b356573b5e238af2bb1e4221c6b467 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:40:48 -0800 Subject: [PATCH 173/920] Harden, simplify, and shrink the kvecs The initial kvec code was a bit wobbly. It had raw loops, some weird constructs, and had more elements than we need. Add some iterator helpers that make it less likely that we'll screw up iterating over different length vectors. Get rid of reliance on a tailing null pointer and always use the count of elements to stop iterating. With that in place we can shrink the number of elements to just the greatest user. Signed-off-by: Zach Brown --- kmod/src/kvec.c | 120 ++++++++++++++++++++++++++++++------------------ kmod/src/kvec.h | 14 ++---- 2 files changed, 81 insertions(+), 53 deletions(-) diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 0db41c87..f87e318d 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -31,6 +31,48 @@ #include "kvec.h" #include "scoutfs_trace.h" +struct iter { + struct kvec *kvec; + size_t off; + size_t i; +}; + +static void iter_advance(struct iter *iter, size_t len) +{ + iter->off += len; + + while (iter->i < SCOUTFS_KVEC_NR && iter->off >= iter->kvec->iov_len) { + iter->off -= iter->kvec->iov_len; + iter->kvec++; + iter->i++; + } +} + +static void iter_init(struct iter *iter, struct kvec *kvec) +{ + iter->kvec = kvec; + iter->i = 0; + iter->off = 0; + + iter_advance(iter, 0); +} + +static void *iter_ptr(struct iter *iter) +{ + if (iter->i < SCOUTFS_KVEC_NR) + return iter->kvec->iov_base + iter->off; + else + return NULL; +} + +static size_t iter_contig(struct iter *iter) +{ + if (iter->i < SCOUTFS_KVEC_NR) + return iter->kvec->iov_len - iter->off; + else + return 0; +} + /* * Return the result of memcmp between the min of the two total lengths. * If their shorter lengths are equal than the shorter length is considered @@ -38,30 +80,24 @@ */ int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) { - int b_off = 0; - int a_off = 0; - int len; + struct iter a_iter; + struct iter b_iter; + size_t len; int ret; - while (a->iov_base && b->iov_base) { - len = min(a->iov_len - a_off, b->iov_len - b_off); - ret = memcmp(a->iov_base + a_off, b->iov_base + b_off, len); + iter_init(&a_iter, a); + iter_init(&b_iter, b); + + while ((len = min(iter_contig(&a_iter), iter_contig(&b_iter)))) { + ret = memcmp(iter_ptr(&a_iter), iter_ptr(&b_iter), len); if (ret) return ret; - b_off += len; - if (b_off == b->iov_len) { - b++; - b_off = 0; - } - a_off += len; - if (a_off == a->iov_len) { - a++; - a_off = 0; - } + iter_advance(&a_iter, len); + iter_advance(&b_iter, len); } - return a->iov_base ? 1 : b->iov_base ? -1 : 0; + return iter_contig(&a_iter) ? 1 : iter_contig(&b_iter) ? -1 : 0; } /* @@ -84,7 +120,7 @@ void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src) int i; for (i = 0; i < SCOUTFS_KVEC_NR; i++) - *(dst++) = *(src++); + dst[i] = src[i]; } /* @@ -94,27 +130,20 @@ void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src) */ int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src) { - int src_off = 0; - int dst_off = 0; - int copied = 0; - int len; + struct iter dst_iter; + struct iter src_iter; + size_t copied = 0; + size_t len; - while (dst->iov_base && src->iov_base) { - len = min(dst->iov_len - dst_off, src->iov_len - src_off); - memcpy(dst->iov_base + dst_off, src->iov_base + src_off, len); + iter_init(&dst_iter, dst); + iter_init(&src_iter, src); + + while ((len = min(iter_contig(&dst_iter), iter_contig(&src_iter)))) { + memcpy(iter_ptr(&dst_iter), iter_ptr(&src_iter), len); copied += len; - - src_off += len; - if (src_off == src->iov_len) { - src++; - src_off = 0; - } - dst_off += len; - if (dst_off == dst->iov_len) { - dst++; - dst_off = 0; - } + iter_advance(&dst_iter, len); + iter_advance(&src_iter, len); } return copied; @@ -161,25 +190,28 @@ int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src) } /* - * Free all the set pointers in the kvec. The pointer values aren't modified - * if they're freed. + * Free all the set pointers in the kvec. */ void scoutfs_kvec_kfree(struct kvec *kvec) { - while (kvec->iov_base) - kfree((kvec++)->iov_base); + int i; + + for (i = 0; i < SCOUTFS_KVEC_NR; i++) { + kfree(kvec[i].iov_base); + kvec[i].iov_base = NULL; + } } void scoutfs_kvec_init_null(struct kvec *kvec) { - memset(kvec, 0, SCOUTFS_KVEC_NR * sizeof(kvec[0])); + memset(kvec, 0, SCOUTFS_KVEC_BYTES); } void scoutfs_kvec_swap(struct kvec *a, struct kvec *b) { SCOUTFS_DECLARE_KVEC(tmp); - memcpy(tmp, a, sizeof(tmp)); - memcpy(a, b, sizeof(tmp)); - memcpy(b, tmp, sizeof(tmp)); + memcpy(tmp, a, SCOUTFS_KVEC_BYTES); + memcpy(a, b, SCOUTFS_KVEC_BYTES); + memcpy(b, tmp, SCOUTFS_KVEC_BYTES); } diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index 49d51ae9..2eb6528b 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -11,7 +11,8 @@ /* * This ends up defining the max item size as nr - 1 * page _size. */ -#define SCOUTFS_KVEC_NR 4 +#define SCOUTFS_KVEC_NR 2 +#define SCOUTFS_KVEC_BYTES (SCOUTFS_KVEC_NR * sizeof(struct kvec)) #define SCOUTFS_DECLARE_KVEC(name) \ struct kvec name[SCOUTFS_KVEC_NR] @@ -19,19 +20,14 @@ static inline void scoutfs_kvec_init_all(struct kvec *kvec, void *ptr0, size_t len0, void *ptr1, size_t len1, - void *ptr2, size_t len2, - void *ptr3, size_t len3, ...) + void *ptr2, ...) { - BUG_ON(ptr3 != NULL); + BUG_ON(ptr2 != NULL); kvec[0].iov_base = ptr0; kvec[0].iov_len = len0; kvec[1].iov_base = ptr1; kvec[1].iov_len = len1; - kvec[2].iov_base = ptr2; - kvec[2].iov_len = len2; - kvec[3].iov_base = ptr3; - kvec[3].iov_len = len3; } /* @@ -43,7 +39,7 @@ static inline void scoutfs_kvec_init_all(struct kvec *kvec, * arguments in the static inline. */ #define scoutfs_kvec_init(val, ...) \ - scoutfs_kvec_init_all(val, __VA_ARGS__, NULL, 0, NULL, 0, NULL, 0) + scoutfs_kvec_init_all(val, __VA_ARGS__, NULL, 0, NULL, 0) static inline int scoutfs_kvec_length(struct kvec *kvec) { From c8d61c2e013f7a8fa0dbe25e935f01663eceab01 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:43:10 -0800 Subject: [PATCH 174/920] Manifest item reading tracked wrong key When iterating over items the manifest would always insert whatever values it found at the caller's key, instead of the key that it found in the segment. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 9858a168..cfc46d0b 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -425,7 +425,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) } /* remember new least key */ - scoutfs_kvec_clone(found_key, key); + scoutfs_kvec_clone(found_key, item_key); scoutfs_kvec_clone(found_val, item_val); found = true; had_found = 0; From 1b4bab3217b9d9a7610979b4304e2cdb2e42d900 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Dec 2016 23:43:54 -0800 Subject: [PATCH 175/920] Fix ring read nr/part confusion Some parts of the ring reading were still using the old 'nr' for the number of blocks to read, but it's now the total number of blocks in the ring. Use part instead. Signed-off-by: Zach Brown --- kmod/src/ring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index bc5e7db7..050fa49c 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -258,15 +258,15 @@ int scoutfs_ring_read(struct super_block *sb) part = min3(nr, (u64)NR_BLOCKS, le64_to_cpu(super->ring_blocks) - index); - trace_printk("index %llu nr %llu\n", index, nr); + trace_printk("index %llu part %llu\n", index, part); - ret = scoutfs_bio_read(sb, pages, blkno, nr); + ret = scoutfs_bio_read(sb, pages, blkno, part); if (ret) goto out; /* XXX verify block header */ - for (i = 0; i < nr; i++) { + for (i = 0; i < part; i++) { ring = scoutfs_page_block_address(pages, i); ret = read_entries(sb, ring); if (ret) From 57a6ff087fc0335a777f4be47b0f28cd19cfb22f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Dec 2016 14:57:38 -0800 Subject: [PATCH 176/920] Add max key and max key size to format We're going to use these to support tracking cached item ranges. Signed-off-by: Zach Brown --- kmod/src/format.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/format.h b/kmod/src/format.h index 2124d970..71027e90 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -213,6 +213,7 @@ struct scoutfs_key { #define SCOUTFS_SYMLINK_KEY 8 #define SCOUTFS_EXTENT_KEY 9 #define SCOUTFS_ORPHAN_KEY 10 +#define SCOUTFS_MAX_UNUSED_KEY 255 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -441,4 +442,7 @@ struct scoutfs_link_backref { __le64 offset; } __packed; +#define SCOUTFS_MAX_KEY_SIZE \ + offsetof(struct scoutfs_dirent_key, name[SCOUTFS_NAME_LEN]) + #endif From cbb4282429e30a11c554032009cbce1c3a886489 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Dec 2016 15:01:44 -0800 Subject: [PATCH 177/920] Add kvec key helpers Add a suite of simple kvec functions to work with kvecs that point to file system keys. The ones worth mentioning format keys into strings. They're used to add formatted strings for the keys to tracepoints. They're still a little rough but this is a functional first step. Signed-off-by: Zach Brown --- kmod/src/kvec.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++- kmod/src/kvec.h | 6 ++ 2 files changed, 204 insertions(+), 3 deletions(-) diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index f87e318d..349e55b6 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -33,6 +34,7 @@ struct iter { struct kvec *kvec; + size_t count; size_t off; size_t i; }; @@ -40,6 +42,7 @@ struct iter { static void iter_advance(struct iter *iter, size_t len) { iter->off += len; + iter->count -= len; while (iter->i < SCOUTFS_KVEC_NR && iter->off >= iter->kvec->iov_len) { iter->off -= iter->kvec->iov_len; @@ -53,6 +56,7 @@ static void iter_init(struct iter *iter, struct kvec *kvec) iter->kvec = kvec; iter->i = 0; iter->off = 0; + iter->count = scoutfs_kvec_length(kvec); iter_advance(iter, 0); } @@ -65,6 +69,7 @@ static void *iter_ptr(struct iter *iter) return NULL; } +/* count of contiguous bytes available at the next vector */ static size_t iter_contig(struct iter *iter) { if (iter->i < SCOUTFS_KVEC_NR) @@ -73,6 +78,12 @@ static size_t iter_contig(struct iter *iter) return 0; } +/* count of bytes remaining in the iteration */ +static size_t iter_count(struct iter *iter) +{ + return iter->count; +} + /* * Return the result of memcmp between the min of the two total lengths. * If their shorter lengths are equal than the shorter length is considered @@ -101,8 +112,9 @@ int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) } /* - * Returns 0 if [a,b] overlaps with [c,d]. Returns -1 if a < c and - * 1 if b > d. + * Return -1 if [a,b] doesn't overlap with and is to the left of [c,d], + * 1 if it doesn't overlap and is to the right of, and 0 if they + * overlap. */ int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, struct kvec *c, struct kvec *d) @@ -181,8 +193,10 @@ int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src) size_t len = scoutfs_kvec_length(src); ptr = kmalloc(len, GFP_NOFS); - if (!ptr) + if (!ptr) { + scoutfs_kvec_init_null(dst); return -ENOMEM; + } scoutfs_kvec_init(dst, ptr, len); scoutfs_kvec_memcpy(dst, src); @@ -215,3 +229,184 @@ void scoutfs_kvec_swap(struct kvec *a, struct kvec *b) memcpy(a, b, SCOUTFS_KVEC_BYTES); memcpy(b, tmp, SCOUTFS_KVEC_BYTES); } + +int scoutfs_kvec_alloc_key(struct kvec *kvec) +{ + const size_t len = SCOUTFS_MAX_KEY_SIZE; + void *ptr; + + ptr = kzalloc(len, GFP_NOFS); + if (!ptr) { + scoutfs_kvec_init_null(kvec); + return -ENOMEM; + } + + scoutfs_kvec_init(kvec, ptr, len); + return 0; +} + +void scoutfs_kvec_init_key(struct kvec *kvec) +{ + scoutfs_kvec_init(kvec, kvec[0].iov_base, SCOUTFS_MAX_KEY_SIZE); +} + +void scoutfs_kvec_set_max_key(struct kvec *kvec) +{ + __u8 *type = kvec[0].iov_base; + + *type = SCOUTFS_MAX_UNUSED_KEY; + scoutfs_kvec_init(kvec, type, 1); +} + +/* + * Clone the source kvec into the dst if the dst is empty or if + * the src kvec is less than the dst. + */ +void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src) +{ + if (scoutfs_kvec_length(dst) == 0 || + scoutfs_kvec_memcmp(src, dst) < 0) + scoutfs_kvec_clone(dst, src); +} + +/* + * Copy bytes from the kvec iterator into the dest buffer, zeroing the + * remainder of the buffer if there aren't enough bytes available in + * the iterator. If the tail bool is set then the kvec data is copied + * into the tail of the buffer and the head is zeroed. + */ +static bool iter_memcpy_zero(void *dst, struct iter *src, size_t len, bool tail) +{ + size_t ctg; + size_t diff; + + if (len == 0 || iter_count(src) == 0) + return false; + + if (iter_count(src) < len) { + diff = len - iter_count(src); + if (tail) { + memset(dst, 0, diff); + dst += diff; + } else { + memset(dst + len - diff, 0, diff); + } + len = iter_count(src); + } + + while ((ctg = min(len, iter_contig(src)))) { + memcpy(dst, iter_ptr(src), ctg); + iter_advance(src, ctg); + dst += ctg; + len -= ctg; + } + + return true; +} + +static int iter_puts_printable(char *dst, struct iter *src) +{ + int len = iter_count(src); + size_t ctg; + int i; + + while ((ctg = iter_contig(src))) { + memcpy(dst, iter_ptr(src), ctg); + iter_advance(src, ctg); + + for (i = 0; i < ctg; i++) { + if (!isprint(dst[i])) + dst[i] = '_'; + } + + dst += ctg; + } + + return len; +} + +#define EMPTY_STR "''" +#define U64_U_BYTES 20 +#define U64_D_BYTES 21 +#define U64_X_BYTES 16 + +/* + * XXX figure out what to do about corrupt keys. + */ + +unsigned scoutfs_kvec_key_strlen(struct kvec *key) +{ + struct iter iter; + unsigned len = 0; + u8 type; + + iter_init(&iter, key); + + if (iter_count(&iter) == 0) { + len = sizeof(EMPTY_STR) - 1; + goto out; + } + + iter_memcpy_zero(&type, &iter, sizeof(type), false); + + len = 4; /* "typ." */ + + switch(type) { + case SCOUTFS_INODE_KEY: + len += U64_U_BYTES; + break; + case SCOUTFS_DIRENT_KEY: + len += U64_U_BYTES + (iter_count(&iter) - 8); + break; + case SCOUTFS_MAX_UNUSED_KEY: + break; + default: + /* hex of everything after the type */ + len += (scoutfs_kvec_length(key) - 1) * 2; + break; + } + +out: + return len + 1; /* null term */ +} + +void scoutfs_kvec_key_sprintf(char *buf, struct kvec *key) +{ + struct iter iter; + __be64 be; + u8 type; + + iter_init(&iter, key); + + if (iter_contig(&iter) == 0) { + buf += sprintf(buf, EMPTY_STR); + goto done; + } + + iter_memcpy_zero(&type, &iter, sizeof(type), false); + + switch(type) { + case SCOUTFS_INODE_KEY: + buf += sprintf(buf, "ino."); + iter_memcpy_zero(&be, &iter, sizeof(be), false); + buf += sprintf(buf, "%llu", be64_to_cpu(be)); + break; + case SCOUTFS_DIRENT_KEY: + buf += sprintf(buf, "den."); + iter_memcpy_zero(&be, &iter, sizeof(be), false); + buf += sprintf(buf, "%llu.", be64_to_cpu(be)); + buf += iter_puts_printable(buf, &iter); + break; + case SCOUTFS_MAX_UNUSED_KEY: + buf += sprintf(buf, "max"); + break; + default: + buf += sprintf(buf, "unk."); + while (iter_memcpy_zero(&be, &iter, sizeof(be), true)) + buf += sprintf(buf, "%llx", be64_to_cpu(be)); + break; + } + +done: + *buf = '\0'; +} diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index 2eb6528b..93ddcd36 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -62,5 +62,11 @@ int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src); void scoutfs_kvec_kfree(struct kvec *kvec); void scoutfs_kvec_init_null(struct kvec *kvec); void scoutfs_kvec_swap(struct kvec *a, struct kvec *b); +int scoutfs_kvec_alloc_key(struct kvec *kvec); +void scoutfs_kvec_init_key(struct kvec *kvec); +void scoutfs_kvec_set_max_key(struct kvec *kvec); +void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src); +unsigned scoutfs_kvec_key_strlen(struct kvec *key); +void scoutfs_kvec_key_sprintf(char *buf, struct kvec *key); #endif From b8d7e042620997ee83196674e16b565dbb38d7ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Dec 2016 16:28:58 -0800 Subject: [PATCH 178/920] Add negative caching of item ranges The item cache only knew about present items in the rbtree. Attempts to read items that didn't exist would always trigger expensive manfifest and segment searches. This reworks the item cache and item reading code to support the notion of cached ranges of keys. When we read items we also communicate the range of keys that we searched. This lets the cache return negative lookups for key values in the search that don't have items. The item cache gets an rbtree of key ranges. Each item lookup method now uses it to determine if a missing item needs to trigger a read. Item reading is now performed in batches instead of one at a time. This lets us specify the cache range along with the batch and apply them all atomically under the lock. The item range code is much more robust now that it has to track the range of keys that it searches. The read items call now takes a range. It knows to look for all level0 segments that interesect that range, not just the first key. The manifest segment references now include the min and max keys for the segment so we can use those to define the item search range. Since the refs now include keys we no longer have them as a dumb allocated array but instead have a list of alloced ref structs. Signed-off-by: Zach Brown --- kmod/src/item.c | 567 ++++++++++++++++++++++++++++++--------- kmod/src/item.h | 6 + kmod/src/manifest.c | 320 +++++++++++++++------- kmod/src/manifest.h | 3 +- kmod/src/scoutfs_trace.h | 52 ++++ 5 files changed, 720 insertions(+), 228 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index ca9a555d..86a7e1a5 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -22,10 +22,22 @@ #include "manifest.h" #include "item.h" #include "seg.h" +#include "scoutfs_trace.h" + +/* + * A simple rbtree of cached items isolates the item API callers from + * the relatively expensive segment searches. + * + * The item cache uses an rbtree of key ranges to record regions of keys + * that are completely described by the items. This lets it return + * negative lookups cache hits for items that don't exist without having + * to constantly perform expensive segment searches. + */ struct item_cache { spinlock_t lock; - struct rb_root root; + struct rb_root items; + struct rb_root ranges; long nr_dirty_items; long dirty_key_bytes; @@ -35,38 +47,77 @@ struct item_cache { /* * The dirty bits track if the given item is dirty and if its child * subtrees contain any dirty items. + * + * The entry is only used when the items are in a private batch list + * before insertion. */ struct cached_item { - struct rb_node node; + union { + struct rb_node node; + struct list_head entry; + }; long dirty; SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); }; -static struct cached_item *find_item(struct rb_root *root, struct kvec *key) +struct cached_range { + struct rb_node node; + + SCOUTFS_DECLARE_KVEC(start); + SCOUTFS_DECLARE_KVEC(end); +}; + +/* + * Walk the item rbtree and return the item found and the next and + * prev items. + */ +static struct cached_item *walk_items(struct rb_root *root, struct kvec *key, + struct cached_item **prev, + struct cached_item **next) { struct rb_node *node = root->rb_node; - struct rb_node *parent = NULL; struct cached_item *item; int cmp; + *prev = NULL; + *next = NULL; + while (node) { - parent = node; item = container_of(node, struct cached_item, node); cmp = scoutfs_kvec_memcmp(key, item->key); - if (cmp < 0) + if (cmp < 0) { + *next = item; node = node->rb_left; - else if (cmp > 0) + } else if (cmp > 0) { + *prev = item; node = node->rb_right; - else + } else { return item; + } } return NULL; } +static struct cached_item *find_item(struct rb_root *root, struct kvec *key) +{ + struct cached_item *prev; + struct cached_item *next; + + return walk_items(root, key, &prev, &next); +} + +static struct cached_item *next_item(struct rb_root *root, struct kvec *key) +{ + struct cached_item *prev; + struct cached_item *next; + + return walk_items(root, key, &prev, &next) ?: next; +} + /* * We store the dirty bits in a single value so that the simple * augmented rbtree implementation gets a single scalar value to compare @@ -159,16 +210,13 @@ static const struct rb_augment_callbacks scoutfs_item_rb_cb = { }; /* - * Always insert the given item. If there's an existing item it is - * returned. This can briefly leave duplicate items in the tree until - * the caller removes the existing item. + * Try to insert the given item. If there's already an item with the + * insertion key then return -EEXIST. */ -static struct cached_item *insert_item(struct rb_root *root, - struct cached_item *ins) +static int insert_item(struct rb_root *root, struct cached_item *ins) { struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; - struct cached_item *existing = NULL; struct cached_item *item; int cmp; @@ -177,57 +225,176 @@ static struct cached_item *insert_item(struct rb_root *root, item = container_of(*node, struct cached_item, node); cmp = scoutfs_kvec_memcmp(ins->key, item->key); - if (cmp == 0) { - BUG_ON(existing); - existing = item; - } - if (cmp < 0) { if (ins->dirty) item->dirty |= LEFT_DIRTY; node = &(*node)->rb_left; - } else { + } else if (cmp > 0) { if (ins->dirty) item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; + } else { + return -EEXIST; } } rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); - return existing; + return 0; +} + +/* + * Return true if the given key is covered by a cached range. end is + * set to the end of the cached range. + * + * Return false if the given key isn't covered by a cached range and is + * instead in an uncached hole. end is set to the start of the next + * cached range. + */ +static bool check_range(struct rb_root *root, struct kvec *key, + struct kvec *end) +{ + struct rb_node *node = root->rb_node; + struct cached_range *next = NULL; + struct cached_range *rng; + int cmp; + + while (node) { + rng = container_of(node, struct cached_range, node); + + cmp = scoutfs_kvec_cmp_overlap(key, key, + rng->start, rng->end); + if (cmp < 0) { + next = rng; + node = node->rb_left; + } else if (cmp > 0) { + node = node->rb_right; + } else { + scoutfs_kvec_memcpy_truncate(end, rng->end); + return true; + } + } + + if (next) + scoutfs_kvec_memcpy_truncate(end, next->start); + else + scoutfs_kvec_set_max_key(end); + + return false; +} + +static void free_range(struct cached_range *rng) +{ + if (!IS_ERR_OR_NULL(rng)) { + scoutfs_kvec_kfree(rng->start); + scoutfs_kvec_kfree(rng->end); + kfree(rng); + } +} + +/* + * Insert a new cached range. It might overlap with any number of + * existing cached ranges. As we descend we combine with and free any + * overlapping ranges before restarting the descent. + * + * We're responsible for the ins allocation. We free it if we don't + * insert it in the tree. + */ +static void insert_range(struct rb_root *root, struct cached_range *ins) +{ + struct cached_range *rng; + struct rb_node *parent; + struct rb_node **node; + int start_cmp; + int end_cmp; + int cmp; + +restart: + parent = NULL; + node = &root->rb_node; + while (*node) { + parent = *node; + rng = container_of(*node, struct cached_range, node); + + cmp = scoutfs_kvec_cmp_overlap(ins->start, ins->end, + rng->start, rng->end); + /* simple iteration until we overlap */ + if (cmp < 0) { + node = &(*node)->rb_left; + continue; + } else if (cmp > 0) { + node = &(*node)->rb_right; + continue; + } + + start_cmp = scoutfs_kvec_memcmp(ins->start, rng->start); + end_cmp = scoutfs_kvec_memcmp(ins->end, rng->end); + + /* free our insertion if we're entirely within an existing */ + if (start_cmp >= 0 && end_cmp <= 0) { + free_range(ins); + return; + } + + /* expand to cover partial overlap before freeing */ + if (start_cmp < 0 && end_cmp < 0) + scoutfs_kvec_swap(ins->end, rng->end); + else if (start_cmp > 0 && end_cmp > 0) + scoutfs_kvec_swap(ins->start, rng->start); + + /* remove and free all overlaps and restart the descent */ + rb_erase(&rng->node, root); + free_range(rng); + goto restart; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); } /* * Find an item with the given key and copy its value into the caller's - * value vector. The amount of bytes copied is returned which can be - * 0 or truncated if the caller's buffer isn't big enough. + * value vector. The amount of bytes copied is returned which can be 0 + * or truncated if the caller's buffer isn't big enough. */ int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + SCOUTFS_DECLARE_KVEC(end); struct cached_item *item; unsigned long flags; int ret; + trace_scoutfs_item_lookup(sb, key, val); + + ret = scoutfs_kvec_alloc_key(end); + if (ret) + goto out; + do { + scoutfs_kvec_init_key(end); + spin_lock_irqsave(&cac->lock, flags); - item = find_item(&cac->root, key); + item = find_item(&cac->items, key); if (item) ret = scoutfs_kvec_memcpy(val, item->val); - else + else if (check_range(&cac->ranges, key, end)) ret = -ENOENT; + else + ret = -ENODATA; spin_unlock_irqrestore(&cac->lock, flags); - } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + scoutfs_kvec_kfree(end); +out: trace_printk("ret %d\n", ret); - return ret; } @@ -256,59 +423,98 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, } /* - * Return the next cached item starting with the given key. + * Return the next item starting with the given key, returning the last + * key at the most. * - * -ENOENT is returned if there are no cached items past the given key. - * If the last key is specified then -ENOENT is returned if there are no - * cached items up until that last key, inclusive. + * -ENOENT is returned if there are no items between the given and last + * keys. * - * The found key is copied to the caller's key. -ENOBUFS is returned if - * the found key didn't fit in the caller's key. + * The next item's key is copied to the caller's key. -ENOBUFS is + * returned if the item's key didn't fit in the caller's key. * - * The found value is copied into the callers value. The number of - * value bytes copied is returned. The copied value can be truncated by - * the caller's value buffer length. + * The next item's value is copied into the callers value. The number + * of value bytes copied is returned. The copied value can be truncated + * by the caller's value buffer length. */ int scoutfs_item_next(struct super_block *sb, struct kvec *key, struct kvec *last, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + SCOUTFS_DECLARE_KVEC(read_start); + SCOUTFS_DECLARE_KVEC(read_end); + SCOUTFS_DECLARE_KVEC(range_end); struct cached_item *item; unsigned long flags; + bool cached; int ret; - /* - * This partial copy and paste of lookup is stubbed out for now. - * we'll want the negative caching fixes to be able to iterate - * without constantly searching the manifest between cached - * items. - */ - return -EINVAL; + /* convenience to avoid searching if caller iterates past their last */ + if (scoutfs_kvec_length(key) > scoutfs_kvec_length(last)) { + ret = -ENOENT; + goto out; + } - do { - spin_lock_irqsave(&cac->lock, flags); + ret = scoutfs_kvec_alloc_key(range_end); + if (ret) + goto out; + + spin_lock_irqsave(&cac->lock, flags); + + for(;;) { + scoutfs_kvec_init_key(range_end); + + /* see if we have a usable item in cache and before last */ + cached = check_range(&cac->ranges, key, range_end); + + if (cached && (item = next_item(&cac->items, key)) && + scoutfs_kvec_memcmp(item->key, range_end) <= 0 && + scoutfs_kvec_memcmp(item->key, last) <= 0) { + + if (scoutfs_kvec_length(item->key) > + scoutfs_kvec_length(key)) { + ret = -ENOBUFS; + break; + } - item = find_item(&cac->root, key); - if (!item) { - ret = -ENOENT; - } else if (scoutfs_kvec_length(item->key) > - scoutfs_kvec_length(key)) { - ret = -ENOBUFS; - } else { scoutfs_kvec_memcpy_truncate(key, item->key); if (val) ret = scoutfs_kvec_memcpy(val, item->val); else ret = 0; + break; + } + + if (!cached) { + /* missing cache starts at key */ + scoutfs_kvec_clone(read_start, key); + scoutfs_kvec_clone(read_end, range_end); + + } else if (scoutfs_kvec_memcmp(range_end, last) < 0) { + /* missing cache starts at range_end */ + scoutfs_kvec_clone(read_start, range_end); + scoutfs_kvec_clone(read_end, last); + + } else { + /* no items and we have cache between key and last */ + ret = -ENOENT; + break; } spin_unlock_irqrestore(&cac->lock, flags); - } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + ret = scoutfs_manifest_read_items(sb, read_start, read_end); + spin_lock_irqsave(&cac->lock, flags); + if (ret) + break; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + scoutfs_kvec_kfree(range_end); +out: trace_printk("ret %d\n", ret); - return ret; } @@ -396,94 +602,188 @@ static void clear_item_dirty(struct item_cache *cac, update_dirty_parents(item); } -/* - * Add an item with the key and value to the item cache. The new item - * is clean. Any existing item at the key will be removed and freed. - */ -static int add_item(struct super_block *sb, struct kvec *key, struct kvec *val, - bool dirty) +static struct cached_item *alloc_item(struct kvec *key, struct kvec *val) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *existing; struct cached_item *item; - unsigned long flags; - int ret; item = kzalloc(sizeof(struct cached_item), GFP_NOFS); - if (!item) - return -ENOMEM; - - ret = scoutfs_kvec_dup_flatten(item->key, key) ?: - scoutfs_kvec_dup_flatten(item->val, val); - if (ret) { - free_item(item); - return ret; + if (item) { + if (scoutfs_kvec_dup_flatten(item->key, key) || + scoutfs_kvec_dup_flatten(item->val, val)) { + free_item(item); + item = NULL; + } } - spin_lock_irqsave(&cac->lock, flags); - existing = insert_item(&cac->root, item); - if (existing) { - clear_item_dirty(cac, existing); - rb_erase_augmented(&existing->node, &cac->root, - &scoutfs_item_rb_cb); - } - if (dirty) - mark_item_dirty(cac, item); - spin_unlock_irqrestore(&cac->lock, flags); - free_item(existing); - - return 0; + return item; } /* - * Add a clean item to the cache. This is used to populate items while - * reading segments. - */ -int scoutfs_item_insert(struct super_block *sb, struct kvec *key, - struct kvec *val) -{ - return add_item(sb, key, val, false); -} - -/* - * Create a new dirty item in the cache. + * Create a new dirty item in the cache. Returns -EEXIST if an item + * already exists with the given key. + * + * XXX but it doesn't read.. is that weird? Seems weird. */ int scoutfs_item_create(struct super_block *sb, struct kvec *key, struct kvec *val) { - return add_item(sb, key, val, true); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + item = alloc_item(key, val); + if (!item) + return -ENOMEM; + + spin_lock_irqsave(&cac->lock, flags); + ret = insert_item(&cac->items, item); + if (!ret) + mark_item_dirty(cac, item); + spin_unlock_irqrestore(&cac->lock, flags); + + if (ret) + free_item(item); + + return ret; } /* - * If the item with the key exists make sure it's cached and dirty. -ENOENT - * will be returned if it doesn't exist. + * Allocate an item with the key and value and add it to the list of + * items to be inserted as a batch later. The caller adds in sort order + * and we add with _tail to maintain that order. + */ +int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, + struct kvec *key, struct kvec *val) +{ + struct cached_item *item; + int ret; + + item = alloc_item(key, val); + if (item) { + list_add_tail(&item->entry, list); + ret = 0; + } else { + ret = -ENOMEM; + } + + return ret; +} + + +/* + * Insert a batch of clean read items from segments into the item cache. + * + * The caller hasn't been locked so the cached items could have changed + * since they were asked to read. If there are duplicates in the item + * cache they might be newer than what was read so we must drop them on + * the floor. + * + * The batch atomically adds the items and updates the cached range to + * include the callers range that covers the items. + * + * It's safe to re-add items to the batch list after they aren't + * inserted because _safe iteration will always be past the head entry + * that will be inserted. + */ +int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, + struct kvec *start, struct kvec *end) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_range *rng; + struct cached_item *item; + struct cached_item *tmp; + unsigned long flags; + int ret; + + trace_scoutfs_item_insert_batch(sb, start, end); + + if (WARN_ON_ONCE(scoutfs_kvec_memcmp(start, end) > 0)) + return -EINVAL; + + rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); + if (rng && (scoutfs_kvec_dup_flatten(rng->start, start) || + scoutfs_kvec_dup_flatten(rng->end, end))) { + free_range(rng); + rng = NULL; + } + if (!rng) { + ret = -ENOMEM; + goto out; + } + + spin_lock_irqsave(&cac->lock, flags); + + insert_range(&cac->ranges, rng); + + list_for_each_entry_safe(item, tmp, list, entry) { + list_del(&item->entry); + if (insert_item(&cac->items, item)) + list_add(&item->entry, list); + } + + spin_unlock_irqrestore(&cac->lock, flags); + + ret = 0; +out: + scoutfs_item_free_batch(list); + return ret; +} + +void scoutfs_item_free_batch(struct list_head *list) +{ + struct cached_item *item; + struct cached_item *tmp; + + list_for_each_entry_safe(item, tmp, list, entry) { + list_del_init(&item->entry); + free_item(item); + } +} + + +/* + * If the item exists make sure it's dirty and pinned. It can be read + * if it wasn't cached. -ENOENT is returned if the item doesn't exist. */ int scoutfs_item_dirty(struct super_block *sb, struct kvec *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + SCOUTFS_DECLARE_KVEC(end); struct cached_item *item; unsigned long flags; int ret; + ret = scoutfs_kvec_alloc_key(end); + if (ret) + goto out; + do { + scoutfs_kvec_init_key(end); + spin_lock_irqsave(&cac->lock, flags); - item = find_item(&cac->root, key); + item = find_item(&cac->items, key); if (item) { mark_item_dirty(cac, item); ret = 0; - } else { + } else if (check_range(&cac->ranges, key, end)) { ret = -ENOENT; + } else { + ret = -ENODATA; } spin_unlock_irqrestore(&cac->lock, flags); - } while (!item && ((ret = scoutfs_manifest_read_items(sb, key)) == 0)); + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + scoutfs_kvec_kfree(end); +out: trace_printk("ret %d\n", ret); - return ret; } @@ -499,37 +799,49 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; SCOUTFS_DECLARE_KVEC(up_val); + SCOUTFS_DECLARE_KVEC(end); struct cached_item *item; unsigned long flags; int ret; + ret = scoutfs_kvec_alloc_key(end); + if (ret) + goto out; + if (val) { ret = scoutfs_kvec_dup_flatten(up_val, val); if (ret) - return -ENOMEM; + goto out; } else { scoutfs_kvec_init_null(up_val); } - spin_lock_irqsave(&cac->lock, flags); + do { + scoutfs_kvec_init_key(end); - /* XXX update seq */ - item = find_item(&cac->root, key); - if (item) { - /* keep dirty counters in sync */ - clear_item_dirty(cac, item); - scoutfs_kvec_swap(up_val, item->val); - mark_item_dirty(cac, item); - } else { - ret = -ENOENT; - } + spin_lock_irqsave(&cac->lock, flags); - spin_unlock_irqrestore(&cac->lock, flags); + item = find_item(&cac->items, key); + if (item) { + clear_item_dirty(cac, item); + scoutfs_kvec_swap(up_val, item->val); + mark_item_dirty(cac, item); + ret = 0; + } else if (check_range(&cac->ranges, key, end)) { + ret = -ENOENT; + } else { + ret = -ENODATA; + } + spin_unlock_irqrestore(&cac->lock, flags); + + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); +out: + scoutfs_kvec_kfree(end); scoutfs_kvec_kfree(up_val); trace_printk("ret %d\n", ret); - return ret; } @@ -645,7 +957,7 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, *key_bytes = 0; total = sizeof(struct scoutfs_segment_block); - for (item = first_dirty(cac->root.rb_node); item; + for (item = first_dirty(cac->items.rb_node); item; item = next_dirty(item)) { total += sizeof(struct scoutfs_segment_item) + @@ -676,7 +988,7 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) count_seg_items(cac, &nr_items, &key_bytes); if (nr_items) { - item = first_dirty(cac->root.rb_node); + item = first_dirty(cac->items.rb_node); scoutfs_seg_first_item(sb, seg, item->key, item->val, nr_items, key_bytes); clear_item_dirty(cac, item); @@ -701,7 +1013,8 @@ int scoutfs_item_setup(struct super_block *sb) sbi->item_cache = cac; spin_lock_init(&cac->lock); - cac->root = RB_ROOT; + cac->items = RB_ROOT; + cac->ranges = RB_ROOT; return 0; } @@ -711,16 +1024,24 @@ void scoutfs_item_destroy(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct cached_item *item; + struct cached_range *rng; struct rb_node *node; if (cac) { - for (node = rb_first(&cac->root); node; ) { + for (node = rb_first(&cac->items); node; ) { item = container_of(node, struct cached_item, node); node = rb_next(node); - rb_erase(&item->node, &cac->root); + rb_erase(&item->node, &cac->items); free_item(item); } + for (node = rb_first(&cac->ranges); node; ) { + rng = container_of(node, struct cached_range, node); + node = rb_next(node); + rb_erase(&rng->node, &cac->items); + free_range(rng); + } + kfree(cac); } } diff --git a/kmod/src/item.h b/kmod/src/item.h index 62d93815..81746822 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -22,6 +22,12 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, struct kvec *val); int scoutfs_item_delete(struct super_block *sb, struct kvec *key); +int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, + struct kvec *key, struct kvec *val); +int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, + struct kvec *start, struct kvec *end); +void scoutfs_item_free_batch(struct list_head *list); + long scoutfs_item_dirty_bytes(struct super_block *sb); int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index cfc46d0b..4980109c 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -22,6 +22,7 @@ #include "item.h" #include "ring.h" #include "manifest.h" +#include "scoutfs_trace.h" struct manifest { spinlock_t lock; @@ -51,15 +52,26 @@ struct manifest_entry { }; /* - * A path tracks all the segments from level 0 to the last level that - * overlap with the search key. + * A reader uses references to segments copied from a walk of the + * manifest. The references are a point in time sample of the manifest. + * The manifest and segments can change while the reader uses their + * references. Locking ensures that the items they're reading will be + * stable while the manifest and segments change, and the segment + * allocator gives readers time to use immutable stale segments before + * their reallocated and reused. */ struct manifest_ref { + struct list_head entry; + u64 segno; u64 seq; struct scoutfs_segment *seg; + int found_ctr; int pos; + u16 first_key_len; + u16 last_key_len; u8 level; + u8 keys[SCOUTFS_MAX_KEY_SIZE * 2]; }; static void init_ment_keys(struct manifest_entry *ment, struct kvec *first, @@ -72,20 +84,25 @@ static void init_ment_keys(struct manifest_entry *ment, struct kvec *first, le16_to_cpu(ment->am.last_key_len)); } -/* - * returns: - * < 0 : key < ment->first_key - * > 0 : key > ment->first_key - * == 0 : ment->first_key <= key <= ment->last_key - */ -static bool cmp_key_ment(struct kvec *key, struct manifest_entry *ment) +static void init_ref_keys(struct manifest_ref *ref, struct kvec *first, + struct kvec *last) +{ + if (first) + scoutfs_kvec_init(first, ref->keys, ref->first_key_len); + if (last) + scoutfs_kvec_init(last, ref->keys + ref->first_key_len, + ref->last_key_len); +} + +static bool cmp_range_ment(struct kvec *key, struct kvec *end, + struct manifest_entry *ment) { SCOUTFS_DECLARE_KVEC(first); SCOUTFS_DECLARE_KVEC(last); init_ment_keys(ment, first, last); - return scoutfs_kvec_cmp_overlap(key, key, first, last); + return scoutfs_kvec_cmp_overlap(key, end, first, last); } static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) @@ -97,7 +114,7 @@ static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) while (node) { ment = container_of(node, struct manifest_entry, node); - cmp = cmp_key_ment(key, ment); + cmp = cmp_range_ment(key, key, ment); if (cmp < 0) node = node->rb_left; else if (cmp > 0) @@ -119,16 +136,16 @@ static int insert_ment(struct rb_root *root, struct manifest_entry *ins) struct rb_node *parent = NULL; struct manifest_entry *ment; SCOUTFS_DECLARE_KVEC(key); + SCOUTFS_DECLARE_KVEC(end); int cmp; - /* either first or last works */ - init_ment_keys(ins, key, key); + init_ment_keys(ins, key, end); while (*node) { parent = *node; ment = container_of(*node, struct manifest_entry, node); - cmp = cmp_key_ment(key, ment); + cmp = cmp_range_ment(key, end, ment); if (cmp < 0) { node = &(*node)->rb_left; } else if (cmp > 0) { @@ -215,6 +232,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, int key_bytes; int ret; + trace_scoutfs_manifest_add(sb, first, last, segno, seq, level, dirty); + key_bytes = scoutfs_kvec_length(first) + scoutfs_kvec_length(last); ment = kmalloc(sizeof(struct manifest_entry) + key_bytes, GFP_NOFS); if (!ment) @@ -249,57 +268,97 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, return ret; } -static void set_ref(struct manifest_ref *ref, struct manifest_entry *ment) +/* + * Grab an allocated ref from the src list, fill it with the details + * from the ment, and add it to the dst list. The ref is added to the + * tail of the dst list so that we maintain the caller's manifest walk + * order. + */ +static void fill_ref_tail(struct list_head *dst, struct list_head *src, + struct manifest_entry *ment) { + SCOUTFS_DECLARE_KVEC(ment_first); + SCOUTFS_DECLARE_KVEC(ment_last); + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); + struct manifest_ref *ref; + + ref = list_first_entry(src, struct manifest_ref, entry); + ref->segno = le64_to_cpu(ment->am.segno); ref->seq = le64_to_cpu(ment->am.seq); ref->level = ment->am.level; + ref->first_key_len = le16_to_cpu(ment->am.first_key_len); + ref->last_key_len = le16_to_cpu(ment->am.last_key_len); + + init_ment_keys(ment, ment_first, ment_last); + init_ref_keys(ref, first, last); + + scoutfs_kvec_memcpy(first, ment_first); + scoutfs_kvec_memcpy(last, ment_last); + + list_move_tail(&ref->entry, dst); } /* - * Returns refs if intersecting segments are found, NULL if none intersect, - * and PTR_ERR on failure. + * Get refs on all the segments in the manifest that we'll need to + * search to populate the cache with the given range. + * + * We have to get all the level 0 segments that intersect with the range + * of items that we want to search because the level 0 segments can + * arbitrarily overlap with each other. + * + * We only need to search for the starting key in all the higher order + * levels. They do not overlap so we can iterate through the key space + * in each segment starting with the key. */ -static struct manifest_ref *get_key_refs(struct manifest *mani, - struct kvec *key, - unsigned int *nr_ret) +static int get_range_refs(struct manifest *mani, struct kvec *key, + struct kvec *end, struct list_head *ref_list) { - struct manifest_ref *refs = NULL; struct manifest_entry *ment; + struct manifest_ref *ref; + struct manifest_ref *tmp; struct rb_root *root; unsigned long flags; unsigned int total; - unsigned int nr; + unsigned int nr = 0; + LIST_HEAD(alloced); + int ret; int i; trace_printk("getting refs\n"); spin_lock_irqsave(&mani->lock, flags); + /* allocate enough refs for the of segments */ total = mani->level0_nr + mani->last_level; - while (nr != total) { - nr = total; + while (nr < total) { spin_unlock_irqrestore(&mani->lock, flags); - kfree(refs); - refs = kcalloc(total, sizeof(struct manifest_ref), GFP_NOFS); - trace_printk("alloc refs %p total %u\n", refs, total); - if (!refs) - return ERR_PTR(-ENOMEM); + for (i = nr; i < total; i++) { + ref = kmalloc(sizeof(struct manifest_ref), GFP_NOFS); + if (!ref) { + ret = -ENOMEM; + goto out; + } + + memset(ref, 0, offsetof(struct manifest_ref, keys)); + list_add(&ref->entry, &alloced); + } + nr = total; spin_lock_irqsave(&mani->lock, flags); } - nr = 0; - + /* find all the overlapping level 0 segments */ list_for_each_entry(ment, &mani->level0_list, level0_entry) { - trace_printk("trying l0 ment %p\n", ment); - if (cmp_key_ment(key, ment)) + if (cmp_range_ment(key, end, ment)) continue; - set_ref(&refs[nr++], ment); + fill_ref_tail(ref_list, &alloced, ment); } + /* find each segment containing the key at the higher orders */ for (i = 1; i <= mani->last_level; i++) { root = &mani->level_roots[i]; if (RB_EMPTY_ROOT(root)) @@ -307,119 +366,151 @@ static struct manifest_ref *get_key_refs(struct manifest *mani, ment = find_ment(root, key); if (ment) - set_ref(&refs[nr++], ment); + fill_ref_tail(ref_list, &alloced, ment); } spin_unlock_irqrestore(&mani->lock, flags); + ret = 0; - *nr_ret = nr; - if (!nr) { - kfree(refs); - refs = NULL; +out: + if (ret) { + list_splice_init(ref_list, &alloced); + list_for_each_entry_safe(ref, tmp, &alloced, entry) { + list_del_init(&ref->entry); + kfree(ref); + } } - - trace_printk("refs %p (err %ld)\n", - refs, IS_ERR(refs) ? PTR_ERR(refs) : 0); - - return refs; + trace_printk("ret %d\n", ret); + return ret; } /* - * The caller didn't find an item for the given key in the item cache - * and wants us to search for it in the lsm segments. We search the - * manifest for all the segments that contain the key. We then read the - * segments and iterate over their items looking for ours. We insert it - * and some number of other surrounding items to amortize the relatively - * expensive multi-segment searches. + * The caller found a hole in the item cache that they'd like populated. + * + * We search the manifest for all the segments we'll need to iterate + * from the key to the end key. We walk the segments and insert as many + * items as we can from the segments, trying to amortize the per-item + * cost of segment searching. + * + * As we insert the batch of items we give the item cache the range of + * keys that contain these items. This lets the cache return negative + * cache lookups for missing items within the range. + * + * Returns 0 if we inserted items with a range covering the starting + * key. The caller should be able to make progress. + * + * Returns -errno if we failed to make any change in the cache. * * This is asking the seg code to read each entire segment. The seg * code could give it it helpers to submit and wait on blocks within the - * segment so that we don't have wild bandwidth amplification in the - * cold random read case. + * segment so that we don't have wild bandwidth amplification for cold + * random reads. * * The segments are immutable at this point so we can use their contents * as long as we hold refs. */ -int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) +#define MAX_ITEMS_READ 32 + +int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, + struct kvec *end) { DECLARE_MANIFEST(sb, mani); SCOUTFS_DECLARE_KVEC(item_key); SCOUTFS_DECLARE_KVEC(item_val); SCOUTFS_DECLARE_KVEC(found_key); SCOUTFS_DECLARE_KVEC(found_val); + SCOUTFS_DECLARE_KVEC(batch_end); + SCOUTFS_DECLARE_KVEC(seg_end); struct scoutfs_segment *seg; - struct manifest_ref *refs; - unsigned long had_found; + struct manifest_ref *ref; + struct manifest_ref *tmp; + LIST_HEAD(ref_list); + LIST_HEAD(batch); + int found_ctr; bool found; int ret = 0; int err; - int nr_refs; int cmp; - int last; - int i; int n; trace_printk("reading items\n"); - refs = get_key_refs(mani, key, &nr_refs); - if (IS_ERR(refs)) - return PTR_ERR(refs); - if (!refs) - return -ENOENT; + /* get refs on all the segments */ + ret = get_range_refs(mani, key, end, &ref_list); + if (ret) + return ret; /* submit reads for all the segments */ - for (i = 0; i < nr_refs; i++) { - seg = scoutfs_seg_submit_read(sb, refs[i].segno); + list_for_each_entry(ref, &ref_list, entry) { + seg = scoutfs_seg_submit_read(sb, ref->segno); if (IS_ERR(seg)) { ret = PTR_ERR(seg); break; } - refs[i].seg = seg; + ref->seg = seg; } - last = i; - /* wait for submitted segments and search if we haven't seen failure */ - for (i = 0; i < last; i++) { - seg = refs[i].seg; + /* wait for submitted segments and search for starting pos */ + list_for_each_entry(ref, &ref_list, entry) { + if (!ref->seg) + break; - err = scoutfs_seg_wait(sb, seg); + err = scoutfs_seg_wait(sb, ref->seg); if (err && !ret) ret = err; - if (!ret) - refs[i].pos = scoutfs_seg_find_pos(seg, key); + if (ret == 0) + ref->pos = scoutfs_seg_find_pos(ref->seg, key); } - - /* done if we saw errors */ if (ret) goto out; - /* walk sorted items, resolving across segments, and insert */ - for (n = 0; n < 16; n++) { + scoutfs_kvec_init_null(batch_end); + scoutfs_kvec_init_null(seg_end); + found_ctr = 0; + + for (n = 0; n < MAX_ITEMS_READ; n++) { found = false; + found_ctr++; - /* find the most recent least key */ - for (i = 0; i < nr_refs; i++) { - seg = refs[i].seg; - if (!seg) - continue; + /* find the next least key from the pos in each segment */ + list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - /* get kvecs, removing if we ran out of items */ - ret = scoutfs_seg_item_kvecs(seg, refs[i].pos, + /* + * Check the next item in the segment. We're + * done with the segment if there are no more + * items or if the next item is past the + * caller's end. We record either the caller's + * end or the segment end if it's a l1+ segment for + * use as the batch end if we don't see more items. + */ + ret = scoutfs_seg_item_kvecs(ref->seg, ref->pos, item_key, item_val); + if (ret < 0) { + if (ref->level > 0) { + init_ref_keys(ref, NULL, item_key); + scoutfs_kvec_clone_less(seg_end, + item_key); + } + } else if (scoutfs_kvec_memcmp(item_key, end) > 0) { + scoutfs_kvec_clone_less(seg_end, end); + ret = -ENOENT; + } if (ret < 0) { - scoutfs_seg_put(seg); - refs[i].seg = NULL; + list_del_init(&ref->entry); + scoutfs_seg_put(ref->seg); + kfree(ref); continue; } + /* see if it's the new least item */ if (found) { cmp = scoutfs_kvec_memcmp(item_key, found_key); if (cmp >= 0) { if (cmp == 0) - set_bit(i, &had_found); + ref->found_ctr = found_ctr; continue; } } @@ -427,37 +518,58 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key) /* remember new least key */ scoutfs_kvec_clone(found_key, item_key); scoutfs_kvec_clone(found_val, item_val); + ref->found_ctr = ++found_ctr; found = true; - had_found = 0; - set_bit(i, &had_found); - } - - /* return -ENOENT if we didn't find any or the callers item */ - if (n == 0 && - (!found || scoutfs_kvec_memcmp(key, found_key))) { - ret = -ENOENT; - break; } + /* ran out of keys in segs, range extends to seg end */ if (!found) { + scoutfs_kvec_clone(batch_end, seg_end); ret = 0; break; } - ret = scoutfs_item_insert(sb, item_key, item_val); - if (ret) + /* + * If we fail to add an item we're done. If we already + * have items it's not a failure and the end of the cached + * range is the last successfully added item. + */ + ret = scoutfs_item_add_batch(sb, &batch, found_key, found_val); + if (ret) { + if (n > 0) + ret = 0; break; + } - /* advance all the positions past the found key */ - for_each_set_bit(i, &had_found, BITS_PER_LONG) - refs[i].pos++; + /* the last successful key determines the range */ + scoutfs_kvec_clone(batch_end, found_key); + + /* if we just saw the end key then we're done */ + if (scoutfs_kvec_memcmp(found_key, end) == 0) { + ret = 0; + break; + } + + /* advance all the positions that had the found key */ + list_for_each_entry(ref, &ref_list, entry) { + if (ref->found_ctr == found_ctr) + ref->pos++; + } + + ret = 0; } + if (ret) + scoutfs_item_free_batch(&batch); + else + ret = scoutfs_item_insert_batch(sb, &batch, key, batch_end); out: - for (i = 0; i < nr_refs; i++) - scoutfs_seg_put(refs[i].seg); + list_for_each_entry_safe(ref, tmp, &ref_list, entry) { + list_del_init(&ref->entry); + scoutfs_seg_put(ref->seg); + kfree(ref); + } - kfree(refs); return ret; } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index f3bea21a..9f1477b4 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -7,7 +7,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, int scoutfs_manifest_has_dirty(struct super_block *sb); int scoutfs_manifest_dirty_ring(struct super_block *sb); -int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key); +int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, + struct kvec *until); int scoutfs_manifest_setup(struct super_block *sb); void scoutfs_manifest_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a9a18118..8793a544 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -27,6 +27,7 @@ #include "key.h" #include "format.h" +#include "kvec.h" struct scoutfs_sb_info; @@ -346,6 +347,57 @@ DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_since, TP_ARGS(sb, first, last) ); +TRACE_EVENT(scoutfs_manifest_add, + TP_PROTO(struct super_block *sb, struct kvec *first, + struct kvec *last, u64 segno, u64 seq, u8 level, bool dirty), + TP_ARGS(sb, first, last, segno, seq, level, dirty), + TP_STRUCT__entry( + __dynamic_array(char, first, scoutfs_kvec_key_strlen(first)) + __dynamic_array(char, last, scoutfs_kvec_key_strlen(last)) + __field(u64, segno) + __field(u64, seq) + __field(u8, level) + __field(u8, dirty) + ), + TP_fast_assign( + scoutfs_kvec_key_sprintf(__get_dynamic_array(first), first); + scoutfs_kvec_key_sprintf(__get_dynamic_array(last), last); + __entry->segno = segno; + __entry->seq = seq; + __entry->level = level; + __entry->dirty = dirty; + ), + TP_printk("first %s last %s segno %llu seq %llu level %u dirty %u", + __get_str(first), __get_str(last), __entry->segno, + __entry->seq, __entry->level, __entry->dirty) +); + +TRACE_EVENT(scoutfs_item_lookup, + TP_PROTO(struct super_block *sb, struct kvec *key, struct kvec *val), + TP_ARGS(sb, key, val), + TP_STRUCT__entry( + __dynamic_array(char, key, scoutfs_kvec_key_strlen(key)) + ), + TP_fast_assign( + scoutfs_kvec_key_sprintf(__get_dynamic_array(key), key); + ), + TP_printk("key %s", __get_str(key)) +); + +TRACE_EVENT(scoutfs_item_insert_batch, + TP_PROTO(struct super_block *sb, struct kvec *start, struct kvec *end), + TP_ARGS(sb, start, end), + TP_STRUCT__entry( + __dynamic_array(char, start, scoutfs_kvec_key_strlen(start)) + __dynamic_array(char, end, scoutfs_kvec_key_strlen(end)) + ), + TP_fast_assign( + scoutfs_kvec_key_sprintf(__get_dynamic_array(start), start); + scoutfs_kvec_key_sprintf(__get_dynamic_array(end), end); + ), + TP_printk("start %s end %s", __get_str(start), __get_str(end)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 3dfe8e10dfdd6e12b95a901d1ec8d60f44542c2e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 29 Dec 2016 16:01:51 -0800 Subject: [PATCH 179/920] Add little cmp u64 helper We have use of this little u64 comparison function in a few more places so let's make it a proper usable inline function in a header. Signed-off-by: Zach Brown --- kmod/src/cmp.h | 11 +++++++++++ kmod/src/seg.c | 10 +++------- 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 kmod/src/cmp.h diff --git a/kmod/src/cmp.h b/kmod/src/cmp.h new file mode 100644 index 00000000..3230c043 --- /dev/null +++ b/kmod/src/cmp.h @@ -0,0 +1,11 @@ +#ifndef _SCOUTFS_CMP_H_ +#define _SCOUTFS_CMP_H_ + +#include + +static inline int scoutfs_cmp_u64s(u64 a, u64 b) +{ + return a < b ? -1 : a > b ? 1 : 0; +} + +#endif diff --git a/kmod/src/seg.c b/kmod/src/seg.c index d4eeb32f..37277096 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -21,6 +21,7 @@ #include "seg.h" #include "bio.h" #include "kvec.h" +#include "cmp.h" #include "manifest.h" #include "alloc.h" @@ -99,11 +100,6 @@ void scoutfs_seg_put(struct scoutfs_segment *seg) } } -static int cmp_u64s(u64 a, u64 b) -{ - return a < b ? -1 : a > b ? 1 : 0; -} - static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) { struct rb_node *node = root->rb_node; @@ -115,7 +111,7 @@ static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) parent = node; seg = container_of(node, struct scoutfs_segment, node); - cmp = cmp_u64s(segno, seg->segno); + cmp = scoutfs_cmp_u64s(segno, seg->segno); if (cmp < 0) node = node->rb_left; else if (cmp > 0) @@ -146,7 +142,7 @@ static struct scoutfs_segment *replace_seg(struct rb_root *root, parent = *node; seg = container_of(*node, struct scoutfs_segment, node); - cmp = cmp_u64s(ins->segno, seg->segno); + cmp = scoutfs_cmp_u64s(ins->segno, seg->segno); if (cmp < 0) { node = &(*node)->rb_left; } else if (cmp > 0) { From db9f2be728113f5fd19d05de180f41937dc7f7fd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 29 Dec 2016 16:06:12 -0800 Subject: [PATCH 180/920] Switch to indexed manifest using treap ring The first pass manifest and allocator storage used a simple ring log that was entirely replayed into memory to be used. That risked the manifest being too large to fit in memory, especially with large keys and large volumes. So we move to using an indexed persistent structure that can be read on demand and cached. We use a treap of byte referenced nodoes stored in a circular ring. The code interface is modeled a bit on the in-memory rbtree interface. Except that we can get IO errors and manage allocation so we return data pointers to the item payload istead of item structs and we can return errors. The manifest and allocator are converted over and the old ring code is removed entirely. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/alloc.c | 398 ++++++------ kmod/src/alloc.h | 3 +- kmod/src/format.h | 82 ++- kmod/src/manifest.c | 536 ++++++++-------- kmod/src/manifest.h | 3 +- kmod/src/ring.c | 326 ---------- kmod/src/ring.h | 18 - kmod/src/scoutfs_trace.h | 10 +- kmod/src/seg.c | 2 +- kmod/src/super.c | 7 +- kmod/src/super.h | 4 +- kmod/src/trans.c | 15 +- kmod/src/treap.c | 1271 ++++++++++++++++++++++++++++++++++++++ kmod/src/treap.h | 44 ++ 15 files changed, 1848 insertions(+), 873 deletions(-) delete mode 100644 kmod/src/ring.c delete mode 100644 kmod/src/ring.h create mode 100644 kmod/src/treap.c create mode 100644 kmod/src/treap.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index bffe9ec6..cb9b29c9 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -4,4 +4,4 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o block.o btree.o buddy.o counters.o crc.o dir.o \ filerw.o kvec.o inode.o ioctl.o item.o manifest.o msg.o name.o \ - ring.o seg.o scoutfs_trace.o super.o trans.o xattr.o + seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index dbacf288..709f3b78 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -17,77 +17,128 @@ #include "super.h" #include "format.h" -#include "ring.h" +#include "treap.h" +#include "cmp.h" #include "alloc.h" /* - * scoutfs allocates segments by storing regions of a bitmap in a radix. - * As the regions are modified their index in the radix is marked dirty - * for writeout. + * scoutfs allocates segments by storing regions of a bitmap in treap + * nodes. * - * Frees are tracked in a separate radix. They're only applied to the - * free regions as a transaction is written. The frees can't satisfy - * allocation until they're committed so that we don't overwrite stable - * referenced data. + * Freed segments are recorded in nodes in an rbtree. The frees can't + * satisfy allocation until they're committed to prevent overwriting + * live data so they're only applied to the region nodes as their + * transaction is written. * - * The allocated segments are large enough to be effectively - * independent. We allocate by sweeping a cursor through the volume. - * This gives racing unlocked readers more time to try to sample a stale - * freed segment, when its safe to do so, before it is reallocated and + * We allocate by sweeping a cursor through the volume. This gives + * racing unlocked readers more time to try to sample a stale freed + * segment, when its safe to do so, before it is reallocated and * rewritten and they're forced to retry their racey read. - * - * XXX - * - make sure seg fits in long index - * - frees can delete region, leave non-NULL nul behind for logging */ struct seg_alloc { - spinlock_t lock; - struct radix_tree_root regs; - struct radix_tree_root pending; + struct rw_semaphore rwsem; + struct rb_root pending_root; + struct scoutfs_treap *treap; u64 next_segno; }; #define DECLARE_SEG_ALLOC(sb, name) \ struct seg_alloc *name = SCOUTFS_SB(sb)->seg_alloc -enum { - DIRTY_RADIX_TAG = 0, +struct pending_region { + struct rb_node node; + struct scoutfs_alloc_region reg; }; +static struct pending_region *find_pending(struct rb_root *root, u64 ind) +{ + struct rb_node *node = root->rb_node; + struct pending_region *pend; + + while (node) { + pend = container_of(node, struct pending_region, node); + + if (ind < le64_to_cpu(pend->reg.index)) + node = node->rb_left; + else if (ind > le64_to_cpu(pend->reg.index)) + node = node->rb_right; + else + return pend; + } + + return NULL; +} + +static void insert_pending(struct rb_root *root, struct pending_region *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct pending_region *pend; + u64 ind = le64_to_cpu(ins->reg.index); + + while (*node) { + parent = *node; + pend = container_of(*node, struct pending_region, node); + + if (ind < le64_to_cpu(pend->reg.index)) + node = &(*node)->rb_left; + else if (ind > le64_to_cpu(pend->reg.index)) + node = &(*node)->rb_right; + else + BUG(); + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); +} + +static bool empty_region(struct scoutfs_alloc_region *reg) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(reg->bits); i++) { + if (reg->bits[i]) + return false; + } + + return true; +} + int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_ring_alloc_region *reg; + struct scoutfs_alloc_region *reg; DECLARE_SEG_ALLOC(sb, sal); - unsigned long flags; - unsigned long ind; + u64 ind; int ret; int nr; - spin_lock_irqsave(&sal->lock, flags); + down_write(&sal->rwsem); - /* start by sweeping through the device for the first time */ - if (sal->next_segno == le64_to_cpu(super->alloc_uninit)) { + /* initially sweep through all segments */ + if (super->alloc_uninit != super->total_segs) { + *segno = le64_to_cpu(super->alloc_uninit); + /* done when inc hits total_segs */ le64_add_cpu(&super->alloc_uninit, 1); - *segno = sal->next_segno++; - if (sal->next_segno == le64_to_cpu(super->total_segs)) - sal->next_segno = 0; ret = 0; goto out; } - /* then fall back to the allocator */ + /* but usually search for region nodes */ ind = sal->next_segno >> SCOUTFS_ALLOC_REGION_SHIFT; nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; do { - ret = radix_tree_gang_lookup(&sal->regs, (void **)®, ind, 1); - } while (ret == 0 && ind && (ind = 0, nr = 0, 1)); + reg = scoutfs_treap_lookup_next_dirty(sal->treap, &ind); + } while (reg == NULL && ind && (ind = 0, nr = 0, 1)); - if (ret == 0) { - ret = -ENOSPC; + if (IS_ERR_OR_NULL(reg)) { + if (IS_ERR(reg)) + ret = PTR_ERR(reg); + else + ret = -ENOSPC; goto out; } @@ -98,237 +149,200 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) goto out; } + ind = le64_to_cpu(reg->index); + clear_bit_le(nr, reg->bits); - radix_tree_tag_set(&sal->regs, ind, DIRTY_RADIX_TAG); + + if (empty_region(reg)) { + ret = scoutfs_treap_delete(sal->treap, &ind); + /* XXX figure out what to do about this inconsistency */ + if (WARN_ON_ONCE(ret)) + goto out; + } *segno = (ind << SCOUTFS_ALLOC_REGION_SHIFT) + nr; - - /* once this wraps it will never equal alloc_uninit */ sal->next_segno = *segno + 1; - if (sal->next_segno == le64_to_cpu(super->total_segs)) - sal->next_segno = 0; ret = 0; out: - spin_unlock_irqrestore(&sal->lock, flags); + up_write(&sal->rwsem); trace_printk("segno %llu ret %d\n", *segno, ret); return ret; } /* - * Record newly freed sgements in pending regions. These can't be - * applied to the main allocator regions until the next commit so that - * they're not still referenced by the stable tree in event of a crash. - * - * The pending regions are merged into dirty regions for the next commit. + * Record newly freed sgements in pending regions. These are applied to + * treap nodes as the transaction commits. */ int scoutfs_alloc_free(struct super_block *sb, u64 segno) { - struct scoutfs_ring_alloc_region *reg; - struct scoutfs_ring_alloc_region *ins; + struct pending_region *pend; DECLARE_SEG_ALLOC(sb, sal); - unsigned long flags; - unsigned long ind; + u64 ind; int ret; int nr; ind = segno >> SCOUTFS_ALLOC_REGION_SHIFT; nr = segno & SCOUTFS_ALLOC_REGION_MASK; - ins = kzalloc(sizeof(struct scoutfs_ring_alloc_region), GFP_NOFS); - if (!ins) { - ret = -ENOMEM; - goto out; + down_write(&sal->rwsem); + + pend = find_pending(&sal->pending_root, ind); + if (!pend) { + pend = kzalloc(sizeof(struct pending_region), GFP_NOFS); + if (!pend) { + ret = -ENOMEM; + goto out; + } + + pend->reg.index = cpu_to_le64(ind); + insert_pending(&sal->pending_root, pend); } - ins->eh.type = SCOUTFS_RING_ADD_ALLOC; - ins->eh.len = cpu_to_le16(sizeof(struct scoutfs_ring_alloc_region)); - ins->index = cpu_to_le64(ind); - - ret = radix_tree_preload(GFP_NOFS); - if (ret) { - goto out; - } - - spin_lock_irqsave(&sal->lock, flags); - - reg = radix_tree_lookup(&sal->pending, ind); - if (!reg) { - reg = ins; - ins = NULL; - radix_tree_insert(&sal->pending, ind, reg); - } - - set_bit_le(nr, reg->bits); - - spin_unlock_irqrestore(&sal->lock, flags); - radix_tree_preload_end(); + set_bit_le(nr, pend->reg.bits); + ret = 0; out: - kfree(ins); - trace_printk("freeing segno %llu ind %lu nr %d ret %d\n", + up_write(&sal->rwsem); + + trace_printk("freeing segno %llu ind %llu nr %d ret %d\n", segno, ind, nr, ret); return ret; } -/* - * Add a new clean region from the ring. It can be replacing existing - * clean stale entries during replay as we make our way through the - * ring. - */ -int scoutfs_alloc_add(struct super_block *sb, - struct scoutfs_ring_alloc_region *ins) +static void or_region_bits(struct scoutfs_alloc_region *dst, + struct scoutfs_alloc_region *src) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dst->bits); i++) + dst->bits[i] |= src->bits[i]; +} + +int scoutfs_alloc_has_dirty(struct super_block *sb) { - struct scoutfs_ring_alloc_region *existing; - struct scoutfs_ring_alloc_region *reg; DECLARE_SEG_ALLOC(sb, sal); - unsigned long flags; int ret; - reg = kmalloc(sizeof(struct scoutfs_ring_alloc_region), GFP_NOFS); - if (!reg) { - ret = -ENOMEM; - goto out; - } + down_write(&sal->rwsem); + ret = scoutfs_treap_has_dirty(sal->treap); + up_write(&sal->rwsem); - memcpy(reg, ins, sizeof(struct scoutfs_ring_alloc_region)); - - ret = radix_tree_preload(GFP_NOFS); - if (ret) { - kfree(reg); - goto out; - } - - spin_lock_irqsave(&sal->lock, flags); - - existing = radix_tree_lookup(&sal->regs, le64_to_cpu(reg->index)); - if (existing) - radix_tree_delete(&sal->regs, le64_to_cpu(reg->index)); - radix_tree_insert(&sal->regs, le64_to_cpu(reg->index), reg); - - spin_unlock_irqrestore(&sal->lock, flags); - radix_tree_preload_end(); - - if (existing) - kfree(existing); - - ret = 0; -out: - trace_printk("inserted reg ind %llu ret %d\n", - le64_to_cpu(ins->index), ret); return ret; } /* - * Append all the dirty alloc regions to the end of the ring. First we - * apply the pending frees to create the final set of dirty regions. - * - * This can't fail and always returns 0. + * First we apply the pending frees to create the final set of dirty + * region nodes and then ask the treap to write them to ring pages. */ int scoutfs_alloc_dirty_ring(struct super_block *sb) { - struct scoutfs_ring_alloc_region *regs[16]; - struct scoutfs_ring_alloc_region *reg; DECLARE_SEG_ALLOC(sb, sal); - unsigned long start; - unsigned long ind; - int nr; - int i; - int b; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_alloc_region *reg; + struct pending_region *pend; + struct rb_node *node; + u64 ind; + int ret; - /* - * Merge pending free regions into dirty regions. If the dirty - * region doesn't exist we can just move the pending region over. - * If it does we or the pending bits in the region. - */ - start = 0; - do { - nr = radix_tree_gang_lookup(&sal->pending, (void **)regs, - start, ARRAY_SIZE(regs)); - for (i = 0; i < nr; i++) { - ind = le64_to_cpu(regs[i]->index); + down_write(&sal->rwsem); - reg = radix_tree_lookup(&sal->regs, ind); - if (!reg) { - radix_tree_insert(&sal->regs, ind, regs[i]); - } else { - for (b = 0; b < ARRAY_SIZE(reg->bits); b++) - reg->bits[i] |= regs[i]->bits[i]; - kfree(regs[i]); - } + while ((node = rb_first(&sal->pending_root))) { + pend = container_of(node, struct pending_region, node); - radix_tree_delete(&sal->pending, ind); - radix_tree_tag_set(&sal->regs, ind, DIRTY_RADIX_TAG); - start = ind + 1; + ind = le64_to_cpu(pend->reg.index); + + reg = scoutfs_treap_lookup_dirty(sal->treap, &ind); + if (!reg) + reg = scoutfs_treap_insert(sal->treap, &ind, + sizeof(struct scoutfs_alloc_region), + &ind); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + goto out; } - } while (nr); - /* and append all the dirty regions to the ring */ - start = 0; - do { - nr = radix_tree_gang_lookup_tag(&sal->regs, (void **)regs, - start, ARRAY_SIZE(regs), - DIRTY_RADIX_TAG); - for (i = 0; i < nr; i++) { - reg = regs[i]; - ind = le64_to_cpu(reg->index); + reg->index = pend->reg.index; + or_region_bits(reg, &pend->reg); - scoutfs_ring_append(sb, ®->eh); - radix_tree_tag_clear(&sal->regs, ind, DIRTY_RADIX_TAG); - start = ind + 1; - } - } while (nr); + rb_erase(&pend->node, &sal->pending_root); + kfree(pend); + } - return 0; + scoutfs_treap_dirty_ring(sal->treap); + scoutfs_treap_update_root(&super->alloc_treap_root, sal->treap); + ret = 0; +out: + up_write(&sal->rwsem); + return ret; } +static int alloc_treap_compare(void *key, void *data) +{ + u64 *ind = key; + struct scoutfs_alloc_region *reg = data; + + return scoutfs_cmp_u64s(*ind, le64_to_cpu(reg->index)); +} + +static void alloc_treap_fill(void *data, void *fill_arg) +{ + struct scoutfs_alloc_region *reg = data; + u64 *ind = fill_arg; + + memset(reg, 0, sizeof(struct scoutfs_alloc_region)); + reg->index = cpu_to_le64p(ind); +} + +static struct scoutfs_treap_ops alloc_treap_ops = { + .compare = alloc_treap_compare, + .fill = alloc_treap_fill, +}; + int scoutfs_alloc_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; struct seg_alloc *sal; /* bits need to be aligned so hosts can use native bitops */ - BUILD_BUG_ON(offsetof(struct scoutfs_ring_alloc_region, bits) & + BUILD_BUG_ON(offsetof(struct scoutfs_alloc_region, bits) & (sizeof(long) - 1)); sal = kzalloc(sizeof(struct seg_alloc), GFP_KERNEL); if (!sal) return -ENOMEM; - sbi->seg_alloc = sal; - spin_lock_init(&sal->lock); - /* inserts preload with _NOFS */ - INIT_RADIX_TREE(&sal->pending, GFP_ATOMIC); - INIT_RADIX_TREE(&sal->regs, GFP_ATOMIC); + init_rwsem(&sal->rwsem); + sal->pending_root = RB_ROOT; + sal->treap = scoutfs_treap_alloc(sb, &alloc_treap_ops, + &super->alloc_treap_root); + if (!sal->treap) { + kfree(sal); + return -ENOMEM; + } + /* XXX read next_segno from super? */ + sbi->seg_alloc = sal; + return 0; } -static void destroy_radix_regs(struct radix_tree_root *radix) -{ - struct scoutfs_ring_alloc_region *regs[16]; - int nr; - int i; - - - do { - nr = radix_tree_gang_lookup(radix, (void **)regs, - 0, ARRAY_SIZE(regs)); - for (i = 0; i < nr; i++) { - radix_tree_delete(radix, le64_to_cpu(regs[i]->index)); - kfree(regs[i]); - } - } while (nr); -} - void scoutfs_alloc_destroy(struct super_block *sb) { DECLARE_SEG_ALLOC(sb, sal); + struct pending_region *pend; + struct rb_node *node; if (sal) { - destroy_radix_regs(&sal->pending); - destroy_radix_regs(&sal->regs); + scoutfs_treap_free(sal->treap); + while ((node = rb_first(&sal->pending_root))) { + pend = container_of(node, struct pending_region, node); + rb_erase(&pend->node, &sal->pending_root); + kfree(pend); + } kfree(sal); } } diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index 4d3d398b..453d667a 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -6,8 +6,7 @@ struct scoutfs_alloc_region; int scoutfs_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_alloc_free(struct super_block *sb, u64 segno); -int scoutfs_alloc_add(struct super_block *sb, - struct scoutfs_ring_alloc_region *ins); +int scoutfs_alloc_has_dirty(struct super_block *sb); int scoutfs_alloc_dirty_ring(struct super_block *sb); int scoutfs_alloc_setup(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 71027e90..c611ea32 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -51,22 +51,63 @@ struct scoutfs_block_header { __le64 blkno; } __packed; -struct scoutfs_ring_entry_header { - __u8 type; - __le16 len; +struct scoutfs_treap_ref { + __le64 off; + __le64 gen; + __u8 aug_bits; } __packed; -#define SCOUTFS_RING_ADD_MANIFEST 1 -#define SCOUTFS_RING_ADD_ALLOC 2 +/* + * The lesser and greater bits are persistent on disk so that we can migrate + * nodes from the older half of the ring. + * + * The dirty bit is only used for in-memory nodes. + */ +#define SCOUTFS_TREAP_AUG_LESSER (1 << 0) +#define SCOUTFS_TREAP_AUG_GREATER (1 << 1) +#define SCOUTFS_TREAP_AUG_HALVES (SCOUTFS_TREAP_AUG_LESSER | \ + SCOUTFS_TREAP_AUG_GREATER) +#define SCOUTFS_TREAP_AUG_DIRTY (1 << 2) -struct scoutfs_ring_add_manifest { - struct scoutfs_ring_entry_header eh; +/* + * Treap nodes are stored at byte offset in the ring of blocks described + * by the super block. Each reference contains the off and gen that it + * will find in the node for verification. Each node has the header + * and data payload covered by a crc. + */ +struct scoutfs_treap_node { + __le32 crc; + __le64 off; + __le64 gen; + __le64 prio; + struct scoutfs_treap_ref left; + struct scoutfs_treap_ref right; + __le16 bytes; + u8 data[0]; +} __packed; + +struct scoutfs_treap_root { + struct scoutfs_treap_ref ref; +} __packed; + +/* + * This is absurdly huge. If there was only ever 1 item per segment and + * 2^64 items the tree could get this deep. + */ +#define SCOUTFS_MANIFEST_MAX_LEVEL 20 + +struct scoutfs_manifest { + struct scoutfs_treap_root root; + __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; +} __packed; + +struct scoutfs_manifest_entry { __le64 segno; __le64 seq; __le16 first_key_len; __le16 last_key_len; __u8 level; - /* first and last key bytes */ + __u8 keys[0]; } __packed; #define SCOUTFS_ALLOC_REGION_SHIFT 8 @@ -77,27 +118,11 @@ struct scoutfs_ring_add_manifest { * The bits need to be aligned so that the host can use native long * bitops on the bits in memory. */ -struct scoutfs_ring_alloc_region { - struct scoutfs_ring_entry_header eh; +struct scoutfs_alloc_region { __le64 index; - __u8 pad[5]; __le64 bits[SCOUTFS_ALLOC_REGION_BITS / 64]; } __packed; -/* - * This is absurdly huge. If there was only ever 1 item per segment and - * 2^64 items the tree could get this deep. - */ -#define SCOUTFS_MANIFEST_MAX_LEVEL 20 - -/* - * The packed entries in the block are terminated by a header with a 0 length. - */ -struct scoutfs_ring_block { - struct scoutfs_block_header hdr; - struct scoutfs_ring_entry_header entries[0]; -} __packed; - /* * We really want these to be a power of two size so that they're naturally * aligned. This ensures that they won't cross page boundaries and we @@ -315,12 +340,13 @@ struct scoutfs_super_block { __le64 free_blocks; __le64 ring_blkno; __le64 ring_blocks; - __le64 ring_index; - __le64 ring_nr; - __le64 ring_seq; + __le64 ring_tail_block; + __le64 ring_gen; __le64 buddy_blocks; struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; + struct scoutfs_treap_root alloc_treap_root; + struct scoutfs_manifest manifest; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 4980109c..51ff1b0e 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -14,43 +14,42 @@ #include #include #include +#include #include "super.h" #include "format.h" #include "kvec.h" #include "seg.h" #include "item.h" -#include "ring.h" +#include "treap.h" +#include "cmp.h" #include "manifest.h" #include "scoutfs_trace.h" +/* + * Manifest entries are stored as treap nodes in the ring. + * + * They're sorted first by level then by their first key. This enables + * the primary searches based on key value for looking up items in + * segments via the manifest. + * + * The treap also supports augmented searches. We get callbacks as the + * tree structure which lets us maintain data in nodes that describe + * subtrees to accelerate searches. We will record the max sequence + * numbers in subtrees for all the seq queries. We'll probably also + * have bits that direct us towards segments that contain deletion items + * for prioritized compaction. + */ + struct manifest { - spinlock_t lock; - - struct list_head level0_list; - unsigned int level0_nr; - - u8 last_level; - struct rb_root level_roots[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; - - struct list_head dirty_list; + struct rw_semaphore rwsem; + struct scoutfs_treap *treap; + u8 nr_levels; }; #define DECLARE_MANIFEST(sb, name) \ struct manifest *name = SCOUTFS_SB(sb)->manifest -struct manifest_entry { - union { - struct list_head level0_entry; - struct rb_node node; - }; - struct list_head dirty_entry; - - struct scoutfs_ring_add_manifest am; - /* u8 key_bytes[am.first_key_len]; */ - /* u8 val_bytes[am.last_key_len]; */ -}; - /* * A reader uses references to segments copied from a walk of the * manifest. The references are a point in time sample of the manifest. @@ -71,17 +70,31 @@ struct manifest_ref { u16 first_key_len; u16 last_key_len; u8 level; - u8 keys[SCOUTFS_MAX_KEY_SIZE * 2]; + u8 keys[0]; }; -static void init_ment_keys(struct manifest_entry *ment, struct kvec *first, - struct kvec *last) +struct manifest_fill_args { + struct scoutfs_manifest_entry ment; + struct kvec *first; + struct kvec *last; +}; + +struct manifest_search_key { + u64 seq; + struct kvec *key; + u8 level; +}; + +static void init_ment_keys(struct scoutfs_manifest_entry *ment, + struct kvec *first, struct kvec *last) { - scoutfs_kvec_init(first, &ment->am + 1, - le16_to_cpu(ment->am.first_key_len)); - scoutfs_kvec_init(last, (void *)(&ment->am + 1) + - le16_to_cpu(ment->am.first_key_len), - le16_to_cpu(ment->am.last_key_len)); + if (first) + scoutfs_kvec_init(first, ment->keys, + le16_to_cpu(ment->first_key_len)); + if (last) + scoutfs_kvec_init(last, ment->keys + + le16_to_cpu(ment->first_key_len), + le16_to_cpu(ment->last_key_len)); } static void init_ref_keys(struct manifest_ref *ref, struct kvec *first, @@ -95,7 +108,7 @@ static void init_ref_keys(struct manifest_ref *ref, struct kvec *first, } static bool cmp_range_ment(struct kvec *key, struct kvec *end, - struct manifest_entry *ment) + struct scoutfs_manifest_entry *ment) { SCOUTFS_DECLARE_KVEC(first); SCOUTFS_DECLARE_KVEC(last); @@ -105,199 +118,102 @@ static bool cmp_range_ment(struct kvec *key, struct kvec *end, return scoutfs_kvec_cmp_overlap(key, end, first, last); } -static struct manifest_entry *find_ment(struct rb_root *root, struct kvec *key) -{ - struct rb_node *node = root->rb_node; - struct manifest_entry *ment; - int cmp; - - while (node) { - ment = container_of(node, struct manifest_entry, node); - - cmp = cmp_range_ment(key, key, ment); - if (cmp < 0) - node = node->rb_left; - else if (cmp > 0) - node = node->rb_right; - else - return ment; - } - - return NULL; -} - /* - * Insert a new entry into one of the L1+ trees. There should never be - * entries that overlap. + * Insert a new manifest entry in the treap. The treap allocates a new + * node for us and we fill it. */ -static int insert_ment(struct rb_root *root, struct manifest_entry *ins) +int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, + struct kvec *last, u64 segno, u64 seq, u8 level) { - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct manifest_entry *ment; - SCOUTFS_DECLARE_KVEC(key); - SCOUTFS_DECLARE_KVEC(end); - int cmp; - - init_ment_keys(ins, key, end); - - while (*node) { - parent = *node; - ment = container_of(*node, struct manifest_entry, node); - - cmp = cmp_range_ment(key, end, ment); - if (cmp < 0) { - node = &(*node)->rb_left; - } else if (cmp > 0) { - node = &(*node)->rb_right; - } else { - return -EEXIST; - } - } - - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); - - return 0; -} - -static void free_ment(struct manifest_entry *ment) -{ - if (!IS_ERR_OR_NULL(ment)) - kfree(ment); -} - -static int add_ment(struct manifest *mani, struct manifest_entry *ment, - bool dirty) -{ - u8 level = ment->am.level; + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_entry *ment; + struct manifest_fill_args args; + struct manifest_search_key skey; + unsigned key_bytes; + unsigned bytes; int ret; + trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); - trace_printk("adding ment %p level %u\n", ment, level); + key_bytes = scoutfs_kvec_length(first) + scoutfs_kvec_length(last); + bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); - if (level) { - ret = insert_ment(&mani->level_roots[level], ment); - if (!ret) - mani->last_level = max(mani->last_level, level); + args.ment.segno = cpu_to_le64(segno); + args.ment.seq = cpu_to_le64(seq); + args.ment.first_key_len = cpu_to_le16(scoutfs_kvec_length(first)); + args.ment.last_key_len = cpu_to_le16(scoutfs_kvec_length(last)); + args.ment.level = level; + + args.first = first; + args.last = last; + + skey.key = first; + skey.level = level; + skey.seq = seq; + + down_write(&mani->rwsem); + + ment = scoutfs_treap_insert(mani->treap, &skey, bytes, &args); + if (IS_ERR(ment)) { + ret = PTR_ERR(ment); } else { - list_add_tail(&ment->level0_entry, &mani->level0_list); - mani->level0_nr++; + mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); + le64_add_cpu(&super->manifest.level_counts[level], 1); ret = 0; } - if (dirty) - list_add_tail(&ment->dirty_entry, &mani->dirty_list); + up_write(&mani->rwsem); return ret; } -static void update_last_level(struct manifest *mani) -{ - int i; - - for (i = mani->last_level; - i > 0 && RB_EMPTY_ROOT(&mani->level_roots[i]); i--) - ; - - mani->last_level = i; -} - -static void remove_ment(struct manifest *mani, struct manifest_entry *ment) -{ - u8 level = ment->am.level; - - if (level) { - rb_erase(&ment->node, &mani->level_roots[level]); - update_last_level(mani); - } else { - list_del_init(&ment->level0_entry); - mani->level0_nr--; - } - - /* XXX more carefully remove dirty ments.. should be exceptional */ - if (!list_empty(&ment->dirty_entry)) - list_del_init(&ment->dirty_entry); -} - -int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level, - bool dirty) -{ - DECLARE_MANIFEST(sb, mani); - struct manifest_entry *ment; - SCOUTFS_DECLARE_KVEC(ment_first); - SCOUTFS_DECLARE_KVEC(ment_last); - unsigned long flags; - int key_bytes; - int ret; - - trace_scoutfs_manifest_add(sb, first, last, segno, seq, level, dirty); - - key_bytes = scoutfs_kvec_length(first) + scoutfs_kvec_length(last); - ment = kmalloc(sizeof(struct manifest_entry) + key_bytes, GFP_NOFS); - if (!ment) - return -ENOMEM; - - if (level) - RB_CLEAR_NODE(&ment->node); - else - INIT_LIST_HEAD(&ment->level0_entry); - INIT_LIST_HEAD(&ment->dirty_entry); - - ment->am.eh.type = SCOUTFS_RING_ADD_MANIFEST; - ment->am.eh.len = cpu_to_le16(sizeof(struct scoutfs_ring_add_manifest) + - key_bytes); - ment->am.segno = cpu_to_le64(segno); - ment->am.seq = cpu_to_le64(seq); - ment->am.first_key_len = cpu_to_le16(scoutfs_kvec_length(first)); - ment->am.last_key_len = cpu_to_le16(scoutfs_kvec_length(last)); - ment->am.level = level; - - init_ment_keys(ment, ment_first, ment_last); - scoutfs_kvec_memcpy(ment_first, first); - scoutfs_kvec_memcpy(ment_last, last); - - /* XXX think about where to insert level 0 */ - spin_lock_irqsave(&mani->lock, flags); - ret = add_ment(mani, ment, dirty); - spin_unlock_irqrestore(&mani->lock, flags); - if (WARN_ON_ONCE(ret)) /* XXX can this happen? ring corruption? */ - free_ment(ment); - - return ret; -} - -/* - * Grab an allocated ref from the src list, fill it with the details - * from the ment, and add it to the dst list. The ref is added to the - * tail of the dst list so that we maintain the caller's manifest walk - * order. - */ -static void fill_ref_tail(struct list_head *dst, struct list_head *src, - struct manifest_entry *ment) +static int alloc_add_ref(struct list_head *list, + struct scoutfs_manifest_entry *ment) { SCOUTFS_DECLARE_KVEC(ment_first); SCOUTFS_DECLARE_KVEC(ment_last); SCOUTFS_DECLARE_KVEC(first); SCOUTFS_DECLARE_KVEC(last); struct manifest_ref *ref; - - ref = list_first_entry(src, struct manifest_ref, entry); - - ref->segno = le64_to_cpu(ment->am.segno); - ref->seq = le64_to_cpu(ment->am.seq); - ref->level = ment->am.level; - ref->first_key_len = le16_to_cpu(ment->am.first_key_len); - ref->last_key_len = le16_to_cpu(ment->am.last_key_len); + unsigned bytes; init_ment_keys(ment, ment_first, ment_last); - init_ref_keys(ref, first, last); + bytes = scoutfs_kvec_length(ment_first) + + scoutfs_kvec_length(ment_first); + + ref = kmalloc(offsetof(struct manifest_ref, keys[bytes]), GFP_NOFS); + if (!ref) + return -ENOMEM; + + memset(ref, 0, offsetof(struct manifest_ref, keys)); + + ref->segno = le64_to_cpu(ment->segno); + ref->seq = le64_to_cpu(ment->seq); + ref->level = ment->level; + ref->first_key_len = le16_to_cpu(ment->first_key_len); + ref->last_key_len = le16_to_cpu(ment->last_key_len); + + init_ref_keys(ref, first, last); scoutfs_kvec_memcpy(first, ment_first); scoutfs_kvec_memcpy(last, ment_last); - list_move_tail(&ref->entry, dst); + list_add_tail(&ref->entry, list); + + return 0; + +} + +/* sort level 0 segments of the list from greatest to least seq */ +static int cmp_ref_list_seqs(void *priv, struct list_head *A, + struct list_head *B) +{ + struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); + struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); + + return -scoutfs_cmp_u64s(a->seq, b->seq); } /* @@ -308,78 +224,85 @@ static void fill_ref_tail(struct list_head *dst, struct list_head *src, * of items that we want to search because the level 0 segments can * arbitrarily overlap with each other. * - * We only need to search for the starting key in all the higher order - * levels. They do not overlap so we can iterate through the key space - * in each segment starting with the key. + * We only need to search for the starting key in all the higher levels. + * They do not overlap so we can iterate through the key space in each + * segment starting with the key. */ -static int get_range_refs(struct manifest *mani, struct kvec *key, - struct kvec *end, struct list_head *ref_list) +static int get_range_refs(struct super_block *sb, struct manifest *mani, + struct kvec *key, struct kvec *end, + struct list_head *ref_list) { - struct manifest_entry *ment; + struct scoutfs_manifest_entry *ment; + struct manifest_search_key skey; + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); struct manifest_ref *ref; struct manifest_ref *tmp; - struct rb_root *root; - unsigned long flags; - unsigned int total; - unsigned int nr = 0; - LIST_HEAD(alloced); + int cmp; int ret; int i; - trace_printk("getting refs\n"); + down_write(&mani->rwsem); - spin_lock_irqsave(&mani->lock, flags); + /* get level 0 segments that overlap with the missing range */ + ment = scoutfs_treap_first(mani->treap); + while (!IS_ERR_OR_NULL(ment)) { + if (ment->level > 0) + break; - /* allocate enough refs for the of segments */ - total = mani->level0_nr + mani->last_level; - while (nr < total) { - spin_unlock_irqrestore(&mani->lock, flags); + cmp = cmp_range_ment(key, end, ment); + if (cmp < 0) + break; - for (i = nr; i < total; i++) { - ref = kmalloc(sizeof(struct manifest_ref), GFP_NOFS); - if (!ref) { - ret = -ENOMEM; + if (cmp == 0) { + ret = alloc_add_ref(ref_list, ment); + if (ret) goto out; - } - - memset(ref, 0, offsetof(struct manifest_ref, keys)); - list_add(&ref->entry, &alloced); } - nr = total; - spin_lock_irqsave(&mani->lock, flags); + ment = scoutfs_treap_next(mani->treap, ment); + } + if (IS_ERR(ment)) { + ret = PTR_ERR(ment); + goto out; } - /* find all the overlapping level 0 segments */ - list_for_each_entry(ment, &mani->level0_list, level0_entry) { - if (cmp_range_ment(key, end, ment)) - continue; + /* level0s are sorted by key, reverse sort by seq */ + list_sort(NULL, ref_list, cmp_ref_list_seqs); - fill_ref_tail(ref_list, &alloced, ment); + /* get higher level segments that overlap with the starting key */ + for (i = 1; i < mani->nr_levels; i++) { + skey.key = key; + skey.level = i; + + /* XXX should use level counts to skip searches */ + + ment = scoutfs_treap_lookup(mani->treap, &skey); + if (IS_ERR(ment)) { + ret = PTR_ERR(ment); + goto out; + } + + if (ment) { + init_ment_keys(ment, first, last); + ret = alloc_add_ref(ref_list, ment); + if (ret) + goto out; + } } - /* find each segment containing the key at the higher orders */ - for (i = 1; i <= mani->last_level; i++) { - root = &mani->level_roots[i]; - if (RB_EMPTY_ROOT(root)) - continue; - - ment = find_ment(root, key); - if (ment) - fill_ref_tail(ref_list, &alloced, ment); - } - - spin_unlock_irqrestore(&mani->lock, flags); ret = 0; out: + up_write(&mani->rwsem); + if (ret) { - list_splice_init(ref_list, &alloced); - list_for_each_entry_safe(ref, tmp, &alloced, entry) { + list_for_each_entry_safe(ref, tmp, ref_list, entry) { list_del_init(&ref->entry); kfree(ref); } } + trace_printk("ret %d\n", ret); return ret; } @@ -436,7 +359,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, trace_printk("reading items\n"); /* get refs on all the segments */ - ret = get_range_refs(mani, key, end, &ref_list); + ret = get_range_refs(sb, mani, key, end, &ref_list); if (ret) return ret; @@ -576,8 +499,13 @@ out: int scoutfs_manifest_has_dirty(struct super_block *sb) { DECLARE_MANIFEST(sb, mani); + int ret; - return !list_empty_careful(&mani->dirty_list); + down_write(&mani->rwsem); + ret = scoutfs_treap_has_dirty(mani->treap); + up_write(&mani->rwsem); + + return ret; } /* @@ -588,33 +516,100 @@ int scoutfs_manifest_has_dirty(struct super_block *sb) int scoutfs_manifest_dirty_ring(struct super_block *sb) { DECLARE_MANIFEST(sb, mani); - struct manifest_entry *ment; - struct manifest_entry *tmp; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; - list_for_each_entry_safe(ment, tmp, &mani->dirty_list, dirty_entry) { - scoutfs_ring_append(sb, &ment->am.eh); - list_del_init(&ment->dirty_entry); - } + down_write(&mani->rwsem); + scoutfs_treap_dirty_ring(mani->treap); + scoutfs_treap_update_root(&super->manifest.root, mani->treap); + up_write(&mani->rwsem); return 0; } +/* + * Manifest entries are first sorted by their level. + * + * Level 0 segments can arbitrarily overlap. Their manifest entries are + * sorted by their first key so that searches can iterate over the + * entries until first shows that no more segments can overlap. We then + * sort by the sequence so that we can manage entries that have + * identical keys. + * + * Higher level segments don't overlap. There will never be manifest + * entries with the same key at a given level. + */ +static int manifest_treap_compare(void *key, void *data) +{ + struct manifest_search_key *skey = key; + struct scoutfs_manifest_entry *ment = data; + SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); + + if (skey->level < ment->level) + return -1; + if (skey->level > ment->level) + return 1; + + init_ment_keys(ment, first, NULL); + + if (skey->level == 0) + return scoutfs_kvec_memcmp(skey->key, first) ?: + scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); + + init_ment_keys(ment, NULL, last); + + return scoutfs_kvec_cmp_overlap(skey->key, skey->key, first, last); +} + +static void manifest_treap_fill(void *data, void *arg) +{ + struct scoutfs_manifest_entry *ment = data; + struct manifest_fill_args *args = arg; + SCOUTFS_DECLARE_KVEC(ment_first); + SCOUTFS_DECLARE_KVEC(ment_last); + + *ment = args->ment; + + init_ment_keys(ment, ment_first, ment_last); + scoutfs_kvec_memcpy(ment_first, args->first); + scoutfs_kvec_memcpy(ment_last, args->last); +} + +static struct scoutfs_treap_ops manifest_treap_ops = { + .compare = manifest_treap_compare, + .fill = manifest_treap_fill, + /* update aug when we track left and right max seq */ +}; + + int scoutfs_manifest_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; struct manifest *mani; int i; mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); if (!mani) return -ENOMEM; - sbi->manifest = mani; - spin_lock_init(&mani->lock); - INIT_LIST_HEAD(&mani->level0_list); - INIT_LIST_HEAD(&mani->dirty_list); - for (i = 0; i < ARRAY_SIZE(mani->level_roots); i++) - mani->level_roots[i] = RB_ROOT; + init_rwsem(&mani->rwsem); + mani->treap = scoutfs_treap_alloc(sb, &manifest_treap_ops, + &super->manifest.root); + if (!mani->treap) { + kfree(mani); + return -ENOMEM; + } + + for (i = ARRAY_SIZE(super->manifest.level_counts) - 1; i >= 0; i--) { + if (super->manifest.level_counts[i]) { + mani->nr_levels = i + 1; + break; + } + } + + sbi->manifest = mani; return 0; } @@ -623,30 +618,9 @@ void scoutfs_manifest_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct manifest *mani = sbi->manifest; - struct manifest_entry *ment; - struct manifest_entry *tmp; - struct rb_node *node; - struct rb_root *root; - int i; - if (!mani) - return; - - for (i = 1; i <= mani->last_level; i++) { - root = &mani->level_roots[i]; - - for (node = rb_first(root); node; ) { - ment = container_of(node, struct manifest_entry, node); - node = rb_next(node); - remove_ment(mani, ment); - free_ment(ment); - } + if (mani) { + scoutfs_treap_free(mani->treap); + kfree(mani); } - - list_for_each_entry_safe(ment, tmp, &mani->level0_list, level0_entry) { - remove_ment(mani, ment); - free_ment(ment); - } - - kfree(mani); } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 9f1477b4..021bdf14 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -2,8 +2,7 @@ #define _SCOUTFS_MANIFEST_H_ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level, - bool dirty); + struct kvec *last, u64 segno, u64 seq, u8 level); int scoutfs_manifest_has_dirty(struct super_block *sb); int scoutfs_manifest_dirty_ring(struct super_block *sb); diff --git a/kmod/src/ring.c b/kmod/src/ring.c deleted file mode 100644 index 050fa49c..00000000 --- a/kmod/src/ring.c +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "kvec.h" -#include "bio.h" -#include "manifest.h" -#include "alloc.h" -#include "ring.h" -#include "crc.h" - - -/* - * Right now we're only writing a segment a time. The entries needed to - * write a segment will always be smaller than a segment itself. - * - * XXX This'll get more clever as we can write multiple segments and build - * up dirty entries while processing compaction results. - */ -struct ring_info { - struct page *pages[SCOUTFS_SEGMENT_PAGES]; - struct scoutfs_ring_block *ring; - struct scoutfs_ring_entry_header *next_eh; - unsigned int nr_blocks; - unsigned int space; -}; - -#define DECLARE_RING_INFO(sb, name) \ - struct ring_info *name = SCOUTFS_SB(sb)->ring_info - -/* - * XXX - * - verify blocks - * - could compress - * - have all entry sources dirty at cursors before dirtying - * - advancing cursor updates head as cursor wraps - */ - -/* - * The space calculation when starting a block included a final empty - * entry header. That is zeroed here. - */ -static void finish_block(struct scoutfs_ring_block *ring, unsigned int tail) -{ - memset((char *)ring + SCOUTFS_BLOCK_SIZE - tail, 0, tail); - scoutfs_crc_block(&ring->hdr); -} - -void scoutfs_ring_append(struct super_block *sb, - struct scoutfs_ring_entry_header *eh) -{ - DECLARE_RING_INFO(sb, rinf); - struct scoutfs_ring_block *ring = rinf->ring; - unsigned int len = le16_to_cpu(eh->len); - - if (rinf->space < (len + sizeof(struct scoutfs_ring_entry_header))) { - if (rinf->space) - finish_block(ring, rinf->space); - ring = scoutfs_page_block_address(rinf->pages, rinf->nr_blocks); - rinf->ring = ring; - - memset(ring, 0, sizeof(struct scoutfs_ring_block)); - - rinf->nr_blocks++; - rinf->next_eh = ring->entries; - rinf->space = SCOUTFS_BLOCK_SIZE - - offsetof(struct scoutfs_ring_block, entries); - } - - memcpy(rinf->next_eh, eh, len); - rinf->next_eh = (void *)rinf->next_eh + len; - rinf->space -= len; -} - -static u64 ring_ind_wrap(struct scoutfs_super_block *super, u64 ind) -{ - u64 ring_blocks = le64_to_cpu(super->ring_blocks); - - while (ind >= ring_blocks) - ind -= ring_blocks; - - return ind; -} - -/* - * Submit writes for all the dirty ring blocks that accumulated as dirty - * entries were appended. The dirty ring blocks are contiguous in the - * page array but can wrap in the block ring on disk. - * - * If it wraps then we submit the earlier fragment at the head of the - * ring first. - * - * The wrapped fragment starts at some block offset in the page array. - * The hacky page array math only works when our fixed 4k block size == - * page_size. To fix it we'd add a offset block to the bio submit loop - * which could add an initial partial page vec to the bios. - */ -int scoutfs_ring_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_RING_INFO(sb, rinf); - u64 wrapped_blocks; - u64 index_blocks; - u64 index; - - if (!rinf->nr_blocks) - return 0; - - if (rinf->space) - finish_block(rinf->ring, rinf->space); - - /* first and last ring block indexes that will be written */ - index = ring_ind_wrap(super, le64_to_cpu(super->ring_index) + - le64_to_cpu(super->ring_nr)); - index_blocks = min_t(u64, rinf->nr_blocks, - le64_to_cpu(super->ring_blocks) - index); - wrapped_blocks = rinf->nr_blocks - index_blocks; - - if (wrapped_blocks) { - BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); - scoutfs_bio_submit_comp(sb, WRITE, rinf->pages + index_blocks, - le64_to_cpu(super->ring_blkno), - wrapped_blocks, comp); - } - - scoutfs_bio_submit_comp(sb, WRITE, rinf->pages, - le64_to_cpu(super->ring_blkno) + index, - index_blocks, comp); - - /* record new tail index in super and reset for next trans */ - le64_add_cpu(&super->ring_nr, rinf->nr_blocks); - rinf->nr_blocks = 0; - rinf->space = 0; - - return 0; -} - -static int read_one_entry(struct super_block *sb, - struct scoutfs_ring_entry_header *eh) -{ - struct scoutfs_ring_alloc_region *reg; - struct scoutfs_ring_add_manifest *am; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); - int ret; - - trace_printk("type %u len %u\n", eh->type, le16_to_cpu(eh->len)); - - switch(eh->type) { - case SCOUTFS_RING_ADD_MANIFEST: - am = container_of(eh, struct scoutfs_ring_add_manifest, eh); - - trace_printk("lens %u %u\n", - le16_to_cpu(am->first_key_len), - le16_to_cpu(am->last_key_len)); - - scoutfs_kvec_init(first, am + 1, - le16_to_cpu(am->first_key_len)); - scoutfs_kvec_init(last, - first[0].iov_base + first[0].iov_len, - le16_to_cpu(am->last_key_len)); - - ret = scoutfs_manifest_add(sb, first, last, - le64_to_cpu(am->segno), - le64_to_cpu(am->seq), am->level, - false); - break; - - case SCOUTFS_RING_ADD_ALLOC: - reg = container_of(eh, struct scoutfs_ring_alloc_region, eh); - ret = scoutfs_alloc_add(sb, reg); - break; - - default: - ret = -EINVAL; - } - - return ret; -} - -static int read_entries(struct super_block *sb, - struct scoutfs_ring_block *ring) -{ - struct scoutfs_ring_entry_header *eh; - int ret = 0; - - for (eh = ring->entries; eh->len; - eh = (void *)eh + le16_to_cpu(eh->len)) { - - ret = read_one_entry(sb, eh); - if (ret) - break; - } - - return ret; -} - - -/* read in a meg at a time */ -#define NR_PAGES DIV_ROUND_UP(1024 * 1024, PAGE_SIZE) -#define NR_BLOCKS (NR_PAGES * SCOUTFS_BLOCKS_PER_PAGE) - -int scoutfs_ring_read(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_ring_block *ring; - struct page **pages; - struct page *page; - u64 index; - u64 blkno; - u64 part; - u64 seq; - u64 nr; - int ret; - int i; - - /* nr_blocks/pages calc doesn't handle multiple pages per block */ - BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SIZE); - - pages = kcalloc(NR_PAGES, sizeof(struct page *), GFP_NOFS); - if (!pages) - return -ENOMEM; - - for (i = 0; i < NR_PAGES; i++) { - page = alloc_page(GFP_NOFS); - if (!page) { - ret = -ENOMEM; - goto out; - } - - pages[i] = page; - } - - index = le64_to_cpu(super->ring_index); - nr = le64_to_cpu(super->ring_nr); - seq = le64_to_cpu(super->ring_seq); - - while (nr) { - blkno = le64_to_cpu(super->ring_blkno) + index; - /* XXX min3_t should be a thing */ - part = min3(nr, (u64)NR_BLOCKS, - le64_to_cpu(super->ring_blocks) - index); - - trace_printk("index %llu part %llu\n", index, part); - - ret = scoutfs_bio_read(sb, pages, blkno, part); - if (ret) - goto out; - - /* XXX verify block header */ - - for (i = 0; i < part; i++) { - ring = scoutfs_page_block_address(pages, i); - ret = read_entries(sb, ring); - if (ret) - goto out; - } - - index = ring_ind_wrap(super, index + part); - nr -= part; - } - -out: - for (i = 0; i < NR_PAGES && pages && pages[i]; i++) - __free_page(pages[i]); - kfree(pages); - - return ret; -} - -int scoutfs_ring_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct ring_info *rinf; - struct page *page; - int i; - - rinf = kzalloc(sizeof(struct ring_info), GFP_KERNEL); - if (!rinf) - return -ENOMEM; - sbi->ring_info = rinf; - - for (i = 0; i < ARRAY_SIZE(rinf->pages); i++) { - page = alloc_page(GFP_KERNEL); - if (!page) { - while (--i >= 0) - __free_page(rinf->pages[i]); - return -ENOMEM; - } - - rinf->pages[i] = page; - } - - return 0; -} - -void scoutfs_ring_destroy(struct super_block *sb) -{ - DECLARE_RING_INFO(sb, rinf); - int i; - - if (rinf) { - for (i = 0; i < ARRAY_SIZE(rinf->pages); i++) - __free_page(rinf->pages[i]); - - kfree(rinf); - } -} - diff --git a/kmod/src/ring.h b/kmod/src/ring.h deleted file mode 100644 index 94eb84c3..00000000 --- a/kmod/src/ring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _SCOUTFS_RING_H_ -#define _SCOUTFS_RING_H_ - -#include - -struct scoutfs_bio_completion; - -int scoutfs_ring_read(struct super_block *sb); -void scoutfs_ring_append(struct super_block *sb, - struct scoutfs_ring_entry_header *eh); - -int scoutfs_ring_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp); - -int scoutfs_ring_setup(struct super_block *sb); -void scoutfs_ring_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 8793a544..7b10ae90 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -349,15 +349,14 @@ DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_since, TRACE_EVENT(scoutfs_manifest_add, TP_PROTO(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level, bool dirty), - TP_ARGS(sb, first, last, segno, seq, level, dirty), + struct kvec *last, u64 segno, u64 seq, u8 level), + TP_ARGS(sb, first, last, segno, seq, level), TP_STRUCT__entry( __dynamic_array(char, first, scoutfs_kvec_key_strlen(first)) __dynamic_array(char, last, scoutfs_kvec_key_strlen(last)) __field(u64, segno) __field(u64, seq) __field(u8, level) - __field(u8, dirty) ), TP_fast_assign( scoutfs_kvec_key_sprintf(__get_dynamic_array(first), first); @@ -365,11 +364,10 @@ TRACE_EVENT(scoutfs_manifest_add, __entry->segno = segno; __entry->seq = seq; __entry->level = level; - __entry->dirty = dirty; ), - TP_printk("first %s last %s segno %llu seq %llu level %u dirty %u", + TP_printk("first %s last %s segno %llu seq %llu level %u", __get_str(first), __get_str(last), __entry->segno, - __entry->seq, __entry->level, __entry->dirty) + __entry->seq, __entry->level) ); TRACE_EVENT(scoutfs_item_lookup, diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 37277096..c16c2e2d 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -540,7 +540,7 @@ int scoutfs_seg_manifest_add(struct super_block *sb, kvec_from_pages(seg, last, item.key_off, item.key_len); return scoutfs_manifest_add(sb, first, last, le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->max_seq), level, true); + le64_to_cpu(sblk->max_seq), level); } int scoutfs_seg_setup(struct super_block *sb) diff --git a/kmod/src/super.c b/kmod/src/super.c index 5ce1a2b7..00cafba4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -28,12 +28,12 @@ #include "counters.h" #include "trans.h" #include "buddy.h" -#include "ring.h" #include "item.h" #include "manifest.h" #include "seg.h" #include "bio.h" #include "alloc.h" +#include "treap.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -228,8 +228,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_alloc_setup(sb) ?: - scoutfs_ring_setup(sb) ?: - scoutfs_ring_read(sb) ?: + scoutfs_treap_setup(sb) ?: // scoutfs_buddy_setup(sb) ?: scoutfs_setup_trans(sb); if (ret) @@ -269,8 +268,8 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); + scoutfs_treap_destroy(sb); scoutfs_seg_destroy(sb); - scoutfs_ring_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index bb803105..f68a1c66 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -12,7 +12,7 @@ struct buddy_info; struct item_cache; struct manifest; struct segment_cache; -struct ring_info; +struct treap_info; struct scoutfs_sb_info { struct super_block *sb; @@ -36,7 +36,7 @@ struct scoutfs_sb_info { struct item_cache *item_cache; struct segment_cache *segment_cache; struct seg_alloc *seg_alloc; - struct ring_info *ring_info; + struct treap_info *treap_info; struct buddy_info *buddy_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 1e23295f..04b1ed52 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -27,7 +27,7 @@ #include "manifest.h" #include "seg.h" #include "alloc.h" -#include "ring.h" +#include "treap.h" #include "scoutfs_trace.h" /* @@ -96,20 +96,15 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); #endif - /* - * We only have to check if there are dirty items or manifest - * entries. You can't have dirty alloc regions without having - * changed references to the allocated segments which produces - * dirty manfiest entries. - */ - if (scoutfs_item_dirty_bytes(sb) || scoutfs_manifest_has_dirty(sb)) { + if (scoutfs_item_dirty_bytes(sb) || scoutfs_manifest_has_dirty(sb) || + scoutfs_alloc_has_dirty(sb)) { ret = scoutfs_seg_alloc(sb, &seg) ?: - scoutfs_item_dirty_seg(sb, seg); + scoutfs_item_dirty_seg(sb, seg) ?: scoutfs_seg_manifest_add(sb, seg, 0) ?: scoutfs_manifest_dirty_ring(sb) ?: scoutfs_alloc_dirty_ring(sb) ?: - scoutfs_ring_submit_write(sb, &comp) ?: + scoutfs_treap_submit_write(sb, &comp) ?: scoutfs_seg_submit_write(sb, seg, &comp) ?: scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_write_dirty_super(sb); diff --git a/kmod/src/treap.c b/kmod/src/treap.c new file mode 100644 index 00000000..81cfe343 --- /dev/null +++ b/kmod/src/treap.c @@ -0,0 +1,1271 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "kvec.h" +#include "bio.h" +#include "treap.h" +#include "scoutfs_trace.h" + +/* + * scoutfs builds a consistent file system out of segments by describing + * them all with the manifest. Typically the manifest will fit in + * memory but in the pathological case it can be much larger. Our task + * is to index the manifest such that the pathological case is possible + * but the typical case isn't unreasonably penalized by the IO cost of + * maintaining the index. + * + * We chose to index the manifest by storing entries in treap nodes in a + * static ring. Updates are large contiguous writes to the ring with + * low amplification. Incremental updates can similarly read-ahead + * large chunks of the ring. Entirely cold reads end up issuing lots of + * small dependent random IOs. + * + * The nodes in the ring are loaded into native copies in memory. + * Having native allocated nodes lets us do things that would be + * unreasonable if we only traversed persistent structures in cached + * blocks: pointers to nodes in memory instead of indirecting through + * block cache lookups, parent pointers for trivial iteration but which + * would would rule out cow updates, and per-node lru tracking so that + * we can reclaim from the leaves of the tree up to the root without + * false pinning based on which nodes happen to share blocks. + * + * As nodes are modified or inserted they're marked dirty. Eventually + * all the dirty nodes are written to the tail of the ring. We ensure + * that new nodes written at the tail never overwrite old live nodes by + * using a large ring and constantly also migrating old nodes in the + * ring to the tail. + * + * Nodes don't span 4k blocks so there will always be at least a node + * struct's worth of blank space in each block, more typically half the + * average item length, and at worst the max item length. + * + * The tree is augmented to enable searches by more than the primary + * sort keys of the tree. The treap itself maintains augmentation in + * memory to track dirty nodes and in the persistent nodes to track old + * nodes for migration. Callers get callbacks to maintain their own + * augmentation in the node payloads. + * + * Each dirty node gets a generation number that is incremented for each + * version of the tree that is written to the tail of the ring. This + * lets traverse cached nodes without needing strong cache coherence + * with other node writers. With the byte offset and generation of root + * node we can traverse our cached nodes and retry the walk when our + * nodes are stale. + * + * XXX + * - add lru list, nodes to tail during walk, shrink from head + * - stale walking needs work: restart walk, get new root sample + * - lru would need to reclaim nodes orphaned by new root ref walk + */ + +/* + * We preallocate sufficient pages to write all the treap nodes to write + * a transactoin. + * + * XXX Today we only ever write a l0 segment or update the manifest and + * allocator for a single compaction. Those events are *well* less than + * the number of pages that make up a large segment. We'll want this to + * be more careful in the future as we batch up updates from lots of + * writers. + */ +struct treap_info { + /* static, derived from the super */ + u64 last_ring_off; + + /* temporarily assigned to each dirty node */ + u64 dirty_off; + u64 dirty_gen; + + /* used to write nodes to the ring */ + struct page *pages[SCOUTFS_SEGMENT_PAGES]; + u64 pages_off; + u64 ring_off; + unsigned int nr_blocks; + unsigned block_space; +}; + +#define DECLARE_TREAP_INFO(sb, name) \ + struct treap_info *name = SCOUTFS_SB(sb)->treap_info + +struct treap_ref { + struct treap_node *node; + u64 off; + u64 gen; + u8 aug_bits; +}; + +struct scoutfs_treap { + struct super_block *sb; + struct scoutfs_super_block *super; + struct scoutfs_treap_ops *ops; + struct treap_ref root_ref; + u64 dirty_bytes; +}; + +/* + * The in-memory node differs in that it uses native endian fields, has + * a parent pointer, and (will some day have) an lru for reclaiming from + * the leaves up. + * + * The data is long aligned so that callers can use native longs to + * manipulate bitmaps in the data. + */ +struct treap_node { + u64 off; + u64 gen; + u64 prio; + u16 bytes; + + struct treap_node *parent; + + struct treap_ref left; + struct treap_ref right; + + u8 data[0] __aligned(sizeof(long)); +}; + +static struct treap_ref *parent_ref(struct scoutfs_treap *treap, + struct treap_node *node) +{ + if (!node->parent) + return &treap->root_ref; + if (node->parent->left.node == node) + return &node->parent->left; + return &node->parent->right; +} + +static u8 off_aug_bit(struct scoutfs_treap *treap, u64 off) +{ + u64 blocks = le64_to_cpu(treap->super->ring_blocks); + u64 mid = (blocks << SCOUTFS_BLOCK_SHIFT) / 2; + + return off < mid ? SCOUTFS_TREAP_AUG_LESSER : + SCOUTFS_TREAP_AUG_GREATER; +} + +static u8 old_aug_bit(struct scoutfs_treap *treap) +{ + DECLARE_TREAP_INFO(treap->sb, tinf); + + return off_aug_bit(treap, tinf->dirty_off) ^ SCOUTFS_TREAP_AUG_HALVES; +} + + +/* Return the aug bits that'll be used to refer to the given node. */ +static u8 node_aug_bits(struct scoutfs_treap *treap, struct treap_node *node) +{ + DECLARE_TREAP_INFO(treap->sb, tinf); + u8 aug_bits = 0; + + if (node->off == tinf->dirty_off) + aug_bits |= SCOUTFS_TREAP_AUG_DIRTY; + + return aug_bits | off_aug_bit(treap, node->off); +} + +/* + * Update the treap augmentation until its back in sync. We can be + * called with a null node to repair a non-existing parent and we just + * have to clear the root aug_bits in that case. + */ +static void update_internal_aug(struct scoutfs_treap *treap, + struct treap_node *node) +{ + struct treap_ref *ref; + u8 bits; + + if (!node) + parent_ref(treap, node)->aug_bits = 0; + + while (node) { + bits = node_aug_bits(treap, node); + ref = parent_ref(treap, node); + if (ref->aug_bits == bits) + break; + ref->aug_bits = bits; + node = node->parent; + } +} + +static bool ops_update_aug(struct scoutfs_treap *treap, + struct treap_node *parent, struct treap_node *node) +{ + if (!treap->ops->update_aug) + return false; + + return treap->ops->update_aug(parent->data, parent->left.node == node, + node->data); +} + +/* + * Update the tree's augmentation stored in the data payloads. The caller + * sets the left or right aug in the parent to match the node. + */ +static void update_data_aug(struct scoutfs_treap *treap, + struct treap_node *node) +{ + struct treap_node *parent; + + while (node && (parent = node->parent)) { + if (!ops_update_aug(treap, parent, node)) + break; + node = node->parent; + } +} + +/* + * G G + * | | + * P N + * / -> \ + * N P + * \ / + * + * parent->left = node->right; + * node->right = parent; + * grand->(left|right) = node + * + * The rotation has the following effect on augmentation: + * - parent ref's aug bits have the same population, no change + * - node left's unchanged + * - parent right's unchanged + * - parent's left just set to the node's right + * - node right's recalculated based on parent + */ +static void rotate_right(struct scoutfs_treap *treap, + struct treap_node *parent, struct treap_node *node) +{ + struct treap_ref *grand_ref; + struct treap_node *grand; + + /* get grandparent ref before clobbering parent */ + grand = parent->parent; + if (grand) { + if (grand->left.node == parent) + grand_ref = &grand->left; + else + grand_ref = &grand->right; + } else { + grand_ref = &treap->root_ref; + } + + /* parent rotates down and points to node's child */ + parent->left = node->right; + if (parent->left.node) + parent->left.node->parent = parent; + + /* node rotates up and points to parent */ + node->right.node = parent; + node->right.off = parent->off; + node->right.gen = parent->gen; + node->right.aug_bits = node_aug_bits(treap, parent); + parent->parent = node; + + /* grand parent points to node */ + grand_ref->node = node; + grand_ref->off = node->off; + grand_ref->gen = node->gen; + grand_ref->aug_bits = node_aug_bits(treap, node); + node->parent = grand; + + ops_update_aug(treap, node, parent); +} + +/* see above: swap left/right */ +static void rotate_left(struct scoutfs_treap *treap, + struct treap_node *parent, struct treap_node *node) +{ + struct treap_ref *grand_ref; + struct treap_node *grand; + + grand = parent->parent; + if (grand) { + if (grand->right.node == parent) + grand_ref = &grand->right; + else + grand_ref = &grand->left; + } else { + grand_ref = &treap->root_ref; + } + + parent->right = node->left; + if (parent->right.node) + parent->right.node->parent = parent; + + node->left.node = parent; + node->left.off = parent->off; + node->left.gen = parent->gen; + node->left.aug_bits = node_aug_bits(treap, parent); + parent->parent = node; + + grand_ref->node = node; + grand_ref->off = node->off; + grand_ref->gen = node->gen; + grand_ref->aug_bits = node_aug_bits(treap, node); + node->parent = grand; + + ops_update_aug(treap, node, parent); +} + +/* + * Rebalance the tree by rotating the parent and child as long as the + * child has a higher random priority. + */ +static void rebalance(struct scoutfs_treap *treap, struct treap_node *node) +{ + struct treap_node *parent; + + while (node && (parent = node->parent) && node->prio > parent->prio) { + if (parent->left.node == node) + rotate_right(treap, parent, node); + else + rotate_left(treap, parent, node); + } +} + + +/* + * The caller has mucked with a node. We make sure all of our internal + * augmentation, the op data's augmentation, and the treap prio balance + * is repaired. + */ +static void repair(struct scoutfs_treap *treap, struct treap_node *node) +{ + update_internal_aug(treap, node); + update_data_aug(treap, node); + rebalance(treap, node); +} + +static struct treap_node *alloc_node(u16 bytes) +{ + struct treap_node *node; + + node = kmalloc(offsetof(struct treap_node, data[bytes]), GFP_NOFS); + if (node) + memset(node, 0, offsetof(struct treap_node, data)); + + return node; +} + +/* + * bytes in the persistent ring taken up by a node with the given number + * of data bytes. + */ +static unsigned node_ring_bytes(struct treap_node *node) +{ + return offsetof(struct scoutfs_treap_node, data[node->bytes]); +} + +static bool dirty_node(struct scoutfs_treap *treap, struct treap_node *node) +{ + DECLARE_TREAP_INFO(treap->sb, tinf); + + return node->off == tinf->dirty_off; +} + +/* + * Ensure that the given node is dirty. If it isn't we need to mark it + * dirty and augment the tree. Transaction limits and preallocation + * make sure that we always have resources to write nodes that are + * dirtied. + * + * When we dirty old nodes we temporarily set their offset to the + * current half of the ring so that they won't show up in augmented + * searches for old nodes. + */ +static bool mark_node_dirty(struct scoutfs_treap *treap, struct treap_ref *ref, + struct treap_node *node) +{ + DECLARE_TREAP_INFO(treap->sb, tinf); + + if (dirty_node(treap, node)) + return false; + + treap->dirty_bytes += node_ring_bytes(node); + + node->off = tinf->dirty_off; + node->gen = tinf->dirty_gen; + ref->off = node->off; + ref->gen = node->gen; + repair(treap, node); + + return true; +} + +static int dirty_old_nodes(struct scoutfs_treap *treap, unsigned old_target, + unsigned dirty_limit); + +static struct scoutfs_treap_node *read_ring_node(struct scoutfs_treap *treap, + u64 off) +{ + struct address_space *mapping = treap->sb->s_bdev->bd_inode->i_mapping; + struct scoutfs_treap_node *tnode = NULL; + struct page *page = NULL; + unsigned pg_off; + unsigned bytes; + pgoff_t pg_ind; + int ret; + + off += le64_to_cpu(treap->super->ring_blkno) << SCOUTFS_BLOCK_SHIFT; + pg_ind = off >> PAGE_CACHE_SHIFT; + pg_off = off & ~PAGE_CACHE_MASK; + + if (pg_off + sizeof(struct scoutfs_treap_node) > PAGE_CACHE_SIZE) { + ret = -EIO; + goto out; + } + +retry: + page = find_or_create_page(mapping, pg_ind, GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + + tnode = page_address(page) + pg_off; + + if (PageUptodate(page)) { + unlock_page(page); + ret = 0; + goto out; + } + + ClearPageError(page); + ret = mapping->a_ops->readpage(NULL, page); + if (ret) { + if (ret == AOP_TRUNCATED_PAGE) { + page_cache_release(page); + goto retry; + } + goto out; + } + + wait_on_page_locked(page); + if (!PageUptodate(page)) { + if (page->mapping != mapping) { + page_cache_release(page); + goto retry; + } + ret = -EIO; + goto out; + } else { + ret = 0; + } + + bytes = le16_to_cpu(tnode->bytes); + + if (pg_off + offsetof(struct scoutfs_treap_node, data[bytes]) > + PAGE_CACHE_SIZE) { + ret = -EIO; + } + +out: + if (ret) { + if (page) + page_cache_release(page); + return ERR_PTR(ret); + } + + return tnode; +} + +static void release_ring_node(struct scoutfs_treap_node *tnode) +{ + if (!IS_ERR_OR_NULL(tnode)) + page_cache_release(virt_to_page(tnode)); +} + +/* + * We write to ring blocks from preallocated private pages with bios but read + * through the bdev page cache. Invalidate the blocks we're about to write + * so we'll read them later. + */ +static void invalidate_blocks(struct super_block *sb, u64 blkno, u64 nr) +{ + struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; + loff_t lstart = blkno << SCOUTFS_BLOCK_SHIFT; + loff_t lend = lstart + (nr << SCOUTFS_BLOCK_SHIFT) - 1; + + truncate_inode_pages_range(mapping, lstart, lend); +} + +static void invalidate_ring_block(struct scoutfs_treap *treap, u64 off) +{ + invalidate_blocks(treap->sb, le64_to_cpu(treap->super->ring_blkno) + + (off >> SCOUTFS_BLOCK_SHIFT), 1); +} + +static __le32 tnode_crc(struct scoutfs_treap_node *tnode) +{ + u16 bytes = le16_to_cpu(tnode->bytes); + unsigned skip = sizeof(tnode->crc); + + return cpu_to_le32(crc32c(~0, (void *)tnode + skip, + offsetof(struct scoutfs_treap_node, + data[bytes]) - skip)); +} + +/* + * Give the caller the node pointed to by their reference. If the node + * isn't already in the tree then we link it in and update augmentation. + * + * XXX what's the consequence of failing to also dirty old ring nodes? + * The ring gets out of balance but we do nothing about it. + */ +static struct treap_node *read_node(struct scoutfs_treap *treap, + struct treap_node *parent, + struct treap_ref *ref, bool dirty) +{ + struct scoutfs_treap_node *tnode = NULL; + struct treap_node *node = NULL; + unsigned retries = 3; + u16 bytes; + int ret; + + if (ref->node) { + node = ref->node; + ret = 0; + goto out; + } + +retry: + tnode = read_ring_node(treap, ref->off); + if (IS_ERR(tnode)) { + ret = PTR_ERR(tnode); + goto out; + } + + if (tnode->crc != tnode_crc(tnode) || + le64_to_cpu(tnode->off) != ref->off || + le64_to_cpu(tnode->gen) != ref->gen) { + invalidate_ring_block(treap, ref->off); + if (retries--) { + /* XXX restart search, not just this read */ + release_ring_node(tnode); + goto retry; + } else { + ret = -EIO; + goto out; + } + } + + bytes = le16_to_cpu(tnode->bytes); + + node = alloc_node(bytes); + if (!node) { + ret = -ENOMEM; + goto out; + } + + node->off = le64_to_cpu(tnode->off); + node->gen = le64_to_cpu(tnode->gen); + node->prio = le64_to_cpu(tnode->prio); + node->left.off = le64_to_cpu(tnode->left.off); + node->left.gen = le64_to_cpu(tnode->left.gen); + node->left.aug_bits = tnode->left.aug_bits; + node->right.off = le64_to_cpu(tnode->right.off); + node->right.gen = le64_to_cpu(tnode->right.gen); + node->right.aug_bits = tnode->right.aug_bits; + node->bytes = bytes; + memcpy(node->data, tnode->data, bytes); + + node->parent = parent; + parent_ref(treap, node)->node = node; + ret = 0; +out: + release_ring_node(tnode); + if (!ret && dirty && mark_node_dirty(treap, ref, node)) + ret = dirty_old_nodes(treap, node_ring_bytes(node), 0); + if (ret) + return ERR_PTR(ret); + + return node; +} + +/* + * Find nodes in the older half of the ring and mark them dirty. Stop + * when we don't have any more older nodes, after dirtying enough old + * nodes, or before dirtying too many nodes. + */ +static int dirty_old_nodes(struct scoutfs_treap *treap, unsigned old_target, + unsigned dirty_limit) +{ + u8 bit = old_aug_bit(treap); + struct treap_node *parent; + struct treap_node *node; + struct treap_ref *ref; + unsigned dirty = 0; + unsigned old = 0; + unsigned bytes; + int ret = 0; + +restart: + parent = NULL; + ref = &treap->root_ref; + + while (ref->aug_bits & bit) { + node = read_node(treap, parent, ref, false); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + break; + } + + bytes = node_ring_bytes(node); + + if (!dirty_node(treap, node) && dirty_limit) { + dirty += bytes; + if (dirty > dirty_limit) + break; + } + + if (old_target && off_aug_bit(treap, node->off) == bit) + old += bytes; + + /* sets dirty, sets current half aug bit, repairs */ + mark_node_dirty(treap, ref, node); + + if (old_target && old >= old_target) + break; + + if (node->left.aug_bits & bit) + ref = &node->left; + else if (node->right.aug_bits & bit) + ref = &node->right; + else + goto restart; + } + + return ret; +} + +/* + * Return the dirty node identified by the given key, creating it if it + * doesn't exist. + * + * Returns ERR -EEXIST if a node already exists at the given key. + */ +void *scoutfs_treap_insert(struct scoutfs_treap *treap, void *key, u16 bytes, + void *fill_arg) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + int cmp; + + while (ref->gen) { + node = read_node(treap, parent, ref, true); + if (IS_ERR(node)) + goto out; + + cmp = treap->ops->compare(key, node->data); + if (cmp < 0) { + ref = &node->left; + } else if (cmp > 0) { + ref = &node->right; + } else { + node = ERR_PTR(-EEXIST); + goto out; + } + + parent = node; + node = NULL; + } + + node = alloc_node(bytes); + if (!node) { + node = ERR_PTR(-ENOMEM); + goto out; + } + + node->parent = parent; + node->bytes = bytes; + get_random_bytes_arch(&node->prio, sizeof(node->prio)); + + ref->node = node; + + /* filling here instead of in caller for aug update in repair */ + treap->ops->fill(node->data, fill_arg); + + /* sets off and gen and repairs */ + mark_node_dirty(treap, ref, node); +out: + if (IS_ERR(node)) + return ERR_CAST(node); + + return node->data; +} + +/* + * Deletion is easy if the node to delete doesn't have both children. + * We just point its parent at its child, if it has one. If it has both + * children we relocate it in the tree so that it doesn't. We find its + * next least successor which by definition won't have a left child. We + * can swap them and maintain key ordering in the tree. But we have to + * repair balance and augmentation. + */ +int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_ref *child_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + struct treap_node *child; + int cmp; + int ret; + + /* find node to delete */ + while (ref->gen) { + node = read_node(treap, parent, ref, true); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto out; + } + + cmp = treap->ops->compare(key, node->data); + if (cmp < 0) + ref = &node->left; + else if (cmp > 0) + ref = &node->right; + else + break; + + parent = node; + node = NULL; + } + + if (!node) { + ret = -ENOENT; + goto out; + } + + /* if it has both children find next successor and swap */ + if (node->left.gen && node->right.gen) { + child_ref = &node->right; + child = read_node(treap, node, child_ref, true); + if (IS_ERR(child)) { + ret = PTR_ERR(child); + goto out; + } + + while (child->left.gen) { + child_ref = &child->left; + child = read_node(treap, child, child_ref, true); + if (IS_ERR(child)) { + ret = PTR_ERR(child); + goto out; + } + } + + /* + * ref points to node, child_ref points to the child. + * We swap the node and child's position in the tree by + * updating refs in and out of the nodes. + * + * Repair after deletion catches the aug and prio + * inconsistencies from moving the child up the tree and + * removing the node. + */ + + swap(*ref, *child_ref); + swap(node->parent, child->parent); + swap(node->left, child->left); + swap(node->right, child->right); + if (node->left.node) + node->left.node->parent = node; + if (node->right.node) + node->right.node->parent = node; + if (child->left.node) + child->left.node->parent = child; + if (child->right.node) + child->right.node->parent = child; + + } + + /* delete the node, might have to point parent at child */ + if (node->left.gen) + child_ref = &node->left; + else + child_ref = &node->right; + + *ref = *child_ref; + if (ref->node) + ref->node->parent = parent; + + if (dirty_node(treap, node)) + treap->dirty_bytes -= node_ring_bytes(node); + + kfree(node); + + repair(treap, parent); + ret = 0; +out: + return ret; +} + +static void *treap_lookup(struct scoutfs_treap *treap, void *key, bool dirty) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + int cmp; + + while (ref->gen) { + node = read_node(treap, parent, ref, dirty); + if (IS_ERR(node)) + break; + + cmp = treap->ops->compare(key, node->data); + if (cmp < 0) + ref = &node->left; + else if (cmp > 0) + ref = &node->right; + else + break; + + parent = node; + node = NULL; + } + + if (IS_ERR(node)) + return ERR_CAST(node); + if (node) + return node->data; + return NULL; +} + +void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, false); +} + +void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, true); +} + +void *scoutfs_treap_first(struct scoutfs_treap *treap) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + + while (ref->gen) { + node = read_node(treap, parent, ref, false); + if (IS_ERR(node)) + break; + + ref = &node->left; + parent = node; + } + + if (IS_ERR(node)) + return ERR_CAST(node); + if (node) + return node->data; + return NULL; +} + +void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data) +{ + struct treap_node *node = container_of(data, struct treap_node, data); + struct treap_node *parent; + + if (node->right.gen) { + node = read_node(treap, node, &node->right, false); + if (IS_ERR(node)) + goto out; + + while (node->left.gen) { + node = read_node(treap, node, &node->left, false); + if (IS_ERR(node)) + goto out; + } + + goto out; + } + + while (((parent = node->parent)) && node == parent->left.node) + node = parent; + node = parent; + +out: + if (IS_ERR(node)) + return ERR_CAST(node); + if (node) + return node->data; + return NULL; +} + +static void *lookup_next(struct scoutfs_treap *treap, void *key, bool dirty) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + struct treap_node *next = NULL; + int cmp; + + while (ref->off) { + node = read_node(treap, parent, ref, dirty); + if (IS_ERR(node)) + break; + + cmp = treap->ops->compare(key, node->data); + if (cmp < 0) { + ref = &node->left; + next = node; + } else if (cmp > 0) { + ref = &node->right; + } else { + next = node; + break; + } + + parent = node; + node = NULL; + } + + if (IS_ERR(node)) + return ERR_CAST(node); + if (next) + return next->data; + return NULL; +} + +void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key) +{ + return lookup_next(treap, key, false); +} + +void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key) +{ + return lookup_next(treap, key, true); +} + +int scoutfs_treap_has_dirty(struct scoutfs_treap *treap) +{ + return !!(treap->root_ref.aug_bits & SCOUTFS_TREAP_AUG_DIRTY); +} + +static void *pages_off_ptr(struct treap_info *tinf) +{ + return page_address(tinf->pages[tinf->pages_off >> PAGE_SHIFT]) + + (tinf->pages_off % ~PAGE_MASK); +} + +/* + * The dirty offset is carefully chosen so that it will consider dirty + * nodes part of the current half of the ring but is an offset that will + * never be actually written. That way it is overwritten as dirty nodes + * are copied to the ring and get their final offset and aren't considered + * dirty. Nodes never span blocks so we set the dirty offset to the final + * byte of the next block in the ring. + */ +static void init_writer(struct treap_info *tinf, + struct scoutfs_super_block *super) +{ + tinf->ring_off = le64_to_cpu(super->ring_tail_block) << + SCOUTFS_BLOCK_SHIFT; + tinf->pages_off = 0; + tinf->block_space = 0; + tinf->nr_blocks = 0; + + tinf->dirty_gen = le64_to_cpu(super->ring_gen) + 1; + tinf->dirty_off = tinf->ring_off + SCOUTFS_BLOCK_MASK; +} + +static void try_zero_block_tail(struct treap_info *tinf) +{ + if (tinf->block_space != SCOUTFS_BLOCK_SIZE) + memset(pages_off_ptr(tinf), 0, tinf->block_space); +} + +/* + * Copy the node to the page at the next free tail offset. The + * in-memory node's offset is set to its final ring offset and its + * parent ref is updated. Thus it will no longer have the magic dirty + * offset and won't be considered dirty by the tree augmentation. + */ +static void copy_node_to_ring(struct scoutfs_treap *treap, + struct treap_node *node) +{ + DECLARE_TREAP_INFO(treap->sb, tinf); + struct scoutfs_treap_node *tnode; + u32 bytes = node_ring_bytes(node); + u32 skip; + + if (tinf->block_space < bytes) { + try_zero_block_tail(tinf); + + skip = ALIGN(tinf->ring_off, SCOUTFS_BLOCK_SIZE) - + tinf->ring_off; + tinf->ring_off += skip; + tinf->pages_off += skip; + + tinf->block_space = SCOUTFS_BLOCK_SIZE; + tinf->nr_blocks++; + + /* see if we're wrapping */ + if (tinf->ring_off == tinf->last_ring_off) + tinf->ring_off = 0; + } + + node->off = tinf->ring_off; + parent_ref(treap, node)->off = node->off; + + tnode = pages_off_ptr(tinf); + tinf->ring_off += bytes; + tinf->pages_off += bytes; + tinf->block_space -= bytes; + + tnode->off = cpu_to_le64(node->off); + tnode->gen = cpu_to_le64(node->gen); + tnode->prio = cpu_to_le64(node->prio); + tnode->left.off = cpu_to_le64(node->left.off); + tnode->left.gen = cpu_to_le64(node->left.gen); + tnode->left.aug_bits = node->left.aug_bits; + tnode->right.off = cpu_to_le64(node->right.off); + tnode->right.gen = cpu_to_le64(node->right.gen); + tnode->right.aug_bits = node->right.aug_bits; + tnode->bytes = cpu_to_le16(node->bytes); + memcpy(tnode->data, node->data, node->bytes); + + tnode->crc = tnode_crc(tnode); +} + +/* + * Copy the currently dirty nodes into preallocated pages for writing. + * + * We can consider the nodes clean as we copy them to the pages. The + * caller is responsible for ensuring forward progress or aborting. + * + * As nodes are copied to the pages they are assigned their final offset + * in the ring. We have to update their parent refs with the new + * offset. (We also could have them cross a half ring, getting new off + * aug bits that bubble up). + * + * All that means that we copy from the leaves up to the root so that we + * capture the modifications to parents as we copy children. + * + * This is called for multiple treaps before the ring is written. + */ +int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap) +{ + struct treap_node *node; + unsigned bytes; + int ret; + + /* first fill final partial block with old nodes */ + bytes = SCOUTFS_BLOCK_SIZE - (treap->dirty_bytes & SCOUTFS_BLOCK_MASK); + if (bytes != SCOUTFS_BLOCK_SIZE) { + ret = dirty_old_nodes(treap, 0, bytes); + if (ret) + goto out; + } + + node = treap->root_ref.node; + while (node) { + /* follow dirty links first */ + if (node->left.aug_bits & SCOUTFS_TREAP_AUG_DIRTY) { + node = node->left.node; + } else if (node->right.aug_bits & SCOUTFS_TREAP_AUG_DIRTY) { + node = node->right.node; + } else { + /* node doesn't have dirty children, append if dirty */ + if (dirty_node(treap, node)) { + copy_node_to_ring(treap, node); + repair(treap, node); + } + + /* ascend back up through parents */ + node = node->parent; + } + } + + treap->dirty_bytes = 0; + ret = 0; +out: + return ret; +} + +/* + * Submit writes for all the dirty nodes that have been copied into the + * preallocated pages. + * entries were appended. The dirty ring blocks are contiguous in the + * page array but can wrap in the block ring on disk. + * + * If it wraps then we submit the earlier fragment at the head of the + * ring first. + * + * The wrapped fragment starts at some block offset in the page array. + * The hacky page array math only works when our fixed 4k block size == + * page_size. To fix it we'd add a offset block to the bio submit loop + * which could add an initial partial page vec to the bios. + * + * XXX figure out where to write. I guess we have a write ring block + * in the super? + */ +int scoutfs_treap_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_TREAP_INFO(sb, tinf); + u64 head_blocks; + u64 tail_blocks; + u64 blkno; + u64 tail; + + if (!tinf->nr_blocks) + return 0; + + try_zero_block_tail(tinf); + + tail = le64_to_cpu(super->ring_tail_block); + tail_blocks = min_t(u64, tinf->nr_blocks, + le64_to_cpu(super->ring_blocks) - tail); + + head_blocks = tinf->nr_blocks - tail_blocks; + + if (head_blocks) { + BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); + invalidate_blocks(sb, le64_to_cpu(super->ring_blkno), + head_blocks); + scoutfs_bio_submit_comp(sb, WRITE, tinf->pages + tail_blocks, + le64_to_cpu(super->ring_blkno), + head_blocks, comp); + } + + blkno = le64_to_cpu(super->ring_blkno) + tail; + invalidate_blocks(sb, blkno, tail_blocks); + scoutfs_bio_submit_comp(sb, WRITE, tinf->pages, blkno, tail_blocks, + comp); + + /* record new tail index in super and reset for next trans */ + super->ring_tail_block = cpu_to_le64(tail + tail_blocks); + if (super->ring_tail_block == super->ring_blocks) + super->ring_tail_block = cpu_to_le64(head_blocks); + + super->ring_gen = cpu_to_le64(tinf->dirty_gen); + + init_writer(tinf, super); + + return 0; +} + +struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, + struct scoutfs_treap_ops *ops, + struct scoutfs_treap_root *root) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_treap *treap; + + treap = kzalloc(sizeof(struct scoutfs_treap), GFP_NOFS); + if (treap) { + treap->sb = sb; + treap->super = &sbi->super; + treap->ops = ops; + treap->root_ref.off = le64_to_cpu(root->ref.off); + treap->root_ref.gen = le64_to_cpu(root->ref.gen); + treap->root_ref.aug_bits = root->ref.aug_bits; + } + + return treap; +} + +void scoutfs_treap_update_root(struct scoutfs_treap_root *root, + struct scoutfs_treap *treap) +{ + root->ref.off = cpu_to_le64(treap->root_ref.off); + root->ref.gen = cpu_to_le64(treap->root_ref.gen); + root->ref.aug_bits = treap->root_ref.aug_bits; +} + +/* + * Free all the allocated nodes in the treap and clear the root. + */ +void scoutfs_treap_free(struct scoutfs_treap *treap) +{ + struct treap_node *node = treap->root_ref.node; + struct treap_node *fre; + + while (node) { + if (node->left.node) { + node = node->left.node; + node->parent->left.node = NULL; + } if (node->right.node) { + node = node->right.node; + node->parent->right.node = NULL; + } else { + fre = node; + node = node->parent; + kfree(fre); + } + } + + kfree(treap); +} + +int scoutfs_treap_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct treap_info *tinf; + struct page *page; + int i; + + BUILD_BUG_ON(offsetof(struct treap_node, data) & (sizeof(long) - 1)); + + tinf = kzalloc(sizeof(struct treap_info), GFP_KERNEL); + if (!tinf) + return -ENOMEM; + + tinf->last_ring_off = le64_to_cpu(super->ring_blocks) << + SCOUTFS_BLOCK_SHIFT; + init_writer(tinf, super); + + for (i = 0; i < ARRAY_SIZE(tinf->pages); i++) { + page = alloc_page(GFP_KERNEL); + if (!page) { + while (--i >= 0) + __free_page(tinf->pages[i]); + kfree(tinf); + return -ENOMEM; + } + + tinf->pages[i] = page; + } + + sbi->treap_info = tinf; + + return 0; +} + +void scoutfs_treap_destroy(struct super_block *sb) +{ + DECLARE_TREAP_INFO(sb, tinf); + int i; + + if (tinf) { + for (i = 0; i < ARRAY_SIZE(tinf->pages); i++) + __free_page(tinf->pages[i]); + + kfree(tinf); + } +} diff --git a/kmod/src/treap.h b/kmod/src/treap.h new file mode 100644 index 00000000..0265611e --- /dev/null +++ b/kmod/src/treap.h @@ -0,0 +1,44 @@ +#ifndef _SCOUTFS_TREAP_H_ +#define _SCOUTFS_TREAP_H_ + +struct scoutfs_bio_completion; + +/* + * The runtime root that's used by operations. It's loaded and stored + * from the persistent root in the super block as transactions are written. + */ +struct scoutfs_treap; + +struct scoutfs_treap_ops { + int (*compare)(void *key, void *data); + void (*fill)(void *data, void *fill_arg); + bool (*update_aug)(void *parent_data, bool left, void *node_data); +}; + +struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, + struct scoutfs_treap_ops *ops, + struct scoutfs_treap_root *root); +void scoutfs_treap_update_root(struct scoutfs_treap_root *root, + struct scoutfs_treap *treap); +void scoutfs_treap_free(struct scoutfs_treap *treap); + +void *scoutfs_treap_insert(struct scoutfs_treap *treap, void *key, u16 bytes, + void *fill_arg); +int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key); + +void *scoutfs_treap_first(struct scoutfs_treap *treap); +void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data); + +int scoutfs_treap_has_dirty(struct scoutfs_treap *treap); +int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap); +int scoutfs_treap_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp); + +int scoutfs_treap_setup(struct super_block *sb); +void scoutfs_treap_destroy(struct super_block *sb); + +#endif From 3333d89f82b0ae54a9738dc2a9a73886afa8cd99 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 Jan 2017 09:03:19 -0800 Subject: [PATCH 181/920] Assign next seg seq from super We hadn't yet assigned real sequence numbers to the segments. Let's track the next sequence in the super block and assign it to segments as we write the first new item in each. Signed-off-by: Zach Brown --- kmod/src/format.h | 3 ++- kmod/src/seg.c | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index c611ea32..6495d457 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -151,7 +151,7 @@ struct scoutfs_segment_block { __le32 crc; __le32 _padding; __le64 segno; - __le64 max_seq; + __le64 seq; __le32 nr_items; __le32 _moar_pads; struct scoutfs_segment_item items[0]; @@ -342,6 +342,7 @@ struct scoutfs_super_block { __le64 ring_blocks; __le64 ring_tail_block; __le64 ring_gen; + __le64 next_seg_seq; __le64 buddy_blocks; struct scoutfs_buddy_root buddy_root; struct scoutfs_btree_root btree_root; diff --git a/kmod/src/seg.c b/kmod/src/seg.c index c16c2e2d..520ed9e3 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -466,6 +466,8 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, struct kvec *key, struct kvec *val, unsigned int nr_items, unsigned int key_bytes) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; SCOUTFS_DECLARE_KVEC(item_key); @@ -475,7 +477,8 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, /* XXX the segment block header is a mess, be better */ sblk->segno = cpu_to_le64(seg->segno); - sblk->max_seq = cpu_to_le64(1); + sblk->seq = super->next_seg_seq; + le64_add_cpu(&super->next_seg_seq, 1); key_off = pos_off(seg, nr_items); val_off = key_off + key_bytes; @@ -540,7 +543,7 @@ int scoutfs_seg_manifest_add(struct super_block *sb, kvec_from_pages(seg, last, item.key_off, item.key_len); return scoutfs_manifest_add(sb, first, last, le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->max_seq), level); + le64_to_cpu(sblk->seq), level); } int scoutfs_seg_setup(struct super_block *sb) From be497f3fcf1b1f80c1cb3946aab8087ea800e37c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 Jan 2017 10:33:02 -0800 Subject: [PATCH 182/920] Make sure to bubble the node aug bits up the treap We forgot to or in a node's children's augmentation bits when setting the augmentation bits up in the parent's ref. This stopped ring dirtying from finding all the dirty nodes in the treap. Signed-off-by: Zach Brown --- kmod/src/treap.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 81cfe343..3e241298 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -168,17 +168,19 @@ static u8 old_aug_bit(struct scoutfs_treap *treap) return off_aug_bit(treap, tinf->dirty_off) ^ SCOUTFS_TREAP_AUG_HALVES; } - -/* Return the aug bits that'll be used to refer to the given node. */ +/* + * Return the aug bits that'll be used to refer to the given node. + * We calculate the bits for the node itself and then or those with the + * bits in its references to its children. + */ static u8 node_aug_bits(struct scoutfs_treap *treap, struct treap_node *node) { DECLARE_TREAP_INFO(treap->sb, tinf); - u8 aug_bits = 0; - if (node->off == tinf->dirty_off) - aug_bits |= SCOUTFS_TREAP_AUG_DIRTY; - - return aug_bits | off_aug_bit(treap, node->off); + return (node->off == tinf->dirty_off ? SCOUTFS_TREAP_AUG_DIRTY : 0) | + off_aug_bit(treap, node->off) | + node->left.aug_bits | + node->right.aug_bits; } /* From 157b9294fa47258fa6f18fd223ed366cc204f54b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 Jan 2017 11:59:37 -0800 Subject: [PATCH 183/920] Be sure not to overfill a segment with items The segment writing loop was assuming that the currently dirty items will fit in a segment. That's not true. Signed-off-by: Zach Brown --- kmod/src/item.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 86a7e1a5..69204518 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -977,6 +977,17 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, * * The caller is responsible for the consistency of the dirty items once * they're in its seg. We can consider them clean once we store them. + * + * Today entering a transaction doesn't ensure that there's never more + * than a segment's worth of dirty items. As we release a trans we kick + * off an async sync. By the time we get here we can have a lot more + * than a segments worth of dirty items. + * + * XXX This is unacceptable because multiple segment writes are not + * atomic. We can have the items that make up an atomic change span + * segments and can be partially visible if we only write the first + * segment. We probably want to throttle trans enters once we have as + * many dirty items as our atomic segment updates can write. */ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) { @@ -987,16 +998,18 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) u32 nr_items; count_seg_items(cac, &nr_items, &key_bytes); - if (nr_items) { - item = first_dirty(cac->items.rb_node); + + item = first_dirty(cac->items.rb_node); + if (item) { scoutfs_seg_first_item(sb, seg, item->key, item->val, nr_items, key_bytes); clear_item_dirty(cac, item); + nr_items--; + } - while ((item = next_dirty(item))) { - scoutfs_seg_append_item(sb, seg, item->key, item->val); - clear_item_dirty(cac, item); - } + while (nr_items-- && (item = next_dirty(item))) { + scoutfs_seg_append_item(sb, seg, item->key, item->val); + clear_item_dirty(cac, item); } return 0; From a15d37783ef8c3deba7b6e15ab23658d4ff091d0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 3 Jan 2017 15:56:53 -0800 Subject: [PATCH 184/920] Set read node parent through ref The code was using parent_ref() to set the parent ref's node pointer. But parent_ref() uses the parent's left node pointer to determine which ref points to the node. If we were setting the left it would return the right because the left isn't set yet. This messed up the tree shape and all hell broke loose. Just set it through the ref, we have it anyway. Signed-off-by: Zach Brown --- kmod/src/treap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 3e241298..d92511de 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -590,7 +590,7 @@ retry: memcpy(node->data, tnode->data, bytes); node->parent = parent; - parent_ref(treap, node)->node = node; + ref->node = node; ret = 0; out: release_ring_node(tnode); From eb94092f2f4b2a216927129ca4178c389e81fdc1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:13:20 -0800 Subject: [PATCH 185/920] Add kvec big endian inc and dec Add helpers that increment or decrement kvec vectors as theough they're big endian values. Signed-off-by: Zach Brown --- kmod/src/kvec.c | 30 ++++++++++++++++++++++++++++++ kmod/src/kvec.h | 2 ++ 2 files changed, 32 insertions(+) diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 349e55b6..5e49c6a3 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -258,6 +258,36 @@ void scoutfs_kvec_set_max_key(struct kvec *kvec) scoutfs_kvec_init(kvec, type, 1); } +/* + * Increase the kvec as though it is a big endian value. Carry + * increments of the least significant byte as long as it wraps. + */ +void scoutfs_kvec_be_inc(struct kvec *kvec) +{ + int i; + int b; + + for (i = SCOUTFS_KVEC_NR - 1; i >= 0; i--) { + for (b = (int)kvec[i].iov_len - 1; b >= 0; b--) { + if (++((u8 *)kvec[i].iov_base)[b]) + return; + } + } +} + +void scoutfs_kvec_be_dec(struct kvec *kvec) +{ + int i; + int b; + + for (i = SCOUTFS_KVEC_NR - 1; i >= 0; i--) { + for (b = (int)kvec[i].iov_len - 1; b >= 0; b--) { + if (--((u8 *)kvec[i].iov_base)[b] != 0xff) + return; + } + } +} + /* * Clone the source kvec into the dst if the dst is empty or if * the src kvec is less than the dst. diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index 93ddcd36..444a8116 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -68,5 +68,7 @@ void scoutfs_kvec_set_max_key(struct kvec *kvec); void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src); unsigned scoutfs_kvec_key_strlen(struct kvec *key); void scoutfs_kvec_key_sprintf(char *buf, struct kvec *key); +void scoutfs_kvec_be_inc(struct kvec *kvec); +void scoutfs_kvec_be_dec(struct kvec *kvec); #endif From cd6cd000cea7f73e86604be6fdf2ffc8795601e2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:24:51 -0800 Subject: [PATCH 186/920] Add ifdefed out quick treap printer This was pretty handy for debugging weird failure cases. Signed-off-by: Zach Brown --- kmod/src/treap.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index d92511de..85e75add 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -142,6 +142,28 @@ struct treap_node { u8 data[0] __aligned(sizeof(long)); }; +#if 0 +static void print_treap_node(struct treap_ref *ref, u64 loc) +{ + struct treap_node *node = ref->node; + + if (!node) + return; + + printk("loc %llx node %p: off %llu gen %llu prio %016llx bytes %u\n", + loc, node, node->off, node->gen, node->prio, node->bytes); + printk(" left: off %llu gen %llu aug %u node %p\n", + node->left.off, node->left.gen, node->left.aug_bits, + node->left.node); + printk(" right: off %llu gen %llu aug %u node %p\n", + node->right.off, node->right.gen, node->right.aug_bits, + node->right.node); + + print_treap_node(&node->left, (loc << 4) | 1); + print_treap_node(&node->right, (loc << 4) | 2); +} +#endif + static struct treap_ref *parent_ref(struct scoutfs_treap *treap, struct treap_node *node) { From e5e7a25ecd54f1f94e8b119488bfc44a3464da02 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:25:31 -0800 Subject: [PATCH 187/920] Don't use null node when repairing aug We were derefing the null parent when deleting a single node in a tree. There's no need to use parent_ref() here, we know that there's no node and we can just clear the root's aug bits. Signed-off-by: Zach Brown --- kmod/src/treap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 85e75add..87ebc4f1 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -217,7 +217,7 @@ static void update_internal_aug(struct scoutfs_treap *treap, u8 bits; if (!node) - parent_ref(treap, node)->aug_bits = 0; + treap->root_ref.aug_bits = 0; while (node) { bits = node_aug_bits(treap, node); From 80da7fefa75876e7b5155122be20ae35c24861fa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:28:11 -0800 Subject: [PATCH 188/920] fix treap deletion Treap deletion was pretty messed up. It forgot to reset parent and ref for the swapped node before using them to finally delete. And it didn't get all the weird cases right where the child node to swap is the direct child of the node. In that case we can't just swap the parent pointers and node pointers, they need to be special cased. So nuts to all that. We'll just rotate the node down until it doesn't have both children. They result in pretty similar patterns and the rotation mechanism is much simpler to understand. Signed-off-by: Zach Brown --- kmod/src/treap.c | 85 ++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 87ebc4f1..59e76e2a 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -738,20 +738,25 @@ out: } /* - * Deletion is easy if the node to delete doesn't have both children. - * We just point its parent at its child, if it has one. If it has both - * children we relocate it in the tree so that it doesn't. We find its - * next least successor which by definition won't have a left child. We - * can swap them and maintain key ordering in the tree. But we have to - * repair balance and augmentation. + * Delete a node with the given key. + * + * It's easy when the node doesn't have two children. We remove the + * node and point it's parent ref at either of the child's refs that + * might have been populated. + * + * Deletion's a little tricker when we have both children. We could + * find an ancestor and swap but that's fiddly to get right with all our + * rich node pointers. Instead we can reuse rotation to rotate the node + * down until it doesn't have both children. */ int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key) { struct treap_ref *ref = &treap->root_ref; - struct treap_ref *child_ref; struct treap_node *parent = NULL; struct treap_node *node = NULL; - struct treap_node *child; + struct treap_ref *child_ref; + struct treap_node *left; + struct treap_node *right; int cmp; int ret; @@ -780,47 +785,43 @@ int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key) goto out; } - /* if it has both children find next successor and swap */ - if (node->left.gen && node->right.gen) { - child_ref = &node->right; - child = read_node(treap, node, child_ref, true); - if (IS_ERR(child)) { - ret = PTR_ERR(child); + /* + * Rotate the node down with its higher priority child until it + * doesn't have both children. Dirtying tries to repair which + * can try to repair priority imbalance with rotation so we swap + * priorities first. Unfortunately we need to read both + * children to get their priorities but we only try to dirty the + * rotation child. It's messy but dirtying both can double + * write amplification. + */ + while (node->left.gen && node->right.gen) { + left = read_node(treap, node, &node->left, false); + right = read_node(treap, node, &node->right, false); + if (IS_ERR(left) || IS_ERR(right)) { + ret = IS_ERR(left) ? PTR_ERR(left) : PTR_ERR(right); goto out; } - while (child->left.gen) { - child_ref = &child->left; - child = read_node(treap, child, child_ref, true); - if (IS_ERR(child)) { - ret = PTR_ERR(child); + if (left->prio > right->prio) { + left = read_node(treap, node, &node->left, true); + if (IS_ERR(left)) { + ret = IS_ERR(left); goto out; } + swap(node->prio, left->prio); + rotate_right(treap, node, left); + } else { + right = read_node(treap, node, &node->right, true); + if (IS_ERR(right)) { + ret = IS_ERR(right); + goto out; + } + swap(node->prio, right->prio); + rotate_left(treap, node, right); } - /* - * ref points to node, child_ref points to the child. - * We swap the node and child's position in the tree by - * updating refs in and out of the nodes. - * - * Repair after deletion catches the aug and prio - * inconsistencies from moving the child up the tree and - * removing the node. - */ - - swap(*ref, *child_ref); - swap(node->parent, child->parent); - swap(node->left, child->left); - swap(node->right, child->right); - if (node->left.node) - node->left.node->parent = node; - if (node->right.node) - node->right.node->parent = node; - if (child->left.node) - child->left.node->parent = child; - if (child->right.node) - child->right.node->parent = child; - + parent = node->parent; + ref = parent_ref(treap, node); } /* delete the node, might have to point parent at child */ From 2522509ec8146240a271d3e617ac02dc8bcc0ba7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:31:55 -0800 Subject: [PATCH 189/920] Fix scoutfs_treap_next() parent walk comparision While walking up parents looking for the next node we were comparing the child with the wrong parent pointer. This is easily verified by glancing at rb_next() :). Signed-off-by: Zach Brown --- kmod/src/treap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 59e76e2a..41bbb40c 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -927,7 +927,7 @@ void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data) goto out; } - while (((parent = node->parent)) && node == parent->left.node) + while (((parent = node->parent)) && node == parent->right.node) node = parent; node = parent; From a45661e5b62fa8761bb268dc81058a3fa4263836 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 14:34:14 -0800 Subject: [PATCH 190/920] Add _prev version of treap lookup and iteration _lookup() and _lookup_next() each had nearly identical loops that took a dirty boolean. We combine them into one walker with flags for dirty and next and add a prev prev as well, giving us all the exported functions with combinations of the flags. We also add _last() to match _first() and _prev() to match _next(). Signed-off-by: Zach Brown --- kmod/src/treap.c | 122 ++++++++++++++++++++++++++++++++--------------- kmod/src/treap.h | 4 ++ 2 files changed, 87 insertions(+), 39 deletions(-) diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 41bbb40c..692b25a2 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -845,30 +845,46 @@ out: return ret; } -static void *treap_lookup(struct scoutfs_treap *treap, void *key, bool dirty) +enum { + LU_DIRTY, + LU_NEXT, + LU_PREV, +}; + +static void *treap_lookup(struct scoutfs_treap *treap, void *key, int flags) { struct treap_ref *ref = &treap->root_ref; struct treap_node *parent = NULL; struct treap_node *node = NULL; + struct treap_node *prev = NULL; + struct treap_node *next = NULL; int cmp; while (ref->gen) { - node = read_node(treap, parent, ref, dirty); + node = read_node(treap, parent, ref, flags & LU_DIRTY); if (IS_ERR(node)) break; cmp = treap->ops->compare(key, node->data); - if (cmp < 0) + if (cmp < 0) { ref = &node->left; - else if (cmp > 0) + next = node; + } else if (cmp > 0) { ref = &node->right; - else + prev = node; + } else { break; + } parent = node; node = NULL; } + if (!node && (flags & LU_PREV) && prev) + node = prev; + else if (!node && (flags & LU_NEXT) && next) + node = next; + if (IS_ERR(node)) return ERR_CAST(node); if (node) @@ -878,12 +894,32 @@ static void *treap_lookup(struct scoutfs_treap *treap, void *key, bool dirty) void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key) { - return treap_lookup(treap, key, false); + return treap_lookup(treap, key, 0); } void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key) { - return treap_lookup(treap, key, true); + return treap_lookup(treap, key, LU_DIRTY); +} + +void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, LU_NEXT); +} + +void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, LU_NEXT | LU_DIRTY); +} + +void *scoutfs_treap_lookup_prev(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, LU_PREV); +} + +void *scoutfs_treap_lookup_prev_dirty(struct scoutfs_treap *treap, void *key) +{ + return treap_lookup(treap, key, LU_PREV | LU_DIRTY); } void *scoutfs_treap_first(struct scoutfs_treap *treap) @@ -908,6 +944,28 @@ void *scoutfs_treap_first(struct scoutfs_treap *treap) return NULL; } +void *scoutfs_treap_last(struct scoutfs_treap *treap) +{ + struct treap_ref *ref = &treap->root_ref; + struct treap_node *parent = NULL; + struct treap_node *node = NULL; + + while (ref->gen) { + node = read_node(treap, parent, ref, false); + if (IS_ERR(node)) + break; + + ref = &node->right; + parent = node; + } + + if (IS_ERR(node)) + return ERR_CAST(node); + if (node) + return node->data; + return NULL; +} + void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data) { struct treap_node *node = container_of(data, struct treap_node, data); @@ -939,51 +997,37 @@ out: return NULL; } -static void *lookup_next(struct scoutfs_treap *treap, void *key, bool dirty) +void *scoutfs_treap_prev(struct scoutfs_treap *treap, void *data) { - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - struct treap_node *next = NULL; - int cmp; + struct treap_node *node = container_of(data, struct treap_node, data); + struct treap_node *parent; - while (ref->off) { - node = read_node(treap, parent, ref, dirty); + if (node->left.gen) { + node = read_node(treap, node, &node->left, false); if (IS_ERR(node)) - break; + goto out; - cmp = treap->ops->compare(key, node->data); - if (cmp < 0) { - ref = &node->left; - next = node; - } else if (cmp > 0) { - ref = &node->right; - } else { - next = node; - break; + while (node->right.gen) { + node = read_node(treap, node, &node->right, false); + if (IS_ERR(node)) + goto out; } - parent = node; - node = NULL; + goto out; } + while (((parent = node->parent)) && node == parent->left.node) + node = parent; + node = parent; + +out: if (IS_ERR(node)) return ERR_CAST(node); - if (next) - return next->data; + if (node) + return node->data; return NULL; } -void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key) -{ - return lookup_next(treap, key, false); -} - -void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key) -{ - return lookup_next(treap, key, true); -} - int scoutfs_treap_has_dirty(struct scoutfs_treap *treap) { return !!(treap->root_ref.aug_bits & SCOUTFS_TREAP_AUG_DIRTY); diff --git a/kmod/src/treap.h b/kmod/src/treap.h index 0265611e..e69fbde1 100644 --- a/kmod/src/treap.h +++ b/kmod/src/treap.h @@ -29,9 +29,13 @@ void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key); void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key); void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key); void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup_prev(struct scoutfs_treap *treap, void *key); +void *scoutfs_treap_lookup_prev_dirty(struct scoutfs_treap *treap, void *key); void *scoutfs_treap_first(struct scoutfs_treap *treap); +void *scoutfs_treap_last(struct scoutfs_treap *treap); void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data); +void *scoutfs_treap_prev(struct scoutfs_treap *treap, void *data); int scoutfs_treap_has_dirty(struct scoutfs_treap *treap); int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap); From 2083793ae00e82312b7e249f0757268db82a673c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Jan 2017 15:06:06 -0800 Subject: [PATCH 191/920] Add first pass at segment compaction This is the first draft of compaction which has the core mechanics. Add segment functions to free a segment's segno and to delete the entry that refers to the given segment. Add manifest functions that lock the manifest and dirty and delete manifest entries. These are used by the compaction thread to atomically modify the manfiest with the result of a compaction. Sort the level 0 entries in the manifest by their sequence. This lets compaction use the first oldest entry and reading can walk them backwards to get them in order and not have to sort. We also more carefully use the sequence field in the manifest search key to differentiate between finding high level entries that overlap and finding specific entries identified by their seq. Add some fields to the per-super compact_info struct which support compaction. We need to know the limit on the number of segments per level and we record keys per level which tell us which segment to use next time that level is compacted. We kick a compaction thread when we add a manifest entry and that brings the level count over the limit. scoutfs_manifest_next_compact() is the first meaty function. The compaction thread uses this to get all the segments involved in a compaction. It does a quick manifest update if the next manifest candidate doesn't overlap with any sgements in the next level. The compaction operation itself is a pretty straight forward read-modify-write operation. It asks the manifest to give it references to the segments it'll need, reads them in, iterates over them to count and copies items in order to output segments, and atomically updates the manifest. Now that the manifest can be dirty without any dirty segments we need to fix the transaction writing function's assumption that everything flows from dirty segments. It also has to now lock and unlock the manifest as it adds the entry for its level 0 segment. Signed-off-by: Zach Brown --- kmod/src/Makefile | 6 +- kmod/src/compact.c | 531 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/compact.h | 12 + kmod/src/format.h | 2 + kmod/src/manifest.c | 357 +++++++++++++++++++++++++---- kmod/src/manifest.h | 9 + kmod/src/seg.c | 23 ++ kmod/src/seg.h | 4 + kmod/src/super.c | 3 + kmod/src/super.h | 2 + kmod/src/trans.c | 45 +++- 11 files changed, 933 insertions(+), 61 deletions(-) create mode 100644 kmod/src/compact.c create mode 100644 kmod/src/compact.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index cb9b29c9..798ba33a 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += alloc.o bio.o block.o btree.o buddy.o counters.o crc.o dir.o \ - filerw.o kvec.o inode.o ioctl.o item.o manifest.o msg.o name.o \ - seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o +scoutfs-y += alloc.o bio.o block.o btree.o buddy.o compact.o counters.o crc.o \ + dir.o filerw.o kvec.o inode.o ioctl.o item.o manifest.o msg.o \ + name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/compact.c b/kmod/src/compact.c new file mode 100644 index 00000000..5933af25 --- /dev/null +++ b/kmod/src/compact.c @@ -0,0 +1,531 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "kvec.h" +#include "seg.h" +#include "bio.h" +#include "cmp.h" +#include "compact.h" +#include "manifest.h" +#include "scoutfs_trace.h" + +/* + * Compaction is what maintains the exponentially increasing number of + * segments in each level of the lsm tree and is what merges duplicate + * and deletion keys. + * + * When the manifest is modified in a way that requires compaction it + * kicks the compaction thread. The compaction thread calls into the + * manifest to find the segments that need to be compaction. + * + * The compaction operation itself always involves a single "upper" + * segment at a given level and a limited number of "lower" segments at + * the next higher level whose key range intersects with the upper + * segment. + * + * Compaction proceeds by iterating over the items in the upper segment + * and items in each of the lower segments in sort order. The items + * from the two input segments are copied into new output segments in + * sorted order. Item space is reclaimed as duplicate or deletion items + * are removed. + * + * Once the compaction is completed the manifest is updated to remove + * the input segments and add the output segments. Here segment space + * is reclaimed when the input items fit in fewer output segments. + * + * XXX today we only know how to skip duplicate individual items. We'll + * need to know how to skip lower based on upper range deletion items + * and to combine incremental update items. + */ + +struct compact_info { + struct super_block *sb; + struct workqueue_struct *workq; + struct work_struct work; +}; + +#define DECLARE_COMPACT_INFO(sb, name) \ + struct compact_info *name = SCOUTFS_SB(sb)->compact_info + +struct compact_seg { + struct list_head entry; + + u64 segno; + u64 seq; + u8 level; + SCOUTFS_DECLARE_KVEC(first); + struct scoutfs_segment *seg; + int pos; + int saved_pos; +}; + +/* + * A compaction request. It's filled up in scoutfs_compact_add() as + * the manifest is wlaked and it finds segments involved in the compaction. + */ +struct compact_cursor { + struct list_head csegs; + + u8 lower_level; + + struct compact_seg *upper; + struct compact_seg *saved_upper; + struct compact_seg *lower; + struct compact_seg *saved_lower; +}; + +static void save_pos(struct compact_cursor *curs) +{ + struct compact_seg *cseg; + + list_for_each_entry(cseg, &curs->csegs, entry) + cseg->saved_pos = cseg->pos; + + curs->saved_upper = curs->upper; + curs->saved_lower = curs->lower; +} + +static void restore_pos(struct compact_cursor *curs) +{ + struct compact_seg *cseg; + + list_for_each_entry(cseg, &curs->csegs, entry) + cseg->pos = cseg->saved_pos; + + curs->upper = curs->saved_upper; + curs->lower = curs->saved_lower; +} + +/* + * There's some common patterns with scoutfs_manifest_read_items().. may + * want some sharing if it's clean. + */ +static int read_segments(struct super_block *sb, struct compact_cursor *curs) +{ + struct scoutfs_segment *seg; + struct compact_seg *cseg; + int ret = 0; + int err; + + list_for_each_entry(cseg, &curs->csegs, entry) { + seg = scoutfs_seg_submit_read(sb, cseg->segno); + if (IS_ERR(seg)) { + ret = PTR_ERR(seg); + break; + } + + cseg->seg = seg; + } + + list_for_each_entry(cseg, &curs->csegs, entry) { + if (!cseg->seg) + break; + + err = scoutfs_seg_wait(sb, cseg->seg); + if (err && !ret) + ret = err; + + /* XXX verify segs */ + } + + return ret; +} + +/* + * This is synchronous for now. We're just ensuring that the segments + * are stable on disk so that the references to them in the dirty manifest + * are safe without having to associate dirty segments and manifest entries. + */ +static int write_segments(struct super_block *sb, struct list_head *results) +{ + struct scoutfs_bio_completion comp; + struct compact_seg *cseg; + int ret = 0; + int err; + + scoutfs_bio_init_comp(&comp); + + list_for_each_entry(cseg, results, entry) { + ret = scoutfs_seg_submit_write(sb, cseg->seg, &comp); + if (ret) + break; + } + + err = scoutfs_bio_wait_comp(sb, &comp); + if (err && !ret) + ret = err; + + return ret; +} + +static struct compact_seg *next_spos(struct compact_cursor *curs, + struct compact_seg *cseg) +{ + if (cseg->entry.next == &curs->csegs) + return NULL; + + return list_next_entry(cseg, entry); +} + +/* + * Point the caller's key and value kvecs at the next item that should + * be copied from the segment's position in the upper and lower + * segments. We use the item that has the lowest key or the upper if + * they're the same. We advance the cursor past the item that is + * returned. + * + * XXX this will get fancier as we get range deletion items and incremental + * update items. + */ +static bool next_item(struct compact_cursor *curs, + struct kvec *item_key, struct kvec *item_val) +{ + struct compact_seg *upper = curs->upper; + struct compact_seg *lower = curs->lower; + SCOUTFS_DECLARE_KVEC(lower_key); + SCOUTFS_DECLARE_KVEC(lower_val); + bool found = false; + int cmp; + int ret; + + if (upper) { + ret = scoutfs_seg_item_kvecs(upper->seg, upper->pos, + item_key, item_val); + if (ret < 0) + upper = NULL; + } + + while (lower) { + ret = scoutfs_seg_item_kvecs(lower->seg, lower->pos, + lower_key, lower_val); + if (ret == 0) + break; + lower = next_spos(curs, lower); + } + + /* we're done if all are empty */ + if (!upper && !lower) { + found = false; + goto out; + } + + /* + * < 0: return upper, advance upper + * == 0: return upper, advance both + * > 0: return lower, advance lower + */ + if (upper && lower) + cmp = scoutfs_kvec_memcmp(item_key, lower_key); + else if (upper) + cmp = -1; + else + cmp = 1; + + if (cmp > 0) { + scoutfs_kvec_clone(item_key, lower_key); + scoutfs_kvec_clone(item_val, lower_val); + } + + if (cmp <= 0) + upper->pos++; + if (cmp >= 0) + lower->pos++; + + found = true; +out: + curs->upper = upper; + curs->lower = lower; + + return found; +} + +/* + * Figure out how many items and bytes of keys we're going to try and + * compact into the next segment. + */ +static void count_items(struct super_block *sb, struct compact_cursor *curs, + u32 *nr_items, u32 *key_bytes) +{ + SCOUTFS_DECLARE_KVEC(item_key); + SCOUTFS_DECLARE_KVEC(item_val); + u32 total; + + *nr_items = 0; + *key_bytes = 0; + total = sizeof(struct scoutfs_segment_block); + + while (next_item(curs, item_key, item_val)) { + + total += sizeof(struct scoutfs_segment_item) + + scoutfs_kvec_length(item_key) + + scoutfs_kvec_length(item_val); + + if (total > SCOUTFS_SEGMENT_SIZE) + break; + + (*nr_items)++; + (*key_bytes) += scoutfs_kvec_length(item_key); + } +} + +static void compact_items(struct super_block *sb, struct compact_cursor *curs, + struct scoutfs_segment *seg, u32 nr_items, + u32 key_bytes) +{ + SCOUTFS_DECLARE_KVEC(item_key); + SCOUTFS_DECLARE_KVEC(item_val); + + next_item(curs, item_key, item_val); + scoutfs_seg_first_item(sb, seg, item_key, item_val, + nr_items, key_bytes); + + while (--nr_items && next_item(curs, item_key, item_val)) + scoutfs_seg_append_item(sb, seg, item_key, item_val); +} + +static int compact_segments(struct super_block *sb, + struct compact_cursor *curs, + struct list_head *results) +{ + struct scoutfs_segment *seg; + struct compact_seg *cseg; + u32 key_bytes; + u32 nr_items; + int ret; + + for (;;) { + + save_pos(curs); + count_items(sb, curs, &nr_items, &key_bytes); + restore_pos(curs); + + if (nr_items == 0) { + ret = 0; + break; + } + + cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); + if (!cseg) { + ret = -ENOMEM; + break; + } + + ret = scoutfs_seg_alloc(sb, &seg); + if (ret) { + kfree(cseg); + break; + } + + cseg->level = curs->lower_level; + cseg->seg = seg; + list_add_tail(&cseg->entry, results); + + compact_items(sb, curs, seg, nr_items, key_bytes); + } + + return ret; +} + +static void free_csegs(struct list_head *list) +{ + struct compact_seg *cseg; + struct compact_seg *tmp; + + list_for_each_entry_safe(cseg, tmp, list, entry) { + list_del_init(&cseg->entry); + scoutfs_seg_put(cseg->seg); + scoutfs_kvec_kfree(cseg->first); + kfree(cseg); + } +} + +int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, + u64 segno, u64 seq, u8 level) +{ + struct compact_cursor *curs = data; + struct compact_seg *cseg; + int ret; + + cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); + if (!cseg) { + ret = -ENOMEM; + goto out; + } + + list_add_tail(&cseg->entry, &curs->csegs); + + ret = scoutfs_kvec_dup_flatten(cseg->first, first); + if (ret) + goto out; + + cseg->segno = segno; + cseg->seq = seq; + cseg->level = level; + + if (!curs->upper) { + curs->upper = cseg; + } else if (!curs->lower) { + curs->lower = cseg; + curs->lower_level = level; + } + + ret = 0; +out: + return ret; +} + +/* + * Atomically update the manifest. We lock down the manifest so no one + * can use it while we're mucking with it. We can always delete dirty + * treap nodes without failure. So we first dirty the deletion nodes + * before modifying anything. Then we add and if any of those fail we + * can delete the dirty previous additions. Then we can delete the + * dirty existing entries without failure. + * + * XXX does locking the manifest prevent commits? I would think so? + */ +static int update_manifest(struct super_block *sb, struct compact_cursor *curs, + struct list_head *results) +{ + struct compact_seg *cseg; + struct compact_seg *until; + int ret = 0; + int err; + + scoutfs_manifest_lock(sb); + + list_for_each_entry(cseg, &curs->csegs, entry) { + ret = scoutfs_manifest_dirty(sb, cseg->first, + cseg->seq, cseg->level); + if (ret) + goto out; + } + + list_for_each_entry(cseg, results, entry) { + ret = scoutfs_seg_manifest_add(sb, cseg->seg, cseg->level); + if (ret) { + until = cseg; + list_for_each_entry(cseg, results, entry) { + if (cseg == until) + break; + err = scoutfs_seg_manifest_del(sb, cseg->seg, + cseg->level); + BUG_ON(err); + } + goto out; + } + } + + list_for_each_entry(cseg, &curs->csegs, entry) { + ret = scoutfs_manifest_del(sb, cseg->first, + cseg->seq, cseg->level); + BUG_ON(ret); + } + +out: + scoutfs_manifest_unlock(sb); + + return ret; +} + +static int free_result_segnos(struct super_block *sb, + struct list_head *results) +{ + struct compact_seg *cseg; + int ret = 0; + int err; + + list_for_each_entry(cseg, results, entry) { + /* XXX failure here would be an inconsistency */ + err = scoutfs_seg_free_segno(sb, cseg->seg); + if (err && !ret) + ret = err; + } + + return ret; +} + +static void scoutfs_compact_func(struct work_struct *work) +{ + struct compact_info *ci = container_of(work, struct compact_info, work); + struct super_block *sb = ci->sb; + struct compact_cursor curs = {{NULL,}}; + LIST_HEAD(results); + int ret; + + INIT_LIST_HEAD(&curs.csegs); + + ret = scoutfs_manifest_next_compact(sb, (void *)&curs) ?: + read_segments(sb, &curs) ?: + compact_segments(sb, &curs, &results) ?: + write_segments(sb, &results) ?: + update_manifest(sb, &curs, &results); + + if (ret) + free_result_segnos(sb, &results); + + free_csegs(&curs.csegs); + free_csegs(&results); + + WARN_ON_ONCE(ret); + trace_printk("ret %d\n", ret); +} + +void scoutfs_compact_kick(struct super_block *sb) +{ + DECLARE_COMPACT_INFO(sb, ci); + + queue_work(ci->workq, &ci->work); +} + +int scoutfs_compact_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct compact_info *ci; + + ci = kzalloc(sizeof(struct compact_info), GFP_KERNEL); + if (!ci) + return -ENOMEM; + + ci->sb = sb; + INIT_WORK(&ci->work, scoutfs_compact_func); + + ci->workq = alloc_workqueue("scoutfs_compact", 0, 1); + if (!ci->workq) { + kfree(ci); + return -ENOMEM; + } + + sbi->compact_info = ci; + + return 0; +} + +/* + * The system should be idle, there should not be any more manifest + * modification which would kick compaction. + */ +void scoutfs_compact_destroy(struct super_block *sb) +{ + DECLARE_COMPACT_INFO(sb, ci); + + if (ci->workq) { + flush_work(&ci->work); + destroy_workqueue(ci->workq); + } +} diff --git a/kmod/src/compact.h b/kmod/src/compact.h new file mode 100644 index 00000000..9e2778e2 --- /dev/null +++ b/kmod/src/compact.h @@ -0,0 +1,12 @@ +#ifndef _SCOUTFS_COMPACT_H_ +#define _SCOUTFS_COMPACT_H_ + +void scoutfs_compact_kick(struct super_block *sb); + +int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, + u64 segno, u64 seq, u8 level); + +int scoutfs_compact_setup(struct super_block *sb); +void scoutfs_compact_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 6495d457..05033817 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -96,6 +96,8 @@ struct scoutfs_treap_root { */ #define SCOUTFS_MANIFEST_MAX_LEVEL 20 +#define SCOUTFS_MANIFEST_FANOUT 10 + struct scoutfs_manifest { struct scoutfs_treap_root root; __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 51ff1b0e..d4a9a58c 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -14,7 +14,6 @@ #include #include #include -#include #include "super.h" #include "format.h" @@ -23,6 +22,7 @@ #include "item.h" #include "treap.h" #include "cmp.h" +#include "compact.h" #include "manifest.h" #include "scoutfs_trace.h" @@ -45,6 +45,11 @@ struct manifest { struct rw_semaphore rwsem; struct scoutfs_treap *treap; u8 nr_levels; + + /* calculated on mount, const thereafter */ + u64 level_limits[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + + SCOUTFS_DECLARE_KVEC(compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]); }; #define DECLARE_MANIFEST(sb, name) \ @@ -79,6 +84,10 @@ struct manifest_fill_args { struct kvec *last; }; +/* + * Seq is only specified for operations that differentiate between + * segments with identical items by their sequence number. + */ struct manifest_search_key { u64 seq; struct kvec *key; @@ -121,6 +130,8 @@ static bool cmp_range_ment(struct kvec *key, struct kvec *end, /* * Insert a new manifest entry in the treap. The treap allocates a new * node for us and we fill it. + * + * This must be called with the manifest lock held. */ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, struct kvec *last, u64 segno, u64 seq, u8 level) @@ -153,22 +164,92 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, skey.level = level; skey.seq = seq; - down_write(&mani->rwsem); - ment = scoutfs_treap_insert(mani->treap, &skey, bytes, &args); if (IS_ERR(ment)) { ret = PTR_ERR(ment); } else { mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); le64_add_cpu(&super->manifest.level_counts[level], 1); + + if (le64_to_cpu(super->manifest.level_counts[level]) > + mani->level_limits[level]) + scoutfs_compact_kick(sb); + ret = 0; } - up_write(&mani->rwsem); + return ret; +} + +/* + * This must be called with the manifest lock held. + */ +int scoutfs_manifest_dirty(struct super_block *sb, struct kvec *first, u64 seq, + u8 level) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_manifest_entry *ment; + struct manifest_search_key skey; + + skey.key = first; + skey.level = level; + skey.seq = seq; + + ment = scoutfs_treap_lookup_dirty(mani->treap, &skey); + if (IS_ERR(ment)) + return PTR_ERR(ment); + if (!ment) + return -ENOENT; + return 0; +} + +/* + * This must be called with the manifest lock held. + */ +int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, + u8 level) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct manifest_search_key skey; + int ret; + + skey.key = first; + skey.level = level; + skey.seq = seq; + + ret = scoutfs_treap_delete(mani->treap, &skey); + if (ret == 0) + le64_add_cpu(&super->manifest.level_counts[level], -1ULL); return ret; } +/* + * XXX This feels pretty gross, but it's a simple way to give compaction + * atomic updates. It'll go away once compactions go to the trouble of + * communicating their atomic results in a message instead of a series + * of function calls. + */ +int scoutfs_manifest_lock(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + + down_write(&mani->rwsem); + + return 0; +} + +int scoutfs_manifest_unlock(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + + up_write(&mani->rwsem); + + return 0; +} + static int alloc_add_ref(struct list_head *list, struct scoutfs_manifest_entry *ment) { @@ -206,16 +287,6 @@ static int alloc_add_ref(struct list_head *list, } -/* sort level 0 segments of the list from greatest to least seq */ -static int cmp_ref_list_seqs(void *priv, struct list_head *A, - struct list_head *B) -{ - struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); - struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); - - return -scoutfs_cmp_u64s(a->seq, b->seq); -} - /* * Get refs on all the segments in the manifest that we'll need to * search to populate the cache with the given range. @@ -238,42 +309,34 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, SCOUTFS_DECLARE_KVEC(last); struct manifest_ref *ref; struct manifest_ref *tmp; - int cmp; int ret; int i; down_write(&mani->rwsem); /* get level 0 segments that overlap with the missing range */ - ment = scoutfs_treap_first(mani->treap); + skey.level = 0; + skey.seq = ~0ULL; + ment = scoutfs_treap_lookup_prev(mani->treap, &skey); while (!IS_ERR_OR_NULL(ment)) { - if (ment->level > 0) - break; - - cmp = cmp_range_ment(key, end, ment); - if (cmp < 0) - break; - - if (cmp == 0) { + if (cmp_range_ment(key, end, ment) == 0) { ret = alloc_add_ref(ref_list, ment); if (ret) goto out; } - ment = scoutfs_treap_next(mani->treap, ment); + ment = scoutfs_treap_prev(mani->treap, ment); } if (IS_ERR(ment)) { ret = PTR_ERR(ment); goto out; } - /* level0s are sorted by key, reverse sort by seq */ - list_sort(NULL, ref_list, cmp_ref_list_seqs); - /* get higher level segments that overlap with the starting key */ for (i = 1; i < mani->nr_levels; i++) { skey.key = key; skey.level = i; + skey.seq = 0; /* XXX should use level counts to skip searches */ @@ -528,16 +591,182 @@ int scoutfs_manifest_dirty_ring(struct super_block *sb) } /* - * Manifest entries are first sorted by their level. + * Give the caller the segments that will be involved in the next + * compaction. * - * Level 0 segments can arbitrarily overlap. Their manifest entries are - * sorted by their first key so that searches can iterate over the - * entries until first shows that no more segments can overlap. We then - * sort by the sequence so that we can manage entries that have - * identical keys. + * For now we have a simple candidate search. We only initiate + * compaction when a level has exceeded its exponentially increasing + * limit on the number of segments. Once we have a level we use keys at + * each level to chose the next segment. This results in a pattern + * where clock hands sweep through each level. The hands wrap much + * faster on the higher levels. * - * Higher level segments don't overlap. There will never be manifest - * entries with the same key at a given level. + * If the candidate segment doesn't overlap with any higher level + * segments then just move it down a level. + * + * If the candidate does overlap then we add all the segments to the + * compaction caller's data and let it do its thing. It'll allocate and + * free segments and update the manifest. + * + * XXX this will get a lot more clever: + * - ensuring concurrent compactions don't overlap + * - prioritize segments with deletion or incremental records + * - prioritize partial segments + * - maybe compact segments by age in a given level + */ +int scoutfs_manifest_next_compact(struct super_block *sb, void *data) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_entry *ment; + struct scoutfs_manifest_entry *over; + struct manifest_search_key skey; + SCOUTFS_DECLARE_KVEC(ment_first); + SCOUTFS_DECLARE_KVEC(ment_last); + SCOUTFS_DECLARE_KVEC(over_first); + SCOUTFS_DECLARE_KVEC(over_last); + int level; + int err; + int ret; + int i; + + down_write(&mani->rwsem); + + for (level = mani->nr_levels - 1; level >= 0; level--) { + if (le64_to_cpu(super->manifest.level_counts[level]) >= + mani->level_limits[level]) + break; + } + + if (level < 0) { + ret = 0; + goto out; + } + + /* find the oldest level 0 or the next higher order level by key */ + if (level == 0) { + ment = scoutfs_treap_first(mani->treap); + if (!IS_ERR_OR_NULL(ment) && ment->level) + ment = NULL; + } else { + skey.key = mani->compact_keys[level]; + skey.level = level; + skey.seq = 0; + ment = scoutfs_treap_lookup_next(mani->treap, &skey); + if (ment == NULL && scoutfs_kvec_length(skey.key)) { + /* XXX ugh, these kvecs are the worst */ + scoutfs_kvec_init(skey.key, + skey.key[0].iov_base, 0); + ment = scoutfs_treap_lookup_next(mani->treap, &skey); + } + } + if (IS_ERR(ment)) { + ret = PTR_ERR(ment); + goto out; + } + if (ment == NULL || ment->level != level) { + /* XXX shouldn't be possible */ + ret = 0; + goto out; + } + + init_ment_keys(ment, ment_first, ment_last); + + /* find first overlapping at the next level */ + skey.key = ment_first; + skey.level = level + 1; + skey.seq = 0; + over = scoutfs_treap_lookup(mani->treap, &skey); + if (IS_ERR(over)) { + ret = PTR_ERR(over); + goto out; + } + + /* if there's no overlap we can just move it down a level */ + if (!over) { + ret = scoutfs_manifest_add(sb, ment_first, ment_last, + le64_to_cpu(ment->segno), + le64_to_cpu(ment->seq), + ment->level + 1); + if (ret) + goto out; + + ret = scoutfs_manifest_del(sb, ment_first, + le64_to_cpu(ment->seq), + ment->level); + if (ret) { + err = scoutfs_manifest_del(sb, ment_first, + le64_to_cpu(ment->seq), + ment->level + 1); + BUG_ON(err); + goto out; + } + + goto done; + } + + /* add the upper input segment */ + ret = scoutfs_compact_add(sb, data, ment_first, + le64_to_cpu(ment->segno), + le64_to_cpu(ment->seq), level); + if (ret) + goto out; + + /* add a fanout's worth of lower overlapping segments */ + init_ment_keys(over, over_first, over_last); + for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { + ret = scoutfs_compact_add(sb, data, over_first, + le64_to_cpu(over->segno), + le64_to_cpu(over->seq), level + 1); + if (ret) + goto out; + + over = scoutfs_treap_next(mani->treap, over); + if (IS_ERR(over)) { + ret = PTR_ERR(over); + goto out; + } + if (!over || over->level != (ment->level + 1)) + break; + + init_ment_keys(over, over_first, over_last); + if (scoutfs_kvec_cmp_overlap(ment_first, ment_last, + over_first, over_last) != 0) + break; + } + +done: + /* record the next key to start from, not exact */ + scoutfs_kvec_init_key(mani->compact_keys[level]); + scoutfs_kvec_memcpy_truncate(mani->compact_keys[level], ment_last); + scoutfs_kvec_be_inc(mani->compact_keys[level]); + + ret = 0; +out: + up_write(&mani->rwsem); + return ret; +} + +/* + * Manifest entries for all levels are stored in a single treap. + * + * First they're sorted by their level. + * + * Level 0 segments can contain any items which overlap so they are + * sorted by their sequence number. Compaction can find the first node + * and reading walks backwards through level 0 to get them from newest + * to oldest to resolve matching items. + * + * Higher level segments don't overlap. They are sorted by their first + * key. + * + * Searching comparisons are different than insertion and deletion + * comparisons for higher level segments. Searches want to find the + * segment that intersects with a given key. Insertions and deletions + * want to operate on the segment with a specific first key and sequence + * number. We tell the difference by the presence of a sequence number. + * A segment will never have a seq of 0. */ static int manifest_treap_compare(void *key, void *data) { @@ -545,21 +774,34 @@ static int manifest_treap_compare(void *key, void *data) struct scoutfs_manifest_entry *ment = data; SCOUTFS_DECLARE_KVEC(first); SCOUTFS_DECLARE_KVEC(last); + int cmp; - if (skey->level < ment->level) - return -1; - if (skey->level > ment->level) - return 1; + if (skey->level < ment->level) { + cmp = -1; + goto out; + } + if (skey->level > ment->level) { + cmp = 1; + goto out; + } - init_ment_keys(ment, first, NULL); + if (skey->level == 0) { + cmp = scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); + goto out; + } - if (skey->level == 0) - return scoutfs_kvec_memcmp(skey->key, first) ?: - scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); + init_ment_keys(ment, first, last); - init_ment_keys(ment, NULL, last); + if (skey->seq == 0) { + cmp = scoutfs_kvec_cmp_overlap(skey->key, skey->key, + first, last); + } else { + cmp = scoutfs_kvec_memcmp(skey->key, first) ?: + scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); + } - return scoutfs_kvec_cmp_overlap(skey->key, skey->key, first, last); +out: + return cmp; } static void manifest_treap_fill(void *data, void *arg) @@ -588,6 +830,7 @@ int scoutfs_manifest_setup(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct manifest *mani; + int ret; int i; mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); @@ -602,6 +845,17 @@ int scoutfs_manifest_setup(struct super_block *sb) return -ENOMEM; } + for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) { + ret = scoutfs_kvec_alloc_key(mani->compact_keys[i]); + if (ret) { + while (--i >= 0) + scoutfs_kvec_kfree(mani->compact_keys[i]); + scoutfs_treap_free(mani->treap); + kfree(mani); + return -ENOMEM; + } + } + for (i = ARRAY_SIZE(super->manifest.level_counts) - 1; i >= 0; i--) { if (super->manifest.level_counts[i]) { mani->nr_levels = i + 1; @@ -609,6 +863,14 @@ int scoutfs_manifest_setup(struct super_block *sb) } } + /* always trigger a compaction if there's a single l0 segment? */ + mani->level_limits[0] = 0; + mani->level_limits[1] = SCOUTFS_MANIFEST_FANOUT; + for (i = 2; i < ARRAY_SIZE(mani->level_limits); i++) { + mani->level_limits[i] = mani->level_limits[i - 1] * + SCOUTFS_MANIFEST_FANOUT; + } + sbi->manifest = mani; return 0; @@ -618,9 +880,12 @@ void scoutfs_manifest_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct manifest *mani = sbi->manifest; + int i; if (mani) { scoutfs_treap_free(mani->treap); + for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) + scoutfs_kvec_kfree(mani->compact_keys[i]); kfree(mani); } } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 021bdf14..2db63bb1 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -3,12 +3,21 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, struct kvec *last, u64 segno, u64 seq, u8 level); +int scoutfs_manifest_dirty(struct super_block *sb, struct kvec *first, u64 seq, + u8 level); +int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, + u8 level); int scoutfs_manifest_has_dirty(struct super_block *sb); int scoutfs_manifest_dirty_ring(struct super_block *sb); +int scoutfs_manifest_lock(struct super_block *sb); +int scoutfs_manifest_unlock(struct super_block *sb); + int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, struct kvec *until); +int scoutfs_manifest_next_compact(struct super_block *sb, void *data); + int scoutfs_manifest_setup(struct super_block *sb); void scoutfs_manifest_destroy(struct super_block *sb); diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 520ed9e3..b3e2a1bd 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -246,6 +246,16 @@ out: } +/* + * This just frees the segno for the given seg. It's gross but + * symmetrical with only being able to allocate segnos by allocating a + * seg. We'll probably have to do better. + */ +int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg) +{ + return scoutfs_alloc_free(sb, seg->segno); +} + /* * The bios submitted by this don't have page references themselves. If * this succeeds then the caller must call _wait before putting their @@ -546,6 +556,19 @@ int scoutfs_seg_manifest_add(struct super_block *sb, le64_to_cpu(sblk->seq), level); } +int scoutfs_seg_manifest_del(struct super_block *sb, + struct scoutfs_segment *seg, u8 level) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct native_item item; + SCOUTFS_DECLARE_KVEC(first); + + load_item(seg, 0, &item); + kvec_from_pages(seg, first, item.key_off, item.key_len); + + return scoutfs_manifest_del(sb, first, le64_to_cpu(sblk->seq), level); +} + int scoutfs_seg_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 683d9a3e..106215bc 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -16,6 +16,8 @@ int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, void scoutfs_seg_put(struct scoutfs_segment *seg); int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); +int scoutfs_seg_free_segno(struct super_block *sb, + struct scoutfs_segment *seg); void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, struct kvec *key, struct kvec *val, unsigned int nr_items, unsigned int key_bytes); @@ -24,6 +26,8 @@ void scoutfs_seg_append_item(struct super_block *sb, struct kvec *key, struct kvec *val); int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level); +int scoutfs_seg_manifest_del(struct super_block *sb, + struct scoutfs_segment *seg, u8 level); int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, diff --git a/kmod/src/super.c b/kmod/src/super.c index 00cafba4..4c4abdfa 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -34,6 +34,7 @@ #include "bio.h" #include "alloc.h" #include "treap.h" +#include "compact.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -230,6 +231,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_alloc_setup(sb) ?: scoutfs_treap_setup(sb) ?: // scoutfs_buddy_setup(sb) ?: + scoutfs_compact_setup(sb) ?: scoutfs_setup_trans(sb); if (ret) return ret; @@ -261,6 +263,7 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_buddy_destroy(sb); if (sbi->block_shrinker.shrink == scoutfs_block_shrink) diff --git a/kmod/src/super.h b/kmod/src/super.h index f68a1c66..d93a296b 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -13,6 +13,7 @@ struct item_cache; struct manifest; struct segment_cache; struct treap_info; +struct compact_info; struct scoutfs_sb_info { struct super_block *sb; @@ -37,6 +38,7 @@ struct scoutfs_sb_info { struct segment_cache *segment_cache; struct seg_alloc *seg_alloc; struct treap_info *treap_info; + struct compact_info *compact_info; struct buddy_info *buddy_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 04b1ed52..e1a96471 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -84,6 +84,7 @@ void scoutfs_trans_write_func(struct work_struct *work) struct scoutfs_segment *seg; bool advance = false; int ret = 0; + int err; scoutfs_bio_init_comp(&comp); sbi->trans_task = NULL; @@ -96,24 +97,44 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); #endif - if (scoutfs_item_dirty_bytes(sb) || scoutfs_manifest_has_dirty(sb) || - scoutfs_alloc_has_dirty(sb)) { - + /* + * XXX this needs serious work to handle errors. + */ + while (scoutfs_item_dirty_bytes(sb)) { + advance = true; + seg = NULL; ret = scoutfs_seg_alloc(sb, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: + scoutfs_manifest_lock(sb) ?: scoutfs_seg_manifest_add(sb, seg, 0) ?: - scoutfs_manifest_dirty_ring(sb) ?: - scoutfs_alloc_dirty_ring(sb) ?: - scoutfs_treap_submit_write(sb, &comp) ?: - scoutfs_seg_submit_write(sb, seg, &comp) ?: - scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_write_dirty_super(sb); - BUG_ON(ret); - + scoutfs_manifest_unlock(sb) ?: + scoutfs_seg_submit_write(sb, seg, &comp); scoutfs_seg_put(seg); - advance = true; + if (ret) + goto out; } + if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { + advance = true; + ret = scoutfs_manifest_dirty_ring(sb) ?: + scoutfs_alloc_dirty_ring(sb) ?: + scoutfs_treap_submit_write(sb, &comp); + if (ret) + goto out; + } + +out: + err = scoutfs_bio_wait_comp(sb, &comp) ?: + scoutfs_write_dirty_super(sb); + if (err && !ret) + ret = err; + + /* XXX this all needs serious work for dealing with errors */ + WARN_ON_ONCE(ret); + + if (advance && ret) + advance = false; + spin_lock(&sbi->trans_write_lock); if (advance) scoutfs_advance_dirty_super(sb); From c21dc4ec20dd4f1dcbf8707599ab3fc43a2390b1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Jan 2017 14:13:56 -0800 Subject: [PATCH 192/920] Refactor level_count and protect with seqcount We were manually manipulating the level counts in the super in a bunch of places under the manifest rwsem. This refactors them into simple get and add functions. We protect them with a seqcount so that we'll be able read them without blocking (from trans hold attempts). We also add a helper for testing that a level is full because we already used different comparisons in two call sites. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index d4a9a58c..e10d811e 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -43,6 +43,7 @@ struct manifest { struct rw_semaphore rwsem; + seqcount_t seqcount; struct scoutfs_treap *treap; u8 nr_levels; @@ -127,6 +128,35 @@ static bool cmp_range_ment(struct kvec *key, struct kvec *end, return scoutfs_kvec_cmp_overlap(key, end, first, last); } +static u64 get_level_count(struct manifest *mani, + struct scoutfs_super_block *super, u8 level) +{ + unsigned int sc; + u64 count; + + do { + sc = read_seqcount_begin(&mani->seqcount); + count = le64_to_cpu(super->manifest.level_counts[level]); + } while (read_seqcount_retry(&mani->seqcount, sc)); + + return count; +} + +static void add_level_count(struct manifest *mani, + struct scoutfs_super_block *super, u8 level, + s64 val) +{ + write_seqcount_begin(&mani->seqcount); + le64_add_cpu(&super->manifest.level_counts[level], val); + write_seqcount_end(&mani->seqcount); +} + +static bool level_full(struct manifest *mani, + struct scoutfs_super_block *super, u8 level) +{ + return get_level_count(mani, super, level) > mani->level_limits[level]; +} + /* * Insert a new manifest entry in the treap. The treap allocates a new * node for us and we fill it. @@ -169,10 +199,9 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, ret = PTR_ERR(ment); } else { mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - le64_add_cpu(&super->manifest.level_counts[level], 1); + add_level_count(mani, super, level, 1); - if (le64_to_cpu(super->manifest.level_counts[level]) > - mani->level_limits[level]) + if (level_full(mani, super, level)) scoutfs_compact_kick(sb); ret = 0; @@ -221,7 +250,7 @@ int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, ret = scoutfs_treap_delete(mani->treap, &skey); if (ret == 0) - le64_add_cpu(&super->manifest.level_counts[level], -1ULL); + add_level_count(mani, super, level, -1ULL); return ret; } @@ -634,8 +663,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) down_write(&mani->rwsem); for (level = mani->nr_levels - 1; level >= 0; level--) { - if (le64_to_cpu(super->manifest.level_counts[level]) >= - mani->level_limits[level]) + if (level_full(mani, super, level)) break; } @@ -838,6 +866,8 @@ int scoutfs_manifest_setup(struct super_block *sb) return -ENOMEM; init_rwsem(&mani->rwsem); + seqcount_init(&mani->seqcount); + mani->treap = scoutfs_treap_alloc(sb, &manifest_treap_ops, &super->manifest.root); if (!mani->treap) { From 30b088377f9ab1aec574a0a107c144cfa1ffac7c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 10:59:19 -0800 Subject: [PATCH 193/920] Fix setting trans_task Some recent refactoring accidentally set the trans task to null instead of the current task. It's not used but until it's removed it should be correct. Signed-off-by: Zach Brown --- kmod/src/trans.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e1a96471..6ea4b5f8 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -87,7 +87,7 @@ void scoutfs_trans_write_func(struct work_struct *work) int err; scoutfs_bio_init_comp(&comp); - sbi->trans_task = NULL; + sbi->trans_task = current; wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); From a333c507fb468e8657355de097108280fa8d9e82 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 11:02:41 -0800 Subject: [PATCH 194/920] Fix how dirty treap is tracked The transaction writing thread tests if the manifest and alloc treaps are dirty. It did this by testing if there were any dirty nodes in the treap. But this misses the case where the treap has been modified and all nodes have been removed. In that case the root references no dirty nodes but needs to be written. Instead let's specifically mark the treap dirty when it's modified. From then on sync will always try to write it out. We also integrate updating the persistent root as part of writing the dirty nodes to the persistent ring. It's required and every caller did it so it was silly to make it a separate step. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 3 +-- kmod/src/manifest.c | 3 +-- kmod/src/treap.c | 21 +++++++++++---------- kmod/src/treap.h | 5 ++--- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 709f3b78..48307ebc 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -270,8 +270,7 @@ int scoutfs_alloc_dirty_ring(struct super_block *sb) kfree(pend); } - scoutfs_treap_dirty_ring(sal->treap); - scoutfs_treap_update_root(&super->alloc_treap_root, sal->treap); + scoutfs_treap_dirty_ring(sal->treap, &super->alloc_treap_root); ret = 0; out: up_write(&sal->rwsem); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index e10d811e..01266a52 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -612,8 +612,7 @@ int scoutfs_manifest_dirty_ring(struct super_block *sb) struct scoutfs_super_block *super = &sbi->super; down_write(&mani->rwsem); - scoutfs_treap_dirty_ring(mani->treap); - scoutfs_treap_update_root(&super->manifest.root, mani->treap); + scoutfs_treap_dirty_ring(mani->treap, &super->manifest.root); up_write(&mani->rwsem); return 0; diff --git a/kmod/src/treap.c b/kmod/src/treap.c index 692b25a2..f458aa76 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -117,6 +117,7 @@ struct scoutfs_treap { struct scoutfs_super_block *super; struct scoutfs_treap_ops *ops; struct treap_ref root_ref; + bool dirty; u64 dirty_bytes; }; @@ -424,6 +425,7 @@ static bool mark_node_dirty(struct scoutfs_treap *treap, struct treap_ref *ref, return false; treap->dirty_bytes += node_ring_bytes(node); + treap->dirty = true; node->off = tinf->dirty_off; node->gen = tinf->dirty_gen; @@ -1030,7 +1032,7 @@ out: int scoutfs_treap_has_dirty(struct scoutfs_treap *treap) { - return !!(treap->root_ref.aug_bits & SCOUTFS_TREAP_AUG_DIRTY); + return treap->dirty; } static void *pages_off_ptr(struct treap_info *tinf) @@ -1135,7 +1137,8 @@ static void copy_node_to_ring(struct scoutfs_treap *treap, * * This is called for multiple treaps before the ring is written. */ -int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap) +int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap, + struct scoutfs_treap_root *root) { struct treap_node *node; unsigned bytes; @@ -1168,7 +1171,13 @@ int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap) } } + /* point the persistent super root at the treap we wrote to the ring */ + root->ref.off = cpu_to_le64(treap->root_ref.off); + root->ref.gen = cpu_to_le64(treap->root_ref.gen); + root->ref.aug_bits = treap->root_ref.aug_bits; + treap->dirty_bytes = 0; + treap->dirty = false; ret = 0; out: return ret; @@ -1258,14 +1267,6 @@ struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, return treap; } -void scoutfs_treap_update_root(struct scoutfs_treap_root *root, - struct scoutfs_treap *treap) -{ - root->ref.off = cpu_to_le64(treap->root_ref.off); - root->ref.gen = cpu_to_le64(treap->root_ref.gen); - root->ref.aug_bits = treap->root_ref.aug_bits; -} - /* * Free all the allocated nodes in the treap and clear the root. */ diff --git a/kmod/src/treap.h b/kmod/src/treap.h index e69fbde1..497d742a 100644 --- a/kmod/src/treap.h +++ b/kmod/src/treap.h @@ -18,8 +18,6 @@ struct scoutfs_treap_ops { struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, struct scoutfs_treap_ops *ops, struct scoutfs_treap_root *root); -void scoutfs_treap_update_root(struct scoutfs_treap_root *root, - struct scoutfs_treap *treap); void scoutfs_treap_free(struct scoutfs_treap *treap); void *scoutfs_treap_insert(struct scoutfs_treap *treap, void *key, u16 bytes, @@ -38,7 +36,8 @@ void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data); void *scoutfs_treap_prev(struct scoutfs_treap *treap, void *data); int scoutfs_treap_has_dirty(struct scoutfs_treap *treap); -int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap); +int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap, + struct scoutfs_treap_root *root); int scoutfs_treap_submit_write(struct super_block *sb, struct scoutfs_bio_completion *comp); From aad5a34290a19be56474c5f02f118cae4f08c61c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 11:17:46 -0800 Subject: [PATCH 195/920] Don't prematurely write dirty super A previous refactoring messed up and had scoutfs_trans_write_func() always write the dirty super even when nothing was dirty and there was nothing for the sync attempt to do. This was very confusing and made it look like the segment and treap writes were being lost when in fact it was the super write that shouldn't have happened. Signed-off-by: Zach Brown --- kmod/src/trans.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 6ea4b5f8..5987646c 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -84,7 +84,6 @@ void scoutfs_trans_write_func(struct work_struct *work) struct scoutfs_segment *seg; bool advance = false; int ret = 0; - int err; scoutfs_bio_init_comp(&comp); sbi->trans_task = current; @@ -101,7 +100,6 @@ void scoutfs_trans_write_func(struct work_struct *work) * XXX this needs serious work to handle errors. */ while (scoutfs_item_dirty_bytes(sb)) { - advance = true; seg = NULL; ret = scoutfs_seg_alloc(sb, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: @@ -115,26 +113,21 @@ void scoutfs_trans_write_func(struct work_struct *work) } if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { - advance = true; ret = scoutfs_manifest_dirty_ring(sb) ?: scoutfs_alloc_dirty_ring(sb) ?: - scoutfs_treap_submit_write(sb, &comp); + scoutfs_treap_submit_write(sb, &comp) ?: + scoutfs_bio_wait_comp(sb, &comp) ?: + scoutfs_write_dirty_super(sb); if (ret) goto out; + + advance = true; } out: - err = scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_write_dirty_super(sb); - if (err && !ret) - ret = err; - /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); - if (advance && ret) - advance = false; - spin_lock(&sbi->trans_write_lock); if (advance) scoutfs_advance_dirty_super(sb); From 3f812fa9a70fc1b8baa8d7fa2bd8d3128976ebb5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 11:35:55 -0800 Subject: [PATCH 196/920] More thoroughly integrate compaction The first pass at compaction just kicked a thread any time we added a segment that brought its level's count over the limit. Tasks could create dirty items and write level0 segments regardless of the progress of compaction. This ties the writing rate to compaction. Writers have to wait to hold a transaction until the dirty item count is under a segment and there's no level0 segments. Usualy more level0 segments are allowed but we're aggressively pushing compaction, we'll relax this later. This also more forcefully ensures that compaction makes forward progress. We kick the compaction thread if we exceed the level count, wait for level0 to drain, or successfully complete a compaction. We tweak scoutfs_manifest_next_compact() to return 0 if there's no compaction work to do so the the compaction thread can exit without triggering another. For clarity we also kick off a sync after compaction so that we don't sit around with a dirty manifest until the next sync. This may not be wise. Signed-off-by: Zach Brown --- kmod/src/compact.c | 17 ++++++--- kmod/src/manifest.c | 58 ++++++++++++++++++++++------- kmod/src/manifest.h | 1 + kmod/src/trans.c | 89 ++++++++++++++++++++++++++++++++++----------- kmod/src/trans.h | 1 + 5 files changed, 126 insertions(+), 40 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 5933af25..8e00b576 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -22,6 +22,7 @@ #include "cmp.h" #include "compact.h" #include "manifest.h" +#include "trans.h" #include "scoutfs_trace.h" /* @@ -470,15 +471,21 @@ static void scoutfs_compact_func(struct work_struct *work) INIT_LIST_HEAD(&curs.csegs); - ret = scoutfs_manifest_next_compact(sb, (void *)&curs) ?: - read_segments(sb, &curs) ?: + ret = scoutfs_manifest_next_compact(sb, (void *)&curs); + if (ret <= 0) + goto out; + + ret = read_segments(sb, &curs) ?: compact_segments(sb, &curs, &results) ?: write_segments(sb, &results) ?: update_manifest(sb, &curs, &results); - - if (ret) + if (ret) { free_result_segnos(sb, &results); - + } else { + scoutfs_sync_fs(sb, 0); + scoutfs_compact_kick(sb); + } +out: free_csegs(&curs.csegs); free_csegs(&results); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 01266a52..48277ab0 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -24,6 +24,7 @@ #include "cmp.h" #include "compact.h" #include "manifest.h" +#include "trans.h" #include "scoutfs_trace.h" /* @@ -142,19 +143,41 @@ static u64 get_level_count(struct manifest *mani, return count; } -static void add_level_count(struct manifest *mani, - struct scoutfs_super_block *super, u8 level, - s64 val) +static bool past_limit(struct manifest *mani, u8 level, u64 count) { - write_seqcount_begin(&mani->seqcount); - le64_add_cpu(&super->manifest.level_counts[level], val); - write_seqcount_end(&mani->seqcount); + return count > mani->level_limits[level]; } static bool level_full(struct manifest *mani, struct scoutfs_super_block *super, u8 level) { - return get_level_count(mani, super, level) > mani->level_limits[level]; + return past_limit(mani, level, get_level_count(mani, super, level)); +} + +static void add_level_count(struct super_block *sb, struct manifest *mani, + struct scoutfs_super_block *super, u8 level, + s64 val) +{ + bool was_full; + bool now_full; + u64 count; + + write_seqcount_begin(&mani->seqcount); + + count = le64_to_cpu(super->manifest.level_counts[level]); + was_full = past_limit(mani, level, count); + + count += val; + now_full = past_limit(mani, level, count); + super->manifest.level_counts[level] = cpu_to_le64(count); + + write_seqcount_end(&mani->seqcount); + + if (was_full && !now_full) + scoutfs_trans_wake_holders(sb); + + if (now_full) + scoutfs_compact_kick(sb); } /* @@ -199,11 +222,7 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, ret = PTR_ERR(ment); } else { mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - add_level_count(mani, super, level, 1); - - if (level_full(mani, super, level)) - scoutfs_compact_kick(sb); - + add_level_count(sb, mani, super, level, 1); ret = 0; } @@ -250,7 +269,7 @@ int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, ret = scoutfs_treap_delete(mani->treap, &skey); if (ret == 0) - add_level_count(mani, super, level, -1ULL); + add_level_count(sb, mani, super, level, -1ULL); return ret; } @@ -618,6 +637,15 @@ int scoutfs_manifest_dirty_ring(struct super_block *sb) return 0; } +u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + return get_level_count(mani, super, level); +} + /* * Give the caller the segments that will be involved in the next * compaction. @@ -636,6 +664,8 @@ int scoutfs_manifest_dirty_ring(struct super_block *sb) * compaction caller's data and let it do its thing. It'll allocate and * free segments and update the manifest. * + * Returns 1 if there's compaction work to do, 0 if not, or -errno. + * * XXX this will get a lot more clever: * - ensuring concurrent compactions don't overlap * - prioritize segments with deletion or incremental records @@ -769,7 +799,7 @@ done: scoutfs_kvec_memcpy_truncate(mani->compact_keys[level], ment_last); scoutfs_kvec_be_inc(mani->compact_keys[level]); - ret = 0; + ret = 1; out: up_write(&mani->rwsem); return ret; diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 2db63bb1..5e529cd5 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -16,6 +16,7 @@ int scoutfs_manifest_unlock(struct super_block *sb); int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, struct kvec *until); +u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); int scoutfs_manifest_setup(struct super_block *sb); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 5987646c..6b10cd55 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -28,6 +28,7 @@ #include "seg.h" #include "alloc.h" #include "treap.h" +#include "compact.h" #include "scoutfs_trace.h" /* @@ -210,9 +211,56 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, } /* - * The first holders race to try and allocate the segment that will be - * written by the next commit. + * I think the holder that creates the most dirty item data is + * symlinking, which can create all the entry items and a symlink target + * item with a full 4k path. We go a little nuts and just set it to two + * blocks. + * + * XXX This divides the segment size to set the hard limit on the number of + * concurrent holders so we'll want this to be more precise. */ +#define MOST_DIRTY (2 * SCOUTFS_BLOCK_SIZE) + +/* + * We're able to hold the transaction if the current dirty item bytes + * and the presumed worst case item dirtying of all the holders, + * including us, all fit in a segment. + */ +static bool hold_acquired(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + long bytes; + int with_us; + int holds; + int before; + + holds = atomic_read(&sbi->trans_holds); + for (;;) { + /* transaction is being committed */ + if (holds < 0) + return false; + + /* only hold when there's no level 0 segments, XXX for now */ + if (scoutfs_manifest_level_count(sb, 0) > 0) { + scoutfs_compact_kick(sb); + return false; + } + + /* see if we all would fill the segment */ + with_us = holds + 1; + bytes = (with_us * MOST_DIRTY) + scoutfs_item_dirty_bytes(sb); + if (bytes > SCOUTFS_SEGMENT_SIZE) { + scoutfs_sync_fs(sb, 0); + return false; + } + + before = atomic_cmpxchg(&sbi->trans_holds, holds, with_us); + if (before == holds) + return true; + holds = before; + } +} + int scoutfs_hold_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -220,37 +268,36 @@ int scoutfs_hold_trans(struct super_block *sb) if (current == sbi->trans_task) return 0; - return wait_event_interruptible(sbi->trans_hold_wq, - atomic_add_unless(&sbi->trans_holds, 1, -1)); + return wait_event_interruptible(sbi->trans_hold_wq, hold_acquired(sb)); } /* - * As we release we kick off a commit if we have a segment's worth of - * dirty items. - * - * Right now it's conservatively kicking off writes at ~95% full blocks. - * This leaves a lot of slop for the largest item bytes created by a - * holder and overrun by concurrent holders (who aren't accounted - * today). - * - * It should more precisely know the worst case item byte consumption of - * holders and only kick off a write when someone tries to hold who - * might fill the segment. + * As we release we'll almost certainly have dirtied less than the + * worst case dirty assumption that holders might be throttled waiting + * for. We always try and wake blocked holders in case they now have + * room to dirty. */ void scoutfs_release_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - unsigned int target = (SCOUTFS_SEGMENT_SIZE * 95 / 100); if (current == sbi->trans_task) return; - if (atomic_sub_return(1, &sbi->trans_holds) == 0) { - if (scoutfs_item_dirty_bytes(sb) >= target) - scoutfs_sync_fs(sb, 0); + atomic_dec(&sbi->trans_holds); + wake_up(&sbi->trans_hold_wq); +} - wake_up(&sbi->trans_hold_wq); - } +/* + * This is called to wake people waiting on holders when the conditions + * that they're waiting on change: levels being full, dirty count falling + * under a segment, or holders falling to 0. + */ +void scoutfs_trans_wake_holders(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + wake_up(&sbi->trans_hold_wq); } int scoutfs_setup_trans(struct super_block *sb) diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 22c5755a..f1ecbc51 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -8,6 +8,7 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int scoutfs_hold_trans(struct super_block *sb); void scoutfs_release_trans(struct super_block *sb); +void scoutfs_trans_wake_holders(struct super_block *sb); int scoutfs_setup_trans(struct super_block *sb); void scoutfs_shutdown_trans(struct super_block *sb); From ded184b481d81535ebc67e657c1cff489e05be9f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 11:36:10 -0800 Subject: [PATCH 197/920] Add a pile of tracing printks Signed-off-by: Zach Brown --- kmod/src/manifest.c | 2 ++ kmod/src/trans.c | 7 +++++++ kmod/src/treap.c | 8 ++++++++ 3 files changed, 17 insertions(+) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 48277ab0..7ad89c35 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -696,6 +696,8 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) break; } + trace_printk("level %d\n", level); + if (level < 0) { ret = 0; goto out; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 6b10cd55..adbab3cb 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -97,6 +97,11 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); #endif + trace_printk("dirty bytes %ld manifest dirty %d alloc dirty %d\n", + scoutfs_item_dirty_bytes(sb), + scoutfs_manifest_has_dirty(sb), + scoutfs_alloc_has_dirty(sb)); + /* * XXX this needs serious work to handle errors. */ @@ -184,6 +189,8 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) struct write_attempt attempt; int ret; + trace_printk("wait %d\n", wait); + if (!wait) { queue_trans_work(sbi); return 0; diff --git a/kmod/src/treap.c b/kmod/src/treap.c index f458aa76..b6346df3 100644 --- a/kmod/src/treap.c +++ b/kmod/src/treap.c @@ -223,6 +223,8 @@ static void update_internal_aug(struct scoutfs_treap *treap, while (node) { bits = node_aug_bits(treap, node); ref = parent_ref(treap, node); + trace_printk("node %p bits %x parent %p ref bits %x\n", + node, bits, node->parent, ref->aug_bits); if (ref->aug_bits == bits) break; ref->aug_bits = bits; @@ -377,6 +379,9 @@ static void repair(struct scoutfs_treap *treap, struct treap_node *node) update_internal_aug(treap, node); update_data_aug(treap, node); rebalance(treap, node); + + trace_printk("treap %p root aug %x\n", + treap, treap->root_ref.aug_bits); } static struct treap_node *alloc_node(u16 bytes) @@ -424,6 +429,9 @@ static bool mark_node_dirty(struct scoutfs_treap *treap, struct treap_ref *ref, if (dirty_node(treap, node)) return false; + trace_printk("node %p off %llu gen %llu now dirty\n", + node, node->off, node->gen); + treap->dirty_bytes += node_ring_bytes(node); treap->dirty = true; From 0a5fb7fd832214a33c4dd1ea8af08135fa44a1ab Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 15:38:28 -0800 Subject: [PATCH 198/920] Add some counters Signed-off-by: Zach Brown --- kmod/src/alloc.c | 4 ++++ kmod/src/compact.c | 5 +++++ kmod/src/counters.h | 11 +++++++++-- kmod/src/manifest.c | 2 ++ kmod/src/trans.c | 3 +++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 48307ebc..184493c9 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -20,6 +20,7 @@ #include "treap.h" #include "cmp.h" #include "alloc.h" +#include "counters.h" /* * scoutfs allocates segments by storing regions of a bitmap in treap @@ -165,6 +166,8 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) ret = 0; out: + if (ret == 0) + scoutfs_inc_counter(sb, alloc_alloc); up_write(&sal->rwsem); trace_printk("segno %llu ret %d\n", *segno, ret); @@ -201,6 +204,7 @@ int scoutfs_alloc_free(struct super_block *sb, u64 segno) } set_bit_le(nr, pend->reg.bits); + scoutfs_inc_counter(sb, alloc_free); ret = 0; out: up_write(&sal->rwsem); diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 8e00b576..dd62debf 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -23,6 +23,7 @@ #include "compact.h" #include "manifest.h" #include "trans.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -131,6 +132,7 @@ static int read_segments(struct super_block *sb, struct compact_cursor *curs) } cseg->seg = seg; + scoutfs_inc_counter(sb, compact_segment_read); } list_for_each_entry(cseg, &curs->csegs, entry) { @@ -165,6 +167,7 @@ static int write_segments(struct super_block *sb, struct list_head *results) ret = scoutfs_seg_submit_write(sb, cseg->seg, &comp); if (ret) break; + scoutfs_inc_counter(sb, compact_segment_write); } err = scoutfs_bio_wait_comp(sb, &comp); @@ -475,6 +478,8 @@ static void scoutfs_compact_func(struct work_struct *work) if (ret <= 0) goto out; + scoutfs_inc_counter(sb, compact_compactions); + ret = read_segments(sb, &curs) ?: compact_segments(sb, &curs, &results) ?: write_segments(sb, &results) ?: diff --git a/kmod/src/counters.h b/kmod/src/counters.h index f6d630c1..09d48252 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,11 +12,18 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(alloc_alloc) \ + EXPAND_COUNTER(alloc_free) \ EXPAND_COUNTER(block_mem_alloc) \ - EXPAND_COUNTER(block_mem_free) + EXPAND_COUNTER(block_mem_free) \ + EXPAND_COUNTER(trans_level0_seg_write) \ + EXPAND_COUNTER(manifest_compact_migrate) \ + EXPAND_COUNTER(compact_compactions) \ + EXPAND_COUNTER(compact_segment_read) \ + EXPAND_COUNTER(compact_segment_write) #define FIRST_COUNTER block_mem_alloc -#define LAST_COUNTER block_mem_free +#define LAST_COUNTER compact_segment_write #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 7ad89c35..2f586466 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -25,6 +25,7 @@ #include "compact.h" #include "manifest.h" #include "trans.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -762,6 +763,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } + scoutfs_inc_counter(sb, manifest_compact_migrate); goto done; } diff --git a/kmod/src/trans.c b/kmod/src/trans.c index adbab3cb..bbd543ff 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -29,6 +29,7 @@ #include "alloc.h" #include "treap.h" #include "compact.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -116,6 +117,8 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_seg_put(seg); if (ret) goto out; + + scoutfs_inc_counter(sb, trans_level0_seg_write); } if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { From 963b04701f5d7b9d71cf11604cdbd5f8796aea79 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 15:40:46 -0800 Subject: [PATCH 199/920] Add some bio tracing Signed-off-by: Zach Brown --- kmod/src/bio.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index 119cd13e..c6be2e87 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -32,6 +32,9 @@ static void dec_end_io(struct bio_end_io_args *args, size_t bytes, int err) if (err && !args->err) args->err = err; + trace_printk("args %p bytes %zu in_flight %d err %d\n", + args, bytes, atomic_read(&args->bytes_in_flight), err); + if (atomic_sub_return(bytes, &args->bytes_in_flight) == 0) { args->end_io(args->sb, args->data, args->err); kfree(args); @@ -42,7 +45,7 @@ static void bio_end_io(struct bio *bio, int err) { struct bio_end_io_args *args = bio->bi_private; - trace_printk("bio %p end io\n", bio); + trace_printk("bio %p size %u err %d \n", bio, bio->bi_size, err); dec_end_io(args, bio->bi_size, err); bio_put(bio); @@ -110,6 +113,9 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, if (bio_add_page(bio, page, bytes, 0) != bytes) { /* submit the full bio and retry this page */ atomic_add(bio->bi_size, &args->bytes_in_flight); + trace_printk("bio %p args %p size %u in_flight %d\n", + bio, args, bio->bi_size, + atomic_read(&args->bytes_in_flight)); submit_bio(rw, bio); bio = NULL; i--; @@ -124,6 +130,9 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, if (bio) { atomic_add(bio->bi_size, &args->bytes_in_flight); + trace_printk("bio %p args %p size %u in_flight %d\n", + bio, args, bio->bi_size, + atomic_read(&args->bytes_in_flight)); submit_bio(rw, bio); } @@ -137,6 +146,7 @@ void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp) atomic_set(&comp->pending, 1); init_completion(&comp->comp); comp->err = 0; + trace_printk("initing comp %p\n", comp); } static void comp_end_io(struct super_block *sb, void *data, int err) @@ -146,6 +156,9 @@ static void comp_end_io(struct super_block *sb, void *data, int err) if (err && !comp->err) comp->err = err; + trace_printk("ending comp %p pending before %d\n", + comp, atomic_read(&comp->pending)); + if (atomic_dec_and_test(&comp->pending)) complete(&comp->comp); } @@ -156,6 +169,9 @@ void scoutfs_bio_submit_comp(struct super_block *sb, int rw, struct scoutfs_bio_completion *comp) { atomic_inc(&comp->pending); + trace_printk("submitting comp %p pending before %d\n", + comp, atomic_read(&comp->pending)); + scoutfs_bio_submit(sb, rw, pages, blkno, nr_blocks, comp_end_io, comp); } @@ -163,6 +179,7 @@ int scoutfs_bio_wait_comp(struct super_block *sb, struct scoutfs_bio_completion *comp) { comp_end_io(sb, comp, 0); + trace_printk("waiting for comp %p\n", comp); wait_for_completion(&comp->comp); return comp->err; } From 3407576ced7d488b9a73348d6429b1ec306fc500 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Jan 2017 16:00:55 -0800 Subject: [PATCH 200/920] Don't use bio size in end_io Some drives don't set bi_size so just track the number of IOs. (And the size argument to end_io has been removed in recent kernels.) Signed-off-by: Zach Brown --- kmod/src/bio.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index c6be2e87..bb42b02a 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -21,21 +21,21 @@ struct bio_end_io_args { struct super_block *sb; - atomic_t bytes_in_flight; + atomic_t in_flight; int err; scoutfs_bio_end_io_t end_io; void *data; }; -static void dec_end_io(struct bio_end_io_args *args, size_t bytes, int err) +static void dec_end_io(struct bio_end_io_args *args, int err) { if (err && !args->err) args->err = err; - trace_printk("args %p bytes %zu in_flight %d err %d\n", - args, bytes, atomic_read(&args->bytes_in_flight), err); + trace_printk("args %p in_flight %d err %d\n", + args, atomic_read(&args->in_flight), err); - if (atomic_sub_return(bytes, &args->bytes_in_flight) == 0) { + if (atomic_dec_and_test(&args->in_flight)) { args->end_io(args->sb, args->data, args->err); kfree(args); } @@ -47,7 +47,7 @@ static void bio_end_io(struct bio *bio, int err) trace_printk("bio %p size %u err %d \n", bio, bio->bi_size, err); - dec_end_io(args, bio->bi_size, err); + dec_end_io(args, err); bio_put(bio); } @@ -83,7 +83,7 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, } args->sb = sb; - atomic_set(&args->bytes_in_flight, 1); + atomic_set(&args->in_flight, 1); args->err = 0; args->end_io = end_io; args->data = data; @@ -112,10 +112,9 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, if (bio_add_page(bio, page, bytes, 0) != bytes) { /* submit the full bio and retry this page */ - atomic_add(bio->bi_size, &args->bytes_in_flight); - trace_printk("bio %p args %p size %u in_flight %d\n", - bio, args, bio->bi_size, - atomic_read(&args->bytes_in_flight)); + atomic_inc(&args->in_flight); + trace_printk("bio %p args %p in_flight %d\n", + bio, args, atomic_read(&args->in_flight)); submit_bio(rw, bio); bio = NULL; i--; @@ -129,15 +128,14 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, } if (bio) { - atomic_add(bio->bi_size, &args->bytes_in_flight); - trace_printk("bio %p args %p size %u in_flight %d\n", - bio, args, bio->bi_size, - atomic_read(&args->bytes_in_flight)); + atomic_inc(&args->in_flight); + trace_printk("bio %p args %p in_flight %d\n", + bio, args, atomic_read(&args->in_flight)); submit_bio(rw, bio); } blk_finish_plug(&plug); - dec_end_io(args, 1, ret); + dec_end_io(args, ret); } void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp) From 930f541c7be70443079c1cae393061cc5184e50f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Jan 2017 10:38:39 -0800 Subject: [PATCH 201/920] Add a scoutfs_seg_get Compaction is going to want to get additional references on a segment. It could just "read" it again while holding a reference but this is more clear. Signed-off-by: Zach Brown --- kmod/src/seg.c | 5 +++++ kmod/src/seg.h | 1 + 2 files changed, 6 insertions(+) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index b3e2a1bd..f2034b8e 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -87,6 +87,11 @@ static struct scoutfs_segment *alloc_seg(u64 segno) return seg; } +void scoutfs_seg_get(struct scoutfs_segment *seg) +{ + atomic_inc(&seg->refcount); +} + void scoutfs_seg_put(struct scoutfs_segment *seg) { int i; diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 106215bc..597e9955 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -13,6 +13,7 @@ int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key); int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, struct kvec *key, struct kvec *val); +void scoutfs_seg_get(struct scoutfs_segment *seg); void scoutfs_seg_put(struct scoutfs_segment *seg); int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); From 519b9c35c4deb0e7347ae58a2c5d5383d6b85b57 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Jan 2017 14:03:33 -0800 Subject: [PATCH 202/920] Correcly wrap when finding compaction entries Compaction looks for the next entry at a given level to compact. It only tested for not finding a next entry when it needs to wrap the key and start over in the level, it missed the case where the next entry is at a greater level. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2f586466..b67c1f9b 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -714,7 +714,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) skey.level = level; skey.seq = 0; ment = scoutfs_treap_lookup_next(mani->treap, &skey); - if (ment == NULL && scoutfs_kvec_length(skey.key)) { + if (ment == NULL || ment->level != level) { /* XXX ugh, these kvecs are the worst */ scoutfs_kvec_init(skey.key, skey.key[0].iov_base, 0); From 822ce205c5649ecf488bc22d821ffe785630b364 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Jan 2017 14:06:17 -0800 Subject: [PATCH 203/920] Let compaction skip segments as needed Previously the only clever compaction avoidance we'd try was in the manifest walk. If we found that there were no overlapping segments in the next level we'd just move the entry down a level and skip compaction entirely. But that's just one specific instance of the general case: either of the lower or upper segments don't overlap with each other. There can be many lower level segments that intersect with the full range of keys in the upper level segment but which don't actually intersect with any items in the upper segment. So we refactor the compaction to notice this case. We get the first and last keys and use them to skip each segment as we first start to iterate through it. We don't want to read segments that we never actually have to copy items from so we read each segment on demand instead of concurrently as the compaction starts. This means that item iteration can now have to read a segment and can now return errors. Signed-off-by: Zach Brown --- kmod/src/compact.c | 303 +++++++++++++++++++++++++++++--------------- kmod/src/compact.h | 2 +- kmod/src/counters.h | 12 +- kmod/src/manifest.c | 75 +++-------- 4 files changed, 232 insertions(+), 160 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index dd62debf..14c35c98 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -71,6 +71,7 @@ struct compact_seg { u64 seq; u8 level; SCOUTFS_DECLARE_KVEC(first); + SCOUTFS_DECLARE_KVEC(last); struct scoutfs_segment *seg; int pos; int saved_pos; @@ -91,6 +92,28 @@ struct compact_cursor { struct compact_seg *saved_lower; }; +static void free_cseg(struct compact_seg *cseg) +{ + WARN_ON_ONCE(!list_empty(&cseg->entry)); + + scoutfs_seg_put(cseg->seg); + scoutfs_kvec_kfree(cseg->first); + scoutfs_kvec_kfree(cseg->last); + + kfree(cseg); +} + +static void free_cseg_list(struct list_head *list) +{ + struct compact_seg *cseg; + struct compact_seg *tmp; + + list_for_each_entry_safe(cseg, tmp, list, entry) { + list_del_init(&cseg->entry); + free_cseg(cseg); + } +} + static void save_pos(struct compact_cursor *curs) { struct compact_seg *cseg; @@ -113,66 +136,24 @@ static void restore_pos(struct compact_cursor *curs) curs->lower = curs->saved_lower; } -/* - * There's some common patterns with scoutfs_manifest_read_items().. may - * want some sharing if it's clean. - */ -static int read_segments(struct super_block *sb, struct compact_cursor *curs) +static int read_segment(struct super_block *sb, struct compact_seg *cseg) { struct scoutfs_segment *seg; - struct compact_seg *cseg; - int ret = 0; - int err; + int ret; - list_for_each_entry(cseg, &curs->csegs, entry) { - seg = scoutfs_seg_submit_read(sb, cseg->segno); - if (IS_ERR(seg)) { - ret = PTR_ERR(seg); - break; - } + if (cseg == NULL || cseg->seg) + return 0; + seg = scoutfs_seg_submit_read(sb, cseg->segno); + if (IS_ERR(seg)) { + ret = PTR_ERR(seg); + } else { cseg->seg = seg; scoutfs_inc_counter(sb, compact_segment_read); + ret = scoutfs_seg_wait(sb, cseg->seg); } - list_for_each_entry(cseg, &curs->csegs, entry) { - if (!cseg->seg) - break; - - err = scoutfs_seg_wait(sb, cseg->seg); - if (err && !ret) - ret = err; - - /* XXX verify segs */ - } - - return ret; -} - -/* - * This is synchronous for now. We're just ensuring that the segments - * are stable on disk so that the references to them in the dirty manifest - * are safe without having to associate dirty segments and manifest entries. - */ -static int write_segments(struct super_block *sb, struct list_head *results) -{ - struct scoutfs_bio_completion comp; - struct compact_seg *cseg; - int ret = 0; - int err; - - scoutfs_bio_init_comp(&comp); - - list_for_each_entry(cseg, results, entry) { - ret = scoutfs_seg_submit_write(sb, cseg->seg, &comp); - if (ret) - break; - scoutfs_inc_counter(sb, compact_segment_write); - } - - err = scoutfs_bio_wait_comp(sb, &comp); - if (err && !ret) - ret = err; + /* XXX verify read segment metadata */ return ret; } @@ -188,22 +169,20 @@ static struct compact_seg *next_spos(struct compact_cursor *curs, /* * Point the caller's key and value kvecs at the next item that should - * be copied from the segment's position in the upper and lower - * segments. We use the item that has the lowest key or the upper if - * they're the same. We advance the cursor past the item that is - * returned. + * be copied from the upper or lower segments. We use the item that has + * the lowest key or the upper if they're the same. We advance the + * cursor past the item that is returned. * * XXX this will get fancier as we get range deletion items and incremental * update items. */ -static bool next_item(struct compact_cursor *curs, - struct kvec *item_key, struct kvec *item_val) +static int next_item(struct super_block *sb, struct compact_cursor *curs, + struct kvec *item_key, struct kvec *item_val) { struct compact_seg *upper = curs->upper; struct compact_seg *lower = curs->lower; SCOUTFS_DECLARE_KVEC(lower_key); SCOUTFS_DECLARE_KVEC(lower_val); - bool found = false; int cmp; int ret; @@ -215,6 +194,10 @@ static bool next_item(struct compact_cursor *curs, } while (lower) { + ret = read_segment(sb, lower); + if (ret) + goto out; + ret = scoutfs_seg_item_kvecs(lower->seg, lower->pos, lower_key, lower_val); if (ret == 0) @@ -224,7 +207,7 @@ static bool next_item(struct compact_cursor *curs, /* we're done if all are empty */ if (!upper && !lower) { - found = false; + ret = 0; goto out; } @@ -250,73 +233,171 @@ static bool next_item(struct compact_cursor *curs, if (cmp >= 0) lower->pos++; - found = true; + ret = 1; out: curs->upper = upper; curs->lower = lower; - return found; + return ret; } /* * Figure out how many items and bytes of keys we're going to try and * compact into the next segment. */ -static void count_items(struct super_block *sb, struct compact_cursor *curs, - u32 *nr_items, u32 *key_bytes) +static int count_items(struct super_block *sb, struct compact_cursor *curs, + u32 *nr_items, u32 *key_bytes) { SCOUTFS_DECLARE_KVEC(item_key); SCOUTFS_DECLARE_KVEC(item_val); u32 total; + int ret; *nr_items = 0; *key_bytes = 0; total = sizeof(struct scoutfs_segment_block); - while (next_item(curs, item_key, item_val)) { + while ((ret = next_item(sb, curs, item_key, item_val)) > 0) { total += sizeof(struct scoutfs_segment_item) + scoutfs_kvec_length(item_key) + scoutfs_kvec_length(item_val); - if (total > SCOUTFS_SEGMENT_SIZE) + if (total > SCOUTFS_SEGMENT_SIZE) { + ret = 0; break; + } (*nr_items)++; (*key_bytes) += scoutfs_kvec_length(item_key); } + + return ret; } -static void compact_items(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_segment *seg, u32 nr_items, - u32 key_bytes) +static int compact_items(struct super_block *sb, struct compact_cursor *curs, + struct scoutfs_segment *seg, u32 nr_items, + u32 key_bytes) { SCOUTFS_DECLARE_KVEC(item_key); SCOUTFS_DECLARE_KVEC(item_val); + int ret; + + ret = next_item(sb, curs, item_key, item_val); + if (ret <= 0) + goto out; - next_item(curs, item_key, item_val); scoutfs_seg_first_item(sb, seg, item_key, item_val, nr_items, key_bytes); - while (--nr_items && next_item(curs, item_key, item_val)) + while (--nr_items) { + ret = next_item(sb, curs, item_key, item_val); + if (ret <= 0) + break; + scoutfs_seg_append_item(sb, seg, item_key, item_val); + } + +out: + return ret; } static int compact_segments(struct super_block *sb, struct compact_cursor *curs, + struct scoutfs_bio_completion *comp, struct list_head *results) { struct scoutfs_segment *seg; struct compact_seg *cseg; + struct compact_seg *upper; + struct compact_seg *lower; + SCOUTFS_DECLARE_KVEC(upper_next); u32 key_bytes; u32 nr_items; int ret; + scoutfs_inc_counter(sb, compact_operations); + for (;;) { + upper = curs->upper; + lower = curs->lower; + + /* + * We can just move the upper segment down a level if it + * doesn't intersect any lower segments. + */ + if (upper && upper->pos == 0 && + (!lower || + scoutfs_kvec_memcmp(upper->last, lower->first) < 0)) { + + cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); + if (!cseg) { + ret = -ENOMEM; + break; + } + + /* + * XXX blah! these csegs are getting + * ridiculous. We should have a robust manifest + * entry iterator that reading and compacting + * can use. + */ + ret = scoutfs_kvec_dup_flatten(cseg->first, + upper->first) ?: + scoutfs_kvec_dup_flatten(cseg->last, upper->last); + if (ret) { + kfree(cseg); + ret = -ENOMEM; + break; + } + + cseg->segno = upper->segno; + cseg->seq = upper->seq; + cseg->level = upper->level + 1; + cseg->seg = upper->seg; + if (cseg->seg) + scoutfs_seg_get(cseg->seg); + list_add_tail(&cseg->entry, results); + + curs->upper = NULL; + upper = NULL; + + scoutfs_inc_counter(sb, compact_segment_moved); + } + + /* we're going to need its next key */ + ret = read_segment(sb, upper); + if (ret) + break; + + /* + * We can skip a lower segment if there's no upper segment + * or the next upper item is past the last in the lower. + */ + if (lower && lower->pos == 0 && + (!upper || + (!scoutfs_seg_item_kvecs(upper->seg, upper->pos, + upper_next, NULL) && + scoutfs_kvec_memcmp(upper_next, lower->last) > 0))) { + + curs->lower = next_spos(curs, lower); + + list_del_init(&lower->entry); + free_cseg(lower); + + scoutfs_inc_counter(sb, compact_segment_skipped); + continue; + } + + ret = read_segment(sb, lower); + if (ret) + break; save_pos(curs); - count_items(sb, curs, &nr_items, &key_bytes); + ret = count_items(sb, curs, &nr_items, &key_bytes); restore_pos(curs); + if (ret < 0) + break; if (nr_items == 0) { ret = 0; @@ -335,31 +416,28 @@ static int compact_segments(struct super_block *sb, break; } + /* csegs will be claned up once they're on the list */ cseg->level = curs->lower_level; cseg->seg = seg; list_add_tail(&cseg->entry, results); - compact_items(sb, curs, seg, nr_items, key_bytes); + ret = compact_items(sb, curs, seg, nr_items, key_bytes); + if (ret < 0) + break; + + /* start a complete segment write now, we'll wait later */ + ret = scoutfs_seg_submit_write(sb, seg, comp); + if (ret) + break; + + scoutfs_inc_counter(sb, compact_segment_written); } return ret; } -static void free_csegs(struct list_head *list) -{ - struct compact_seg *cseg; - struct compact_seg *tmp; - - list_for_each_entry_safe(cseg, tmp, list, entry) { - list_del_init(&cseg->entry); - scoutfs_seg_put(cseg->seg); - scoutfs_kvec_kfree(cseg->first); - kfree(cseg); - } -} - int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, - u64 segno, u64 seq, u8 level) + struct kvec *last, u64 segno, u64 seq, u8 level) { struct compact_cursor *curs = data; struct compact_seg *cseg; @@ -373,7 +451,8 @@ int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, list_add_tail(&cseg->entry, &curs->csegs); - ret = scoutfs_kvec_dup_flatten(cseg->first, first); + ret = scoutfs_kvec_dup_flatten(cseg->first, first) ?: + scoutfs_kvec_dup_flatten(cseg->last, last); if (ret) goto out; @@ -421,7 +500,14 @@ static int update_manifest(struct super_block *sb, struct compact_cursor *curs, } list_for_each_entry(cseg, results, entry) { - ret = scoutfs_seg_manifest_add(sb, cseg->seg, cseg->level); + /* XXX moved upper segments won't have read the segment :P */ + if (cseg->seg) + ret = scoutfs_seg_manifest_add(sb, cseg->seg, + cseg->level); + else + ret = scoutfs_manifest_add(sb, cseg->first, + cseg->last, cseg->segno, + cseg->seq, cseg->level); if (ret) { until = cseg; list_for_each_entry(cseg, results, entry) { @@ -464,35 +550,52 @@ static int free_result_segnos(struct super_block *sb, return ret; } +/* + * The compaction worker tries to make forward progress with compaction + * every time its kicked. It asks the manifest for segments to compact. + * + * If it succeeds in doing work then it kicks itself again to see if there's + * more work to do. + * + * XXX worry about forward progress in the case of errors. + */ static void scoutfs_compact_func(struct work_struct *work) { struct compact_info *ci = container_of(work, struct compact_info, work); struct super_block *sb = ci->sb; struct compact_cursor curs = {{NULL,}}; + struct scoutfs_bio_completion comp; LIST_HEAD(results); int ret; + int err; INIT_LIST_HEAD(&curs.csegs); + scoutfs_bio_init_comp(&comp); ret = scoutfs_manifest_next_compact(sb, (void *)&curs); - if (ret <= 0) + if (list_empty(&curs.csegs)) goto out; - scoutfs_inc_counter(sb, compact_compactions); + ret = compact_segments(sb, &curs, &comp, &results); - ret = read_segments(sb, &curs) ?: - compact_segments(sb, &curs, &results) ?: - write_segments(sb, &results) ?: - update_manifest(sb, &curs, &results); - if (ret) { - free_result_segnos(sb, &results); - } else { + /* always wait for io completion */ + err = scoutfs_bio_wait_comp(sb, &comp); + if (!ret && err) + ret = err; + if (ret) + goto out; + + ret = update_manifest(sb, &curs, &results); + if (ret == 0) { scoutfs_sync_fs(sb, 0); + scoutfs_trans_wake_holders(sb); scoutfs_compact_kick(sb); } out: - free_csegs(&curs.csegs); - free_csegs(&results); + if (ret) + free_result_segnos(sb, &results); + free_cseg_list(&curs.csegs); + free_cseg_list(&results); WARN_ON_ONCE(ret); trace_printk("ret %d\n", ret); diff --git a/kmod/src/compact.h b/kmod/src/compact.h index 9e2778e2..48312d57 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -4,7 +4,7 @@ void scoutfs_compact_kick(struct super_block *sb); int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, - u64 segno, u64 seq, u8 level); + struct kvec *last, u64 segno, u64 seq, u8 level); int scoutfs_compact_setup(struct super_block *sb); void scoutfs_compact_destroy(struct super_block *sb); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 09d48252..e23b859c 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -18,12 +18,14 @@ EXPAND_COUNTER(block_mem_free) \ EXPAND_COUNTER(trans_level0_seg_write) \ EXPAND_COUNTER(manifest_compact_migrate) \ - EXPAND_COUNTER(compact_compactions) \ - EXPAND_COUNTER(compact_segment_read) \ - EXPAND_COUNTER(compact_segment_write) + EXPAND_COUNTER(compact_operations) \ + EXPAND_COUNTER(compact_segment_moved) \ + EXPAND_COUNTER(compact_segment_skipped) \ + EXPAND_COUNTER(compact_segment_read) \ + EXPAND_COUNTER(compact_segment_written) -#define FIRST_COUNTER block_mem_alloc -#define LAST_COUNTER compact_segment_write +#define FIRST_COUNTER alloc_alloc +#define LAST_COUNTER compact_segment_written #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index b67c1f9b..2a96d9da 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -658,14 +658,10 @@ u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level) * where clock hands sweep through each level. The hands wrap much * faster on the higher levels. * - * If the candidate segment doesn't overlap with any higher level - * segments then just move it down a level. + * We add all the segments to the compaction caller's data and let it do + * its thing. It'll allocate and free segments and update the manifest. * - * If the candidate does overlap then we add all the segments to the - * compaction caller's data and let it do its thing. It'll allocate and - * free segments and update the manifest. - * - * Returns 1 if there's compaction work to do, 0 if not, or -errno. + * Returns 0 or -errno. The caller will see if any segments were added. * * XXX this will get a lot more clever: * - ensuring concurrent compactions don't overlap @@ -686,7 +682,6 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) SCOUTFS_DECLARE_KVEC(over_first); SCOUTFS_DECLARE_KVEC(over_last); int level; - int err; int ret; int i; @@ -733,57 +728,21 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) init_ment_keys(ment, ment_first, ment_last); - /* find first overlapping at the next level */ - skey.key = ment_first; - skey.level = level + 1; - skey.seq = 0; - over = scoutfs_treap_lookup(mani->treap, &skey); - if (IS_ERR(over)) { - ret = PTR_ERR(over); - goto out; - } - - /* if there's no overlap we can just move it down a level */ - if (!over) { - ret = scoutfs_manifest_add(sb, ment_first, ment_last, - le64_to_cpu(ment->segno), - le64_to_cpu(ment->seq), - ment->level + 1); - if (ret) - goto out; - - ret = scoutfs_manifest_del(sb, ment_first, - le64_to_cpu(ment->seq), - ment->level); - if (ret) { - err = scoutfs_manifest_del(sb, ment_first, - le64_to_cpu(ment->seq), - ment->level + 1); - BUG_ON(err); - goto out; - } - - scoutfs_inc_counter(sb, manifest_compact_migrate); - goto done; - } - /* add the upper input segment */ - ret = scoutfs_compact_add(sb, data, ment_first, + ret = scoutfs_compact_add(sb, data, ment_first, ment_last, le64_to_cpu(ment->segno), le64_to_cpu(ment->seq), level); if (ret) goto out; - /* add a fanout's worth of lower overlapping segments */ - init_ment_keys(over, over_first, over_last); - for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { - ret = scoutfs_compact_add(sb, data, over_first, - le64_to_cpu(over->segno), - le64_to_cpu(over->seq), level + 1); - if (ret) - goto out; + /* start with the first overlapping at the next level */ + skey.key = ment_first; + skey.level = level + 1; + skey.seq = 0; + over = scoutfs_treap_lookup(mani->treap, &skey); - over = scoutfs_treap_next(mani->treap, over); + /* and add a fanout's worth of lower overlapping segments */ + for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { if (IS_ERR(over)) { ret = PTR_ERR(over); goto out; @@ -792,18 +751,26 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) break; init_ment_keys(over, over_first, over_last); + if (scoutfs_kvec_cmp_overlap(ment_first, ment_last, over_first, over_last) != 0) break; + + ret = scoutfs_compact_add(sb, data, over_first, over_last, + le64_to_cpu(over->segno), + le64_to_cpu(over->seq), level + 1); + if (ret) + goto out; + + over = scoutfs_treap_next(mani->treap, over); } -done: /* record the next key to start from, not exact */ scoutfs_kvec_init_key(mani->compact_keys[level]); scoutfs_kvec_memcpy_truncate(mani->compact_keys[level], ment_last); scoutfs_kvec_be_inc(mani->compact_keys[level]); - ret = 1; + ret = 0; out: up_write(&mani->rwsem); return ret; From 8a302609f243e6a6da0a5d434fc46d1b56e14a9b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Jan 2017 14:46:37 -0800 Subject: [PATCH 204/920] Add some item cache/range counters Signed-off-by: Zach Brown --- kmod/src/counters.h | 10 ++++++++-- kmod/src/item.c | 45 +++++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e23b859c..e01c2e1b 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -22,10 +22,16 @@ EXPAND_COUNTER(compact_segment_moved) \ EXPAND_COUNTER(compact_segment_skipped) \ EXPAND_COUNTER(compact_segment_read) \ - EXPAND_COUNTER(compact_segment_written) + EXPAND_COUNTER(compact_segment_written) \ + EXPAND_COUNTER(item_create) \ + EXPAND_COUNTER(item_lookup_hit) \ + EXPAND_COUNTER(item_lookup_miss) \ + EXPAND_COUNTER(item_range_hit) \ + EXPAND_COUNTER(item_range_miss) \ + EXPAND_COUNTER(item_range_insert) #define FIRST_COUNTER alloc_alloc -#define LAST_COUNTER compact_segment_written +#define LAST_COUNTER item_range_insert #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/item.c b/kmod/src/item.c index 69204518..b5bb2c5e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -22,6 +22,7 @@ #include "manifest.h" #include "item.h" #include "seg.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -102,12 +103,21 @@ static struct cached_item *walk_items(struct rb_root *root, struct kvec *key, return NULL; } -static struct cached_item *find_item(struct rb_root *root, struct kvec *key) +static struct cached_item *find_item(struct super_block *sb, + struct rb_root *root, struct kvec *key) { struct cached_item *prev; struct cached_item *next; + struct cached_item *item; - return walk_items(root, key, &prev, &next); + item = walk_items(root, key, &prev, &next); + + if (item) + scoutfs_inc_counter(sb, item_lookup_hit); + else + scoutfs_inc_counter(sb, item_lookup_miss); + + return item; } static struct cached_item *next_item(struct rb_root *root, struct kvec *key) @@ -252,8 +262,8 @@ static int insert_item(struct rb_root *root, struct cached_item *ins) * instead in an uncached hole. end is set to the start of the next * cached range. */ -static bool check_range(struct rb_root *root, struct kvec *key, - struct kvec *end) +static bool check_range(struct super_block *sb, struct rb_root *root, + struct kvec *key, struct kvec *end) { struct rb_node *node = root->rb_node; struct cached_range *next = NULL; @@ -272,6 +282,7 @@ static bool check_range(struct rb_root *root, struct kvec *key, node = node->rb_right; } else { scoutfs_kvec_memcpy_truncate(end, rng->end); + scoutfs_inc_counter(sb, item_range_hit); return true; } } @@ -281,6 +292,7 @@ static bool check_range(struct rb_root *root, struct kvec *key, else scoutfs_kvec_set_max_key(end); + scoutfs_inc_counter(sb, item_range_miss); return false; } @@ -301,7 +313,8 @@ static void free_range(struct cached_range *rng) * We're responsible for the ins allocation. We free it if we don't * insert it in the tree. */ -static void insert_range(struct rb_root *root, struct cached_range *ins) +static void insert_range(struct super_block *sb, struct rb_root *root, + struct cached_range *ins) { struct cached_range *rng; struct rb_node *parent; @@ -310,6 +323,8 @@ static void insert_range(struct rb_root *root, struct cached_range *ins) int end_cmp; int cmp; + scoutfs_inc_counter(sb, item_range_insert); + restart: parent = NULL; node = &root->rb_node; @@ -379,10 +394,10 @@ int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, spin_lock_irqsave(&cac->lock, flags); - item = find_item(&cac->items, key); + item = find_item(sb, &cac->items, key); if (item) ret = scoutfs_kvec_memcpy(val, item->val); - else if (check_range(&cac->ranges, key, end)) + else if (check_range(sb, &cac->ranges, key, end)) ret = -ENOENT; else ret = -ENODATA; @@ -465,7 +480,7 @@ int scoutfs_item_next(struct super_block *sb, struct kvec *key, scoutfs_kvec_init_key(range_end); /* see if we have a usable item in cache and before last */ - cached = check_range(&cac->ranges, key, range_end); + cached = check_range(sb, &cac->ranges, key, range_end); if (cached && (item = next_item(&cac->items, key)) && scoutfs_kvec_memcmp(item->key, range_end) <= 0 && @@ -639,8 +654,10 @@ int scoutfs_item_create(struct super_block *sb, struct kvec *key, spin_lock_irqsave(&cac->lock, flags); ret = insert_item(&cac->items, item); - if (!ret) + if (!ret) { + scoutfs_inc_counter(sb, item_create); mark_item_dirty(cac, item); + } spin_unlock_irqrestore(&cac->lock, flags); if (ret) @@ -716,7 +733,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, spin_lock_irqsave(&cac->lock, flags); - insert_range(&cac->ranges, rng); + insert_range(sb, &cac->ranges, rng); list_for_each_entry_safe(item, tmp, list, entry) { list_del(&item->entry); @@ -766,11 +783,11 @@ int scoutfs_item_dirty(struct super_block *sb, struct kvec *key) spin_lock_irqsave(&cac->lock, flags); - item = find_item(&cac->items, key); + item = find_item(sb, &cac->items, key); if (item) { mark_item_dirty(cac, item); ret = 0; - } else if (check_range(&cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -821,13 +838,13 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, spin_lock_irqsave(&cac->lock, flags); - item = find_item(&cac->items, key); + item = find_item(sb, &cac->items, key); if (item) { clear_item_dirty(cac, item); scoutfs_kvec_swap(up_val, item->val); mark_item_dirty(cac, item); ret = 0; - } else if (check_range(&cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; } else { ret = -ENODATA; From 2bc16172808f4fc01607d74976ff31114fc9e1c6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Jan 2017 13:30:04 -0800 Subject: [PATCH 205/920] Use contiguous key struct instead of kvecs Using kvecs for keys seemed like a good idea because there were a few uses that had keys in fragmented memory: dirent keys made up of an on-stack struct and the file name in the dentry, and keys straddling the pages that make up a cached segment. But it hasn't worked out very well. The code to perform ops on keys by iterating over vectors is pretty fiddly. And the raw kvecs only describe the actively referenced key, they know nothing about the total size of the buffer that the key resides in. Some ops can't check that they're not clobbering things, they're relying on callers not to mess up. And critically, the kvec iteration's become a bottleneck. It turns out that comparing keys is a very hot path in the item cache. All the code to initialize and iterate over two key vectors adds up when each high level fs operation is a few tree descents and each tree descent is a bunch of compares. So let's back off and have a specific struct for tracking keys that are stored in contiguous memory regions. Users ensure that keys are contiguous. The code ends up being a lot clearer, code now can see how big the full key buffer is, and the rbtree node comparison fast path is now just a memcmp. Almost all of the changes in the patch are mechanical semantic changes involving types, function names, args, and occasionaly slightly different return conventions. A slightly more involved change is that now dirent key users have to manage an allocated contiguous key with a copy of the path from the dentry. Item reading is now a little more clever about calculating the greatest range it can cache by initially walking all the segments instead of trying to do it as it runs out of items in each segment. The largest meaningful change is that now keys can't straddle page boundaries in memory which means they can't cross block boundaries in the segment. We align key offsets to the next block as we write keys to segments that would have straddled a block. We then also have to account for that padding when building segments. We add a helper that calculates if a given number of items will fit in a segment which is used by item dirtying, segment writing, and compaction. I left the tracepoint formatting for another patch. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/compact.c | 126 +++++++++++--------- kmod/src/compact.h | 6 +- kmod/src/dir.c | 88 +++++++++----- kmod/src/inode.c | 33 +++-- kmod/src/item.c | 285 +++++++++++++++++++++++--------------------- kmod/src/item.h | 39 +++--- kmod/src/key.c | 92 ++++++++++++++ kmod/src/key.h | 114 ++++++++++++++++++ kmod/src/manifest.c | 283 ++++++++++++++++++++++--------------------- kmod/src/manifest.h | 19 +-- kmod/src/seg.c | 110 ++++++++++++----- kmod/src/seg.h | 13 +- kmod/src/trans.c | 41 +++++-- 14 files changed, 794 insertions(+), 459 deletions(-) create mode 100644 kmod/src/key.c diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 798ba33a..828cce93 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o block.o btree.o buddy.o compact.o counters.o crc.o \ - dir.o filerw.o kvec.o inode.o ioctl.o item.o manifest.o msg.o \ - name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o + dir.o filerw.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ + msg.o name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 14c35c98..b6e0965d 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -70,8 +70,8 @@ struct compact_seg { u64 segno; u64 seq; u8 level; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf *first; + struct scoutfs_key_buf *last; struct scoutfs_segment *seg; int pos; int saved_pos; @@ -92,25 +92,45 @@ struct compact_cursor { struct compact_seg *saved_lower; }; -static void free_cseg(struct compact_seg *cseg) +static void free_cseg(struct super_block *sb, struct compact_seg *cseg) { WARN_ON_ONCE(!list_empty(&cseg->entry)); scoutfs_seg_put(cseg->seg); - scoutfs_kvec_kfree(cseg->first); - scoutfs_kvec_kfree(cseg->last); + scoutfs_key_free(sb, cseg->first); + scoutfs_key_free(sb, cseg->last); kfree(cseg); } -static void free_cseg_list(struct list_head *list) +static struct compact_seg *alloc_cseg(struct super_block *sb, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) +{ + struct compact_seg *cseg; + + cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); + if (cseg) { + INIT_LIST_HEAD(&cseg->entry); + cseg->first = scoutfs_key_dup(sb, first); + cseg->last = scoutfs_key_dup(sb, last); + if (!cseg->first || !cseg->last) { + free_cseg(sb, cseg); + cseg = NULL; + } + } + + return cseg; +} + +static void free_cseg_list(struct super_block *sb, struct list_head *list) { struct compact_seg *cseg; struct compact_seg *tmp; list_for_each_entry_safe(cseg, tmp, list, entry) { list_del_init(&cseg->entry); - free_cseg(cseg); + free_cseg(sb, cseg); } } @@ -177,18 +197,18 @@ static struct compact_seg *next_spos(struct compact_cursor *curs, * update items. */ static int next_item(struct super_block *sb, struct compact_cursor *curs, - struct kvec *item_key, struct kvec *item_val) + struct scoutfs_key_buf *item_key, struct kvec *item_val) { struct compact_seg *upper = curs->upper; struct compact_seg *lower = curs->lower; - SCOUTFS_DECLARE_KVEC(lower_key); + struct scoutfs_key_buf lower_key; SCOUTFS_DECLARE_KVEC(lower_val); int cmp; int ret; if (upper) { - ret = scoutfs_seg_item_kvecs(upper->seg, upper->pos, - item_key, item_val); + ret = scoutfs_seg_item_ptrs(upper->seg, upper->pos, + item_key, item_val); if (ret < 0) upper = NULL; } @@ -198,8 +218,8 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, if (ret) goto out; - ret = scoutfs_seg_item_kvecs(lower->seg, lower->pos, - lower_key, lower_val); + ret = scoutfs_seg_item_ptrs(lower->seg, lower->pos, + &lower_key, lower_val); if (ret == 0) break; lower = next_spos(curs, lower); @@ -217,14 +237,14 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, * > 0: return lower, advance lower */ if (upper && lower) - cmp = scoutfs_kvec_memcmp(item_key, lower_key); + cmp = scoutfs_key_compare(item_key, &lower_key); else if (upper) cmp = -1; else cmp = 1; if (cmp > 0) { - scoutfs_kvec_clone(item_key, lower_key); + scoutfs_key_clone(item_key, &lower_key); scoutfs_kvec_clone(item_val, lower_val); } @@ -248,28 +268,27 @@ out: static int count_items(struct super_block *sb, struct compact_cursor *curs, u32 *nr_items, u32 *key_bytes) { - SCOUTFS_DECLARE_KVEC(item_key); + struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); - u32 total; + u32 items = 0; + u32 keys = 0; + u32 vals = 0; int ret; *nr_items = 0; *key_bytes = 0; - total = sizeof(struct scoutfs_segment_block); - while ((ret = next_item(sb, curs, item_key, item_val)) > 0) { + while ((ret = next_item(sb, curs, &item_key, item_val)) > 0) { - total += sizeof(struct scoutfs_segment_item) + - scoutfs_kvec_length(item_key) + - scoutfs_kvec_length(item_val); + items++; + keys += item_key.key_len; + vals += scoutfs_kvec_length(item_val); - if (total > SCOUTFS_SEGMENT_SIZE) { - ret = 0; + if (!scoutfs_seg_fits_single(items, keys, vals)) break; - } - (*nr_items)++; - (*key_bytes) += scoutfs_kvec_length(item_key); + *nr_items = items; + *key_bytes = keys; } return ret; @@ -279,23 +298,23 @@ static int compact_items(struct super_block *sb, struct compact_cursor *curs, struct scoutfs_segment *seg, u32 nr_items, u32 key_bytes) { - SCOUTFS_DECLARE_KVEC(item_key); + struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); int ret; - ret = next_item(sb, curs, item_key, item_val); + ret = next_item(sb, curs, &item_key, item_val); if (ret <= 0) goto out; - scoutfs_seg_first_item(sb, seg, item_key, item_val, + scoutfs_seg_first_item(sb, seg, &item_key, item_val, nr_items, key_bytes); while (--nr_items) { - ret = next_item(sb, curs, item_key, item_val); + ret = next_item(sb, curs, &item_key, item_val); if (ret <= 0) break; - scoutfs_seg_append_item(sb, seg, item_key, item_val); + scoutfs_seg_append_item(sb, seg, &item_key, item_val); } out: @@ -307,11 +326,11 @@ static int compact_segments(struct super_block *sb, struct scoutfs_bio_completion *comp, struct list_head *results) { + struct scoutfs_key_buf upper_next; struct scoutfs_segment *seg; struct compact_seg *cseg; struct compact_seg *upper; struct compact_seg *lower; - SCOUTFS_DECLARE_KVEC(upper_next); u32 key_bytes; u32 nr_items; int ret; @@ -328,13 +347,7 @@ static int compact_segments(struct super_block *sb, */ if (upper && upper->pos == 0 && (!lower || - scoutfs_kvec_memcmp(upper->last, lower->first) < 0)) { - - cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); - if (!cseg) { - ret = -ENOMEM; - break; - } + scoutfs_key_compare(upper->last, lower->first) < 0)) { /* * XXX blah! these csegs are getting @@ -342,11 +355,8 @@ static int compact_segments(struct super_block *sb, * entry iterator that reading and compacting * can use. */ - ret = scoutfs_kvec_dup_flatten(cseg->first, - upper->first) ?: - scoutfs_kvec_dup_flatten(cseg->last, upper->last); - if (ret) { - kfree(cseg); + cseg = alloc_cseg(sb, upper->first, upper->last); + if (!cseg) { ret = -ENOMEM; break; } @@ -376,14 +386,14 @@ static int compact_segments(struct super_block *sb, */ if (lower && lower->pos == 0 && (!upper || - (!scoutfs_seg_item_kvecs(upper->seg, upper->pos, - upper_next, NULL) && - scoutfs_kvec_memcmp(upper_next, lower->last) > 0))) { + (!scoutfs_seg_item_ptrs(upper->seg, upper->pos, + &upper_next, NULL) && + scoutfs_key_compare(&upper_next, lower->last) > 0))) { curs->lower = next_spos(curs, lower); list_del_init(&lower->entry); - free_cseg(lower); + free_cseg(sb, lower); scoutfs_inc_counter(sb, compact_segment_skipped); continue; @@ -404,6 +414,7 @@ static int compact_segments(struct super_block *sb, break; } + /* no cseg keys, manifest update uses seg item keys */ cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); if (!cseg) { ret = -ENOMEM; @@ -436,14 +447,16 @@ static int compact_segments(struct super_block *sb, return ret; } -int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level) +int scoutfs_compact_add(struct super_block *sb, void *data, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, + u8 level) { struct compact_cursor *curs = data; struct compact_seg *cseg; int ret; - cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); + cseg = alloc_cseg(sb, first, last); if (!cseg) { ret = -ENOMEM; goto out; @@ -451,11 +464,6 @@ int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, list_add_tail(&cseg->entry, &curs->csegs); - ret = scoutfs_kvec_dup_flatten(cseg->first, first) ?: - scoutfs_kvec_dup_flatten(cseg->last, last); - if (ret) - goto out; - cseg->segno = segno; cseg->seq = seq; cseg->level = level; @@ -594,8 +602,8 @@ static void scoutfs_compact_func(struct work_struct *work) out: if (ret) free_result_segnos(sb, &results); - free_cseg_list(&curs.csegs); - free_cseg_list(&results); + free_cseg_list(sb, &curs.csegs); + free_cseg_list(sb, &results); WARN_ON_ONCE(ret); trace_printk("ret %d\n", ret); diff --git a/kmod/src/compact.h b/kmod/src/compact.h index 48312d57..5241ff11 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -3,8 +3,10 @@ void scoutfs_compact_kick(struct super_block *sb); -int scoutfs_compact_add(struct super_block *sb, void *data, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level); +int scoutfs_compact_add(struct super_block *sb, void *data, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, + u8 level); int scoutfs_compact_setup(struct super_block *sb); void scoutfs_compact_destroy(struct super_block *sb); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index f979fec6..c67675b5 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -97,13 +97,32 @@ static unsigned int dentry_type(unsigned int type) return DT_UNKNOWN; } +static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, + struct inode *dir, + struct dentry *dentry) +{ + struct scoutfs_dirent_key *dkey; + struct scoutfs_key_buf *key; + + key = scoutfs_key_alloc(sb, offsetof(struct scoutfs_dirent_key, + name[dentry->d_name.len])); + if (key) { + dkey = key->data; + dkey->type = SCOUTFS_DIRENT_KEY; + dkey->ino = cpu_to_be64(scoutfs_ino(dir)); + memcpy(dkey->name, (void *)dentry->d_name.name, + dentry->d_name.len); + } + + return key; +} + static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct super_block *sb = dir->i_sb; - struct scoutfs_dirent_key dkey; + struct scoutfs_key_buf *key = NULL; struct scoutfs_dirent dent; - SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); struct inode *inode; u64 ino = 0; @@ -114,10 +133,11 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } - dkey.type = SCOUTFS_DIRENT_KEY; - dkey.ino = cpu_to_be64(scoutfs_ino(dir)); - scoutfs_kvec_init(key, &dkey, sizeof(dkey), - (void *)dentry->d_name.name, dentry->d_name.len); + key = alloc_dirent_key(sb, dir, dentry); + if (!key) { + ret = -ENOMEM; + goto out; + } scoutfs_kvec_init(val, &dent, sizeof(dent)); @@ -137,6 +157,8 @@ out: else inode = scoutfs_iget(sb, ino); + scoutfs_key_free(sb, key); + return d_splice_alias(inode, dentry); } @@ -162,6 +184,17 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) return 1; } +static void init_readdir_key(struct scoutfs_key_buf *key, + struct scoutfs_readdir_key *rkey, + struct inode *inode, loff_t pos) +{ + rkey->type = SCOUTFS_READDIR_KEY; + rkey->ino = cpu_to_be64(scoutfs_ino(inode)); + rkey->pos = cpu_to_be64(pos); + + scoutfs_key_init(key, rkey, sizeof(struct scoutfs_readdir_key)); +} + /* * readdir simply iterates over the dirent items for the dir inode and * uses their offset as the readdir position. @@ -174,10 +207,10 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; struct scoutfs_dirent *dent; + struct scoutfs_key_buf key; + struct scoutfs_key_buf last_key; struct scoutfs_readdir_key rkey; struct scoutfs_readdir_key last_rkey; - SCOUTFS_DECLARE_KVEC(key); - SCOUTFS_DECLARE_KVEC(last_key); SCOUTFS_DECLARE_KVEC(val); unsigned int item_len; unsigned int name_len; @@ -187,15 +220,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (!dir_emit_dots(file, dirent, filldir)) return 0; - rkey.type = SCOUTFS_READDIR_KEY; - rkey.ino = cpu_to_be64(scoutfs_ino(inode)); - /* pos set in each loop */ - scoutfs_kvec_init(key, &rkey, sizeof(rkey)); - - last_rkey.type = SCOUTFS_READDIR_KEY; - last_rkey.ino = cpu_to_be64(scoutfs_ino(inode)); - last_rkey.pos = cpu_to_be64(SCOUTFS_DIRENT_LAST_POS); - scoutfs_kvec_init(last_key, &last_rkey, sizeof(last_rkey)); + init_readdir_key(&last_key, &last_rkey, inode, SCOUTFS_DIRENT_LAST_POS); item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); dent = kmalloc(item_len, GFP_KERNEL); @@ -203,9 +228,10 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) return -ENOMEM; for (;;) { - rkey.pos = cpu_to_be64(file->f_pos); + init_readdir_key(&key, &rkey, inode, file->f_pos); + scoutfs_kvec_init(val, dent, item_len); - ret = scoutfs_item_next_same_min(sb, key, last_key, val, + ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, offsetof(struct scoutfs_dirent, name[1])); if (ret < 0) { if (ret == -ENOENT) @@ -261,9 +287,8 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, struct inode *inode) { struct super_block *sb = dir->i_sb; - struct scoutfs_dirent_key dkey; + struct scoutfs_key_buf *key; struct scoutfs_dirent dent; - SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); int ret; @@ -275,10 +300,9 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, return ret; /* dirent item for lookup */ - dkey.type = SCOUTFS_DIRENT_KEY; - dkey.ino = cpu_to_be64(scoutfs_ino(dir)); - scoutfs_kvec_init(key, &dkey, sizeof(dkey), - (void *)dentry->d_name.name, dentry->d_name.len); + key = alloc_dirent_key(sb, dir, dentry); + if (!key) + return -ENOMEM; dent.ino = cpu_to_le64(scoutfs_ino(inode)); dent.type = mode_to_type(inode->i_mode); @@ -323,6 +347,7 @@ out_dent: } #endif + scoutfs_key_free(sb, key); return ret; } @@ -423,8 +448,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_dirent_key dkey; - SCOUTFS_DECLARE_KVEC(key); + struct scoutfs_key_buf *key = NULL; int ret = 0; /* will need to add deletion items */ @@ -443,10 +467,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto out; /* XXX same items as add_entry_items */ - dkey.type = SCOUTFS_DIRENT_KEY; - dkey.ino = cpu_to_be64(scoutfs_ino(dir)); - scoutfs_kvec_init(key, &dkey, sizeof(dkey), - (void *)dentry->d_name.name, dentry->d_name.len); + key = alloc_dirent_key(sb, dir, dentry); + if (!key) { + ret = -ENOMEM; + goto out; + } ret = scoutfs_item_delete(sb, key); if (ret) @@ -478,6 +503,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(dir); out: + scoutfs_key_free(sb, key); scoutfs_release_trans(sb); return ret; } diff --git a/kmod/src/inode.c b/kmod/src/inode.c index c34babf7..48864f94 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -127,26 +127,28 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->data_version = le64_to_cpu(cinode->data_version); } -static void set_inode_key(struct scoutfs_inode_key *ikey, u64 ino) +static void init_inode_key(struct scoutfs_key_buf *key, + struct scoutfs_inode_key *ikey, u64 ino) { ikey->type = SCOUTFS_INODE_KEY; ikey->ino = cpu_to_be64(ino); + + scoutfs_key_init(key, ikey, sizeof(struct scoutfs_inode_key)); } static int scoutfs_read_locked_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; + struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); int ret; - set_inode_key(&ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + init_inode_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(sinode)); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); if (ret == 0) load_inode(inode, &sinode); @@ -269,16 +271,15 @@ int scoutfs_dirty_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; + struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(key); int ret; store_inode(&sinode, inode); - set_inode_key(&ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + init_inode_key(&key, &ikey, scoutfs_ino(inode)); - ret = scoutfs_item_dirty(sb, key); + ret = scoutfs_item_dirty(sb, &key); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -297,18 +298,17 @@ void scoutfs_update_inode_item(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; + struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); int err; store_inode(&sinode, inode); - set_inode_key(&ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + init_inode_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, key, val); + err = scoutfs_item_update(sb, &key, val); BUG_ON(err); trace_scoutfs_update_inode(inode); @@ -388,8 +388,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, { struct scoutfs_inode_info *ci; struct scoutfs_inode_key ikey; + struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(key); SCOUTFS_DECLARE_KVEC(val); struct inode *inode; u64 ino; @@ -419,11 +419,10 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, set_inode_ops(inode); store_inode(&sinode, inode); - set_inode_key(&ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(key, &ikey, sizeof(ikey)); + init_inode_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_create(sb, key, val); + ret = scoutfs_item_create(sb, &key, val); if (ret) { iput(inode); return ERR_PTR(ret); diff --git a/kmod/src/item.c b/kmod/src/item.c index b5bb2c5e..85c4942b 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -59,22 +59,24 @@ struct cached_item { }; long dirty; - SCOUTFS_DECLARE_KVEC(key); + struct scoutfs_key_buf *key; + SCOUTFS_DECLARE_KVEC(val); }; struct cached_range { struct rb_node node; - SCOUTFS_DECLARE_KVEC(start); - SCOUTFS_DECLARE_KVEC(end); + struct scoutfs_key_buf *start; + struct scoutfs_key_buf *end; }; /* * Walk the item rbtree and return the item found and the next and * prev items. */ -static struct cached_item *walk_items(struct rb_root *root, struct kvec *key, +static struct cached_item *walk_items(struct rb_root *root, + struct scoutfs_key_buf *key, struct cached_item **prev, struct cached_item **next) { @@ -88,7 +90,7 @@ static struct cached_item *walk_items(struct rb_root *root, struct kvec *key, while (node) { item = container_of(node, struct cached_item, node); - cmp = scoutfs_kvec_memcmp(key, item->key); + cmp = scoutfs_key_compare(key, item->key); if (cmp < 0) { *next = item; node = node->rb_left; @@ -104,7 +106,8 @@ static struct cached_item *walk_items(struct rb_root *root, struct kvec *key, } static struct cached_item *find_item(struct super_block *sb, - struct rb_root *root, struct kvec *key) + struct rb_root *root, + struct scoutfs_key_buf *key) { struct cached_item *prev; struct cached_item *next; @@ -120,7 +123,8 @@ static struct cached_item *find_item(struct super_block *sb, return item; } -static struct cached_item *next_item(struct rb_root *root, struct kvec *key) +static struct cached_item *next_item(struct rb_root *root, + struct scoutfs_key_buf *key) { struct cached_item *prev; struct cached_item *next; @@ -234,7 +238,7 @@ static int insert_item(struct rb_root *root, struct cached_item *ins) parent = *node; item = container_of(*node, struct cached_item, node); - cmp = scoutfs_kvec_memcmp(ins->key, item->key); + cmp = scoutfs_key_compare(ins->key, item->key); if (cmp < 0) { if (ins->dirty) item->dirty |= LEFT_DIRTY; @@ -263,7 +267,8 @@ static int insert_item(struct rb_root *root, struct cached_item *ins) * cached range. */ static bool check_range(struct super_block *sb, struct rb_root *root, - struct kvec *key, struct kvec *end) + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end) { struct rb_node *node = root->rb_node; struct cached_range *next = NULL; @@ -273,34 +278,34 @@ static bool check_range(struct super_block *sb, struct rb_root *root, while (node) { rng = container_of(node, struct cached_range, node); - cmp = scoutfs_kvec_cmp_overlap(key, key, - rng->start, rng->end); + cmp = scoutfs_key_compare_ranges(key, key, + rng->start, rng->end); if (cmp < 0) { next = rng; node = node->rb_left; } else if (cmp > 0) { node = node->rb_right; } else { - scoutfs_kvec_memcpy_truncate(end, rng->end); + scoutfs_key_copy(end, rng->end); scoutfs_inc_counter(sb, item_range_hit); return true; } } if (next) - scoutfs_kvec_memcpy_truncate(end, next->start); + scoutfs_key_copy(end, next->start); else - scoutfs_kvec_set_max_key(end); + scoutfs_key_set_max(end); scoutfs_inc_counter(sb, item_range_miss); return false; } -static void free_range(struct cached_range *rng) +static void free_range(struct super_block *sb, struct cached_range *rng) { if (!IS_ERR_OR_NULL(rng)) { - scoutfs_kvec_kfree(rng->start); - scoutfs_kvec_kfree(rng->end); + scoutfs_key_free(sb, rng->start); + scoutfs_key_free(sb, rng->end); kfree(rng); } } @@ -332,8 +337,8 @@ restart: parent = *node; rng = container_of(*node, struct cached_range, node); - cmp = scoutfs_kvec_cmp_overlap(ins->start, ins->end, - rng->start, rng->end); + cmp = scoutfs_key_compare_ranges(ins->start, ins->end, + rng->start, rng->end); /* simple iteration until we overlap */ if (cmp < 0) { node = &(*node)->rb_left; @@ -343,24 +348,24 @@ restart: continue; } - start_cmp = scoutfs_kvec_memcmp(ins->start, rng->start); - end_cmp = scoutfs_kvec_memcmp(ins->end, rng->end); + start_cmp = scoutfs_key_compare(ins->start, rng->start); + end_cmp = scoutfs_key_compare(ins->end, rng->end); /* free our insertion if we're entirely within an existing */ if (start_cmp >= 0 && end_cmp <= 0) { - free_range(ins); + free_range(sb, ins); return; } /* expand to cover partial overlap before freeing */ if (start_cmp < 0 && end_cmp < 0) - scoutfs_kvec_swap(ins->end, rng->end); + swap(ins->end, rng->end); else if (start_cmp > 0 && end_cmp > 0) - scoutfs_kvec_swap(ins->start, rng->start); + swap(ins->start, rng->start); /* remove and free all overlaps and restart the descent */ rb_erase(&rng->node, root); - free_range(rng); + free_range(sb, rng); goto restart; } @@ -373,25 +378,25 @@ restart: * value vector. The amount of bytes copied is returned which can be 0 * or truncated if the caller's buffer isn't big enough. */ -int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - SCOUTFS_DECLARE_KVEC(end); + struct scoutfs_key_buf *end; struct cached_item *item; unsigned long flags; int ret; - trace_scoutfs_item_lookup(sb, key, val); +// trace_scoutfs_item_lookup(sb, key, val); - ret = scoutfs_kvec_alloc_key(end); - if (ret) + end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!end) { + ret = -ENOMEM; goto out; + } do { - scoutfs_kvec_init_key(end); - spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); @@ -407,7 +412,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - scoutfs_kvec_kfree(end); + scoutfs_key_free(sb, end); out: trace_printk("ret %d\n", ret); return ret; @@ -423,8 +428,9 @@ out: * * Returns 0 or -errno. */ -int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, - struct kvec *val, int size) +int scoutfs_item_lookup_exact(struct super_block *sb, + struct scoutfs_key_buf *key, struct kvec *val, + int size) { int ret; @@ -444,55 +450,51 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, * -ENOENT is returned if there are no items between the given and last * keys. * - * The next item's key is copied to the caller's key. -ENOBUFS is - * returned if the item's key didn't fit in the caller's key. + * The next item's key is copied to the caller's key. The caller is + * responsible for dealing with key lengths and truncation. * * The next item's value is copied into the callers value. The number * of value bytes copied is returned. The copied value can be truncated * by the caller's value buffer length. */ -int scoutfs_item_next(struct super_block *sb, struct kvec *key, - struct kvec *last, struct kvec *val) +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - SCOUTFS_DECLARE_KVEC(read_start); - SCOUTFS_DECLARE_KVEC(read_end); - SCOUTFS_DECLARE_KVEC(range_end); + struct scoutfs_key_buf *read_start = NULL; + struct scoutfs_key_buf *read_end = NULL; + struct scoutfs_key_buf *range_end = NULL; struct cached_item *item; unsigned long flags; bool cached; int ret; /* convenience to avoid searching if caller iterates past their last */ - if (scoutfs_kvec_length(key) > scoutfs_kvec_length(last)) { + if (scoutfs_key_compare(key, last) > 0) { ret = -ENOENT; goto out; } - ret = scoutfs_kvec_alloc_key(range_end); - if (ret) + read_start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + read_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!read_start || !read_end || !range_end) { + ret = -ENOMEM; goto out; + } spin_lock_irqsave(&cac->lock, flags); for(;;) { - scoutfs_kvec_init_key(range_end); - /* see if we have a usable item in cache and before last */ cached = check_range(sb, &cac->ranges, key, range_end); if (cached && (item = next_item(&cac->items, key)) && - scoutfs_kvec_memcmp(item->key, range_end) <= 0 && - scoutfs_kvec_memcmp(item->key, last) <= 0) { + scoutfs_key_compare(item->key, range_end) <= 0 && + scoutfs_key_compare(item->key, last) <= 0) { - if (scoutfs_kvec_length(item->key) > - scoutfs_kvec_length(key)) { - ret = -ENOBUFS; - break; - } - - scoutfs_kvec_memcpy_truncate(key, item->key); + scoutfs_key_copy(key, item->key); if (val) ret = scoutfs_kvec_memcpy(val, item->val); else @@ -502,13 +504,13 @@ int scoutfs_item_next(struct super_block *sb, struct kvec *key, if (!cached) { /* missing cache starts at key */ - scoutfs_kvec_clone(read_start, key); - scoutfs_kvec_clone(read_end, range_end); + scoutfs_key_copy(read_start, key); + scoutfs_key_copy(read_end, range_end); - } else if (scoutfs_kvec_memcmp(range_end, last) < 0) { + } else if (scoutfs_key_compare(range_end, last) < 0) { /* missing cache starts at range_end */ - scoutfs_kvec_clone(read_start, range_end); - scoutfs_kvec_clone(read_end, last); + scoutfs_key_copy(read_start, range_end); + scoutfs_key_copy(read_end, last); } else { /* no items and we have cache between key and last */ @@ -526,9 +528,11 @@ int scoutfs_item_next(struct super_block *sb, struct kvec *key, } spin_unlock_irqrestore(&cac->lock, flags); - - scoutfs_kvec_kfree(range_end); out: + scoutfs_key_free(sb, read_start); + scoutfs_key_free(sb, read_end); + scoutfs_key_free(sb, range_end); + trace_printk("ret %d\n", ret); return ret; } @@ -539,10 +543,12 @@ out: * size mismatches as a sign of corruption. A found key larger than the * found key buffer gives -ENOBUFS and is a sign of corruption. */ -int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, - struct kvec *last, struct kvec *val, int len) +int scoutfs_item_next_same_min(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, + struct kvec *val, int len) { - int key_len = scoutfs_kvec_length(key); + int key_len = key->key_len; int ret; trace_printk("key len %u min val len %d\n", key_len, len); @@ -551,8 +557,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, return -EINVAL; ret = scoutfs_item_next(sb, key, last, val); - if (ret == -ENOBUFS || - (ret >= 0 && (scoutfs_kvec_length(key) != key_len || ret < len))) + if (ret >= 0 && (key->key_len != key_len || ret < len)) ret = -EIO; trace_printk("ret %d\n", ret); @@ -560,10 +565,10 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, return ret; } -static void free_item(struct cached_item *item) +static void free_item(struct super_block *sb, struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { - scoutfs_kvec_kfree(item->key); + scoutfs_key_free(sb, item->key); scoutfs_kvec_kfree(item->val); kfree(item); } @@ -591,7 +596,7 @@ static void mark_item_dirty(struct item_cache *cac, item->dirty |= ITEM_DIRTY; cac->nr_dirty_items++; - cac->dirty_key_bytes += scoutfs_kvec_length(item->key); + cac->dirty_key_bytes += item->key->key_len; cac->dirty_val_bytes += scoutfs_kvec_length(item->val); update_dirty_parents(item); @@ -608,7 +613,7 @@ static void clear_item_dirty(struct item_cache *cac, item->dirty &= ~ITEM_DIRTY; cac->nr_dirty_items--; - cac->dirty_key_bytes -= scoutfs_kvec_length(item->key); + cac->dirty_key_bytes -= item->key->key_len; cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || @@ -617,15 +622,17 @@ static void clear_item_dirty(struct item_cache *cac, update_dirty_parents(item); } -static struct cached_item *alloc_item(struct kvec *key, struct kvec *val) +static struct cached_item *alloc_item(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val) { struct cached_item *item; item = kzalloc(sizeof(struct cached_item), GFP_NOFS); if (item) { - if (scoutfs_kvec_dup_flatten(item->key, key) || - scoutfs_kvec_dup_flatten(item->val, val)) { - free_item(item); + item->key = scoutfs_key_dup(sb, key); + if (!item->key || scoutfs_kvec_dup_flatten(item->val, val)) { + free_item(sb, item); item = NULL; } } @@ -639,7 +646,7 @@ static struct cached_item *alloc_item(struct kvec *key, struct kvec *val) * * XXX but it doesn't read.. is that weird? Seems weird. */ -int scoutfs_item_create(struct super_block *sb, struct kvec *key, +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -648,7 +655,7 @@ int scoutfs_item_create(struct super_block *sb, struct kvec *key, unsigned long flags; int ret; - item = alloc_item(key, val); + item = alloc_item(sb, key, val); if (!item) return -ENOMEM; @@ -661,7 +668,7 @@ int scoutfs_item_create(struct super_block *sb, struct kvec *key, spin_unlock_irqrestore(&cac->lock, flags); if (ret) - free_item(item); + free_item(sb, item); return ret; } @@ -672,12 +679,12 @@ int scoutfs_item_create(struct super_block *sb, struct kvec *key, * and we add with _tail to maintain that order. */ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct kvec *key, struct kvec *val) + struct scoutfs_key_buf *key, struct kvec *val) { struct cached_item *item; int ret; - item = alloc_item(key, val); + item = alloc_item(sb, key, val); if (item) { list_add_tail(&item->entry, list); ret = 0; @@ -705,7 +712,8 @@ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, * that will be inserted. */ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct kvec *start, struct kvec *end) + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -715,18 +723,18 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, unsigned long flags; int ret; - trace_scoutfs_item_insert_batch(sb, start, end); +// trace_scoutfs_item_insert_batch(sb, start, end); - if (WARN_ON_ONCE(scoutfs_kvec_memcmp(start, end) > 0)) + if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) return -EINVAL; rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); - if (rng && (scoutfs_kvec_dup_flatten(rng->start, start) || - scoutfs_kvec_dup_flatten(rng->end, end))) { - free_range(rng); - rng = NULL; + if (rng) { + rng->start = scoutfs_key_dup(sb, start); + rng->end = scoutfs_key_dup(sb, end); } - if (!rng) { + if (!rng || !rng->start || !rng->end) { + free_range(sb, rng); ret = -ENOMEM; goto out; } @@ -745,18 +753,18 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, ret = 0; out: - scoutfs_item_free_batch(list); + scoutfs_item_free_batch(sb, list); return ret; } -void scoutfs_item_free_batch(struct list_head *list) +void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) { struct cached_item *item; struct cached_item *tmp; list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); - free_item(item); + free_item(sb, item); } } @@ -765,22 +773,22 @@ void scoutfs_item_free_batch(struct list_head *list) * If the item exists make sure it's dirty and pinned. It can be read * if it wasn't cached. -ENOENT is returned if the item doesn't exist. */ -int scoutfs_item_dirty(struct super_block *sb, struct kvec *key) +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - SCOUTFS_DECLARE_KVEC(end); + struct scoutfs_key_buf *end; struct cached_item *item; unsigned long flags; int ret; - ret = scoutfs_kvec_alloc_key(end); - if (ret) + end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!end) { + ret = -ENOMEM; goto out; + } do { - scoutfs_kvec_init_key(end); - spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); @@ -798,7 +806,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct kvec *key) } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - scoutfs_kvec_kfree(end); + scoutfs_key_free(sb, end); out: trace_printk("ret %d\n", ret); return ret; @@ -810,20 +818,22 @@ out: * * Returns -ENOENT if the item doesn't exist. */ -int scoutfs_item_update(struct super_block *sb, struct kvec *key, +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + struct scoutfs_key_buf *end; SCOUTFS_DECLARE_KVEC(up_val); - SCOUTFS_DECLARE_KVEC(end); struct cached_item *item; unsigned long flags; int ret; - ret = scoutfs_kvec_alloc_key(end); - if (ret) + end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!end) { + ret = -ENOMEM; goto out; + } if (val) { ret = scoutfs_kvec_dup_flatten(up_val, val); @@ -834,8 +844,6 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, } do { - scoutfs_kvec_init_key(end); - spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); @@ -855,7 +863,7 @@ int scoutfs_item_update(struct super_block *sb, struct kvec *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); out: - scoutfs_kvec_kfree(end); + scoutfs_key_free(sb, end); scoutfs_kvec_kfree(up_val); trace_printk("ret %d\n", ret); @@ -866,7 +874,7 @@ out: * XXX how nice, it'd just creates a cached deletion item. It doesn't * have to read. */ -int scoutfs_item_delete(struct super_block *sb, struct kvec *key) +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) { return WARN_ON_ONCE(-EINVAL); } @@ -931,33 +939,39 @@ static struct cached_item *next_dirty(struct cached_item *item) return NULL; } -/* - * The total number of bytes that will be stored in segments if we were - * to write out all the currently dirty items. - * - * XXX this isn't strictly correct because item's aren't of a uniform - * size. We might need more segments when large items leave gaps at the - * tail of each segment as it is filled with sorted items. It's close - * enough for now. - */ -long scoutfs_item_dirty_bytes(struct super_block *sb) +bool scoutfs_item_has_dirty(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; unsigned long flags; - long bytes; + bool has; spin_lock_irqsave(&cac->lock, flags); - - bytes = (cac->nr_dirty_items * sizeof(struct scoutfs_segment_item)) + - cac->dirty_key_bytes + cac->dirty_val_bytes; - + has = cac->nr_dirty_items != 0; spin_unlock_irqrestore(&cac->lock, flags); - bytes += DIV_ROUND_UP(bytes, SCOUTFS_SEGMENT_SIZE) * - sizeof(struct scoutfs_segment_block); + return has; +} - return bytes; +/* + * Returns true if adding more items with the given count, keys, and values + * still fits in a single item along with the current dirty items. + */ +bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, + u32 key_bytes, u32 val_bytes) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + unsigned long flags; + bool fits; + + spin_lock_irqsave(&cac->lock, flags); + fits = scoutfs_seg_fits_single(nr_items + cac->nr_dirty_items, + key_bytes + cac->dirty_key_bytes, + val_bytes + cac->dirty_val_bytes); + spin_unlock_irqrestore(&cac->lock, flags); + + return fits; } /* @@ -968,24 +982,25 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, u32 *key_bytes) { struct cached_item *item; - u32 total; + u32 items = 0; + u32 keys = 0; + u32 vals = 0; *nr_items = 0; *key_bytes = 0; - total = sizeof(struct scoutfs_segment_block); for (item = first_dirty(cac->items.rb_node); item; item = next_dirty(item)) { - total += sizeof(struct scoutfs_segment_item) + - scoutfs_kvec_length(item->key) + - scoutfs_kvec_length(item->val); + items++; + keys += item->key->key_len; + vals += scoutfs_kvec_length(item->val); - if (total > SCOUTFS_SEGMENT_SIZE) + if (!scoutfs_seg_fits_single(items, keys, vals)) break; - (*nr_items)++; - (*key_bytes) += scoutfs_kvec_length(item->key); + *nr_items = items; + *key_bytes = keys; } } @@ -1062,14 +1077,14 @@ void scoutfs_item_destroy(struct super_block *sb) item = container_of(node, struct cached_item, node); node = rb_next(node); rb_erase(&item->node, &cac->items); - free_item(item); + free_item(sb, item); } for (node = rb_first(&cac->ranges); node; ) { rng = container_of(node, struct cached_range, node); node = rb_next(node); rb_erase(&rng->node, &cac->items); - free_range(rng); + free_range(sb, rng); } kfree(cac); diff --git a/kmod/src/item.h b/kmod/src/item.h index 81746822..3c7c6057 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -4,31 +4,38 @@ #include struct scoutfs_segment; +struct scoutfs_key_buf; -int scoutfs_item_lookup(struct super_block *sb, struct kvec *key, +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_lookup_exact(struct super_block *sb, struct kvec *key, - struct kvec *val, int size); -int scoutfs_item_next(struct super_block *sb, struct kvec *key, - struct kvec *last, struct kvec *val); -int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *key, - struct kvec *last, struct kvec *val, int len); -int scoutfs_item_insert(struct super_block *sb, struct kvec *key, +int scoutfs_item_lookup_exact(struct super_block *sb, + struct scoutfs_key_buf *key, struct kvec *val, + int size); +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, struct kvec *val); +int scoutfs_item_next_same_min(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, + struct kvec *val, int len); +int scoutfs_item_insert(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_create(struct super_block *sb, struct kvec *key, +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_dirty(struct super_block *sb, struct kvec *key); -int scoutfs_item_update(struct super_block *sb, struct kvec *key, +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_delete(struct super_block *sb, struct kvec *key); +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct kvec *key, struct kvec *val); + struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct kvec *start, struct kvec *end); -void scoutfs_item_free_batch(struct list_head *list); + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end); +void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); -long scoutfs_item_dirty_bytes(struct super_block *sb); +bool scoutfs_item_has_dirty(struct super_block *sb); +bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, + u32 key_bytes, u32 val_bytes); int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_item_setup(struct super_block *sb); diff --git a/kmod/src/key.c b/kmod/src/key.c new file mode 100644 index 00000000..9795f797 --- /dev/null +++ b/kmod/src/key.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "key.h" + +struct scoutfs_key_buf *scoutfs_key_alloc(struct super_block *sb, u16 len) +{ + struct scoutfs_key_buf *key; + + if (WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE)) + return NULL; + + key = kmalloc(sizeof(struct scoutfs_key_buf) + len, GFP_NOFS); + if (key) { + key->data = key + 1; + key->key_len = len; + key->buf_len = len; + } + + return key; +} + +struct scoutfs_key_buf *scoutfs_key_dup(struct super_block *sb, + struct scoutfs_key_buf *key) +{ + struct scoutfs_key_buf *dup; + + dup = scoutfs_key_alloc(sb, key->key_len); + if (dup) + memcpy(dup->data, key->data, dup->key_len); + return dup; +} + +void scoutfs_key_free(struct super_block *sb, struct scoutfs_key_buf *key) +{ + kfree(key); +} + +/* + * Keys are large multi-byte big-endian values. To correctly increase + * or decrease keys we need to start by extending the key to the full + * precision using the max key size, setting the least significant bytes + * to 0. + */ +static void extend_zeros(struct scoutfs_key_buf *key) +{ + if (key->key_len < SCOUTFS_MAX_KEY_SIZE && + !WARN_ON_ONCE(key->buf_len != SCOUTFS_MAX_KEY_SIZE)) { + memset(key->data + key->key_len, 0, + key->buf_len - key->key_len); + key->key_len = key->buf_len; + } +} + +void scoutfs_key_inc(struct scoutfs_key_buf *key) +{ + u8 *bytes = key->data; + int i; + + extend_zeros(key); + + for (i = key->key_len - 1; i >= 0; i--) { + if (++bytes[i] != 0) + break; + } +} + +void scoutfs_key_dec(struct scoutfs_key_buf *key) +{ + u8 *bytes = key->data; + int i; + + extend_zeros(key); + + for (i = key->key_len - 1; i >= 0; i--) { + if (--bytes[i] != 255) + break; + } +} diff --git a/kmod/src/key.h b/kmod/src/key.h index cb8460d6..a63244ba 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -4,6 +4,120 @@ #include #include "format.h" +struct scoutfs_key_buf { + void *data; + u16 key_len; + u16 buf_len; +}; + +struct scoutfs_key_buf *scoutfs_key_alloc(struct super_block *sb, u16 len); +struct scoutfs_key_buf *scoutfs_key_dup(struct super_block *sb, + struct scoutfs_key_buf *key); +void scoutfs_key_free(struct super_block *sb, struct scoutfs_key_buf *key); +void scoutfs_key_inc(struct scoutfs_key_buf *key); +void scoutfs_key_dec(struct scoutfs_key_buf *key); + + +/* + * Point the key buf, usually statically allocated, at an existing + * contiguous key stored elsewhere. + */ +static inline void scoutfs_key_init(struct scoutfs_key_buf *key, + void *data, u16 len) +{ + WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE); + + key->data = data; + key->key_len = len; + key->buf_len = len; +} + +/* + * Compare the fs keys in segment sort order. + */ +static inline int scoutfs_key_compare(struct scoutfs_key_buf *a, + struct scoutfs_key_buf *b) +{ + return memcmp(a->data, b->data, min(a->key_len, b->key_len)) ?: + a->key_len < b->key_len ? -1 : a->key_len > b->key_len ? 1 : 0; +} + +/* + * Compare ranges of keys where overlapping is equality. Returns: + * -1: a_end < b_start + * 1: a_start > b_end + * else 0: ranges overlap + */ +static inline int scoutfs_key_compare_ranges(struct scoutfs_key_buf *a_start, + struct scoutfs_key_buf *a_end, + struct scoutfs_key_buf *b_start, + struct scoutfs_key_buf *b_end) +{ + return scoutfs_key_compare(a_end, b_start) < 0 ? -1 : + scoutfs_key_compare(a_start, b_end) > 0 ? 1 : + 0; +} + +/* + * Copy as much of the contents of the source buffer that fits into the + * dest buffer. + */ +static inline void scoutfs_key_copy(struct scoutfs_key_buf *dst, + struct scoutfs_key_buf *src) +{ + dst->key_len = min(dst->buf_len, src->key_len); + memcpy(dst->data, src->data, dst->key_len); +} + +/* + * Initialize the dst buffer to point to the source buffer in all ways, + * including the buf len. The contents of the buffer are shared by the + * fields describing the buffers are not. + */ +static inline void scoutfs_key_clone(struct scoutfs_key_buf *dst, + struct scoutfs_key_buf *src) +{ + *dst = *src; +} + +/* + * Memset as much of the length as fits in the buffer and set that to + * the new key length. + */ +static inline void scoutfs_key_memset(struct scoutfs_key_buf *key, int c, + u16 len) +{ + if (WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE)) + return; + + key->key_len = min(key->buf_len, len); + memset(key->data, c, key->key_len); +} + +/* + * Set the contents of the buffer to the smallest possible key by sort + * order. It might be truncated if the buffer isn't large enough. + */ +static inline void scoutfs_key_set_min(struct scoutfs_key_buf *key) +{ + scoutfs_key_memset(key, 0, sizeof(struct scoutfs_inode_key)); +} + +/* + * Set the contents of the buffer to the largest possible key by sort + * order. It might be truncated if the buffer isn't large enough. + */ +static inline void scoutfs_key_set_max(struct scoutfs_key_buf *key) +{ + scoutfs_key_memset(key, 0xff, sizeof(struct scoutfs_inode_key)); +} + +/* + * What follows are the key functions for the small fixed size btree + * keys. It will all be removed once the callers are converted from + * the btree to the item cache. + */ + #define CKF "%llu.%u.%llu" #define CKA(key) \ le64_to_cpu((key)->inode), (key)->type, le64_to_cpu((key)->offset) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2a96d9da..98ac6234 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -52,7 +52,7 @@ struct manifest { /* calculated on mount, const thereafter */ u64 level_limits[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; - SCOUTFS_DECLARE_KVEC(compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]); + struct scoutfs_key_buf *compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; }; #define DECLARE_MANIFEST(sb, name) \ @@ -75,16 +75,16 @@ struct manifest_ref { struct scoutfs_segment *seg; int found_ctr; int pos; - u16 first_key_len; - u16 last_key_len; u8 level; - u8 keys[0]; + + struct scoutfs_key_buf *first; + struct scoutfs_key_buf *last; }; struct manifest_fill_args { struct scoutfs_manifest_entry ment; - struct kvec *first; - struct kvec *last; + struct scoutfs_key_buf *first; + struct scoutfs_key_buf *last; }; /* @@ -93,41 +93,33 @@ struct manifest_fill_args { */ struct manifest_search_key { u64 seq; - struct kvec *key; + struct scoutfs_key_buf *key; u8 level; }; static void init_ment_keys(struct scoutfs_manifest_entry *ment, - struct kvec *first, struct kvec *last) + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) { if (first) - scoutfs_kvec_init(first, ment->keys, - le16_to_cpu(ment->first_key_len)); + scoutfs_key_init(first, ment->keys, + le16_to_cpu(ment->first_key_len)); if (last) - scoutfs_kvec_init(last, ment->keys + - le16_to_cpu(ment->first_key_len), - le16_to_cpu(ment->last_key_len)); + scoutfs_key_init(last, ment->keys + + le16_to_cpu(ment->first_key_len), + le16_to_cpu(ment->last_key_len)); } -static void init_ref_keys(struct manifest_ref *ref, struct kvec *first, - struct kvec *last) -{ - if (first) - scoutfs_kvec_init(first, ref->keys, ref->first_key_len); - if (last) - scoutfs_kvec_init(last, ref->keys + ref->first_key_len, - ref->last_key_len); -} - -static bool cmp_range_ment(struct kvec *key, struct kvec *end, +static bool cmp_range_ment(struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, struct scoutfs_manifest_entry *ment) { - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; - init_ment_keys(ment, first, last); + init_ment_keys(ment, &first, &last); - return scoutfs_kvec_cmp_overlap(key, end, first, last); + return scoutfs_key_compare_ranges(key, end, &first, &last); } static u64 get_level_count(struct manifest *mani, @@ -187,8 +179,10 @@ static void add_level_count(struct super_block *sb, struct manifest *mani, * * This must be called with the manifest lock held. */ -int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level) +int scoutfs_manifest_add(struct super_block *sb, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, + u8 level) { DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -200,15 +194,15 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, unsigned bytes; int ret; - trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); +// trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); - key_bytes = scoutfs_kvec_length(first) + scoutfs_kvec_length(last); + key_bytes = first->key_len + last->key_len; bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); args.ment.segno = cpu_to_le64(segno); args.ment.seq = cpu_to_le64(seq); - args.ment.first_key_len = cpu_to_le16(scoutfs_kvec_length(first)); - args.ment.last_key_len = cpu_to_le16(scoutfs_kvec_length(last)); + args.ment.first_key_len = cpu_to_le16(first->key_len); + args.ment.last_key_len = cpu_to_le16(last->key_len); args.ment.level = level; args.first = first; @@ -233,8 +227,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, /* * This must be called with the manifest lock held. */ -int scoutfs_manifest_dirty(struct super_block *sb, struct kvec *first, u64 seq, - u8 level) +int scoutfs_manifest_dirty(struct super_block *sb, + struct scoutfs_key_buf *first, u64 seq, u8 level) { DECLARE_MANIFEST(sb, mani); struct scoutfs_manifest_entry *ment; @@ -255,8 +249,8 @@ int scoutfs_manifest_dirty(struct super_block *sb, struct kvec *first, u64 seq, /* * This must be called with the manifest lock held. */ -int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, - u8 level) +int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, + u64 seq, u8 level) { DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -299,41 +293,43 @@ int scoutfs_manifest_unlock(struct super_block *sb) return 0; } -static int alloc_add_ref(struct list_head *list, +static void free_ref(struct super_block *sb, struct manifest_ref *ref) +{ + if (!IS_ERR_OR_NULL(ref)) { + WARN_ON_ONCE(!list_empty(&ref->entry)); + scoutfs_seg_put(ref->seg); + scoutfs_key_free(sb, ref->first); + scoutfs_key_free(sb, ref->last); + kfree(ref); + } +} + +static int alloc_add_ref(struct super_block *sb, struct list_head *list, struct scoutfs_manifest_entry *ment) { - SCOUTFS_DECLARE_KVEC(ment_first); - SCOUTFS_DECLARE_KVEC(ment_last); - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf ment_first; + struct scoutfs_key_buf ment_last; struct manifest_ref *ref; - unsigned bytes; - init_ment_keys(ment, ment_first, ment_last); + init_ment_keys(ment, &ment_first, &ment_last); - bytes = scoutfs_kvec_length(ment_first) + - scoutfs_kvec_length(ment_first); - - ref = kmalloc(offsetof(struct manifest_ref, keys[bytes]), GFP_NOFS); - if (!ref) + ref = kzalloc(sizeof(struct manifest_ref), GFP_NOFS); + if (ref) { + ref->first = scoutfs_key_dup(sb, &ment_first); + ref->last = scoutfs_key_dup(sb, &ment_last); + } + if (!ref || !ref->first || !ref->last) { + free_ref(sb, ref); return -ENOMEM; - - memset(ref, 0, offsetof(struct manifest_ref, keys)); + } ref->segno = le64_to_cpu(ment->segno); ref->seq = le64_to_cpu(ment->seq); ref->level = ment->level; - ref->first_key_len = le16_to_cpu(ment->first_key_len); - ref->last_key_len = le16_to_cpu(ment->last_key_len); - - init_ref_keys(ref, first, last); - scoutfs_kvec_memcpy(first, ment_first); - scoutfs_kvec_memcpy(last, ment_last); list_add_tail(&ref->entry, list); return 0; - } /* @@ -349,13 +345,14 @@ static int alloc_add_ref(struct list_head *list, * segment starting with the key. */ static int get_range_refs(struct super_block *sb, struct manifest *mani, - struct kvec *key, struct kvec *end, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, struct list_head *ref_list) { struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; struct manifest_ref *ref; struct manifest_ref *tmp; int ret; @@ -369,7 +366,7 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, ment = scoutfs_treap_lookup_prev(mani->treap, &skey); while (!IS_ERR_OR_NULL(ment)) { if (cmp_range_ment(key, end, ment) == 0) { - ret = alloc_add_ref(ref_list, ment); + ret = alloc_add_ref(sb, ref_list, ment); if (ret) goto out; } @@ -396,8 +393,8 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, } if (ment) { - init_ment_keys(ment, first, last); - ret = alloc_add_ref(ref_list, ment); + init_ment_keys(ment, &first, &last); + ret = alloc_add_ref(sb, ref_list, ment); if (ret) goto out; } @@ -411,7 +408,7 @@ out: if (ret) { list_for_each_entry_safe(ref, tmp, ref_list, entry) { list_del_init(&ref->entry); - kfree(ref); + free_ref(sb, ref); } } @@ -446,16 +443,17 @@ out: */ #define MAX_ITEMS_READ 32 -int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, - struct kvec *end) +int scoutfs_manifest_read_items(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end) { DECLARE_MANIFEST(sb, mani); - SCOUTFS_DECLARE_KVEC(item_key); + struct scoutfs_key_buf item_key; + struct scoutfs_key_buf found_key; + struct scoutfs_key_buf batch_end; + struct scoutfs_key_buf seg_end; SCOUTFS_DECLARE_KVEC(item_val); - SCOUTFS_DECLARE_KVEC(found_key); SCOUTFS_DECLARE_KVEC(found_val); - SCOUTFS_DECLARE_KVEC(batch_end); - SCOUTFS_DECLARE_KVEC(seg_end); struct scoutfs_segment *seg; struct manifest_ref *ref; struct manifest_ref *tmp; @@ -486,7 +484,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, ref->seg = seg; } - /* wait for submitted segments and search for starting pos */ + /* always wait for submitted segments */ list_for_each_entry(ref, &ref_list, entry) { if (!ref->seg) break; @@ -494,15 +492,29 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, err = scoutfs_seg_wait(sb, ref->seg); if (err && !ret) ret = err; - - if (ret == 0) - ref->pos = scoutfs_seg_find_pos(ref->seg, key); } if (ret) goto out; - scoutfs_kvec_init_null(batch_end); - scoutfs_kvec_init_null(seg_end); + /* start from the next item from the key in each segment */ + list_for_each_entry(ref, &ref_list, entry) + ref->pos = scoutfs_seg_find_pos(ref->seg, key); + + /* + * Find the greatest range we can cover if we walk all the + * segments. We only have level 0 segments for the missing + * range so that's the greatest. Then we shrink the range by + * the limit of each higher level segment that intersected with + * our starting key. + */ + scoutfs_key_clone(&seg_end, end); + list_for_each_entry(ref, &ref_list, entry) { + if (ref->level > 0 && + scoutfs_key_compare(ref->last, &seg_end) < 0) { + scoutfs_key_clone(&seg_end, ref->last); + } + } + found_ctr = 0; for (n = 0; n < MAX_ITEMS_READ; n++) { @@ -512,37 +524,26 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, /* find the next least key from the pos in each segment */ list_for_each_entry_safe(ref, tmp, &ref_list, entry) { + if (ref->pos == -1) + continue; /* * Check the next item in the segment. We're * done with the segment if there are no more * items or if the next item is past the - * caller's end. We record either the caller's - * end or the segment end if it's a l1+ segment for - * use as the batch end if we don't see more items. + * caller's end. */ - ret = scoutfs_seg_item_kvecs(ref->seg, ref->pos, - item_key, item_val); - if (ret < 0) { - if (ref->level > 0) { - init_ref_keys(ref, NULL, item_key); - scoutfs_kvec_clone_less(seg_end, - item_key); - } - } else if (scoutfs_kvec_memcmp(item_key, end) > 0) { - scoutfs_kvec_clone_less(seg_end, end); - ret = -ENOENT; - } - if (ret < 0) { - list_del_init(&ref->entry); - scoutfs_seg_put(ref->seg); - kfree(ref); + ret = scoutfs_seg_item_ptrs(ref->seg, ref->pos, + &item_key, item_val); + if (ret < 0 || scoutfs_key_compare(&item_key, end) > 0){ + ref->pos = -1; continue; } /* see if it's the new least item */ if (found) { - cmp = scoutfs_kvec_memcmp(item_key, found_key); + cmp = scoutfs_key_compare(&item_key, + &found_key); if (cmp >= 0) { if (cmp == 0) ref->found_ctr = found_ctr; @@ -551,7 +552,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, } /* remember new least key */ - scoutfs_kvec_clone(found_key, item_key); + scoutfs_key_clone(&found_key, &item_key); scoutfs_kvec_clone(found_val, item_val); ref->found_ctr = ++found_ctr; found = true; @@ -559,7 +560,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, /* ran out of keys in segs, range extends to seg end */ if (!found) { - scoutfs_kvec_clone(batch_end, seg_end); + scoutfs_key_clone(&batch_end, &seg_end); ret = 0; break; } @@ -569,18 +570,18 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, * have items it's not a failure and the end of the cached * range is the last successfully added item. */ - ret = scoutfs_item_add_batch(sb, &batch, found_key, found_val); + ret = scoutfs_item_add_batch(sb, &batch, &found_key, found_val); if (ret) { if (n > 0) ret = 0; break; } - /* the last successful key determines the range */ - scoutfs_kvec_clone(batch_end, found_key); + /* the last successful key determines range end until run out */ + scoutfs_key_clone(&batch_end, &found_key); /* if we just saw the end key then we're done */ - if (scoutfs_kvec_memcmp(found_key, end) == 0) { + if (scoutfs_key_compare(&found_key, end) == 0) { ret = 0; break; } @@ -595,14 +596,13 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, } if (ret) - scoutfs_item_free_batch(&batch); + scoutfs_item_free_batch(sb, &batch); else - ret = scoutfs_item_insert_batch(sb, &batch, key, batch_end); + ret = scoutfs_item_insert_batch(sb, &batch, key, &batch_end); out: list_for_each_entry_safe(ref, tmp, &ref_list, entry) { list_del_init(&ref->entry); - scoutfs_seg_put(ref->seg); - kfree(ref); + free_ref(sb, ref); } return ret; @@ -677,10 +677,10 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) struct scoutfs_manifest_entry *ment; struct scoutfs_manifest_entry *over; struct manifest_search_key skey; - SCOUTFS_DECLARE_KVEC(ment_first); - SCOUTFS_DECLARE_KVEC(ment_last); - SCOUTFS_DECLARE_KVEC(over_first); - SCOUTFS_DECLARE_KVEC(over_last); + struct scoutfs_key_buf ment_first; + struct scoutfs_key_buf ment_last; + struct scoutfs_key_buf over_first; + struct scoutfs_key_buf over_last; int level; int ret; int i; @@ -710,9 +710,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) skey.seq = 0; ment = scoutfs_treap_lookup_next(mani->treap, &skey); if (ment == NULL || ment->level != level) { - /* XXX ugh, these kvecs are the worst */ - scoutfs_kvec_init(skey.key, - skey.key[0].iov_base, 0); + scoutfs_key_set_min(skey.key); ment = scoutfs_treap_lookup_next(mani->treap, &skey); } } @@ -726,17 +724,17 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } - init_ment_keys(ment, ment_first, ment_last); + init_ment_keys(ment, &ment_first, &ment_last); /* add the upper input segment */ - ret = scoutfs_compact_add(sb, data, ment_first, ment_last, + ret = scoutfs_compact_add(sb, data, &ment_first, &ment_last, le64_to_cpu(ment->segno), le64_to_cpu(ment->seq), level); if (ret) goto out; /* start with the first overlapping at the next level */ - skey.key = ment_first; + skey.key = &ment_first; skey.level = level + 1; skey.seq = 0; over = scoutfs_treap_lookup(mani->treap, &skey); @@ -750,13 +748,13 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) if (!over || over->level != (ment->level + 1)) break; - init_ment_keys(over, over_first, over_last); + init_ment_keys(over, &over_first, &over_last); - if (scoutfs_kvec_cmp_overlap(ment_first, ment_last, - over_first, over_last) != 0) + if (scoutfs_key_compare_ranges(&ment_first, &ment_last, + &over_first, &over_last) != 0) break; - ret = scoutfs_compact_add(sb, data, over_first, over_last, + ret = scoutfs_compact_add(sb, data, &over_first, &over_last, le64_to_cpu(over->segno), le64_to_cpu(over->seq), level + 1); if (ret) @@ -765,10 +763,9 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) over = scoutfs_treap_next(mani->treap, over); } - /* record the next key to start from, not exact */ - scoutfs_kvec_init_key(mani->compact_keys[level]); - scoutfs_kvec_memcpy_truncate(mani->compact_keys[level], ment_last); - scoutfs_kvec_be_inc(mani->compact_keys[level]); + /* record the next key to start from */ + scoutfs_key_copy(mani->compact_keys[level], &ment_last); + scoutfs_key_inc(mani->compact_keys[level]); ret = 0; out: @@ -800,8 +797,8 @@ static int manifest_treap_compare(void *key, void *data) { struct manifest_search_key *skey = key; struct scoutfs_manifest_entry *ment = data; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; int cmp; if (skey->level < ment->level) { @@ -818,13 +815,13 @@ static int manifest_treap_compare(void *key, void *data) goto out; } - init_ment_keys(ment, first, last); + init_ment_keys(ment, &first, &last); if (skey->seq == 0) { - cmp = scoutfs_kvec_cmp_overlap(skey->key, skey->key, - first, last); + cmp = scoutfs_key_compare_ranges(skey->key, skey->key, + &first, &last); } else { - cmp = scoutfs_kvec_memcmp(skey->key, first) ?: + cmp = scoutfs_key_compare(skey->key, &first) ?: scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); } @@ -836,14 +833,14 @@ static void manifest_treap_fill(void *data, void *arg) { struct scoutfs_manifest_entry *ment = data; struct manifest_fill_args *args = arg; - SCOUTFS_DECLARE_KVEC(ment_first); - SCOUTFS_DECLARE_KVEC(ment_last); + struct scoutfs_key_buf ment_first; + struct scoutfs_key_buf ment_last; *ment = args->ment; - init_ment_keys(ment, ment_first, ment_last); - scoutfs_kvec_memcpy(ment_first, args->first); - scoutfs_kvec_memcpy(ment_last, args->last); + init_ment_keys(ment, &ment_first, &ment_last); + scoutfs_key_copy(&ment_first, args->first); + scoutfs_key_copy(&ment_last, args->last); } static struct scoutfs_treap_ops manifest_treap_ops = { @@ -858,7 +855,6 @@ int scoutfs_manifest_setup(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct manifest *mani; - int ret; int i; mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); @@ -876,14 +872,17 @@ int scoutfs_manifest_setup(struct super_block *sb) } for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) { - ret = scoutfs_kvec_alloc_key(mani->compact_keys[i]); - if (ret) { + mani->compact_keys[i] = scoutfs_key_alloc(sb, + SCOUTFS_MAX_KEY_SIZE); + if (!mani->compact_keys[i]) { while (--i >= 0) - scoutfs_kvec_kfree(mani->compact_keys[i]); + scoutfs_key_free(sb, mani->compact_keys[i]); scoutfs_treap_free(mani->treap); kfree(mani); return -ENOMEM; } + + scoutfs_key_set_min(mani->compact_keys[i]); } for (i = ARRAY_SIZE(super->manifest.level_counts) - 1; i >= 0; i--) { @@ -915,7 +914,7 @@ void scoutfs_manifest_destroy(struct super_block *sb) if (mani) { scoutfs_treap_free(mani->treap); for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) - scoutfs_kvec_kfree(mani->compact_keys[i]); + scoutfs_key_free(sb, mani->compact_keys[i]); kfree(mani); } } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 5e529cd5..d788aeaf 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -1,20 +1,25 @@ #ifndef _SCOUTFS_MANIFEST_H_ #define _SCOUTFS_MANIFEST_H_ -int scoutfs_manifest_add(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level); -int scoutfs_manifest_dirty(struct super_block *sb, struct kvec *first, u64 seq, - u8 level); -int scoutfs_manifest_del(struct super_block *sb, struct kvec *first, u64 seq, +struct scoutfs_key_buf; + +int scoutfs_manifest_add(struct super_block *sb, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, u8 level); +int scoutfs_manifest_dirty(struct super_block *sb, + struct scoutfs_key_buf *first, u64 seq, u8 level); +int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, + u64 seq, u8 level); int scoutfs_manifest_has_dirty(struct super_block *sb); int scoutfs_manifest_dirty_ring(struct super_block *sb); int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); -int scoutfs_manifest_read_items(struct super_block *sb, struct kvec *key, - struct kvec *until); +int scoutfs_manifest_read_items(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end); u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); diff --git a/kmod/src/seg.c b/kmod/src/seg.c index f2034b8e..f881e701 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -24,6 +24,7 @@ #include "cmp.h" #include "manifest.h" #include "alloc.h" +#include "key.h" /* * seg.c should just be about the cache and io, and maybe @@ -339,7 +340,7 @@ static void *off_ptr(struct scoutfs_segment *seg, u32 off) return page_address(seg->pages[pg]) + pg_off; } -static u32 pos_off(struct scoutfs_segment *seg, u32 pos) +static u32 pos_off(u32 pos) { /* items need of be a power of two */ BUILD_BUG_ON(!is_power_of_2(sizeof(struct scoutfs_segment_item))); @@ -352,7 +353,7 @@ static u32 pos_off(struct scoutfs_segment *seg, u32 pos) static void *pos_ptr(struct scoutfs_segment *seg, u32 pos) { - return off_ptr(seg, pos_off(seg, pos)); + return off_ptr(seg, pos_off(pos)); } /* @@ -416,8 +417,8 @@ static void kvec_from_pages(struct scoutfs_segment *seg, off_ptr(seg, off + first), len - first); } -int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, - struct kvec *key, struct kvec *val) +int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, + struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; @@ -428,7 +429,7 @@ int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, load_item(seg, pos, &item); if (key) - kvec_from_pages(seg, key, item.key_off, item.key_len); + scoutfs_key_init(key, off_ptr(seg, item.key_off), item.key_len); if (val) kvec_from_pages(seg, val, item.val_off, item.val_len); @@ -440,10 +441,11 @@ int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, * This can return the number of positions if the key is greater than * all the keys. */ -static int find_key_pos(struct scoutfs_segment *seg, struct kvec *search) +static int find_key_pos(struct scoutfs_segment *seg, + struct scoutfs_key_buf *search) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - SCOUTFS_DECLARE_KVEC(key); + struct scoutfs_key_buf key; unsigned int start = 0; unsigned int end = le32_to_cpu(sblk->nr_items); unsigned int pos = 0; @@ -451,9 +453,9 @@ static int find_key_pos(struct scoutfs_segment *seg, struct kvec *search) while (start < end) { pos = start + (end - start) / 2; - scoutfs_seg_item_kvecs(seg, pos, key, NULL); + scoutfs_seg_item_ptrs(seg, pos, &key, NULL); - cmp = scoutfs_kvec_memcmp(search, key); + cmp = scoutfs_key_compare(search, &key); if (cmp < 0) end = pos; else if (cmp > 0) @@ -465,11 +467,51 @@ static int find_key_pos(struct scoutfs_segment *seg, struct kvec *search) return pos; } -int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key) +int scoutfs_seg_find_pos(struct scoutfs_segment *seg, + struct scoutfs_key_buf *key) { return find_key_pos(seg, key); } +/* + * Keys are aligned to the next block boundary if they'd cross a block + * boundary. To find the first value offset we have to assume that + * there will be a worst case key alignment at every block boundary. + */ +static u32 first_val_off(u32 nr_items, u32 key_bytes) +{ + u32 key_padding = SCOUTFS_MAX_KEY_SIZE - 1; + u32 partial_block = SCOUTFS_BLOCK_SIZE - key_padding; + u32 first_key_off = pos_off(nr_items); + u32 block_off = first_key_off & SCOUTFS_BLOCK_MASK; + u32 total_padding = ((block_off + key_bytes) / partial_block) * + key_padding; + + return first_key_off + key_bytes + total_padding; +} + +/* + * Returns true if the given number of items with the given total byte + * counts of keys and values fits inside a single segment. + */ +bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes) +{ + return (first_val_off(nr_items, key_bytes) + val_bytes) + <= SCOUTFS_SEGMENT_SIZE; +} + +static u32 align_key_off(struct scoutfs_segment *seg, u32 key_off, u32 len) +{ + u32 space = SCOUTFS_BLOCK_SIZE - (key_off & SCOUTFS_BLOCK_MASK); + + if (len > space) { + memset(off_ptr(seg, key_off), 0, space); + return key_off + space; + } + + return key_off; +} + /* * Store the first item in the segment. The caller knows the number * of items and bytes of keys that determine where the keys and values @@ -478,14 +520,14 @@ int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key) * This should never fail because any item must always fit in a segment. */ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, - struct kvec *key, struct kvec *val, + struct scoutfs_key_buf *key, struct kvec *val, unsigned int nr_items, unsigned int key_bytes) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; - SCOUTFS_DECLARE_KVEC(item_key); + struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); u32 key_off; u32 val_off; @@ -495,31 +537,33 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, sblk->seq = super->next_seg_seq; le64_add_cpu(&super->next_seg_seq, 1); - key_off = pos_off(seg, nr_items); - val_off = key_off + key_bytes; + key_off = align_key_off(seg, pos_off(nr_items), key->key_len); + val_off = first_val_off(nr_items, key_bytes); sblk->nr_items = cpu_to_le32(1); + trace_printk("first item offs key %u val %u\n", key_off, val_off); + item.seq = 1; item.key_off = key_off; item.val_off = val_off; - item.key_len = scoutfs_kvec_length(key); + item.key_len = key->key_len; item.val_len = scoutfs_kvec_length(val); store_item(seg, 0, &item); - scoutfs_seg_item_kvecs(seg, 0, item_key, item_val); - scoutfs_kvec_memcpy(item_key, key); + scoutfs_seg_item_ptrs(seg, 0, &item_key, item_val); + scoutfs_key_copy(&item_key, key); scoutfs_kvec_memcpy(item_val, val); } void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct kvec *key, struct kvec *val) + struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; struct native_item prev; - SCOUTFS_DECLARE_KVEC(item_key); + struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); u32 pos; @@ -529,14 +573,18 @@ void scoutfs_seg_append_item(struct super_block *sb, load_item(seg, pos - 1, &prev); item.seq = 1; - item.key_off = prev.key_off + prev.key_len; - item.key_len = scoutfs_kvec_length(key); + item.key_off = align_key_off(seg, prev.key_off + prev.key_len, + key->key_len); + item.key_len = key->key_len; item.val_off = prev.val_off + prev.val_len; item.val_len = scoutfs_kvec_length(val); store_item(seg, pos, &item); - scoutfs_seg_item_kvecs(seg, pos, item_key, item_val); - scoutfs_kvec_memcpy(item_key, key); + trace_printk("item %u offs key %u val %u\n", + pos, item.key_off, item.val_off); + + scoutfs_seg_item_ptrs(seg, pos, &item_key, item_val); + scoutfs_key_copy(&item_key, key); scoutfs_kvec_memcpy(item_val, val); } @@ -548,16 +596,16 @@ int scoutfs_seg_manifest_add(struct super_block *sb, { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; - SCOUTFS_DECLARE_KVEC(first); - SCOUTFS_DECLARE_KVEC(last); + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; load_item(seg, 0, &item); - kvec_from_pages(seg, first, item.key_off, item.key_len); + scoutfs_key_init(&first, off_ptr(seg, item.key_off), item.key_len); load_item(seg, le32_to_cpu(sblk->nr_items) - 1, &item); - kvec_from_pages(seg, last, item.key_off, item.key_len); + scoutfs_key_init(&last, off_ptr(seg, item.key_off), item.key_len); - return scoutfs_manifest_add(sb, first, last, le64_to_cpu(sblk->segno), + return scoutfs_manifest_add(sb, &first, &last, le64_to_cpu(sblk->segno), le64_to_cpu(sblk->seq), level); } @@ -566,12 +614,12 @@ int scoutfs_seg_manifest_del(struct super_block *sb, { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct native_item item; - SCOUTFS_DECLARE_KVEC(first); + struct scoutfs_key_buf first; load_item(seg, 0, &item); - kvec_from_pages(seg, first, item.key_off, item.key_len); + scoutfs_key_init(&first, off_ptr(seg, item.key_off), item.key_len); - return scoutfs_manifest_del(sb, first, le64_to_cpu(sblk->seq), level); + return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); } int scoutfs_seg_setup(struct super_block *sb) diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 597e9955..e43b2268 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -3,15 +3,17 @@ struct scoutfs_bio_completion; struct scoutfs_segment; +struct scoutfs_key_buf; struct kvec; struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, u64 segno); int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); -int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct kvec *key); -int scoutfs_seg_item_kvecs(struct scoutfs_segment *seg, int pos, - struct kvec *key, struct kvec *val); +int scoutfs_seg_find_pos(struct scoutfs_segment *seg, + struct scoutfs_key_buf *key); +int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, + struct scoutfs_key_buf *key, struct kvec *val); void scoutfs_seg_get(struct scoutfs_segment *seg); void scoutfs_seg_put(struct scoutfs_segment *seg); @@ -19,12 +21,13 @@ void scoutfs_seg_put(struct scoutfs_segment *seg); int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); +bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, - struct kvec *key, struct kvec *val, + struct scoutfs_key_buf *key, struct kvec *val, unsigned int nr_items, unsigned int key_bytes); void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct kvec *key, struct kvec *val); + struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level); int scoutfs_seg_manifest_del(struct super_block *sb, diff --git a/kmod/src/trans.c b/kmod/src/trans.c index bbd543ff..65db65ec 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -98,15 +98,15 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_filerw_free_alloc(sb); #endif - trace_printk("dirty bytes %ld manifest dirty %d alloc dirty %d\n", - scoutfs_item_dirty_bytes(sb), + trace_printk("items dirty %d manifest dirty %d alloc dirty %d\n", + scoutfs_item_has_dirty(sb), scoutfs_manifest_has_dirty(sb), scoutfs_alloc_has_dirty(sb)); /* * XXX this needs serious work to handle errors. */ - while (scoutfs_item_dirty_bytes(sb)) { + while (scoutfs_item_has_dirty(sb)) { seg = NULL; ret = scoutfs_seg_alloc(sb, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: @@ -222,14 +222,27 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, /* * I think the holder that creates the most dirty item data is - * symlinking, which can create all the entry items and a symlink target - * item with a full 4k path. We go a little nuts and just set it to two - * blocks. + * symlinking which can create an inode, the three dirent items with a + * full file name, and a symlink item with a full path. * - * XXX This divides the segment size to set the hard limit on the number of - * concurrent holders so we'll want this to be more precise. + * XXX Assuming the worst case here too aggressively limits the number + * of concurrent holders that can work without being blocked when they + * know they'll dirty much less. We may want to have callers pass in + * their item, key, and val budgets if that's not too fragile. + * + * XXX fix to use real backref and symlink items, placeholders for now */ -#define MOST_DIRTY (2 * SCOUTFS_BLOCK_SIZE) +#define HOLD_WORST_ITEMS 5 +#define HOLD_WORST_KEYS (sizeof(struct scoutfs_inode_key) + \ + sizeof(struct scoutfs_dirent_key) + SCOUTFS_NAME_LEN +\ + sizeof(struct scoutfs_readdir_key) + \ + sizeof(struct scoutfs_readdir_key) + \ + sizeof(struct scoutfs_inode_key)) +#define HOLD_WORST_VALS (sizeof(struct scoutfs_inode) + \ + sizeof(struct scoutfs_dirent) + \ + sizeof(struct scoutfs_dirent) + SCOUTFS_NAME_LEN + \ + sizeof(struct scoutfs_dirent) + SCOUTFS_NAME_LEN + \ + SCOUTFS_SYMLINK_MAX_SIZE) /* * We're able to hold the transaction if the current dirty item bytes @@ -239,10 +252,12 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, static bool hold_acquired(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - long bytes; int with_us; int holds; int before; + u32 items; + u32 keys; + u32 vals; holds = atomic_read(&sbi->trans_holds); for (;;) { @@ -258,8 +273,10 @@ static bool hold_acquired(struct super_block *sb) /* see if we all would fill the segment */ with_us = holds + 1; - bytes = (with_us * MOST_DIRTY) + scoutfs_item_dirty_bytes(sb); - if (bytes > SCOUTFS_SEGMENT_SIZE) { + items = with_us * HOLD_WORST_ITEMS; + keys = with_us * HOLD_WORST_KEYS; + vals = with_us * HOLD_WORST_VALS; + if (!scoutfs_item_dirty_fits_single(sb, items, keys, vals)) { scoutfs_sync_fs(sb, 0); return false; } From 736d5765fccc9d2c7dfb0bd654abd00b414ae403 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Jan 2017 20:46:52 -0800 Subject: [PATCH 206/920] Add a shrinker for the segment cache After segments have finished IO and while they're in the rbtree we track them with an LRU. Under memory pressure we can remove the oldest segments from the rbtree and free them. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/seg.c | 139 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 125 insertions(+), 15 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e01c2e1b..a473523d 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -16,6 +16,7 @@ EXPAND_COUNTER(alloc_free) \ EXPAND_COUNTER(block_mem_alloc) \ EXPAND_COUNTER(block_mem_free) \ + EXPAND_COUNTER(seg_lru_shrink) \ EXPAND_COUNTER(trans_level0_seg_write) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(compact_operations) \ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index f881e701..18378c2e 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -25,6 +25,7 @@ #include "manifest.h" #include "alloc.h" #include "key.h" +#include "counters.h" /* * seg.c should just be about the cache and io, and maybe @@ -38,13 +39,19 @@ */ struct segment_cache { + struct super_block *sb; spinlock_t lock; struct rb_root root; wait_queue_head_t waitq; + + struct shrinker shrinker; + struct list_head lru_list; + unsigned long lru_nr; }; struct scoutfs_segment { struct rb_node node; + struct list_head lru_entry; atomic_t refcount; u64 segno; unsigned long flags; @@ -70,6 +77,7 @@ static struct scoutfs_segment *alloc_seg(u64 segno) return seg; RB_CLEAR_NODE(&seg->node); + INIT_LIST_HEAD(&seg->lru_entry); atomic_set(&seg->refcount, 1); seg->segno = segno; @@ -99,6 +107,7 @@ void scoutfs_seg_put(struct scoutfs_segment *seg) if (!IS_ERR_OR_NULL(seg) && atomic_dec_and_test(&seg->refcount)) { WARN_ON_ONCE(!RB_EMPTY_NODE(&seg->node)); + WARN_ON_ONCE(!list_empty(&seg->lru_entry)); for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) if (seg->pages[i]) __free_page(seg->pages[i]); @@ -129,15 +138,33 @@ static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) return NULL; } +static void lru_check(struct segment_cache *cac, struct scoutfs_segment *seg) +{ + if (RB_EMPTY_NODE(&seg->node)) { + if (!list_empty(&seg->lru_entry)) { + list_del_init(&seg->lru_entry); + cac->lru_nr--; + } + } else { + if (list_empty(&seg->lru_entry)) { + list_add_tail(&seg->lru_entry, &cac->lru_list); + cac->lru_nr++; + } else { + list_move_tail(&seg->lru_entry, &cac->lru_list); + } + } +} + /* * This always inserts the segment into the rbtree. If there's already * a segment at the given seg then it is removed and returned. The * caller doesn't have to erase it from the tree if it's returned but it * does have to put the reference that it's given. */ -static struct scoutfs_segment *replace_seg(struct rb_root *root, +static struct scoutfs_segment *replace_seg(struct segment_cache *cac, struct scoutfs_segment *ins) { + struct rb_root *root = &cac->root; struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; struct scoutfs_segment *seg; @@ -155,6 +182,8 @@ static struct scoutfs_segment *replace_seg(struct rb_root *root, node = &(*node)->rb_right; } else { rb_replace_node(&seg->node, &ins->node, root); + lru_check(cac, seg); + lru_check(cac, ins); found = seg; break; } @@ -163,16 +192,18 @@ static struct scoutfs_segment *replace_seg(struct rb_root *root, if (!found) { rb_link_node(&ins->node, parent, node); rb_insert_color(&ins->node, root); + lru_check(cac, ins); } return found; } -static bool erase_seg(struct rb_root *root, struct scoutfs_segment *seg) +static bool erase_seg(struct segment_cache *cac, struct scoutfs_segment *seg) { if (!RB_EMPTY_NODE(&seg->node)) { - rb_erase(&seg->node, root); + rb_erase(&seg->node, &cac->root); RB_CLEAR_NODE(&seg->node); + lru_check(cac, seg); return true; } @@ -185,23 +216,27 @@ static void seg_end_io(struct super_block *sb, void *data, int err) struct segment_cache *cac = sbi->segment_cache; struct scoutfs_segment *seg = data; unsigned long flags; - bool erased; + bool erased = false; + + spin_lock_irqsave(&cac->lock, flags); + + set_bit(SF_END_IO, &seg->flags); if (err) { seg->err = err; - - spin_lock_irqsave(&cac->lock, flags); - erased = erase_seg(&cac->root, seg); - spin_unlock_irqrestore(&cac->lock, flags); - if (erased) - scoutfs_seg_put(seg); + erased = erase_seg(cac, seg); + } else { + lru_check(cac, seg); } - set_bit(SF_END_IO, &seg->flags); + spin_unlock_irqrestore(&cac->lock, flags); + smp_mb__after_atomic(); if (waitqueue_active(&cac->waitq)) wake_up(&cac->waitq); + if (erased) + scoutfs_seg_put(seg); scoutfs_seg_put(seg); } @@ -239,8 +274,9 @@ int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret) /* XXX always remove existing segs, is that necessary? */ spin_lock_irqsave(&cac->lock, flags); + atomic_inc(&seg->refcount); - existing = replace_seg(&cac->root, seg); + existing = replace_seg(cac, seg); spin_unlock_irqrestore(&cac->lock, flags); if (existing) scoutfs_seg_put(existing); @@ -280,8 +316,10 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, spin_lock_irqsave(&cac->lock, flags); seg = find_seg(&cac->root, segno); - if (seg) + if (seg) { + lru_check(cac, seg); atomic_inc(&seg->refcount); + } spin_unlock_irqrestore(&cac->lock, flags); if (seg) return seg; @@ -293,7 +331,7 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, /* always drop existing segs, could compare seqs */ spin_lock_irqsave(&cac->lock, flags); atomic_inc(&seg->refcount); - existing = replace_seg(&cac->root, seg); + existing = replace_seg(cac, seg); spin_unlock_irqrestore(&cac->lock, flags); if (existing) scoutfs_seg_put(existing); @@ -622,6 +660,68 @@ int scoutfs_seg_manifest_del(struct super_block *sb, return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); } +/* + * We maintain an LRU of segments so that the shrinker can free the + * oldest under memory pressure. Segments are only present in the LRU + * after their IO has completed and while they're in the rbtree. This + * shrink only removes them from the rbtree and drops the reference it + * held. They may be freed a bit later once all their active references + * are dropped. + * + * If this is called with nr_to_scan == 0 then it only returns the nr. + * We avoid acquiring the lock in that case. + * + * Lookup code only uses the lru entry to change position in the LRU while + * the segment is in the rbtree. Once we remove it no one else will use + * the LRU entry and we can use it to track all the segments that we're + * going to put outside of the lock. + * + * XXX: + * - are sc->nr_to_scan and our return meant to be in units of pages? + * - should we sync a transaction here? + */ +static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) +{ + struct segment_cache *cac = container_of(shrink, struct segment_cache, + shrinker); + struct super_block *sb = cac->sb; + struct scoutfs_segment *seg; + struct scoutfs_segment *tmp; + unsigned long flags; + unsigned long nr; + LIST_HEAD(list); + + nr = sc->nr_to_scan; + if (!nr) + goto out; + + spin_lock_irqsave(&cac->lock, flags); + + list_for_each_entry_safe(seg, tmp, &cac->lru_list, lru_entry) { + /* shouldn't be possible */ + if (WARN_ON_ONCE(RB_EMPTY_NODE(&seg->node))) + continue; + + if (nr-- == 0) + break; + + /* using ref that rb tree presence had */ + erase_seg(cac, seg); + list_add_tail(&seg->lru_entry, &list); + } + + spin_unlock_irqrestore(&cac->lock, flags); + + list_for_each_entry_safe(seg, tmp, &list, lru_entry) { + scoutfs_inc_counter(sb, seg_lru_shrink); + list_del_init(&seg->lru_entry); + scoutfs_seg_put(seg); + } + +out: + return min_t(unsigned long, cac->lru_nr, INT_MAX); +} + int scoutfs_seg_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -632,10 +732,16 @@ int scoutfs_seg_setup(struct super_block *sb) return -ENOMEM; sbi->segment_cache = cac; + cac->sb = sb; spin_lock_init(&cac->lock); cac->root = RB_ROOT; init_waitqueue_head(&cac->waitq); + cac->shrinker.shrink = seg_lru_shrink; + cac->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&cac->shrinker); + INIT_LIST_HEAD(&cac->lru_list); + return 0; } @@ -647,10 +753,13 @@ void scoutfs_seg_destroy(struct super_block *sb) struct rb_node *node; if (cac) { + if (cac->shrinker.shrink == seg_lru_shrink) + unregister_shrinker(&cac->shrinker); + for (node = rb_first(&cac->root); node; ) { seg = container_of(node, struct scoutfs_segment, node); node = rb_next(node); - erase_seg(&cac->root, seg); + erase_seg(cac, seg); scoutfs_seg_put(seg); } From 685eb1f2dc760fea0ff61bb5dd6896346b883554 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 20 Jan 2017 13:52:46 -0800 Subject: [PATCH 207/920] Fix segment block item alignemnt build bug The BUILD_BUG_ON() to test that the start of the items in the segment header is naturally aligned had a typo that masked the length instead of checking the remainder of division by the length. Signed-off-by: Zach Brown --- kmod/src/seg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 18378c2e..b016be8f 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -383,7 +383,7 @@ static u32 pos_off(u32 pos) /* items need of be a power of two */ BUILD_BUG_ON(!is_power_of_2(sizeof(struct scoutfs_segment_item))); /* and the first item has to be naturally aligned */ - BUILD_BUG_ON(offsetof(struct scoutfs_segment_block, items) & + BUILD_BUG_ON(offsetof(struct scoutfs_segment_block, items) % sizeof(struct scoutfs_segment_item)); return offsetof(struct scoutfs_segment_block, items[pos]); From cfc6d7226389009fdc70ffa0437ea3496358a081 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 20 Jan 2017 13:53:43 -0800 Subject: [PATCH 208/920] Remove item off and len packing The key and value offsets and lengths were aggressively packed into the item structs in the segments. This saved a few bytes per item but didn't leave any room left for expansion without growing the item. We want to add a deletion item flag so let's just grow the item struct. It now has room for full precision offsets and lengths that we can access natively so we can get rid fo the packing and unpacking functions. Signed-off-by: Zach Brown --- kmod/src/format.h | 15 ++---- kmod/src/seg.c | 118 ++++++++++++++++------------------------------ 2 files changed, 46 insertions(+), 87 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 05033817..901d4a66 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -130,21 +130,16 @@ struct scoutfs_alloc_region { * aligned. This ensures that they won't cross page boundaries and we * can use pointers to them in the page vecs that make up segments without * funny business. - * - * We limit segment sizes to 8 megs (23 bits) and value lengths to 512 bytes - * (9 bits). The item offsets and lengths then take up 64 bits. - * - * We then operate on the items in on-stack nice native structs. */ struct scoutfs_segment_item { __le64 seq; - __le32 key_off_len; - __le32 val_off_len; + __le32 key_off; + __le32 val_off; + __le16 key_len; + __le16 val_len; + __u8 padding[12]; } __packed; -#define SCOUTFS_SEGMENT_ITEM_OFF_SHIFT 9 -#define SCOUTFS_SEGMENT_ITEM_LEN_MASK ((1 << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT)-1) - /* * Each large segment starts with a segment block that describes the * rest of the blocks that make up the segment. diff --git a/kmod/src/seg.c b/kmod/src/seg.c index b016be8f..4687b7af 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -394,53 +394,6 @@ static void *pos_ptr(struct scoutfs_segment *seg, u32 pos) return off_ptr(seg, pos_off(pos)); } -/* - * The persistent item fields that are stored in the segment are packed - * with funny precision. We translate those to and from a much more - * natural native representation of the fields. - */ -struct native_item { - u64 seq; - u32 key_off; - u32 val_off; - u16 key_len; - u16 val_len; -}; - -static void load_item(struct scoutfs_segment *seg, u32 pos, - struct native_item *item) -{ - struct scoutfs_segment_item *sitem = pos_ptr(seg, pos); - u32 packed; - - item->seq = le64_to_cpu(sitem->seq); - - packed = le32_to_cpu(sitem->key_off_len); - item->key_off = packed >> SCOUTFS_SEGMENT_ITEM_OFF_SHIFT; - item->key_len = packed & SCOUTFS_SEGMENT_ITEM_LEN_MASK; - - packed = le32_to_cpu(sitem->val_off_len); - item->val_off = packed >> SCOUTFS_SEGMENT_ITEM_OFF_SHIFT; - item->val_len = packed & SCOUTFS_SEGMENT_ITEM_LEN_MASK; -} - -static void store_item(struct scoutfs_segment *seg, u32 pos, - struct native_item *item) -{ - struct scoutfs_segment_item *sitem = pos_ptr(seg, pos); - u32 packed; - - sitem->seq = cpu_to_le64(item->seq); - - packed = (item->key_off << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT) | - (item->key_len & SCOUTFS_SEGMENT_ITEM_LEN_MASK); - sitem->key_off_len = cpu_to_le32(packed); - - packed = (item->val_off << SCOUTFS_SEGMENT_ITEM_OFF_SHIFT) | - (item->val_len & SCOUTFS_SEGMENT_ITEM_LEN_MASK); - sitem->val_off_len = cpu_to_le32(packed); -} - static void kvec_from_pages(struct scoutfs_segment *seg, struct kvec *kvec, u32 off, u16 len) { @@ -459,17 +412,19 @@ int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct native_item item; + struct scoutfs_segment_item *item; if (pos < 0 || pos >= le32_to_cpu(sblk->nr_items)) return -ENOENT; - load_item(seg, pos, &item); + item = pos_ptr(seg, pos); if (key) - scoutfs_key_init(key, off_ptr(seg, item.key_off), item.key_len); + scoutfs_key_init(key, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); if (val) - kvec_from_pages(seg, val, item.val_off, item.val_len); + kvec_from_pages(seg, val, le32_to_cpu(item->val_off), + le16_to_cpu(item->val_len)); return 0; } @@ -564,7 +519,7 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct native_item item; + struct scoutfs_segment_item *item; struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); u32 key_off; @@ -582,12 +537,12 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, trace_printk("first item offs key %u val %u\n", key_off, val_off); - item.seq = 1; - item.key_off = key_off; - item.val_off = val_off; - item.key_len = key->key_len; - item.val_len = scoutfs_kvec_length(val); - store_item(seg, 0, &item); + item = pos_ptr(seg, 0); + item->seq = cpu_to_le64(1); + item->key_off = cpu_to_le32(key_off); + item->val_off = cpu_to_le32(val_off); + item->key_len = cpu_to_le16(key->key_len); + item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); scoutfs_seg_item_ptrs(seg, 0, &item_key, item_val); scoutfs_key_copy(&item_key, key); @@ -599,27 +554,33 @@ void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct native_item item; - struct native_item prev; + struct scoutfs_segment_item *item; + struct scoutfs_segment_item *prev; struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); + u32 key_off; + u32 val_off; u32 pos; pos = le32_to_cpu(sblk->nr_items); sblk->nr_items = cpu_to_le32(pos + 1); - load_item(seg, pos - 1, &prev); + prev = pos_ptr(seg, pos - 1); + item = pos_ptr(seg, pos); - item.seq = 1; - item.key_off = align_key_off(seg, prev.key_off + prev.key_len, - key->key_len); - item.key_len = key->key_len; - item.val_off = prev.val_off + prev.val_len; - item.val_len = scoutfs_kvec_length(val); - store_item(seg, pos, &item); + key_off = le32_to_cpu(prev->key_off) + le16_to_cpu(prev->key_len); + val_off = le32_to_cpu(prev->val_off) + le16_to_cpu(prev->val_len); + + key_off = align_key_off(seg, key_off, key->key_len); + + item->seq = cpu_to_le64(1); + item->key_off = cpu_to_le32(key_off); + item->val_off = cpu_to_le32(val_off); + item->key_len = cpu_to_le16(key->key_len); + item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); trace_printk("item %u offs key %u val %u\n", - pos, item.key_off, item.val_off); + pos, key_off, val_off); scoutfs_seg_item_ptrs(seg, pos, &item_key, item_val); scoutfs_key_copy(&item_key, key); @@ -633,15 +594,17 @@ int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct native_item item; + struct scoutfs_segment_item *item; struct scoutfs_key_buf first; struct scoutfs_key_buf last; - load_item(seg, 0, &item); - scoutfs_key_init(&first, off_ptr(seg, item.key_off), item.key_len); + item = pos_ptr(seg, 0); + scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); - load_item(seg, le32_to_cpu(sblk->nr_items) - 1, &item); - scoutfs_key_init(&last, off_ptr(seg, item.key_off), item.key_len); + item = pos_ptr(seg, le32_to_cpu(sblk->nr_items) - 1); + scoutfs_key_init(&last, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); return scoutfs_manifest_add(sb, &first, &last, le64_to_cpu(sblk->segno), le64_to_cpu(sblk->seq), level); @@ -651,11 +614,12 @@ int scoutfs_seg_manifest_del(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct native_item item; + struct scoutfs_segment_item *item; struct scoutfs_key_buf first; - load_item(seg, 0, &item); - scoutfs_key_init(&first, off_ptr(seg, item.key_off), item.key_len); + item = pos_ptr(seg, 0); + scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); } From 2ac239a4cb793f73518460c21b3eb2af78efe240 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 23 Jan 2017 17:54:18 -0800 Subject: [PATCH 209/920] Add deletion items So far we were only able to add items to the segments. To support deletion we have to insert deletion items and then remove them and the item they reference when their segments are compacted. As callers attempt to delete items from the item cache we replace the existing item with a deletion marker with the key but no value. Now that there are deletion items in the cache we have to teach the other item cache operations to skip them. There's some noise in the patch from moving functions around so that item insertion can free a deletion item it finds. The deletion items are written out to the segment as usual except now the in-segment item struct has a flag to mark a deletion item and the deletion item is removed from the cache once its written to the segment. Item reading knows to skip deletion items and not add them back into the cache. Compaction proceeds as usual for most of the levels with the deletion item clobbering any older higher level items with the same key. Eventually the deletion item itself is removed by skipping over it when compacting to the largest final level. We support this by adding a little call that describes the max level of the tree at the time the compaction starts so that compaction can tell when it should skip copying the deletion item to the final lower level. All of this is for deletion of items with a precise key. In the future we'll expand the deletion items so that they can reference a contiguous range of keys. Signed-off-by: Zach Brown --- kmod/src/compact.c | 78 ++++++++--- kmod/src/compact.h | 2 + kmod/src/counters.h | 1 + kmod/src/format.h | 5 +- kmod/src/item.c | 332 +++++++++++++++++++++++++++++++------------- kmod/src/manifest.c | 31 +++-- kmod/src/seg.c | 22 ++- kmod/src/seg.h | 12 +- 8 files changed, 345 insertions(+), 138 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index b6e0965d..a0f951cd 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -49,10 +49,6 @@ * Once the compaction is completed the manifest is updated to remove * the input segments and add the output segments. Here segment space * is reclaimed when the input items fit in fewer output segments. - * - * XXX today we only know how to skip duplicate individual items. We'll - * need to know how to skip lower based on upper range deletion items - * and to combine incremental update items. */ struct compact_info { @@ -85,6 +81,7 @@ struct compact_cursor { struct list_head csegs; u8 lower_level; + u8 last_level; struct compact_seg *upper; struct compact_seg *saved_upper; @@ -193,22 +190,25 @@ static struct compact_seg *next_spos(struct compact_cursor *curs, * the lowest key or the upper if they're the same. We advance the * cursor past the item that is returned. * - * XXX this will get fancier as we get range deletion items and incremental - * update items. + * XXX this will get fancier as we get range deletion items and + * incremental update items. */ static int next_item(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_key_buf *item_key, struct kvec *item_val) + struct scoutfs_key_buf *item_key, struct kvec *item_val, + u8 *item_flags) { struct compact_seg *upper = curs->upper; struct compact_seg *lower = curs->lower; struct scoutfs_key_buf lower_key; SCOUTFS_DECLARE_KVEC(lower_val); + u8 lower_flags; int cmp; int ret; +retry: if (upper) { ret = scoutfs_seg_item_ptrs(upper->seg, upper->pos, - item_key, item_val); + item_key, item_val, item_flags); if (ret < 0) upper = NULL; } @@ -219,7 +219,8 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, goto out; ret = scoutfs_seg_item_ptrs(lower->seg, lower->pos, - &lower_key, lower_val); + &lower_key, lower_val, + &lower_flags); if (ret == 0) break; lower = next_spos(curs, lower); @@ -246,6 +247,7 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, if (cmp > 0) { scoutfs_key_clone(item_key, &lower_key); scoutfs_kvec_clone(item_val, lower_val); + *item_flags = lower_flags; } if (cmp <= 0) @@ -253,6 +255,16 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, if (cmp >= 0) lower->pos++; + /* + * Deletion items make their way down all the levels, replacing + * all the duplicate items that they find. When we're + * compacting to the last level we can remove them by retrying + * the search after we've advanced past them. + */ + if ((curs->lower_level == curs->last_level) && + ((*item_flags) & SCOUTFS_ITEM_FLAG_DELETION)) + goto retry; + ret = 1; out: curs->upper = upper; @@ -273,12 +285,13 @@ static int count_items(struct super_block *sb, struct compact_cursor *curs, u32 items = 0; u32 keys = 0; u32 vals = 0; + u8 flags; int ret; *nr_items = 0; *key_bytes = 0; - while ((ret = next_item(sb, curs, &item_key, item_val)) > 0) { + while ((ret = next_item(sb, curs, &item_key, item_val, &flags)) > 0) { items++; keys += item_key.key_len; @@ -300,21 +313,22 @@ static int compact_items(struct super_block *sb, struct compact_cursor *curs, { struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); + u8 flags; int ret; - ret = next_item(sb, curs, &item_key, item_val); + ret = next_item(sb, curs, &item_key, item_val, &flags); if (ret <= 0) goto out; - scoutfs_seg_first_item(sb, seg, &item_key, item_val, + scoutfs_seg_first_item(sb, seg, &item_key, item_val, flags, nr_items, key_bytes); while (--nr_items) { - ret = next_item(sb, curs, &item_key, item_val); + ret = next_item(sb, curs, &item_key, item_val, &flags); if (ret <= 0) break; - scoutfs_seg_append_item(sb, seg, &item_key, item_val); + scoutfs_seg_append_item(sb, seg, &item_key, item_val, flags); } out: @@ -344,6 +358,12 @@ static int compact_segments(struct super_block *sb, /* * We can just move the upper segment down a level if it * doesn't intersect any lower segments. + * + * XXX we can't do this if the segment we're moving has + * deletion items. We need to copy the non-deletion items + * and drop the deletion items in that case. To do that + * we'll need the manifest to count the number of deletion + * and non-deletion items. */ if (upper && upper->pos == 0 && (!lower || @@ -383,11 +403,14 @@ static int compact_segments(struct super_block *sb, /* * We can skip a lower segment if there's no upper segment * or the next upper item is past the last in the lower. + * + * XXX this will need to test for intersection with range + * deletion items. */ if (lower && lower->pos == 0 && (!upper || (!scoutfs_seg_item_ptrs(upper->seg, upper->pos, - &upper_next, NULL) && + &upper_next, NULL, NULL) && scoutfs_key_compare(&upper_next, lower->last) > 0))) { curs->lower = next_spos(curs, lower); @@ -447,6 +470,25 @@ static int compact_segments(struct super_block *sb, return ret; } +/* + * Manifest walking is providing the details of the overall compaction + * operation. It'll then add all the segments involved. + */ +void scoutfs_compact_describe(struct super_block *sb, void *data, + u8 upper_level, u8 last_level) +{ + struct compact_cursor *curs = data; + + curs->lower_level = upper_level + 1; + curs->last_level = last_level; +} + +/* + * Add a segment involved in the compaction operation. + * + * XXX Today we know that the caller is always adding only one upper segment + * and is then possibly adding all the lower overlapping segments. + */ int scoutfs_compact_add(struct super_block *sb, void *data, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, u64 segno, u64 seq, @@ -468,12 +510,10 @@ int scoutfs_compact_add(struct super_block *sb, void *data, cseg->seq = seq; cseg->level = level; - if (!curs->upper) { + if (!curs->upper) curs->upper = cseg; - } else if (!curs->lower) { + else if (!curs->lower) curs->lower = cseg; - curs->lower_level = level; - } ret = 0; out: diff --git a/kmod/src/compact.h b/kmod/src/compact.h index 5241ff11..d3654fd3 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -3,6 +3,8 @@ void scoutfs_compact_kick(struct super_block *sb); +void scoutfs_compact_describe(struct super_block *sb, void *data, + u8 upper_level, u8 last_level); int scoutfs_compact_add(struct super_block *sb, void *data, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, u64 segno, u64 seq, diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a473523d..86787796 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -27,6 +27,7 @@ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ + EXPAND_COUNTER(item_delete) \ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ EXPAND_COUNTER(item_range_insert) diff --git a/kmod/src/format.h b/kmod/src/format.h index 901d4a66..a2931169 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -137,9 +137,12 @@ struct scoutfs_segment_item { __le32 val_off; __le16 key_len; __le16 val_len; - __u8 padding[12]; + __u8 padding[11]; + __u8 flags; } __packed; +#define SCOUTFS_ITEM_FLAG_DELETION (1 << 0) + /* * Each large segment starts with a segment block that describes the * rest of the blocks that make up the segment. diff --git a/kmod/src/item.c b/kmod/src/item.c index 85c4942b..b3300598 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -33,6 +33,11 @@ * that are completely described by the items. This lets it return * negative lookups cache hits for items that don't exist without having * to constantly perform expensive segment searches. + * + * Deletions are recorded with items in the rbtree which record the key + * of the deletion. They're removed once they're written to a level0 + * segment. While they're present in the cache we have to be careful to + * clobber them in creation and skip them in lookups. */ struct item_cache { @@ -57,7 +62,9 @@ struct cached_item { struct rb_node node; struct list_head entry; }; + long dirty; + unsigned deletion:1; struct scoutfs_key_buf *key; @@ -71,6 +78,38 @@ struct cached_range { struct scoutfs_key_buf *end; }; +static u8 item_flags(struct cached_item *item) +{ + return item->deletion ? SCOUTFS_ITEM_FLAG_DELETION : 0; +} + +static void free_item(struct super_block *sb, struct cached_item *item) +{ + if (!IS_ERR_OR_NULL(item)) { + scoutfs_key_free(sb, item->key); + scoutfs_kvec_kfree(item->val); + kfree(item); + } +} + +static struct cached_item *alloc_item(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val) +{ + struct cached_item *item; + + item = kzalloc(sizeof(struct cached_item), GFP_NOFS); + if (item) { + item->key = scoutfs_key_dup(sb, key); + if (!item->key || scoutfs_kvec_dup_flatten(item->val, val)) { + free_item(sb, item); + item = NULL; + } + } + + return item; +} + /* * Walk the item rbtree and return the item found and the next and * prev items. @@ -105,6 +144,14 @@ static struct cached_item *walk_items(struct rb_root *root, return NULL; } +/* + * Look for the item with the given key. Callers of this are looking + * for existing items. They would just return -ENOENT from a deletion + * item if we gave it to them so we return null for deletion items. + * Callers that would remove a deletion item before inserting a new + * version of the item do so by having insert_item() replace existing + * deleted items on their behalf. + */ static struct cached_item *find_item(struct super_block *sb, struct rb_root *root, struct scoutfs_key_buf *key) @@ -115,6 +162,9 @@ static struct cached_item *find_item(struct super_block *sb, item = walk_items(root, key, &prev, &next); + if (item && item->deletion) + item = NULL; + if (item) scoutfs_inc_counter(sb, item_lookup_hit); else @@ -224,11 +274,64 @@ static const struct rb_augment_callbacks scoutfs_item_rb_cb = { }; /* - * Try to insert the given item. If there's already an item with the - * insertion key then return -EEXIST. + * The caller has changed an item's dirty bit. Its child dirty bits are + * still consistent. But its parent's bits might need to be updated. + * Its bits are consistent so we don't propagate from the node itself + * because it would immediately terminate. */ -static int insert_item(struct rb_root *root, struct cached_item *ins) +static void update_dirty_parents(struct cached_item *item) { + scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); +} + +static void mark_item_dirty(struct item_cache *cac, + struct cached_item *item) +{ + if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) + return; + + if (item->dirty & ITEM_DIRTY) + return; + + item->dirty |= ITEM_DIRTY; + cac->nr_dirty_items++; + cac->dirty_key_bytes += item->key->key_len; + cac->dirty_val_bytes += scoutfs_kvec_length(item->val); + + update_dirty_parents(item); +} + +static void clear_item_dirty(struct item_cache *cac, + struct cached_item *item) +{ + if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) + return; + + if (!(item->dirty & ITEM_DIRTY)) + return; + + item->dirty &= ~ITEM_DIRTY; + cac->nr_dirty_items--; + cac->dirty_key_bytes -= item->key->key_len; + cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); + + WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || + cac->dirty_val_bytes < 0); + + update_dirty_parents(item); +} + +/* + * Try to insert the given item. If there's already a non-deletion item + * with the insertion key then return -EEXIST. An existing deletion + * item is replaced and freed. + * + * The caller is responsible for marking the newly inserted item dirty. + */ +static int insert_item(struct super_block *sb, struct item_cache *cac, + struct cached_item *ins) +{ + struct rb_root *root = &cac->items; struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; struct cached_item *item; @@ -248,7 +351,13 @@ static int insert_item(struct rb_root *root, struct cached_item *ins) item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; } else { - return -EEXIST; + if (!item->deletion) + return -EEXIST; + + clear_item_dirty(cac, item); + rb_replace_node(&item->node, &ins->node, root); + free_item(sb, item); + return 0; } } @@ -443,6 +552,43 @@ int scoutfs_item_lookup_exact(struct super_block *sb, return ret; } +/* + * Find the next item to return from the "_next" item interface. It's the + * next item from the key that isn't a deletion item and is within the + * bounds of the end of the cache and the caller's last key. + */ +static struct cached_item *item_for_next(struct rb_root *root, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *range_end, + struct scoutfs_key_buf *last) +{ + struct cached_item *item; + struct rb_node *node; + + /* limit by the lesser of the two */ + if (scoutfs_key_compare(range_end, last) < 0) + last = range_end; + + item = next_item(root, key); + while (item) { + if (scoutfs_key_compare(item->key, last) > 0) { + item = NULL; + break; + } + + if (!item->deletion) + break; + + node = rb_next(&item->node); + if (node) + item = container_of(node, struct cached_item, node); + else + item = NULL; + } + + return item; +} + /* * Return the next item starting with the given key, returning the last * key at the most. @@ -490,10 +636,8 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, /* see if we have a usable item in cache and before last */ cached = check_range(sb, &cac->ranges, key, range_end); - if (cached && (item = next_item(&cac->items, key)) && - scoutfs_key_compare(item->key, range_end) <= 0 && - scoutfs_key_compare(item->key, last) <= 0) { - + if (cached && (item = item_for_next(&cac->items, key, + range_end, last))) { scoutfs_key_copy(key, item->key); if (val) ret = scoutfs_kvec_memcpy(val, item->val); @@ -565,81 +709,6 @@ int scoutfs_item_next_same_min(struct super_block *sb, return ret; } -static void free_item(struct super_block *sb, struct cached_item *item) -{ - if (!IS_ERR_OR_NULL(item)) { - scoutfs_key_free(sb, item->key); - scoutfs_kvec_kfree(item->val); - kfree(item); - } -} - -/* - * The caller has changed an item's dirty bit. Its child dirty bits are - * still consistent. But its parent's bits might need to be updated. - * Its bits are consistent so we don't propagate from the node itself - * because it would immediately terminate. - */ -static void update_dirty_parents(struct cached_item *item) -{ - scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); -} - -static void mark_item_dirty(struct item_cache *cac, - struct cached_item *item) -{ - if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) - return; - - if (item->dirty & ITEM_DIRTY) - return; - - item->dirty |= ITEM_DIRTY; - cac->nr_dirty_items++; - cac->dirty_key_bytes += item->key->key_len; - cac->dirty_val_bytes += scoutfs_kvec_length(item->val); - - update_dirty_parents(item); -} - -static void clear_item_dirty(struct item_cache *cac, - struct cached_item *item) -{ - if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) - return; - - if (!(item->dirty & ITEM_DIRTY)) - return; - - item->dirty &= ~ITEM_DIRTY; - cac->nr_dirty_items--; - cac->dirty_key_bytes -= item->key->key_len; - cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); - - WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || - cac->dirty_val_bytes < 0); - - update_dirty_parents(item); -} - -static struct cached_item *alloc_item(struct super_block *sb, - struct scoutfs_key_buf *key, - struct kvec *val) -{ - struct cached_item *item; - - item = kzalloc(sizeof(struct cached_item), GFP_NOFS); - if (item) { - item->key = scoutfs_key_dup(sb, key); - if (!item->key || scoutfs_kvec_dup_flatten(item->val, val)) { - free_item(sb, item); - item = NULL; - } - } - - return item; -} - /* * Create a new dirty item in the cache. Returns -EEXIST if an item * already exists with the given key. @@ -660,7 +729,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return -ENOMEM; spin_lock_irqsave(&cac->lock, flags); - ret = insert_item(&cac->items, item); + ret = insert_item(sb, cac, item); if (!ret) { scoutfs_inc_counter(sb, item_create); mark_item_dirty(cac, item); @@ -745,7 +814,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del(&item->entry); - if (insert_item(&cac->items, item)) + if (insert_item(sb, cac, item)) list_add(&item->entry, list); } @@ -871,12 +940,61 @@ out: } /* - * XXX how nice, it'd just creates a cached deletion item. It doesn't - * have to read. + * Delete an existing item with the given key. + * + * If a non-deletion item is present then we mark it dirty and deleted + * and free it's value. + * + * Returns -ENOENT if an item doesn't exist at the key. This forces us + * to read the item before creating a deletion item for it. XXX If we + * relaxed this we'd need to see if callers make use of -ENOENT and if + * there are any ways for userspace to overwhelm the system with + * deletion items for items that didn't exist in the first place. */ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) { - return WARN_ON_ONCE(-EINVAL); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct scoutfs_key_buf *end; + struct cached_item *item; + SCOUTFS_DECLARE_KVEC(del_val); + unsigned long flags; + int ret; + + scoutfs_kvec_init_null(del_val); + + end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!end) { + ret = -ENOMEM; + goto out; + } + + do { + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + if (item) { + scoutfs_kvec_swap(item->val, del_val); + item->deletion = 1; + mark_item_dirty(cac, item); + scoutfs_inc_counter(sb, item_delete); + ret = 0; + } else if (check_range(sb, &cac->ranges, key, end)) { + ret = -ENOENT; + } else { + ret = -ENODATA; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + + scoutfs_key_free(sb, end); + scoutfs_kvec_kfree(del_val); +out: + trace_printk("ret %d\n", ret); + return ret; } /* @@ -1020,28 +1138,46 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, * segments and can be partially visible if we only write the first * segment. We probably want to throttle trans enters once we have as * many dirty items as our atomic segment updates can write. + * + * XXX this first/append pattern will go away once we can write a stream + * of items to a segment without needing to know the item count to + * find the starting key and value offsets. */ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct cached_item *item; + struct cached_item *item = NULL; + struct cached_item *del; u32 key_bytes; u32 nr_items; count_seg_items(cac, &nr_items, &key_bytes); - item = first_dirty(cac->items.rb_node); - if (item) { - scoutfs_seg_first_item(sb, seg, item->key, item->val, - nr_items, key_bytes); - clear_item_dirty(cac, item); - nr_items--; - } + /* remember nr_items is passed to _first_item */ + while (nr_items) { + + if (!item) { + item = first_dirty(cac->items.rb_node); + scoutfs_seg_first_item(sb, seg, item->key, item->val, + item_flags(item), nr_items, + key_bytes); + } else { + scoutfs_seg_append_item(sb, seg, item->key, item->val, + item_flags(item)); + } - while (nr_items-- && (item = next_dirty(item))) { - scoutfs_seg_append_item(sb, seg, item->key, item->val); clear_item_dirty(cac, item); + + del = item; + item = next_dirty(item); + + if (del->deletion) { + rb_erase(&del->node, &cac->items); + free_item(sb, del); + } + + nr_items--; } return 0; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 98ac6234..f8c4edb5 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -459,6 +459,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct manifest_ref *tmp; LIST_HEAD(ref_list); LIST_HEAD(batch); + u8 found_flags = 0; + u8 item_flags; int found_ctr; bool found; int ret = 0; @@ -534,7 +536,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, * caller's end. */ ret = scoutfs_seg_item_ptrs(ref->seg, ref->pos, - &item_key, item_val); + &item_key, item_val, + &item_flags); if (ret < 0 || scoutfs_key_compare(&item_key, end) > 0){ ref->pos = -1; continue; @@ -554,6 +557,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, /* remember new least key */ scoutfs_key_clone(&found_key, &item_key); scoutfs_kvec_clone(found_val, item_val); + found_flags = item_flags; ref->found_ctr = ++found_ctr; found = true; } @@ -566,15 +570,22 @@ int scoutfs_manifest_read_items(struct super_block *sb, } /* + * Add the next found item to the batch if it's not a + * deletion item. We still need to use their key to + * remember the end of the batch for negative caching. + * * If we fail to add an item we're done. If we already - * have items it's not a failure and the end of the cached - * range is the last successfully added item. + * have items it's not a failure and the end of the + * cached range is the last successfully added item. */ - ret = scoutfs_item_add_batch(sb, &batch, &found_key, found_val); - if (ret) { - if (n > 0) - ret = 0; - break; + if (!(found_flags & SCOUTFS_ITEM_FLAG_DELETION)) { + ret = scoutfs_item_add_batch(sb, &batch, &found_key, + found_val); + if (ret) { + if (n > 0) + ret = 0; + break; + } } /* the last successful key determines range end until run out */ @@ -699,6 +710,8 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } + scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1); + /* find the oldest level 0 or the next higher order level by key */ if (level == 0) { ment = scoutfs_treap_first(mani->treap); @@ -737,7 +750,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) skey.key = &ment_first; skey.level = level + 1; skey.seq = 0; - over = scoutfs_treap_lookup(mani->treap, &skey); + over = scoutfs_treap_lookup_next(mani->treap, &skey); /* and add a fanout's worth of lower overlapping segments */ for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 4687b7af..b343cb5e 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -409,7 +409,8 @@ static void kvec_from_pages(struct scoutfs_segment *seg, } int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, - struct scoutfs_key_buf *key, struct kvec *val) + struct scoutfs_key_buf *key, struct kvec *val, + u8 *flags) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; @@ -425,6 +426,8 @@ int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, if (val) kvec_from_pages(seg, val, le32_to_cpu(item->val_off), le16_to_cpu(item->val_len)); + if (flags) + *flags = item->flags; return 0; } @@ -446,7 +449,7 @@ static int find_key_pos(struct scoutfs_segment *seg, while (start < end) { pos = start + (end - start) / 2; - scoutfs_seg_item_ptrs(seg, pos, &key, NULL); + scoutfs_seg_item_ptrs(seg, pos, &key, NULL, NULL); cmp = scoutfs_key_compare(search, &key); if (cmp < 0) @@ -512,9 +515,11 @@ static u32 align_key_off(struct scoutfs_segment *seg, u32 key_off, u32 len) * * This should never fail because any item must always fit in a segment. */ -void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, +void scoutfs_seg_first_item(struct super_block *sb, + struct scoutfs_segment *seg, struct scoutfs_key_buf *key, struct kvec *val, - unsigned int nr_items, unsigned int key_bytes) + u8 flags, unsigned int nr_items, + unsigned int key_bytes) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; @@ -543,15 +548,17 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, item->val_off = cpu_to_le32(val_off); item->key_len = cpu_to_le16(key->key_len); item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); + item->flags = flags; - scoutfs_seg_item_ptrs(seg, 0, &item_key, item_val); + scoutfs_seg_item_ptrs(seg, 0, &item_key, item_val, NULL); scoutfs_key_copy(&item_key, key); scoutfs_kvec_memcpy(item_val, val); } void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val) + struct scoutfs_key_buf *key, struct kvec *val, + u8 flags) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; @@ -578,11 +585,12 @@ void scoutfs_seg_append_item(struct super_block *sb, item->val_off = cpu_to_le32(val_off); item->key_len = cpu_to_le16(key->key_len); item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); + item->flags = flags; trace_printk("item %u offs key %u val %u\n", pos, key_off, val_off); - scoutfs_seg_item_ptrs(seg, pos, &item_key, item_val); + scoutfs_seg_item_ptrs(seg, pos, &item_key, item_val, NULL); scoutfs_key_copy(&item_key, key); scoutfs_kvec_memcpy(item_val, val); } diff --git a/kmod/src/seg.h b/kmod/src/seg.h index e43b2268..15c6834f 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -13,7 +13,8 @@ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_seg_find_pos(struct scoutfs_segment *seg, struct scoutfs_key_buf *key); int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, - struct scoutfs_key_buf *key, struct kvec *val); + struct scoutfs_key_buf *key, struct kvec *val, + u8 *flags); void scoutfs_seg_get(struct scoutfs_segment *seg); void scoutfs_seg_put(struct scoutfs_segment *seg); @@ -22,12 +23,15 @@ int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); -void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment *seg, +void scoutfs_seg_first_item(struct super_block *sb, + struct scoutfs_segment *seg, struct scoutfs_key_buf *key, struct kvec *val, - unsigned int nr_items, unsigned int key_bytes); + u8 flags, unsigned int nr_items, + unsigned int key_bytes); void scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val); + struct scoutfs_key_buf *key, struct kvec *val, + u8 flags); int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level); int scoutfs_seg_manifest_del(struct super_block *sb, From 8f631963186e5b4a3b1d5a875511b83aa9cfeb3a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jan 2017 14:42:56 -0800 Subject: [PATCH 210/920] Add key inc/dec variants for partial keys Some callers know that it's safe to increment their partial keys. Let them do so without trying to expand the keys to full precision and triggering warnings that their buffers aren't large enough. Signed-off-by: Zach Brown --- kmod/src/key.c | 24 ++++++++++++++++++++---- kmod/src/key.h | 2 ++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/kmod/src/key.c b/kmod/src/key.c index 9795f797..04891839 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -65,20 +65,30 @@ static void extend_zeros(struct scoutfs_key_buf *key) } } -void scoutfs_key_inc(struct scoutfs_key_buf *key) +/* + * There are callers that work with a range of keys of a uniform length + * who know that it's safe to increment their keys that aren't full + * precision. These are exceptional so a specific function variant + * marks them. + */ +void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key) { u8 *bytes = key->data; int i; - extend_zeros(key); - for (i = key->key_len - 1; i >= 0; i--) { if (++bytes[i] != 0) break; } } -void scoutfs_key_dec(struct scoutfs_key_buf *key) +void scoutfs_key_inc(struct scoutfs_key_buf *key) +{ + extend_zeros(key); + scoutfs_key_inc_cur_len(key); +} + +void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key) { u8 *bytes = key->data; int i; @@ -90,3 +100,9 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) break; } } + +void scoutfs_key_dec(struct scoutfs_key_buf *key) +{ + extend_zeros(key); + scoutfs_key_dec_cur_len(key); +} diff --git a/kmod/src/key.h b/kmod/src/key.h index a63244ba..2ed2f3f6 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -15,7 +15,9 @@ struct scoutfs_key_buf *scoutfs_key_dup(struct super_block *sb, struct scoutfs_key_buf *key); void scoutfs_key_free(struct super_block *sb, struct scoutfs_key_buf *key); void scoutfs_key_inc(struct scoutfs_key_buf *key); +void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key); void scoutfs_key_dec(struct scoutfs_key_buf *key); +void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); /* From 9d68e272ccdb2c6d3b017819abb61485b394e1ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jan 2017 14:44:11 -0800 Subject: [PATCH 211/920] Allow creation of items with no value Item creation always tried to allocate a value. We have some item types which don't have values. Signed-off-by: Zach Brown --- kmod/src/item.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index b3300598..45755724 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -100,8 +100,12 @@ static struct cached_item *alloc_item(struct super_block *sb, item = kzalloc(sizeof(struct cached_item), GFP_NOFS); if (item) { + if (!val) + scoutfs_kvec_init_null(item->val); + item->key = scoutfs_key_dup(sb, key); - if (!item->key || scoutfs_kvec_dup_flatten(item->val, val)) { + if (!item->key || + (val && scoutfs_kvec_dup_flatten(item->val, val))) { free_item(sb, item); item = NULL; } From 9d6d70bd891d6afa97887040bb1119681572ffe7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jan 2017 14:46:04 -0800 Subject: [PATCH 212/920] Add an item next for key len ignoring val Add scoutfs_item_next_same() which requires that the key lengths be identical but which allows any values, including no value by way of a null kvec. Signed-off-by: Zach Brown --- kmod/src/item.c | 21 +++++++++++++++++++++ kmod/src/item.h | 2 ++ 2 files changed, 23 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 45755724..c4dff761 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -713,6 +713,27 @@ int scoutfs_item_next_same_min(struct super_block *sb, return ret; } +/* + * Like _next but requires that the found keys be the same length as the + * search key. It treats size mismatches as a sign of corruption. + */ +int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, struct kvec *val) +{ + int key_len = key->key_len; + int ret; + + trace_printk("key len %u\n", key_len); + + ret = scoutfs_item_next(sb, key, last, val); + if (ret >= 0 && (key->key_len != key_len)) + ret = -EIO; + + trace_printk("ret %d\n", ret); + + return ret; +} + /* * Create a new dirty item in the cache. Returns -EEXIST if an item * already exists with the given key. diff --git a/kmod/src/item.h b/kmod/src/item.h index 3c7c6057..02afc91c 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -17,6 +17,8 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, int len); +int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *last, struct kvec *val); int scoutfs_item_insert(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, From f139cf4a5e6b9c8b617b8cfca7167a28e8ba9bc5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jan 2017 14:47:31 -0800 Subject: [PATCH 213/920] Convert unlink and orphan processing Restore unlink functionality by converting unlink and orphan item processing from the old btree interface to the new item cache interface. Signed-off-by: Zach Brown --- kmod/src/dir.c | 3 -- kmod/src/format.h | 6 ++++ kmod/src/inode.c | 86 ++++++++++++++++++++++++++--------------------- 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index c67675b5..fca91feb 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -451,9 +451,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct scoutfs_key_buf *key = NULL; int ret = 0; - /* will need to add deletion items */ - return -EINVAL; - if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; diff --git a/kmod/src/format.h b/kmod/src/format.h index a2931169..28125b6c 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -262,6 +262,12 @@ struct scoutfs_readdir_key { __be64 pos; } __packed; +/* no value */ +struct scoutfs_orphan_key { + __u8 type; + __be64 ino; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 48864f94..eeda6cb7 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -431,27 +431,35 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, return inode; } +static void init_orphan_key(struct scoutfs_key_buf *key, + struct scoutfs_orphan_key *okey, u64 ino) +{ + okey->type = SCOUTFS_ORPHAN_KEY; + okey->ino = cpu_to_be64(ino); + + scoutfs_key_init(key, okey, sizeof(struct scoutfs_orphan_key)); +} + static int remove_orphan_item(struct super_block *sb, u64 ino) { - struct scoutfs_key key; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_orphan_key okey; + struct scoutfs_key_buf key; int ret; - scoutfs_set_key(&key, ino, SCOUTFS_ORPHAN_KEY, 0); + init_orphan_key(&key, &okey, ino); - ret = scoutfs_btree_delete(sb, meta, &key); + ret = scoutfs_item_delete(sb, &key); if (ret == -ENOENT) ret = 0; return ret; } -static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, +static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, u64 ino, umode_t mode) { - int ret; bool release = false; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + int ret; trace_delete_inode(sb, ino, mode); @@ -460,6 +468,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, goto out; release = true; +#if 0 ret = scoutfs_xattr_drop(sb, ino); if (ret) goto out; @@ -471,7 +480,8 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key *key, if (ret) goto out; - ret = scoutfs_btree_delete(sb, meta, key); +#endif + ret = scoutfs_item_delete(sb, key); if (ret) goto out; @@ -487,19 +497,18 @@ out: */ static void delete_inode(struct super_block *sb, u64 ino) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; struct scoutfs_inode sinode; - struct scoutfs_key key; + struct scoutfs_inode_key ikey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); umode_t mode; int ret; /* sample the inode mode, XXX don't need to copy whole thing here */ - scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); - val.check_size_eq = 1; + init_inode_key(&key, &ikey, ino); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_btree_lookup(sb, meta, &key, &val); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); if (ret < 0) goto out; @@ -544,17 +553,16 @@ int scoutfs_drop_inode(struct inode *inode) static int process_orphaned_inode(struct super_block *sb, u64 ino) { - int ret; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; + struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - struct scoutfs_key key; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + int ret; - scoutfs_set_key(&key, ino, SCOUTFS_INODE_KEY, 0); - scoutfs_btree_init_val(&val, &sinode, sizeof(sinode)); - val.check_size_eq = 1; + init_inode_key(&key, &ikey, ino); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_btree_lookup(sb, meta, &key, &val); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -570,7 +578,7 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) } /* - * Scan the metadata tree for orphan items and process each one. + * Find orphan items and process each one. * * Runtime of this will be bounded by the number of orphans, which could * theoretically be very large. If that becomes a problem we might want to push @@ -578,28 +586,30 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) */ int scoutfs_scan_orphans(struct super_block *sb) { - int ret, err = 0; - struct scoutfs_key first, last, found; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_orphan_key okey; + struct scoutfs_orphan_key last_okey; + struct scoutfs_key_buf key; + struct scoutfs_key_buf last; + int err = 0; + int ret; trace_scoutfs_scan_orphans(sb); - scoutfs_set_key(&first, 0, SCOUTFS_ORPHAN_KEY, 0); - scoutfs_set_key(&last, ~0ULL, SCOUTFS_ORPHAN_KEY, 0); + init_orphan_key(&key, &okey, 0); + init_orphan_key(&last, &last_okey, ~0ULL); while (1) { - ret = scoutfs_btree_next(sb, meta, &first, &last, &found, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) goto out; - ret = process_orphaned_inode(sb, le64_to_cpu(found.inode)); + ret = process_orphaned_inode(sb, be64_to_cpu(okey.ino)); if (ret && ret != -ENOENT && !err) err = ret; - first = found; - scoutfs_inc_key(&first); + scoutfs_key_inc_cur_len(&key); } ret = 0; @@ -609,16 +619,16 @@ out: int scoutfs_orphan_inode(struct inode *inode) { - int ret; struct super_block *sb = inode->i_sb; - struct scoutfs_key key; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_orphan_key okey; + struct scoutfs_key_buf key; + int ret; trace_scoutfs_orphan_inode(sb, inode); - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_ORPHAN_KEY, 0); + init_orphan_key(&key, &okey, scoutfs_ino(inode)); - ret = scoutfs_btree_insert(sb, meta, &key, NULL); + ret = scoutfs_item_create(sb, &key, NULL); return ret; } From 9a293bfa758797ecb92c1aaf22e092dd6b4cf5ff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 25 Jan 2017 11:08:43 -0800 Subject: [PATCH 214/920] Add item delete dirty and many interfaces Add item functions for deleting items that we know to be dirty and add a user in another function that deletes many items without leaving parial deletions behind in the case of errors. Signed-off-by: Zach Brown --- kmod/src/item.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++--- kmod/src/item.h | 4 +++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index c4dff761..d471cab6 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -964,6 +964,23 @@ out: return ret; } +/* + * Turn an item that the caller has found while holding the lock into a + * deletion item. The caller will free whatever we put in the deletion + * value after releasing the lock. + */ +static void become_deletion_item(struct super_block *sb, + struct item_cache *cac, + struct cached_item *item, + struct kvec *del_val) +{ + scoutfs_kvec_clone(del_val, item->val); + scoutfs_kvec_init_null(item->val); + item->deletion = 1; + mark_item_dirty(cac, item); + scoutfs_inc_counter(sb, item_delete); +} + /* * Delete an existing item with the given key. * @@ -999,10 +1016,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) item = find_item(sb, &cac->items, key); if (item) { - scoutfs_kvec_swap(item->val, del_val); - item->deletion = 1; - mark_item_dirty(cac, item); - scoutfs_inc_counter(sb, item_delete); + become_deletion_item(sb, cac, item, del_val); ret = 0; } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; @@ -1022,6 +1036,62 @@ out: return ret; } +/* + * Delete an item that the caller knows must be dirty because they hold + * locks and the transaction and have created or dirtied it. This can't + * fail. + */ +void scoutfs_item_delete_dirty(struct super_block *sb, + struct scoutfs_key_buf *key) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + SCOUTFS_DECLARE_KVEC(del_val); + struct cached_item *item; + unsigned long flags; + + scoutfs_kvec_init_null(del_val); + + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + if (item) + become_deletion_item(sb, cac, item, del_val); + + spin_unlock_irqrestore(&cac->lock, flags); + + scoutfs_kvec_kfree(del_val); +} + +/* + * A helper that deletes a set of items. It first dirties the items + * will be pinned so that deletion won't fail as it tries to read and + * populate the items. + * + * It's a little cleaner to have this helper than have the caller + * iterate, but it could also give us the opportunity to reduce item + * searches if we remembered the items we dirtied. + */ +int scoutfs_item_delete_many(struct super_block *sb, + struct scoutfs_key_buf **keys, unsigned nr) +{ + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + ret = scoutfs_item_dirty(sb, keys[i]); + if (ret) + goto out; + } + + for (i = 0; i < nr; i++) + scoutfs_item_delete_dirty(sb, keys[i]); + +out: + trace_printk("ret %d\n", ret); + return ret; +} + /* * Return the first dirty node in the subtree starting at the given node. */ diff --git a/kmod/src/item.h b/kmod/src/item.h index 02afc91c..8f84389a 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -26,6 +26,10 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); +void scoutfs_item_delete_dirty(struct super_block *sb, + struct scoutfs_key_buf *key); +int scoutfs_item_delete_many(struct super_block *sb, + struct scoutfs_key_buf **keys, unsigned nr); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, From 67aec72c77d257b32df77e2a29f59235f43b1cb9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 25 Jan 2017 11:12:22 -0800 Subject: [PATCH 215/920] Add readdir items Restore readdir functionality by adding readdir items. The readdir items are keyed by an increasing position in the parent dir's inode. We track it in our inode info. To delete the readdir items we restore the dentry_info and put the pos in the dentry so unlink can build the readdir item key. And finally we put the pos in the lookup dirent so that it can populate the dentry info on lookup. Signed-off-by: Zach Brown --- kmod/src/dir.c | 194 +++++++++++++++++++++++++++++++++++++--------- kmod/src/dir.h | 3 + kmod/src/format.h | 3 + kmod/src/inode.c | 3 + kmod/src/inode.h | 1 + kmod/src/super.c | 2 + 6 files changed, 170 insertions(+), 36 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index fca91feb..0cf55a13 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -97,6 +97,77 @@ static unsigned int dentry_type(unsigned int type) return DT_UNKNOWN; } +/* + * Each dentry stores the values that are needed to build the keys of + * the items that are removed on unlink so that we don't to search + * through items on unlink. + */ +struct dentry_info { + u64 readdir_pos; +}; + +static struct kmem_cache *dentry_info_cache; + +static void scoutfs_d_release(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (di) { + kmem_cache_free(dentry_info_cache, di); + dentry->d_fsdata = NULL; + } +} + +static const struct dentry_operations scoutfs_dentry_ops = { + .d_release = scoutfs_d_release, +}; + +static int alloc_dentry_info(struct dentry *dentry) +{ + struct dentry_info *di; + + /* XXX read mb? */ + if (dentry->d_fsdata) + return 0; + + di = kmem_cache_zalloc(dentry_info_cache, GFP_NOFS); + if (!di) + return -ENOMEM; + + spin_lock(&dentry->d_lock); + if (!dentry->d_fsdata) { + dentry->d_fsdata = di; + d_set_d_op(dentry, &scoutfs_dentry_ops); + } + spin_unlock(&dentry->d_lock); + + if (di != dentry->d_fsdata) + kmem_cache_free(dentry_info_cache, di); + + return 0; +} + +static void update_dentry_info(struct dentry *dentry, + struct scoutfs_dirent *dent) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return; + + di->readdir_pos = le64_to_cpu(dent->readdir_pos); +} + +static u64 dentry_info_pos(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return 0; + + return di->readdir_pos; +} + static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, struct inode *dir, struct dentry *dentry) @@ -133,6 +204,10 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } + ret = alloc_dentry_info(dentry); + if (ret) + goto out; + key = alloc_dirent_key(sb, dir, dentry); if (!key) { ret = -ENOMEM; @@ -147,6 +222,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, ret = 0; } else if (ret == 0) { ino = le64_to_cpu(dent.ino); + update_dentry_info(dentry, &dent); } out: @@ -286,11 +362,23 @@ static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, static int add_entry_items(struct inode *dir, struct dentry *dentry, struct inode *inode) { + struct scoutfs_inode_info *si = SCOUTFS_I(dir); + struct dentry_info *di = dentry->d_fsdata; struct super_block *sb = dir->i_sb; - struct scoutfs_key_buf *key; + struct scoutfs_key_buf *ent_key = NULL; + struct scoutfs_key_buf *del_keys[3]; + struct scoutfs_key_buf rdir_key; + struct scoutfs_readdir_key rkey; struct scoutfs_dirent dent; SCOUTFS_DECLARE_KVEC(val); + int del = 0; + u64 pos; int ret; + int err; + + /* caller should have allocated the dentry info */ + if (WARN_ON_ONCE(di == NULL)) + return -EINVAL; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; @@ -299,55 +387,54 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) return ret; + /* initialize the dent */ + pos = si->next_readdir_pos++; + dent.ino = cpu_to_le64(scoutfs_ino(inode)); + dent.readdir_pos = cpu_to_le64(pos); + dent.type = mode_to_type(inode->i_mode); + /* dirent item for lookup */ - key = alloc_dirent_key(sb, dir, dentry); - if (!key) + ent_key = alloc_dirent_key(sb, dir, dentry); + if (!ent_key) return -ENOMEM; - dent.ino = cpu_to_le64(scoutfs_ino(inode)); - dent.type = mode_to_type(inode->i_mode); scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_create(sb, key, val); + ret = scoutfs_item_create(sb, ent_key, val); if (ret) - return ret; - -#if 0 - struct scoutfs_inode_info *si = SCOUTFS_I(dir); + goto out; + del_keys[del++] = ent_key; /* readdir item for .. readdir */ - si->readdir_pos++; - rkey.type = SCOUTFS_READDIR_KEY; - rkey.ino = cpu_to_le64(scoutfs_ino(dir)); - rkey.pos = cpu_to_le64(si->readdir_pos); - scoutfs_kvec_init(key, &rkey, sizeof(rkey)); - + init_readdir_key(&rdir_key, &rkey, dir, pos); scoutfs_kvec_init(val, &dent, sizeof(dent), - dentry->d_name.name, dentry->d_name.len); + (void *)dentry->d_name.name, dentry->d_name.len); - ret = scoutfs_item_create(sb, key, val); + ret = scoutfs_item_create(sb, &rdir_key, val); if (ret) - goto out_dent; + goto out; + del_keys[del++] = &rdir_key; +#if 0 /* backref item for inode to path resolution */ lrkey.type = SCOUTFS_LINK_BACKREF_KEY; lrey.ino = cpu_to_le64(scoutfs_ino(inode)); lrey.dir = cpu_to_le64(scoutfs_ino(dir)); scoutfs_kvec_init(key, &lrkey, sizeof(lrkey), dentry->d_name.name, dentry->d_name.len); - - ret = scoutfs_item_create(sb, key, NULL); - if (ret) { - scoutfs_kvec_init(key, &rkey, sizeof(rkey)); - scoutfs_item_delete(sb, key); -out_dent: - scoutfs_kvec_init(key, &dkey, sizeof(dkey), - dentry->d_name.name, dentry->d_name.len); - scoutfs_item_delete(sb, key); - } #endif - scoutfs_key_free(sb, key); + update_dentry_info(dentry, &dent); + ret = 0; +out: + while (ret < 0 && --del >= 0) { + err = scoutfs_item_delete(sb, del_keys[del]); + /* can always delete dirty while holding */ + BUG_ON(err); + } + + scoutfs_key_free(sb, ent_key); + return ret; } @@ -358,6 +445,10 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct inode *inode; int ret; + ret = alloc_dentry_info(dentry); + if (ret) + return ret; + ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -416,6 +507,10 @@ static int scoutfs_link(struct dentry *old_dentry, if (inode->i_nlink >= SCOUTFS_LINK_MAX) return -EMLINK; + ret = alloc_dentry_info(dentry); + if (ret) + return ret; + ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -448,7 +543,9 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_key_buf *key = NULL; + struct scoutfs_key_buf *keys[2] = {NULL,}; + struct scoutfs_key_buf rdir_key; + struct scoutfs_readdir_key rkey; int ret = 0; if (S_ISDIR(inode->i_mode) && i_size_read(inode)) @@ -463,14 +560,16 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (ret) goto out; - /* XXX same items as add_entry_items */ - key = alloc_dirent_key(sb, dir, dentry); - if (!key) { + keys[0] = alloc_dirent_key(sb, dir, dentry); + if (!keys[0]) { ret = -ENOMEM; goto out; } - ret = scoutfs_item_delete(sb, key); + init_readdir_key(&rdir_key, &rkey, dir, dentry_info_pos(dentry)); + keys[1] = &rdir_key; + + ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys)); if (ret) goto out; @@ -500,7 +599,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(dir); out: - scoutfs_key_free(sb, key); + scoutfs_key_free(sb, keys[0]); scoutfs_release_trans(sb); return ret; } @@ -618,6 +717,10 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) return -ENAMETOOLONG; + ret = alloc_dentry_info(dentry); + if (ret) + return ret; + ret = scoutfs_hold_trans(sb); if (ret) return ret; @@ -911,3 +1014,22 @@ const struct inode_operations scoutfs_dir_iops = { .removexattr = scoutfs_removexattr, .symlink = scoutfs_symlink, }; + +void scoutfs_dir_exit(void) +{ + if (dentry_info_cache) { + kmem_cache_destroy(dentry_info_cache); + dentry_info_cache = NULL; + } +} + +int scoutfs_dir_init(void) +{ + dentry_info_cache = kmem_cache_create("scoutfs_dentry_info", + sizeof(struct dentry_info), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (!dentry_info_cache) + return -ENOMEM; + + return 0; +} diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 2327518b..1221846e 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -18,4 +18,7 @@ int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, int scoutfs_symlink_drop(struct super_block *sb, u64 ino); +int scoutfs_dir_init(void); +void scoutfs_dir_exit(void); + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 28125b6c..f2480fc3 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -380,6 +380,7 @@ struct scoutfs_inode { __le64 blocks; __le64 link_counter; __le64 data_version; + __le64 next_readdir_pos; __le32 nlink; __le32 uid; __le32 gid; @@ -403,6 +404,7 @@ struct scoutfs_inode { struct scoutfs_dirent { __le64 ino; __le64 counter; + __le64 readdir_pos; __u8 type; __u8 name[0]; } __packed; @@ -435,6 +437,7 @@ struct scoutfs_dirent { #define SCOUTFS_DIRENT_OFF_BITS 31 #define SCOUTFS_DIRENT_OFF_MASK ((1U << SCOUTFS_DIRENT_OFF_BITS) - 1) /* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +#define SCOUTFS_DIRENT_FIRST_POS 2 #define SCOUTFS_DIRENT_LAST_POS (INT_MAX - 1) enum { diff --git a/kmod/src/inode.c b/kmod/src/inode.c index eeda6cb7..87e6e063 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -125,6 +125,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->salt = le32_to_cpu(cinode->salt); atomic64_set(&ci->link_counter, le64_to_cpu(cinode->link_counter)); ci->data_version = le64_to_cpu(cinode->data_version); + ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); } static void init_inode_key(struct scoutfs_key_buf *key, @@ -245,6 +246,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->salt = cpu_to_le32(ci->salt); cinode->link_counter = cpu_to_le64(atomic64_read(&ci->link_counter)); cinode->data_version = cpu_to_le64(ci->data_version); + cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); } /* @@ -407,6 +409,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->ino = ino; seqcount_init(&ci->seqcount); ci->data_version = 0; + ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->staging = false; get_random_bytes(&ci->salt, sizeof(ci->salt)); atomic64_set(&ci->link_counter, 0); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 0d48f158..93f6d276 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -7,6 +7,7 @@ struct scoutfs_inode_info { seqcount_t seqcount; u64 data_version; + u64 next_readdir_pos; /* holder of i_mutex is staging */ bool staging; diff --git a/kmod/src/super.c b/kmod/src/super.c index 4c4abdfa..2fd01d8c 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -292,6 +292,7 @@ static struct file_system_type scoutfs_fs_type = { /* safe to call at any failure point in _init */ static void teardown_module(void) { + scoutfs_dir_exit(); scoutfs_inode_exit(); if (scoutfs_kset) kset_unregister(scoutfs_kset); @@ -308,6 +309,7 @@ static int __init scoutfs_module_init(void) return -ENOMEM; ret = scoutfs_inode_init() ?: + scoutfs_dir_init() ?: register_filesystem(&scoutfs_fs_type); if (ret) teardown_module(); From 0298cbb562cb41681075969036bbeec3aa175356 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 10:49:49 -0800 Subject: [PATCH 216/920] Fix compact cleanup on mount failure scoutfs_compact_destroy() was testing the wrong pointer to see if _setup() had built up resources that needed to be torn down. It'd crash on mount failure. Signed-off-by: Zach Brown --- kmod/src/compact.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index a0f951cd..cce6715f 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -687,7 +687,7 @@ void scoutfs_compact_destroy(struct super_block *sb) { DECLARE_COMPACT_INFO(sb, ci); - if (ci->workq) { + if (ci) { flush_work(&ci->work); destroy_workqueue(ci->workq); } From 7045e3a6e87dd7eed2a7206d86d5112eac6e6545 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 09:40:23 -0800 Subject: [PATCH 217/920] More efficiently destroy item rbtrees I was auditing rb_erase() use and noticed that we we don't need to fully tear down the item trees. We can just blow them away with postorder traversal and raw frees of the nodes. Signed-off-by: Zach Brown --- kmod/src/item.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index d471cab6..41cb9fa6 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1295,26 +1295,27 @@ int scoutfs_item_setup(struct super_block *sb) return 0; } +/* + * There's no more users of the items and ranges at this point. We can + * destroy them without locking and ignoring augmentation. + */ void scoutfs_item_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct cached_item *item; + struct cached_item *pos_item; struct cached_range *rng; - struct rb_node *node; + struct cached_range *pos_rng; if (cac) { - for (node = rb_first(&cac->items); node; ) { - item = container_of(node, struct cached_item, node); - node = rb_next(node); - rb_erase(&item->node, &cac->items); + rbtree_postorder_for_each_entry_safe(item, pos_item, + &cac->items, node) { free_item(sb, item); } - for (node = rb_first(&cac->ranges); node; ) { - rng = container_of(node, struct cached_range, node); - node = rb_next(node); - rb_erase(&rng->node, &cac->items); + rbtree_postorder_for_each_entry_safe(rng, pos_rng, + &cac->ranges, node) { free_range(sb, rng); } From 568cefa4db85430842aa4dfeddeb765243adcc3c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 09:46:31 -0800 Subject: [PATCH 218/920] Add some item debugging tracing to seg writing Trace the items that we count and then write to the segment. Signed-off-by: Zach Brown --- kmod/src/item.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 41cb9fa6..fbd7dc36 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1214,6 +1214,9 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, *nr_items = items; *key_bytes = keys; + + trace_printk("counted item %p nr %u keys %u\n", + item, items, keys); } } @@ -1252,6 +1255,9 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) /* remember nr_items is passed to _first_item */ while (nr_items) { + trace_printk("copying item %p nr %u keys %u\n", + item, nr_items, key_bytes); + if (!item) { item = first_dirty(cac->items.rb_node); scoutfs_seg_first_item(sb, seg, item->key, item->val, @@ -1262,6 +1268,8 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) item_flags(item)); } + key_bytes -= item->key->key_len; + clear_item_dirty(cac, item); del = item; From 9f885b4c12cdb0b43ab83fe74886d80cf4faa6c5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 09:48:09 -0800 Subject: [PATCH 219/920] Fix item erase augmentation The item cache was getting inconsistent as items were removed. This would manifest in failing to find dirty items that it had counted as it was writing items into the segment and removing deletion items. For a start it wasn't using the augmented rb_erase(). We make a function that everyone uses. There's no augmented rb_replace() so We just augment erase, restart, and insert. (We could probably augment on descent and replace/propagate but that can come later.) Then the augmentation callbacks got the semantics slightly wrong. The rotation callback is named after a caller that happens to use it, not on any implied relationship between the nodes. It actually just recalculates the augmentation value for the two subtrees. Mischief managed. (We'll probably rework the augmentation so the value is for the node and its children and we can get rid of the extra code we have today to support our augmentation value that is sensitive to the difference between the left and write subtrees.) Signed-off-by: Zach Brown --- kmod/src/item.c | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index fbd7dc36..538b3ce1 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -239,10 +239,9 @@ static void scoutfs_item_rb_propagate(struct rb_node *node, static void scoutfs_item_rb_copy(struct rb_node *old, struct rb_node *new) { - struct cached_item *o = container_of(old, struct cached_item, node); struct cached_item *n = container_of(new, struct cached_item, node); - n->dirty = o->dirty; + n->dirty = compute_item_dirty(n); } /* calculate the new parent last as it depends on the old parent */ @@ -251,8 +250,6 @@ static void scoutfs_item_rb_rotate(struct rb_node *old, struct rb_node *new) struct cached_item *o = container_of(old, struct cached_item, node); struct cached_item *n = container_of(new, struct cached_item, node); - BUG_ON(rb_parent(old) != new); - o->dirty = compute_item_dirty(o); n->dirty = compute_item_dirty(n); } @@ -325,6 +322,20 @@ static void clear_item_dirty(struct item_cache *cac, update_dirty_parents(item); } +/* + * Safely erase an item from the tree. Make sure to remove its dirty + * accounting, use the augmented erase, and free it. + */ +static void erase_item(struct super_block *sb, struct item_cache *cac, + struct cached_item *item) +{ + trace_printk("erasing item %p\n", item); + + clear_item_dirty(cac, item); + rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); + free_item(sb, item); +} + /* * Try to insert the given item. If there's already a non-deletion item * with the insertion key then return -EEXIST. An existing deletion @@ -336,11 +347,14 @@ static int insert_item(struct super_block *sb, struct item_cache *cac, struct cached_item *ins) { struct rb_root *root = &cac->items; - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; struct cached_item *item; + struct rb_node *parent; + struct rb_node **node; int cmp; +restart: + node = &root->rb_node; + parent = NULL; while (*node) { parent = *node; item = container_of(*node, struct cached_item, node); @@ -358,10 +372,9 @@ static int insert_item(struct super_block *sb, struct item_cache *cac, if (!item->deletion) return -EEXIST; - clear_item_dirty(cac, item); - rb_replace_node(&item->node, &ins->node, root); - free_item(sb, item); - return 0; + /* sadly there's no augmented replace */ + erase_item(sb, cac, item); + goto restart; } } @@ -1275,10 +1288,8 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) del = item; item = next_dirty(item); - if (del->deletion) { - rb_erase(&del->node, &cac->items); - free_item(sb, del); - } + if (del->deletion) + erase_item(sb, cac, del); nr_items--; } From c3307e941b7b1e15f060552f931b5df5aec706a7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 10:01:28 -0800 Subject: [PATCH 220/920] Add scoutfs_item_forget() Add a forget call which forcefully removes an item, no matter it's state. The page cache will use this in invalidate page to drop ephemeral items that reference a dirty page that's being truncated. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/item.c | 26 ++++++++++++++++++++++++++ kmod/src/item.h | 1 + 3 files changed, 28 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 86787796..186a57d7 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -28,6 +28,7 @@ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ EXPAND_COUNTER(item_delete) \ + EXPAND_COUNTER(item_forget) \ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ EXPAND_COUNTER(item_range_insert) diff --git a/kmod/src/item.c b/kmod/src/item.c index 538b3ce1..23516168 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1105,6 +1105,32 @@ out: return ret; } +/* + * Forcefully remove an item from the cache regardless of its state or + * relationship to persistent items. + * + * The caller is entirely responsible for the correctness of having this + * item vanish. + */ +void scoutfs_item_forget(struct super_block *sb, struct scoutfs_key_buf *key) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + if (item) { + trace_printk("forgetting item %p\n", item); + scoutfs_inc_counter(sb, item_forget); + erase_item(sb, cac, item); + } + + spin_unlock_irqrestore(&cac->lock, flags); +} + /* * Return the first dirty node in the subtree starting at the given node. */ diff --git a/kmod/src/item.h b/kmod/src/item.h index 8f84389a..4ca4d088 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -31,6 +31,7 @@ void scoutfs_item_delete_dirty(struct super_block *sb, int scoutfs_item_delete_many(struct super_block *sb, struct scoutfs_key_buf **keys, unsigned nr); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); +void scoutfs_item_forget(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); From 1ad479a1af8fb8bcf2d3120f8972915bd31ba185 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 10:02:27 -0800 Subject: [PATCH 221/920] Add ephemeral items Ephemeral items exist to reference external values. They're going to be used by the page cache to reference dirty pages for writeback. Signed-off-by: Zach Brown --- kmod/src/counters.h | 2 ++ kmod/src/item.c | 84 +++++++++++++++++++++++++++++++++++++++++---- kmod/src/item.h | 6 ++++ 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 186a57d7..1ccb08d4 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -25,6 +25,8 @@ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_written) \ EXPAND_COUNTER(item_create) \ + EXPAND_COUNTER(item_create_ephemeral) \ + EXPAND_COUNTER(item_update_ephemeral) \ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ EXPAND_COUNTER(item_delete) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index 23516168..e5b5970f 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -64,7 +64,8 @@ struct cached_item { }; long dirty; - unsigned deletion:1; + unsigned deletion:1, + ephemeral:1; struct scoutfs_key_buf *key; @@ -87,7 +88,8 @@ static void free_item(struct super_block *sb, struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { scoutfs_key_free(sb, item->key); - scoutfs_kvec_kfree(item->val); + if (!item->ephemeral) + scoutfs_kvec_kfree(item->val); kfree(item); } } @@ -344,7 +346,7 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, * The caller is responsible for marking the newly inserted item dirty. */ static int insert_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *ins) + struct cached_item *ins, bool overwrite) { struct rb_root *root = &cac->items; struct cached_item *item; @@ -369,7 +371,7 @@ restart: item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; } else { - if (!item->deletion) + if (!item->deletion && !overwrite) return -EEXIST; /* sadly there's no augmented replace */ @@ -767,7 +769,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return -ENOMEM; spin_lock_irqsave(&cac->lock, flags); - ret = insert_item(sb, cac, item); + ret = insert_item(sb, cac, item, false); if (!ret) { scoutfs_inc_counter(sb, item_create); mark_item_dirty(cac, item); @@ -780,6 +782,74 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return ret; } +/* + * Ephemeral items are slightly magical and used to track file contents + * without copying the data into an allocated value. + * + * Their value kvec clones the callers which means they reference + * external data. They're freed after items are copied into segments so + * that callers can know that no items reference their structures after + * a commit finishes. + * + * They forcefully clobber any existing item at their key without + * reading the existing item. + */ +int scoutfs_item_create_ephemeral(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + item = alloc_item(sb, key, NULL); + if (!item) + return -ENOMEM; + + scoutfs_kvec_clone(item->val, val); + item->ephemeral = 1; + + spin_lock_irqsave(&cac->lock, flags); + + ret = insert_item(sb, cac, item, true); + BUG_ON(ret); + + scoutfs_inc_counter(sb, item_create_ephemeral); + mark_item_dirty(cac, item); + + spin_unlock_irqrestore(&cac->lock, flags); + + return ret; +} + +/* + * Update the value for an ephemeral item if it exists. + */ +void scoutfs_item_update_ephemeral(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + if (item && item->ephemeral) { + trace_printk("updating ephemeral item %p\n", item); + scoutfs_inc_counter(sb, item_update_ephemeral); + clear_item_dirty(cac, item); + scoutfs_kvec_clone(item->val, val); + mark_item_dirty(cac, item); + } + + spin_unlock_irqrestore(&cac->lock, flags); +} + /* * Allocate an item with the key and value and add it to the list of * items to be inserted as a batch later. The caller adds in sort order @@ -852,7 +922,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del(&item->entry); - if (insert_item(sb, cac, item)) + if (insert_item(sb, cac, item, false)) list_add(&item->entry, list); } @@ -1314,7 +1384,7 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) del = item; item = next_dirty(item); - if (del->deletion) + if (del->deletion || del->ephemeral) erase_item(sb, cac, del); nr_items--; diff --git a/kmod/src/item.h b/kmod/src/item.h index 4ca4d088..6021e3c3 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -23,9 +23,15 @@ int scoutfs_item_insert(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); +int scoutfs_item_create_ephemeral(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); +void scoutfs_item_update_ephemeral(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_delete_many(struct super_block *sb, From 9f5e42f7ddc22f57fb625e5d177b0d75018fad38 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 10:53:30 -0800 Subject: [PATCH 222/920] Add simple data items Add basic file data support by managing file data items from the page cache address space callbacks. Data is read by copying from cached items into page contents in readpage. Writes create new ephemeral items which reference dirty pages. The items are deleted once they're written in a transaction or if invalidatepage removes the dirty page they reference. There's a lot more to do to remove data copies, avoid compaction bw overhead, and add support for truncate, o_direct, and mmap. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/counters.h | 6 + kmod/src/data.c | 616 ++++++++++++++++++++++++++++++++++++++++ kmod/src/data.h | 14 + kmod/src/filerw.c | 667 -------------------------------------------- kmod/src/filerw.h | 11 - kmod/src/format.h | 8 + kmod/src/inode.c | 6 +- kmod/src/ioctl.c | 6 +- kmod/src/super.c | 4 +- kmod/src/super.h | 7 +- kmod/src/trans.c | 10 +- 12 files changed, 662 insertions(+), 695 deletions(-) create mode 100644 kmod/src/data.c create mode 100644 kmod/src/data.h delete mode 100644 kmod/src/filerw.c delete mode 100644 kmod/src/filerw.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 828cce93..3e7c9b35 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o block.o btree.o buddy.o compact.o counters.o crc.o \ - dir.o filerw.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ + data.o dir.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ msg.o name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 1ccb08d4..137ebbae 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -24,6 +24,12 @@ EXPAND_COUNTER(compact_segment_skipped) \ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_written) \ + EXPAND_COUNTER(data_readpage) \ + EXPAND_COUNTER(data_write_begin) \ + EXPAND_COUNTER(data_write_end) \ + EXPAND_COUNTER(data_invalidatepage) \ + EXPAND_COUNTER(data_writepage) \ + EXPAND_COUNTER(data_end_writeback_page) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_create_ephemeral) \ EXPAND_COUNTER(item_update_ephemeral) \ diff --git a/kmod/src/data.c b/kmod/src/data.c new file mode 100644 index 00000000..88843d69 --- /dev/null +++ b/kmod/src/data.c @@ -0,0 +1,616 @@ +/* +* Copyright (C) 2017 Versity Software, Inc. All rights reserved. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public +* License v2 as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* General Public License for more details. +*/ +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "inode.h" +#include "key.h" +#include "data.h" +#include "trans.h" +#include "counters.h" +#include "scoutfs_trace.h" +#include "btree.h" +#include "item.h" +#include "ioctl.h" + +/* + * scoutfs stores data in items that can be up to the small 4K block + * size. The page cache address space callbacks work with the item + * cache. Each OS page can be stored in multiple of our smaller fixed + * size items. The code doesn't understand OS pages that are smaller + * than our block size. + * + * readpage does a blocking read of the item and then copies its + * contents into the page. Since the segments are huge we sort of get + * limited read-ahead by reading in segments at a time. + * + * Writing is quite a bit more fiddly. We want to pack small files. + * The item cache and transactions want to accurately track the size of + * dirty items to fill the next segment. And we would like to minimize + * cpu copying as much as we can. + * + * This simplest first pass creates dirty items as pages are dirtied + * whose values reference the page contents. They're freed after + * they're written to the segment so that we don't have to worry about + * items that reference clean pages. Invalidatepage forgets any items + * if a dirty page is truncated away. + * + * Writeback is built around all the dirty items being written by a + * commit. This can happen naturally in the backgroud. Or writepage + * can initiate it to start by kicking the commit thread. In either + * case our dirty pages are "in writeback" by being put on a list that + * is walked by the end of the commit. Because writes and page dirtying + * are serialized with the commit we know that there can be no dirty + * pages after the commit and we can mark writeback complete on all the + * pages that started writeback before the commit finished. motivate + * having items in the item cache while there are dirty pages. + * + * Data is copied from the dirty page contents into the segment pages + * for writing. This lets us easily pack small files without worrying + * about DMA alignment and avoids the stable page problem of the page + * being modified after the cpu calculates the checksum but before the + * DMA reads to the device. + * + * XXX + * - truncate + * - mmap + * - better io error propagation + * - async readpages for more concurrent readahead + * - forced unmount with dirty data + * - direct IO + * - probably stitch page vecs into block struct page fragments for bios + * - maybe cut segment boundaries on aligned data offsets + * - maybe decouple metadata and data segment writes + */ + +struct data_info { + struct llist_head writeback_pages; +}; + +#define DECLARE_DATA_INFO(sb, name) \ + struct data_info *name = SCOUTFS_SB(sb)->data_info + +/* + * trace_printk() doesn't support %c? + * + * 1 - 1ocked + * a - uptodAte + * d - Dirty + * b - writeBack + * e - Error + */ +#define page_hexflag(page, name, val, shift) \ + (Page##name(page) ? (val << (shift * 4)) : 0) + +#define page_hexflags(page) \ + (page_hexflag(page, Locked, 0x1, 4) | \ + page_hexflag(page, Uptodate, 0xa, 3) | \ + page_hexflag(page, Dirty, 0xd, 2) | \ + page_hexflag(page, Writeback, 0xb, 1) | \ + page_hexflag(page, Error, 0xe, 0)) + +#define PGF "page %p [index %lu flags %x]" +#define PGA(page) \ + (page), (page)->index, page_hexflags(page) \ + +#define BHF "bh %p [blocknr %llu size %zu state %lx]" +#define BHA(bh) \ + (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ + +/* + * Free extents whose blocks fall inside the specified blocks. The + * caller holds a transaction. + * + * If 'release' is given then blocks are freed inside i_size but the + * extent items are left behind and their _OFFLINE flag is set. + * + * This is the low level extent item truncate code. Callers manage + * higher order truncation and orphan cleanup. + */ +int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline) +{ + struct scoutfs_btree_root *meta = SCOUTFS_META(sb); + struct scoutfs_extent extent; + struct scoutfs_btree_val val; + struct scoutfs_key key; + struct scoutfs_key first; + u64 seq; + int ret; + + /* XXX not yet updated */ + + scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); + scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); + + trace_printk("iblock %llu\n", iblock); + + scoutfs_btree_init_val(&val, &extent, sizeof(extent)); + val.check_size_eq = 1; + + for (;;) { + ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, + &val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + len = le64_to_cpu(extent.len); + if (WARN_ON_ONCE(len != 1)) { + ret = -EIO; + break; + } + + /* XXX corruption: offline and allocation are exclusive */ + if (!!extent.blkno == + !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { + ret = -EIO; + break; + } + + if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) + continue; + + /* make sure we can delete the extent after freeing */ + if (extent.blkno) { + ret = scoutfs_btree_dirty(sb, meta, &key); + if (ret) + break; + + ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), + le64_to_cpu(extent.blkno), 0); + if (ret) + break; + } + + if (offline) { + extent.blkno = 0; + extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; + scoutfs_btree_update(sb, meta, &key, &val); + } else { + ret = scoutfs_btree_delete(sb, meta, &key); + if (ret) + break; + } + + /* XXX sync transaction if it's enormous */ + scoutfs_dec_key(&key); + } + + return ret; +} + +static inline struct page *page_from_llist_node(struct llist_node *node) +{ + BUILD_BUG_ON(member_sizeof(struct page, private) != + sizeof(struct llist_node)); + + return container_of((void *)node, struct page, private); +} + +static inline struct llist_node *llist_node_from_page(struct page *page) +{ + return (void *)&page->private; +} + +static inline void page_llist_add(struct page *page, struct llist_head *head) +{ + llist_add(llist_node_from_page(page), head); +} + +/* + * The transaction has committed so there are no more dirty items. End + * writeback on all the dirty pages that started writeback before the + * commit finished. The commit doesn't start until all holders which + * could dirty are released so there couldn't have been new dirty pages + * and writeback entries while the commit was in flight. + */ +void scoutfs_data_end_writeback(struct super_block *sb, int err) +{ + DECLARE_DATA_INFO(sb, datinf); + struct llist_node *node; + struct page *page; + + /* XXX haven't thought about errors here */ + BUG_ON(err); + + node = llist_del_all(&datinf->writeback_pages); + + while (node) { + page = page_from_llist_node(node); + node = llist_next(node); + + trace_printk("ending writeback "PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_end_writeback_page); + + + set_page_private(page, 0); + end_page_writeback(page); + page_cache_release(page); + } +} + +static void init_data_key(struct scoutfs_key_buf *key, + struct scoutfs_data_key *dkey, + struct inode *inode, u64 block) +{ + dkey->type = SCOUTFS_DATA_KEY; + dkey->ino = cpu_to_be64(scoutfs_ino(inode)); + dkey->block = cpu_to_be64(block); + + scoutfs_key_init(key, dkey, sizeof(struct scoutfs_data_key)); +} + +/* Iterate over all the data block items that make up the page. */ +#define for_each_page_block(page, start, loff, block, key, dkey, val) \ + for (start = 0; \ + start < PAGE_CACHE_SIZE && \ + (loff = ((loff_t)page->index << PAGE_CACHE_SHIFT) + start, \ + block = loff >> SCOUTFS_BLOCK_SHIFT, \ + init_data_key(&key, &dkey, page->mapping->host, block), \ + scoutfs_kvec_init(val, page_address(page) + start, \ + SCOUTFS_BLOCK_SIZE), \ + 1); \ + start += SCOUTFS_BLOCK_SIZE) + +/* + * Copy the contents of each item that makes up the page into their + * regions of the page, zeroing any page contents not covered by items. + * + * This is the simplest loop that looks up every possible block. We + * could instead have a readpages() that iterates over present items and + * puts them in the pages in the batch. + */ +static int scoutfs_readpage(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + loff_t size = i_size_read(inode); + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned start; + loff_t loff; + u64 block; + int ret = 0; + + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_readpage); + + for_each_page_block(page, start, loff, block, key, dkey, val) { + /* the rest of the page is zero when block is past i_size */ + if (loff >= size) + break; + + /* copy the block item contents into the page */ + ret = scoutfs_item_lookup(sb, &key, val); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + else + break; + } + + /* + * XXX do we need to clamp the item length by i_size? + * truncate should purge the item cache and create + * truncation range items that'd merge away old data + * items, and invalidatepage should shrink any ephemeral + * vecs. Seems like the item length should be accurate? + */ + + /* zero the tail of the block */ + if (ret < SCOUTFS_BLOCK_SIZE) + zero_user(page, start, SCOUTFS_BLOCK_SIZE - ret); + } + + /* zero any remaining tail blocks */ + if (start < PAGE_CACHE_SIZE) + zero_user(page, start, PAGE_CACHE_SIZE - start); + + if (ret == 0) + SetPageUptodate(page); + else + SetPageError(page); + + trace_printk("ret %d\n", ret); + unlock_page(page); + return ret; +} + +/* + * Start writeback on a dirty page. We always try to kick off a commit. + * Repeated calls harmlessly bounce off the thread work's pending bit. + * (we could probably test that the writeback pgaes list is empty before + * trying to kick off a commit.) + * + * We add ourselves to a list of pages that the commit will end + * writeback on once its done. If there's no dirty data the commit + * thread will end writeback after not doing anything. + */ +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + DECLARE_DATA_INFO(sb, datinf); + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_writepage); + + BUG_ON(PageWriteback(page)); + BUG_ON(page->private != 0); + + ClearPagePrivate(page); /* invalidatepage not needed */ + set_page_writeback(page); + page_cache_get(page); + page_llist_add(page, &datinf->writeback_pages); + unlock_page(page); + scoutfs_sync_fs(sb, 0); + + return 0; +} + +/* + * Truncate is invalidating part of the contents of a page. + * + * We can't return errors here so our job is not to create dirty items + * that end up executing the truncate. That's the job of higher level + * callers. Our job is to make sure that we update references to the + * page from existing ephemeral items if they already exist. + */ +static void scoutfs_invalidatepage(struct page *page, unsigned long offset) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned start; + loff_t loff; + u64 block; + + trace_printk(PGF"\n", PGA(page)); + scoutfs_inc_counter(sb, data_invalidatepage); + + for_each_page_block(page, start, loff, block, key, dkey, val) { + if (offset) { + /* XXX maybe integrate offset into foreach */ + /* XXX ugh, kvecs are still clumsy :) */ + if (start + SCOUTFS_BLOCK_SIZE > offset) + val[0].iov_len = offset - start; + scoutfs_item_update_ephemeral(sb, &key, val); + } else { + scoutfs_item_forget(sb, &key); + } + } +} + +/* + * Start modifying a page cache page. + * + * We hold the transaction for write_end's inode updates before + * acquiring the page lock. + * + * We give the writer the current page contents in the relatively rare + * case of writing a partial page inside i_size. write_end will zero + * any region around the write if the page isn't uptodate. + */ +static int scoutfs_write_begin(struct file *file, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; + pgoff_t index = pos >> PAGE_SHIFT; + loff_t size = i_size_read(inode); + struct page *page; + int ret; + + trace_printk("ino %llu pos %llu len %u flags %x\n", + scoutfs_ino(inode), (u64)pos, len, flags); + scoutfs_inc_counter(sb, data_write_begin); + + ret = scoutfs_hold_trans(sb); + if (ret) + return ret; + + /* can't re-enter fs, have trans */ + flags |= AOP_FLAG_NOFS; + + ret = scoutfs_dirty_inode_item(inode); + if (ret) + goto out; + +retry: + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) { + ret = -ENOMEM; + goto out; + } + + trace_printk(PGF"\n", PGA(page)); + + if (!PageUptodate(page) && (pos < size && len < PAGE_CACHE_SIZE)) { + ClearPageError(page); + ret = scoutfs_readpage(file, page); + if (!ret) { + wait_on_page_locked(page); + if (!PageUptodate(page)) + ret = -EIO; + } + page_cache_release(page); + if (ret) + goto out; + + /* let grab_ lock and check for truncated pages */ + goto retry; + } + + *pagep = page; + ret = 0; +out: + if (ret) + scoutfs_release_trans(sb); + + trace_printk("ret %d\n", ret); + return ret; +} + +/* + * Finish modification of a page cache page. + * + * write_begin has held the transaction and dirtied the inode. We + * create items for each dirty block whose value references the page + * contents that will be written. + * + * We Modify the dirty item and its dependent metadata items while + * holding the transaction so that we never get missing data. + * + * XXX + * - detect no change with copied == 0? + * - only iterate over written blocks, not the whole page? + * - make sure page granular locking and concurrent extending writes works + * - error handling needs work, truncate partial writes on failure? + */ +static int scoutfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = page->mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_data_key dkey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + loff_t old_size = i_size_read(inode); + bool update_inode = false; + loff_t new_size; + unsigned start; + loff_t loff; + u64 block; + int ret; + + trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", + scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); + scoutfs_inc_counter(sb, data_write_end); + + /* zero any unwritten portions of a new page around the write */ + if (!PageUptodate(page)) { + if (copied != PAGE_CACHE_SIZE) { + start = pos & ~PAGE_CACHE_MASK; + zero_user_segments(page, 0, start, + start + copied, PAGE_CACHE_SIZE); + } + SetPageUptodate(page); + } + + new_size = pos + copied; + + for_each_page_block(page, start, loff, block, key, dkey, val) { + + /* only put data inside i_size in items */ + /* XXX ugh, kvecs are still clumsy :) */ + if (loff + SCOUTFS_BLOCK_SIZE > new_size) + val[0].iov_len = new_size - loff; + + ret = scoutfs_item_create_ephemeral(sb, &key, val); + if (ret) + goto out; + } + + /* update i_size if we extended */ + if (new_size > inode->i_size) { + i_size_write(inode, new_size); + update_inode = true; + } + + if (old_size < pos) + pagecache_isize_extended(inode, old_size, pos); + + if (copied) { + scoutfs_inode_inc_data_version(inode); + update_inode = true; + } + + if (update_inode) + scoutfs_update_inode_item(inode); + + flush_dcache_page(page); + set_page_dirty(page); + SetPagePrivate(page); /* call invalidatepage */ + + ret = copied; +out: + unlock_page(page); + scoutfs_release_trans(sb); + + /* XXX error handling needs work */ + WARN_ON_ONCE(ret < 0); + return ret; +} + +const struct address_space_operations scoutfs_file_aops = { + .readpage = scoutfs_readpage, + .writepage = scoutfs_writepage, + .set_page_dirty = __set_page_dirty_nobuffers, + .invalidatepage = scoutfs_invalidatepage, + .write_begin = scoutfs_write_begin, + .write_end = scoutfs_write_end, +}; + +const struct file_operations scoutfs_file_fops = { + .read = do_sync_read, + .write = do_sync_write, + .aio_read = generic_file_aio_read, + .aio_write = generic_file_aio_write, + .unlocked_ioctl = scoutfs_ioctl, + .fsync = scoutfs_file_fsync, +}; + +int scoutfs_data_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf; + + /* page block iteration doesn't understand multiple pages per block */ + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SIZE); + + datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); + if (!datinf) + return -ENOMEM; + sbi->data_info = datinf; + + init_llist_head(&datinf->writeback_pages); + + return 0; +} + +void scoutfs_data_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct data_info *datinf = sbi->data_info; + + if (datinf) { + WARN_ON_ONCE(!llist_empty(&datinf->writeback_pages)); + kfree(datinf); + } +} diff --git a/kmod/src/data.h b/kmod/src/data.h new file mode 100644 index 00000000..189b2cba --- /dev/null +++ b/kmod/src/data.h @@ -0,0 +1,14 @@ +#ifndef _SCOUTFS_FILERW_H_ +#define _SCOUTFS_FILERW_H_ + +extern const struct address_space_operations scoutfs_file_aops; +extern const struct file_operations scoutfs_file_fops; + +int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline); +void scoutfs_data_end_writeback(struct super_block *sb, int err); + +int scoutfs_data_setup(struct super_block *sb); +void scoutfs_data_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/filerw.c b/kmod/src/filerw.c deleted file mode 100644 index 5cdce097..00000000 --- a/kmod/src/filerw.c +++ /dev/null @@ -1,667 +0,0 @@ -/* -* Copyright (C) 2016 Versity Software, Inc. All rights reserved. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public -* License v2 as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* General Public License for more details. -*/ -#include -#include -#include -#include -#include - -#include "format.h" -#include "super.h" -#include "inode.h" -#include "key.h" -#include "filerw.h" -#include "trans.h" -#include "scoutfs_trace.h" -#include "btree.h" -#include "ioctl.h" - -/* - * scoutfs uses an extent item to map logical file data blocks to - * physical block locations. - * - * The small block size is set to the smallest supported page size. - * This means that our file IO code never has to worry about the - * situation where a page write is smaller than the block size. We - * never have to perform RMW of blocks larger than pages, nor do we have - * to punch a whole and worry about block tracking items that could be - * sharing references to a block on either side of a smaller dirty page. - * We can simply use the kernel's buffer head code, loathed though it - * is, and have a 1:1 relationship between block writes and block - * mapping item entries. - * - * Dirty extents are only written to free space. The first time a block - * hits write_page in a transaction it gets a newly allocated block. We - * get decent contiguous allocations by having per-task preallocation - * streams. These are trimmed back as the transaction is committed. We - * don't bother worrying about small transactions. - * - * Because we only write to allocated space we can't naively use the - * buffer head get_blocks support functions. They assume that they can - * write dirty buffers to existing clean mappings which is absolutely - * not true for us. We clear mappings for clean pages before we call - * block_write_begin() so that it won't write to blocks that were caned - * from previous reads. We make sure that the page is uptodate ourself - * so that it won't use readpage to read the existing block and then - * turn around and write to it. - * - * Data blocks aren't pinned for the duration of the transaction. They - * can be written out and read back in and redirtied during the lifetime - * of a transaction. As we map dirty pages we see if its current allocation - * is newly allocated in the transaction and can reuse it. - * - * XXX - * - need to wire up dirty inode? - * - enforce writing to free blknos - * - per-task allocation regions - * - tear down dirty extents left by write errors on unmount - * - should invalidate dirty blocks if freed - * - data block checksumming (stable pages) - * - mmap creating dirty unmapped pages at writepage - * - pack small tails into inline items - * - direct IO - */ - - -/* - * trace_printk() doesn't support %c? - * - * 1 - 1ocked - * a - uptodAte - * d - Dirty - * b - writeBack - * e - Error - */ -#define page_hexflag(page, name, val, shift) \ - (Page##name(page) ? (val << (shift * 4)) : 0) - -#define page_hexflags(page) \ - (page_hexflag(page, Locked, 0x1, 4) | \ - page_hexflag(page, Uptodate, 0xa, 3) | \ - page_hexflag(page, Dirty, 0xd, 2) | \ - page_hexflag(page, Writeback, 0xb, 1) | \ - page_hexflag(page, Error, 0xe, 0)) - -#define PGF "page %p [index %lu flags %x]" -#define PGA(page) \ - (page), (page)->index, page_hexflags(page) \ - -#define BHF "bh %p [blocknr %llu size %zu state %lx]" -#define BHA(bh) \ - (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ - -/* - * For now this is super cheesy. We just have one allocation on the - * super that is consumed as buffered writes make their way through unmapped - * buffer heads and alloc in get_block. - */ -static int alloc_file_block(struct super_block *sb, u64 *blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u64 alloc_blkno; - int order = 0; - int ret; - - *blkno = 0; - - spin_lock(&sbi->file_alloc_lock); - - if (sbi->file_alloc_count == 0) { - spin_unlock(&sbi->file_alloc_lock); - - order = scoutfs_buddy_alloc(sb, &alloc_blkno, - SCOUTFS_BUDDY_ORDERS - 1); - if (order < 0) { - ret = order; - goto out; - } - - spin_lock(&sbi->file_alloc_lock); - - if (sbi->file_alloc_count == 0) { - sbi->file_alloc_blkno = alloc_blkno; - sbi->file_alloc_count = 1 << order; - order = -1; - } - } - - if (sbi->file_alloc_count) { - *blkno = sbi->file_alloc_blkno; - sbi->file_alloc_blkno++; - sbi->file_alloc_count--; - ret = 0; - } else { - ret = -ENOSPC; - } - - spin_unlock(&sbi->file_alloc_lock); - - if (order > 0) - scoutfs_buddy_free(sb, sbi->super.hdr.seq, alloc_blkno, order); - -out: - trace_printk("allocated blkno %llu ret %d\n", *blkno, ret); - return ret; -} - -/* - * The caller didn't need an allocated file block after all. We return - * it to the pool. This has to succeed because it's called after we've - * done things that would be annoying to revert. - */ -static void return_file_block(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - spin_lock(&sbi->file_alloc_lock); - - BUG_ON(sbi->file_alloc_count && - sbi->file_alloc_blkno != (blkno + 1)); - - if (sbi->file_alloc_count == 0) - sbi->file_alloc_blkno = blkno + 1; - - sbi->file_alloc_blkno--; - sbi->file_alloc_count++; - - spin_unlock(&sbi->file_alloc_lock); -} - -/* - * Free extents whose blocks fall inside the specified blocks. The - * caller holds a transaction. - * - * If 'release' is given then blocks are freed inside i_size but the - * extent items are left behind and their _OFFLINE flag is set. - * - * This is the low level extent item truncate code. Callers manage - * higher order truncation and orphan cleanup. - */ -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline) -{ - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_extent extent; - struct scoutfs_btree_val val; - struct scoutfs_key key; - struct scoutfs_key first; - u64 seq; - int ret; - - scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); - scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); - - trace_printk("iblock %llu\n", iblock); - - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - val.check_size_eq = 1; - - for (;;) { - ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, - &val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - len = le64_to_cpu(extent.len); - if (WARN_ON_ONCE(len != 1)) { - ret = -EIO; - break; - } - - /* XXX corruption: offline and allocation are exclusive */ - if (!!extent.blkno == - !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { - ret = -EIO; - break; - } - - if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) - continue; - - /* make sure we can delete the extent after freeing */ - if (extent.blkno) { - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - break; - - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), - le64_to_cpu(extent.blkno), 0); - if (ret) - break; - } - - if (offline) { - extent.blkno = 0; - extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; - scoutfs_btree_update(sb, meta, &key, &val); - } else { - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret) - break; - } - - /* XXX sync transaction if it's enormous */ - scoutfs_dec_key(&key); - } - - return ret; -} - -/* - * The caller ensures that this is serialized against all other callers - * and writers. - */ -void scoutfs_filerw_free_alloc(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - trace_printk("blkno %llu count %llu\n", sbi->file_alloc_blkno, - sbi->file_alloc_count); - - if (sbi->file_alloc_count) - scoutfs_buddy_free_extent(sb, sbi->file_alloc_blkno, - sbi->file_alloc_count); - - sbi->file_alloc_blkno = 0; - sbi->file_alloc_count = 0; -} - -/* - * Return the number of contiguously mapped blocks starting from the - * given logical block in the inode. - */ -static int contig_mapped_blocks(struct inode *inode, u64 iblock, u64 *blkno) -{ - struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_extent extent; - struct scoutfs_key key; - int ret; - - *blkno = 0; - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret == sizeof(extent)) { - if (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) { - ret = 0; - } else { - *blkno = le64_to_cpu(extent.blkno); - ret = min_t(u64, le64_to_cpu(extent.len), INT_MAX); - } - } else if (ret >= 0) { - /* XXX corruption */ - ret = -EIO; - } else if (ret == -ENOENT) { - ret = 0; - } - - trace_printk("ino %llu iblock %llu blkno %llu ret %d\n", - scoutfs_ino(inode), iblock, *blkno, ret); - - return ret; -} - -/* - * Make sure that the mapped block at the given logical block number is - * writable in this transaction. If it's not we allocate and reference - * a new block. If there was a previous stable block we free it. We - * give the caller the writable block number. - * - * Writeback is allowed during a transaction so we can get here with - * buffer heads that are newly allocated and being written to but for - * blocks that were allocated in the current transacation. In that - * case we re-use the existing mapping. None of it will be stable until - * there's a sync that writes all the referencing metadata. - */ -static int map_writable_block(struct inode *inode, u64 iblock, u64 *blkno_ret) -{ - struct scoutfs_inode_info *si = SCOUTFS_I(inode); - struct super_block *sb = inode->i_sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->stable_super; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_extent extent; - struct scoutfs_btree_val val; - struct scoutfs_key first; - struct scoutfs_key key; - bool inserted = false; - u64 old_blkno = 0; - u64 new_blkno = 0; - u64 seq; - int ret; - int err; - - scoutfs_set_key(&first, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, 0); - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_EXTENT_KEY, iblock); - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - val.check_size_eq = 1; - - /* see if there's an existing mapping */ - ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, &val); - if (ret == 0 && ((le64_to_cpu(key.offset) + - le64_to_cpu(extent.len)) <= iblock)) - ret = -ENOENT; - if (ret < 0 && ret != -ENOENT) - goto out; - - /* make sure that updating the extent item won't fail */ - if (ret == -ENOENT) { - memset(&extent, 0, sizeof(extent)); - ret = scoutfs_btree_insert(sb, meta, &key, &val); - if (ret) - goto out; - inserted = true; - } else { - if ((extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE) && - !si->staging) { - ret = -EINVAL; - goto out; - } - - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - goto out; - } - - old_blkno = le64_to_cpu(extent.blkno); - - /* If the existing block is dirty then we can use it */ - if (old_blkno && cpu_to_le64(seq) == super->hdr.seq) { - *blkno_ret = old_blkno; - ret = 0; - goto out; - } - - ret = alloc_file_block(sb, &new_blkno); - if (ret < 0) - goto out; - - if (old_blkno) { - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), old_blkno, 0); - if (ret) - goto out; - } - - extent.blkno = cpu_to_le64(new_blkno); - extent.len = cpu_to_le64(1); - extent.flags &= ~SCOUTFS_EXTENT_FLAG_OFFLINE; - - /* dirtying guarantees success */ - err = scoutfs_btree_update(sb, meta, &key, &val); - BUG_ON(err); - - *blkno_ret = new_blkno; - new_blkno = 0; - ret = 0; -out: - if (ret) { - if (new_blkno) - return_file_block(sb, new_blkno); - if (inserted) { - err = scoutfs_btree_delete(sb, meta, &key); - BUG_ON(err); /* always succeeds */ - } - } - - return ret; -} - -static int scoutfs_readpage_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - u64 blkno; - int ret; - - if (WARN_ON_ONCE(create)) - return -EINVAL; - - ret = contig_mapped_blocks(inode, iblock, &blkno); - if (ret > 0) { - map_bh(bh, inode->i_sb, blkno); - bh->b_size = min_t(u64, bh->b_size, - (u64)ret << inode->i_blkbits); - ret = 0; - } - - trace_printk("ino %llu iblock %llu create %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); - - return ret; -} - -static int scoutfs_readpage(struct file *file, struct page *page) -{ - trace_printk(PGF"\n", PGA(page)); - - return mpage_readpage(page, scoutfs_readpage_get_block); -} - -static int scoutfs_readpages(struct file *file, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages) -{ - return mpage_readpages(mapping, pages, nr_pages, - scoutfs_readpage_get_block); -} - -/* - * For now we don't know what to do if unmapped blocks make it to - * writepage (mmap?). - */ -static int scoutfs_writepage_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - trace_printk("ino %llu iblock %llu create %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, BHA(bh)); - - return WARN_ON_ONCE(-EINVAL); -} - -/* - * Dirty file pages can be written to their newly allocated free extents - * at any time. They won't be referenced by metadata until the current - * transaction is committed. They can be re-read and re-dirtied at - * their free block number in this transaction. - */ -static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) -{ - trace_printk(PGF"\n", PGA(page)); - - return block_write_full_page(page, scoutfs_writepage_get_block, wbc); -} - -static int scoutfs_writepages(struct address_space *mapping, - struct writeback_control *wbc) -{ - trace_printk("mapping %p\n", mapping); - - return mpage_writepages(mapping, wbc, scoutfs_writepage_get_block); -} - -/* - * Extent allocation during buffered writes needs to make sure that the - * dirty blocks will be written to free space. - */ -static int scoutfs_write_begin_get_block(struct inode *inode, sector_t iblock, - struct buffer_head *bh, int create) -{ - u64 blkno = 0; - int ret; - - if (WARN_ON_ONCE(!create)) - return -EINVAL; - - ret = map_writable_block(inode, iblock, &blkno); - if (ret == 0) { - map_bh(bh, inode->i_sb, blkno); - bh->b_size = SCOUTFS_BLOCK_SIZE; - ret = 0; - } - - trace_printk("ino %llu iblock %llu create %d ret %d "BHF"\n", - scoutfs_ino(inode), (u64)iblock, create, ret, BHA(bh)); - return ret; -} - -/* XXX could make a for_each wrapper if we get a few of these */ -static inline void clear_mapped_page_buffers(struct page *page) -{ - struct buffer_head *head; - struct buffer_head *bh; - - if (!page_has_buffers(page)) - return; - - head = page_buffers(page); - bh = head; - do { - if (buffer_mapped(bh)) { - trace_printk(BHF"\n", BHA(bh)); - clear_buffer_mapped(bh); - } - - bh = bh->b_this_page; - } while (bh != head); -} - -/* - * Dirty blocks have to be mapped to be written out to free space so - * that we don't overwrite live data. We're relying on - * block_write_begin() to call get_block(). There are two problems with - * this. - * - * First, if it's going to be trying to read a partial block before writing - * then we can't give it the location to read. It'll just mark the - * block dirty and write to that same location. We use readpage to make - * the page uptodate if it's going to be satisfying a partial overwrite. - * - * Second, we can't let it use mappings that were used by readpage to - * read the current stable data. We need to have get_block be called - * for existing clean uptodate pages so that we can reallocate them to - * free space. We do this by clearing the buffer mappings for every buffer - * on the page for every call. This is probably unnecessarily expensive - * because we don't need to do it for clean buffers. That optimization - * would need to be done very carefully. - */ -static int scoutfs_write_begin(struct file *file, - struct address_space *mapping, loff_t pos, - unsigned len, unsigned flags, - struct page **pagep, void **fsdata) -{ - struct inode *inode = mapping->host; - struct super_block *sb = inode->i_sb; - pgoff_t index = pos >> PAGE_SHIFT; - struct page *page; - int ret; - - ret = scoutfs_hold_trans(sb); - if (ret) - return ret; - - /* can't re-enter fs, have trans */ - flags |= AOP_FLAG_NOFS; - - /* generic write_end updates i_size and calls dirty_inode */ - ret = scoutfs_dirty_inode_item(inode); - if (ret) - goto out; - -retry: - page = grab_cache_page_write_begin(mapping, index, flags); - if (!page) { - ret = -ENOMEM; - goto out; - } - - /* - * read in the page if we're going to be dirtying part of the - * page. readpage catches when this is a read past i_size or - * from a hole and zeros the buffer. We try to grab the page - * again to let it deal with locking and races. - */ - if (!PageUptodate(page) && !IS_ALIGNED(pos | len, SCOUTFS_BLOCK_SIZE)) { - ClearPageError(page); - ret = scoutfs_readpage(file, page); - if (!ret) { - wait_on_page_locked(page); - if (!PageUptodate(page)) - ret = -EIO; - } - page_cache_release(page); - if (ret) - goto out; - goto retry; - } - - /* make sure our get_block gets a chance to alloc */ - clear_mapped_page_buffers(page); - - ret = __block_write_begin(page, pos, len, - scoutfs_write_begin_get_block); - if (ret < 0) { - /* XXX handle truncating? */ - unlock_page(page); - put_page(page); - page = NULL; - } - - *pagep = page; -out: - if (ret) - scoutfs_release_trans(sb); - return ret; -} - -static int scoutfs_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, - struct page *page, void *fsdata) -{ - struct inode *inode = mapping->host; - struct super_block *sb = inode->i_sb; - int ret; - - trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", - scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); - - ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); - if (ret > 0) { - scoutfs_inode_inc_data_version(inode); - /* XXX kind of a big hammer, inode life cycle needs work */ - scoutfs_update_inode_item(inode); - } - scoutfs_release_trans(sb); - return ret; -} - -const struct address_space_operations scoutfs_file_aops = { - .readpage = scoutfs_readpage, - .readpages = scoutfs_readpages, - .writepage = scoutfs_writepage, - .writepages = scoutfs_writepages, - .write_begin = scoutfs_write_begin, - .write_end = scoutfs_write_end, -}; - -const struct file_operations scoutfs_file_fops = { - .read = do_sync_read, - .write = do_sync_write, - .aio_read = generic_file_aio_read, - .aio_write = generic_file_aio_write, - .unlocked_ioctl = scoutfs_ioctl, - .fsync = scoutfs_file_fsync, -}; diff --git a/kmod/src/filerw.h b/kmod/src/filerw.h deleted file mode 100644 index f5924d71..00000000 --- a/kmod/src/filerw.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _SCOUTFS_FILERW_H_ -#define _SCOUTFS_FILERW_H_ - -extern const struct address_space_operations scoutfs_file_aops; -extern const struct file_operations scoutfs_file_fops; - -void scoutfs_filerw_free_alloc(struct super_block *sb); -int scoutfs_truncate_extent_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline); - -#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index f2480fc3..2ba9cbd8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -238,6 +238,7 @@ struct scoutfs_key { #define SCOUTFS_SYMLINK_KEY 8 #define SCOUTFS_EXTENT_KEY 9 #define SCOUTFS_ORPHAN_KEY 10 +#define SCOUTFS_DATA_KEY 11 #define SCOUTFS_MAX_UNUSED_KEY 255 #define SCOUTFS_MAX_ITEM_LEN 512 @@ -268,6 +269,13 @@ struct scoutfs_orphan_key { __be64 ino; } __packed; +/* value is data payload bytes */ +struct scoutfs_data_key { + __u8 type; + __be64 ino; + __be64 block; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 87e6e063..b773f0ee 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -23,7 +24,7 @@ #include "inode.h" #include "btree.h" #include "dir.h" -#include "filerw.h" +#include "data.h" #include "scoutfs_trace.h" #include "xattr.h" #include "trans.h" @@ -103,6 +104,9 @@ static void set_inode_ops(struct inode *inode) init_special_inode(inode, inode->i_mode, inode->i_rdev); break; } + + /* ephemeral data items avoid kmap for pointers to page contents */ + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); } static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index d40cfae1..4c1b039a 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -30,7 +30,7 @@ #include "super.h" #include "inode.h" #include "trans.h" -#include "filerw.h" +#include "data.h" /* * Find all the inodes that have had keys of a given type modified since @@ -365,8 +365,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (ret) goto out; - ret = scoutfs_truncate_extent_items(sb, scoutfs_ino(inode), - iblock, len, true); + ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), iblock, len, + true); scoutfs_release_trans(sb); out: mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/super.c b/kmod/src/super.c index 2fd01d8c..c96bbbb4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -35,6 +35,7 @@ #include "alloc.h" #include "treap.h" #include "compact.h" +#include "data.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -212,7 +213,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - spin_lock_init(&sbi->file_alloc_lock); sbi->block_shrinker.shrink = scoutfs_block_shrink; sbi->block_shrinker.seeks = DEFAULT_SEEKS; @@ -228,6 +228,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_seg_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: + scoutfs_data_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_treap_setup(sb) ?: // scoutfs_buddy_setup(sb) ?: @@ -268,6 +269,7 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_buddy_destroy(sb); if (sbi->block_shrinker.shrink == scoutfs_block_shrink) unregister_shrinker(&sbi->block_shrinker); + scoutfs_data_destroy(sb); scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index d93a296b..82eb6bba 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,7 @@ struct manifest; struct segment_cache; struct treap_info; struct compact_info; +struct data_info; struct scoutfs_sb_info { struct super_block *sb; @@ -39,6 +40,7 @@ struct scoutfs_sb_info { struct seg_alloc *seg_alloc; struct treap_info *treap_info; struct compact_info *compact_info; + struct data_info *data_info; struct buddy_info *buddy_info; @@ -59,11 +61,6 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; - - /* XXX we'd like this to be per task, not per super */ - spinlock_t file_alloc_lock; - u64 file_alloc_blkno; - u64 file_alloc_count; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 65db65ec..26e30367 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -21,7 +21,7 @@ #include "block.h" #include "trans.h" #include "buddy.h" -#include "filerw.h" +#include "data.h" #include "bio.h" #include "item.h" #include "manifest.h" @@ -93,11 +93,6 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); - /* XXX file data needs to be updated to the new item api */ -#if 0 - scoutfs_filerw_free_alloc(sb); -#endif - trace_printk("items dirty %d manifest dirty %d alloc dirty %d\n", scoutfs_item_has_dirty(sb), scoutfs_manifest_has_dirty(sb), @@ -137,6 +132,9 @@ out: /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); + /* must be done before waking waiting trans holders who might dirty */ + scoutfs_data_end_writeback(sb, ret); + spin_lock(&sbi->trans_write_lock); if (advance) scoutfs_advance_dirty_super(sb); From 6516ce7d575d7236337c720cc0d7cd4df45737e7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Feb 2017 13:56:57 -0800 Subject: [PATCH 223/920] Report free blocks in statfs Our statfs callback was still using the old buddy allocator. We add a free segments field to the super and have it track the number of free segments in the allocator. We then use that to calculate the number of free blocks for statfs. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 24 +++++++++++++++++++++++- kmod/src/alloc.h | 1 + kmod/src/format.h | 3 +++ kmod/src/super.c | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 184493c9..5be9d490 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -166,8 +166,10 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) ret = 0; out: - if (ret == 0) + if (ret == 0) { scoutfs_inc_counter(sb, alloc_alloc); + le64_add_cpu(&super->free_segs, -1); + } up_write(&sal->rwsem); trace_printk("segno %llu ret %d\n", *segno, ret); @@ -180,6 +182,8 @@ out: */ int scoutfs_alloc_free(struct super_block *sb, u64 segno) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; struct pending_region *pend; DECLARE_SEG_ALLOC(sb, sal); u64 ind; @@ -205,6 +209,7 @@ int scoutfs_alloc_free(struct super_block *sb, u64 segno) set_bit_le(nr, pend->reg.bits); scoutfs_inc_counter(sb, alloc_free); + le64_add_cpu(&super->free_segs, 1); ret = 0; out: up_write(&sal->rwsem); @@ -281,6 +286,23 @@ out: return ret; } +/* + * Return the number of blocks free for statfs. + */ +u64 scoutfs_alloc_bfree(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + DECLARE_SEG_ALLOC(sb, sal); + u64 bfree; + + down_read(&sal->rwsem); + bfree = le64_to_cpu(super->free_segs) << SCOUTFS_SEGMENT_BLOCK_SHIFT; + up_read(&sal->rwsem); + + return bfree; +} + static int alloc_treap_compare(void *key, void *data) { u64 *ind = key; diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index 453d667a..2a400e64 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -8,6 +8,7 @@ int scoutfs_alloc_free(struct super_block *sb, u64 segno); int scoutfs_alloc_has_dirty(struct super_block *sb); int scoutfs_alloc_dirty_ring(struct super_block *sb); +u64 scoutfs_alloc_bfree(struct super_block *sb); int scoutfs_alloc_setup(struct super_block *sb); void scoutfs_alloc_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 2ba9cbd8..4a6b15bf 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -23,6 +23,8 @@ #define SCOUTFS_SEGMENT_MASK (SCOUTFS_SEGMENT_SIZE - 1) #define SCOUTFS_SEGMENT_PAGES (SCOUTFS_SEGMENT_SIZE / PAGE_SIZE) #define SCOUTFS_SEGMENT_BLOCKS (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_SEGMENT_BLOCK_SHIFT \ + (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) #define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) @@ -350,6 +352,7 @@ struct scoutfs_super_block { __le64 next_ino; __le64 alloc_uninit; __le64 total_segs; + __le64 free_segs; __le64 total_blocks; __le64 free_blocks; __le64 ring_blkno; diff --git a/kmod/src/super.c b/kmod/src/super.c index c96bbbb4..11e959a3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -56,7 +56,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) struct scoutfs_super_block *super = &sbi->super; __le32 * __packed uuid = (void *)super->uuid; - kst->f_bfree = scoutfs_buddy_bfree(sb); + kst->f_bfree = scoutfs_alloc_bfree(sb); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; kst->f_blocks = le64_to_cpu(super->total_blocks); From 8def9141bc86d3179e8b87739d9e1ba48086d545 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 3 Feb 2017 14:21:57 -0800 Subject: [PATCH 224/920] Add scoutfs_key_init_buf_len() As of yet the static key users have key and buffer lengths that match. We're about to add a link backref caller who searches with a small key but gets a result copied into a larger buffer. Signed-off-by: Zach Brown --- kmod/src/key.h | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/kmod/src/key.h b/kmod/src/key.h index 2ed2f3f6..7d3b2230 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -19,6 +19,22 @@ void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key); void scoutfs_key_dec(struct scoutfs_key_buf *key); void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); +/* + * Initialize a small key in a larger allocated buffer. This lets + * callers, for example, search for a small key and get a larger key + * copied in. + */ +static inline void scoutfs_key_init_buf_len(struct scoutfs_key_buf *key, + void *data, u16 key_len, + u16 buf_len) +{ + WARN_ON_ONCE(buf_len > SCOUTFS_MAX_KEY_SIZE); + WARN_ON_ONCE(key_len > buf_len); + + key->data = data; + key->key_len = key_len; + key->buf_len = buf_len; +} /* * Point the key buf, usually statically allocated, at an existing @@ -27,11 +43,7 @@ void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); static inline void scoutfs_key_init(struct scoutfs_key_buf *key, void *data, u16 len) { - WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE); - - key->data = data; - key->key_len = len; - key->buf_len = len; + scoutfs_key_init_buf_len(key, data, len, len); } /* From fff6fb474037c42d73cd87ca6fa5638b54b43528 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 3 Feb 2017 14:23:15 -0800 Subject: [PATCH 225/920] Restore link backref items Convert the link backref code from btree items to the item cache. Now that the backref items have the full entry name we can traverse a link with one item lookup. We don't need to lock the inode and verify that the entry at the backref offset really points to our inode. The link backref walk gets a lot simpler. But we have to widen the ioctl cursor to store a full dir ino and path name isntead of just the dir's backref counter. Signed-off-by: Zach Brown --- kmod/src/dir.c | 384 ++++++++++++++++++++++------------------------ kmod/src/dir.h | 13 +- kmod/src/format.h | 21 ++- kmod/src/ioctl.c | 134 +++++++++------- kmod/src/ioctl.h | 48 +++++- 5 files changed, 327 insertions(+), 273 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0cf55a13..16dce5df 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -188,6 +188,40 @@ static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, return key; } +static void init_link_backref_key(struct scoutfs_key_buf *key, + struct scoutfs_link_backref_key *lbrkey, + u64 ino, u64 dir_ino, + char *name, unsigned name_len) +{ + lbrkey->type = SCOUTFS_LINK_BACKREF_KEY; + lbrkey->ino = cpu_to_be64(ino); + lbrkey->dir_ino = cpu_to_be64(dir_ino); + if (name_len) + memcpy(lbrkey->name, name, name_len); + + scoutfs_key_init(key, lbrkey, offsetof(struct scoutfs_link_backref_key, + name[name_len])); +} + +static struct scoutfs_key_buf *alloc_link_backref_key(struct super_block *sb, + u64 ino, u64 dir_ino, + char *name, + unsigned name_len) +{ + struct scoutfs_link_backref_key *lbkey; + struct scoutfs_key_buf *key; + + key = scoutfs_key_alloc(sb, offsetof(struct scoutfs_link_backref_key, + name[name_len])); + if (key) { + lbkey = key->data; + init_link_backref_key(key, lbkey, ino, dir_ino, + name, name_len); + } + + return key; +} + static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { @@ -331,34 +365,6 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) return ret; } -#if 0 -static void set_lref_key(struct scoutfs_key *key, u64 ino, u64 ctr) -{ - scoutfs_set_key(key, ino, SCOUTFS_LINK_BACKREF_KEY, ctr); -} - -static int update_lref_item(struct super_block *sb, struct scoutfs_key *key, - u64 dir_ino, u64 dir_off, bool update) -{ - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_link_backref lref; - struct scoutfs_btree_val val; - int ret; - - lref.ino = cpu_to_le64(dir_ino); - lref.offset = cpu_to_le64(dir_off); - - scoutfs_btree_init_val(&val, &lref, sizeof(lref)); - - if (update) - ret = scoutfs_btree_update(sb, meta, key, &val); - else - ret = scoutfs_btree_insert(sb, meta, key, &val); - - return ret; -} -#endif - static int add_entry_items(struct inode *dir, struct dentry *dentry, struct inode *inode) { @@ -366,6 +372,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, struct dentry_info *di = dentry->d_fsdata; struct super_block *sb = dir->i_sb; struct scoutfs_key_buf *ent_key = NULL; + struct scoutfs_key_buf *lb_key = NULL; struct scoutfs_key_buf *del_keys[3]; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; @@ -415,14 +422,20 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, goto out; del_keys[del++] = &rdir_key; -#if 0 - /* backref item for inode to path resolution */ - lrkey.type = SCOUTFS_LINK_BACKREF_KEY; - lrey.ino = cpu_to_le64(scoutfs_ino(inode)); - lrey.dir = cpu_to_le64(scoutfs_ino(dir)); - scoutfs_kvec_init(key, &lrkey, sizeof(lrkey), - dentry->d_name.name, dentry->d_name.len); -#endif + /* link backref item for inode to path resolution */ + lb_key = alloc_link_backref_key(sb, scoutfs_ino(inode), + scoutfs_ino(dir), + (void *)dentry->d_name.name, + dentry->d_name.len); + if (!lb_key) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_item_create(sb, lb_key, NULL); + if (ret) + goto out; + del_keys[del++] = lb_key; update_dentry_info(dentry, &dent); ret = 0; @@ -434,6 +447,7 @@ out: } scoutfs_key_free(sb, ent_key); + scoutfs_key_free(sb, lb_key); return ret; } @@ -543,7 +557,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_key_buf *keys[2] = {NULL,}; + struct scoutfs_key_buf *keys[3] = {NULL,}; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; int ret = 0; @@ -569,6 +583,15 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) init_readdir_key(&rdir_key, &rkey, dir, dentry_info_pos(dentry)); keys[1] = &rdir_key; + keys[2] = alloc_link_backref_key(sb, scoutfs_ino(inode), + scoutfs_ino(dir), + (void *)dentry->d_name.name, + dentry->d_name.len); + if (!keys[2]) { + ret = -ENOMEM; + goto out; + } + ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys)); if (ret) goto out; @@ -600,6 +623,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) out: scoutfs_key_free(sb, keys[0]); + scoutfs_key_free(sb, keys[2]); scoutfs_release_trans(sb); return ret; } @@ -804,194 +828,154 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) } /* - * Store the null terminated path component that links to the inode at - * the given counter in the callers buffer. + * Find the next link backref key for the given ino starting from the + * given dir inode and null terminated name. If we find a backref item + * we add an allocated copy of it to the head of the caller's list. * - * This is implemented by searching for link backrefs on the inode - * starting from the given counter. Those contain references to the - * parent directory and dirent key offset that contain the link to the - * inode. - * - * The caller holds no locks that protect components in the path. We - * search the link backref to find the parent dir then acquire it's - * i_mutex to make sure that its entries and backrefs are stable. If - * the next backref points to a different dir after we acquire the lock - * we bounce off and retry. - * - * Backref counters are never reused and rename only modifies the - * existing backref counter under the dir's mutex. + * Returns 0 if we added an entry, -ENOENT if we didn't, and -errno for + * search errors. */ -static int append_linkref_name(struct super_block *sb, u64 *dir_ino, u64 ino, - u64 *ctr, char *path, unsigned int bytes) +static int add_next_linkref(struct super_block *sb, u64 ino, + u64 dir_ino, char *name, unsigned int name_len, + struct list_head *list) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_link_backref lref; - struct scoutfs_btree_val val; - struct scoutfs_dirent dent; - struct inode *inode = NULL; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_key key; - u64 retried = 0; - u64 off; + struct scoutfs_link_backref_key last_lbkey; + struct scoutfs_link_backref_entry *ent; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; int len; int ret; -retry: - scoutfs_set_key(&first, ino, SCOUTFS_LINK_BACKREF_KEY, *ctr); - scoutfs_set_key(&last, ino, SCOUTFS_LINK_BACKREF_KEY, ~0ULL); + ent = kmalloc(offsetof(struct scoutfs_link_backref_entry, + lbkey.name[SCOUTFS_NAME_LEN + 1]), GFP_KERNEL); + if (!ent) + return -ENOMEM; - scoutfs_btree_init_val(&val, &lref, sizeof(lref)); - val.check_size_eq = 1; + INIT_LIST_HEAD(&ent->head); - ret = scoutfs_btree_next(sb, meta, &first, &last, &key, &val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; + /* put search key in ent */ + init_link_backref_key(&key, &ent->lbkey, ino, dir_ino, name, name_len); + /* we actually have room for a full backref item */ + scoutfs_key_init_buf_len(&key, key.data, key.key_len, + offsetof(struct scoutfs_link_backref_key, + name[SCOUTFS_NAME_LEN + 1])); + + /* small last key to avoid full name copy, XXX enforce no U64_MAX ino */ + init_link_backref_key(&last, &last_lbkey, ino, U64_MAX, NULL, 0); + + /* next backref key is now in ent */ + ret = scoutfs_item_next(sb, &key, &last, NULL); + trace_printk("ino %llu dir_ino %llu ret %d key_len %u\n", + ino, dir_ino, ret, key.key_len); + if (ret < 0) goto out; - } - *dir_ino = le64_to_cpu(lref.ino), - off = le64_to_cpu(lref.offset); - *ctr = scoutfs_key_offset(&key); - trace_printk("ino %llu ctr %llu dir_ino %llu off %llu\n", - ino, *ctr, *dir_ino, off); - - /* XXX corruption, should never be key == U64_MAX */ - if (*ctr == U64_MAX) { + len = (int)key.key_len - sizeof(struct scoutfs_link_backref_key); + /* XXX corruption */ + if (len < 1 || len > SCOUTFS_NAME_LEN) { ret = -EIO; goto out; } - /* XXX should verify ino and offset, too */ + ent->name_len = len; + list_add(&ent->head, list); + ret = 0; +out: + if (list_empty(&ent->head)) + kfree(ent); + return ret; +} - if (inode && scoutfs_ino(inode) != *dir_ino) { - mutex_unlock(&inode->i_mutex); - iput(inode); - inode = NULL; +static u64 first_backref_dir_ino(struct list_head *list) +{ + struct scoutfs_link_backref_entry *ent; + + ent = list_first_entry(list, struct scoutfs_link_backref_entry, head); + return be64_to_cpu(ent->lbkey.dir_ino); +} + +void scoutfs_dir_free_backref_path(struct super_block *sb, + struct list_head *list) +{ + struct scoutfs_link_backref_entry *ent; + struct scoutfs_link_backref_entry *pos; + + list_for_each_entry_safe(ent, pos, list, head) { + list_del_init(&ent->head); + kfree(ent); } +} - if (!inode) { - inode = scoutfs_iget(sb, *dir_ino); - if (IS_ERR(inode)) { - ret = PTR_ERR(inode); - inode = NULL; - if (ret == -ENOENT && retried != *dir_ino) { - retried = *dir_ino; +/* + * Give the caller the next path from the root to the inode by walking + * backref items from the dir and name position, putting the backref keys + * we find in the caller's list. + * + * Return 0 if we found a path, -ENOENT if we didn't, and -errno on error. + * + * If parents get unlinked while we're searching we can fail to make it + * up to the root. We restart the search in that case. Parent dirs + * couldn't have been unlinked while they still had entries and we won't + * see links to the inode that have been unlinked. + * + * XXX Each path component traversal is consistent but that doesn't mean + * that the total traversed path is consistent. If renames hit dirs + * that have been visited and then dirs to be visited we can return a + * path that was never present in the system: + * + * path to inode mv performed built up path + * ---- + * a/b/c/d/e/f + * d/e/f + * mv a/b/c/d/e a/b/c/ + * a/b/c/e/f + * mv a/b/c a/ + * a/c/e/f + * a/c/d/e/f + * + * XXX We'll protect against this by sampling the seq before the + * traversal and restarting if we saw backref items whose seq was + * greater than the start point. It's not precise in that it doesn't + * also capture the rename of a dir that we already traversed but it + * lets us complete the traversal in one pass that very rarely restarts. + * + * XXX and worry about traversing entirely dirty backref items with + * equal seqs that have seen crazy modification? seems like we have to + * sync if we see our dirty seq. + */ +int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, + char *name, u16 name_len, + struct list_head *list) +{ + u64 par_ino; + int ret; + +retry: + /* get the next link name to the given inode */ + ret = add_next_linkref(sb, ino, dir_ino, name, name_len, list); + if (ret < 0) + goto out; + + /* then get the names of all the parent dirs */ + par_ino = first_backref_dir_ino(list); + while (par_ino != SCOUTFS_ROOT_INO) { + + ret = add_next_linkref(sb, par_ino, 0, NULL, 0, list); + if (ret < 0) { + if (ret == -ENOENT) { + /* restart if there was no parent component */ + scoutfs_dir_free_backref_path(sb, list); goto retry; } goto out; } - mutex_lock(&inode->i_mutex); - goto retry; + par_ino = first_backref_dir_ino(list); } - - scoutfs_set_key(&key, *dir_ino, SCOUTFS_DIRENT_KEY, off); - scoutfs_btree_init_val(&val, &dent, sizeof(dent), path, bytes - 1); - val.check_size_lte = 1; - - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret < 0) { - /* XXX corruption, should always have dirent for backref */ - if (ret == -ENOENT) - ret = -EIO; - else if (ret == -EOVERFLOW) - ret = -ENAMETOOLONG; - goto out; - } - - /* XXX corruption */ - if (ret <= sizeof(dent)) { - ret = -EIO; - goto out; - } - - len = ret - sizeof(dent); /* just name len, no null term */ - - /* XXX corruption */ - if (len > SCOUTFS_NAME_LEN || le64_to_cpu(dent.ino) != ino) { - ret = -EIO; - goto out; - } - - trace_printk("dent ino %llu len %d\n", le64_to_cpu(dent.ino), len); - - (*ctr)++; - path[len] = '\0'; - ret = len + 1; out: - if (inode) { - mutex_unlock(&inode->i_mutex); - iput(inode); - } - - return ret; -} - -/* - * Fill the caller's buffer with the null terminated path components - * from the target inode to the root. These will be in the opposite - * order of a typical slash delimited path. The caller's ctr gives the - * specific link to start from. - * - * This is racing with modification of components in the path. We can - * traverse a partial path only to find that it's been blown away - * entirely. If we see a component go missing we retry. The removal of - * the final link to the inode should prevent repeatedly traversing - * paths that no longer exist. - * - * Returns > 0 and *ctr is updated if a full path from the link to the - * root dir was filled, 0 if no name past *ctr was found, or -errno on - * errors. - */ -int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, - char *path, unsigned int bytes) -{ - u64 final_ctr; - u64 par_ctr; - u64 par_ino; - int ret; - int nr; - - /* update for kvec items */ - return -EINVAL; - - if (*ctr == U64_MAX) - return 0; - -retry: - final_ctr = *ctr; - ret = 0; - - /* get the next link name to the given inode */ - nr = append_linkref_name(sb, &par_ino, ino, &final_ctr, path, bytes); - if (nr <= 0) { - ret = nr; - goto out; - } - ret += nr; - - /* then get the names of all the parent dirs */ - while (par_ino != SCOUTFS_ROOT_INO) { - par_ctr = 0; - nr = append_linkref_name(sb, &par_ino, par_ino, &par_ctr, - path + ret, bytes - ret); - if (nr < 0) { - ret = nr; - goto out; - } - - /* restart if there was no parent component */ - if (nr == 0) - goto retry; - - ret += nr; - } - -out: - *ctr = final_ctr; + if (ret < 0) + scoutfs_dir_free_backref_path(sb, list); return ret; } diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 1221846e..273d1f54 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -7,14 +7,17 @@ extern const struct file_operations scoutfs_dir_fops; extern const struct inode_operations scoutfs_dir_iops; extern const struct inode_operations scoutfs_symlink_iops; -struct scoutfs_path_component { +struct scoutfs_link_backref_entry { struct list_head head; - unsigned int len; - char name[SCOUTFS_NAME_LEN]; + u16 name_len; + struct scoutfs_link_backref_key lbkey; }; -int scoutfs_dir_get_ino_path(struct super_block *sb, u64 ino, u64 *ctr, - char *path, unsigned int bytes); +int scoutfs_dir_get_backref_path(struct super_block *sb, u64 target_ino, + u64 dir_ino, char *name, u16 name_len, + struct list_head *list); +void scoutfs_dir_free_backref_path(struct super_block *sb, + struct list_head *list); int scoutfs_symlink_drop(struct super_block *sb, u64 ino); diff --git a/kmod/src/format.h b/kmod/src/format.h index 4a6b15bf..0d5edcfc 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -265,6 +265,14 @@ struct scoutfs_readdir_key { __be64 pos; } __packed; +/* value is empty */ +struct scoutfs_link_backref_key { + __u8 type; + __be64 ino; + __be64 dir_ino; + __u8 name[0]; +} __packed; + /* no value */ struct scoutfs_orphan_key { __u8 type; @@ -479,17 +487,8 @@ struct scoutfs_extent { #define SCOUTFS_EXTENT_FLAG_OFFLINE (1 << 0) -/* - * link backrefs give us a way to find all the hard links that refer - * to a target inode. They're stored at an offset determined by an - * advancing counter in their inode. - */ -struct scoutfs_link_backref { - __le64 ino; - __le64 offset; -} __packed; - +/* ino_path can search for backref items with a null term */ #define SCOUTFS_MAX_KEY_SIZE \ - offsetof(struct scoutfs_dirent_key, name[SCOUTFS_NAME_LEN]) + offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) #endif diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 4c1b039a..ff409346 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -106,90 +106,116 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, return ret; } +struct ino_path_cursor { + __u64 dir_ino; + __u8 name[SCOUTFS_NAME_LEN + 1]; +} __packed; + /* - * Fill the caller's buffer with one of the paths from the on-disk root - * directory to the target inode. + * see the definition of scoutfs_ioctl_ino_path for ioctl semantics. * - * Userspace provides a u64 counter used to chose which path to return. - * It should be initialized to zero to start iterating. After each path - * it is set to the next counter to search from. - * - * This only walks back through full hard links. None of the returned - * paths will reflect symlinks to components in the path. - * - * This doesn't ensure that the caller has permissions to traverse the - * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which - * bypasses permissions checking. - * - * ENAMETOOLONG is returned when the next path from the given counter - * doesn't fit in the buffer. Providing a buffer of PATH_MAX should - * succeed. - * - * This call is not serialized with any modification (create, rename, - * unlink) of the path components. It will return all the paths that - * were stable both before and after the call. It may or may not return - * paths which are created or unlinked during the call. - * - * The number of bytes in the path, including the null terminator, are - * returned when a path is found. 0 is returned when there are no more - * paths to the link from the given counter. -errno is returned on - * errors. - * - * XXX - * - can dir renaming trick us into returning garbage paths? seems likely. + * The null termination of the cursor name is a trick to skip past the + * last name we read without having to try and "increment" the name. + * Adding a null sorts the cursor after the non-null name and before all + * the next names because the item names aren't null terminated. */ static long scoutfs_ioc_ino_path(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_ino_path __user *uargs = (void __user *)arg; + struct scoutfs_ioctl_ino_path __user *uargs; + struct scoutfs_link_backref_entry *ent; + struct ino_path_cursor __user *ucurs; struct scoutfs_ioctl_ino_path args; - unsigned int bytes; char __user *upath; - char *comp; - char *path; + LIST_HEAD(list); + u64 dir_ino; + u16 name_len; + char term; + char *name; int ret; - int len; + + BUILD_BUG_ON(SCOUTFS_IOC_INO_PATH_CURSOR_BYTES != + sizeof(struct ino_path_cursor)); if (!capable(CAP_DAC_READ_SEARCH)) return -EPERM; + uargs = (void __user *)arg; if (copy_from_user(&args, uargs, sizeof(args))) return -EFAULT; - if (args.path_bytes <= 1) + if (args.cursor_bytes != sizeof(struct ino_path_cursor)) return -EINVAL; - bytes = min_t(unsigned int, args.path_bytes, PATH_MAX); - path = kmalloc(bytes, GFP_KERNEL); - if (path == NULL) + ucurs = (void __user *)(unsigned long)args.cursor_ptr; + upath = (void __user *)(unsigned long)args.path_ptr; + + if (get_user(dir_ino, &ucurs->dir_ino)) + return -EFAULT; + + /* alloc/copy the small cursor name, requires and includes null */ + name_len = strnlen_user(ucurs->name, sizeof(ucurs->name)); + if (name_len < 1 || name_len > sizeof(ucurs->name)) + return -EINVAL; + + name = kmalloc(name_len, GFP_KERNEL); + if (!name) return -ENOMEM; - /* positive ret is len of all components including null terminators */ - ret = scoutfs_dir_get_ino_path(sb, args.ino, &args.ctr, path, bytes); - if (ret <= 0) + if (copy_from_user(name, ucurs->name, name_len)) { + ret = -EFAULT; goto out; + } - /* reverse the components from backref order to path/ order */ - comp = path; - upath = (void __user *)((unsigned long)args.path_ptr + ret); - while (comp < (path + ret)) { - len = strlen(comp); - if (comp != path) - comp[len] = '/'; - len++; + ret = scoutfs_dir_get_backref_path(sb, args.ino, dir_ino, name, + name_len, &list); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } - upath -= len; - if (copy_to_user(upath, comp, len)) { + ret = 0; + list_for_each_entry(ent, &list, head) { + if (ret + ent->name_len + 1 > args.path_bytes) { + ret = -ENAMETOOLONG; + goto out; + } + + if (copy_to_user(upath, ent->lbkey.name, ent->name_len)) { + ret = -EFAULT; + goto out; + } + + upath += ent->name_len; + ret += ent->name_len; + + if (ent->head.next == &list) + term = '\0'; + else + term = '/'; + + if (put_user(term, upath)) { ret = -EFAULT; break; } - comp += len; + + upath++; + ret++; } - if (ret > 0 && put_user(args.ctr, &uargs->ctr)) + /* copy the last entry into the cursor */ + ent = list_last_entry(&list, struct scoutfs_link_backref_entry, head); + + if (put_user(be64_to_cpu(ent->lbkey.dir_ino), &ucurs->dir_ino) || + copy_to_user(ucurs->name, ent->lbkey.name, ent->name_len) || + put_user('\0', &ucurs->name[ent->name_len])) { ret = -EFAULT; + } + out: - kfree(path); + scoutfs_dir_free_backref_path(sb, &list); + kfree(name); return ret; } diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index d39c6272..e95a7ba2 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -26,14 +26,56 @@ struct scoutfs_ioctl_inodes_since { #define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ struct scoutfs_ioctl_inodes_since) -/* returns bytes of path buffer set starting at _off, including null */ +/* + * Fill the path buffer with the next path to the target inode. An + * iteration cursor is stored in the cursor buffer which advances + * through the paths to the inode at each call. + * + * @ino: The target ino that we're finding paths to. Constant across + * all the calls that make up an iteration over all the inode's paths. + * + * @cursor_ptr: A pointer to the buffer that will hold the iteration + * cursor. It must be initialized to 0 before iterating. Each call + * modifies it to skip past the result of that call. + * + * @cusur_bytes: The length of the cursor buffer. Must be + * SCOUTFS_IOC_INO_PATH_CURSOR_BYTES. + * + * @path_ptr: The buffer to store each found path. + * + * @path_bytes: The size of the buffer that will the found path + * including null termination. (PATH_MAX is a solid choice.) + * + * This only walks back through full hard links. None of the returned + * paths will reflect symlinks to components in the path. + * + * This doesn't ensure that the caller has permissions to traverse the + * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which + * bypasses permissions checking. + * + * ENAMETOOLONG is returned when the next path found from the cursor + * doesn't fit in the path buffer. + * + * This call is not serialized with any modification (create, rename, + * unlink) of the path components. It will return all the paths that + * were stable both before and after the call. It may or may not return + * paths which are created or unlinked during the call. + * + * The number of bytes in the path, including the null terminator, are + * returned when a path is found. 0 is returned when there are no more + * paths to the link to the inode from the cursor. + */ struct scoutfs_ioctl_ino_path { __u64 ino; - __u64 ctr; /* init to 0, set to next */ + __u64 cursor_ptr; __u64 path_ptr; - __u16 path_bytes; /* total buffer space, including null term */ + __u16 cursor_bytes; + __u16 path_bytes; } __packed; +#define SCOUTFS_IOC_INO_PATH_CURSOR_BYTES \ + (sizeof(u64) + SCOUTFS_NAME_LEN + 1) + /* Get a single path from the root to the given inode number */ #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ struct scoutfs_ioctl_ino_path) From a310027380eb48856cbf5786f8a9a28f30b03ed2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Feb 2017 14:38:39 -0800 Subject: [PATCH 226/920] Remove the find xattr ioctls The current plan for finding populations of inodes to search no longer involves xattr backrefs. We're about to change the xattr storage format so let's remove these interfaces so we don't have to update them. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 92 ------------------------------------------------ kmod/src/ioctl.h | 23 +++--------- 2 files changed, 4 insertions(+), 111 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index ff409346..843d82ec 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -219,94 +219,6 @@ out: return ret; } -/* - * Find inodes that might contain a given xattr name or value. - * - * The inodes are filled in sorted order from the first to the last - * inode. The number of found inodes is returned. If an error is hit - * it can return the number of inodes found before the error. - * - * The search can be continued from the next inode after the last - * returned. - */ -static long scoutfs_ioc_find_xattr(struct file *file, unsigned long arg, - bool find_name) -{ - struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_STABLE_META(sb); - struct scoutfs_ioctl_find_xattr args; - struct scoutfs_key key; - struct scoutfs_key last; - char __user *ustr; - u64 __user *uino; - char *str; - int copied = 0; - int ret = 0; - u64 ino; - u8 type; - u64 h; - - if (copy_from_user(&args, (void __user *)arg, sizeof(args))) - return -EFAULT; - - if (args.str_len > SCOUTFS_MAX_XATTR_LEN || args.ino_count > INT_MAX) - return -EINVAL; - - if (args.first_ino > args.last_ino) - return -EINVAL; - - if (args.ino_count == 0) - return 0; - - ustr = (void __user *)(unsigned long)args.str_ptr; - uino = (void __user *)(unsigned long)args.ino_ptr; - - str = kmalloc(args.str_len, GFP_KERNEL); - if (!str) - return -ENOMEM; - - if (copy_from_user(str, ustr, args.str_len)) { - ret = -EFAULT; - goto out; - } - - h = scoutfs_name_hash(str, args.str_len); - - if (find_name) { - h &= ~SCOUTFS_XATTR_NAME_HASH_MASK; - type = SCOUTFS_XATTR_NAME_HASH_KEY; - } else { - type = SCOUTFS_XATTR_VAL_HASH_KEY; - } - - scoutfs_set_key(&key, h, type, args.first_ino); - scoutfs_set_key(&last, h, type, args.last_ino); - - while (copied < args.ino_count) { - - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - ino = scoutfs_key_offset(&key); - if (put_user(ino, uino)) { - ret = -EFAULT; - break; - } - - uino++; - copied++; - scoutfs_inc_key(&key); - } - -out: - kfree(str); - return copied ?: ret; -} - /* * Sample the inode's data_version. It is not strictly serialized with * writes that are in flight. @@ -505,10 +417,6 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_INODE_KEY); case SCOUTFS_IOC_INO_PATH: return scoutfs_ioc_ino_path(file, arg); - case SCOUTFS_IOC_FIND_XATTR_NAME: - return scoutfs_ioc_find_xattr(file, arg, true); - case SCOUTFS_IOC_FIND_XATTR_VAL: - return scoutfs_ioc_find_xattr(file, arg, false); case SCOUTFS_IOC_INODE_DATA_SINCE: return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_EXTENT_KEY); case SCOUTFS_IOC_DATA_VERSION: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index e95a7ba2..be5e5d9f 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -80,25 +80,10 @@ struct scoutfs_ioctl_ino_path { #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ struct scoutfs_ioctl_ino_path) -/* XXX might as well include a seq? 0 for current behaviour? */ -struct scoutfs_ioctl_find_xattr { - __u64 first_ino; - __u64 last_ino; - __u64 str_ptr; - __u32 str_len; - __u64 ino_ptr; - __u32 ino_count; -} __packed; - -#define SCOUTFS_IOC_FIND_XATTR_NAME _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ - struct scoutfs_ioctl_find_xattr) -#define SCOUTFS_IOC_FIND_XATTR_VAL _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ - struct scoutfs_ioctl_find_xattr) - -#define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ +#define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ struct scoutfs_ioctl_inodes_since) -#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 6, u64) +#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, u64) struct scoutfs_ioctl_release { __u64 offset; @@ -106,7 +91,7 @@ struct scoutfs_ioctl_release { __u64 data_version; } __packed; -#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ +#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ struct scoutfs_ioctl_release) struct scoutfs_ioctl_stage { @@ -116,7 +101,7 @@ struct scoutfs_ioctl_stage { __s32 count; } __packed; -#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ +#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 6, \ struct scoutfs_ioctl_stage) #endif From 64bc145e3cf7365a74ad695bcc038b9ea0b04b7c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Feb 2017 14:54:49 -0800 Subject: [PATCH 227/920] Add scoutfs_item_set_batch() We're about to update xattrs to use the item cache API and xattrs want to be pretty big. scoutfs_item_set_batch() let's the xattr code atomically update xattrs made up of multiple items. Signed-off-by: Zach Brown --- kmod/src/item.c | 207 ++++++++++++++++++++++++++++++++++++++++-------- kmod/src/item.h | 9 +++ 2 files changed, 184 insertions(+), 32 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index e5b5970f..83e1a09c 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -40,6 +40,11 @@ * clobber them in creation and skip them in lookups. */ +static bool invalid_flags(int sif) +{ + return (sif & SIF_EXCLUSIVE) && (sif & SIF_REPLACE); +} + struct item_cache { spinlock_t lock; struct rb_root items; @@ -338,6 +343,23 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, free_item(sb, item); } +/* + * Turn an item that the caller has found while holding the lock into a + * deletion item. The caller will free whatever we put in the deletion + * value after releasing the lock. + */ +static void become_deletion_item(struct super_block *sb, + struct item_cache *cac, + struct cached_item *item, + struct kvec *del_val) +{ + scoutfs_kvec_clone(del_val, item->val); + scoutfs_kvec_init_null(item->val); + item->deletion = 1; + mark_item_dirty(cac, item); + scoutfs_inc_counter(sb, item_delete); +} + /* * Try to insert the given item. If there's already a non-deletion item * with the insertion key then return -EEXIST. An existing deletion @@ -571,6 +593,37 @@ int scoutfs_item_lookup_exact(struct super_block *sb, return ret; } +/* + * Return the next linked node in the tree that isn't a deletion item + * and which is still within the last allowed key value. + */ +static struct cached_item *next_item_node(struct rb_root *root, + struct cached_item *item, + struct scoutfs_key_buf *last) +{ + struct rb_node *node; + + while (item) { + node = rb_next(&item->node); + if (!node) { + item = NULL; + break; + } + + item = container_of(node, struct cached_item, node); + + if (scoutfs_key_compare(item->key, last) > 0) { + item = NULL; + break; + } + + if (!item->deletion) + break; + } + + return item; +} + /* * Find the next item to return from the "_next" item interface. It's the * next item from the key that isn't a deletion item and is within the @@ -582,27 +635,17 @@ static struct cached_item *item_for_next(struct rb_root *root, struct scoutfs_key_buf *last) { struct cached_item *item; - struct rb_node *node; /* limit by the lesser of the two */ - if (scoutfs_key_compare(range_end, last) < 0) + if (range_end && scoutfs_key_compare(range_end, last) < 0) last = range_end; item = next_item(root, key); - while (item) { - if (scoutfs_key_compare(item->key, last) > 0) { - item = NULL; - break; - } - - if (!item->deletion) - break; - - node = rb_next(&item->node); - if (node) - item = container_of(node, struct cached_item, node); - else + if (item) { + if (scoutfs_key_compare(item->key, last) > 0) item = NULL; + else if (item->deletion) + item = next_item_node(root, item, last); } return item; @@ -934,6 +977,123 @@ out: return ret; } +/* + * Atomically set the caller's items to be the only cached items in the + * caller's range. Any existing items that overlap with the caller's + * items are replaced. Any existing items in the range that aren't in + * the caller's list will be replaced with deletion items. The deletion + * items and the caller's inserted items will all be marked dirty. + * + * In practice this is used for relatively few items at a time, at most + * on the order of 16. So we're not too worried with it walking a small + * number of items a few times when the caller provides flags that have + * to check for existing items. + * + * Returns -ENODATA if SIF_REPLACE is set and a batch item doesn't have + * a matching existing item or -EEXIST if SIF_EXCLUSIVE is set and a + * batch item does have an existing item. + */ +int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, int sif) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct scoutfs_key_buf *missing; + SCOUTFS_DECLARE_KVEC(del_val); + struct cached_item *exist; + struct cached_item *item; + struct cached_item *tmp; + unsigned long flags; + int cmp; + int ret; + + if (WARN_ON_ONCE(invalid_flags(sif))) + return -EINVAL; + +// trace_scoutfs_item_set_batch(sb, start, end); + + if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) + return -EINVAL; + + missing = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!missing) + return -ENOMEM; + + spin_lock_irqsave(&cac->lock, flags); + + while (!check_range(sb, &cac->ranges, start, missing)) { + + spin_unlock_irqrestore(&cac->lock, flags); + ret = scoutfs_manifest_read_items(sb, start, missing); + spin_lock_irqsave(&cac->lock, flags); + + if (ret) + goto out; + } + + /* check for _EXCLUSIVE or _REPLACE errors before destroying items */ + if (!list_empty(list) && (sif & (SIF_EXCLUSIVE | SIF_REPLACE))) { + + item = list_first_entry(list, struct cached_item, entry); + exist = item_for_next(&cac->items, start, NULL, end); + + while (item) { + /* compare keys, with bias to finding _REPLACE err */ + if (exist) + cmp = scoutfs_key_compare(item->key, + exist->key); + else + cmp = -1; + + if (cmp < 0) { + if (sif & SIF_REPLACE) { + ret = -ENODATA; + goto out; + } + if (item->entry.next != list) + item = list_next_entry(item, entry); + else + item = NULL; + + } else if (cmp > 0) { + exist = next_item_node(&cac->items, exist, end); + + } else { + /* cmp == 0 */ + if (sif & SIF_EXCLUSIVE) { + ret = -EEXIST; + goto out; + } + } + } + + } + + /* delete everything in the range */ + for (exist = item_for_next(&cac->items, start, NULL, end); + exist; exist = next_item_node(&cac->items, exist, end)) { + + scoutfs_kvec_init_null(del_val); + become_deletion_item(sb, cac, exist, del_val); + scoutfs_kvec_kfree(del_val); + } + + /* insert the caller's items, overwriting any existing */ + list_for_each_entry_safe(item, tmp, list, entry) { + list_del_init(&item->entry); + insert_item(sb, cac, item, true); + mark_item_dirty(cac, item); + } + + ret = 0; +out: + spin_unlock_irqrestore(&cac->lock, flags); + scoutfs_key_free(sb, missing); + + return ret; +} + void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) { struct cached_item *item; @@ -1047,23 +1207,6 @@ out: return ret; } -/* - * Turn an item that the caller has found while holding the lock into a - * deletion item. The caller will free whatever we put in the deletion - * value after releasing the lock. - */ -static void become_deletion_item(struct super_block *sb, - struct item_cache *cac, - struct cached_item *item, - struct kvec *del_val) -{ - scoutfs_kvec_clone(del_val, item->val); - scoutfs_kvec_init_null(item->val); - item->deletion = 1; - mark_item_dirty(cac, item); - scoutfs_inc_counter(sb, item_delete); -} - /* * Delete an existing item with the given key. * diff --git a/kmod/src/item.h b/kmod/src/item.h index 6021e3c3..2e02e596 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -3,6 +3,12 @@ #include +/* behavioural flags for the item functions */ +enum { + SIF_EXCLUSIVE = (1 << 1), + SIF_REPLACE = (1 << 2), +}; + struct scoutfs_segment; struct scoutfs_key_buf; @@ -44,6 +50,9 @@ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); +int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, int sif); void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); From 54e07470f176feeda0a86dff641dd79fd77600e6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Feb 2017 15:38:45 -0800 Subject: [PATCH 228/920] Update xattrs to use the item cache Update the xattrs to use the item cache. Because we now have large keys we can store the xattr at its full name instead of having to deal with hashing the name and addressing collisions. Now that we don't have the find xattr ioctls we don't need to maintain backrefs. We also add support for large xattrs that span multiple items. The key footer and value header give us the metadata we need to iterate over the items that make up an xattr. Signed-off-by: Zach Brown --- kmod/src/format.h | 37 ++- kmod/src/scoutfs_trace.h | 2 - kmod/src/super.c | 1 + kmod/src/trans.c | 31 +- kmod/src/xattr.c | 687 +++++++++++++++------------------------ kmod/src/xattr.h | 2 + 6 files changed, 304 insertions(+), 456 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 0d5edcfc..83c4d5d8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -231,9 +231,7 @@ struct scoutfs_key { * have to stress about cleverly allocating the types. */ #define SCOUTFS_INODE_KEY 1 -#define SCOUTFS_XATTR_KEY 2 -#define SCOUTFS_XATTR_NAME_HASH_KEY 3 -#define SCOUTFS_XATTR_VAL_HASH_KEY 4 +#define SCOUTFS_XATTR_KEY 3 #define SCOUTFS_DIRENT_KEY 5 #define SCOUTFS_READDIR_KEY 6 #define SCOUTFS_LINK_BACKREF_KEY 7 @@ -286,6 +284,23 @@ struct scoutfs_data_key { __be64 block; } __packed; +/* value is each item's part of the full xattr value for the off/len */ +struct scoutfs_xattr_key { + __u8 type; + __be64 ino; + __u8 name[0]; +} __packed; + +struct scoutfs_xattr_key_footer { + __u8 null; + __u8 part; +} __packed; + +struct scoutfs_xattr_val_header { + __le16 part_len; + __u8 last_part; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; @@ -447,6 +462,13 @@ struct scoutfs_dirent { /* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ #define SCOUTFS_LINK_MAX S32_MAX +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_SIZE 65536 +#define SCOUTFS_XATTR_PART_SIZE \ + (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_xattr_val_header)) +#define SCOUTFS_XATTR_MAX_PARTS \ + DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) + /* * We only use 31 bits for readdir positions so that we don't confuse * old signed 32bit f_pos applications or those on the other side of @@ -470,15 +492,6 @@ enum { SCOUTFS_DT_WHT, }; -#define SCOUTFS_MAX_XATTR_LEN 255 -#define SCOUTFS_XATTR_NAME_HASH_MASK 7ULL - -struct scoutfs_xattr { - __u8 name_len; - __u8 value_len; - __u8 name[0]; -} __packed; - struct scoutfs_extent { __le64 blkno; __le64 len; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7b10ae90..4a1d7a7f 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -35,8 +35,6 @@ struct scoutfs_sb_info; __print_symbolic(type, \ { SCOUTFS_INODE_KEY, "INODE" }, \ { SCOUTFS_XATTR_KEY, "XATTR" }, \ - { SCOUTFS_XATTR_NAME_HASH_KEY, "XATTR_NAME_HASH"}, \ - { SCOUTFS_XATTR_VAL_HASH_KEY, "XATTR_VAL_HASH" }, \ { SCOUTFS_DIRENT_KEY, "DIRENT" }, \ { SCOUTFS_LINK_BACKREF_KEY, "LINK_BACKREF"}, \ { SCOUTFS_SYMLINK_KEY, "SYMLINK" }, \ diff --git a/kmod/src/super.c b/kmod/src/super.c index 11e959a3..792d25ae 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -312,6 +312,7 @@ static int __init scoutfs_module_init(void) ret = scoutfs_inode_init() ?: scoutfs_dir_init() ?: + scoutfs_xattr_init() ?: register_filesystem(&scoutfs_fs_type); if (ret) teardown_module(); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 26e30367..487514f6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -219,28 +219,27 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, } /* - * I think the holder that creates the most dirty item data is - * symlinking which can create an inode, the three dirent items with a - * full file name, and a symlink item with a full path. + * The holder that creates the most dirty item data is adding a full + * size xattr. The largest xattr can have a 255 byte name and 64KB + * value. * * XXX Assuming the worst case here too aggressively limits the number * of concurrent holders that can work without being blocked when they * know they'll dirty much less. We may want to have callers pass in * their item, key, and val budgets if that's not too fragile. - * - * XXX fix to use real backref and symlink items, placeholders for now */ -#define HOLD_WORST_ITEMS 5 -#define HOLD_WORST_KEYS (sizeof(struct scoutfs_inode_key) + \ - sizeof(struct scoutfs_dirent_key) + SCOUTFS_NAME_LEN +\ - sizeof(struct scoutfs_readdir_key) + \ - sizeof(struct scoutfs_readdir_key) + \ - sizeof(struct scoutfs_inode_key)) -#define HOLD_WORST_VALS (sizeof(struct scoutfs_inode) + \ - sizeof(struct scoutfs_dirent) + \ - sizeof(struct scoutfs_dirent) + SCOUTFS_NAME_LEN + \ - sizeof(struct scoutfs_dirent) + SCOUTFS_NAME_LEN + \ - SCOUTFS_SYMLINK_MAX_SIZE) +#define HOLD_WORST_ITEMS \ + SCOUTFS_XATTR_MAX_PARTS + +#define HOLD_WORST_KEYS \ + (SCOUTFS_XATTR_MAX_PARTS * \ + (sizeof(struct scoutfs_xattr_key) + \ + SCOUTFS_XATTR_MAX_NAME_LEN + \ + sizeof(struct scoutfs_xattr_key_footer))) + +#define HOLD_WORST_VALS \ + (sizeof(struct scoutfs_xattr_val_header) + \ + SCOUTFS_XATTR_MAX_SIZE) /* * We're able to hold the transaction if the current dirty item bytes diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 4a0a2a7c..52f1acd7 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -19,291 +19,115 @@ #include "inode.h" #include "key.h" #include "super.h" -#include "btree.h" +#include "kvec.h" +#include "item.h" #include "trans.h" #include "name.h" #include "xattr.h" /* - * xattrs are stored in items with offsets set to the hash of their - * name. The item's value contains the xattr name and value. + * In the simple case an xattr is stored in a single item whose key and + * value contain the key and value from the xattr. * - * We reserve a few low bits of the key offset for hash collisions. - * Lookup walks collisions looking for an xattr with its name and create - * looks for a hole in the colliding key space for the new xattr. + * But xattr values can be larger than our max item value length. In + * that case the rest of the xattr value is stored in additional items. + * Each item key contains a footer struct after the name which + * identifies the position of the item in the series that make up the + * total xattr. * - * Usually btree block locking would protect the atomicity of xattr - * value updates. Lookups would have to wait for modification to - * finish. But the collision items are updated with multiple btree - * operations. And we insert new items before deleting the old so that - * we can always unwind on errors. This means that there can be - * multiple versions of an xattr in the btree. So we add an inode rw - * semaphore around xattr operations. - * - * We support ioctls which find inodes that may contain xattrs with - * either a given name or value. A name hash item is created for a - * given hash value with no collision bits as long as there are any - * names at that hash value. A value hash item is created but it - * contains a refcount in its value to track the number of values with - * that hash value because we can't use the xattr keys to determine if - * there are matching values or not. + * That xattrs are then spread out across multiple items does mean that + * we need locking other than the item cache locking which only protects + * each item call, the i_mutex which isn't held on getxattr, and cluster + * locking which doesn't serialize local matches on the same node. We + * use a rwsem in the inode. * * XXX * - add acl support and call generic xattr->handlers for SYSTEM - * - remove all xattrs on unlink */ -/* the value immediately follows the name and there is no null termination */ -static char *xat_value(struct scoutfs_xattr *xat) +/* + * We have a static full xattr name with all 1s so that we can construct + * precise final keys for the range of items that cover all the xattrs + * on an inode. We could instead construct a smaller last key for the + * next inode with a null name but that could be accidentally create + * lock contention with that next inode. We want lock ranges to be as + * precise as possible. + */ +static char last_xattr_name[SCOUTFS_XATTR_MAX_NAME_LEN]; + +/* account for the footer after the name */ +static unsigned xattr_key_bytes(unsigned name_len) { - return &xat->name[xat->name_len]; + return offsetof(struct scoutfs_xattr_key, name[name_len]) + + sizeof(struct scoutfs_xattr_key_footer); } -static unsigned int xat_bytes(unsigned int name_len, unsigned int value_len) +static unsigned xattr_key_name_len(struct scoutfs_key_buf *key) { - return offsetof(struct scoutfs_xattr, name[name_len + value_len]); + return key->key_len - xattr_key_bytes(0); } -static void set_xattr_keys(struct inode *inode, struct scoutfs_key *first, - struct scoutfs_key *last, const char *name, - unsigned int name_len) +static struct scoutfs_xattr_key_footer * +xattr_key_footer(struct scoutfs_key_buf *key) { - u64 h = scoutfs_name_hash(name, name_len) & - ~SCOUTFS_XATTR_NAME_HASH_MASK; - - scoutfs_set_key(first, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, h); - scoutfs_set_key(last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, - h | SCOUTFS_XATTR_NAME_HASH_MASK); + return key->data + key->key_len - + sizeof(struct scoutfs_xattr_key_footer); } -static void set_name_val_keys(struct scoutfs_key *name_key, - struct scoutfs_key *val_key, - struct scoutfs_key *key, u64 val_hash) +static struct scoutfs_key_buf *alloc_xattr_key(struct super_block *sb, + u64 ino, const char *name, + unsigned int name_len, u8 part) { - u64 h = scoutfs_key_offset(key) & ~SCOUTFS_XATTR_NAME_HASH_MASK; + struct scoutfs_xattr_key_footer *foot; + struct scoutfs_xattr_key *xkey; + struct scoutfs_key_buf *key; - scoutfs_set_key(name_key, h, SCOUTFS_XATTR_NAME_HASH_KEY, - scoutfs_key_inode(key)); + key = scoutfs_key_alloc(sb, xattr_key_bytes(name_len)); + if (key) { + xkey = key->data; + foot = xattr_key_footer(key); - scoutfs_set_key(val_key, val_hash, SCOUTFS_XATTR_VAL_HASH_KEY, - scoutfs_key_inode(key)); + xkey->type = SCOUTFS_XATTR_KEY; + xkey->ino = cpu_to_be64(ino); + + if (name && name_len) + memcpy(xkey->name, name, name_len); + + foot->null = '\0'; + foot->part = part; + } + + return key; +} + +static void set_xattr_key_part(struct scoutfs_key_buf *key, u8 part) +{ + struct scoutfs_xattr_key_footer *foot = xattr_key_footer(key); + + foot->part = part; } /* - * Before insertion we perform a pretty through search of the xattr - * items whose offset collides with the name to be inserted. + * This walks the keys and values for the items that make up the xattr + * items that describe the value in the caller's buffer. The caller is + * responsible for breaking out when it hits an existing final item that + * hasn't consumed the buffer. * - * We try to find the item with the matching item so it can be removed. - * We notice if there are other colliding names so that the caller can - * correctly maintain the name hash items. We calculate the value hash - * of the existing item so that the caller can maintain the value hash - * items. And we notice if there are any free colliding items that are - * available for new item insertion. + * Each iteration sets the val header in case the caller is writing + * items. If they're reading items they'll just overwrite it. */ -struct xattr_search_results { - bool found; - bool other_coll; - struct scoutfs_key key; - u64 val_hash; - bool found_hole; - struct scoutfs_key hole_key; -}; - -static int search_xattr_items(struct inode *inode, const char *name, - unsigned int name_len, - struct xattr_search_results *res) -{ - struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_xattr *xat; - struct scoutfs_key last; - struct scoutfs_key key; - unsigned int max_len; - int ret; - - max_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, SCOUTFS_MAX_XATTR_LEN), - xat = kmalloc(max_len, GFP_KERNEL); - if (!xat) - return -ENOMEM; - - set_xattr_keys(inode, &key, &last, name, name_len); - scoutfs_btree_init_val(&val, xat, max_len); - - res->found = false; - res->other_coll = false; - res->found_hole = false; - res->hole_key = key; - - for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - /* XXX corruption */ - if (ret < sizeof(struct scoutfs_xattr) || - ret != xat_bytes(xat->name_len, xat->value_len)) { - ret = -EIO; - break; - } - - /* found a hole when we skip past next expected key */ - if (!res->found_hole && - scoutfs_key_cmp(&res->hole_key, &key) < 0) - res->found_hole = true; - - /* keep searching for a hole past this key */ - if (!res->found_hole) { - res->hole_key = key; - scoutfs_inc_key(&res->hole_key); - } - - /* only compare the names until we find our given name */ - if (!res->found && - scoutfs_names_equal(name, name_len, xat->name, - xat->name_len)) { - res->found = true; - res->key = key; - res->val_hash = scoutfs_name_hash(xat_value(xat), - xat->value_len); - } else { - res->other_coll = true; - } - - /* finished once we have all the caller needs */ - if (res->found && res->other_coll && res->found_hole) { - ret = 0; - break; - } - - scoutfs_inc_key(&key); - } - - kfree(xat); - return ret; -} - -/* - * Inset a new xattr item, updating the name and value hash items as - * needed. The caller is responsible for managing transactions and - * locking. If this returns an error then no changes will have been - * made. - */ -static int insert_xattr(struct inode *inode, const char *name, - unsigned int name_len, const void *value, size_t size, - struct scoutfs_key *key, bool other_coll, - u64 val_hash) -{ - struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - bool inserted_name_hash_item = false; - struct scoutfs_btree_val val; - __le64 refcount; - struct scoutfs_key name_key; - struct scoutfs_key val_key; - struct scoutfs_xattr xat; - int ret; - - /* insert the main xattr item */ - set_name_val_keys(&name_key, &val_key, key, val_hash); - scoutfs_btree_init_val(&val, &xat, sizeof(xat), (void *)name, name_len, - (void *)value, size); - - xat.name_len = name_len; - xat.value_len = size; - - ret = scoutfs_btree_insert(sb, meta, key, &val); - if (ret) - return ret; - - /* insert the name hash item for find_xattr if we're first */ - if (!other_coll) { - ret = scoutfs_btree_insert(sb, meta, &name_key, NULL); - /* XXX eexist would be corruption */ - if (ret) - goto out; - inserted_name_hash_item = true; - } - - /* increment the val hash item for find_xattr, inserting if first */ - scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); - val.check_size_eq = 1; - - ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); - if (ret < 0 && ret != -ENOENT) - goto out; - - if (ret == -ENOENT) { - refcount = cpu_to_le64(1); - ret = scoutfs_btree_insert(sb, meta, &val_key, &val); - } else { - le64_add_cpu(&refcount, 1); - ret = scoutfs_btree_update(sb, meta, &val_key, &val); - } -out: - if (ret) { - scoutfs_btree_delete(sb, meta, key); - if (inserted_name_hash_item) - scoutfs_btree_delete(sb, meta, &name_key); - } - return ret; -} - -/* - * Remove an xattr. Remove the name hash item if there are no more xattrs - * in the inode that hash to the name's hash value. Remove the value hash - * item if there are no more xattr values in the inode with this value - * hash. - */ -static int delete_xattr(struct super_block *sb, struct scoutfs_key *key, - bool other_coll, u64 val_hash) -{ - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_key name_key; - struct scoutfs_key val_key; - __le64 refcount; - int ret; - - set_name_val_keys(&name_key, &val_key, key, val_hash); - - /* update the val_hash refcount, making sure it's not nonsense */ - scoutfs_btree_init_val(&val, &refcount, sizeof(refcount)); - val.check_size_eq = 1; - ret = scoutfs_btree_lookup(sb, meta, &val_key, &val); - if (ret < 0) - goto out; - - le64_add_cpu(&refcount, -1ULL); - - /* ensure that we can update and delete name_ and val_ keys */ - if (!other_coll) { - ret = scoutfs_btree_dirty(sb, meta, &name_key); - if (ret) - goto out; - } - ret = scoutfs_btree_dirty(sb, meta, &val_key); - if (ret) - goto out; - - ret = scoutfs_btree_delete(sb, meta, key); - if (ret) - goto out; - - if (!other_coll) - scoutfs_btree_delete(sb, meta, &name_key); - - if (refcount) - scoutfs_btree_update(sb, meta, &val_key, &val); - else - scoutfs_btree_delete(sb, meta, &val_key); - ret = 0; -out: - return ret; -} +#define for_each_xattr_item(key, val, vh, buffer, size, part, off, bytes) \ + for (part = 0, off = 0; \ + off < size && \ + (bytes = min_t(size_t, SCOUTFS_XATTR_PART_SIZE, size - off), \ + set_xattr_key_part(key, part), \ + (vh)->part_len = cpu_to_le16(bytes), \ + (vh)->last_part = off + bytes == size ? 1 : 0, \ + scoutfs_kvec_init(val, vh, \ + sizeof(struct scoutfs_xattr_val_header), \ + buffer + off, bytes), \ + 1); \ + part++, off += bytes) /* * This will grow to have all the supported prefixes (then will turn @@ -315,76 +139,89 @@ static int unknown_prefix(const char *name) } /* - * Look up an xattr matching the given name. We walk our xattr items stored - * at the hashed name. We'll only be able to copy out a value that fits - * in the callers buffer. + * Copy the value for the given xattr name into the caller's buffer, if it + * fits. Return the bytes copied or -ERANGE if it doesn't fit. */ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, size_t size) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); struct scoutfs_inode_info *si = SCOUTFS_I(inode); - size_t name_len = strlen(name); - struct scoutfs_btree_val val; - struct scoutfs_xattr *xat; - struct scoutfs_key key; - struct scoutfs_key last; - unsigned int item_len; + struct scoutfs_xattr_val_header vh; + struct scoutfs_key_buf *key = NULL; + SCOUTFS_DECLARE_KVEC(val); + unsigned int total; + unsigned int bytes; + unsigned int off; + size_t name_len; + u8 part; int ret; if (unknown_prefix(name)) return -EOPNOTSUPP; - /* make sure we don't allocate an enormous item */ - if (name_len > SCOUTFS_MAX_XATTR_LEN) + name_len = strlen(name); + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) return -ENODATA; - size = min_t(size_t, size, SCOUTFS_MAX_XATTR_LEN); - item_len = xat_bytes(name_len, size); - xat = kmalloc(item_len, GFP_KERNEL); - if (!xat) + /* honestly, userspace, just alloc a max size buffer */ + if (size == 0) + return SCOUTFS_XATTR_MAX_SIZE; + + key = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0); + if (!key) return -ENOMEM; - set_xattr_keys(inode, &key, &last, name, name_len); - scoutfs_btree_init_val(&val, xat, item_len); - down_read(&si->xattr_rwsem); - for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + total = 0; + vh.last_part = 0; + + for_each_xattr_item(key, val, &vh, buffer, size, part, off, bytes) { + + ret = scoutfs_item_lookup(sb, key, val); if (ret < 0) { if (ret == -ENOENT) - ret = -ENODATA; + ret = -EIO; break; } - /* XXX corruption */ - if (ret < sizeof(struct scoutfs_xattr)) { + /* XXX corruption: no header, more val than header len */ + ret -= sizeof(struct scoutfs_xattr_val_header); + if (ret < 0 || ret > le16_to_cpu(vh.part_len)) { ret = -EIO; break; } - if (!scoutfs_names_equal(name, name_len, xat->name, - xat->name_len)) { - scoutfs_inc_key(&key); - continue; + /* not enough buffer if we didn't copy the part */ + if (ret < le16_to_cpu(vh.part_len)) { + ret = -ERANGE; + break; } - ret = xat->value_len; - if (buffer) { - if (ret <= size) - memcpy(buffer, xat_value(xat), ret); - else - ret = -ERANGE; + total += ret; + + /* XXX corruption: total xattr val too long */ + if (total > SCOUTFS_XATTR_MAX_SIZE) { + ret = -EIO; + break; + } + + /* done if we fully copied last part */ + if (vh.last_part) { + ret = total; + break; } - break; } + /* not enough buffer if we didn't see last */ + if (ret >= 0 && !vh.last_part) + ret = -ERANGE; + up_read(&si->xattr_rwsem); + scoutfs_key_free(sb, key); - kfree(xat); return ret; } @@ -392,95 +229,98 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, * The confusing swiss army knife of creating, modifying, and deleting * xattrs. * - * If the value pointer is non-null then we always create a new item. The - * value can have a size of 0. We create a new item before possibly - * deleting an old item. + * This always removes the old existing xattr. If value is set then + * we're replacing it with a new xattr. The flags cause creation to + * fail if the xattr already exists (_CREATE) or doesn't already exist + * (_REPLACE). xattrs can have a zero length value. * - * We always delete the old xattr item. If we have a null value then we're - * deleting the xattr. If there's a value then we're effectively updating - * the xattr by deleting old and creating new. + * To modify xattrs built of individual items we use the batch + * interface. It provides atomic transitions from one group of items to + * another. */ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) { struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct xattr_search_results old = {0,}; + struct scoutfs_key_buf *last; + struct scoutfs_key_buf *key; + struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); - u64 new_val_hash = 0; + SCOUTFS_DECLARE_KVEC(val); + unsigned int bytes; + unsigned int off; + LIST_HEAD(list); + u8 part; + int sif; int ret; - if (name_len > SCOUTFS_MAX_XATTR_LEN || - (value && size > SCOUTFS_MAX_XATTR_LEN)) + trace_printk("name_len %zu value %p size %zu flags 0x%x\n", + name_len, value, size, flags); + + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN || + (value && size > SCOUTFS_XATTR_MAX_SIZE)) return -EINVAL; if (unknown_prefix(name)) return -EOPNOTSUPP; - ret = scoutfs_hold_trans(sb); - if (ret) - return ret; - - ret = scoutfs_dirty_inode_item(inode); - if (ret) - goto out; - - /* might as well do this outside locking */ - if (value) - new_val_hash = scoutfs_name_hash(value, size); - - down_write(&si->xattr_rwsem); - - /* - * The presence of other colliding names is a little tricky. - * Searching will set it if there are other non-matching names. - * It will be false if we only found the old matching name. That - * old match is also considered a collision for later insertion. - * Then *that* insertion is considered a collision for deletion - * of the existing old matching name. - */ - ret = search_xattr_items(inode, name, name_len, &old); - if (ret) - goto out; - - if (old.found && (flags & XATTR_CREATE)) { - ret = -EEXIST; - goto out; - } - if (!old.found && (flags & XATTR_REPLACE)) { - ret = -ENODATA; + key = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0); + last = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0xff); + if (!key || !last) { + ret = -ENOMEM; goto out; } + /* build up batch of new items for the new xattr */ if (value) { - ret = insert_xattr(inode, name, name_len, value, size, - &old.hole_key, old.other_coll || old.found, - new_val_hash); - if (ret) - goto out; - } + for_each_xattr_item(key, val, &vh, (void *)value, size, + part, off, bytes) { - if (old.found) { - ret = delete_xattr(sb, &old.key, old.other_coll || value, - old.val_hash); - if (ret) { - if (value) - delete_xattr(sb, &old.hole_key, true, - new_val_hash); - goto out; + ret = scoutfs_item_add_batch(sb, &list, key, val); + if (ret) + goto out; } } - inode_inc_iversion(inode); - inode->i_ctime = CURRENT_TIME; - scoutfs_update_inode_item(inode); - ret = 0; -out: + /* XXX could add range deletion items around xattr items here */ + + /* reset key to first */ + set_xattr_key_part(key, 0); + + if (flags & XATTR_CREATE) + sif = SIF_EXCLUSIVE; + else if (flags & XATTR_REPLACE) + sif = SIF_REPLACE; + else + sif = 0; + + ret = scoutfs_hold_trans(sb); + if (ret) + goto out; + + down_write(&si->xattr_rwsem); + + ret = scoutfs_dirty_inode_item(inode) ?: + scoutfs_item_set_batch(sb, &list, key, last, sif); + if (ret == 0) { + /* XXX do these want i_mutex or anything? */ + inode_inc_iversion(inode); + inode->i_ctime = CURRENT_TIME; + scoutfs_update_inode_item(inode); + } + up_write(&si->xattr_rwsem); scoutfs_release_trans(sb); +out: + scoutfs_item_free_batch(sb, &list); + scoutfs_key_free(sb, key); + scoutfs_key_free(sb, last); + return ret; } @@ -503,134 +343,129 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_xattr *xat; - struct scoutfs_key key; - struct scoutfs_key last; - unsigned int item_len; + struct scoutfs_xattr_key_footer *foot; + struct scoutfs_xattr_key *xkey; + struct scoutfs_key_buf *key; + struct scoutfs_key_buf *last; ssize_t total; + int name_len; int ret; - item_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, 0); - xat = kmalloc(item_len, GFP_KERNEL); - if (!xat) - return -ENOMEM; + key = alloc_xattr_key(sb, scoutfs_ino(inode), + NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); + last = alloc_xattr_key(sb, scoutfs_ino(inode), last_xattr_name, + SCOUTFS_XATTR_MAX_NAME_LEN, 0xff); + if (!key || !last) { + ret = -ENOMEM; + goto out; + } - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, 0); - scoutfs_set_key(&last, scoutfs_ino(inode), SCOUTFS_XATTR_KEY, ~0ULL); - scoutfs_btree_init_val(&val, xat, item_len); + xkey = key->data; + xkey->name[0] = '\0'; down_read(&si->xattr_rwsem); total = 0; for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + ret = scoutfs_item_next(sb, key, last, NULL); if (ret < 0) { if (ret == -ENOENT) - ret = 0; + ret = total; break; } + /* not used until we verify key len */ + foot = xattr_key_footer(key); + /* XXX corruption */ - if (ret < sizeof(struct scoutfs_xattr)) { + if (key->key_len < xattr_key_bytes(1) || + foot->null != '\0' || foot->part != 0) { ret = -EIO; break; } - total += xat->name_len + 1; + name_len = xattr_key_name_len(key); + + /* XXX corruption? */ + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) { + ret = -EIO; + break; + } + + total += name_len + 1; if (size) { - if (!buffer || total > size) { + if (total > size) { ret = -ERANGE; break; } - memcpy(buffer, xat->name, xat->name_len); - buffer += xat->name_len; + memcpy(buffer, xkey->name, name_len); + buffer += name_len; *(buffer++) = '\0'; } - scoutfs_inc_key(&key); + set_xattr_key_part(key, 0xff); } up_read(&si->xattr_rwsem); +out: + scoutfs_key_free(sb, key); + scoutfs_key_free(sb, last); - kfree(xat); - - return ret < 0 ? ret : total; + return ret; } /* - * Delete all the xattr items associted with this inode. The caller + * Delete all the xattr items associated with this inode. The caller * holds a transaction. * - * The name and value hashes are sorted by the hash value instead of the - * inode so we have to use the inode's xattr items to find them. We - * only remove the xattr item once the hash items are removed. - * - * Hash items can be shared amongst xattrs whose names or values hash to - * the same hash value. We don't bother trying to remove the hash items - * as the last xattr is removed. We always try to remove them and allow - * failure when we try to remove a hash item that wasn't found. + * XXX This isn't great because it reads in all the items so that it can + * create deletion items for each. It would be better to have the + * caller create range deletion items for all the items covered by the + * inode. That wouldn't require reading at all. */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct scoutfs_xattr *xat; - struct scoutfs_key last; - struct scoutfs_key key; - struct scoutfs_key name_key; - struct scoutfs_key val_key; - unsigned int item_len; - u64 val_hash; + struct scoutfs_key_buf *key; + struct scoutfs_key_buf *last; int ret; - scoutfs_set_key(&key, ino, SCOUTFS_XATTR_KEY, 0); - scoutfs_set_key(&last, ino, SCOUTFS_XATTR_KEY, ~0ULL); + key = alloc_xattr_key(sb, ino, NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); + last = alloc_xattr_key(sb, ino, last_xattr_name, + SCOUTFS_XATTR_MAX_NAME_LEN, 0xff); + if (!key || !last) { + ret = -ENOMEM; + goto out; + } - item_len = xat_bytes(SCOUTFS_MAX_XATTR_LEN, SCOUTFS_MAX_XATTR_LEN), - xat = kmalloc(item_len, GFP_KERNEL); - if (!xat) - return -ENOMEM; - - scoutfs_btree_init_val(&val, xat, item_len); + /* the inode is dead so we don't need the xattr sem */ for (;;) { - ret = scoutfs_btree_next(sb, meta, &key, &last, &key, &val); + ret = scoutfs_item_next(sb, key, last, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - /* XXX corruption */ - if (ret < sizeof(struct scoutfs_xattr) || - ret != xat_bytes(xat->name_len, xat->value_len)) { - ret = -EIO; - break; - } - - val_hash = scoutfs_name_hash(xat_value(xat), xat->value_len); - set_name_val_keys(&name_key, &val_key, &key, val_hash); - - ret = scoutfs_btree_delete(sb, meta, &name_key); - if (ret && ret != -ENOENT) + ret = scoutfs_item_delete(sb, key); + if (ret) break; - ret = scoutfs_btree_delete(sb, meta, &val_key); - if (ret && ret != -ENOENT) - break; - - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret && ret != -ENOENT) - break; - - scoutfs_inc_key(&key); + /* don't need to increment past deleted key */ } - kfree(xat); +out: + scoutfs_key_free(sb, key); + scoutfs_key_free(sb, last); return ret; } + +int scoutfs_xattr_init(void) +{ + memset(last_xattr_name, 0xff, sizeof(last_xattr_name)); + + return 0; +} diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index e0fadf32..1035d622 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -10,4 +10,6 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); int scoutfs_xattr_drop(struct super_block *sb, u64 ino); +int scoutfs_xattr_init(void); + #endif From 75b018a0e7564e46c456e940d47d3591c7eeca8f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Feb 2017 16:14:09 -0800 Subject: [PATCH 229/920] Add symlinks back Convert symlinks to use the new item cache API. This is so much easier because our max item size matches the symlink size. Signed-off-by: Zach Brown --- kmod/src/dir.c | 108 +++++++++++++++------------------------------- kmod/src/format.h | 6 +++ kmod/src/item.h | 2 - 3 files changed, 41 insertions(+), 75 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 16dce5df..0d5f0bb2 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -628,6 +628,15 @@ out: return ret; } +static void init_symlink_key(struct scoutfs_key_buf *key, + struct scoutfs_symlink_key *skey, u64 ino) +{ + skey->type = SCOUTFS_SYMLINK_KEY; + skey->ino = cpu_to_be64(ino); + + scoutfs_key_init(key, skey, sizeof(struct scoutfs_symlink_key)); +} + /* * Full a buffer with the null terminated symlink, point nd at it, and * return it so put_link can free it once the vfs is done. @@ -640,18 +649,12 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); loff_t size = i_size_read(inode); - struct scoutfs_btree_val val; - struct scoutfs_key key; + struct scoutfs_symlink_key skey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); char *path; - int bytes; - int off; int ret; - int k; - - /* update for kvec items */ - return ERR_PTR(-EINVAL); /* XXX corruption */ if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) @@ -665,30 +668,17 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (!path) return ERR_PTR(-ENOMEM); - for (off = 0, k = 0; off < size ; k++) { - scoutfs_set_key(&key, scoutfs_ino(inode), - SCOUTFS_SYMLINK_KEY, k); - bytes = min_t(int, size - off, SCOUTFS_MAX_ITEM_LEN); - scoutfs_btree_init_val(&val, path + off, bytes); - val.check_size_eq = 1; + init_symlink_key(&key, &skey, scoutfs_ino(inode)); + scoutfs_kvec_init(val, path, size); - ret = scoutfs_btree_lookup(sb, meta, &key, &val); - if (ret < 0) { - /* XXX corruption */ - if (ret == -ENOENT) - ret = -EIO; - break; - } + ret = scoutfs_item_lookup(sb, &key, val); - off += bytes; - ret = 0; - } - - /* XXX corruption */ - if (ret == 0 && (off != size || path[off - 1] != '\0')) + /* XXX corruption: missing item, wrong size, not null term */ + if (ret == -ENOENT || + (ret >= 0 && (ret != size || path[size - 1] != '\0'))) ret = -EIO; - if (ret) { + if (ret < 0) { kfree(path); path = ERR_PTR(ret); } else { @@ -724,18 +714,12 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) { struct super_block *sb = dir->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_btree_val val; - struct inode *inode = NULL; - struct scoutfs_key key; const int name_len = strlen(symname) + 1; - int off; - int bytes; + struct scoutfs_symlink_key skey; + struct scoutfs_key_buf key; + struct inode *inode = NULL; + SCOUTFS_DECLARE_KVEC(val); int ret; - int k = 0; - - /* update for kvec items */ - return -EINVAL; /* path_max includes null as does our value for nd_set_link */ if (name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) @@ -755,17 +739,12 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, goto out; } - for (k = 0, off = 0; off < name_len; off += bytes, k++) { - scoutfs_set_key(&key, scoutfs_ino(inode), SCOUTFS_SYMLINK_KEY, - k); - bytes = min(name_len - off, SCOUTFS_MAX_ITEM_LEN); + init_symlink_key(&key, &skey, scoutfs_ino(inode)); + scoutfs_kvec_init(val, (void *)symname, name_len); - scoutfs_btree_init_val(&val, (char *)symname + off, bytes); - - ret = scoutfs_btree_insert(sb, meta, &key, &val); - if (ret) - goto out; - } + ret = scoutfs_item_create(sb, &key, val); + if (ret) + goto out; ret = add_entry_items(dir, dentry, inode); if (ret) @@ -788,41 +767,24 @@ out: if (!IS_ERR_OR_NULL(inode)) iput(inode); - while (k--) { - scoutfs_set_key(&key, scoutfs_ino(inode), - SCOUTFS_SYMLINK_KEY, k); - scoutfs_btree_delete(sb, meta, &key); - } + scoutfs_item_delete(sb, &key); } scoutfs_release_trans(sb); return ret; } -/* - * Delete all the symlink items. There should only ever be a handful of - * these that contain the target path of the symlink. - */ int scoutfs_symlink_drop(struct super_block *sb, u64 ino) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_key key; + struct scoutfs_symlink_key skey; + struct scoutfs_key_buf key; int ret; - int nr; - int k; - nr = DIV_ROUND_UP(SCOUTFS_SYMLINK_MAX_SIZE, SCOUTFS_MAX_ITEM_LEN); + init_symlink_key(&key, &skey, ino); - for (k = 0; k < nr; k++) { - scoutfs_set_key(&key, ino, SCOUTFS_SYMLINK_KEY, k); - - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - } + ret = scoutfs_item_delete(sb, &key); + if (ret == -ENOENT) + ret = 0; return ret; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 83c4d5d8..77945486 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -301,6 +301,12 @@ struct scoutfs_xattr_val_header { __u8 last_part; } __packed; +/* value is the null terminated target path */ +struct scoutfs_symlink_key { + __u8 type; + __be64 ino; +} __packed; + struct scoutfs_btree_root { u8 height; struct scoutfs_block_ref ref; diff --git a/kmod/src/item.h b/kmod/src/item.h index 2e02e596..bd3e3f73 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -25,8 +25,6 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct kvec *val, int len); int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val); -int scoutfs_item_insert(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_create_ephemeral(struct super_block *sb, From 92b10e8270607987bdb4bf4ce3889fc66b5683d5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 08:44:14 -0800 Subject: [PATCH 230/920] Write super with bio functions Write our super block from an allocated page with our bio functions instead of relying on the old block cache layer which is going away. Signed-off-by: Zach Brown --- kmod/src/bio.c | 11 +++++++++++ kmod/src/bio.h | 2 ++ kmod/src/super.c | 23 +++++++++++++---------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index bb42b02a..916c5be7 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -197,6 +197,17 @@ int scoutfs_bio_read(struct super_block *sb, struct page **pages, return scoutfs_bio_wait_comp(sb, &comp); } +int scoutfs_bio_write(struct super_block *sb, struct page **pages, + u64 blkno, unsigned int nr_blocks) +{ + struct scoutfs_bio_completion comp; + + scoutfs_bio_init_comp(&comp); + scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr_blocks, &comp); + + return scoutfs_bio_wait_comp(sb, &comp); +} + /* return pointer to the blk 4k block offset amongst the pages */ void *scoutfs_page_block_address(struct page **pages, unsigned int blk) { diff --git a/kmod/src/bio.h b/kmod/src/bio.h index d2e3390a..93439775 100644 --- a/kmod/src/bio.h +++ b/kmod/src/bio.h @@ -35,6 +35,8 @@ int scoutfs_bio_wait_comp(struct super_block *sb, int scoutfs_bio_read(struct super_block *sb, struct page **pages, u64 blkno, unsigned int nr_blocks); +int scoutfs_bio_write(struct super_block *sb, struct page **pages, + u64 blkno, unsigned int nr_blocks); void *scoutfs_page_block_address(struct page **pages, unsigned int blk); diff --git a/kmod/src/super.c b/kmod/src/super.c index 792d25ae..00b428a7 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -110,26 +111,28 @@ void scoutfs_advance_dirty_super(struct super_block *sb) /* * The caller is responsible for setting the super header's blkno * and seq to something reasonable. + * + * XXX it'd be pretty easy to preallocate to avoid failure here. */ int scoutfs_write_dirty_super(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; - struct scoutfs_block *bl; + struct page *page; int ret; - /* XXX prealloc? */ - bl = scoutfs_block_dirty(sb, le64_to_cpu(sbi->super.hdr.blkno)); - if (WARN_ON_ONCE(IS_ERR(bl))) - return PTR_ERR(bl); - super = scoutfs_block_data(bl); + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return -ENOMEM; + super = page_address(page); memcpy(super, &sbi->super, sizeof(*super)); - scoutfs_block_zero(bl, sizeof(*super)); - scoutfs_block_set_crc(bl); - ret = scoutfs_block_write_sync(bl); - scoutfs_block_put(bl); + ret = scoutfs_bio_write(sb, &page, le64_to_cpu(super->hdr.blkno), 1); + WARN_ON_ONCE(ret); + + __free_page(page); + return ret; } From 429e1b6eb435a10988132360559b19402cf8a696 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 09:15:27 -0800 Subject: [PATCH 231/920] Truncate data items scoutfs_data_truncate_items() was still using the btree. This updates it to use the item cache but doesn't yet support regions being offline. Signed-off-by: Zach Brown --- kmod/src/data.c | 107 +++++++++++++++--------------------------------- 1 file changed, 34 insertions(+), 73 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 88843d69..b37d2f10 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -25,7 +25,6 @@ #include "trans.h" #include "counters.h" #include "scoutfs_trace.h" -#include "btree.h" #include "item.h" #include "ioctl.h" @@ -113,86 +112,59 @@ struct data_info { #define BHA(bh) \ (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ +static void init_data_key(struct scoutfs_key_buf *key, + struct scoutfs_data_key *dkey, u64 ino, u64 block) +{ + dkey->type = SCOUTFS_DATA_KEY; + dkey->ino = cpu_to_be64(ino); + dkey->block = cpu_to_be64(block); + + scoutfs_key_init(key, dkey, sizeof(struct scoutfs_data_key)); +} + /* - * Free extents whose blocks fall inside the specified blocks. The - * caller holds a transaction. - * - * If 'release' is given then blocks are freed inside i_size but the - * extent items are left behind and their _OFFLINE flag is set. + * Delete the data block items in the given region. * * This is the low level extent item truncate code. Callers manage * higher order truncation and orphan cleanup. + * + * XXX + * - restore support for releasing data. + * - for final unlink this would be better as a range deletion + * - probably don't want to read items to find them for removal */ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline) { - struct scoutfs_btree_root *meta = SCOUTFS_META(sb); - struct scoutfs_extent extent; - struct scoutfs_btree_val val; - struct scoutfs_key key; - struct scoutfs_key first; - u64 seq; + struct scoutfs_data_key last_dkey; + struct scoutfs_data_key dkey; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; int ret; - /* XXX not yet updated */ + trace_printk("iblock %llu len %llu offline %u\n", + iblock, len, offline); - scoutfs_set_key(&first, ino, SCOUTFS_EXTENT_KEY, iblock); - scoutfs_set_key(&key, ino, SCOUTFS_EXTENT_KEY, iblock + len - 1); + if (WARN_ON_ONCE(iblock + len <= iblock) || + WARN_ON_ONCE(offline)) + return -EINVAL; - trace_printk("iblock %llu\n", iblock); - - scoutfs_btree_init_val(&val, &extent, sizeof(extent)); - val.check_size_eq = 1; + init_data_key(&key, &dkey, ino, iblock); + init_data_key(&last, &last_dkey, ino, iblock + len - 1); for (;;) { - ret = scoutfs_btree_prev(sb, meta, &first, &key, &key, &seq, - &val); + ret = scoutfs_item_next(sb, &key, &last, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - len = le64_to_cpu(extent.len); - if (WARN_ON_ONCE(len != 1)) { - ret = -EIO; + /* XXX would set offline bit items here */ + + ret = scoutfs_item_delete(sb, &key); + if (ret) break; - } - - /* XXX corruption: offline and allocation are exclusive */ - if (!!extent.blkno == - !!(extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) { - ret = -EIO; - break; - } - - if (offline && (extent.flags & SCOUTFS_EXTENT_FLAG_OFFLINE)) - continue; - - /* make sure we can delete the extent after freeing */ - if (extent.blkno) { - ret = scoutfs_btree_dirty(sb, meta, &key); - if (ret) - break; - - ret = scoutfs_buddy_free(sb, cpu_to_le64(seq), - le64_to_cpu(extent.blkno), 0); - if (ret) - break; - } - - if (offline) { - extent.blkno = 0; - extent.flags |= SCOUTFS_EXTENT_FLAG_OFFLINE; - scoutfs_btree_update(sb, meta, &key, &val); - } else { - ret = scoutfs_btree_delete(sb, meta, &key); - if (ret) - break; - } - - /* XXX sync transaction if it's enormous */ - scoutfs_dec_key(&key); } return ret; @@ -248,24 +220,13 @@ void scoutfs_data_end_writeback(struct super_block *sb, int err) } } -static void init_data_key(struct scoutfs_key_buf *key, - struct scoutfs_data_key *dkey, - struct inode *inode, u64 block) -{ - dkey->type = SCOUTFS_DATA_KEY; - dkey->ino = cpu_to_be64(scoutfs_ino(inode)); - dkey->block = cpu_to_be64(block); - - scoutfs_key_init(key, dkey, sizeof(struct scoutfs_data_key)); -} - -/* Iterate over all the data block items that make up the page. */ #define for_each_page_block(page, start, loff, block, key, dkey, val) \ for (start = 0; \ start < PAGE_CACHE_SIZE && \ (loff = ((loff_t)page->index << PAGE_CACHE_SHIFT) + start, \ block = loff >> SCOUTFS_BLOCK_SHIFT, \ - init_data_key(&key, &dkey, page->mapping->host, block), \ + init_data_key(&key, &dkey, \ + scoutfs_ino(page->mapping->host), block), \ scoutfs_kvec_init(val, page_address(page) + start, \ SCOUTFS_BLOCK_SIZE), \ 1); \ From 02af35a98e187020b204c671d53895300fae0855 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 09:26:55 -0800 Subject: [PATCH 232/920] Convert inode since ioctl to the item API The inode since ioctl was the last user of the btree. It doesn't yet work because the item cache doesn't know how to search for items by sequence yet. It's not yet clear exactly how we'll build the data since ioctls. It'll be easy enough to refactor the inode since item walk if they follow a similar pattern again. Signed-off-by: Zach Brown --- kmod/src/inode.c | 16 ++++++++-------- kmod/src/inode.h | 5 +++++ kmod/src/ioctl.c | 27 +++++++++++++++------------ 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index b773f0ee..ad1caa79 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -132,8 +132,8 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); } -static void init_inode_key(struct scoutfs_key_buf *key, - struct scoutfs_inode_key *ikey, u64 ino) +void scoutfs_inode_init_key(struct scoutfs_key_buf *key, + struct scoutfs_inode_key *ikey, u64 ino) { ikey->type = SCOUTFS_INODE_KEY; ikey->ino = cpu_to_be64(ino); @@ -150,7 +150,7 @@ static int scoutfs_read_locked_inode(struct inode *inode) SCOUTFS_DECLARE_KVEC(val); int ret; - init_inode_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); @@ -283,7 +283,7 @@ int scoutfs_dirty_inode_item(struct inode *inode) store_inode(&sinode, inode); - init_inode_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); ret = scoutfs_item_dirty(sb, &key); if (!ret) @@ -311,7 +311,7 @@ void scoutfs_update_inode_item(struct inode *inode) store_inode(&sinode, inode); - init_inode_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); err = scoutfs_item_update(sb, &key, val); @@ -426,7 +426,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, set_inode_ops(inode); store_inode(&sinode, inode); - init_inode_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); ret = scoutfs_item_create(sb, &key, val); @@ -512,7 +512,7 @@ static void delete_inode(struct super_block *sb, u64 ino) int ret; /* sample the inode mode, XXX don't need to copy whole thing here */ - init_inode_key(&key, &ikey, ino); + scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); @@ -566,7 +566,7 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) SCOUTFS_DECLARE_KVEC(val); int ret; - init_inode_key(&key, &ikey, ino); + scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 93f6d276..f3badfb4 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -1,6 +1,8 @@ #ifndef _SCOUTFS_INODE_H_ #define _SCOUTFS_INODE_H_ +#include "key.h" + struct scoutfs_inode_info { u64 ino; u32 salt; @@ -28,6 +30,9 @@ static inline u64 scoutfs_ino(struct inode *inode) return SCOUTFS_I(inode)->ino; } +void scoutfs_inode_init_key(struct scoutfs_key_buf *key, + struct scoutfs_inode_key *ikey, u64 ino); + struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); int scoutfs_drop_inode(struct inode *inode); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 843d82ec..255e167f 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -22,7 +22,6 @@ #include #include "format.h" -#include "btree.h" #include "key.h" #include "dir.h" #include "name.h" @@ -51,15 +50,16 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, u8 type) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_btree_root *meta = SCOUTFS_STABLE_META(sb); struct scoutfs_ioctl_inodes_since __user *uargs = (void __user *)arg; struct scoutfs_ioctl_inodes_since args; struct scoutfs_ioctl_ino_seq __user *uiseq; struct scoutfs_ioctl_ino_seq iseq; - struct scoutfs_key key; - struct scoutfs_key last; - u64 seq; + struct scoutfs_inode_key last_ikey; + struct scoutfs_inode_key ikey; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; long bytes; + u64 seq; int ret; if (copy_from_user(&args, uargs, sizeof(args))) @@ -69,20 +69,23 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, if (args.buf_len < sizeof(iseq) || args.buf_len > INT_MAX) return -EINVAL; - scoutfs_set_key(&key, args.first_ino, type, 0); - scoutfs_set_key(&last, args.last_ino, type, 0); + scoutfs_inode_init_key(&key, &ikey, args.first_ino); + scoutfs_inode_init_key(&last, &last_ikey, args.last_ino); bytes = 0; for (;;) { - ret = scoutfs_btree_since(sb, meta, &key, &last, args.seq, - &key, &seq, NULL); + + /* XXX item cache needs to search by seq */ + seq = !!sb; + ret = WARN_ON_ONCE(-EINVAL); +// ret = scoutfs_item_since(sb, &key, &last, args.seq, &seq, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - iseq.ino = scoutfs_key_inode(&key); + iseq.ino = be64_to_cpu(ikey.ino); iseq.seq = seq; if (copy_to_user(uiseq, &iseq, sizeof(iseq))) { @@ -97,7 +100,7 @@ static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, break; } - key.inode = cpu_to_le64(iseq.ino + 1); + last_ikey.ino = cpu_to_be64(iseq.ino + 1); } if (bytes) @@ -418,7 +421,7 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case SCOUTFS_IOC_INO_PATH: return scoutfs_ioc_ino_path(file, arg); case SCOUTFS_IOC_INODE_DATA_SINCE: - return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_EXTENT_KEY); + return WARN_ON_ONCE(-EINVAL); case SCOUTFS_IOC_DATA_VERSION: return scoutfs_ioc_data_version(file, arg); case SCOUTFS_IOC_RELEASE: From 00fed84c6808b11636d9ed618089a0b0f3b0223d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 09:55:11 -0800 Subject: [PATCH 233/920] Build statfs f_blocks from total_segs Use the current total_segs field to calculate the total number of blocks in the system instead of the old and redundant total_segs field which is going away. Signed-off-by: Zach Brown --- kmod/src/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 00b428a7..57dcbd46 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -60,7 +60,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_bfree = scoutfs_alloc_bfree(sb); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; - kst->f_blocks = le64_to_cpu(super->total_blocks); + kst->f_blocks = le64_to_cpu(super->total_segs) * SCOUTFS_SEGMENT_BLOCKS; kst->f_bavail = kst->f_bfree; kst->f_ffree = kst->f_bfree * 17; From 6bcdca3cf9fa6a640060c3e7602be8cf183fa7f4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 09:56:45 -0800 Subject: [PATCH 234/920] Update dirent last pos and update first comment The last valid pos for us is now a full u64 because we're storing entries at an increasing counter instead of at a hahs of the entry name. And might as well add a clarifying comment to the first pos while we're here. Signed-off-by: Zach Brown --- kmod/src/format.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 77945486..a3784bcb 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -483,9 +483,10 @@ struct scoutfs_dirent { #define SCOUTFS_DIRENT_OFF_BITS 31 #define SCOUTFS_DIRENT_OFF_MASK ((1U << SCOUTFS_DIRENT_OFF_BITS) - 1) -/* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +/* entries begin after . and .. */ #define SCOUTFS_DIRENT_FIRST_POS 2 -#define SCOUTFS_DIRENT_LAST_POS (INT_MAX - 1) +/* getdents returns next pos with an entry, no entry at (f_pos)~0 */ +#define SCOUTFS_DIRENT_LAST_POS (U64_MAX - 1) enum { SCOUTFS_DT_FIFO = 0, From 97cb75bd88ba0de3f0aacfb8d82ab3b0441db758 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Feb 2017 09:58:37 -0800 Subject: [PATCH 235/920] Remove dead btree, block, and buddy code Remove all the unused dead code from the previous btree block design. Signed-off-by: Zach Brown --- kmod/src/Makefile | 6 +- kmod/src/block.c | 786 ------------------- kmod/src/block.h | 38 - kmod/src/btree.c | 1582 -------------------------------------- kmod/src/btree.h | 77 -- kmod/src/buddy.c | 1063 ------------------------- kmod/src/buddy.h | 20 - kmod/src/counters.h | 2 - kmod/src/crc.c | 23 - kmod/src/crc.h | 6 - kmod/src/dir.c | 2 - kmod/src/format.h | 163 ---- kmod/src/inode.c | 7 - kmod/src/inode.h | 2 - kmod/src/ioctl.c | 1 - kmod/src/key.h | 123 --- kmod/src/kvec.c | 2 - kmod/src/name.c | 35 - kmod/src/name.h | 8 - kmod/src/scoutfs_trace.c | 1 - kmod/src/scoutfs_trace.h | 165 ---- kmod/src/super.c | 21 - kmod/src/super.h | 28 - kmod/src/trans.c | 2 - kmod/src/xattr.c | 1 - 25 files changed, 3 insertions(+), 4161 deletions(-) delete mode 100644 kmod/src/block.c delete mode 100644 kmod/src/block.h delete mode 100644 kmod/src/btree.c delete mode 100644 kmod/src/btree.h delete mode 100644 kmod/src/buddy.c delete mode 100644 kmod/src/buddy.h delete mode 100644 kmod/src/crc.c delete mode 100644 kmod/src/crc.h delete mode 100644 kmod/src/name.c delete mode 100644 kmod/src/name.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 3e7c9b35..e31924ed 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,6 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += alloc.o bio.o block.o btree.o buddy.o compact.o counters.o crc.o \ - data.o dir.o kvec.o inode.o ioctl.o item.o key.o manifest.o \ - msg.o name.o seg.o scoutfs_trace.o super.o trans.o treap.o xattr.o +scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ + ioctl.o item.o key.o manifest.o msg.o seg.o scoutfs_trace.o \ + super.o trans.o treap.o xattr.o diff --git a/kmod/src/block.c b/kmod/src/block.c deleted file mode 100644 index 3ecf6a7b..00000000 --- a/kmod/src/block.c +++ /dev/null @@ -1,786 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "block.h" -#include "crc.h" -#include "counters.h" -#include "buddy.h" - -/* - * scoutfs maintains a cache of metadata blocks in a radix tree. This - * gives us blocks bigger than page size and avoids fixing the location - * of a logical cached block in one possible position in a larger block - * device page cache page. - * - * This does the work to cow dirty blocks, track dirty blocks, generate - * checksums as they're written, only write them in transactions, verify - * checksums on read, and invalidate and retry reads of stale cached - * blocks. (That last bit only has a hint of an implementation.) - * - * XXX - * - tear down dirty blocks left by write errors on unmount - * - multiple smaller page allocs - * - vmalloc? vm_map_ram? - * - blocks allocated from per-cpu pages when page size > block size - * - cmwq crc calcs if that makes sense - * - slab of block structs - * - don't verify checksums in end_io context? - * - fall back to multiple single bios per block io if bio alloc fails? - * - fail mount if total_blocks is greater than long radix blkno - */ - -struct scoutfs_block { - struct rw_semaphore rwsem; - atomic_t refcount; - struct list_head lru_entry; - u64 blkno; - - unsigned long bits; - - struct super_block *sb; - struct page *page; - void *data; -}; - -#define DIRTY_RADIX_TAG 0 - -enum { - BLOCK_BIT_UPTODATE = 0, - BLOCK_BIT_ERROR, - BLOCK_BIT_CLASS_SET, -}; - -static struct scoutfs_block *alloc_block(struct super_block *sb, u64 blkno) -{ - struct scoutfs_block *bl; - struct page *page; - - /* we'd need to be just a bit more careful */ - BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); - - bl = kzalloc(sizeof(struct scoutfs_block), GFP_NOFS); - if (bl) { - /* change _from_contents if allocs not aligned */ - page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); - WARN_ON_ONCE(!page); - if (page) { - init_rwsem(&bl->rwsem); - atomic_set(&bl->refcount, 1); - INIT_LIST_HEAD(&bl->lru_entry); - bl->blkno = blkno; - bl->sb = sb; - bl->page = page; - bl->data = page_address(page); - trace_printk("allocated bl %p\n", bl); - } else { - kfree(bl); - bl = NULL; - } - } - - return bl; -} - -void scoutfs_block_put(struct scoutfs_block *bl) -{ - if (!IS_ERR_OR_NULL(bl) && atomic_dec_and_test(&bl->refcount)) { - trace_printk("freeing bl %p\n", bl); - WARN_ON_ONCE(!list_empty(&bl->lru_entry)); - __free_pages(bl->page, SCOUTFS_BLOCK_PAGE_ORDER); - kfree(bl); - scoutfs_inc_counter(bl->sb, block_mem_free); - } -} - -static void lru_add(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) -{ - if (list_empty(&bl->lru_entry)) { - list_add_tail(&bl->lru_entry, &sbi->block_lru_list); - sbi->block_lru_nr++; - } -} - -static void lru_del(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) -{ - if (!list_empty(&bl->lru_entry)) { - list_del_init(&bl->lru_entry); - sbi->block_lru_nr--; - } -} - -/* - * The caller is referencing a block but doesn't know if its in the LRU - * or not. If it is move it to the tail so it's last to be dropped by - * the shrinker. - */ -static void lru_move(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) -{ - if (!list_empty(&bl->lru_entry)) - list_move_tail(&bl->lru_entry, &sbi->block_lru_list); -} - -static void radix_insert(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl, - bool dirty) -{ - radix_tree_insert(&sbi->block_radix, bl->blkno, bl); - if (dirty) - radix_tree_tag_set(&sbi->block_radix, bl->blkno, - DIRTY_RADIX_TAG); - else - lru_add(sbi, bl); - atomic_inc(&bl->refcount); -} - -/* deleting the blkno from the radix also clears the dirty tag if it was set */ -static void radix_delete(struct scoutfs_sb_info *sbi, struct scoutfs_block *bl) -{ - lru_del(sbi, bl); - radix_tree_delete(&sbi->block_radix, bl->blkno); - scoutfs_block_put(bl); -} - -static int verify_block_header(struct super_block *sb, struct scoutfs_block *bl) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_block_header *hdr = bl->data; - u32 crc = scoutfs_crc_block(hdr); - int ret = -EIO; - - if (le32_to_cpu(hdr->crc) != crc) { - printk("blkno %llu hdr crc %x != calculated %x\n", bl->blkno, - le32_to_cpu(hdr->crc), crc); - } else if (super->hdr.fsid && hdr->fsid != super->hdr.fsid) { - printk("blkno %llu fsid %llx != super fsid %llx\n", bl->blkno, - le64_to_cpu(hdr->fsid), le64_to_cpu(super->hdr.fsid)); - } else if (le64_to_cpu(hdr->blkno) != bl->blkno) { - printk("blkno %llu invalid hdr blkno %llx\n", bl->blkno, - le64_to_cpu(hdr->blkno)); - } else { - ret = 0; - } - - return ret; -} - -static void block_read_end_io(struct bio *bio, int err) -{ - struct scoutfs_block *bl = bio->bi_private; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - - if (!err && !verify_block_header(bl->sb, bl)) - set_bit(BLOCK_BIT_UPTODATE, &bl->bits); - else - set_bit(BLOCK_BIT_ERROR, &bl->bits); - - /* - * uncontended spin_lock in wake_up and unconditional smp_mb to - * make waitqueue_active safe are about the same cost, so we - * prefer the obviously safe choice. - */ - wake_up(&sbi->block_wq); - - scoutfs_block_put(bl); - bio_put(bio); -} - -/* - * Once a transaction block is persistent it's fine to drop the dirty - * tag. It's been checksummed so it can be read in again. It's seq - * will be in the current transaction so it'll simply be dirtied and - * checksummed and written out again. - */ -static void block_write_end_io(struct bio *bio, int err) -{ - struct scoutfs_block *bl = bio->bi_private; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - unsigned long flags; - - if (!err) { - spin_lock_irqsave(&sbi->block_lock, flags); - radix_tree_tag_clear(&sbi->block_radix, - bl->blkno, DIRTY_RADIX_TAG); - lru_add(sbi, bl); - spin_unlock_irqrestore(&sbi->block_lock, flags); - } - - /* not too worried about racing ints */ - if (err && !sbi->block_write_err) - sbi->block_write_err = err; - - if (atomic_dec_and_test(&sbi->block_writes)) - wake_up(&sbi->block_wq); - - scoutfs_block_put(bl); - bio_put(bio); - -} - -static int block_submit_bio(struct scoutfs_block *bl, int rw) -{ - struct super_block *sb = bl->sb; - struct bio *bio; - int ret; - - bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); - if (WARN_ON_ONCE(!bio)) - return -ENOMEM; - - bio->bi_sector = bl->blkno << (SCOUTFS_BLOCK_SHIFT - 9); - bio->bi_bdev = sb->s_bdev; - if (rw & WRITE) { - bio->bi_end_io = block_write_end_io; - } else - bio->bi_end_io = block_read_end_io; - bio->bi_private = bl; - - ret = bio_add_page(bio, bl->page, SCOUTFS_BLOCK_SIZE, 0); - if (WARN_ON_ONCE(ret != SCOUTFS_BLOCK_SIZE)) { - bio_put(bio); - return -ENOMEM; - } - - atomic_inc(&bl->refcount); - submit_bio(rw, bio); - - return 0; -} - -/* - * Read an existing block from the device and verify its metadata header. - */ -struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *found; - struct scoutfs_block *bl; - unsigned long flags; - int ret; - - /* find an existing block, dropping if it's errored */ - spin_lock_irqsave(&sbi->block_lock, flags); - - bl = radix_tree_lookup(&sbi->block_radix, blkno); - if (bl) { - if (test_bit(BLOCK_BIT_ERROR, &bl->bits)) { - radix_delete(sbi, bl); - bl = NULL; - } else { - lru_move(sbi, bl); - atomic_inc(&bl->refcount); - } - } - spin_unlock_irqrestore(&sbi->block_lock, flags); - if (bl) - goto wait; - - /* allocate a new block and try to insert it */ - bl = alloc_block(sb, blkno); - if (!bl) { - ret = -EIO; - goto out; - } - - ret = radix_tree_preload(GFP_NOFS); - if (ret) - goto out; - - spin_lock_irqsave(&sbi->block_lock, flags); - - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) { - scoutfs_block_put(bl); - bl = found; - lru_move(sbi, bl); - atomic_inc(&bl->refcount); - } else { - radix_insert(sbi, bl, false); - } - - spin_unlock_irqrestore(&sbi->block_lock, flags); - radix_tree_preload_end(); - - if (!found) { - ret = block_submit_bio(bl, READ_SYNC | REQ_META); - if (ret) - goto out; - } - -wait: - ret = wait_event_interruptible(sbi->block_wq, - test_bit(BLOCK_BIT_UPTODATE, &bl->bits) || - test_bit(BLOCK_BIT_ERROR, &bl->bits)); - if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bl->bits)) - ret = -EIO; -out: - if (ret) { - scoutfs_block_put(bl); - bl = ERR_PTR(ret); - } - - return bl; -} - -/* - * Read an existing block from the device described by the caller's - * reference. - * - * If the reference sequence numbers don't match then we could be racing - * with another writer. We back off and try again. If it happens too - * many times the caller assumes that we've hit persistent corruption - * and returns an error. - * - * XXX: - * - actually implement this - * - reads that span transactions? - * - writers creating a new dirty block? - */ -struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) -{ - struct scoutfs_block_header *hdr; - struct scoutfs_block *bl; - - bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); - if (!IS_ERR(bl)) { - hdr = scoutfs_block_data(bl); - if (WARN_ON_ONCE(hdr->seq != ref->seq)) { - scoutfs_block_put(bl); - bl = ERR_PTR(-EAGAIN); - } - } - - return bl; -} - -/* - * The caller knows that it's not racing with writers. - */ -int scoutfs_block_has_dirty(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - return radix_tree_tagged(&sbi->block_radix, DIRTY_RADIX_TAG); -} - -/* - * Submit writes for all the blocks in the radix with their dirty tag - * set. The transaction machinery ensures that the dirty blocks form a - * consistent image and excludes future dirtying while IO is in flight. - * - * Presence in the dirty tree holds a reference. Blocks are only - * removed from the tree which drops the ref when IO completes. - * - * Blocks that see write errors remain in the dirty tree and will try to - * be written again in the next transaction commit. - * - * Reads can traverse the blocks while they're in flight. - */ -int scoutfs_block_write_dirty(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *blocks[16]; - struct scoutfs_block *bl; - struct blk_plug plug; - unsigned long flags; - u64 blkno; - int ret; - int nr; - int i; - - atomic_set(&sbi->block_writes, 1); - sbi->block_write_err = 0; - blkno = 0; - ret = 0; - - blk_start_plug(&plug); - - do { - /* get refs to a bunch of dirty blocks */ - spin_lock_irqsave(&sbi->block_lock, flags); - nr = radix_tree_gang_lookup_tag(&sbi->block_radix, - (void **)blocks, blkno, - ARRAY_SIZE(blocks), - DIRTY_RADIX_TAG); - if (nr > 0) - blkno = blocks[nr - 1]->blkno + 1; - for (i = 0; i < nr; i++) - atomic_inc(&blocks[i]->refcount); - spin_unlock_irqrestore(&sbi->block_lock, flags); - - /* submit them in order, being careful to put all on err */ - for (i = 0; i < nr; i++) { - bl = blocks[i]; - - if (ret == 0) { - scoutfs_block_set_crc(bl); - atomic_inc(&sbi->block_writes); - ret = block_submit_bio(bl, WRITE); - if (ret) - atomic_dec(&sbi->block_writes); - } - scoutfs_block_put(bl); - } - } while (nr && !ret); - - blk_finish_plug(&plug); - - /* wait for all io to drain */ - atomic_dec(&sbi->block_writes); - wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); - - return ret ?: sbi->block_write_err; -} - -/* - * XXX This is a gross hack for writing the super. It doesn't have - * per-block write completion indication. It knows that it's the only - * thing that will be writing. - */ -int scoutfs_block_write_sync(struct scoutfs_block *bl) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - int ret; - - BUG_ON(atomic_read(&sbi->block_writes) != 0); - - atomic_inc(&sbi->block_writes); - ret = block_submit_bio(bl, WRITE); - if (ret) - atomic_dec(&sbi->block_writes); - else - wait_event(sbi->block_wq, atomic_read(&sbi->block_writes) == 0); - - return ret ?: sbi->block_write_err; -} - -/* - * Give the caller a dirty block that they can safely modify. If the - * reference refers to a stable clean block then we allocate a new block - * and update the reference. - * - * Blocks are dirtied and modified within a transaction that has a given - * sequence number which we use to determine if the block is currently - * dirty or not. - * - * For now we're using the dirty super block in the sb_info to track the - * dirty seq. That'll be different when we have multiple btrees. - * - * Callers are responsible for serializing modification to the reference - * which is probably embedded in some other dirty persistent structure. - */ -struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block_header *hdr; - struct scoutfs_block *copy_bl = NULL; - struct scoutfs_block *bl; - u64 blkno = 0; - int ret; - int err; - - bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); - if (IS_ERR(bl) || ref->seq == sbi->super.hdr.seq) - return bl; - - ret = scoutfs_buddy_alloc_same(sb, &blkno, le64_to_cpu(ref->blkno)); - if (ret < 0) - goto out; - - copy_bl = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(copy_bl)) { - ret = PTR_ERR(copy_bl); - goto out; - } - - hdr = scoutfs_block_data(bl); - ret = scoutfs_buddy_free(sb, hdr->seq, le64_to_cpu(hdr->blkno), 0); - if (ret) - goto out; - - memcpy(scoutfs_block_data(copy_bl), scoutfs_block_data(bl), - SCOUTFS_BLOCK_SIZE); - - hdr = scoutfs_block_data(copy_bl); - hdr->blkno = cpu_to_le64(blkno); - hdr->seq = sbi->super.hdr.seq; - ref->blkno = hdr->blkno; - ref->seq = hdr->seq; - - ret = 0; -out: - scoutfs_block_put(bl); - if (ret) { - if (!IS_ERR_OR_NULL(copy_bl)) { - err = scoutfs_buddy_free(sb, sbi->super.hdr.seq, - blkno, 0); - WARN_ON_ONCE(err); /* freeing dirty must work */ - } - scoutfs_block_put(copy_bl); - copy_bl = ERR_PTR(ret); - } - - return copy_bl; -} - -/* - * Return a dirty metadata block with an updated block header to match - * the current dirty seq. Callers are responsible for serializing - * access to the block and for zeroing unwritten block contents. - * - * Always allocating a new block and replacing any old cached block - * serves a very specific purpose. We can have an unlocked reader - * traversing stable structures actively using a clean block while a - * writer gets that same blkno from the allocator and starts modifying - * it. By always allocating a new block we let the reader continue - * safely using their old immutable block while the writer works on the - * newly allocated block. The old stable block will be freed once the - * reader drops their reference. - */ -struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block_header *hdr; - struct scoutfs_block *found; - struct scoutfs_block *bl; - unsigned long flags; - int ret; - - /* allocate a new block and try to insert it */ - bl = alloc_block(sb, blkno); - if (!bl) { - ret = -EIO; - goto out; - } - - set_bit(BLOCK_BIT_UPTODATE, &bl->bits); - - ret = radix_tree_preload(GFP_NOFS); - if (ret) - goto out; - - hdr = bl->data; - *hdr = sbi->super.hdr; - hdr->blkno = cpu_to_le64(blkno); - hdr->seq = sbi->super.hdr.seq; - - spin_lock_irqsave(&sbi->block_lock, flags); - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found) - radix_delete(sbi, found); - radix_insert(sbi, bl, true); - spin_unlock_irqrestore(&sbi->block_lock, flags); - - radix_tree_preload_end(); - ret = 0; -out: - if (ret) { - scoutfs_block_put(bl); - bl = ERR_PTR(ret); - } - - return bl; -} - -/* - * Allocate a new dirty writable block. The caller must be in a - * transaction so that we can assign the dirty seq. - */ -struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->stable_super; - struct scoutfs_block *bl; - u64 blkno; - int ret; - int err; - - ret = scoutfs_buddy_alloc(sb, &blkno, 0); - if (ret < 0) - return ERR_PTR(ret); - - bl = scoutfs_block_dirty(sb, blkno); - if (IS_ERR(bl)) { - err = scoutfs_buddy_free(sb, super->hdr.seq, blkno, 0); - WARN_ON_ONCE(err); /* freeing dirty must work */ - } - return bl; -} - -/* - * Forget the given block by removing it from the radix and clearing its - * dirty tag. It will not be found by future lookups and will not be - * written out. The caller can still use it until it drops its - * reference. - */ -void scoutfs_block_forget(struct scoutfs_block *bl) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(bl->sb); - struct scoutfs_block *found; - unsigned long flags; - u64 blkno = bl->blkno; - - spin_lock_irqsave(&sbi->block_lock, flags); - found = radix_tree_lookup(&sbi->block_radix, blkno); - if (found == bl) - radix_delete(sbi, bl); - spin_unlock_irqrestore(&sbi->block_lock, flags); -} - -/* - * We maintain an LRU of blocks so that the shrinker can free the oldest - * under memory pressure. We can't reclaim dirty blocks so only clean - * blocks are kept in the LRU. Blocks are only in the LRU while their - * presence in the radix holds a reference. We don't care if a reader - * has an active ref on a clean block that gets reclaimed. All we're - * doing is removing from the radix. The caller can still work with the - * block and it will be freed once they drop their ref. - * - * If this is called with nr_to_scan == 0 then it only returns the nr. - * We avoid acquiring the lock in that case. - * - * Lookup code only moves blocks around in the LRU while they're in the - * radix. Once we remove the block from the radix we're able to use the - * lru_entry to drop all the blocks outside the lock. - * - * XXX: - * - are sc->nr_to_scan and our return meant to be in units of pages? - * - should we sync a transaction here? - */ -int scoutfs_block_shrink(struct shrinker *shrink, struct shrink_control *sc) -{ - struct scoutfs_sb_info *sbi = container_of(shrink, - struct scoutfs_sb_info, - block_shrinker); - struct scoutfs_block *tmp; - struct scoutfs_block *bl; - unsigned long flags; - unsigned long nr; - LIST_HEAD(list); - - nr = sc->nr_to_scan; - if (!nr) - goto out; - - spin_lock_irqsave(&sbi->block_lock, flags); - - list_for_each_entry_safe(bl, tmp, &sbi->block_lru_list, lru_entry) { - if (nr-- == 0) - break; - atomic_inc(&bl->refcount); - radix_delete(sbi, bl); - list_add(&bl->lru_entry, &list); - } - - spin_unlock_irqrestore(&sbi->block_lock, flags); - - list_for_each_entry_safe(bl, tmp, &list, lru_entry) { - list_del_init(&bl->lru_entry); - scoutfs_block_put(bl); - } - -out: - return min_t(unsigned long, sbi->block_lru_nr, INT_MAX); -} - -void scoutfs_block_set_crc(struct scoutfs_block *bl) -{ - struct scoutfs_block_header *hdr = scoutfs_block_data(bl); - - hdr->crc = cpu_to_le32(scoutfs_crc_block(hdr)); -} - -/* - * Zero the block from the given byte to the end of the block. - */ -void scoutfs_block_zero(struct scoutfs_block *bl, size_t off) -{ - if (WARN_ON_ONCE(off > SCOUTFS_BLOCK_SIZE)) - return; - - if (off < SCOUTFS_BLOCK_SIZE) - memset(scoutfs_block_data(bl) + off, 0, - SCOUTFS_BLOCK_SIZE - off); -} - -/* - * Zero the block from the given byte to the end of the block. - */ -void scoutfs_block_zero_from(struct scoutfs_block *bl, void *ptr) -{ - return scoutfs_block_zero(bl, (char *)ptr - - (char *)scoutfs_block_data(bl)); -} - -void scoutfs_block_set_lock_class(struct scoutfs_block *bl, - struct lock_class_key *class) -{ - if (!test_bit(BLOCK_BIT_CLASS_SET, &bl->bits)) { - lockdep_set_class(&bl->rwsem, class); - set_bit(BLOCK_BIT_CLASS_SET, &bl->bits); - } -} - -void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass) -{ - if (write) - down_write_nested(&bl->rwsem, subclass); - else - down_read_nested(&bl->rwsem, subclass); -} - -void scoutfs_block_unlock(struct scoutfs_block *bl, bool write) -{ - if (write) - up_write(&bl->rwsem); - else - up_read(&bl->rwsem); -} - -void *scoutfs_block_data(struct scoutfs_block *bl) -{ - return bl->data; -} - -void *scoutfs_block_data_from_contents(const void *ptr) -{ - unsigned long addr = (unsigned long)ptr; - - return (void *)(addr & ~((unsigned long)SCOUTFS_BLOCK_MASK)); -} - -void scoutfs_block_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_block *blocks[16]; - struct scoutfs_block *bl; - unsigned long blkno = 0; - int nr; - int i; - - do { - nr = radix_tree_gang_lookup(&sbi->block_radix, (void **)blocks, - blkno, ARRAY_SIZE(blocks)); - for (i = 0; i < nr; i++) { - bl = blocks[i]; - blkno = bl->blkno + 1; - radix_delete(sbi, bl); - } - } while (nr); -} diff --git a/kmod/src/block.h b/kmod/src/block.h deleted file mode 100644 index 0eb86837..00000000 --- a/kmod/src/block.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef _SCOUTFS_BLOCK_H_ -#define _SCOUTFS_BLOCK_H_ - -struct scoutfs_block; - -#include - -struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); -struct scoutfs_block *scoutfs_block_read_ref(struct super_block *sb, - struct scoutfs_block_ref *ref); - -struct scoutfs_block *scoutfs_block_dirty(struct super_block *sb, u64 blkno); -struct scoutfs_block *scoutfs_block_dirty_alloc(struct super_block *sb); -struct scoutfs_block *scoutfs_block_dirty_ref(struct super_block *sb, - struct scoutfs_block_ref *ref); - -int scoutfs_block_has_dirty(struct super_block *sb); -int scoutfs_block_write_dirty(struct super_block *sb); -int scoutfs_block_write_sync(struct scoutfs_block *bl); - -void scoutfs_block_set_crc(struct scoutfs_block *bl); -void scoutfs_block_zero(struct scoutfs_block *bl, size_t off); -void scoutfs_block_zero_from(struct scoutfs_block *bl, void *ptr); - -void scoutfs_block_set_lock_class(struct scoutfs_block *bl, - struct lock_class_key *class); -void scoutfs_block_lock(struct scoutfs_block *bl, bool write, int subclass); -void scoutfs_block_unlock(struct scoutfs_block *bl, bool write); - -void *scoutfs_block_data(struct scoutfs_block *bl); -void *scoutfs_block_data_from_contents(const void *ptr); -void scoutfs_block_forget(struct scoutfs_block *bl); -void scoutfs_block_put(struct scoutfs_block *bl); - -int scoutfs_block_shrink(struct shrinker *shrink, struct shrink_control *sc); -void scoutfs_block_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c deleted file mode 100644 index a1410134..00000000 --- a/kmod/src/btree.c +++ /dev/null @@ -1,1582 +0,0 @@ -/* - * Copyright (C) 2016 Zach Brown. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "block.h" -#include "key.h" -#include "btree.h" - -#include "scoutfs_trace.h" - -/* - * scoutfs stores file system metadata in btrees whose items have fixed - * sized keys and variable length values. - * - * Items are stored as a small header with the key followed by the - * value. New items are allocated from the back of the block towards - * the front. Deleted items can be reclaimed by packing items towards - * the back of the block by walking them in reverse offset order. - * - * A dense array of item offsets after the btree block header header - * maintains the sorted order of the items by their keys. The array is - * small enough that the memmoves to keep it dense involves a few cache - * lines at most. - * - * Parent blocks in the btree have the same format as leaf blocks. - * There's one key for every child reference instead of having separator - * keys between child references. The key in a child reference contains - * the largest key that may be found in the child subtree. The right - * spine of the tree has maximal keys so that they don't have to be - * updated if we insert an item with a key greater than everything in - * the tree. - * - * btree blocks, block references, and items all have sequence numbers - * that are set to the current dirty btree sequence number when they're - * modified. This lets us efficiently search a range of keys for items - * that are newer than a given sequence number. - * - * Operations are performed in one pass down the tree. This lets us - * cascade locks from the root down to the leaves and avoids having to - * maintain a record of the path down the tree. Splits and merges are - * performed as we descend. - * - * XXX - * - do we want a level in the btree header? seems like we would? - * - validate structures on read? - * - internal bl/pos/cmp interface is clumsy.. - */ - -/* number of contiguous bytes used by the item header and val of given len */ -static inline unsigned int val_bytes(unsigned int val_len) -{ - return sizeof(struct scoutfs_btree_item) + val_len; -} - -/* number of contiguous bytes used by the item header its current value */ -static inline unsigned int item_bytes(struct scoutfs_btree_item *item) -{ - return val_bytes(le16_to_cpu(item->val_len)); -} - -/* total bytes consumed by an item with given val len: offset, header, value */ -static inline unsigned int all_val_bytes(unsigned int val_len) -{ - return sizeof(((struct scoutfs_btree_block *)NULL)->item_offs[0]) + - val_bytes(val_len); -} - -/* total bytes consumed by an item with its current value */ -static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) -{ - return all_val_bytes(le16_to_cpu(item->val_len)); -} - -/* number of contig free bytes between item offset and first item */ -static inline unsigned int contig_free(struct scoutfs_btree_block *bt) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - - return le16_to_cpu(bt->free_end) - - offsetof(struct scoutfs_btree_block, item_offs[nr]); -} - -/* number of contig bytes free after reclaiming free amongst items */ -static inline unsigned int reclaimable_free(struct scoutfs_btree_block *bt) -{ - return contig_free(bt) + le16_to_cpu(bt->free_reclaim); -} - -/* all bytes used by item offsets, headers, and values */ -static inline unsigned int used_total(struct scoutfs_btree_block *bt) -{ - return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - - reclaimable_free(bt); -} - -static inline struct scoutfs_btree_item * -off_item(struct scoutfs_btree_block *bt, __le16 off) -{ - return (void *)bt + le16_to_cpu(off); -} - -static inline struct scoutfs_btree_item * -pos_item(struct scoutfs_btree_block *bt, unsigned int pos) -{ - return off_item(bt, bt->item_offs[pos]); -} - -static inline struct scoutfs_key *greatest_key(struct scoutfs_btree_block *bt) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - - return &pos_item(bt, nr - 1)->key; -} - -/* - * Copy as much of the item as fits in the value vector. The min of the - * value vec length and the item length is returned, including possibly - * 0. - */ -static int copy_to_val(struct scoutfs_btree_val *val, - struct scoutfs_btree_item *item) -{ - size_t val_len = le16_to_cpu(item->val_len); - char *val_ptr = item->val; - struct kvec *kv; - size_t bytes; - size_t off; - int i; - - /* - * Corruption check, right now we just return -EIO if the - * caller wants this. In the future we can grow this to do - * different things (go readonly, ignore, return error) based - * on the severity of the problem. - */ - /* XXX corruption */ - if (val->check_size_eq && val_len != scoutfs_btree_val_length(val)) - return -EIO; - if (val->check_size_lte && val_len > scoutfs_btree_val_length(val)) - return -EOVERFLOW; - - for (i = 0, off = 0; val_len > 0 && i < ARRAY_SIZE(val->vec); i++) { - kv = &val->vec[i]; - - if (WARN_ON_ONCE(kv->iov_len && !kv->iov_base)) - return -EINVAL; - - bytes = min(val_len, kv->iov_len); - if (bytes) - memcpy(kv->iov_base, val_ptr + off, bytes); - - val_len -= bytes; - off += bytes; - } - - return off; -} - -/* - * Copy the caller's value vector into the item in the tree block. This - * is only called when the item should exactly match the value vector. - * - * -EINVAL is returned if the lengths don't match. - */ -static int copy_to_item(struct scoutfs_btree_item *item, - struct scoutfs_btree_val *val) -{ - size_t val_len = le16_to_cpu(item->val_len); - char *val_ptr = item->val; - struct kvec *kv; - size_t bytes; - int i; - - if (val_len != scoutfs_btree_val_length(val)) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(val->vec); i++) { - kv = &val->vec[i]; - - if (WARN_ON_ONCE(kv->iov_len && !kv->iov_base)) - return -EINVAL; - - bytes = min(val_len, kv->iov_len); - if (bytes) - memcpy(val_ptr, kv->iov_base, bytes); - - val_len -= bytes; - val_ptr += bytes; - } - - return 0; -} - -/* - * Returns the sorted item position that an item with the given key - * should occupy. - * - * It sets *cmp to the final comparison of the given key and the - * position's item key. - * - * If the given key is greater then all items' keys then the number of - * items can be returned. Callers need to be careful to test for this - * invalid index. - */ -static int find_pos(struct scoutfs_btree_block *bt, struct scoutfs_key *key, - int *cmp) -{ - unsigned int start = 0; - unsigned int end = le16_to_cpu(bt->nr_items); - unsigned int pos = 0; - - *cmp = -1; - - while (start < end) { - pos = start + (end - start) / 2; - - *cmp = scoutfs_key_cmp(key, &pos_item(bt, pos)->key); - if (*cmp < 0) { - end = pos; - } else if (*cmp > 0) { - start = ++pos; - *cmp = -1; - } else { - break; - } - } - - return pos; -} - -/* move a number of contigous elements from the src index to the dst index */ -#define memmove_arr(arr, dst, src, nr) \ - memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) - -/* - * Allocate and insert a new item into the block. The caller has made - * sure that there's room for everything. The caller is responsible for - * initializing the value. - */ -static struct scoutfs_btree_item *create_item(struct scoutfs_btree_block *bt, - unsigned int pos, - struct scoutfs_key *key, - unsigned int val_len) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - struct scoutfs_btree_item *item; - - if (pos < nr) - memmove_arr(bt->item_offs, pos + 1, pos, nr - pos); - - le16_add_cpu(&bt->free_end, -val_bytes(val_len)); - bt->item_offs[pos] = bt->free_end; - nr++; - bt->nr_items = cpu_to_le16(nr); - - BUG_ON(le16_to_cpu(bt->free_end) < - offsetof(struct scoutfs_btree_block, item_offs[nr])); - - item = pos_item(bt, pos); - item->key = *key; - item->seq = bt->hdr.seq; - item->val_len = cpu_to_le16(val_len); - - trace_printk("pos %u off %u\n", pos, le16_to_cpu(bt->item_offs[pos])); - - return item; -} - -/* - * Delete an item from a btree block. We record the amount of space it - * frees to later decide if we can satisfy an insertion by compaction - * instead of splitting. - */ -static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) -{ - struct scoutfs_btree_item *item = pos_item(bt, pos); - unsigned int nr = le16_to_cpu(bt->nr_items); - - trace_printk("pos %u off %u\n", pos, le16_to_cpu(bt->item_offs[pos])); - - if (pos < (nr - 1)) - memmove_arr(bt->item_offs, pos, pos + 1, nr - 1 - pos); - - le16_add_cpu(&bt->free_reclaim, item_bytes(item)); - nr--; - bt->nr_items = cpu_to_le16(nr); - - /* wipe deleted items to avoid leaking data */ - memset(item, 0, item_bytes(item)); -} - -/* - * Move items from a source block to a destination block. The caller - * tells us if we're moving from the tail of the source block right to - * the head of the destination block, or vice versa. We stop moving - * once we've moved enough bytes of items. - */ -static void move_items(struct scoutfs_btree_block *dst, - struct scoutfs_btree_block *src, bool move_right, - int to_move) -{ - struct scoutfs_btree_item *from; - struct scoutfs_btree_item *to; - unsigned int t; - unsigned int f; - - if (move_right) { - f = le16_to_cpu(src->nr_items) - 1; - t = 0; - } else { - f = 0; - t = le16_to_cpu(dst->nr_items); - } - - while (f < le16_to_cpu(src->nr_items) && to_move > 0) { - from = pos_item(src, f); - - to = create_item(dst, t, &from->key, - le16_to_cpu(from->val_len)); - - memcpy(to, from, item_bytes(from)); - to_move -= all_item_bytes(from); - - delete_item(src, f); - if (move_right) - f--; - else - t++; - } -} - -static int sort_key_cmp(const void *A, const void *B) -{ - struct scoutfs_btree_block *bt = scoutfs_block_data_from_contents(A); - const __le16 * __packed a = A; - const __le16 * __packed b = B; - - return scoutfs_key_cmp(&off_item(bt, *a)->key, &off_item(bt, *b)->key); -} - -static int sort_off_cmp(const void *A, const void *B) -{ - const __le16 * __packed a = A; - const __le16 * __packed b = B; - - return (int)le16_to_cpu(*a) - (int)le16_to_cpu(*b); -} - -static void sort_off_swap(void *A, void *B, int size) -{ - __le16 * __packed a = A; - __le16 * __packed b = B; - - swap(*a, *b); -} - -/* - * As items are deleted they create fragmented free space. Even if we - * indexed free space in the block it could still get sufficiently - * fragmented to force a split on insertion even though the two - * resulting blocks would have less than the minimum space consumed by - * items. - * - * We don't bother implementing free space indexing and addressing that - * corner case. Instead we track the number of bytes that could be - * reclaimed if we compacted the item space after the free_end offset. - * block. If this additional free space would satisfy an insertion then - * we compact the items instead of splitting the block. - * - * We move the free space to the center of the block by walking - * backwards through the items in offset order, moving items into free - * space between items towards the end of the block. - * - * We don't have specific metadata to either walk the items in offset - * order or to update the item offsets as we move items. We sort the - * item offset array to achieve both ends. First we sort it by offset - * so we can walk in reverse order. As we move items we update their - * position and then sort by keys once we're done. - * - * Compaction is only attempted during descent as we find a block that - * needs more or less free space. The caller has the parent locked for - * writing and there are no references to the items at this point so - * it's safe to scramble the block contents. - */ -static void compact_items(struct scoutfs_btree_block *bt) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - struct scoutfs_btree_item *from; - struct scoutfs_btree_item *to; - unsigned int bytes; - __le16 end; - int i; - - trace_printk("free_reclaim %u\n", le16_to_cpu(bt->free_reclaim)); - - sort(bt->item_offs, nr, sizeof(bt->item_offs[0]), - sort_off_cmp, sort_off_swap); - - end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); - - for (i = nr - 1; i >= 0; i--) { - from = pos_item(bt, i); - - bytes = item_bytes(from); - le16_add_cpu(&end, -bytes); - to = off_item(bt, end); - bt->item_offs[i] = end; - - if (from != to) - memmove(to, from, bytes); - } - - bt->free_end = end; - bt->free_reclaim = 0; - - sort(bt->item_offs, nr, sizeof(bt->item_offs[0]), - sort_key_cmp, sort_off_swap); -} - - -/* - * Let's talk about btree locking. - * - * The main metadata btree has lots of callers who want concurrency. - * They have their own locks that protect multi item consistency -- say - * an inode's i_mutex protecting the items related to a given inode. - * But it's our responsibility to lock the btree itself. - * - * Our btree operations are implemented with a single walk down the - * tree. This gives us the opportunity to cascade block locks down the - * tree. We first lock the root. Then we lock the first block and - * unlock the root. Then lock the next block and unlock the first - * block. And so on down the tree. After contention on the root and - * first block we have lots of concurrency down paths of the tree to the - * leaves. - * - * Merging during descent has to lock the sibling block that it's - * pulling items from. It has to acquire these nested locks in - * consistent tree order. - * - * The cow tree updates let us skip block locking entirely for stable - * blocks because they're read only. All the blocks in the stable - * super tree are stable so we don't have to lock that tree at all. - * We let the block layer use the header's seq to avoid locking - * stable blocks. - * - * lockdep has to not be freaked out by all of this. The cascading - * block locks really make it angry without annotation so we add classes - * for each level and use nested subclasses for the locking of siblings - * during merge. - */ - -static void set_block_lock_class(struct scoutfs_block *bl, int level) -{ -#ifdef CONFIG_LOCKDEP - static struct lock_class_key tree_depth_classes[SCOUTFS_BTREE_MAX_DEPTH]; - - scoutfs_block_set_lock_class(bl, &tree_depth_classes[level]); -#endif -} - -static void lock_tree_block(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_block *bl, bool write, int subclass) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (root == &sbi->super.btree_root) { - if (bl) { - scoutfs_block_lock(bl, write, subclass); - } else { - if (write) - down_write(&sbi->btree_rwsem); - else - down_read(&sbi->btree_rwsem); - } - } -} - -static void unlock_tree_block(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_block *bl, bool write) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - if (root == &sbi->super.btree_root) { - if (bl) { - scoutfs_block_unlock(bl, write); - } else { - if (write) - up_write(&sbi->btree_rwsem); - else - up_read(&sbi->btree_rwsem); - } - } -} - -/* - * Allocate and initialize a new tree block. The caller adds references - * to it. - */ -static struct scoutfs_block *alloc_tree_block(struct super_block *sb, int level) -{ - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - - bl = scoutfs_block_dirty_alloc(sb); - if (!IS_ERR(bl)) { - bt = scoutfs_block_data(bl); - - bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); - bt->free_reclaim = 0; - bt->nr_items = 0; - - set_block_lock_class(bl, level); - } - - return bl; -} - -/* the caller has ensured that the free must succeed */ -static void free_tree_block(struct super_block *sb, struct scoutfs_block *bl) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_btree_block *bt = scoutfs_block_data(bl); - int err; - - BUG_ON(bt->hdr.seq != super->hdr.seq); - - scoutfs_block_forget(bl); - err = scoutfs_buddy_free(sb, bt->hdr.seq, - le64_to_cpu(bt->hdr.blkno), 0); - BUG_ON(err); -} - -/* - * Allocate a new tree block and point the root at it. The caller - * is responsible for the items in the new root block. - */ -static struct scoutfs_block *grow_tree(struct super_block *sb, - struct scoutfs_btree_root *root) -{ - struct scoutfs_block_header *hdr; - struct scoutfs_block *bl; - - bl = alloc_tree_block(sb, root->height); - if (!IS_ERR(bl)) { - hdr = scoutfs_block_data(bl); - - root->height++; - root->ref.blkno = hdr->blkno; - root->ref.seq = hdr->seq; - - set_block_lock_class(bl, root->height - 1); - } - - return bl; -} - -static struct scoutfs_block *get_block_ref(struct super_block *sb, int level, - struct scoutfs_block_ref *ref, - bool dirty) -{ - struct scoutfs_block *bl; - - if (dirty) - bl = scoutfs_block_dirty_ref(sb, ref); - else - bl = scoutfs_block_read_ref(sb, ref); - - if (!IS_ERR(bl)) - set_block_lock_class(bl, level); - - return bl; -} - -/* - * Create a new item in the parent which references the child. The caller - * specifies the key in the item that describes the items in the child. - */ -static void create_parent_item(struct scoutfs_btree_block *parent, - unsigned int pos, - struct scoutfs_btree_block *child, - struct scoutfs_key *key) -{ - struct scoutfs_btree_item *item; - struct scoutfs_block_ref ref = { - .blkno = child->hdr.blkno, - .seq = child->hdr.seq, - }; - - item = create_item(parent, pos, key, sizeof(ref)); - memcpy(&item->val, &ref, sizeof(ref)); -} - -/* - * See if we need to split this block while descending for insertion so - * that we have enough space to insert. - * - * Parent blocks need enough space for a new item and child ref if a - * child block splits. Leaf blocks need enough space to insert the new - * item with its value. - * - * We split to the left so that the greatest key in the existing block - * doesn't change so we don't have to update the key in its parent item. - * - * If the search key falls in the new split block then we return it to - * the caller to walk through. - * - * The caller has the parent (or root) and our block locked. We don't - * have to lock the blocks we allocate while we have the references to - * them locked. We only need to lock the new sibling if we return it - * instead of our given block for the caller to continue descent. - */ -static struct scoutfs_block *try_split(struct super_block *sb, - struct scoutfs_btree_root *root, - int level, struct scoutfs_key *key, - unsigned int val_len, - struct scoutfs_btree_block *parent, - unsigned int parent_pos, - struct scoutfs_block *right_bl) -{ - struct scoutfs_btree_block *right = scoutfs_block_data(right_bl); - struct scoutfs_btree_block *left; - struct scoutfs_block *left_bl; - struct scoutfs_block *par_bl = NULL; - struct scoutfs_key maximal; - unsigned int all_bytes; - - if (level) - val_len = sizeof(struct scoutfs_block_ref); - all_bytes = all_val_bytes(val_len); - - if (contig_free(right) >= all_bytes) - return right_bl; - - if (reclaimable_free(right) >= all_bytes) { - compact_items(right); - return right_bl; - } - - /* alloc split neighbour first to avoid unwinding tree growth */ - left_bl = alloc_tree_block(sb, level); - if (IS_ERR(left_bl)) { - unlock_tree_block(sb, root, right_bl, true); - scoutfs_block_put(right_bl); - return left_bl; - } - left = scoutfs_block_data(left_bl); - - if (!parent) { - par_bl = grow_tree(sb, root); - if (IS_ERR(par_bl)) { - free_tree_block(sb, left_bl); - scoutfs_block_put(left_bl); - unlock_tree_block(sb, root, right_bl, true); - scoutfs_block_put(right_bl); - return par_bl; - } - - parent = scoutfs_block_data(par_bl); - parent_pos = 0; - - scoutfs_set_max_key(&maximal); - create_parent_item(parent, parent_pos, right, &maximal); - } - - move_items(left, right, false, used_total(right) / 2); - create_parent_item(parent, parent_pos, left, greatest_key(left)); - parent_pos++; /* not that anything uses it again :P */ - - if (scoutfs_key_cmp(key, greatest_key(left)) <= 0) { - /* insertion will go to the new left block */ - unlock_tree_block(sb, root, right_bl, true); - lock_tree_block(sb, root, left_bl, true, 0); - swap(right_bl, left_bl); - } else { - /* insertion will still go through us, might need to compact */ - if (contig_free(right) < all_bytes) - compact_items(right); - } - - scoutfs_block_put(par_bl); - scoutfs_block_put(left_bl); - - return right_bl; -} - -/* - * This is called during descent for deletion when we have a parent and - * might need to merge items from a sibling block if this block has too - * much free space. Eventually we'll be able to fit all of the - * sibling's items in our free space which lets us delete the sibling - * block. - * - * The error handling here is a little weird. We're returning an - * ERR_PTR buffer to match splitting so that the walk can handle errors - * from both easily. We have to unlock and release our buffer to return - * an error. - * - * The caller locks the parent and our given block. We need to - * lock sibling blocks in consistent tree order. Our common case - * has us pulling from our left sibling so we prefer to lock blocks - * from right to left. Splitting doesn't hold both sibling locks. - * - * We free sibling or parent btree block blknos if we drain them of items. - * They're dirtied either by descent or before we start migrating items - * so freeing their blkno must succeed. - * - * XXX this could more cleverly chose a merge candidate sibling - */ -static struct scoutfs_block *try_merge(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_block *par_bl, - int level, unsigned int pos, - struct scoutfs_block *bl) -{ - struct scoutfs_btree_block *parent = scoutfs_block_data(par_bl); - struct scoutfs_btree_block *bt = scoutfs_block_data(bl); - struct scoutfs_btree_item *sib_item; - struct scoutfs_btree_block *sib_bt; - struct scoutfs_block *sib_bl; - unsigned int sib_pos; - bool move_right; - int to_move; - - if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) - return bl; - - /* move items right into our block if we have a left sibling */ - if (pos) { - sib_pos = pos - 1; - move_right = true; - } else { - sib_pos = pos + 1; - move_right = false; - } - sib_item = pos_item(parent, sib_pos); - - sib_bl = get_block_ref(sb, level, (void *)sib_item->val, true); - if (IS_ERR(sib_bl)) { - /* XXX do we need to unlock this? don't think so */ - scoutfs_block_put(bl); - return sib_bl; - } - sib_bt = scoutfs_block_data(sib_bl); - - if (!move_right) { - unlock_tree_block(sb, root, bl, true); - lock_tree_block(sb, root, sib_bl, true, 0); - lock_tree_block(sb, root, bl, true, 1); - - if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) { - unlock_tree_block(sb, root, sib_bl, true); - scoutfs_block_put(sib_bl); - return bl; - } - } else { - lock_tree_block(sb, root, sib_bl, true, 1); - } - - if (used_total(sib_bt) <= reclaimable_free(bt)) - to_move = used_total(sib_bt); - else - to_move = reclaimable_free(bt) - SCOUTFS_BTREE_FREE_LIMIT; - - /* - * Make sure there's room to move a max size item if it's the - * next in line when we only have one byte left to try and move. - * - * XXX This is getting awfully fiddly. Should we be refactoring - * item insertion/deletion to do this for us? - */ - if (contig_free(bt) < (to_move + (SCOUTFS_MAX_ITEM_LEN - 1))) - compact_items(bt); - - trace_printk("sib_pos %d move_right %u to_move %u\n", - sib_pos, move_right, to_move); - - move_items(bt, sib_bt, move_right, to_move); - - /* update our parent's ref if we changed our greatest key */ - if (!move_right) - pos_item(parent, pos)->key = *greatest_key(bt); - - /* delete an empty sib or update if we changed its greatest key */ - if (le16_to_cpu(sib_bt->nr_items) == 0) { - delete_item(parent, sib_pos); - free_tree_block(sb, sib_bl); - } else if (move_right) { - sib_item->key = *greatest_key(sib_bt); - } - - /* and finally shrink the tree if our parent is the root with 1 */ - if (le16_to_cpu(parent->nr_items) == 1) { - root->height--; - root->ref.blkno = bt->hdr.blkno; - root->ref.seq = bt->hdr.seq; - free_tree_block(sb, par_bl); - /* caller just unlocks and drops parent */ - } - - unlock_tree_block(sb, root, sib_bl, true); - scoutfs_block_put(sib_bl); - - return bl; -} - -enum { - WALK_INSERT = 1, - WALK_DELETE, - WALK_NEXT_SEQ, - WALK_DIRTY, -}; - -static u64 item_block_ref_seq(struct scoutfs_btree_item *item) -{ - struct scoutfs_block_ref *ref = (void *)item->val; - - return le64_to_cpu(ref->seq); -} - -/* - * Return true if we should skip this item while iterating by sequence - * number. If it's a parent then we test the block ref's seq, if it's a - * leaf item then we check the item's seq. - */ -static bool skip_pos_seq(struct scoutfs_btree_block *bt, unsigned int pos, - int level, u64 seq, int op) -{ - struct scoutfs_btree_item *item; - - if (op != WALK_NEXT_SEQ || pos >= le16_to_cpu(bt->nr_items)) - return false; - - item = pos_item(bt, pos); - - return ((level > 0 && item_block_ref_seq(item) < seq) || - (level == 0 && le64_to_cpu(item->seq) < seq)); -} - -/* - * Return the next sorted item position, possibly skipping those with - * sequence numbers less than the desired sequence number. - */ -static unsigned int next_pos_seq(struct scoutfs_btree_block *bt, - unsigned int pos, int level, u64 seq, int op) -{ - do { - pos++; - } while (skip_pos_seq(bt, pos, level, seq, op)); - - return pos; -} - -/* - * Return the first item after the given key, possibly skipping those - * with sequence numbers less than the desired sequence number. - */ -static unsigned int find_pos_after_seq(struct scoutfs_btree_block *bt, - struct scoutfs_key *key, int level, - u64 seq, int op) -{ - unsigned int pos; - int cmp; - - pos = find_pos(bt, key, &cmp); - if (skip_pos_seq(bt, pos, level, seq, op)) - pos = next_pos_seq(bt, pos, level, seq, op); - - return pos; -} - -/* - * Verify that the btree block isn't corrupt. This is way too expensive - * to do for each block access though that's very helpful for debugging - * btree block corruption. - * - * It should be done the first time we read blocks and it doing it for - * every block access should be hidden behind runtime options. - * - * XXX - * - make sure items don't overlap - * - make sure offs point to live items - * - do things with level - * - see if item keys make sense - */ -static int verify_btree_block(struct scoutfs_btree_block *bt, int level, - struct scoutfs_key *small, - struct scoutfs_key *large) -{ - struct scoutfs_btree_item *item; - struct scoutfs_key *prev; - unsigned int bytes = 0; - unsigned int after_offs = sizeof(struct scoutfs_btree_block); - unsigned int first_off; - unsigned int off; - unsigned int nr; - unsigned int i = 0; - int bad = 1; - - nr = le16_to_cpu(bt->nr_items); - if (nr == 0) - goto out; - - if (nr > SCOUTFS_BTREE_MAX_ITEMS) { - nr = SCOUTFS_BTREE_MAX_ITEMS; - goto out; - } - - after_offs = offsetof(struct scoutfs_btree_block, item_offs[nr]); - first_off = SCOUTFS_BLOCK_SIZE; - - for (i = 0; i < nr; i++) { - - off = le16_to_cpu(bt->item_offs[i]); - if (off >= SCOUTFS_BLOCK_SIZE || off < after_offs) - goto out; - - first_off = min(first_off, off); - - item = pos_item(bt, i); - bytes += item_bytes(item); - - if ((i == 0 && scoutfs_key_cmp(&item->key, small) < 0) || - (i > 0 && scoutfs_key_cmp(&item->key, prev) <= 0) || - (i == (nr - 1) && scoutfs_key_cmp(&item->key, large) > 0)) - goto out; - - prev = &item->key; - } - - if (first_off < le16_to_cpu(bt->free_end)) - goto out; - - if ((le16_to_cpu(bt->free_end) + bytes + - le16_to_cpu(bt->free_reclaim)) != SCOUTFS_BLOCK_SIZE) - goto out; - - bad = 0; -out: - if (bad) { - printk("bt %p blkno %llu level %d small "CKF" large "CKF" end %u reclaim %u nr %u (max %lu after %u bytes %u)\n", - bt, le64_to_cpu(bt->hdr.blkno), level, - CKA(small), CKA(large), le16_to_cpu(bt->free_end), - le16_to_cpu(bt->free_reclaim), bt->nr_items, - SCOUTFS_BTREE_MAX_ITEMS, after_offs, bytes); - for (i = 0; i < nr; i++) { - item = pos_item(bt, i); - off = le16_to_cpu(bt->item_offs[i]); - printk(" [%u] off %u key "CKF" len %u\n", - i, off, CKA(&item->key), - le16_to_cpu(item->val_len)); - } - BUG_ON(bad); - } - - return 0; -} - -/* - * Return the leaf block that should contain the given key. The caller - * is responsible for searching the leaf block and performing their - * operation. The block is returned locked for either reading or - * writing depending on the operation. - * - * As we descend through parent items we set prev_key or next_key to the - * last key in the previous sibling's block or to the first key in the - * next sibling's block, respectively. This is used by iteration to - * keep searching sibling blocks if their search key falls at the end of - * a leaf in their search direction. - */ -static struct scoutfs_block *btree_walk(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_key *prev_key, - struct scoutfs_key *next_key, - unsigned int val_len, u64 seq, int op) -{ - struct scoutfs_btree_block *parent = NULL; - struct scoutfs_block *par_bl = NULL; - struct scoutfs_block *bl = NULL; - struct scoutfs_btree_item *item = NULL; - struct scoutfs_block_ref *ref; - struct scoutfs_key small; - struct scoutfs_key large; - unsigned int level; - unsigned int pos = 0; - const bool dirty = op == WALK_INSERT || op == WALK_DELETE || - op == WALK_DIRTY; - int ret; - - /* no sibling blocks if we don't have parent blocks */ - if (next_key) - scoutfs_set_max_key(next_key); - if (prev_key) - scoutfs_key_set_zero(prev_key); - - lock_tree_block(sb, root, NULL, dirty, 0); - - ref = &root->ref; - level = root->height; - - if (!root->height) { - if (op == WALK_INSERT) { - bl = ERR_PTR(-ENOENT); - } else { - bl = grow_tree(sb, root); - if (!IS_ERR(bl)) { - lock_tree_block(sb, root, bl, dirty, 0); - unlock_tree_block(sb, root, NULL, dirty); - } - } - goto out; - } - - - /* skip the whole tree if the root ref's seq is old */ - if (op == WALK_NEXT_SEQ && le64_to_cpu(ref->seq) < seq) { - bl = ERR_PTR(-ENOENT); - goto out; - } - - scoutfs_set_key(&small, 0, 0, 0); - scoutfs_set_key(&large, ~0ULL, ~0, ~0ULL); - - while (level--) { - /* XXX hmm, need to think about retry */ - bl = get_block_ref(sb, level, ref, dirty); - if (IS_ERR(bl)) - break; - - /* XXX enable this */ - ret = 0 && verify_btree_block(scoutfs_block_data(bl), level, - &small, &large); - if (ret) { - scoutfs_block_put(bl); - bl = ERR_PTR(ret); - break; - } - - lock_tree_block(sb, root, bl, dirty, 0); - - if (op == WALK_INSERT) - bl = try_split(sb, root, level, key, val_len, parent, - pos, bl); - if ((op == WALK_DELETE) && parent) - bl = try_merge(sb, root, par_bl, level, pos, bl); - if (IS_ERR(bl)) - break; - - unlock_tree_block(sb, root, par_bl, dirty); - - if (!level) - break; - - scoutfs_block_put(par_bl); - par_bl = bl; - parent = scoutfs_block_data(par_bl); - - /* - * Find the parent item that references the next child - * block to search. If we're skipping items with old - * seqs then we might not have any child items to - * search. - */ - pos = find_pos_after_seq(parent, key, level, seq, op); - if (pos >= le16_to_cpu(parent->nr_items)) { - /* current block dropped as parent below */ - if (op == WALK_NEXT_SEQ) - bl = ERR_PTR(-ENOENT); - else - bl = ERR_PTR(-EIO); - break; - } - - /* XXX verify sane length */ - item = pos_item(parent, pos); - ref = (void *)item->val; - - /* - * Update the keys that iterators should continue - * searching from. Keep in mind that iteration is read - * only so the parent item won't be changed splitting or - * merging. - */ - if (next_key) { - *next_key = item->key; - scoutfs_inc_key(next_key); - } - - if (pos) { - small = pos_item(parent, pos - 1)->key; - if (prev_key) - *prev_key = small; - } - large = item->key; - } - -out: - if (IS_ERR(bl)) - unlock_tree_block(sb, root, par_bl, dirty); - scoutfs_block_put(par_bl); - - return bl; -} - -/* - * Copy the given value identified by the given key into the caller's - * buffer. The number of bytes copied is returned, -ENOENT if the key - * wasn't found, or -errno on errors. - */ -int scoutfs_btree_lookup(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - unsigned int pos; - int cmp; - int ret; - - trace_scoutfs_btree_lookup(sb, key, scoutfs_btree_val_length(val)); - - bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, 0); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = scoutfs_block_data(bl); - - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - item = pos_item(bt, pos); - ret = copy_to_val(val, item); - } else { - ret = -ENOENT; - } - - unlock_tree_block(sb, root, bl, false); - scoutfs_block_put(bl); - - trace_printk("key "CKF" ret %d\n", CKA(key), ret); - - return ret; -} - -/* - * Insert a new item in the tree. - * - * 0 is returned on success. -EEXIST is returned if the key is already - * present in the tree. - * - * If no value pointer is given then the item is created with a zero - * length value. - */ -int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - unsigned int val_len; - int pos; - int cmp; - int ret; - - if (val) - val_len = scoutfs_btree_val_length(val); - else - val_len = 0; - - trace_scoutfs_btree_insert(sb, key, val_len); - - if (WARN_ON_ONCE(val_len > SCOUTFS_MAX_ITEM_LEN)) - return -EINVAL; - - bl = btree_walk(sb, root, key, NULL, NULL, val_len, 0, WALK_INSERT); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = scoutfs_block_data(bl); - - pos = find_pos(bt, key, &cmp); - if (cmp) { - item = create_item(bt, pos, key, val_len); - if (val) - ret = copy_to_item(item, val); - else - ret = 0; - } else { - ret = -EEXIST; - } - - unlock_tree_block(sb, root, bl, true); - scoutfs_block_put(bl); - - return ret; -} - -/* - * Delete an item from the tree. -ENOENT is returned if the key isn't - * found. - */ -int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key) -{ - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - int pos; - int cmp; - int ret; - - trace_scoutfs_btree_delete(sb, key, 0); - - bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DELETE); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - goto out; - } - bt = scoutfs_block_data(bl); - - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - delete_item(bt, pos); - ret = 0; - - /* XXX this locking is broken.. hold root rwsem? */ - - /* delete the final block in the tree */ - if (bt->nr_items == 0) { - root->height = 0; - root->ref.blkno = 0; - root->ref.seq = 0; - - free_tree_block(sb, bl); - } - } else { - ret = -ENOENT; - } - - unlock_tree_block(sb, root, bl, true); - scoutfs_block_put(bl); - -out: - trace_printk("key "CKF" ret %d\n", CKA(key), ret); - return ret; -} - -/* - * Find the next key in the tree starting from 'first', and ending at - * 'last'. 'found', 'found_seq', and 'val' are set to the discovered - * item if they're provided. - * - * The caller can limit results to items with a sequence number greater - * than or equal to their sequence number. - * - * The only tricky bit is that they key we're searching for might not - * exist in the tree. We can get to the leaf and find that there are no - * greater items in the leaf. We have to search again from the keys - * greater than the parent item's keys which the walk gives us. We also - * star the search over from this next key if walking while filtering - * based on seqs terminates early. - * - * Returns the bytes copied into the value (0 if not provided), -ENOENT - * if there is no item past first until last, or -errno on errors. - * - * It's a common pattern to use the same key for first and found so we're - * careful to copy first before we modify found. - */ -static int btree_next(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, int op, struct scoutfs_key *found, - u64 *found_seq, struct scoutfs_btree_val *val) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_block *bt; - struct scoutfs_key start = *first; - struct scoutfs_key key = *first; - struct scoutfs_key next_key; - struct scoutfs_block *bl; - int pos; - int ret; - - trace_printk("finding next first "CKF" last "CKF"\n", - CKA(&start), CKA(last)); - - /* find the leaf that contains the next item after the key */ - ret = -ENOENT; - while (scoutfs_key_cmp(&key, last) <= 0) { - - bl = btree_walk(sb, root, &key, NULL, &next_key, 0, seq, op); - - /* next seq walks can terminate in parents with old seqs */ - if (op == WALK_NEXT_SEQ && bl == ERR_PTR(-ENOENT)) { - key = next_key; - continue; - } - - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - break; - } - bt = scoutfs_block_data(bl); - - /* keep trying leaves until next_key passes last */ - pos = find_pos_after_seq(bt, &key, 0, seq, op); - if (pos >= le16_to_cpu(bt->nr_items)) { - key = next_key; - unlock_tree_block(sb, root, bl, false); - scoutfs_block_put(bl); - continue; - } - - item = pos_item(bt, pos); - if (scoutfs_key_cmp(&item->key, last) <= 0) { - *found = item->key; - if (found_seq) - *found_seq = le64_to_cpu(item->seq); - if (val) - ret = copy_to_val(val, item); - else - ret = 0; - } else { - ret = -ENOENT; - } - - unlock_tree_block(sb, root, bl, false); - scoutfs_block_put(bl); - break; - } - - trace_printk("next first "CKF" last "CKF" found "CKF" ret %d\n", - CKA(&start), CKA(last), CKA(found), ret); - return ret; -} - -int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, - struct scoutfs_btree_val *val) -{ - trace_scoutfs_btree_next(sb, first, last); - - return btree_next(sb, root, first, last, 0, 0, found, NULL, val); -} - -int scoutfs_btree_since(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, struct scoutfs_key *found, u64 *found_seq, - struct scoutfs_btree_val *val) -{ - trace_scoutfs_btree_since(sb, first, last); - - return btree_next(sb, root, first, last, seq, WALK_NEXT_SEQ, - found, found_seq, val); -} - -/* - * Find the greatest key that is >= first and <= last, starting at last. - * For each search cursor key we descend to the leaf and find its - * position in the items. The item binary search returns the position - * that the key would be inserted into, so if we didn't find the key - * specifically we go to the previous position. The btree walk gives us - * the previous key to search from if we fall off the front of the - * block. - * - * This doesn't support filtering the tree traversal by seqs. - */ -int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, u64 *found_seq, - struct scoutfs_btree_val *val) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_block *bt; - struct scoutfs_key key = *last; - struct scoutfs_key prev_key; - struct scoutfs_block *bl; - int pos; - int cmp; - int ret; - - trace_scoutfs_btree_prev(sb, first, last); - - /* find the leaf that contains the next item after the key */ - ret = -ENOENT; - while (scoutfs_key_cmp(&key, first) >= 0) { - - bl = btree_walk(sb, root, &key, &prev_key, NULL, 0, 0, 0); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - break; - } - bt = scoutfs_block_data(bl); - - pos = find_pos(bt, &key, &cmp); - - /* walk to the prev leaf if we hit the front of this leaf */ - if (pos == 0 && cmp != 0) { - unlock_tree_block(sb, root, bl, false); - scoutfs_block_put(bl); - if (scoutfs_key_is_zero(&key)) - break; - key = prev_key; - continue; - } - - /* we want the item before a non-matching position */ - if (pos && cmp) - pos--; - - /* return the item if it's still within our first bound */ - item = pos_item(bt, pos); - if (cmp == 0 || scoutfs_key_cmp(&item->key, first) >= 0) { - *found = item->key; - if (found_seq) - *found_seq = le64_to_cpu(item->seq); - if (val) - ret = copy_to_val(val, item); - else - ret = 0; - } - - unlock_tree_block(sb, root, bl, false); - scoutfs_block_put(bl); - break; - } - - return ret; -} - -/* - * Ensure that the blocks that lead to the item with the given key are - * dirty. caller can hold a transaction to pin the dirty blocks and - * guarantee that later updates of the item will succeed. - * - * <0 is returned on error, including -ENOENT if the key isn't present. - */ -int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key) -{ - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - int cmp; - int ret; - - trace_scoutfs_btree_dirty(sb, key, 0); - - bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = scoutfs_block_data(bl); - - find_pos(bt, key, &cmp); - if (cmp == 0) { - ret = 0; - } else { - ret = -ENOENT; - } - - unlock_tree_block(sb, root, bl, true); - scoutfs_block_put(bl); - - trace_printk("key "CKF" ret %d\n", CKA(key), ret); - - return ret; -} - -/* - * This is guaranteed not to fail if the caller has already dirtied the - * block that contains the item in the current transaction. - * - * 0 is returned on success. -EINVAL is returned if the caller's value - * length doesn't match the existing item's value length. - */ -int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; - int pos; - int cmp; - int ret; - - trace_scoutfs_btree_update(sb, key, - val ? scoutfs_btree_val_length(val) : 0); - - bl = btree_walk(sb, root, key, NULL, NULL, 0, 0, WALK_DIRTY); - if (IS_ERR(bl)) - return PTR_ERR(bl); - bt = scoutfs_block_data(bl); - - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - item = pos_item(bt, pos); - ret = copy_to_item(item, val); - if (ret == 0) - item->seq = bt->hdr.seq; - } else { - ret = -ENOENT; - } - - unlock_tree_block(sb, root, bl, true); - scoutfs_block_put(bl); - - return ret; -} - -/* - * Set hole to a missing key in the caller's range. - * - * 0 is returned if we find a missing key, -ENOSPC is returned if all - * the keys in the range are present in the tree, and -errno is returned - * if we saw an error. - * - * We try to find the first key in the range. If the next key is past - * the first key then we return the key before the found key. This will - * tend to let us find the hole with one btree search. - * - * We keep searching as long as we keep finding the first key and will - * return -ENOSPC if we fall off the end of the range doing so. - */ -int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, - struct scoutfs_key *last, struct scoutfs_key *hole) -{ - struct scoutfs_key key = *first; - struct scoutfs_key found; - int ret; - - trace_scoutfs_btree_hole(sb, first, last); - - if (WARN_ON_ONCE(scoutfs_key_cmp(first, last) > 0)) { - scoutfs_key_set_zero(hole); - return -EINVAL; - } - - /* search as long as we keep finding our first key */ - do { - ret = scoutfs_btree_next(sb, root, &key, last, &found, NULL); - } while (ret == 0 && - scoutfs_key_cmp(&found, &key) == 0 && - (scoutfs_inc_key(&key), ret = -ENOSPC, - scoutfs_key_cmp(&key, last) <= 0)); - - if (ret == 0) { - *hole = found; - scoutfs_dec_key(hole); - } else if (ret == -ENOENT) { - *hole = *last; - ret = 0; - } - - trace_printk("first "CKF" last "CKF" hole "CKF" ret %d\n", - CKA(first), CKA(last), CKA(hole), ret); - - return ret; -} diff --git a/kmod/src/btree.h b/kmod/src/btree.h deleted file mode 100644 index dec2310c..00000000 --- a/kmod/src/btree.h +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef _SCOUTFS_BTREE_H_ -#define _SCOUTFS_BTREE_H_ - -#include - -struct scoutfs_btree_val { - struct kvec vec[3]; - unsigned int check_size_eq:1; - unsigned int check_size_lte:1; -}; - -static inline void __scoutfs_btree_init_val(struct scoutfs_btree_val *val, - void *ptr0, unsigned int len0, - void *ptr1, unsigned int len1, - void *ptr2, unsigned int len2) -{ - *val = (struct scoutfs_btree_val) { - { { ptr0, len0 }, { ptr1, len1 }, { ptr2, len2 } } - }; -} - -#define _scoutfs_btree_init_val(v, p0, l0, p1, l1, p2, l2, ...) \ - __scoutfs_btree_init_val(v, p0, l0, p1, l1, p2, l2) - -/* - * Provide a nice variadic initialization function without having to - * iterate over the callers arg types. We play some macro games to pad - * out the callers ptr/len pairs to the full possible number. This will - * produce confusing errors if an odd number of arguments is given and - * the padded ptr/length types aren't compatible with the fixed - * arguments in the static inline. - */ -#define scoutfs_btree_init_val(val, ...) \ - _scoutfs_btree_init_val(val, __VA_ARGS__, NULL, 0, NULL, 0, NULL, 0) - -static inline int scoutfs_btree_val_length(struct scoutfs_btree_val *val) -{ - - return iov_length((struct iovec *)val->vec, ARRAY_SIZE(val->vec)); -} - -int scoutfs_btree_lookup(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val); -int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val); -int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key); -int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, - struct scoutfs_btree_val *val); -int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - struct scoutfs_key *found, u64 *found_seq, - struct scoutfs_btree_val *val); -int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key); -int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_val *val); -int scoutfs_btree_hole(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *first, - struct scoutfs_key *last, struct scoutfs_key *hole); -int scoutfs_btree_since(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *first, struct scoutfs_key *last, - u64 seq, struct scoutfs_key *found, u64 *found_seq, - struct scoutfs_btree_val *val); - -#endif diff --git a/kmod/src/buddy.c b/kmod/src/buddy.c deleted file mode 100644 index 9f4a16fd..00000000 --- a/kmod/src/buddy.c +++ /dev/null @@ -1,1063 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "block.h" -#include "buddy.h" -#include "scoutfs_trace.h" - -/* - * scoutfs uses buddy bitmaps in an augmented radix to index free space. - * - * At the heart of the allocator are the buddy bitmaps in the radix - * leaves. For a given region of blocks there are bitmaps for each - * power of two order of blocks that can be allocated. N bits record - * whether each order 0 size block region is allocated or freed, then - * N/2 bits describe order 1 regions that span pairs of order 0 blocks, - * and so on. This ends up using two bits in the bitmaps for each - * device block that's managed. - * - * An order bit is set when it is free. All of its lower order bits - * will be clear. To allocate we clear a bit. A partial allocation - * clears the higher order bit and each buddy for each lower order until - * the allocated order. Freeing sets an order bit. Then if it's buddy - * order is also set we clear both and set their higher order bit. This - * proceeds to the highest order. - * - * Each buddy block records the first set bit in each order bitmap. As - * bits are set they update these first set records if they're before - * the previous value. As bits are cleared we find the next set if it - * was the first. - * - * These buddy bitmap blocks that each fully describe a region of blocks - * are assembled into a radix tree. Each reference to a leaf block in - * parent blocks have a bitmap of the orders that are free in its leaf - * block. The parent blocks then also record the first slot that has - * each order bit set in its child references. This indexing holds all - * the way to the root. This lets us quickly determine an order that - * will satisfy an allocation and descend to the leaf that contains the - * first free region of that order. - * - * These buddy blocks themselves are located in preallocated space. Each - * logical position in the tree occupies two blocks on the device. In - * each transaction we use the currently referenced block to cow into - * its partner. Since the block positions are calculated the block - * references only need a bit to specify which of the pair is being - * referenced. The number of blocks needed is precisely calculated by - * taking the number of leaf blocks needed to track the device blocks - * and dividing by the radix fanout until we have a single root block. - * - * Each aligned block allocation order is stored in a path down the - * radix to a leaf that's a function of the block offset. This lets us - * ensure that we can allocate or free a given allocation order by - * dirtying those blocks. If we've allocated an order in a transaction - * it can always be freed (or re-allocated) while the transaction holds - * the dirty buddy blocks. - * - * We use that property to ensure that frees of stable data don't - * satisfy allocation until the next transaction. When we free stable - * data we dirty the path to its position in the radix and record the - * free in an rbtree. We can then apply these frees as we commit the - * transaction. If the transaction fails we can undo the frees and let - * the file system carry on. We'll try to reapply the frees before the - * next transaction commits. The allocator never introduces - * unrecoverable errors. - * - * The radix isn't fully populated when it's created. mkfs only - * initializes the two paths down the tree that have partially - * initialized parent slots and leaf bitmaps. The path down the left - * spine has the initial file system blocks allocated. The path down - * the right spine can have partial parent slots and bits set in the - * leaf when device sizes aren't multiples of the leaf block bit count - * and radix fanout. The kernel then only has to initialize the rest of - * the buddy blocks blocks which have fully populated parent slots and - * leaf bitmaps. - * - * XXX - * - resize is going to be a thing. figure out that thing. - */ - -struct buddy_info { - struct mutex mutex; - - atomic_t alloc_count; - struct rb_root pending_frees; - - /* max height given total blocks */ - u8 max_height; - /* the device blkno of the first block of a given level */ - u64 level_blkno[SCOUTFS_BUDDY_MAX_HEIGHT]; - /* blk divisor to find slot index at each level */ - u64 level_div[SCOUTFS_BUDDY_MAX_HEIGHT]; - - struct buddy_stack { - struct scoutfs_block *bl[SCOUTFS_BUDDY_MAX_HEIGHT]; - u16 sl[SCOUTFS_BUDDY_MAX_HEIGHT]; - int nr; - } stack; -}; - -/* the first device blkno covered by the buddy allocator */ -static u64 first_blkno(struct scoutfs_super_block *super) -{ - return SCOUTFS_BUDDY_BLKNO + le64_to_cpu(super->buddy_blocks); -} - -/* the last device blkno covered by the buddy allocator */ -static u64 last_blkno(struct scoutfs_super_block *super) -{ - return le64_to_cpu(super->total_blocks) - 1; -} - -/* the last relative blkno covered by the buddy allocator */ -static u64 last_blk(struct scoutfs_super_block *super) -{ - return last_blkno(super) - first_blkno(super); -} - -/* true when the device blkno is covered by the allocator */ -static bool device_blkno(struct scoutfs_super_block *super, u64 blkno) -{ - return blkno >= first_blkno(super) && blkno <= last_blkno(super); -} - -/* true when the device blkno is used for buddy blocks */ -static bool buddy_blkno(struct scoutfs_super_block *super, u64 blkno) -{ - return blkno < first_blkno(super); -} - -/* the order 0 bit offset in a buddy block of a given relative blk */ -static int buddy_bit(u64 blk) -{ - return do_div(blk, SCOUTFS_BUDDY_ORDER0_BITS); -} - -/* true if the rel blk could be the start of an allocation of the order */ -static bool valid_order(u64 blk, int order) -{ - return (buddy_bit(blk) & ((1 << order) - 1)) == 0; -} - -/* the block bit offset of the first bit of the given order's bitmap */ -static int order_off(int order) -{ - if (order == 0) - return 0; - - return (2 * SCOUTFS_BUDDY_ORDER0_BITS) - - (SCOUTFS_BUDDY_ORDER0_BITS / (1 << (order - 1))); -} - -/* the bit offset in the block bitmap of an order's bit */ -static int order_nr(int order, int nr) -{ - return order_off(order) + nr; -} - -static void stack_push(struct buddy_stack *sta, struct scoutfs_block *bl, - u16 sl) -{ - sta->bl[sta->nr] = bl; - sta->sl[sta->nr++] = sl; -} - -/* sl isn't returned because callers peek the leaf where sl is meaningless */ -static struct scoutfs_block *stack_peek(struct buddy_stack *sta) -{ - if (sta->nr) - return sta->bl[sta->nr - 1]; - - return NULL; -} - -static struct scoutfs_block *stack_pop(struct buddy_stack *sta, u16 *sl) -{ - if (sta->nr) { - *sl = sta->sl[--sta->nr]; - return sta->bl[sta->nr]; - } - - return NULL; -} - -/* update first_set if the caller set an earlier nr for the given order */ -static void set_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) -{ - u16 first = le16_to_cpu(bud->first_set[order]); - - trace_printk("set level %u order %d nr %u first %u\n", - bud->level, order, nr, first); - - if (nr <= first) - bud->first_set[order] = cpu_to_le16(nr); -} - -/* find the next first set if the caller just cleared the current first_set */ -static void clear_order_nr(struct scoutfs_buddy_block *bud, int order, u16 nr) -{ - u16 first = le16_to_cpu(bud->first_set[order]); - int size; - int i; - - trace_printk("cleared level %u order %d nr %u first %u\n", - bud->level, order, nr, first); - - if (nr != first) - return; - - if (bud->level) { - for (i = nr + 1; i < SCOUTFS_BUDDY_SLOTS; i++) { - if (le16_to_cpu(bud->slots[i].free_orders) & - (1 << order)) - break; - } - if (i == SCOUTFS_BUDDY_SLOTS) - i = U16_MAX; - - } else { - size = order_off(order + 1); - i = find_next_bit_le(bud->bits, size, - order_nr(order, first) + 1); - if (i >= size) - i = U16_MAX; - else - i -= order_off(order); - } - - bud->first_set[order] = cpu_to_le16(i); - -} - -#define for_each_changed_bit(nr, bit, old, new, tmp) \ - for (tmp = old ^ new; \ - tmp && (nr = ffs(tmp) - 1, bit = 1 << nr, 1); \ - tmp ^= bit) - -/* - * Set a slot's free_orders value and update first_set for each order - * that it changes. Returns true of the slot's free_orders was changed. - */ -static bool set_slot_free_orders(struct scoutfs_buddy_block *bud, u16 sl, - u16 free_orders) -{ - u16 old = le16_to_cpu(bud->slots[sl].free_orders); - int order; - int tmp; - int bit; - - if (old == free_orders) - return false; - - for_each_changed_bit(order, bit, old, free_orders, tmp) { - if (old & bit) - clear_order_nr(bud, order, sl); - else - set_order_nr(bud, order, sl); - } - - bud->slots[sl].free_orders = cpu_to_le16(free_orders); - return true; -} - -/* - * The block at the top of the stack has changed its bits or slots and - * updated its first set. We propagate those changes up through - * free_orders in parents slots and their first_set up through the tree - * to free_orders in the root. We can stop when a block's first_set - * values don't change free_orders in their parent's slot. - */ -static void stack_cleanup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct buddy_stack *sta = &binf->stack; - struct scoutfs_buddy_root *root = &sbi->super.buddy_root; - struct scoutfs_buddy_block *bud; - struct scoutfs_block *bl; - u16 free_orders = 0; - bool parent; - u16 sl; - int i; - - parent = false; - while ((bl = stack_pop(sta, &sl))) { - - bud = scoutfs_block_data(bl); - if (parent && !set_slot_free_orders(bud, sl, free_orders)) { - scoutfs_block_put(bl); - break; - } - - free_orders = 0; - for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) { - if (bud->first_set[i] != cpu_to_le16(U16_MAX)) - free_orders |= 1 << i; - } - - scoutfs_block_put(bl); - parent = true; - } - - /* set root if we got that far */ - if (bl == NULL) - root->slot.free_orders = cpu_to_le16(free_orders); - - /* put any remaining blocks */ - while ((bl = stack_pop(sta, &sl))) - scoutfs_block_put(bl); - -} - -static int test_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) -{ - return !!test_bit_le(order_nr(order, nr), bud->bits); -} - -static void set_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) -{ - if (!test_and_set_bit_le(order_nr(order, nr), bud->bits)) - set_order_nr(bud, order, nr); -} - -static void clear_buddy_bit(struct scoutfs_buddy_block *bud, int order, int nr) -{ - if (test_and_clear_bit_le(order_nr(order, nr), bud->bits)) - clear_order_nr(bud, order, nr); -} - -/* - * mkfs always writes the paths down the sides of the radix that have - * partially populated blocks. We only have to initialize full blocks - * in the middle of the tree. - */ -static void init_buddy_block(struct buddy_info *binf, - struct scoutfs_super_block *super, - struct scoutfs_block *bl, int level) -{ - struct scoutfs_buddy_block *bud = scoutfs_block_data(bl); - u16 count; - int nr; - int i; - - scoutfs_block_zero(bl, sizeof(bud->hdr)); - - for (i = 0; i < ARRAY_SIZE(bud->first_set); i++) - bud->first_set[i] = cpu_to_le16(U16_MAX); - - bud->level = level; - - if (level) { - for (i = 0; i < SCOUTFS_BUDDY_SLOTS; i++) - set_slot_free_orders(bud, i, SCOUTFS_BUDDY_ORDER0_BITS); - } else { - /* ensure that there aren't multiple highest orders */ - BUILD_BUG_ON((SCOUTFS_BUDDY_ORDER0_BITS / - (1 << (SCOUTFS_BUDDY_ORDERS - 1))) > 1); - - count = SCOUTFS_BUDDY_ORDER0_BITS; - nr = 0; - for (i = SCOUTFS_BUDDY_ORDERS - 1; i >= 0; i--) { - if (count & (1 << i)) { - set_buddy_bit(bud, i, nr); - nr = (nr + 1) << 1; - } else { - nr <<= 1; - } - } - } -} - -/* - * Give the caller the block referenced by the given slot. They've - * calculated the blkno of the pair of blocks while walking the tree. - * The slot describes which of the pair its referencing. The caller is - * always going to modify the block so we always try and cow it. We - * construct a fake ref so we can re-use the block ref cow code. When - * we initialize the first use of a block we use the first of the pair. - */ -static struct scoutfs_block *get_buddy_block(struct super_block *sb, - struct scoutfs_buddy_slot *slot, - u64 blkno, int level) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_info *binf = sbi->buddy_info; - struct scoutfs_buddy_block *bud; - struct scoutfs_block_ref ref; - struct scoutfs_block *bl; - - trace_printk("getting block level %d blkno %llu slot seq %llu off %u\n", - level, blkno, le64_to_cpu(slot->seq), slot->blkno_off); - - /* init a new block for an unused slot */ - if (slot->seq == 0) { - bl = scoutfs_block_dirty(sb, blkno); - if (!IS_ERR(bl)) - init_buddy_block(binf, super, bl, level); - } else { - /* construct block ref from tree walk blkno and slot ref */ - ref.blkno = cpu_to_le64(blkno + slot->blkno_off); - ref.seq = slot->seq; - bl = scoutfs_block_dirty_ref(sb, &ref); - } - - if (!IS_ERR(bl)) { - bud = scoutfs_block_data(bl); - - /* rebuild slot ref to blkno */ - if (slot->seq != bud->hdr.seq) { - slot->blkno_off = le64_to_cpu(bud->hdr.blkno) - blkno; - /* alloc_same only xors low bit */ - BUG_ON(slot->blkno_off > 1); - slot->seq = bud->hdr.seq; - } - } - - return bl; -} - -/* - * Walk the buddy block radix to the leaf that contains either the given - * relative blk or the first free given order. The radix is of a fixed - * depth and we initialize new blocks as we descend through - * uninitialized refs. - * - * If order is -1 then we search for the blk. - * - * As we descend we calculate the base blk offset of the path we're - * taking down the tree. This is used to find the blkno of the next - * block relative to the blkno of the given level. It's then used by - * the caller to calculate the total blk offset by adding the bit they - * find in the block. - * - * The path through the tree is recorded in the stack in the buddy info. - * The caller is responsible for cleaning up the stack and must do so - * even if we return an error. - */ -static int buddy_walk(struct super_block *sb, u64 blk, int order, u64 *base) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_info *binf = sbi->buddy_info; - struct buddy_stack *sta = &binf->stack; - struct scoutfs_buddy_root *root = &sbi->super.buddy_root; - struct scoutfs_buddy_block *bud; - struct scoutfs_buddy_slot *slot; - struct scoutfs_block *bl; - u64 blkno; - int level; - int ret = 0; - int sl = 0; - - /* XXX corruption? */ - if (blk > last_blk(super) || root->height == 0 || - root->height > SCOUTFS_BUDDY_MAX_HEIGHT) - return -EIO; - - slot = &root->slot; - level = root->height; - blkno = SCOUTFS_BUDDY_BLKNO; - *base = 0; - - while (level--) { - /* XXX do base and level make sense here? */ - bl = get_buddy_block(sb, slot, blkno, level); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - break; - } - - trace_printk("before blk %llu order %d level %d blkno %llu base %llu sl %d\n", - blk, order, level, blkno, *base, sl); - - bud = scoutfs_block_data(bl); - - if (level) { - if (order >= 0) { - /* find first slot with order free */ - sl = le16_to_cpu(bud->first_set[order]); - /* XXX corruption */ - if (sl == U16_MAX) { - scoutfs_block_put(bl); - ret = -EIO; - break; - } - } else { - /* find slot based on blk */ - sl = div64_u64_rem(blk, binf->level_div[level], - &blk); - } - - /* shouldn't be sl * 2, right? */ - *base = (*base * SCOUTFS_BUDDY_SLOTS) + sl; - /* this is the only place we * 2 */ - blkno = binf->level_blkno[level - 1] + (*base * 2); - slot = &bud->slots[sl]; - } else { - *base *= SCOUTFS_BUDDY_ORDER0_BITS; - /* sl in stack is 0 for final leaf block */ - sl = 0; - } - - trace_printk("after blk %llu order %d level %d blkno %llu base %llu sl %d\n", - blk, order, level, blkno, *base, sl); - - - stack_push(sta, bl, sl); - } - - trace_printk("walking ret %d\n", ret); - - return ret; -} - -/* - * Find the order to search for to allocate a requested order. We try - * to use the smallest greater or equal order and then the largest - * smaller order. - */ -static int find_free_order(struct scoutfs_buddy_root *root, int order) -{ - u16 free = le16_to_cpu(root->slot.free_orders); - u16 smaller_mask = (1 << order) - 1; - u16 larger = free & ~smaller_mask; - u16 smaller = free & smaller_mask; - - if (larger) - return ffs(larger) - 1; - if (smaller) - return fls(smaller) - 1; - - return -ENOSPC; -} - -/* - * Walk to the leaf that contains the found order and allocate a region - * of the given order, returning the relative blk to the caller. - */ -static int buddy_alloc(struct super_block *sb, u64 *blk, int order, int found) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct buddy_stack *sta = &binf->stack; - struct scoutfs_buddy_block *bud; - struct scoutfs_block *bl; - u64 base; - int ret; - int nr; - int i; - - trace_printk("alloc order %d found %d\n", order, found); - - if (WARN_ON_ONCE(found >= 0 && order > found)) - return -EINVAL; - - ret = buddy_walk(sb, *blk, found, &base); - if (ret) - goto out; - - bl = stack_peek(sta); - bud = scoutfs_block_data(bl); - - if (found >= 0) { - nr = le16_to_cpu(bud->first_set[found]); - /* XXX corruption */ - if (nr == U16_MAX) { - ret = -EIO; - goto out; - } - - /* give caller the found blk for the order */ - *blk = base + (nr << found); - } else { - nr = buddy_bit(*blk) >> found; - } - - /* always allocate the higher or equal found order */ - clear_buddy_bit(bud, found, nr); - - /* and maybe free our buddies between smaller order and larger found */ - nr = buddy_bit(*blk) >> order; - for (i = order; i < found; i++) { - set_buddy_bit(bud, i, nr ^ 1); - nr >>= 1; - } - - ret = 0; -out: - trace_printk("alloc order %d found %d blk %llu ret %d\n", - order, found, *blk, ret); - stack_cleanup(sb); - return ret; -} - -/* - * Free a given order by setting its order bit. If the order's buddy - * isn't set then it isn't free and we can't merge so we set our order - * and are done. If the buddy is free then we can clear it and ascend - * up to try and set the next higher order. That performs the same - * buddy merging test. Eventually we make it to the highest order which - * doesn't have a buddy so we can always set it. - * - * As we're freeing orders in the final buddy bitmap that only partially - * covers the end of the device we might try to test buddies which are - * past the end of the device. The test will still fall within the leaf - * block bitmap and those bits past the device will never be set so we - * will fail the merge and correctly set the orders free. - */ -static int buddy_free(struct super_block *sb, u64 blk, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct buddy_stack *sta = &binf->stack; - struct scoutfs_buddy_block *bud; - struct scoutfs_block *bl; - u64 unused; - int ret; - int nr; - int i; - - ret = buddy_walk(sb, blk, -1, &unused); - if (ret) - goto out; - - bl = stack_peek(sta); - bud = scoutfs_block_data(bl); - - nr = buddy_bit(blk) >> order; - for (i = order; i < SCOUTFS_BUDDY_ORDERS - 2; i++) { - - if (!test_buddy_bit(bud, i, nr ^ 1)) - break; - - clear_buddy_bit(bud, i, nr ^ 1); - nr >>= 1; - } - - set_buddy_bit(bud, i, nr); - - ret = 0; -out: - stack_cleanup(sb); - return ret; -} - -/* - * Try to allocate an extent with the size number of blocks. blkno is - * set to the start of the extent and the order of the block count is - * returned. - */ -int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_info *binf = sbi->buddy_info; - int found; - u64 blk; - int ret; - - trace_printk("order %d\n", order); - - mutex_lock(&binf->mutex); - - found = find_free_order(&super->buddy_root, order); - if (found < 0) { - ret = found; - goto out; - } - - if (found < order) - order = found; - - blk = 0; - ret = buddy_alloc(sb, &blk, order, found); - if (ret) - goto out; - - *blkno = first_blkno(super) + blk; - le64_add_cpu(&super->free_blocks, -(1ULL << order)); - atomic_add((1ULL << order), &binf->alloc_count); - ret = order; - -out: - trace_printk("blkno %llu order %d ret %d\n", *blkno, order, ret); - mutex_unlock(&binf->mutex); - return ret; -} - -/* - * We use the block _ref() routines to dirty existing blocks to reuse - * all the block verification and cow machinery. During cow this is - * called to allocate a new blkno to cow an existing buddy block. We - * use the existing blkno to see if we have to return the other mirrored - * buddy blkno or do a real allocation for every other kind of block - * being cowed. - */ -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - if (buddy_blkno(super, existing)) { - *blkno = existing ^ 1; - trace_printk("existing %llu ret blkno %llu\n", - existing, *blkno); - return 0; - } - - return scoutfs_buddy_alloc(sb, blkno, 0); -} - -struct extent_node { - struct rb_node node; - u64 start; - u64 len; -}; - -static int add_enode_extent(struct rb_root *root, u64 start, u64 len) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct extent_node *left = NULL; - struct extent_node *right = NULL; - struct extent_node *enode; - - trace_printk("adding enode [%llu,%llu]\n", start, len); - - while (*node && !(left && right)) { - parent = *node; - enode = container_of(*node, struct extent_node, node); - - if (start < enode->start) { - if (!right && start + len == enode->start) - right = enode; - node = &(*node)->rb_left; - } else { - if (!left && enode->start + enode->len == start) - left = enode; - node = &(*node)->rb_right; - } - } - - if (right) { - right->start = start; - right->len += len; - trace_printk("right now [%llu, %llu]\n", - right->start, right->len); - } - - if (left) { - if (right) { - left->len += right->len; - rb_erase(&right->node, root); - kfree(right); - } else { - left->len += len; - } - trace_printk("left now [%llu, %llu]\n", left->start, left->len); - } - - if (left || right) - return 0; - - enode = kmalloc(sizeof(struct extent_node), GFP_NOFS); - if (!enode) - return -ENOMEM; - - enode->start = start; - enode->len = len; - - trace_printk("inserted new [%llu, %llu]\n", enode->start, enode->len); - - rb_link_node(&enode->node, parent, node); - rb_insert_color(&enode->node, root); - - return 0; -} - -static void destroy_pending_frees(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct extent_node *enode; - struct rb_node *node; - - for (node = rb_first(&binf->pending_frees); node;) { - enode = rb_entry(node, struct extent_node, node); - node = rb_next(node); - - rb_erase(&enode->node, &binf->pending_frees); - kfree(enode); - } -} - -/* XXX this should be generic */ -#define min3_t(t, a, b, c) min3((t)(a), (t)(b), (t)(c)) - -/* - * Allocate or free all the orders that make up a given arbitrary block - * extent. Today this is used by callers who know that the blocks for - * the extent have already been pinned so we BUG on error. - */ -static void apply_extent(struct super_block *sb, bool alloc, u64 blk, u64 len) -{ - unsigned int blk_order; - unsigned int blk_bit; - unsigned int size; - int order; - int ret; - - trace_printk("applying extent blk %llu len %llu\n", blk, len); - - while (len) { - /* buddy bit might be 0, len always has a bit set */ - blk_bit = buddy_bit(blk); - blk_order = blk_bit ? ffs(blk_bit) - 1 : 0; - order = min3_t(int, blk_order, fls64(len) - 1, - SCOUTFS_BUDDY_ORDERS - 1); - size = 1 << order; - - trace_printk("applying blk %llu order %d\n", blk, order); - - if (alloc) - ret = buddy_alloc(sb, &blk, order, -1); - else - ret = buddy_free(sb, blk, order); - BUG_ON(ret); - - blk += size; - len -= size; - } -} - -/* - * The pending rbtree has recorded frees of stable data that we had to - * wait until transaction commit to record. Once these are tracked in - * the allocator we can't use the allocator until the commit succeeds. - * This is called by transaction commit to get these pending frees into - * the current commit. If it fails they pull them back out. - */ -int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct extent_node *enode; - struct rb_node *node; - - for (node = rb_first(&binf->pending_frees); node;) { - enode = rb_entry(node, struct extent_node, node); - node = rb_next(node); - - apply_extent(sb, alloc, enode->start, enode->len); - } - - return 0; -} - -/* - * Free a given allocated extent. The seq tells us which transaction - * first allocated the extent. If it was allocated in this transaction - * then we can return it to the free buddy and that must succeed. - * - * If it was allocated in a previous transaction then we dirty the - * blocks it will take to free it then record it in an rbtree. The - * rbtree entries are replayed into the dirty blocks as the transaction - * commits. - * - * Buddy block numbers are preallocated and calculated from the radix - * tree structure so we can ignore the block layer's calls to free buddy - * blocks during cow. - */ -int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, int order) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_info *binf = sbi->buddy_info; - u64 unused; - u64 blk; - int ret; - - trace_printk("seq %llu blkno %llu order %d rsv %u\n", - le64_to_cpu(seq), blkno, order, buddy_blkno(super, blkno)); - - /* no specific free tracking for buddy blocks */ - if (buddy_blkno(super, blkno)) - return 0; - - /* XXX corruption? */ - if (!device_blkno(super, blkno)) - return -EINVAL; - - blk = blkno - first_blkno(super); - - if (!valid_order(blk, order)) - return -EINVAL; - - mutex_lock(&binf->mutex); - - if (seq == super->hdr.seq) { - ret = buddy_free(sb, blk, order); - /* - * If this order was allocated in this transaction then its - * blocks should be pinned and we should always be able - * to free it. - */ - BUG_ON(ret); - } else { - ret = buddy_walk(sb, blk, -1, &unused) ?: - add_enode_extent(&binf->pending_frees, blk, 1 << order); - if (ret == 0) - trace_printk("added blk %llu order %d\n", blk, order); - stack_cleanup(sb); - } - - if (ret == 0) - le64_add_cpu(&super->free_blocks, 1ULL << order); - - mutex_unlock(&binf->mutex); - - return ret; -} - -/* - * This is current only used to return partial extents from larger - * allocations in this transaction. - */ -void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct scoutfs_super_block *super = &sbi->stable_super; - u64 blk; - - BUG_ON(!device_blkno(super, blkno)); - - blk = blkno - first_blkno(super); - - mutex_lock(&binf->mutex); - - apply_extent(sb, false, blkno - first_blkno(super), count); - le64_add_cpu(&super->free_blocks, count); - - mutex_unlock(&binf->mutex); -} - -/* - * Return the number of block allocations since the last time the - * counter was reset. This count doesn't include dirty buddy blocks. - */ -unsigned int scoutfs_buddy_alloc_count(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - - return atomic_read(&binf->alloc_count); -} - -u64 scoutfs_buddy_bfree(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - struct scoutfs_super_block *super = &sbi->super; - u64 ret; - - mutex_lock(&binf->mutex); - ret = le64_to_cpu(super->free_blocks); - mutex_unlock(&binf->mutex); - - return ret; -} - -void scoutfs_buddy_committed(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - - atomic_set(&binf->alloc_count, 0); - destroy_pending_frees(sb); -} - -int scoutfs_buddy_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct buddy_info *binf = sbi->buddy_info; - u64 level_blocks[SCOUTFS_BUDDY_MAX_HEIGHT]; - u64 blocks; - int i; - - /* first bit offsets in blocks are __le16 */ - BUILD_BUG_ON(SCOUTFS_BUDDY_ORDER0_BITS >= U16_MAX); - - /* bits need to be naturally aligned to long for _le bitops */ - BUILD_BUG_ON(offsetof(struct scoutfs_buddy_block, bits) & - (sizeof(long) - 1)); - - binf = kzalloc(sizeof(struct buddy_info), GFP_KERNEL); - if (!binf) - return -ENOMEM; - sbi->buddy_info = binf; - - mutex_init(&binf->mutex); - atomic_set(&binf->alloc_count, 0); - binf->pending_frees = RB_ROOT; - - /* calculate blocks at each level */ - blocks = DIV_ROUND_UP_ULL(last_blk(super) + 1, - SCOUTFS_BUDDY_ORDER0_BITS); - for (i = 0; i < SCOUTFS_BUDDY_MAX_HEIGHT; i++) { - level_blocks[i] = (blocks * 2); - if (blocks == 1) { - binf->max_height = i + 1; - break; - } - blocks = DIV_ROUND_UP_ULL(blocks, SCOUTFS_BUDDY_SLOTS); - } - - /* calculate device blkno of first block in each level */ - binf->level_blkno[binf->max_height - 1] = SCOUTFS_BUDDY_BLKNO; - for (i = (binf->max_height - 2); i >= 0; i--) { - binf->level_blkno[i] = binf->level_blkno[i + 1] + - level_blocks[i + 1]; - } - - /* calculate blk divisor to find slot at a given level */ - binf->level_div[1] = SCOUTFS_BUDDY_ORDER0_BITS; - for (i = 2; i < binf->max_height; i++) { - binf->level_div[i] = binf->level_div[i - 1] * - SCOUTFS_BUDDY_SLOTS; - } - - for (i = 0; i < binf->max_height; i++) - trace_printk("level %d div %llu blkno %llu blocks %llu\n", - i, binf->level_div[i], binf->level_blkno[i], - level_blocks[i]); - - return 0; -} - -void scoutfs_buddy_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct buddy_info *binf = sbi->buddy_info; - - if (binf) - WARN_ON_ONCE(!RB_EMPTY_ROOT(&binf->pending_frees)); - kfree(binf); -} - diff --git a/kmod/src/buddy.h b/kmod/src/buddy.h deleted file mode 100644 index 24c0ed0c..00000000 --- a/kmod/src/buddy.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _SCOUTFS_BUDDY_H_ -#define _SCOUTFS_BUDDY_H_ - -int scoutfs_buddy_alloc(struct super_block *sb, u64 *blkno, int order); -int scoutfs_buddy_alloc_same(struct super_block *sb, u64 *blkno, u64 existing); -int scoutfs_buddy_free(struct super_block *sb, __le64 seq, u64 blkno, - int order); -void scoutfs_buddy_free_extent(struct super_block *sb, u64 blkno, u64 count); - -int scoutfs_buddy_was_free(struct super_block *sb, u64 blkno, int order); -u64 scoutfs_buddy_bfree(struct super_block *sb); - -unsigned int scoutfs_buddy_alloc_count(struct super_block *sb); -int scoutfs_buddy_apply_pending(struct super_block *sb, bool alloc); -void scoutfs_buddy_committed(struct super_block *sb); - -int scoutfs_buddy_setup(struct super_block *sb); -void scoutfs_buddy_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 137ebbae..c9d081a6 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -14,8 +14,6 @@ #define EXPAND_EACH_COUNTER \ EXPAND_COUNTER(alloc_alloc) \ EXPAND_COUNTER(alloc_free) \ - EXPAND_COUNTER(block_mem_alloc) \ - EXPAND_COUNTER(block_mem_free) \ EXPAND_COUNTER(seg_lru_shrink) \ EXPAND_COUNTER(trans_level0_seg_write) \ EXPAND_COUNTER(manifest_compact_migrate) \ diff --git a/kmod/src/crc.c b/kmod/src/crc.c deleted file mode 100644 index cde9a1ae..00000000 --- a/kmod/src/crc.c +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2015 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include - -#include "format.h" -#include "crc.h" - -u32 scoutfs_crc_block(struct scoutfs_block_header *hdr) -{ - return crc32c(~0, (char *)hdr + sizeof(hdr->crc), - SCOUTFS_BLOCK_SIZE - sizeof(hdr->crc)); -} diff --git a/kmod/src/crc.h b/kmod/src/crc.h deleted file mode 100644 index 7f1fbf56..00000000 --- a/kmod/src/crc.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _SCOUTFS_CRC_H_ -#define _SCOUTFS_CRC_H_ - -u32 scoutfs_crc_block(struct scoutfs_block_header *hdr); - -#endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0d5f0bb2..79b75dcc 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -23,9 +23,7 @@ #include "inode.h" #include "key.h" #include "super.h" -#include "btree.h" #include "trans.h" -#include "name.h" #include "xattr.h" #include "kvec.h" #include "item.h" diff --git a/kmod/src/format.h b/kmod/src/format.h index a3784bcb..b877d4d8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -35,9 +35,6 @@ */ #define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_SUPER_NR 2 -#define SCOUTFS_BUDDY_BLKNO (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR) - -#define SCOUTFS_MAX_TRANS_BLOCKS (128 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) /* * This header is found at the start of every block so that we can @@ -161,70 +158,6 @@ struct scoutfs_segment_block { /* packed vals */ } __packed; -/* - * Block references include the sequence number so that we can detect - * readers racing with writers and so that we can tell that we don't - * need to follow a reference when traversing based on seqs. - */ -struct scoutfs_block_ref { - __le64 blkno; - __le64 seq; -} __packed; - -/* - * If the block was full of bits the largest possible order would be - * the block size shift + 3 (BITS_PER_BYTE). But the header uses - * up some space and then the buddy bits mean two bits per block. - * Then +1 for this being the number, not the greatest order. - */ -#define SCOUTFS_BUDDY_ORDERS (SCOUTFS_BLOCK_SHIFT + 3 - 2 + 1) - -struct scoutfs_buddy_block { - struct scoutfs_block_header hdr; - __le16 first_set[SCOUTFS_BUDDY_ORDERS]; - __u8 level; - __u8 __pad[3]; /* naturally align bits */ - union { - struct scoutfs_buddy_slot { - __le64 seq; - __le16 free_orders; - /* XXX seems like we could hide a bit somewhere */ - __u8 blkno_off; - } __packed slots[0]; - __le64 bits[0]; - } __packed; -} __packed; - -/* - * Each buddy leaf block references order 0 blocks with half of its - * bitmap. The other half of the bits are used for the higher order - * bits. - */ -#define SCOUTFS_BUDDY_ORDER0_BITS \ - (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) * 8) / 2) - -#define SCOUTFS_BUDDY_SLOTS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_buddy_block)) / \ - sizeof(struct scoutfs_buddy_slot)) - -struct scoutfs_buddy_root { - struct scoutfs_buddy_slot slot; - __u8 height; -} __packed; - -/* ((SCOUTFS_BUDDY_SLOTS^5) * SCOUTFS_BUDDY_ORDER0_BITS) > 2^52 */ -#define SCOUTFS_BUDDY_MAX_HEIGHT 6 - -/* - * We should be able to make the offset smaller if neither dirents nor - * data items use the full 64 bits. - */ -struct scoutfs_key { - __le64 inode; - u8 type; - __le64 offset; -} __packed; - /* * Currently we sort keys by the numeric value of the types, but that * isn't necessary. We could have an arbitrary sort order. So we don't @@ -241,8 +174,6 @@ struct scoutfs_key { #define SCOUTFS_DATA_KEY 11 #define SCOUTFS_MAX_UNUSED_KEY 255 -#define SCOUTFS_MAX_ITEM_LEN 512 - /* value is struct scoutfs_inode */ struct scoutfs_inode_key { __u8 type; @@ -307,66 +238,9 @@ struct scoutfs_symlink_key { __be64 ino; } __packed; -struct scoutfs_btree_root { - u8 height; - struct scoutfs_block_ref ref; -} __packed; - -/* - * @free_end: records the byte offset of the first byte after the free - * space in the block between the header and the first item. New items - * are allocated by subtracting the space they need. - * - * @free_reclaim: records the number of bytes of free space amongst the - * items after free_end. If a block is compacted then this much new - * free space would be reclaimed. - */ -struct scoutfs_btree_block { - struct scoutfs_block_header hdr; - __le16 free_end; - __le16 free_reclaim; - __le16 nr_items; - __le16 item_offs[0]; -} __packed; - -/* - * The item sequence number is set to the dirty block's sequence number - * when the item is modified. It is not changed by splits or merges. - */ -struct scoutfs_btree_item { - struct scoutfs_key key; - __le64 seq; - __le16 val_len; - char val[0]; -} __packed; - -/* Blocks are no more than half free. */ -#define SCOUTFS_BTREE_FREE_LIMIT \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) - /* XXX does this exist upstream somewhere? */ #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) -#define SCOUTFS_BTREE_MAX_ITEMS \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / \ - (member_sizeof(struct scoutfs_btree_block, item_offs[0]) + \ - sizeof(struct scoutfs_btree_item))) - -/* - * We can calculate the max tree depth by calculating how many leaf - * blocks the tree could reference. The block device can only reference - * 2^64 bytes. The tallest parent tree has half full parent blocks. - * - * So we have the relation: - * - * ceil(max_items / 2) ^ (max_depth - 1) >= 2^64 / block_size - * - * and solve for depth: - * - * max_depth = log(ceil(max_items / 2), 2^64 / block_size) + 1 - */ -#define SCOUTFS_BTREE_MAX_DEPTH 10 - #define SCOUTFS_UUID_BYTES 16 /* @@ -382,16 +256,11 @@ struct scoutfs_super_block { __le64 alloc_uninit; __le64 total_segs; __le64 free_segs; - __le64 total_blocks; - __le64 free_blocks; __le64 ring_blkno; __le64 ring_blocks; __le64 ring_tail_block; __le64 ring_gen; __le64 next_seg_seq; - __le64 buddy_blocks; - struct scoutfs_buddy_root buddy_root; - struct scoutfs_btree_root btree_root; struct scoutfs_treap_root alloc_treap_root; struct scoutfs_manifest manifest; } __packed; @@ -418,7 +287,6 @@ struct scoutfs_timespec { struct scoutfs_inode { __le64 size; __le64 blocks; - __le64 link_counter; __le64 data_version; __le64 next_readdir_pos; __le32 nlink; @@ -426,7 +294,6 @@ struct scoutfs_inode { __le32 gid; __le32 mode; __le32 rdev; - __le32 salt; struct scoutfs_timespec atime; struct scoutfs_timespec ctime; struct scoutfs_timespec mtime; @@ -449,20 +316,6 @@ struct scoutfs_dirent { __u8 name[0]; } __packed; -/* - * Dirent items are stored at keys with the offset set to the hash of - * the name. Creation can find that hash values collide and will - * attempt to linearly probe this many following hash values looking for - * an unused value. - * - * In small directories this doesn't really matter because hash values - * will so very rarely collide. At around 50k items we start to see our - * first collisions. 16 slots is still pretty quick to scan in the - * btree and it gets us up into the hundreds of millions of entries - * before enospc is returned as we run out of hash values. - */ -#define SCOUTFS_DIRENT_COLL_NR 16 - #define SCOUTFS_NAME_LEN 255 /* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ @@ -475,14 +328,6 @@ struct scoutfs_dirent { #define SCOUTFS_XATTR_MAX_PARTS \ DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) -/* - * We only use 31 bits for readdir positions so that we don't confuse - * old signed 32bit f_pos applications or those on the other side of - * network protocols that have limited readir positions. - */ - -#define SCOUTFS_DIRENT_OFF_BITS 31 -#define SCOUTFS_DIRENT_OFF_MASK ((1U << SCOUTFS_DIRENT_OFF_BITS) - 1) /* entries begin after . and .. */ #define SCOUTFS_DIRENT_FIRST_POS 2 /* getdents returns next pos with an entry, no entry at (f_pos)~0 */ @@ -499,14 +344,6 @@ enum { SCOUTFS_DT_WHT, }; -struct scoutfs_extent { - __le64 blkno; - __le64 len; - __u8 flags; -} __packed; - -#define SCOUTFS_EXTENT_FLAG_OFFLINE (1 << 0) - /* ino_path can search for backref items with a null term */ #define SCOUTFS_MAX_KEY_SIZE \ offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index ad1caa79..e34441de 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -22,7 +22,6 @@ #include "super.h" #include "key.h" #include "inode.h" -#include "btree.h" #include "dir.h" #include "data.h" #include "scoutfs_trace.h" @@ -126,8 +125,6 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_ctime.tv_sec = le64_to_cpu(cinode->ctime.sec); inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); - ci->salt = le32_to_cpu(cinode->salt); - atomic64_set(&ci->link_counter, le64_to_cpu(cinode->link_counter)); ci->data_version = le64_to_cpu(cinode->data_version); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); } @@ -247,8 +244,6 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->mtime.sec = cpu_to_le64(inode->i_mtime.tv_sec); cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); - cinode->salt = cpu_to_le32(ci->salt); - cinode->link_counter = cpu_to_le64(atomic64_read(&ci->link_counter)); cinode->data_version = cpu_to_le64(ci->data_version); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); } @@ -415,8 +410,6 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->staging = false; - get_random_bytes(&ci->salt, sizeof(ci->salt)); - atomic64_set(&ci->link_counter, 0); inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f3badfb4..6dcb03d8 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -5,7 +5,6 @@ struct scoutfs_inode_info { u64 ino; - u32 salt; seqcount_t seqcount; u64 data_version; @@ -14,7 +13,6 @@ struct scoutfs_inode_info { /* holder of i_mutex is staging */ bool staging; - atomic64_t link_counter; struct rw_semaphore xattr_rwsem; struct inode inode; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 255e167f..82375047 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -24,7 +24,6 @@ #include "format.h" #include "key.h" #include "dir.h" -#include "name.h" #include "ioctl.h" #include "super.h" #include "inode.h" diff --git a/kmod/src/key.h b/kmod/src/key.h index 7d3b2230..3c108555 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -126,127 +126,4 @@ static inline void scoutfs_key_set_max(struct scoutfs_key_buf *key) scoutfs_key_memset(key, 0xff, sizeof(struct scoutfs_inode_key)); } -/* - * What follows are the key functions for the small fixed size btree - * keys. It will all be removed once the callers are converted from - * the btree to the item cache. - */ - -#define CKF "%llu.%u.%llu" -#define CKA(key) \ - le64_to_cpu((key)->inode), (key)->type, le64_to_cpu((key)->offset) - -static inline u64 scoutfs_key_inode(struct scoutfs_key *key) -{ - return le64_to_cpu(key->inode); -} - -static inline u64 scoutfs_key_offset(struct scoutfs_key *key) -{ - return le64_to_cpu(key->offset); -} - -static inline int le64_cmp(__le64 a, __le64 b) -{ - return le64_to_cpu(a) < le64_to_cpu(b) ? -1 : - le64_to_cpu(a) > le64_to_cpu(b) ? 1 : 0; -} - -/* - * Items are sorted by type and then by inode to reflect the relative - * frequency of use. Inodes and xattrs are hot, then dirents, then file - * data extents. We want each use class to be hot and dense, we don't - * want a scan of the inodes to have to skip over each inode's extent - * items. - */ -static inline int scoutfs_key_cmp(struct scoutfs_key *a, struct scoutfs_key *b) -{ - return ((short)a->type - (short)b->type) ?: - le64_cmp(a->inode, b->inode) ?: - le64_cmp(a->offset, b->offset); -} - -/* - * return -ve if the first range is completely before the second, +ve for - * completely after, and 0 if they intersect. - */ -static inline int scoutfs_cmp_key_ranges(struct scoutfs_key *a_first, - struct scoutfs_key *a_last, - struct scoutfs_key *b_first, - struct scoutfs_key *b_last) -{ - if (scoutfs_key_cmp(a_last, b_first) < 0) - return -1; - if (scoutfs_key_cmp(a_first, b_last) > 0) - return 1; - return 0; -} - -static inline int scoutfs_cmp_key_range(struct scoutfs_key *key, - struct scoutfs_key *first, - struct scoutfs_key *last) -{ - return scoutfs_cmp_key_ranges(key, key, first, last); -} - -static inline void scoutfs_set_key(struct scoutfs_key *key, u64 inode, u8 type, - u64 offset) -{ - key->inode = cpu_to_le64(inode); - key->type = type; - key->offset = cpu_to_le64(offset); -} - -static inline void scoutfs_set_max_key(struct scoutfs_key *key) -{ - scoutfs_set_key(key, ~0ULL, ~0, ~0ULL); -} - -/* - * This saturates at (~0,~0,~0) instead of wrapping. This will never be - * an issue for real item keys but parent item keys along the right - * spine of the tree have maximal key values that could wrap if - * incremented. - */ -static inline void scoutfs_inc_key(struct scoutfs_key *key) -{ - if (key->inode == cpu_to_le64(~0ULL) && - key->type == (u8)~0 && - key->offset == cpu_to_le64(~0ULL)) - return; - - le64_add_cpu(&key->offset, 1); - if (!key->offset) { - if (++key->type == 0) - le64_add_cpu(&key->inode, 1); - } -} - -static inline void scoutfs_dec_key(struct scoutfs_key *key) -{ - le64_add_cpu(&key->offset, -1ULL); - if (key->offset == cpu_to_le64(~0ULL)) { - if (key->type-- == 0) - le64_add_cpu(&key->inode, -1ULL); - } -} - -static inline struct scoutfs_key *scoutfs_max_key(struct scoutfs_key *a, - struct scoutfs_key *b) -{ - return scoutfs_key_cmp(a, b) > 0 ? a : b; -} - -static inline bool scoutfs_key_is_zero(struct scoutfs_key *key) -{ - return key->inode == 0 && key->type == 0 && key->offset == 0; -} - -static inline void scoutfs_key_set_zero(struct scoutfs_key *key) -{ - key->inode = 0; - key->type = 0; - key->offset = 0; -} - #endif diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 5e49c6a3..422a4fc5 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -25,10 +25,8 @@ #include "dir.h" #include "xattr.h" #include "msg.h" -#include "block.h" #include "counters.h" #include "trans.h" -#include "buddy.h" #include "kvec.h" #include "scoutfs_trace.h" diff --git a/kmod/src/name.c b/kmod/src/name.c deleted file mode 100644 index e14f52bd..00000000 --- a/kmod/src/name.c +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include - -#include "name.h" - -/* - * XXX This crc nonsense is a quick hack. We'll want something a - * lot stronger like siphash. - */ -u64 scoutfs_name_hash(const char *name, unsigned int len) -{ - unsigned int half = (len + 1) / 2; - - return crc32c(~0, name, half) | - ((u64)crc32c(~0, name + len - half, half) << 32); -} - -int scoutfs_names_equal(const char *name_a, int len_a, - const char *name_b, int len_b) -{ - return (len_a == len_b) && !memcmp(name_a, name_b, len_a); -} diff --git a/kmod/src/name.h b/kmod/src/name.h deleted file mode 100644 index 020ecb0f..00000000 --- a/kmod/src/name.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SCOUTFS_NAME_H_ -#define _SCOUTFS_NAME_H_ - -u64 scoutfs_name_hash(const char *data, unsigned int len); -int scoutfs_names_equal(const char *name_a, int len_a, - const char *name_b, int len_b); - -#endif diff --git a/kmod/src/scoutfs_trace.c b/kmod/src/scoutfs_trace.c index 038eb228..6c775b9f 100644 --- a/kmod/src/scoutfs_trace.c +++ b/kmod/src/scoutfs_trace.c @@ -23,7 +23,6 @@ #include "inode.h" #include "dir.h" #include "msg.h" -#include "block.h" #define CREATE_TRACE_POINTS #include "scoutfs_trace.h" diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 4a1d7a7f..669b99f4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -180,171 +180,6 @@ TRACE_EVENT(scoutfs_scan_orphans, TP_printk("dev %d,%d", MAJOR(__entry->dev), MINOR(__entry->dev)) ); -TRACE_EVENT(scoutfs_buddy_alloc, - TP_PROTO(u64 blkno, int order, int region, int ret), - - TP_ARGS(blkno, order, region, ret), - - TP_STRUCT__entry( - __field(u64, blkno) - __field(int, order) - __field(int, region) - __field(int, ret) - ), - - TP_fast_assign( - __entry->blkno = blkno; - __entry->order = order; - __entry->region = region; - __entry->ret = ret; - ), - - TP_printk("blkno %llu order %d region %d ret %d", - __entry->blkno, __entry->order, __entry->region, __entry->ret) -); - - -TRACE_EVENT(scoutfs_buddy_free, - TP_PROTO(u64 blkno, int order, int region, int ret), - - TP_ARGS(blkno, order, region, ret), - - TP_STRUCT__entry( - __field(u64, blkno) - __field(int, order) - __field(int, region) - __field(int, ret) - ), - - TP_fast_assign( - __entry->blkno = blkno; - __entry->order = order; - __entry->region = region; - __entry->ret = ret; - ), - - TP_printk("blkno %llu order %d region %d ret %d", - __entry->blkno, __entry->order, __entry->region, __entry->ret) -); - -DECLARE_EVENT_CLASS(scoutfs_btree_op, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len), - - TP_STRUCT__entry( - __field( dev_t, dev ) - __field( u64, key_ino ) - __field( u64, key_off ) - __field( u8, key_type ) - __field( int, val_len ) - ), - - TP_fast_assign( - __entry->dev = sb->s_dev; - __entry->key_ino = le64_to_cpu(key->inode); - __entry->key_off = le64_to_cpu(key->offset); - __entry->key_type = key->type; - __entry->val_len = len; - ), - - TP_printk("dev %d,%d key "TRACE_KEYF" size %d", - MAJOR(__entry->dev), MINOR(__entry->dev), - __entry->key_ino, show_key_type(__entry->key_type), - __entry->key_off, __entry->val_len) -); - -DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_lookup, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len) -); - -DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_insert, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len) -); - -DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_delete, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len) -); - -DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_dirty, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len) -); - -DEFINE_EVENT(scoutfs_btree_op, scoutfs_btree_update, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int len), - - TP_ARGS(sb, key, len) -); - -DECLARE_EVENT_CLASS(scoutfs_btree_ranged_op, - TP_PROTO(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last), - - TP_ARGS(sb, first, last), - - TP_STRUCT__entry( - __field( dev_t, dev ) - __field( u64, first_ino ) - __field( u64, first_off ) - __field( u8, first_type ) - __field( u64, last_ino ) - __field( u64, last_off ) - __field( u8, last_type ) - ), - - TP_fast_assign( - __entry->dev = sb->s_dev; - __entry->first_ino = le64_to_cpu(first->inode); - __entry->first_off = le64_to_cpu(first->offset); - __entry->first_type = first->type; - __entry->last_ino = le64_to_cpu(last->inode); - __entry->last_off = le64_to_cpu(last->offset); - __entry->last_type = last->type; - ), - - TP_printk("dev %d,%d first key "TRACE_KEYF" last key "TRACE_KEYF, - MAJOR(__entry->dev), MINOR(__entry->dev), __entry->first_ino, - show_key_type(__entry->first_type), __entry->first_off, - __entry->last_ino, show_key_type(__entry->last_type), - __entry->last_off) -); - -DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_hole, - TP_PROTO(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last), - - TP_ARGS(sb, first, last) -); - -DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_next, - TP_PROTO(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last), - - TP_ARGS(sb, first, last) -); - -DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_prev, - TP_PROTO(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last), - - TP_ARGS(sb, first, last) -); - -DEFINE_EVENT(scoutfs_btree_ranged_op, scoutfs_btree_since, - TP_PROTO(struct super_block *sb, struct scoutfs_key *first, - struct scoutfs_key *last), - - TP_ARGS(sb, first, last) -); - TRACE_EVENT(scoutfs_manifest_add, TP_PROTO(struct super_block *sb, struct kvec *first, struct kvec *last, u64 segno, u64 seq, u8 level), diff --git a/kmod/src/super.c b/kmod/src/super.c index 57dcbd46..158052c1 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -25,10 +25,8 @@ #include "dir.h" #include "xattr.h" #include "msg.h" -#include "block.h" #include "counters.h" #include "trans.h" -#include "buddy.h" #include "item.h" #include "manifest.h" #include "seg.h" @@ -96,8 +94,6 @@ void scoutfs_advance_dirty_super(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - sbi->stable_super = sbi->super; - le64_add_cpu(&super->hdr.blkno, 1); if (le64_to_cpu(super->hdr.blkno) == (SCOUTFS_SUPER_BLKNO + SCOUTFS_SUPER_NR)) @@ -182,8 +178,6 @@ static int read_supers(struct super_block *sb) scoutfs_info(sb, "using super %u with seq %llu", found, le64_to_cpu(sbi->super.hdr.seq)); - sbi->stable_super = sbi->super; - return 0; } @@ -204,23 +198,12 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -ENOMEM; spin_lock_init(&sbi->next_ino_lock); - spin_lock_init(&sbi->block_lock); - /* radix only inserted with NOFS _preload */ - INIT_RADIX_TREE(&sbi->block_radix, GFP_ATOMIC); - init_waitqueue_head(&sbi->block_wq); - atomic_set(&sbi->block_writes, 0); - INIT_LIST_HEAD(&sbi->block_lru_list); - init_rwsem(&sbi->btree_rwsem); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - sbi->block_shrinker.shrink = scoutfs_block_shrink; - sbi->block_shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&sbi->block_shrinker); - /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); if (!sbi->kset) @@ -269,16 +252,12 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sbi) { scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); - scoutfs_buddy_destroy(sb); - if (sbi->block_shrinker.shrink == scoutfs_block_shrink) - unregister_shrinker(&sbi->block_shrinker); scoutfs_data_destroy(sb); scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); scoutfs_treap_destroy(sb); scoutfs_seg_destroy(sb); - scoutfs_block_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); diff --git a/kmod/src/super.h b/kmod/src/super.h index 82eb6bba..e791e76d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,10 +5,8 @@ #include #include "format.h" -#include "buddy.h" struct scoutfs_counters; -struct buddy_info; struct item_cache; struct manifest; struct segment_cache; @@ -20,20 +18,9 @@ struct scoutfs_sb_info { struct super_block *sb; struct scoutfs_super_block super; - struct scoutfs_super_block stable_super; spinlock_t next_ino_lock; - spinlock_t block_lock; - struct radix_tree_root block_radix; - wait_queue_head_t block_wq; - atomic_t block_writes; - int block_write_err; - /* block cache lru */ - struct shrinker block_shrinker; - struct list_head block_lru_list; - unsigned long block_lru_nr; - struct manifest *manifest; struct item_cache *item_cache; struct segment_cache *segment_cache; @@ -42,10 +29,6 @@ struct scoutfs_sb_info { struct compact_info *compact_info; struct data_info *data_info; - struct buddy_info *buddy_info; - - struct rw_semaphore btree_rwsem; - atomic_t trans_holds; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; @@ -68,17 +51,6 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } -/* The root of the metadata btree */ -static inline struct scoutfs_btree_root *SCOUTFS_META(struct super_block *sb) -{ - return &SCOUTFS_SB(sb)->super.btree_root; -} - -static inline struct scoutfs_btree_root *SCOUTFS_STABLE_META(struct super_block *sb) -{ - return &SCOUTFS_SB(sb)->stable_super.btree_root; -} - void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 487514f6..d596bf68 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -18,9 +18,7 @@ #include #include "super.h" -#include "block.h" #include "trans.h" -#include "buddy.h" #include "data.h" #include "bio.h" #include "item.h" diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 52f1acd7..afe1cc14 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -22,7 +22,6 @@ #include "kvec.h" #include "item.h" #include "trans.h" -#include "name.h" #include "xattr.h" /* From f373f05fb7b82ba7145a33c38e1447b97b3da0af Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 14 Feb 2017 11:35:30 -0800 Subject: [PATCH 236/920] Add engineering markdown document Let's put the engineering doc in the source tree so that eventually it'll be easily found upstream. Signed-off-by: Zach Brown --- kmod/Documentation/scoutfs.md | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 kmod/Documentation/scoutfs.md diff --git a/kmod/Documentation/scoutfs.md b/kmod/Documentation/scoutfs.md new file mode 100644 index 00000000..baf92616 --- /dev/null +++ b/kmod/Documentation/scoutfs.md @@ -0,0 +1,80 @@ + +# scoutfs Engineering Compendium + +----- + +## Document Overview + +This document is intended to be a relatively unstructured but thorough +coverage of the design, implementation, and deployment of scoutfs. + +*Not Yet Discussed: repair, dump/restore, remote namespace +synchronization, compression, encryption, trim, dedup, hole punching, +SMR, iops v. bw, range locking, sorting keys by type/inode, enospc, +compaction priority, manifest server, manifest network protocol, inode +allocation, clustered open-unlink, seq queries, offline data, LSM, +forward/back compat.* + +## Raison D'être + +scoutfs is an archival posix file system. It's built to provide a posix +interface to petabytes of data in trillions of files through thousands +of nodes. + +scoutfs uses log-structured merge trees to achieve high operation +throughput with low device command rates. It uses ranged locking to +maintain consistent POSIX semantics amongst clustered nodes with minimum +synchronization overhead. It offers additional metadata indexing and +data residency interfaces for efficiently executing archival policies. +It is deployed on a shared block fabric for high bandwidth and low +latency. + +## Indexing Inodes by Modification Time + +As files are modified archival agents need to find these modified files +so that the archive can be updated. As inode counts explode it becomes +infeasible to scan the entire inode population and meet archival +deadlines. + +scoutfs maintains an index of inodes by modification time. An ioctl is +offered which iterates over the inodes in the order that they were +modified. The ioctl takes a timespec cursor from which to walk. It +fills a buffer with inodes and the time they were modified, sorted by +time. + +The ioctl results are inherently racey. There's nothing to stop an +inode from being modified and moved in the index between when the call +returns and the caller operates on the inode. + +This index is maintained by having time fields in the inode and +modification time items at those time values. The item key sorts the +items by time for the ioctl to iterate over. The items have no value. + + .type = SCOUTFS_MODTIME_KEY, + .ino = inode, + .ts.tv_sec = seconds, + .ts.tv_nsec = nanoseconds, + +As inodes are modified deletion items are created for the old time and +new items are inserted. LSM's ability to let us create items without +strictly locking their key value keeps these items from creating +unacceptable lock contention. If the modifying task has sufficient +locking on the inode it can modify these items and LSM will eventually +merge them into place. + +The index is keyed on real world time so that we don't have to create +our own consistent advancing clock. The clock only needs to be as +accurate as the users of the index require (this often doesn't add +unreasonable requirements, it's often already the case that arhicval +policies involve time and motivate a reasonably synchronized clock +across the cluster.) + +As inodes are deleted their modification items are deleted. + +> *XXX Need to figure out how to resolve multiple items created by +> concurrent writers. We want concurrent parallel writers, say, and +> they'll all way to create their own items at their write times. We'd +> need to be able to find those to delete them during future +> modification or deletion. Sort of sounds like we want +> per-node-identity backrefs for each to maintain and to purge as nodes +> leave the cluster. From b3b26939398dea9e90c6bc99bbc43f7fa4121103 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Feb 2017 08:29:45 -0800 Subject: [PATCH 237/920] Add simple debugging range locking layer We can work on shared mechanics without requiring a full locking server. We can stand up a simple layer which uses shared data structures in a kernel image to lock between mounts in the same kernel. On mount we add supers to a list. Held locks are tracked in a rbtree. A lock attempt blocks until it doesn't conflict with anything in the rbtree. As locks are acquired we walk all the other supers and write/invaludate any items they have which intersect with the acquired range. This is easier to implement and less efficient than caching locks after they're unlocked and implementing downconvert/blocking/revoke. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/item.c | 74 ++++++++++++ kmod/src/item.h | 6 + kmod/src/lock.c | 282 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 26 +++++ kmod/src/super.c | 5 +- kmod/src/super.h | 3 + 7 files changed, 397 insertions(+), 3 deletions(-) create mode 100644 kmod/src/lock.c create mode 100644 kmod/src/lock.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index e31924ed..83da3832 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ - ioctl.o item.o key.o manifest.o msg.o seg.o scoutfs_trace.o \ - super.o trans.o treap.o xattr.o + ioctl.o item.o key.o lock.o manifest.o msg.o seg.o \ + scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/item.c b/kmod/src/item.c index 83e1a09c..e489005f 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -24,6 +24,7 @@ #include "seg.h" #include "counters.h" #include "scoutfs_trace.h" +#include "trans.h" /* * A simple rbtree of cached items isolates the item API callers from @@ -1536,6 +1537,79 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) return 0; } +/* + * The caller wants us to write out any dirty items within the given + * range. We look for any dirty items within the range and if we find + * any we issue a sync which writes out all the dirty items. + */ +int scoutfs_item_writeback(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + bool sync = false; + int ret = 0; + + /* XXX think about racing with trans write */ + + spin_lock_irqsave(&cac->lock, flags); + + if (cac->nr_dirty_items) { + item = next_item(&cac->items, start); + if (item && !(item->dirty & ITEM_DIRTY)) + item = next_dirty(item); + if (item && scoutfs_key_compare(item->key, end) <= 0) + sync = true; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + if (sync) + ret = scoutfs_sync_fs(sb, 1); + + return ret; +} + +/* + * The caller wants us to drop any items within the range on the floor. + * They should have ensured that items in this range won't be dirty. + */ +void scoutfs_item_invalidate(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *next; + struct cached_item *item; + struct rb_node *node; + unsigned long flags; + + /* XXX think about racing with trans write */ + + spin_lock_irqsave(&cac->lock, flags); + + for (item = next_item(&cac->items, start); + item && scoutfs_key_compare(item->key, end) <= 0; + item = next) { + + /* XXX seems like this should be a helper? */ + node = rb_next(&item->node); + if (node) + next = container_of(node, struct cached_item, node); + else + next = NULL; + + WARN_ON_ONCE(item->dirty & ITEM_DIRTY); + erase_item(sb, cac, item); + } + + spin_unlock_irqrestore(&cac->lock, flags); +} + int scoutfs_item_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/item.h b/kmod/src/item.h index bd3e3f73..fced5957 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -57,6 +57,12 @@ bool scoutfs_item_has_dirty(struct super_block *sb); bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, u32 key_bytes, u32 val_bytes); int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); +int scoutfs_item_writeback(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end); +void scoutfs_item_invalidate(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end); int scoutfs_item_setup(struct super_block *sb); void scoutfs_item_destroy(struct super_block *sb); diff --git a/kmod/src/lock.c b/kmod/src/lock.c new file mode 100644 index 00000000..62f09129 --- /dev/null +++ b/kmod/src/lock.c @@ -0,0 +1,282 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "lock.h" +#include "item.h" +#include "scoutfs_trace.h" + +/* + * This is meant to be simple and correct, not performant. + */ + +static DECLARE_RWSEM(global_rwsem); +static LIST_HEAD(global_super_list); + +/* + * Allocated once and pointed to by the lock info of all the supers with + * the same fsid. Freed as the last super unmounts. + */ +struct held_locks { + spinlock_t lock; + struct list_head list; + wait_queue_head_t waitq; +}; + + +/* + * allocated per-super. Stored in the global list for finding supers + * with fsids and stored in a list with others with the same fsid for + * invalidation. Freed on unmount. + */ +struct lock_info { + struct super_block *sb; + struct held_locks *held; + struct list_head id_head; + struct list_head global_head; +}; + +#define DECLARE_LOCK_INFO(sb, name) \ + struct lock_info *name = SCOUTFS_SB(sb)->lock_info + +/* + * locks are compatible if they're from the same super, or are both reads, + * or don't overlap. + */ +static bool compatible_locks(struct scoutfs_lock *a, struct scoutfs_lock *b) +{ + return a->sb == b->sb || + (a->mode == SCOUTFS_LOCK_MODE_READ && + b->mode == SCOUTFS_LOCK_MODE_READ) || + scoutfs_key_compare_ranges(a->start, a->end, b->start, b->end); +} + +static bool lock_added(struct held_locks *held, struct scoutfs_lock *add) +{ + struct scoutfs_lock *lck; + bool added = true; + + spin_lock(&held->lock); + + list_for_each_entry(lck, &held->list, head) { + if (!compatible_locks(lck, add)) { + added = false; + break; + } + } + + if (added) + list_add(&add->head, &held->list); + + spin_unlock(&held->lock); + + return added; +} + +/* + * Invalidate caches on this super because another super has acquired + * a lock with the given mode and range. We always have to write out + * dirty overlapping items. If they're writing then we need to also + * invalidate all cached overlapping structures. + */ +static int invalidate_caches(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + int ret; + + ret = scoutfs_item_writeback(sb, start, end); + if (ret) + return ret; + + if (mode == SCOUTFS_LOCK_MODE_WRITE) { + scoutfs_item_invalidate(sb, start, end); +#if 0 + scoutfs_dir_invalidate(sb, start, end) ?: + scoutfs_inode_invalidate(sb, start, end) ?: + scoutfs_data_invalidate(sb, start, end); +#endif + } + + return 0; +} + +#define for_each_other_linf(linf, from_linf) \ + for (linf = list_entry(from_linf->id_head.next, struct lock_info, \ + id_head); \ + linf != from_linf; \ + linf = list_entry(linf->id_head.next, struct lock_info, \ + id_head)) + +static int invalidate_others(struct super_block *from, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + DECLARE_LOCK_INFO(from, from_linf); + struct lock_info *linf; + int ret; + + down_read(&global_rwsem); + + for_each_other_linf(linf, from_linf) { + ret = invalidate_caches(linf->sb, mode, start, end); + if (ret) + break; + } + + up_read(&global_rwsem); + + return ret; +} + +static void unlock(struct held_locks *held, struct scoutfs_lock *lck) +{ + spin_lock(&held->lock); + list_del_init(&lck->head); + spin_unlock(&held->lock); + + wake_up(&held->waitq); +} + +/* + * Acquire a coherent lock on the given range of keys. While the lock + * is held other lockers are serialized. Cache coherency is maintained + * by the locking infrastructure. Lock acquisition causes writeout from + * or invalidation of other caches. + * + * The caller provides the opaque lock structure used for storage and + * their start and end pointers will be accessed while the lock is held. + */ +int scoutfs_lock_range(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_lock *lck) +{ + DECLARE_LOCK_INFO(sb, linf); + struct held_locks *held = linf->held; + int ret; + + INIT_LIST_HEAD(&lck->head); + lck->sb = sb; + lck->start = start; + lck->end = end; + lck->mode = mode; + + ret = wait_event_interruptible(held->waitq, lock_added(held, lck)); + if (ret == 0) { + ret = invalidate_others(sb, mode, start, end); + if (ret) + unlock(held, lck); + } + + return ret; +} + +void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) +{ + DECLARE_LOCK_INFO(sb, linf); + struct held_locks *held = linf->held; + + unlock(held, lck); +} + +/* + * The moment this is done we can have other mounts start asking + * us to write back and invalidate, so do this very very late. + */ +int scoutfs_lock_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_sb_info *other_sbi; + struct lock_info *other_linf; + struct held_locks *held; + struct lock_info *linf; + + linf = kmalloc(sizeof(struct lock_info), GFP_KERNEL); + if (!linf) + return -ENOMEM; + + held = kmalloc(sizeof(struct held_locks), GFP_KERNEL); + if (!held) { + kfree(linf); + return -ENOMEM; + } + + spin_lock_init(&held->lock); + INIT_LIST_HEAD(&held->list); + init_waitqueue_head(&held->waitq); + + linf->sb = sb; + linf->held = held; + INIT_LIST_HEAD(&linf->id_head); + INIT_LIST_HEAD(&linf->global_head); + + sbi->lock_info = linf; + + trace_printk("sb %p id %016llx allocated linf %p held %p\n", + sb, le64_to_cpu(sbi->super.id), linf, held); + + down_write(&global_rwsem); + + list_for_each_entry(other_linf, &global_super_list, global_head) { + other_sbi = SCOUTFS_SB(other_linf->sb); + if (other_sbi->super.id == sbi->super.id) { + list_add(&linf->id_head, &other_linf->id_head); + linf->held = other_linf->held; + trace_printk("sharing held %p\n", linf->held); + break; + } + } + + /* add to global list after walking so we don't see ourselves */ + list_add(&linf->global_head, &global_super_list); + + up_write(&global_rwsem); + + if (linf->held != held) + kfree(held); + + return 0; +} + +void scoutfs_lock_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_LOCK_INFO(sb, linf); + struct held_locks *held; + + if (linf) { + down_write(&global_rwsem); + + list_del_init(&linf->global_head); + + if (!list_empty(&linf->id_head)) { + list_del_init(&linf->id_head); + held = NULL; + } else { + held = linf->held; + } + + up_write(&global_rwsem); + + trace_printk("sb %p id %016llx freeing linf %p held %p\n", + sb, le64_to_cpu(sbi->super.id), linf, held); + + kfree(held); + kfree(linf); + } +} diff --git a/kmod/src/lock.h b/kmod/src/lock.h new file mode 100644 index 00000000..1f7d2681 --- /dev/null +++ b/kmod/src/lock.h @@ -0,0 +1,26 @@ +#ifndef _SCOUTFS_LOCK_H_ +#define _SCOUTFS_LOCK_H_ + +struct scoutfs_lock { + struct list_head head; + struct super_block *sb; + struct scoutfs_key_buf *start; + struct scoutfs_key_buf *end; + int mode; +}; + +enum { + SCOUTFS_LOCK_MODE_READ, + SCOUTFS_LOCK_MODE_WRITE, +}; + +int scoutfs_lock_range(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_lock *lck); +void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck); + +int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 158052c1..4aa4d28f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -35,6 +35,7 @@ #include "treap.h" #include "compact.h" #include "data.h" +#include "lock.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -219,7 +220,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_treap_setup(sb) ?: // scoutfs_buddy_setup(sb) ?: scoutfs_compact_setup(sb) ?: - scoutfs_setup_trans(sb); + scoutfs_setup_trans(sb) ?: + scoutfs_lock_setup(sb); if (ret) return ret; @@ -250,6 +252,7 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_lock_destroy(sb); scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_data_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index e791e76d..5f1b468d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -13,6 +13,7 @@ struct segment_cache; struct treap_info; struct compact_info; struct data_info; +struct lock_info; struct scoutfs_sb_info { struct super_block *sb; @@ -40,6 +41,8 @@ struct scoutfs_sb_info { wait_queue_head_t trans_write_wq; struct workqueue_struct *trans_write_workq; + struct lock_info *lock_info; + /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; From 607eff9b7cb03edeaca42e0f38a4c0bf1567fc0d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Feb 2017 08:49:47 -0800 Subject: [PATCH 238/920] Add range locking to xattr ops We can use easy xattrs to test range locking and item consistency between mounts. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index afe1cc14..c3d10c2c 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -23,6 +23,7 @@ #include "item.h" #include "trans.h" #include "xattr.h" +#include "lock.h" /* * In the simple case an xattr is stored in a single item whose key and @@ -149,7 +150,9 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct scoutfs_xattr_val_header vh; struct scoutfs_key_buf *key = NULL; + struct scoutfs_key_buf *last = NULL; SCOUTFS_DECLARE_KVEC(val); + struct scoutfs_lock lck; unsigned int total; unsigned int bytes; unsigned int off; @@ -169,8 +172,15 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, return SCOUTFS_XATTR_MAX_SIZE; key = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0); - if (!key) - return -ENOMEM; + last = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0xff); + if (!key || !last) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_READ, key, last, &lck); + if (ret) + goto out; down_read(&si->xattr_rwsem); @@ -219,8 +229,11 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = -ERANGE; up_read(&si->xattr_rwsem); - scoutfs_key_free(sb, key); + scoutfs_unlock_range(sb, &lck); +out: + scoutfs_key_free(sb, key); + scoutfs_key_free(sb, last); return ret; } @@ -250,6 +263,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); + struct scoutfs_lock lck; unsigned int bytes; unsigned int off; LIST_HEAD(list); @@ -274,6 +288,10 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } + ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, key, last, &lck); + if (ret) + goto out; + /* build up batch of new items for the new xattr */ if (value) { for_each_xattr_item(key, val, &vh, (void *)value, size, @@ -281,7 +299,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, ret = scoutfs_item_add_batch(sb, &list, key, val); if (ret) - goto out; + goto unlock; } } @@ -299,7 +317,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, ret = scoutfs_hold_trans(sb); if (ret) - goto out; + goto unlock; down_write(&si->xattr_rwsem); @@ -315,6 +333,9 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, up_write(&si->xattr_rwsem); scoutfs_release_trans(sb); +unlock: + scoutfs_unlock_range(sb, &lck); + out: scoutfs_item_free_batch(sb, &list); scoutfs_key_free(sb, key); @@ -346,6 +367,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct scoutfs_xattr_key *xkey; struct scoutfs_key_buf *key; struct scoutfs_key_buf *last; + struct scoutfs_lock lck; ssize_t total; int name_len; int ret; @@ -362,6 +384,10 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) xkey = key->data; xkey->name[0] = '\0'; + ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_READ, key, last, &lck); + if (ret) + goto out; + down_read(&si->xattr_rwsem); total = 0; @@ -408,6 +434,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) } up_read(&si->xattr_rwsem); + scoutfs_unlock_range(sb, &lck); out: scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); @@ -428,6 +455,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { struct scoutfs_key_buf *key; struct scoutfs_key_buf *last; + struct scoutfs_lock lck; int ret; key = alloc_xattr_key(sb, ino, NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); @@ -438,6 +466,11 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) goto out; } + /* while we read to delete we need to writeback others */ + ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, key, last, &lck); + if (ret) + goto out; + /* the inode is dead so we don't need the xattr sem */ for (;;) { @@ -455,6 +488,8 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* don't need to increment past deleted key */ } + scoutfs_unlock_range(sb, &lck); + out: scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); From 955d940c647b9ef5d33a0ecded6443a0942abcfe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Feb 2017 13:10:27 -0800 Subject: [PATCH 239/920] Restore key tracing Now that the keys are a contiguous buffer we can format them for the trace buffers with a much more straight forward type check around per-key snprintfs. We can get rid of all the weird kvec code that tried to deal with keys that straddled vectors. With that fixed we can uncomment out the tracing statements that were waiting the key formatting. I was testing with xattr keys so they're added as the code is updated. The rest of the key types will be added seperately as they're used. Signed-off-by: Zach Brown --- kmod/src/item.c | 4 +- kmod/src/key.c | 67 ++++++++++++++++++ kmod/src/key.h | 2 + kmod/src/kvec.c | 148 --------------------------------------- kmod/src/kvec.h | 2 - kmod/src/manifest.c | 2 +- kmod/src/scoutfs_trace.h | 30 ++++---- 7 files changed, 88 insertions(+), 167 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index e489005f..606c80e9 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -539,7 +539,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; -// trace_scoutfs_item_lookup(sb, key, val); + trace_scoutfs_item_lookup(sb, key, val); end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); if (!end) { @@ -944,7 +944,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, unsigned long flags; int ret; -// trace_scoutfs_item_insert_batch(sb, start, end); + trace_scoutfs_item_insert_batch(sb, start, end); if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) return -EINVAL; diff --git a/kmod/src/key.c b/kmod/src/key.c index 04891839..abcfac9f 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -106,3 +106,70 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) extend_zeros(key); scoutfs_key_dec_cur_len(key); } + +/* return the bytes of the string including the null term */ +#define snprintf_null(buf, size, fmt, args...) \ + (snprintf((buf), (size), fmt, ##args) + 1) + +/* + * Write the null-terminated string that describes the key to the + * buffer. The bytes copied (including the null) is returned. A null + * buffer can be used to find the string size without writing anything. + * + * XXX nonprintable characters in the trace? + */ +int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) +{ + size_t size = buf ? INT_MAX : 0; + int len; + u8 type; + + if (key->key_len == 0) + return snprintf_null(buf, size, "[0 len]"); + + type = *(u8 *)key->data; + + switch(type) { + + case SCOUTFS_INODE_KEY: { + struct scoutfs_inode_key *ikey = key->data; + + if (key->key_len < sizeof(struct scoutfs_inode_key)) + break; + + return snprintf_null(buf, size, "ino.%llu", + be64_to_cpu(ikey->ino)); + } + + case SCOUTFS_XATTR_KEY: { + struct scoutfs_xattr_key *xkey = key->data; + + len = (int)key->key_len - offsetof(struct scoutfs_xattr_key, + name[1]); + if (len <= 0) + break; + + return snprintf_null(buf, size, "xat.%llu.%.*s", + be64_to_cpu(xkey->ino), len, xkey->name); + } + + case SCOUTFS_DIRENT_KEY: { + struct scoutfs_dirent_key *dkey = key->data; + + len = (int)key->key_len - offsetof(struct scoutfs_dirent_key, + name[1]); + if (len <= 0) + break; + + return snprintf_null(buf, size, "dnt.%llu.%.*s", + be64_to_cpu(dkey->ino), len, dkey->name); + } + + default: + return snprintf_null(buf, size, "[unknown type %u len %u]", + type, key->key_len); + } + + return snprintf_null(buf, size, "[truncated type %u len %u]", + type, key->key_len); +} diff --git a/kmod/src/key.h b/kmod/src/key.h index 3c108555..26f4b499 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -19,6 +19,8 @@ void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key); void scoutfs_key_dec(struct scoutfs_key_buf *key); void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); +int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key); + /* * Initialize a small key in a larger allocated buffer. This lets * callers, for example, search for a small key and get a larger key diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index 422a4fc5..ca4bcf22 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -76,12 +76,6 @@ static size_t iter_contig(struct iter *iter) return 0; } -/* count of bytes remaining in the iteration */ -static size_t iter_count(struct iter *iter) -{ - return iter->count; -} - /* * Return the result of memcmp between the min of the two total lengths. * If their shorter lengths are equal than the shorter length is considered @@ -296,145 +290,3 @@ void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src) scoutfs_kvec_memcmp(src, dst) < 0) scoutfs_kvec_clone(dst, src); } - -/* - * Copy bytes from the kvec iterator into the dest buffer, zeroing the - * remainder of the buffer if there aren't enough bytes available in - * the iterator. If the tail bool is set then the kvec data is copied - * into the tail of the buffer and the head is zeroed. - */ -static bool iter_memcpy_zero(void *dst, struct iter *src, size_t len, bool tail) -{ - size_t ctg; - size_t diff; - - if (len == 0 || iter_count(src) == 0) - return false; - - if (iter_count(src) < len) { - diff = len - iter_count(src); - if (tail) { - memset(dst, 0, diff); - dst += diff; - } else { - memset(dst + len - diff, 0, diff); - } - len = iter_count(src); - } - - while ((ctg = min(len, iter_contig(src)))) { - memcpy(dst, iter_ptr(src), ctg); - iter_advance(src, ctg); - dst += ctg; - len -= ctg; - } - - return true; -} - -static int iter_puts_printable(char *dst, struct iter *src) -{ - int len = iter_count(src); - size_t ctg; - int i; - - while ((ctg = iter_contig(src))) { - memcpy(dst, iter_ptr(src), ctg); - iter_advance(src, ctg); - - for (i = 0; i < ctg; i++) { - if (!isprint(dst[i])) - dst[i] = '_'; - } - - dst += ctg; - } - - return len; -} - -#define EMPTY_STR "''" -#define U64_U_BYTES 20 -#define U64_D_BYTES 21 -#define U64_X_BYTES 16 - -/* - * XXX figure out what to do about corrupt keys. - */ - -unsigned scoutfs_kvec_key_strlen(struct kvec *key) -{ - struct iter iter; - unsigned len = 0; - u8 type; - - iter_init(&iter, key); - - if (iter_count(&iter) == 0) { - len = sizeof(EMPTY_STR) - 1; - goto out; - } - - iter_memcpy_zero(&type, &iter, sizeof(type), false); - - len = 4; /* "typ." */ - - switch(type) { - case SCOUTFS_INODE_KEY: - len += U64_U_BYTES; - break; - case SCOUTFS_DIRENT_KEY: - len += U64_U_BYTES + (iter_count(&iter) - 8); - break; - case SCOUTFS_MAX_UNUSED_KEY: - break; - default: - /* hex of everything after the type */ - len += (scoutfs_kvec_length(key) - 1) * 2; - break; - } - -out: - return len + 1; /* null term */ -} - -void scoutfs_kvec_key_sprintf(char *buf, struct kvec *key) -{ - struct iter iter; - __be64 be; - u8 type; - - iter_init(&iter, key); - - if (iter_contig(&iter) == 0) { - buf += sprintf(buf, EMPTY_STR); - goto done; - } - - iter_memcpy_zero(&type, &iter, sizeof(type), false); - - switch(type) { - case SCOUTFS_INODE_KEY: - buf += sprintf(buf, "ino."); - iter_memcpy_zero(&be, &iter, sizeof(be), false); - buf += sprintf(buf, "%llu", be64_to_cpu(be)); - break; - case SCOUTFS_DIRENT_KEY: - buf += sprintf(buf, "den."); - iter_memcpy_zero(&be, &iter, sizeof(be), false); - buf += sprintf(buf, "%llu.", be64_to_cpu(be)); - buf += iter_puts_printable(buf, &iter); - break; - case SCOUTFS_MAX_UNUSED_KEY: - buf += sprintf(buf, "max"); - break; - default: - buf += sprintf(buf, "unk."); - while (iter_memcpy_zero(&be, &iter, sizeof(be), true)) - buf += sprintf(buf, "%llx", be64_to_cpu(be)); - break; - } - -done: - *buf = '\0'; -} diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index 444a8116..c078e802 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -66,8 +66,6 @@ int scoutfs_kvec_alloc_key(struct kvec *kvec); void scoutfs_kvec_init_key(struct kvec *kvec); void scoutfs_kvec_set_max_key(struct kvec *kvec); void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src); -unsigned scoutfs_kvec_key_strlen(struct kvec *key); -void scoutfs_kvec_key_sprintf(char *buf, struct kvec *key); void scoutfs_kvec_be_inc(struct kvec *kvec); void scoutfs_kvec_be_dec(struct kvec *kvec); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index f8c4edb5..fd2aeda8 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -194,7 +194,7 @@ int scoutfs_manifest_add(struct super_block *sb, unsigned bytes; int ret; -// trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); + trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); key_bytes = first->key_len + last->key_len; bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 669b99f4..6b5a828c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -181,19 +181,19 @@ TRACE_EVENT(scoutfs_scan_orphans, ); TRACE_EVENT(scoutfs_manifest_add, - TP_PROTO(struct super_block *sb, struct kvec *first, - struct kvec *last, u64 segno, u64 seq, u8 level), + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, u8 level), TP_ARGS(sb, first, last, segno, seq, level), TP_STRUCT__entry( - __dynamic_array(char, first, scoutfs_kvec_key_strlen(first)) - __dynamic_array(char, last, scoutfs_kvec_key_strlen(last)) + __dynamic_array(char, first, scoutfs_key_str(NULL, first)) + __dynamic_array(char, last, scoutfs_key_str(NULL, last)) __field(u64, segno) __field(u64, seq) __field(u8, level) ), TP_fast_assign( - scoutfs_kvec_key_sprintf(__get_dynamic_array(first), first); - scoutfs_kvec_key_sprintf(__get_dynamic_array(last), last); + scoutfs_key_str(__get_dynamic_array(first), first); + scoutfs_key_str(__get_dynamic_array(last), last); __entry->segno = segno; __entry->seq = seq; __entry->level = level; @@ -204,27 +204,29 @@ TRACE_EVENT(scoutfs_manifest_add, ); TRACE_EVENT(scoutfs_item_lookup, - TP_PROTO(struct super_block *sb, struct kvec *key, struct kvec *val), + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key, + struct kvec *val), TP_ARGS(sb, key, val), TP_STRUCT__entry( - __dynamic_array(char, key, scoutfs_kvec_key_strlen(key)) + __dynamic_array(char, key, scoutfs_key_str(NULL, key)) ), TP_fast_assign( - scoutfs_kvec_key_sprintf(__get_dynamic_array(key), key); + scoutfs_key_str(__get_dynamic_array(key), key); ), TP_printk("key %s", __get_str(key)) ); TRACE_EVENT(scoutfs_item_insert_batch, - TP_PROTO(struct super_block *sb, struct kvec *start, struct kvec *end), + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), TP_ARGS(sb, start, end), TP_STRUCT__entry( - __dynamic_array(char, start, scoutfs_kvec_key_strlen(start)) - __dynamic_array(char, end, scoutfs_kvec_key_strlen(end)) + __dynamic_array(char, start, scoutfs_key_str(NULL, start)) + __dynamic_array(char, end, scoutfs_key_str(NULL, end)) ), TP_fast_assign( - scoutfs_kvec_key_sprintf(__get_dynamic_array(start), start); - scoutfs_kvec_key_sprintf(__get_dynamic_array(end), end); + scoutfs_key_str(__get_dynamic_array(start), start); + scoutfs_key_str(__get_dynamic_array(end), end); ), TP_printk("start %s end %s", __get_str(start), __get_str(end)) ); From 392ed81c43a95e4d2db88cce1672f2f51fe8a12b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Feb 2017 15:40:45 -0800 Subject: [PATCH 240/920] Add some simple lock/invalidation tracing Signed-off-by: Zach Brown --- kmod/src/lock.c | 6 +++++ kmod/src/scoutfs_trace.h | 54 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 62f09129..f9eae7f2 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -99,6 +99,8 @@ static int invalidate_caches(struct super_block *sb, int mode, { int ret; + trace_scoutfs_lock_invalidate_sb(sb, mode, start, end); + ret = scoutfs_item_writeback(sb, start, end); if (ret) return ret; @@ -176,6 +178,8 @@ int scoutfs_lock_range(struct super_block *sb, int mode, lck->end = end; lck->mode = mode; + trace_scoutfs_lock_range(sb, lck); + ret = wait_event_interruptible(held->waitq, lock_added(held, lck)); if (ret == 0) { ret = invalidate_others(sb, mode, start, end); @@ -191,6 +195,8 @@ void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) DECLARE_LOCK_INFO(sb, linf); struct held_locks *held = linf->held; + trace_scoutfs_unlock_range(sb, lck); + unlock(held, lck); } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 6b5a828c..ae1c4c16 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -28,6 +28,7 @@ #include "key.h" #include "format.h" #include "kvec.h" +#include "lock.h" struct scoutfs_sb_info; @@ -231,6 +232,59 @@ TRACE_EVENT(scoutfs_item_insert_batch, TP_printk("start %s end %s", __get_str(start), __get_str(end)) ); +#define lock_mode(mode) \ + __print_symbolic(mode, \ + { SCOUTFS_LOCK_MODE_READ, "READ" }, \ + { SCOUTFS_LOCK_MODE_WRITE, "WRITE" }) + +DECLARE_EVENT_CLASS(scoutfs_lock_class, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck), + TP_STRUCT__entry( + __field(int, mode) + __dynamic_array(char, start, scoutfs_key_str(NULL, lck->start)) + __dynamic_array(char, end, scoutfs_key_str(NULL, lck->end)) + ), + TP_fast_assign( + __entry->mode = lck->mode; + scoutfs_key_str(__get_dynamic_array(start), lck->start); + scoutfs_key_str(__get_dynamic_array(end), lck->end); + ), + TP_printk("mode %s start %s end %s", + lock_mode(__entry->mode), __get_str(start), __get_str(end)) +); + +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_range, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock_range, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +TRACE_EVENT(scoutfs_lock_invalidate_sb, + TP_PROTO(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, mode, start, end), + TP_STRUCT__entry( + __field(void *, sb) + __field(int, mode) + __dynamic_array(char, start, scoutfs_key_str(NULL, start)) + __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->mode = mode; + scoutfs_key_str(__get_dynamic_array(start), start); + scoutfs_key_str(__get_dynamic_array(end), end); + ), + TP_printk("sb %p mode %s start %s end %s", + __entry->sb, lock_mode(__entry->mode), + __get_str(start), __get_str(end)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 39ae89d85fd9a138da4ccc06fe435a20d069705e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Mar 2017 10:39:53 -0800 Subject: [PATCH 241/920] Add network messaging between mounts We're going to need communication between mounts to update and distribute the manifest and allocators in the treap ring. This adds a netwoking core where one mount becomes the server and other mounts send requests to it. The messaging semantics are pretty simple in that clients reliably send requests and the server passively reply to requests. Complexity beyond that is up to the callers implementing the requests. It relies on locking to establish the server role and to broadcast the address of the server socket. We add a trivial lvb back to our local test locking implementation to store the address. We also add the ability to shut down locking so that the locking networking work stops blocking. A little demonstration request is included which just gives visibility into client and server clocks in the trace logs. Next up we'll add the requests that do real work. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/format.h | 43 +- kmod/src/lock.c | 130 ++++- kmod/src/lock.h | 6 + kmod/src/net.c | 1177 +++++++++++++++++++++++++++++++++++++++++++++ kmod/src/net.h | 9 + kmod/src/super.c | 8 +- kmod/src/super.h | 2 + 8 files changed, 1351 insertions(+), 26 deletions(-) create mode 100644 kmod/src/net.c create mode 100644 kmod/src/net.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 83da3832..1448caa6 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ - ioctl.o item.o key.o lock.o manifest.o msg.o seg.o \ + ioctl.o item.o key.o lock.o manifest.o msg.o net.o seg.o \ scoutfs_trace.o super.o trans.o treap.o xattr.o diff --git a/kmod/src/format.h b/kmod/src/format.h index b877d4d8..0fbe3c40 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -172,7 +172,10 @@ struct scoutfs_segment_block { #define SCOUTFS_EXTENT_KEY 9 #define SCOUTFS_ORPHAN_KEY 10 #define SCOUTFS_DATA_KEY 11 -#define SCOUTFS_MAX_UNUSED_KEY 255 +/* not found in the fs */ +#define SCOUTFS_MAX_UNUSED_KEY 253 +#define SCOUTFS_NET_ADDR_KEY 254 +#define SCOUTFS_NET_LISTEN_KEY 255 /* value is struct scoutfs_inode */ struct scoutfs_inode_key { @@ -348,4 +351,42 @@ enum { #define SCOUTFS_MAX_KEY_SIZE \ offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) +/* + * messages over the wire. + */ + +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; +} __packed; + +/* + * This header precedes and describes all network messages sent over + * sockets. The id is set by the request and sent in the reply. The + * type is strictly redundant in the reply because the id will find the + * send but we include it in both packets to make it easier to observe + * replies without having the id from their previous request. + */ +struct scoutfs_net_header { + __le64 id; + __le16 data_len; + __u8 type; + __u8 status; + __u8 data[0]; +}; + +enum { + /* sends and receives a struct scoutfs_timeval */ + SCOUTFS_NET_TRADE_TIME = 0, + SCOUTFS_NET_UNKNOWN, +}; + +enum { + SCOUTFS_NET_STATUS_REQUEST = 0, + SCOUTFS_NET_STATUS_SUCCESS, + SCOUTFS_NET_STATUS_ERROR, + SCOUTFS_NET_STATUS_UNKNOWN, +}; + #endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c index f9eae7f2..876d7690 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -35,8 +35,13 @@ struct held_locks { spinlock_t lock; struct list_head list; wait_queue_head_t waitq; -}; + /* super hacky fake lvb that only allows one specific key */ + char fake_lvb[sizeof(struct scoutfs_inet_addr)]; + struct scoutfs_key_buf fake_lvb_key; + char fake_lvb_key_data[SCOUTFS_MAX_KEY_SIZE]; + +}; /* * allocated per-super. Stored in the global list for finding supers @@ -45,6 +50,7 @@ struct held_locks { */ struct lock_info { struct super_block *sb; + bool shutdown; struct held_locks *held; struct list_head id_head; struct list_head global_head; @@ -65,13 +71,20 @@ static bool compatible_locks(struct scoutfs_lock *a, struct scoutfs_lock *b) scoutfs_key_compare_ranges(a->start, a->end, b->start, b->end); } -static bool lock_added(struct held_locks *held, struct scoutfs_lock *add) +/* also returns true if we're shutting down, caller tests after waiting */ +static bool lock_added(struct lock_info *linf, struct scoutfs_lock *add) { + struct held_locks *held = linf->held; struct scoutfs_lock *lck; bool added = true; spin_lock(&held->lock); + if (linf->shutdown) { + added = true; + goto out; + } + list_for_each_entry(lck, &held->list, head) { if (!compatible_locks(lck, add)) { added = false; @@ -82,6 +95,7 @@ static bool lock_added(struct held_locks *held, struct scoutfs_lock *add) if (added) list_add(&add->head, &held->list); +out: spin_unlock(&held->lock); return added; @@ -154,6 +168,74 @@ static void unlock(struct held_locks *held, struct scoutfs_lock *lck) wake_up(&held->waitq); } +static void assert_fake_lvb(struct held_locks *held, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, unsigned lvb_len) +{ + + BUG_ON(scoutfs_key_compare(start, end)); + BUG_ON(lvb_len != sizeof(held->fake_lvb)); + BUG_ON(held->fake_lvb_key.key_len && + scoutfs_key_compare(&held->fake_lvb_key, start)); +} + +/* + * Acquire a coherent lock on the given range of keys. While the lock + * is held other lockers are serialized. Cache coherency is maintained + * by the locking infrastructure. Lock acquisition causes writeout from + * or invalidation of other caches. + * + * The caller provides the opaque lock structure used for storage and + * their start and end pointers will be accessed while the lock is held. + */ +int scoutfs_lock_range_lvb(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + void *caller_lvb, unsigned lvb_len, + struct scoutfs_lock *lck) +{ + DECLARE_LOCK_INFO(sb, linf); + struct held_locks *held = linf->held; + int ret; + + INIT_LIST_HEAD(&lck->head); + lck->sb = sb; + lck->start = start; + lck->end = end; + lck->mode = mode; + + trace_scoutfs_lock_range(sb, lck); + + ret = wait_event_interruptible(held->waitq, lock_added(linf, lck)); + if (ret) + goto out; + + if (linf->shutdown) { + /* unlocked, but we own it */ + if (!list_empty(&lck->head)) + unlock(held, lck); + ret = -ESHUTDOWN; + goto out; + } + + ret = invalidate_others(sb, mode, start, end); + if (ret) + goto out; + + if (caller_lvb) { + assert_fake_lvb(held, start, end, lvb_len); + if (mode == SCOUTFS_LOCK_MODE_WRITE) { + memcpy(held->fake_lvb, caller_lvb, lvb_len); + scoutfs_key_copy(&held->fake_lvb_key, start); + } else { + memcpy(caller_lvb, held->fake_lvb, lvb_len); + } + } + +out: + return ret; +} + /* * Acquire a coherent lock on the given range of keys. While the lock * is held other lockers are serialized. Cache coherency is maintained @@ -168,26 +250,7 @@ int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *end, struct scoutfs_lock *lck) { - DECLARE_LOCK_INFO(sb, linf); - struct held_locks *held = linf->held; - int ret; - - INIT_LIST_HEAD(&lck->head); - lck->sb = sb; - lck->start = start; - lck->end = end; - lck->mode = mode; - - trace_scoutfs_lock_range(sb, lck); - - ret = wait_event_interruptible(held->waitq, lock_added(held, lck)); - if (ret == 0) { - ret = invalidate_others(sb, mode, start, end); - if (ret) - unlock(held, lck); - } - - return ret; + return scoutfs_lock_range_lvb(sb, mode, start, end, NULL, 0, lck); } void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) @@ -216,7 +279,7 @@ int scoutfs_lock_setup(struct super_block *sb) if (!linf) return -ENOMEM; - held = kmalloc(sizeof(struct held_locks), GFP_KERNEL); + held = kzalloc(sizeof(struct held_locks), GFP_KERNEL); if (!held) { kfree(linf); return -ENOMEM; @@ -225,8 +288,11 @@ int scoutfs_lock_setup(struct super_block *sb) spin_lock_init(&held->lock); INIT_LIST_HEAD(&held->list); init_waitqueue_head(&held->waitq); + scoutfs_key_init_buf_len(&held->fake_lvb_key, &held->fake_lvb_key_data, + 0, sizeof(held->fake_lvb_key_data)); linf->sb = sb; + linf->shutdown = false; linf->held = held; INIT_LIST_HEAD(&linf->id_head); INIT_LIST_HEAD(&linf->global_head); @@ -259,6 +325,24 @@ int scoutfs_lock_setup(struct super_block *sb) return 0; } +/* + * Cause all lock attempts from our super to fail, waking anyone who is + * currently blocked attempting to lock. Now that locks can't block we + * can easily tear down subsystems that use locking before freeing lock + * infrastructure. + */ +void scoutfs_lock_shutdown(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linf); + struct held_locks *held = linf->held; + + spin_lock(&held->lock); + linf->shutdown = true; + spin_unlock(&held->lock); + + wake_up(&held->waitq); +} + void scoutfs_lock_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 1f7d2681..fe011988 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -18,9 +18,15 @@ int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, struct scoutfs_lock *lck); +int scoutfs_lock_range_lvb(struct super_block *sb, int mode, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + void *caller_lvb, unsigned lvb_len, + struct scoutfs_lock *lck); void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck); int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); #endif diff --git a/kmod/src/net.c b/kmod/src/net.c new file mode 100644 index 00000000..a6ecdde2 --- /dev/null +++ b/kmod/src/net.c @@ -0,0 +1,1177 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "net.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * scoutfs mounts use a simple client-server model to send and process + * requests to maintain consistency with lighter overhead than full + * locking. + * + * All mounts try to establish themselves as a server. They try to + * acquire an exclusive lock that allows them to act as the server. + * While they hold that lock they broadcast their listening address with + * an address lock's lvb. The server only accepts client connections, + * processes requests, and sends replies. It never sends requests to + * clients. The client is responsible for reliability and forward + * progress. + * + * All mounts must connect to the server to function. They sample the + * address lock's lvb to find an address and try to connect to it. + * Callers enqueue reqeust messages with a reply function. The requests + * are sent down each re-established connection to the server. If the + * client receives a reply it frees the request and calls the reply + * function. + * + * All kernel socket calls are non-blocking and made from work functions + * in a single threaded workqueue. This makes it easy to stop all work + * on the socket before shutting it down. + * + * XXX: + * - include mount id in the workqueue names? + * - set recv buf size to multiple of largest message size + */ + +struct net_info { + struct super_block *sb; + + /* protects lists and sock info pointers */ + struct mutex mutex; + + /* client connects and sends requests */ + struct delayed_work client_work; + struct sock_info *connected_sinf; + struct list_head to_send; + u64 next_id; + + /* server listens and processes requests */ + struct delayed_work server_work; + struct sock_info *listening_sinf; + + /* both track active sockets for destruction */ + struct list_head active_socks; + + /* non-blocking sock work is serialized, one at a time */ + struct workqueue_struct *sock_wq; + /* processing is unlimited and concurrent but each is non-reentrant */ + struct workqueue_struct *proc_wq; +}; + +#define DECLARE_NET_INFO(sb, name) \ + struct net_info *name = SCOUTFS_SB(sb)->net_info + +typedef int (*reply_func_t)(struct super_block *sb, void *recv, int bytes); + +/* + * Send buffers are allocated either by clients who send requests or by + * the server who sends replies. Request sends are freed when they get + * a reply and reply sends are freed either after they're sent or when + * their accepted client socket is shut down. + */ +struct send_buf { + struct list_head head; + reply_func_t func; + struct scoutfs_net_header nh[0]; +}; + +/* + * Receive bufs hold messages from the socket while they're being + * processed. They have embedded work so we can have easy concurrent + * processing. Their processing can block for IO. Their sending socket + * can be torn down during their processing in which case no reply is + * sent. + */ +struct recv_buf { + struct net_info *nti; + struct sock_info *sinf; + struct list_head head; + struct work_struct proc_work; + struct scoutfs_net_header nh[0]; +}; + +struct sock_info { + struct super_block *sb; + struct list_head head; + bool shutting_down; + + unsigned send_pos; + struct list_head to_send; + struct list_head have_sent; + struct list_head active_rbufs; + + struct scoutfs_lock listen_lck; + struct scoutfs_inet_addr addr; + + struct work_struct listen_work; + struct work_struct accept_work; + struct work_struct connect_work; + struct work_struct send_work; + struct work_struct recv_work; + struct work_struct shutdown_work; + + struct socket *sock; + void (*orig_state_change)(struct sock *sk); + void (*orig_data_ready)(struct sock *sk, int bytes); + void (*orig_write_space)(struct sock *sk); +}; + +/* + * XXX instead of magic keys in the main fs resource we could have + * another resource that contains the server locks. + */ +static u8 listen_type = SCOUTFS_NET_LISTEN_KEY; +static struct scoutfs_key_buf listen_key; +static u8 addr_type = SCOUTFS_NET_ADDR_KEY; +static struct scoutfs_key_buf addr_key; + +static int send_msg(struct socket *sock, void *buf, unsigned len) +{ + struct kvec kvec = { .iov_base = buf, .iov_len = len }; + struct msghdr msg = { + .msg_iov = (struct iovec *)&kvec, + .msg_iovlen = 1, + .msg_flags = MSG_NOSIGNAL | MSG_DONTWAIT, + }; + + return kernel_sendmsg(sock, &msg, &kvec, 1, len); +} + +static int recv_msg(struct socket *sock, void *buf, unsigned len, int flags) +{ + struct kvec kvec = { .iov_base = buf, .iov_len = len }; + struct msghdr msg = { + .msg_iov = (struct iovec *)&kvec, + .msg_iovlen = 1, + .msg_flags = MSG_NOSIGNAL | MSG_DONTWAIT | flags, + }; + + return kernel_recvmsg(sock, &msg, &kvec, 1, len, msg.msg_flags); +} + +/* + * Don't queue work on the socket if it's shutting down so that the + * shutdown work knows it can free the socket without work pending. + */ +static void queue_sock_work(struct sock_info *sinf, struct work_struct *work) +{ + DECLARE_NET_INFO(sinf->sb, nti); + + if (!sinf->shutting_down) + queue_work(nti->sock_wq, work); +} + +/* + * By giving all the sockets all the work funcs we can have one set of + * socket callbacks that queue the appropriate work only if the func has + * been set. + */ +static void queue_sock_work_if_func(struct sock_info *sinf, + struct work_struct *work) +{ + if (work->func) + queue_sock_work(sinf, work); +} + +/* + * This non-blocking work consumes the send queue in the socket info as + * messages are sent out. If the messages have a reply function then + * they're requests that are resent until we receive a reply. If they + * don't then they're one-off replies that we free once they're sent. + */ +static void scoutfs_net_send_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + send_work); + DECLARE_NET_INFO(sinf->sb, nti); + struct send_buf *sbuf; + struct send_buf *pos; + char *buf; + int total; + int len; + int ret = 0; + + mutex_lock(&nti->mutex); + + list_for_each_entry_safe(sbuf, pos, &sinf->to_send, head) { + total = sizeof(struct scoutfs_net_header) + + le16_to_cpu(sbuf->nh->data_len); + + buf = (char *)sbuf->nh + sinf->send_pos; + len = total - sinf->send_pos; + + ret = send_msg(sinf->sock, buf, len); + trace_printk("sinf %p sock %p send len %d ret %d\n", + sinf, sinf->sock, len, ret); + if (ret < 0) { + if (ret == -EAGAIN) + ret = 0; + break; + } + if (ret == 0 || ret > len) { + ret = -EINVAL; + break; + } + + sinf->send_pos += ret; + + if (sinf->send_pos == total) { + sinf->send_pos = 0; + list_del_init(&sbuf->head); + + if (sbuf->func) + list_add_tail(&sbuf->head, &sinf->have_sent); + else + kfree(sbuf); + } + } + + if (ret < 0) { + trace_printk("ret %d\n", ret); + queue_sock_work(sinf, &sinf->shutdown_work); + } + + mutex_unlock(&nti->mutex); +} + +static struct send_buf *alloc_sbuf(unsigned data_len) +{ + unsigned len = offsetof(struct send_buf, nh[0].data[data_len]); + struct send_buf *sbuf; + + sbuf = kmalloc(len, GFP_NOFS); + if (sbuf) { + INIT_LIST_HEAD(&sbuf->head); + sbuf->nh->data_len = cpu_to_le16(data_len); + } + + return sbuf; +} + +/* + * Log the time in the request and reply with our current time. + */ +static struct send_buf *process_trade_time(struct super_block *sb, + struct scoutfs_timespec *req, + int req_len) +{ + struct scoutfs_timespec *reply; + struct send_buf *sbuf; + struct timespec64 ts; + + if (req_len != sizeof(*req)) + return ERR_PTR(-EINVAL); + + sbuf = alloc_sbuf(sizeof(struct scoutfs_timespec)); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + getnstimeofday64(&ts); + trace_printk("req %llu.%u replying %llu.%lu\n", + le64_to_cpu(req->sec), le32_to_cpu(req->nsec), + (u64)ts.tv_sec, ts.tv_nsec); + + reply = (void *)sbuf->nh->data; + reply->sec = cpu_to_le64(ts.tv_sec); + reply->nsec = cpu_to_le32(ts.tv_nsec); + + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + +/* + * Process an incoming request and queue its reply to send if the socket + * is still open by the time we have the reply. + */ +static int process_request(struct net_info *nti, struct recv_buf *rbuf) +{ + struct super_block *sb = nti->sb; + struct send_buf *sbuf; + unsigned data_len; + + data_len = le16_to_cpu(rbuf->nh->data_len); + + if (rbuf->nh->type == SCOUTFS_NET_TRADE_TIME) + sbuf = process_trade_time(sb, (void *)rbuf->nh->data, + data_len); + else + sbuf = ERR_PTR(-EINVAL); + + if (IS_ERR(sbuf)) + return PTR_ERR(sbuf); + + /* processing sets data_len and status */ + sbuf->func = NULL; + sbuf->nh->id = rbuf->nh->id; + sbuf->nh->type = rbuf->nh->type; + + mutex_lock(&nti->mutex); + if (rbuf->sinf) { + list_add(&sbuf->head, &rbuf->sinf->to_send); + queue_sock_work(rbuf->sinf, &rbuf->sinf->send_work); + sbuf = NULL; + } + mutex_unlock(&nti->mutex); + + kfree(sbuf); + + return 0; +} + +/* + * The server only sends replies down the socket on which it receives + * the request. If we receive a reply we must have sent the request + * down the socket and the send buf will be found on the have_sent list. + */ +static int process_reply(struct net_info *nti, struct recv_buf *rbuf) +{ + struct super_block *sb = nti->sb; + reply_func_t func = NULL; + struct send_buf *sbuf; + int ret; + + mutex_lock(&nti->mutex); + + if (rbuf->sinf) { + list_for_each_entry(sbuf, &rbuf->sinf->have_sent, head) { + if (sbuf->nh->id == rbuf->nh->id) { + list_del_init(&sbuf->head); + func = sbuf->func; + kfree(sbuf); + sbuf = NULL; + break; + } + } + } + + mutex_unlock(&nti->mutex); + + if (func == NULL) + return 0; + + if (rbuf->nh->status == SCOUTFS_NET_STATUS_SUCCESS) + ret = le16_to_cpu(rbuf->nh->data_len); + else + ret = -EIO; + + return func(sb, rbuf->nh->data, ret); +} + +/* + * Process each received message in its own non-reentrant work so we get + * concurrent request processing. + */ +static void scoutfs_net_proc_func(struct work_struct *work) +{ + struct recv_buf *rbuf = container_of(work, struct recv_buf, proc_work); + struct net_info *nti = rbuf->nti; + int ret; + + if (rbuf->nh->status == SCOUTFS_NET_STATUS_REQUEST) + ret = process_request(nti, rbuf); + else + ret = process_reply(nti, rbuf); + + if (ret) + trace_printk("type %u id %llu status %u ret %d\n", + rbuf->nh->type, le64_to_cpu(rbuf->nh->id), + rbuf->nh->status, ret); + + mutex_lock(&nti->mutex); + + if (ret < 0 && rbuf->sinf) + queue_sock_work(rbuf->sinf, &rbuf->sinf->shutdown_work); + + if (!list_empty(&rbuf->head)) + list_del_init(&rbuf->head); + + mutex_unlock(&nti->mutex); + + kfree(rbuf); +} + +/* + * only accepted (not listening or connected) sockets receive requests + * and only connected sockets receive replies. This is running in the + * single threaded socket workqueue so it isn't racing with the shutdown + * work that would null the sinf pointer if it matches this sinf. + */ +static bool inappropriate_message(struct net_info *nti, struct sock_info *sinf, + struct recv_buf *rbuf) +{ + if (rbuf->nh->status == SCOUTFS_NET_STATUS_REQUEST && + (sinf == nti->listening_sinf || sinf == nti->connected_sinf)) + return true; + + if (rbuf->nh->status != SCOUTFS_NET_STATUS_REQUEST && + sinf != nti->connected_sinf) + return true; + + return false; +} + +/* + * Parse an incoming message on a socket. We peek at the socket buffer + * until it has the whole message. Then we queue request or reply + * processing work and shut down the socket if anything weird happens. + */ +static void scoutfs_net_recv_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + recv_work); + DECLARE_NET_INFO(sinf->sb, nti); + struct scoutfs_net_header nh; + struct recv_buf *rbuf; + int len; + int inq; + int ret; + + for (;;) { + /* peek to see data_len in the header */ + ret = recv_msg(sinf->sock, &nh, sizeof(nh), MSG_PEEK); + trace_printk("sinf %p sock %p peek ret %d\n", + sinf, sinf->sock, ret); + if (ret != sizeof(nh)) { + if (ret > 0 || ret == -EAGAIN) + ret = 0; + else if (ret == 0) + ret = -EIO; + break; + } + + /* XXX verify data_len isn't insane */ + + len = sizeof(struct scoutfs_net_header) + + le16_to_cpu(nh.data_len); + + /* XXX rx buf has to be > max packet len */ + ret = kernel_sock_ioctl(sinf->sock, SIOCINQ, + (unsigned long)&inq); + trace_printk("sinf %p sock %p ioctl ret %d\n", + sinf, sinf->sock, ret); + if (ret < 0 || inq < len) + break; + + rbuf = kmalloc(sizeof(struct recv_buf) + len, GFP_NOFS); + if (!rbuf) { + ret = -ENOMEM; + break; + } + + ret = recv_msg(sinf->sock, rbuf->nh, len, 0); + trace_printk("sinf %p sock %p recv len %d ret %d\n", + sinf, sinf->sock, len, ret); + if (ret != len) { + if (ret >= 0) + ret = -EIO; + break; + } + + if (inappropriate_message(nti, sinf, rbuf)) { + ret = -EINVAL; + break; + } + + rbuf->nti = nti; + rbuf->sinf = sinf; + INIT_LIST_HEAD(&rbuf->head); + INIT_WORK(&rbuf->proc_work, scoutfs_net_proc_func); + + mutex_lock(&nti->mutex); + list_add(&rbuf->head, &sinf->active_rbufs); + mutex_unlock(&nti->mutex); + queue_work(nti->proc_wq, &rbuf->proc_work); + rbuf = NULL; + } + + if (ret < 0) { + kfree(rbuf); + trace_printk("ret %d\n", ret); + queue_sock_work(sinf, &sinf->shutdown_work); + } +} + + +/* + * Connecting sockets kick off send and recv work once the socket is + * connected and all sockets shutdown when closed. + */ +static void scoutfs_net_state_change(struct sock *sk) +{ + void (*state_change)(struct sock *sk); + struct sock_info *sinf; + + read_lock(&sk->sk_callback_lock); + + sinf = sk->sk_user_data; + if (sinf == NULL) { + state_change = sk->sk_state_change; + goto out; + } + + trace_printk("sinf %p state %u\n", sinf, sk->sk_state); + + switch(sk->sk_state) { + case TCP_ESTABLISHED: + queue_sock_work_if_func(sinf, &sinf->send_work); + queue_sock_work_if_func(sinf, &sinf->recv_work); + break; + case TCP_CLOSE: + queue_sock_work(sinf, &sinf->shutdown_work); + break; + } + state_change = sinf->orig_state_change; +out: + read_unlock(&sk->sk_callback_lock); + state_change(sk); +} + +/* + * Listening sockets accept incoming sockets and accepted and connected + * sockets recv data. + */ +static void scoutfs_net_data_ready(struct sock *sk, int bytes) +{ + void (*data_ready)(struct sock *sk, int bytes); + struct sock_info *sinf; + + read_lock(&sk->sk_callback_lock); + + sinf = sk->sk_user_data; + if (sinf == NULL) { + data_ready = sk->sk_data_ready; + goto out; + } + + trace_printk("sinf %p bytes %d\n", sinf, bytes); + + queue_sock_work_if_func(sinf, &sinf->recv_work); + queue_sock_work_if_func(sinf, &sinf->accept_work); + data_ready = sinf->orig_data_ready; +out: + read_unlock(&sk->sk_callback_lock); + data_ready(sk, bytes); +} + +/* + * Connected and accepted sockets send once there's space again in the + * tx buffer. + */ +static void scoutfs_net_write_space(struct sock *sk) +{ + void (*write_space)(struct sock *sk); + struct sock_info *sinf; + + read_lock(&sk->sk_callback_lock); + + sinf = sk->sk_user_data; + if (sinf == NULL) { + write_space = sk->sk_write_space; + goto out; + } + + trace_printk("sinf %p\n", sinf); + + queue_sock_work_if_func(sinf, &sinf->send_work); + write_space = sinf->orig_write_space; +out: + read_unlock(&sk->sk_callback_lock); + write_space(sk); +} + +/* + * For accepted sockets our callbacks can execute and queue work the + * moment user_data is set so this should only be called once the socket + * info is fully initialized. + */ +static void set_sock_callbacks(struct sock_info *sinf) +{ + struct socket *sock = sinf->sock; + struct sock *sk = sock->sk; + + write_lock_bh(&sk->sk_callback_lock); + + sinf->orig_state_change = sk->sk_state_change; + sinf->orig_data_ready = sk->sk_data_ready; + sinf->orig_write_space = sk->sk_write_space; + + sk->sk_state_change = scoutfs_net_state_change; + sk->sk_data_ready = scoutfs_net_data_ready; + sk->sk_write_space = scoutfs_net_write_space; + sk->sk_user_data = sinf; + + write_unlock_bh(&sk->sk_callback_lock); +} + +/* get or set the address of the listening server depending on mode */ +static int lock_addr_lvb(struct super_block *sb, int mode, + struct scoutfs_inet_addr *addr) +{ + struct scoutfs_lock lck; + int ret; + + ret = scoutfs_lock_range_lvb(sb, mode, &addr_key, &addr_key, + addr, sizeof(*addr), &lck); + if (ret == 0) + scoutfs_unlock_range(sb, &lck); + + return ret; +} + +/* + * The caller can provide an error to give to pending sends before + * freeing them. + */ +static void free_sbuf_list(struct super_block *sb, struct list_head *list, + int ret) +{ + struct send_buf *sbuf; + struct send_buf *pos; + + list_for_each_entry_safe(sbuf, pos, list, head) { + list_del_init(&sbuf->head); + if (ret && sbuf->func) + sbuf->func(sb, NULL, ret); + kfree(sbuf); + } +} + +/* + * Remove the rbufs from the list and clear their sinf pointers so that + * they can't reference a sinf that's being freed. + */ +static void empty_rbuf_list(struct list_head *list) +{ + struct recv_buf *rbuf; + struct recv_buf *pos; + + list_for_each_entry_safe(rbuf, pos, list, head) { + list_del_init(&rbuf->head); + rbuf->sinf = NULL; + } +} + +/* + * Shutdown and free a socket. This can be queued from most all socket + * work. It executes in the single socket workqueue context so we know + * that we're serialized with all other socket work. Listening, + * connecting, and accepting don't reference the socket once it's + * possible for this work to execute. + * + * Other work won't be executing but could be queued when we get here. + * We can't cancel other work from inside their workqueue (until later + * kernels when cancel_work() comes back). So we have a two phase + * shutdown where we first prevent additional work from being queued and + * then queue the work again. By the time the work executes again we + * know that none of our work will be pending. + */ +static void scoutfs_net_shutdown_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + shutdown_work); + struct super_block *sb = sinf->sb; + DECLARE_NET_INFO(sb, nti); + struct socket *sock = sinf->sock; + struct sock *sk; + + trace_printk("sinf %p sock %p shutting_down %d\n", + sinf, sock, sinf->shutting_down); + + if (!sinf->shutting_down) { + sinf->shutting_down = true; + queue_work(nti->sock_wq, &sinf->shutdown_work); + return; + } + + if (sock) { + sk = sock->sk; + + write_lock_bh(&sk->sk_callback_lock); + sk->sk_state_change = sinf->orig_state_change; + sk->sk_data_ready = sinf->orig_data_ready; + sk->sk_write_space = sinf->orig_write_space; + sk->sk_user_data = NULL; + write_unlock_bh(&sk->sk_callback_lock); + } + + mutex_lock(&nti->mutex); + + if (sinf == nti->listening_sinf) { + /* clear addr lvb and try to reacquire lock and listen */ + nti->listening_sinf = NULL; + memset(&sinf->addr, 0, sizeof(sinf->addr)); + lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_WRITE, &sinf->addr); + scoutfs_unlock_range(sb, &sinf->listen_lck); + queue_delayed_work(nti->proc_wq, &nti->server_work, 0); + + } if (sinf == nti->connected_sinf) { + /* save reliable sends and try to reconnect */ + nti->connected_sinf = NULL; + list_splice_init(&sinf->have_sent, &nti->to_send); + list_splice_init(&sinf->to_send, &nti->to_send); + queue_delayed_work(nti->proc_wq, &nti->client_work, 0); + + } else { + /* free reply sends and stop rbuf socket refs */ + free_sbuf_list(sb, &sinf->to_send, 0); + empty_rbuf_list(&sinf->active_rbufs); + } + + list_del_init(&sinf->head); + + mutex_unlock(&nti->mutex); + + sock_release(sock); + kfree(sinf); +} + +static int add_send_buf(struct super_block *sb, int type, void *data, + unsigned data_len, reply_func_t func) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_net_header *nh; + struct sock_info *sinf; + struct send_buf *sbuf; + + sbuf = alloc_sbuf(data_len); + if (!sbuf) + return -ENOMEM; + + sbuf->func = func; + sbuf->nh->status = SCOUTFS_NET_STATUS_REQUEST; + + nh = sbuf->nh; + nh->type = type; + memcpy(nh->data, data, data_len); + + mutex_lock(&nti->mutex); + + nh->id = cpu_to_le64(nti->next_id++); + + sinf = nti->connected_sinf; + if (sinf) { + list_add_tail(&sbuf->head, &sinf->to_send); + queue_sock_work(sinf, &sinf->send_work); + } else { + list_add_tail(&sbuf->head, &nti->to_send); + } + + mutex_unlock(&nti->mutex); + + return 0; +} + +static int trade_time_reply(struct super_block *sb, void *reply, int ret) +{ + struct scoutfs_timespec *ts = reply; + + if (ret != sizeof(*ts)) + return -EINVAL; + + trace_printk("reply %llu.%u\n", + le64_to_cpu(ts->sec), le32_to_cpu(ts->nsec)); + + return 0; +} + +int scoutfs_net_trade_time(struct super_block *sb) +{ + struct scoutfs_timespec send; + struct timespec64 ts; + int ret; + + getnstimeofday64(&ts); + send.sec = cpu_to_le64(ts.tv_sec); + send.nsec = cpu_to_le32(ts.tv_nsec); + + ret = add_send_buf(sb, SCOUTFS_NET_TRADE_TIME, &send, + sizeof(send), trade_time_reply); + + trace_printk("sent %llu.%lu ret %d\n", + (u64)ts.tv_sec, ts.tv_nsec, ret); + + return ret; +} + +static struct sock_info *alloc_sinf(struct super_block *sb) +{ + struct sock_info *sinf; + + sinf = kzalloc(sizeof(struct sock_info), GFP_NOFS); + if (sinf) { + sinf->sb = sb; + INIT_LIST_HEAD(&sinf->head); + INIT_LIST_HEAD(&sinf->to_send); + INIT_LIST_HEAD(&sinf->have_sent); + INIT_LIST_HEAD(&sinf->active_rbufs); + + /* callers set other role specific work as appropriate */ + INIT_WORK(&sinf->shutdown_work, scoutfs_net_shutdown_func); + } + + return sinf; +} + +static void scoutfs_net_accept_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + accept_work); + struct super_block *sb = sinf->sb; + DECLARE_NET_INFO(sb, nti); + struct sock_info *new_sinf; + struct socket *new_sock; + int ret; + + for (;;) { + ret = kernel_accept(sinf->sock, &new_sock, O_NONBLOCK); + trace_printk("nti %p accept sock %p ret %d\n", + nti, new_sock, ret); + if (ret < 0) { + if (ret == -EAGAIN) + ret = 0; + break; + } + + new_sinf = alloc_sinf(sb); + if (!new_sinf) { + ret = -ENOMEM; + sock_release(new_sock); + break; + } + + new_sinf->sock = new_sock; + INIT_WORK(&new_sinf->send_work, scoutfs_net_send_func); + INIT_WORK(&new_sinf->recv_work, scoutfs_net_recv_func); + + mutex_lock(&nti->mutex); + list_add(&new_sinf->head, &nti->active_socks); + queue_sock_work(new_sinf, &new_sinf->recv_work); + mutex_unlock(&nti->mutex); + + set_sock_callbacks(new_sinf); + } + + if (ret) { + trace_printk("ret %d\n", ret); + queue_sock_work(sinf, &sinf->shutdown_work); + } +} + +/* + * The server work has acquired the listen lock. We create a socket and + * publish its bound address in the addr lock's lvb. + * + * This can block in the otherwise non-blocking socket workqueue while + * acquiring the addr lock but it should be brief and doesn't matter + * much given that we're bringing up a new server. This should happen + * rarely. + */ +static void scoutfs_net_listen_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + listen_work); + struct super_block *sb = sinf->sb; + struct scoutfs_inet_addr addr; + struct sockaddr_in sin; + struct socket *sock; + int addrlen; + int ret; + + /* XXX option to set listening address */ + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = cpu_to_be32(INADDR_LOOPBACK); + sin.sin_port = 0; + + trace_printk("binding to %pIS:%u\n", + &sin, be16_to_cpu(sin.sin_port)); + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + sinf->sock = sock; + INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); + + addrlen = sizeof(sin); + ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen) ?: + kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); + if (ret) + goto out; + + trace_printk("sock %p listening on %pIS:%u\n", + sock, &sin, be16_to_cpu(sin.sin_port)); + + addr.addr = cpu_to_le32(be32_to_cpu(sin.sin_addr.s_addr)); + addr.port = cpu_to_le16(be16_to_cpu(sin.sin_port)); + + set_sock_callbacks(sinf); + + ret = kernel_listen(sock, 255) ?: + lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_WRITE, &addr); + if (ret == 0) + queue_sock_work(sinf, &sinf->accept_work); + +out: + if (ret) { + trace_printk("ret %d\n", ret); + queue_sock_work(sinf, &sinf->shutdown_work); + } +} + +/* + * The client work has found an address to try and connect to. Create a + * connecting socket and wire up its callbacks. + */ +static void scoutfs_net_connect_func(struct work_struct *work) +{ + struct sock_info *sinf = container_of(work, struct sock_info, + connect_work); + struct sockaddr_in sin; + struct socket *sock; + int addrlen; + int ret; + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + sinf->sock = sock; + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = cpu_to_be32(le32_to_cpu(sinf->addr.addr)); + sin.sin_port = cpu_to_be16(le16_to_cpu(sinf->addr.port)); + + trace_printk("connecting to %pIS:%u\n", + &sin, be16_to_cpu(sin.sin_port)); + + /* callbacks can fire once inside connect that'll succeed */ + set_sock_callbacks(sinf); + + addrlen = sizeof(sin); + ret = kernel_connect(sock, (struct sockaddr *)&sin, addrlen, + O_NONBLOCK); + if (ret == -EINPROGRESS) + ret = 0; +out: + if (ret) { + trace_printk("ret %d\n", ret); + queue_sock_work(sinf, &sinf->shutdown_work); + } +} + +/* + * This work executes whenever there isn't a socket on the client connected + * to the server: on mount, after the connected socket is shut down, and + * when we can't find an address in the addr lock's lvb. + */ +static void scoutfs_net_client_func(struct work_struct *work) +{ + struct net_info *nti = container_of(work, struct net_info, + client_work.work); + struct super_block *sb = nti->sb; + struct sock_info *sinf = NULL; + int ret; + + BUG_ON(nti->connected_sinf); + + sinf = alloc_sinf(sb); + if (!sinf) { + ret = -ENOMEM; + goto out; + } + + INIT_WORK(&sinf->connect_work, scoutfs_net_connect_func); + INIT_WORK(&sinf->send_work, scoutfs_net_send_func); + INIT_WORK(&sinf->recv_work, scoutfs_net_recv_func); + + ret = lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_READ, &sinf->addr); + if (ret == 0 && sinf->addr.addr == cpu_to_le32(INADDR_ANY)) + ret = -ENOENT; + if (ret < 0) { + kfree(sinf); + goto out; + } + + mutex_lock(&nti->mutex); + nti->connected_sinf = sinf; + list_splice_init(&nti->to_send, &sinf->to_send); + list_add(&sinf->head, &nti->active_socks); + queue_sock_work(sinf, &sinf->connect_work); + mutex_unlock(&nti->mutex); + +out: + if (ret < 0 && ret != -ESHUTDOWN) { + trace_printk("ret %d\n", ret); + queue_delayed_work(nti->proc_wq, &nti->client_work, HZ / 2); + } +} + +/* + * This very long running blocking work just sits trying to acquire a + * lock on the listening key which marks it as the active server. When + * it does that it queues off work to build up the listening socket. + * The lock is associated with the listening socket and is unlocked when + * the socket is shut down. + * + * This work is queued by mount, shutdown of the listening socket, and + * errors. It stops re-arming itself if it sees that locking has been + * shut down. + */ +static void scoutfs_net_server_func(struct work_struct *work) +{ + struct net_info *nti = container_of(work, struct net_info, + server_work.work); + struct super_block *sb = nti->sb; + struct sock_info *sinf = NULL; + int ret; + + BUG_ON(nti->listening_sinf); + + sinf = alloc_sinf(sb); + if (!sinf) { + ret = -ENOMEM; + goto out; + } + + INIT_WORK(&sinf->listen_work, scoutfs_net_listen_func); + INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); + + ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, &listen_key, + &listen_key, &sinf->listen_lck); + if (ret) { + kfree(sinf); + goto out; + } + + mutex_lock(&nti->mutex); + nti->listening_sinf = sinf; + list_add(&sinf->head, &nti->active_socks); + queue_sock_work(sinf, &sinf->listen_work); + mutex_unlock(&nti->mutex); + +out: + if (ret < 0 && ret != -ESHUTDOWN) { + trace_printk("ret %d\n", ret); + queue_delayed_work(nti->proc_wq, &nti->server_work, HZ / 2); + } +} + +static void free_nti(struct net_info *nti) +{ + if (nti) { + if (nti->sock_wq) + destroy_workqueue(nti->sock_wq); + if (nti->proc_wq) + destroy_workqueue(nti->proc_wq); + kfree(nti); + } +} + +int scoutfs_net_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct net_info *nti; + + scoutfs_key_init(&listen_key, &listen_type, sizeof(listen_type)); + scoutfs_key_init(&addr_key, &addr_type, sizeof(addr_type)); + + nti = kzalloc(sizeof(struct net_info), GFP_KERNEL); + if (nti) { + nti->sock_wq = alloc_workqueue("scoutfs_net_sock", + WQ_UNBOUND, 1); + nti->proc_wq = alloc_workqueue("scoutfs_net_proc", + WQ_NON_REENTRANT, 0); + } + if (!nti || !nti->sock_wq || !nti->proc_wq) { + free_nti(nti); + return -ENOMEM; + } + + nti->sb = sb; + mutex_init(&nti->mutex); + INIT_DELAYED_WORK(&nti->client_work, scoutfs_net_client_func); + INIT_LIST_HEAD(&nti->to_send); + nti->next_id = 1; + INIT_DELAYED_WORK(&nti->server_work, scoutfs_net_server_func); + INIT_LIST_HEAD(&nti->active_socks); + + sbi->net_info = nti; + + queue_delayed_work(nti->proc_wq, &nti->server_work, 0); + queue_delayed_work(nti->proc_wq, &nti->client_work, 0); + + return 0; +} + +/* + * Shutdown and destroy all our socket communications. + * + * This is called after locking has been shutdown. Client and server + * work that executes from this point on will fail with -ESHUTDOWN and + * won't rearm itself. That prevents new sockets from being created so + * our job is to shutdown all the existing sockets. + * + * We'll have to be careful to shut down any non-vfs callers of ours + * that might try to send requests during destruction. + */ +void scoutfs_net_destroy(struct super_block *sb) +{ + DECLARE_NET_INFO(sb, nti); + struct sock_info *sinf; + struct sock_info *pos; + + if (nti) { + /* let any currently executing client/server work finish */ + flush_workqueue(nti->proc_wq); + + /* stop any additional incoming accepted sockets */ + mutex_lock(&nti->mutex); + sinf = nti->listening_sinf; + if (sinf) + queue_sock_work(sinf, &sinf->shutdown_work); + mutex_unlock(&nti->mutex); + drain_workqueue(nti->sock_wq); + + /* shutdown all the remaining sockets */ + mutex_lock(&nti->mutex); + list_for_each_entry_safe(sinf, pos, &nti->active_socks, head) + queue_sock_work(sinf, &sinf->shutdown_work); + mutex_unlock(&nti->mutex); + drain_workqueue(nti->sock_wq); + + /* wait for processing to finish and free rbufs */ + flush_workqueue(nti->proc_wq); + + /* make sure client/server work isn't queued */ + cancel_delayed_work_sync(&nti->server_work); + cancel_delayed_work_sync(&nti->client_work); + + /* call all pending replies with errors */ + list_for_each_entry_safe(sinf, pos, &nti->active_socks, head) + + /* and free all resources */ + free_sbuf_list(sb, &nti->to_send, -ESHUTDOWN); + free_nti(nti); + } +} diff --git a/kmod/src/net.h b/kmod/src/net.h new file mode 100644 index 00000000..382b686e --- /dev/null +++ b/kmod/src/net.h @@ -0,0 +1,9 @@ +#ifndef _SCOUTFS_NET_H_ +#define _SCOUTFS_NET_H_ + +int scoutfs_net_trade_time(struct super_block *sb); + +int scoutfs_net_setup(struct super_block *sb); +void scoutfs_net_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 4aa4d28f..15c0778e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -36,6 +36,7 @@ #include "compact.h" #include "data.h" #include "lock.h" +#include "net.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -221,10 +222,13 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) // scoutfs_buddy_setup(sb) ?: scoutfs_compact_setup(sb) ?: scoutfs_setup_trans(sb) ?: - scoutfs_lock_setup(sb); + scoutfs_lock_setup(sb) ?: + scoutfs_net_setup(sb); if (ret) return ret; + scoutfs_net_trade_time(sb); + scoutfs_advance_dirty_super(sb); inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); @@ -252,6 +256,8 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_lock_shutdown(sb); + scoutfs_net_destroy(sb); scoutfs_lock_destroy(sb); scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 5f1b468d..e4f48514 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,7 @@ struct treap_info; struct compact_info; struct data_info; struct lock_info; +struct net_info; struct scoutfs_sb_info { struct super_block *sb; @@ -42,6 +43,7 @@ struct scoutfs_sb_info { struct workqueue_struct *trans_write_workq; struct lock_info *lock_info; + struct net_info *net_info; /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; From 27e55eb43c5f63d6d484c2132d66de0573a7f4ff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Mar 2017 15:34:29 -0800 Subject: [PATCH 242/920] Flesh out some pieces of the scoutfs.md doc Trying to keep adding coverage across the design. Signed-off-by: Zach Brown --- kmod/Documentation/scoutfs.md | 274 ++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) diff --git a/kmod/Documentation/scoutfs.md b/kmod/Documentation/scoutfs.md index baf92616..e4387e7a 100644 --- a/kmod/Documentation/scoutfs.md +++ b/kmod/Documentation/scoutfs.md @@ -29,6 +29,280 @@ data residency interfaces for efficiently executing archival policies. It is deployed on a shared block fabric for high bandwidth and low latency. +## Super Block + +The super block is the anchor of all the persistent storage in the block +device. It contains volume-wide configuration information and +references to the current stable versions of persistent data structures +in the rest of the block device. The super block is stored in two 4KB +blocks at a known location at the start of the device. + +To read the current super block both block locations are read. The +valid super block with the most recent sequence number is used. Either +of the super blocks can be corrupt because they're overwritten in place +and a crash during a write could scramble the block. + +Each new version of the super block is written to the block that doesn't +contain the current super block. If this new super block write fails +then the old super block can still be used and no data is lost. + +The super block, and indeed all file system data, doesn't touch a few +blocks at the start of the device to avoid corrupting blocks that are +used by host platforms that store data inside devices to manage them. + +## Inodes + +Inodes are stored in items identified by the inode number. + + key = struct scoutfs_inode_key { + .type = SCOUTFS_INODE_KEY, + .ino, + } + + val = struct scoutfs_inode { + size, nlink, uid, gid, atime, mtime, ..., + } + +The variable length value that stores the item struct gives us dense +inode packing without having to predefine an inode storage size when the +file system is created and gives us a future expansion mechanism that +uses the item length to determine the version of the inode struct that +is written. + +Inode numbers are 64bit and are never re-used. By never re-using inode +numbers we don't need to manage an inode number allocator that would +need to be consistent across nodes. We can grant large ranges of +numbers to mount clients for allocation. Each inode number uniquely +identify the lifetime of a file and avoids having to store a seperate +generation number for each inode number. + +## Extended Attributes + +Extended attributes are stored in items on the inode at the full name of +the attribute. The attribute name is limited to 255 bytes and the +attribute values is limited to 64KB. The max xattr value size is larger +than our max item size so we can store an xattr in multiple items, but +in the common case a single xattr is efficiently stored in a single +item. + + key = struct scoutfs_xattr_key { + .type = SCOUTFS_XATTR_KEY, + .ino, + .name, + struct scoutfs_xattr_key_footer { + .null = '\0', + .part, + } + } + +Storing the null after the attribute name, which can't be found in any +name, lets us accurately locate a given name in the presence of other +names that share partial prefixes. The part identifies each key's +position in the set of keys that make up the large value. Storing the +full name in each key ensures that all the keys that make up an +attribute are stored adjacent to each other. + +Each item's value starts with a header which describes portion of the +attribute value stored in the item. + + val = struct scoutfs_xattr_val_header { + .part_len, + .last_part, + .data, + } + +The result of all this is that operations on xattrs iterate over keys +starting with the name and part 0 and stop when they hit the final part +(or error on corruption if the parts aren't consistent.) + +## Directory Entries + +Directory entry items store the target inode number referred to by a +given entry name in a parent directory. The name is limited to 255 +non-null bytes. The large keys supported by our items let us store +directory entries in items indexed by the full entry name itself. + + key = struct scoutfs_dirent_key { + .type = SCOUTFS_DIRENT_KEY, + .ino, + .name, + } + + val = struct scoutfs_dirent { + .ino, + .readdir_pos, + .type, + } + +These full precision items let us work on each item for a given name +directly rather than scrambling their sorting by storing them at a hash +value of their name. Storing at a hash value not only adds the +complexity of collisions, it critically causes entry lock attempts in a +directory between mounts to be perfectly randomly distributed and +constantly conflicting with each other. Storing and range locking the +directory entries at their full name preserves non-overlapping patterns +between mounts and gives them a chance to efficiently operate on +disjoint sets of names. + +We index the directory entry items by the full name of the entry so +there is no limit imposed on the number of entries in a directory. The +system will run out of blocks to store entries long before the index is +incapable of storing them. + +While we can satisfy lookups with a full precision index, readdir +doesn't use a full precision iterator. It forces us to describe each +entry with a small scalar directory position. We use a separate item +that's indexed by this readdir position instead of the file name. + + key = struct scoutfs_readdir_key { + .type = SCOUTFS_DIRENT_KEY, + .ino, + .readdir_pos, + } + + val = struct scoutfs_dirent { + .ino, + .readdir_pos, + .type, + .name, + } + +The key's position is allocated as each entry is created. This results +in readdir returning entries ordered by creation time. Like inode +numbers, readdir positions are never re-used so that we don't have to +risk contention by maintaining a consistent free position index across +nodes. + +## Directory Entry Link Backrefs + +The third and final item used by each directory entry is an item that is +stored at the target inode instead of in the parent directory. These +backref items can be traversed to find the full paths from the root +inode to all the entries that link to the target inode. + + key = struct scoutfs_link_backref_key { + .type = SCOUTFS_LINK_BACKREF_KEY, + .ino, + .dir_ino, + .name, + } + + /* no value */ + +Iterating over these items for a given target ino yields the parent +dir_ino and full file name of every entry that references the target +inode. The entry items in the parent dir are stored at the full file +name so the only way for us to reference them is with another copy of +the file name, brining the total to three full copies of the name stored +for every directory entry. + +Because we store the full name for these backref items they do not +impose a limit on the number of hard links to an inode. + +## Regular File Data Extents + +scoutfs stores file data in block extents at 4KB granularity. Items +describe the extents of 4KB blocks that map logical file offsets to +physical block extents in the device: + + key = struct scoutfs_extent_key { + .type = SCOUTFS_EXTENT_KEY, + .ino, + .iblock, + .blkno, + .count, + .flags, + } + + /* no value */ + +The flags field indicates the state of the extent, for example it can be +preallocated but unwritten or offline. If the extent is offline then +the blkno is unused and should be zero. + +Checksums of file data are contained in items at the physical block +offset of the checksumed blocks. Each item contains a fixed number of +checksums for a given group of blocks. + + key = struct scoutfs_checksum_key { + .type = SCOUTFS_CHECKSUM_KEY, + .blkno, + } + + val = { + .crcs[8], + } + +The checksum items are keyed by the physical block number instead of the +logical file position so that the checksum items are only written as new +data is written. The checksum items are left alone as the file data +references change: truncate, unlink, hole punching, and cloning don't +have to modify checksum items. + +With these structures in place the file read and write paths in scoutfs +look very much like most other block file systems in Linux. The generic +buffer_head support code is used and our get_blocks callback reads and +writes the extent items that reference block extents. Write and sync +patterns, with the help of delalloc, preallocation, and fallocate, +determine the physical contiguity of extent allocations. Buffered +read-ahead and O_DIRECT reads walk the extent items and build large +efficient bios if the extents are physically contiguous. + +## Allocating Regular File Data Extents + +The primary persistent allocator for blocks on the device uses an +efficient bitmap with a bit for each 1MB segment. File data allocation +wants to track extents at 4KB granularity and also index them by the +size of the free extent, neither of which the segment bitmap allocator +supports. + +We have free extent items that track free block extents in the device at +the finer 4K granularity. There are two keys for each free extent: one +indexed by the block location and one by the size of the free extent. +Modifying a free extent can thus modify three different positions in the +key namespace: the block location, the old size location, and the new +size location. LSM lets us generate and merge these disjoint items +across different mounts efficiently. + +To avoid the prohibitively expensive lock contention of modifying these +items from multiple mounts, we first create groups of free extents and +assign a given mount to a group for the lifetime of its mount. + + key = struct scoutfs_free_extent_loc_key + .type = SCOUTFS_FREE_EXTENT_LOC_KEY, + .group, + .blkno, + .count, + } + + key = struct scoutfs_free_extent_len_key + .type = SCOUTFS_FREE_EXTENT_LEN_KEY, + .group, + .count, + .blkno, + } + +Mounts are responsible for mangement of the free extent items. They're +populated with the result from requests from the manifest server for +free segment blocks. They're consumed as file data is written and +logical extents are allocated. They're repopulated as file data is +truncated and its extents are freed. They're returned to the segment +allocator when they contain aligned 1MB free extents. + +Like all persistent filesystem items, the free extent items are +protected by range locks. In the common case a single mount will be +operating on its group and having all the lock operations satisfied by +range matches. Any mount can modify any group's extents by acquiring +the right locks, but this should be limited to rare attempts to +defragment or migrate free extents between groups. + +The manifest server is responsible for tracking the assigment of mounts +to groups as mounts come and go through clean mounts and unclean crashes +and recovery. Free extents can get stranded in groups that don't have +an assigned mount. A mount scrambling to find free space in other +groups would need a mechanism to discover other groups, perhaps with a +set of keys that record the presence of extents in each group. + ## Indexing Inodes by Modification Time As files are modified archival agents need to find these modified files From 8c59902b7002d6aeb24a194d14cd8f0a73a1564b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 Mar 2017 10:49:43 -0700 Subject: [PATCH 243/920] scoutfs: cleanup socket callbacks The first attempt at wiring up the socket callbacks was a bit too precious. We can simplify and do what other modern socket callback users do: don't bother with the callback locks and call shutdown before release. We also protect against spurious callbacks by only doing work in the callbacks when the sk user_data points to a sock_info which points back to the socket. Signed-off-by: Zach Brown --- kmod/src/net.c | 132 +++++++++++++++---------------------------------- 1 file changed, 40 insertions(+), 92 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index a6ecdde2..52dc18aa 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -131,9 +131,6 @@ struct sock_info { struct work_struct shutdown_work; struct socket *sock; - void (*orig_state_change)(struct sock *sk); - void (*orig_data_ready)(struct sock *sk, int bytes); - void (*orig_write_space)(struct sock *sk); }; /* @@ -181,18 +178,6 @@ static void queue_sock_work(struct sock_info *sinf, struct work_struct *work) queue_work(nti->sock_wq, work); } -/* - * By giving all the sockets all the work funcs we can have one set of - * socket callbacks that queue the appropriate work only if the func has - * been set. - */ -static void queue_sock_work_if_func(struct sock_info *sinf, - struct work_struct *work) -{ - if (work->func) - queue_sock_work(sinf, work); -} - /* * This non-blocking work consumes the send queue in the socket info as * messages are sent out. If the messages have a reply function then @@ -519,32 +504,21 @@ static void scoutfs_net_recv_func(struct work_struct *work) */ static void scoutfs_net_state_change(struct sock *sk) { - void (*state_change)(struct sock *sk); - struct sock_info *sinf; + struct sock_info *sinf = sk->sk_user_data; - read_lock(&sk->sk_callback_lock); + trace_printk("sk %p state %u sinf %p\n", sk, sk->sk_state, sinf); - sinf = sk->sk_user_data; - if (sinf == NULL) { - state_change = sk->sk_state_change; - goto out; + if (sinf && sinf->sock->sk == sk) { + switch(sk->sk_state) { + case TCP_ESTABLISHED: + queue_sock_work(sinf, &sinf->send_work); + queue_sock_work(sinf, &sinf->recv_work); + break; + case TCP_CLOSE: + queue_sock_work(sinf, &sinf->shutdown_work); + break; + } } - - trace_printk("sinf %p state %u\n", sinf, sk->sk_state); - - switch(sk->sk_state) { - case TCP_ESTABLISHED: - queue_sock_work_if_func(sinf, &sinf->send_work); - queue_sock_work_if_func(sinf, &sinf->recv_work); - break; - case TCP_CLOSE: - queue_sock_work(sinf, &sinf->shutdown_work); - break; - } - state_change = sinf->orig_state_change; -out: - read_unlock(&sk->sk_callback_lock); - state_change(sk); } /* @@ -553,25 +527,16 @@ out: */ static void scoutfs_net_data_ready(struct sock *sk, int bytes) { - void (*data_ready)(struct sock *sk, int bytes); - struct sock_info *sinf; + struct sock_info *sinf = sk->sk_user_data; - read_lock(&sk->sk_callback_lock); + trace_printk("sk %p bytes %d sinf %p\n", sk, bytes, sinf); - sinf = sk->sk_user_data; - if (sinf == NULL) { - data_ready = sk->sk_data_ready; - goto out; + if (sinf && sinf->sock->sk == sk) { + if (sk->sk_state == TCP_LISTEN) + queue_sock_work(sinf, &sinf->accept_work); + else + queue_sock_work(sinf, &sinf->recv_work); } - - trace_printk("sinf %p bytes %d\n", sinf, bytes); - - queue_sock_work_if_func(sinf, &sinf->recv_work); - queue_sock_work_if_func(sinf, &sinf->accept_work); - data_ready = sinf->orig_data_ready; -out: - read_unlock(&sk->sk_callback_lock); - data_ready(sk, bytes); } /* @@ -580,48 +545,32 @@ out: */ static void scoutfs_net_write_space(struct sock *sk) { - void (*write_space)(struct sock *sk); - struct sock_info *sinf; + struct sock_info *sinf = sk->sk_user_data; - read_lock(&sk->sk_callback_lock); + trace_printk("sk %p sinf %p\n", sk, sinf); - sinf = sk->sk_user_data; - if (sinf == NULL) { - write_space = sk->sk_write_space; - goto out; + if (sinf && sinf->sock->sk == sk) { + if (sk_stream_is_writeable(sk)) + clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); + queue_sock_work(sinf, &sinf->send_work); } - - trace_printk("sinf %p\n", sinf); - - queue_sock_work_if_func(sinf, &sinf->send_work); - write_space = sinf->orig_write_space; -out: - read_unlock(&sk->sk_callback_lock); - write_space(sk); } /* - * For accepted sockets our callbacks can execute and queue work the - * moment user_data is set so this should only be called once the socket - * info is fully initialized. + * Accepted sockets inherit the sk fields from the listening socket so + * all the callbacks check that the sinf they're working on points to + * the socket executing the callback. This ensures that we'll only get + * callbacks doing work once we've initialized sinf for the socket. */ static void set_sock_callbacks(struct sock_info *sinf) { - struct socket *sock = sinf->sock; - struct sock *sk = sock->sk; - - write_lock_bh(&sk->sk_callback_lock); - - sinf->orig_state_change = sk->sk_state_change; - sinf->orig_data_ready = sk->sk_data_ready; - sinf->orig_write_space = sk->sk_write_space; + struct sock *sk = sinf->sock->sk; sk->sk_state_change = scoutfs_net_state_change; sk->sk_data_ready = scoutfs_net_data_ready; sk->sk_write_space = scoutfs_net_write_space; sk->sk_user_data = sinf; - write_unlock_bh(&sk->sk_callback_lock); } /* get or set the address of the listening server depending on mode */ @@ -693,7 +642,6 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) struct super_block *sb = sinf->sb; DECLARE_NET_INFO(sb, nti); struct socket *sock = sinf->sock; - struct sock *sk; trace_printk("sinf %p sock %p shutting_down %d\n", sinf, sock, sinf->shutting_down); @@ -704,16 +652,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) return; } - if (sock) { - sk = sock->sk; - - write_lock_bh(&sk->sk_callback_lock); - sk->sk_state_change = sinf->orig_state_change; - sk->sk_data_ready = sinf->orig_data_ready; - sk->sk_write_space = sinf->orig_write_space; - sk->sk_user_data = NULL; - write_unlock_bh(&sk->sk_callback_lock); - } + kernel_sock_shutdown(sock, SHUT_RDWR); mutex_lock(&nti->mutex); @@ -860,6 +799,9 @@ static void scoutfs_net_accept_func(struct work_struct *work) break; } + trace_printk("accepted sinf %p sock %p sk %p\n", + new_sinf, new_sock, new_sock->sk); + new_sinf->sock = new_sock; INIT_WORK(&new_sinf->send_work, scoutfs_net_send_func); INIT_WORK(&new_sinf->recv_work, scoutfs_net_recv_func); @@ -910,6 +852,9 @@ static void scoutfs_net_listen_func(struct work_struct *work) if (ret) goto out; + trace_printk("listening sinf %p sock %p sk %p\n", + sinf, sock, sock->sk); + sinf->sock = sock; INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); @@ -956,6 +901,9 @@ static void scoutfs_net_connect_func(struct work_struct *work) if (ret) goto out; + trace_printk("connecting sinf %p sock %p sk %p\n", + sinf, sock, sock->sk); + sinf->sock = sock; sin.sin_family = AF_INET; From 2ea5f1d734f7db63d478877a66be22b15729fc3b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 17 Mar 2017 09:35:38 -0700 Subject: [PATCH 244/920] invalidate_others could return uninit ret Make sure to initialize ret in case there aren't other mounts. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 876d7690..dd5bdb52 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -144,7 +144,7 @@ static int invalidate_others(struct super_block *from, int mode, { DECLARE_LOCK_INFO(from, from_linf); struct lock_info *linf; - int ret; + int ret = 0; down_read(&global_rwsem); From 104bbb06a90fc865ff5a747cd29b417205b544e0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 17 Mar 2017 09:38:50 -0700 Subject: [PATCH 245/920] Remove cached range when invalidating items When invalidating items we need to remove the cached range that covers the range of keys that we're removing so that the removed items aren't then considered negative cached items. Signed-off-by: Zach Brown --- kmod/src/item.c | 114 ++++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/item.h | 6 +-- kmod/src/lock.c | 4 +- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 606c80e9..7dfcc71c 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -524,6 +524,92 @@ restart: rb_insert_color(&ins->node, root); } +/* + * Remove a given cached range. The caller has already removed all the + * items that fell within the range. There can be any number of + * existing cached ranges that overlap with the range that should be + * removed. + * + * The caller's range has full precision keys that specify the endpoints + * that will not be considered cached. If we use them to set the new + * bounds of existing ranges then we have to dec/inc them into the range + * to have them represent the last/first valid key, not the first/last + * key to be removed. + * + * Like insert_, we're responsible for freeing the caller's range. We + * might insert it into the tree to track the other half of a range + * that's split by the removal. + */ +static void remove_range(struct super_block *sb, struct rb_root *root, + struct cached_range *rem) +{ + struct cached_range *rng; + struct rb_node *parent; + struct rb_node **node; + bool insert = false; + int start_cmp; + int end_cmp; + int cmp; + +restart: + parent = NULL; + node = &root->rb_node; + while (*node) { + parent = *node; + rng = container_of(*node, struct cached_range, node); + + cmp = scoutfs_key_compare_ranges(rem->start, rem->end, + rng->start, rng->end); + /* simple iteration until we overlap */ + if (cmp < 0) { + node = &(*node)->rb_left; + continue; + } else if (cmp > 0) { + node = &(*node)->rb_right; + continue; + } + + start_cmp = scoutfs_key_compare(rem->start, rng->start); + end_cmp = scoutfs_key_compare(rem->end, rng->end); + + /* remove the middle of an existing range, insert other half */ + if (start_cmp > 0 && end_cmp < 0) { + swap(rng->end, rem->start); + scoutfs_key_dec(rng->end); + + swap(rem->start, rem->end); + scoutfs_key_inc(rem->start); + insert = true; + goto restart; + } + + /* remove partial overlap from existing */ + if (start_cmp < 0 && end_cmp < 0) { + swap(rem->end, rng->start); + scoutfs_key_inc(rng->start); + continue; + } + + if (start_cmp > 0 && end_cmp > 0) { + swap(rem->start, rng->end); + scoutfs_key_dec(rng->end); + continue; + } + + /* erase and free existing surrounded by removal */ + rb_erase(&rng->node, root); + free_range(sb, rng); + goto restart; + } + + if (insert) { + rb_link_node(&rem->node, parent, node); + rb_insert_color(&rem->node, root); + } else { + free_range(sb, rem); + } +} + /* * Find an item with the given key and copy its value into the caller's * value vector. The amount of bytes copied is returned which can be 0 @@ -1577,19 +1663,35 @@ int scoutfs_item_writeback(struct super_block *sb, * The caller wants us to drop any items within the range on the floor. * They should have ensured that items in this range won't be dirty. */ -void scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) +int scoutfs_item_invalidate(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + struct cached_range *rng; struct cached_item *next; struct cached_item *item; struct rb_node *node; unsigned long flags; + int ret; /* XXX think about racing with trans write */ + rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); + if (rng) { + rng->start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + rng->end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + } + if (!rng || !rng->start || !rng->end) { + free_range(sb, rng); + ret = -ENOMEM; + goto out; + } + + scoutfs_key_copy(rng->start, start); + scoutfs_key_copy(rng->end, end); + spin_lock_irqsave(&cac->lock, flags); for (item = next_item(&cac->items, start); @@ -1607,7 +1709,13 @@ void scoutfs_item_invalidate(struct super_block *sb, erase_item(sb, cac, item); } + remove_range(sb, &cac->ranges, rng); + spin_unlock_irqrestore(&cac->lock, flags); + + ret = 0; +out: + return ret; } int scoutfs_item_setup(struct super_block *sb) diff --git a/kmod/src/item.h b/kmod/src/item.h index fced5957..868aeeb6 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -60,9 +60,9 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_item_writeback(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); -void scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end); +int scoutfs_item_invalidate(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end); int scoutfs_item_setup(struct super_block *sb); void scoutfs_item_destroy(struct super_block *sb); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index dd5bdb52..1ded4be4 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -120,7 +120,7 @@ static int invalidate_caches(struct super_block *sb, int mode, return ret; if (mode == SCOUTFS_LOCK_MODE_WRITE) { - scoutfs_item_invalidate(sb, start, end); + ret = scoutfs_item_invalidate(sb, start, end); #if 0 scoutfs_dir_invalidate(sb, start, end) ?: scoutfs_inode_invalidate(sb, start, end) ?: @@ -128,7 +128,7 @@ static int invalidate_caches(struct super_block *sb, int mode, #endif } - return 0; + return ret; } #define for_each_other_linf(linf, from_linf) \ From 86d30909829a75c4160a7a8c25fc95704ba857df Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 17 Mar 2017 09:41:42 -0700 Subject: [PATCH 246/920] Tighten lock range error handling If lock_range returns an error then the caller won't unlock the range. Make sure to unlock the range if we have it locked when we get errors that we're going to return to the caller. Signed-off-by: Zach Brown --- kmod/src/lock.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 1ded4be4..e9f1c5a4 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -211,9 +211,6 @@ int scoutfs_lock_range_lvb(struct super_block *sb, int mode, goto out; if (linf->shutdown) { - /* unlocked, but we own it */ - if (!list_empty(&lck->head)) - unlock(held, lck); ret = -ESHUTDOWN; goto out; } @@ -231,8 +228,11 @@ int scoutfs_lock_range_lvb(struct super_block *sb, int mode, memcpy(caller_lvb, held->fake_lvb, lvb_len); } } + ret = 0; out: + if (ret < 0 && !list_empty(&lck->head)) + unlock(held, lck); return ret; } From 5e0e9ac12eeceada5bfe83e1273226f03c101e75 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 10 Apr 2017 10:09:52 -0700 Subject: [PATCH 247/920] Move to much simpler manifest/alloc storage Using the treap to be able to incrementally read and write the manifest and allocation storage from all nodes wasn't quite ready for prime time. The biggest problem is that invalidating cached nodes which are the target of native pointers, either for consistency or memory pressure, is problematic. This was getting in the way of adding shared support as readers and writers try to use as much of their treap caches as they can. There were other serious problems that we'd run into eventually: memory pressure from duplicate caching in native nodes and the page cache, small IOs from reading a page at a time, the risk of pathologically imbalanced treaps, and the ring being corrupted if the migration balancing doesn't work (the model assumed you could always dirty an individual node in a transaction, you have to dirty all the parents in each new transaction). Let's back off to a much simpler mechanism while we build the rest of the system around it. We can revisit aggressively optimizing this when it's our worst problem. We'll store the indexes that the manifest server needs in simple preallocated rings with log entries. The server has to read the index in its entirety into a native rbtree before it can work on it. We won't access the physical ring from mounts anymore, they'll send messages to the server. The ring callers are now working with a pinned tree in memory so the interface can be a bit simpler. By storing the indexes in their own rings the code and write path become a lot simper: we have an IO submission path for each index instead of "dirtying" calls per index and then a writing call. All this is much more robust and much less likely to get in our way as we stand up the rest of the system around it. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/alloc.c | 92 +-- kmod/src/alloc.h | 5 +- kmod/src/compact.c | 11 +- kmod/src/format.h | 54 +- kmod/src/manifest.c | 186 +++--- kmod/src/manifest.h | 5 +- kmod/src/ring.c | 803 ++++++++++++++++++++++++++ kmod/src/ring.h | 55 ++ kmod/src/super.c | 4 - kmod/src/super.h | 2 - kmod/src/trans.c | 9 +- kmod/src/treap.c | 1349 ------------------------------------------- kmod/src/treap.h | 47 -- 14 files changed, 1027 insertions(+), 1599 deletions(-) create mode 100644 kmod/src/ring.c create mode 100644 kmod/src/ring.h delete mode 100644 kmod/src/treap.c delete mode 100644 kmod/src/treap.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 1448caa6..ecb5f39b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ - ioctl.o item.o key.o lock.o manifest.o msg.o net.o seg.o \ - scoutfs_trace.o super.o trans.o treap.o xattr.o + ioctl.o item.o key.o lock.o manifest.o msg.o net.o ring.o seg.o \ + scoutfs_trace.o super.o trans.o xattr.o diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 5be9d490..fa6676d1 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -17,13 +17,13 @@ #include "super.h" #include "format.h" -#include "treap.h" +#include "ring.h" #include "cmp.h" #include "alloc.h" #include "counters.h" /* - * scoutfs allocates segments by storing regions of a bitmap in treap + * scoutfs allocates segments by storing regions of a bitmap in ring * nodes. * * Freed segments are recorded in nodes in an rbtree. The frees can't @@ -40,7 +40,7 @@ struct seg_alloc { struct rw_semaphore rwsem; struct rb_root pending_root; - struct scoutfs_treap *treap; + struct scoutfs_ring_info ring; u64 next_segno; }; @@ -132,7 +132,7 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; do { - reg = scoutfs_treap_lookup_next_dirty(sal->treap, &ind); + reg = scoutfs_ring_lookup_next(&sal->ring, &ind); } while (reg == NULL && ind && (ind = 0, nr = 0, 1)); if (IS_ERR_OR_NULL(reg)) { @@ -143,6 +143,8 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) goto out; } + scoutfs_ring_dirty(&sal->ring, reg); + nr = find_next_bit_le(reg->bits, SCOUTFS_ALLOC_REGION_BITS, nr); if (nr >= SCOUTFS_ALLOC_REGION_BITS) { /* XXX corruption? shouldn't find empty regions */ @@ -154,12 +156,8 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) clear_bit_le(nr, reg->bits); - if (empty_region(reg)) { - ret = scoutfs_treap_delete(sal->treap, &ind); - /* XXX figure out what to do about this inconsistency */ - if (WARN_ON_ONCE(ret)) - goto out; - } + if (empty_region(reg)) + scoutfs_ring_delete(&sal->ring, reg); *segno = (ind << SCOUTFS_ALLOC_REGION_SHIFT) + nr; sal->next_segno = *segno + 1; @@ -178,7 +176,7 @@ out: /* * Record newly freed sgements in pending regions. These are applied to - * treap nodes as the transaction commits. + * ring nodes as the transaction commits. */ int scoutfs_alloc_free(struct super_block *sb, u64 segno) { @@ -234,7 +232,8 @@ int scoutfs_alloc_has_dirty(struct super_block *sb) int ret; down_write(&sal->rwsem); - ret = scoutfs_treap_has_dirty(sal->treap); + ret = !!(scoutfs_ring_has_dirty(&sal->ring) || + !RB_EMPTY_ROOT(&sal->pending_root)); up_write(&sal->rwsem); return ret; @@ -242,13 +241,12 @@ int scoutfs_alloc_has_dirty(struct super_block *sb) /* * First we apply the pending frees to create the final set of dirty - * region nodes and then ask the treap to write them to ring pages. + * region nodes and then ask the ring to write them to the ring. */ -int scoutfs_alloc_dirty_ring(struct super_block *sb) +int scoutfs_alloc_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp) { DECLARE_SEG_ALLOC(sb, sal); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct scoutfs_alloc_region *reg; struct pending_region *pend; struct rb_node *node; @@ -262,30 +260,41 @@ int scoutfs_alloc_dirty_ring(struct super_block *sb) ind = le64_to_cpu(pend->reg.index); - reg = scoutfs_treap_lookup_dirty(sal->treap, &ind); - if (!reg) - reg = scoutfs_treap_insert(sal->treap, &ind, - sizeof(struct scoutfs_alloc_region), - &ind); - if (IS_ERR(reg)) { - ret = PTR_ERR(reg); - goto out; + reg = scoutfs_ring_lookup(&sal->ring, &ind); + if (!reg) { + reg = scoutfs_ring_insert(&sal->ring, &ind, + sizeof(struct scoutfs_alloc_region)); + if (!reg) { + ret = -ENOMEM; + goto out; + } + + memset(reg, 0, sizeof(struct scoutfs_alloc_region)); + reg->index = cpu_to_le64(ind); } - reg->index = pend->reg.index; or_region_bits(reg, &pend->reg); + scoutfs_ring_dirty(&sal->ring, reg); rb_erase(&pend->node, &sal->pending_root); kfree(pend); } - scoutfs_treap_dirty_ring(sal->treap, &super->alloc_treap_root); - ret = 0; + ret = scoutfs_ring_submit_write(sb, &sal->ring, comp); out: up_write(&sal->rwsem); return ret; } +void scoutfs_alloc_write_complete(struct super_block *sb) +{ + DECLARE_SEG_ALLOC(sb, sal); + + down_write(&sal->rwsem); + scoutfs_ring_write_complete(&sal->ring); + up_write(&sal->rwsem); +} + /* * Return the number of blocks free for statfs. */ @@ -303,7 +312,7 @@ u64 scoutfs_alloc_bfree(struct super_block *sb) return bfree; } -static int alloc_treap_compare(void *key, void *data) +static int alloc_ring_compare_key(void *key, void *data) { u64 *ind = key; struct scoutfs_alloc_region *reg = data; @@ -311,25 +320,20 @@ static int alloc_treap_compare(void *key, void *data) return scoutfs_cmp_u64s(*ind, le64_to_cpu(reg->index)); } -static void alloc_treap_fill(void *data, void *fill_arg) +static int alloc_ring_compare_data(void *A, void *B) { - struct scoutfs_alloc_region *reg = data; - u64 *ind = fill_arg; + struct scoutfs_alloc_region *a = A; + struct scoutfs_alloc_region *b = B; - memset(reg, 0, sizeof(struct scoutfs_alloc_region)); - reg->index = cpu_to_le64p(ind); + return scoutfs_cmp_u64s(le64_to_cpu(a->index), le64_to_cpu(b->index)); } -static struct scoutfs_treap_ops alloc_treap_ops = { - .compare = alloc_treap_compare, - .fill = alloc_treap_fill, -}; - int scoutfs_alloc_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct seg_alloc *sal; + int ret; /* bits need to be aligned so hosts can use native bitops */ BUILD_BUG_ON(offsetof(struct scoutfs_alloc_region, bits) & @@ -341,11 +345,13 @@ int scoutfs_alloc_setup(struct super_block *sb) init_rwsem(&sal->rwsem); sal->pending_root = RB_ROOT; - sal->treap = scoutfs_treap_alloc(sb, &alloc_treap_ops, - &super->alloc_treap_root); - if (!sal->treap) { + scoutfs_ring_init(&sal->ring, &super->alloc_ring, + alloc_ring_compare_key, alloc_ring_compare_data); + + ret = scoutfs_ring_load(sb, &sal->ring); + if (ret) { kfree(sal); - return -ENOMEM; + return ret; } /* XXX read next_segno from super? */ @@ -362,7 +368,7 @@ void scoutfs_alloc_destroy(struct super_block *sb) struct rb_node *node; if (sal) { - scoutfs_treap_free(sal->treap); + scoutfs_ring_destroy(&sal->ring); while ((node = rb_first(&sal->pending_root))) { pend = container_of(node, struct pending_region, node); rb_erase(&pend->node, &sal->pending_root); diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index 2a400e64..bb185d90 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -2,12 +2,15 @@ #define _SCOUTFS_ALLOC_H_ struct scoutfs_alloc_region; +struct scoutfs_bio_completion; int scoutfs_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_alloc_free(struct super_block *sb, u64 segno); int scoutfs_alloc_has_dirty(struct super_block *sb); -int scoutfs_alloc_dirty_ring(struct super_block *sb); +int scoutfs_alloc_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp); +void scoutfs_alloc_write_complete(struct super_block *sb); u64 scoutfs_alloc_bfree(struct super_block *sb); int scoutfs_alloc_setup(struct super_block *sb); diff --git a/kmod/src/compact.c b/kmod/src/compact.c index cce6715f..cae9d7f6 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -522,11 +522,12 @@ out: /* * Atomically update the manifest. We lock down the manifest so no one - * can use it while we're mucking with it. We can always delete dirty - * treap nodes without failure. So we first dirty the deletion nodes - * before modifying anything. Then we add and if any of those fail we - * can delete the dirty previous additions. Then we can delete the - * dirty existing entries without failure. + * can use it while we're mucking with it. While the current ring can + * always delete without failure we will probably have a manifest + * storage layer eventually that could return errors on deletion. We + * also also have corrupted something and try to delete an entry that + * doesn't exist. So we use an initial dirtying step to ensure that our + * later deletions succeed. * * XXX does locking the manifest prevent commits? I would think so? */ diff --git a/kmod/src/format.h b/kmod/src/format.h index 0fbe3c40..25cb3878 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -50,43 +50,30 @@ struct scoutfs_block_header { __le64 blkno; } __packed; -struct scoutfs_treap_ref { - __le64 off; - __le64 gen; - __u8 aug_bits; +struct scoutfs_ring_entry { + __le16 data_len; + __u8 flags; + __u8 data[0]; } __packed; -/* - * The lesser and greater bits are persistent on disk so that we can migrate - * nodes from the older half of the ring. - * - * The dirty bit is only used for in-memory nodes. - */ -#define SCOUTFS_TREAP_AUG_LESSER (1 << 0) -#define SCOUTFS_TREAP_AUG_GREATER (1 << 1) -#define SCOUTFS_TREAP_AUG_HALVES (SCOUTFS_TREAP_AUG_LESSER | \ - SCOUTFS_TREAP_AUG_GREATER) -#define SCOUTFS_TREAP_AUG_DIRTY (1 << 2) +#define SCOUTFS_RING_ENTRY_FLAG_DELETION (1 << 0) -/* - * Treap nodes are stored at byte offset in the ring of blocks described - * by the super block. Each reference contains the off and gen that it - * will find in the node for verification. Each node has the header - * and data payload covered by a crc. - */ -struct scoutfs_treap_node { +struct scoutfs_ring_block { __le32 crc; - __le64 off; - __le64 gen; - __le64 prio; - struct scoutfs_treap_ref left; - struct scoutfs_treap_ref right; - __le16 bytes; - u8 data[0]; + __le32 pad; + __le64 fsid; + __le64 seq; + __le64 block; + __le32 nr_entries; + struct scoutfs_ring_entry entries[0]; } __packed; -struct scoutfs_treap_root { - struct scoutfs_treap_ref ref; +struct scoutfs_ring_descriptor { + __le64 blkno; + __le64 total_blocks; + __le64 first_block; + __le64 first_seq; + __le64 nr_blocks; } __packed; /* @@ -98,7 +85,7 @@ struct scoutfs_treap_root { #define SCOUTFS_MANIFEST_FANOUT 10 struct scoutfs_manifest { - struct scoutfs_treap_root root; + struct scoutfs_ring_descriptor ring; __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; } __packed; @@ -246,6 +233,7 @@ struct scoutfs_symlink_key { #define SCOUTFS_UUID_BYTES 16 + /* * The ring fields describe the statically allocated ring log. The * head and tail indexes are logical 4k blocks offsets inside the ring. @@ -264,7 +252,7 @@ struct scoutfs_super_block { __le64 ring_tail_block; __le64 ring_gen; __le64 next_seg_seq; - struct scoutfs_treap_root alloc_treap_root; + struct scoutfs_ring_descriptor alloc_ring; struct scoutfs_manifest manifest; } __packed; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index fd2aeda8..6c9e72cb 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -20,7 +20,7 @@ #include "kvec.h" #include "seg.h" #include "item.h" -#include "treap.h" +#include "ring.h" #include "cmp.h" #include "compact.h" #include "manifest.h" @@ -29,24 +29,17 @@ #include "scoutfs_trace.h" /* - * Manifest entries are stored as treap nodes in the ring. + * Manifest entries are stored in ring nodes. * * They're sorted first by level then by their first key. This enables * the primary searches based on key value for looking up items in * segments via the manifest. - * - * The treap also supports augmented searches. We get callbacks as the - * tree structure which lets us maintain data in nodes that describe - * subtrees to accelerate searches. We will record the max sequence - * numbers in subtrees for all the seq queries. We'll probably also - * have bits that direct us towards segments that contain deletion items - * for prioritized compaction. */ struct manifest { struct rw_semaphore rwsem; seqcount_t seqcount; - struct scoutfs_treap *treap; + struct scoutfs_ring_info ring; u8 nr_levels; /* calculated on mount, const thereafter */ @@ -81,12 +74,6 @@ struct manifest_ref { struct scoutfs_key_buf *last; }; -struct manifest_fill_args { - struct scoutfs_manifest_entry ment; - struct scoutfs_key_buf *first; - struct scoutfs_key_buf *last; -}; - /* * Seq is only specified for operations that differentiate between * segments with identical items by their sequence number. @@ -174,7 +161,7 @@ static void add_level_count(struct super_block *sb, struct manifest *mani, } /* - * Insert a new manifest entry in the treap. The treap allocates a new + * Insert a new manifest entry in the ring. The ring allocates a new * node for us and we fill it. * * This must be called with the manifest lock held. @@ -188,40 +175,38 @@ int scoutfs_manifest_add(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_manifest_entry *ment; - struct manifest_fill_args args; + struct scoutfs_key_buf ment_first; + struct scoutfs_key_buf ment_last; struct manifest_search_key skey; unsigned key_bytes; unsigned bytes; - int ret; trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); key_bytes = first->key_len + last->key_len; bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); - args.ment.segno = cpu_to_le64(segno); - args.ment.seq = cpu_to_le64(seq); - args.ment.first_key_len = cpu_to_le16(first->key_len); - args.ment.last_key_len = cpu_to_le16(last->key_len); - args.ment.level = level; - - args.first = first; - args.last = last; - skey.key = first; skey.level = level; skey.seq = seq; - ment = scoutfs_treap_insert(mani->treap, &skey, bytes, &args); - if (IS_ERR(ment)) { - ret = PTR_ERR(ment); - } else { - mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - add_level_count(sb, mani, super, level, 1); - ret = 0; - } + ment = scoutfs_ring_insert(&mani->ring, &skey, bytes); + if (!ment) + return -ENOMEM; - return ret; + ment->segno = cpu_to_le64(segno); + ment->seq = cpu_to_le64(seq); + ment->first_key_len = cpu_to_le16(first->key_len); + ment->last_key_len = cpu_to_le16(last->key_len); + ment->level = level; + + init_ment_keys(ment, &ment_first, &ment_last); + scoutfs_key_copy(&ment_first, first); + scoutfs_key_copy(&ment_last, last); + + mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); + add_level_count(sb, mani, super, level, 1); + return 0; } /* @@ -238,11 +223,11 @@ int scoutfs_manifest_dirty(struct super_block *sb, skey.level = level; skey.seq = seq; - ment = scoutfs_treap_lookup_dirty(mani->treap, &skey); - if (IS_ERR(ment)) - return PTR_ERR(ment); + ment = scoutfs_ring_lookup(&mani->ring, &skey); if (!ment) return -ENOENT; + + scoutfs_ring_dirty(&mani->ring, ment); return 0; } @@ -255,18 +240,20 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; - int ret; skey.key = first; skey.level = level; skey.seq = seq; - ret = scoutfs_treap_delete(mani->treap, &skey); - if (ret == 0) - add_level_count(sb, mani, super, level, -1ULL); + ment = scoutfs_ring_lookup(&mani->ring, &skey); + if (!ment) + return -ENOENT; - return ret; + scoutfs_ring_delete(&mani->ring, ment); + add_level_count(sb, mani, super, level, -1ULL); + return 0; } /* @@ -363,19 +350,15 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, /* get level 0 segments that overlap with the missing range */ skey.level = 0; skey.seq = ~0ULL; - ment = scoutfs_treap_lookup_prev(mani->treap, &skey); - while (!IS_ERR_OR_NULL(ment)) { + ment = scoutfs_ring_lookup_prev(&mani->ring, &skey); + while (ment) { if (cmp_range_ment(key, end, ment) == 0) { ret = alloc_add_ref(sb, ref_list, ment); if (ret) goto out; } - ment = scoutfs_treap_prev(mani->treap, ment); - } - if (IS_ERR(ment)) { - ret = PTR_ERR(ment); - goto out; + ment = scoutfs_ring_prev(&mani->ring, ment); } /* get higher level segments that overlap with the starting key */ @@ -386,12 +369,7 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, /* XXX should use level counts to skip searches */ - ment = scoutfs_treap_lookup(mani->treap, &skey); - if (IS_ERR(ment)) { - ret = PTR_ERR(ment); - goto out; - } - + ment = scoutfs_ring_lookup(&mani->ring, &skey); if (ment) { init_ment_keys(ment, &first, &last); ret = alloc_add_ref(sb, ref_list, ment); @@ -625,28 +603,32 @@ int scoutfs_manifest_has_dirty(struct super_block *sb) int ret; down_write(&mani->rwsem); - ret = scoutfs_treap_has_dirty(mani->treap); + ret = scoutfs_ring_has_dirty(&mani->ring); up_write(&mani->rwsem); return ret; } -/* - * Append the dirty manifest entries to the end of the ring. - * - * This returns 0 but can't fail. - */ -int scoutfs_manifest_dirty_ring(struct super_block *sb) +int scoutfs_manifest_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + int ret; down_write(&mani->rwsem); - scoutfs_treap_dirty_ring(mani->treap, &super->manifest.root); + ret = scoutfs_ring_submit_write(sb, &mani->ring, comp); up_write(&mani->rwsem); - return 0; + return ret; +} + +void scoutfs_manifest_write_complete(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + + down_write(&mani->rwsem); + scoutfs_ring_write_complete(&mani->ring); + up_write(&mani->rwsem); } u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level) @@ -714,23 +696,19 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) /* find the oldest level 0 or the next higher order level by key */ if (level == 0) { - ment = scoutfs_treap_first(mani->treap); - if (!IS_ERR_OR_NULL(ment) && ment->level) + ment = scoutfs_ring_first(&mani->ring); + if (ment && ment->level) ment = NULL; } else { skey.key = mani->compact_keys[level]; skey.level = level; skey.seq = 0; - ment = scoutfs_treap_lookup_next(mani->treap, &skey); + ment = scoutfs_ring_lookup_next(&mani->ring, &skey); if (ment == NULL || ment->level != level) { scoutfs_key_set_min(skey.key); - ment = scoutfs_treap_lookup_next(mani->treap, &skey); + ment = scoutfs_ring_lookup_next(&mani->ring, &skey); } } - if (IS_ERR(ment)) { - ret = PTR_ERR(ment); - goto out; - } if (ment == NULL || ment->level != level) { /* XXX shouldn't be possible */ ret = 0; @@ -750,14 +728,10 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) skey.key = &ment_first; skey.level = level + 1; skey.seq = 0; - over = scoutfs_treap_lookup_next(mani->treap, &skey); + over = scoutfs_ring_lookup_next(&mani->ring, &skey); /* and add a fanout's worth of lower overlapping segments */ for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { - if (IS_ERR(over)) { - ret = PTR_ERR(over); - goto out; - } if (!over || over->level != (ment->level + 1)) break; @@ -773,7 +747,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) if (ret) goto out; - over = scoutfs_treap_next(mani->treap, over); + over = scoutfs_ring_next(&mani->ring, over); } /* record the next key to start from */ @@ -787,7 +761,7 @@ out: } /* - * Manifest entries for all levels are stored in a single treap. + * Manifest entries for all levels are stored in a single ring. * * First they're sorted by their level. * @@ -806,7 +780,7 @@ out: * number. We tell the difference by the presence of a sequence number. * A segment will never have a seq of 0. */ -static int manifest_treap_compare(void *key, void *data) +static int manifest_ring_compare_key(void *key, void *data) { struct manifest_search_key *skey = key; struct scoutfs_manifest_entry *ment = data; @@ -842,32 +816,27 @@ out: return cmp; } -static void manifest_treap_fill(void *data, void *arg) +static int manifest_ring_compare_data(void *a, void *b) { - struct scoutfs_manifest_entry *ment = data; - struct manifest_fill_args *args = arg; - struct scoutfs_key_buf ment_first; - struct scoutfs_key_buf ment_last; + struct manifest_search_key skey; + struct scoutfs_manifest_entry *ment = a; + struct scoutfs_key_buf key; - *ment = args->ment; + init_ment_keys(ment, &key, NULL); - init_ment_keys(ment, &ment_first, &ment_last); - scoutfs_key_copy(&ment_first, args->first); - scoutfs_key_copy(&ment_last, args->last); + skey.seq = le64_to_cpu(ment->seq); + skey.key = &key; + skey.level = ment->level; + + return manifest_ring_compare_key(&skey, b); } -static struct scoutfs_treap_ops manifest_treap_ops = { - .compare = manifest_treap_compare, - .fill = manifest_treap_fill, - /* update aug when we track left and right max seq */ -}; - - int scoutfs_manifest_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct manifest *mani; + int ret; int i; mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); @@ -876,12 +845,13 @@ int scoutfs_manifest_setup(struct super_block *sb) init_rwsem(&mani->rwsem); seqcount_init(&mani->seqcount); - - mani->treap = scoutfs_treap_alloc(sb, &manifest_treap_ops, - &super->manifest.root); - if (!mani->treap) { + scoutfs_ring_init(&mani->ring, &super->manifest.ring, + manifest_ring_compare_key, + manifest_ring_compare_data); + ret = scoutfs_ring_load(sb, &mani->ring); + if (ret) { kfree(mani); - return -ENOMEM; + return ret; } for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) { @@ -890,7 +860,7 @@ int scoutfs_manifest_setup(struct super_block *sb) if (!mani->compact_keys[i]) { while (--i >= 0) scoutfs_key_free(sb, mani->compact_keys[i]); - scoutfs_treap_free(mani->treap); + scoutfs_ring_destroy(&mani->ring); kfree(mani); return -ENOMEM; } @@ -925,7 +895,7 @@ void scoutfs_manifest_destroy(struct super_block *sb) int i; if (mani) { - scoutfs_treap_free(mani->treap); + scoutfs_ring_destroy(&mani->ring); for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) scoutfs_key_free(sb, mani->compact_keys[i]); kfree(mani); diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index d788aeaf..25cf236a 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -2,6 +2,7 @@ #define _SCOUTFS_MANIFEST_H_ struct scoutfs_key_buf; +struct scoutfs_bio_completion; int scoutfs_manifest_add(struct super_block *sb, struct scoutfs_key_buf *first, @@ -12,7 +13,9 @@ int scoutfs_manifest_dirty(struct super_block *sb, int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, u64 seq, u8 level); int scoutfs_manifest_has_dirty(struct super_block *sb); -int scoutfs_manifest_dirty_ring(struct super_block *sb); +int scoutfs_manifest_submit_write(struct super_block *sb, + struct scoutfs_bio_completion *comp); +void scoutfs_manifest_write_complete(struct super_block *sb); int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); diff --git a/kmod/src/ring.c b/kmod/src/ring.c new file mode 100644 index 00000000..00809b19 --- /dev/null +++ b/kmod/src/ring.c @@ -0,0 +1,803 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "bio.h" +#include "ring.h" + +/* + * scoutfs stores the persistent indexes for the server in a simple log + * entries in a preallocated ring of blocks. + * + * The index is read from the log and loaded in to an rbtree in memory. + * Callers then lock around operations that work on the rbtrees. Dirty + * and deleted nodes are tracked and are eventually copied to pages that + * are written to the tail of the log. + * + * This has the great benefit of updating an index with very few (often + * one) contiguous block writes with low write amplification. + * + * This has the significant cost of requiring reading the indexes in to + * memory before doing any work and then having to hold them resident. + * This is fine for now but we'll have to address these latency and + * capacity limitations before too long. + * + * Callers are entirely responsible for locking. + */ + +/* + * XXX + * - deletion entries could be smaller if we understood keys + * - shouldn't be too hard to compress + */ + +/* + * @block records the logical ring index of the block that contained the + * node. As we commit a ring update we can look at the clean list to + * find the first block that we have to read out of the ring. This + * helps minimize the active region of the ring. + * + * @in_ring is used to mark nodes that were present in the ring and + * which need deletion entries written to the ring before they can be + * freed. + */ +struct ring_node { + struct rb_node rb_node; + struct list_head head; + u64 block; + + u16 data_len; + + u8 dirty:1, + deleted:1, + in_ring:1; + + /* data is packed but callers perform native long bitops */ + u8 data[0] __aligned(__alignof__(long)); +}; + +static struct ring_node *data_rnode(void *data) +{ + return data ? container_of(data, struct ring_node, data) : NULL; +} + +static void *rnode_data(struct ring_node *rnode) +{ + return rnode ? rnode->data : NULL; +} + +static unsigned total_entry_bytes(unsigned data_len) +{ + return offsetof(struct scoutfs_ring_entry, data[data_len]); +} + +/* + * Each time we mark a node dirty we also dirty the oldest clean entry. + * This ensures that we never overwrite stable data. + * + * Picture a ring of blocks where the first half of the ring is full of + * existing entries. Imagine that we continuously update a set of + * entries that make up a single block. Each new update block + * invalidates the previous update block but it advances through the + * ring while the old entries are sitting idle in the first half. + * Eventually the new update blocks wrap around and clobber the old + * blocks. + * + * Now instead imagine that each time we dirty an entry in this set of + * constantly changing entries that we also go and dirty the earliest + * existing entry in the ring. Now each update is a block of the + * useless updating entries and a block of old entries that have been + * migrated. Each time we write two blocks to the ring we migrate one + * block from the start of the ring. Now by the time we fill the second + * half of the ring we've reclaimed half of the first half of the ring. + * + * So we size the ring to fit 4x the largest possible index. Now we're + * sure that we'll be able to fully migrate the index from the first + * half of the ring into the second half before it wraps around and + * starts overwriting the first. + */ +static void mark_node_dirty(struct scoutfs_ring_info *ring, + struct ring_node *rnode, bool migrate) +{ + struct ring_node *pos; + long total; + + if (!rnode || rnode->dirty) + return; + + list_move_tail(&rnode->head, &ring->dirty_list); + rnode->dirty = 1; + ring->dirty_bytes += total_entry_bytes(rnode->data_len); + + if (migrate) { + total = total_entry_bytes(rnode->data_len); + + list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { + mark_node_dirty(ring, rnode, false); + total -= total_entry_bytes(rnode->data_len); + if (total < 0) + break; + } + } +} + +static void mark_node_clean(struct scoutfs_ring_info *ring, + struct ring_node *rnode) +{ + if (!rnode || !rnode->dirty) + return; + + list_move_tail(&rnode->head, &ring->clean_list); + rnode->dirty = 0; + ring->dirty_bytes -= total_entry_bytes(rnode->data_len); +} + +static void free_node(struct scoutfs_ring_info *ring, + struct ring_node *rnode) +{ + if (rnode) { + mark_node_clean(ring, rnode); + + if (!list_empty(&rnode->head)) + list_del_init(&rnode->head); + if (!RB_EMPTY_NODE(&rnode->rb_node)) + rb_erase(&rnode->rb_node, &ring->rb_root); + + kfree(rnode); + } +} + +/* + * Walk the tree and return the last node traversed. cmp gives the + * caller the comparison between their key and the returned node. The + * caller can provide either their key or another nodes data to compare + * with during descent. If we're asked to insert we replace any node we + * find in the key's place. + */ +static struct ring_node *ring_rb_walk(struct scoutfs_ring_info *ring, + void *key, void *data, + struct ring_node *ins, + int *cmp) +{ + struct rb_node **node = &ring->rb_root.rb_node; + struct rb_node *parent = NULL; + struct ring_node *found = NULL; + struct ring_node *rnode; + + /* only provide one or the other */ + BUG_ON(!!key == !!data); + + while (*node) { + parent = *node; + rnode = container_of(*node, struct ring_node, rb_node); + + if (key) + *cmp = ring->compare_key(key, &rnode->data); + else + *cmp = ring->compare_data(data, &rnode->data); + + if (*cmp < 0) { + node = &(*node)->rb_left; + } else if (*cmp > 0) { + node = &(*node)->rb_right; + } else { + found = rnode; + break; + } + } + + if (ins) { + if (found) { + rb_replace_node(&found->rb_node, &ins->rb_node, + &ring->rb_root); + RB_CLEAR_NODE(&found->rb_node); + free_node(ring, found); + } else { + rb_link_node(&ins->rb_node, parent, node); + rb_insert_color(&ins->rb_node, &ring->rb_root); + } + found = ins; + } + + return found; +} + +static struct ring_node *ring_rb_entry(struct rb_node *node) +{ + return node ? rb_entry(node, struct ring_node, rb_node) : NULL; +} + +/* return the next node, skipping deleted */ +static struct ring_node *ring_rb_next(struct ring_node *rnode) +{ + do { + if (rnode) + rnode = ring_rb_entry(rb_next(&rnode->rb_node)); + } while (rnode && rnode->deleted); + + return rnode; +} + +/* return the prev node, skipping deleted */ +static struct ring_node *ring_rb_prev(struct ring_node *rnode) +{ + do { + if (rnode) + rnode = ring_rb_entry(rb_prev(&rnode->rb_node)); + } while (rnode && rnode->deleted); + + return rnode; +} + +/* return the first node, skipping deleted */ +static struct ring_node *ring_rb_first(struct scoutfs_ring_info *ring) +{ + struct ring_node *rnode; + + rnode = ring_rb_entry(rb_first(&ring->rb_root)); + if (rnode && rnode->deleted) + rnode = ring_rb_next(rnode); + return rnode; +} + +static struct ring_node *alloc_node(unsigned data_len) +{ + struct ring_node *rnode; + + rnode = kzalloc(offsetof(struct ring_node, data[data_len]), GFP_NOFS); + if (rnode) { + RB_CLEAR_NODE(&rnode->rb_node); + INIT_LIST_HEAD(&rnode->head); + rnode->data_len = data_len; + } + + return rnode; +} + +/* + * Insert a new node. This will replace any existing node which could + * be in any state. + */ +void *scoutfs_ring_insert(struct scoutfs_ring_info *ring, void *key, + unsigned data_len) +{ + struct ring_node *rnode; + int cmp; + + rnode = alloc_node(data_len); + if (!rnode) + return NULL; + + ring_rb_walk(ring, key, NULL, rnode, &cmp); + /* just put it on a list, dirtying moves it to dirty */ + list_add_tail(&rnode->head, &ring->dirty_list); + mark_node_dirty(ring, rnode, true); + + return rnode->data; +} + +void *scoutfs_ring_first(struct scoutfs_ring_info *ring) +{ + return rnode_data(ring_rb_first(ring)); +} + +void *scoutfs_ring_lookup(struct scoutfs_ring_info *ring, void *key) +{ + struct ring_node *rnode; + int cmp; + + rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); + if (rnode && (cmp || rnode->deleted)) + rnode = NULL; + + return rnode_data(rnode); +} + +void *scoutfs_ring_lookup_next(struct scoutfs_ring_info *ring, void *key) +{ + struct ring_node *rnode; + int cmp; + + rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); + if (rnode && (cmp > 1 || rnode->deleted)) + rnode = ring_rb_next(rnode); + + return rnode_data(rnode); +} + +void *scoutfs_ring_lookup_prev(struct scoutfs_ring_info *ring, void *key) +{ + struct ring_node *rnode; + int cmp; + + rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); + if (rnode && (cmp < 1 || rnode->deleted)) + rnode = ring_rb_prev(rnode); + + return rnode_data(rnode); +} + +void *scoutfs_ring_next(struct scoutfs_ring_info *ring, void *data) +{ + return rnode_data(ring_rb_next(data_rnode(data))); +} + +void *scoutfs_ring_prev(struct scoutfs_ring_info *ring, void *data) +{ + return rnode_data(ring_rb_prev(data_rnode(data))); +} + +/* + * Calculate the most blocks we could have to use to store a given number + * of bytes of entries. At worst each block has a header and leaves one + * less than the max manifest entry unused. + */ +static unsigned most_blocks(unsigned long bytes) +{ + unsigned long space; + + space = SCOUTFS_BLOCK_SIZE - + sizeof(struct scoutfs_ring_block) - + (sizeof(struct scoutfs_manifest_entry) + + (2 * SCOUTFS_MAX_KEY_SIZE) - 1); + + return DIV_ROUND_UP(bytes, space); +} + +static u64 wrap_ring_block(struct scoutfs_ring_descriptor *rdesc, u64 block) +{ + if (block >= le64_to_cpu(rdesc->total_blocks)) + block -= le64_to_cpu(rdesc->total_blocks); + + /* XXX callers should have verified on load */ + BUG_ON(block >= le64_to_cpu(rdesc->total_blocks)); + + return block; +} + +static u64 calc_first_dirty_block(struct scoutfs_ring_descriptor *rdesc) +{ + return wrap_ring_block(rdesc, le64_to_cpu(rdesc->first_block) + + le64_to_cpu(rdesc->nr_blocks)); +} + +static __le32 rblk_crc(struct scoutfs_ring_block *rblk) +{ + unsigned long skip = (char *)(&rblk->crc + 1) - (char *)rblk; + + return cpu_to_le32(crc32c(~0, (char *)rblk + skip, + SCOUTFS_BLOCK_SIZE - skip)); +} + +/* + * This is called after the caller has copied all the dirty nodes into + * blocks in pages for writing. We might be able to dirty a few more + * clean nodes to fill up the end of the last dirty block to keep the + * ring blocks densely populated. + */ +static void fill_last_dirty_block(struct scoutfs_ring_info *ring, + unsigned space) +{ + struct ring_node *rnode; + struct ring_node *pos; + unsigned tot; + + list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { + + tot = total_entry_bytes(rnode->data_len); + if (tot > space) + break; + + mark_node_dirty(ring, rnode, false); + space -= tot; + } +} + +void scoutfs_ring_dirty(struct scoutfs_ring_info *ring, void *data) +{ + struct ring_node *rnode; + + rnode = data_rnode(data); + if (rnode) + mark_node_dirty(ring, rnode, true); +} + +/* + * Delete the given node. This can free the node so the caller cannot + * use the data after calling this. + * + * If the node previously existed in the ring then we have to save it and + * write a deletion entry before freeing it. + */ +void scoutfs_ring_delete(struct scoutfs_ring_info *ring, void *data) +{ + struct ring_node *rnode = data_rnode(data); + + BUG_ON(rnode->deleted); + + if (rnode->in_ring) { + rnode->deleted = 1; + mark_node_dirty(ring, rnode, true); + } else { + free_node(ring, rnode); + } +} + +static struct scoutfs_ring_block *block_in_pages(struct page **pages, + unsigned i) +{ + return page_address(pages[i / SCOUTFS_BLOCKS_PER_PAGE]) + + ((i % SCOUTFS_BLOCKS_PER_PAGE) << SCOUTFS_BLOCK_SHIFT); +} + +static int load_ring_block(struct scoutfs_ring_info *ring, + struct scoutfs_ring_block *rblk) +{ + struct scoutfs_ring_entry *rent; + struct ring_node *rnode; + unsigned data_len; + unsigned i; + int ret = 0; + int cmp; + + rent = rblk->entries; + for (i = 0; i < le32_to_cpu(rblk->nr_entries); i++) { + + /* XXX verify fields? */ + data_len = le16_to_cpu(rent->data_len); + + if (rent->flags & SCOUTFS_RING_ENTRY_FLAG_DELETION) { + rnode = ring_rb_walk(ring, NULL, rent->data, NULL, + &cmp); + if (rnode && cmp == 0) + free_node(ring, rnode); + } else { + rnode = alloc_node(data_len); + if (!rnode) { + ret = -ENOMEM; + break; + } + + rnode->block = le64_to_cpu(rblk->block); + rnode->in_ring = 1; + memcpy(rnode->data, rent->data, data_len); + + ring_rb_walk(ring, NULL, rnode->data, rnode, &cmp); + list_add_tail(&rnode->head, &ring->clean_list); + } + + rent = (void *)&rent->data[data_len]; + } + + return ret; +} + +/* + * Read the ring entries into rb nodes with nice large synchronous reads. + */ +#define LOAD_BYTES (4 * 1024 * 1024) +#define LOAD_BLOCKS DIV_ROUND_UP(LOAD_BYTES, SCOUTFS_BLOCK_SIZE) +#define LOAD_PAGES DIV_ROUND_UP(LOAD_BYTES, PAGE_SIZE) +int scoutfs_ring_load(struct super_block *sb, struct scoutfs_ring_info *ring) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_ring_descriptor *rdesc = ring->rdesc; + struct scoutfs_ring_block *rblk; + struct page **pages; + unsigned read_nr; + unsigned i; + __le32 crc; + u64 block; + u64 total; + u64 seq; + u64 nr; + int ret; + + pages = kcalloc(LOAD_PAGES, sizeof(struct page *), GFP_NOFS); + if (!pages) + return -ENOMEM; + + for (i = 0; i < LOAD_PAGES; i++) { + pages[i] = alloc_page(GFP_NOFS); + if (!pages[i]) { + ret = -ENOMEM; + goto out; + } + } + + block = le64_to_cpu(rdesc->first_block); + seq = le64_to_cpu(rdesc->first_seq); + total = le64_to_cpu(rdesc->total_blocks); + nr = le64_to_cpu(rdesc->nr_blocks); + + while (nr) { + read_nr = min3(nr, (u64)LOAD_BLOCKS, total - block); + + ret = scoutfs_bio_read(sb, pages, le64_to_cpu(rdesc->blkno) + + block, read_nr); + if (ret) + goto out; + + for (i = 0; i < read_nr; i++) { + rblk = block_in_pages(pages, i); + crc = rblk_crc(rblk); + + if (rblk->fsid != super->hdr.fsid || + le64_to_cpu(rblk->block) != (block + i) || + le64_to_cpu(rblk->seq) != (seq + i) || + rblk->crc != crc) { + ret = -EIO; + goto out; + } + + ret = load_ring_block(ring, rblk); + if (ret) + goto out; + } + + block = wrap_ring_block(rdesc, block + read_nr); + seq += read_nr; + nr -= read_nr; + } + ret = 0; + +out: + for (i = 0; pages && i < LOAD_PAGES && pages[i]; i++) + __free_page(pages[i]); + kfree(pages); + + if (ret) + scoutfs_ring_destroy(ring); + + return ret; +} + +static struct ring_node *first_dirty_node(struct scoutfs_ring_info *ring) +{ + return list_first_entry_or_null(&ring->dirty_list, struct ring_node, + head); +} + +static struct ring_node *next_dirty_node(struct scoutfs_ring_info *ring, + struct ring_node *rnode) +{ + if (rnode->head.next == &ring->dirty_list) + return NULL; + + return list_next_entry(rnode, head); +} + +static void ring_free_pages(struct scoutfs_ring_info *ring) +{ + unsigned i; + + if (!ring->pages) + return; + + for (i = 0; i < ring->nr_pages; i++) { + if (ring->pages[i]) + __free_page(ring->pages[i]); + } + + kfree(ring->pages); + + ring->pages = NULL; + ring->nr_pages = 0; +} + +int scoutfs_ring_has_dirty(struct scoutfs_ring_info *ring) +{ + return !!ring->dirty_bytes; +} + +int scoutfs_ring_submit_write(struct super_block *sb, + struct scoutfs_ring_info *ring, + struct scoutfs_bio_completion *comp) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_ring_descriptor *rdesc = ring->rdesc; + struct scoutfs_ring_block *rblk; + struct scoutfs_ring_entry *rent; + struct ring_node *rnode; + struct ring_node *next; + struct page **pages; + unsigned nr_blocks; + unsigned nr_pages; + unsigned i; + u64 blkno; + u64 block; + u64 first; + u64 last; + u64 seq; + u64 nr; + u8 *end; + int ret; + + if (ring->dirty_bytes == 0) + return 0; + + nr_blocks = most_blocks(ring->dirty_bytes); + nr_pages = DIV_ROUND_UP(nr_blocks, SCOUTFS_BLOCKS_PER_PAGE); + + ring_free_pages(ring); + + pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); + if (!pages) + return -ENOMEM; + + ring->pages = pages; + ring->nr_pages = nr_pages; + + for (i = 0; i < nr_pages; i++) { + pages[i] = alloc_page(GFP_NOFS | __GFP_ZERO); + if (!pages[i]) { + ret = -ENOMEM; + goto out; + } + } + + block = ring->first_dirty_block; + seq = ring->first_dirty_seq; + rnode = first_dirty_node(ring); + + for (i = 0; rnode && i < nr_blocks; i++) { + + rblk = block_in_pages(pages, i); + end = (u8 *)rblk + SCOUTFS_BLOCK_SIZE; + + rblk->fsid = super->hdr.fsid; + rblk->seq = cpu_to_le64(seq); + rblk->block = cpu_to_le64(block); + + rent = rblk->entries; + + while (rnode && &rent->data[rnode->data_len] <= end) { + + rent->data_len = cpu_to_le16(rnode->data_len); + if (rnode->deleted) + rent->flags = SCOUTFS_RING_ENTRY_FLAG_DELETION; + memcpy(rent->data, rnode->data, rnode->data_len); + + le32_add_cpu(&rblk->nr_entries, 1); + + rnode->block = block; + + rent = (void *)&rent->data[le16_to_cpu(rent->data_len)]; + + next = next_dirty_node(ring, rnode); + if (!next) { + fill_last_dirty_block(ring, (char *)end - + (char *)rent); + next = next_dirty_node(ring, rnode); + } + rnode = next; + } + + rblk->crc = rblk_crc(rblk); + + block = wrap_ring_block(rdesc, block + 1); + seq++; + } + + /* update the number of blocks we actually filled */ + nr_blocks = i; + + /* point the descriptor at the new active region of the ring */ + rnode = list_first_entry_or_null(&ring->clean_list, struct ring_node, + head); + if (rnode) + first = rnode->block; + else + first = ring->first_dirty_block; + + last = wrap_ring_block(rdesc, ring->first_dirty_block + nr_blocks); + + if (first < last) + nr = last - first; + else + nr = last + le64_to_cpu(rdesc->total_blocks) - first; + + rdesc->first_block = cpu_to_le64(first); + rdesc->first_seq = cpu_to_le64(ring->first_dirty_seq); + rdesc->nr_blocks = cpu_to_le64(nr); + + /* the contig dirty blocks in pages might wrap around ring */ + blkno = le64_to_cpu(rdesc->blkno) + ring->first_dirty_block; + nr = min_t(u64, nr_blocks, + le64_to_cpu(rdesc->total_blocks) - ring->first_dirty_block); + + scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr, comp); + + if (nr != nr_blocks) { + pages += nr / SCOUTFS_BLOCKS_PER_PAGE; + blkno = le64_to_cpu(rdesc->blkno); + nr = nr_blocks - nr; + + scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr, comp); + } + + ret = 0; + +out: + if (ret) + ring_free_pages(ring); + + return ret; +} + +void scoutfs_ring_write_complete(struct scoutfs_ring_info *ring) +{ + struct ring_node *rnode; + struct ring_node *pos; + + list_for_each_entry_safe(rnode, pos, &ring->dirty_list, head) { + if (rnode->deleted) { + free_node(ring, rnode); + } else { + mark_node_clean(ring, rnode); + rnode->in_ring = 1; + } + } + + ring_free_pages(ring); + + ring->dirty_bytes = 0; + ring->first_dirty_block = calc_first_dirty_block(ring->rdesc); + ring->first_dirty_seq = le64_to_cpu(ring->rdesc->first_seq) + + le64_to_cpu(ring->rdesc->nr_blocks); +} + +void scoutfs_ring_init(struct scoutfs_ring_info *ring, + struct scoutfs_ring_descriptor *rdesc, + scoutfs_ring_cmp_t compare_key, + scoutfs_ring_cmp_t compare_data) +{ + ring->rdesc = rdesc; + ring->compare_key = compare_key; + ring->compare_data = compare_data; + ring->rb_root = RB_ROOT; + INIT_LIST_HEAD(&ring->clean_list); + INIT_LIST_HEAD(&ring->dirty_list); + ring->dirty_bytes = 0; + ring->first_dirty_block = calc_first_dirty_block(rdesc); + ring->first_dirty_seq = le64_to_cpu(rdesc->first_seq) + + le64_to_cpu(rdesc->nr_blocks); + ring->pages = NULL; + ring->nr_pages = 0; +} + +void scoutfs_ring_destroy(struct scoutfs_ring_info *ring) +{ + struct ring_node *rnode; + struct ring_node *pos; + + /* XXX we don't really have a coherent forced dirty unmount story */ + WARN_ON_ONCE(!list_empty(&ring->dirty_list)); + + list_splice_init(&ring->dirty_list, &ring->clean_list); + + list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { + list_del_init(&rnode->head); + kfree(rnode); + } + + ring_free_pages(ring); + scoutfs_ring_init(ring, ring->rdesc, ring->compare_key, + ring->compare_data); +} diff --git a/kmod/src/ring.h b/kmod/src/ring.h new file mode 100644 index 00000000..a9341092 --- /dev/null +++ b/kmod/src/ring.h @@ -0,0 +1,55 @@ +#ifndef _SCOUTFS_RING_H_ +#define _SCOUTFS_RING_H_ + +struct scoutfs_bio_completion; + +typedef int (*scoutfs_ring_cmp_t)(void *a, void *b); + +struct scoutfs_ring_info { + struct scoutfs_ring_descriptor *rdesc; + + scoutfs_ring_cmp_t compare_key; + scoutfs_ring_cmp_t compare_data; + + struct rb_root rb_root; + + struct list_head clean_list; + struct list_head dirty_list; + + unsigned long dirty_bytes; + u64 first_dirty_block; + u64 first_dirty_seq; + + struct page **pages; + unsigned long nr_pages; +}; + +void scoutfs_ring_init(struct scoutfs_ring_info *ring, + struct scoutfs_ring_descriptor *rdesc, + scoutfs_ring_cmp_t compare_key, + scoutfs_ring_cmp_t compare_data); + +int scoutfs_ring_load(struct super_block *sb, struct scoutfs_ring_info *ring); + +void *scoutfs_ring_insert(struct scoutfs_ring_info *ring, void *key, + unsigned data_len); + +void *scoutfs_ring_first(struct scoutfs_ring_info *ring); +void *scoutfs_ring_lookup(struct scoutfs_ring_info *ring, void *key); +void *scoutfs_ring_lookup_next(struct scoutfs_ring_info *ring, void *key); +void *scoutfs_ring_lookup_prev(struct scoutfs_ring_info *ring, void *key); + +void *scoutfs_ring_next(struct scoutfs_ring_info *ring, void *rdata); +void *scoutfs_ring_prev(struct scoutfs_ring_info *ring, void *rdata); +void scoutfs_ring_dirty(struct scoutfs_ring_info *ring, void *rdata); +void scoutfs_ring_delete(struct scoutfs_ring_info *ring, void *rdata); + +int scoutfs_ring_has_dirty(struct scoutfs_ring_info *ring); +int scoutfs_ring_submit_write(struct super_block *sb, + struct scoutfs_ring_info *ring, + struct scoutfs_bio_completion *comp); +void scoutfs_ring_write_complete(struct scoutfs_ring_info *ring); + +void scoutfs_ring_destroy(struct scoutfs_ring_info *ring); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 15c0778e..b15c9339 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -32,7 +32,6 @@ #include "seg.h" #include "bio.h" #include "alloc.h" -#include "treap.h" #include "compact.h" #include "data.h" #include "lock.h" @@ -218,8 +217,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_item_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_alloc_setup(sb) ?: - scoutfs_treap_setup(sb) ?: -// scoutfs_buddy_setup(sb) ?: scoutfs_compact_setup(sb) ?: scoutfs_setup_trans(sb) ?: scoutfs_lock_setup(sb) ?: @@ -265,7 +262,6 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); - scoutfs_treap_destroy(sb); scoutfs_seg_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) diff --git a/kmod/src/super.h b/kmod/src/super.h index e4f48514..458345fd 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -10,7 +10,6 @@ struct scoutfs_counters; struct item_cache; struct manifest; struct segment_cache; -struct treap_info; struct compact_info; struct data_info; struct lock_info; @@ -27,7 +26,6 @@ struct scoutfs_sb_info { struct item_cache *item_cache; struct segment_cache *segment_cache; struct seg_alloc *seg_alloc; - struct treap_info *treap_info; struct compact_info *compact_info; struct data_info *data_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index d596bf68..aa927fb2 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -25,7 +25,7 @@ #include "manifest.h" #include "seg.h" #include "alloc.h" -#include "treap.h" +#include "ring.h" #include "compact.h" #include "counters.h" #include "scoutfs_trace.h" @@ -115,14 +115,15 @@ void scoutfs_trans_write_func(struct work_struct *work) } if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { - ret = scoutfs_manifest_dirty_ring(sb) ?: - scoutfs_alloc_dirty_ring(sb) ?: - scoutfs_treap_submit_write(sb, &comp) ?: + ret = scoutfs_manifest_submit_write(sb, &comp) ?: + scoutfs_alloc_submit_write(sb, &comp) ?: scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_write_dirty_super(sb); if (ret) goto out; + scoutfs_manifest_write_complete(sb); + scoutfs_alloc_write_complete(sb); advance = true; } diff --git a/kmod/src/treap.c b/kmod/src/treap.c deleted file mode 100644 index b6346df3..00000000 --- a/kmod/src/treap.c +++ /dev/null @@ -1,1349 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "kvec.h" -#include "bio.h" -#include "treap.h" -#include "scoutfs_trace.h" - -/* - * scoutfs builds a consistent file system out of segments by describing - * them all with the manifest. Typically the manifest will fit in - * memory but in the pathological case it can be much larger. Our task - * is to index the manifest such that the pathological case is possible - * but the typical case isn't unreasonably penalized by the IO cost of - * maintaining the index. - * - * We chose to index the manifest by storing entries in treap nodes in a - * static ring. Updates are large contiguous writes to the ring with - * low amplification. Incremental updates can similarly read-ahead - * large chunks of the ring. Entirely cold reads end up issuing lots of - * small dependent random IOs. - * - * The nodes in the ring are loaded into native copies in memory. - * Having native allocated nodes lets us do things that would be - * unreasonable if we only traversed persistent structures in cached - * blocks: pointers to nodes in memory instead of indirecting through - * block cache lookups, parent pointers for trivial iteration but which - * would would rule out cow updates, and per-node lru tracking so that - * we can reclaim from the leaves of the tree up to the root without - * false pinning based on which nodes happen to share blocks. - * - * As nodes are modified or inserted they're marked dirty. Eventually - * all the dirty nodes are written to the tail of the ring. We ensure - * that new nodes written at the tail never overwrite old live nodes by - * using a large ring and constantly also migrating old nodes in the - * ring to the tail. - * - * Nodes don't span 4k blocks so there will always be at least a node - * struct's worth of blank space in each block, more typically half the - * average item length, and at worst the max item length. - * - * The tree is augmented to enable searches by more than the primary - * sort keys of the tree. The treap itself maintains augmentation in - * memory to track dirty nodes and in the persistent nodes to track old - * nodes for migration. Callers get callbacks to maintain their own - * augmentation in the node payloads. - * - * Each dirty node gets a generation number that is incremented for each - * version of the tree that is written to the tail of the ring. This - * lets traverse cached nodes without needing strong cache coherence - * with other node writers. With the byte offset and generation of root - * node we can traverse our cached nodes and retry the walk when our - * nodes are stale. - * - * XXX - * - add lru list, nodes to tail during walk, shrink from head - * - stale walking needs work: restart walk, get new root sample - * - lru would need to reclaim nodes orphaned by new root ref walk - */ - -/* - * We preallocate sufficient pages to write all the treap nodes to write - * a transactoin. - * - * XXX Today we only ever write a l0 segment or update the manifest and - * allocator for a single compaction. Those events are *well* less than - * the number of pages that make up a large segment. We'll want this to - * be more careful in the future as we batch up updates from lots of - * writers. - */ -struct treap_info { - /* static, derived from the super */ - u64 last_ring_off; - - /* temporarily assigned to each dirty node */ - u64 dirty_off; - u64 dirty_gen; - - /* used to write nodes to the ring */ - struct page *pages[SCOUTFS_SEGMENT_PAGES]; - u64 pages_off; - u64 ring_off; - unsigned int nr_blocks; - unsigned block_space; -}; - -#define DECLARE_TREAP_INFO(sb, name) \ - struct treap_info *name = SCOUTFS_SB(sb)->treap_info - -struct treap_ref { - struct treap_node *node; - u64 off; - u64 gen; - u8 aug_bits; -}; - -struct scoutfs_treap { - struct super_block *sb; - struct scoutfs_super_block *super; - struct scoutfs_treap_ops *ops; - struct treap_ref root_ref; - bool dirty; - u64 dirty_bytes; -}; - -/* - * The in-memory node differs in that it uses native endian fields, has - * a parent pointer, and (will some day have) an lru for reclaiming from - * the leaves up. - * - * The data is long aligned so that callers can use native longs to - * manipulate bitmaps in the data. - */ -struct treap_node { - u64 off; - u64 gen; - u64 prio; - u16 bytes; - - struct treap_node *parent; - - struct treap_ref left; - struct treap_ref right; - - u8 data[0] __aligned(sizeof(long)); -}; - -#if 0 -static void print_treap_node(struct treap_ref *ref, u64 loc) -{ - struct treap_node *node = ref->node; - - if (!node) - return; - - printk("loc %llx node %p: off %llu gen %llu prio %016llx bytes %u\n", - loc, node, node->off, node->gen, node->prio, node->bytes); - printk(" left: off %llu gen %llu aug %u node %p\n", - node->left.off, node->left.gen, node->left.aug_bits, - node->left.node); - printk(" right: off %llu gen %llu aug %u node %p\n", - node->right.off, node->right.gen, node->right.aug_bits, - node->right.node); - - print_treap_node(&node->left, (loc << 4) | 1); - print_treap_node(&node->right, (loc << 4) | 2); -} -#endif - -static struct treap_ref *parent_ref(struct scoutfs_treap *treap, - struct treap_node *node) -{ - if (!node->parent) - return &treap->root_ref; - if (node->parent->left.node == node) - return &node->parent->left; - return &node->parent->right; -} - -static u8 off_aug_bit(struct scoutfs_treap *treap, u64 off) -{ - u64 blocks = le64_to_cpu(treap->super->ring_blocks); - u64 mid = (blocks << SCOUTFS_BLOCK_SHIFT) / 2; - - return off < mid ? SCOUTFS_TREAP_AUG_LESSER : - SCOUTFS_TREAP_AUG_GREATER; -} - -static u8 old_aug_bit(struct scoutfs_treap *treap) -{ - DECLARE_TREAP_INFO(treap->sb, tinf); - - return off_aug_bit(treap, tinf->dirty_off) ^ SCOUTFS_TREAP_AUG_HALVES; -} - -/* - * Return the aug bits that'll be used to refer to the given node. - * We calculate the bits for the node itself and then or those with the - * bits in its references to its children. - */ -static u8 node_aug_bits(struct scoutfs_treap *treap, struct treap_node *node) -{ - DECLARE_TREAP_INFO(treap->sb, tinf); - - return (node->off == tinf->dirty_off ? SCOUTFS_TREAP_AUG_DIRTY : 0) | - off_aug_bit(treap, node->off) | - node->left.aug_bits | - node->right.aug_bits; -} - -/* - * Update the treap augmentation until its back in sync. We can be - * called with a null node to repair a non-existing parent and we just - * have to clear the root aug_bits in that case. - */ -static void update_internal_aug(struct scoutfs_treap *treap, - struct treap_node *node) -{ - struct treap_ref *ref; - u8 bits; - - if (!node) - treap->root_ref.aug_bits = 0; - - while (node) { - bits = node_aug_bits(treap, node); - ref = parent_ref(treap, node); - trace_printk("node %p bits %x parent %p ref bits %x\n", - node, bits, node->parent, ref->aug_bits); - if (ref->aug_bits == bits) - break; - ref->aug_bits = bits; - node = node->parent; - } -} - -static bool ops_update_aug(struct scoutfs_treap *treap, - struct treap_node *parent, struct treap_node *node) -{ - if (!treap->ops->update_aug) - return false; - - return treap->ops->update_aug(parent->data, parent->left.node == node, - node->data); -} - -/* - * Update the tree's augmentation stored in the data payloads. The caller - * sets the left or right aug in the parent to match the node. - */ -static void update_data_aug(struct scoutfs_treap *treap, - struct treap_node *node) -{ - struct treap_node *parent; - - while (node && (parent = node->parent)) { - if (!ops_update_aug(treap, parent, node)) - break; - node = node->parent; - } -} - -/* - * G G - * | | - * P N - * / -> \ - * N P - * \ / - * - * parent->left = node->right; - * node->right = parent; - * grand->(left|right) = node - * - * The rotation has the following effect on augmentation: - * - parent ref's aug bits have the same population, no change - * - node left's unchanged - * - parent right's unchanged - * - parent's left just set to the node's right - * - node right's recalculated based on parent - */ -static void rotate_right(struct scoutfs_treap *treap, - struct treap_node *parent, struct treap_node *node) -{ - struct treap_ref *grand_ref; - struct treap_node *grand; - - /* get grandparent ref before clobbering parent */ - grand = parent->parent; - if (grand) { - if (grand->left.node == parent) - grand_ref = &grand->left; - else - grand_ref = &grand->right; - } else { - grand_ref = &treap->root_ref; - } - - /* parent rotates down and points to node's child */ - parent->left = node->right; - if (parent->left.node) - parent->left.node->parent = parent; - - /* node rotates up and points to parent */ - node->right.node = parent; - node->right.off = parent->off; - node->right.gen = parent->gen; - node->right.aug_bits = node_aug_bits(treap, parent); - parent->parent = node; - - /* grand parent points to node */ - grand_ref->node = node; - grand_ref->off = node->off; - grand_ref->gen = node->gen; - grand_ref->aug_bits = node_aug_bits(treap, node); - node->parent = grand; - - ops_update_aug(treap, node, parent); -} - -/* see above: swap left/right */ -static void rotate_left(struct scoutfs_treap *treap, - struct treap_node *parent, struct treap_node *node) -{ - struct treap_ref *grand_ref; - struct treap_node *grand; - - grand = parent->parent; - if (grand) { - if (grand->right.node == parent) - grand_ref = &grand->right; - else - grand_ref = &grand->left; - } else { - grand_ref = &treap->root_ref; - } - - parent->right = node->left; - if (parent->right.node) - parent->right.node->parent = parent; - - node->left.node = parent; - node->left.off = parent->off; - node->left.gen = parent->gen; - node->left.aug_bits = node_aug_bits(treap, parent); - parent->parent = node; - - grand_ref->node = node; - grand_ref->off = node->off; - grand_ref->gen = node->gen; - grand_ref->aug_bits = node_aug_bits(treap, node); - node->parent = grand; - - ops_update_aug(treap, node, parent); -} - -/* - * Rebalance the tree by rotating the parent and child as long as the - * child has a higher random priority. - */ -static void rebalance(struct scoutfs_treap *treap, struct treap_node *node) -{ - struct treap_node *parent; - - while (node && (parent = node->parent) && node->prio > parent->prio) { - if (parent->left.node == node) - rotate_right(treap, parent, node); - else - rotate_left(treap, parent, node); - } -} - - -/* - * The caller has mucked with a node. We make sure all of our internal - * augmentation, the op data's augmentation, and the treap prio balance - * is repaired. - */ -static void repair(struct scoutfs_treap *treap, struct treap_node *node) -{ - update_internal_aug(treap, node); - update_data_aug(treap, node); - rebalance(treap, node); - - trace_printk("treap %p root aug %x\n", - treap, treap->root_ref.aug_bits); -} - -static struct treap_node *alloc_node(u16 bytes) -{ - struct treap_node *node; - - node = kmalloc(offsetof(struct treap_node, data[bytes]), GFP_NOFS); - if (node) - memset(node, 0, offsetof(struct treap_node, data)); - - return node; -} - -/* - * bytes in the persistent ring taken up by a node with the given number - * of data bytes. - */ -static unsigned node_ring_bytes(struct treap_node *node) -{ - return offsetof(struct scoutfs_treap_node, data[node->bytes]); -} - -static bool dirty_node(struct scoutfs_treap *treap, struct treap_node *node) -{ - DECLARE_TREAP_INFO(treap->sb, tinf); - - return node->off == tinf->dirty_off; -} - -/* - * Ensure that the given node is dirty. If it isn't we need to mark it - * dirty and augment the tree. Transaction limits and preallocation - * make sure that we always have resources to write nodes that are - * dirtied. - * - * When we dirty old nodes we temporarily set their offset to the - * current half of the ring so that they won't show up in augmented - * searches for old nodes. - */ -static bool mark_node_dirty(struct scoutfs_treap *treap, struct treap_ref *ref, - struct treap_node *node) -{ - DECLARE_TREAP_INFO(treap->sb, tinf); - - if (dirty_node(treap, node)) - return false; - - trace_printk("node %p off %llu gen %llu now dirty\n", - node, node->off, node->gen); - - treap->dirty_bytes += node_ring_bytes(node); - treap->dirty = true; - - node->off = tinf->dirty_off; - node->gen = tinf->dirty_gen; - ref->off = node->off; - ref->gen = node->gen; - repair(treap, node); - - return true; -} - -static int dirty_old_nodes(struct scoutfs_treap *treap, unsigned old_target, - unsigned dirty_limit); - -static struct scoutfs_treap_node *read_ring_node(struct scoutfs_treap *treap, - u64 off) -{ - struct address_space *mapping = treap->sb->s_bdev->bd_inode->i_mapping; - struct scoutfs_treap_node *tnode = NULL; - struct page *page = NULL; - unsigned pg_off; - unsigned bytes; - pgoff_t pg_ind; - int ret; - - off += le64_to_cpu(treap->super->ring_blkno) << SCOUTFS_BLOCK_SHIFT; - pg_ind = off >> PAGE_CACHE_SHIFT; - pg_off = off & ~PAGE_CACHE_MASK; - - if (pg_off + sizeof(struct scoutfs_treap_node) > PAGE_CACHE_SIZE) { - ret = -EIO; - goto out; - } - -retry: - page = find_or_create_page(mapping, pg_ind, GFP_NOFS); - if (!page) { - ret = -ENOMEM; - goto out; - } - - tnode = page_address(page) + pg_off; - - if (PageUptodate(page)) { - unlock_page(page); - ret = 0; - goto out; - } - - ClearPageError(page); - ret = mapping->a_ops->readpage(NULL, page); - if (ret) { - if (ret == AOP_TRUNCATED_PAGE) { - page_cache_release(page); - goto retry; - } - goto out; - } - - wait_on_page_locked(page); - if (!PageUptodate(page)) { - if (page->mapping != mapping) { - page_cache_release(page); - goto retry; - } - ret = -EIO; - goto out; - } else { - ret = 0; - } - - bytes = le16_to_cpu(tnode->bytes); - - if (pg_off + offsetof(struct scoutfs_treap_node, data[bytes]) > - PAGE_CACHE_SIZE) { - ret = -EIO; - } - -out: - if (ret) { - if (page) - page_cache_release(page); - return ERR_PTR(ret); - } - - return tnode; -} - -static void release_ring_node(struct scoutfs_treap_node *tnode) -{ - if (!IS_ERR_OR_NULL(tnode)) - page_cache_release(virt_to_page(tnode)); -} - -/* - * We write to ring blocks from preallocated private pages with bios but read - * through the bdev page cache. Invalidate the blocks we're about to write - * so we'll read them later. - */ -static void invalidate_blocks(struct super_block *sb, u64 blkno, u64 nr) -{ - struct address_space *mapping = sb->s_bdev->bd_inode->i_mapping; - loff_t lstart = blkno << SCOUTFS_BLOCK_SHIFT; - loff_t lend = lstart + (nr << SCOUTFS_BLOCK_SHIFT) - 1; - - truncate_inode_pages_range(mapping, lstart, lend); -} - -static void invalidate_ring_block(struct scoutfs_treap *treap, u64 off) -{ - invalidate_blocks(treap->sb, le64_to_cpu(treap->super->ring_blkno) + - (off >> SCOUTFS_BLOCK_SHIFT), 1); -} - -static __le32 tnode_crc(struct scoutfs_treap_node *tnode) -{ - u16 bytes = le16_to_cpu(tnode->bytes); - unsigned skip = sizeof(tnode->crc); - - return cpu_to_le32(crc32c(~0, (void *)tnode + skip, - offsetof(struct scoutfs_treap_node, - data[bytes]) - skip)); -} - -/* - * Give the caller the node pointed to by their reference. If the node - * isn't already in the tree then we link it in and update augmentation. - * - * XXX what's the consequence of failing to also dirty old ring nodes? - * The ring gets out of balance but we do nothing about it. - */ -static struct treap_node *read_node(struct scoutfs_treap *treap, - struct treap_node *parent, - struct treap_ref *ref, bool dirty) -{ - struct scoutfs_treap_node *tnode = NULL; - struct treap_node *node = NULL; - unsigned retries = 3; - u16 bytes; - int ret; - - if (ref->node) { - node = ref->node; - ret = 0; - goto out; - } - -retry: - tnode = read_ring_node(treap, ref->off); - if (IS_ERR(tnode)) { - ret = PTR_ERR(tnode); - goto out; - } - - if (tnode->crc != tnode_crc(tnode) || - le64_to_cpu(tnode->off) != ref->off || - le64_to_cpu(tnode->gen) != ref->gen) { - invalidate_ring_block(treap, ref->off); - if (retries--) { - /* XXX restart search, not just this read */ - release_ring_node(tnode); - goto retry; - } else { - ret = -EIO; - goto out; - } - } - - bytes = le16_to_cpu(tnode->bytes); - - node = alloc_node(bytes); - if (!node) { - ret = -ENOMEM; - goto out; - } - - node->off = le64_to_cpu(tnode->off); - node->gen = le64_to_cpu(tnode->gen); - node->prio = le64_to_cpu(tnode->prio); - node->left.off = le64_to_cpu(tnode->left.off); - node->left.gen = le64_to_cpu(tnode->left.gen); - node->left.aug_bits = tnode->left.aug_bits; - node->right.off = le64_to_cpu(tnode->right.off); - node->right.gen = le64_to_cpu(tnode->right.gen); - node->right.aug_bits = tnode->right.aug_bits; - node->bytes = bytes; - memcpy(node->data, tnode->data, bytes); - - node->parent = parent; - ref->node = node; - ret = 0; -out: - release_ring_node(tnode); - if (!ret && dirty && mark_node_dirty(treap, ref, node)) - ret = dirty_old_nodes(treap, node_ring_bytes(node), 0); - if (ret) - return ERR_PTR(ret); - - return node; -} - -/* - * Find nodes in the older half of the ring and mark them dirty. Stop - * when we don't have any more older nodes, after dirtying enough old - * nodes, or before dirtying too many nodes. - */ -static int dirty_old_nodes(struct scoutfs_treap *treap, unsigned old_target, - unsigned dirty_limit) -{ - u8 bit = old_aug_bit(treap); - struct treap_node *parent; - struct treap_node *node; - struct treap_ref *ref; - unsigned dirty = 0; - unsigned old = 0; - unsigned bytes; - int ret = 0; - -restart: - parent = NULL; - ref = &treap->root_ref; - - while (ref->aug_bits & bit) { - node = read_node(treap, parent, ref, false); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - break; - } - - bytes = node_ring_bytes(node); - - if (!dirty_node(treap, node) && dirty_limit) { - dirty += bytes; - if (dirty > dirty_limit) - break; - } - - if (old_target && off_aug_bit(treap, node->off) == bit) - old += bytes; - - /* sets dirty, sets current half aug bit, repairs */ - mark_node_dirty(treap, ref, node); - - if (old_target && old >= old_target) - break; - - if (node->left.aug_bits & bit) - ref = &node->left; - else if (node->right.aug_bits & bit) - ref = &node->right; - else - goto restart; - } - - return ret; -} - -/* - * Return the dirty node identified by the given key, creating it if it - * doesn't exist. - * - * Returns ERR -EEXIST if a node already exists at the given key. - */ -void *scoutfs_treap_insert(struct scoutfs_treap *treap, void *key, u16 bytes, - void *fill_arg) -{ - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - int cmp; - - while (ref->gen) { - node = read_node(treap, parent, ref, true); - if (IS_ERR(node)) - goto out; - - cmp = treap->ops->compare(key, node->data); - if (cmp < 0) { - ref = &node->left; - } else if (cmp > 0) { - ref = &node->right; - } else { - node = ERR_PTR(-EEXIST); - goto out; - } - - parent = node; - node = NULL; - } - - node = alloc_node(bytes); - if (!node) { - node = ERR_PTR(-ENOMEM); - goto out; - } - - node->parent = parent; - node->bytes = bytes; - get_random_bytes_arch(&node->prio, sizeof(node->prio)); - - ref->node = node; - - /* filling here instead of in caller for aug update in repair */ - treap->ops->fill(node->data, fill_arg); - - /* sets off and gen and repairs */ - mark_node_dirty(treap, ref, node); -out: - if (IS_ERR(node)) - return ERR_CAST(node); - - return node->data; -} - -/* - * Delete a node with the given key. - * - * It's easy when the node doesn't have two children. We remove the - * node and point it's parent ref at either of the child's refs that - * might have been populated. - * - * Deletion's a little tricker when we have both children. We could - * find an ancestor and swap but that's fiddly to get right with all our - * rich node pointers. Instead we can reuse rotation to rotate the node - * down until it doesn't have both children. - */ -int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key) -{ - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - struct treap_ref *child_ref; - struct treap_node *left; - struct treap_node *right; - int cmp; - int ret; - - /* find node to delete */ - while (ref->gen) { - node = read_node(treap, parent, ref, true); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - goto out; - } - - cmp = treap->ops->compare(key, node->data); - if (cmp < 0) - ref = &node->left; - else if (cmp > 0) - ref = &node->right; - else - break; - - parent = node; - node = NULL; - } - - if (!node) { - ret = -ENOENT; - goto out; - } - - /* - * Rotate the node down with its higher priority child until it - * doesn't have both children. Dirtying tries to repair which - * can try to repair priority imbalance with rotation so we swap - * priorities first. Unfortunately we need to read both - * children to get their priorities but we only try to dirty the - * rotation child. It's messy but dirtying both can double - * write amplification. - */ - while (node->left.gen && node->right.gen) { - left = read_node(treap, node, &node->left, false); - right = read_node(treap, node, &node->right, false); - if (IS_ERR(left) || IS_ERR(right)) { - ret = IS_ERR(left) ? PTR_ERR(left) : PTR_ERR(right); - goto out; - } - - if (left->prio > right->prio) { - left = read_node(treap, node, &node->left, true); - if (IS_ERR(left)) { - ret = IS_ERR(left); - goto out; - } - swap(node->prio, left->prio); - rotate_right(treap, node, left); - } else { - right = read_node(treap, node, &node->right, true); - if (IS_ERR(right)) { - ret = IS_ERR(right); - goto out; - } - swap(node->prio, right->prio); - rotate_left(treap, node, right); - } - - parent = node->parent; - ref = parent_ref(treap, node); - } - - /* delete the node, might have to point parent at child */ - if (node->left.gen) - child_ref = &node->left; - else - child_ref = &node->right; - - *ref = *child_ref; - if (ref->node) - ref->node->parent = parent; - - if (dirty_node(treap, node)) - treap->dirty_bytes -= node_ring_bytes(node); - - kfree(node); - - repair(treap, parent); - ret = 0; -out: - return ret; -} - -enum { - LU_DIRTY, - LU_NEXT, - LU_PREV, -}; - -static void *treap_lookup(struct scoutfs_treap *treap, void *key, int flags) -{ - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - struct treap_node *prev = NULL; - struct treap_node *next = NULL; - int cmp; - - while (ref->gen) { - node = read_node(treap, parent, ref, flags & LU_DIRTY); - if (IS_ERR(node)) - break; - - cmp = treap->ops->compare(key, node->data); - if (cmp < 0) { - ref = &node->left; - next = node; - } else if (cmp > 0) { - ref = &node->right; - prev = node; - } else { - break; - } - - parent = node; - node = NULL; - } - - if (!node && (flags & LU_PREV) && prev) - node = prev; - else if (!node && (flags & LU_NEXT) && next) - node = next; - - if (IS_ERR(node)) - return ERR_CAST(node); - if (node) - return node->data; - return NULL; -} - -void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, 0); -} - -void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, LU_DIRTY); -} - -void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, LU_NEXT); -} - -void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, LU_NEXT | LU_DIRTY); -} - -void *scoutfs_treap_lookup_prev(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, LU_PREV); -} - -void *scoutfs_treap_lookup_prev_dirty(struct scoutfs_treap *treap, void *key) -{ - return treap_lookup(treap, key, LU_PREV | LU_DIRTY); -} - -void *scoutfs_treap_first(struct scoutfs_treap *treap) -{ - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - - while (ref->gen) { - node = read_node(treap, parent, ref, false); - if (IS_ERR(node)) - break; - - ref = &node->left; - parent = node; - } - - if (IS_ERR(node)) - return ERR_CAST(node); - if (node) - return node->data; - return NULL; -} - -void *scoutfs_treap_last(struct scoutfs_treap *treap) -{ - struct treap_ref *ref = &treap->root_ref; - struct treap_node *parent = NULL; - struct treap_node *node = NULL; - - while (ref->gen) { - node = read_node(treap, parent, ref, false); - if (IS_ERR(node)) - break; - - ref = &node->right; - parent = node; - } - - if (IS_ERR(node)) - return ERR_CAST(node); - if (node) - return node->data; - return NULL; -} - -void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data) -{ - struct treap_node *node = container_of(data, struct treap_node, data); - struct treap_node *parent; - - if (node->right.gen) { - node = read_node(treap, node, &node->right, false); - if (IS_ERR(node)) - goto out; - - while (node->left.gen) { - node = read_node(treap, node, &node->left, false); - if (IS_ERR(node)) - goto out; - } - - goto out; - } - - while (((parent = node->parent)) && node == parent->right.node) - node = parent; - node = parent; - -out: - if (IS_ERR(node)) - return ERR_CAST(node); - if (node) - return node->data; - return NULL; -} - -void *scoutfs_treap_prev(struct scoutfs_treap *treap, void *data) -{ - struct treap_node *node = container_of(data, struct treap_node, data); - struct treap_node *parent; - - if (node->left.gen) { - node = read_node(treap, node, &node->left, false); - if (IS_ERR(node)) - goto out; - - while (node->right.gen) { - node = read_node(treap, node, &node->right, false); - if (IS_ERR(node)) - goto out; - } - - goto out; - } - - while (((parent = node->parent)) && node == parent->left.node) - node = parent; - node = parent; - -out: - if (IS_ERR(node)) - return ERR_CAST(node); - if (node) - return node->data; - return NULL; -} - -int scoutfs_treap_has_dirty(struct scoutfs_treap *treap) -{ - return treap->dirty; -} - -static void *pages_off_ptr(struct treap_info *tinf) -{ - return page_address(tinf->pages[tinf->pages_off >> PAGE_SHIFT]) + - (tinf->pages_off % ~PAGE_MASK); -} - -/* - * The dirty offset is carefully chosen so that it will consider dirty - * nodes part of the current half of the ring but is an offset that will - * never be actually written. That way it is overwritten as dirty nodes - * are copied to the ring and get their final offset and aren't considered - * dirty. Nodes never span blocks so we set the dirty offset to the final - * byte of the next block in the ring. - */ -static void init_writer(struct treap_info *tinf, - struct scoutfs_super_block *super) -{ - tinf->ring_off = le64_to_cpu(super->ring_tail_block) << - SCOUTFS_BLOCK_SHIFT; - tinf->pages_off = 0; - tinf->block_space = 0; - tinf->nr_blocks = 0; - - tinf->dirty_gen = le64_to_cpu(super->ring_gen) + 1; - tinf->dirty_off = tinf->ring_off + SCOUTFS_BLOCK_MASK; -} - -static void try_zero_block_tail(struct treap_info *tinf) -{ - if (tinf->block_space != SCOUTFS_BLOCK_SIZE) - memset(pages_off_ptr(tinf), 0, tinf->block_space); -} - -/* - * Copy the node to the page at the next free tail offset. The - * in-memory node's offset is set to its final ring offset and its - * parent ref is updated. Thus it will no longer have the magic dirty - * offset and won't be considered dirty by the tree augmentation. - */ -static void copy_node_to_ring(struct scoutfs_treap *treap, - struct treap_node *node) -{ - DECLARE_TREAP_INFO(treap->sb, tinf); - struct scoutfs_treap_node *tnode; - u32 bytes = node_ring_bytes(node); - u32 skip; - - if (tinf->block_space < bytes) { - try_zero_block_tail(tinf); - - skip = ALIGN(tinf->ring_off, SCOUTFS_BLOCK_SIZE) - - tinf->ring_off; - tinf->ring_off += skip; - tinf->pages_off += skip; - - tinf->block_space = SCOUTFS_BLOCK_SIZE; - tinf->nr_blocks++; - - /* see if we're wrapping */ - if (tinf->ring_off == tinf->last_ring_off) - tinf->ring_off = 0; - } - - node->off = tinf->ring_off; - parent_ref(treap, node)->off = node->off; - - tnode = pages_off_ptr(tinf); - tinf->ring_off += bytes; - tinf->pages_off += bytes; - tinf->block_space -= bytes; - - tnode->off = cpu_to_le64(node->off); - tnode->gen = cpu_to_le64(node->gen); - tnode->prio = cpu_to_le64(node->prio); - tnode->left.off = cpu_to_le64(node->left.off); - tnode->left.gen = cpu_to_le64(node->left.gen); - tnode->left.aug_bits = node->left.aug_bits; - tnode->right.off = cpu_to_le64(node->right.off); - tnode->right.gen = cpu_to_le64(node->right.gen); - tnode->right.aug_bits = node->right.aug_bits; - tnode->bytes = cpu_to_le16(node->bytes); - memcpy(tnode->data, node->data, node->bytes); - - tnode->crc = tnode_crc(tnode); -} - -/* - * Copy the currently dirty nodes into preallocated pages for writing. - * - * We can consider the nodes clean as we copy them to the pages. The - * caller is responsible for ensuring forward progress or aborting. - * - * As nodes are copied to the pages they are assigned their final offset - * in the ring. We have to update their parent refs with the new - * offset. (We also could have them cross a half ring, getting new off - * aug bits that bubble up). - * - * All that means that we copy from the leaves up to the root so that we - * capture the modifications to parents as we copy children. - * - * This is called for multiple treaps before the ring is written. - */ -int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap, - struct scoutfs_treap_root *root) -{ - struct treap_node *node; - unsigned bytes; - int ret; - - /* first fill final partial block with old nodes */ - bytes = SCOUTFS_BLOCK_SIZE - (treap->dirty_bytes & SCOUTFS_BLOCK_MASK); - if (bytes != SCOUTFS_BLOCK_SIZE) { - ret = dirty_old_nodes(treap, 0, bytes); - if (ret) - goto out; - } - - node = treap->root_ref.node; - while (node) { - /* follow dirty links first */ - if (node->left.aug_bits & SCOUTFS_TREAP_AUG_DIRTY) { - node = node->left.node; - } else if (node->right.aug_bits & SCOUTFS_TREAP_AUG_DIRTY) { - node = node->right.node; - } else { - /* node doesn't have dirty children, append if dirty */ - if (dirty_node(treap, node)) { - copy_node_to_ring(treap, node); - repair(treap, node); - } - - /* ascend back up through parents */ - node = node->parent; - } - } - - /* point the persistent super root at the treap we wrote to the ring */ - root->ref.off = cpu_to_le64(treap->root_ref.off); - root->ref.gen = cpu_to_le64(treap->root_ref.gen); - root->ref.aug_bits = treap->root_ref.aug_bits; - - treap->dirty_bytes = 0; - treap->dirty = false; - ret = 0; -out: - return ret; -} - -/* - * Submit writes for all the dirty nodes that have been copied into the - * preallocated pages. - * entries were appended. The dirty ring blocks are contiguous in the - * page array but can wrap in the block ring on disk. - * - * If it wraps then we submit the earlier fragment at the head of the - * ring first. - * - * The wrapped fragment starts at some block offset in the page array. - * The hacky page array math only works when our fixed 4k block size == - * page_size. To fix it we'd add a offset block to the bio submit loop - * which could add an initial partial page vec to the bios. - * - * XXX figure out where to write. I guess we have a write ring block - * in the super? - */ -int scoutfs_treap_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_TREAP_INFO(sb, tinf); - u64 head_blocks; - u64 tail_blocks; - u64 blkno; - u64 tail; - - if (!tinf->nr_blocks) - return 0; - - try_zero_block_tail(tinf); - - tail = le64_to_cpu(super->ring_tail_block); - tail_blocks = min_t(u64, tinf->nr_blocks, - le64_to_cpu(super->ring_blocks) - tail); - - head_blocks = tinf->nr_blocks - tail_blocks; - - if (head_blocks) { - BUILD_BUG_ON(SCOUTFS_BLOCK_SIZE != PAGE_SIZE); - invalidate_blocks(sb, le64_to_cpu(super->ring_blkno), - head_blocks); - scoutfs_bio_submit_comp(sb, WRITE, tinf->pages + tail_blocks, - le64_to_cpu(super->ring_blkno), - head_blocks, comp); - } - - blkno = le64_to_cpu(super->ring_blkno) + tail; - invalidate_blocks(sb, blkno, tail_blocks); - scoutfs_bio_submit_comp(sb, WRITE, tinf->pages, blkno, tail_blocks, - comp); - - /* record new tail index in super and reset for next trans */ - super->ring_tail_block = cpu_to_le64(tail + tail_blocks); - if (super->ring_tail_block == super->ring_blocks) - super->ring_tail_block = cpu_to_le64(head_blocks); - - super->ring_gen = cpu_to_le64(tinf->dirty_gen); - - init_writer(tinf, super); - - return 0; -} - -struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, - struct scoutfs_treap_ops *ops, - struct scoutfs_treap_root *root) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_treap *treap; - - treap = kzalloc(sizeof(struct scoutfs_treap), GFP_NOFS); - if (treap) { - treap->sb = sb; - treap->super = &sbi->super; - treap->ops = ops; - treap->root_ref.off = le64_to_cpu(root->ref.off); - treap->root_ref.gen = le64_to_cpu(root->ref.gen); - treap->root_ref.aug_bits = root->ref.aug_bits; - } - - return treap; -} - -/* - * Free all the allocated nodes in the treap and clear the root. - */ -void scoutfs_treap_free(struct scoutfs_treap *treap) -{ - struct treap_node *node = treap->root_ref.node; - struct treap_node *fre; - - while (node) { - if (node->left.node) { - node = node->left.node; - node->parent->left.node = NULL; - } if (node->right.node) { - node = node->right.node; - node->parent->right.node = NULL; - } else { - fre = node; - node = node->parent; - kfree(fre); - } - } - - kfree(treap); -} - -int scoutfs_treap_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct treap_info *tinf; - struct page *page; - int i; - - BUILD_BUG_ON(offsetof(struct treap_node, data) & (sizeof(long) - 1)); - - tinf = kzalloc(sizeof(struct treap_info), GFP_KERNEL); - if (!tinf) - return -ENOMEM; - - tinf->last_ring_off = le64_to_cpu(super->ring_blocks) << - SCOUTFS_BLOCK_SHIFT; - init_writer(tinf, super); - - for (i = 0; i < ARRAY_SIZE(tinf->pages); i++) { - page = alloc_page(GFP_KERNEL); - if (!page) { - while (--i >= 0) - __free_page(tinf->pages[i]); - kfree(tinf); - return -ENOMEM; - } - - tinf->pages[i] = page; - } - - sbi->treap_info = tinf; - - return 0; -} - -void scoutfs_treap_destroy(struct super_block *sb) -{ - DECLARE_TREAP_INFO(sb, tinf); - int i; - - if (tinf) { - for (i = 0; i < ARRAY_SIZE(tinf->pages); i++) - __free_page(tinf->pages[i]); - - kfree(tinf); - } -} diff --git a/kmod/src/treap.h b/kmod/src/treap.h deleted file mode 100644 index 497d742a..00000000 --- a/kmod/src/treap.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef _SCOUTFS_TREAP_H_ -#define _SCOUTFS_TREAP_H_ - -struct scoutfs_bio_completion; - -/* - * The runtime root that's used by operations. It's loaded and stored - * from the persistent root in the super block as transactions are written. - */ -struct scoutfs_treap; - -struct scoutfs_treap_ops { - int (*compare)(void *key, void *data); - void (*fill)(void *data, void *fill_arg); - bool (*update_aug)(void *parent_data, bool left, void *node_data); -}; - -struct scoutfs_treap *scoutfs_treap_alloc(struct super_block *sb, - struct scoutfs_treap_ops *ops, - struct scoutfs_treap_root *root); -void scoutfs_treap_free(struct scoutfs_treap *treap); - -void *scoutfs_treap_insert(struct scoutfs_treap *treap, void *key, u16 bytes, - void *fill_arg); -int scoutfs_treap_delete(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup_dirty(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup_next(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup_next_dirty(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup_prev(struct scoutfs_treap *treap, void *key); -void *scoutfs_treap_lookup_prev_dirty(struct scoutfs_treap *treap, void *key); - -void *scoutfs_treap_first(struct scoutfs_treap *treap); -void *scoutfs_treap_last(struct scoutfs_treap *treap); -void *scoutfs_treap_next(struct scoutfs_treap *treap, void *data); -void *scoutfs_treap_prev(struct scoutfs_treap *treap, void *data); - -int scoutfs_treap_has_dirty(struct scoutfs_treap *treap); -int scoutfs_treap_dirty_ring(struct scoutfs_treap *treap, - struct scoutfs_treap_root *root); -int scoutfs_treap_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp); - -int scoutfs_treap_setup(struct super_block *sb); -void scoutfs_treap_destroy(struct super_block *sb); - -#endif From 45882f5a77f62a233b317752071a01c4bfc528ba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Apr 2017 12:54:44 -0700 Subject: [PATCH 248/920] Add some ring tracing Signed-off-by: Zach Brown --- kmod/src/ring.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 00809b19..6df1c04c 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -289,6 +289,10 @@ void *scoutfs_ring_insert(struct scoutfs_ring_info *ring, void *key, list_add_tail(&rnode->head, &ring->dirty_list); mark_node_dirty(ring, rnode, true); + trace_printk("inserted rnode %p in %u deleted %u dirty %u\n", + rnode, rnode->in_ring, rnode->deleted, + rnode->dirty); + return rnode->data; } @@ -429,6 +433,9 @@ void scoutfs_ring_delete(struct scoutfs_ring_info *ring, void *data) { struct ring_node *rnode = data_rnode(data); + trace_printk("deleting rnode %p in %u deleted %u dirty %u\n", + rnode, rnode->in_ring, rnode->deleted, rnode->dirty); + BUG_ON(rnode->deleted); if (rnode->in_ring) { @@ -456,12 +463,16 @@ static int load_ring_block(struct scoutfs_ring_info *ring, int ret = 0; int cmp; + trace_printk("block %llu\n", le64_to_cpu(rblk->block)); + rent = rblk->entries; for (i = 0; i < le32_to_cpu(rblk->nr_entries); i++) { /* XXX verify fields? */ data_len = le16_to_cpu(rent->data_len); + trace_printk("rent %u data_len %u\n", i, data_len); + if (rent->flags & SCOUTFS_RING_ENTRY_FLAG_DELETION) { rnode = ring_rb_walk(ring, NULL, rent->data, NULL, &cmp); @@ -669,6 +680,11 @@ int scoutfs_ring_submit_write(struct super_block *sb, while (rnode && &rent->data[rnode->data_len] <= end) { + trace_printk("writing ent %u rnode %p in %u deleted %u dirty %u\n", + le32_to_cpu(rblk->nr_entries), + rnode, rnode->in_ring, rnode->deleted, + rnode->dirty); + rent->data_len = cpu_to_le16(rnode->data_len); if (rnode->deleted) rent->flags = SCOUTFS_RING_ENTRY_FLAG_DELETION; From 453715a78df69ef2948d7f49fc8b74a6a3ec6237 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Apr 2017 16:56:39 -0700 Subject: [PATCH 249/920] Only shutdown locks that were setup Lock shutdown was crashing trying to deref a null linf on cleanup from mont errors that happened before locks were setup. Make sure lock shutdown only tries to do work if the locks have been setup. Signed-off-by: Zach Brown --- kmod/src/lock.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e9f1c5a4..6078b024 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -336,11 +336,14 @@ void scoutfs_lock_shutdown(struct super_block *sb) DECLARE_LOCK_INFO(sb, linf); struct held_locks *held = linf->held; - spin_lock(&held->lock); - linf->shutdown = true; - spin_unlock(&held->lock); + if (linf) { + held = linf->held; + spin_lock(&held->lock); + linf->shutdown = true; + spin_unlock(&held->lock); - wake_up(&held->waitq); + wake_up(&held->waitq); + } } void scoutfs_lock_destroy(struct super_block *sb) From b50de9019629d68379a112cb73a5fd78ed86da40 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Apr 2017 16:58:07 -0700 Subject: [PATCH 250/920] Alloc inodes from pool from server Inode allocation was always modifying the in-memory super block. This doesn't work when the server is solely responsible for modifying the super blocks. We add network messages to have mounts send a message to the server to request inodes that they can use to satisfy allocation. Signed-off-by: Zach Brown --- kmod/src/format.h | 12 ++++- kmod/src/inode.c | 128 ++++++++++++++++++++++++++++++++++++++++++---- kmod/src/inode.h | 5 +- kmod/src/net.c | 85 +++++++++++++++++++++++++++++- kmod/src/net.h | 1 + kmod/src/super.c | 2 + kmod/src/super.h | 2 + 7 files changed, 221 insertions(+), 14 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 25cb3878..25663ac9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -362,11 +362,21 @@ struct scoutfs_net_header { __u8 type; __u8 status; __u8 data[0]; -}; +} __packed; + +/* + * When there's no more free inodes this will be sent with ino = ~0 and + * nr = 0. + */ +struct scoutfs_net_inode_alloc { + __le64 ino; + __le64 nr; +} __packed; enum { /* sends and receives a struct scoutfs_timeval */ SCOUTFS_NET_TRADE_TIME = 0, + SCOUTFS_NET_ALLOC_INODES, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e34441de..560f1d2a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -30,6 +31,7 @@ #include "msg.h" #include "kvec.h" #include "item.h" +#include "net.h" /* * XXX @@ -37,6 +39,14 @@ * - use inode item value lengths for forward/back compat */ +struct free_ino_pool { + wait_queue_head_t waitq; + spinlock_t lock; + u64 ino; + u64 nr; + bool in_flight; +}; + static struct kmem_cache *scoutfs_inode_cachep; static void scoutfs_inode_ctor(void *obj) @@ -359,24 +369,98 @@ u64 scoutfs_last_ino(struct super_block *sb) return last; } +/* + * Network replies refill the pool, providing ino = ~0ULL nr = 0 when + * there's no more inodes (which should never happen in practice.) + */ +void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr) +{ + struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + + trace_printk("filling ino %llu nr %llu\n", ino, nr); + + spin_lock(&pool->lock); + + pool->ino = ino; + pool->nr = nr; + pool->in_flight = false; + + spin_unlock(&pool->lock); + + wake_up(&pool->waitq); +} + +static bool pool_in_flight(struct free_ino_pool *pool) +{ + bool in_flight; + + spin_lock(&pool->lock); + in_flight = pool->in_flight; + spin_unlock(&pool->lock); + + return in_flight; +} + +/* + * We have a pool of free inodes given to us by the server. If it + * empties we only ever have one request for new inodes in flight. The + * net layer calls us when it gets a reply. If there's no more inodes + * we'll get ino == ~0 and nr == 0. + */ static int alloc_ino(struct super_block *sb, u64 *ino) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + bool request; int ret; - spin_lock(&sbi->next_ino_lock); + *ino = 0; - if (super->next_ino == 0) { - ret = -ENOSPC; - } else { - *ino = le64_to_cpu(super->next_ino); - le64_add_cpu(&super->next_ino, 1); - ret = 0; + spin_lock(&pool->lock); + + while (pool->nr == 0 && pool->ino != ~0ULL) { + if (pool->in_flight) { + request = false; + } else { + pool->in_flight = true; + request = true; + } + + spin_unlock(&pool->lock); + + if (request) { + ret = scoutfs_net_alloc_inodes(sb); + if (ret) { + spin_lock(&pool->lock); + pool->in_flight = false; + spin_unlock(&pool->lock); + wake_up(&pool->waitq); + goto out; + } + } + + ret = wait_event_interruptible(pool->waitq, + !pool_in_flight(pool)); + if (ret) + goto out; + + spin_lock(&pool->lock); } - spin_unlock(&sbi->next_ino_lock); + if (pool->nr == 0) { + *ino = 0; + ret = -ENOSPC; + } else { + *ino = pool->ino++; + pool->nr--; + ret = 0; + } + + spin_unlock(&pool->lock); + +out: + trace_printk("ret %d ino %llu pool ino %llu nr %llu req %u (racey)\n", + ret, *ino, pool->ino, pool->nr, pool->in_flight); return ret; } @@ -633,6 +717,30 @@ int scoutfs_orphan_inode(struct inode *inode) return ret; } +int scoutfs_inode_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct free_ino_pool *pool; + + pool = kzalloc(sizeof(struct free_ino_pool), GFP_KERNEL); + if (!pool) + return -ENOMEM; + + init_waitqueue_head(&pool->waitq); + spin_lock_init(&pool->lock); + + sbi->free_ino_pool = pool; + + return 0; +} + +void scoutfs_inode_destroy(struct super_block *sb) +{ + struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + + kfree(pool); +} + void scoutfs_inode_exit(void) { if (scoutfs_inode_cachep) { diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 6dcb03d8..b282960a 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -41,6 +41,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode); void scoutfs_dirty_inode(struct inode *inode, int flags); void scoutfs_update_inode_item(struct inode *inode); +void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); void scoutfs_inode_inc_data_version(struct inode *inode); @@ -53,7 +54,7 @@ u64 scoutfs_last_ino(struct super_block *sb); void scoutfs_inode_exit(void); int scoutfs_inode_init(void); -int scoutfs_item_setup(struct super_block *sb); -void scoutfs_item_destroy(struct super_block *sb); +int scoutfs_inode_setup(struct super_block *sb); +void scoutfs_inode_destroy(struct super_block *sb); #endif diff --git a/kmod/src/net.c b/kmod/src/net.c index 52dc18aa..6458c40e 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -22,6 +22,7 @@ #include "format.h" #include "net.h" #include "counters.h" +#include "inode.h" #include "scoutfs_trace.h" /* @@ -253,6 +254,47 @@ static struct send_buf *alloc_sbuf(unsigned data_len) return sbuf; } +/* + * XXX should this call into inodes? not sure about the layering here. + */ +static struct send_buf *process_alloc_inodes(struct super_block *sb, + void *req, int req_len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_inode_alloc *ial; + struct send_buf *sbuf; + int ret; + u64 ino; + u64 nr; + + if (req_len != 0) + return ERR_PTR(-EINVAL); + + sbuf = alloc_sbuf(sizeof(struct scoutfs_net_inode_alloc)); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + spin_lock(&sbi->next_ino_lock); + + ino = le64_to_cpu(super->next_ino); + nr = min(100000ULL, ~0ULL - ino); + le64_add_cpu(&super->next_ino, nr); + + spin_unlock(&sbi->next_ino_lock); + + /* XXX think about server ring commits */ + ret = 0; //sync_or_something(); + + ial = (void *)sbuf->nh->data; + ial->ino = cpu_to_le64(ino); + ial->nr = cpu_to_le64(nr); + + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + /* * Log the time in the request and reply with our current time. */ @@ -300,6 +342,9 @@ static int process_request(struct net_info *nti, struct recv_buf *rbuf) if (rbuf->nh->type == SCOUTFS_NET_TRADE_TIME) sbuf = process_trade_time(sb, (void *)rbuf->nh->data, data_len); + else if (rbuf->nh->type == SCOUTFS_NET_ALLOC_INODES) + sbuf = process_alloc_inodes(sb, (void *)rbuf->nh->data, + data_len); else sbuf = ERR_PTR(-EINVAL); @@ -702,7 +747,8 @@ static int add_send_buf(struct super_block *sb, int type, void *data, nh = sbuf->nh; nh->type = type; - memcpy(nh->data, data, data_len); + if (data_len) + memcpy(nh->data, data, data_len); mutex_lock(&nti->mutex); @@ -721,6 +767,43 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return 0; } +static int alloc_inodes_reply(struct super_block *sb, void *reply, int ret) +{ + struct scoutfs_net_inode_alloc *ial = reply; + u64 ino; + u64 nr; + + if (ret != sizeof(*ial)) { + ret = -EINVAL; + goto out; + } + + ino = le64_to_cpu(ial->ino); + nr = le64_to_cpu(ial->nr); + + /* catch wrapping */ + if (ino + nr < ino) { + ret = -EINVAL; + goto out; + } + + /* XXX compare to greatest inode we've seen? */ + + ret = 0; +out: + if (ret < 0) + scoutfs_inode_fill_pool(sb, 0, 0); + else + scoutfs_inode_fill_pool(sb, ino, nr); + return ret; +} + +int scoutfs_net_alloc_inodes(struct super_block *sb) +{ + return add_send_buf(sb, SCOUTFS_NET_ALLOC_INODES, NULL, 0, + alloc_inodes_reply); +} + static int trade_time_reply(struct super_block *sb, void *reply, int ret) { struct scoutfs_timespec *ts = reply; diff --git a/kmod/src/net.h b/kmod/src/net.h index 382b686e..d58c23c9 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -2,6 +2,7 @@ #define _SCOUTFS_NET_H_ int scoutfs_net_trade_time(struct super_block *sb); +int scoutfs_net_alloc_inodes(struct super_block *sb); int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index b15c9339..50daa887 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -215,6 +215,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_seg_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: + scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_compact_setup(sb) ?: @@ -259,6 +260,7 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_data_destroy(sb); + scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 458345fd..d3e1237a 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,6 +14,7 @@ struct compact_info; struct data_info; struct lock_info; struct net_info; +struct free_ino_pool; struct scoutfs_sb_info { struct super_block *sb; @@ -28,6 +29,7 @@ struct scoutfs_sb_info { struct seg_alloc *seg_alloc; struct compact_info *compact_info; struct data_info *data_info; + struct free_ino_pool *free_ino_pool; atomic_t trans_holds; wait_queue_head_t trans_hold_wq; From 5487aee6a773bef0f774effb1ba44d4b73fd6ac2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Apr 2017 14:21:08 -0700 Subject: [PATCH 251/920] Read items with manifest entries from server Item reading tries to directly walk the manifest to find segments to read. That doesn't work when only the server has read the ring and loaded the manifest. This adds a network message to ask the server for the manifest entries that describe the segments that will be needed to read items. Previously item reading would walk the manifest and build up native manifest references in a list that it'd use to read. To implement the network message we add request sending, processing, and reply parsing around those original functions. Item reading now packs its key range and sends it to the server. The server walks the manifest and sends the entries that intersect with the key range. Then the reply function builds up the native manifest references that item reading will use. The net reply functions needed an argument so that the manifest reading request could pass in the caller's list that the native manifest references should be added to. Signed-off-by: Zach Brown --- kmod/src/format.h | 12 +++ kmod/src/manifest.c | 103 ++++++++++++++-------- kmod/src/manifest.h | 11 +++ kmod/src/net.c | 208 ++++++++++++++++++++++++++++++++++++++++++-- kmod/src/net.h | 6 ++ 5 files changed, 296 insertions(+), 44 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 25663ac9..fd361c03 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -373,10 +373,22 @@ struct scoutfs_net_inode_alloc { __le64 nr; } __packed; +struct scoutfs_net_key_range { + __le16 start_len; + __le16 end_len; + __u8 key_bytes[0]; +} __packed; + +struct scoutfs_net_manifest_entries { + __le16 nr; + struct scoutfs_manifest_entry ments[0]; +} __packed; + enum { /* sends and receives a struct scoutfs_timeval */ SCOUTFS_NET_TRADE_TIME = 0, SCOUTFS_NET_ALLOC_INODES, + SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 6c9e72cb..bdd2f641 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -26,6 +26,7 @@ #include "manifest.h" #include "trans.h" #include "counters.h" +#include "net.h" #include "scoutfs_trace.h" /* @@ -256,6 +257,17 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, return 0; } +/* + * Return the total number of bytes used by the given manifest entry, + * including its struct. + */ +int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment) +{ + return sizeof(struct scoutfs_manifest_entry) + + le16_to_cpu(ment->first_key_len) + + le16_to_cpu(ment->last_key_len); +} + /* * XXX This feels pretty gross, but it's a simple way to give compaction * atomic updates. It'll go away once compactions go to the trouble of @@ -291,12 +303,20 @@ static void free_ref(struct super_block *sb, struct manifest_ref *ref) } } -static int alloc_add_ref(struct super_block *sb, struct list_head *list, - struct scoutfs_manifest_entry *ment) +/* + * Allocate a native manifest ref so that we can work with segments described + * by the callers manifest entry. + (* + * This frees all the elements on the list if it returns an error. + */ +int scoutfs_manifest_add_ment_ref(struct super_block *sb, + struct list_head *list, + struct scoutfs_manifest_entry *ment) { struct scoutfs_key_buf ment_first; struct scoutfs_key_buf ment_last; struct manifest_ref *ref; + struct manifest_ref *tmp; init_ment_keys(ment, &ment_first, &ment_last); @@ -307,6 +327,10 @@ static int alloc_add_ref(struct super_block *sb, struct list_head *list, } if (!ref || !ref->first || !ref->last) { free_ref(sb, ref); + list_for_each_entry_safe(ref, tmp, list, entry) { + list_del_init(&ref->entry); + free_ref(sb, ref); + } return -ENOMEM; } @@ -320,8 +344,10 @@ static int alloc_add_ref(struct super_block *sb, struct list_head *list, } /* - * Get refs on all the segments in the manifest that we'll need to - * search to populate the cache with the given range. + * Return an array of pointers to the entries in the manifest that + * intersect with the given key range. The entries will be ordered by + * the order that they should be read: level 0 from newest to oldest + * then increasing higher order levels. * * We have to get all the level 0 segments that intersect with the range * of items that we want to search because the level 0 segments can @@ -330,22 +356,40 @@ static int alloc_add_ref(struct super_block *sb, struct list_head *list, * We only need to search for the starting key in all the higher levels. * They do not overlap so we can iterate through the key space in each * segment starting with the key. + * + * This is called by the server who is processing manifest search + * messages from mounts. The server locks down the manifest while it + * gets these pointers and then uses them to allocate and fill a reply + * message. */ -static int get_range_refs(struct super_block *sb, struct manifest *mani, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - struct list_head *ref_list) +struct scoutfs_manifest_entry ** +scoutfs_manifest_find_range_entries(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, + unsigned *found_bytes) { + DECLARE_MANIFEST(sb, mani); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_manifest_entry **found; struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - struct manifest_ref *ref; - struct manifest_ref *tmp; - int ret; + unsigned nr; int i; - down_write(&mani->rwsem); + lockdep_assert_held(&mani->rwsem); + + *found_bytes = 0; + + /* at most we get all level 0, one from other levels, and null term */ + nr = get_level_count(mani, super, 0) + mani->nr_levels + 1; + + found = kcalloc(nr, sizeof(struct scoutfs_manifest_entry *), GFP_NOFS); + if (!found) { + found = ERR_PTR(-ENOMEM); + goto out; + } + + nr = 0; /* get level 0 segments that overlap with the missing range */ skey.level = 0; @@ -353,9 +397,8 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, ment = scoutfs_ring_lookup_prev(&mani->ring, &skey); while (ment) { if (cmp_range_ment(key, end, ment) == 0) { - ret = alloc_add_ref(sb, ref_list, ment); - if (ret) - goto out; + found[nr++] = ment; + *found_bytes += scoutfs_manifest_bytes(ment); } ment = scoutfs_ring_prev(&mani->ring, ment); @@ -371,27 +414,16 @@ static int get_range_refs(struct super_block *sb, struct manifest *mani, ment = scoutfs_ring_lookup(&mani->ring, &skey); if (ment) { - init_ment_keys(ment, &first, &last); - ret = alloc_add_ref(sb, ref_list, ment); - if (ret) - goto out; + found[nr++] = ment; + *found_bytes += scoutfs_manifest_bytes(ment); } } - ret = 0; + /* null terminate */ + found[nr++] = NULL; out: - up_write(&mani->rwsem); - - if (ret) { - list_for_each_entry_safe(ref, tmp, ref_list, entry) { - list_del_init(&ref->entry); - free_ref(sb, ref); - } - } - - trace_printk("ret %d\n", ret); - return ret; + return found; } /* @@ -425,7 +457,6 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end) { - DECLARE_MANIFEST(sb, mani); struct scoutfs_key_buf item_key; struct scoutfs_key_buf found_key; struct scoutfs_key_buf batch_end; @@ -449,9 +480,9 @@ int scoutfs_manifest_read_items(struct super_block *sb, trace_printk("reading items\n"); /* get refs on all the segments */ - ret = get_range_refs(sb, mani, key, end, &ref_list); + ret = scoutfs_net_manifest_range_entries(sb, key, end, &ref_list); if (ret) - return ret; + goto out; /* submit reads for all the segments */ list_for_each_entry(ref, &ref_list, entry) { diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 25cf236a..64936e59 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -17,12 +17,23 @@ int scoutfs_manifest_submit_write(struct super_block *sb, struct scoutfs_bio_completion *comp); void scoutfs_manifest_write_complete(struct super_block *sb); +int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment); + int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); +struct scoutfs_manifest_entry ** +scoutfs_manifest_find_range_entries(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, + unsigned *found_bytes); + int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); +int scoutfs_manifest_add_ment_ref(struct super_block *sb, + struct list_head *list, + struct scoutfs_manifest_entry *ment); u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); diff --git a/kmod/src/net.c b/kmod/src/net.c index 6458c40e..85143934 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -23,6 +23,7 @@ #include "net.h" #include "counters.h" #include "inode.h" +#include "manifest.h" #include "scoutfs_trace.h" /* @@ -82,7 +83,8 @@ struct net_info { #define DECLARE_NET_INFO(sb, name) \ struct net_info *name = SCOUTFS_SB(sb)->net_info -typedef int (*reply_func_t)(struct super_block *sb, void *recv, int bytes); +typedef int (*reply_func_t)(struct super_block *sb, void *recv, int bytes, + void *arg); /* * Send buffers are allocated either by clients who send requests or by @@ -93,6 +95,7 @@ typedef int (*reply_func_t)(struct super_block *sb, void *recv, int bytes); struct send_buf { struct list_head head; reply_func_t func; + void *arg; struct scoutfs_net_header nh[0]; }; @@ -254,6 +257,73 @@ static struct send_buf *alloc_sbuf(unsigned data_len) return sbuf; } +/* + * Find the manifest entries that intersect with the request's key + * range. We lock the manifest and get pointers to the manifest entries + * that intersect. We then allocate a reply buffer and copy them over. + */ +static struct send_buf *process_manifest_range_entries(struct super_block *sb, + void *req, int req_len) +{ + struct scoutfs_net_key_range *kr = req; + struct scoutfs_net_manifest_entries *ments; + struct scoutfs_manifest_entry **found = NULL; + struct scoutfs_manifest_entry *ment; + struct scoutfs_key_buf start; + struct scoutfs_key_buf end; + struct send_buf *sbuf; + unsigned total; + unsigned bytes; + int i; + + /* XXX this is a write lock and should be a read lock */ + scoutfs_manifest_lock(sb); + + if (req_len < sizeof(struct scoutfs_net_key_range) || + req_len < offsetof(struct scoutfs_net_key_range, + key_bytes[le16_to_cpu(kr->start_len) + + le16_to_cpu(kr->end_len)])) { + sbuf = ERR_PTR(-EINVAL); + goto out; + } + + scoutfs_key_init(&start, kr->key_bytes, le16_to_cpu(kr->start_len)); + scoutfs_key_init(&end, kr->key_bytes + le16_to_cpu(kr->start_len), + le16_to_cpu(kr->end_len)); + + found = scoutfs_manifest_find_range_entries(sb, &start, &end, &total); + if (IS_ERR(found)) { + sbuf = ERR_CAST(found); + goto out; + } + + total += sizeof(struct scoutfs_net_manifest_entries); + + sbuf = alloc_sbuf(total); + if (!sbuf) { + sbuf = ERR_PTR(-ENOMEM); + goto out; + } + + ments = (void *)sbuf->nh->data; + ment = ments->ments; + + for (i = 0; found[i]; i++) { + bytes = scoutfs_manifest_bytes(found[i]); + memcpy(ment, found[i], bytes); + ment = (void *)((char *)ment + bytes); + } + + ments->nr = cpu_to_le16(i); + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + +out: + scoutfs_manifest_unlock(sb); + if (!IS_ERR_OR_NULL(found)) + kfree(found); + return sbuf; +} + /* * XXX should this call into inodes? not sure about the layering here. */ @@ -345,6 +415,10 @@ static int process_request(struct net_info *nti, struct recv_buf *rbuf) else if (rbuf->nh->type == SCOUTFS_NET_ALLOC_INODES) sbuf = process_alloc_inodes(sb, (void *)rbuf->nh->data, data_len); + else if (rbuf->nh->type == SCOUTFS_NET_MANIFEST_RANGE_ENTRIES) + sbuf = process_manifest_range_entries(sb, + (void *)rbuf->nh->data, + data_len); else sbuf = ERR_PTR(-EINVAL); @@ -379,6 +453,7 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) struct super_block *sb = nti->sb; reply_func_t func = NULL; struct send_buf *sbuf; + void *arg; int ret; mutex_lock(&nti->mutex); @@ -388,6 +463,7 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) if (sbuf->nh->id == rbuf->nh->id) { list_del_init(&sbuf->head); func = sbuf->func; + arg = sbuf->arg; kfree(sbuf); sbuf = NULL; break; @@ -405,7 +481,7 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) else ret = -EIO; - return func(sb, rbuf->nh->data, ret); + return func(sb, rbuf->nh->data, ret, arg); } /* @@ -646,7 +722,7 @@ static void free_sbuf_list(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(sbuf, pos, list, head) { list_del_init(&sbuf->head); if (ret && sbuf->func) - sbuf->func(sb, NULL, ret); + sbuf->func(sb, NULL, ret, sbuf->arg); kfree(sbuf); } } @@ -731,7 +807,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) } static int add_send_buf(struct super_block *sb, int type, void *data, - unsigned data_len, reply_func_t func) + unsigned data_len, reply_func_t func, void *arg) { DECLARE_NET_INFO(sb, nti); struct scoutfs_net_header *nh; @@ -743,6 +819,7 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return -ENOMEM; sbuf->func = func; + sbuf->arg = arg; sbuf->nh->status = SCOUTFS_NET_STATUS_REQUEST; nh = sbuf->nh; @@ -767,7 +844,121 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return 0; } -static int alloc_inodes_reply(struct super_block *sb, void *reply, int ret) +struct manifest_range_entries_args { + struct list_head *list; + struct completion comp; + int ret; +}; + +/* + * The server has given us entries that intersect with our request's + * key range. Our caller is still blocked waiting for our completion. + * We walk the manifest entries and add native manifest refs to their + * list and wake them. + */ +static int manifest_range_entries_reply(struct super_block *sb, void *reply, + int reply_bytes, void *arg) +{ + struct manifest_range_entries_args *args = arg; + struct scoutfs_net_manifest_entries *ments = reply; + struct scoutfs_manifest_entry *ment; + unsigned bytes; + int ret = 0; + int i; + + if (reply_bytes < 0) { + ret = reply_bytes; + goto out; + } + + reply_bytes -= sizeof(struct scoutfs_net_manifest_entries); + if (reply_bytes < 0) { + ret = -EINVAL; + goto out; + } + + ment = ments->ments; + for (i = 0; i < le16_to_cpu(ments->nr); i++) { + + + if (reply_bytes < sizeof(struct scoutfs_manifest_entry)) { + ret = -EINVAL; + goto out; + } + + bytes = scoutfs_manifest_bytes(ment); + reply_bytes -= bytes; + if (reply_bytes < 0) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_manifest_add_ment_ref(sb, args->list, ment); + if (ret) + break; + + ment = (void *)((char *)ment + bytes); + } + +out: + args->ret = ret; + complete(&args->comp); /* args can be freed from this point */ + return ret; +} + +/* + * Ask the manifest server for the manifest entries whose key range + * intersects with the callers key range. The reply func will fill the + * caller's list with the reply's entries. + * + * XXX for now this can't be interrupted. The reply func which is off + * in work in a worker thread is blocking to allocate and put things on + * a list in our stack. We'd need better lifetime support to let it + * find out that we've returned and that it should stop processing the + * reply. + */ +int scoutfs_net_manifest_range_entries(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct list_head *list) +{ + struct manifest_range_entries_args args; + struct scoutfs_net_key_range *kr; + struct scoutfs_key_buf start_key; + struct scoutfs_key_buf end_key; + unsigned len; + int ret; + + len = sizeof(struct scoutfs_net_key_range) + + start->key_len + end->key_len; + kr = kmalloc(len, GFP_NOFS); + if (!kr) + return -ENOMEM; + + kr->start_len = cpu_to_le16(start->key_len); + kr->end_len = cpu_to_le16(end->key_len); + + scoutfs_key_init(&start_key, kr->key_bytes, start->key_len); + scoutfs_key_init(&end_key, kr->key_bytes + start->key_len, + end->key_len); + scoutfs_key_copy(&start_key, start); + scoutfs_key_copy(&end_key, end); + + args.list = list; + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, kr, len, + manifest_range_entries_reply, &args); + kfree(kr); + if (ret) + return ret; + + wait_for_completion(&args.comp); + return args.ret; +} + +static int alloc_inodes_reply(struct super_block *sb, void *reply, int ret, + void *arg) { struct scoutfs_net_inode_alloc *ial = reply; u64 ino; @@ -801,10 +992,11 @@ out: int scoutfs_net_alloc_inodes(struct super_block *sb) { return add_send_buf(sb, SCOUTFS_NET_ALLOC_INODES, NULL, 0, - alloc_inodes_reply); + alloc_inodes_reply, NULL); } -static int trade_time_reply(struct super_block *sb, void *reply, int ret) +static int trade_time_reply(struct super_block *sb, void *reply, int ret, + void *arg) { struct scoutfs_timespec *ts = reply; @@ -828,7 +1020,7 @@ int scoutfs_net_trade_time(struct super_block *sb) send.nsec = cpu_to_le32(ts.tv_nsec); ret = add_send_buf(sb, SCOUTFS_NET_TRADE_TIME, &send, - sizeof(send), trade_time_reply); + sizeof(send), trade_time_reply, NULL); trace_printk("sent %llu.%lu ret %d\n", (u64)ts.tv_sec, ts.tv_nsec, ret); diff --git a/kmod/src/net.h b/kmod/src/net.h index d58c23c9..aee93bfd 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -1,8 +1,14 @@ #ifndef _SCOUTFS_NET_H_ #define _SCOUTFS_NET_H_ +struct scoutfs_key_buf; + int scoutfs_net_trade_time(struct super_block *sb); int scoutfs_net_alloc_inodes(struct super_block *sb); +int scoutfs_net_manifest_range_entries(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct list_head *list); int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); From 5eefaf34f82d7a9f7aed5f8b3ae0a6fae0c4f570 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Apr 2017 11:07:37 -0700 Subject: [PATCH 252/920] Server updates ring for level0 segment writes Transaction commits currently directly modify the ring and super block as segments are written. As we introduce shared mounts only the server can modify the ring and super blocks. This adds network messages to let mounts write items in a level 0 segment while the server modifies the allocator and manifest. The item transaction commit now sends a message to the server to get an allocated segno for its new level0 segment and sends a manifest entry to the server once the segment is written. The request and reply handlers for the functions are straight forward. The processing paths are simple wrappers around the allocation and update functions that transaction writing used to call directly. Now that the item transactions aren't updating the super sync can't work with the super sequence numbers. The server needs to make both allocations and manifest updates persistent before it sends replies to the client. We add the ability for the server processing paths to queue and wait for commits of the rings and super block. We can hopefull get reasonable batching by using a work struct for the commit. We update the other processing path callers that modify the rings to use the new commit mechanism. We add a few segment and manifest functions to work with manifest entries that describe segments. This creats a bit of similar looking code thorughout the segment and manifest code but we'll come back and clean this up once we see what the final shared support looks like. scoutfs_seg_alloc() now takes the segno from the caller for the segment it's allocating and inserting into the cache. Transaction commit uses the segno it got from the server while compaction still allocates locally. Signed-off-by: Zach Brown --- kmod/src/compact.c | 11 +- kmod/src/format.h | 2 + kmod/src/manifest.c | 74 ++++++++++ kmod/src/manifest.h | 8 ++ kmod/src/net.c | 344 +++++++++++++++++++++++++++++++++++++++++--- kmod/src/net.h | 4 + kmod/src/seg.c | 40 ++++-- kmod/src/seg.h | 7 +- kmod/src/trans.c | 64 +++------ 9 files changed, 477 insertions(+), 77 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index cae9d7f6..4a9792c5 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -24,6 +24,7 @@ #include "manifest.h" #include "trans.h" #include "counters.h" +#include "alloc.h" #include "scoutfs_trace.h" /* @@ -347,6 +348,7 @@ static int compact_segments(struct super_block *sb, struct compact_seg *lower; u32 key_bytes; u32 nr_items; + u64 segno; int ret; scoutfs_inc_counter(sb, compact_operations); @@ -444,12 +446,19 @@ static int compact_segments(struct super_block *sb, break; } - ret = scoutfs_seg_alloc(sb, &seg); + ret = scoutfs_alloc_segno(sb, &segno); if (ret) { kfree(cseg); break; } + ret = scoutfs_seg_alloc(sb, segno, &seg); + if (ret) { + scoutfs_alloc_free(sb, segno); + kfree(cseg); + break; + } + /* csegs will be claned up once they're on the list */ cseg->level = curs->lower_level; cseg->seg = seg; diff --git a/kmod/src/format.h b/kmod/src/format.h index fd361c03..5d3c184b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -389,6 +389,8 @@ enum { SCOUTFS_NET_TRADE_TIME = 0, SCOUTFS_NET_ALLOC_INODES, SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, + SCOUTFS_NET_ALLOC_SEGNO, + SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index bdd2f641..869cc60f 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -210,6 +210,45 @@ int scoutfs_manifest_add(struct super_block *sb, return 0; } +/* + * Add a manifest entry as provided by the caller instead of exploded + * out into arguments. + * + * This must be called with the manifest lock held. + */ +int scoutfs_manifest_add_ment(struct super_block *sb, + struct scoutfs_manifest_entry *add) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_entry *ment; + struct manifest_search_key skey; + struct scoutfs_key_buf first; + unsigned bytes; + + lockdep_assert_held(&mani->rwsem); + + init_ment_keys(add, &first, NULL); + + skey.key = &first; + skey.level = add->level; + skey.seq = le64_to_cpu(add->seq); + + bytes = scoutfs_manifest_bytes(add); + + ment = scoutfs_ring_insert(&mani->ring, &skey, bytes); + if (!ment) + return -ENOMEM; + + memcpy(ment, add, bytes); + + mani->nr_levels = max_t(u8, mani->nr_levels, add->level + 1); + add_level_count(sb, mani, super, add->level, 1); + + return 0; +} + /* * This must be called with the manifest lock held. */ @@ -268,6 +307,41 @@ int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment) le16_to_cpu(ment->last_key_len); } +/* + * Return an allocated and filled in manifest entry. + */ +struct scoutfs_manifest_entry * +scoutfs_manifest_alloc_entry(struct super_block *sb, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, + u8 level) +{ + struct scoutfs_manifest_entry *ment; + struct scoutfs_key_buf ment_first; + struct scoutfs_key_buf ment_last; + unsigned key_bytes; + unsigned bytes; + + key_bytes = first->key_len + last->key_len; + bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); + + ment = kmalloc(bytes, GFP_NOFS); + if (!ment) + return NULL; + + ment->segno = cpu_to_le64(segno); + ment->seq = cpu_to_le64(seq); + ment->first_key_len = cpu_to_le16(first->key_len); + ment->last_key_len = cpu_to_le16(last->key_len); + ment->level = level; + + init_ment_keys(ment, &ment_first, &ment_last); + scoutfs_key_copy(&ment_first, first); + scoutfs_key_copy(&ment_last, last); + + return ment; +} + /* * XXX This feels pretty gross, but it's a simple way to give compaction * atomic updates. It'll go away once compactions go to the trouble of diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 64936e59..e0547c26 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -8,6 +8,8 @@ int scoutfs_manifest_add(struct super_block *sb, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, u64 segno, u64 seq, u8 level); +int scoutfs_manifest_add_ment(struct super_block *sb, + struct scoutfs_manifest_entry *add); int scoutfs_manifest_dirty(struct super_block *sb, struct scoutfs_key_buf *first, u64 seq, u8 level); int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, @@ -19,6 +21,12 @@ void scoutfs_manifest_write_complete(struct super_block *sb); int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment); +struct scoutfs_manifest_entry * +scoutfs_manifest_alloc_entry(struct super_block *sb, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, u64 segno, u64 seq, + u8 level); + int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index 85143934..bbe081cf 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -24,6 +24,9 @@ #include "counters.h" #include "inode.h" #include "manifest.h" +#include "bio.h" +#include "alloc.h" +#include "seg.h" #include "scoutfs_trace.h" /* @@ -71,6 +74,11 @@ struct net_info { struct delayed_work server_work; struct sock_info *listening_sinf; + /* server commits ring changes while processing requests */ + struct rw_semaphore ring_commit_rwsem; + struct llist_head ring_commit_waiters; + struct work_struct ring_commit_work; + /* both track active sockets for destruction */ struct list_head active_socks; @@ -243,6 +251,102 @@ static void scoutfs_net_send_func(struct work_struct *work) mutex_unlock(&nti->mutex); } +struct commit_waiter { + struct completion comp; + struct llist_node node; + int ret; +}; + +/* + * This is called while still holding the rwsem that prevents commits so + * that the caller can be sure to be woken by the next commit after they + * queue and release the lock. + * + * This could queue delayed work but we're first trying to have batching + * work by having concurrent modification line up behind a commit in + * flight. Once the commit finishes it'll unlock and hopefully everyone + * will race to make their changes and they'll all be applied by the + * next commit after that. + */ +static void queue_commit_work(struct net_info *nti, struct commit_waiter *cw) +{ + lockdep_assert_held(&nti->ring_commit_rwsem); + + cw->ret = 0; + init_completion(&cw->comp); + llist_add(&cw->node, &nti->ring_commit_waiters); + queue_work(nti->proc_wq, &nti->ring_commit_work); +} + +static int wait_for_commit(struct commit_waiter *cw) +{ + wait_for_completion(&cw->comp); + return cw->ret; +} + +/* + * A core function of request processing is to modify the manifest and + * allocator. Often the processing needs to make the modifications + * persistent before replying. We'd like to batch these commits as much + * as is reasonable so that we don't degrade to a few IO round trips per + * request. + * + * Getting that batching right is bound up in the concurrency of request + * processing so a clear way to implement the batched commits is to + * implement commits with work funcs like the processing. This ring + * commit work is queued on the non-reentrant proc_wq so there will only + * ever be one commit executing at a time. + * + * Processing paths acquire the rwsem for reading while they're making + * multiple dependent changes. When they're done and want it persistent + * they add themselves to the list of waiters and queue the commit work. + * This work runs, acquires the lock to exclude other writers, and + * performs the commit. Readers can run concurrently with these + * commits. + */ +static void scoutfs_net_ring_commit_func(struct work_struct *work) +{ + struct net_info *nti = container_of(work, struct net_info, + ring_commit_work); + struct super_block *sb = nti->sb; + struct scoutfs_bio_completion comp; + struct commit_waiter *cw; + struct commit_waiter *pos; + struct llist_node *node; + int ret; + + scoutfs_bio_init_comp(&comp); + + down_write(&nti->ring_commit_rwsem); + + if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { + ret = scoutfs_manifest_submit_write(sb, &comp) ?: + scoutfs_alloc_submit_write(sb, &comp) ?: + scoutfs_bio_wait_comp(sb, &comp) ?: + scoutfs_write_dirty_super(sb); + + /* we'd need to loop or something */ + BUG_ON(ret); + + scoutfs_manifest_write_complete(sb); + scoutfs_alloc_write_complete(sb); + + scoutfs_advance_dirty_super(sb); + } else { + ret = 0; + } + + node = llist_del_all(&nti->ring_commit_waiters); + + /* waiters always wait on completion, cw could be free after complete */ + llist_for_each_entry_safe(cw, pos, node, node) { + cw->ret = ret; + complete(&cw->comp); + } + + up_write(&nti->ring_commit_rwsem); +} + static struct send_buf *alloc_sbuf(unsigned data_len) { unsigned len = offsetof(struct send_buf, nh[0].data[data_len]); @@ -257,6 +361,98 @@ static struct send_buf *alloc_sbuf(unsigned data_len) return sbuf; } +static struct send_buf *process_record_segment(struct super_block *sb, + void *req, int req_len) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_manifest_entry *ment; + struct commit_waiter cw; + struct send_buf *sbuf; + int ret; + + if (req_len < sizeof(struct scoutfs_manifest_entry)) { + sbuf = ERR_PTR(-EINVAL); + goto out; + } + + ment = req; + + if (req_len != scoutfs_manifest_bytes(ment)) { + sbuf = ERR_PTR(-EINVAL); + goto out; + } + + down_read(&nti->ring_commit_rwsem); + + scoutfs_manifest_lock(sb); + ret = scoutfs_manifest_add_ment(sb, ment); + scoutfs_manifest_unlock(sb); + + if (ret == 0) + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + sbuf = alloc_sbuf(0); + if (!sbuf) { + sbuf = ERR_PTR(-ENOMEM); + goto out; + } + + if (ret) + sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; + else + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; +out: + return sbuf; +} + +static struct send_buf *process_alloc_segno(struct super_block *sb, + void *req, int req_len) +{ + DECLARE_NET_INFO(sb, nti); + __le64 * __packed lesegno; + struct commit_waiter cw; + struct send_buf *sbuf; + u64 segno; + int ret; + + if (req_len != 0) { + sbuf = ERR_PTR(-EINVAL); + goto out; + } + + down_read(&nti->ring_commit_rwsem); + + ret = scoutfs_alloc_segno(sb, &segno); + if (ret == 0) + queue_commit_work(nti, &cw); + + up_read(&nti->ring_commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + sbuf = alloc_sbuf(sizeof(__le64)); + if (!sbuf) { + sbuf = ERR_PTR(-ENOMEM); + goto out; + } + + if (ret) { + sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; + } else { + lesegno = (void *)sbuf->nh->data; + *lesegno = cpu_to_le64(segno); + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + } + +out: + return sbuf; +} + /* * Find the manifest entries that intersect with the request's key * range. We lock the manifest and get pointers to the manifest entries @@ -330,9 +526,11 @@ out: static struct send_buf *process_alloc_inodes(struct super_block *sb, void *req, int req_len) { + DECLARE_NET_INFO(sb, nti); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_inode_alloc *ial; + struct commit_waiter cw; struct send_buf *sbuf; int ret; u64 ino; @@ -345,22 +543,27 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, if (!sbuf) return ERR_PTR(-ENOMEM); - spin_lock(&sbi->next_ino_lock); + down_read(&nti->ring_commit_rwsem); + spin_lock(&sbi->next_ino_lock); ino = le64_to_cpu(super->next_ino); nr = min(100000ULL, ~0ULL - ino); le64_add_cpu(&super->next_ino, nr); - spin_unlock(&sbi->next_ino_lock); - /* XXX think about server ring commits */ - ret = 0; //sync_or_something(); + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + ret = wait_for_commit(&cw); ial = (void *)sbuf->nh->data; ial->ino = cpu_to_le64(ino); ial->nr = cpu_to_le64(nr); - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + if (ret < 0) + sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; + else + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; return sbuf; } @@ -369,9 +572,9 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, * Log the time in the request and reply with our current time. */ static struct send_buf *process_trade_time(struct super_block *sb, - struct scoutfs_timespec *req, - int req_len) + void *r, int req_len) { + struct scoutfs_timespec *req = r; struct scoutfs_timespec *reply; struct send_buf *sbuf; struct timespec64 ts; @@ -397,6 +600,23 @@ static struct send_buf *process_trade_time(struct super_block *sb, return sbuf; } +typedef struct send_buf *(*proc_func_t)(struct super_block *sb, void *req, + int req_len); + +static proc_func_t type_proc_func(u8 type) +{ + static proc_func_t funcs[] = { + [SCOUTFS_NET_TRADE_TIME] = process_trade_time, + [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, + [SCOUTFS_NET_MANIFEST_RANGE_ENTRIES] = + process_manifest_range_entries, + [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, + [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, + }; + + return type < SCOUTFS_NET_UNKNOWN ? funcs[type] : NULL; +} + /* * Process an incoming request and queue its reply to send if the socket * is still open by the time we have the reply. @@ -405,23 +625,15 @@ static int process_request(struct net_info *nti, struct recv_buf *rbuf) { struct super_block *sb = nti->sb; struct send_buf *sbuf; + proc_func_t proc; unsigned data_len; data_len = le16_to_cpu(rbuf->nh->data_len); - - if (rbuf->nh->type == SCOUTFS_NET_TRADE_TIME) - sbuf = process_trade_time(sb, (void *)rbuf->nh->data, - data_len); - else if (rbuf->nh->type == SCOUTFS_NET_ALLOC_INODES) - sbuf = process_alloc_inodes(sb, (void *)rbuf->nh->data, - data_len); - else if (rbuf->nh->type == SCOUTFS_NET_MANIFEST_RANGE_ENTRIES) - sbuf = process_manifest_range_entries(sb, - (void *)rbuf->nh->data, - data_len); + proc = type_proc_func(rbuf->nh->type); + if (proc) + sbuf = proc(sb, (void *)rbuf->nh->data, data_len); else sbuf = ERR_PTR(-EINVAL); - if (IS_ERR(sbuf)) return PTR_ERR(sbuf); @@ -844,6 +1056,93 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return 0; } +struct record_segment_args { + struct completion comp; + int ret; +}; + +static int record_segment_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct record_segment_args *args = arg; + + if (ret > 0) + ret = -EINVAL; + + args->ret = ret; + complete(&args->comp); + return args->ret; +} + +int scoutfs_net_record_segment(struct super_block *sb, + struct scoutfs_segment *seg, u8 level) +{ + struct scoutfs_manifest_entry *ment; + struct record_segment_args args; + int ret; + + ment = scoutfs_seg_manifest_entry(sb, seg, level); + if (!ment) { + ret = -ENOMEM; + goto out; + } + + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_RECORD_SEGMENT, ment, + scoutfs_manifest_bytes(ment), + record_segment_reply, &args); + kfree(ment); + if (ret == 0) { + wait_for_completion(&args.comp); + ret = args.ret; + } +out: + return ret; +} + +struct alloc_segno_args { + u64 segno; + struct completion comp; + int ret; +}; + +static int alloc_segno_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct alloc_segno_args *args = arg; + __le64 * __packed segno = reply; + + if (ret == sizeof(__le64)) { + args->segno = le64_to_cpup(segno); + args->ret = 0; + } else { + args->ret = -EINVAL; + } + + complete(&args->comp); /* args can be freed from this point */ + return args->ret; +} + +int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno) +{ + struct alloc_segno_args args; + int ret; + + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_ALLOC_SEGNO, NULL, 0, + alloc_segno_reply, &args); + if (ret == 0) { + wait_for_completion(&args.comp); + *segno = args.segno; + ret = args.ret; + if (ret == 0 && *segno == 0) + ret = -ENOSPC; + } + return ret; +} + struct manifest_range_entries_args { struct list_head *list; struct completion comp; @@ -1337,6 +1636,9 @@ int scoutfs_net_setup(struct super_block *sb) INIT_LIST_HEAD(&nti->to_send); nti->next_id = 1; INIT_DELAYED_WORK(&nti->server_work, scoutfs_net_server_func); + init_rwsem(&nti->ring_commit_rwsem); + init_llist_head(&nti->ring_commit_waiters); + INIT_WORK(&nti->ring_commit_work, scoutfs_net_ring_commit_func); INIT_LIST_HEAD(&nti->active_socks); sbi->net_info = nti; @@ -1383,8 +1685,8 @@ void scoutfs_net_destroy(struct super_block *sb) mutex_unlock(&nti->mutex); drain_workqueue(nti->sock_wq); - /* wait for processing to finish and free rbufs */ - flush_workqueue(nti->proc_wq); + /* wait for processing (and commits) to finish and free rbufs */ + drain_workqueue(nti->proc_wq); /* make sure client/server work isn't queued */ cancel_delayed_work_sync(&nti->server_work); diff --git a/kmod/src/net.h b/kmod/src/net.h index aee93bfd..973b0530 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -2,6 +2,7 @@ #define _SCOUTFS_NET_H_ struct scoutfs_key_buf; +struct scoutfs_segment; int scoutfs_net_trade_time(struct super_block *sb); int scoutfs_net_alloc_inodes(struct super_block *sb); @@ -9,6 +10,9 @@ int scoutfs_net_manifest_range_entries(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, struct list_head *list); +int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno); +int scoutfs_net_record_segment(struct super_block *sb, + struct scoutfs_segment *seg, u8 level); int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); diff --git a/kmod/src/seg.c b/kmod/src/seg.c index b343cb5e..6e0fc04a 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -245,26 +245,18 @@ static u64 segno_to_blkno(u64 blkno) return blkno << (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT); } -int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret) +int scoutfs_seg_alloc(struct super_block *sb, u64 segno, + struct scoutfs_segment **seg_ret) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct segment_cache *cac = sbi->segment_cache; struct scoutfs_segment *existing; struct scoutfs_segment *seg; unsigned long flags; - u64 segno; int ret; - *seg_ret = NULL; - - ret = scoutfs_alloc_segno(sb, &segno); - if (ret) - goto out; - seg = alloc_seg(segno); if (!seg) { - ret = scoutfs_alloc_free(sb, segno); - BUG_ON(ret); /* XXX could make pending when allocating */ ret = -ENOMEM; goto out; } @@ -281,9 +273,9 @@ int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret) if (existing) scoutfs_seg_put(existing); - *seg_ret = seg; ret = 0; out: + *seg_ret = seg; return ret; } @@ -632,6 +624,32 @@ int scoutfs_seg_manifest_del(struct super_block *sb, return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); } +/* + * Return an allocated manifest entry that describes the segment, returns + * NULL if it couldn't allocate. + */ +struct scoutfs_manifest_entry * +scoutfs_seg_manifest_entry(struct super_block *sb, + struct scoutfs_segment *seg, u8 level) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct scoutfs_segment_item *item; + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; + + item = pos_ptr(seg, 0); + scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); + + item = pos_ptr(seg, le32_to_cpu(sblk->nr_items) - 1); + scoutfs_key_init(&last, off_ptr(seg, le32_to_cpu(item->key_off)), + le16_to_cpu(item->key_len)); + + return scoutfs_manifest_alloc_entry(sb, &first, &last, + le64_to_cpu(sblk->segno), + le64_to_cpu(sblk->seq), level); +} + /* * We maintain an LRU of segments so that the shrinker can free the * oldest under memory pressure. Segments are only present in the LRU diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 15c6834f..5ce0c076 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -19,7 +19,8 @@ int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, void scoutfs_seg_get(struct scoutfs_segment *seg); void scoutfs_seg_put(struct scoutfs_segment *seg); -int scoutfs_seg_alloc(struct super_block *sb, struct scoutfs_segment **seg_ret); +int scoutfs_seg_alloc(struct super_block *sb, u64 segno, + struct scoutfs_segment **seg_ret); int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); @@ -41,6 +42,10 @@ int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_bio_completion *comp); +struct scoutfs_manifest_entry * +scoutfs_seg_manifest_entry(struct super_block *sb, + struct scoutfs_segment *seg, u8 level); + int scoutfs_seg_setup(struct super_block *sb); void scoutfs_seg_destroy(struct super_block *sb); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index aa927fb2..5661398f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -24,10 +24,9 @@ #include "item.h" #include "manifest.h" #include "seg.h" -#include "alloc.h" -#include "ring.h" #include "compact.h" #include "counters.h" +#include "net.h" #include "scoutfs_trace.h" /* @@ -82,7 +81,7 @@ void scoutfs_trans_write_func(struct work_struct *work) struct super_block *sb = sbi->sb; struct scoutfs_bio_completion comp; struct scoutfs_segment *seg; - bool advance = false; + u64 segno; int ret = 0; scoutfs_bio_init_comp(&comp); @@ -91,42 +90,25 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); - trace_printk("items dirty %d manifest dirty %d alloc dirty %d\n", - scoutfs_item_has_dirty(sb), - scoutfs_manifest_has_dirty(sb), - scoutfs_alloc_has_dirty(sb)); + trace_printk("items dirty %d\n", scoutfs_item_has_dirty(sb)); - /* - * XXX this needs serious work to handle errors. - */ - while (scoutfs_item_has_dirty(sb)) { - seg = NULL; - ret = scoutfs_seg_alloc(sb, &seg) ?: + if (scoutfs_item_has_dirty(sb)) { + /* + * XXX only straight pass through, we're not worrying + * about leaking segnos nor duplicate manifest entries + * on crashes between us and the server. + */ + ret = scoutfs_net_alloc_segno(sb, &segno) ?: + scoutfs_seg_alloc(sb, segno, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: - scoutfs_manifest_lock(sb) ?: - scoutfs_seg_manifest_add(sb, seg, 0) ?: - scoutfs_manifest_unlock(sb) ?: - scoutfs_seg_submit_write(sb, seg, &comp); - scoutfs_seg_put(seg); + scoutfs_seg_submit_write(sb, seg, &comp) ?: + scoutfs_bio_wait_comp(sb, &comp) ?: + scoutfs_net_record_segment(sb, seg, 0); if (ret) goto out; scoutfs_inc_counter(sb, trans_level0_seg_write); } - - if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { - ret = scoutfs_manifest_submit_write(sb, &comp) ?: - scoutfs_alloc_submit_write(sb, &comp) ?: - scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_write_dirty_super(sb); - if (ret) - goto out; - - scoutfs_manifest_write_complete(sb); - scoutfs_alloc_write_complete(sb); - advance = true; - } - out: /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); @@ -135,8 +117,6 @@ out: scoutfs_data_end_writeback(sb, ret); spin_lock(&sbi->trans_write_lock); - if (advance) - scoutfs_advance_dirty_super(sb); sbi->trans_write_count++; sbi->trans_write_ret = ret; spin_unlock(&sbi->trans_write_lock); @@ -149,7 +129,6 @@ out: } struct write_attempt { - u64 seq; u64 count; int ret; }; @@ -161,9 +140,7 @@ static int write_attempted(struct scoutfs_sb_info *sbi, int done = 1; spin_lock(&sbi->trans_write_lock); - if (le64_to_cpu(sbi->super.hdr.seq) > attempt->seq) - attempt->ret = 0; - else if (sbi->trans_write_count > attempt->count) + if (sbi->trans_write_count > attempt->count) attempt->ret = sbi->trans_write_ret; else done = 0; @@ -178,10 +155,12 @@ static void queue_trans_work(struct scoutfs_sb_info *sbi) } /* - * sync records the current dirty seq and write count and waits for - * either to change. If there's nothing to write or the write returned - * an error then only the write count advances and sets the appropriate - * return code. + * Wait for a trans commit to finish and return its error code. There + * can already be one in flight that we end up waiting for the + * completion of. This is safe because dirtying and trans commits are + * serialized. There's no way that there could have been dirty data + * before the caller got here that wouldn't be covered by a commit + * that's in flight. */ int scoutfs_sync_fs(struct super_block *sb, int wait) { @@ -197,7 +176,6 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) } spin_lock(&sbi->trans_write_lock); - attempt.seq = le64_to_cpu(sbi->super.hdr.seq); attempt.count = sbi->trans_write_count; spin_unlock(&sbi->trans_write_lock); From cec3f9468af15d014492ee3015667b6e2f705493 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Apr 2017 15:25:30 -0700 Subject: [PATCH 253/920] Further isolate rings and compaction Each mount was still loading the manifest and allocator rings and starting compaction, even if they were coordinating segment reads and writes with the server. This moves ring and compaction setup and teardown from on mount and unmount to as the server starts up and shuts down. Now only the server has the rings resident and is running compaction. We had to null some of the super info fields so that we can repeatedly load and destroy the ring indices over the lifetime of a mount. We also have to be careful not to call between item transactions and compaction. We'll restore this functionality with the server in the future. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 2 ++ kmod/src/compact.c | 8 +++++++- kmod/src/manifest.c | 1 + kmod/src/net.c | 46 +++++++++++++++++++++++++++++++++++++++++++-- kmod/src/super.c | 15 ++++++++------- kmod/src/super.h | 1 + kmod/src/trans.c | 3 ++- 7 files changed, 65 insertions(+), 11 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index fa6676d1..75aa408e 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -363,6 +363,7 @@ int scoutfs_alloc_setup(struct super_block *sb) void scoutfs_alloc_destroy(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_SEG_ALLOC(sb, sal); struct pending_region *pend; struct rb_node *node; @@ -375,5 +376,6 @@ void scoutfs_alloc_destroy(struct super_block *sb) kfree(pend); } kfree(sal); + sbi->seg_alloc = NULL; } } diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 4a9792c5..1f3835c1 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -22,7 +22,6 @@ #include "cmp.h" #include "compact.h" #include "manifest.h" -#include "trans.h" #include "counters.h" #include "alloc.h" #include "scoutfs_trace.h" @@ -645,8 +644,13 @@ static void scoutfs_compact_func(struct work_struct *work) ret = update_manifest(sb, &curs, &results); if (ret == 0) { +#if 0 /* XXX this is busted, fixing soon */ scoutfs_sync_fs(sb, 0); +#endif + +#if 0 /* XXX where do we do this in shared? */ scoutfs_trans_wake_holders(sb); +#endif scoutfs_compact_kick(sb); } out: @@ -695,10 +699,12 @@ int scoutfs_compact_setup(struct super_block *sb) */ void scoutfs_compact_destroy(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_COMPACT_INFO(sb, ci); if (ci) { flush_work(&ci->work); destroy_workqueue(ci->workq); + sbi->compact_info = NULL; } } diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 869cc60f..b3c7a258 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -1004,5 +1004,6 @@ void scoutfs_manifest_destroy(struct super_block *sb) for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) scoutfs_key_free(sb, mani->compact_keys[i]); kfree(mani); + sbi->manifest = NULL; } } diff --git a/kmod/src/net.c b/kmod/src/net.c index bbe081cf..ad22ea7c 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -27,6 +27,7 @@ #include "bio.h" #include "alloc.h" #include "seg.h" +#include "compact.h" #include "scoutfs_trace.h" /* @@ -73,6 +74,7 @@ struct net_info { /* server listens and processes requests */ struct delayed_work server_work; struct sock_info *listening_sinf; + bool server_loaded; /* server commits ring changes while processing requests */ struct rw_semaphore ring_commit_rwsem; @@ -696,6 +698,13 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) return func(sb, rbuf->nh->data, ret, arg); } +static void destroy_server_state(struct super_block *sb) +{ + scoutfs_alloc_destroy(sb); + scoutfs_manifest_destroy(sb); + scoutfs_compact_destroy(sb); +} + /* * Process each received message in its own non-reentrant work so we get * concurrent request processing. @@ -704,7 +713,35 @@ static void scoutfs_net_proc_func(struct work_struct *work) { struct recv_buf *rbuf = container_of(work, struct recv_buf, proc_work); struct net_info *nti = rbuf->nti; - int ret; + struct super_block *sb = nti->sb; + int ret = 0; + + /* + * This is the first blocking context we have once all the + * server locking and networking is set up so we bring up the + * rest of the server state the first time we get here. + */ + while (!nti->server_loaded) { + mutex_lock(&nti->mutex); + if (!nti->server_loaded) { + ret = scoutfs_read_supers(sb) ?: + scoutfs_manifest_setup(sb) ?: + scoutfs_alloc_setup(sb) ?: + scoutfs_compact_setup(sb); + if (ret == 0) { + scoutfs_advance_dirty_super(sb); + nti->server_loaded = true; + } else { + destroy_server_state(sb); + } + } + mutex_unlock(&nti->mutex); + if (ret) { + trace_printk("server setup failed %d\n", ret); + queue_sock_work(rbuf->sinf, &rbuf->sinf->shutdown_work); + return; + } + } if (rbuf->nh->status == SCOUTFS_NET_STATUS_REQUEST) ret = process_request(nti, rbuf); @@ -990,8 +1027,13 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) mutex_lock(&nti->mutex); if (sinf == nti->listening_sinf) { - /* clear addr lvb and try to reacquire lock and listen */ nti->listening_sinf = NULL; + + /* shutdown the server, processing won't leave rings dirty */ + destroy_server_state(sb); + nti->server_loaded = false; + + /* clear addr lvb and try to reacquire lock and listen */ memset(&sinf->addr, 0, sizeof(sinf->addr)); lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_WRITE, &sinf->addr); scoutfs_unlock_range(sb, &sinf->listen_lck); diff --git a/kmod/src/super.c b/kmod/src/super.c index 50daa887..5a43a426 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -133,7 +133,13 @@ int scoutfs_write_dirty_super(struct super_block *sb) return ret; } -static int read_supers(struct super_block *sb) +/* + * Read the pair of super blocks and store the most recent one in the sb + * info. Clients reference but don't modify the super. The server has + * to re-read the super every time it comes up so that it can work from + * the most recent persistent state. + */ +int scoutfs_read_supers(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; @@ -211,14 +217,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -ENOMEM; ret = scoutfs_setup_counters(sb) ?: - read_supers(sb) ?: + scoutfs_read_supers(sb) ?: scoutfs_seg_setup(sb) ?: - scoutfs_manifest_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: - scoutfs_alloc_setup(sb) ?: - scoutfs_compact_setup(sb) ?: scoutfs_setup_trans(sb) ?: scoutfs_lock_setup(sb) ?: scoutfs_net_setup(sb); @@ -227,8 +230,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_net_trade_time(sb); - scoutfs_advance_dirty_super(sb); - inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) return PTR_ERR(inode); diff --git a/kmod/src/super.h b/kmod/src/super.h index d3e1237a..184c92b8 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -56,6 +56,7 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } +int scoutfs_read_supers(struct super_block *sb); void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 5661398f..e6247bc0 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -24,7 +24,6 @@ #include "item.h" #include "manifest.h" #include "seg.h" -#include "compact.h" #include "counters.h" #include "net.h" #include "scoutfs_trace.h" @@ -239,11 +238,13 @@ static bool hold_acquired(struct super_block *sb) if (holds < 0) return false; +#if 0 /* XXX where will we do this in the shared universe? */ /* only hold when there's no level 0 segments, XXX for now */ if (scoutfs_manifest_level_count(sb, 0) > 0) { scoutfs_compact_kick(sb); return false; } +#endif /* see if we all would fill the segment */ with_us = holds + 1; From efd95688d37f902a5c475aca3031d177c071d99b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 17 Nov 2016 15:55:25 -0800 Subject: [PATCH 254/920] Add printf format checking to scoutfs msg funcs scoutfs_msg() was missing the attribute to check printf formats and arguments. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/msg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 64376f9a..8e53290f 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -1,8 +1,8 @@ #ifndef _SCOUTFS_MSG_H_ #define _SCOUTFS_MSG_H_ -void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, - const char *fmt, ...); +void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, + const char *str, const char *fmt, ...); #define scoutfs_err(sb, fmt, args...) \ scoutfs_msg(sb, KERN_ERR, " error", fmt, ##args) From e61697a54e9a0d622edb03ad3b98904364a2371b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Nov 2016 12:51:35 -0800 Subject: [PATCH 255/920] Add generic file and dir seek methods Two more xfstests pass when we can seek in files and dirs. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/data.c | 1 + kmod/src/dir.c | 1 + 2 files changed, 2 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index b37d2f10..61547c2e 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -545,6 +545,7 @@ const struct file_operations scoutfs_file_fops = { .aio_write = generic_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, + .llseek = generic_file_llseek, }; int scoutfs_data_setup(struct super_block *sb) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 79b75dcc..d912b136 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -942,6 +942,7 @@ out: const struct file_operations scoutfs_dir_fops = { .readdir = scoutfs_readdir, .fsync = scoutfs_file_fsync, + .llseek = generic_file_llseek, }; const struct inode_operations scoutfs_dir_iops = { From 9fc47dedf80ce220673a9d47525f63a0eddd4c28 Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Fri, 18 Nov 2016 10:05:57 -0700 Subject: [PATCH 256/920] Add unlocked ioctls for directories. The use of the Scout ioctls for inode-since and data-since on the root directory is a rather helpful boost. This allows user code to start on blank filesystems and monitor activity without needing to create files. The existing ioctl code was already present, so wiring into the directory file operations was all that needed to happen. Signed-off-by: Nic Henke Reviewed-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/dir.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index d912b136..ce579e8c 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -21,6 +21,7 @@ #include "format.h" #include "dir.h" #include "inode.h" +#include "ioctl.h" #include "key.h" #include "super.h" #include "trans.h" @@ -941,6 +942,7 @@ out: const struct file_operations scoutfs_dir_fops = { .readdir = scoutfs_readdir, + .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, .llseek = generic_file_llseek, }; From 2591e54fdcbb24ea0e8105a4c8f8f2b786bea6aa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Nov 2016 13:46:21 -0800 Subject: [PATCH 257/920] Make it easier to build scoutfs.ko We were duplicating the make args a few times so make a little ARGS variable. Default to the /lib/modules/$(uname -r) installed kernel source if SK_KSRC isn't set. And only try a sparse build that can fail if we can execute the sparse command. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/Makefile | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/kmod/Makefile b/kmod/Makefile index d0abca9b..54aafb03 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -1,38 +1,24 @@ ALL: module -# -# SK_KSRC points to the kernel header build dir to build against. -# On a running machine this could be /lib/modules/$(uname -r)/build with -# the right kernel-headers package installed. I tend to build on other -# hosts so I extract the kernel-headers package for the target machine's -# kernel in a dir somehere. -# -# sparse is critical for avoiding endian mistakes. It should just work -# if the sparse package is installed. -# -# but sometimes kernel-headers are broken. For example, the -# rhel 3.10.0-327.el7.x86_64 kernel needs the following patch. -# We'll try to have a git tree with fixed headers. -# -# -# diff --git a/include/linux/rh_kabi.h b/include/linux/rh_kabi.h -# index 1767770..0a8e5f3 100644 -# --- a/include/linux/rh_kabi.h -# +++ b/include/linux/rh_kabi.h -# @@ -73,7 +73,6 @@ -# struct { \ -# _orig; \ -# } __UNIQUE_ID(rh_kabi_hide); \ -# - __RH_KABI_CHECK_SIZE_ALIGN(_orig, _new); \ -# } -# -# #define _RH_KABI_REPLACE_UNSAFE(_orig, _new) _new +# default to building against the installed source for the running kernel +ifeq ($(SK_KSRC),) +SK_KSRC := $(shell echo /lib/modules/`uname -r`/build) +endif + +# fail if sparse fails if we find it +ifeq ($(shell sparse && echo found),found) +SP = +else +SP = @: +endif + +ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src all: module module: - make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src - make C=2 CF="-D__CHECK_ENDIAN__" CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src + make $(ARGS) + $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(ARGS) clean: - make CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src clean + make $(ARGS) clean From 78d15a019c7cbe7d8ff9899291fe779c24423fb8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 22 Nov 2016 10:28:56 -0800 Subject: [PATCH 258/920] Print inode nr and err on inode upate error We're currently excessively freaking out if inode updates fail. Let's add a little more context to help us track down what goes wrong. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/inode.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 560f1d2a..76b9048c 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -320,7 +320,11 @@ void scoutfs_update_inode_item(struct inode *inode) scoutfs_kvec_init(val, &sinode, sizeof(sinode)); err = scoutfs_item_update(sb, &key, val); - BUG_ON(err); + if (err) { + scoutfs_err(sb, "inode %llu update err %d", + scoutfs_ino(inode), err); + BUG_ON(err); + } trace_scoutfs_update_inode(inode); } From 2aa274b38b3096fa8fb57e2ea3d879d95d9fe1ce Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 22 Nov 2016 13:41:27 -0800 Subject: [PATCH 259/920] Add xattr iops for special files xfstests generic/062 was failing because it was getting an unexpected error code when trying to work with xattrs on special files. Adding our ops gives it the errnos it expects. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/inode.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 76b9048c..046de8e5 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -89,6 +89,13 @@ static const struct inode_operations scoutfs_file_iops = { .removexattr = scoutfs_removexattr, }; +static const struct inode_operations scoutfs_special_iops = { + .setxattr = scoutfs_setxattr, + .getxattr = scoutfs_getxattr, + .listxattr = scoutfs_listxattr, + .removexattr = scoutfs_removexattr, +}; + /* * Called once new inode allocation or inode reading has initialized * enough of the inode for us to set the ops based on the mode. @@ -109,7 +116,7 @@ static void set_inode_ops(struct inode *inode) inode->i_op = &scoutfs_symlink_iops; break; default: -// inode->i_op = &scoutfs_special_iops; + inode->i_op = &scoutfs_special_iops; init_special_inode(inode, inode->i_mode, inode->i_rdev); break; } From 37ba46213c84da56f9e8f220ee8b2829ee6a92aa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 22 Nov 2016 17:12:02 -0800 Subject: [PATCH 260/920] Add suport for more xattr namespaces Add support for more of the known xattr namespaces. This helps generic/062 in xfstests pass. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/xattr.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index c3d10c2c..f2fd9289 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -129,13 +129,12 @@ static void set_xattr_key_part(struct scoutfs_key_buf *key, u8 part) 1); \ part++, off += bytes) -/* - * This will grow to have all the supported prefixes (then will turn - * into xattr_handlers with prefixes upstream). - */ static int unknown_prefix(const char *name) { - return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); + return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) && + strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN) && + strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); } /* From 5c54bdbf855ac26d154ad8b8f22a71edf03651ba Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Fri, 9 Dec 2016 15:18:01 -0700 Subject: [PATCH 261/920] Change type for DATA_VERSION ioctl to __u64 For consistency and to keep upstream users (scout-utils, etc) from needing to include different type headers, we'll change the type to match the rest of the header. Signed-off-by: Nic Henke --- kmod/src/ioctl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index be5e5d9f..e84db63c 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -83,7 +83,7 @@ struct scoutfs_ioctl_ino_path { #define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ struct scoutfs_ioctl_inodes_since) -#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, u64) +#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, __u64) struct scoutfs_ioctl_release { __u64 offset; From 8b82aa7f18950cfa328538be77236270fab1c47d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 19 Dec 2016 13:38:24 -0800 Subject: [PATCH 262/920] Consistently initialize inode fields Inode info struct initialization spread out over three places: - once for the memory of a slab obect - when reading an existing inode from items - when initializing a newly allocated inode Over time field initializtion got out of sync with these rules. This makes it more clear which fields get initialized where. In the inode info struct we group fields by where there initialized. We order the fields by size and location in the inode struct. Then we make sure that all the initialization sites have everything covered. Doing everything in consistent struct order makes it easier to audit that we haven't missed anything. What lead to this was realizing that we missed initializing the seqcount when reading existing inodes. It should have been initialized in the slab object constructor. The 'staging' boolean has the same problem. Signed-off-by: Zach Brown Reviewed-by: Mark Fasheh --- kmod/src/inode.c | 9 +++++++-- kmod/src/inode.h | 9 ++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 046de8e5..7f61f552 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -49,10 +49,17 @@ struct free_ino_pool { static struct kmem_cache *scoutfs_inode_cachep; +/* + * This is called once before all the allocations and frees of a inode + * object within a slab. It's for inode fields that don't need to be + * initialized for a given instance of an inode. + */ static void scoutfs_inode_ctor(void *obj) { struct scoutfs_inode_info *ci = obj; + seqcount_init(&ci->seqcount); + ci->staging = false; init_rwsem(&ci->xattr_rwsem); inode_init_once(&ci->inode); @@ -501,10 +508,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; - seqcount_init(&ci->seqcount); ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; - ci->staging = false; inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index b282960a..da24e9af 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -4,15 +4,14 @@ #include "key.h" struct scoutfs_inode_info { + /* read or initialized for each inode instance */ u64 ino; - - seqcount_t seqcount; u64 data_version; u64 next_readdir_pos; - /* holder of i_mutex is staging */ - bool staging; - + /* initialized once for slab object */ + seqcount_t seqcount; + bool staging; /* holder of i_mutex is staging */ struct rw_semaphore xattr_rwsem; struct inode inode; From d5a2b0a6dbf17335de17d4433ab63fa50b52c714 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Apr 2017 11:27:30 -0700 Subject: [PATCH 263/920] Move towards compaction messages The compaction code is still directly referencing the super block and calling sync methods as though it was still standalone. This is mostly OK because only the server runs it. But it isn't quite right because the sync methods no longer make the rings persistent as they write the item transaction. The server is in control of that now. Eventually we'll have compaction messages being sent between the mount clients and the server. Let's take a step in that direction by having the compaction work call net methods to get its compaction parameters and finish the compaction. Eventually these would be marshalled through request/process/reply code. But in this first step we know that the compaction code is running on the server so we can forgo all the messaging and just call in to and out of compaction. The net calls just holds the ring consistency locks in the server and call into the manifest to do the work, commiting the changes when its done. This is more careful about segno alloction and freeing. Compaction doesn't call the allocator directly. It gets allocaitons from the messages and returns them if it doesn't use them. We actually now free segnos as they're removed from the manifest. With the server controlling compaction and can tear all the fiddly level count watching code out of the manifest. Item transactions can't care about the level counts and the server always tries compaction after the manifest is updated intead of having the manifest watch the level counts and call compaction. Now that the server owns the rings they should not be torn down as the super is torn down, net does that now. And we need to be more careful to be sure that writes from dirtying and compaction are stable before killing the super. With all this in place moving to shared compaction involves adding the messages and negotiating concurrent compactions in the manifest. Signed-off-by: Zach Brown --- kmod/src/compact.c | 176 ++++++++++++++++++++++---------------------- kmod/src/compact.h | 2 + kmod/src/manifest.c | 80 +++----------------- kmod/src/manifest.h | 1 - kmod/src/net.c | 85 ++++++++++++++++++++- kmod/src/net.h | 4 + kmod/src/super.c | 12 ++- 7 files changed, 197 insertions(+), 163 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 1f3835c1..f27fb1c5 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -24,6 +24,7 @@ #include "manifest.h" #include "counters.h" #include "alloc.h" +#include "net.h" #include "scoutfs_trace.h" /* @@ -71,6 +72,7 @@ struct compact_seg { struct scoutfs_segment *seg; int pos; int saved_pos; + bool part_of_move; }; /* @@ -80,6 +82,10 @@ struct compact_seg { struct compact_cursor { struct list_head csegs; + /* buffer holds allocations and our returning them */ + u64 segnos[2 * (1 + SCOUTFS_MANIFEST_FANOUT)]; + unsigned nr_segnos; + u8 lower_level; u8 last_level; @@ -345,9 +351,9 @@ static int compact_segments(struct super_block *sb, struct compact_seg *cseg; struct compact_seg *upper; struct compact_seg *lower; + unsigned next_segno = 0; u32 key_bytes; u32 nr_items; - u64 segno; int ret; scoutfs_inc_counter(sb, compact_operations); @@ -390,6 +396,10 @@ static int compact_segments(struct super_block *sb, scoutfs_seg_get(cseg->seg); list_add_tail(&cseg->entry, results); + /* don't mess with its segno */ + upper->part_of_move = true; + cseg->part_of_move = true; + curs->upper = NULL; upper = NULL; @@ -445,15 +455,14 @@ static int compact_segments(struct super_block *sb, break; } - ret = scoutfs_alloc_segno(sb, &segno); - if (ret) { - kfree(cseg); - break; - } + cseg->segno = curs->segnos[next_segno]; + curs->segnos[next_segno] = 0; + next_segno++; - ret = scoutfs_seg_alloc(sb, segno, &seg); + ret = scoutfs_seg_alloc(sb, cseg->segno, &seg); if (ret) { - scoutfs_alloc_free(sb, segno); + next_segno--; + curs->segnos[next_segno] = cseg->segno; kfree(cseg); break; } @@ -529,33 +538,55 @@ out: } /* - * Atomically update the manifest. We lock down the manifest so no one - * can use it while we're mucking with it. While the current ring can - * always delete without failure we will probably have a manifest - * storage layer eventually that could return errors on deletion. We - * also also have corrupted something and try to delete an entry that - * doesn't exist. So we use an initial dirtying step to ensure that our - * later deletions succeed. - * - * XXX does locking the manifest prevent commits? I would think so? + * Give the compaction cursor a segno to allocate from. */ -static int update_manifest(struct super_block *sb, struct compact_cursor *curs, - struct list_head *results) +void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno) { + struct compact_cursor *curs = data; + + curs->segnos[curs->nr_segnos++] = segno; +} + +/* + * Commit the result of a compaction based on the state of the cursor. + * The net caller stops the rings from being written while we're making + * changes. We lock the manifest to atomically make our changes. + * + * The erorr handling is sketchy here because calling the manifest from + * here is temporary. We should be sending a message to the server + * instead of calling the allocator and manifest. + */ +int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) +{ + struct compact_cursor *curs = c; + struct list_head *results = r; struct compact_seg *cseg; - struct compact_seg *until; - int ret = 0; - int err; + int ret; + int i; + + /* free unused segnos that were allocated for the compaction */ + for (i = 0; i < curs->nr_segnos; i++) { + if (curs->segnos[i]) { + ret = scoutfs_alloc_free(sb, curs->segnos[i]); + BUG_ON(ret); + } + } scoutfs_manifest_lock(sb); + /* delete input segments, probably freeing their segnos */ list_for_each_entry(cseg, &curs->csegs, entry) { - ret = scoutfs_manifest_dirty(sb, cseg->first, - cseg->seq, cseg->level); - if (ret) - goto out; + if (!cseg->part_of_move) { + ret = scoutfs_alloc_free(sb, cseg->segno); + BUG_ON(ret); + } + + ret = scoutfs_manifest_del(sb, cseg->first, + cseg->seq, cseg->level); + BUG_ON(ret); } + /* add output entries */ list_for_each_entry(cseg, results, entry) { /* XXX moved upper segments won't have read the segment :P */ if (cseg->seg) @@ -565,56 +596,22 @@ static int update_manifest(struct super_block *sb, struct compact_cursor *curs, ret = scoutfs_manifest_add(sb, cseg->first, cseg->last, cseg->segno, cseg->seq, cseg->level); - if (ret) { - until = cseg; - list_for_each_entry(cseg, results, entry) { - if (cseg == until) - break; - err = scoutfs_seg_manifest_del(sb, cseg->seg, - cseg->level); - BUG_ON(err); - } - goto out; - } - } - - list_for_each_entry(cseg, &curs->csegs, entry) { - ret = scoutfs_manifest_del(sb, cseg->first, - cseg->seq, cseg->level); BUG_ON(ret); } -out: scoutfs_manifest_unlock(sb); - return ret; -} - -static int free_result_segnos(struct super_block *sb, - struct list_head *results) -{ - struct compact_seg *cseg; - int ret = 0; - int err; - - list_for_each_entry(cseg, results, entry) { - /* XXX failure here would be an inconsistency */ - err = scoutfs_seg_free_segno(sb, cseg->seg); - if (err && !ret) - ret = err; - } - - return ret; + return 0; } /* * The compaction worker tries to make forward progress with compaction - * every time its kicked. It asks the manifest for segments to compact. + * every time its kicked. It pretends to send a message requesting + * compaction parameters but in reality the net request function there + * is calling directly into the manifest and back into our compaction + * add routines. * - * If it succeeds in doing work then it kicks itself again to see if there's - * more work to do. - * - * XXX worry about forward progress in the case of errors. + * We always try to clean up everything on errors. */ static void scoutfs_compact_func(struct work_struct *work) { @@ -622,6 +619,7 @@ static void scoutfs_compact_func(struct work_struct *work) struct super_block *sb = ci->sb; struct compact_cursor curs = {{NULL,}}; struct scoutfs_bio_completion comp; + struct compact_seg *cseg; LIST_HEAD(results); int ret; int err; @@ -629,33 +627,35 @@ static void scoutfs_compact_func(struct work_struct *work) INIT_LIST_HEAD(&curs.csegs); scoutfs_bio_init_comp(&comp); - ret = scoutfs_manifest_next_compact(sb, (void *)&curs); - if (list_empty(&curs.csegs)) - goto out; + ret = scoutfs_net_get_compaction(sb, (void *)&curs); - ret = compact_segments(sb, &curs, &comp, &results); + /* short circuit no compaction work to do */ + if (ret == 0 && list_empty(&curs.csegs)) + return; - /* always wait for io completion */ - err = scoutfs_bio_wait_comp(sb, &comp); + if (ret == 0 && !list_empty(&curs.csegs)) { + ret = compact_segments(sb, &curs, &comp, &results); + + /* always wait for io completion */ + err = scoutfs_bio_wait_comp(sb, &comp); + if (!ret && err) + ret = err; + } + + /* don't update manifest on error, just free segnos */ + if (ret) { + list_for_each_entry(cseg, &results, entry) { + if (!cseg->part_of_move) + curs.segnos[curs.nr_segnos++] = cseg->segno; + } + free_cseg_list(sb, &curs.csegs); + free_cseg_list(sb, &results); + } + + err = scoutfs_net_finish_compaction(sb, &curs, &results); if (!ret && err) ret = err; - if (ret) - goto out; - ret = update_manifest(sb, &curs, &results); - if (ret == 0) { -#if 0 /* XXX this is busted, fixing soon */ - scoutfs_sync_fs(sb, 0); -#endif - -#if 0 /* XXX where do we do this in shared? */ - scoutfs_trans_wake_holders(sb); -#endif - scoutfs_compact_kick(sb); - } -out: - if (ret) - free_result_segnos(sb, &results); free_cseg_list(sb, &curs.csegs); free_cseg_list(sb, &results); diff --git a/kmod/src/compact.h b/kmod/src/compact.h index d3654fd3..e017dd87 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -9,6 +9,8 @@ int scoutfs_compact_add(struct super_block *sb, void *data, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, u64 segno, u64 seq, u8 level); +void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno); +int scoutfs_compact_commit(struct super_block *sb, void *c, void *r); int scoutfs_compact_setup(struct super_block *sb); void scoutfs_compact_destroy(struct super_block *sb); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index b3c7a258..e0eb9a9e 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -39,7 +39,6 @@ struct manifest { struct rw_semaphore rwsem; - seqcount_t seqcount; struct scoutfs_ring_info ring; u8 nr_levels; @@ -110,57 +109,6 @@ static bool cmp_range_ment(struct scoutfs_key_buf *key, return scoutfs_key_compare_ranges(key, end, &first, &last); } -static u64 get_level_count(struct manifest *mani, - struct scoutfs_super_block *super, u8 level) -{ - unsigned int sc; - u64 count; - - do { - sc = read_seqcount_begin(&mani->seqcount); - count = le64_to_cpu(super->manifest.level_counts[level]); - } while (read_seqcount_retry(&mani->seqcount, sc)); - - return count; -} - -static bool past_limit(struct manifest *mani, u8 level, u64 count) -{ - return count > mani->level_limits[level]; -} - -static bool level_full(struct manifest *mani, - struct scoutfs_super_block *super, u8 level) -{ - return past_limit(mani, level, get_level_count(mani, super, level)); -} - -static void add_level_count(struct super_block *sb, struct manifest *mani, - struct scoutfs_super_block *super, u8 level, - s64 val) -{ - bool was_full; - bool now_full; - u64 count; - - write_seqcount_begin(&mani->seqcount); - - count = le64_to_cpu(super->manifest.level_counts[level]); - was_full = past_limit(mani, level, count); - - count += val; - now_full = past_limit(mani, level, count); - super->manifest.level_counts[level] = cpu_to_le64(count); - - write_seqcount_end(&mani->seqcount); - - if (was_full && !now_full) - scoutfs_trans_wake_holders(sb); - - if (now_full) - scoutfs_compact_kick(sb); -} - /* * Insert a new manifest entry in the ring. The ring allocates a new * node for us and we fill it. @@ -206,7 +154,7 @@ int scoutfs_manifest_add(struct super_block *sb, scoutfs_key_copy(&ment_last, last); mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - add_level_count(sb, mani, super, level, 1); + le64_add_cpu(&super->manifest.level_counts[level], 1); return 0; } @@ -244,7 +192,7 @@ int scoutfs_manifest_add_ment(struct super_block *sb, memcpy(ment, add, bytes); mani->nr_levels = max_t(u8, mani->nr_levels, add->level + 1); - add_level_count(sb, mani, super, add->level, 1); + le64_add_cpu(&super->manifest.level_counts[add->level], 1); return 0; } @@ -292,7 +240,7 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, return -ENOENT; scoutfs_ring_delete(&mani->ring, ment); - add_level_count(sb, mani, super, level, -1ULL); + le64_add_cpu(&super->manifest.level_counts[level], -1ULL); return 0; } @@ -455,7 +403,7 @@ scoutfs_manifest_find_range_entries(struct super_block *sb, *found_bytes = 0; /* at most we get all level 0, one from other levels, and null term */ - nr = get_level_count(mani, super, 0) + mani->nr_levels + 1; + nr = le64_to_cpu(super->manifest.level_counts[0]) + mani->nr_levels + 1; found = kcalloc(nr, sizeof(struct scoutfs_manifest_entry *), GFP_NOFS); if (!found) { @@ -736,15 +684,6 @@ void scoutfs_manifest_write_complete(struct super_block *sb) up_write(&mani->rwsem); } -u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - return get_level_count(mani, super, level); -} - /* * Give the caller the segments that will be involved in the next * compaction. @@ -759,7 +698,7 @@ u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level) * We add all the segments to the compaction caller's data and let it do * its thing. It'll allocate and free segments and update the manifest. * - * Returns 0 or -errno. The caller will see if any segments were added. + * Returns the number of input segments or -errno. * * XXX this will get a lot more clever: * - ensuring concurrent compactions don't overlap @@ -781,12 +720,14 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) struct scoutfs_key_buf over_last; int level; int ret; + int nr = 0; int i; down_write(&mani->rwsem); for (level = mani->nr_levels - 1; level >= 0; level--) { - if (level_full(mani, super, level)) + if (le64_to_cpu(super->manifest.level_counts[level]) > + mani->level_limits[level]) break; } @@ -828,6 +769,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) le64_to_cpu(ment->seq), level); if (ret) goto out; + nr++; /* start with the first overlapping at the next level */ skey.key = &ment_first; @@ -851,6 +793,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) le64_to_cpu(over->seq), level + 1); if (ret) goto out; + nr++; over = scoutfs_ring_next(&mani->ring, over); } @@ -862,7 +805,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) ret = 0; out: up_write(&mani->rwsem); - return ret; + return ret ?: nr; } /* @@ -949,7 +892,6 @@ int scoutfs_manifest_setup(struct super_block *sb) return -ENOMEM; init_rwsem(&mani->rwsem); - seqcount_init(&mani->seqcount); scoutfs_ring_init(&mani->ring, &super->manifest.ring, manifest_ring_compare_key, manifest_ring_compare_data); diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index e0547c26..b65860a2 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -43,7 +43,6 @@ int scoutfs_manifest_add_ment_ref(struct super_block *sb, struct list_head *list, struct scoutfs_manifest_entry *ment); -u64 scoutfs_manifest_level_count(struct super_block *sb, u8 level); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); int scoutfs_manifest_setup(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index ad22ea7c..11e5fe30 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -397,6 +397,8 @@ static struct send_buf *process_record_segment(struct super_block *sb, if (ret == 0) ret = wait_for_commit(&cw); + scoutfs_compact_kick(sb); + sbuf = alloc_sbuf(0); if (!sbuf) { sbuf = ERR_PTR(-ENOMEM); @@ -700,9 +702,9 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) static void destroy_server_state(struct super_block *sb) { + scoutfs_compact_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); - scoutfs_compact_destroy(sb); } /* @@ -1098,6 +1100,87 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return 0; } +/* + * Eventually we're going to have messages that control compaction. + * Each client mount would have long-lived work that sends requests + * which are stuck in processing until there's work to do. They'd get + * their entries, perform the compaction, and send a reply. But we're + * not there yet. + * + * This is a short circuit that's called directly by a work function + * that's only queued on the server. It makes compaction work inside + * the ring update consistency mechanics inside net message processing + * and demonstrates the moving pieces that we'd need to cut up into a + * series of messages and replies. + * + * The compaction work caller cleans up everything on errors. + */ +int scoutfs_net_get_compaction(struct super_block *sb, void *curs) +{ + DECLARE_NET_INFO(sb, nti); + struct commit_waiter cw; + u64 segno; + int ret = 0; + int nr; + int i; + + down_read(&nti->ring_commit_rwsem); + + nr = scoutfs_manifest_next_compact(sb, curs); + if (nr <= 0) { + up_read(&nti->ring_commit_rwsem); + return nr; + } + + for (i = 0; i < nr; i++) { + ret = scoutfs_alloc_segno(sb, &segno); + if (ret < 0) + break; + scoutfs_compact_add_segno(sb, curs, segno); + } + + if (ret == 0) + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + return ret; +} + +/* + * This is a stub for recording the results of a compaction. We just + * call back into compaction to have it call the manifest and allocator + * updates. + * + * In the future we'd encode the manifest and segnos in requests sent to + * the server who'd update the manifest and allocator in request + * processing. + */ +int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, + void *list) +{ + DECLARE_NET_INFO(sb, nti); + struct commit_waiter cw; + int ret; + + down_read(&nti->ring_commit_rwsem); + + ret = scoutfs_compact_commit(sb, curs, list); + + if (ret == 0) + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + scoutfs_compact_kick(sb); + + return ret; +} + struct record_segment_args { struct completion comp; int ret; diff --git a/kmod/src/net.h b/kmod/src/net.h index 973b0530..d48fc2b7 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -14,6 +14,10 @@ int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_net_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); +int scoutfs_net_get_compaction(struct super_block *sb, void *curs); +int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, + void *list); + int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index 5a43a426..48fb27d2 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -253,18 +253,22 @@ static void scoutfs_kill_sb(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - kill_block_super(sb); + /* make sure all dirty work is settled before killing the super */ if (sbi) { + sync_filesystem(sb); + scoutfs_lock_shutdown(sb); scoutfs_net_destroy(sb); + } + + kill_block_super(sb); + + if (sbi) { scoutfs_lock_destroy(sb); - scoutfs_compact_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_data_destroy(sb); scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); - scoutfs_alloc_destroy(sb); - scoutfs_manifest_destroy(sb); scoutfs_seg_destroy(sb); scoutfs_destroy_counters(sb); if (sbi->kset) From 6719733ddc199376fede0e43490e423ecab3e887 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 28 Apr 2017 16:32:24 -0700 Subject: [PATCH 264/920] scoutfs: output full dirent name when tracing The dirent name formatting code accidentally copied the calculation for the length of the name from the xattrs, which are null terminated. The durents are not, their length is just the value length minus the dirent header. Signed-off-by: Zach Brown --- kmod/src/key.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/key.c b/kmod/src/key.c index abcfac9f..584fa2fb 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -156,8 +156,7 @@ int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) case SCOUTFS_DIRENT_KEY: { struct scoutfs_dirent_key *dkey = key->data; - len = (int)key->key_len - offsetof(struct scoutfs_dirent_key, - name[1]); + len = (int)key->key_len - sizeof(struct scoutfs_dirent_key); if (len <= 0) break; From 6afeb978028e1b6efcac93d6af945ee201ba04e7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 13:57:59 -0700 Subject: [PATCH 265/920] scoutfs: reference file data with extent items Our first attempt at storing file data put them in items. This was easy to implement but won't be acceptable in the long term. The cost of the power of LSM indexing is compaction overhead. That's acceptable for fine grained metadata but is totally unacceptable for bulk file data. This switches to storing file data in seperate block allocations which are referenced by extent items. The bulk of the change is the mechanics of working with extents. We have high level callers which add or remove logical extents and then underlying mechanisms that insert, merge, or split the items that the extents are stored in. We have three types of extent items. The primary type maps logical file regions to physical block extents. The next two store free extents per-node so that clients don't create lock and LSM contention as they try and allocate extents. To fill those per-node free extents we add messages that communcate free extents in the form of lists of segment allocations from the server. We don't do any fancy multi-block allocation yet. We only allocate blocks in get_blocks as writes find unmapped blocks. We do use some per-task cursors to cache block allocation positions so that these single block allocations are very likely to merge into larger extents as tasks stream wites. This is just the first chunk of the extent work that's coming. A later patch adds offline flags and fixes up the change nonsense that seemed like a good idea here. The final moving part is that we initiate writeback on all newly allocated extents before we commit the metadata that references the new blocks. We do this with our own dirty inode tracking because the high level vfs methods are unusably slow in some upstream kernels (they walk all inodes, not just dirty inodes.) Signed-off-by: Zach Brown --- kmod/src/data.c | 1278 ++++++++++++++++++++++++++++++--------------- kmod/src/data.h | 1 - kmod/src/format.h | 34 +- kmod/src/inode.c | 158 +++++- kmod/src/inode.h | 4 + kmod/src/net.c | 164 ++++++ kmod/src/net.h | 1 + kmod/src/super.c | 6 + kmod/src/super.h | 6 +- kmod/src/trans.c | 8 +- 10 files changed, 1224 insertions(+), 436 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 61547c2e..8533a3c5 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -14,8 +14,12 @@ #include #include #include -#include -#include +#include +#include +#include +#include +#include +#include #include "format.h" #include "super.h" @@ -27,354 +31,844 @@ #include "scoutfs_trace.h" #include "item.h" #include "ioctl.h" +#include "net.h" /* - * scoutfs stores data in items that can be up to the small 4K block - * size. The page cache address space callbacks work with the item - * cache. Each OS page can be stored in multiple of our smaller fixed - * size items. The code doesn't understand OS pages that are smaller - * than our block size. + * scoutfs uses extent records to reference file data. * - * readpage does a blocking read of the item and then copies its - * contents into the page. Since the segments are huge we sort of get - * limited read-ahead by reading in segments at a time. + * The extent items map logical file regions to device blocks at at 4K + * block granularity. File data isn't overwritten so that overwriting + * doesn't generate extent item locking and modification. * - * Writing is quite a bit more fiddly. We want to pack small files. - * The item cache and transactions want to accurately track the size of - * dirty items to fill the next segment. And we would like to minimize - * cpu copying as much as we can. + * Nodes have their own free extent items stored at their node id to + * avoid lock contention during allocation and freeing. These pools are + * filled and drained with RPCs to the server who allocates blocks in + * segment-sized regions. * - * This simplest first pass creates dirty items as pages are dirtied - * whose values reference the page contents. They're freed after - * they're written to the segment so that we don't have to worry about - * items that reference clean pages. Invalidatepage forgets any items - * if a dirty page is truncated away. + * Block allocation maintains a fixed number of allocation cursors that + * remember the position of tasks within free regions. This is very + * simple and maintains decent extents for simple streaming writes. It + * eventually won't be good enough and we'll spend complexity on + * delalloc but we want to put that off as long as possible. * - * Writeback is built around all the dirty items being written by a - * commit. This can happen naturally in the backgroud. Or writepage - * can initiate it to start by kicking the commit thread. In either - * case our dirty pages are "in writeback" by being put on a list that - * is walked by the end of the commit. Because writes and page dirtying - * are serialized with the commit we know that there can be no dirty - * pages after the commit and we can mark writeback complete on all the - * pages that started writeback before the commit finished. motivate - * having items in the item cache while there are dirty pages. + * There's no unwritten extents. As we dirty file data pages, possibly + * allocating extents for the first time, we track their inodes. Before + * we commit dirty metadata we write out all tracked inodes. This + * ensures that data is persistent before the metadata that references + * it is usable. * - * Data is copied from the dirty page contents into the segment pages - * for writing. This lets us easily pack small files without worrying - * about DMA alignment and avoids the stable page problem of the page - * being modified after the cpu calculates the checksum but before the - * DMA reads to the device. + * Weirdly, the extents are indexed by the *final* logical block and + * blkno of the extent. This lets us search for neighbouring previous + * extents with a _next() call and avoids having to implement item + * reading that iterates backwards through the manifest and segments. + * + * There are two items that track free extents, one indexed by the block + * location of the free extent and one indexed by the size of the free + * region. This means that one allocation can update a great number of + * items throughout the tree as file and both kinds of free extents + * split and merge. The code goes to great lengths to stage these + * updates so that it can always unwind and return errors without + * leaving the items inconsistent. * * XXX * - truncate * - mmap * - better io error propagation - * - async readpages for more concurrent readahead * - forced unmount with dirty data * - direct IO - * - probably stitch page vecs into block struct page fragments for bios - * - maybe cut segment boundaries on aligned data offsets - * - maybe decouple metadata and data segment writes */ struct data_info { - struct llist_head writeback_pages; + struct rw_semaphore alloc_rwsem; + u64 next_large_blkno; + struct rhashtable cursors; + struct list_head cursor_lru; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -/* - * trace_printk() doesn't support %c? - * - * 1 - 1ocked - * a - uptodAte - * d - Dirty - * b - writeBack - * e - Error - */ -#define page_hexflag(page, name, val, shift) \ - (Page##name(page) ? (val << (shift * 4)) : 0) - -#define page_hexflags(page) \ - (page_hexflag(page, Locked, 0x1, 4) | \ - page_hexflag(page, Uptodate, 0xa, 3) | \ - page_hexflag(page, Dirty, 0xd, 2) | \ - page_hexflag(page, Writeback, 0xb, 1) | \ - page_hexflag(page, Error, 0xe, 0)) - -#define PGF "page %p [index %lu flags %x]" -#define PGA(page) \ - (page), (page)->index, page_hexflags(page) \ - -#define BHF "bh %p [blocknr %llu size %zu state %lx]" -#define BHA(bh) \ - (bh), (u64)(bh)->b_blocknr, (bh)->b_size, (bh)->b_state \ - -static void init_data_key(struct scoutfs_key_buf *key, - struct scoutfs_data_key *dkey, u64 ino, u64 block) -{ - dkey->type = SCOUTFS_DATA_KEY; - dkey->ino = cpu_to_be64(ino); - dkey->block = cpu_to_be64(block); - - scoutfs_key_init(key, dkey, sizeof(struct scoutfs_data_key)); -} +/* more than enough for a few tasks per core on moderate hardware */ +#define NR_CURSORS 4096 /* - * Delete the data block items in the given region. + * This is the size of extents that are tracked by a cursor and so end + * up being the largest file item extent length given concurrent + * streaming writes. * - * This is the low level extent item truncate code. Callers manage - * higher order truncation and orphan cleanup. - * - * XXX - * - restore support for releasing data. - * - for final unlink this would be better as a range deletion - * - probably don't want to read items to find them for removal + * XXX We probably want this to be a bit larger to further reduce the + * amount of item churn involved in truncating tremendous files. */ -int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline) -{ - struct scoutfs_data_key last_dkey; - struct scoutfs_data_key dkey; - struct scoutfs_key_buf last; - struct scoutfs_key_buf key; - int ret; +#define LARGE_EXTENT_BLOCKS SCOUTFS_SEGMENT_BLOCKS - trace_printk("iblock %llu len %llu offline %u\n", - iblock, len, offline); +struct cursor_id { + struct task_struct *task; + pid_t pid; +} __packed; /* rhashtable_lookup() always memcmp()s, avoid padding */ - if (WARN_ON_ONCE(iblock + len <= iblock) || - WARN_ON_ONCE(offline)) - return -EINVAL; - - init_data_key(&key, &dkey, ino, iblock); - init_data_key(&last, &last_dkey, ino, iblock + len - 1); - - for (;;) { - ret = scoutfs_item_next(sb, &key, &last, NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - /* XXX would set offline bit items here */ - - ret = scoutfs_item_delete(sb, &key); - if (ret) - break; - } - - return ret; -} - -static inline struct page *page_from_llist_node(struct llist_node *node) -{ - BUILD_BUG_ON(member_sizeof(struct page, private) != - sizeof(struct llist_node)); - - return container_of((void *)node, struct page, private); -} - -static inline struct llist_node *llist_node_from_page(struct page *page) -{ - return (void *)&page->private; -} - -static inline void page_llist_add(struct page *page, struct llist_head *head) -{ - llist_add(llist_node_from_page(page), head); -} +struct task_cursor { + u64 blkno; + u64 blocks; + struct rhash_head hash_head; + struct list_head list_head; + struct cursor_id id; +}; /* - * The transaction has committed so there are no more dirty items. End - * writeback on all the dirty pages that started writeback before the - * commit finished. The commit doesn't start until all holders which - * could dirty are released so there couldn't have been new dirty pages - * and writeback entries while the commit was in flight. + * Both file extent and free extent keys are converted into this native + * form for manipulation. The free extents set blk_off to blkno. */ -void scoutfs_data_end_writeback(struct super_block *sb, int err) +struct native_extent { + u64 blk_off; + u64 blkno; + u64 blocks; +}; + +/* These are stored in a (type==0) terminated array on caller's stacks */ +struct extent_change { + struct native_extent ext; + u64 arg; + unsigned ins:1, + type; +}; + +/* insert file extent + remove both blkno and blocks extents + 0 term */ +#define MAX_CHANGES (3 + 3 + 3 + 1) + +/* XXX avoiding dynamic on-stack array initializers :/ */ +union extent_key_union { + struct scoutfs_file_extent_key file; + struct scoutfs_free_extent_blkno_key blkno; + struct scoutfs_free_extent_blocks_key blocks; +} __packed; +#define MAX_KEY_BYTES sizeof(union extent_key_union) + +static void init_file_extent_key(struct scoutfs_key_buf *key, void *key_bytes, + struct native_extent *ext, u64 arg) { - DECLARE_DATA_INFO(sb, datinf); - struct llist_node *node; - struct page *page; + struct scoutfs_file_extent_key *fkey = key_bytes; - /* XXX haven't thought about errors here */ - BUG_ON(err); + fkey->type = SCOUTFS_FILE_EXTENT_KEY; + fkey->ino = cpu_to_be64(arg); + fkey->last_blk_off = cpu_to_be64(ext->blk_off + ext->blocks - 1); + fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); + fkey->blocks = cpu_to_be64(ext->blocks); - node = llist_del_all(&datinf->writeback_pages); - - while (node) { - page = page_from_llist_node(node); - node = llist_next(node); - - trace_printk("ending writeback "PGF"\n", PGA(page)); - scoutfs_inc_counter(sb, data_end_writeback_page); - - - set_page_private(page, 0); - end_page_writeback(page); - page_cache_release(page); - } + scoutfs_key_init(key, fkey, sizeof(struct scoutfs_file_extent_key)); } -#define for_each_page_block(page, start, loff, block, key, dkey, val) \ - for (start = 0; \ - start < PAGE_CACHE_SIZE && \ - (loff = ((loff_t)page->index << PAGE_CACHE_SHIFT) + start, \ - block = loff >> SCOUTFS_BLOCK_SHIFT, \ - init_data_key(&key, &dkey, \ - scoutfs_ino(page->mapping->host), block), \ - scoutfs_kvec_init(val, page_address(page) + start, \ - SCOUTFS_BLOCK_SIZE), \ - 1); \ - start += SCOUTFS_BLOCK_SIZE) +#define INIT_FREE_EXTENT_KEY(which_type, key, key_bytes, ext, arg, type) \ +do { \ + struct which_type *fkey = key_bytes; \ + \ + fkey->type = type; \ + fkey->node_id = cpu_to_be64(arg); \ + fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); \ + fkey->blocks = cpu_to_be64(ext->blocks); \ + \ + scoutfs_key_init(key, fkey, sizeof(struct which_type)); \ +} while (0) -/* - * Copy the contents of each item that makes up the page into their - * regions of the page, zeroing any page contents not covered by items. - * - * This is the simplest loop that looks up every possible block. We - * could instead have a readpages() that iterates over present items and - * puts them in the pages in the batch. - */ -static int scoutfs_readpage(struct file *file, struct page *page) +static void init_extent_key(struct scoutfs_key_buf *key, void *key_bytes, + struct native_extent *ext, u64 arg, u8 type) { - struct inode *inode = page->mapping->host; - struct super_block *sb = inode->i_sb; - loff_t size = i_size_read(inode); - struct scoutfs_data_key dkey; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); - unsigned start; - loff_t loff; - u64 block; - int ret = 0; - - - trace_printk(PGF"\n", PGA(page)); - scoutfs_inc_counter(sb, data_readpage); - - for_each_page_block(page, start, loff, block, key, dkey, val) { - /* the rest of the page is zero when block is past i_size */ - if (loff >= size) - break; - - /* copy the block item contents into the page */ - ret = scoutfs_item_lookup(sb, &key, val); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - else - break; - } - - /* - * XXX do we need to clamp the item length by i_size? - * truncate should purge the item cache and create - * truncation range items that'd merge away old data - * items, and invalidatepage should shrink any ephemeral - * vecs. Seems like the item length should be accurate? - */ - - /* zero the tail of the block */ - if (ret < SCOUTFS_BLOCK_SIZE) - zero_user(page, start, SCOUTFS_BLOCK_SIZE - ret); - } - - /* zero any remaining tail blocks */ - if (start < PAGE_CACHE_SIZE) - zero_user(page, start, PAGE_CACHE_SIZE - start); - - if (ret == 0) - SetPageUptodate(page); + if (type == SCOUTFS_FILE_EXTENT_KEY) + init_file_extent_key(key, key_bytes, ext, arg); + else if(type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) + INIT_FREE_EXTENT_KEY(scoutfs_free_extent_blkno_key, + key, key_bytes, ext, arg, type); else - SetPageError(page); + INIT_FREE_EXTENT_KEY(scoutfs_free_extent_blocks_key, + key, key_bytes, ext, arg, type); +} - trace_printk("ret %d\n", ret); - unlock_page(page); - return ret; +/* XXX could have some sanity checks */ +static void load_file_extent(struct native_extent *ext, + struct scoutfs_key_buf *key) +{ + struct scoutfs_file_extent_key *fkey = key->data; + + ext->blocks = be64_to_cpu(fkey->blocks); + ext->blk_off = be64_to_cpu(fkey->last_blk_off) - ext->blocks + 1; + ext->blkno = be64_to_cpu(fkey->last_blkno) - ext->blocks + 1; +} + +#define LOAD_FREE_EXTENT(which_type, ext, key) \ +do { \ + struct which_type *fkey = key->data; \ + \ + ext->blkno = be64_to_cpu(fkey->last_blkno) - \ + be64_to_cpu(fkey->blocks) + 1; \ + ext->blk_off = ext->blkno; \ + ext->blocks = be64_to_cpu(fkey->blocks); \ +} while (0) + +static void load_extent(struct native_extent *ext, struct scoutfs_key_buf *key) +{ + struct scoutfs_free_extent_blocks_key *fkey = key->data; + + BUILD_BUG_ON(offsetof(struct scoutfs_file_extent_key, type) != + offsetof(struct scoutfs_free_extent_blkno_key, type) || + offsetof(struct scoutfs_file_extent_key, type) != + offsetof(struct scoutfs_free_extent_blocks_key, type)); + + if (fkey->type == SCOUTFS_FILE_EXTENT_KEY) + load_file_extent(ext, key); + else if (fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) + LOAD_FREE_EXTENT(scoutfs_free_extent_blkno_key, ext, key); + else + LOAD_FREE_EXTENT(scoutfs_free_extent_blocks_key, ext, key); } /* - * Start writeback on a dirty page. We always try to kick off a commit. - * Repeated calls harmlessly bounce off the thread work's pending bit. - * (we could probably test that the writeback pgaes list is empty before - * trying to kick off a commit.) - * - * We add ourselves to a list of pages that the commit will end - * writeback on once its done. If there's no dirty data the commit - * thread will end writeback after not doing anything. + * Merge two extents if they're adjacent. First we arrange them to + * only test their adjoining endpoints, then are careful to not reference + * fields after we've modified them. */ -static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +static int merge_extents(struct native_extent *mod, + struct native_extent *ext) { - struct inode *inode = page->mapping->host; - struct super_block *sb = inode->i_sb; - DECLARE_DATA_INFO(sb, datinf); + struct native_extent *left; + struct native_extent *right; - trace_printk(PGF"\n", PGA(page)); - scoutfs_inc_counter(sb, data_writepage); + if (mod->blk_off < ext->blk_off) { + left = mod; + right = ext; + } else { + left = ext; + right = mod; + } - BUG_ON(PageWriteback(page)); - BUG_ON(page->private != 0); - - ClearPagePrivate(page); /* invalidatepage not needed */ - set_page_writeback(page); - page_cache_get(page); - page_llist_add(page, &datinf->writeback_pages); - unlock_page(page); - scoutfs_sync_fs(sb, 0); + if (left->blk_off + left->blocks == right->blk_off && + left->blkno + left->blocks == right->blkno) { + mod->blk_off = left->blk_off; + mod->blkno = left->blkno; + mod->blocks = left->blocks + right->blocks; + return 1; + } return 0; } /* - * Truncate is invalidating part of the contents of a page. - * - * We can't return errors here so our job is not to create dirty items - * that end up executing the truncate. That's the job of higher level - * callers. Our job is to make sure that we update references to the - * page from existing ephemeral items if they already exist. + * The caller has ensured that the inner extent is entirely within + * the outer extent. Fill out the left and right regions of outter + * that don't overlap with inner. */ -static void scoutfs_invalidatepage(struct page *page, unsigned long offset) +static void trim_extents(struct native_extent *left, + struct native_extent *right, + struct native_extent *outer, + struct native_extent *inner) { - struct inode *inode = page->mapping->host; - struct super_block *sb = inode->i_sb; - struct scoutfs_data_key dkey; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); - unsigned start; - loff_t loff; - u64 block; + left->blk_off = outer->blk_off; + left->blkno = outer->blkno; + left->blocks = inner->blk_off - outer->blk_off; - trace_printk(PGF"\n", PGA(page)); - scoutfs_inc_counter(sb, data_invalidatepage); + right->blk_off = inner->blk_off + inner->blocks; + right->blkno = inner->blkno + inner->blocks; + right->blocks = (outer->blk_off + outer->blocks) - right->blk_off; +} - for_each_page_block(page, start, loff, block, key, dkey, val) { - if (offset) { - /* XXX maybe integrate offset into foreach */ - /* XXX ugh, kvecs are still clumsy :) */ - if (start + SCOUTFS_BLOCK_SIZE > offset) - val[0].iov_len = offset - start; - scoutfs_item_update_ephemeral(sb, &key, val); - } else { - scoutfs_item_forget(sb, &key); - } - } +/* return true if inner is fully contained by outer */ +static bool extents_within(struct native_extent *outer, + struct native_extent *inner) +{ + u64 outer_end = outer->blk_off + outer->blocks - 1; + u64 inner_end = inner->blk_off + inner->blocks - 1; + + return outer->blk_off <= inner_end && outer_end >= inner_end; } /* - * Start modifying a page cache page. - * - * We hold the transaction for write_end's inode updates before - * acquiring the page lock. - * - * We give the writer the current page contents in the relatively rare - * case of writing a partial page inside i_size. write_end will zero - * any region around the write if the page isn't uptodate. + * Add a new entry to the array of changes. The _BLOCKS extent items + * exactly match the _BLKNO items but with different field order for + * searching by size. We keep them in sync by always adding a _BLOCKS + * change for every _BLKNO change. */ +static struct extent_change *append_change(struct extent_change *chg, + bool ins, struct native_extent *ext, + u64 arg, u8 type) +{ + trace_printk("appending ins %d blk_off %llu blkno %llu blocks %llu arg %llu type %u\n", + ins, ext->blk_off, ext->blkno, ext->blocks, + arg, type); + + chg->ext = *ext; + chg->arg = arg; + chg->ins = ins; + chg->type = type; + + if (type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) { + chg++; + *chg = *(chg - 1); + chg->type = SCOUTFS_FREE_EXTENT_BLOCKS_KEY; + } + + return chg + 1; +} + +/* + * Find an adjacent extent in the direction of the delta. If we can + * merge with it then we modify the incoming cur extent. nei is set to + * the neighbour we found. > 0 is returned if we merged, 0 if not, and + * < 0 on error. + */ +static int try_merge(struct super_block *sb, struct native_extent *cur, + s64 delta, struct native_extent *nei, u64 arg, u8 type) +{ + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent ext; + int ret; + + /* short circuit prev search for common first block alloc */ + if (cur->blk_off == 0 && delta < 0) + return 0; + + trace_printk("nei %lld from blk_off %llu blkno %llu blocks %llu\n", + delta, cur->blk_off, cur->blkno, cur->blocks); + + memset(&ext, ~0, sizeof(ext)); + init_extent_key(&last, last_bytes, &ext, arg, type); + + ext.blk_off = cur->blk_off + delta; + ext.blkno = cur->blkno + delta; + ext.blocks = 1; + init_extent_key(&key, key_bytes, &ext, arg, type); + + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + load_extent(nei, &key); + trace_printk("found nei blk_off %llu blkno %llu blocks %llu\n", + nei->blk_off, nei->blkno, nei->blocks); + + ret = merge_extents(cur, nei); +out: + return ret; +} + +/* + * Build the changes needed to insert the given extent. The semantics + * of the extents and callers means that we should not find existing extents + * that overlap the insertion. + */ +static int record_insert_changes(struct super_block *sb, + struct extent_change *chg, + struct native_extent *caller_ins, + u64 arg, u8 type) +{ + struct native_extent ins = *caller_ins; + struct native_extent ext; + int ret; + + trace_printk("inserting arg %llu type %u blk_off %llu blkno %llu blocks %llu\n", + arg, type, ins.blk_off, ins.blkno, ins.blocks); + + /* find the end */ + while (chg->type) + chg++; + + /* find previous that might be adjacent */ + ret = try_merge(sb, &ins, -1, &ext, arg, type); + if (ret < 0) + goto out; + else if (ret > 0) + chg = append_change(chg, false, &ext, arg, type); + + /* find next that might be adjacent */ + ret = try_merge(sb, &ins, 1, &ext, arg, type); + if (ret < 0) + goto out; + else if (ret > 0) + chg = append_change(chg, false, &ext, arg, type); + + /* and insert the new extent, possibly including merged neighbours */ + chg = append_change(chg, true, &ins, arg, type); + ret = 0; +out: + return ret; +} + +/* + * Record the changes needed to remove a portion of an existing extent. + */ +static int record_remove_changes(struct super_block *sb, + struct extent_change *chg, + struct native_extent *rem, u64 arg, + u8 type) +{ + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent left; + struct native_extent right; + struct native_extent outer; + int ret; + + trace_printk("removing arg %llu type %u blk_off %llu blkno %llu blocks %llu\n", + arg, type, rem->blk_off, rem->blkno, rem->blocks); + + /* find the end */ + while (chg->type) + chg++; + + memset(&outer, ~0, sizeof(outer)); + init_extent_key(&last, last_bytes, &outer, arg, type); + + /* find outer existing extent that contains removal extent */ + init_extent_key(&key, key_bytes, rem, arg, type); + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + if (ret) + goto out; + + load_extent(&outer, &key); + + trace_printk("found outer blk_off %llu blkno %llu blocks %llu\n", + outer.blk_off, outer.blkno, outer.blocks); + + if (!extents_within(&outer, rem)) { + ret = -EIO; + goto out; + } + + trim_extents(&left, &right, &outer, rem); + + chg = append_change(chg, false, &outer, arg, type); + + if (left.blocks) { + trace_printk("left trim blk_off %llu blkno %llu blocks %llu\n", + left.blk_off, left.blkno, left.blocks); + chg = append_change(chg, true, &left, arg, type); + } + + if (right.blocks) { + trace_printk("right trim blk_off %llu blkno %llu blocks %llu\n", + right.blk_off, right.blkno, right.blocks); + chg = append_change(chg, true, &right, arg, type); + } + + ret = 0; +out: + if (ret) + trace_printk("ret %d\n", ret); + return ret; +} + +/* + * Any given allocation or free of a file data extent can involve both + * insertion and deletion of both file extent and free extent items. To + * make these atomic we record all the insertions and deletions that are + * performed. We first dirty the deletions, then insert, then delete. + * This lets us always safely unwind on failure. + */ +static int apply_changes(struct super_block *sb, struct extent_change *changes) +{ + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf key; + struct extent_change *chg; + int ret; + int err; + + for (chg = changes; chg->type; chg++) { + if (chg->ins) + continue; + + init_extent_key(&key, key_bytes, &chg->ext, chg->arg, + chg->type); + ret = scoutfs_item_dirty(sb, &key); + if (ret) + goto out; + } + + for (chg = changes; chg->type; chg++) { + if (!chg->ins) + continue; + + init_extent_key(&key, key_bytes, &chg->ext, chg->arg, + chg->type); + ret = scoutfs_item_create(sb, &key, NULL); + if (ret) { + while ((--chg) >= changes) { + if (!chg->ins) + continue; + init_extent_key(&key, key_bytes, &chg->ext, + chg->arg, chg->type); + err = scoutfs_item_delete(sb, &key); + BUG_ON(err); + } + goto out; + } + } + + for (chg = changes; chg->type; chg++) { + if (chg->ins) + continue; + + init_extent_key(&key, key_bytes, &chg->ext, chg->arg, + chg->type); + ret = scoutfs_item_delete(sb, &key); + BUG_ON(ret); + } + +out: + return ret; +} + +int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, + u64 len, bool offline) +{ + BUG(); /* NYI */ +} + +/* + * These cheesy cursors are only meant to encourage nice IO patterns for + * concurrent tasks either streaming large file writes or creating lots + * of small files. It will do very poorly in many other situations. To + * do better we'd need to go further down the road to delalloc and take + * more surrounding context into account. + */ +static struct task_cursor *get_cursor(struct data_info *datinf) +{ + struct task_cursor *curs; + struct cursor_id id = { + .task = current, + .pid = current->pid, + }; + + curs = rhashtable_lookup(&datinf->cursors, &id); + if (!curs) { + curs = list_last_entry(&datinf->cursor_lru, + struct task_cursor, list_head); + trace_printk("resetting curs %p was task %p pid %u\n", + curs, curs->id.task, curs->id.pid); + rhashtable_remove(&datinf->cursors, &curs->hash_head, GFP_NOFS); + curs->id = id; + rhashtable_insert(&datinf->cursors, &curs->hash_head, GFP_NOFS); + curs->blkno = 0; + curs->blocks = 0; + } + + list_move(&curs->list_head, &datinf->cursor_lru); + + return curs; +} + +static int bulk_alloc(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct extent_change changes[MAX_CHANGES]; + struct native_extent ext; + u64 *segnos = NULL; + int ret; + int i; + + segnos = scoutfs_net_bulk_alloc(sb); + if (IS_ERR(segnos)) { + ret = PTR_ERR(segnos); + goto out; + } + + for (i = 0; segnos[i]; i++) { + memset(changes, 0, sizeof(changes)); + + /* merge or set this one */ + if (i > 0 && (segnos[i] == segnos[i - 1] + 1)) { + ext.blocks += SCOUTFS_SEGMENT_BLOCKS; + trace_printk("merged segno [%u] %llu blocks %llu\n", + i, segnos[i], ext.blocks); + } else { + ext.blkno = segnos[i] << SCOUTFS_SEGMENT_BLOCK_SHIFT; + ext.blocks = SCOUTFS_SEGMENT_BLOCKS; + trace_printk("set extent segno [%u] %llu blkno %llu\n", + i, segnos[i], ext.blkno); + } + + /* don't write if we merge with the next one */ + if ((segnos[i] + 1) == segnos[i + 1]) + continue; + + trace_printk("inserting extent [%u] blkno %llu blocks %llu\n", + i, ext.blkno, ext.blocks); + + ext.blk_off = ext.blkno; + ret = record_insert_changes(sb, changes, &ext, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY) ?: + apply_changes(sb, changes); + /* XXX error here leaks segnos */ + if (ret) + break; + } + +out: + if (!IS_ERR_OR_NULL(segnos)) + kfree(segnos); + + return ret; +} + +/* + * Allocate a single block for the logical block offset in the file. + * + * We try to merge single block allocations into large extents by using + * per-task cursors. Each cursor tracks a block region that should be + * searched for free extents. If we don't have a cursor, or we find + * free space outside of our cursor, then we look for the next large + * free extent. + */ +static int allocate_block(struct inode *inode, sector_t iblock, u64 *blkno) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_DATA_INFO(sb, datinf); + struct extent_change changes[MAX_CHANGES] = {{{0,}}}; + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent last_ext; + struct native_extent found; + struct native_extent ext; + struct task_cursor *curs; + bool alloced = false; + u8 type; + int ret; + + memset(&last_ext, ~0, sizeof(last_ext)); + + down_write(&datinf->alloc_rwsem); + + curs = get_cursor(datinf); + + /* start from the cursor or look for the next large extent */ +reset_cursor: + if (curs->blocks) { + ext.blkno = curs->blkno; + ext.blocks = 0; + type = SCOUTFS_FREE_EXTENT_BLKNO_KEY; + } else { + ext.blkno = datinf->next_large_blkno; + ext.blocks = LARGE_EXTENT_BLOCKS; + type = SCOUTFS_FREE_EXTENT_BLOCKS_KEY; + } + +retry: + trace_printk("searching %llu,%llu curs %p task %p pid %u %llu,%llu\n", + ext.blkno, ext.blocks, curs, curs->id.task, curs->id.pid, + curs->blkno, curs->blocks); + + ext.blk_off = ext.blkno; + init_extent_key(&key, key_bytes, &ext, sbi->node_id, type); + init_extent_key(&last, last_bytes, &last_ext, sbi->node_id, type); + + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + if (ret < 0) { + if (ret == -ENOENT) { + /* if the cursor's empty fall back to next large */ + if (ext.blkno && ext.blocks == 0) { + curs->blkno = 0; + curs->blocks = 0; + goto reset_cursor; + } + + /* wrap the search for large extents */ + if (ext.blkno > LARGE_EXTENT_BLOCKS && ext.blocks) { + datinf->next_large_blkno = LARGE_EXTENT_BLOCKS; + ext.blkno = datinf->next_large_blkno; + goto retry; + } + + /* ask the server for more extents */ + if (ext.blocks && !alloced) { + ret = bulk_alloc(sb); + if (ret < 0) + goto out; + alloced = true; + goto retry; + } + + /* finally look for any free block at all */ + if (ext.blocks) { + ext.blkno = 0; + ext.blocks = 0; + type = SCOUTFS_FREE_EXTENT_BLKNO_KEY; + goto retry; + } + + /* after all that return -ENOSPC */ + ret = -ENOSPC; + } + goto out; + } + + load_extent(&found, &key); + trace_printk("found %llu,%llu\n", found.blkno, found.blocks); + + /* look for a new large extent if found is outside cursor */ + if (curs->blocks && + (found.blkno + found.blocks <= curs->blkno || + found.blkno >= curs->blkno + curs->blocks)) { + curs->blkno = 0; + curs->blocks = 0; + goto reset_cursor; + } + + /* + * Set the cursor if: + * - we didn't already have one + * - it's large enough for a large extent with alignment padding + * - the sufficiently large free region is past next large + */ + if (!curs->blocks && + found.blocks >= (2 * LARGE_EXTENT_BLOCKS) && + (found.blkno + found.blocks - (2 * LARGE_EXTENT_BLOCKS) >= + datinf->next_large_blkno)) { + + curs->blkno = ALIGN(max(found.blkno, datinf->next_large_blkno), + LARGE_EXTENT_BLOCKS); + curs->blocks = LARGE_EXTENT_BLOCKS; + found.blkno = curs->blkno; + found.blocks = curs->blocks; + + datinf->next_large_blkno = curs->blkno + LARGE_EXTENT_BLOCKS; + } + + trace_printk("using %llu,%llu curs %llu,%llu\n", + found.blkno, found.blocks, curs->blkno, curs->blocks); + + *blkno = found.blkno; + ext.blk_off = iblock; + ext.blkno = found.blkno; + ext.blocks = 1; + ret = record_insert_changes(sb, changes, &ext, scoutfs_ino(inode), + SCOUTFS_FILE_EXTENT_KEY); + if (ret < 0) + goto out; + + ext.blk_off = ext.blkno; + ret = record_remove_changes(sb, changes, &ext, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY) ?: + apply_changes(sb, changes); + + /* advance cursor if we're using it */ + if (ret == 0 && curs->blocks) { + if (--curs->blocks == 0) + curs->blkno = 0; + else + curs->blkno++; + } + +out: + up_write(&datinf->alloc_rwsem); + return ret; +} + +static int scoutfs_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + struct super_block *sb = inode->i_sb; + DECLARE_DATA_INFO(sb, datinf); + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent ext; + u64 blocks; + u64 blkno; + u64 off; + int ret; + + bh->b_blocknr = 0; + bh->b_size = 0; + blocks = 0; + + ext.blk_off = iblock; + ext.blocks = 1; + ext.blkno = 0; + init_extent_key(&key, key_bytes, &ext, scoutfs_ino(inode), + SCOUTFS_FILE_EXTENT_KEY); + + ext.blk_off = ~0ULL; + ext.blkno = ~0ULL; + ext.blocks = ~0ULL; + init_extent_key(&last, last_bytes, &ext, scoutfs_ino(inode), + SCOUTFS_FILE_EXTENT_KEY); + + /* + * XXX think about how far this next can go, given locking and + * item consistency. + */ + down_read(&datinf->alloc_rwsem); + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + up_read(&datinf->alloc_rwsem); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + else + goto out; + } else { + load_extent(&ext, &key); + trace_printk("found blk_off %llu blkno %llu blocks %llu\n", + ext.blk_off, ext.blkno, ext.blocks); + if (iblock >= ext.blk_off && + iblock < (ext.blk_off + ext.blocks)) { + off = iblock - ext.blk_off; + blkno = ext.blkno + off; + blocks = ext.blocks - off; + } + } + + if (blocks == 0 && create) { + ret = allocate_block(inode, iblock, &blkno); + if (ret) + goto out; + + blocks = 1; + } + + if (blocks) { + map_bh(bh, inode->i_sb, blkno); + bh->b_size = min_t(u64, SIZE_MAX, + blocks << SCOUTFS_BLOCK_SHIFT); + } + +out: + trace_printk("ino %llu iblock %llu create %d ret %d bnr %llu size %zu\n", + scoutfs_ino(inode), (u64)iblock, create, ret, + (u64)bh->b_blocknr, bh->b_size); + + return ret; +} + +static int scoutfs_readpage(struct file *file, struct page *page) +{ + return mpage_readpage(page, scoutfs_get_block); +} + +static int scoutfs_readpages(struct file *file, struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) +{ + return mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); +} + +static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) +{ + return block_write_full_page(page, scoutfs_get_block, wbc); +} + +static int scoutfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, scoutfs_get_block); +} + static int scoutfs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, @@ -382,158 +876,56 @@ static int scoutfs_write_begin(struct file *file, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; - pgoff_t index = pos >> PAGE_SHIFT; - loff_t size = i_size_read(inode); - struct page *page; int ret; - trace_printk("ino %llu pos %llu len %u flags %x\n", - scoutfs_ino(inode), (u64)pos, len, flags); - scoutfs_inc_counter(sb, data_write_begin); + trace_printk("ino %llu pos %llu len %u\n", + scoutfs_ino(inode), (u64)pos, len); ret = scoutfs_hold_trans(sb); if (ret) - return ret; + goto out; /* can't re-enter fs, have trans */ flags |= AOP_FLAG_NOFS; + /* generic write_end updates i_size and calls dirty_inode */ ret = scoutfs_dirty_inode_item(inode); - if (ret) - goto out; - -retry: - page = grab_cache_page_write_begin(mapping, index, flags); - if (!page) { - ret = -ENOMEM; - goto out; - } - - trace_printk(PGF"\n", PGA(page)); - - if (!PageUptodate(page) && (pos < size && len < PAGE_CACHE_SIZE)) { - ClearPageError(page); - ret = scoutfs_readpage(file, page); - if (!ret) { - wait_on_page_locked(page); - if (!PageUptodate(page)) - ret = -EIO; - } - page_cache_release(page); - if (ret) - goto out; - - /* let grab_ lock and check for truncated pages */ - goto retry; - } - - *pagep = page; - ret = 0; -out: + if (ret == 0) + ret = block_write_begin(mapping, pos, len, flags, pagep, + scoutfs_get_block); if (ret) scoutfs_release_trans(sb); - - trace_printk("ret %d\n", ret); +out: return ret; } -/* - * Finish modification of a page cache page. - * - * write_begin has held the transaction and dirtied the inode. We - * create items for each dirty block whose value references the page - * contents that will be written. - * - * We Modify the dirty item and its dependent metadata items while - * holding the transaction so that we never get missing data. - * - * XXX - * - detect no change with copied == 0? - * - only iterate over written blocks, not the whole page? - * - make sure page granular locking and concurrent extending writes works - * - error handling needs work, truncate partial writes on failure? - */ static int scoutfs_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { - struct inode *inode = page->mapping->host; + struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; - struct scoutfs_data_key dkey; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); - loff_t old_size = i_size_read(inode); - bool update_inode = false; - loff_t new_size; - unsigned start; - loff_t loff; - u64 block; int ret; - trace_printk("ino %llu "PGF" pos %llu len %u copied %d\n", - scoutfs_ino(inode), PGA(page), (u64)pos, len, copied); - scoutfs_inc_counter(sb, data_write_end); + trace_printk("ino %llu pgind %lu pos %llu len %u copied %d\n", + scoutfs_ino(inode), page->index, (u64)pos, len, copied); - /* zero any unwritten portions of a new page around the write */ - if (!PageUptodate(page)) { - if (copied != PAGE_CACHE_SIZE) { - start = pos & ~PAGE_CACHE_MASK; - zero_user_segments(page, 0, start, - start + copied, PAGE_CACHE_SIZE); - } - SetPageUptodate(page); - } - - new_size = pos + copied; - - for_each_page_block(page, start, loff, block, key, dkey, val) { - - /* only put data inside i_size in items */ - /* XXX ugh, kvecs are still clumsy :) */ - if (loff + SCOUTFS_BLOCK_SIZE > new_size) - val[0].iov_len = new_size - loff; - - ret = scoutfs_item_create_ephemeral(sb, &key, val); - if (ret) - goto out; - } - - /* update i_size if we extended */ - if (new_size > inode->i_size) { - i_size_write(inode, new_size); - update_inode = true; - } - - if (old_size < pos) - pagecache_isize_extended(inode, old_size, pos); - - if (copied) { + ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); + if (ret > 0) { scoutfs_inode_inc_data_version(inode); - update_inode = true; - } - - if (update_inode) + /* XXX kind of a big hammer, inode life cycle needs work */ scoutfs_update_inode_item(inode); - - flush_dcache_page(page); - set_page_dirty(page); - SetPagePrivate(page); /* call invalidatepage */ - - ret = copied; -out: - unlock_page(page); + scoutfs_inode_queue_writeback(inode); + } scoutfs_release_trans(sb); - - /* XXX error handling needs work */ - WARN_ON_ONCE(ret < 0); return ret; } const struct address_space_operations scoutfs_file_aops = { .readpage = scoutfs_readpage, + .readpages = scoutfs_readpages, .writepage = scoutfs_writepage, - .set_page_dirty = __set_page_dirty_nobuffers, - .invalidatepage = scoutfs_invalidatepage, + .writepages = scoutfs_writepages, .write_begin = scoutfs_write_begin, .write_end = scoutfs_write_end, }; @@ -545,23 +937,75 @@ const struct file_operations scoutfs_file_fops = { .aio_write = generic_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, - .llseek = generic_file_llseek, }; +static int derpy_global_mutex_is_held(void) +{ + return 1; +} + +static struct rhashtable_params cursor_hash_params = { + .key_len = member_sizeof(struct task_cursor, id), + .key_offset = offsetof(struct task_cursor, id), + .head_offset = offsetof(struct task_cursor, hash_head), + .hashfn = arch_fast_hash, + .grow_decision = rht_grow_above_75, + .shrink_decision = rht_shrink_below_30, + + .mutex_is_held = derpy_global_mutex_is_held, +}; + +static void destroy_cursors(struct data_info *datinf) +{ + struct task_cursor *curs; + struct task_cursor *pos; + + list_for_each_entry_safe(curs, pos, &datinf->cursor_lru, list_head) { + list_del_init(&curs->list_head); + kfree(curs); + } + rhashtable_destroy(&datinf->cursors); +} + int scoutfs_data_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct data_info *datinf; - - /* page block iteration doesn't understand multiple pages per block */ - BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SIZE); + struct task_cursor *curs; + int ret; + int i; datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); if (!datinf) return -ENOMEM; - sbi->data_info = datinf; - init_llist_head(&datinf->writeback_pages); + init_rwsem(&datinf->alloc_rwsem); + INIT_LIST_HEAD(&datinf->cursor_lru); + /* always search for large aligned extents */ + datinf->next_large_blkno = LARGE_EXTENT_BLOCKS; + + ret = rhashtable_init(&datinf->cursors, &cursor_hash_params); + if (ret) { + kfree(datinf); + return -ENOMEM; + } + + /* just allocate all of these up front */ + for (i = 0; i < NR_CURSORS; i++) { + curs = kzalloc(sizeof(struct task_cursor), GFP_KERNEL); + if (!curs) { + destroy_cursors(datinf); + kfree(datinf); + return -ENOMEM; + } + + curs->id.pid = i; + rhashtable_insert(&datinf->cursors, &curs->hash_head, + GFP_KERNEL); + list_add(&curs->list_head, &datinf->cursor_lru); + } + + sbi->data_info = datinf; return 0; } @@ -572,7 +1016,7 @@ void scoutfs_data_destroy(struct super_block *sb) struct data_info *datinf = sbi->data_info; if (datinf) { - WARN_ON_ONCE(!llist_empty(&datinf->writeback_pages)); + destroy_cursors(datinf); kfree(datinf); } } diff --git a/kmod/src/data.h b/kmod/src/data.h index 189b2cba..1319d100 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -6,7 +6,6 @@ extern const struct file_operations scoutfs_file_fops; int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline); -void scoutfs_data_end_writeback(struct super_block *sb, int err); int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 5d3c184b..a58d12b6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -156,9 +156,10 @@ struct scoutfs_segment_block { #define SCOUTFS_READDIR_KEY 6 #define SCOUTFS_LINK_BACKREF_KEY 7 #define SCOUTFS_SYMLINK_KEY 8 -#define SCOUTFS_EXTENT_KEY 9 +#define SCOUTFS_FILE_EXTENT_KEY 9 #define SCOUTFS_ORPHAN_KEY 10 -#define SCOUTFS_DATA_KEY 11 +#define SCOUTFS_FREE_EXTENT_BLKNO_KEY 11 +#define SCOUTFS_FREE_EXTENT_BLOCKS_KEY 12 /* not found in the fs */ #define SCOUTFS_MAX_UNUSED_KEY 253 #define SCOUTFS_NET_ADDR_KEY 254 @@ -198,11 +199,28 @@ struct scoutfs_orphan_key { __be64 ino; } __packed; -/* value is data payload bytes */ -struct scoutfs_data_key { +/* no value */ +struct scoutfs_file_extent_key { __u8 type; __be64 ino; - __be64 block; + __be64 last_blk_off; + __be64 last_blkno; + __be64 blocks; +} __packed; + +/* no value */ +struct scoutfs_free_extent_blkno_key { + __u8 type; + __be64 node_id; + __be64 last_blkno; + __be64 blocks; +} __packed; + +struct scoutfs_free_extent_blocks_key { + __u8 type; + __be64 node_id; + __be64 blocks; + __be64 last_blkno; } __packed; /* value is each item's part of the full xattr value for the off/len */ @@ -384,6 +402,11 @@ struct scoutfs_net_manifest_entries { struct scoutfs_manifest_entry ments[0]; } __packed; +struct scoutfs_net_segnos { + __le16 nr; + __le64 segnos[0]; +} __packed; + enum { /* sends and receives a struct scoutfs_timeval */ SCOUTFS_NET_TRADE_TIME = 0, @@ -391,6 +414,7 @@ enum { SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, + SCOUTFS_NET_BULK_ALLOC, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 7f61f552..71a98d33 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -47,6 +47,16 @@ struct free_ino_pool { bool in_flight; }; +struct inode_sb_info { + struct free_ino_pool pool; + + spinlock_t writeback_lock; + struct rb_root writeback_inodes; +}; + +#define DECLARE_INODE_SB_INFO(sb, name) \ + struct inode_sb_info *name = SCOUTFS_SB(sb)->inode_sb_info + static struct kmem_cache *scoutfs_inode_cachep; /* @@ -61,6 +71,7 @@ static void scoutfs_inode_ctor(void *obj) seqcount_init(&ci->seqcount); ci->staging = false; init_rwsem(&ci->xattr_rwsem); + RB_CLEAR_NODE(&ci->writeback_node); inode_init_once(&ci->inode); } @@ -84,8 +95,48 @@ static void scoutfs_i_callback(struct rcu_head *head) kmem_cache_free(scoutfs_inode_cachep, SCOUTFS_I(inode)); } +static void insert_writeback_inode(struct inode_sb_info *inf, + struct scoutfs_inode_info *ins) +{ + struct rb_root *root = &inf->writeback_inodes; + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_inode_info *si; + + while (*node) { + parent = *node; + si = container_of(*node, struct scoutfs_inode_info, + writeback_node); + + if (ins->ino < si->ino) + node = &(*node)->rb_left; + else if (ins->ino > si->ino) + node = &(*node)->rb_right; + else + BUG(); + } + + rb_link_node(&ins->writeback_node, parent, node); + rb_insert_color(&ins->writeback_node, root); +} + +static void remove_writeback_inode(struct inode_sb_info *inf, + struct scoutfs_inode_info *si) +{ + if (!RB_EMPTY_NODE(&si->writeback_node)) { + rb_erase(&si->writeback_node, &inf->writeback_inodes); + RB_CLEAR_NODE(&si->writeback_node); + } +} + void scoutfs_destroy_inode(struct inode *inode) { + DECLARE_INODE_SB_INFO(inode->i_sb, inf); + + spin_lock(&inf->writeback_lock); + remove_writeback_inode(inf, SCOUTFS_I(inode)); + spin_unlock(&inf->writeback_lock); + call_rcu(&inode->i_rcu, scoutfs_i_callback); } @@ -393,7 +444,7 @@ u64 scoutfs_last_ino(struct super_block *sb) */ void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr) { - struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; trace_printk("filling ino %llu nr %llu\n", ino, nr); @@ -427,7 +478,7 @@ static bool pool_in_flight(struct free_ino_pool *pool) */ static int alloc_ino(struct super_block *sb, u64 *ino) { - struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; bool request; int ret; @@ -733,28 +784,121 @@ int scoutfs_orphan_inode(struct inode *inode) return ret; } +/* + * Track an inode that could have dirty pages. Used to kick off writeback + * on all dirty pages during transaction commit without tying ourselves in + * knots trying to call through the high level vfs sync methods. + */ +void scoutfs_inode_queue_writeback(struct inode *inode) +{ + DECLARE_INODE_SB_INFO(inode->i_sb, inf); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + spin_lock(&inf->writeback_lock); + if (RB_EMPTY_NODE(&si->writeback_node)) + insert_writeback_inode(inf, si); + spin_unlock(&inf->writeback_lock); +} + +/* + * Walk our dirty inodes in ino order and either start dirty page + * writeback or wait for writeback to complete. + * + * This is called by transaction commiting so other writers are + * excluded. We're still very careful to iterate over the tree while it + * and the inodes could be changing. + * + * Because writes are excluded we know that there's no remaining dirty + * pages once waiting returns successfully. + * + * XXX not sure what to do about retrying io errors. + */ +int scoutfs_inode_walk_writeback(struct super_block *sb, bool write) +{ + DECLARE_INODE_SB_INFO(sb, inf); + struct scoutfs_inode_info *si; + struct rb_node *node; + struct inode *inode; + struct inode *defer_iput = NULL; + int ret; + + spin_lock(&inf->writeback_lock); + + node = rb_first(&inf->writeback_inodes); + while (node) { + si = container_of(node, struct scoutfs_inode_info, + writeback_node); + node = rb_next(node); + inode = igrab(&si->inode); + if (!inode) + continue; + + spin_unlock(&inf->writeback_lock); + + if (defer_iput) { + iput(defer_iput); + defer_iput = NULL; + } + + if (write) + ret = filemap_fdatawrite(inode->i_mapping); + else + ret = filemap_fdatawait(inode->i_mapping); + trace_printk("ino %llu write %d ret %d\n", + scoutfs_ino(inode), write, ret); + if (ret) { + iput(inode); + goto out; + } + + spin_lock(&inf->writeback_lock); + + if (WARN_ON_ONCE(RB_EMPTY_NODE(&si->writeback_node))) + node = rb_first(&inf->writeback_inodes); + else + node = rb_next(&si->writeback_node); + + if (!write) + remove_writeback_inode(inf, si); + + /* avoid iput->destroy lock deadlock */ + defer_iput = inode; + } + + spin_unlock(&inf->writeback_lock); +out: + if (defer_iput) + iput(defer_iput); + return ret; +} + int scoutfs_inode_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct free_ino_pool *pool; + struct inode_sb_info *inf; - pool = kzalloc(sizeof(struct free_ino_pool), GFP_KERNEL); - if (!pool) + inf = kzalloc(sizeof(struct inode_sb_info), GFP_KERNEL); + if (!inf) return -ENOMEM; + pool = &inf->pool; init_waitqueue_head(&pool->waitq); spin_lock_init(&pool->lock); - sbi->free_ino_pool = pool; + spin_lock_init(&inf->writeback_lock); + inf->writeback_inodes = RB_ROOT; + + sbi->inode_sb_info = inf; return 0; } void scoutfs_inode_destroy(struct super_block *sb) { - struct free_ino_pool *pool = SCOUTFS_SB(sb)->free_ino_pool; + struct inode_sb_info *inf = SCOUTFS_SB(sb)->inode_sb_info; - kfree(pool); + kfree(inf); } void scoutfs_inode_exit(void) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index da24e9af..59da8f7a 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -13,6 +13,7 @@ struct scoutfs_inode_info { seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ struct rw_semaphore xattr_rwsem; + struct rb_node writeback_node; struct inode inode; }; @@ -48,6 +49,9 @@ u64 scoutfs_inode_get_data_version(struct inode *inode); int scoutfs_scan_orphans(struct super_block *sb); +void scoutfs_inode_queue_writeback(struct inode *inode); +int scoutfs_inode_walk_writeback(struct super_block *sb, bool write); + u64 scoutfs_last_ino(struct super_block *sb); void scoutfs_inode_exit(void); diff --git a/kmod/src/net.c b/kmod/src/net.c index 11e5fe30..33d2da3c 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "format.h" #include "net.h" @@ -363,6 +364,61 @@ static struct send_buf *alloc_sbuf(unsigned data_len) return sbuf; } +/* XXX I dunno, totally made up */ +#define BULK_COUNT 32 + +static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, + int req_len) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_net_segnos *ns; + struct commit_waiter cw; + struct send_buf *sbuf; + u64 segno; + int ret; + int i; + + if (req_len != 0) + return ERR_PTR(-EINVAL); + + sbuf = alloc_sbuf(offsetof(struct scoutfs_net_segnos, + segnos[BULK_COUNT])); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + ns = (void *)sbuf->nh->data; + ns->nr = cpu_to_le16(BULK_COUNT); + + down_read(&nti->ring_commit_rwsem); + + for (i = 0; i < BULK_COUNT; i++) { + ret = scoutfs_alloc_segno(sb, &segno); + if (ret) { + while (i-- > 0) + scoutfs_alloc_free(sb, + le64_to_cpu(ns->segnos[i])); + break; + } + + ns->segnos[i] = cpu_to_le64(segno); + } + + + if (ret == 0) + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + if (ret) + sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; + else + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + static struct send_buf *process_record_segment(struct super_block *sb, void *req, int req_len) { @@ -616,6 +672,7 @@ static proc_func_t type_proc_func(u8 type) process_manifest_range_entries, [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, + [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, }; return type < SCOUTFS_NET_UNKNOWN ? funcs[type] : NULL; @@ -1100,6 +1157,113 @@ static int add_send_buf(struct super_block *sb, int type, void *data, return 0; } +struct bulk_alloc_args { + struct completion comp; + u64 *segnos; + int ret; +}; + +static int sort_cmp_u64s(const void *A, const void *B) +{ + const u64 *a = A; + const u64 *b = B; + + return *a < *b ? -1 : *a > *b ? 1 : 0; +} + +static void sort_swap_u64s(void *A, void *B, int size) +{ + u64 *a = A; + u64 *b = B; + + swap(*a, *b); +} + +static int bulk_alloc_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct bulk_alloc_args *args = arg; + struct scoutfs_net_segnos *ns = reply; + u16 nr; + int i; + + if (ret < sizeof(struct scoutfs_net_segnos) || + ret != offsetof(struct scoutfs_net_segnos, + segnos[le16_to_cpu(ns->nr)])) { + ret = -EINVAL; + goto out; + } + + nr = le16_to_cpu(ns->nr); + + args->segnos = kmalloc((nr + 1) * sizeof(args->segnos[0]), GFP_NOFS); + if (args->segnos == NULL) { + ret = -ENOMEM; /* XXX hmm. */ + goto out; + } + + for (i = 0; i < nr; i++) { + args->segnos[i] = le64_to_cpu(ns->segnos[i]); + + /* make sure they're all non-zero */ + if (args->segnos[i] == 0) { + ret = -EINVAL; + goto out; + } + } + + sort(args->segnos, nr, sizeof(args->segnos[0]), + sort_cmp_u64s, sort_swap_u64s); + + /* make sure they're all unique */ + for (i = 1; i < nr; i++) { + if (args->segnos[i] == args->segnos[i - 1]) { + ret = -EINVAL; + goto out; + } + } + + args->segnos[nr] = 0; + ret = 0; +out: + if (ret && args->segnos) { + kfree(args->segnos); + args->segnos = NULL; + } + args->ret = ret; + complete(&args->comp); + return args->ret; +} + +/* + * Returns a 0-terminated allocated array of segnos, the caller is + * responsible for freeing it. + */ +u64 *scoutfs_net_bulk_alloc(struct super_block *sb) +{ + struct bulk_alloc_args args; + int ret; + + args.segnos = NULL; + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_BULK_ALLOC, NULL, 0, + bulk_alloc_reply, &args); + if (ret == 0) { + wait_for_completion(&args.comp); + ret = args.ret; + if (ret == 0 && (args.segnos == NULL || args.segnos[0] == 0)) + ret = -ENOSPC; + } + + if (ret) { + kfree(args.segnos); + args.segnos = ERR_PTR(ret); + } + + return args.segnos; +} + /* * Eventually we're going to have messages that control compaction. * Each client mount would have long-lived work that sends requests diff --git a/kmod/src/net.h b/kmod/src/net.h index d48fc2b7..125bf327 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -13,6 +13,7 @@ int scoutfs_net_manifest_range_entries(struct super_block *sb, int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_net_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); +u64 *scoutfs_net_bulk_alloc(struct super_block *sb); int scoutfs_net_get_compaction(struct super_block *sb, void *curs); int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, diff --git a/kmod/src/super.c b/kmod/src/super.c index 48fb27d2..0fe2ed52 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -204,6 +204,12 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; + /* + * XXX this is random today for initial testing, but we'll want + * it to be assigned by the server. + */ + get_random_bytes_arch(&sbi->node_id, sizeof(sbi->node_id)); + spin_lock_init(&sbi->next_ino_lock); atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); diff --git a/kmod/src/super.h b/kmod/src/super.h index 184c92b8..5b6d5903 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -14,11 +14,13 @@ struct compact_info; struct data_info; struct lock_info; struct net_info; -struct free_ino_pool; +struct inode_sb_info; struct scoutfs_sb_info { struct super_block *sb; + u64 node_id; + struct scoutfs_super_block super; spinlock_t next_ino_lock; @@ -29,7 +31,7 @@ struct scoutfs_sb_info { struct seg_alloc *seg_alloc; struct compact_info *compact_info; struct data_info *data_info; - struct free_ino_pool *free_ino_pool; + struct inode_sb_info *inode_sb_info; atomic_t trans_holds; wait_queue_head_t trans_hold_wq; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e6247bc0..11941c7e 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -26,6 +26,7 @@ #include "seg.h" #include "counters.h" #include "net.h" +#include "inode.h" #include "scoutfs_trace.h" /* @@ -97,10 +98,12 @@ void scoutfs_trans_write_func(struct work_struct *work) * about leaking segnos nor duplicate manifest entries * on crashes between us and the server. */ - ret = scoutfs_net_alloc_segno(sb, &segno) ?: + ret = scoutfs_inode_walk_writeback(sb, true) ?: + scoutfs_net_alloc_segno(sb, &segno) ?: scoutfs_seg_alloc(sb, segno, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: scoutfs_seg_submit_write(sb, seg, &comp) ?: + scoutfs_inode_walk_writeback(sb, false) ?: scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_net_record_segment(sb, seg, 0); if (ret) @@ -112,9 +115,6 @@ out: /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); - /* must be done before waking waiting trans holders who might dirty */ - scoutfs_data_end_writeback(sb, ret); - spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; sbi->trans_write_ret = ret; From 723e0368f8c63a09a2fe2d440f74b02538df8c93 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 14:30:02 -0700 Subject: [PATCH 266/920] scoutfs: add a trace point for item insertion Signed-off-by: Zach Brown --- kmod/src/item.c | 2 ++ kmod/src/scoutfs_trace.h | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 7dfcc71c..071942e8 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -403,6 +403,8 @@ restart: } } + trace_scoutfs_item_insertion(sb, ins->key, ins->val); + rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index ae1c4c16..f77cce2c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -217,6 +217,19 @@ TRACE_EVENT(scoutfs_item_lookup, TP_printk("key %s", __get_str(key)) ); +TRACE_EVENT(scoutfs_item_insertion, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key, + struct kvec *val), + TP_ARGS(sb, key, val), + TP_STRUCT__entry( + __dynamic_array(char, key, scoutfs_key_str(NULL, key)) + ), + TP_fast_assign( + scoutfs_key_str(__get_dynamic_array(key), key); + ), + TP_printk("key %s", __get_str(key)) +); + TRACE_EVENT(scoutfs_item_insert_batch, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), From 66dd35b9a54301d63860d45729680e846fbbef07 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 14:30:20 -0700 Subject: [PATCH 267/920] scoutfs: fix ring next/prev The ring node rb walker was returning an exact match for the search key instead of the last node that was traversed. This stopped callers from then iterating from the traversed node to find the next or previous node. Signed-off-by: Zach Brown --- kmod/src/ring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 6df1c04c..7eb61b0a 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -178,7 +178,7 @@ static struct ring_node *ring_rb_walk(struct scoutfs_ring_info *ring, struct rb_node **node = &ring->rb_root.rb_node; struct rb_node *parent = NULL; struct ring_node *found = NULL; - struct ring_node *rnode; + struct ring_node *rnode = NULL; /* only provide one or the other */ BUG_ON(!!key == !!data); @@ -213,9 +213,10 @@ static struct ring_node *ring_rb_walk(struct scoutfs_ring_info *ring, rb_insert_color(&ins->rb_node, &ring->rb_root); } found = ins; + *cmp = 0; } - return found; + return rnode; } static struct ring_node *ring_rb_entry(struct rb_node *node) From c67892340104751e1c33aec88dca98cd2bb52861 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 14:49:14 -0700 Subject: [PATCH 268/920] scoutfs: don't try to sync on mount errors kill_sb tries to sync before calling kill_block_super. It shouldn't do this on mount errors that wouldn't have initialized the higher level systems needed for syncing. Signed-off-by: Zach Brown --- kmod/src/net.c | 2 ++ kmod/src/super.c | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 33d2da3c..14f1d4f3 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1951,6 +1951,7 @@ int scoutfs_net_setup(struct super_block *sb) */ void scoutfs_net_destroy(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_NET_INFO(sb, nti); struct sock_info *sinf; struct sock_info *pos; @@ -1987,5 +1988,6 @@ void scoutfs_net_destroy(struct super_block *sb) /* and free all resources */ free_sbuf_list(sb, &nti->to_send, -ESHUTDOWN); free_nti(nti); + sbi->net_info = NULL; } } diff --git a/kmod/src/super.c b/kmod/src/super.c index 0fe2ed52..bcf831fc 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -259,8 +259,12 @@ static void scoutfs_kill_sb(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - /* make sure all dirty work is settled before killing the super */ - if (sbi) { + /* + * If we had successfully mounted then make sure dirty data + * writeback and compaction is done before we kill the block + * super and start tearing everything down. + */ + if (sb->s_root) { sync_filesystem(sb); scoutfs_lock_shutdown(sb); @@ -271,6 +275,7 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sbi) { scoutfs_lock_destroy(sb); + scoutfs_net_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_data_destroy(sb); scoutfs_inode_destroy(sb); From 81866620a905bce245d4545208d6537ca41e105c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 14:57:46 -0700 Subject: [PATCH 269/920] scoutfs: allow xattrs with 0 length values xattrs can have 0 lenth values so fix the item iterator to emit a single item in the case where the size is 0. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index f2fd9289..a427763d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -118,7 +118,7 @@ static void set_xattr_key_part(struct scoutfs_key_buf *key, u8 part) */ #define for_each_xattr_item(key, val, vh, buffer, size, part, off, bytes) \ for (part = 0, off = 0; \ - off < size && \ + ((off < size) || (part == 0 && size == 0)) && \ (bytes = min_t(size_t, SCOUTFS_XATTR_PART_SIZE, size - off), \ set_xattr_key_part(key, part), \ (vh)->part_len = cpu_to_le16(bytes), \ From a262a158ce152567aba664668ce3dfab5fe63e84 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 May 2017 08:55:57 -0700 Subject: [PATCH 270/920] scoutfs: fix single block release The offset comparison in release that was meant to catch wrapping was inverted and accidentally prevented releasing a single block. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 82375047..39f10c2c 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -268,7 +268,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) start = round_up(args.offset, SCOUTFS_BLOCK_SIZE); end_inc = round_down(args.offset + args.count, SCOUTFS_BLOCK_SIZE) - 1; - if (end_inc > start) + if (end_inc < start) return 0; iblock = start >> SCOUTFS_BLOCK_SHIFT; From e34f8db4a944850a0fed0485b3c974ab7f2bc512 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 May 2017 08:57:08 -0700 Subject: [PATCH 271/920] scoutfs: add release argument and result tracing Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 39f10c2c..48f814d0 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -261,6 +261,9 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; + trace_printk("offset %llu count %llu vers %llu\n", + args.offset, args.count, args.data_version); + if (args.count == 0) return 0; if ((args.offset + args.count) < args.offset) @@ -312,6 +315,7 @@ out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); + trace_printk("ret %d\n", ret); return ret; } From 4084d3d9dc04565fdb58d8f86ce4b7cd121d960f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 May 2017 15:19:02 -0700 Subject: [PATCH 272/920] scoutfs: add offline flag, releasing, and fiemap Now that we have basic file extents we can add a flag to extents to track offline extents. We have to initialize and test the flags as we work with extents. Truncation can be told to leave removed extents around with no block mapping and the offline bit set. Only staging with the correct data version can write to the offline regions. Demand staging isn't implemented yet. Reads from offline extents are treated like sparse regions. Truncation is a straight forward iteration over the portions of existing extents which overlap with the truncated blocks. Writing to offline extents has to first remove the existing offline extent before then adding the new allocated extents. The 'changes' mechanism relied on being able to search the current items to find the changes that should be made before making any changes. This doesn't work for finding merge candidates for the new allocated insertion because the old offline extent change won't have been applied yet. We replace the change mechanism with straight forward item modification and unwinding. The generic block fiemap can't communicate offline extents and iterates over blocks instead of extents. We add our fiemap that iterates over extents and sets the 'UNKNOWN' flag on offline extents. Signed-off-by: Zach Brown --- kmod/src/data.c | 691 +++++++++++++++++++++++++++++++--------------- kmod/src/data.h | 2 + kmod/src/format.h | 3 + kmod/src/inode.c | 1 + 4 files changed, 473 insertions(+), 224 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 8533a3c5..98dc2143 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -19,7 +19,6 @@ #include #include #include -#include #include "format.h" #include "super.h" @@ -33,16 +32,19 @@ #include "ioctl.h" #include "net.h" +#define EXTF "[off %llu bno %llu bks %llu fl %x]" +#define EXTA(ne) (ne)->blk_off, (ne)->blkno, (ne)->blocks, (ne)->flags + /* - * scoutfs uses extent records to reference file data. + * scoutfs uses extent items to reference file data. * - * The extent items map logical file regions to device blocks at at 4K + * The extent items map logical file regions to device blocks at 4K * block granularity. File data isn't overwritten so that overwriting * doesn't generate extent item locking and modification. * * Nodes have their own free extent items stored at their node id to * avoid lock contention during allocation and freeing. These pools are - * filled and drained with RPCs to the server who allocates blocks in + * filled and drained with messages to the server who allocates * segment-sized regions. * * Block allocation maintains a fixed number of allocation cursors that @@ -55,7 +57,7 @@ * allocating extents for the first time, we track their inodes. Before * we commit dirty metadata we write out all tracked inodes. This * ensures that data is persistent before the metadata that references - * it is usable. + * it is visible. * * Weirdly, the extents are indexed by the *final* logical block and * blkno of the extent. This lets us search for neighbouring previous @@ -64,11 +66,10 @@ * * There are two items that track free extents, one indexed by the block * location of the free extent and one indexed by the size of the free - * region. This means that one allocation can update a great number of - * items throughout the tree as file and both kinds of free extents - * split and merge. The code goes to great lengths to stage these - * updates so that it can always unwind and return errors without - * leaving the items inconsistent. + * extent. This means that one allocation can update a great number of + * items throughout the tree as items are created and deleted as extents + * are split and merged. This can introduce inconsistent failure + * states. We'll some day address that with preallocation and pinning. * * XXX * - truncate @@ -122,20 +123,10 @@ struct native_extent { u64 blk_off; u64 blkno; u64 blocks; + u8 flags; }; -/* These are stored in a (type==0) terminated array on caller's stacks */ -struct extent_change { - struct native_extent ext; - u64 arg; - unsigned ins:1, - type; -}; - -/* insert file extent + remove both blkno and blocks extents + 0 term */ -#define MAX_CHANGES (3 + 3 + 3 + 1) - -/* XXX avoiding dynamic on-stack array initializers :/ */ +/* avoiding dynamic on-stack array initializers :/ */ union extent_key_union { struct scoutfs_file_extent_key file; struct scoutfs_free_extent_blkno_key blkno; @@ -153,6 +144,7 @@ static void init_file_extent_key(struct scoutfs_key_buf *key, void *key_bytes, fkey->last_blk_off = cpu_to_be64(ext->blk_off + ext->blocks - 1); fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); fkey->blocks = cpu_to_be64(ext->blocks); + fkey->flags = ext->flags; scoutfs_key_init(key, fkey, sizeof(struct scoutfs_file_extent_key)); } @@ -191,6 +183,7 @@ static void load_file_extent(struct native_extent *ext, ext->blocks = be64_to_cpu(fkey->blocks); ext->blk_off = be64_to_cpu(fkey->last_blk_off) - ext->blocks + 1; ext->blkno = be64_to_cpu(fkey->last_blkno) - ext->blocks + 1; + ext->flags = fkey->flags; } #define LOAD_FREE_EXTENT(which_type, ext, key) \ @@ -201,6 +194,7 @@ do { \ be64_to_cpu(fkey->blocks) + 1; \ ext->blk_off = ext->blkno; \ ext->blocks = be64_to_cpu(fkey->blocks); \ + ext->flags = 0; \ } while (0) static void load_extent(struct native_extent *ext, struct scoutfs_key_buf *key) @@ -240,7 +234,8 @@ static int merge_extents(struct native_extent *mod, } if (left->blk_off + left->blocks == right->blk_off && - left->blkno + left->blocks == right->blkno) { + left->blkno + left->blocks == right->blkno && + left->flags == right->flags) { mod->blk_off = left->blk_off; mod->blkno = left->blkno; mod->blocks = left->blocks + right->blocks; @@ -263,10 +258,12 @@ static void trim_extents(struct native_extent *left, left->blk_off = outer->blk_off; left->blkno = outer->blkno; left->blocks = inner->blk_off - outer->blk_off; + left->flags = outer->flags; right->blk_off = inner->blk_off + inner->blocks; right->blkno = inner->blkno + inner->blocks; right->blocks = (outer->blk_off + outer->blocks) - right->blk_off; + right->flags = outer->flags; } /* return true if inner is fully contained by outer */ @@ -279,39 +276,11 @@ static bool extents_within(struct native_extent *outer, return outer->blk_off <= inner_end && outer_end >= inner_end; } -/* - * Add a new entry to the array of changes. The _BLOCKS extent items - * exactly match the _BLKNO items but with different field order for - * searching by size. We keep them in sync by always adding a _BLOCKS - * change for every _BLKNO change. - */ -static struct extent_change *append_change(struct extent_change *chg, - bool ins, struct native_extent *ext, - u64 arg, u8 type) -{ - trace_printk("appending ins %d blk_off %llu blkno %llu blocks %llu arg %llu type %u\n", - ins, ext->blk_off, ext->blkno, ext->blocks, - arg, type); - - chg->ext = *ext; - chg->arg = arg; - chg->ins = ins; - chg->type = type; - - if (type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) { - chg++; - *chg = *(chg - 1); - chg->type = SCOUTFS_FREE_EXTENT_BLOCKS_KEY; - } - - return chg + 1; -} - /* * Find an adjacent extent in the direction of the delta. If we can * merge with it then we modify the incoming cur extent. nei is set to - * the neighbour we found. > 0 is returned if we merged, 0 if not, and - * < 0 on error. + * the neighbour we found. If we didn't merge then nei's blocks is set + * to 0. */ static int try_merge(struct super_block *sb, struct native_extent *cur, s64 delta, struct native_extent *nei, u64 arg, u8 type) @@ -323,19 +292,19 @@ static int try_merge(struct super_block *sb, struct native_extent *cur, struct native_extent ext; int ret; + memset(nei, 0, sizeof(struct native_extent)); + /* short circuit prev search for common first block alloc */ if (cur->blk_off == 0 && delta < 0) return 0; - trace_printk("nei %lld from blk_off %llu blkno %llu blocks %llu\n", - delta, cur->blk_off, cur->blkno, cur->blocks); - memset(&ext, ~0, sizeof(ext)); init_extent_key(&last, last_bytes, &ext, arg, type); ext.blk_off = cur->blk_off + delta; ext.blkno = cur->blkno + delta; ext.blocks = 1; + ext.flags = 0; init_extent_key(&key, key_bytes, &ext, arg, type); ret = scoutfs_item_next_same(sb, &key, &last, NULL); @@ -345,80 +314,139 @@ static int try_merge(struct super_block *sb, struct native_extent *cur, goto out; } - load_extent(nei, &key); - trace_printk("found nei blk_off %llu blkno %llu blocks %llu\n", - nei->blk_off, nei->blkno, nei->blocks); + load_extent(&ext, &key); + trace_printk("merge nei "EXTF"\n", EXTA(&ext)); - ret = merge_extents(cur, nei); -out: - return ret; -} - -/* - * Build the changes needed to insert the given extent. The semantics - * of the extents and callers means that we should not find existing extents - * that overlap the insertion. - */ -static int record_insert_changes(struct super_block *sb, - struct extent_change *chg, - struct native_extent *caller_ins, - u64 arg, u8 type) -{ - struct native_extent ins = *caller_ins; - struct native_extent ext; - int ret; - - trace_printk("inserting arg %llu type %u blk_off %llu blkno %llu blocks %llu\n", - arg, type, ins.blk_off, ins.blkno, ins.blocks); - - /* find the end */ - while (chg->type) - chg++; - - /* find previous that might be adjacent */ - ret = try_merge(sb, &ins, -1, &ext, arg, type); - if (ret < 0) - goto out; - else if (ret > 0) - chg = append_change(chg, false, &ext, arg, type); - - /* find next that might be adjacent */ - ret = try_merge(sb, &ins, 1, &ext, arg, type); - if (ret < 0) - goto out; - else if (ret > 0) - chg = append_change(chg, false, &ext, arg, type); - - /* and insert the new extent, possibly including merged neighbours */ - chg = append_change(chg, true, &ins, arg, type); + if (merge_extents(cur, &ext)) + *nei = ext; ret = 0; out: return ret; } /* - * Record the changes needed to remove a portion of an existing extent. + * We have two item types for indexing free extents by either the + * location of the extent or the size of the extent. When we create + * logical extents we might be finding neighbouring extents that could + * be merged. We can only search for neighbours in the location items. + * Once we find them we mirror the item modifications for both the + * location and size items. + * + * If this returns an error then nothing will have changed. */ -static int record_remove_changes(struct super_block *sb, - struct extent_change *chg, - struct native_extent *rem, u64 arg, - u8 type) +static int modify_items(struct super_block *sb, struct native_extent *ext, + u64 arg, u8 type, bool create) +{ + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf key; + int ret; + int err; + + trace_printk("mod cre %u "EXTF"\n", create, EXTA(ext)); + + BUG_ON(type != SCOUTFS_FILE_EXTENT_KEY && + type != SCOUTFS_FREE_EXTENT_BLKNO_KEY); + + init_extent_key(&key, key_bytes, ext, arg, type); + ret = create ? scoutfs_item_create(sb, &key, NULL) : + scoutfs_item_delete(sb, &key); + + if (ret == 0 && type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) { + init_extent_key(&key, key_bytes, ext, arg, + SCOUTFS_FREE_EXTENT_BLOCKS_KEY); + ret = create ? scoutfs_item_create(sb, &key, NULL) : + scoutfs_item_delete(sb, &key); + if (ret) { + init_extent_key(&key, key_bytes, ext, arg, type); + err = create ? scoutfs_item_delete(sb, &key) : + scoutfs_item_create(sb, &key, NULL); + BUG_ON(err); + } + } + + return ret; +} + +/* + * Insert a new extent. We see if it can be merged with adjacent + * existing extents. If this returns an error then the existing extents + * will not have changed. + */ +static int insert_extent(struct super_block *sb, + struct native_extent *caller_ins, + u64 arg, u8 type) +{ + struct native_extent left; + struct native_extent right; + struct native_extent ins = *caller_ins; + bool del_ins = false; + bool ins_left = false; + int err; + int ret; + + trace_printk("inserting "EXTF"\n", EXTA(caller_ins)); + + /* find previous that might be adjacent */ + ret = try_merge(sb, &ins, -1, &left, arg, type); + try_merge(sb, &ins, 1, &right, arg, type); + if (ret < 0) + goto out; + + trace_printk("merge left "EXTF"\n", EXTA(&left)); + trace_printk("merge right "EXTF"\n", EXTA(&right)); + + ret = modify_items(sb, &ins, arg, type, true); + if (ret) + goto out; + del_ins = true; + + if (left.blocks) { + ret = modify_items(sb, &left, arg, type, false); + if (ret) + goto undo; + ins_left = true; + } + + if (right.blocks) + ret = modify_items(sb, &right, arg, type, false); + +undo: + if (ret) { + if (ins_left) { + err = modify_items(sb, &left, arg, type, true); + BUG_ON(err); + } + if (del_ins) { + err = modify_items(sb, &ins, arg, type, false); + BUG_ON(err); + } + } + +out: + return ret; +} + +/* + * Remove a portion of an existing extent. The removal might leave + * behind non-overlapping edges of the existing extent. If this returns + * an error then the existing extent will not have changed. + */ +static int remove_extent(struct super_block *sb, + struct native_extent *rem, u64 arg, u8 type) { u8 last_bytes[MAX_KEY_BYTES]; u8 key_bytes[MAX_KEY_BYTES]; struct scoutfs_key_buf last; struct scoutfs_key_buf key; - struct native_extent left; - struct native_extent right; + struct native_extent left = {0,}; + struct native_extent right = {0,}; struct native_extent outer; + bool rem_left = false; + bool rem_right = false; + int err = 0; int ret; - trace_printk("removing arg %llu type %u blk_off %llu blkno %llu blocks %llu\n", - arg, type, rem->blk_off, rem->blkno, rem->blocks); - - /* find the end */ - while (chg->type) - chg++; + trace_printk("removing "EXTF"\n", EXTA(rem)); memset(&outer, ~0, sizeof(outer)); init_extent_key(&last, last_bytes, &outer, arg, type); @@ -431,101 +459,181 @@ static int record_remove_changes(struct super_block *sb, load_extent(&outer, &key); - trace_printk("found outer blk_off %llu blkno %llu blocks %llu\n", - outer.blk_off, outer.blkno, outer.blocks); + trace_printk("outer "EXTF"\n", EXTA(&outer)); - if (!extents_within(&outer, rem)) { + if (!extents_within(&outer, rem) || outer.flags != rem->flags) { ret = -EIO; goto out; } trim_extents(&left, &right, &outer, rem); - chg = append_change(chg, false, &outer, arg, type); + trace_printk("trim left "EXTF"\n", EXTA(&left)); + trace_printk("trim right "EXTF"\n", EXTA(&right)); if (left.blocks) { - trace_printk("left trim blk_off %llu blkno %llu blocks %llu\n", - left.blk_off, left.blkno, left.blocks); - chg = append_change(chg, true, &left, arg, type); + ret = modify_items(sb, &left, arg, type, true); + if (ret) + goto out; + rem_left = true; } if (right.blocks) { - trace_printk("right trim blk_off %llu blkno %llu blocks %llu\n", - right.blk_off, right.blkno, right.blocks); - chg = append_change(chg, true, &right, arg, type); + ret = modify_items(sb, &right, arg, type, true); + if (ret) + goto out; + rem_right = true; } - ret = 0; + ret = modify_items(sb, &outer, arg, type, false); + out: - if (ret) - trace_printk("ret %d\n", ret); + if (ret) { + if (rem_right) { + err = modify_items(sb, &right, arg, type, false); + BUG_ON(err); + } + if (rem_left) { + err = modify_items(sb, &left, arg, type, false); + BUG_ON(err); + } + } + + trace_printk("ret %d\n", ret); return ret; } /* - * Any given allocation or free of a file data extent can involve both - * insertion and deletion of both file extent and free extent items. To - * make these atomic we record all the insertions and deletions that are - * performed. We first dirty the deletions, then insert, then delete. - * This lets us always safely unwind on failure. + * Free extents whose blocks fall inside the specified logical block + * range. + * + * If 'offline' is given then blocks are freed but the extent items are + * left behind and their _OFFLINE flag is set. + * + * This is the low level extent item manipulation code. Callers manage + * higher order locking and transactional consistency. */ -static int apply_changes(struct super_block *sb, struct extent_change *changes) -{ - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf key; - struct extent_change *chg; - int ret; - int err; - - for (chg = changes; chg->type; chg++) { - if (chg->ins) - continue; - - init_extent_key(&key, key_bytes, &chg->ext, chg->arg, - chg->type); - ret = scoutfs_item_dirty(sb, &key); - if (ret) - goto out; - } - - for (chg = changes; chg->type; chg++) { - if (!chg->ins) - continue; - - init_extent_key(&key, key_bytes, &chg->ext, chg->arg, - chg->type); - ret = scoutfs_item_create(sb, &key, NULL); - if (ret) { - while ((--chg) >= changes) { - if (!chg->ins) - continue; - init_extent_key(&key, key_bytes, &chg->ext, - chg->arg, chg->type); - err = scoutfs_item_delete(sb, &key); - BUG_ON(err); - } - goto out; - } - } - - for (chg = changes; chg->type; chg++) { - if (chg->ins) - continue; - - init_extent_key(&key, key_bytes, &chg->ext, chg->arg, - chg->type); - ret = scoutfs_item_delete(sb, &key); - BUG_ON(ret); - } - -out: - return ret; -} - int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline) { - BUG(); /* NYI */ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent found; + struct native_extent rng; + struct native_extent ext; + struct native_extent ofl; + struct native_extent fr; + bool rem_fr = false; + bool ins_ext = false; + int ret = 0; + int err; + + trace_printk("iblock %llu len %llu offline %u\n", + iblock, len, offline); + + memset(&ext, ~0, sizeof(ext)); + init_extent_key(&last, last_bytes, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); + + rng.blk_off = iblock; + rng.blocks = len; + rng.blkno = 0; + rng.flags = 0; + + while (rng.blocks) { + /* find the next extent that could include our first block */ + init_extent_key(&key, key_bytes, &rng, ino, + SCOUTFS_FILE_EXTENT_KEY); + + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + load_extent(&found, &key); + trace_printk("found "EXTF"\n", EXTA(&found)); + + /* XXX corruption: offline and allocation are exclusive */ + if (!!found.blkno == + !!(found.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) { + ret = -EIO; + break; + } + + /* we're done if the found extent is past us */ + if (found.blk_off >= rng.blk_off + rng.blocks) { + ret = 0; + break; + } + + /* find the intersection */ + ext.blk_off = max(rng.blk_off, found.blk_off); + ext.blocks = min(rng.blk_off + rng.blocks, + found.blk_off + found.blocks) - ext.blk_off; + ext.blkno = found.blkno + (ext.blk_off - found.blk_off); + ext.flags = found.flags; + + /* next search will be past the extent we truncate */ + rng.blk_off = ext.blk_off + ext.blocks; + if (rng.blk_off < iblock + len) + rng.blocks = (iblock + len) - rng.blk_off; + else + rng.blocks = 0; + + /* done if already offline */ + if (offline && (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) + continue; + + /* free the old extent if it was allocated */ + if (ext.blkno) { + fr = ext; + fr.blk_off = fr.blkno; + ret = insert_extent(sb, &fr, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY); + if (ret) + break; + rem_fr = true; + } + + /* always remove the overlapping file extent */ + ret = remove_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); + if (ret) + break; + ins_ext = true; + + /* maybe add new file extents with the offline flag set */ + if (offline) { + ofl = ext; + ofl.blkno = 0; + ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; + ret = insert_extent(sb, &ofl, sbi->node_id, + SCOUTFS_FILE_EXTENT_KEY); + if (ret) + break; + } + + rem_fr = false; + ins_ext = false; + } + + if (ret) { + if (ins_ext) { + err = insert_extent(sb, &ext, ino, + SCOUTFS_FILE_EXTENT_KEY); + BUG_ON(err); + } + if (rem_fr) { + err = remove_extent(sb, &fr, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY); + BUG_ON(err); + } + } + + return ret; } /* @@ -564,7 +672,6 @@ static struct task_cursor *get_cursor(struct data_info *datinf) static int bulk_alloc(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct extent_change changes[MAX_CHANGES]; struct native_extent ext; u64 *segnos = NULL; int ret; @@ -577,7 +684,6 @@ static int bulk_alloc(struct super_block *sb) } for (i = 0; segnos[i]; i++) { - memset(changes, 0, sizeof(changes)); /* merge or set this one */ if (i > 0 && (segnos[i] == segnos[i - 1] + 1)) { @@ -595,14 +701,12 @@ static int bulk_alloc(struct super_block *sb) if ((segnos[i] + 1) == segnos[i + 1]) continue; - trace_printk("inserting extent [%u] blkno %llu blocks %llu\n", - i, ext.blkno, ext.blocks); + trace_printk("inserting [%u] "EXTF"\n", i, EXTA(&ext)); ext.blk_off = ext.blkno; - ret = record_insert_changes(sb, changes, &ext, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY) ?: - apply_changes(sb, changes); - /* XXX error here leaks segnos */ + ext.flags = 0; + ret = insert_extent(sb, &ext, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY); if (ret) break; } @@ -611,6 +715,8 @@ out: if (!IS_ERR_OR_NULL(segnos)) kfree(segnos); + /* XXX don't orphan segnos on error, crash recovery with server */ + return ret; } @@ -623,12 +729,12 @@ out: * free space outside of our cursor, then we look for the next large * free extent. */ -static int allocate_block(struct inode *inode, sector_t iblock, u64 *blkno) +static int allocate_block(struct inode *inode, sector_t iblock, u64 *blkno, + bool was_offline) { struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); - struct extent_change changes[MAX_CHANGES] = {{{0,}}}; u8 last_bytes[MAX_KEY_BYTES]; u8 key_bytes[MAX_KEY_BYTES]; struct scoutfs_key_buf last; @@ -636,9 +742,15 @@ static int allocate_block(struct inode *inode, sector_t iblock, u64 *blkno) struct native_extent last_ext; struct native_extent found; struct native_extent ext; + struct native_extent ofl; + struct native_extent fr; struct task_cursor *curs; bool alloced = false; + const u64 ino = scoutfs_ino(inode); + bool rem_ext = false; + bool ins_ofl = false; u8 type; + int err; int ret; memset(&last_ext, ~0, sizeof(last_ext)); @@ -658,6 +770,7 @@ reset_cursor: ext.blocks = LARGE_EXTENT_BLOCKS; type = SCOUTFS_FREE_EXTENT_BLOCKS_KEY; } + ext.flags = 0; retry: trace_printk("searching %llu,%llu curs %p task %p pid %u %llu,%llu\n", @@ -709,7 +822,7 @@ retry: } load_extent(&found, &key); - trace_printk("found %llu,%llu\n", found.blkno, found.blocks); + trace_printk("found nei "EXTF"\n", EXTA(&found)); /* look for a new large extent if found is outside cursor */ if (curs->blocks && @@ -743,36 +856,69 @@ retry: trace_printk("using %llu,%llu curs %llu,%llu\n", found.blkno, found.blocks, curs->blkno, curs->blocks); + /* remove old offline block if we're staging */ + if (was_offline) { + ofl.blk_off = iblock; + ofl.blkno = 0; + ofl.blocks = 1; + ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; + ret = remove_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_KEY); + if (ret < 0) + goto out; + ins_ofl = true; + } + + /* insert new file extent */ *blkno = found.blkno; ext.blk_off = iblock; ext.blkno = found.blkno; ext.blocks = 1; - ret = record_insert_changes(sb, changes, &ext, scoutfs_ino(inode), - SCOUTFS_FILE_EXTENT_KEY); + ext.flags = 0; + ret = insert_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); if (ret < 0) goto out; + rem_ext = true; - ext.blk_off = ext.blkno; - ret = record_remove_changes(sb, changes, &ext, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY) ?: - apply_changes(sb, changes); + /* and remove free extents */ + fr = ext; + fr.blk_off = ext.blkno; + ret = remove_extent(sb, &fr, sbi->node_id, + SCOUTFS_FREE_EXTENT_BLKNO_KEY); + if (ret) + goto out; /* advance cursor if we're using it */ - if (ret == 0 && curs->blocks) { + if (curs->blocks) { if (--curs->blocks == 0) curs->blkno = 0; else curs->blkno++; } + ret = 0; out: + if (ret) { + if (rem_ext) { + err = remove_extent(sb, &ext, ino, + SCOUTFS_FILE_EXTENT_KEY); + BUG_ON(err); + } + if (ins_ofl) { + err = insert_extent(sb, &ofl, ino, + SCOUTFS_FILE_EXTENT_KEY); + BUG_ON(err); + } + } + up_write(&datinf->alloc_rwsem); + trace_printk("ret %d\n", ret); return ret; } static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; DECLARE_DATA_INFO(sb, datinf); u8 last_bytes[MAX_KEY_BYTES]; @@ -780,24 +926,22 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct scoutfs_key_buf last; struct scoutfs_key_buf key; struct native_extent ext; - u64 blocks; + bool was_offline = false; u64 blkno; u64 off; int ret; bh->b_blocknr = 0; bh->b_size = 0; - blocks = 0; ext.blk_off = iblock; ext.blocks = 1; ext.blkno = 0; + ext.flags = 0; init_extent_key(&key, key_bytes, &ext, scoutfs_ino(inode), SCOUTFS_FILE_EXTENT_KEY); - ext.blk_off = ~0ULL; - ext.blkno = ~0ULL; - ext.blocks = ~0ULL; + memset(&ext, ~0, sizeof(ext)); init_extent_key(&last, last_bytes, &ext, scoutfs_ino(inode), SCOUTFS_FILE_EXTENT_KEY); @@ -810,35 +954,49 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, up_read(&datinf->alloc_rwsem); if (ret < 0) { if (ret == -ENOENT) - ret = 0; + memset(&ext, 0, sizeof(ext)); else goto out; } else { load_extent(&ext, &key); - trace_printk("found blk_off %llu blkno %llu blocks %llu\n", - ext.blk_off, ext.blkno, ext.blocks); - if (iblock >= ext.blk_off && - iblock < (ext.blk_off + ext.blocks)) { + trace_printk("found nei "EXTF"\n", EXTA(&ext)); + } + + if ((ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) && !si->staging) { + ret = -EINVAL; + goto out; + } + + /* use the extent if it intersects */ + if (iblock >= ext.blk_off && iblock < (ext.blk_off + ext.blocks)) { + + if (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) { + /* non-stage can't write to offline */ + if (!si->staging) { + ret = -EINVAL; + goto out; + } + was_offline = true; + } else { + /* found online extent */ off = iblock - ext.blk_off; - blkno = ext.blkno + off; - blocks = ext.blocks - off; + map_bh(bh, inode->i_sb, ext.blkno + off); + bh->b_size = min_t(u64, SIZE_MAX, + (ext.blocks - off) << SCOUTFS_BLOCK_SHIFT); } } - if (blocks == 0 && create) { - ret = allocate_block(inode, iblock, &blkno); + if (!buffer_mapped(bh) && create) { + ret = allocate_block(inode, iblock, &blkno, was_offline); if (ret) goto out; - blocks = 1; - } - - if (blocks) { map_bh(bh, inode->i_sb, blkno); - bh->b_size = min_t(u64, SIZE_MAX, - blocks << SCOUTFS_BLOCK_SHIFT); + bh->b_size = SCOUTFS_BLOCK_SHIFT; + set_buffer_new(bh); } + ret = 0; out: trace_printk("ino %llu iblock %llu create %d ret %d bnr %llu size %zu\n", scoutfs_ino(inode), (u64)iblock, create, ret, @@ -921,6 +1079,91 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, return ret; } +/* + * Return the extents that intersect with the given byte range. It doesn't + * trim the returned extents to the byte range. + */ +int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct super_block *sb = inode->i_sb; + const u8 type = SCOUTFS_FILE_EXTENT_KEY; + const u64 ino = scoutfs_ino(inode); + u8 last_bytes[MAX_KEY_BYTES]; + u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + struct native_extent ext; + u64 logical; + u64 blk_off; + u64 final; + u64 phys; + u64 size; + u32 flags; + int ret = 0; + + ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); + if (ret) + goto out; + + memset(&ext, ~0, sizeof(ext)); + init_extent_key(&last, last_bytes, &ext, ino, type); + + blk_off = start >> SCOUTFS_BLOCK_SHIFT; + final = (start + len - 1) >> SCOUTFS_BLOCK_SHIFT; + size = 0; + flags = 0; + + /* XXX overkill? */ + mutex_lock(&inode->i_mutex); + + for (;;) { + ext.blk_off = blk_off; + ext.blkno = 0; + ext.blocks = 1; + ext.flags = 0; + init_extent_key(&key, key_bytes, &ext, ino, type); + + ret = scoutfs_item_next_same(sb, &key, &last, NULL); + if (ret < 0) { + if (ret != -ENOENT) + break; + flags |= FIEMAP_EXTENT_LAST; + ret = 0; + } + + load_extent(&ext, &key); + + if (ext.blk_off > final) + flags |= FIEMAP_EXTENT_LAST; + + if (size) { + ret = fiemap_fill_next_extent(fieinfo, logical, phys, + size, flags); + if (ret != 0) { + if (ret == 1) + ret = 0; + break; + } + } + + if (flags & FIEMAP_EXTENT_LAST) + break; + + logical = ext.blk_off << SCOUTFS_BLOCK_SHIFT; + phys = ext.blkno << SCOUTFS_BLOCK_SHIFT; + size = ext.blocks << SCOUTFS_BLOCK_SHIFT; + flags = ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE ? + FIEMAP_EXTENT_UNKNOWN : 0; + + blk_off = ext.blk_off + ext.blocks; + } + + mutex_unlock(&inode->i_mutex); +out: + return ret; +} + const struct address_space_operations scoutfs_file_aops = { .readpage = scoutfs_readpage, .readpages = scoutfs_readpages, diff --git a/kmod/src/data.h b/kmod/src/data.h index 1319d100..da624a80 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -6,6 +6,8 @@ extern const struct file_operations scoutfs_file_fops; int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline); +int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index a58d12b6..63cc07be 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -206,8 +206,11 @@ struct scoutfs_file_extent_key { __be64 last_blk_off; __be64 last_blkno; __be64 blocks; + __u8 flags; } __packed; +#define SCOUTFS_FILE_EXTENT_OFFLINE (1 << 0) + /* no value */ struct scoutfs_free_extent_blkno_key { __u8 type; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 71a98d33..506cdd74 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -145,6 +145,7 @@ static const struct inode_operations scoutfs_file_iops = { .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, .removexattr = scoutfs_removexattr, + .fiemap = scoutfs_data_fiemap, }; static const struct inode_operations scoutfs_special_iops = { From b97587b8fa47d6aea6b33ea0bf11a29e89a428c4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 15 May 2017 09:37:36 -0700 Subject: [PATCH 273/920] scoutfs: add indexing of inodes by fields Add items for indexing inodes by their fields. When we update the inode item we also delete the old index items and create the new items. We rename and refactor the old inode since ioctl to now walk the inode index items. Signed-off-by: Zach Brown --- kmod/src/format.h | 15 +++++++ kmod/src/inode.c | 97 ++++++++++++++++++++++++++++++++++++++++++ kmod/src/inode.h | 4 ++ kmod/src/ioctl.c | 105 ++++++++++++++++++++++------------------------ kmod/src/ioctl.h | 60 ++++++++++++++++++-------- 5 files changed, 209 insertions(+), 72 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 63cc07be..7e91afa1 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -160,6 +160,9 @@ struct scoutfs_segment_block { #define SCOUTFS_ORPHAN_KEY 10 #define SCOUTFS_FREE_EXTENT_BLKNO_KEY 11 #define SCOUTFS_FREE_EXTENT_BLOCKS_KEY 12 +#define SCOUTFS_INODE_INDEX_CTIME_KEY 13 +#define SCOUTFS_INODE_INDEX_MTIME_KEY 14 +#define SCOUTFS_INODE_INDEX_SIZE_KEY 15 /* not found in the fs */ #define SCOUTFS_MAX_UNUSED_KEY 253 #define SCOUTFS_NET_ADDR_KEY 254 @@ -249,6 +252,18 @@ struct scoutfs_symlink_key { __be64 ino; } __packed; +struct scoutfs_betimespec { + __be64 sec; + __be32 nsec; +} __packed; + +struct scoutfs_inode_index_key { + __u8 type; + __be64 major; + __be32 minor; + __be64 ino; +} __packed; + /* XXX does this exist upstream somewhere? */ #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 506cdd74..43c6c730 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -184,6 +184,16 @@ static void set_inode_ops(struct inode *inode) mapping_set_gfp_mask(inode->i_mapping, GFP_USER); } +static void set_item_info(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + si->have_item = true; + si->item_size = i_size_read(inode); + si->item_ctime = inode->i_ctime; + si->item_mtime = inode->i_mtime; +} + static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) { struct scoutfs_inode_info *ci = SCOUTFS_I(inode); @@ -203,6 +213,8 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->data_version = le64_to_cpu(cinode->data_version); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); + + set_item_info(inode); } void scoutfs_inode_init_key(struct scoutfs_key_buf *key, @@ -362,6 +374,77 @@ int scoutfs_dirty_inode_item(struct inode *inode) return ret; } +/* + * Make sure inode index items are kept in sync with the fields that are + * set in the inode items. This must be called any time the contents of + * the inode items are updated. + * + * This is effectively a RMW on the inode fields so the caller needs to + * lock the inode so that it's the only one working with the index items + * for a given set of fields in the inode. + * + * But it doesn't need to lock the index item keys. By locking the + * inode we've ensured that we can safely log deletion and insertion + * items in our log. The indexes are eventually consistent so we don't + * need to wrap them locks. + * + * XXX this needs more supporting work from the rest of the + * infrastructure: + * + * - Deleting and creating the items needs to forcefully set those dirty + * items in the cache without first trying to read them from segments. + * - the reading ioctl needs to forcefully invalidate the index items + * as it walks. + * - maybe the reading ioctl needs to verify fields with inodes? + * - final inode deletion needs to invalidate the index items for + * each inode as it deletes items based on the locked inode fields. + * - make sure deletion items safely vanish w/o finding existing item + * - ... error handling :( + */ +static int update_index(struct inode *inode, u8 type, u64 now_major, + u32 now_minor, u64 then_major, u32 then_minor) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_inode_index_key ins_ikey; + struct scoutfs_inode_index_key del_ikey; + struct scoutfs_key_buf ins; + struct scoutfs_key_buf del; + int ret; + int err; + + trace_printk("ino %llu have %u now %llu.%u then %llu.%u \n", + scoutfs_ino(inode), si->have_item, + now_major, now_minor, then_major, then_minor); + + if (si->have_item && now_major == then_major && now_minor == then_minor) + return 0; + + ins_ikey.type = type; + ins_ikey.major = cpu_to_be64(now_major); + ins_ikey.minor = cpu_to_be32(now_minor); + ins_ikey.ino = cpu_to_be64(scoutfs_ino(inode)); + scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); + + ret = scoutfs_item_create(sb, &ins, NULL); + if (ret || !si->have_item) + return ret; + + del_ikey.type = type; + del_ikey.major = cpu_to_be64(then_major); + del_ikey.minor = cpu_to_be32(then_minor); + del_ikey.ino = cpu_to_be64(scoutfs_ino(inode)); + scoutfs_key_init(&del, &del_ikey, sizeof(del_ikey)); + + ret = scoutfs_item_delete(sb, &del); + if (ret) { + err = scoutfs_item_delete(sb, &ins); + BUG_ON(err); + } + + return ret; +} + /* * Every time we modify the inode in memory we copy it to its inode * item. This lets us write out items without having to track down @@ -373,13 +456,25 @@ int scoutfs_dirty_inode_item(struct inode *inode) */ void scoutfs_update_inode_item(struct inode *inode) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; struct scoutfs_key_buf key; struct scoutfs_inode sinode; SCOUTFS_DECLARE_KVEC(val); + int ret; int err; + ret = update_index(inode, SCOUTFS_INODE_INDEX_CTIME_KEY, + inode->i_ctime.tv_sec, inode->i_ctime.tv_nsec, + si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: + update_index(inode, SCOUTFS_INODE_INDEX_MTIME_KEY, + inode->i_mtime.tv_sec, inode->i_mtime.tv_nsec, + si->item_mtime.tv_sec, si->item_mtime.tv_nsec) ?: + update_index(inode, SCOUTFS_INODE_INDEX_SIZE_KEY, + i_size_read(inode), 0, si->item_size, 0); + BUG_ON(ret); + store_inode(&sinode, inode); scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); @@ -392,6 +487,7 @@ void scoutfs_update_inode_item(struct inode *inode) BUG_ON(err); } + set_item_info(inode); trace_scoutfs_update_inode(inode); } @@ -562,6 +658,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->ino = ino; ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; + ci->have_item = false; inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 59da8f7a..5f453996 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -8,6 +8,10 @@ struct scoutfs_inode_info { u64 ino; u64 data_version; u64 next_readdir_pos; + bool have_item; + u64 item_size; + struct timespec item_ctime; + struct timespec item_mtime; /* initialized once for slab object */ seqcount_t seqcount; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 48f814d0..44a24682 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -28,84 +28,81 @@ #include "super.h" #include "inode.h" #include "trans.h" +#include "item.h" #include "data.h" /* - * Find all the inodes that have had keys of a given type modified since - * a given sequence number. The user's arg struct specifies the inode - * range to search within and the sequence value to return results from. - * Different ioctls call this for different key types. - * - * When this is used for file data items the user is trying to find - * inodes whose data has changed since a given time in the past. - * - * XXX We'll need to improve the walk and search to notice when file - * data items have been truncated away. - * - * Inodes and their sequence numbers are copied out to userspace in - * inode order, not sequence order. + * Walk one of the inode index items. This is a thin ioctl wrapper + * around the core item interface. */ -static long scoutfs_ioc_inodes_since(struct file *file, unsigned long arg, - u8 type) +static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_inodes_since __user *uargs = (void __user *)arg; - struct scoutfs_ioctl_inodes_since args; - struct scoutfs_ioctl_ino_seq __user *uiseq; - struct scoutfs_ioctl_ino_seq iseq; - struct scoutfs_inode_key last_ikey; - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf last; + struct scoutfs_ioctl_walk_inodes __user *uwalk = (void __user *)arg; + struct scoutfs_ioctl_walk_inodes walk; + struct scoutfs_ioctl_walk_inodes_entry ent; + struct scoutfs_inode_index_key last_ikey; + struct scoutfs_inode_index_key ikey; + struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; - long bytes; - u64 seq; - int ret; + int ret = 0; + u32 nr; - if (copy_from_user(&args, uargs, sizeof(args))) + if (copy_from_user(&walk, uwalk, sizeof(walk))) return -EFAULT; - uiseq = (void __user *)(unsigned long)args.buf_ptr; - if (args.buf_len < sizeof(iseq) || args.buf_len > INT_MAX) + trace_printk("index %u first %llu.%u.%llu last %llu.%u.%llu\n", + walk.index, walk.first.major, walk.first.minor, + walk.first.ino, walk.last.major, walk.last.minor, + walk.last.ino); + + if (walk.index == SCOUTFS_IOC_WALK_INODES_CTIME) + ikey.type = SCOUTFS_INODE_INDEX_CTIME_KEY; + else if (walk.index == SCOUTFS_IOC_WALK_INODES_MTIME) + ikey.type = SCOUTFS_INODE_INDEX_MTIME_KEY; + else if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) + ikey.type = SCOUTFS_INODE_INDEX_SIZE_KEY; + else return -EINVAL; - scoutfs_inode_init_key(&key, &ikey, args.first_ino); - scoutfs_inode_init_key(&last, &last_ikey, args.last_ino); + ikey.major = cpu_to_be64(walk.first.major); + ikey.minor = cpu_to_be32(walk.first.minor); + ikey.ino = cpu_to_be64(walk.first.ino); + scoutfs_key_init(&key, &ikey, sizeof(ikey)); - bytes = 0; - for (;;) { + last_ikey.type = ikey.type; + last_ikey.major = cpu_to_be64(walk.last.major); + last_ikey.minor = cpu_to_be32(walk.last.minor); + last_ikey.ino = cpu_to_be64(walk.last.ino); + scoutfs_key_init(&last_key, &last_ikey, sizeof(last_ikey)); - /* XXX item cache needs to search by seq */ - seq = !!sb; - ret = WARN_ON_ONCE(-EINVAL); -// ret = scoutfs_item_since(sb, &key, &last, args.seq, &seq, NULL); + /* cap nr to the max the ioctl can return to a compat task */ + walk.nr_entries = min_t(u64, walk.nr_entries, INT_MAX); + + for (nr = 0; nr < walk.nr_entries; + nr++, walk.entries_ptr += sizeof(ent)) { + + ret = scoutfs_item_next_same(sb, &key, &last_key, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - iseq.ino = be64_to_cpu(ikey.ino); - iseq.seq = seq; + ent.major = be64_to_cpu(ikey.major); + ent.minor = be32_to_cpu(ikey.minor); + ent.ino = be64_to_cpu(ikey.ino); - if (copy_to_user(uiseq, &iseq, sizeof(iseq))) { + if (copy_to_user((void __user *)walk.entries_ptr, &ent, + sizeof(ent))) { ret = -EFAULT; break; } - uiseq++; - bytes += sizeof(iseq); - if (bytes + sizeof(iseq) > args.buf_len) { - ret = 0; - break; - } - - last_ikey.ino = cpu_to_be64(iseq.ino + 1); + scoutfs_key_inc_cur_len(&key); } - if (bytes) - ret = bytes; - - return ret; + return nr ?: ret; } struct ino_path_cursor { @@ -419,12 +416,10 @@ out: long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { - case SCOUTFS_IOC_INODES_SINCE: - return scoutfs_ioc_inodes_since(file, arg, SCOUTFS_INODE_KEY); + case SCOUTFS_IOC_WALK_INODES: + return scoutfs_ioc_walk_inodes(file, arg); case SCOUTFS_IOC_INO_PATH: return scoutfs_ioc_ino_path(file, arg); - case SCOUTFS_IOC_INODE_DATA_SINCE: - return WARN_ON_ONCE(-EINVAL); case SCOUTFS_IOC_DATA_VERSION: return scoutfs_ioc_data_version(file, arg); case SCOUTFS_IOC_RELEASE: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index e84db63c..4d529550 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -6,25 +6,54 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); /* XXX I have no idea how these are chosen. */ #define SCOUTFS_IOCTL_MAGIC 's' -struct scoutfs_ioctl_ino_seq { +struct scoutfs_ioctl_walk_inodes_entry { + __u64 major; + __u32 minor; __u64 ino; - __u64 seq; -} __packed; - -struct scoutfs_ioctl_inodes_since { - __u64 first_ino; - __u64 last_ino; - __u64 seq; - __u64 buf_ptr; - __u32 buf_len; } __packed; /* - * Adds entries to the user's buffer for each inode whose sequence - * number is greater than or equal to the given seq. + * Walk inodes in an index that is sorted by one of their fields. + * + * Each index is built from generic index items that have major and + * minor values that are set to the field being indexed. In time + * indices, for example, major is seconds and minor is nanoseconds. + * + * @first The first index entry that can be returned. + * @last The last index entry that can be returned. + * @entries_ptr Pointer to emory containing buffer for entry results. + * @nr_entries The number of entries that can fit in the buffer. + * @index Which index to walk, enumerated in _WALK_INODES_ constants. + * + * To start iterating first can be memset to 0 and last to 0xff. Then + * after each set of results first can be set to the last entry returned + * and then the fields can be incremented in reverse sort order (ino < + * minor < major) as each increasingly significant value wraps around to + * 0. + * + * If first is greater than last then the walk will return 0 entries. */ -#define SCOUTFS_IOC_INODES_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ - struct scoutfs_ioctl_inodes_since) +struct scoutfs_ioctl_walk_inodes { + struct scoutfs_ioctl_walk_inodes_entry first; + struct scoutfs_ioctl_walk_inodes_entry last; + __u64 entries_ptr; + __u32 nr_entries; + __u8 index; +} __packed; + +enum { + SCOUTFS_IOC_WALK_INODES_CTIME = 0, + SCOUTFS_IOC_WALK_INODES_MTIME, + SCOUTFS_IOC_WALK_INODES_SIZE, + SCOUTFS_IOC_WALK_INODES_UNKNOWN, +}; + +/* + * Adds entries to the user's buffer for each inode that is found in the + * given index between the first and last positions. + */ +#define SCOUTFS_IOC_WALK_INODES _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ + struct scoutfs_ioctl_walk_inodes) /* * Fill the path buffer with the next path to the target inode. An @@ -80,9 +109,6 @@ struct scoutfs_ioctl_ino_path { #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ struct scoutfs_ioctl_ino_path) -#define SCOUTFS_IOC_INODE_DATA_SINCE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ - struct scoutfs_ioctl_inodes_since) - #define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, __u64) struct scoutfs_ioctl_release { From 5307c56954aef244c757a4d7317fa49e307df416 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 16 May 2017 14:11:56 -0700 Subject: [PATCH 274/920] scoutfs: add a stat_more ioctl We have inode fields that we want to return to userspace with very low overhead. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 20 ++++++++++++++++++++ kmod/src/ioctl.h | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 44a24682..3e0807cc 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -413,6 +413,24 @@ out: return ret; } +static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct scoutfs_ioctl_stat_more stm; + + if (get_user(stm.valid_bytes, (__u64 __user *)arg)) + return -EFAULT; + + stm.valid_bytes = min_t(u64, stm.valid_bytes, + sizeof(struct scoutfs_ioctl_stat_more)); + stm.data_version = scoutfs_inode_get_data_version(inode); + + if (copy_to_user((void __user *)arg, &stm, stm.valid_bytes)) + return -EFAULT; + + return 0; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -426,6 +444,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_release(file, arg); case SCOUTFS_IOC_STAGE: return scoutfs_ioc_stage(file, arg); + case SCOUTFS_IOC_STAT_MORE: + return scoutfs_ioc_stat_more(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 4d529550..a32a6724 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -130,4 +130,25 @@ struct scoutfs_ioctl_stage { #define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 6, \ struct scoutfs_ioctl_stage) +/* + * Give the user inode fields that are not otherwise visible. statx() + * isn't always available and xattrs are relatively expensive. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_stat_more { + __u64 valid_bytes; + __u64 data_version; +} __packed; + +#define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ + struct scoutfs_ioctl_stat_more) + #endif From 8ea414ac687b460c2fb824113d1d1e2d47c3ec22 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 16 May 2017 14:51:36 -0700 Subject: [PATCH 275/920] scoutfs: clear seg rb node after replacing When inserting a newly allocated segment we might find an existing cached stale segment. We replace it in the cache so that its user can keep using its stale contents while we work on the new segment. Replacing doesn't clear the rb_node, though, so we trip over a warning when we finally free the segment and it looks like it's still present in the rb tree. Clear the node after we replace it so that freeing sees a clear node and doesn't issue a warning. Signed-off-by: Zach Brown --- kmod/src/seg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 6e0fc04a..767bab2e 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -182,6 +182,7 @@ static struct scoutfs_segment *replace_seg(struct segment_cache *cac, node = &(*node)->rb_right; } else { rb_replace_node(&seg->node, &ins->node, root); + RB_CLEAR_NODE(&seg->node); lru_check(cac, seg); lru_check(cac, ins); found = seg; From 373def02f080e9bd11f596fdd4cbec88a9ce948e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 18 May 2017 10:51:09 -0700 Subject: [PATCH 276/920] scoutfs: remove trade_time message This was mostly just a demonstration for how to add messages. We're about to add a message that we always send on mount so this becomes completely redundant. Signed-off-by: Zach Brown --- kmod/src/format.h | 4 +-- kmod/src/net.c | 66 ----------------------------------------------- kmod/src/net.h | 1 - kmod/src/super.c | 2 -- 4 files changed, 1 insertion(+), 72 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 7e91afa1..4afd26f6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -426,9 +426,7 @@ struct scoutfs_net_segnos { } __packed; enum { - /* sends and receives a struct scoutfs_timeval */ - SCOUTFS_NET_TRADE_TIME = 0, - SCOUTFS_NET_ALLOC_INODES, + SCOUTFS_NET_ALLOC_INODES = 0, SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, diff --git a/kmod/src/net.c b/kmod/src/net.c index 14f1d4f3..eb34d12b 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -628,45 +628,12 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, return sbuf; } -/* - * Log the time in the request and reply with our current time. - */ -static struct send_buf *process_trade_time(struct super_block *sb, - void *r, int req_len) -{ - struct scoutfs_timespec *req = r; - struct scoutfs_timespec *reply; - struct send_buf *sbuf; - struct timespec64 ts; - - if (req_len != sizeof(*req)) - return ERR_PTR(-EINVAL); - - sbuf = alloc_sbuf(sizeof(struct scoutfs_timespec)); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - getnstimeofday64(&ts); - trace_printk("req %llu.%u replying %llu.%lu\n", - le64_to_cpu(req->sec), le32_to_cpu(req->nsec), - (u64)ts.tv_sec, ts.tv_nsec); - - reply = (void *)sbuf->nh->data; - reply->sec = cpu_to_le64(ts.tv_sec); - reply->nsec = cpu_to_le32(ts.tv_nsec); - - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - typedef struct send_buf *(*proc_func_t)(struct super_block *sb, void *req, int req_len); static proc_func_t type_proc_func(u8 type) { static proc_func_t funcs[] = { - [SCOUTFS_NET_TRADE_TIME] = process_trade_time, [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, [SCOUTFS_NET_MANIFEST_RANGE_ENTRIES] = process_manifest_range_entries, @@ -1583,39 +1550,6 @@ int scoutfs_net_alloc_inodes(struct super_block *sb) alloc_inodes_reply, NULL); } -static int trade_time_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct scoutfs_timespec *ts = reply; - - if (ret != sizeof(*ts)) - return -EINVAL; - - trace_printk("reply %llu.%u\n", - le64_to_cpu(ts->sec), le32_to_cpu(ts->nsec)); - - return 0; -} - -int scoutfs_net_trade_time(struct super_block *sb) -{ - struct scoutfs_timespec send; - struct timespec64 ts; - int ret; - - getnstimeofday64(&ts); - send.sec = cpu_to_le64(ts.tv_sec); - send.nsec = cpu_to_le32(ts.tv_nsec); - - ret = add_send_buf(sb, SCOUTFS_NET_TRADE_TIME, &send, - sizeof(send), trade_time_reply, NULL); - - trace_printk("sent %llu.%lu ret %d\n", - (u64)ts.tv_sec, ts.tv_nsec, ret); - - return ret; -} - static struct sock_info *alloc_sinf(struct super_block *sb) { struct sock_info *sinf; diff --git a/kmod/src/net.h b/kmod/src/net.h index 125bf327..e51d9266 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -4,7 +4,6 @@ struct scoutfs_key_buf; struct scoutfs_segment; -int scoutfs_net_trade_time(struct super_block *sb); int scoutfs_net_alloc_inodes(struct super_block *sb); int scoutfs_net_manifest_range_entries(struct super_block *sb, struct scoutfs_key_buf *start, diff --git a/kmod/src/super.c b/kmod/src/super.c index bcf831fc..847b79b8 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -234,8 +234,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (ret) return ret; - scoutfs_net_trade_time(sb); - inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); if (IS_ERR(inode)) return PTR_ERR(inode); From b291818448b01d52ae90ca819b8a4249ffcf41d0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 May 2017 11:19:56 -0700 Subject: [PATCH 277/920] scoutfs: add sync deadline timer Make sure that data is regularly synced. We switch to a delayed work struct that is always queued with the sync deadline. If we need an immediate sync we mod it to now. Signed-off-by: Zach Brown --- kmod/src/super.c | 3 ++- kmod/src/super.h | 2 +- kmod/src/trans.c | 24 +++++++++++++++++++++--- kmod/src/trans.h | 1 + 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 847b79b8..f89bad20 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -214,7 +214,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); - INIT_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); + INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); /* XXX can have multiple mounts of a device, need mount id */ @@ -242,6 +242,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; + scoutfs_trans_restart_sync_deadline(sb); // scoutfs_scan_orphans(sb); return 0; diff --git a/kmod/src/super.h b/kmod/src/super.h index 5b6d5903..8bbd313b 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -40,7 +40,7 @@ struct scoutfs_sb_info { spinlock_t trans_write_lock; u64 trans_write_count; int trans_write_ret; - struct work_struct trans_write_work; + struct delayed_work trans_write_work; wait_queue_head_t trans_write_wq; struct workqueue_struct *trans_write_workq; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 11941c7e..58c12465 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -50,6 +50,9 @@ * very long time. */ +/* sync dirty data at least this often */ +#define TRANS_SYNC_DELAY (HZ * 10) + /* * This work func is responsible for writing out all the dirty blocks * that make up the current dirty transaction. It prevents writers from @@ -77,7 +80,7 @@ void scoutfs_trans_write_func(struct work_struct *work) { struct scoutfs_sb_info *sbi = container_of(work, struct scoutfs_sb_info, - trans_write_work); + trans_write_work.work); struct super_block *sb = sbi->sb; struct scoutfs_bio_completion comp; struct scoutfs_segment *seg; @@ -125,6 +128,8 @@ out: wake_up(&sbi->trans_hold_wq); sbi->trans_task = NULL; + + scoutfs_trans_restart_sync_deadline(sb); } struct write_attempt { @@ -148,9 +153,14 @@ static int write_attempted(struct scoutfs_sb_info *sbi, return done; } + +/* + * We always have delayed sync work pending but the caller wants it + * to execute immediately. + */ static void queue_trans_work(struct scoutfs_sb_info *sbi) { - queue_work(sbi->trans_write_workq, &sbi->trans_write_work); + mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, 0); } /* @@ -194,6 +204,14 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, return scoutfs_sync_fs(file->f_inode->i_sb, 1); } +void scoutfs_trans_restart_sync_deadline(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, + TRANS_SYNC_DELAY); +} + /* * The holder that creates the most dirty item data is adding a full * size xattr. The largest xattr can have a 255 byte name and 64KB @@ -322,7 +340,7 @@ void scoutfs_shutdown_trans(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); if (sbi->trans_write_workq) { - flush_work(&sbi->trans_write_work); + cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); } } diff --git a/kmod/src/trans.h b/kmod/src/trans.h index f1ecbc51..396ad6be 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -5,6 +5,7 @@ void scoutfs_trans_write_func(struct work_struct *work); int scoutfs_sync_fs(struct super_block *sb, int wait); int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); +void scoutfs_trans_restart_sync_deadline(struct super_block *sb); int scoutfs_hold_trans(struct super_block *sb); void scoutfs_release_trans(struct super_block *sb); From 5f11cdbfe53da21d23c7aa7974537f8b07165eea Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 May 2017 13:51:00 -0700 Subject: [PATCH 278/920] scoutfs: add and index inode meta and data seqs For each transaction we send a message to to the server asking for a unique sequence number to associate with the transaction. When we change metadata or data of an inode we store the current transaction seq in the inode and we index it with index items like the other inode fields. The server remembers the sequences it gives out. When we go to walk the inode sequence indexes we ask the server for the largest stable seq and limit results to that seq. This ensures that we never return seqs that are past dirty items so never have inodes and seqs appear in the past. Nodes use the sync timer to regularly cycle through seqs and ensure that inode seq index walks don't get stuck on their otherwise idle seq. Signed-off-by: Zach Brown --- kmod/src/data.c | 6 +- kmod/src/format.h | 15 ++++ kmod/src/inode.c | 88 ++++++++++++++++-- kmod/src/inode.h | 12 ++- kmod/src/ioctl.c | 46 +++++----- kmod/src/ioctl.h | 15 ++++ kmod/src/net.c | 225 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/net.h | 2 + kmod/src/super.c | 4 + kmod/src/super.h | 2 + kmod/src/trans.c | 13 +++ 11 files changed, 396 insertions(+), 32 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 98dc2143..76cdd2a5 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1062,6 +1062,7 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct page *page, void *fsdata) { struct inode *inode = mapping->host; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; int ret; @@ -1070,7 +1071,10 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); if (ret > 0) { - scoutfs_inode_inc_data_version(inode); + if (!si->staging) { + scoutfs_inode_set_data_seq(inode); + scoutfs_inode_inc_data_version(inode); + } /* XXX kind of a big hammer, inode life cycle needs work */ scoutfs_update_inode_item(inode); scoutfs_inode_queue_writeback(inode); diff --git a/kmod/src/format.h b/kmod/src/format.h index 4afd26f6..dd991c2d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -163,6 +163,8 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_CTIME_KEY 13 #define SCOUTFS_INODE_INDEX_MTIME_KEY 14 #define SCOUTFS_INODE_INDEX_SIZE_KEY 15 +#define SCOUTFS_INODE_INDEX_META_SEQ_KEY 16 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_KEY 17 /* not found in the fs */ #define SCOUTFS_MAX_UNUSED_KEY 253 #define SCOUTFS_NET_ADDR_KEY 254 @@ -280,6 +282,7 @@ struct scoutfs_super_block { __le64 id; __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; + __le64 next_seq; __le64 alloc_uninit; __le64 total_segs; __le64 free_segs; @@ -300,6 +303,14 @@ struct scoutfs_timespec { } __packed; /* + * @meta_seq: advanced the first time an inode is updated in a given + * transaction. It can only advance again after the inode is written + * and a new transaction opens. + * + * @data_seq: advanced the first time a file's data (or size) is + * modified in a given transaction. It can only advance again after the + * file is written and a new transaction opens. + * * @data_version: incremented every time the contents of a file could * have changed. It is exposed via an ioctl and is then provided as an * argument to data functions to protect racing modification. @@ -314,6 +325,8 @@ struct scoutfs_timespec { struct scoutfs_inode { __le64 size; __le64 blocks; + __le64 meta_seq; + __le64 data_seq; __le64 data_version; __le64 next_readdir_pos; __le32 nlink; @@ -431,6 +444,8 @@ enum { SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_BULK_ALLOC, + SCOUTFS_NET_ADVANCE_SEQ, + SCOUTFS_NET_GET_LAST_SEQ, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 43c6c730..c1a65eef 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -192,6 +192,8 @@ static void set_item_info(struct inode *inode) si->item_size = i_size_read(inode); si->item_ctime = inode->i_ctime; si->item_mtime = inode->i_mtime; + si->item_meta_seq = scoutfs_inode_meta_seq(inode); + si->item_data_seq = scoutfs_inode_data_seq(inode); } static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) @@ -211,6 +213,8 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) inode->i_ctime.tv_sec = le64_to_cpu(cinode->ctime.sec); inode->i_ctime.tv_nsec = le32_to_cpu(cinode->ctime.nsec); + ci->meta_seq = le64_to_cpu(cinode->meta_seq); + ci->data_seq = le64_to_cpu(cinode->data_seq); ci->data_version = le64_to_cpu(cinode->data_version); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); @@ -245,31 +249,84 @@ static int scoutfs_read_locked_inode(struct inode *inode) return ret; } -void scoutfs_inode_inc_data_version(struct inode *inode) +/* + * Set a given seq to the current trans seq if it differs. The caller + * holds locks and a transaction which prevents the transaction from + * committing and refreshing the seq. + */ +static void set_trans_seq(struct inode *inode, u64 *seq) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - if (!si->staging) { + if (*seq != sbi->trans_seq) { preempt_disable(); write_seqcount_begin(&si->seqcount); - si->data_version++; + *seq = sbi->trans_seq; write_seqcount_end(&si->seqcount); preempt_enable(); } } -u64 scoutfs_inode_get_data_version(struct inode *inode) +void scoutfs_inode_set_meta_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + set_trans_seq(inode, &si->meta_seq); +} + +void scoutfs_inode_set_data_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + set_trans_seq(inode, &si->data_seq); +} + +void scoutfs_inode_inc_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version++; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + +static u64 read_seqcount_u64(struct inode *inode, u64 *val) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); unsigned int seq; - u64 vers; + u64 v; do { seq = read_seqcount_begin(&si->seqcount); - vers = si->data_version; + v = *val; } while (read_seqcount_retry(&si->seqcount, seq)); - return vers; + return v; +} + +u64 scoutfs_inode_meta_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->meta_seq); +} + +u64 scoutfs_inode_data_seq(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->data_seq); +} + +u64 scoutfs_inode_data_version(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->data_version); } static int scoutfs_iget_test(struct inode *inode, void *arg) @@ -332,7 +389,9 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->mtime.sec = cpu_to_le64(inode->i_mtime.tv_sec); cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); - cinode->data_version = cpu_to_le64(ci->data_version); + cinode->meta_seq = cpu_to_le64(scoutfs_inode_meta_seq(inode)); + cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); + cinode->data_version = cpu_to_le64(scoutfs_inode_data_version(inode)); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); } @@ -465,6 +524,9 @@ void scoutfs_update_inode_item(struct inode *inode) int ret; int err; + /* set the meta version once per trans for any inode updates */ + scoutfs_inode_set_meta_seq(inode); + ret = update_index(inode, SCOUTFS_INODE_INDEX_CTIME_KEY, inode->i_ctime.tv_sec, inode->i_ctime.tv_nsec, si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: @@ -472,7 +534,13 @@ void scoutfs_update_inode_item(struct inode *inode) inode->i_mtime.tv_sec, inode->i_mtime.tv_nsec, si->item_mtime.tv_sec, si->item_mtime.tv_nsec) ?: update_index(inode, SCOUTFS_INODE_INDEX_SIZE_KEY, - i_size_read(inode), 0, si->item_size, 0); + i_size_read(inode), 0, si->item_size, 0) ?: + update_index(inode, SCOUTFS_INODE_INDEX_META_SEQ_KEY, + scoutfs_inode_meta_seq(inode), 0, + si->item_meta_seq, 0) ?: + update_index(inode, SCOUTFS_INODE_INDEX_DATA_SEQ_KEY, + scoutfs_inode_data_seq(inode), 0, + si->item_data_seq, 0); BUG_ON(ret); store_inode(&sinode, inode); @@ -656,6 +724,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; + ci->meta_seq = 0; + ci->data_seq = 0; ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 5f453996..d95139c0 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -6,12 +6,16 @@ struct scoutfs_inode_info { /* read or initialized for each inode instance */ u64 ino; - u64 data_version; u64 next_readdir_pos; + u64 meta_seq; + u64 data_seq; + u64 data_version; bool have_item; u64 item_size; struct timespec item_ctime; struct timespec item_mtime; + u64 item_meta_seq; + u64 item_data_seq; /* initialized once for slab object */ seqcount_t seqcount; @@ -48,8 +52,12 @@ void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev); +void scoutfs_inode_set_meta_seq(struct inode *inode); +void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); -u64 scoutfs_inode_get_data_version(struct inode *inode); +u64 scoutfs_inode_meta_seq(struct inode *inode); +u64 scoutfs_inode_data_seq(struct inode *inode); +u64 scoutfs_inode_data_version(struct inode *inode); int scoutfs_scan_orphans(struct super_block *sb); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 3e0807cc..e5109853 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -30,6 +30,7 @@ #include "trans.h" #include "item.h" #include "data.h" +#include "net.h" /* * Walk one of the inode index items. This is a thin ioctl wrapper @@ -45,6 +46,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) struct scoutfs_inode_index_key ikey; struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; + u64 last_seq; int ret = 0; u32 nr; @@ -62,9 +64,28 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) ikey.type = SCOUTFS_INODE_INDEX_MTIME_KEY; else if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) ikey.type = SCOUTFS_INODE_INDEX_SIZE_KEY; + else if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) + ikey.type = SCOUTFS_INODE_INDEX_META_SEQ_KEY; + else if (walk.index == SCOUTFS_IOC_WALK_INODES_DATA_SEQ) + ikey.type = SCOUTFS_INODE_INDEX_DATA_SEQ_KEY; else return -EINVAL; + /* clamp results to the inodes in the farthest stable seq */ + if (ikey.type == SCOUTFS_INODE_INDEX_META_SEQ_KEY || + ikey.type == SCOUTFS_INODE_INDEX_DATA_SEQ_KEY) { + + ret = scoutfs_net_get_last_seq(sb, &last_seq); + if (ret) + return ret; + + if (last_seq < walk.last.major) { + walk.last.major = last_seq; + walk.last.minor = ~0; + walk.last.ino = ~0ULL; + } + } + ikey.major = cpu_to_be64(walk.first.major); ikey.minor = cpu_to_be32(walk.first.minor); ikey.ino = cpu_to_be64(walk.first.ino); @@ -218,21 +239,6 @@ out: return ret; } -/* - * Sample the inode's data_version. It is not strictly serialized with - * writes that are in flight. - */ -static long scoutfs_ioc_data_version(struct file *file, unsigned long arg) -{ - u64 __user *uvers = (void __user *)arg; - u64 vers = scoutfs_inode_get_data_version(file_inode(file)); - - if (put_user(vers, uvers)) - return -EFAULT; - - return 0; -} - /* * The caller has a version of the data available in the given byte * range in an external archive. As long as the data version still @@ -291,7 +297,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) goto out; } - if (scoutfs_inode_get_data_version(inode) != args.data_version) { + if (scoutfs_inode_data_version(inode) != args.data_version) { ret = -ESTALE; goto out; } @@ -386,7 +392,7 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) goto out; } - if (scoutfs_inode_get_data_version(inode) != args.data_version) { + if (scoutfs_inode_data_version(inode) != args.data_version) { ret = -ESTALE; goto out; } @@ -423,7 +429,9 @@ static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) stm.valid_bytes = min_t(u64, stm.valid_bytes, sizeof(struct scoutfs_ioctl_stat_more)); - stm.data_version = scoutfs_inode_get_data_version(inode); + stm.meta_seq = scoutfs_inode_meta_seq(inode); + stm.data_seq = scoutfs_inode_data_seq(inode); + stm.data_version = scoutfs_inode_data_version(inode); if (copy_to_user((void __user *)arg, &stm, stm.valid_bytes)) return -EFAULT; @@ -438,8 +446,6 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_walk_inodes(file, arg); case SCOUTFS_IOC_INO_PATH: return scoutfs_ioc_ino_path(file, arg); - case SCOUTFS_IOC_DATA_VERSION: - return scoutfs_ioc_data_version(file, arg); case SCOUTFS_IOC_RELEASE: return scoutfs_ioc_release(file, arg); case SCOUTFS_IOC_STAGE: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index a32a6724..d1814b8c 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -31,7 +31,18 @@ struct scoutfs_ioctl_walk_inodes_entry { * minor < major) as each increasingly significant value wraps around to * 0. * + * These indexes are not strictly consistent. The items that back these + * index entries aren't updated with cluster locks so they're not + * guaranteed to be visible the moment you read after writing. They're + * only visible when the transaction that updated them is synced. + * + * In addition, the seq indexes will only allow walking through sequence + * space that has been consistent. This prevents old dirty entries from + * becoming visible after newer stable entries are displayed. + * * If first is greater than last then the walk will return 0 entries. + * + * XXX invalidate before reading. */ struct scoutfs_ioctl_walk_inodes { struct scoutfs_ioctl_walk_inodes_entry first; @@ -45,6 +56,8 @@ enum { SCOUTFS_IOC_WALK_INODES_CTIME = 0, SCOUTFS_IOC_WALK_INODES_MTIME, SCOUTFS_IOC_WALK_INODES_SIZE, + SCOUTFS_IOC_WALK_INODES_META_SEQ, + SCOUTFS_IOC_WALK_INODES_DATA_SEQ, SCOUTFS_IOC_WALK_INODES_UNKNOWN, }; @@ -145,6 +158,8 @@ struct scoutfs_ioctl_stage { */ struct scoutfs_ioctl_stat_more { __u64 valid_bytes; + __u64 meta_seq; + __u64 data_seq; __u64 data_version; } __packed; diff --git a/kmod/src/net.c b/kmod/src/net.c index eb34d12b..1d2662c2 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -82,6 +82,10 @@ struct net_info { struct llist_head ring_commit_waiters; struct work_struct ring_commit_work; + /* server tracks seq use */ + spinlock_t seq_lock; + struct list_head pending_seqs; + /* both track active sockets for destruction */ struct list_head active_socks; @@ -628,6 +632,132 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, return sbuf; } +struct pending_seq { + struct list_head head; + u64 seq; +}; + +/* + * Give the client the next seq for it to use in items in its + * transaction. They tell us the seq they just used so we can remove it + * from pending tracking and possibly include it in get_last_seq + * replies. + * + * The list walk is O(clients) and the message processing rate goes from + * every committed segment to every sync deadline interval. + * + * XXX The pending seq tracking should be persistent so that it survives + * server failover. + */ +static struct send_buf *process_advance_seq(struct super_block *sb, + void *req, int req_len) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct pending_seq *next_ps; + struct pending_seq *ps; + struct commit_waiter cw; + __le64 * __packed prev; + __le64 * __packed next; + struct send_buf *sbuf; + int ret; + + if (req_len != sizeof(__le64)) + return ERR_PTR(-EINVAL); + + prev = req; + + sbuf = alloc_sbuf(sizeof(__le64)); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + next = (void *)sbuf->nh->data; + + next_ps = kmalloc(sizeof(struct pending_seq), GFP_NOFS); + if (!next_ps) { + ret = -ENOMEM; + goto out; + } + + down_read(&nti->ring_commit_rwsem); + + spin_lock(&nti->seq_lock); + + list_for_each_entry(ps, &nti->pending_seqs, head) { + if (ps->seq == le64_to_cpu(*prev)) { + list_del_init(&ps->head); + kfree(ps); + break; + } + } + + *next = super->next_seq; + le64_add_cpu(&super->next_seq, 1); + + trace_printk("prev %llu next %llu, super next_seq %llu\n", + le64_to_cpup(prev), le64_to_cpup(next), + le64_to_cpu(super->next_seq)); + + next_ps->seq = le64_to_cpup(next); + list_add_tail(&next_ps->head, &nti->pending_seqs); + + spin_unlock(&nti->seq_lock); + + queue_commit_work(nti, &cw); + up_read(&nti->ring_commit_rwsem); + + ret = wait_for_commit(&cw); +out: + if (ret < 0) + sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; + else + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + +/* + * Give the client the last seq that is stable before the lowest seq + * that is still dirty out at a client. + */ +static struct send_buf *process_get_last_seq(struct super_block *sb, + void *req, int req_len) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct pending_seq *ps; + __le64 * __packed last; + struct send_buf *sbuf; + + if (req_len != 0) + return ERR_PTR(-EINVAL); + + sbuf = alloc_sbuf(sizeof(__le64)); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + last = (void *)sbuf->nh->data; + + spin_lock(&nti->seq_lock); + ps = list_first_entry_or_null(&nti->pending_seqs, + struct pending_seq, head); + if (ps) { + *last = cpu_to_le64(ps->seq - 1); + } else { + *last = super->next_seq; + le64_add_cpu(last, -1ULL); + } + spin_unlock(&nti->seq_lock); + + trace_printk("last %llu\n", le64_to_cpup(last)); + + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + typedef struct send_buf *(*proc_func_t)(struct super_block *sb, void *req, int req_len); @@ -640,6 +770,8 @@ static proc_func_t type_proc_func(u8 type) [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, + [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, + [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, }; return type < SCOUTFS_NET_UNKNOWN ? funcs[type] : NULL; @@ -726,9 +858,19 @@ static int process_reply(struct net_info *nti, struct recv_buf *rbuf) static void destroy_server_state(struct super_block *sb) { + DECLARE_NET_INFO(sb, nti); + struct pending_seq *ps; + struct pending_seq *tmp; + scoutfs_compact_destroy(sb); scoutfs_alloc_destroy(sb); scoutfs_manifest_destroy(sb); + + /* XXX these should be persistent and reclaimed during recovery */ + list_for_each_entry_safe(ps, tmp, &nti->pending_seqs, head) { + list_del_init(&ps->head); + kfree(ps); + } } /* @@ -1550,6 +1692,87 @@ int scoutfs_net_alloc_inodes(struct super_block *sb) alloc_inodes_reply, NULL); } +struct advance_seq_args { + u64 seq; + struct completion comp; + int ret; +}; + +static int advance_seq_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct advance_seq_args *args = arg; + __le64 * __packed seq = reply; + + if (ret == sizeof(__le64)) { + args->seq = le64_to_cpup(seq); + args->ret = 0; + } else { + args->ret = -EINVAL; + } + + complete(&args->comp); /* args can be freed from this point */ + return args->ret; +} + +int scoutfs_net_advance_seq(struct super_block *sb, u64 *seq) +{ + struct advance_seq_args args; + __le64 leseq = cpu_to_le64p(seq); + int ret; + + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_ADVANCE_SEQ, &leseq, + sizeof(leseq), advance_seq_reply, &args); + if (ret == 0) { + wait_for_completion(&args.comp); + *seq = args.seq; + ret = args.ret; + } + return ret; +} + +struct get_last_seq_args { + u64 seq; + struct completion comp; + int ret; +}; + +static int get_last_seq_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct get_last_seq_args *args = arg; + __le64 * __packed seq = reply; + + if (ret == sizeof(__le64)) { + args->seq = le64_to_cpup(seq); + args->ret = 0; + } else { + args->ret = -EINVAL; + } + + complete(&args->comp); /* args can be freed from this point */ + return args->ret; +} + +int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq) +{ + struct get_last_seq_args args; + int ret; + + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_GET_LAST_SEQ, NULL, 0, + get_last_seq_reply, &args); + if (ret == 0) { + wait_for_completion(&args.comp); + *seq = args.seq; + ret = args.ret; + } + return ret; +} + static struct sock_info *alloc_sinf(struct super_block *sb) { struct sock_info *sinf; @@ -1862,6 +2085,8 @@ int scoutfs_net_setup(struct super_block *sb) init_rwsem(&nti->ring_commit_rwsem); init_llist_head(&nti->ring_commit_waiters); INIT_WORK(&nti->ring_commit_work, scoutfs_net_ring_commit_func); + spin_lock_init(&nti->seq_lock); + INIT_LIST_HEAD(&nti->pending_seqs); INIT_LIST_HEAD(&nti->active_socks); sbi->net_info = nti; diff --git a/kmod/src/net.h b/kmod/src/net.h index e51d9266..ea131144 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -17,6 +17,8 @@ u64 *scoutfs_net_bulk_alloc(struct super_block *sb); int scoutfs_net_get_compaction(struct super_block *sb, void *curs); int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, void *list); +int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq); +int scoutfs_net_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index f89bad20..aa2870dc 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -242,6 +242,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sb->s_root) return -ENOMEM; + ret = scoutfs_net_advance_seq(sb, &sbi->trans_seq); + if (ret) + return ret; + scoutfs_trans_restart_sync_deadline(sb); // scoutfs_scan_orphans(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 8bbd313b..39cc354a 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -39,10 +39,12 @@ struct scoutfs_sb_info { spinlock_t trans_write_lock; u64 trans_write_count; + u64 trans_seq; int trans_write_ret; struct delayed_work trans_write_work; wait_queue_head_t trans_write_wq; struct workqueue_struct *trans_write_workq; + bool trans_deadline_expired; struct lock_info *lock_info; struct net_info *net_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 58c12465..c55da90b 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -113,7 +113,18 @@ void scoutfs_trans_write_func(struct work_struct *work) goto out; scoutfs_inc_counter(sb, trans_level0_seg_write); + + } else if (sbi->trans_deadline_expired) { + /* + * If we're not writing data then we only advance the + * seq at the sync deadline interval. This keeps idle + * mounts from pinning a seq and stopping readers of the + * seq indices but doesn't send a message for every sync + * syscall. + */ + ret = scoutfs_net_advance_seq(sb, &sbi->trans_seq); } + out: /* XXX this all needs serious work for dealing with errors */ WARN_ON_ONCE(ret); @@ -160,6 +171,7 @@ static int write_attempted(struct scoutfs_sb_info *sbi, */ static void queue_trans_work(struct scoutfs_sb_info *sbi) { + sbi->trans_deadline_expired = false; mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, 0); } @@ -208,6 +220,7 @@ void scoutfs_trans_restart_sync_deadline(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + sbi->trans_deadline_expired = true; mod_delayed_work(sbi->trans_write_workq, &sbi->trans_write_work, TRANS_SYNC_DELAY); } From 297b859577276220e92deb030c6fe14a0c24d95f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 May 2017 11:45:56 -0700 Subject: [PATCH 279/920] scoutfs: deletion items maintain counts When we turned existing items into deletion items we'd remove their values. But we didn't update the count of dirty values to reflect that removal so the dirty value count would slowly grow without bound. Signed-off-by: Zach Brown --- kmod/src/item.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 071942e8..4acf1abd 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -354,6 +354,7 @@ static void become_deletion_item(struct super_block *sb, struct cached_item *item, struct kvec *del_val) { + clear_item_dirty(cac, item); scoutfs_kvec_clone(del_val, item->val); scoutfs_kvec_init_null(item->val); item->deletion = 1; From b7bbad1fba67771358ed401a61cce4ded308bd5b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 May 2017 11:47:39 -0700 Subject: [PATCH 280/920] scoutfs: add precise transation item reservations We had a simple mechanism for ensuring that transaction didn't create more items than would fit in a single written segment. We calculated the most dirty items that a holder could generate and assumed that all holders dirtied that much. This had two big problems. The first was that it wasn't accounting for nested holds. write_begin/end calls the generic inode dirtying path whild holding a transaction. This ended up deadlocking as the dirty inode waited to be able to write while its trans held back in write_begin prevented writeout. The second was that the worst case (full size xattr) item dirtying is enormous and meaningfully restricts concurrent transaction holders. With no currently dirty items you can have less than 16 full size xattr writes. This concurrency limit only gets worse as the transaction fills up with dirty items. This fixes those problems. It adds precise accounting of the dirty items that can be created while a transaction is held. These reservations are tracked in journal_info so that they can be used by nested holds. The precision allows much greater concurrency as something like a create will try to reserve a few hundreds bytes instead of 64k. Normal sized xattr operations won't try to reserve the largest possible space. We add some feedback from the item cache to the transaction to issue warnings if a holder dirties more items than it reserved. Now that we have precise item/key/value counts (segment space consumption is a function of all three :/) we can't have a single atomic track transaction holders. We add a long-overdue trans_info and put a proper lock and fields there and much more clearly track transaction serialization amongst the holders and writer. Signed-off-by: Zach Brown --- kmod/src/count.h | 169 ++++++++++++++++++++++++++++ kmod/src/data.c | 22 +++- kmod/src/dir.c | 16 ++- kmod/src/format.h | 10 +- kmod/src/inode.c | 9 +- kmod/src/ioctl.c | 6 - kmod/src/item.c | 34 +++--- kmod/src/net.c | 9 +- kmod/src/super.c | 1 - kmod/src/super.h | 3 +- kmod/src/trans.c | 273 ++++++++++++++++++++++++++++++++++------------ kmod/src/trans.h | 7 +- kmod/src/xattr.c | 4 +- 13 files changed, 452 insertions(+), 111 deletions(-) create mode 100644 kmod/src/count.h diff --git a/kmod/src/count.h b/kmod/src/count.h new file mode 100644 index 00000000..921dea89 --- /dev/null +++ b/kmod/src/count.h @@ -0,0 +1,169 @@ +#ifndef _SCOUTFS_COUNT_H_ +#define _SCOUTFS_COUNT_H_ + +struct scoutfs_item_count { + signed items; + signed keys; + signed vals; +}; + +#define DECLARE_ITEM_COUNT(name) \ + struct scoutfs_item_count name = { 0, } + +/* + * Allocating an inode creates a new set of indexed items. + */ +static inline void scoutfs_count_alloc_inode(struct scoutfs_item_count *cnt) +{ + const int nr_indices = SCOUTFS_INODE_INDEX_NR; + + cnt->items += 1 + nr_indices; + cnt->keys += sizeof(struct scoutfs_inode_key) + + (nr_indices * sizeof(struct scoutfs_inode_index_key)); + cnt->vals += sizeof(struct scoutfs_inode); +} + +/* + * Dirtying an inode dirties the inode item and can delete and create + * the full set of indexed items. + */ +static inline void scoutfs_count_dirty_inode(struct scoutfs_item_count *cnt) +{ + const int nr_indices = 2 * SCOUTFS_INODE_INDEX_NR; + + cnt->items += 1 + nr_indices; + cnt->keys += sizeof(struct scoutfs_inode_key) + + (nr_indices * sizeof(struct scoutfs_inode_index_key)); + cnt->vals += sizeof(struct scoutfs_inode); +} + +/* + * Adding a dirent adds the entry key, readdir key, and backref. + */ +static inline void scoutfs_count_dirents(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + + cnt->items += 3; + cnt->keys += offsetof(struct scoutfs_dirent_key, name[name_len]) + + sizeof(struct scoutfs_readdir_key) + + offsetof(struct scoutfs_link_backref_key, name[name_len]); + cnt->vals += 2 * offsetof(struct scoutfs_dirent, name[name_len]); +} + +static inline void scoutfs_count_sym_target(struct scoutfs_item_count *cnt, + unsigned size) +{ + + cnt->items += 1; + cnt->keys += sizeof(struct scoutfs_symlink_key); + cnt->vals += size; +} + +static inline void scoutfs_count_orphan(struct scoutfs_item_count *cnt) +{ + + cnt->items += 1; + cnt->keys += sizeof(struct scoutfs_orphan_key); +} + +static inline void scoutfs_count_mknod(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + scoutfs_count_alloc_inode(cnt); + scoutfs_count_dirents(cnt, name_len); + scoutfs_count_dirty_inode(cnt); +} + +static inline void scoutfs_count_link(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + scoutfs_count_dirents(cnt, name_len); + scoutfs_count_dirty_inode(cnt); + scoutfs_count_dirty_inode(cnt); +} + +/* + * Unlink can add orphan items. + */ +static inline void scoutfs_count_unlink(struct scoutfs_item_count *cnt, + unsigned name_len) +{ + scoutfs_count_dirents(cnt, name_len); + scoutfs_count_dirty_inode(cnt); + scoutfs_count_dirty_inode(cnt); + scoutfs_count_orphan(cnt); +} + +static inline void scoutfs_count_symlink(struct scoutfs_item_count *cnt, + unsigned name_len, unsigned size) +{ + scoutfs_count_mknod(cnt, name_len); + scoutfs_count_sym_target(cnt, size); +} + +/* + * Setting an xattr can create a full set of items for an xattr with a + * max name and length. Any existing items will be dirtied rather than + * deleted so we won't have more items than a max xattr's worth. + */ +static inline void scoutfs_count_xattr_set(struct scoutfs_item_count *cnt, + unsigned name_len, unsigned size) +{ + unsigned parts = DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE); + + scoutfs_count_dirty_inode(cnt); + + cnt->items += parts; + cnt->keys += parts * (offsetof(struct scoutfs_xattr_key, + name[name_len]) + + sizeof(struct scoutfs_xattr_key_footer)); + cnt->vals += parts * (sizeof(struct scoutfs_xattr_val_header) + + SCOUTFS_XATTR_PART_SIZE); +} + +/* + * Both insertion and removal modifications can dirty three extents + * at most: insertion can delete two existing neighbours and create a + * third new extent and removal can delete an existing extent and create + * two new remaining extents. + */ +static inline void scoutfs_count_extents(struct scoutfs_item_count *cnt, + unsigned nr_mod, unsigned sz) +{ + + cnt->items += nr_mod * 3; + cnt->keys += (nr_mod * 3) * sz; +} + +/* + * write_begin can refill local free extents after a bulk alloc rpc, + * alloc an block, delete an offline mapping, and insert the new allocated + * mapping. + */ +static inline void scoutfs_count_write_begin(struct scoutfs_item_count *cnt) +{ + BUILD_BUG_ON(sizeof(struct scoutfs_free_extent_blkno_key) != + sizeof(struct scoutfs_free_extent_blocks_key)); + + scoutfs_count_dirty_inode(cnt); + + scoutfs_count_extents(cnt, 2 * (SCOUTFS_BULK_ALLOC_COUNT + 1), + sizeof(struct scoutfs_free_extent_blkno_key)); + scoutfs_count_extents(cnt, 2, + sizeof(struct scoutfs_file_extent_key)); +} + +/* + * Truncating a block can free an allocated block, delete an online + * mapping, and create an offline mapping. + */ +static inline void scoutfs_count_trunc_block(struct scoutfs_item_count *cnt) +{ + scoutfs_count_extents(cnt, 2 * 1, + sizeof(struct scoutfs_free_extent_blkno_key)); + scoutfs_count_extents(cnt, 2, + sizeof(struct scoutfs_file_extent_key)); +} + +#endif diff --git a/kmod/src/data.c b/kmod/src/data.c index 76cdd2a5..77dc5f98 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -510,8 +510,9 @@ out: * If 'offline' is given then blocks are freed but the extent items are * left behind and their _OFFLINE flag is set. * - * This is the low level extent item manipulation code. Callers manage - * higher order locking and transactional consistency. + * This is the low level extent item manipulation code. We hold and + * release the transaction so the caller doesn't have to deal with + * partial progress. */ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline) @@ -526,8 +527,10 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, struct native_extent ext; struct native_extent ofl; struct native_extent fr; + DECLARE_ITEM_COUNT(cnt); bool rem_fr = false; bool ins_ext = false; + bool holding = false; int ret = 0; int err; @@ -588,6 +591,12 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, if (offline && (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) continue; + scoutfs_count_trunc_block(&cnt); + ret = scoutfs_hold_trans(sb, &cnt); + if (ret) + break; + holding = true; + /* free the old extent if it was allocated */ if (ext.blkno) { fr = ext; @@ -618,8 +627,13 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, rem_fr = false; ins_ext = false; + scoutfs_release_trans(sb); + holding = false; } + if (holding) + scoutfs_release_trans(sb); + if (ret) { if (ins_ext) { err = insert_extent(sb, &ext, ino, @@ -1034,12 +1048,14 @@ static int scoutfs_write_begin(struct file *file, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; + DECLARE_ITEM_COUNT(cnt); int ret; trace_printk("ino %llu pos %llu len %u\n", scoutfs_ino(inode), (u64)pos, len); - ret = scoutfs_hold_trans(sb); + scoutfs_count_write_begin(&cnt); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index ce579e8c..0d77d95a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -455,6 +455,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct super_block *sb = dir->i_sb; + DECLARE_ITEM_COUNT(cnt); struct inode *inode; int ret; @@ -462,7 +463,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) return ret; - ret = scoutfs_hold_trans(sb); + scoutfs_count_mknod(&cnt, dentry->d_name.len); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) return ret; @@ -515,6 +517,7 @@ static int scoutfs_link(struct dentry *old_dentry, { struct inode *inode = old_dentry->d_inode; struct super_block *sb = dir->i_sb; + DECLARE_ITEM_COUNT(cnt); int ret; if (inode->i_nlink >= SCOUTFS_LINK_MAX) @@ -524,7 +527,8 @@ static int scoutfs_link(struct dentry *old_dentry, if (ret) return ret; - ret = scoutfs_hold_trans(sb); + scoutfs_count_link(&cnt, dentry->d_name.len); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) return ret; @@ -559,12 +563,14 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct scoutfs_key_buf *keys[3] = {NULL,}; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; + DECLARE_ITEM_COUNT(cnt); int ret = 0; if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - ret = scoutfs_hold_trans(sb); + scoutfs_count_unlink(&cnt, dentry->d_name.len); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) return ret; @@ -718,6 +724,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct scoutfs_key_buf key; struct inode *inode = NULL; SCOUTFS_DECLARE_KVEC(val); + DECLARE_ITEM_COUNT(cnt); int ret; /* path_max includes null as does our value for nd_set_link */ @@ -728,7 +735,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - ret = scoutfs_hold_trans(sb); + scoutfs_count_symlink(&cnt, dentry->d_name.len, name_len); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) return ret; diff --git a/kmod/src/format.h b/kmod/src/format.h index dd991c2d..58adf6ba 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -160,7 +160,7 @@ struct scoutfs_segment_block { #define SCOUTFS_ORPHAN_KEY 10 #define SCOUTFS_FREE_EXTENT_BLKNO_KEY 11 #define SCOUTFS_FREE_EXTENT_BLOCKS_KEY 12 -#define SCOUTFS_INODE_INDEX_CTIME_KEY 13 +#define SCOUTFS_INODE_INDEX_CTIME_KEY 13 /* don't forget first and last */ #define SCOUTFS_INODE_INDEX_MTIME_KEY 14 #define SCOUTFS_INODE_INDEX_SIZE_KEY 15 #define SCOUTFS_INODE_INDEX_META_SEQ_KEY 16 @@ -170,6 +170,11 @@ struct scoutfs_segment_block { #define SCOUTFS_NET_ADDR_KEY 254 #define SCOUTFS_NET_LISTEN_KEY 255 +#define SCOUTFS_INODE_INDEX_FIRST SCOUTFS_INODE_INDEX_CTIME_KEY +#define SCOUTFS_INODE_INDEX_LAST SCOUTFS_INODE_INDEX_DATA_SEQ_KEY +#define SCOUTFS_INODE_INDEX_NR \ + (SCOUTFS_INODE_INDEX_LAST - SCOUTFS_INODE_INDEX_FIRST + 1) + /* value is struct scoutfs_inode */ struct scoutfs_inode_key { __u8 type; @@ -433,6 +438,9 @@ struct scoutfs_net_manifest_entries { struct scoutfs_manifest_entry ments[0]; } __packed; +/* XXX I dunno, totally made up */ +#define SCOUTFS_BULK_ALLOC_COUNT 32 + struct scoutfs_net_segnos { __le16 nr; __le64 segnos[0]; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index c1a65eef..b5774682 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -574,9 +574,11 @@ void scoutfs_update_inode_item(struct inode *inode) void scoutfs_dirty_inode(struct inode *inode, int flags) { struct super_block *sb = inode->i_sb; + DECLARE_ITEM_COUNT(cnt); int ret; - ret = scoutfs_hold_trans(sb); + scoutfs_count_dirty_inode(&cnt); + ret = scoutfs_hold_trans(sb, &cnt); if (ret == 0) { ret = scoutfs_dirty_inode_item(inode); if (ret == 0) @@ -777,12 +779,15 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, u64 ino, umode_t mode) { + DECLARE_ITEM_COUNT(cnt); bool release = false; int ret; trace_delete_inode(sb, ino, mode); - ret = scoutfs_hold_trans(sb); + /* XXX this is obviously not done yet :) */ + scoutfs_count_dirty_inode(&cnt); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) goto out; release = true; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index e5109853..e4cdeb7d 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -27,7 +27,6 @@ #include "ioctl.h" #include "super.h" #include "inode.h" -#include "trans.h" #include "item.h" #include "data.h" #include "net.h" @@ -307,13 +306,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) /* drop all clean and dirty cached blocks in the range */ truncate_inode_pages_range(&inode->i_data, start, end_inc); - ret = scoutfs_hold_trans(sb); - if (ret) - goto out; - ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), iblock, len, true); - scoutfs_release_trans(sb); out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); diff --git a/kmod/src/item.c b/kmod/src/item.c index 4acf1abd..ba213f8c 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -293,7 +293,7 @@ static void update_dirty_parents(struct cached_item *item) scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); } -static void mark_item_dirty(struct item_cache *cac, +static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, struct cached_item *item) { if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) @@ -307,10 +307,13 @@ static void mark_item_dirty(struct item_cache *cac, cac->dirty_key_bytes += item->key->key_len; cac->dirty_val_bytes += scoutfs_kvec_length(item->val); + scoutfs_trans_track_item(sb, 1, item->key->key_len, + scoutfs_kvec_length(item->val)); + update_dirty_parents(item); } -static void clear_item_dirty(struct item_cache *cac, +static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, struct cached_item *item) { if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) @@ -324,6 +327,9 @@ static void clear_item_dirty(struct item_cache *cac, cac->dirty_key_bytes -= item->key->key_len; cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); + scoutfs_trans_track_item(sb, -1, -item->key->key_len, + -scoutfs_kvec_length(item->val)); + WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || cac->dirty_val_bytes < 0); @@ -339,7 +345,7 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, { trace_printk("erasing item %p\n", item); - clear_item_dirty(cac, item); + clear_item_dirty(sb, cac, item); rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); free_item(sb, item); } @@ -354,11 +360,11 @@ static void become_deletion_item(struct super_block *sb, struct cached_item *item, struct kvec *del_val) { - clear_item_dirty(cac, item); + clear_item_dirty(sb, cac, item); scoutfs_kvec_clone(del_val, item->val); scoutfs_kvec_init_null(item->val); item->deletion = 1; - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); scoutfs_inc_counter(sb, item_delete); } @@ -905,7 +911,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, ret = insert_item(sb, cac, item, false); if (!ret) { scoutfs_inc_counter(sb, item_create); - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); } spin_unlock_irqrestore(&cac->lock, flags); @@ -950,7 +956,7 @@ int scoutfs_item_create_ephemeral(struct super_block *sb, BUG_ON(ret); scoutfs_inc_counter(sb, item_create_ephemeral); - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); @@ -975,9 +981,9 @@ void scoutfs_item_update_ephemeral(struct super_block *sb, if (item && item->ephemeral) { trace_printk("updating ephemeral item %p\n", item); scoutfs_inc_counter(sb, item_update_ephemeral); - clear_item_dirty(cac, item); + clear_item_dirty(sb, cac, item); scoutfs_kvec_clone(item->val, val); - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); } spin_unlock_irqrestore(&cac->lock, flags); @@ -1173,7 +1179,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); insert_item(sb, cac, item, true); - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); } ret = 0; @@ -1220,7 +1226,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key) item = find_item(sb, &cac->items, key); if (item) { - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); ret = 0; } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; @@ -1275,9 +1281,9 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, item = find_item(sb, &cac->items, key); if (item) { - clear_item_dirty(cac, item); + clear_item_dirty(sb, cac, item); scoutfs_kvec_swap(up_val, item->val); - mark_item_dirty(cac, item); + mark_item_dirty(sb, cac, item); ret = 0; } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; @@ -1612,7 +1618,7 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) key_bytes -= item->key->key_len; - clear_item_dirty(cac, item); + clear_item_dirty(sb, cac, item); del = item; item = next_dirty(item); diff --git a/kmod/src/net.c b/kmod/src/net.c index 1d2662c2..d7b16b8a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -368,9 +368,6 @@ static struct send_buf *alloc_sbuf(unsigned data_len) return sbuf; } -/* XXX I dunno, totally made up */ -#define BULK_COUNT 32 - static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, int req_len) { @@ -386,16 +383,16 @@ static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, return ERR_PTR(-EINVAL); sbuf = alloc_sbuf(offsetof(struct scoutfs_net_segnos, - segnos[BULK_COUNT])); + segnos[SCOUTFS_BULK_ALLOC_COUNT])); if (!sbuf) return ERR_PTR(-ENOMEM); ns = (void *)sbuf->nh->data; - ns->nr = cpu_to_le16(BULK_COUNT); + ns->nr = cpu_to_le16(SCOUTFS_BULK_ALLOC_COUNT); down_read(&nti->ring_commit_rwsem); - for (i = 0; i < BULK_COUNT; i++) { + for (i = 0; i < SCOUTFS_BULK_ALLOC_COUNT; i++) { ret = scoutfs_alloc_segno(sb, &segno); if (ret) { while (i-- > 0) diff --git a/kmod/src/super.c b/kmod/src/super.c index aa2870dc..90367575 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -211,7 +211,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) get_random_bytes_arch(&sbi->node_id, sizeof(sbi->node_id)); spin_lock_init(&sbi->next_ino_lock); - atomic_set(&sbi->trans_holds, 0); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); diff --git a/kmod/src/super.h b/kmod/src/super.h index 39cc354a..350dbca9 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -12,6 +12,7 @@ struct manifest; struct segment_cache; struct compact_info; struct data_info; +struct trans_info; struct lock_info; struct net_info; struct inode_sb_info; @@ -33,7 +34,6 @@ struct scoutfs_sb_info { struct data_info *data_info; struct inode_sb_info *inode_sb_info; - atomic_t trans_holds; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; @@ -46,6 +46,7 @@ struct scoutfs_sb_info { struct workqueue_struct *trans_write_workq; bool trans_deadline_expired; + struct trans_info *trans_info; struct lock_info *lock_info; struct net_info *net_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index c55da90b..4c41de88 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "super.h" #include "trans.h" @@ -53,6 +54,33 @@ /* sync dirty data at least this often */ #define TRANS_SYNC_DELAY (HZ * 10) +/* + * XXX move the rest of the super trans_ fields here. + */ +struct trans_info { + spinlock_t lock; + unsigned reserved_items; + unsigned reserved_keys; + unsigned reserved_vals; + unsigned holders; + bool writing; +}; + +#define DECLARE_TRANS_INFO(sb, name) \ + struct trans_info *name = SCOUTFS_SB(sb)->trans_info + +static bool drained_holders(struct trans_info *tri) +{ + bool drained; + + spin_lock(&tri->lock); + tri->writing = true; + drained = tri->holders == 0; + spin_unlock(&tri->lock); + + return drained; +} + /* * This work func is responsible for writing out all the dirty blocks * that make up the current dirty transaction. It prevents writers from @@ -82,6 +110,7 @@ void scoutfs_trans_write_func(struct work_struct *work) struct scoutfs_sb_info *sbi = container_of(work, struct scoutfs_sb_info, trans_write_work.work); struct super_block *sb = sbi->sb; + DECLARE_TRANS_INFO(sb, tri); struct scoutfs_bio_completion comp; struct scoutfs_segment *seg; u64 segno; @@ -90,8 +119,7 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_bio_init_comp(&comp); sbi->trans_task = current; - wait_event(sbi->trans_hold_wq, - atomic_cmpxchg(&sbi->trans_holds, 0, -1) == 0); + wait_event(sbi->trans_hold_wq, drained_holders(tri)); trace_printk("items dirty %d\n", scoutfs_item_has_dirty(sb)); @@ -108,7 +136,8 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_seg_submit_write(sb, seg, &comp) ?: scoutfs_inode_walk_writeback(sb, false) ?: scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_net_record_segment(sb, seg, 0); + scoutfs_net_record_segment(sb, seg, 0) ?: + scoutfs_net_advance_seq(sb, &sbi->trans_seq); if (ret) goto out; @@ -135,7 +164,10 @@ out: spin_unlock(&sbi->trans_write_lock); wake_up(&sbi->trans_write_wq); - atomic_set(&sbi->trans_holds, 0); + spin_lock(&tri->lock); + tri->writing = false; + spin_unlock(&tri->lock); + wake_up(&sbi->trans_hold_wq); sbi->trans_task = NULL; @@ -226,99 +258,184 @@ void scoutfs_trans_restart_sync_deadline(struct super_block *sb) } /* - * The holder that creates the most dirty item data is adding a full - * size xattr. The largest xattr can have a 255 byte name and 64KB - * value. - * - * XXX Assuming the worst case here too aggressively limits the number - * of concurrent holders that can work without being blocked when they - * know they'll dirty much less. We may want to have callers pass in - * their item, key, and val budgets if that's not too fragile. + * Each thread reserves space in the segment for their dirty items while + * they hold the transaction. This is calculated before the first + * transaction hold is acquired. It includes all the potential nested + * item manipulation that could happen with the transaction held. + * Including nested holds avoids having to deal with writing out partial + * transactions while a caller still holds the transaction. */ -#define HOLD_WORST_ITEMS \ - SCOUTFS_XATTR_MAX_PARTS - -#define HOLD_WORST_KEYS \ - (SCOUTFS_XATTR_MAX_PARTS * \ - (sizeof(struct scoutfs_xattr_key) + \ - SCOUTFS_XATTR_MAX_NAME_LEN + \ - sizeof(struct scoutfs_xattr_key_footer))) - -#define HOLD_WORST_VALS \ - (sizeof(struct scoutfs_xattr_val_header) + \ - SCOUTFS_XATTR_MAX_SIZE) +#define SCOUTFS_RESERVATION_MAGIC 0xd57cd13b +struct scoutfs_reservation { + unsigned magic; + unsigned holders; + struct scoutfs_item_count reserved; + struct scoutfs_item_count actual; +}; /* - * We're able to hold the transaction if the current dirty item bytes - * and the presumed worst case item dirtying of all the holders, - * including us, all fit in a segment. + * Try to hold the transaction. If a caller already holds the trans then + * we piggy back on their hold. We wait if the writer is trying to + * write out the transation. And if our items won't fit then we kick off + * a write. */ -static bool hold_acquired(struct super_block *sb) +static bool acquired_hold(struct super_block *sb, + struct scoutfs_reservation *rsv, + struct scoutfs_item_count *cnt) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int with_us; - int holds; - int before; - u32 items; - u32 keys; - u32 vals; + DECLARE_TRANS_INFO(sb, tri); + bool acquired = false; + unsigned items; + unsigned keys; + unsigned vals; + bool fits; - holds = atomic_read(&sbi->trans_holds); - for (;;) { - /* transaction is being committed */ - if (holds < 0) - return false; + spin_lock(&tri->lock); -#if 0 /* XXX where will we do this in the shared universe? */ - /* only hold when there's no level 0 segments, XXX for now */ - if (scoutfs_manifest_level_count(sb, 0) > 0) { - scoutfs_compact_kick(sb); - return false; - } -#endif + trace_printk("cnt %u.%u.%u, rsv %p holders %u reserved %u.%u.%u actual %d.%d.%d, trans holders %u writing %u reserved %u.%u.%u\n", + cnt->items, cnt->keys, cnt->vals, rsv, rsv->holders, + rsv->reserved.items, rsv->reserved.keys, + rsv->reserved.vals, rsv->actual.items, rsv->actual.keys, + rsv->actual.vals, tri->holders, tri->writing, + tri->reserved_items, tri->reserved_keys, + tri->reserved_vals); - /* see if we all would fill the segment */ - with_us = holds + 1; - items = with_us * HOLD_WORST_ITEMS; - keys = with_us * HOLD_WORST_KEYS; - vals = with_us * HOLD_WORST_VALS; - if (!scoutfs_item_dirty_fits_single(sb, items, keys, vals)) { - scoutfs_sync_fs(sb, 0); - return false; - } + /* use a caller's existing reservation */ + if (rsv->holders) + goto hold; - before = atomic_cmpxchg(&sbi->trans_holds, holds, with_us); - if (before == holds) - return true; - holds = before; + /* wait until the writing thread is finished */ + if (tri->writing) + goto out; + + /* see if we can reserve space for our item count */ + items = tri->reserved_items + cnt->items; + keys = tri->reserved_keys + cnt->keys; + vals = tri->reserved_vals + cnt->vals; + fits = scoutfs_item_dirty_fits_single(sb, items, keys, vals); + if (!fits) { + queue_trans_work(sbi); + goto out; } + + tri->reserved_items = items; + tri->reserved_keys = keys; + tri->reserved_vals = vals; + + rsv->reserved.items = cnt->items; + rsv->reserved.keys = cnt->keys; + rsv->reserved.vals = cnt->vals; + +hold: + rsv->holders++; + tri->holders++; + acquired = true; + +out: + + spin_unlock(&tri->lock); + + return acquired; } -int scoutfs_hold_trans(struct super_block *sb) +int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv; + int ret; if (current == sbi->trans_task) return 0; - return wait_event_interruptible(sbi->trans_hold_wq, hold_acquired(sb)); + rsv = current->journal_info; + if (rsv == NULL) { + rsv = kzalloc(sizeof(struct scoutfs_reservation), GFP_NOFS); + if (!rsv) + return -ENOMEM; + + rsv->magic = SCOUTFS_RESERVATION_MAGIC; + current->journal_info = rsv; + } + + BUG_ON(rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + ret = wait_event_interruptible(sbi->trans_hold_wq, + acquired_hold(sb, rsv, cnt)); + if (ret && rsv->holders == 0) { + current->journal_info = NULL; + kfree(rsv); + } + return ret; } -/* - * As we release we'll almost certainly have dirtied less than the - * worst case dirty assumption that holders might be throttled waiting - * for. We always try and wake blocked holders in case they now have - * room to dirty. - */ -void scoutfs_release_trans(struct super_block *sb) +void scoutfs_trans_track_item(struct super_block *sb, signed items, + signed keys, signed vals) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv = current->journal_info; if (current == sbi->trans_task) return; - atomic_dec(&sbi->trans_holds); - wake_up(&sbi->trans_hold_wq); + BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + rsv->actual.items += items; + rsv->actual.keys += keys; + rsv->actual.vals += vals; + + WARN_ON_ONCE(rsv->actual.items > rsv->reserved.items); + WARN_ON_ONCE(rsv->actual.keys > rsv->reserved.keys); + WARN_ON_ONCE(rsv->actual.vals > rsv->reserved.vals); +} + +/* + * As we drop the last hold in the reservation we try and wake other + * hold attempts that were waiting for space. As we drop the last trans + * holder we try to wake a writing thread that was waiting for us to + * finish. + */ +void scoutfs_release_trans(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_reservation *rsv; + DECLARE_TRANS_INFO(sb, tri); + bool wake = false; + + if (current == sbi->trans_task) + return; + + rsv = current->journal_info; + BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC); + + spin_lock(&tri->lock); + + trace_printk("rsv %p holders %u reserved %u.%u.%u actual %d.%d.%d, trans holders %u writing %u reserved %u.%u.%u\n", + rsv, rsv->holders, rsv->reserved.items, + rsv->reserved.keys, rsv->reserved.vals, + rsv->actual.items, rsv->actual.keys, rsv->actual.vals, + tri->holders, tri->writing, tri->reserved_items, + tri->reserved_keys, tri->reserved_vals); + + BUG_ON(rsv->holders <= 0); + BUG_ON(tri->holders <= 0); + + if (--rsv->holders == 0) { + tri->reserved_items -= rsv->reserved.items; + tri->reserved_keys -= rsv->reserved.keys; + tri->reserved_vals -= rsv->reserved.vals; + current->journal_info = NULL; + kfree(rsv); + wake = true; + } + + if (--tri->holders == 0) + wake = true; + + spin_unlock(&tri->lock); + + if (wake) + wake_up(&sbi->trans_hold_wq); } /* @@ -336,10 +453,21 @@ void scoutfs_trans_wake_holders(struct super_block *sb) int scoutfs_setup_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct trans_info *tri; + + tri = kzalloc(sizeof(struct trans_info), GFP_KERNEL); + if (!tri) + return -ENOMEM; + + spin_lock_init(&tri->lock); sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", 0, 1); - if (!sbi->trans_write_workq) + if (!sbi->trans_write_workq) { + kfree(tri); return -ENOMEM; + } + + sbi->trans_info = tri; return 0; } @@ -351,9 +479,12 @@ int scoutfs_setup_trans(struct super_block *sb) void scoutfs_shutdown_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_TRANS_INFO(sb, tri); if (sbi->trans_write_workq) { cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); } + + kfree(tri); } diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 396ad6be..6f52553e 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -1,15 +1,20 @@ #ifndef _SCOUTFS_TRANS_H_ #define _SCOUTFS_TRANS_H_ +#include "net.h" +#include "count.h" + void scoutfs_trans_write_func(struct work_struct *work); int scoutfs_sync_fs(struct super_block *sb, int wait); int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); void scoutfs_trans_restart_sync_deadline(struct super_block *sb); -int scoutfs_hold_trans(struct super_block *sb); +int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt); void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_wake_holders(struct super_block *sb); +void scoutfs_trans_track_item(struct super_block *sb, signed items, + signed keys, signed vals); int scoutfs_setup_trans(struct super_block *sb); void scoutfs_shutdown_trans(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index a427763d..2bd066d4 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -262,6 +262,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); + DECLARE_ITEM_COUNT(cnt); struct scoutfs_lock lck; unsigned int bytes; unsigned int off; @@ -314,7 +315,8 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, else sif = 0; - ret = scoutfs_hold_trans(sb); + scoutfs_count_xattr_set(&cnt, name_len, size); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) goto unlock; From 1f933016f09aa35419b616a1dae1c327c6053820 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 May 2017 12:58:09 -0700 Subject: [PATCH 281/920] scoutfs: remove ephemeral items Ephemeral items were only used by the page cache which tracked page contents in items whose values pointed to the pages. Remove their special case. Signed-off-by: Zach Brown --- kmod/src/counters.h | 2 -- kmod/src/item.c | 76 ++------------------------------------------- kmod/src/item.h | 6 ---- 3 files changed, 3 insertions(+), 81 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index c9d081a6..3e35a1be 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -29,8 +29,6 @@ EXPAND_COUNTER(data_writepage) \ EXPAND_COUNTER(data_end_writeback_page) \ EXPAND_COUNTER(item_create) \ - EXPAND_COUNTER(item_create_ephemeral) \ - EXPAND_COUNTER(item_update_ephemeral) \ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ EXPAND_COUNTER(item_delete) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index ba213f8c..440912fe 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -70,8 +70,7 @@ struct cached_item { }; long dirty; - unsigned deletion:1, - ephemeral:1; + unsigned deletion:1; struct scoutfs_key_buf *key; @@ -94,8 +93,7 @@ static void free_item(struct super_block *sb, struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { scoutfs_key_free(sb, item->key); - if (!item->ephemeral) - scoutfs_kvec_kfree(item->val); + scoutfs_kvec_kfree(item->val); kfree(item); } } @@ -921,74 +919,6 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return ret; } -/* - * Ephemeral items are slightly magical and used to track file contents - * without copying the data into an allocated value. - * - * Their value kvec clones the callers which means they reference - * external data. They're freed after items are copied into segments so - * that callers can know that no items reference their structures after - * a commit finishes. - * - * They forcefully clobber any existing item at their key without - * reading the existing item. - */ -int scoutfs_item_create_ephemeral(struct super_block *sb, - struct scoutfs_key_buf *key, - struct kvec *val) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - item = alloc_item(sb, key, NULL); - if (!item) - return -ENOMEM; - - scoutfs_kvec_clone(item->val, val); - item->ephemeral = 1; - - spin_lock_irqsave(&cac->lock, flags); - - ret = insert_item(sb, cac, item, true); - BUG_ON(ret); - - scoutfs_inc_counter(sb, item_create_ephemeral); - mark_item_dirty(sb, cac, item); - - spin_unlock_irqrestore(&cac->lock, flags); - - return ret; -} - -/* - * Update the value for an ephemeral item if it exists. - */ -void scoutfs_item_update_ephemeral(struct super_block *sb, - struct scoutfs_key_buf *key, - struct kvec *val) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item && item->ephemeral) { - trace_printk("updating ephemeral item %p\n", item); - scoutfs_inc_counter(sb, item_update_ephemeral); - clear_item_dirty(sb, cac, item); - scoutfs_kvec_clone(item->val, val); - mark_item_dirty(sb, cac, item); - } - - spin_unlock_irqrestore(&cac->lock, flags); -} - /* * Allocate an item with the key and value and add it to the list of * items to be inserted as a batch later. The caller adds in sort order @@ -1623,7 +1553,7 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) del = item; item = next_dirty(item); - if (del->deletion || del->ephemeral) + if (del->deletion) erase_item(sb, cac, del); nr_items--; diff --git a/kmod/src/item.h b/kmod/src/item.h index 868aeeb6..97ef8bc1 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -27,15 +27,9 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_create_ephemeral(struct super_block *sb, - struct scoutfs_key_buf *key, - struct kvec *val); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -void scoutfs_item_update_ephemeral(struct super_block *sb, - struct scoutfs_key_buf *key, - struct kvec *val); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_delete_many(struct super_block *sb, From a2ef5ecb330ebe23f94938c0ee074d799d633d5e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 May 2017 12:59:24 -0700 Subject: [PATCH 282/920] scoutfs: remove item_forget It's pretty dangerous to forcefully remove items without writing deletion items to lsm segments. This was only used for magical ephemeral items when we were having them store file data. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 - kmod/src/item.c | 26 -------------------------- kmod/src/item.h | 1 - 3 files changed, 28 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 3e35a1be..0369bb4c 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -32,7 +32,6 @@ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ EXPAND_COUNTER(item_delete) \ - EXPAND_COUNTER(item_forget) \ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ EXPAND_COUNTER(item_range_insert) diff --git a/kmod/src/item.c b/kmod/src/item.c index 440912fe..cd2b6cac 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1344,32 +1344,6 @@ out: return ret; } -/* - * Forcefully remove an item from the cache regardless of its state or - * relationship to persistent items. - * - * The caller is entirely responsible for the correctness of having this - * item vanish. - */ -void scoutfs_item_forget(struct super_block *sb, struct scoutfs_key_buf *key) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - trace_printk("forgetting item %p\n", item); - scoutfs_inc_counter(sb, item_forget); - erase_item(sb, cac, item); - } - - spin_unlock_irqrestore(&cac->lock, flags); -} - /* * Return the first dirty node in the subtree starting at the given node. */ diff --git a/kmod/src/item.h b/kmod/src/item.h index 97ef8bc1..68f14f8c 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -35,7 +35,6 @@ void scoutfs_item_delete_dirty(struct super_block *sb, int scoutfs_item_delete_many(struct super_block *sb, struct scoutfs_key_buf **keys, unsigned nr); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); -void scoutfs_item_forget(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); From c84250b8c69da71b97082d54854fbb76dbb600d3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 2 Jun 2017 09:20:02 -0700 Subject: [PATCH 283/920] scoutfs: add item_set_batch trace point Restore the item_set_batch trace point by changing the current insert_batch tracepoint to a class and defining insert and set as class trace points. Signed-off-by: Zach Brown --- kmod/src/item.c | 2 +- kmod/src/scoutfs_trace.h | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index cd2b6cac..3aaa9864 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1037,7 +1037,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, if (WARN_ON_ONCE(invalid_flags(sif))) return -EINVAL; -// trace_scoutfs_item_set_batch(sb, start, end); + trace_scoutfs_item_set_batch(sb, start, end); if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) return -EINVAL; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index f77cce2c..24348d6b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -230,7 +230,7 @@ TRACE_EVENT(scoutfs_item_insertion, TP_printk("key %s", __get_str(key)) ); -TRACE_EVENT(scoutfs_item_insert_batch, +DECLARE_EVENT_CLASS(scoutfs_range_class, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), TP_ARGS(sb, start, end), @@ -245,6 +245,18 @@ TRACE_EVENT(scoutfs_item_insert_batch, TP_printk("start %s end %s", __get_str(start), __get_str(end)) ); +DEFINE_EVENT(scoutfs_range_class, scoutfs_item_set_batch, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), + TP_ARGS(sb, start, end) +); + +DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), + TP_ARGS(sb, start, end) +); + #define lock_mode(mode) \ __print_symbolic(mode, \ { SCOUTFS_LOCK_MODE_READ, "READ" }, \ From 2bd698b60417a97f83dbeb69c4b487d823bf56bf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 14:23:03 -0700 Subject: [PATCH 284/920] scoutfs: set NODELAY and REUSEADDR on net sockets Add a helper that creates a socket and sets nodelay for all sockets and set reuseaddr in listening sockets. Signed-off-by: Zach Brown --- kmod/src/net.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index d7b16b8a..f915e24a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1837,6 +1837,38 @@ static void scoutfs_net_accept_func(struct work_struct *work) } } +/* + * Create a new TCP socket and set all the options that are used for + * both connecting and listening sockets. + */ +static int create_sock_setopts(struct socket **sock_ret) +{ + struct socket *sock; + int optval; + int ret; + + *sock_ret = NULL; + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) { + trace_printk("sock create ret %d\n", ret); + return ret; + } + + optval = 1; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&optval, + sizeof(optval)); + if (ret) { + trace_printk("nodelay ret %d\n", ret); + sock_release(sock); + return ret; + } + + *sock_ret = sock; + + return 0; +} + /* * The server work has acquired the listen lock. We create a socket and * publish its bound address in the addr lock's lvb. @@ -1855,6 +1887,7 @@ static void scoutfs_net_listen_func(struct work_struct *work) struct sockaddr_in sin; struct socket *sock; int addrlen; + int optval; int ret; /* XXX option to set listening address */ @@ -1865,13 +1898,22 @@ static void scoutfs_net_listen_func(struct work_struct *work) trace_printk("binding to %pIS:%u\n", &sin, be16_to_cpu(sin.sin_port)); - ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + ret = create_sock_setopts(&sock); if (ret) goto out; trace_printk("listening sinf %p sock %p sk %p\n", sinf, sock, sock->sk); + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&optval, + sizeof(optval)); + if (ret) { + trace_printk("reuseaddr ret %d\n", ret); + sock_release(sock); + goto out; + } + sinf->sock = sock; INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); @@ -1914,7 +1956,7 @@ static void scoutfs_net_connect_func(struct work_struct *work) int addrlen; int ret; - ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + ret = create_sock_setopts(&sock); if (ret) goto out; From 79de18443b022df9a35ca24184dee6892929f134 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 14:30:43 -0700 Subject: [PATCH 285/920] scoutfs: don't extend key in dec_cur_len A copy and paste bug had us extending the length of keys that were decremented at their previous length. The whole point of the _cur_len functions is that they don't have to extend the key buf out to full precision. Signed-off-by: Zach Brown --- kmod/src/key.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kmod/src/key.c b/kmod/src/key.c index 584fa2fb..5b33d637 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -93,8 +93,6 @@ void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key) u8 *bytes = key->data; int i; - extend_zeros(key); - for (i = key->key_len - 1; i >= 0; i--) { if (--bytes[i] != 255) break; From a05015225482b16e05ee1fa1faaafa2a15daa8e0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 14:32:29 -0700 Subject: [PATCH 286/920] scoutfs: fix ring next/prev walk comparison test The scoutfs_ring_next() and _prev() functions had a really dumb bug where they check the sign of comparisons by comparing with 1. For example, next would miss that the walk traversed a lesser item and wouldn't return the next item. This was causing compaction to miss underlying segments, creating segments in levels that had overlapping keys, which then totally confused reading and kept it from finding the items it was looking for. Signed-off-by: Zach Brown --- kmod/src/ring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 7eb61b0a..47a21bc9 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -320,7 +320,7 @@ void *scoutfs_ring_lookup_next(struct scoutfs_ring_info *ring, void *key) int cmp; rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); - if (rnode && (cmp > 1 || rnode->deleted)) + if (rnode && (cmp > 0 || rnode->deleted)) rnode = ring_rb_next(rnode); return rnode_data(rnode); @@ -332,7 +332,7 @@ void *scoutfs_ring_lookup_prev(struct scoutfs_ring_info *ring, void *key) int cmp; rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); - if (rnode && (cmp < 1 || rnode->deleted)) + if (rnode && (cmp < 0 || rnode->deleted)) rnode = ring_rb_prev(rnode); return rnode_data(rnode); From 1485b0255482bf579698db46071af3a1860f0a9f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 14:48:09 -0700 Subject: [PATCH 287/920] scoutfs: add SK_ helpers for printing keys Add some percpu string buffers so that we can pass formatted strings as arguments when printing keys. The percpu struct uses a different buffer for each argument. We wrap the whole print call in a wrapper that disables and enables preemption. Signed-off-by: Zach Brown --- kmod/src/key.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/key.h | 29 ++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/kmod/src/key.c b/kmod/src/key.c index 5b33d637..63963782 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -116,9 +116,8 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) * * XXX nonprintable characters in the trace? */ -int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) +int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) { - size_t size = buf ? INT_MAX : 0; int len; u8 type; @@ -170,3 +169,55 @@ int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) return snprintf_null(buf, size, "[truncated type %u len %u]", type, key->key_len); } + +/* + * A null buf can be set to find the length of the formatted string. + */ +int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) +{ + return scoutfs_key_str_size(buf, key, buf ? INT_MAX : 0); +} + +#define MAX_STR_COUNT 10 + +struct key_strings { + bool started; + int next_str; + char strings[MAX_STR_COUNT][SK_STR_BYTES]; +}; + +static DEFINE_PER_CPU(struct key_strings, percpu_key_strings); + +void scoutfs_key_start_percpu(void) +{ + struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); + + BUG_ON(ks->started); + ks->started = true; + get_cpu(); +} + +char *scoutfs_key_percpu_string(void) +{ + struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); + char *str; + + BUG_ON(!ks->started); + + str = ks->strings[ks->next_str++]; + BUG_ON(ks->next_str >= MAX_STR_COUNT); + + return str; +} + +void scoutfs_key_finish_percpu(void) +{ + struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); + + BUG_ON(!ks->started); + + ks->next_str = 0; + ks->started = false; + + put_cpu(); +} diff --git a/kmod/src/key.h b/kmod/src/key.h index 26f4b499..bdeafb57 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -19,7 +19,36 @@ void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key); void scoutfs_key_dec(struct scoutfs_key_buf *key); void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); +int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size); int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key); +void scoutfs_key_start_percpu(void); +char *scoutfs_key_percpu_string(void); +void scoutfs_key_finish_percpu(void); + +#define SK_PCPU(statements) do { \ + scoutfs_key_start_percpu(); \ + { statements; } \ + scoutfs_key_finish_percpu(); \ +} while (0) + +/* + * The biggest keys are typically a little struct then a large name. The + * string representation will tend to be mostly the name, but some of the + * strict fields can blow up from say 8 bytes to 20 bytes. So we give + * a lot of padding for that. + */ +#define SK_STR_BYTES (100 + SCOUTFS_MAX_KEY_SIZE) + +#define SK_FMT "%s" +#define SK_ARG(k) \ +({ \ + char *__str = scoutfs_key_percpu_string(); \ + scoutfs_key_str_size(__str, k, SK_STR_BYTES); \ + __str; \ +}) + +#define SK_TRACE_PRINTK(args...) SK_PCPU(trace_printk(args)) +#define SK_PRINTK(args...) SK_PCPU(printk(args)) /* * Initialize a small key in a larger allocated buffer. This lets From 54d286d91c47f1afaba18849b7aed4eb7086e7b9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:01:37 -0700 Subject: [PATCH 288/920] scoutfs: format strings for all key types Add all the missing key types to scoutfs_key_str() so that we can get traces and printks of all key types. Signed-off-by: Zach Brown --- kmod/src/key.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/kmod/src/key.c b/kmod/src/key.c index 63963782..6135b983 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -121,6 +121,9 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) int len; u8 type; + if (key == NULL || key->data == NULL) + return snprintf_null(buf, size, "[NULL]"); + if (key->key_len == 0) return snprintf_null(buf, size, "[0 len]"); @@ -161,6 +164,82 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) be64_to_cpu(dkey->ino), len, dkey->name); } + case SCOUTFS_READDIR_KEY: { + struct scoutfs_readdir_key *rkey = key->data; + + return snprintf_null(buf, size, "rdr.%llu.%llu", + be64_to_cpu(rkey->ino), + be64_to_cpu(rkey->pos)); + } + + case SCOUTFS_LINK_BACKREF_KEY: { + struct scoutfs_link_backref_key *lkey = key->data; + + len = (int)key->key_len - sizeof(*lkey); + if (len <= 0) + break; + + return snprintf_null(buf, size, "lbr.%llu.%llu.%.*s", + be64_to_cpu(lkey->ino), + be64_to_cpu(lkey->dir_ino), len, + lkey->name); + } + + case SCOUTFS_SYMLINK_KEY: { + struct scoutfs_symlink_key *skey = key->data; + + return snprintf_null(buf, size, "sym.%llu", + be64_to_cpu(skey->ino)); + } + + case SCOUTFS_FILE_EXTENT_KEY: { + struct scoutfs_file_extent_key *ekey = key->data; + + return snprintf_null(buf, size, "ext.%llu.%llu.%llu.%llu.%x", + be64_to_cpu(ekey->ino), + be64_to_cpu(ekey->last_blk_off), + be64_to_cpu(ekey->last_blkno), + be64_to_cpu(ekey->blocks), + ekey->flags); + } + + case SCOUTFS_ORPHAN_KEY: { + struct scoutfs_orphan_key *okey = key->data; + + return snprintf_null(buf, size, "orp.%llu", + be64_to_cpu(okey->ino)); + } + + case SCOUTFS_FREE_EXTENT_BLKNO_KEY: + case SCOUTFS_FREE_EXTENT_BLOCKS_KEY: { + struct scoutfs_free_extent_blkno_key *fkey = key->data; + + return snprintf_null(buf, size, "%s.%llu.%llu.%llu", + fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_KEY ? "fel" : + "fes", + be64_to_cpu(fkey->node_id), + be64_to_cpu(fkey->last_blkno), + be64_to_cpu(fkey->blocks)); + } + + case SCOUTFS_INODE_INDEX_CTIME_KEY: + case SCOUTFS_INODE_INDEX_MTIME_KEY: + case SCOUTFS_INODE_INDEX_SIZE_KEY: + case SCOUTFS_INODE_INDEX_META_SEQ_KEY: + case SCOUTFS_INODE_INDEX_DATA_SEQ_KEY: { + struct scoutfs_inode_index_key *ikey = key->data; + + return snprintf_null(buf, size, "%s.%llu.%u.%llu", + ikey->type == SCOUTFS_INODE_INDEX_CTIME_KEY ? "ctm" : + ikey->type == SCOUTFS_INODE_INDEX_MTIME_KEY ? "mtm" : + ikey->type == SCOUTFS_INODE_INDEX_SIZE_KEY ? "siz" : + ikey->type == SCOUTFS_INODE_INDEX_META_SEQ_KEY ? "msq" : + ikey->type == SCOUTFS_INODE_INDEX_DATA_SEQ_KEY ? "dsq" : + "uii", be64_to_cpu(ikey->major), + be32_to_cpu(ikey->minor), + be64_to_cpu(ikey->ino)); + } + default: return snprintf_null(buf, size, "[unknown type %u len %u]", type, key->key_len); From b5ee282f6b6e22e6c49ccac3e4011432820c54dc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:30:21 -0700 Subject: [PATCH 289/920] scoutfs: minor manifest ring comparison tracing It was nice to watch the ring compare nodes so leave behind the trace and clean up the callers so that uninitialized keys are cleanly null. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index e0eb9a9e..a04fbe50 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -414,6 +414,7 @@ scoutfs_manifest_find_range_entries(struct super_block *sb, nr = 0; /* get level 0 segments that overlap with the missing range */ + skey.key = NULL; skey.level = 0; skey.seq = ~0ULL; ment = scoutfs_ring_lookup_prev(&mani->ring, &skey); @@ -836,6 +837,8 @@ static int manifest_ring_compare_key(void *key, void *data) struct scoutfs_key_buf last; int cmp; + scoutfs_key_init(&first, NULL, 0); + if (skey->level < ment->level) { cmp = -1; goto out; @@ -861,6 +864,13 @@ static int manifest_ring_compare_key(void *key, void *data) } out: +#if 0 + /* pretty expensive to be on by default */ + SK_TRACE_PRINTK("%u,%llu,"SK_FMT" %c %u,%llu,"SK_FMT"\n", + skey->level, skey->seq, SK_ARG(skey->key), + cmp < 0 ? '<' : cmp == 0 ? '=' : '>', + ment->level, le64_to_cpu(ment->seq), SK_ARG(&first)); +#endif return cmp; } From 43e9d2caa2433d841112799e7e3ab01fce617876 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:31:47 -0700 Subject: [PATCH 290/920] scoutfs: trace compaction manifest entries Trace the manifest entries compaction received from the server. Signed-off-by: Zach Brown --- kmod/src/compact.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index f27fb1c5..f02234e0 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -633,6 +633,13 @@ static void scoutfs_compact_func(struct work_struct *work) if (ret == 0 && list_empty(&curs.csegs)) return; + /* trace compaction ranges */ + list_for_each_entry(cseg, &curs.csegs, entry) { + SK_TRACE_PRINTK("level %u segno %llu first "SK_FMT" last "SK_FMT"\n", + cseg->level, cseg->segno, SK_ARG(cseg->first), + SK_ARG(cseg->last)); + } + if (ret == 0 && !list_empty(&curs.csegs)) { ret = compact_segments(sb, &curs, &comp, &results); From 1652512af735b439765c9209cf27e1307e2444f9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:34:42 -0700 Subject: [PATCH 291/920] scoutfs: remove ancient dirty item comment This is just old and wrong. Signed-off-by: Zach Brown --- kmod/src/item.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 3aaa9864..68895042 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1478,17 +1478,6 @@ static void count_seg_items(struct item_cache *cac, u32 *nr_items, * The caller is responsible for the consistency of the dirty items once * they're in its seg. We can consider them clean once we store them. * - * Today entering a transaction doesn't ensure that there's never more - * than a segment's worth of dirty items. As we release a trans we kick - * off an async sync. By the time we get here we can have a lot more - * than a segments worth of dirty items. - * - * XXX This is unacceptable because multiple segment writes are not - * atomic. We can have the items that make up an atomic change span - * segments and can be partially visible if we only write the first - * segment. We probably want to throttle trans enters once we have as - * many dirty items as our atomic segment updates can write. - * * XXX this first/append pattern will go away once we can write a stream * of items to a segment without needing to know the item count to * find the starting key and value offsets. From a1dadd9763e69e21dd3f863993032f904eda9d88 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:35:05 -0700 Subject: [PATCH 292/920] scoutfs: lock around dirty item writing Writing dirty items into a segment wasn't protected by locking. It's not racing with item dirtying, bit it's absolutely racing with reads while modifying the rbtree. And shrinking will be modifying the item cache at any old time in the future. Signed-off-by: Zach Brown --- kmod/src/item.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 68895042..b3e9d8f5 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1488,9 +1488,12 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) struct item_cache *cac = sbi->item_cache; struct cached_item *item = NULL; struct cached_item *del; + unsigned long flags; u32 key_bytes; u32 nr_items; + spin_lock_irqsave(&cac->lock, flags); + count_seg_items(cac, &nr_items, &key_bytes); /* remember nr_items is passed to _first_item */ @@ -1522,6 +1525,8 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) nr_items--; } + spin_unlock_irqrestore(&cac->lock, flags); + return 0; } From 0280971faba00c5590f9e49a4c950a9131015786 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Jun 2017 15:45:15 -0700 Subject: [PATCH 293/920] scoutfs: add bug on for out of order seg items We've seen some cases where compaction writes a new segment that contains items that aren't sorted. This eventually leads to read being mislead in its binary search of the items in a segment and failing to find the items it was looking for. Signed-off-by: Zach Brown --- kmod/src/seg.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 767bab2e..1441d152 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -565,6 +565,16 @@ void scoutfs_seg_append_item(struct super_block *sb, pos = le32_to_cpu(sblk->nr_items); sblk->nr_items = cpu_to_le32(pos + 1); + /* + * It's very bad data corruption if we write out of order items + * to a segment. It'll mislead the key search during read and + * stop it from finding its items. + */ + if (pos) { + scoutfs_seg_item_ptrs(seg, pos - 1, &item_key, NULL, NULL); + BUG_ON(scoutfs_key_compare(key, &item_key) <= 0); + } + prev = pos_ptr(seg, pos - 1); item = pos_ptr(seg, pos); From 85cbe7dc97208913e526a3816ec9a2ec83eec0fd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Jun 2017 08:52:11 -0700 Subject: [PATCH 294/920] scoutfs: add a counter add macro to match inc Just a quick wrapper around the related percpu_counter call. Signed-off-by: Zach Brown --- kmod/src/counters.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 0369bb4c..e5d31e5a 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -58,6 +58,9 @@ struct scoutfs_counters { #define scoutfs_inc_counter(sb, which) \ percpu_counter_inc(&SCOUTFS_SB(sb)->counters->which) +#define scoutfs_add_counter(sb, which, cnt) \ + percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, cnt) + void __init scoutfs_init_counters(void); int scoutfs_setup_counters(struct super_block *sb); void scoutfs_destroy_counters(struct super_block *sb); From dfc220ad6fdd6b8d0db1290faf8979ed9e2320c9 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 4 Jan 2017 15:31:26 -0600 Subject: [PATCH 295/920] Import fs/dlm/* from linux-3.10.0-327.36.1.el7 Also wire it into the build system. We have to figure out how to get scoutfs pulling in the right headers but that can wait until we have something more usable. Signed-off-by: Mark Fasheh --- kmod/.gitignore | 12 +- kmod/Makefile | 13 +- kmod/dlm/Kconfig | 16 + kmod/dlm/Makefile | 20 + kmod/dlm/ast.c | 316 + kmod/dlm/ast.h | 32 + kmod/dlm/config.c | 1025 ++++ kmod/dlm/config.h | 53 + kmod/dlm/debug_fs.c | 815 +++ kmod/dlm/dir.c | 308 + kmod/dlm/dir.h | 25 + kmod/dlm/dlm_internal.h | 727 +++ kmod/dlm/include/linux/dlm.h | 172 + kmod/dlm/include/linux/dlm_plock.h | 19 + kmod/dlm/include/uapi/linux/dlm.h | 75 + kmod/dlm/include/uapi/linux/dlm_device.h | 108 + kmod/dlm/include/uapi/linux/dlm_netlink.h | 58 + kmod/dlm/include/uapi/linux/dlm_plock.h | 45 + kmod/dlm/include/uapi/linux/dlmconstants.h | 163 + kmod/dlm/lock.c | 6303 ++++++++++++++++++++ kmod/dlm/lock.h | 80 + kmod/dlm/lockspace.c | 906 +++ kmod/dlm/lockspace.h | 26 + kmod/dlm/lowcomms.c | 1726 ++++++ kmod/dlm/lowcomms.h | 27 + kmod/dlm/lvb_table.h | 18 + kmod/dlm/main.c | 97 + kmod/dlm/member.c | 725 +++ kmod/dlm/member.h | 33 + kmod/dlm/memory.c | 96 + kmod/dlm/memory.h | 27 + kmod/dlm/midcomms.c | 137 + kmod/dlm/midcomms.h | 21 + kmod/dlm/netlink.c | 141 + kmod/dlm/plock.c | 515 ++ kmod/dlm/rcom.c | 656 ++ kmod/dlm/rcom.h | 26 + kmod/dlm/recover.c | 955 +++ kmod/dlm/recover.h | 34 + kmod/dlm/recoverd.c | 342 ++ kmod/dlm/recoverd.h | 23 + kmod/dlm/requestqueue.c | 171 + kmod/dlm/requestqueue.h | 22 + kmod/dlm/user.c | 1028 ++++ kmod/dlm/user.h | 19 + kmod/dlm/util.c | 154 + kmod/dlm/util.h | 22 + 47 files changed, 18324 insertions(+), 8 deletions(-) create mode 100644 kmod/dlm/Kconfig create mode 100644 kmod/dlm/Makefile create mode 100644 kmod/dlm/ast.c create mode 100644 kmod/dlm/ast.h create mode 100644 kmod/dlm/config.c create mode 100644 kmod/dlm/config.h create mode 100644 kmod/dlm/debug_fs.c create mode 100644 kmod/dlm/dir.c create mode 100644 kmod/dlm/dir.h create mode 100644 kmod/dlm/dlm_internal.h create mode 100644 kmod/dlm/include/linux/dlm.h create mode 100644 kmod/dlm/include/linux/dlm_plock.h create mode 100644 kmod/dlm/include/uapi/linux/dlm.h create mode 100644 kmod/dlm/include/uapi/linux/dlm_device.h create mode 100644 kmod/dlm/include/uapi/linux/dlm_netlink.h create mode 100644 kmod/dlm/include/uapi/linux/dlm_plock.h create mode 100644 kmod/dlm/include/uapi/linux/dlmconstants.h create mode 100644 kmod/dlm/lock.c create mode 100644 kmod/dlm/lock.h create mode 100644 kmod/dlm/lockspace.c create mode 100644 kmod/dlm/lockspace.h create mode 100644 kmod/dlm/lowcomms.c create mode 100644 kmod/dlm/lowcomms.h create mode 100644 kmod/dlm/lvb_table.h create mode 100644 kmod/dlm/main.c create mode 100644 kmod/dlm/member.c create mode 100644 kmod/dlm/member.h create mode 100644 kmod/dlm/memory.c create mode 100644 kmod/dlm/memory.h create mode 100644 kmod/dlm/midcomms.c create mode 100644 kmod/dlm/midcomms.h create mode 100644 kmod/dlm/netlink.c create mode 100644 kmod/dlm/plock.c create mode 100644 kmod/dlm/rcom.c create mode 100644 kmod/dlm/rcom.h create mode 100644 kmod/dlm/recover.c create mode 100644 kmod/dlm/recover.h create mode 100644 kmod/dlm/recoverd.c create mode 100644 kmod/dlm/recoverd.h create mode 100644 kmod/dlm/requestqueue.c create mode 100644 kmod/dlm/requestqueue.h create mode 100644 kmod/dlm/user.c create mode 100644 kmod/dlm/user.h create mode 100644 kmod/dlm/util.c create mode 100644 kmod/dlm/util.h diff --git a/kmod/.gitignore b/kmod/.gitignore index 50873cec..03621d2d 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -1,8 +1,12 @@ -src/*.o -src/*.ko -src/*.mod.c -src/*.cmd +*.o +*.ko +*.mod.c +*.cmd +*~ src/.tmp_versions/ +dlm/.tmp_versions/ src/Module.symvers +dlm/Module.symvers src/modules.order +dlm/modules.order cscope.* diff --git a/kmod/Makefile b/kmod/Makefile index 54aafb03..bc37906e 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -12,13 +12,18 @@ else SP = @: endif -ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src +SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src +DLM_ARGS := CONFIG_DLM=m CONFIG_DLM_DEBUG=y -C $(SK_KSRC) M=$(CURDIR)/dlm all: module module: - make $(ARGS) - $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(ARGS) + make $(SCOUTFS_ARGS) + make $(DLM_ARGS) + $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) +# Do not enable until we can clean up some warnings +# $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(DLM_ARGS) clean: - make $(ARGS) clean + make $(SCOUTFS_ARGS) clean + make $(DLM_ARGS) clean diff --git a/kmod/dlm/Kconfig b/kmod/dlm/Kconfig new file mode 100644 index 00000000..e4242c3f --- /dev/null +++ b/kmod/dlm/Kconfig @@ -0,0 +1,16 @@ +menuconfig DLM + tristate "Distributed Lock Manager (DLM)" + depends on INET + depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n) + select IP_SCTP + help + A general purpose distributed lock manager for kernel or userspace + applications. + +config DLM_DEBUG + bool "DLM debugging" + depends on DLM + help + Under the debugfs mount point, the name of each lockspace will + appear as a file in the "dlm" directory. The output is the + list of resource and locks the local node knows about. diff --git a/kmod/dlm/Makefile b/kmod/dlm/Makefile new file mode 100644 index 00000000..c43e89cd --- /dev/null +++ b/kmod/dlm/Makefile @@ -0,0 +1,20 @@ +obj-$(CONFIG_DLM) += dlm.o +dlm-y := ast.o \ + config.o \ + dir.o \ + lock.o \ + lockspace.o \ + main.o \ + member.o \ + memory.o \ + midcomms.o \ + netlink.o \ + lowcomms.o \ + plock.o \ + rcom.o \ + recover.o \ + recoverd.o \ + requestqueue.o \ + user.o \ + util.o +dlm-$(CONFIG_DLM_DEBUG) += debug_fs.o diff --git a/kmod/dlm/ast.c b/kmod/dlm/ast.c new file mode 100644 index 00000000..27a6ba9a --- /dev/null +++ b/kmod/dlm/ast.c @@ -0,0 +1,316 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lock.h" +#include "user.h" + +static uint64_t dlm_cb_seq; +static DEFINE_SPINLOCK(dlm_cb_seq_spin); + +static void dlm_dump_lkb_callbacks(struct dlm_lkb *lkb) +{ + int i; + + log_print("last_bast %x %llu flags %x mode %d sb %d %x", + lkb->lkb_id, + (unsigned long long)lkb->lkb_last_bast.seq, + lkb->lkb_last_bast.flags, + lkb->lkb_last_bast.mode, + lkb->lkb_last_bast.sb_status, + lkb->lkb_last_bast.sb_flags); + + log_print("last_cast %x %llu flags %x mode %d sb %d %x", + lkb->lkb_id, + (unsigned long long)lkb->lkb_last_cast.seq, + lkb->lkb_last_cast.flags, + lkb->lkb_last_cast.mode, + lkb->lkb_last_cast.sb_status, + lkb->lkb_last_cast.sb_flags); + + for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { + log_print("cb %x %llu flags %x mode %d sb %d %x", + lkb->lkb_id, + (unsigned long long)lkb->lkb_callbacks[i].seq, + lkb->lkb_callbacks[i].flags, + lkb->lkb_callbacks[i].mode, + lkb->lkb_callbacks[i].sb_status, + lkb->lkb_callbacks[i].sb_flags); + } +} + +int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, uint64_t seq) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + uint64_t prev_seq; + int prev_mode; + int i, rv; + + for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { + if (lkb->lkb_callbacks[i].seq) + continue; + + /* + * Suppress some redundant basts here, do more on removal. + * Don't even add a bast if the callback just before it + * is a bast for the same mode or a more restrictive mode. + * (the addional > PR check is needed for PR/CW inversion) + */ + + if ((i > 0) && (flags & DLM_CB_BAST) && + (lkb->lkb_callbacks[i-1].flags & DLM_CB_BAST)) { + + prev_seq = lkb->lkb_callbacks[i-1].seq; + prev_mode = lkb->lkb_callbacks[i-1].mode; + + if ((prev_mode == mode) || + (prev_mode > mode && prev_mode > DLM_LOCK_PR)) { + + log_debug(ls, "skip %x add bast %llu mode %d " + "for bast %llu mode %d", + lkb->lkb_id, + (unsigned long long)seq, + mode, + (unsigned long long)prev_seq, + prev_mode); + rv = 0; + goto out; + } + } + + lkb->lkb_callbacks[i].seq = seq; + lkb->lkb_callbacks[i].flags = flags; + lkb->lkb_callbacks[i].mode = mode; + lkb->lkb_callbacks[i].sb_status = status; + lkb->lkb_callbacks[i].sb_flags = (sbflags & 0x000000FF); + rv = 0; + break; + } + + if (i == DLM_CALLBACKS_SIZE) { + log_error(ls, "no callbacks %x %llu flags %x mode %d sb %d %x", + lkb->lkb_id, (unsigned long long)seq, + flags, mode, status, sbflags); + dlm_dump_lkb_callbacks(lkb); + rv = -1; + goto out; + } + out: + return rv; +} + +int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_callback *cb, int *resid) +{ + int i, rv; + + *resid = 0; + + if (!lkb->lkb_callbacks[0].seq) { + rv = -ENOENT; + goto out; + } + + /* oldest undelivered cb is callbacks[0] */ + + memcpy(cb, &lkb->lkb_callbacks[0], sizeof(struct dlm_callback)); + memset(&lkb->lkb_callbacks[0], 0, sizeof(struct dlm_callback)); + + /* shift others down */ + + for (i = 1; i < DLM_CALLBACKS_SIZE; i++) { + if (!lkb->lkb_callbacks[i].seq) + break; + memcpy(&lkb->lkb_callbacks[i-1], &lkb->lkb_callbacks[i], + sizeof(struct dlm_callback)); + memset(&lkb->lkb_callbacks[i], 0, sizeof(struct dlm_callback)); + (*resid)++; + } + + /* if cb is a bast, it should be skipped if the blocking mode is + compatible with the last granted mode */ + + if ((cb->flags & DLM_CB_BAST) && lkb->lkb_last_cast.seq) { + if (dlm_modes_compat(cb->mode, lkb->lkb_last_cast.mode)) { + cb->flags |= DLM_CB_SKIP; + + log_debug(ls, "skip %x bast %llu mode %d " + "for cast %llu mode %d", + lkb->lkb_id, + (unsigned long long)cb->seq, + cb->mode, + (unsigned long long)lkb->lkb_last_cast.seq, + lkb->lkb_last_cast.mode); + rv = 0; + goto out; + } + } + + if (cb->flags & DLM_CB_CAST) { + memcpy(&lkb->lkb_last_cast, cb, sizeof(struct dlm_callback)); + lkb->lkb_last_cast_time = ktime_get(); + } + + if (cb->flags & DLM_CB_BAST) { + memcpy(&lkb->lkb_last_bast, cb, sizeof(struct dlm_callback)); + lkb->lkb_last_bast_time = ktime_get(); + } + rv = 0; + out: + return rv; +} + +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, + uint32_t sbflags) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + uint64_t new_seq, prev_seq; + int rv; + + spin_lock(&dlm_cb_seq_spin); + new_seq = ++dlm_cb_seq; + spin_unlock(&dlm_cb_seq_spin); + + if (lkb->lkb_flags & DLM_IFL_USER) { + dlm_user_add_ast(lkb, flags, mode, status, sbflags, new_seq); + return; + } + + mutex_lock(&lkb->lkb_cb_mutex); + prev_seq = lkb->lkb_callbacks[0].seq; + + rv = dlm_add_lkb_callback(lkb, flags, mode, status, sbflags, new_seq); + if (rv < 0) + goto out; + + if (!prev_seq) { + kref_get(&lkb->lkb_ref); + + if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) { + mutex_lock(&ls->ls_cb_mutex); + list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay); + mutex_unlock(&ls->ls_cb_mutex); + } else { + queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work); + } + } + out: + mutex_unlock(&lkb->lkb_cb_mutex); +} + +void dlm_callback_work(struct work_struct *work) +{ + struct dlm_lkb *lkb = container_of(work, struct dlm_lkb, lkb_cb_work); + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + void (*castfn) (void *astparam); + void (*bastfn) (void *astparam, int mode); + struct dlm_callback callbacks[DLM_CALLBACKS_SIZE]; + int i, rv, resid; + + memset(&callbacks, 0, sizeof(callbacks)); + + mutex_lock(&lkb->lkb_cb_mutex); + if (!lkb->lkb_callbacks[0].seq) { + /* no callback work exists, shouldn't happen */ + log_error(ls, "dlm_callback_work %x no work", lkb->lkb_id); + dlm_print_lkb(lkb); + dlm_dump_lkb_callbacks(lkb); + } + + for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { + rv = dlm_rem_lkb_callback(ls, lkb, &callbacks[i], &resid); + if (rv < 0) + break; + } + + if (resid) { + /* cbs remain, loop should have removed all, shouldn't happen */ + log_error(ls, "dlm_callback_work %x resid %d", lkb->lkb_id, + resid); + dlm_print_lkb(lkb); + dlm_dump_lkb_callbacks(lkb); + } + mutex_unlock(&lkb->lkb_cb_mutex); + + castfn = lkb->lkb_astfn; + bastfn = lkb->lkb_bastfn; + + for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { + if (!callbacks[i].seq) + break; + if (callbacks[i].flags & DLM_CB_SKIP) { + continue; + } else if (callbacks[i].flags & DLM_CB_BAST) { + bastfn(lkb->lkb_astparam, callbacks[i].mode); + } else if (callbacks[i].flags & DLM_CB_CAST) { + lkb->lkb_lksb->sb_status = callbacks[i].sb_status; + lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags; + castfn(lkb->lkb_astparam); + } + } + + /* undo kref_get from dlm_add_callback, may cause lkb to be freed */ + dlm_put_lkb(lkb); +} + +int dlm_callback_start(struct dlm_ls *ls) +{ + ls->ls_callback_wq = alloc_workqueue("dlm_callback", + WQ_UNBOUND | + WQ_MEM_RECLAIM | + WQ_NON_REENTRANT, + 0); + if (!ls->ls_callback_wq) { + log_print("can't start dlm_callback workqueue"); + return -ENOMEM; + } + return 0; +} + +void dlm_callback_stop(struct dlm_ls *ls) +{ + if (ls->ls_callback_wq) + destroy_workqueue(ls->ls_callback_wq); +} + +void dlm_callback_suspend(struct dlm_ls *ls) +{ + set_bit(LSFL_CB_DELAY, &ls->ls_flags); + + if (ls->ls_callback_wq) + flush_workqueue(ls->ls_callback_wq); +} + +void dlm_callback_resume(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb, *safe; + int count = 0; + + clear_bit(LSFL_CB_DELAY, &ls->ls_flags); + + if (!ls->ls_callback_wq) + return; + + mutex_lock(&ls->ls_cb_mutex); + list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) { + list_del_init(&lkb->lkb_cb_list); + queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work); + count++; + } + mutex_unlock(&ls->ls_cb_mutex); + + if (count) + log_debug(ls, "dlm_callback_resume %d", count); +} + diff --git a/kmod/dlm/ast.h b/kmod/dlm/ast.h new file mode 100644 index 00000000..757b551c --- /dev/null +++ b/kmod/dlm/ast.h @@ -0,0 +1,32 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __ASTD_DOT_H__ +#define __ASTD_DOT_H__ + +void dlm_del_ast(struct dlm_lkb *lkb); +int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, uint64_t seq); +int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_callback *cb, int *resid); +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, + uint32_t sbflags); + +void dlm_callback_work(struct work_struct *work); +int dlm_callback_start(struct dlm_ls *ls); +void dlm_callback_stop(struct dlm_ls *ls); +void dlm_callback_suspend(struct dlm_ls *ls); +void dlm_callback_resume(struct dlm_ls *ls); + +#endif + + diff --git a/kmod/dlm/config.c b/kmod/dlm/config.c new file mode 100644 index 00000000..7d58d5b1 --- /dev/null +++ b/kmod/dlm/config.c @@ -0,0 +1,1025 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "lowcomms.h" + +/* + * /config/dlm//spaces//nodes//nodeid + * /config/dlm//spaces//nodes//weight + * /config/dlm//comms//nodeid + * /config/dlm//comms//local + * /config/dlm//comms//addr (write only) + * /config/dlm//comms//addr_list (read only) + * The level is useless, but I haven't figured out how to avoid it. + */ + +static struct config_group *space_list; +static struct config_group *comm_list; +static struct dlm_comm *local_comm; +static uint32_t dlm_comm_count; + +struct dlm_clusters; +struct dlm_cluster; +struct dlm_spaces; +struct dlm_space; +struct dlm_comms; +struct dlm_comm; +struct dlm_nodes; +struct dlm_node; + +static struct config_group *make_cluster(struct config_group *, const char *); +static void drop_cluster(struct config_group *, struct config_item *); +static void release_cluster(struct config_item *); +static struct config_group *make_space(struct config_group *, const char *); +static void drop_space(struct config_group *, struct config_item *); +static void release_space(struct config_item *); +static struct config_item *make_comm(struct config_group *, const char *); +static void drop_comm(struct config_group *, struct config_item *); +static void release_comm(struct config_item *); +static struct config_item *make_node(struct config_group *, const char *); +static void drop_node(struct config_group *, struct config_item *); +static void release_node(struct config_item *); + +static ssize_t show_cluster(struct config_item *i, struct configfs_attribute *a, + char *buf); +static ssize_t store_cluster(struct config_item *i, + struct configfs_attribute *a, + const char *buf, size_t len); +static ssize_t show_comm(struct config_item *i, struct configfs_attribute *a, + char *buf); +static ssize_t store_comm(struct config_item *i, struct configfs_attribute *a, + const char *buf, size_t len); +static ssize_t show_node(struct config_item *i, struct configfs_attribute *a, + char *buf); +static ssize_t store_node(struct config_item *i, struct configfs_attribute *a, + const char *buf, size_t len); + +static ssize_t comm_nodeid_read(struct dlm_comm *cm, char *buf); +static ssize_t comm_nodeid_write(struct dlm_comm *cm, const char *buf, + size_t len); +static ssize_t comm_local_read(struct dlm_comm *cm, char *buf); +static ssize_t comm_local_write(struct dlm_comm *cm, const char *buf, + size_t len); +static ssize_t comm_addr_write(struct dlm_comm *cm, const char *buf, + size_t len); +static ssize_t comm_addr_list_read(struct dlm_comm *cm, char *buf); +static ssize_t node_nodeid_read(struct dlm_node *nd, char *buf); +static ssize_t node_nodeid_write(struct dlm_node *nd, const char *buf, + size_t len); +static ssize_t node_weight_read(struct dlm_node *nd, char *buf); +static ssize_t node_weight_write(struct dlm_node *nd, const char *buf, + size_t len); + +struct dlm_cluster { + struct config_group group; + unsigned int cl_tcp_port; + unsigned int cl_buffer_size; + unsigned int cl_rsbtbl_size; + unsigned int cl_recover_timer; + unsigned int cl_toss_secs; + unsigned int cl_scan_secs; + unsigned int cl_log_debug; + unsigned int cl_protocol; + unsigned int cl_timewarn_cs; + unsigned int cl_waitwarn_us; + unsigned int cl_new_rsb_count; + unsigned int cl_recover_callbacks; + char cl_cluster_name[DLM_LOCKSPACE_LEN]; +}; + +enum { + CLUSTER_ATTR_TCP_PORT = 0, + CLUSTER_ATTR_BUFFER_SIZE, + CLUSTER_ATTR_RSBTBL_SIZE, + CLUSTER_ATTR_RECOVER_TIMER, + CLUSTER_ATTR_TOSS_SECS, + CLUSTER_ATTR_SCAN_SECS, + CLUSTER_ATTR_LOG_DEBUG, + CLUSTER_ATTR_PROTOCOL, + CLUSTER_ATTR_TIMEWARN_CS, + CLUSTER_ATTR_WAITWARN_US, + CLUSTER_ATTR_NEW_RSB_COUNT, + CLUSTER_ATTR_RECOVER_CALLBACKS, + CLUSTER_ATTR_CLUSTER_NAME, +}; + +struct cluster_attribute { + struct configfs_attribute attr; + ssize_t (*show)(struct dlm_cluster *, char *); + ssize_t (*store)(struct dlm_cluster *, const char *, size_t); +}; + +static ssize_t cluster_cluster_name_read(struct dlm_cluster *cl, char *buf) +{ + return sprintf(buf, "%s\n", cl->cl_cluster_name); +} + +static ssize_t cluster_cluster_name_write(struct dlm_cluster *cl, + const char *buf, size_t len) +{ + strncpy(dlm_config.ci_cluster_name, buf, DLM_LOCKSPACE_LEN); + strncpy(cl->cl_cluster_name, buf, DLM_LOCKSPACE_LEN); + return len; +} + +static struct cluster_attribute cluster_attr_cluster_name = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "cluster_name", + .ca_mode = S_IRUGO | S_IWUSR }, + .show = cluster_cluster_name_read, + .store = cluster_cluster_name_write, +}; + +static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field, + int *info_field, int check_zero, + const char *buf, size_t len) +{ + unsigned int x; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + x = simple_strtoul(buf, NULL, 0); + + if (check_zero && !x) + return -EINVAL; + + *cl_field = x; + *info_field = x; + + return len; +} + +#define CLUSTER_ATTR(name, check_zero) \ +static ssize_t name##_write(struct dlm_cluster *cl, const char *buf, size_t len) \ +{ \ + return cluster_set(cl, &cl->cl_##name, &dlm_config.ci_##name, \ + check_zero, buf, len); \ +} \ +static ssize_t name##_read(struct dlm_cluster *cl, char *buf) \ +{ \ + return snprintf(buf, PAGE_SIZE, "%u\n", cl->cl_##name); \ +} \ +static struct cluster_attribute cluster_attr_##name = \ +__CONFIGFS_ATTR(name, 0644, name##_read, name##_write) + +CLUSTER_ATTR(tcp_port, 1); +CLUSTER_ATTR(buffer_size, 1); +CLUSTER_ATTR(rsbtbl_size, 1); +CLUSTER_ATTR(recover_timer, 1); +CLUSTER_ATTR(toss_secs, 1); +CLUSTER_ATTR(scan_secs, 1); +CLUSTER_ATTR(log_debug, 0); +CLUSTER_ATTR(protocol, 0); +CLUSTER_ATTR(timewarn_cs, 1); +CLUSTER_ATTR(waitwarn_us, 0); +CLUSTER_ATTR(new_rsb_count, 0); +CLUSTER_ATTR(recover_callbacks, 0); + +static struct configfs_attribute *cluster_attrs[] = { + [CLUSTER_ATTR_TCP_PORT] = &cluster_attr_tcp_port.attr, + [CLUSTER_ATTR_BUFFER_SIZE] = &cluster_attr_buffer_size.attr, + [CLUSTER_ATTR_RSBTBL_SIZE] = &cluster_attr_rsbtbl_size.attr, + [CLUSTER_ATTR_RECOVER_TIMER] = &cluster_attr_recover_timer.attr, + [CLUSTER_ATTR_TOSS_SECS] = &cluster_attr_toss_secs.attr, + [CLUSTER_ATTR_SCAN_SECS] = &cluster_attr_scan_secs.attr, + [CLUSTER_ATTR_LOG_DEBUG] = &cluster_attr_log_debug.attr, + [CLUSTER_ATTR_PROTOCOL] = &cluster_attr_protocol.attr, + [CLUSTER_ATTR_TIMEWARN_CS] = &cluster_attr_timewarn_cs.attr, + [CLUSTER_ATTR_WAITWARN_US] = &cluster_attr_waitwarn_us.attr, + [CLUSTER_ATTR_NEW_RSB_COUNT] = &cluster_attr_new_rsb_count.attr, + [CLUSTER_ATTR_RECOVER_CALLBACKS] = &cluster_attr_recover_callbacks.attr, + [CLUSTER_ATTR_CLUSTER_NAME] = &cluster_attr_cluster_name.attr, + NULL, +}; + +enum { + COMM_ATTR_NODEID = 0, + COMM_ATTR_LOCAL, + COMM_ATTR_ADDR, + COMM_ATTR_ADDR_LIST, +}; + +struct comm_attribute { + struct configfs_attribute attr; + ssize_t (*show)(struct dlm_comm *, char *); + ssize_t (*store)(struct dlm_comm *, const char *, size_t); +}; + +static struct comm_attribute comm_attr_nodeid = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "nodeid", + .ca_mode = S_IRUGO | S_IWUSR }, + .show = comm_nodeid_read, + .store = comm_nodeid_write, +}; + +static struct comm_attribute comm_attr_local = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "local", + .ca_mode = S_IRUGO | S_IWUSR }, + .show = comm_local_read, + .store = comm_local_write, +}; + +static struct comm_attribute comm_attr_addr = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "addr", + .ca_mode = S_IWUSR }, + .store = comm_addr_write, +}; + +static struct comm_attribute comm_attr_addr_list = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "addr_list", + .ca_mode = S_IRUGO }, + .show = comm_addr_list_read, +}; + +static struct configfs_attribute *comm_attrs[] = { + [COMM_ATTR_NODEID] = &comm_attr_nodeid.attr, + [COMM_ATTR_LOCAL] = &comm_attr_local.attr, + [COMM_ATTR_ADDR] = &comm_attr_addr.attr, + [COMM_ATTR_ADDR_LIST] = &comm_attr_addr_list.attr, + NULL, +}; + +enum { + NODE_ATTR_NODEID = 0, + NODE_ATTR_WEIGHT, +}; + +struct node_attribute { + struct configfs_attribute attr; + ssize_t (*show)(struct dlm_node *, char *); + ssize_t (*store)(struct dlm_node *, const char *, size_t); +}; + +static struct node_attribute node_attr_nodeid = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "nodeid", + .ca_mode = S_IRUGO | S_IWUSR }, + .show = node_nodeid_read, + .store = node_nodeid_write, +}; + +static struct node_attribute node_attr_weight = { + .attr = { .ca_owner = THIS_MODULE, + .ca_name = "weight", + .ca_mode = S_IRUGO | S_IWUSR }, + .show = node_weight_read, + .store = node_weight_write, +}; + +static struct configfs_attribute *node_attrs[] = { + [NODE_ATTR_NODEID] = &node_attr_nodeid.attr, + [NODE_ATTR_WEIGHT] = &node_attr_weight.attr, + NULL, +}; + +struct dlm_clusters { + struct configfs_subsystem subsys; +}; + +struct dlm_spaces { + struct config_group ss_group; +}; + +struct dlm_space { + struct config_group group; + struct list_head members; + struct mutex members_lock; + int members_count; +}; + +struct dlm_comms { + struct config_group cs_group; +}; + +struct dlm_comm { + struct config_item item; + int seq; + int nodeid; + int local; + int addr_count; + struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; +}; + +struct dlm_nodes { + struct config_group ns_group; +}; + +struct dlm_node { + struct config_item item; + struct list_head list; /* space->members */ + int nodeid; + int weight; + int new; + int comm_seq; /* copy of cm->seq when nd->nodeid is set */ +}; + +static struct configfs_group_operations clusters_ops = { + .make_group = make_cluster, + .drop_item = drop_cluster, +}; + +static struct configfs_item_operations cluster_ops = { + .release = release_cluster, + .show_attribute = show_cluster, + .store_attribute = store_cluster, +}; + +static struct configfs_group_operations spaces_ops = { + .make_group = make_space, + .drop_item = drop_space, +}; + +static struct configfs_item_operations space_ops = { + .release = release_space, +}; + +static struct configfs_group_operations comms_ops = { + .make_item = make_comm, + .drop_item = drop_comm, +}; + +static struct configfs_item_operations comm_ops = { + .release = release_comm, + .show_attribute = show_comm, + .store_attribute = store_comm, +}; + +static struct configfs_group_operations nodes_ops = { + .make_item = make_node, + .drop_item = drop_node, +}; + +static struct configfs_item_operations node_ops = { + .release = release_node, + .show_attribute = show_node, + .store_attribute = store_node, +}; + +static struct config_item_type clusters_type = { + .ct_group_ops = &clusters_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type cluster_type = { + .ct_item_ops = &cluster_ops, + .ct_attrs = cluster_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type spaces_type = { + .ct_group_ops = &spaces_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type space_type = { + .ct_item_ops = &space_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type comms_type = { + .ct_group_ops = &comms_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type comm_type = { + .ct_item_ops = &comm_ops, + .ct_attrs = comm_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type nodes_type = { + .ct_group_ops = &nodes_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_item_type node_type = { + .ct_item_ops = &node_ops, + .ct_attrs = node_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct dlm_cluster *config_item_to_cluster(struct config_item *i) +{ + return i ? container_of(to_config_group(i), struct dlm_cluster, group) : + NULL; +} + +static struct dlm_space *config_item_to_space(struct config_item *i) +{ + return i ? container_of(to_config_group(i), struct dlm_space, group) : + NULL; +} + +static struct dlm_comm *config_item_to_comm(struct config_item *i) +{ + return i ? container_of(i, struct dlm_comm, item) : NULL; +} + +static struct dlm_node *config_item_to_node(struct config_item *i) +{ + return i ? container_of(i, struct dlm_node, item) : NULL; +} + +static struct config_group *make_cluster(struct config_group *g, + const char *name) +{ + struct dlm_cluster *cl = NULL; + struct dlm_spaces *sps = NULL; + struct dlm_comms *cms = NULL; + void *gps = NULL; + + cl = kzalloc(sizeof(struct dlm_cluster), GFP_NOFS); + gps = kcalloc(3, sizeof(struct config_group *), GFP_NOFS); + sps = kzalloc(sizeof(struct dlm_spaces), GFP_NOFS); + cms = kzalloc(sizeof(struct dlm_comms), GFP_NOFS); + + if (!cl || !gps || !sps || !cms) + goto fail; + + config_group_init_type_name(&cl->group, name, &cluster_type); + config_group_init_type_name(&sps->ss_group, "spaces", &spaces_type); + config_group_init_type_name(&cms->cs_group, "comms", &comms_type); + + cl->group.default_groups = gps; + cl->group.default_groups[0] = &sps->ss_group; + cl->group.default_groups[1] = &cms->cs_group; + cl->group.default_groups[2] = NULL; + + cl->cl_tcp_port = dlm_config.ci_tcp_port; + cl->cl_buffer_size = dlm_config.ci_buffer_size; + cl->cl_rsbtbl_size = dlm_config.ci_rsbtbl_size; + cl->cl_recover_timer = dlm_config.ci_recover_timer; + cl->cl_toss_secs = dlm_config.ci_toss_secs; + cl->cl_scan_secs = dlm_config.ci_scan_secs; + cl->cl_log_debug = dlm_config.ci_log_debug; + cl->cl_protocol = dlm_config.ci_protocol; + cl->cl_timewarn_cs = dlm_config.ci_timewarn_cs; + cl->cl_waitwarn_us = dlm_config.ci_waitwarn_us; + cl->cl_new_rsb_count = dlm_config.ci_new_rsb_count; + cl->cl_recover_callbacks = dlm_config.ci_recover_callbacks; + memcpy(cl->cl_cluster_name, dlm_config.ci_cluster_name, + DLM_LOCKSPACE_LEN); + + space_list = &sps->ss_group; + comm_list = &cms->cs_group; + return &cl->group; + + fail: + kfree(cl); + kfree(gps); + kfree(sps); + kfree(cms); + return ERR_PTR(-ENOMEM); +} + +static void drop_cluster(struct config_group *g, struct config_item *i) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + struct config_item *tmp; + int j; + + for (j = 0; cl->group.default_groups[j]; j++) { + tmp = &cl->group.default_groups[j]->cg_item; + cl->group.default_groups[j] = NULL; + config_item_put(tmp); + } + + space_list = NULL; + comm_list = NULL; + + config_item_put(i); +} + +static void release_cluster(struct config_item *i) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + kfree(cl->group.default_groups); + kfree(cl); +} + +static struct config_group *make_space(struct config_group *g, const char *name) +{ + struct dlm_space *sp = NULL; + struct dlm_nodes *nds = NULL; + void *gps = NULL; + + sp = kzalloc(sizeof(struct dlm_space), GFP_NOFS); + gps = kcalloc(2, sizeof(struct config_group *), GFP_NOFS); + nds = kzalloc(sizeof(struct dlm_nodes), GFP_NOFS); + + if (!sp || !gps || !nds) + goto fail; + + config_group_init_type_name(&sp->group, name, &space_type); + config_group_init_type_name(&nds->ns_group, "nodes", &nodes_type); + + sp->group.default_groups = gps; + sp->group.default_groups[0] = &nds->ns_group; + sp->group.default_groups[1] = NULL; + + INIT_LIST_HEAD(&sp->members); + mutex_init(&sp->members_lock); + sp->members_count = 0; + return &sp->group; + + fail: + kfree(sp); + kfree(gps); + kfree(nds); + return ERR_PTR(-ENOMEM); +} + +static void drop_space(struct config_group *g, struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(i); + struct config_item *tmp; + int j; + + /* assert list_empty(&sp->members) */ + + for (j = 0; sp->group.default_groups[j]; j++) { + tmp = &sp->group.default_groups[j]->cg_item; + sp->group.default_groups[j] = NULL; + config_item_put(tmp); + } + + config_item_put(i); +} + +static void release_space(struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(i); + kfree(sp->group.default_groups); + kfree(sp); +} + +static struct config_item *make_comm(struct config_group *g, const char *name) +{ + struct dlm_comm *cm; + + cm = kzalloc(sizeof(struct dlm_comm), GFP_NOFS); + if (!cm) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&cm->item, name, &comm_type); + + cm->seq = dlm_comm_count++; + if (!cm->seq) + cm->seq = dlm_comm_count++; + + cm->nodeid = -1; + cm->local = 0; + cm->addr_count = 0; + return &cm->item; +} + +static void drop_comm(struct config_group *g, struct config_item *i) +{ + struct dlm_comm *cm = config_item_to_comm(i); + if (local_comm == cm) + local_comm = NULL; + dlm_lowcomms_close(cm->nodeid); + while (cm->addr_count--) + kfree(cm->addr[cm->addr_count]); + config_item_put(i); +} + +static void release_comm(struct config_item *i) +{ + struct dlm_comm *cm = config_item_to_comm(i); + kfree(cm); +} + +static struct config_item *make_node(struct config_group *g, const char *name) +{ + struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); + struct dlm_node *nd; + + nd = kzalloc(sizeof(struct dlm_node), GFP_NOFS); + if (!nd) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&nd->item, name, &node_type); + nd->nodeid = -1; + nd->weight = 1; /* default weight of 1 if none is set */ + nd->new = 1; /* set to 0 once it's been read by dlm_nodeid_list() */ + + mutex_lock(&sp->members_lock); + list_add(&nd->list, &sp->members); + sp->members_count++; + mutex_unlock(&sp->members_lock); + + return &nd->item; +} + +static void drop_node(struct config_group *g, struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); + struct dlm_node *nd = config_item_to_node(i); + + mutex_lock(&sp->members_lock); + list_del(&nd->list); + sp->members_count--; + mutex_unlock(&sp->members_lock); + + config_item_put(i); +} + +static void release_node(struct config_item *i) +{ + struct dlm_node *nd = config_item_to_node(i); + kfree(nd); +} + +static struct dlm_clusters clusters_root = { + .subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "dlm", + .ci_type = &clusters_type, + }, + }, + }, +}; + +int __init dlm_config_init(void) +{ + config_group_init(&clusters_root.subsys.su_group); + mutex_init(&clusters_root.subsys.su_mutex); + return configfs_register_subsystem(&clusters_root.subsys); +} + +void dlm_config_exit(void) +{ + configfs_unregister_subsystem(&clusters_root.subsys); +} + +/* + * Functions for user space to read/write attributes + */ + +static ssize_t show_cluster(struct config_item *i, struct configfs_attribute *a, + char *buf) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + struct cluster_attribute *cla = + container_of(a, struct cluster_attribute, attr); + return cla->show ? cla->show(cl, buf) : 0; +} + +static ssize_t store_cluster(struct config_item *i, + struct configfs_attribute *a, + const char *buf, size_t len) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + struct cluster_attribute *cla = + container_of(a, struct cluster_attribute, attr); + return cla->store ? cla->store(cl, buf, len) : -EINVAL; +} + +static ssize_t show_comm(struct config_item *i, struct configfs_attribute *a, + char *buf) +{ + struct dlm_comm *cm = config_item_to_comm(i); + struct comm_attribute *cma = + container_of(a, struct comm_attribute, attr); + return cma->show ? cma->show(cm, buf) : 0; +} + +static ssize_t store_comm(struct config_item *i, struct configfs_attribute *a, + const char *buf, size_t len) +{ + struct dlm_comm *cm = config_item_to_comm(i); + struct comm_attribute *cma = + container_of(a, struct comm_attribute, attr); + return cma->store ? cma->store(cm, buf, len) : -EINVAL; +} + +static ssize_t comm_nodeid_read(struct dlm_comm *cm, char *buf) +{ + return sprintf(buf, "%d\n", cm->nodeid); +} + +static ssize_t comm_nodeid_write(struct dlm_comm *cm, const char *buf, + size_t len) +{ + cm->nodeid = simple_strtol(buf, NULL, 0); + return len; +} + +static ssize_t comm_local_read(struct dlm_comm *cm, char *buf) +{ + return sprintf(buf, "%d\n", cm->local); +} + +static ssize_t comm_local_write(struct dlm_comm *cm, const char *buf, + size_t len) +{ + cm->local= simple_strtol(buf, NULL, 0); + if (cm->local && !local_comm) + local_comm = cm; + return len; +} + +static ssize_t comm_addr_write(struct dlm_comm *cm, const char *buf, size_t len) +{ + struct sockaddr_storage *addr; + int rv; + + if (len != sizeof(struct sockaddr_storage)) + return -EINVAL; + + if (cm->addr_count >= DLM_MAX_ADDR_COUNT) + return -ENOSPC; + + addr = kzalloc(sizeof(*addr), GFP_NOFS); + if (!addr) + return -ENOMEM; + + memcpy(addr, buf, len); + + rv = dlm_lowcomms_addr(cm->nodeid, addr, len); + if (rv) { + kfree(addr); + return rv; + } + + cm->addr[cm->addr_count++] = addr; + return len; +} + +static ssize_t comm_addr_list_read(struct dlm_comm *cm, char *buf) +{ + ssize_t s; + ssize_t allowance; + int i; + struct sockaddr_storage *addr; + struct sockaddr_in *addr_in; + struct sockaddr_in6 *addr_in6; + + /* Taken from ip6_addr_string() defined in lib/vsprintf.c */ + char buf0[sizeof("AF_INET6 xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255\n")]; + + + /* Derived from SIMPLE_ATTR_SIZE of fs/configfs/file.c */ + allowance = 4096; + buf[0] = '\0'; + + for (i = 0; i < cm->addr_count; i++) { + addr = cm->addr[i]; + + switch(addr->ss_family) { + case AF_INET: + addr_in = (struct sockaddr_in *)addr; + s = sprintf(buf0, "AF_INET %pI4\n", &addr_in->sin_addr.s_addr); + break; + case AF_INET6: + addr_in6 = (struct sockaddr_in6 *)addr; + s = sprintf(buf0, "AF_INET6 %pI6\n", &addr_in6->sin6_addr); + break; + default: + s = sprintf(buf0, "%s\n", ""); + break; + } + allowance -= s; + if (allowance >= 0) + strcat(buf, buf0); + else { + allowance += s; + break; + } + } + return 4096 - allowance; +} + +static ssize_t show_node(struct config_item *i, struct configfs_attribute *a, + char *buf) +{ + struct dlm_node *nd = config_item_to_node(i); + struct node_attribute *nda = + container_of(a, struct node_attribute, attr); + return nda->show ? nda->show(nd, buf) : 0; +} + +static ssize_t store_node(struct config_item *i, struct configfs_attribute *a, + const char *buf, size_t len) +{ + struct dlm_node *nd = config_item_to_node(i); + struct node_attribute *nda = + container_of(a, struct node_attribute, attr); + return nda->store ? nda->store(nd, buf, len) : -EINVAL; +} + +static ssize_t node_nodeid_read(struct dlm_node *nd, char *buf) +{ + return sprintf(buf, "%d\n", nd->nodeid); +} + +static ssize_t node_nodeid_write(struct dlm_node *nd, const char *buf, + size_t len) +{ + uint32_t seq = 0; + nd->nodeid = simple_strtol(buf, NULL, 0); + dlm_comm_seq(nd->nodeid, &seq); + nd->comm_seq = seq; + return len; +} + +static ssize_t node_weight_read(struct dlm_node *nd, char *buf) +{ + return sprintf(buf, "%d\n", nd->weight); +} + +static ssize_t node_weight_write(struct dlm_node *nd, const char *buf, + size_t len) +{ + nd->weight = simple_strtol(buf, NULL, 0); + return len; +} + +/* + * Functions for the dlm to get the info that's been configured + */ + +static struct dlm_space *get_space(char *name) +{ + struct config_item *i; + + if (!space_list) + return NULL; + + mutex_lock(&space_list->cg_subsys->su_mutex); + i = config_group_find_item(space_list, name); + mutex_unlock(&space_list->cg_subsys->su_mutex); + + return config_item_to_space(i); +} + +static void put_space(struct dlm_space *sp) +{ + config_item_put(&sp->group.cg_item); +} + +static struct dlm_comm *get_comm(int nodeid) +{ + struct config_item *i; + struct dlm_comm *cm = NULL; + int found = 0; + + if (!comm_list) + return NULL; + + mutex_lock(&clusters_root.subsys.su_mutex); + + list_for_each_entry(i, &comm_list->cg_children, ci_entry) { + cm = config_item_to_comm(i); + + if (cm->nodeid != nodeid) + continue; + found = 1; + config_item_get(i); + break; + } + mutex_unlock(&clusters_root.subsys.su_mutex); + + if (!found) + cm = NULL; + return cm; +} + +static void put_comm(struct dlm_comm *cm) +{ + config_item_put(&cm->item); +} + +/* caller must free mem */ +int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, + int *count_out) +{ + struct dlm_space *sp; + struct dlm_node *nd; + struct dlm_config_node *nodes, *node; + int rv, count; + + sp = get_space(lsname); + if (!sp) + return -EEXIST; + + mutex_lock(&sp->members_lock); + if (!sp->members_count) { + rv = -EINVAL; + printk(KERN_ERR "dlm: zero members_count\n"); + goto out; + } + + count = sp->members_count; + + nodes = kcalloc(count, sizeof(struct dlm_config_node), GFP_NOFS); + if (!nodes) { + rv = -ENOMEM; + goto out; + } + + node = nodes; + list_for_each_entry(nd, &sp->members, list) { + node->nodeid = nd->nodeid; + node->weight = nd->weight; + node->new = nd->new; + node->comm_seq = nd->comm_seq; + node++; + + nd->new = 0; + } + + *count_out = count; + *nodes_out = nodes; + rv = 0; + out: + mutex_unlock(&sp->members_lock); + put_space(sp); + return rv; +} + +int dlm_comm_seq(int nodeid, uint32_t *seq) +{ + struct dlm_comm *cm = get_comm(nodeid); + if (!cm) + return -EEXIST; + *seq = cm->seq; + put_comm(cm); + return 0; +} + +int dlm_our_nodeid(void) +{ + return local_comm ? local_comm->nodeid : 0; +} + +/* num 0 is first addr, num 1 is second addr */ +int dlm_our_addr(struct sockaddr_storage *addr, int num) +{ + if (!local_comm) + return -1; + if (num + 1 > local_comm->addr_count) + return -1; + memcpy(addr, local_comm->addr[num], sizeof(*addr)); + return 0; +} + +/* Config file defaults */ +#define DEFAULT_TCP_PORT 21064 +#define DEFAULT_BUFFER_SIZE 4096 +#define DEFAULT_RSBTBL_SIZE 1024 +#define DEFAULT_RECOVER_TIMER 5 +#define DEFAULT_TOSS_SECS 10 +#define DEFAULT_SCAN_SECS 5 +#define DEFAULT_LOG_DEBUG 0 +#define DEFAULT_PROTOCOL 0 +#define DEFAULT_TIMEWARN_CS 500 /* 5 sec = 500 centiseconds */ +#define DEFAULT_WAITWARN_US 0 +#define DEFAULT_NEW_RSB_COUNT 128 +#define DEFAULT_RECOVER_CALLBACKS 0 +#define DEFAULT_CLUSTER_NAME "" + +struct dlm_config_info dlm_config = { + .ci_tcp_port = DEFAULT_TCP_PORT, + .ci_buffer_size = DEFAULT_BUFFER_SIZE, + .ci_rsbtbl_size = DEFAULT_RSBTBL_SIZE, + .ci_recover_timer = DEFAULT_RECOVER_TIMER, + .ci_toss_secs = DEFAULT_TOSS_SECS, + .ci_scan_secs = DEFAULT_SCAN_SECS, + .ci_log_debug = DEFAULT_LOG_DEBUG, + .ci_protocol = DEFAULT_PROTOCOL, + .ci_timewarn_cs = DEFAULT_TIMEWARN_CS, + .ci_waitwarn_us = DEFAULT_WAITWARN_US, + .ci_new_rsb_count = DEFAULT_NEW_RSB_COUNT, + .ci_recover_callbacks = DEFAULT_RECOVER_CALLBACKS, + .ci_cluster_name = DEFAULT_CLUSTER_NAME +}; + diff --git a/kmod/dlm/config.h b/kmod/dlm/config.h new file mode 100644 index 00000000..f30697bc --- /dev/null +++ b/kmod/dlm/config.h @@ -0,0 +1,53 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __CONFIG_DOT_H__ +#define __CONFIG_DOT_H__ + +struct dlm_config_node { + int nodeid; + int weight; + int new; + uint32_t comm_seq; +}; + +#define DLM_MAX_ADDR_COUNT 3 + +struct dlm_config_info { + int ci_tcp_port; + int ci_buffer_size; + int ci_rsbtbl_size; + int ci_recover_timer; + int ci_toss_secs; + int ci_scan_secs; + int ci_log_debug; + int ci_protocol; + int ci_timewarn_cs; + int ci_waitwarn_us; + int ci_new_rsb_count; + int ci_recover_callbacks; + char ci_cluster_name[DLM_LOCKSPACE_LEN]; +}; + +extern struct dlm_config_info dlm_config; + +int dlm_config_init(void); +void dlm_config_exit(void); +int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, + int *count_out); +int dlm_comm_seq(int nodeid, uint32_t *seq); +int dlm_our_nodeid(void); +int dlm_our_addr(struct sockaddr_storage *addr, int num); + +#endif /* __CONFIG_DOT_H__ */ + diff --git a/kmod/dlm/debug_fs.c b/kmod/dlm/debug_fs.c new file mode 100644 index 00000000..b969deef --- /dev/null +++ b/kmod/dlm/debug_fs.c @@ -0,0 +1,815 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "dlm_internal.h" +#include "lock.h" + +#define DLM_DEBUG_BUF_LEN 4096 +static char debug_buf[DLM_DEBUG_BUF_LEN]; +static struct mutex debug_buf_lock; + +static struct dentry *dlm_root; + +static char *print_lockmode(int mode) +{ + switch (mode) { + case DLM_LOCK_IV: + return "--"; + case DLM_LOCK_NL: + return "NL"; + case DLM_LOCK_CR: + return "CR"; + case DLM_LOCK_CW: + return "CW"; + case DLM_LOCK_PR: + return "PR"; + case DLM_LOCK_PW: + return "PW"; + case DLM_LOCK_EX: + return "EX"; + default: + return "??"; + } +} + +static int print_format1_lock(struct seq_file *s, struct dlm_lkb *lkb, + struct dlm_rsb *res) +{ + seq_printf(s, "%08x %s", lkb->lkb_id, print_lockmode(lkb->lkb_grmode)); + + if (lkb->lkb_status == DLM_LKSTS_CONVERT || + lkb->lkb_status == DLM_LKSTS_WAITING) + seq_printf(s, " (%s)", print_lockmode(lkb->lkb_rqmode)); + + if (lkb->lkb_nodeid) { + if (lkb->lkb_nodeid != res->res_nodeid) + seq_printf(s, " Remote: %3d %08x", lkb->lkb_nodeid, + lkb->lkb_remid); + else + seq_printf(s, " Master: %08x", lkb->lkb_remid); + } + + if (lkb->lkb_wait_type) + seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); + + return seq_printf(s, "\n"); +} + +static int print_format1(struct dlm_rsb *res, struct seq_file *s) +{ + struct dlm_lkb *lkb; + int i, lvblen = res->res_ls->ls_lvblen, recover_list, root_list; + int rv; + + lock_rsb(res); + + rv = seq_printf(s, "\nResource %p Name (len=%d) \"", + res, res->res_length); + if (rv) + goto out; + + for (i = 0; i < res->res_length; i++) { + if (isprint(res->res_name[i])) + seq_printf(s, "%c", res->res_name[i]); + else + seq_printf(s, "%c", '.'); + } + + if (res->res_nodeid > 0) + rv = seq_printf(s, "\" \nLocal Copy, Master is node %d\n", + res->res_nodeid); + else if (res->res_nodeid == 0) + rv = seq_printf(s, "\" \nMaster Copy\n"); + else if (res->res_nodeid == -1) + rv = seq_printf(s, "\" \nLooking up master (lkid %x)\n", + res->res_first_lkid); + else + rv = seq_printf(s, "\" \nInvalid master %d\n", + res->res_nodeid); + if (rv) + goto out; + + /* Print the LVB: */ + if (res->res_lvbptr) { + seq_printf(s, "LVB: "); + for (i = 0; i < lvblen; i++) { + if (i == lvblen / 2) + seq_printf(s, "\n "); + seq_printf(s, "%02x ", + (unsigned char) res->res_lvbptr[i]); + } + if (rsb_flag(res, RSB_VALNOTVALID)) + seq_printf(s, " (INVALID)"); + rv = seq_printf(s, "\n"); + if (rv) + goto out; + } + + root_list = !list_empty(&res->res_root_list); + recover_list = !list_empty(&res->res_recover_list); + + if (root_list || recover_list) { + rv = seq_printf(s, "Recovery: root %d recover %d flags %lx " + "count %d\n", root_list, recover_list, + res->res_flags, res->res_recover_locks_count); + if (rv) + goto out; + } + + /* Print the locks attached to this resource */ + seq_printf(s, "Granted Queue\n"); + list_for_each_entry(lkb, &res->res_grantqueue, lkb_statequeue) { + rv = print_format1_lock(s, lkb, res); + if (rv) + goto out; + } + + seq_printf(s, "Conversion Queue\n"); + list_for_each_entry(lkb, &res->res_convertqueue, lkb_statequeue) { + rv = print_format1_lock(s, lkb, res); + if (rv) + goto out; + } + + seq_printf(s, "Waiting Queue\n"); + list_for_each_entry(lkb, &res->res_waitqueue, lkb_statequeue) { + rv = print_format1_lock(s, lkb, res); + if (rv) + goto out; + } + + if (list_empty(&res->res_lookup)) + goto out; + + seq_printf(s, "Lookup Queue\n"); + list_for_each_entry(lkb, &res->res_lookup, lkb_rsb_lookup) { + rv = seq_printf(s, "%08x %s", lkb->lkb_id, + print_lockmode(lkb->lkb_rqmode)); + if (lkb->lkb_wait_type) + seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); + rv = seq_printf(s, "\n"); + } + out: + unlock_rsb(res); + return rv; +} + +static int print_format2_lock(struct seq_file *s, struct dlm_lkb *lkb, + struct dlm_rsb *r) +{ + u64 xid = 0; + u64 us; + int rv; + + if (lkb->lkb_flags & DLM_IFL_USER) { + if (lkb->lkb_ua) + xid = lkb->lkb_ua->xid; + } + + /* microseconds since lkb was added to current queue */ + us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_timestamp)); + + /* id nodeid remid pid xid exflags flags sts grmode rqmode time_us + r_nodeid r_len r_name */ + + rv = seq_printf(s, "%x %d %x %u %llu %x %x %d %d %d %llu %u %d \"%s\"\n", + lkb->lkb_id, + lkb->lkb_nodeid, + lkb->lkb_remid, + lkb->lkb_ownpid, + (unsigned long long)xid, + lkb->lkb_exflags, + lkb->lkb_flags, + lkb->lkb_status, + lkb->lkb_grmode, + lkb->lkb_rqmode, + (unsigned long long)us, + r->res_nodeid, + r->res_length, + r->res_name); + return rv; +} + +static int print_format2(struct dlm_rsb *r, struct seq_file *s) +{ + struct dlm_lkb *lkb; + int rv = 0; + + lock_rsb(r); + + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + rv = print_format2_lock(s, lkb, r); + if (rv) + goto out; + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + rv = print_format2_lock(s, lkb, r); + if (rv) + goto out; + } + + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { + rv = print_format2_lock(s, lkb, r); + if (rv) + goto out; + } + out: + unlock_rsb(r); + return rv; +} + +static int print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb, + int rsb_lookup) +{ + u64 xid = 0; + int rv; + + if (lkb->lkb_flags & DLM_IFL_USER) { + if (lkb->lkb_ua) + xid = lkb->lkb_ua->xid; + } + + rv = seq_printf(s, "lkb %x %d %x %u %llu %x %x %d %d %d %d %d %d %u %llu %llu\n", + lkb->lkb_id, + lkb->lkb_nodeid, + lkb->lkb_remid, + lkb->lkb_ownpid, + (unsigned long long)xid, + lkb->lkb_exflags, + lkb->lkb_flags, + lkb->lkb_status, + lkb->lkb_grmode, + lkb->lkb_rqmode, + lkb->lkb_last_bast.mode, + rsb_lookup, + lkb->lkb_wait_type, + lkb->lkb_lvbseq, + (unsigned long long)ktime_to_ns(lkb->lkb_timestamp), + (unsigned long long)ktime_to_ns(lkb->lkb_last_bast_time)); + return rv; +} + +static int print_format3(struct dlm_rsb *r, struct seq_file *s) +{ + struct dlm_lkb *lkb; + int i, lvblen = r->res_ls->ls_lvblen; + int print_name = 1; + int rv; + + lock_rsb(r); + + rv = seq_printf(s, "rsb %p %d %x %lx %d %d %u %d ", + r, + r->res_nodeid, + r->res_first_lkid, + r->res_flags, + !list_empty(&r->res_root_list), + !list_empty(&r->res_recover_list), + r->res_recover_locks_count, + r->res_length); + if (rv) + goto out; + + for (i = 0; i < r->res_length; i++) { + if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) + print_name = 0; + } + + seq_printf(s, "%s", print_name ? "str " : "hex"); + + for (i = 0; i < r->res_length; i++) { + if (print_name) + seq_printf(s, "%c", r->res_name[i]); + else + seq_printf(s, " %02x", (unsigned char)r->res_name[i]); + } + rv = seq_printf(s, "\n"); + if (rv) + goto out; + + if (!r->res_lvbptr) + goto do_locks; + + seq_printf(s, "lvb %u %d", r->res_lvbseq, lvblen); + + for (i = 0; i < lvblen; i++) + seq_printf(s, " %02x", (unsigned char)r->res_lvbptr[i]); + rv = seq_printf(s, "\n"); + if (rv) + goto out; + + do_locks: + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + rv = print_format3_lock(s, lkb, 0); + if (rv) + goto out; + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + rv = print_format3_lock(s, lkb, 0); + if (rv) + goto out; + } + + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { + rv = print_format3_lock(s, lkb, 0); + if (rv) + goto out; + } + + list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) { + rv = print_format3_lock(s, lkb, 1); + if (rv) + goto out; + } + out: + unlock_rsb(r); + return rv; +} + +static int print_format4(struct dlm_rsb *r, struct seq_file *s) +{ + int our_nodeid = dlm_our_nodeid(); + int print_name = 1; + int i, rv; + + lock_rsb(r); + + rv = seq_printf(s, "rsb %p %d %d %d %d %lu %lx %d ", + r, + r->res_nodeid, + r->res_master_nodeid, + r->res_dir_nodeid, + our_nodeid, + r->res_toss_time, + r->res_flags, + r->res_length); + if (rv) + goto out; + + for (i = 0; i < r->res_length; i++) { + if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) + print_name = 0; + } + + seq_printf(s, "%s", print_name ? "str " : "hex"); + + for (i = 0; i < r->res_length; i++) { + if (print_name) + seq_printf(s, "%c", r->res_name[i]); + else + seq_printf(s, " %02x", (unsigned char)r->res_name[i]); + } + rv = seq_printf(s, "\n"); + out: + unlock_rsb(r); + return rv; +} + +struct rsbtbl_iter { + struct dlm_rsb *rsb; + unsigned bucket; + int format; + int header; +}; + +/* seq_printf returns -1 if the buffer is full, and 0 otherwise. + If the buffer is full, seq_printf can be called again, but it + does nothing and just returns -1. So, the these printing routines + periodically check the return value to avoid wasting too much time + trying to print to a full buffer. */ + +static int table_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct rsbtbl_iter *ri = iter_ptr; + int rv = 0; + + switch (ri->format) { + case 1: + rv = print_format1(ri->rsb, seq); + break; + case 2: + if (ri->header) { + seq_printf(seq, "id nodeid remid pid xid exflags " + "flags sts grmode rqmode time_ms " + "r_nodeid r_len r_name\n"); + ri->header = 0; + } + rv = print_format2(ri->rsb, seq); + break; + case 3: + if (ri->header) { + seq_printf(seq, "version rsb 1.1 lvb 1.1 lkb 1.1\n"); + ri->header = 0; + } + rv = print_format3(ri->rsb, seq); + break; + case 4: + if (ri->header) { + seq_printf(seq, "version 4 rsb 2\n"); + ri->header = 0; + } + rv = print_format4(ri->rsb, seq); + break; + } + + return rv; +} + +static const struct seq_operations format1_seq_ops; +static const struct seq_operations format2_seq_ops; +static const struct seq_operations format3_seq_ops; +static const struct seq_operations format4_seq_ops; + +static void *table_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct rb_root *tree; + struct rb_node *node; + struct dlm_ls *ls = seq->private; + struct rsbtbl_iter *ri; + struct dlm_rsb *r; + loff_t n = *pos; + unsigned bucket, entry; + int toss = (seq->op == &format4_seq_ops); + + bucket = n >> 32; + entry = n & ((1LL << 32) - 1); + + if (bucket >= ls->ls_rsbtbl_size) + return NULL; + + ri = kzalloc(sizeof(struct rsbtbl_iter), GFP_NOFS); + if (!ri) + return NULL; + if (n == 0) + ri->header = 1; + if (seq->op == &format1_seq_ops) + ri->format = 1; + if (seq->op == &format2_seq_ops) + ri->format = 2; + if (seq->op == &format3_seq_ops) + ri->format = 3; + if (seq->op == &format4_seq_ops) + ri->format = 4; + + tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + if (!RB_EMPTY_ROOT(tree)) { + for (node = rb_first(tree); node; node = rb_next(node)) { + r = rb_entry(node, struct dlm_rsb, res_hashnode); + if (!entry--) { + dlm_hold_rsb(r); + ri->rsb = r; + ri->bucket = bucket; + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + return ri; + } + } + } + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + + /* + * move to the first rsb in the next non-empty bucket + */ + + /* zero the entry */ + n &= ~((1LL << 32) - 1); + + while (1) { + bucket++; + n += 1LL << 32; + + if (bucket >= ls->ls_rsbtbl_size) { + kfree(ri); + return NULL; + } + tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + if (!RB_EMPTY_ROOT(tree)) { + node = rb_first(tree); + r = rb_entry(node, struct dlm_rsb, res_hashnode); + dlm_hold_rsb(r); + ri->rsb = r; + ri->bucket = bucket; + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + *pos = n; + return ri; + } + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + } +} + +static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos) +{ + struct dlm_ls *ls = seq->private; + struct rsbtbl_iter *ri = iter_ptr; + struct rb_root *tree; + struct rb_node *next; + struct dlm_rsb *r, *rp; + loff_t n = *pos; + unsigned bucket; + int toss = (seq->op == &format4_seq_ops); + + bucket = n >> 32; + + /* + * move to the next rsb in the same bucket + */ + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + rp = ri->rsb; + next = rb_next(&rp->res_hashnode); + + if (next) { + r = rb_entry(next, struct dlm_rsb, res_hashnode); + dlm_hold_rsb(r); + ri->rsb = r; + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + dlm_put_rsb(rp); + ++*pos; + return ri; + } + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + dlm_put_rsb(rp); + + /* + * move to the first rsb in the next non-empty bucket + */ + + /* zero the entry */ + n &= ~((1LL << 32) - 1); + + while (1) { + bucket++; + n += 1LL << 32; + + if (bucket >= ls->ls_rsbtbl_size) { + kfree(ri); + return NULL; + } + tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + if (!RB_EMPTY_ROOT(tree)) { + next = rb_first(tree); + r = rb_entry(next, struct dlm_rsb, res_hashnode); + dlm_hold_rsb(r); + ri->rsb = r; + ri->bucket = bucket; + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + *pos = n; + return ri; + } + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + } +} + +static void table_seq_stop(struct seq_file *seq, void *iter_ptr) +{ + struct rsbtbl_iter *ri = iter_ptr; + + if (ri) { + dlm_put_rsb(ri->rsb); + kfree(ri); + } +} + +static const struct seq_operations format1_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format2_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format3_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format4_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct file_operations format1_fops; +static const struct file_operations format2_fops; +static const struct file_operations format3_fops; +static const struct file_operations format4_fops; + +static int table_open(struct inode *inode, struct file *file) +{ + struct seq_file *seq; + int ret = -1; + + if (file->f_op == &format1_fops) + ret = seq_open(file, &format1_seq_ops); + else if (file->f_op == &format2_fops) + ret = seq_open(file, &format2_seq_ops); + else if (file->f_op == &format3_fops) + ret = seq_open(file, &format3_seq_ops); + else if (file->f_op == &format4_fops) + ret = seq_open(file, &format4_seq_ops); + + if (ret) + return ret; + + seq = file->private_data; + seq->private = inode->i_private; /* the dlm_ls */ + return 0; +} + +static const struct file_operations format1_fops = { + .owner = THIS_MODULE, + .open = table_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format2_fops = { + .owner = THIS_MODULE, + .open = table_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format3_fops = { + .owner = THIS_MODULE, + .open = table_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format4_fops = { + .owner = THIS_MODULE, + .open = table_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +/* + * dump lkb's on the ls_waiters list + */ +static ssize_t waiters_read(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct dlm_ls *ls = file->private_data; + struct dlm_lkb *lkb; + size_t len = DLM_DEBUG_BUF_LEN, pos = 0, ret, rv; + + mutex_lock(&debug_buf_lock); + mutex_lock(&ls->ls_waiters_mutex); + memset(debug_buf, 0, sizeof(debug_buf)); + + list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { + ret = snprintf(debug_buf + pos, len - pos, "%x %d %d %s\n", + lkb->lkb_id, lkb->lkb_wait_type, + lkb->lkb_nodeid, lkb->lkb_resource->res_name); + if (ret >= len - pos) + break; + pos += ret; + } + mutex_unlock(&ls->ls_waiters_mutex); + + rv = simple_read_from_buffer(userbuf, count, ppos, debug_buf, pos); + mutex_unlock(&debug_buf_lock); + return rv; +} + +static const struct file_operations waiters_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = waiters_read, + .llseek = default_llseek, +}; + +void dlm_delete_debug_file(struct dlm_ls *ls) +{ + if (ls->ls_debug_rsb_dentry) + debugfs_remove(ls->ls_debug_rsb_dentry); + if (ls->ls_debug_waiters_dentry) + debugfs_remove(ls->ls_debug_waiters_dentry); + if (ls->ls_debug_locks_dentry) + debugfs_remove(ls->ls_debug_locks_dentry); + if (ls->ls_debug_all_dentry) + debugfs_remove(ls->ls_debug_all_dentry); + if (ls->ls_debug_toss_dentry) + debugfs_remove(ls->ls_debug_toss_dentry); +} + +int dlm_create_debug_file(struct dlm_ls *ls) +{ + char name[DLM_LOCKSPACE_LEN+8]; + + /* format 1 */ + + ls->ls_debug_rsb_dentry = debugfs_create_file(ls->ls_name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format1_fops); + if (!ls->ls_debug_rsb_dentry) + goto fail; + + /* format 2 */ + + memset(name, 0, sizeof(name)); + snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_locks", ls->ls_name); + + ls->ls_debug_locks_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format2_fops); + if (!ls->ls_debug_locks_dentry) + goto fail; + + /* format 3 */ + + memset(name, 0, sizeof(name)); + snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_all", ls->ls_name); + + ls->ls_debug_all_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format3_fops); + if (!ls->ls_debug_all_dentry) + goto fail; + + /* format 4 */ + + memset(name, 0, sizeof(name)); + snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_toss", ls->ls_name); + + ls->ls_debug_toss_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format4_fops); + if (!ls->ls_debug_toss_dentry) + goto fail; + + memset(name, 0, sizeof(name)); + snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_waiters", ls->ls_name); + + ls->ls_debug_waiters_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &waiters_fops); + if (!ls->ls_debug_waiters_dentry) + goto fail; + + return 0; + + fail: + dlm_delete_debug_file(ls); + return -ENOMEM; +} + +int __init dlm_register_debugfs(void) +{ + mutex_init(&debug_buf_lock); + dlm_root = debugfs_create_dir("dlm", NULL); + return dlm_root ? 0 : -ENOMEM; +} + +void dlm_unregister_debugfs(void) +{ + debugfs_remove(dlm_root); +} + diff --git a/kmod/dlm/dir.c b/kmod/dlm/dir.c new file mode 100644 index 00000000..278a75cd --- /dev/null +++ b/kmod/dlm/dir.c @@ -0,0 +1,308 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "lowcomms.h" +#include "rcom.h" +#include "config.h" +#include "memory.h" +#include "recover.h" +#include "util.h" +#include "lock.h" +#include "dir.h" + +/* + * We use the upper 16 bits of the hash value to select the directory node. + * Low bits are used for distribution of rsb's among hash buckets on each node. + * + * To give the exact range wanted (0 to num_nodes-1), we apply a modulus of + * num_nodes to the hash value. This value in the desired range is used as an + * offset into the sorted list of nodeid's to give the particular nodeid. + */ + +int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash) +{ + uint32_t node; + + if (ls->ls_num_nodes == 1) + return dlm_our_nodeid(); + else { + node = (hash >> 16) % ls->ls_total_weight; + return ls->ls_node_array[node]; + } +} + +int dlm_dir_nodeid(struct dlm_rsb *r) +{ + return r->res_dir_nodeid; +} + +void dlm_recover_dir_nodeid(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + + down_read(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + r->res_dir_nodeid = dlm_hash2nodeid(ls, r->res_hash); + } + up_read(&ls->ls_root_sem); +} + +int dlm_recover_directory(struct dlm_ls *ls) +{ + struct dlm_member *memb; + char *b, *last_name = NULL; + int error = -ENOMEM, last_len, nodeid, result; + uint16_t namelen; + unsigned int count = 0, count_match = 0, count_bad = 0, count_add = 0; + + log_debug(ls, "dlm_recover_directory"); + + if (dlm_no_directory(ls)) + goto out_status; + + last_name = kmalloc(DLM_RESNAME_MAXLEN, GFP_NOFS); + if (!last_name) + goto out; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->nodeid == dlm_our_nodeid()) + continue; + + memset(last_name, 0, DLM_RESNAME_MAXLEN); + last_len = 0; + + for (;;) { + int left; + error = dlm_recovery_stopped(ls); + if (error) + goto out_free; + + error = dlm_rcom_names(ls, memb->nodeid, + last_name, last_len); + if (error) + goto out_free; + + cond_resched(); + + /* + * pick namelen/name pairs out of received buffer + */ + + b = ls->ls_recover_buf->rc_buf; + left = ls->ls_recover_buf->rc_header.h_length; + left -= sizeof(struct dlm_rcom); + + for (;;) { + __be16 v; + + error = -EINVAL; + if (left < sizeof(__be16)) + goto out_free; + + memcpy(&v, b, sizeof(__be16)); + namelen = be16_to_cpu(v); + b += sizeof(__be16); + left -= sizeof(__be16); + + /* namelen of 0xFFFFF marks end of names for + this node; namelen of 0 marks end of the + buffer */ + + if (namelen == 0xFFFF) + goto done; + if (!namelen) + break; + + if (namelen > left) + goto out_free; + + if (namelen > DLM_RESNAME_MAXLEN) + goto out_free; + + error = dlm_master_lookup(ls, memb->nodeid, + b, namelen, + DLM_LU_RECOVER_DIR, + &nodeid, &result); + if (error) { + log_error(ls, "recover_dir lookup %d", + error); + goto out_free; + } + + /* The name was found in rsbtbl, but the + * master nodeid is different from + * memb->nodeid which says it is the master. + * This should not happen. */ + + if (result == DLM_LU_MATCH && + nodeid != memb->nodeid) { + count_bad++; + log_error(ls, "recover_dir lookup %d " + "nodeid %d memb %d bad %u", + result, nodeid, memb->nodeid, + count_bad); + print_hex_dump_bytes("dlm_recover_dir ", + DUMP_PREFIX_NONE, + b, namelen); + } + + /* The name was found in rsbtbl, and the + * master nodeid matches memb->nodeid. */ + + if (result == DLM_LU_MATCH && + nodeid == memb->nodeid) { + count_match++; + } + + /* The name was not found in rsbtbl and was + * added with memb->nodeid as the master. */ + + if (result == DLM_LU_ADD) { + count_add++; + } + + last_len = namelen; + memcpy(last_name, b, namelen); + b += namelen; + left -= namelen; + count++; + } + } + done: + ; + } + + out_status: + error = 0; + dlm_set_recover_status(ls, DLM_RS_DIR); + + log_debug(ls, "dlm_recover_directory %u in %u new", + count, count_add); + out_free: + kfree(last_name); + out: + return error; +} + +static struct dlm_rsb *find_rsb_root(struct dlm_ls *ls, char *name, int len) +{ + struct dlm_rsb *r; + uint32_t hash, bucket; + int rv; + + hash = jhash(name, len, 0); + bucket = hash & (ls->ls_rsbtbl_size - 1); + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[bucket].keep, name, len, &r); + if (rv) + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[bucket].toss, + name, len, &r); + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + + if (!rv) + return r; + + down_read(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + if (len == r->res_length && !memcmp(name, r->res_name, len)) { + up_read(&ls->ls_root_sem); + log_debug(ls, "find_rsb_root revert to root_list %s", + r->res_name); + return r; + } + } + up_read(&ls->ls_root_sem); + return NULL; +} + +/* Find the rsb where we left off (or start again), then send rsb names + for rsb's we're master of and whose directory node matches the requesting + node. inbuf is the rsb name last sent, inlen is the name's length */ + +void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen, + char *outbuf, int outlen, int nodeid) +{ + struct list_head *list; + struct dlm_rsb *r; + int offset = 0, dir_nodeid; + __be16 be_namelen; + + down_read(&ls->ls_root_sem); + + if (inlen > 1) { + r = find_rsb_root(ls, inbuf, inlen); + if (!r) { + inbuf[inlen - 1] = '\0'; + log_error(ls, "copy_master_names from %d start %d %s", + nodeid, inlen, inbuf); + goto out; + } + list = r->res_root_list.next; + } else { + list = ls->ls_root_list.next; + } + + for (offset = 0; list != &ls->ls_root_list; list = list->next) { + r = list_entry(list, struct dlm_rsb, res_root_list); + if (r->res_nodeid) + continue; + + dir_nodeid = dlm_dir_nodeid(r); + if (dir_nodeid != nodeid) + continue; + + /* + * The block ends when we can't fit the following in the + * remaining buffer space: + * namelen (uint16_t) + + * name (r->res_length) + + * end-of-block record 0x0000 (uint16_t) + */ + + if (offset + sizeof(uint16_t)*2 + r->res_length > outlen) { + /* Write end-of-block record */ + be_namelen = cpu_to_be16(0); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + ls->ls_recover_dir_sent_msg++; + goto out; + } + + be_namelen = cpu_to_be16(r->res_length); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + memcpy(outbuf + offset, r->res_name, r->res_length); + offset += r->res_length; + ls->ls_recover_dir_sent_res++; + } + + /* + * If we've reached the end of the list (and there's room) write a + * terminating record. + */ + + if ((list == &ls->ls_root_list) && + (offset + sizeof(uint16_t) <= outlen)) { + be_namelen = cpu_to_be16(0xFFFF); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + ls->ls_recover_dir_sent_msg++; + } + out: + up_read(&ls->ls_root_sem); +} + diff --git a/kmod/dlm/dir.h b/kmod/dlm/dir.h new file mode 100644 index 00000000..41750634 --- /dev/null +++ b/kmod/dlm/dir.h @@ -0,0 +1,25 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __DIR_DOT_H__ +#define __DIR_DOT_H__ + +int dlm_dir_nodeid(struct dlm_rsb *rsb); +int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash); +void dlm_recover_dir_nodeid(struct dlm_ls *ls); +int dlm_recover_directory(struct dlm_ls *ls); +void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen, + char *outbuf, int outlen, int nodeid); + +#endif /* __DIR_DOT_H__ */ + diff --git a/kmod/dlm/dlm_internal.h b/kmod/dlm/dlm_internal.h new file mode 100644 index 00000000..e7665c31 --- /dev/null +++ b/kmod/dlm/dlm_internal.h @@ -0,0 +1,727 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __DLM_INTERNAL_DOT_H__ +#define __DLM_INTERNAL_DOT_H__ + +/* + * This is the main header file to be included in each DLM source file. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "config.h" + +/* Size of the temp buffer midcomms allocates on the stack. + We try to make this large enough so most messages fit. + FIXME: should sctp make this unnecessary? */ + +#define DLM_INBUF_LEN 148 + +struct dlm_ls; +struct dlm_lkb; +struct dlm_rsb; +struct dlm_member; +struct dlm_rsbtable; +struct dlm_recover; +struct dlm_header; +struct dlm_message; +struct dlm_rcom; +struct dlm_mhandle; + +#define log_print(fmt, args...) \ + printk(KERN_ERR "dlm: "fmt"\n" , ##args) +#define log_error(ls, fmt, args...) \ + printk(KERN_ERR "dlm: %s: " fmt "\n", (ls)->ls_name , ##args) + +#define log_debug(ls, fmt, args...) \ +do { \ + if (dlm_config.ci_log_debug) \ + printk(KERN_DEBUG "dlm: %s: " fmt "\n", \ + (ls)->ls_name , ##args); \ +} while (0) + +#define log_limit(ls, fmt, args...) \ +do { \ + if (dlm_config.ci_log_debug) \ + printk_ratelimited(KERN_DEBUG "dlm: %s: " fmt "\n", \ + (ls)->ls_name , ##args); \ +} while (0) + +#define DLM_ASSERT(x, do) \ +{ \ + if (!(x)) \ + { \ + printk(KERN_ERR "\nDLM: Assertion failed on line %d of file %s\n" \ + "DLM: assertion: \"%s\"\n" \ + "DLM: time = %lu\n", \ + __LINE__, __FILE__, #x, jiffies); \ + {do} \ + printk("\n"); \ + BUG(); \ + panic("DLM: Record message above and reboot.\n"); \ + } \ +} + + +#define DLM_RTF_SHRINK 0x00000001 + +struct dlm_rsbtable { + struct rb_root keep; + struct rb_root toss; + spinlock_t lock; + uint32_t flags; +}; + + +/* + * Lockspace member (per node in a ls) + */ + +struct dlm_member { + struct list_head list; + int nodeid; + int weight; + int slot; + int slot_prev; + int comm_seq; + uint32_t generation; +}; + +/* + * Save and manage recovery state for a lockspace. + */ + +struct dlm_recover { + struct list_head list; + struct dlm_config_node *nodes; + int nodes_count; + uint64_t seq; +}; + +/* + * Pass input args to second stage locking function. + */ + +struct dlm_args { + uint32_t flags; + void (*astfn) (void *astparam); + void *astparam; + void (*bastfn) (void *astparam, int mode); + int mode; + struct dlm_lksb *lksb; + unsigned long timeout; +}; + + +/* + * Lock block + * + * A lock can be one of three types: + * + * local copy lock is mastered locally + * (lkb_nodeid is zero and DLM_LKF_MSTCPY is not set) + * process copy lock is mastered on a remote node + * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is not set) + * master copy master node's copy of a lock owned by remote node + * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is set) + * + * lkb_exflags: a copy of the most recent flags arg provided to dlm_lock or + * dlm_unlock. The dlm does not modify these or use any private flags in + * this field; it only contains DLM_LKF_ flags from dlm.h. These flags + * are sent as-is to the remote master when the lock is remote. + * + * lkb_flags: internal dlm flags (DLM_IFL_ prefix) from dlm_internal.h. + * Some internal flags are shared between the master and process nodes; + * these shared flags are kept in the lower two bytes. One of these + * flags set on the master copy will be propagated to the process copy + * and v.v. Other internal flags are private to the master or process + * node (e.g. DLM_IFL_MSTCPY). These are kept in the high two bytes. + * + * lkb_sbflags: status block flags. These flags are copied directly into + * the caller's lksb.sb_flags prior to the dlm_lock/dlm_unlock completion + * ast. All defined in dlm.h with DLM_SBF_ prefix. + * + * lkb_status: the lock status indicates which rsb queue the lock is + * on, grant, convert, or wait. DLM_LKSTS_ WAITING/GRANTED/CONVERT + * + * lkb_wait_type: the dlm message type (DLM_MSG_ prefix) for which a + * reply is needed. Only set when the lkb is on the lockspace waiters + * list awaiting a reply from a remote node. + * + * lkb_nodeid: when the lkb is a local copy, nodeid is 0; when the lkb + * is a master copy, nodeid specifies the remote lock holder, when the + * lkb is a process copy, the nodeid specifies the lock master. + */ + +/* lkb_status */ + +#define DLM_LKSTS_WAITING 1 +#define DLM_LKSTS_GRANTED 2 +#define DLM_LKSTS_CONVERT 3 + +/* lkb_flags */ + +#define DLM_IFL_MSTCPY 0x00010000 +#define DLM_IFL_RESEND 0x00020000 +#define DLM_IFL_DEAD 0x00040000 +#define DLM_IFL_OVERLAP_UNLOCK 0x00080000 +#define DLM_IFL_OVERLAP_CANCEL 0x00100000 +#define DLM_IFL_ENDOFLIFE 0x00200000 +#define DLM_IFL_WATCH_TIMEWARN 0x00400000 +#define DLM_IFL_TIMEOUT_CANCEL 0x00800000 +#define DLM_IFL_DEADLOCK_CANCEL 0x01000000 +#define DLM_IFL_STUB_MS 0x02000000 /* magic number for m_flags */ +#define DLM_IFL_USER 0x00000001 +#define DLM_IFL_ORPHAN 0x00000002 + +#define DLM_CALLBACKS_SIZE 6 + +#define DLM_CB_CAST 0x00000001 +#define DLM_CB_BAST 0x00000002 +#define DLM_CB_SKIP 0x00000004 + +struct dlm_callback { + uint64_t seq; + uint32_t flags; /* DLM_CBF_ */ + int sb_status; /* copy to lksb status */ + uint8_t sb_flags; /* copy to lksb flags */ + int8_t mode; /* rq mode of bast, gr mode of cast */ +}; + +struct dlm_lkb { + struct dlm_rsb *lkb_resource; /* the rsb */ + struct kref lkb_ref; + int lkb_nodeid; /* copied from rsb */ + int lkb_ownpid; /* pid of lock owner */ + uint32_t lkb_id; /* our lock ID */ + uint32_t lkb_remid; /* lock ID on remote partner */ + uint32_t lkb_exflags; /* external flags from caller */ + uint32_t lkb_sbflags; /* lksb flags */ + uint32_t lkb_flags; /* internal flags */ + uint32_t lkb_lvbseq; /* lvb sequence number */ + + int8_t lkb_status; /* granted, waiting, convert */ + int8_t lkb_rqmode; /* requested lock mode */ + int8_t lkb_grmode; /* granted lock mode */ + int8_t lkb_highbast; /* highest mode bast sent for */ + + int8_t lkb_wait_type; /* type of reply waiting for */ + int8_t lkb_wait_count; + int lkb_wait_nodeid; /* for debugging */ + + struct list_head lkb_statequeue; /* rsb g/c/w list */ + struct list_head lkb_rsb_lookup; /* waiting for rsb lookup */ + struct list_head lkb_wait_reply; /* waiting for remote reply */ + struct list_head lkb_ownqueue; /* list of locks for a process */ + struct list_head lkb_time_list; + ktime_t lkb_timestamp; + ktime_t lkb_wait_time; + unsigned long lkb_timeout_cs; + + struct mutex lkb_cb_mutex; + struct work_struct lkb_cb_work; + struct list_head lkb_cb_list; /* for ls_cb_delay or proc->asts */ + struct dlm_callback lkb_callbacks[DLM_CALLBACKS_SIZE]; + struct dlm_callback lkb_last_cast; + struct dlm_callback lkb_last_bast; + ktime_t lkb_last_cast_time; /* for debugging */ + ktime_t lkb_last_bast_time; /* for debugging */ + + uint64_t lkb_recover_seq; /* from ls_recover_seq */ + + char *lkb_lvbptr; + struct dlm_lksb *lkb_lksb; /* caller's status block */ + void (*lkb_astfn) (void *astparam); + void (*lkb_bastfn) (void *astparam, int mode); + union { + void *lkb_astparam; /* caller's ast arg */ + struct dlm_user_args *lkb_ua; + }; +}; + +/* + * res_master_nodeid is "normal": 0 is unset/invalid, non-zero is the real + * nodeid, even when nodeid is our_nodeid. + * + * res_nodeid is "odd": -1 is unset/invalid, zero means our_nodeid, + * greater than zero when another nodeid. + * + * (TODO: remove res_nodeid and only use res_master_nodeid) + */ + +struct dlm_rsb { + struct dlm_ls *res_ls; /* the lockspace */ + struct kref res_ref; + struct mutex res_mutex; + unsigned long res_flags; + int res_length; /* length of rsb name */ + int res_nodeid; + int res_master_nodeid; + int res_dir_nodeid; + int res_id; /* for ls_recover_idr */ + uint32_t res_lvbseq; + uint32_t res_hash; + uint32_t res_bucket; /* rsbtbl */ + unsigned long res_toss_time; + uint32_t res_first_lkid; + struct list_head res_lookup; /* lkbs waiting on first */ + union { + struct list_head res_hashchain; + struct rb_node res_hashnode; /* rsbtbl */ + }; + struct list_head res_grantqueue; + struct list_head res_convertqueue; + struct list_head res_waitqueue; + + struct list_head res_root_list; /* used for recovery */ + struct list_head res_recover_list; /* used for recovery */ + int res_recover_locks_count; + + char *res_lvbptr; + char res_name[DLM_RESNAME_MAXLEN+1]; +}; + +/* dlm_master_lookup() flags */ + +#define DLM_LU_RECOVER_DIR 1 +#define DLM_LU_RECOVER_MASTER 2 + +/* dlm_master_lookup() results */ + +#define DLM_LU_MATCH 1 +#define DLM_LU_ADD 2 + +/* find_rsb() flags */ + +#define R_REQUEST 0x00000001 +#define R_RECEIVE_REQUEST 0x00000002 +#define R_RECEIVE_RECOVER 0x00000004 + +/* rsb_flags */ + +enum rsb_flags { + RSB_MASTER_UNCERTAIN, + RSB_VALNOTVALID, + RSB_VALNOTVALID_PREV, + RSB_NEW_MASTER, + RSB_NEW_MASTER2, + RSB_RECOVER_CONVERT, + RSB_RECOVER_GRANT, + RSB_RECOVER_LVB_INVAL, +}; + +static inline void rsb_set_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + __set_bit(flag, &r->res_flags); +} + +static inline void rsb_clear_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + __clear_bit(flag, &r->res_flags); +} + +static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + return test_bit(flag, &r->res_flags); +} + + +/* dlm_header is first element of all structs sent between nodes */ + +#define DLM_HEADER_MAJOR 0x00030000 +#define DLM_HEADER_MINOR 0x00000001 + +#define DLM_HEADER_SLOTS 0x00000001 + +#define DLM_MSG 1 +#define DLM_RCOM 2 + +struct dlm_header { + uint32_t h_version; + uint32_t h_lockspace; + uint32_t h_nodeid; /* nodeid of sender */ + uint16_t h_length; + uint8_t h_cmd; /* DLM_MSG, DLM_RCOM */ + uint8_t h_pad; +}; + + +#define DLM_MSG_REQUEST 1 +#define DLM_MSG_CONVERT 2 +#define DLM_MSG_UNLOCK 3 +#define DLM_MSG_CANCEL 4 +#define DLM_MSG_REQUEST_REPLY 5 +#define DLM_MSG_CONVERT_REPLY 6 +#define DLM_MSG_UNLOCK_REPLY 7 +#define DLM_MSG_CANCEL_REPLY 8 +#define DLM_MSG_GRANT 9 +#define DLM_MSG_BAST 10 +#define DLM_MSG_LOOKUP 11 +#define DLM_MSG_REMOVE 12 +#define DLM_MSG_LOOKUP_REPLY 13 +#define DLM_MSG_PURGE 14 + +struct dlm_message { + struct dlm_header m_header; + uint32_t m_type; /* DLM_MSG_ */ + uint32_t m_nodeid; + uint32_t m_pid; + uint32_t m_lkid; /* lkid on sender */ + uint32_t m_remid; /* lkid on receiver */ + uint32_t m_parent_lkid; + uint32_t m_parent_remid; + uint32_t m_exflags; + uint32_t m_sbflags; + uint32_t m_flags; + uint32_t m_lvbseq; + uint32_t m_hash; + int m_status; + int m_grmode; + int m_rqmode; + int m_bastmode; + int m_asts; + int m_result; /* 0 or -EXXX */ + char m_extra[0]; /* name or lvb */ +}; + + +#define DLM_RS_NODES 0x00000001 +#define DLM_RS_NODES_ALL 0x00000002 +#define DLM_RS_DIR 0x00000004 +#define DLM_RS_DIR_ALL 0x00000008 +#define DLM_RS_LOCKS 0x00000010 +#define DLM_RS_LOCKS_ALL 0x00000020 +#define DLM_RS_DONE 0x00000040 +#define DLM_RS_DONE_ALL 0x00000080 + +#define DLM_RCOM_STATUS 1 +#define DLM_RCOM_NAMES 2 +#define DLM_RCOM_LOOKUP 3 +#define DLM_RCOM_LOCK 4 +#define DLM_RCOM_STATUS_REPLY 5 +#define DLM_RCOM_NAMES_REPLY 6 +#define DLM_RCOM_LOOKUP_REPLY 7 +#define DLM_RCOM_LOCK_REPLY 8 + +struct dlm_rcom { + struct dlm_header rc_header; + uint32_t rc_type; /* DLM_RCOM_ */ + int rc_result; /* multi-purpose */ + uint64_t rc_id; /* match reply with request */ + uint64_t rc_seq; /* sender's ls_recover_seq */ + uint64_t rc_seq_reply; /* remote ls_recover_seq */ + char rc_buf[0]; +}; + +union dlm_packet { + struct dlm_header header; /* common to other two */ + struct dlm_message message; + struct dlm_rcom rcom; +}; + +#define DLM_RSF_NEED_SLOTS 0x00000001 + +/* RCOM_STATUS data */ +struct rcom_status { + __le32 rs_flags; + __le32 rs_unused1; + __le64 rs_unused2; +}; + +/* RCOM_STATUS_REPLY data */ +struct rcom_config { + __le32 rf_lvblen; + __le32 rf_lsflags; + + /* DLM_HEADER_SLOTS adds: */ + __le32 rf_flags; + __le16 rf_our_slot; + __le16 rf_num_slots; + __le32 rf_generation; + __le32 rf_unused1; + __le64 rf_unused2; +}; + +struct rcom_slot { + __le32 ro_nodeid; + __le16 ro_slot; + __le16 ro_unused1; + __le64 ro_unused2; +}; + +struct rcom_lock { + __le32 rl_ownpid; + __le32 rl_lkid; + __le32 rl_remid; + __le32 rl_parent_lkid; + __le32 rl_parent_remid; + __le32 rl_exflags; + __le32 rl_flags; + __le32 rl_lvbseq; + __le32 rl_result; + int8_t rl_rqmode; + int8_t rl_grmode; + int8_t rl_status; + int8_t rl_asts; + __le16 rl_wait_type; + __le16 rl_namelen; + char rl_name[DLM_RESNAME_MAXLEN]; + char rl_lvb[0]; +}; + +/* + * The max number of resources per rsbtbl bucket that shrink will attempt + * to remove in each iteration. + */ + +#define DLM_REMOVE_NAMES_MAX 8 + +struct dlm_ls { + struct list_head ls_list; /* list of lockspaces */ + dlm_lockspace_t *ls_local_handle; + uint32_t ls_global_id; /* global unique lockspace ID */ + uint32_t ls_generation; + uint32_t ls_exflags; + int ls_lvblen; + int ls_count; /* refcount of processes in + the dlm using this ls */ + int ls_create_count; /* create/release refcount */ + unsigned long ls_flags; /* LSFL_ */ + unsigned long ls_scan_time; + struct kobject ls_kobj; + + struct idr ls_lkbidr; + spinlock_t ls_lkbidr_spin; + + struct dlm_rsbtable *ls_rsbtbl; + uint32_t ls_rsbtbl_size; + + struct mutex ls_waiters_mutex; + struct list_head ls_waiters; /* lkbs needing a reply */ + + struct mutex ls_orphans_mutex; + struct list_head ls_orphans; + + struct mutex ls_timeout_mutex; + struct list_head ls_timeout; + + spinlock_t ls_new_rsb_spin; + int ls_new_rsb_count; + struct list_head ls_new_rsb; /* new rsb structs */ + + spinlock_t ls_remove_spin; + char ls_remove_name[DLM_RESNAME_MAXLEN+1]; + char *ls_remove_names[DLM_REMOVE_NAMES_MAX]; + int ls_remove_len; + int ls_remove_lens[DLM_REMOVE_NAMES_MAX]; + + struct list_head ls_nodes; /* current nodes in ls */ + struct list_head ls_nodes_gone; /* dead node list, recovery */ + int ls_num_nodes; /* number of nodes in ls */ + int ls_low_nodeid; + int ls_total_weight; + int *ls_node_array; + + int ls_slot; + int ls_num_slots; + int ls_slots_size; + struct dlm_slot *ls_slots; + + struct dlm_rsb ls_stub_rsb; /* for returning errors */ + struct dlm_lkb ls_stub_lkb; /* for returning errors */ + struct dlm_message ls_stub_ms; /* for faking a reply */ + + struct dentry *ls_debug_rsb_dentry; /* debugfs */ + struct dentry *ls_debug_waiters_dentry; /* debugfs */ + struct dentry *ls_debug_locks_dentry; /* debugfs */ + struct dentry *ls_debug_all_dentry; /* debugfs */ + struct dentry *ls_debug_toss_dentry; /* debugfs */ + + wait_queue_head_t ls_uevent_wait; /* user part of join/leave */ + int ls_uevent_result; + struct completion ls_members_done; + int ls_members_result; + + struct miscdevice ls_device; + + struct workqueue_struct *ls_callback_wq; + + /* recovery related */ + + struct mutex ls_cb_mutex; + struct list_head ls_cb_delay; /* save for queue_work later */ + struct timer_list ls_timer; + struct task_struct *ls_recoverd_task; + struct mutex ls_recoverd_active; + spinlock_t ls_recover_lock; + unsigned long ls_recover_begin; /* jiffies timestamp */ + uint32_t ls_recover_status; /* DLM_RS_ */ + uint64_t ls_recover_seq; + struct dlm_recover *ls_recover_args; + struct rw_semaphore ls_in_recovery; /* block local requests */ + struct rw_semaphore ls_recv_active; /* block dlm_recv */ + struct list_head ls_requestqueue;/* queue remote requests */ + struct mutex ls_requestqueue_mutex; + struct dlm_rcom *ls_recover_buf; + int ls_recover_nodeid; /* for debugging */ + unsigned int ls_recover_dir_sent_res; /* for log info */ + unsigned int ls_recover_dir_sent_msg; /* for log info */ + unsigned int ls_recover_locks_in; /* for log info */ + uint64_t ls_rcom_seq; + spinlock_t ls_rcom_spin; + struct list_head ls_recover_list; + spinlock_t ls_recover_list_lock; + int ls_recover_list_count; + struct idr ls_recover_idr; + spinlock_t ls_recover_idr_lock; + wait_queue_head_t ls_wait_general; + wait_queue_head_t ls_recover_lock_wait; + struct mutex ls_clear_proc_locks; + + struct list_head ls_root_list; /* root resources */ + struct rw_semaphore ls_root_sem; /* protect root_list */ + + const struct dlm_lockspace_ops *ls_ops; + void *ls_ops_arg; + + int ls_namelen; + char ls_name[1]; +}; + +/* + * LSFL_RECOVER_STOP - dlm_ls_stop() sets this to tell dlm recovery routines + * that they should abort what they're doing so new recovery can be started. + * + * LSFL_RECOVER_DOWN - dlm_ls_stop() sets this to tell dlm_recoverd that it + * should do down_write() on the in_recovery rw_semaphore. (doing down_write + * within dlm_ls_stop causes complaints about the lock acquired/released + * in different contexts.) + * + * LSFL_RECOVER_LOCK - dlm_recoverd holds the in_recovery rw_semaphore. + * It sets this after it is done with down_write() on the in_recovery + * rw_semaphore and clears it after it has released the rw_semaphore. + * + * LSFL_RECOVER_WORK - dlm_ls_start() sets this to tell dlm_recoverd that it + * should begin recovery of the lockspace. + * + * LSFL_RUNNING - set when normal locking activity is enabled. + * dlm_ls_stop() clears this to tell dlm locking routines that they should + * quit what they are doing so recovery can run. dlm_recoverd sets + * this after recovery is finished. + */ + +#define LSFL_RECOVER_STOP 0 +#define LSFL_RECOVER_DOWN 1 +#define LSFL_RECOVER_LOCK 2 +#define LSFL_RECOVER_WORK 3 +#define LSFL_RUNNING 4 + +#define LSFL_RCOM_READY 5 +#define LSFL_RCOM_WAIT 6 +#define LSFL_UEVENT_WAIT 7 +#define LSFL_TIMEWARN 8 +#define LSFL_CB_DELAY 9 +#define LSFL_NODIR 10 + +/* much of this is just saving user space pointers associated with the + lock that we pass back to the user lib with an ast */ + +struct dlm_user_args { + struct dlm_user_proc *proc; /* each process that opens the lockspace + device has private data + (dlm_user_proc) on the struct file, + the process's locks point back to it*/ + struct dlm_lksb lksb; + struct dlm_lksb __user *user_lksb; + void __user *castparam; + void __user *castaddr; + void __user *bastparam; + void __user *bastaddr; + uint64_t xid; +}; + +#define DLM_PROC_FLAGS_CLOSING 1 +#define DLM_PROC_FLAGS_COMPAT 2 + +/* locks list is kept so we can remove all a process's locks when it + exits (or orphan those that are persistent) */ + +struct dlm_user_proc { + dlm_lockspace_t *lockspace; + unsigned long flags; /* DLM_PROC_FLAGS */ + struct list_head asts; + spinlock_t asts_spin; + struct list_head locks; + spinlock_t locks_spin; + struct list_head unlocking; + wait_queue_head_t wait; +}; + +static inline int dlm_locking_stopped(struct dlm_ls *ls) +{ + return !test_bit(LSFL_RUNNING, &ls->ls_flags); +} + +static inline int dlm_recovery_stopped(struct dlm_ls *ls) +{ + return test_bit(LSFL_RECOVER_STOP, &ls->ls_flags); +} + +static inline int dlm_no_directory(struct dlm_ls *ls) +{ + return test_bit(LSFL_NODIR, &ls->ls_flags); +} + +int dlm_netlink_init(void); +void dlm_netlink_exit(void); +void dlm_timeout_warn(struct dlm_lkb *lkb); +int dlm_plock_init(void); +void dlm_plock_exit(void); + +#ifdef CONFIG_DLM_DEBUG +int dlm_register_debugfs(void); +void dlm_unregister_debugfs(void); +int dlm_create_debug_file(struct dlm_ls *ls); +void dlm_delete_debug_file(struct dlm_ls *ls); +#else +static inline int dlm_register_debugfs(void) { return 0; } +static inline void dlm_unregister_debugfs(void) { } +static inline int dlm_create_debug_file(struct dlm_ls *ls) { return 0; } +static inline void dlm_delete_debug_file(struct dlm_ls *ls) { } +#endif + +#endif /* __DLM_INTERNAL_DOT_H__ */ + diff --git a/kmod/dlm/include/linux/dlm.h b/kmod/dlm/include/linux/dlm.h new file mode 100644 index 00000000..d02da2c6 --- /dev/null +++ b/kmod/dlm/include/linux/dlm.h @@ -0,0 +1,172 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ +#ifndef __DLM_DOT_H__ +#define __DLM_DOT_H__ + +#include + + +struct dlm_slot { + int nodeid; /* 1 to MAX_INT */ + int slot; /* 1 to MAX_INT */ +}; + +/* + * recover_prep: called before the dlm begins lock recovery. + * Notfies lockspace user that locks from failed members will be granted. + * recover_slot: called after recover_prep and before recover_done. + * Identifies a failed lockspace member. + * recover_done: called after the dlm completes lock recovery. + * Identifies lockspace members and lockspace generation number. + */ + +struct dlm_lockspace_ops { + void (*recover_prep) (void *ops_arg); + void (*recover_slot) (void *ops_arg, struct dlm_slot *slot); + void (*recover_done) (void *ops_arg, struct dlm_slot *slots, + int num_slots, int our_slot, uint32_t generation); +}; + +/* + * dlm_new_lockspace + * + * Create/join a lockspace. + * + * name: lockspace name, null terminated, up to DLM_LOCKSPACE_LEN (not + * including terminating null). + * + * cluster: cluster name, null terminated, up to DLM_LOCKSPACE_LEN (not + * including terminating null). Optional. When cluster is null, it + * is not used. When set, dlm_new_lockspace() returns -EBADR if cluster + * is not equal to the dlm cluster name. + * + * flags: + * DLM_LSFL_NODIR + * The dlm should not use a resource directory, but statically assign + * resource mastery to nodes based on the name hash that is otherwise + * used to select the directory node. Must be the same on all nodes. + * DLM_LSFL_TIMEWARN + * The dlm should emit netlink messages if locks have been waiting + * for a configurable amount of time. (Unused.) + * DLM_LSFL_FS + * The lockspace user is in the kernel (i.e. filesystem). Enables + * direct bast/cast callbacks. + * DLM_LSFL_NEWEXCL + * dlm_new_lockspace() should return -EEXIST if the lockspace exists. + * + * lvblen: length of lvb in bytes. Must be multiple of 8. + * dlm_new_lockspace() returns an error if this does not match + * what other nodes are using. + * + * ops: callbacks that indicate lockspace recovery points so the + * caller can coordinate its recovery and know lockspace members. + * This is only used by the initial dlm_new_lockspace() call. + * Optional. + * + * ops_arg: arg for ops callbacks. + * + * ops_result: tells caller if the ops callbacks (if provided) will + * be used or not. 0: will be used, -EXXX will not be used. + * -EOPNOTSUPP: the dlm does not have recovery_callbacks enabled. + * + * lockspace: handle for dlm functions + */ + +int dlm_new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, void *ops_arg, + int *ops_result, dlm_lockspace_t **lockspace); + +/* + * dlm_release_lockspace + * + * Stop a lockspace. + */ + +int dlm_release_lockspace(dlm_lockspace_t *lockspace, int force); + +/* + * dlm_lock + * + * Make an asynchronous request to acquire or convert a lock on a named + * resource. + * + * lockspace: context for the request + * mode: the requested mode of the lock (DLM_LOCK_) + * lksb: lock status block for input and async return values + * flags: input flags (DLM_LKF_) + * name: name of the resource to lock, can be binary + * namelen: the length in bytes of the resource name (MAX_RESNAME_LEN) + * parent: the lock ID of a parent lock or 0 if none + * lockast: function DLM executes when it completes processing the request + * astarg: argument passed to lockast and bast functions + * bast: function DLM executes when this lock later blocks another request + * + * Returns: + * 0 if request is successfully queued for processing + * -EINVAL if any input parameters are invalid + * -EAGAIN if request would block and is flagged DLM_LKF_NOQUEUE + * -ENOMEM if there is no memory to process request + * -ENOTCONN if there is a communication error + * + * If the call to dlm_lock returns an error then the operation has failed and + * the AST routine will not be called. If dlm_lock returns 0 it is still + * possible that the lock operation will fail. The AST routine will be called + * when the locking is complete and the status is returned in the lksb. + * + * If the AST routines or parameter are passed to a conversion operation then + * they will overwrite those values that were passed to a previous dlm_lock + * call. + * + * AST routines should not block (at least not for long), but may make + * any locking calls they please. + */ + +int dlm_lock(dlm_lockspace_t *lockspace, + int mode, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*lockast) (void *astarg), + void *astarg, + void (*bast) (void *astarg, int mode)); + +/* + * dlm_unlock + * + * Asynchronously release a lock on a resource. The AST routine is called + * when the resource is successfully unlocked. + * + * lockspace: context for the request + * lkid: the lock ID as returned in the lksb + * flags: input flags (DLM_LKF_) + * lksb: if NULL the lksb parameter passed to last lock request is used + * astarg: the arg used with the completion ast for the unlock + * + * Returns: + * 0 if request is successfully queued for processing + * -EINVAL if any input parameters are invalid + * -ENOTEMPTY if the lock still has sublocks + * -EBUSY if the lock is waiting for a remote lock operation + * -ENOTCONN if there is a communication error + */ + +int dlm_unlock(dlm_lockspace_t *lockspace, + uint32_t lkid, + uint32_t flags, + struct dlm_lksb *lksb, + void *astarg); + +#endif /* __DLM_DOT_H__ */ diff --git a/kmod/dlm/include/linux/dlm_plock.h b/kmod/dlm/include/linux/dlm_plock.h new file mode 100644 index 00000000..95ad387a --- /dev/null +++ b/kmod/dlm/include/linux/dlm_plock.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ +#ifndef __DLM_PLOCK_DOT_H__ +#define __DLM_PLOCK_DOT_H__ + +#include + +int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + int cmd, struct file_lock *fl); +int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl); +int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl); +#endif diff --git a/kmod/dlm/include/uapi/linux/dlm.h b/kmod/dlm/include/uapi/linux/dlm.h new file mode 100644 index 00000000..1f73cc06 --- /dev/null +++ b/kmod/dlm/include/uapi/linux/dlm.h @@ -0,0 +1,75 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef _UAPI__DLM_DOT_H__ +#define _UAPI__DLM_DOT_H__ + +/* + * Interface to Distributed Lock Manager (DLM) + * routines and structures to use DLM lockspaces + */ + +/* Lock levels and flags are here */ +#include +#include + +typedef void dlm_lockspace_t; + +/* + * Lock status block + * + * Use this structure to specify the contents of the lock value block. For a + * conversion request, this structure is used to specify the lock ID of the + * lock. DLM writes the status of the lock request and the lock ID assigned + * to the request in the lock status block. + * + * sb_lkid: the returned lock ID. It is set on new (non-conversion) requests. + * It is available when dlm_lock returns. + * + * sb_lvbptr: saves or returns the contents of the lock's LVB according to rules + * shown for the DLM_LKF_VALBLK flag. + * + * sb_flags: DLM_SBF_DEMOTED is returned if in the process of promoting a lock, + * it was first demoted to NL to avoid conversion deadlock. + * DLM_SBF_VALNOTVALID is returned if the resource's LVB is marked invalid. + * + * sb_status: the returned status of the lock request set prior to AST + * execution. Possible return values: + * + * 0 if lock request was successful + * -EAGAIN if request would block and is flagged DLM_LKF_NOQUEUE + * -DLM_EUNLOCK if unlock request was successful + * -DLM_ECANCEL if a cancel completed successfully + * -EDEADLK if a deadlock was detected + * -ETIMEDOUT if the lock request was canceled due to a timeout + */ + +#define DLM_SBF_DEMOTED 0x01 +#define DLM_SBF_VALNOTVALID 0x02 +#define DLM_SBF_ALTMODE 0x04 + +struct dlm_lksb { + int sb_status; + __u32 sb_lkid; + char sb_flags; + char * sb_lvbptr; +}; + +/* dlm_new_lockspace() flags */ + +#define DLM_LSFL_TIMEWARN 0x00000002 +#define DLM_LSFL_FS 0x00000004 +#define DLM_LSFL_NEWEXCL 0x00000008 + + +#endif /* _UAPI__DLM_DOT_H__ */ diff --git a/kmod/dlm/include/uapi/linux/dlm_device.h b/kmod/dlm/include/uapi/linux/dlm_device.h new file mode 100644 index 00000000..3060783c --- /dev/null +++ b/kmod/dlm/include/uapi/linux/dlm_device.h @@ -0,0 +1,108 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef _LINUX_DLM_DEVICE_H +#define _LINUX_DLM_DEVICE_H + +/* This is the device interface for dlm, most users will use a library + * interface. + */ + +#include +#include + +#define DLM_USER_LVB_LEN 32 + +/* Version of the device interface */ +#define DLM_DEVICE_VERSION_MAJOR 6 +#define DLM_DEVICE_VERSION_MINOR 0 +#define DLM_DEVICE_VERSION_PATCH 1 + +/* struct passed to the lock write */ +struct dlm_lock_params { + __u8 mode; + __u8 namelen; + __u16 unused; + __u32 flags; + __u32 lkid; + __u32 parent; + __u64 xid; + __u64 timeout; + void __user *castparam; + void __user *castaddr; + void __user *bastparam; + void __user *bastaddr; + struct dlm_lksb __user *lksb; + char lvb[DLM_USER_LVB_LEN]; + char name[0]; +}; + +struct dlm_lspace_params { + __u32 flags; + __u32 minor; + char name[0]; +}; + +struct dlm_purge_params { + __u32 nodeid; + __u32 pid; +}; + +struct dlm_write_request { + __u32 version[3]; + __u8 cmd; + __u8 is64bit; + __u8 unused[2]; + + union { + struct dlm_lock_params lock; + struct dlm_lspace_params lspace; + struct dlm_purge_params purge; + } i; +}; + +struct dlm_device_version { + __u32 version[3]; +}; + +/* struct read from the "device" fd, + consists mainly of userspace pointers for the library to use */ + +struct dlm_lock_result { + __u32 version[3]; + __u32 length; + void __user * user_astaddr; + void __user * user_astparam; + struct dlm_lksb __user * user_lksb; + struct dlm_lksb lksb; + __u8 bast_mode; + __u8 unused[3]; + /* Offsets may be zero if no data is present */ + __u32 lvb_offset; +}; + +/* Commands passed to the device */ +#define DLM_USER_LOCK 1 +#define DLM_USER_UNLOCK 2 +#define DLM_USER_QUERY 3 +#define DLM_USER_CREATE_LOCKSPACE 4 +#define DLM_USER_REMOVE_LOCKSPACE 5 +#define DLM_USER_PURGE 6 +#define DLM_USER_DEADLOCK 7 + +/* Lockspace flags */ +#define DLM_USER_LSFLG_AUTOFREE 1 +#define DLM_USER_LSFLG_FORCEFREE 2 + +#endif + diff --git a/kmod/dlm/include/uapi/linux/dlm_netlink.h b/kmod/dlm/include/uapi/linux/dlm_netlink.h new file mode 100644 index 00000000..647c8ef2 --- /dev/null +++ b/kmod/dlm/include/uapi/linux/dlm_netlink.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2007 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ + +#ifndef _DLM_NETLINK_H +#define _DLM_NETLINK_H + +#include + +enum { + DLM_STATUS_WAITING = 1, + DLM_STATUS_GRANTED = 2, + DLM_STATUS_CONVERT = 3, +}; + +#define DLM_LOCK_DATA_VERSION 1 + +struct dlm_lock_data { + __u16 version; + __u32 lockspace_id; + int nodeid; + int ownpid; + __u32 id; + __u32 remid; + __u64 xid; + __s8 status; + __s8 grmode; + __s8 rqmode; + unsigned long timestamp; + int resource_namelen; + char resource_name[DLM_RESNAME_MAXLEN]; +}; + +enum { + DLM_CMD_UNSPEC = 0, + DLM_CMD_HELLO, /* user->kernel */ + DLM_CMD_TIMEOUT, /* kernel->user */ + __DLM_CMD_MAX, +}; + +#define DLM_CMD_MAX (__DLM_CMD_MAX - 1) + +enum { + DLM_TYPE_UNSPEC = 0, + DLM_TYPE_LOCK, + __DLM_TYPE_MAX, +}; + +#define DLM_TYPE_MAX (__DLM_TYPE_MAX - 1) + +#define DLM_GENL_VERSION 0x1 +#define DLM_GENL_NAME "DLM" + +#endif /* _DLM_NETLINK_H */ diff --git a/kmod/dlm/include/uapi/linux/dlm_plock.h b/kmod/dlm/include/uapi/linux/dlm_plock.h new file mode 100644 index 00000000..6ae692c9 --- /dev/null +++ b/kmod/dlm/include/uapi/linux/dlm_plock.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ + +#ifndef _UAPI__DLM_PLOCK_DOT_H__ +#define _UAPI__DLM_PLOCK_DOT_H__ + +#include + +#define DLM_PLOCK_MISC_NAME "dlm_plock" + +#define DLM_PLOCK_VERSION_MAJOR 1 +#define DLM_PLOCK_VERSION_MINOR 2 +#define DLM_PLOCK_VERSION_PATCH 0 + +enum { + DLM_PLOCK_OP_LOCK = 1, + DLM_PLOCK_OP_UNLOCK, + DLM_PLOCK_OP_GET, +}; + +#define DLM_PLOCK_FL_CLOSE 1 + +struct dlm_plock_info { + __u32 version[3]; + __u8 optype; + __u8 ex; + __u8 wait; + __u8 flags; + __u32 pid; + __s32 nodeid; + __s32 rv; + __u32 fsid; + __u64 number; + __u64 start; + __u64 end; + __u64 owner; +}; + + +#endif /* _UAPI__DLM_PLOCK_DOT_H__ */ diff --git a/kmod/dlm/include/uapi/linux/dlmconstants.h b/kmod/dlm/include/uapi/linux/dlmconstants.h new file mode 100644 index 00000000..2857bdc5 --- /dev/null +++ b/kmod/dlm/include/uapi/linux/dlmconstants.h @@ -0,0 +1,163 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __DLMCONSTANTS_DOT_H__ +#define __DLMCONSTANTS_DOT_H__ + +/* + * Constants used by DLM interface. + */ + +#define DLM_LOCKSPACE_LEN 64 +#define DLM_RESNAME_MAXLEN 64 + + +/* + * Lock Modes + */ + +#define DLM_LOCK_IV (-1) /* invalid */ +#define DLM_LOCK_NL 0 /* null */ +#define DLM_LOCK_CR 1 /* concurrent read */ +#define DLM_LOCK_CW 2 /* concurrent write */ +#define DLM_LOCK_PR 3 /* protected read */ +#define DLM_LOCK_PW 4 /* protected write */ +#define DLM_LOCK_EX 5 /* exclusive */ + + +/* + * Flags to dlm_lock + * + * DLM_LKF_NOQUEUE + * + * Do not queue the lock request on the wait queue if it cannot be granted + * immediately. If the lock cannot be granted because of this flag, DLM will + * either return -EAGAIN from the dlm_lock call or will return 0 from + * dlm_lock and -EAGAIN in the lock status block when the AST is executed. + * + * DLM_LKF_CANCEL + * + * Used to cancel a pending lock request or conversion. A converting lock is + * returned to its previously granted mode. + * + * DLM_LKF_CONVERT + * + * Indicates a lock conversion request. For conversions the name and namelen + * are ignored and the lock ID in the LKSB is used to identify the lock. + * + * DLM_LKF_VALBLK + * + * Requests DLM to return the current contents of the lock value block in the + * lock status block. When this flag is set in a lock conversion from PW or EX + * modes, DLM assigns the value specified in the lock status block to the lock + * value block of the lock resource. The LVB is a DLM_LVB_LEN size array + * containing application-specific information. + * + * DLM_LKF_QUECVT + * + * Force a conversion request to be queued, even if it is compatible with + * the granted modes of other locks on the same resource. + * + * DLM_LKF_IVVALBLK + * + * Invalidate the lock value block. + * + * DLM_LKF_CONVDEADLK + * + * Allows the dlm to resolve conversion deadlocks internally by demoting the + * granted mode of a converting lock to NL. The DLM_SBF_DEMOTED flag is + * returned for a conversion that's been effected by this. + * + * DLM_LKF_PERSISTENT + * + * Only relevant to locks originating in userspace. A persistent lock will not + * be removed if the process holding the lock exits. + * + * DLM_LKF_NODLCKWT + * + * Do not cancel the lock if it gets into conversion deadlock. + * Exclude this lock from being monitored due to DLM_LSFL_TIMEWARN. + * + * DLM_LKF_NODLCKBLK + * + * net yet implemented + * + * DLM_LKF_EXPEDITE + * + * Used only with new requests for NL mode locks. Tells the lock manager + * to grant the lock, ignoring other locks in convert and wait queues. + * + * DLM_LKF_NOQUEUEBAST + * + * Send blocking AST's before returning -EAGAIN to the caller. It is only + * used along with the NOQUEUE flag. Blocking AST's are not sent for failed + * NOQUEUE requests otherwise. + * + * DLM_LKF_HEADQUE + * + * Add a lock to the head of the convert or wait queue rather than the tail. + * + * DLM_LKF_NOORDER + * + * Disregard the standard grant order rules and grant a lock as soon as it + * is compatible with other granted locks. + * + * DLM_LKF_ORPHAN + * + * Acquire an orphan lock. + * + * DLM_LKF_ALTPR + * + * If the requested mode cannot be granted immediately, try to grant the lock + * in PR mode instead. If this alternate mode is granted instead of the + * requested mode, DLM_SBF_ALTMODE is returned in the lksb. + * + * DLM_LKF_ALTCW + * + * The same as ALTPR, but the alternate mode is CW. + * + * DLM_LKF_FORCEUNLOCK + * + * Unlock the lock even if it is converting or waiting or has sublocks. + * Only really for use by the userland device.c code. + * + */ + +#define DLM_LKF_NOQUEUE 0x00000001 +#define DLM_LKF_CANCEL 0x00000002 +#define DLM_LKF_CONVERT 0x00000004 +#define DLM_LKF_VALBLK 0x00000008 +#define DLM_LKF_QUECVT 0x00000010 +#define DLM_LKF_IVVALBLK 0x00000020 +#define DLM_LKF_CONVDEADLK 0x00000040 +#define DLM_LKF_PERSISTENT 0x00000080 +#define DLM_LKF_NODLCKWT 0x00000100 +#define DLM_LKF_NODLCKBLK 0x00000200 +#define DLM_LKF_EXPEDITE 0x00000400 +#define DLM_LKF_NOQUEUEBAST 0x00000800 +#define DLM_LKF_HEADQUE 0x00001000 +#define DLM_LKF_NOORDER 0x00002000 +#define DLM_LKF_ORPHAN 0x00004000 +#define DLM_LKF_ALTPR 0x00008000 +#define DLM_LKF_ALTCW 0x00010000 +#define DLM_LKF_FORCEUNLOCK 0x00020000 +#define DLM_LKF_TIMEOUT 0x00040000 + +/* + * Some return codes that are not in errno.h + */ + +#define DLM_ECANCEL 0x10001 +#define DLM_EUNLOCK 0x10002 + +#endif /* __DLMCONSTANTS_DOT_H__ */ diff --git a/kmod/dlm/lock.c b/kmod/dlm/lock.c new file mode 100644 index 00000000..275e171c --- /dev/null +++ b/kmod/dlm/lock.c @@ -0,0 +1,6303 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +/* Central locking logic has four stages: + + dlm_lock() + dlm_unlock() + + request_lock(ls, lkb) + convert_lock(ls, lkb) + unlock_lock(ls, lkb) + cancel_lock(ls, lkb) + + _request_lock(r, lkb) + _convert_lock(r, lkb) + _unlock_lock(r, lkb) + _cancel_lock(r, lkb) + + do_request(r, lkb) + do_convert(r, lkb) + do_unlock(r, lkb) + do_cancel(r, lkb) + + Stage 1 (lock, unlock) is mainly about checking input args and + splitting into one of the four main operations: + + dlm_lock = request_lock + dlm_lock+CONVERT = convert_lock + dlm_unlock = unlock_lock + dlm_unlock+CANCEL = cancel_lock + + Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is + provided to the next stage. + + Stage 3, _xxxx_lock(), determines if the operation is local or remote. + When remote, it calls send_xxxx(), when local it calls do_xxxx(). + + Stage 4, do_xxxx(), is the guts of the operation. It manipulates the + given rsb and lkb and queues callbacks. + + For remote operations, send_xxxx() results in the corresponding do_xxxx() + function being executed on the remote node. The connecting send/receive + calls on local (L) and remote (R) nodes: + + L: send_xxxx() -> R: receive_xxxx() + R: do_xxxx() + L: receive_xxxx_reply() <- R: send_xxxx_reply() +*/ +#include +#include +#include +#include "dlm_internal.h" +#include +#include "memory.h" +#include "lowcomms.h" +#include "requestqueue.h" +#include "util.h" +#include "dir.h" +#include "member.h" +#include "lockspace.h" +#include "ast.h" +#include "lock.h" +#include "rcom.h" +#include "recover.h" +#include "lvb_table.h" +#include "user.h" +#include "config.h" + +static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode); +static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_remove(struct dlm_rsb *r); +static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms); +static int receive_extralen(struct dlm_message *ms); +static void do_purge(struct dlm_ls *ls, int nodeid, int pid); +static void del_timeout(struct dlm_lkb *lkb); +static void toss_rsb(struct kref *kref); + +/* + * Lock compatibilty matrix - thanks Steve + * UN = Unlocked state. Not really a state, used as a flag + * PD = Padding. Used to make the matrix a nice power of two in size + * Other states are the same as the VMS DLM. + * Usage: matrix[grmode+1][rqmode+1] (although m[rq+1][gr+1] is the same) + */ + +static const int __dlm_compat_matrix[8][8] = { + /* UN NL CR CW PR PW EX PD */ + {1, 1, 1, 1, 1, 1, 1, 0}, /* UN */ + {1, 1, 1, 1, 1, 1, 1, 0}, /* NL */ + {1, 1, 1, 1, 1, 1, 0, 0}, /* CR */ + {1, 1, 1, 1, 0, 0, 0, 0}, /* CW */ + {1, 1, 1, 0, 1, 0, 0, 0}, /* PR */ + {1, 1, 1, 0, 0, 0, 0, 0}, /* PW */ + {1, 1, 0, 0, 0, 0, 0, 0}, /* EX */ + {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ +}; + +/* + * This defines the direction of transfer of LVB data. + * Granted mode is the row; requested mode is the column. + * Usage: matrix[grmode+1][rqmode+1] + * 1 = LVB is returned to the caller + * 0 = LVB is written to the resource + * -1 = nothing happens to the LVB + */ + +const int dlm_lvb_operations[8][8] = { + /* UN NL CR CW PR PW EX PD*/ + { -1, 1, 1, 1, 1, 1, 1, -1 }, /* UN */ + { -1, 1, 1, 1, 1, 1, 1, 0 }, /* NL */ + { -1, -1, 1, 1, 1, 1, 1, 0 }, /* CR */ + { -1, -1, -1, 1, 1, 1, 1, 0 }, /* CW */ + { -1, -1, -1, -1, 1, 1, 1, 0 }, /* PR */ + { -1, 0, 0, 0, 0, 0, 1, 0 }, /* PW */ + { -1, 0, 0, 0, 0, 0, 0, 0 }, /* EX */ + { -1, 0, 0, 0, 0, 0, 0, 0 } /* PD */ +}; + +#define modes_compat(gr, rq) \ + __dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1] + +int dlm_modes_compat(int mode1, int mode2) +{ + return __dlm_compat_matrix[mode1 + 1][mode2 + 1]; +} + +/* + * Compatibility matrix for conversions with QUECVT set. + * Granted mode is the row; requested mode is the column. + * Usage: matrix[grmode+1][rqmode+1] + */ + +static const int __quecvt_compat_matrix[8][8] = { + /* UN NL CR CW PR PW EX PD */ + {0, 0, 0, 0, 0, 0, 0, 0}, /* UN */ + {0, 0, 1, 1, 1, 1, 1, 0}, /* NL */ + {0, 0, 0, 1, 1, 1, 1, 0}, /* CR */ + {0, 0, 0, 0, 1, 1, 1, 0}, /* CW */ + {0, 0, 0, 1, 0, 1, 1, 0}, /* PR */ + {0, 0, 0, 0, 0, 0, 1, 0}, /* PW */ + {0, 0, 0, 0, 0, 0, 0, 0}, /* EX */ + {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ +}; + +void dlm_print_lkb(struct dlm_lkb *lkb) +{ + printk(KERN_ERR "lkb: nodeid %d id %x remid %x exflags %x flags %x " + "sts %d rq %d gr %d wait_type %d wait_nodeid %d seq %llu\n", + lkb->lkb_nodeid, lkb->lkb_id, lkb->lkb_remid, lkb->lkb_exflags, + lkb->lkb_flags, lkb->lkb_status, lkb->lkb_rqmode, + lkb->lkb_grmode, lkb->lkb_wait_type, lkb->lkb_wait_nodeid, + (unsigned long long)lkb->lkb_recover_seq); +} + +static void dlm_print_rsb(struct dlm_rsb *r) +{ + printk(KERN_ERR "rsb: nodeid %d master %d dir %d flags %lx first %x " + "rlc %d name %s\n", + r->res_nodeid, r->res_master_nodeid, r->res_dir_nodeid, + r->res_flags, r->res_first_lkid, r->res_recover_locks_count, + r->res_name); +} + +void dlm_dump_rsb(struct dlm_rsb *r) +{ + struct dlm_lkb *lkb; + + dlm_print_rsb(r); + + printk(KERN_ERR "rsb: root_list empty %d recover_list empty %d\n", + list_empty(&r->res_root_list), list_empty(&r->res_recover_list)); + printk(KERN_ERR "rsb lookup list\n"); + list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb grant queue:\n"); + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb convert queue:\n"); + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb wait queue:\n"); + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) + dlm_print_lkb(lkb); +} + +/* Threads cannot use the lockspace while it's being recovered */ + +static inline void dlm_lock_recovery(struct dlm_ls *ls) +{ + down_read(&ls->ls_in_recovery); +} + +void dlm_unlock_recovery(struct dlm_ls *ls) +{ + up_read(&ls->ls_in_recovery); +} + +int dlm_lock_recovery_try(struct dlm_ls *ls) +{ + return down_read_trylock(&ls->ls_in_recovery); +} + +static inline int can_be_queued(struct dlm_lkb *lkb) +{ + return !(lkb->lkb_exflags & DLM_LKF_NOQUEUE); +} + +static inline int force_blocking_asts(struct dlm_lkb *lkb) +{ + return (lkb->lkb_exflags & DLM_LKF_NOQUEUEBAST); +} + +static inline int is_demoted(struct dlm_lkb *lkb) +{ + return (lkb->lkb_sbflags & DLM_SBF_DEMOTED); +} + +static inline int is_altmode(struct dlm_lkb *lkb) +{ + return (lkb->lkb_sbflags & DLM_SBF_ALTMODE); +} + +static inline int is_granted(struct dlm_lkb *lkb) +{ + return (lkb->lkb_status == DLM_LKSTS_GRANTED); +} + +static inline int is_remote(struct dlm_rsb *r) +{ + DLM_ASSERT(r->res_nodeid >= 0, dlm_print_rsb(r);); + return !!r->res_nodeid; +} + +static inline int is_process_copy(struct dlm_lkb *lkb) +{ + return (lkb->lkb_nodeid && !(lkb->lkb_flags & DLM_IFL_MSTCPY)); +} + +static inline int is_master_copy(struct dlm_lkb *lkb) +{ + return (lkb->lkb_flags & DLM_IFL_MSTCPY) ? 1 : 0; +} + +static inline int middle_conversion(struct dlm_lkb *lkb) +{ + if ((lkb->lkb_grmode==DLM_LOCK_PR && lkb->lkb_rqmode==DLM_LOCK_CW) || + (lkb->lkb_rqmode==DLM_LOCK_PR && lkb->lkb_grmode==DLM_LOCK_CW)) + return 1; + return 0; +} + +static inline int down_conversion(struct dlm_lkb *lkb) +{ + return (!middle_conversion(lkb) && lkb->lkb_rqmode < lkb->lkb_grmode); +} + +static inline int is_overlap_unlock(struct dlm_lkb *lkb) +{ + return lkb->lkb_flags & DLM_IFL_OVERLAP_UNLOCK; +} + +static inline int is_overlap_cancel(struct dlm_lkb *lkb) +{ + return lkb->lkb_flags & DLM_IFL_OVERLAP_CANCEL; +} + +static inline int is_overlap(struct dlm_lkb *lkb) +{ + return (lkb->lkb_flags & (DLM_IFL_OVERLAP_UNLOCK | + DLM_IFL_OVERLAP_CANCEL)); +} + +static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + if (is_master_copy(lkb)) + return; + + del_timeout(lkb); + + DLM_ASSERT(lkb->lkb_lksb, dlm_print_lkb(lkb);); + + /* if the operation was a cancel, then return -DLM_ECANCEL, if a + timeout caused the cancel then return -ETIMEDOUT */ + if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_TIMEOUT_CANCEL)) { + lkb->lkb_flags &= ~DLM_IFL_TIMEOUT_CANCEL; + rv = -ETIMEDOUT; + } + + if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_DEADLOCK_CANCEL)) { + lkb->lkb_flags &= ~DLM_IFL_DEADLOCK_CANCEL; + rv = -EDEADLK; + } + + dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, rv, lkb->lkb_sbflags); +} + +static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + queue_cast(r, lkb, + is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL); +} + +static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode) +{ + if (is_master_copy(lkb)) { + send_bast(r, lkb, rqmode); + } else { + dlm_add_cb(lkb, DLM_CB_BAST, rqmode, 0, 0); + } +} + +/* + * Basic operations on rsb's and lkb's + */ + +/* This is only called to add a reference when the code already holds + a valid reference to the rsb, so there's no need for locking. */ + +static inline void hold_rsb(struct dlm_rsb *r) +{ + kref_get(&r->res_ref); +} + +void dlm_hold_rsb(struct dlm_rsb *r) +{ + hold_rsb(r); +} + +/* When all references to the rsb are gone it's transferred to + the tossed list for later disposal. */ + +static void put_rsb(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + uint32_t bucket = r->res_bucket; + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + kref_put(&r->res_ref, toss_rsb); + spin_unlock(&ls->ls_rsbtbl[bucket].lock); +} + +void dlm_put_rsb(struct dlm_rsb *r) +{ + put_rsb(r); +} + +static int pre_rsb_struct(struct dlm_ls *ls) +{ + struct dlm_rsb *r1, *r2; + int count = 0; + + spin_lock(&ls->ls_new_rsb_spin); + if (ls->ls_new_rsb_count > dlm_config.ci_new_rsb_count / 2) { + spin_unlock(&ls->ls_new_rsb_spin); + return 0; + } + spin_unlock(&ls->ls_new_rsb_spin); + + r1 = dlm_allocate_rsb(ls); + r2 = dlm_allocate_rsb(ls); + + spin_lock(&ls->ls_new_rsb_spin); + if (r1) { + list_add(&r1->res_hashchain, &ls->ls_new_rsb); + ls->ls_new_rsb_count++; + } + if (r2) { + list_add(&r2->res_hashchain, &ls->ls_new_rsb); + ls->ls_new_rsb_count++; + } + count = ls->ls_new_rsb_count; + spin_unlock(&ls->ls_new_rsb_spin); + + if (!count) + return -ENOMEM; + return 0; +} + +/* If ls->ls_new_rsb is empty, return -EAGAIN, so the caller can + unlock any spinlocks, go back and call pre_rsb_struct again. + Otherwise, take an rsb off the list and return it. */ + +static int get_rsb_struct(struct dlm_ls *ls, char *name, int len, + struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r; + int count; + + spin_lock(&ls->ls_new_rsb_spin); + if (list_empty(&ls->ls_new_rsb)) { + count = ls->ls_new_rsb_count; + spin_unlock(&ls->ls_new_rsb_spin); + log_debug(ls, "find_rsb retry %d %d %s", + count, dlm_config.ci_new_rsb_count, name); + return -EAGAIN; + } + + r = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, res_hashchain); + list_del(&r->res_hashchain); + /* Convert the empty list_head to a NULL rb_node for tree usage: */ + memset(&r->res_hashnode, 0, sizeof(struct rb_node)); + ls->ls_new_rsb_count--; + spin_unlock(&ls->ls_new_rsb_spin); + + r->res_ls = ls; + r->res_length = len; + memcpy(r->res_name, name, len); + mutex_init(&r->res_mutex); + + INIT_LIST_HEAD(&r->res_lookup); + INIT_LIST_HEAD(&r->res_grantqueue); + INIT_LIST_HEAD(&r->res_convertqueue); + INIT_LIST_HEAD(&r->res_waitqueue); + INIT_LIST_HEAD(&r->res_root_list); + INIT_LIST_HEAD(&r->res_recover_list); + + *r_ret = r; + return 0; +} + +static int rsb_cmp(struct dlm_rsb *r, const char *name, int nlen) +{ + char maxname[DLM_RESNAME_MAXLEN]; + + memset(maxname, 0, DLM_RESNAME_MAXLEN); + memcpy(maxname, name, nlen); + return memcmp(r->res_name, maxname, DLM_RESNAME_MAXLEN); +} + +int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, + struct dlm_rsb **r_ret) +{ + struct rb_node *node = tree->rb_node; + struct dlm_rsb *r; + int rc; + + while (node) { + r = rb_entry(node, struct dlm_rsb, res_hashnode); + rc = rsb_cmp(r, name, len); + if (rc < 0) + node = node->rb_left; + else if (rc > 0) + node = node->rb_right; + else + goto found; + } + *r_ret = NULL; + return -EBADR; + + found: + *r_ret = r; + return 0; +} + +static int rsb_insert(struct dlm_rsb *rsb, struct rb_root *tree) +{ + struct rb_node **newn = &tree->rb_node; + struct rb_node *parent = NULL; + int rc; + + while (*newn) { + struct dlm_rsb *cur = rb_entry(*newn, struct dlm_rsb, + res_hashnode); + + parent = *newn; + rc = rsb_cmp(cur, rsb->res_name, rsb->res_length); + if (rc < 0) + newn = &parent->rb_left; + else if (rc > 0) + newn = &parent->rb_right; + else { + log_print("rsb_insert match"); + dlm_dump_rsb(rsb); + dlm_dump_rsb(cur); + return -EEXIST; + } + } + + rb_link_node(&rsb->res_hashnode, parent, newn); + rb_insert_color(&rsb->res_hashnode, tree); + return 0; +} + +/* + * Find rsb in rsbtbl and potentially create/add one + * + * Delaying the release of rsb's has a similar benefit to applications keeping + * NL locks on an rsb, but without the guarantee that the cached master value + * will still be valid when the rsb is reused. Apps aren't always smart enough + * to keep NL locks on an rsb that they may lock again shortly; this can lead + * to excessive master lookups and removals if we don't delay the release. + * + * Searching for an rsb means looking through both the normal list and toss + * list. When found on the toss list the rsb is moved to the normal list with + * ref count of 1; when found on normal list the ref count is incremented. + * + * rsb's on the keep list are being used locally and refcounted. + * rsb's on the toss list are not being used locally, and are not refcounted. + * + * The toss list rsb's were either + * - previously used locally but not any more (were on keep list, then + * moved to toss list when last refcount dropped) + * - created and put on toss list as a directory record for a lookup + * (we are the dir node for the res, but are not using the res right now, + * but some other node is) + * + * The purpose of find_rsb() is to return a refcounted rsb for local use. + * So, if the given rsb is on the toss list, it is moved to the keep list + * before being returned. + * + * toss_rsb() happens when all local usage of the rsb is done, i.e. no + * more refcounts exist, so the rsb is moved from the keep list to the + * toss list. + * + * rsb's on both keep and toss lists are used for doing a name to master + * lookups. rsb's that are in use locally (and being refcounted) are on + * the keep list, rsb's that are not in use locally (not refcounted) and + * only exist for name/master lookups are on the toss list. + * + * rsb's on the toss list who's dir_nodeid is not local can have stale + * name/master mappings. So, remote requests on such rsb's can potentially + * return with an error, which means the mapping is stale and needs to + * be updated with a new lookup. (The idea behind MASTER UNCERTAIN and + * first_lkid is to keep only a single outstanding request on an rsb + * while that rsb has a potentially stale master.) + */ + +static int find_rsb_dir(struct dlm_ls *ls, char *name, int len, + uint32_t hash, uint32_t b, + int dir_nodeid, int from_nodeid, + unsigned int flags, struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r = NULL; + int our_nodeid = dlm_our_nodeid(); + int from_local = 0; + int from_other = 0; + int from_dir = 0; + int create = 0; + int error; + + if (flags & R_RECEIVE_REQUEST) { + if (from_nodeid == dir_nodeid) + from_dir = 1; + else + from_other = 1; + } else if (flags & R_REQUEST) { + from_local = 1; + } + + /* + * flags & R_RECEIVE_RECOVER is from dlm_recover_master_copy, so + * from_nodeid has sent us a lock in dlm_recover_locks, believing + * we're the new master. Our local recovery may not have set + * res_master_nodeid to our_nodeid yet, so allow either. Don't + * create the rsb; dlm_recover_process_copy() will handle EBADR + * by resending. + * + * If someone sends us a request, we are the dir node, and we do + * not find the rsb anywhere, then recreate it. This happens if + * someone sends us a request after we have removed/freed an rsb + * from our toss list. (They sent a request instead of lookup + * because they are using an rsb from their toss list.) + */ + + if (from_local || from_dir || + (from_other && (dir_nodeid == our_nodeid))) { + create = 1; + } + + retry: + if (create) { + error = pre_rsb_struct(ls); + if (error < 0) + goto out; + } + + spin_lock(&ls->ls_rsbtbl[b].lock); + + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (error) + goto do_toss; + + /* + * rsb is active, so we can't check master_nodeid without lock_rsb. + */ + + kref_get(&r->res_ref); + error = 0; + goto out_unlock; + + + do_toss: + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (error) + goto do_new; + + /* + * rsb found inactive (master_nodeid may be out of date unless + * we are the dir_nodeid or were the master) No other thread + * is using this rsb because it's on the toss list, so we can + * look at or update res_master_nodeid without lock_rsb. + */ + + if ((r->res_master_nodeid != our_nodeid) && from_other) { + /* our rsb was not master, and another node (not the dir node) + has sent us a request */ + log_debug(ls, "find_rsb toss from_other %d master %d dir %d %s", + from_nodeid, r->res_master_nodeid, dir_nodeid, + r->res_name); + error = -ENOTBLK; + goto out_unlock; + } + + if ((r->res_master_nodeid != our_nodeid) && from_dir) { + /* don't think this should ever happen */ + log_error(ls, "find_rsb toss from_dir %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + /* fix it and go on */ + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = 0; + } + + if (from_local && (r->res_master_nodeid != our_nodeid)) { + /* Because we have held no locks on this rsb, + res_master_nodeid could have become stale. */ + rsb_set_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = 0; + } + + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); + error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); + goto out_unlock; + + + do_new: + /* + * rsb not found + */ + + if (error == -EBADR && !create) + goto out_unlock; + + error = get_rsb_struct(ls, name, len, &r); + if (error == -EAGAIN) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + goto retry; + } + if (error) + goto out_unlock; + + r->res_hash = hash; + r->res_bucket = b; + r->res_dir_nodeid = dir_nodeid; + kref_init(&r->res_ref); + + if (from_dir) { + /* want to see how often this happens */ + log_debug(ls, "find_rsb new from_dir %d recreate %s", + from_nodeid, r->res_name); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + goto out_add; + } + + if (from_other && (dir_nodeid != our_nodeid)) { + /* should never happen */ + log_error(ls, "find_rsb new from_other %d dir %d our %d %s", + from_nodeid, dir_nodeid, our_nodeid, r->res_name); + dlm_free_rsb(r); + error = -ENOTBLK; + goto out_unlock; + } + + if (from_other) { + log_debug(ls, "find_rsb new from_other %d dir %d %s", + from_nodeid, dir_nodeid, r->res_name); + } + + if (dir_nodeid == our_nodeid) { + /* When we are the dir nodeid, we can set the master + node immediately */ + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } else { + /* set_master will send_lookup to dir_nodeid */ + r->res_master_nodeid = 0; + r->res_nodeid = -1; + } + + out_add: + error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); + out_unlock: + spin_unlock(&ls->ls_rsbtbl[b].lock); + out: + *r_ret = r; + return error; +} + +/* During recovery, other nodes can send us new MSTCPY locks (from + dlm_recover_locks) before we've made ourself master (in + dlm_recover_masters). */ + +static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len, + uint32_t hash, uint32_t b, + int dir_nodeid, int from_nodeid, + unsigned int flags, struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r = NULL; + int our_nodeid = dlm_our_nodeid(); + int recover = (flags & R_RECEIVE_RECOVER); + int error; + + retry: + error = pre_rsb_struct(ls); + if (error < 0) + goto out; + + spin_lock(&ls->ls_rsbtbl[b].lock); + + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (error) + goto do_toss; + + /* + * rsb is active, so we can't check master_nodeid without lock_rsb. + */ + + kref_get(&r->res_ref); + goto out_unlock; + + + do_toss: + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (error) + goto do_new; + + /* + * rsb found inactive. No other thread is using this rsb because + * it's on the toss list, so we can look at or update + * res_master_nodeid without lock_rsb. + */ + + if (!recover && (r->res_master_nodeid != our_nodeid) && from_nodeid) { + /* our rsb is not master, and another node has sent us a + request; this should never happen */ + log_error(ls, "find_rsb toss from_nodeid %d master %d dir %d", + from_nodeid, r->res_master_nodeid, dir_nodeid); + dlm_print_rsb(r); + error = -ENOTBLK; + goto out_unlock; + } + + if (!recover && (r->res_master_nodeid != our_nodeid) && + (dir_nodeid == our_nodeid)) { + /* our rsb is not master, and we are dir; may as well fix it; + this should never happen */ + log_error(ls, "find_rsb toss our %d master %d dir %d", + our_nodeid, r->res_master_nodeid, dir_nodeid); + dlm_print_rsb(r); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } + + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); + error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); + goto out_unlock; + + + do_new: + /* + * rsb not found + */ + + error = get_rsb_struct(ls, name, len, &r); + if (error == -EAGAIN) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + goto retry; + } + if (error) + goto out_unlock; + + r->res_hash = hash; + r->res_bucket = b; + r->res_dir_nodeid = dir_nodeid; + r->res_master_nodeid = dir_nodeid; + r->res_nodeid = (dir_nodeid == our_nodeid) ? 0 : dir_nodeid; + kref_init(&r->res_ref); + + error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); + out_unlock: + spin_unlock(&ls->ls_rsbtbl[b].lock); + out: + *r_ret = r; + return error; +} + +static int find_rsb(struct dlm_ls *ls, char *name, int len, int from_nodeid, + unsigned int flags, struct dlm_rsb **r_ret) +{ + uint32_t hash, b; + int dir_nodeid; + + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; + + hash = jhash(name, len, 0); + b = hash & (ls->ls_rsbtbl_size - 1); + + dir_nodeid = dlm_hash2nodeid(ls, hash); + + if (dlm_no_directory(ls)) + return find_rsb_nodir(ls, name, len, hash, b, dir_nodeid, + from_nodeid, flags, r_ret); + else + return find_rsb_dir(ls, name, len, hash, b, dir_nodeid, + from_nodeid, flags, r_ret); +} + +/* we have received a request and found that res_master_nodeid != our_nodeid, + so we need to return an error or make ourself the master */ + +static int validate_master_nodeid(struct dlm_ls *ls, struct dlm_rsb *r, + int from_nodeid) +{ + if (dlm_no_directory(ls)) { + log_error(ls, "find_rsb keep from_nodeid %d master %d dir %d", + from_nodeid, r->res_master_nodeid, + r->res_dir_nodeid); + dlm_print_rsb(r); + return -ENOTBLK; + } + + if (from_nodeid != r->res_dir_nodeid) { + /* our rsb is not master, and another node (not the dir node) + has sent us a request. this is much more common when our + master_nodeid is zero, so limit debug to non-zero. */ + + if (r->res_master_nodeid) { + log_debug(ls, "validate master from_other %d master %d " + "dir %d first %x %s", from_nodeid, + r->res_master_nodeid, r->res_dir_nodeid, + r->res_first_lkid, r->res_name); + } + return -ENOTBLK; + } else { + /* our rsb is not master, but the dir nodeid has sent us a + request; this could happen with master 0 / res_nodeid -1 */ + + if (r->res_master_nodeid) { + log_error(ls, "validate master from_dir %d master %d " + "first %x %s", + from_nodeid, r->res_master_nodeid, + r->res_first_lkid, r->res_name); + } + + r->res_master_nodeid = dlm_our_nodeid(); + r->res_nodeid = 0; + return 0; + } +} + +/* + * We're the dir node for this res and another node wants to know the + * master nodeid. During normal operation (non recovery) this is only + * called from receive_lookup(); master lookups when the local node is + * the dir node are done by find_rsb(). + * + * normal operation, we are the dir node for a resource + * . _request_lock + * . set_master + * . send_lookup + * . receive_lookup + * . dlm_master_lookup flags 0 + * + * recover directory, we are rebuilding dir for all resources + * . dlm_recover_directory + * . dlm_rcom_names + * remote node sends back the rsb names it is master of and we are dir of + * . dlm_master_lookup RECOVER_DIR (fix_master 0, from_master 1) + * we either create new rsb setting remote node as master, or find existing + * rsb and set master to be the remote node. + * + * recover masters, we are finding the new master for resources + * . dlm_recover_masters + * . recover_master + * . dlm_send_rcom_lookup + * . receive_rcom_lookup + * . dlm_master_lookup RECOVER_MASTER (fix_master 1, from_master 0) + */ + +int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, char *name, int len, + unsigned int flags, int *r_nodeid, int *result) +{ + struct dlm_rsb *r = NULL; + uint32_t hash, b; + int from_master = (flags & DLM_LU_RECOVER_DIR); + int fix_master = (flags & DLM_LU_RECOVER_MASTER); + int our_nodeid = dlm_our_nodeid(); + int dir_nodeid, error, toss_list = 0; + + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; + + if (from_nodeid == our_nodeid) { + log_error(ls, "dlm_master_lookup from our_nodeid %d flags %x", + our_nodeid, flags); + return -EINVAL; + } + + hash = jhash(name, len, 0); + b = hash & (ls->ls_rsbtbl_size - 1); + + dir_nodeid = dlm_hash2nodeid(ls, hash); + if (dir_nodeid != our_nodeid) { + log_error(ls, "dlm_master_lookup from %d dir %d our %d h %x %d", + from_nodeid, dir_nodeid, our_nodeid, hash, + ls->ls_num_nodes); + *r_nodeid = -1; + return -EINVAL; + } + + retry: + error = pre_rsb_struct(ls); + if (error < 0) + return error; + + spin_lock(&ls->ls_rsbtbl[b].lock); + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (!error) { + /* because the rsb is active, we need to lock_rsb before + checking/changing re_master_nodeid */ + + hold_rsb(r); + spin_unlock(&ls->ls_rsbtbl[b].lock); + lock_rsb(r); + goto found; + } + + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (error) + goto not_found; + + /* because the rsb is inactive (on toss list), it's not refcounted + and lock_rsb is not used, but is protected by the rsbtbl lock */ + + toss_list = 1; + found: + if (r->res_dir_nodeid != our_nodeid) { + /* should not happen, but may as well fix it and carry on */ + log_error(ls, "dlm_master_lookup res_dir %d our %d %s", + r->res_dir_nodeid, our_nodeid, r->res_name); + r->res_dir_nodeid = our_nodeid; + } + + if (fix_master && dlm_is_removed(ls, r->res_master_nodeid)) { + /* Recovery uses this function to set a new master when + the previous master failed. Setting NEW_MASTER will + force dlm_recover_masters to call recover_master on this + rsb even though the res_nodeid is no longer removed. */ + + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + rsb_set_flag(r, RSB_NEW_MASTER); + + if (toss_list) { + /* I don't think we should ever find it on toss list. */ + log_error(ls, "dlm_master_lookup fix_master on toss"); + dlm_dump_rsb(r); + } + } + + if (from_master && (r->res_master_nodeid != from_nodeid)) { + /* this will happen if from_nodeid became master during + a previous recovery cycle, and we aborted the previous + cycle before recovering this master value */ + + log_limit(ls, "dlm_master_lookup from_master %d " + "master_nodeid %d res_nodeid %d first %x %s", + from_nodeid, r->res_master_nodeid, r->res_nodeid, + r->res_first_lkid, r->res_name); + + if (r->res_master_nodeid == our_nodeid) { + log_error(ls, "from_master %d our_master", from_nodeid); + dlm_dump_rsb(r); + dlm_send_rcom_lookup_dump(r, from_nodeid); + goto out_found; + } + + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + rsb_set_flag(r, RSB_NEW_MASTER); + } + + if (!r->res_master_nodeid) { + /* this will happen if recovery happens while we're looking + up the master for this rsb */ + + log_debug(ls, "dlm_master_lookup master 0 to %d first %x %s", + from_nodeid, r->res_first_lkid, r->res_name); + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + } + + if (!from_master && !fix_master && + (r->res_master_nodeid == from_nodeid)) { + /* this can happen when the master sends remove, the dir node + finds the rsb on the keep list and ignores the remove, + and the former master sends a lookup */ + + log_limit(ls, "dlm_master_lookup from master %d flags %x " + "first %x %s", from_nodeid, flags, + r->res_first_lkid, r->res_name); + } + + out_found: + *r_nodeid = r->res_master_nodeid; + if (result) + *result = DLM_LU_MATCH; + + if (toss_list) { + r->res_toss_time = jiffies; + /* the rsb was inactive (on toss list) */ + spin_unlock(&ls->ls_rsbtbl[b].lock); + } else { + /* the rsb was active */ + unlock_rsb(r); + put_rsb(r); + } + return 0; + + not_found: + error = get_rsb_struct(ls, name, len, &r); + if (error == -EAGAIN) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + goto retry; + } + if (error) + goto out_unlock; + + r->res_hash = hash; + r->res_bucket = b; + r->res_dir_nodeid = our_nodeid; + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + kref_init(&r->res_ref); + r->res_toss_time = jiffies; + + error = rsb_insert(r, &ls->ls_rsbtbl[b].toss); + if (error) { + /* should never happen */ + dlm_free_rsb(r); + spin_unlock(&ls->ls_rsbtbl[b].lock); + goto retry; + } + + if (result) + *result = DLM_LU_ADD; + *r_nodeid = from_nodeid; + error = 0; + out_unlock: + spin_unlock(&ls->ls_rsbtbl[b].lock); + return error; +} + +static void dlm_dump_rsb_hash(struct dlm_ls *ls, uint32_t hash) +{ + struct rb_node *n; + struct dlm_rsb *r; + int i; + + for (i = 0; i < ls->ls_rsbtbl_size; i++) { + spin_lock(&ls->ls_rsbtbl[i].lock); + for (n = rb_first(&ls->ls_rsbtbl[i].keep); n; n = rb_next(n)) { + r = rb_entry(n, struct dlm_rsb, res_hashnode); + if (r->res_hash == hash) + dlm_dump_rsb(r); + } + spin_unlock(&ls->ls_rsbtbl[i].lock); + } +} + +void dlm_dump_rsb_name(struct dlm_ls *ls, char *name, int len) +{ + struct dlm_rsb *r = NULL; + uint32_t hash, b; + int error; + + hash = jhash(name, len, 0); + b = hash & (ls->ls_rsbtbl_size - 1); + + spin_lock(&ls->ls_rsbtbl[b].lock); + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (!error) + goto out_dump; + + error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (error) + goto out; + out_dump: + dlm_dump_rsb(r); + out: + spin_unlock(&ls->ls_rsbtbl[b].lock); +} + +static void toss_rsb(struct kref *kref) +{ + struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref); + struct dlm_ls *ls = r->res_ls; + + DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r);); + kref_init(&r->res_ref); + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[r->res_bucket].keep); + rsb_insert(r, &ls->ls_rsbtbl[r->res_bucket].toss); + r->res_toss_time = jiffies; + ls->ls_rsbtbl[r->res_bucket].flags |= DLM_RTF_SHRINK; + if (r->res_lvbptr) { + dlm_free_lvb(r->res_lvbptr); + r->res_lvbptr = NULL; + } +} + +/* See comment for unhold_lkb */ + +static void unhold_rsb(struct dlm_rsb *r) +{ + int rv; + rv = kref_put(&r->res_ref, toss_rsb); + DLM_ASSERT(!rv, dlm_dump_rsb(r);); +} + +static void kill_rsb(struct kref *kref) +{ + struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref); + + /* All work is done after the return from kref_put() so we + can release the write_lock before the remove and free. */ + + DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r);); +} + +/* Attaching/detaching lkb's from rsb's is for rsb reference counting. + The rsb must exist as long as any lkb's for it do. */ + +static void attach_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + hold_rsb(r); + lkb->lkb_resource = r; +} + +static void detach_lkb(struct dlm_lkb *lkb) +{ + if (lkb->lkb_resource) { + put_rsb(lkb->lkb_resource); + lkb->lkb_resource = NULL; + } +} + +static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret) +{ + struct dlm_lkb *lkb; + int rv; + + lkb = dlm_allocate_lkb(ls); + if (!lkb) + return -ENOMEM; + + lkb->lkb_nodeid = -1; + lkb->lkb_grmode = DLM_LOCK_IV; + kref_init(&lkb->lkb_ref); + INIT_LIST_HEAD(&lkb->lkb_ownqueue); + INIT_LIST_HEAD(&lkb->lkb_rsb_lookup); + INIT_LIST_HEAD(&lkb->lkb_time_list); + INIT_LIST_HEAD(&lkb->lkb_cb_list); + mutex_init(&lkb->lkb_cb_mutex); + INIT_WORK(&lkb->lkb_cb_work, dlm_callback_work); + + idr_preload(GFP_NOFS); + spin_lock(&ls->ls_lkbidr_spin); + rv = idr_alloc(&ls->ls_lkbidr, lkb, 1, 0, GFP_NOWAIT); + if (rv >= 0) + lkb->lkb_id = rv; + spin_unlock(&ls->ls_lkbidr_spin); + idr_preload_end(); + + if (rv < 0) { + log_error(ls, "create_lkb idr error %d", rv); + return rv; + } + + *lkb_ret = lkb; + return 0; +} + +static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret) +{ + struct dlm_lkb *lkb; + + spin_lock(&ls->ls_lkbidr_spin); + lkb = idr_find(&ls->ls_lkbidr, lkid); + if (lkb) + kref_get(&lkb->lkb_ref); + spin_unlock(&ls->ls_lkbidr_spin); + + *lkb_ret = lkb; + return lkb ? 0 : -ENOENT; +} + +static void kill_lkb(struct kref *kref) +{ + struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref); + + /* All work is done after the return from kref_put() so we + can release the write_lock before the detach_lkb */ + + DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); +} + +/* __put_lkb() is used when an lkb may not have an rsb attached to + it so we need to provide the lockspace explicitly */ + +static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + uint32_t lkid = lkb->lkb_id; + + spin_lock(&ls->ls_lkbidr_spin); + if (kref_put(&lkb->lkb_ref, kill_lkb)) { + idr_remove(&ls->ls_lkbidr, lkid); + spin_unlock(&ls->ls_lkbidr_spin); + + detach_lkb(lkb); + + /* for local/process lkbs, lvbptr points to caller's lksb */ + if (lkb->lkb_lvbptr && is_master_copy(lkb)) + dlm_free_lvb(lkb->lkb_lvbptr); + dlm_free_lkb(lkb); + return 1; + } else { + spin_unlock(&ls->ls_lkbidr_spin); + return 0; + } +} + +int dlm_put_lkb(struct dlm_lkb *lkb) +{ + struct dlm_ls *ls; + + DLM_ASSERT(lkb->lkb_resource, dlm_print_lkb(lkb);); + DLM_ASSERT(lkb->lkb_resource->res_ls, dlm_print_lkb(lkb);); + + ls = lkb->lkb_resource->res_ls; + return __put_lkb(ls, lkb); +} + +/* This is only called to add a reference when the code already holds + a valid reference to the lkb, so there's no need for locking. */ + +static inline void hold_lkb(struct dlm_lkb *lkb) +{ + kref_get(&lkb->lkb_ref); +} + +/* This is called when we need to remove a reference and are certain + it's not the last ref. e.g. del_lkb is always called between a + find_lkb/put_lkb and is always the inverse of a previous add_lkb. + put_lkb would work fine, but would involve unnecessary locking */ + +static inline void unhold_lkb(struct dlm_lkb *lkb) +{ + int rv; + rv = kref_put(&lkb->lkb_ref, kill_lkb); + DLM_ASSERT(!rv, dlm_print_lkb(lkb);); +} + +static void lkb_add_ordered(struct list_head *new, struct list_head *head, + int mode) +{ + struct dlm_lkb *lkb = NULL; + + list_for_each_entry(lkb, head, lkb_statequeue) + if (lkb->lkb_rqmode < mode) + break; + + __list_add(new, lkb->lkb_statequeue.prev, &lkb->lkb_statequeue); +} + +/* add/remove lkb to rsb's grant/convert/wait queue */ + +static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) +{ + kref_get(&lkb->lkb_ref); + + DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); + + lkb->lkb_timestamp = ktime_get(); + + lkb->lkb_status = status; + + switch (status) { + case DLM_LKSTS_WAITING: + if (lkb->lkb_exflags & DLM_LKF_HEADQUE) + list_add(&lkb->lkb_statequeue, &r->res_waitqueue); + else + list_add_tail(&lkb->lkb_statequeue, &r->res_waitqueue); + break; + case DLM_LKSTS_GRANTED: + /* convention says granted locks kept in order of grmode */ + lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue, + lkb->lkb_grmode); + break; + case DLM_LKSTS_CONVERT: + if (lkb->lkb_exflags & DLM_LKF_HEADQUE) + list_add(&lkb->lkb_statequeue, &r->res_convertqueue); + else + list_add_tail(&lkb->lkb_statequeue, + &r->res_convertqueue); + break; + default: + DLM_ASSERT(0, dlm_print_lkb(lkb); printk("sts=%d\n", status);); + } +} + +static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + lkb->lkb_status = 0; + list_del(&lkb->lkb_statequeue); + unhold_lkb(lkb); +} + +static void move_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int sts) +{ + hold_lkb(lkb); + del_lkb(r, lkb); + add_lkb(r, lkb, sts); + unhold_lkb(lkb); +} + +static int msg_reply_type(int mstype) +{ + switch (mstype) { + case DLM_MSG_REQUEST: + return DLM_MSG_REQUEST_REPLY; + case DLM_MSG_CONVERT: + return DLM_MSG_CONVERT_REPLY; + case DLM_MSG_UNLOCK: + return DLM_MSG_UNLOCK_REPLY; + case DLM_MSG_CANCEL: + return DLM_MSG_CANCEL_REPLY; + case DLM_MSG_LOOKUP: + return DLM_MSG_LOOKUP_REPLY; + } + return -1; +} + +static int nodeid_warned(int nodeid, int num_nodes, int *warned) +{ + int i; + + for (i = 0; i < num_nodes; i++) { + if (!warned[i]) { + warned[i] = nodeid; + return 0; + } + if (warned[i] == nodeid) + return 1; + } + return 0; +} + +void dlm_scan_waiters(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + ktime_t zero = ktime_set(0, 0); + s64 us; + s64 debug_maxus = 0; + u32 debug_scanned = 0; + u32 debug_expired = 0; + int num_nodes = 0; + int *warned = NULL; + + if (!dlm_config.ci_waitwarn_us) + return; + + mutex_lock(&ls->ls_waiters_mutex); + + list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { + if (ktime_equal(lkb->lkb_wait_time, zero)) + continue; + + debug_scanned++; + + us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_wait_time)); + + if (us < dlm_config.ci_waitwarn_us) + continue; + + lkb->lkb_wait_time = zero; + + debug_expired++; + if (us > debug_maxus) + debug_maxus = us; + + if (!num_nodes) { + num_nodes = ls->ls_num_nodes; + warned = kzalloc(num_nodes * sizeof(int), GFP_KERNEL); + } + if (!warned) + continue; + if (nodeid_warned(lkb->lkb_wait_nodeid, num_nodes, warned)) + continue; + + log_error(ls, "waitwarn %x %lld %d us check connection to " + "node %d", lkb->lkb_id, (long long)us, + dlm_config.ci_waitwarn_us, lkb->lkb_wait_nodeid); + } + mutex_unlock(&ls->ls_waiters_mutex); + kfree(warned); + + if (debug_expired) + log_debug(ls, "scan_waiters %u warn %u over %d us max %lld us", + debug_scanned, debug_expired, + dlm_config.ci_waitwarn_us, (long long)debug_maxus); +} + +/* add/remove lkb from global waiters list of lkb's waiting for + a reply from a remote node */ + +static int add_to_waiters(struct dlm_lkb *lkb, int mstype, int to_nodeid) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int error = 0; + + mutex_lock(&ls->ls_waiters_mutex); + + if (is_overlap_unlock(lkb) || + (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL))) { + error = -EINVAL; + goto out; + } + + if (lkb->lkb_wait_type || is_overlap_cancel(lkb)) { + switch (mstype) { + case DLM_MSG_UNLOCK: + lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; + break; + case DLM_MSG_CANCEL: + lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; + break; + default: + error = -EBUSY; + goto out; + } + lkb->lkb_wait_count++; + hold_lkb(lkb); + + log_debug(ls, "addwait %x cur %d overlap %d count %d f %x", + lkb->lkb_id, lkb->lkb_wait_type, mstype, + lkb->lkb_wait_count, lkb->lkb_flags); + goto out; + } + + DLM_ASSERT(!lkb->lkb_wait_count, + dlm_print_lkb(lkb); + printk("wait_count %d\n", lkb->lkb_wait_count);); + + lkb->lkb_wait_count++; + lkb->lkb_wait_type = mstype; + lkb->lkb_wait_time = ktime_get(); + lkb->lkb_wait_nodeid = to_nodeid; /* for debugging */ + hold_lkb(lkb); + list_add(&lkb->lkb_wait_reply, &ls->ls_waiters); + out: + if (error) + log_error(ls, "addwait error %x %d flags %x %d %d %s", + lkb->lkb_id, error, lkb->lkb_flags, mstype, + lkb->lkb_wait_type, lkb->lkb_resource->res_name); + mutex_unlock(&ls->ls_waiters_mutex); + return error; +} + +/* We clear the RESEND flag because we might be taking an lkb off the waiters + list as part of process_requestqueue (e.g. a lookup that has an optimized + request reply on the requestqueue) between dlm_recover_waiters_pre() which + set RESEND and dlm_recover_waiters_post() */ + +static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype, + struct dlm_message *ms) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int overlap_done = 0; + + if (is_overlap_unlock(lkb) && (mstype == DLM_MSG_UNLOCK_REPLY)) { + log_debug(ls, "remwait %x unlock_reply overlap", lkb->lkb_id); + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; + overlap_done = 1; + goto out_del; + } + + if (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL_REPLY)) { + log_debug(ls, "remwait %x cancel_reply overlap", lkb->lkb_id); + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + overlap_done = 1; + goto out_del; + } + + /* Cancel state was preemptively cleared by a successful convert, + see next comment, nothing to do. */ + + if ((mstype == DLM_MSG_CANCEL_REPLY) && + (lkb->lkb_wait_type != DLM_MSG_CANCEL)) { + log_debug(ls, "remwait %x cancel_reply wait_type %d", + lkb->lkb_id, lkb->lkb_wait_type); + return -1; + } + + /* Remove for the convert reply, and premptively remove for the + cancel reply. A convert has been granted while there's still + an outstanding cancel on it (the cancel is moot and the result + in the cancel reply should be 0). We preempt the cancel reply + because the app gets the convert result and then can follow up + with another op, like convert. This subsequent op would see the + lingering state of the cancel and fail with -EBUSY. */ + + if ((mstype == DLM_MSG_CONVERT_REPLY) && + (lkb->lkb_wait_type == DLM_MSG_CONVERT) && + is_overlap_cancel(lkb) && ms && !ms->m_result) { + log_debug(ls, "remwait %x convert_reply zap overlap_cancel", + lkb->lkb_id); + lkb->lkb_wait_type = 0; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + lkb->lkb_wait_count--; + goto out_del; + } + + /* N.B. type of reply may not always correspond to type of original + msg due to lookup->request optimization, verify others? */ + + if (lkb->lkb_wait_type) { + lkb->lkb_wait_type = 0; + goto out_del; + } + + log_error(ls, "remwait error %x remote %d %x msg %d flags %x no wait", + lkb->lkb_id, ms ? ms->m_header.h_nodeid : 0, lkb->lkb_remid, + mstype, lkb->lkb_flags); + return -1; + + out_del: + /* the force-unlock/cancel has completed and we haven't recvd a reply + to the op that was in progress prior to the unlock/cancel; we + give up on any reply to the earlier op. FIXME: not sure when/how + this would happen */ + + if (overlap_done && lkb->lkb_wait_type) { + log_error(ls, "remwait error %x reply %d wait_type %d overlap", + lkb->lkb_id, mstype, lkb->lkb_wait_type); + lkb->lkb_wait_count--; + lkb->lkb_wait_type = 0; + } + + DLM_ASSERT(lkb->lkb_wait_count, dlm_print_lkb(lkb);); + + lkb->lkb_flags &= ~DLM_IFL_RESEND; + lkb->lkb_wait_count--; + if (!lkb->lkb_wait_count) + list_del_init(&lkb->lkb_wait_reply); + unhold_lkb(lkb); + return 0; +} + +static int remove_from_waiters(struct dlm_lkb *lkb, int mstype) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int error; + + mutex_lock(&ls->ls_waiters_mutex); + error = _remove_from_waiters(lkb, mstype, NULL); + mutex_unlock(&ls->ls_waiters_mutex); + return error; +} + +/* Handles situations where we might be processing a "fake" or "stub" reply in + which we can't try to take waiters_mutex again. */ + +static int remove_from_waiters_ms(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int error; + + if (ms->m_flags != DLM_IFL_STUB_MS) + mutex_lock(&ls->ls_waiters_mutex); + error = _remove_from_waiters(lkb, ms->m_type, ms); + if (ms->m_flags != DLM_IFL_STUB_MS) + mutex_unlock(&ls->ls_waiters_mutex); + return error; +} + +/* If there's an rsb for the same resource being removed, ensure + that the remove message is sent before the new lookup message. + It should be rare to need a delay here, but if not, then it may + be worthwhile to add a proper wait mechanism rather than a delay. */ + +static void wait_pending_remove(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + restart: + spin_lock(&ls->ls_remove_spin); + if (ls->ls_remove_len && + !rsb_cmp(r, ls->ls_remove_name, ls->ls_remove_len)) { + log_debug(ls, "delay lookup for remove dir %d %s", + r->res_dir_nodeid, r->res_name); + spin_unlock(&ls->ls_remove_spin); + msleep(1); + goto restart; + } + spin_unlock(&ls->ls_remove_spin); +} + +/* + * ls_remove_spin protects ls_remove_name and ls_remove_len which are + * read by other threads in wait_pending_remove. ls_remove_names + * and ls_remove_lens are only used by the scan thread, so they do + * not need protection. + */ + +static void shrink_bucket(struct dlm_ls *ls, int b) +{ + struct rb_node *n, *next; + struct dlm_rsb *r; + char *name; + int our_nodeid = dlm_our_nodeid(); + int remote_count = 0; + int need_shrink = 0; + int i, len, rv; + + memset(&ls->ls_remove_lens, 0, sizeof(int) * DLM_REMOVE_NAMES_MAX); + + spin_lock(&ls->ls_rsbtbl[b].lock); + + if (!(ls->ls_rsbtbl[b].flags & DLM_RTF_SHRINK)) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + return; + } + + for (n = rb_first(&ls->ls_rsbtbl[b].toss); n; n = next) { + next = rb_next(n); + r = rb_entry(n, struct dlm_rsb, res_hashnode); + + /* If we're the directory record for this rsb, and + we're not the master of it, then we need to wait + for the master node to send us a dir remove for + before removing the dir record. */ + + if (!dlm_no_directory(ls) && + (r->res_master_nodeid != our_nodeid) && + (dlm_dir_nodeid(r) == our_nodeid)) { + continue; + } + + need_shrink = 1; + + if (!time_after_eq(jiffies, r->res_toss_time + + dlm_config.ci_toss_secs * HZ)) { + continue; + } + + if (!dlm_no_directory(ls) && + (r->res_master_nodeid == our_nodeid) && + (dlm_dir_nodeid(r) != our_nodeid)) { + + /* We're the master of this rsb but we're not + the directory record, so we need to tell the + dir node to remove the dir record. */ + + ls->ls_remove_lens[remote_count] = r->res_length; + memcpy(ls->ls_remove_names[remote_count], r->res_name, + DLM_RESNAME_MAXLEN); + remote_count++; + + if (remote_count >= DLM_REMOVE_NAMES_MAX) + break; + continue; + } + + if (!kref_put(&r->res_ref, kill_rsb)) { + log_error(ls, "tossed rsb in use %s", r->res_name); + continue; + } + + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); + dlm_free_rsb(r); + } + + if (need_shrink) + ls->ls_rsbtbl[b].flags |= DLM_RTF_SHRINK; + else + ls->ls_rsbtbl[b].flags &= ~DLM_RTF_SHRINK; + spin_unlock(&ls->ls_rsbtbl[b].lock); + + /* + * While searching for rsb's to free, we found some that require + * remote removal. We leave them in place and find them again here + * so there is a very small gap between removing them from the toss + * list and sending the removal. Keeping this gap small is + * important to keep us (the master node) from being out of sync + * with the remote dir node for very long. + * + * From the time the rsb is removed from toss until just after + * send_remove, the rsb name is saved in ls_remove_name. A new + * lookup checks this to ensure that a new lookup message for the + * same resource name is not sent just before the remove message. + */ + + for (i = 0; i < remote_count; i++) { + name = ls->ls_remove_names[i]; + len = ls->ls_remove_lens[i]; + + spin_lock(&ls->ls_rsbtbl[b].lock); + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (rv) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_debug(ls, "remove_name not toss %s", name); + continue; + } + + if (r->res_master_nodeid != our_nodeid) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_debug(ls, "remove_name master %d dir %d our %d %s", + r->res_master_nodeid, r->res_dir_nodeid, + our_nodeid, name); + continue; + } + + if (r->res_dir_nodeid == our_nodeid) { + /* should never happen */ + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_error(ls, "remove_name dir %d master %d our %d %s", + r->res_dir_nodeid, r->res_master_nodeid, + our_nodeid, name); + continue; + } + + if (!time_after_eq(jiffies, r->res_toss_time + + dlm_config.ci_toss_secs * HZ)) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_debug(ls, "remove_name toss_time %lu now %lu %s", + r->res_toss_time, jiffies, name); + continue; + } + + if (!kref_put(&r->res_ref, kill_rsb)) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_error(ls, "remove_name in use %s", name); + continue; + } + + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); + + /* block lookup of same name until we've sent remove */ + spin_lock(&ls->ls_remove_spin); + ls->ls_remove_len = len; + memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN); + spin_unlock(&ls->ls_remove_spin); + spin_unlock(&ls->ls_rsbtbl[b].lock); + + send_remove(r); + + /* allow lookup of name again */ + spin_lock(&ls->ls_remove_spin); + ls->ls_remove_len = 0; + memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN); + spin_unlock(&ls->ls_remove_spin); + + dlm_free_rsb(r); + } +} + +void dlm_scan_rsbs(struct dlm_ls *ls) +{ + int i; + + for (i = 0; i < ls->ls_rsbtbl_size; i++) { + shrink_bucket(ls, i); + if (dlm_locking_stopped(ls)) + break; + cond_resched(); + } +} + +static void add_timeout(struct dlm_lkb *lkb) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + + if (is_master_copy(lkb)) + return; + + if (test_bit(LSFL_TIMEWARN, &ls->ls_flags) && + !(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) { + lkb->lkb_flags |= DLM_IFL_WATCH_TIMEWARN; + goto add_it; + } + if (lkb->lkb_exflags & DLM_LKF_TIMEOUT) + goto add_it; + return; + + add_it: + DLM_ASSERT(list_empty(&lkb->lkb_time_list), dlm_print_lkb(lkb);); + mutex_lock(&ls->ls_timeout_mutex); + hold_lkb(lkb); + list_add_tail(&lkb->lkb_time_list, &ls->ls_timeout); + mutex_unlock(&ls->ls_timeout_mutex); +} + +static void del_timeout(struct dlm_lkb *lkb) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + + mutex_lock(&ls->ls_timeout_mutex); + if (!list_empty(&lkb->lkb_time_list)) { + list_del_init(&lkb->lkb_time_list); + unhold_lkb(lkb); + } + mutex_unlock(&ls->ls_timeout_mutex); +} + +/* FIXME: is it safe to look at lkb_exflags, lkb_flags, lkb_timestamp, and + lkb_lksb_timeout without lock_rsb? Note: we can't lock timeout_mutex + and then lock rsb because of lock ordering in add_timeout. We may need + to specify some special timeout-related bits in the lkb that are just to + be accessed under the timeout_mutex. */ + +void dlm_scan_timeout(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + struct dlm_lkb *lkb; + int do_cancel, do_warn; + s64 wait_us; + + for (;;) { + if (dlm_locking_stopped(ls)) + break; + + do_cancel = 0; + do_warn = 0; + mutex_lock(&ls->ls_timeout_mutex); + list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) { + + wait_us = ktime_to_us(ktime_sub(ktime_get(), + lkb->lkb_timestamp)); + + if ((lkb->lkb_exflags & DLM_LKF_TIMEOUT) && + wait_us >= (lkb->lkb_timeout_cs * 10000)) + do_cancel = 1; + + if ((lkb->lkb_flags & DLM_IFL_WATCH_TIMEWARN) && + wait_us >= dlm_config.ci_timewarn_cs * 10000) + do_warn = 1; + + if (!do_cancel && !do_warn) + continue; + hold_lkb(lkb); + break; + } + mutex_unlock(&ls->ls_timeout_mutex); + + if (!do_cancel && !do_warn) + break; + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + if (do_warn) { + /* clear flag so we only warn once */ + lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN; + if (!(lkb->lkb_exflags & DLM_LKF_TIMEOUT)) + del_timeout(lkb); + dlm_timeout_warn(lkb); + } + + if (do_cancel) { + log_debug(ls, "timeout cancel %x node %d %s", + lkb->lkb_id, lkb->lkb_nodeid, r->res_name); + lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN; + lkb->lkb_flags |= DLM_IFL_TIMEOUT_CANCEL; + del_timeout(lkb); + _cancel_lock(r, lkb); + } + + unlock_rsb(r); + unhold_rsb(r); + dlm_put_lkb(lkb); + } +} + +/* This is only called by dlm_recoverd, and we rely on dlm_ls_stop() stopping + dlm_recoverd before checking/setting ls_recover_begin. */ + +void dlm_adjust_timeouts(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + u64 adj_us = jiffies_to_usecs(jiffies - ls->ls_recover_begin); + + ls->ls_recover_begin = 0; + mutex_lock(&ls->ls_timeout_mutex); + list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) + lkb->lkb_timestamp = ktime_add_us(lkb->lkb_timestamp, adj_us); + mutex_unlock(&ls->ls_timeout_mutex); + + if (!dlm_config.ci_waitwarn_us) + return; + + mutex_lock(&ls->ls_waiters_mutex); + list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { + if (ktime_to_us(lkb->lkb_wait_time)) + lkb->lkb_wait_time = ktime_get(); + } + mutex_unlock(&ls->ls_waiters_mutex); +} + +/* lkb is master or local copy */ + +static void set_lvb_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int b, len = r->res_ls->ls_lvblen; + + /* b=1 lvb returned to caller + b=0 lvb written to rsb or invalidated + b=-1 do nothing */ + + b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; + + if (b == 1) { + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + return; + + memcpy(lkb->lkb_lvbptr, r->res_lvbptr, len); + lkb->lkb_lvbseq = r->res_lvbseq; + + } else if (b == 0) { + if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + + if (!r->res_lvbptr) + return; + + memcpy(r->res_lvbptr, lkb->lkb_lvbptr, len); + r->res_lvbseq++; + lkb->lkb_lvbseq = r->res_lvbseq; + rsb_clear_flag(r, RSB_VALNOTVALID); + } + + if (rsb_flag(r, RSB_VALNOTVALID)) + lkb->lkb_sbflags |= DLM_SBF_VALNOTVALID; +} + +static void set_lvb_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + if (lkb->lkb_grmode < DLM_LOCK_PW) + return; + + if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + + if (!r->res_lvbptr) + return; + + memcpy(r->res_lvbptr, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); + r->res_lvbseq++; + rsb_clear_flag(r, RSB_VALNOTVALID); +} + +/* lkb is process copy (pc) */ + +static void set_lvb_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + int b; + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; + if (b == 1) { + int len = receive_extralen(ms); + if (len > DLM_RESNAME_MAXLEN) + len = DLM_RESNAME_MAXLEN; + memcpy(lkb->lkb_lvbptr, ms->m_extra, len); + lkb->lkb_lvbseq = ms->m_lvbseq; + } +} + +/* Manipulate lkb's on rsb's convert/granted/waiting queues + remove_lock -- used for unlock, removes lkb from granted + revert_lock -- used for cancel, moves lkb from convert to granted + grant_lock -- used for request and convert, adds lkb to granted or + moves lkb from convert or waiting to granted + + Each of these is used for master or local copy lkb's. There is + also a _pc() variation used to make the corresponding change on + a process copy (pc) lkb. */ + +static void _remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + del_lkb(r, lkb); + lkb->lkb_grmode = DLM_LOCK_IV; + /* this unhold undoes the original ref from create_lkb() + so this leads to the lkb being freed */ + unhold_lkb(lkb); +} + +static void remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + set_lvb_unlock(r, lkb); + _remove_lock(r, lkb); +} + +static void remove_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + _remove_lock(r, lkb); +} + +/* returns: 0 did nothing + 1 moved lock to granted + -1 removed lock */ + +static int revert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int rv = 0; + + lkb->lkb_rqmode = DLM_LOCK_IV; + + switch (lkb->lkb_status) { + case DLM_LKSTS_GRANTED: + break; + case DLM_LKSTS_CONVERT: + move_lkb(r, lkb, DLM_LKSTS_GRANTED); + rv = 1; + break; + case DLM_LKSTS_WAITING: + del_lkb(r, lkb); + lkb->lkb_grmode = DLM_LOCK_IV; + /* this unhold undoes the original ref from create_lkb() + so this leads to the lkb being freed */ + unhold_lkb(lkb); + rv = -1; + break; + default: + log_print("invalid status for revert %d", lkb->lkb_status); + } + return rv; +} + +static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return revert_lock(r, lkb); +} + +static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + if (lkb->lkb_grmode != lkb->lkb_rqmode) { + lkb->lkb_grmode = lkb->lkb_rqmode; + if (lkb->lkb_status) + move_lkb(r, lkb, DLM_LKSTS_GRANTED); + else + add_lkb(r, lkb, DLM_LKSTS_GRANTED); + } + + lkb->lkb_rqmode = DLM_LOCK_IV; + lkb->lkb_highbast = 0; +} + +static void grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + set_lvb_lock(r, lkb); + _grant_lock(r, lkb); +} + +static void grant_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + set_lvb_lock_pc(r, lkb, ms); + _grant_lock(r, lkb); +} + +/* called by grant_pending_locks() which means an async grant message must + be sent to the requesting node in addition to granting the lock if the + lkb belongs to a remote node. */ + +static void grant_lock_pending(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + grant_lock(r, lkb); + if (is_master_copy(lkb)) + send_grant(r, lkb); + else + queue_cast(r, lkb, 0); +} + +/* The special CONVDEADLK, ALTPR and ALTCW flags allow the master to + change the granted/requested modes. We're munging things accordingly in + the process copy. + CONVDEADLK: our grmode may have been forced down to NL to resolve a + conversion deadlock + ALTPR/ALTCW: our rqmode may have been changed to PR or CW to become + compatible with other granted locks */ + +static void munge_demoted(struct dlm_lkb *lkb) +{ + if (lkb->lkb_rqmode == DLM_LOCK_IV || lkb->lkb_grmode == DLM_LOCK_IV) { + log_print("munge_demoted %x invalid modes gr %d rq %d", + lkb->lkb_id, lkb->lkb_grmode, lkb->lkb_rqmode); + return; + } + + lkb->lkb_grmode = DLM_LOCK_NL; +} + +static void munge_altmode(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + if (ms->m_type != DLM_MSG_REQUEST_REPLY && + ms->m_type != DLM_MSG_GRANT) { + log_print("munge_altmode %x invalid reply type %d", + lkb->lkb_id, ms->m_type); + return; + } + + if (lkb->lkb_exflags & DLM_LKF_ALTPR) + lkb->lkb_rqmode = DLM_LOCK_PR; + else if (lkb->lkb_exflags & DLM_LKF_ALTCW) + lkb->lkb_rqmode = DLM_LOCK_CW; + else { + log_print("munge_altmode invalid exflags %x", lkb->lkb_exflags); + dlm_print_lkb(lkb); + } +} + +static inline int first_in_list(struct dlm_lkb *lkb, struct list_head *head) +{ + struct dlm_lkb *first = list_entry(head->next, struct dlm_lkb, + lkb_statequeue); + if (lkb->lkb_id == first->lkb_id) + return 1; + + return 0; +} + +/* Check if the given lkb conflicts with another lkb on the queue. */ + +static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb) +{ + struct dlm_lkb *this; + + list_for_each_entry(this, head, lkb_statequeue) { + if (this == lkb) + continue; + if (!modes_compat(this, lkb)) + return 1; + } + return 0; +} + +/* + * "A conversion deadlock arises with a pair of lock requests in the converting + * queue for one resource. The granted mode of each lock blocks the requested + * mode of the other lock." + * + * Part 2: if the granted mode of lkb is preventing an earlier lkb in the + * convert queue from being granted, then deadlk/demote lkb. + * + * Example: + * Granted Queue: empty + * Convert Queue: NL->EX (first lock) + * PR->EX (second lock) + * + * The first lock can't be granted because of the granted mode of the second + * lock and the second lock can't be granted because it's not first in the + * list. We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we + * demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK + * flag set and return DEMOTED in the lksb flags. + * + * Originally, this function detected conv-deadlk in a more limited scope: + * - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or + * - if lkb1 was the first entry in the queue (not just earlier), and was + * blocked by the granted mode of lkb2, and there was nothing on the + * granted queue preventing lkb1 from being granted immediately, i.e. + * lkb2 was the only thing preventing lkb1 from being granted. + * + * That second condition meant we'd only say there was conv-deadlk if + * resolving it (by demotion) would lead to the first lock on the convert + * queue being granted right away. It allowed conversion deadlocks to exist + * between locks on the convert queue while they couldn't be granted anyway. + * + * Now, we detect and take action on conversion deadlocks immediately when + * they're created, even if they may not be immediately consequential. If + * lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted + * mode that would prevent lkb1's conversion from being granted, we do a + * deadlk/demote on lkb2 right away and don't let it onto the convert queue. + * I think this means that the lkb_is_ahead condition below should always + * be zero, i.e. there will never be conv-deadlk between two locks that are + * both already on the convert queue. + */ + +static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2) +{ + struct dlm_lkb *lkb1; + int lkb_is_ahead = 0; + + list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) { + if (lkb1 == lkb2) { + lkb_is_ahead = 1; + continue; + } + + if (!lkb_is_ahead) { + if (!modes_compat(lkb2, lkb1)) + return 1; + } else { + if (!modes_compat(lkb2, lkb1) && + !modes_compat(lkb1, lkb2)) + return 1; + } + } + return 0; +} + +/* + * Return 1 if the lock can be granted, 0 otherwise. + * Also detect and resolve conversion deadlocks. + * + * lkb is the lock to be granted + * + * now is 1 if the function is being called in the context of the + * immediate request, it is 0 if called later, after the lock has been + * queued. + * + * recover is 1 if dlm_recover_grant() is trying to grant conversions + * after recovery. + * + * References are from chapter 6 of "VAXcluster Principles" by Roy Davis + */ + +static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, + int recover) +{ + int8_t conv = (lkb->lkb_grmode != DLM_LOCK_IV); + + /* + * 6-10: Version 5.4 introduced an option to address the phenomenon of + * a new request for a NL mode lock being blocked. + * + * 6-11: If the optional EXPEDITE flag is used with the new NL mode + * request, then it would be granted. In essence, the use of this flag + * tells the Lock Manager to expedite theis request by not considering + * what may be in the CONVERTING or WAITING queues... As of this + * writing, the EXPEDITE flag can be used only with new requests for NL + * mode locks. This flag is not valid for conversion requests. + * + * A shortcut. Earlier checks return an error if EXPEDITE is used in a + * conversion or used with a non-NL requested mode. We also know an + * EXPEDITE request is always granted immediately, so now must always + * be 1. The full condition to grant an expedite request: (now && + * !conv && lkb->rqmode == DLM_LOCK_NL && (flags & EXPEDITE)) can + * therefore be shortened to just checking the flag. + */ + + if (lkb->lkb_exflags & DLM_LKF_EXPEDITE) + return 1; + + /* + * A shortcut. Without this, !queue_conflict(grantqueue, lkb) would be + * added to the remaining conditions. + */ + + if (queue_conflict(&r->res_grantqueue, lkb)) + return 0; + + /* + * 6-3: By default, a conversion request is immediately granted if the + * requested mode is compatible with the modes of all other granted + * locks + */ + + if (queue_conflict(&r->res_convertqueue, lkb)) + return 0; + + /* + * The RECOVER_GRANT flag means dlm_recover_grant() is granting + * locks for a recovered rsb, on which lkb's have been rebuilt. + * The lkb's may have been rebuilt on the queues in a different + * order than they were in on the previous master. So, granting + * queued conversions in order after recovery doesn't make sense + * since the order hasn't been preserved anyway. The new order + * could also have created a new "in place" conversion deadlock. + * (e.g. old, failed master held granted EX, with PR->EX, NL->EX. + * After recovery, there would be no granted locks, and possibly + * NL->EX, PR->EX, an in-place conversion deadlock.) So, after + * recovery, grant conversions without considering order. + */ + + if (conv && recover) + return 1; + + /* + * 6-5: But the default algorithm for deciding whether to grant or + * queue conversion requests does not by itself guarantee that such + * requests are serviced on a "first come first serve" basis. This, in + * turn, can lead to a phenomenon known as "indefinate postponement". + * + * 6-7: This issue is dealt with by using the optional QUECVT flag with + * the system service employed to request a lock conversion. This flag + * forces certain conversion requests to be queued, even if they are + * compatible with the granted modes of other locks on the same + * resource. Thus, the use of this flag results in conversion requests + * being ordered on a "first come first servce" basis. + * + * DCT: This condition is all about new conversions being able to occur + * "in place" while the lock remains on the granted queue (assuming + * nothing else conflicts.) IOW if QUECVT isn't set, a conversion + * doesn't _have_ to go onto the convert queue where it's processed in + * order. The "now" variable is necessary to distinguish converts + * being received and processed for the first time now, because once a + * convert is moved to the conversion queue the condition below applies + * requiring fifo granting. + */ + + if (now && conv && !(lkb->lkb_exflags & DLM_LKF_QUECVT)) + return 1; + + /* + * Even if the convert is compat with all granted locks, + * QUECVT forces it behind other locks on the convert queue. + */ + + if (now && conv && (lkb->lkb_exflags & DLM_LKF_QUECVT)) { + if (list_empty(&r->res_convertqueue)) + return 1; + else + return 0; + } + + /* + * The NOORDER flag is set to avoid the standard vms rules on grant + * order. + */ + + if (lkb->lkb_exflags & DLM_LKF_NOORDER) + return 1; + + /* + * 6-3: Once in that queue [CONVERTING], a conversion request cannot be + * granted until all other conversion requests ahead of it are granted + * and/or canceled. + */ + + if (!now && conv && first_in_list(lkb, &r->res_convertqueue)) + return 1; + + /* + * 6-4: By default, a new request is immediately granted only if all + * three of the following conditions are satisfied when the request is + * issued: + * - The queue of ungranted conversion requests for the resource is + * empty. + * - The queue of ungranted new requests for the resource is empty. + * - The mode of the new request is compatible with the most + * restrictive mode of all granted locks on the resource. + */ + + if (now && !conv && list_empty(&r->res_convertqueue) && + list_empty(&r->res_waitqueue)) + return 1; + + /* + * 6-4: Once a lock request is in the queue of ungranted new requests, + * it cannot be granted until the queue of ungranted conversion + * requests is empty, all ungranted new requests ahead of it are + * granted and/or canceled, and it is compatible with the granted mode + * of the most restrictive lock granted on the resource. + */ + + if (!now && !conv && list_empty(&r->res_convertqueue) && + first_in_list(lkb, &r->res_waitqueue)) + return 1; + + return 0; +} + +static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, + int recover, int *err) +{ + int rv; + int8_t alt = 0, rqmode = lkb->lkb_rqmode; + int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV); + + if (err) + *err = 0; + + rv = _can_be_granted(r, lkb, now, recover); + if (rv) + goto out; + + /* + * The CONVDEADLK flag is non-standard and tells the dlm to resolve + * conversion deadlocks by demoting grmode to NL, otherwise the dlm + * cancels one of the locks. + */ + + if (is_convert && can_be_queued(lkb) && + conversion_deadlock_detect(r, lkb)) { + if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) { + lkb->lkb_grmode = DLM_LOCK_NL; + lkb->lkb_sbflags |= DLM_SBF_DEMOTED; + } else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) { + if (err) + *err = -EDEADLK; + else { + log_print("can_be_granted deadlock %x now %d", + lkb->lkb_id, now); + dlm_dump_rsb(r); + } + } + goto out; + } + + /* + * The ALTPR and ALTCW flags are non-standard and tell the dlm to try + * to grant a request in a mode other than the normal rqmode. It's a + * simple way to provide a big optimization to applications that can + * use them. + */ + + if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR)) + alt = DLM_LOCK_PR; + else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW)) + alt = DLM_LOCK_CW; + + if (alt) { + lkb->lkb_rqmode = alt; + rv = _can_be_granted(r, lkb, now, 0); + if (rv) + lkb->lkb_sbflags |= DLM_SBF_ALTMODE; + else + lkb->lkb_rqmode = rqmode; + } + out: + return rv; +} + +/* FIXME: I don't think that can_be_granted() can/will demote or find deadlock + for locks pending on the convert list. Once verified (watch for these + log_prints), we should be able to just call _can_be_granted() and not + bother with the demote/deadlk cases here (and there's no easy way to deal + with a deadlk here, we'd have to generate something like grant_lock with + the deadlk error.) */ + +/* Returns the highest requested mode of all blocked conversions; sets + cw if there's a blocked conversion to DLM_LOCK_CW. */ + +static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, + unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + int recover = rsb_flag(r, RSB_RECOVER_GRANT); + int hi, demoted, quit, grant_restart, demote_restart; + int deadlk; + + quit = 0; + restart: + grant_restart = 0; + demote_restart = 0; + hi = DLM_LOCK_IV; + + list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) { + demoted = is_demoted(lkb); + deadlk = 0; + + if (can_be_granted(r, lkb, 0, recover, &deadlk)) { + grant_lock_pending(r, lkb); + grant_restart = 1; + if (count) + (*count)++; + continue; + } + + if (!demoted && is_demoted(lkb)) { + log_print("WARN: pending demoted %x node %d %s", + lkb->lkb_id, lkb->lkb_nodeid, r->res_name); + demote_restart = 1; + continue; + } + + if (deadlk) { + log_print("WARN: pending deadlock %x node %d %s", + lkb->lkb_id, lkb->lkb_nodeid, r->res_name); + dlm_dump_rsb(r); + continue; + } + + hi = max_t(int, lkb->lkb_rqmode, hi); + + if (cw && lkb->lkb_rqmode == DLM_LOCK_CW) + *cw = 1; + } + + if (grant_restart) + goto restart; + if (demote_restart && !quit) { + quit = 1; + goto restart; + } + + return max_t(int, high, hi); +} + +static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw, + unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + + list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) { + if (can_be_granted(r, lkb, 0, 0, NULL)) { + grant_lock_pending(r, lkb); + if (count) + (*count)++; + } else { + high = max_t(int, lkb->lkb_rqmode, high); + if (lkb->lkb_rqmode == DLM_LOCK_CW) + *cw = 1; + } + } + + return high; +} + +/* cw of 1 means there's a lock with a rqmode of DLM_LOCK_CW that's blocked + on either the convert or waiting queue. + high is the largest rqmode of all locks blocked on the convert or + waiting queue. */ + +static int lock_requires_bast(struct dlm_lkb *gr, int high, int cw) +{ + if (gr->lkb_grmode == DLM_LOCK_PR && cw) { + if (gr->lkb_highbast < DLM_LOCK_EX) + return 1; + return 0; + } + + if (gr->lkb_highbast < high && + !__dlm_compat_matrix[gr->lkb_grmode+1][high+1]) + return 1; + return 0; +} + +static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + int high = DLM_LOCK_IV; + int cw = 0; + + if (!is_master(r)) { + log_print("grant_pending_locks r nodeid %d", r->res_nodeid); + dlm_dump_rsb(r); + return; + } + + high = grant_pending_convert(r, high, &cw, count); + high = grant_pending_wait(r, high, &cw, count); + + if (high == DLM_LOCK_IV) + return; + + /* + * If there are locks left on the wait/convert queue then send blocking + * ASTs to granted locks based on the largest requested mode (high) + * found above. + */ + + list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) { + if (lkb->lkb_bastfn && lock_requires_bast(lkb, high, cw)) { + if (cw && high == DLM_LOCK_PR && + lkb->lkb_grmode == DLM_LOCK_PR) + queue_bast(r, lkb, DLM_LOCK_CW); + else + queue_bast(r, lkb, high); + lkb->lkb_highbast = high; + } + } +} + +static int modes_require_bast(struct dlm_lkb *gr, struct dlm_lkb *rq) +{ + if ((gr->lkb_grmode == DLM_LOCK_PR && rq->lkb_rqmode == DLM_LOCK_CW) || + (gr->lkb_grmode == DLM_LOCK_CW && rq->lkb_rqmode == DLM_LOCK_PR)) { + if (gr->lkb_highbast < DLM_LOCK_EX) + return 1; + return 0; + } + + if (gr->lkb_highbast < rq->lkb_rqmode && !modes_compat(gr, rq)) + return 1; + return 0; +} + +static void send_bast_queue(struct dlm_rsb *r, struct list_head *head, + struct dlm_lkb *lkb) +{ + struct dlm_lkb *gr; + + list_for_each_entry(gr, head, lkb_statequeue) { + /* skip self when sending basts to convertqueue */ + if (gr == lkb) + continue; + if (gr->lkb_bastfn && modes_require_bast(gr, lkb)) { + queue_bast(r, gr, lkb->lkb_rqmode); + gr->lkb_highbast = lkb->lkb_rqmode; + } + } +} + +static void send_blocking_asts(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + send_bast_queue(r, &r->res_grantqueue, lkb); +} + +static void send_blocking_asts_all(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + send_bast_queue(r, &r->res_grantqueue, lkb); + send_bast_queue(r, &r->res_convertqueue, lkb); +} + +/* set_master(r, lkb) -- set the master nodeid of a resource + + The purpose of this function is to set the nodeid field in the given + lkb using the nodeid field in the given rsb. If the rsb's nodeid is + known, it can just be copied to the lkb and the function will return + 0. If the rsb's nodeid is _not_ known, it needs to be looked up + before it can be copied to the lkb. + + When the rsb nodeid is being looked up remotely, the initial lkb + causing the lookup is kept on the ls_waiters list waiting for the + lookup reply. Other lkb's waiting for the same rsb lookup are kept + on the rsb's res_lookup list until the master is verified. + + Return values: + 0: nodeid is set in rsb/lkb and the caller should go ahead and use it + 1: the rsb master is not available and the lkb has been placed on + a wait queue +*/ + +static int set_master(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int our_nodeid = dlm_our_nodeid(); + + if (rsb_flag(r, RSB_MASTER_UNCERTAIN)) { + rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = lkb->lkb_id; + lkb->lkb_nodeid = r->res_nodeid; + return 0; + } + + if (r->res_first_lkid && r->res_first_lkid != lkb->lkb_id) { + list_add_tail(&lkb->lkb_rsb_lookup, &r->res_lookup); + return 1; + } + + if (r->res_master_nodeid == our_nodeid) { + lkb->lkb_nodeid = 0; + return 0; + } + + if (r->res_master_nodeid) { + lkb->lkb_nodeid = r->res_master_nodeid; + return 0; + } + + if (dlm_dir_nodeid(r) == our_nodeid) { + /* This is a somewhat unusual case; find_rsb will usually + have set res_master_nodeid when dir nodeid is local, but + there are cases where we become the dir node after we've + past find_rsb and go through _request_lock again. + confirm_master() or process_lookup_list() needs to be + called after this. */ + log_debug(r->res_ls, "set_master %x self master %d dir %d %s", + lkb->lkb_id, r->res_master_nodeid, r->res_dir_nodeid, + r->res_name); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + lkb->lkb_nodeid = 0; + return 0; + } + + wait_pending_remove(r); + + r->res_first_lkid = lkb->lkb_id; + send_lookup(r, lkb); + return 1; +} + +static void process_lookup_list(struct dlm_rsb *r) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, &r->res_lookup, lkb_rsb_lookup) { + list_del_init(&lkb->lkb_rsb_lookup); + _request_lock(r, lkb); + schedule(); + } +} + +/* confirm_master -- confirm (or deny) an rsb's master nodeid */ + +static void confirm_master(struct dlm_rsb *r, int error) +{ + struct dlm_lkb *lkb; + + if (!r->res_first_lkid) + return; + + switch (error) { + case 0: + case -EINPROGRESS: + r->res_first_lkid = 0; + process_lookup_list(r); + break; + + case -EAGAIN: + case -EBADR: + case -ENOTBLK: + /* the remote request failed and won't be retried (it was + a NOQUEUE, or has been canceled/unlocked); make a waiting + lkb the first_lkid */ + + r->res_first_lkid = 0; + + if (!list_empty(&r->res_lookup)) { + lkb = list_entry(r->res_lookup.next, struct dlm_lkb, + lkb_rsb_lookup); + list_del_init(&lkb->lkb_rsb_lookup); + r->res_first_lkid = lkb->lkb_id; + _request_lock(r, lkb); + } + break; + + default: + log_error(r->res_ls, "confirm_master unknown error %d", error); + } +} + +static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, + int namelen, unsigned long timeout_cs, + void (*ast) (void *astparam), + void *astparam, + void (*bast) (void *astparam, int mode), + struct dlm_args *args) +{ + int rv = -EINVAL; + + /* check for invalid arg usage */ + + if (mode < 0 || mode > DLM_LOCK_EX) + goto out; + + if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN)) + goto out; + + if (flags & DLM_LKF_CANCEL) + goto out; + + if (flags & DLM_LKF_QUECVT && !(flags & DLM_LKF_CONVERT)) + goto out; + + if (flags & DLM_LKF_CONVDEADLK && !(flags & DLM_LKF_CONVERT)) + goto out; + + if (flags & DLM_LKF_CONVDEADLK && flags & DLM_LKF_NOQUEUE) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_CONVERT) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_QUECVT) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_NOQUEUE) + goto out; + + if (flags & DLM_LKF_EXPEDITE && mode != DLM_LOCK_NL) + goto out; + + if (!ast || !lksb) + goto out; + + if (flags & DLM_LKF_VALBLK && !lksb->sb_lvbptr) + goto out; + + if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid) + goto out; + + /* these args will be copied to the lkb in validate_lock_args, + it cannot be done now because when converting locks, fields in + an active lkb cannot be modified before locking the rsb */ + + args->flags = flags; + args->astfn = ast; + args->astparam = astparam; + args->bastfn = bast; + args->timeout = timeout_cs; + args->mode = mode; + args->lksb = lksb; + rv = 0; + out: + return rv; +} + +static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args) +{ + if (flags & ~(DLM_LKF_CANCEL | DLM_LKF_VALBLK | DLM_LKF_IVVALBLK | + DLM_LKF_FORCEUNLOCK)) + return -EINVAL; + + if (flags & DLM_LKF_CANCEL && flags & DLM_LKF_FORCEUNLOCK) + return -EINVAL; + + args->flags = flags; + args->astparam = astarg; + return 0; +} + +static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + int rv = -EINVAL; + + if (args->flags & DLM_LKF_CONVERT) { + if (lkb->lkb_flags & DLM_IFL_MSTCPY) + goto out; + + if (args->flags & DLM_LKF_QUECVT && + !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1]) + goto out; + + rv = -EBUSY; + if (lkb->lkb_status != DLM_LKSTS_GRANTED) + goto out; + + if (lkb->lkb_wait_type) + goto out; + + if (is_overlap(lkb)) + goto out; + } + + lkb->lkb_exflags = args->flags; + lkb->lkb_sbflags = 0; + lkb->lkb_astfn = args->astfn; + lkb->lkb_astparam = args->astparam; + lkb->lkb_bastfn = args->bastfn; + lkb->lkb_rqmode = args->mode; + lkb->lkb_lksb = args->lksb; + lkb->lkb_lvbptr = args->lksb->sb_lvbptr; + lkb->lkb_ownpid = (int) current->pid; + lkb->lkb_timeout_cs = args->timeout; + rv = 0; + out: + if (rv) + log_debug(ls, "validate_lock_args %d %x %x %x %d %d %s", + rv, lkb->lkb_id, lkb->lkb_flags, args->flags, + lkb->lkb_status, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + return rv; +} + +/* when dlm_unlock() sees -EBUSY with CANCEL/FORCEUNLOCK it returns 0 + for success */ + +/* note: it's valid for lkb_nodeid/res_nodeid to be -1 when we get here + because there may be a lookup in progress and it's valid to do + cancel/unlockf on it */ + +static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int rv = -EINVAL; + + if (lkb->lkb_flags & DLM_IFL_MSTCPY) { + log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id); + dlm_print_lkb(lkb); + goto out; + } + + /* an lkb may still exist even though the lock is EOL'ed due to a + cancel, unlock or failed noqueue request; an app can't use these + locks; return same error as if the lkid had not been found at all */ + + if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { + log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id); + rv = -ENOENT; + goto out; + } + + /* an lkb may be waiting for an rsb lookup to complete where the + lookup was initiated by another lock */ + + if (!list_empty(&lkb->lkb_rsb_lookup)) { + if (args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) { + log_debug(ls, "unlock on rsb_lookup %x", lkb->lkb_id); + list_del_init(&lkb->lkb_rsb_lookup); + queue_cast(lkb->lkb_resource, lkb, + args->flags & DLM_LKF_CANCEL ? + -DLM_ECANCEL : -DLM_EUNLOCK); + unhold_lkb(lkb); /* undoes create_lkb() */ + } + /* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */ + rv = -EBUSY; + goto out; + } + + /* cancel not allowed with another cancel/unlock in progress */ + + if (args->flags & DLM_LKF_CANCEL) { + if (lkb->lkb_exflags & DLM_LKF_CANCEL) + goto out; + + if (is_overlap(lkb)) + goto out; + + /* don't let scand try to do a cancel */ + del_timeout(lkb); + + if (lkb->lkb_flags & DLM_IFL_RESEND) { + lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; + rv = -EBUSY; + goto out; + } + + /* there's nothing to cancel */ + if (lkb->lkb_status == DLM_LKSTS_GRANTED && + !lkb->lkb_wait_type) { + rv = -EBUSY; + goto out; + } + + switch (lkb->lkb_wait_type) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; + rv = -EBUSY; + goto out; + case DLM_MSG_UNLOCK: + case DLM_MSG_CANCEL: + goto out; + } + /* add_to_waiters() will set OVERLAP_CANCEL */ + goto out_ok; + } + + /* do we need to allow a force-unlock if there's a normal unlock + already in progress? in what conditions could the normal unlock + fail such that we'd want to send a force-unlock to be sure? */ + + if (args->flags & DLM_LKF_FORCEUNLOCK) { + if (lkb->lkb_exflags & DLM_LKF_FORCEUNLOCK) + goto out; + + if (is_overlap_unlock(lkb)) + goto out; + + /* don't let scand try to do a cancel */ + del_timeout(lkb); + + if (lkb->lkb_flags & DLM_IFL_RESEND) { + lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; + rv = -EBUSY; + goto out; + } + + switch (lkb->lkb_wait_type) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; + rv = -EBUSY; + goto out; + case DLM_MSG_UNLOCK: + goto out; + } + /* add_to_waiters() will set OVERLAP_UNLOCK */ + goto out_ok; + } + + /* normal unlock not allowed if there's any op in progress */ + rv = -EBUSY; + if (lkb->lkb_wait_type || lkb->lkb_wait_count) + goto out; + + out_ok: + /* an overlapping op shouldn't blow away exflags from other op */ + lkb->lkb_exflags |= args->flags; + lkb->lkb_sbflags = 0; + lkb->lkb_astparam = args->astparam; + rv = 0; + out: + if (rv) + log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", rv, + lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, + args->flags, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + return rv; +} + +/* + * Four stage 4 varieties: + * do_request(), do_convert(), do_unlock(), do_cancel() + * These are called on the master node for the given lock and + * from the central locking logic. + */ + +static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error = 0; + + if (can_be_granted(r, lkb, 1, 0, NULL)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + + if (can_be_queued(lkb)) { + error = -EINPROGRESS; + add_lkb(r, lkb, DLM_LKSTS_WAITING); + add_timeout(lkb); + goto out; + } + + error = -EAGAIN; + queue_cast(r, lkb, -EAGAIN); + out: + return error; +} + +static void do_request_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + +static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error = 0; + int deadlk = 0; + + /* changing an existing lock may allow others to be granted */ + + if (can_be_granted(r, lkb, 1, 0, &deadlk)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + + /* can_be_granted() detected that this lock would block in a conversion + deadlock, so we leave it on the granted queue and return EDEADLK in + the ast for the convert. */ + + if (deadlk) { + /* it's left on the granted queue */ + revert_lock(r, lkb); + queue_cast(r, lkb, -EDEADLK); + error = -EDEADLK; + goto out; + } + + /* is_demoted() means the can_be_granted() above set the grmode + to NL, and left us on the granted queue. This auto-demotion + (due to CONVDEADLK) might mean other locks, and/or this lock, are + now grantable. We have to try to grant other converting locks + before we try again to grant this one. */ + + if (is_demoted(lkb)) { + grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL); + if (_can_be_granted(r, lkb, 1, 0)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + /* else fall through and move to convert queue */ + } + + if (can_be_queued(lkb)) { + error = -EINPROGRESS; + del_lkb(r, lkb); + add_lkb(r, lkb, DLM_LKSTS_CONVERT); + add_timeout(lkb); + goto out; + } + + error = -EAGAIN; + queue_cast(r, lkb, -EAGAIN); + out: + return error; +} + +static void do_convert_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case 0: + grant_pending_locks(r, NULL); + /* grant_pending_locks also sends basts */ + break; + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + +static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + remove_lock(r, lkb); + queue_cast(r, lkb, -DLM_EUNLOCK); + return -DLM_EUNLOCK; +} + +static void do_unlock_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + grant_pending_locks(r, NULL); +} + +/* returns: 0 did nothing, -DLM_ECANCEL canceled lock */ + +static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + error = revert_lock(r, lkb); + if (error) { + queue_cast(r, lkb, -DLM_ECANCEL); + return -DLM_ECANCEL; + } + return 0; +} + +static void do_cancel_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + if (error) + grant_pending_locks(r, NULL); +} + +/* + * Four stage 3 varieties: + * _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock() + */ + +/* add a new lkb to a possibly new rsb, called by requesting process */ + +static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + /* set_master: sets lkb nodeid from r */ + + error = set_master(r, lkb); + if (error < 0) + goto out; + if (error) { + error = 0; + goto out; + } + + if (is_remote(r)) { + /* receive_request() calls do_request() on remote node */ + error = send_request(r, lkb); + } else { + error = do_request(r, lkb); + /* for remote locks the request_reply is sent + between do_request and do_request_effects */ + do_request_effects(r, lkb, error); + } + out: + return error; +} + +/* change some property of an existing lkb, e.g. mode */ + +static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_convert() calls do_convert() on remote node */ + error = send_convert(r, lkb); + } else { + error = do_convert(r, lkb); + /* for remote locks the convert_reply is sent + between do_convert and do_convert_effects */ + do_convert_effects(r, lkb, error); + } + + return error; +} + +/* remove an existing lkb from the granted queue */ + +static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_unlock() calls do_unlock() on remote node */ + error = send_unlock(r, lkb); + } else { + error = do_unlock(r, lkb); + /* for remote locks the unlock_reply is sent + between do_unlock and do_unlock_effects */ + do_unlock_effects(r, lkb, error); + } + + return error; +} + +/* remove an existing lkb from the convert or wait queue */ + +static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_cancel() calls do_cancel() on remote node */ + error = send_cancel(r, lkb); + } else { + error = do_cancel(r, lkb); + /* for remote locks the cancel_reply is sent + between do_cancel and do_cancel_effects */ + do_cancel_effects(r, lkb, error); + } + + return error; +} + +/* + * Four stage 2 varieties: + * request_lock(), convert_lock(), unlock_lock(), cancel_lock() + */ + +static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name, + int len, struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + error = validate_lock_args(ls, lkb, args); + if (error) + return error; + + error = find_rsb(ls, name, len, 0, R_REQUEST, &r); + if (error) + return error; + + lock_rsb(r); + + attach_lkb(r, lkb); + lkb->lkb_lksb->sb_lkid = lkb->lkb_id; + + error = _request_lock(r, lkb); + + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int convert_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_lock_args(ls, lkb, args); + if (error) + goto out; + + error = _convert_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int unlock_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, args); + if (error) + goto out; + + error = _unlock_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, args); + if (error) + goto out; + + error = _cancel_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +/* + * Two stage 1 varieties: dlm_lock() and dlm_unlock() + */ + +int dlm_lock(dlm_lockspace_t *lockspace, + int mode, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*ast) (void *astarg), + void *astarg, + void (*bast) (void *astarg, int mode)) +{ + struct dlm_ls *ls; + struct dlm_lkb *lkb; + struct dlm_args args; + int error, convert = flags & DLM_LKF_CONVERT; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + dlm_lock_recovery(ls); + + if (convert) + error = find_lkb(ls, lksb->sb_lkid, &lkb); + else + error = create_lkb(ls, &lkb); + + if (error) + goto out; + + error = set_lock_args(mode, lksb, flags, namelen, 0, ast, + astarg, bast, &args); + if (error) + goto out_put; + + if (convert) + error = convert_lock(ls, lkb, &args); + else + error = request_lock(ls, lkb, name, namelen, &args); + + if (error == -EINPROGRESS) + error = 0; + out_put: + if (convert || error) + __put_lkb(ls, lkb); + if (error == -EAGAIN || error == -EDEADLK) + error = 0; + out: + dlm_unlock_recovery(ls); + dlm_put_lockspace(ls); + return error; +} + +int dlm_unlock(dlm_lockspace_t *lockspace, + uint32_t lkid, + uint32_t flags, + struct dlm_lksb *lksb, + void *astarg) +{ + struct dlm_ls *ls; + struct dlm_lkb *lkb; + struct dlm_args args; + int error; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + error = set_unlock_args(flags, astarg, &args); + if (error) + goto out_put; + + if (flags & DLM_LKF_CANCEL) + error = cancel_lock(ls, lkb, &args); + else + error = unlock_lock(ls, lkb, &args); + + if (error == -DLM_EUNLOCK || error == -DLM_ECANCEL) + error = 0; + if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK))) + error = 0; + out_put: + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + dlm_put_lockspace(ls); + return error; +} + +/* + * send/receive routines for remote operations and replies + * + * send_args + * send_common + * send_request receive_request + * send_convert receive_convert + * send_unlock receive_unlock + * send_cancel receive_cancel + * send_grant receive_grant + * send_bast receive_bast + * send_lookup receive_lookup + * send_remove receive_remove + * + * send_common_reply + * receive_request_reply send_request_reply + * receive_convert_reply send_convert_reply + * receive_unlock_reply send_unlock_reply + * receive_cancel_reply send_cancel_reply + * receive_lookup_reply send_lookup_reply + */ + +static int _create_message(struct dlm_ls *ls, int mb_len, + int to_nodeid, int mstype, + struct dlm_message **ms_ret, + struct dlm_mhandle **mh_ret) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + char *mb; + + /* get_buffer gives us a message handle (mh) that we need to + pass into lowcomms_commit and a message buffer (mb) that we + write our data into */ + + mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb); + if (!mh) + return -ENOBUFS; + + memset(mb, 0, mb_len); + + ms = (struct dlm_message *) mb; + + ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + ms->m_header.h_lockspace = ls->ls_global_id; + ms->m_header.h_nodeid = dlm_our_nodeid(); + ms->m_header.h_length = mb_len; + ms->m_header.h_cmd = DLM_MSG; + + ms->m_type = mstype; + + *mh_ret = mh; + *ms_ret = ms; + return 0; +} + +static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb, + int to_nodeid, int mstype, + struct dlm_message **ms_ret, + struct dlm_mhandle **mh_ret) +{ + int mb_len = sizeof(struct dlm_message); + + switch (mstype) { + case DLM_MSG_REQUEST: + case DLM_MSG_LOOKUP: + case DLM_MSG_REMOVE: + mb_len += r->res_length; + break; + case DLM_MSG_CONVERT: + case DLM_MSG_UNLOCK: + case DLM_MSG_REQUEST_REPLY: + case DLM_MSG_CONVERT_REPLY: + case DLM_MSG_GRANT: + if (lkb && lkb->lkb_lvbptr) + mb_len += r->res_ls->ls_lvblen; + break; + } + + return _create_message(r->res_ls, mb_len, to_nodeid, mstype, + ms_ret, mh_ret); +} + +/* further lowcomms enhancements or alternate implementations may make + the return value from this function useful at some point */ + +static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms) +{ + dlm_message_out(ms); + dlm_lowcomms_commit_buffer(mh); + return 0; +} + +static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + ms->m_nodeid = lkb->lkb_nodeid; + ms->m_pid = lkb->lkb_ownpid; + ms->m_lkid = lkb->lkb_id; + ms->m_remid = lkb->lkb_remid; + ms->m_exflags = lkb->lkb_exflags; + ms->m_sbflags = lkb->lkb_sbflags; + ms->m_flags = lkb->lkb_flags; + ms->m_lvbseq = lkb->lkb_lvbseq; + ms->m_status = lkb->lkb_status; + ms->m_grmode = lkb->lkb_grmode; + ms->m_rqmode = lkb->lkb_rqmode; + ms->m_hash = r->res_hash; + + /* m_result and m_bastmode are set from function args, + not from lkb fields */ + + if (lkb->lkb_bastfn) + ms->m_asts |= DLM_CB_BAST; + if (lkb->lkb_astfn) + ms->m_asts |= DLM_CB_CAST; + + /* compare with switch in create_message; send_remove() doesn't + use send_args() */ + + switch (ms->m_type) { + case DLM_MSG_REQUEST: + case DLM_MSG_LOOKUP: + memcpy(ms->m_extra, r->res_name, r->res_length); + break; + case DLM_MSG_CONVERT: + case DLM_MSG_UNLOCK: + case DLM_MSG_REQUEST_REPLY: + case DLM_MSG_CONVERT_REPLY: + case DLM_MSG_GRANT: + if (!lkb->lkb_lvbptr) + break; + memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); + break; + } +} + +static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = r->res_nodeid; + + error = add_to_waiters(lkb, mstype, to_nodeid); + if (error) + return error; + + error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); + if (error) + goto fail; + + send_args(r, lkb, ms); + + error = send_message(mh, ms); + if (error) + goto fail; + return 0; + + fail: + remove_from_waiters(lkb, msg_reply_type(mstype)); + return error; +} + +static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_REQUEST); +} + +static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + error = send_common(r, lkb, DLM_MSG_CONVERT); + + /* down conversions go without a reply from the master */ + if (!error && down_conversion(lkb)) { + remove_from_waiters(lkb, DLM_MSG_CONVERT_REPLY); + r->res_ls->ls_stub_ms.m_flags = DLM_IFL_STUB_MS; + r->res_ls->ls_stub_ms.m_type = DLM_MSG_CONVERT_REPLY; + r->res_ls->ls_stub_ms.m_result = 0; + __receive_convert_reply(r, lkb, &r->res_ls->ls_stub_ms); + } + + return error; +} + +/* FIXME: if this lkb is the only lock we hold on the rsb, then set + MASTER_UNCERTAIN to force the next request on the rsb to confirm + that the master is still correct. */ + +static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_UNLOCK); +} + +static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_CANCEL); +} + +static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, lkb, to_nodeid, DLM_MSG_GRANT, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_result = 0; + + error = send_message(mh, ms); + out: + return error; +} + +static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, NULL, to_nodeid, DLM_MSG_BAST, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_bastmode = mode; + + error = send_message(mh, ms); + out: + return error; +} + +static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = dlm_dir_nodeid(r); + + error = add_to_waiters(lkb, DLM_MSG_LOOKUP, to_nodeid); + if (error) + return error; + + error = create_message(r, NULL, to_nodeid, DLM_MSG_LOOKUP, &ms, &mh); + if (error) + goto fail; + + send_args(r, lkb, ms); + + error = send_message(mh, ms); + if (error) + goto fail; + return 0; + + fail: + remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); + return error; +} + +static int send_remove(struct dlm_rsb *r) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = dlm_dir_nodeid(r); + + error = create_message(r, NULL, to_nodeid, DLM_MSG_REMOVE, &ms, &mh); + if (error) + goto out; + + memcpy(ms->m_extra, r->res_name, r->res_length); + ms->m_hash = r->res_hash; + + error = send_message(mh, ms); + out: + return error; +} + +static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + int mstype, int rv) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_result = rv; + + error = send_message(mh, ms); + out: + return error; +} + +static int send_request_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_REQUEST_REPLY, rv); +} + +static int send_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_CONVERT_REPLY, rv); +} + +static int send_unlock_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_UNLOCK_REPLY, rv); +} + +static int send_cancel_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_CANCEL_REPLY, rv); +} + +static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in, + int ret_nodeid, int rv) +{ + struct dlm_rsb *r = &ls->ls_stub_rsb; + struct dlm_message *ms; + struct dlm_mhandle *mh; + int error, nodeid = ms_in->m_header.h_nodeid; + + error = create_message(r, NULL, nodeid, DLM_MSG_LOOKUP_REPLY, &ms, &mh); + if (error) + goto out; + + ms->m_lkid = ms_in->m_lkid; + ms->m_result = rv; + ms->m_nodeid = ret_nodeid; + + error = send_message(mh, ms); + out: + return error; +} + +/* which args we save from a received message depends heavily on the type + of message, unlike the send side where we can safely send everything about + the lkb for any type of message */ + +static void receive_flags(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + lkb->lkb_exflags = ms->m_exflags; + lkb->lkb_sbflags = ms->m_sbflags; + lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) | + (ms->m_flags & 0x0000FFFF); +} + +static void receive_flags_reply(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + if (ms->m_flags == DLM_IFL_STUB_MS) + return; + + lkb->lkb_sbflags = ms->m_sbflags; + lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) | + (ms->m_flags & 0x0000FFFF); +} + +static int receive_extralen(struct dlm_message *ms) +{ + return (ms->m_header.h_length - sizeof(struct dlm_message)); +} + +static int receive_lvb(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + int len; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + if (!lkb->lkb_lvbptr) + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + len = receive_extralen(ms); + if (len > DLM_RESNAME_MAXLEN) + len = DLM_RESNAME_MAXLEN; + memcpy(lkb->lkb_lvbptr, ms->m_extra, len); + } + return 0; +} + +static void fake_bastfn(void *astparam, int mode) +{ + log_print("fake_bastfn should not be called"); +} + +static void fake_astfn(void *astparam) +{ + log_print("fake_astfn should not be called"); +} + +static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + lkb->lkb_nodeid = ms->m_header.h_nodeid; + lkb->lkb_ownpid = ms->m_pid; + lkb->lkb_remid = ms->m_lkid; + lkb->lkb_grmode = DLM_LOCK_IV; + lkb->lkb_rqmode = ms->m_rqmode; + + lkb->lkb_bastfn = (ms->m_asts & DLM_CB_BAST) ? &fake_bastfn : NULL; + lkb->lkb_astfn = (ms->m_asts & DLM_CB_CAST) ? &fake_astfn : NULL; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + /* lkb was just created so there won't be an lvb yet */ + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + } + + return 0; +} + +static int receive_convert_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + if (lkb->lkb_status != DLM_LKSTS_GRANTED) + return -EBUSY; + + if (receive_lvb(ls, lkb, ms)) + return -ENOMEM; + + lkb->lkb_rqmode = ms->m_rqmode; + lkb->lkb_lvbseq = ms->m_lvbseq; + + return 0; +} + +static int receive_unlock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + if (receive_lvb(ls, lkb, ms)) + return -ENOMEM; + return 0; +} + +/* We fill in the stub-lkb fields with the info that send_xxxx_reply() + uses to send a reply and that the remote end uses to process the reply. */ + +static void setup_stub_lkb(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb = &ls->ls_stub_lkb; + lkb->lkb_nodeid = ms->m_header.h_nodeid; + lkb->lkb_remid = ms->m_lkid; +} + +/* This is called after the rsb is locked so that we can safely inspect + fields in the lkb. */ + +static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + int from = ms->m_header.h_nodeid; + int error = 0; + + switch (ms->m_type) { + case DLM_MSG_CONVERT: + case DLM_MSG_UNLOCK: + case DLM_MSG_CANCEL: + if (!is_master_copy(lkb) || lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + case DLM_MSG_CONVERT_REPLY: + case DLM_MSG_UNLOCK_REPLY: + case DLM_MSG_CANCEL_REPLY: + case DLM_MSG_GRANT: + case DLM_MSG_BAST: + if (!is_process_copy(lkb) || lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + case DLM_MSG_REQUEST_REPLY: + if (!is_process_copy(lkb)) + error = -EINVAL; + else if (lkb->lkb_nodeid != -1 && lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + default: + error = -EINVAL; + } + + if (error) + log_error(lkb->lkb_resource->res_ls, + "ignore invalid message %d from %d %x %x %x %d", + ms->m_type, from, lkb->lkb_id, lkb->lkb_remid, + lkb->lkb_flags, lkb->lkb_nodeid); + return error; +} + +static void send_repeat_remove(struct dlm_ls *ls, char *ms_name, int len) +{ + char name[DLM_RESNAME_MAXLEN + 1]; + struct dlm_message *ms; + struct dlm_mhandle *mh; + struct dlm_rsb *r; + uint32_t hash, b; + int rv, dir_nodeid; + + memset(name, 0, sizeof(name)); + memcpy(name, ms_name, len); + + hash = jhash(name, len, 0); + b = hash & (ls->ls_rsbtbl_size - 1); + + dir_nodeid = dlm_hash2nodeid(ls, hash); + + log_error(ls, "send_repeat_remove dir %d %s", dir_nodeid, name); + + spin_lock(&ls->ls_rsbtbl[b].lock); + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (!rv) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_error(ls, "repeat_remove on keep %s", name); + return; + } + + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (!rv) { + spin_unlock(&ls->ls_rsbtbl[b].lock); + log_error(ls, "repeat_remove on toss %s", name); + return; + } + + /* use ls->remove_name2 to avoid conflict with shrink? */ + + spin_lock(&ls->ls_remove_spin); + ls->ls_remove_len = len; + memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN); + spin_unlock(&ls->ls_remove_spin); + spin_unlock(&ls->ls_rsbtbl[b].lock); + + rv = _create_message(ls, sizeof(struct dlm_message) + len, + dir_nodeid, DLM_MSG_REMOVE, &ms, &mh); + if (rv) + return; + + memcpy(ms->m_extra, name, len); + ms->m_hash = hash; + + send_message(mh, ms); + + spin_lock(&ls->ls_remove_spin); + ls->ls_remove_len = 0; + memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN); + spin_unlock(&ls->ls_remove_spin); +} + +static int receive_request(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int from_nodeid; + int error, namelen = 0; + + from_nodeid = ms->m_header.h_nodeid; + + error = create_lkb(ls, &lkb); + if (error) + goto fail; + + receive_flags(lkb, ms); + lkb->lkb_flags |= DLM_IFL_MSTCPY; + error = receive_request_args(ls, lkb, ms); + if (error) { + __put_lkb(ls, lkb); + goto fail; + } + + /* The dir node is the authority on whether we are the master + for this rsb or not, so if the master sends us a request, we should + recreate the rsb if we've destroyed it. This race happens when we + send a remove message to the dir node at the same time that the dir + node sends us a request for the rsb. */ + + namelen = receive_extralen(ms); + + error = find_rsb(ls, ms->m_extra, namelen, from_nodeid, + R_RECEIVE_REQUEST, &r); + if (error) { + __put_lkb(ls, lkb); + goto fail; + } + + lock_rsb(r); + + if (r->res_master_nodeid != dlm_our_nodeid()) { + error = validate_master_nodeid(ls, r, from_nodeid); + if (error) { + unlock_rsb(r); + put_rsb(r); + __put_lkb(ls, lkb); + goto fail; + } + } + + attach_lkb(r, lkb); + error = do_request(r, lkb); + send_request_reply(r, lkb, error); + do_request_effects(r, lkb, error); + + unlock_rsb(r); + put_rsb(r); + + if (error == -EINPROGRESS) + error = 0; + if (error) + dlm_put_lkb(lkb); + return 0; + + fail: + /* TODO: instead of returning ENOTBLK, add the lkb to res_lookup + and do this receive_request again from process_lookup_list once + we get the lookup reply. This would avoid a many repeated + ENOTBLK request failures when the lookup reply designating us + as master is delayed. */ + + /* We could repeatedly return -EBADR here if our send_remove() is + delayed in being sent/arriving/being processed on the dir node. + Another node would repeatedly lookup up the master, and the dir + node would continue returning our nodeid until our send_remove + took effect. + + We send another remove message in case our previous send_remove + was lost/ignored/missed somehow. */ + + if (error != -ENOTBLK) { + log_limit(ls, "receive_request %x from %d %d", + ms->m_lkid, from_nodeid, error); + } + + if (namelen && error == -EBADR) { + send_repeat_remove(ls, ms->m_extra, namelen); + msleep(1000); + } + + setup_stub_lkb(ls, ms); + send_request_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); + return error; +} + +static int receive_convert(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, reply = 1; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + goto fail; + + if (lkb->lkb_remid != ms->m_lkid) { + log_error(ls, "receive_convert %x remid %x recover_seq %llu " + "remote %d %x", lkb->lkb_id, lkb->lkb_remid, + (unsigned long long)lkb->lkb_recover_seq, + ms->m_header.h_nodeid, ms->m_lkid); + error = -ENOENT; + goto fail; + } + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags(lkb, ms); + + error = receive_convert_args(ls, lkb, ms); + if (error) { + send_convert_reply(r, lkb, error); + goto out; + } + + reply = !down_conversion(lkb); + + error = do_convert(r, lkb); + if (reply) + send_convert_reply(r, lkb, error); + do_convert_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_stub_lkb(ls, ms); + send_convert_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); + return error; +} + +static int receive_unlock(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + goto fail; + + if (lkb->lkb_remid != ms->m_lkid) { + log_error(ls, "receive_unlock %x remid %x remote %d %x", + lkb->lkb_id, lkb->lkb_remid, + ms->m_header.h_nodeid, ms->m_lkid); + error = -ENOENT; + goto fail; + } + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags(lkb, ms); + + error = receive_unlock_args(ls, lkb, ms); + if (error) { + send_unlock_reply(r, lkb, error); + goto out; + } + + error = do_unlock(r, lkb); + send_unlock_reply(r, lkb, error); + do_unlock_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_stub_lkb(ls, ms); + send_unlock_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); + return error; +} + +static int receive_cancel(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + goto fail; + + receive_flags(lkb, ms); + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + error = do_cancel(r, lkb); + send_cancel_reply(r, lkb, error); + do_cancel_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_stub_lkb(ls, ms); + send_cancel_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); + return error; +} + +static int receive_grant(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags_reply(lkb, ms); + if (is_altmode(lkb)) + munge_altmode(lkb, ms); + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static int receive_bast(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + queue_bast(r, lkb, ms->m_bastmode); + lkb->lkb_highbast = ms->m_bastmode; + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static void receive_lookup(struct dlm_ls *ls, struct dlm_message *ms) +{ + int len, error, ret_nodeid, from_nodeid, our_nodeid; + + from_nodeid = ms->m_header.h_nodeid; + our_nodeid = dlm_our_nodeid(); + + len = receive_extralen(ms); + + error = dlm_master_lookup(ls, from_nodeid, ms->m_extra, len, 0, + &ret_nodeid, NULL); + + /* Optimization: we're master so treat lookup as a request */ + if (!error && ret_nodeid == our_nodeid) { + receive_request(ls, ms); + return; + } + send_lookup_reply(ls, ms, ret_nodeid, error); +} + +static void receive_remove(struct dlm_ls *ls, struct dlm_message *ms) +{ + char name[DLM_RESNAME_MAXLEN+1]; + struct dlm_rsb *r; + uint32_t hash, b; + int rv, len, dir_nodeid, from_nodeid; + + from_nodeid = ms->m_header.h_nodeid; + + len = receive_extralen(ms); + + if (len > DLM_RESNAME_MAXLEN) { + log_error(ls, "receive_remove from %d bad len %d", + from_nodeid, len); + return; + } + + dir_nodeid = dlm_hash2nodeid(ls, ms->m_hash); + if (dir_nodeid != dlm_our_nodeid()) { + log_error(ls, "receive_remove from %d bad nodeid %d", + from_nodeid, dir_nodeid); + return; + } + + /* Look for name on rsbtbl.toss, if it's there, kill it. + If it's on rsbtbl.keep, it's being used, and we should ignore this + message. This is an expected race between the dir node sending a + request to the master node at the same time as the master node sends + a remove to the dir node. The resolution to that race is for the + dir node to ignore the remove message, and the master node to + recreate the master rsb when it gets a request from the dir node for + an rsb it doesn't have. */ + + memset(name, 0, sizeof(name)); + memcpy(name, ms->m_extra, len); + + hash = jhash(name, len, 0); + b = hash & (ls->ls_rsbtbl_size - 1); + + spin_lock(&ls->ls_rsbtbl[b].lock); + + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); + if (rv) { + /* verify the rsb is on keep list per comment above */ + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); + if (rv) { + /* should not happen */ + log_error(ls, "receive_remove from %d not found %s", + from_nodeid, name); + spin_unlock(&ls->ls_rsbtbl[b].lock); + return; + } + if (r->res_master_nodeid != from_nodeid) { + /* should not happen */ + log_error(ls, "receive_remove keep from %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + spin_unlock(&ls->ls_rsbtbl[b].lock); + return; + } + + log_debug(ls, "receive_remove from %d master %d first %x %s", + from_nodeid, r->res_master_nodeid, r->res_first_lkid, + name); + spin_unlock(&ls->ls_rsbtbl[b].lock); + return; + } + + if (r->res_master_nodeid != from_nodeid) { + log_error(ls, "receive_remove toss from %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + spin_unlock(&ls->ls_rsbtbl[b].lock); + return; + } + + if (kref_put(&r->res_ref, kill_rsb)) { + rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); + spin_unlock(&ls->ls_rsbtbl[b].lock); + dlm_free_rsb(r); + } else { + log_error(ls, "receive_remove from %d rsb ref error", + from_nodeid); + dlm_print_rsb(r); + spin_unlock(&ls->ls_rsbtbl[b].lock); + } +} + +static void receive_purge(struct dlm_ls *ls, struct dlm_message *ms) +{ + do_purge(ls, ms->m_nodeid, ms->m_pid); +} + +static int receive_request_reply(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, mstype, result; + int from_nodeid = ms->m_header.h_nodeid; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + mstype = lkb->lkb_wait_type; + error = remove_from_waiters(lkb, DLM_MSG_REQUEST_REPLY); + if (error) { + log_error(ls, "receive_request_reply %x remote %d %x result %d", + lkb->lkb_id, from_nodeid, ms->m_lkid, ms->m_result); + dlm_dump_rsb(r); + goto out; + } + + /* Optimization: the dir node was also the master, so it took our + lookup as a request and sent request reply instead of lookup reply */ + if (mstype == DLM_MSG_LOOKUP) { + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + lkb->lkb_nodeid = from_nodeid; + } + + /* this is the value returned from do_request() on the master */ + result = ms->m_result; + + switch (result) { + case -EAGAIN: + /* request would block (be queued) on remote master */ + queue_cast(r, lkb, -EAGAIN); + confirm_master(r, -EAGAIN); + unhold_lkb(lkb); /* undoes create_lkb() */ + break; + + case -EINPROGRESS: + case 0: + /* request was queued or granted on remote master */ + receive_flags_reply(lkb, ms); + lkb->lkb_remid = ms->m_lkid; + if (is_altmode(lkb)) + munge_altmode(lkb, ms); + if (result) { + add_lkb(r, lkb, DLM_LKSTS_WAITING); + add_timeout(lkb); + } else { + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + } + confirm_master(r, result); + break; + + case -EBADR: + case -ENOTBLK: + /* find_rsb failed to find rsb or rsb wasn't master */ + log_limit(ls, "receive_request_reply %x from %d %d " + "master %d dir %d first %x %s", lkb->lkb_id, + from_nodeid, result, r->res_master_nodeid, + r->res_dir_nodeid, r->res_first_lkid, r->res_name); + + if (r->res_dir_nodeid != dlm_our_nodeid() && + r->res_master_nodeid != dlm_our_nodeid()) { + /* cause _request_lock->set_master->send_lookup */ + r->res_master_nodeid = 0; + r->res_nodeid = -1; + lkb->lkb_nodeid = -1; + } + + if (is_overlap(lkb)) { + /* we'll ignore error in cancel/unlock reply */ + queue_cast_overlap(r, lkb); + confirm_master(r, result); + unhold_lkb(lkb); /* undoes create_lkb() */ + } else { + _request_lock(r, lkb); + + if (r->res_master_nodeid == dlm_our_nodeid()) + confirm_master(r, 0); + } + break; + + default: + log_error(ls, "receive_request_reply %x error %d", + lkb->lkb_id, result); + } + + if (is_overlap_unlock(lkb) && (result == 0 || result == -EINPROGRESS)) { + log_debug(ls, "receive_request_reply %x result %d unlock", + lkb->lkb_id, result); + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + send_unlock(r, lkb); + } else if (is_overlap_cancel(lkb) && (result == -EINPROGRESS)) { + log_debug(ls, "receive_request_reply %x cancel", lkb->lkb_id); + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + send_cancel(r, lkb); + } else { + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; + } + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + /* this is the value returned from do_convert() on the master */ + switch (ms->m_result) { + case -EAGAIN: + /* convert would block (be queued) on remote master */ + queue_cast(r, lkb, -EAGAIN); + break; + + case -EDEADLK: + receive_flags_reply(lkb, ms); + revert_lock_pc(r, lkb); + queue_cast(r, lkb, -EDEADLK); + break; + + case -EINPROGRESS: + /* convert was queued on remote master */ + receive_flags_reply(lkb, ms); + if (is_demoted(lkb)) + munge_demoted(lkb); + del_lkb(r, lkb); + add_lkb(r, lkb, DLM_LKSTS_CONVERT); + add_timeout(lkb); + break; + + case 0: + /* convert was granted on remote master */ + receive_flags_reply(lkb, ms); + if (is_demoted(lkb)) + munge_demoted(lkb); + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + break; + + default: + log_error(r->res_ls, "receive_convert_reply %x remote %d %x %d", + lkb->lkb_id, ms->m_header.h_nodeid, ms->m_lkid, + ms->m_result); + dlm_print_rsb(r); + dlm_print_lkb(lkb); + } +} + +static void _receive_convert_reply(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + /* stub reply can happen with waiters_mutex held */ + error = remove_from_waiters_ms(lkb, ms); + if (error) + goto out; + + __receive_convert_reply(r, lkb, ms); + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_convert_reply(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + _receive_convert_reply(lkb, ms); + dlm_put_lkb(lkb); + return 0; +} + +static void _receive_unlock_reply(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + /* stub reply can happen with waiters_mutex held */ + error = remove_from_waiters_ms(lkb, ms); + if (error) + goto out; + + /* this is the value returned from do_unlock() on the master */ + + switch (ms->m_result) { + case -DLM_EUNLOCK: + receive_flags_reply(lkb, ms); + remove_lock_pc(r, lkb); + queue_cast(r, lkb, -DLM_EUNLOCK); + break; + case -ENOENT: + break; + default: + log_error(r->res_ls, "receive_unlock_reply %x error %d", + lkb->lkb_id, ms->m_result); + } + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_unlock_reply(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + _receive_unlock_reply(lkb, ms); + dlm_put_lkb(lkb); + return 0; +} + +static void _receive_cancel_reply(struct dlm_lkb *lkb, struct dlm_message *ms) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + /* stub reply can happen with waiters_mutex held */ + error = remove_from_waiters_ms(lkb, ms); + if (error) + goto out; + + /* this is the value returned from do_cancel() on the master */ + + switch (ms->m_result) { + case -DLM_ECANCEL: + receive_flags_reply(lkb, ms); + revert_lock_pc(r, lkb); + queue_cast(r, lkb, -DLM_ECANCEL); + break; + case 0: + break; + default: + log_error(r->res_ls, "receive_cancel_reply %x error %d", + lkb->lkb_id, ms->m_result); + } + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_cancel_reply(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, ms->m_remid, &lkb); + if (error) + return error; + + _receive_cancel_reply(lkb, ms); + dlm_put_lkb(lkb); + return 0; +} + +static void receive_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, ret_nodeid; + int do_lookup_list = 0; + + error = find_lkb(ls, ms->m_lkid, &lkb); + if (error) { + log_error(ls, "receive_lookup_reply no lkid %x", ms->m_lkid); + return; + } + + /* ms->m_result is the value returned by dlm_master_lookup on dir node + FIXME: will a non-zero error ever be returned? */ + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); + if (error) + goto out; + + ret_nodeid = ms->m_nodeid; + + /* We sometimes receive a request from the dir node for this + rsb before we've received the dir node's loookup_reply for it. + The request from the dir node implies we're the master, so we set + ourself as master in receive_request_reply, and verify here that + we are indeed the master. */ + + if (r->res_master_nodeid && (r->res_master_nodeid != ret_nodeid)) { + /* This should never happen */ + log_error(ls, "receive_lookup_reply %x from %d ret %d " + "master %d dir %d our %d first %x %s", + lkb->lkb_id, ms->m_header.h_nodeid, ret_nodeid, + r->res_master_nodeid, r->res_dir_nodeid, + dlm_our_nodeid(), r->res_first_lkid, r->res_name); + } + + if (ret_nodeid == dlm_our_nodeid()) { + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = 0; + do_lookup_list = 1; + r->res_first_lkid = 0; + } else if (ret_nodeid == -1) { + /* the remote node doesn't believe it's the dir node */ + log_error(ls, "receive_lookup_reply %x from %d bad ret_nodeid", + lkb->lkb_id, ms->m_header.h_nodeid); + r->res_master_nodeid = 0; + r->res_nodeid = -1; + lkb->lkb_nodeid = -1; + } else { + /* set_master() will set lkb_nodeid from r */ + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = ret_nodeid; + } + + if (is_overlap(lkb)) { + log_debug(ls, "receive_lookup_reply %x unlock %x", + lkb->lkb_id, lkb->lkb_flags); + queue_cast_overlap(r, lkb); + unhold_lkb(lkb); /* undoes create_lkb() */ + goto out_list; + } + + _request_lock(r, lkb); + + out_list: + if (do_lookup_list) + process_lookup_list(r); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); +} + +static void _receive_message(struct dlm_ls *ls, struct dlm_message *ms, + uint32_t saved_seq) +{ + int error = 0, noent = 0; + + if (!dlm_is_member(ls, ms->m_header.h_nodeid)) { + log_limit(ls, "receive %d from non-member %d %x %x %d", + ms->m_type, ms->m_header.h_nodeid, ms->m_lkid, + ms->m_remid, ms->m_result); + return; + } + + switch (ms->m_type) { + + /* messages sent to a master node */ + + case DLM_MSG_REQUEST: + error = receive_request(ls, ms); + break; + + case DLM_MSG_CONVERT: + error = receive_convert(ls, ms); + break; + + case DLM_MSG_UNLOCK: + error = receive_unlock(ls, ms); + break; + + case DLM_MSG_CANCEL: + noent = 1; + error = receive_cancel(ls, ms); + break; + + /* messages sent from a master node (replies to above) */ + + case DLM_MSG_REQUEST_REPLY: + error = receive_request_reply(ls, ms); + break; + + case DLM_MSG_CONVERT_REPLY: + error = receive_convert_reply(ls, ms); + break; + + case DLM_MSG_UNLOCK_REPLY: + error = receive_unlock_reply(ls, ms); + break; + + case DLM_MSG_CANCEL_REPLY: + error = receive_cancel_reply(ls, ms); + break; + + /* messages sent from a master node (only two types of async msg) */ + + case DLM_MSG_GRANT: + noent = 1; + error = receive_grant(ls, ms); + break; + + case DLM_MSG_BAST: + noent = 1; + error = receive_bast(ls, ms); + break; + + /* messages sent to a dir node */ + + case DLM_MSG_LOOKUP: + receive_lookup(ls, ms); + break; + + case DLM_MSG_REMOVE: + receive_remove(ls, ms); + break; + + /* messages sent from a dir node (remove has no reply) */ + + case DLM_MSG_LOOKUP_REPLY: + receive_lookup_reply(ls, ms); + break; + + /* other messages */ + + case DLM_MSG_PURGE: + receive_purge(ls, ms); + break; + + default: + log_error(ls, "unknown message type %d", ms->m_type); + } + + /* + * When checking for ENOENT, we're checking the result of + * find_lkb(m_remid): + * + * The lock id referenced in the message wasn't found. This may + * happen in normal usage for the async messages and cancel, so + * only use log_debug for them. + * + * Some errors are expected and normal. + */ + + if (error == -ENOENT && noent) { + log_debug(ls, "receive %d no %x remote %d %x saved_seq %u", + ms->m_type, ms->m_remid, ms->m_header.h_nodeid, + ms->m_lkid, saved_seq); + } else if (error == -ENOENT) { + log_error(ls, "receive %d no %x remote %d %x saved_seq %u", + ms->m_type, ms->m_remid, ms->m_header.h_nodeid, + ms->m_lkid, saved_seq); + + if (ms->m_type == DLM_MSG_CONVERT) + dlm_dump_rsb_hash(ls, ms->m_hash); + } + + if (error == -EINVAL) { + log_error(ls, "receive %d inval from %d lkid %x remid %x " + "saved_seq %u", + ms->m_type, ms->m_header.h_nodeid, + ms->m_lkid, ms->m_remid, saved_seq); + } +} + +/* If the lockspace is in recovery mode (locking stopped), then normal + messages are saved on the requestqueue for processing after recovery is + done. When not in recovery mode, we wait for dlm_recoverd to drain saved + messages off the requestqueue before we process new ones. This occurs right + after recovery completes when we transition from saving all messages on + requestqueue, to processing all the saved messages, to processing new + messages as they arrive. */ + +static void dlm_receive_message(struct dlm_ls *ls, struct dlm_message *ms, + int nodeid) +{ + if (dlm_locking_stopped(ls)) { + /* If we were a member of this lockspace, left, and rejoined, + other nodes may still be sending us messages from the + lockspace generation before we left. */ + if (!ls->ls_generation) { + log_limit(ls, "receive %d from %d ignore old gen", + ms->m_type, nodeid); + return; + } + + dlm_add_requestqueue(ls, nodeid, ms); + } else { + dlm_wait_requestqueue(ls); + _receive_message(ls, ms, 0); + } +} + +/* This is called by dlm_recoverd to process messages that were saved on + the requestqueue. */ + +void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms, + uint32_t saved_seq) +{ + _receive_message(ls, ms, saved_seq); +} + +/* This is called by the midcomms layer when something is received for + the lockspace. It could be either a MSG (normal message sent as part of + standard locking activity) or an RCOM (recovery message sent as part of + lockspace recovery). */ + +void dlm_receive_buffer(union dlm_packet *p, int nodeid) +{ + struct dlm_header *hd = &p->header; + struct dlm_ls *ls; + int type = 0; + + switch (hd->h_cmd) { + case DLM_MSG: + dlm_message_in(&p->message); + type = p->message.m_type; + break; + case DLM_RCOM: + dlm_rcom_in(&p->rcom); + type = p->rcom.rc_type; + break; + default: + log_print("invalid h_cmd %d from %u", hd->h_cmd, nodeid); + return; + } + + if (hd->h_nodeid != nodeid) { + log_print("invalid h_nodeid %d from %d lockspace %x", + hd->h_nodeid, nodeid, hd->h_lockspace); + return; + } + + ls = dlm_find_lockspace_global(hd->h_lockspace); + if (!ls) { + if (dlm_config.ci_log_debug) { + printk_ratelimited(KERN_DEBUG "dlm: invalid lockspace " + "%u from %d cmd %d type %d\n", + hd->h_lockspace, nodeid, hd->h_cmd, type); + } + + if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS) + dlm_send_ls_not_ready(nodeid, &p->rcom); + return; + } + + /* this rwsem allows dlm_ls_stop() to wait for all dlm_recv threads to + be inactive (in this ls) before transitioning to recovery mode */ + + down_read(&ls->ls_recv_active); + if (hd->h_cmd == DLM_MSG) + dlm_receive_message(ls, &p->message, nodeid); + else + dlm_receive_rcom(ls, &p->rcom, nodeid); + up_read(&ls->ls_recv_active); + + dlm_put_lockspace(ls); +} + +static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms_stub) +{ + if (middle_conversion(lkb)) { + hold_lkb(lkb); + memset(ms_stub, 0, sizeof(struct dlm_message)); + ms_stub->m_flags = DLM_IFL_STUB_MS; + ms_stub->m_type = DLM_MSG_CONVERT_REPLY; + ms_stub->m_result = -EINPROGRESS; + ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; + _receive_convert_reply(lkb, ms_stub); + + /* Same special case as in receive_rcom_lock_args() */ + lkb->lkb_grmode = DLM_LOCK_IV; + rsb_set_flag(lkb->lkb_resource, RSB_RECOVER_CONVERT); + unhold_lkb(lkb); + + } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) { + lkb->lkb_flags |= DLM_IFL_RESEND; + } + + /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down + conversions are async; there's no reply from the remote master */ +} + +/* A waiting lkb needs recovery if the master node has failed, or + the master node is changing (only when no directory is used) */ + +static int waiter_needs_recovery(struct dlm_ls *ls, struct dlm_lkb *lkb, + int dir_nodeid) +{ + if (dlm_no_directory(ls)) + return 1; + + if (dlm_is_removed(ls, lkb->lkb_wait_nodeid)) + return 1; + + return 0; +} + +/* Recovery for locks that are waiting for replies from nodes that are now + gone. We can just complete unlocks and cancels by faking a reply from the + dead node. Requests and up-conversions we flag to be resent after + recovery. Down-conversions can just be completed with a fake reply like + unlocks. Conversions between PR and CW need special attention. */ + +void dlm_recover_waiters_pre(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb, *safe; + struct dlm_message *ms_stub; + int wait_type, stub_unlock_result, stub_cancel_result; + int dir_nodeid; + + ms_stub = kmalloc(sizeof(struct dlm_message), GFP_KERNEL); + if (!ms_stub) { + log_error(ls, "dlm_recover_waiters_pre no mem"); + return; + } + + mutex_lock(&ls->ls_waiters_mutex); + + list_for_each_entry_safe(lkb, safe, &ls->ls_waiters, lkb_wait_reply) { + + dir_nodeid = dlm_dir_nodeid(lkb->lkb_resource); + + /* exclude debug messages about unlocks because there can be so + many and they aren't very interesting */ + + if (lkb->lkb_wait_type != DLM_MSG_UNLOCK) { + log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " + "lkb_nodeid %d wait_nodeid %d dir_nodeid %d", + lkb->lkb_id, + lkb->lkb_remid, + lkb->lkb_wait_type, + lkb->lkb_resource->res_nodeid, + lkb->lkb_nodeid, + lkb->lkb_wait_nodeid, + dir_nodeid); + } + + /* all outstanding lookups, regardless of destination will be + resent after recovery is done */ + + if (lkb->lkb_wait_type == DLM_MSG_LOOKUP) { + lkb->lkb_flags |= DLM_IFL_RESEND; + continue; + } + + if (!waiter_needs_recovery(ls, lkb, dir_nodeid)) + continue; + + wait_type = lkb->lkb_wait_type; + stub_unlock_result = -DLM_EUNLOCK; + stub_cancel_result = -DLM_ECANCEL; + + /* Main reply may have been received leaving a zero wait_type, + but a reply for the overlapping op may not have been + received. In that case we need to fake the appropriate + reply for the overlap op. */ + + if (!wait_type) { + if (is_overlap_cancel(lkb)) { + wait_type = DLM_MSG_CANCEL; + if (lkb->lkb_grmode == DLM_LOCK_IV) + stub_cancel_result = 0; + } + if (is_overlap_unlock(lkb)) { + wait_type = DLM_MSG_UNLOCK; + if (lkb->lkb_grmode == DLM_LOCK_IV) + stub_unlock_result = -ENOENT; + } + + log_debug(ls, "rwpre overlap %x %x %d %d %d", + lkb->lkb_id, lkb->lkb_flags, wait_type, + stub_cancel_result, stub_unlock_result); + } + + switch (wait_type) { + + case DLM_MSG_REQUEST: + lkb->lkb_flags |= DLM_IFL_RESEND; + break; + + case DLM_MSG_CONVERT: + recover_convert_waiter(ls, lkb, ms_stub); + break; + + case DLM_MSG_UNLOCK: + hold_lkb(lkb); + memset(ms_stub, 0, sizeof(struct dlm_message)); + ms_stub->m_flags = DLM_IFL_STUB_MS; + ms_stub->m_type = DLM_MSG_UNLOCK_REPLY; + ms_stub->m_result = stub_unlock_result; + ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; + _receive_unlock_reply(lkb, ms_stub); + dlm_put_lkb(lkb); + break; + + case DLM_MSG_CANCEL: + hold_lkb(lkb); + memset(ms_stub, 0, sizeof(struct dlm_message)); + ms_stub->m_flags = DLM_IFL_STUB_MS; + ms_stub->m_type = DLM_MSG_CANCEL_REPLY; + ms_stub->m_result = stub_cancel_result; + ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; + _receive_cancel_reply(lkb, ms_stub); + dlm_put_lkb(lkb); + break; + + default: + log_error(ls, "invalid lkb wait_type %d %d", + lkb->lkb_wait_type, wait_type); + } + schedule(); + } + mutex_unlock(&ls->ls_waiters_mutex); + kfree(ms_stub); +} + +static struct dlm_lkb *find_resend_waiter(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + int found = 0; + + mutex_lock(&ls->ls_waiters_mutex); + list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { + if (lkb->lkb_flags & DLM_IFL_RESEND) { + hold_lkb(lkb); + found = 1; + break; + } + } + mutex_unlock(&ls->ls_waiters_mutex); + + if (!found) + lkb = NULL; + return lkb; +} + +/* Deal with lookups and lkb's marked RESEND from _pre. We may now be the + master or dir-node for r. Processing the lkb may result in it being placed + back on waiters. */ + +/* We do this after normal locking has been enabled and any saved messages + (in requestqueue) have been processed. We should be confident that at + this point we won't get or process a reply to any of these waiting + operations. But, new ops may be coming in on the rsbs/locks here from + userspace or remotely. */ + +/* there may have been an overlap unlock/cancel prior to recovery or after + recovery. if before, the lkb may still have a pos wait_count; if after, the + overlap flag would just have been set and nothing new sent. we can be + confident here than any replies to either the initial op or overlap ops + prior to recovery have been received. */ + +int dlm_recover_waiters_post(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error = 0, mstype, err, oc, ou; + + while (1) { + if (dlm_locking_stopped(ls)) { + log_debug(ls, "recover_waiters_post aborted"); + error = -EINTR; + break; + } + + lkb = find_resend_waiter(ls); + if (!lkb) + break; + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + mstype = lkb->lkb_wait_type; + oc = is_overlap_cancel(lkb); + ou = is_overlap_unlock(lkb); + err = 0; + + log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " + "lkb_nodeid %d wait_nodeid %d dir_nodeid %d " + "overlap %d %d", lkb->lkb_id, lkb->lkb_remid, mstype, + r->res_nodeid, lkb->lkb_nodeid, lkb->lkb_wait_nodeid, + dlm_dir_nodeid(r), oc, ou); + + /* At this point we assume that we won't get a reply to any + previous op or overlap op on this lock. First, do a big + remove_from_waiters() for all previous ops. */ + + lkb->lkb_flags &= ~DLM_IFL_RESEND; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; + lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; + lkb->lkb_wait_type = 0; + lkb->lkb_wait_count = 0; + mutex_lock(&ls->ls_waiters_mutex); + list_del_init(&lkb->lkb_wait_reply); + mutex_unlock(&ls->ls_waiters_mutex); + unhold_lkb(lkb); /* for waiters list */ + + if (oc || ou) { + /* do an unlock or cancel instead of resending */ + switch (mstype) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + queue_cast(r, lkb, ou ? -DLM_EUNLOCK : + -DLM_ECANCEL); + unhold_lkb(lkb); /* undoes create_lkb() */ + break; + case DLM_MSG_CONVERT: + if (oc) { + queue_cast(r, lkb, -DLM_ECANCEL); + } else { + lkb->lkb_exflags |= DLM_LKF_FORCEUNLOCK; + _unlock_lock(r, lkb); + } + break; + default: + err = 1; + } + } else { + switch (mstype) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + _request_lock(r, lkb); + if (is_master(r)) + confirm_master(r, 0); + break; + case DLM_MSG_CONVERT: + _convert_lock(r, lkb); + break; + default: + err = 1; + } + } + + if (err) { + log_error(ls, "waiter %x msg %d r_nodeid %d " + "dir_nodeid %d overlap %d %d", + lkb->lkb_id, mstype, r->res_nodeid, + dlm_dir_nodeid(r), oc, ou); + } + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + } + + return error; +} + +static void purge_mstcpy_list(struct dlm_ls *ls, struct dlm_rsb *r, + struct list_head *list) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { + if (!is_master_copy(lkb)) + continue; + + /* don't purge lkbs we've added in recover_master_copy for + the current recovery seq */ + + if (lkb->lkb_recover_seq == ls->ls_recover_seq) + continue; + + del_lkb(r, lkb); + + /* this put should free the lkb */ + if (!dlm_put_lkb(lkb)) + log_error(ls, "purged mstcpy lkb not released"); + } +} + +void dlm_purge_mstcpy_locks(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + purge_mstcpy_list(ls, r, &r->res_grantqueue); + purge_mstcpy_list(ls, r, &r->res_convertqueue); + purge_mstcpy_list(ls, r, &r->res_waitqueue); +} + +static void purge_dead_list(struct dlm_ls *ls, struct dlm_rsb *r, + struct list_head *list, + int nodeid_gone, unsigned int *count) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { + if (!is_master_copy(lkb)) + continue; + + if ((lkb->lkb_nodeid == nodeid_gone) || + dlm_is_removed(ls, lkb->lkb_nodeid)) { + + /* tell recover_lvb to invalidate the lvb + because a node holding EX/PW failed */ + if ((lkb->lkb_exflags & DLM_LKF_VALBLK) && + (lkb->lkb_grmode >= DLM_LOCK_PW)) { + rsb_set_flag(r, RSB_RECOVER_LVB_INVAL); + } + + del_lkb(r, lkb); + + /* this put should free the lkb */ + if (!dlm_put_lkb(lkb)) + log_error(ls, "purged dead lkb not released"); + + rsb_set_flag(r, RSB_RECOVER_GRANT); + + (*count)++; + } + } +} + +/* Get rid of locks held by nodes that are gone. */ + +void dlm_recover_purge(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + struct dlm_member *memb; + int nodes_count = 0; + int nodeid_gone = 0; + unsigned int lkb_count = 0; + + /* cache one removed nodeid to optimize the common + case of a single node removed */ + + list_for_each_entry(memb, &ls->ls_nodes_gone, list) { + nodes_count++; + nodeid_gone = memb->nodeid; + } + + if (!nodes_count) + return; + + down_write(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + hold_rsb(r); + lock_rsb(r); + if (is_master(r)) { + purge_dead_list(ls, r, &r->res_grantqueue, + nodeid_gone, &lkb_count); + purge_dead_list(ls, r, &r->res_convertqueue, + nodeid_gone, &lkb_count); + purge_dead_list(ls, r, &r->res_waitqueue, + nodeid_gone, &lkb_count); + } + unlock_rsb(r); + unhold_rsb(r); + cond_resched(); + } + up_write(&ls->ls_root_sem); + + if (lkb_count) + log_debug(ls, "dlm_recover_purge %u locks for %u nodes", + lkb_count, nodes_count); +} + +static struct dlm_rsb *find_grant_rsb(struct dlm_ls *ls, int bucket) +{ + struct rb_node *n; + struct dlm_rsb *r; + + spin_lock(&ls->ls_rsbtbl[bucket].lock); + for (n = rb_first(&ls->ls_rsbtbl[bucket].keep); n; n = rb_next(n)) { + r = rb_entry(n, struct dlm_rsb, res_hashnode); + + if (!rsb_flag(r, RSB_RECOVER_GRANT)) + continue; + if (!is_master(r)) { + rsb_clear_flag(r, RSB_RECOVER_GRANT); + continue; + } + hold_rsb(r); + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + return r; + } + spin_unlock(&ls->ls_rsbtbl[bucket].lock); + return NULL; +} + +/* + * Attempt to grant locks on resources that we are the master of. + * Locks may have become grantable during recovery because locks + * from departed nodes have been purged (or not rebuilt), allowing + * previously blocked locks to now be granted. The subset of rsb's + * we are interested in are those with lkb's on either the convert or + * waiting queues. + * + * Simplest would be to go through each master rsb and check for non-empty + * convert or waiting queues, and attempt to grant on those rsbs. + * Checking the queues requires lock_rsb, though, for which we'd need + * to release the rsbtbl lock. This would make iterating through all + * rsb's very inefficient. So, we rely on earlier recovery routines + * to set RECOVER_GRANT on any rsb's that we should attempt to grant + * locks for. + */ + +void dlm_recover_grant(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + int bucket = 0; + unsigned int count = 0; + unsigned int rsb_count = 0; + unsigned int lkb_count = 0; + + while (1) { + r = find_grant_rsb(ls, bucket); + if (!r) { + if (bucket == ls->ls_rsbtbl_size - 1) + break; + bucket++; + continue; + } + rsb_count++; + count = 0; + lock_rsb(r); + /* the RECOVER_GRANT flag is checked in the grant path */ + grant_pending_locks(r, &count); + rsb_clear_flag(r, RSB_RECOVER_GRANT); + lkb_count += count; + confirm_master(r, 0); + unlock_rsb(r); + put_rsb(r); + cond_resched(); + } + + if (lkb_count) + log_debug(ls, "dlm_recover_grant %u locks on %u resources", + lkb_count, rsb_count); +} + +static struct dlm_lkb *search_remid_list(struct list_head *head, int nodeid, + uint32_t remid) +{ + struct dlm_lkb *lkb; + + list_for_each_entry(lkb, head, lkb_statequeue) { + if (lkb->lkb_nodeid == nodeid && lkb->lkb_remid == remid) + return lkb; + } + return NULL; +} + +static struct dlm_lkb *search_remid(struct dlm_rsb *r, int nodeid, + uint32_t remid) +{ + struct dlm_lkb *lkb; + + lkb = search_remid_list(&r->res_grantqueue, nodeid, remid); + if (lkb) + return lkb; + lkb = search_remid_list(&r->res_convertqueue, nodeid, remid); + if (lkb) + return lkb; + lkb = search_remid_list(&r->res_waitqueue, nodeid, remid); + if (lkb) + return lkb; + return NULL; +} + +/* needs at least dlm_rcom + rcom_lock */ +static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_rsb *r, struct dlm_rcom *rc) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + + lkb->lkb_nodeid = rc->rc_header.h_nodeid; + lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid); + lkb->lkb_remid = le32_to_cpu(rl->rl_lkid); + lkb->lkb_exflags = le32_to_cpu(rl->rl_exflags); + lkb->lkb_flags = le32_to_cpu(rl->rl_flags) & 0x0000FFFF; + lkb->lkb_flags |= DLM_IFL_MSTCPY; + lkb->lkb_lvbseq = le32_to_cpu(rl->rl_lvbseq); + lkb->lkb_rqmode = rl->rl_rqmode; + lkb->lkb_grmode = rl->rl_grmode; + /* don't set lkb_status because add_lkb wants to itself */ + + lkb->lkb_bastfn = (rl->rl_asts & DLM_CB_BAST) ? &fake_bastfn : NULL; + lkb->lkb_astfn = (rl->rl_asts & DLM_CB_CAST) ? &fake_astfn : NULL; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + int lvblen = rc->rc_header.h_length - sizeof(struct dlm_rcom) - + sizeof(struct rcom_lock); + if (lvblen > ls->ls_lvblen) + return -EINVAL; + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + memcpy(lkb->lkb_lvbptr, rl->rl_lvb, lvblen); + } + + /* Conversions between PR and CW (middle modes) need special handling. + The real granted mode of these converting locks cannot be determined + until all locks have been rebuilt on the rsb (recover_conversion) */ + + if (rl->rl_wait_type == cpu_to_le16(DLM_MSG_CONVERT) && + middle_conversion(lkb)) { + rl->rl_status = DLM_LKSTS_CONVERT; + lkb->lkb_grmode = DLM_LOCK_IV; + rsb_set_flag(r, RSB_RECOVER_CONVERT); + } + + return 0; +} + +/* This lkb may have been recovered in a previous aborted recovery so we need + to check if the rsb already has an lkb with the given remote nodeid/lkid. + If so we just send back a standard reply. If not, we create a new lkb with + the given values and send back our lkid. We send back our lkid by sending + back the rcom_lock struct we got but with the remid field filled in. */ + +/* needs at least dlm_rcom + rcom_lock */ +int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + struct dlm_rsb *r; + struct dlm_lkb *lkb; + uint32_t remid = 0; + int from_nodeid = rc->rc_header.h_nodeid; + int error; + + if (rl->rl_parent_lkid) { + error = -EOPNOTSUPP; + goto out; + } + + remid = le32_to_cpu(rl->rl_lkid); + + /* In general we expect the rsb returned to be R_MASTER, but we don't + have to require it. Recovery of masters on one node can overlap + recovery of locks on another node, so one node can send us MSTCPY + locks before we've made ourselves master of this rsb. We can still + add new MSTCPY locks that we receive here without any harm; when + we make ourselves master, dlm_recover_masters() won't touch the + MSTCPY locks we've received early. */ + + error = find_rsb(ls, rl->rl_name, le16_to_cpu(rl->rl_namelen), + from_nodeid, R_RECEIVE_RECOVER, &r); + if (error) + goto out; + + lock_rsb(r); + + if (dlm_no_directory(ls) && (dlm_dir_nodeid(r) != dlm_our_nodeid())) { + log_error(ls, "dlm_recover_master_copy remote %d %x not dir", + from_nodeid, remid); + error = -EBADR; + goto out_unlock; + } + + lkb = search_remid(r, from_nodeid, remid); + if (lkb) { + error = -EEXIST; + goto out_remid; + } + + error = create_lkb(ls, &lkb); + if (error) + goto out_unlock; + + error = receive_rcom_lock_args(ls, lkb, r, rc); + if (error) { + __put_lkb(ls, lkb); + goto out_unlock; + } + + attach_lkb(r, lkb); + add_lkb(r, lkb, rl->rl_status); + error = 0; + ls->ls_recover_locks_in++; + + if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) + rsb_set_flag(r, RSB_RECOVER_GRANT); + + out_remid: + /* this is the new value returned to the lock holder for + saving in its process-copy lkb */ + rl->rl_remid = cpu_to_le32(lkb->lkb_id); + + lkb->lkb_recover_seq = ls->ls_recover_seq; + + out_unlock: + unlock_rsb(r); + put_rsb(r); + out: + if (error && error != -EEXIST) + log_debug(ls, "dlm_recover_master_copy remote %d %x error %d", + from_nodeid, remid, error); + rl->rl_result = cpu_to_le32(error); + return error; +} + +/* needs at least dlm_rcom + rcom_lock */ +int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + struct dlm_rsb *r; + struct dlm_lkb *lkb; + uint32_t lkid, remid; + int error, result; + + lkid = le32_to_cpu(rl->rl_lkid); + remid = le32_to_cpu(rl->rl_remid); + result = le32_to_cpu(rl->rl_result); + + error = find_lkb(ls, lkid, &lkb); + if (error) { + log_error(ls, "dlm_recover_process_copy no %x remote %d %x %d", + lkid, rc->rc_header.h_nodeid, remid, result); + return error; + } + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + if (!is_process_copy(lkb)) { + log_error(ls, "dlm_recover_process_copy bad %x remote %d %x %d", + lkid, rc->rc_header.h_nodeid, remid, result); + dlm_dump_rsb(r); + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return -EINVAL; + } + + switch (result) { + case -EBADR: + /* There's a chance the new master received our lock before + dlm_recover_master_reply(), this wouldn't happen if we did + a barrier between recover_masters and recover_locks. */ + + log_debug(ls, "dlm_recover_process_copy %x remote %d %x %d", + lkid, rc->rc_header.h_nodeid, remid, result); + + dlm_send_rcom_lock(r, lkb); + goto out; + case -EEXIST: + case 0: + lkb->lkb_remid = remid; + break; + default: + log_error(ls, "dlm_recover_process_copy %x remote %d %x %d unk", + lkid, rc->rc_header.h_nodeid, remid, result); + } + + /* an ack for dlm_recover_locks() which waits for replies from + all the locks it sends to new masters */ + dlm_recovered_lock(r); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + + return 0; +} + +int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, + int mode, uint32_t flags, void *name, unsigned int namelen, + unsigned long timeout_cs) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + int error; + + dlm_lock_recovery(ls); + + error = create_lkb(ls, &lkb); + if (error) { + kfree(ua); + goto out; + } + + if (flags & DLM_LKF_VALBLK) { + ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); + if (!ua->lksb.sb_lvbptr) { + kfree(ua); + __put_lkb(ls, lkb); + error = -ENOMEM; + goto out; + } + } + + /* After ua is attached to lkb it will be freed by dlm_free_lkb(). + When DLM_IFL_USER is set, the dlm knows that this is a userspace + lock and that lkb_astparam is the dlm_user_args structure. */ + + error = set_lock_args(mode, &ua->lksb, flags, namelen, timeout_cs, + fake_astfn, ua, fake_bastfn, &args); + lkb->lkb_flags |= DLM_IFL_USER; + + if (error) { + __put_lkb(ls, lkb); + goto out; + } + + error = request_lock(ls, lkb, name, namelen, &args); + + switch (error) { + case 0: + break; + case -EINPROGRESS: + error = 0; + break; + case -EAGAIN: + error = 0; + /* fall through */ + default: + __put_lkb(ls, lkb); + goto out; + } + + /* add this new lkb to the per-process list of locks */ + spin_lock(&ua->proc->locks_spin); + hold_lkb(lkb); + list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); + spin_unlock(&ua->proc->locks_spin); + out: + dlm_unlock_recovery(ls); + return error; +} + +int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, uint32_t lkid, char *lvb_in, + unsigned long timeout_cs) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + /* user can change the params on its lock when it converts it, or + add an lvb that didn't exist before */ + + ua = lkb->lkb_ua; + + if (flags & DLM_LKF_VALBLK && !ua->lksb.sb_lvbptr) { + ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); + if (!ua->lksb.sb_lvbptr) { + error = -ENOMEM; + goto out_put; + } + } + if (lvb_in && ua->lksb.sb_lvbptr) + memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); + + ua->xid = ua_tmp->xid; + ua->castparam = ua_tmp->castparam; + ua->castaddr = ua_tmp->castaddr; + ua->bastparam = ua_tmp->bastparam; + ua->bastaddr = ua_tmp->bastaddr; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_lock_args(mode, &ua->lksb, flags, 0, timeout_cs, + fake_astfn, ua, fake_bastfn, &args); + if (error) + goto out_put; + + error = convert_lock(ls, lkb, &args); + + if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK) + error = 0; + out_put: + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +/* + * The caller asks for an orphan lock on a given resource with a given mode. + * If a matching lock exists, it's moved to the owner's list of locks and + * the lkid is returned. + */ + +int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, void *name, unsigned int namelen, + unsigned long timeout_cs, uint32_t *lkid) +{ + struct dlm_lkb *lkb; + struct dlm_user_args *ua; + int found_other_mode = 0; + int found = 0; + int rv = 0; + + mutex_lock(&ls->ls_orphans_mutex); + list_for_each_entry(lkb, &ls->ls_orphans, lkb_ownqueue) { + if (lkb->lkb_resource->res_length != namelen) + continue; + if (memcmp(lkb->lkb_resource->res_name, name, namelen)) + continue; + if (lkb->lkb_grmode != mode) { + found_other_mode = 1; + continue; + } + + found = 1; + list_del_init(&lkb->lkb_ownqueue); + lkb->lkb_flags &= ~DLM_IFL_ORPHAN; + *lkid = lkb->lkb_id; + break; + } + mutex_unlock(&ls->ls_orphans_mutex); + + if (!found && found_other_mode) { + rv = -EAGAIN; + goto out; + } + + if (!found) { + rv = -ENOENT; + goto out; + } + + lkb->lkb_exflags = flags; + lkb->lkb_ownpid = (int) current->pid; + + ua = lkb->lkb_ua; + + ua->proc = ua_tmp->proc; + ua->xid = ua_tmp->xid; + ua->castparam = ua_tmp->castparam; + ua->castaddr = ua_tmp->castaddr; + ua->bastparam = ua_tmp->bastparam; + ua->bastaddr = ua_tmp->bastaddr; + ua->user_lksb = ua_tmp->user_lksb; + + /* + * The lkb reference from the ls_orphans list was not + * removed above, and is now considered the reference + * for the proc locks list. + */ + + spin_lock(&ua->proc->locks_spin); + list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); + spin_unlock(&ua->proc->locks_spin); + out: + kfree(ua_tmp); + return rv; +} + +int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid, char *lvb_in) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + ua = lkb->lkb_ua; + + if (lvb_in && ua->lksb.sb_lvbptr) + memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); + if (ua_tmp->castparam) + ua->castparam = ua_tmp->castparam; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + error = unlock_lock(ls, lkb, &args); + + if (error == -DLM_EUNLOCK) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY && (flags & DLM_LKF_FORCEUNLOCK)) + error = 0; + if (error) + goto out_put; + + spin_lock(&ua->proc->locks_spin); + /* dlm_user_add_cb() may have already taken lkb off the proc list */ + if (!list_empty(&lkb->lkb_ownqueue)) + list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking); + spin_unlock(&ua->proc->locks_spin); + out_put: + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + ua = lkb->lkb_ua; + if (ua_tmp->castparam) + ua->castparam = ua_tmp->castparam; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + error = cancel_lock(ls, lkb, &args); + + if (error == -DLM_ECANCEL) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY) + error = 0; + out_put: + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + struct dlm_rsb *r; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + ua = lkb->lkb_ua; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + /* same as cancel_lock(), but set DEADLOCK_CANCEL after lock_rsb */ + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, &args); + if (error) + goto out_r; + lkb->lkb_flags |= DLM_IFL_DEADLOCK_CANCEL; + + error = _cancel_lock(r, lkb); + out_r: + unlock_rsb(r); + put_rsb(r); + + if (error == -DLM_ECANCEL) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY) + error = 0; + out_put: + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + return error; +} + +/* lkb's that are removed from the waiters list by revert are just left on the + orphans list with the granted orphan locks, to be freed by purge */ + +static int orphan_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + struct dlm_args args; + int error; + + hold_lkb(lkb); /* reference for the ls_orphans list */ + mutex_lock(&ls->ls_orphans_mutex); + list_add_tail(&lkb->lkb_ownqueue, &ls->ls_orphans); + mutex_unlock(&ls->ls_orphans_mutex); + + set_unlock_args(0, lkb->lkb_ua, &args); + + error = cancel_lock(ls, lkb, &args); + if (error == -DLM_ECANCEL) + error = 0; + return error; +} + +/* The FORCEUNLOCK flag allows the unlock to go ahead even if the lkb isn't + granted. Regardless of what rsb queue the lock is on, it's removed and + freed. The IVVALBLK flag causes the lvb on the resource to be invalidated + if our lock is PW/EX (it's ignored if our granted mode is smaller.) */ + +static int unlock_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + struct dlm_args args; + int error; + + set_unlock_args(DLM_LKF_FORCEUNLOCK | DLM_LKF_IVVALBLK, + lkb->lkb_ua, &args); + + error = unlock_lock(ls, lkb, &args); + if (error == -DLM_EUNLOCK) + error = 0; + return error; +} + +/* We have to release clear_proc_locks mutex before calling unlock_proc_lock() + (which does lock_rsb) due to deadlock with receiving a message that does + lock_rsb followed by dlm_user_add_cb() */ + +static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls, + struct dlm_user_proc *proc) +{ + struct dlm_lkb *lkb = NULL; + + mutex_lock(&ls->ls_clear_proc_locks); + if (list_empty(&proc->locks)) + goto out; + + lkb = list_entry(proc->locks.next, struct dlm_lkb, lkb_ownqueue); + list_del_init(&lkb->lkb_ownqueue); + + if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) + lkb->lkb_flags |= DLM_IFL_ORPHAN; + else + lkb->lkb_flags |= DLM_IFL_DEAD; + out: + mutex_unlock(&ls->ls_clear_proc_locks); + return lkb; +} + +/* The ls_clear_proc_locks mutex protects against dlm_user_add_cb() which + 1) references lkb->ua which we free here and 2) adds lkbs to proc->asts, + which we clear here. */ + +/* proc CLOSING flag is set so no more device_reads should look at proc->asts + list, and no more device_writes should add lkb's to proc->locks list; so we + shouldn't need to take asts_spin or locks_spin here. this assumes that + device reads/writes/closes are serialized -- FIXME: we may need to serialize + them ourself. */ + +void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) +{ + struct dlm_lkb *lkb, *safe; + + dlm_lock_recovery(ls); + + while (1) { + lkb = del_proc_lock(ls, proc); + if (!lkb) + break; + del_timeout(lkb); + if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) + orphan_proc_lock(ls, lkb); + else + unlock_proc_lock(ls, lkb); + + /* this removes the reference for the proc->locks list + added by dlm_user_request, it may result in the lkb + being freed */ + + dlm_put_lkb(lkb); + } + + mutex_lock(&ls->ls_clear_proc_locks); + + /* in-progress unlocks */ + list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { + list_del_init(&lkb->lkb_ownqueue); + lkb->lkb_flags |= DLM_IFL_DEAD; + dlm_put_lkb(lkb); + } + + list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_cb_list) { + memset(&lkb->lkb_callbacks, 0, + sizeof(struct dlm_callback) * DLM_CALLBACKS_SIZE); + list_del_init(&lkb->lkb_cb_list); + dlm_put_lkb(lkb); + } + + mutex_unlock(&ls->ls_clear_proc_locks); + dlm_unlock_recovery(ls); +} + +static void purge_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) +{ + struct dlm_lkb *lkb, *safe; + + while (1) { + lkb = NULL; + spin_lock(&proc->locks_spin); + if (!list_empty(&proc->locks)) { + lkb = list_entry(proc->locks.next, struct dlm_lkb, + lkb_ownqueue); + list_del_init(&lkb->lkb_ownqueue); + } + spin_unlock(&proc->locks_spin); + + if (!lkb) + break; + + lkb->lkb_flags |= DLM_IFL_DEAD; + unlock_proc_lock(ls, lkb); + dlm_put_lkb(lkb); /* ref from proc->locks list */ + } + + spin_lock(&proc->locks_spin); + list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { + list_del_init(&lkb->lkb_ownqueue); + lkb->lkb_flags |= DLM_IFL_DEAD; + dlm_put_lkb(lkb); + } + spin_unlock(&proc->locks_spin); + + spin_lock(&proc->asts_spin); + list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_cb_list) { + memset(&lkb->lkb_callbacks, 0, + sizeof(struct dlm_callback) * DLM_CALLBACKS_SIZE); + list_del_init(&lkb->lkb_cb_list); + dlm_put_lkb(lkb); + } + spin_unlock(&proc->asts_spin); +} + +/* pid of 0 means purge all orphans */ + +static void do_purge(struct dlm_ls *ls, int nodeid, int pid) +{ + struct dlm_lkb *lkb, *safe; + + mutex_lock(&ls->ls_orphans_mutex); + list_for_each_entry_safe(lkb, safe, &ls->ls_orphans, lkb_ownqueue) { + if (pid && lkb->lkb_ownpid != pid) + continue; + unlock_proc_lock(ls, lkb); + list_del_init(&lkb->lkb_ownqueue); + dlm_put_lkb(lkb); + } + mutex_unlock(&ls->ls_orphans_mutex); +} + +static int send_purge(struct dlm_ls *ls, int nodeid, int pid) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int error; + + error = _create_message(ls, sizeof(struct dlm_message), nodeid, + DLM_MSG_PURGE, &ms, &mh); + if (error) + return error; + ms->m_nodeid = nodeid; + ms->m_pid = pid; + + return send_message(mh, ms); +} + +int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, + int nodeid, int pid) +{ + int error = 0; + + if (nodeid && (nodeid != dlm_our_nodeid())) { + error = send_purge(ls, nodeid, pid); + } else { + dlm_lock_recovery(ls); + if (pid == current->pid) + purge_proc_locks(ls, proc); + else + do_purge(ls, nodeid, pid); + dlm_unlock_recovery(ls); + } + return error; +} + diff --git a/kmod/dlm/lock.h b/kmod/dlm/lock.h new file mode 100644 index 00000000..ed8ebd3a --- /dev/null +++ b/kmod/dlm/lock.h @@ -0,0 +1,80 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOCK_DOT_H__ +#define __LOCK_DOT_H__ + +void dlm_dump_rsb(struct dlm_rsb *r); +void dlm_dump_rsb_name(struct dlm_ls *ls, char *name, int len); +void dlm_print_lkb(struct dlm_lkb *lkb); +void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms, + uint32_t saved_seq); +void dlm_receive_buffer(union dlm_packet *p, int nodeid); +int dlm_modes_compat(int mode1, int mode2); +void dlm_put_rsb(struct dlm_rsb *r); +void dlm_hold_rsb(struct dlm_rsb *r); +int dlm_put_lkb(struct dlm_lkb *lkb); +void dlm_scan_rsbs(struct dlm_ls *ls); +int dlm_lock_recovery_try(struct dlm_ls *ls); +void dlm_unlock_recovery(struct dlm_ls *ls); +void dlm_scan_waiters(struct dlm_ls *ls); +void dlm_scan_timeout(struct dlm_ls *ls); +void dlm_adjust_timeouts(struct dlm_ls *ls); +int dlm_master_lookup(struct dlm_ls *ls, int nodeid, char *name, int len, + unsigned int flags, int *r_nodeid, int *result); + +int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, + struct dlm_rsb **r_ret); + +void dlm_recover_purge(struct dlm_ls *ls); +void dlm_purge_mstcpy_locks(struct dlm_rsb *r); +void dlm_recover_grant(struct dlm_ls *ls); +int dlm_recover_waiters_post(struct dlm_ls *ls); +void dlm_recover_waiters_pre(struct dlm_ls *ls); +int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc); +int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc); + +int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, int mode, + uint32_t flags, void *name, unsigned int namelen, + unsigned long timeout_cs); +int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, uint32_t lkid, char *lvb_in, + unsigned long timeout_cs); +int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, void *name, unsigned int namelen, + unsigned long timeout_cs, uint32_t *lkid); +int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid, char *lvb_in); +int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid); +int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, + int nodeid, int pid); +int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid); +void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc); + +static inline int is_master(struct dlm_rsb *r) +{ + return !r->res_nodeid; +} + +static inline void lock_rsb(struct dlm_rsb *r) +{ + mutex_lock(&r->res_mutex); +} + +static inline void unlock_rsb(struct dlm_rsb *r) +{ + mutex_unlock(&r->res_mutex); +} + +#endif + diff --git a/kmod/dlm/lockspace.c b/kmod/dlm/lockspace.c new file mode 100644 index 00000000..88556dc0 --- /dev/null +++ b/kmod/dlm/lockspace.c @@ -0,0 +1,906 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "recoverd.h" +#include "dir.h" +#include "lowcomms.h" +#include "config.h" +#include "memory.h" +#include "lock.h" +#include "recover.h" +#include "requestqueue.h" +#include "user.h" +#include "ast.h" + +static int ls_count; +static struct mutex ls_lock; +static struct list_head lslist; +static spinlock_t lslist_lock; +static struct task_struct * scand_task; + + +static ssize_t dlm_control_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + ssize_t ret = len; + int n = simple_strtol(buf, NULL, 0); + + ls = dlm_find_lockspace_local(ls->ls_local_handle); + if (!ls) + return -EINVAL; + + switch (n) { + case 0: + dlm_ls_stop(ls); + break; + case 1: + dlm_ls_start(ls); + break; + default: + ret = -EINVAL; + } + dlm_put_lockspace(ls); + return ret; +} + +static ssize_t dlm_event_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + ls->ls_uevent_result = simple_strtol(buf, NULL, 0); + set_bit(LSFL_UEVENT_WAIT, &ls->ls_flags); + wake_up(&ls->ls_uevent_wait); + return len; +} + +static ssize_t dlm_id_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%u\n", ls->ls_global_id); +} + +static ssize_t dlm_id_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + ls->ls_global_id = simple_strtoul(buf, NULL, 0); + return len; +} + +static ssize_t dlm_nodir_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%u\n", dlm_no_directory(ls)); +} + +static ssize_t dlm_nodir_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + int val = simple_strtoul(buf, NULL, 0); + if (val == 1) + set_bit(LSFL_NODIR, &ls->ls_flags); + return len; +} + +static ssize_t dlm_recover_status_show(struct dlm_ls *ls, char *buf) +{ + uint32_t status = dlm_recover_status(ls); + return snprintf(buf, PAGE_SIZE, "%x\n", status); +} + +static ssize_t dlm_recover_nodeid_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", ls->ls_recover_nodeid); +} + +struct dlm_attr { + struct attribute attr; + ssize_t (*show)(struct dlm_ls *, char *); + ssize_t (*store)(struct dlm_ls *, const char *, size_t); +}; + +static struct dlm_attr dlm_attr_control = { + .attr = {.name = "control", .mode = S_IWUSR}, + .store = dlm_control_store +}; + +static struct dlm_attr dlm_attr_event = { + .attr = {.name = "event_done", .mode = S_IWUSR}, + .store = dlm_event_store +}; + +static struct dlm_attr dlm_attr_id = { + .attr = {.name = "id", .mode = S_IRUGO | S_IWUSR}, + .show = dlm_id_show, + .store = dlm_id_store +}; + +static struct dlm_attr dlm_attr_nodir = { + .attr = {.name = "nodir", .mode = S_IRUGO | S_IWUSR}, + .show = dlm_nodir_show, + .store = dlm_nodir_store +}; + +static struct dlm_attr dlm_attr_recover_status = { + .attr = {.name = "recover_status", .mode = S_IRUGO}, + .show = dlm_recover_status_show +}; + +static struct dlm_attr dlm_attr_recover_nodeid = { + .attr = {.name = "recover_nodeid", .mode = S_IRUGO}, + .show = dlm_recover_nodeid_show +}; + +static struct attribute *dlm_attrs[] = { + &dlm_attr_control.attr, + &dlm_attr_event.attr, + &dlm_attr_id.attr, + &dlm_attr_nodir.attr, + &dlm_attr_recover_status.attr, + &dlm_attr_recover_nodeid.attr, + NULL, +}; + +static ssize_t dlm_attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); + return a->show ? a->show(ls, buf) : 0; +} + +static ssize_t dlm_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); + return a->store ? a->store(ls, buf, len) : len; +} + +static void lockspace_kobj_release(struct kobject *k) +{ + struct dlm_ls *ls = container_of(k, struct dlm_ls, ls_kobj); + kfree(ls); +} + +static const struct sysfs_ops dlm_attr_ops = { + .show = dlm_attr_show, + .store = dlm_attr_store, +}; + +static struct kobj_type dlm_ktype = { + .default_attrs = dlm_attrs, + .sysfs_ops = &dlm_attr_ops, + .release = lockspace_kobj_release, +}; + +static struct kset *dlm_kset; + +static int do_uevent(struct dlm_ls *ls, int in) +{ + int error; + + if (in) + kobject_uevent(&ls->ls_kobj, KOBJ_ONLINE); + else + kobject_uevent(&ls->ls_kobj, KOBJ_OFFLINE); + + log_debug(ls, "%s the lockspace group...", in ? "joining" : "leaving"); + + /* dlm_controld will see the uevent, do the necessary group management + and then write to sysfs to wake us */ + + error = wait_event_interruptible(ls->ls_uevent_wait, + test_and_clear_bit(LSFL_UEVENT_WAIT, &ls->ls_flags)); + + log_debug(ls, "group event done %d %d", error, ls->ls_uevent_result); + + if (error) + goto out; + + error = ls->ls_uevent_result; + out: + if (error) + log_error(ls, "group %s failed %d %d", in ? "join" : "leave", + error, ls->ls_uevent_result); + return error; +} + +static int dlm_uevent(struct kset *kset, struct kobject *kobj, + struct kobj_uevent_env *env) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + + add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name); + return 0; +} + +static struct kset_uevent_ops dlm_uevent_ops = { + .uevent = dlm_uevent, +}; + +int __init dlm_lockspace_init(void) +{ + ls_count = 0; + mutex_init(&ls_lock); + INIT_LIST_HEAD(&lslist); + spin_lock_init(&lslist_lock); + + dlm_kset = kset_create_and_add("dlm", &dlm_uevent_ops, kernel_kobj); + if (!dlm_kset) { + printk(KERN_WARNING "%s: can not create kset\n", __func__); + return -ENOMEM; + } + return 0; +} + +void dlm_lockspace_exit(void) +{ + kset_unregister(dlm_kset); +} + +static struct dlm_ls *find_ls_to_scan(void) +{ + struct dlm_ls *ls; + + spin_lock(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (time_after_eq(jiffies, ls->ls_scan_time + + dlm_config.ci_scan_secs * HZ)) { + spin_unlock(&lslist_lock); + return ls; + } + } + spin_unlock(&lslist_lock); + return NULL; +} + +static int dlm_scand(void *data) +{ + struct dlm_ls *ls; + + while (!kthread_should_stop()) { + ls = find_ls_to_scan(); + if (ls) { + if (dlm_lock_recovery_try(ls)) { + ls->ls_scan_time = jiffies; + dlm_scan_rsbs(ls); + dlm_scan_timeout(ls); + dlm_scan_waiters(ls); + dlm_unlock_recovery(ls); + } else { + ls->ls_scan_time += HZ; + } + continue; + } + schedule_timeout_interruptible(dlm_config.ci_scan_secs * HZ); + } + return 0; +} + +static int dlm_scand_start(void) +{ + struct task_struct *p; + int error = 0; + + p = kthread_run(dlm_scand, NULL, "dlm_scand"); + if (IS_ERR(p)) + error = PTR_ERR(p); + else + scand_task = p; + return error; +} + +static void dlm_scand_stop(void) +{ + kthread_stop(scand_task); +} + +struct dlm_ls *dlm_find_lockspace_global(uint32_t id) +{ + struct dlm_ls *ls; + + spin_lock(&lslist_lock); + + list_for_each_entry(ls, &lslist, ls_list) { + if (ls->ls_global_id == id) { + ls->ls_count++; + goto out; + } + } + ls = NULL; + out: + spin_unlock(&lslist_lock); + return ls; +} + +struct dlm_ls *dlm_find_lockspace_local(dlm_lockspace_t *lockspace) +{ + struct dlm_ls *ls; + + spin_lock(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (ls->ls_local_handle == lockspace) { + ls->ls_count++; + goto out; + } + } + ls = NULL; + out: + spin_unlock(&lslist_lock); + return ls; +} + +struct dlm_ls *dlm_find_lockspace_device(int minor) +{ + struct dlm_ls *ls; + + spin_lock(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (ls->ls_device.minor == minor) { + ls->ls_count++; + goto out; + } + } + ls = NULL; + out: + spin_unlock(&lslist_lock); + return ls; +} + +void dlm_put_lockspace(struct dlm_ls *ls) +{ + spin_lock(&lslist_lock); + ls->ls_count--; + spin_unlock(&lslist_lock); +} + +static void remove_lockspace(struct dlm_ls *ls) +{ + for (;;) { + spin_lock(&lslist_lock); + if (ls->ls_count == 0) { + WARN_ON(ls->ls_create_count != 0); + list_del(&ls->ls_list); + spin_unlock(&lslist_lock); + return; + } + spin_unlock(&lslist_lock); + ssleep(1); + } +} + +static int threads_start(void) +{ + int error; + + error = dlm_scand_start(); + if (error) { + log_print("cannot start dlm_scand thread %d", error); + goto fail; + } + + /* Thread for sending/receiving messages for all lockspace's */ + error = dlm_lowcomms_start(); + if (error) { + log_print("cannot start dlm lowcomms %d", error); + goto scand_fail; + } + + return 0; + + scand_fail: + dlm_scand_stop(); + fail: + return error; +} + +static void threads_stop(void) +{ + dlm_scand_stop(); + dlm_lowcomms_stop(); +} + +static int new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, void *ops_arg, + int *ops_result, dlm_lockspace_t **lockspace) +{ + struct dlm_ls *ls; + int i, size, error; + int do_unreg = 0; + int namelen = strlen(name); + + if (namelen > DLM_LOCKSPACE_LEN) + return -EINVAL; + + if (!lvblen || (lvblen % 8)) + return -EINVAL; + + if (!try_module_get(THIS_MODULE)) + return -EINVAL; + + if (!dlm_user_daemon_available()) { + log_print("dlm user daemon not available"); + error = -EUNATCH; + goto out; + } + + if (ops && ops_result) { + if (!dlm_config.ci_recover_callbacks) + *ops_result = -EOPNOTSUPP; + else + *ops_result = 0; + } + + if (dlm_config.ci_recover_callbacks && cluster && + strncmp(cluster, dlm_config.ci_cluster_name, DLM_LOCKSPACE_LEN)) { + log_print("dlm cluster name %s mismatch %s", + dlm_config.ci_cluster_name, cluster); + error = -EBADR; + goto out; + } + + error = 0; + + spin_lock(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + WARN_ON(ls->ls_create_count <= 0); + if (ls->ls_namelen != namelen) + continue; + if (memcmp(ls->ls_name, name, namelen)) + continue; + if (flags & DLM_LSFL_NEWEXCL) { + error = -EEXIST; + break; + } + ls->ls_create_count++; + *lockspace = ls; + error = 1; + break; + } + spin_unlock(&lslist_lock); + + if (error) + goto out; + + error = -ENOMEM; + + ls = kzalloc(sizeof(struct dlm_ls) + namelen, GFP_NOFS); + if (!ls) + goto out; + memcpy(ls->ls_name, name, namelen); + ls->ls_namelen = namelen; + ls->ls_lvblen = lvblen; + ls->ls_count = 0; + ls->ls_flags = 0; + ls->ls_scan_time = jiffies; + + if (ops && dlm_config.ci_recover_callbacks) { + ls->ls_ops = ops; + ls->ls_ops_arg = ops_arg; + } + + if (flags & DLM_LSFL_TIMEWARN) + set_bit(LSFL_TIMEWARN, &ls->ls_flags); + + /* ls_exflags are forced to match among nodes, and we don't + need to require all nodes to have some flags set */ + ls->ls_exflags = (flags & ~(DLM_LSFL_TIMEWARN | DLM_LSFL_FS | + DLM_LSFL_NEWEXCL)); + + size = dlm_config.ci_rsbtbl_size; + ls->ls_rsbtbl_size = size; + + ls->ls_rsbtbl = vmalloc(sizeof(struct dlm_rsbtable) * size); + if (!ls->ls_rsbtbl) + goto out_lsfree; + for (i = 0; i < size; i++) { + ls->ls_rsbtbl[i].keep.rb_node = NULL; + ls->ls_rsbtbl[i].toss.rb_node = NULL; + spin_lock_init(&ls->ls_rsbtbl[i].lock); + } + + spin_lock_init(&ls->ls_remove_spin); + + for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) { + ls->ls_remove_names[i] = kzalloc(DLM_RESNAME_MAXLEN+1, + GFP_KERNEL); + if (!ls->ls_remove_names[i]) + goto out_rsbtbl; + } + + idr_init(&ls->ls_lkbidr); + spin_lock_init(&ls->ls_lkbidr_spin); + + INIT_LIST_HEAD(&ls->ls_waiters); + mutex_init(&ls->ls_waiters_mutex); + INIT_LIST_HEAD(&ls->ls_orphans); + mutex_init(&ls->ls_orphans_mutex); + INIT_LIST_HEAD(&ls->ls_timeout); + mutex_init(&ls->ls_timeout_mutex); + + INIT_LIST_HEAD(&ls->ls_new_rsb); + spin_lock_init(&ls->ls_new_rsb_spin); + + INIT_LIST_HEAD(&ls->ls_nodes); + INIT_LIST_HEAD(&ls->ls_nodes_gone); + ls->ls_num_nodes = 0; + ls->ls_low_nodeid = 0; + ls->ls_total_weight = 0; + ls->ls_node_array = NULL; + + memset(&ls->ls_stub_rsb, 0, sizeof(struct dlm_rsb)); + ls->ls_stub_rsb.res_ls = ls; + + ls->ls_debug_rsb_dentry = NULL; + ls->ls_debug_waiters_dentry = NULL; + + init_waitqueue_head(&ls->ls_uevent_wait); + ls->ls_uevent_result = 0; + init_completion(&ls->ls_members_done); + ls->ls_members_result = -1; + + mutex_init(&ls->ls_cb_mutex); + INIT_LIST_HEAD(&ls->ls_cb_delay); + + ls->ls_recoverd_task = NULL; + mutex_init(&ls->ls_recoverd_active); + spin_lock_init(&ls->ls_recover_lock); + spin_lock_init(&ls->ls_rcom_spin); + get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t)); + ls->ls_recover_status = 0; + ls->ls_recover_seq = 0; + ls->ls_recover_args = NULL; + init_rwsem(&ls->ls_in_recovery); + init_rwsem(&ls->ls_recv_active); + INIT_LIST_HEAD(&ls->ls_requestqueue); + mutex_init(&ls->ls_requestqueue_mutex); + mutex_init(&ls->ls_clear_proc_locks); + + ls->ls_recover_buf = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS); + if (!ls->ls_recover_buf) + goto out_lkbidr; + + ls->ls_slot = 0; + ls->ls_num_slots = 0; + ls->ls_slots_size = 0; + ls->ls_slots = NULL; + + INIT_LIST_HEAD(&ls->ls_recover_list); + spin_lock_init(&ls->ls_recover_list_lock); + idr_init(&ls->ls_recover_idr); + spin_lock_init(&ls->ls_recover_idr_lock); + ls->ls_recover_list_count = 0; + ls->ls_local_handle = ls; + init_waitqueue_head(&ls->ls_wait_general); + INIT_LIST_HEAD(&ls->ls_root_list); + init_rwsem(&ls->ls_root_sem); + + spin_lock(&lslist_lock); + ls->ls_create_count = 1; + list_add(&ls->ls_list, &lslist); + spin_unlock(&lslist_lock); + + if (flags & DLM_LSFL_FS) { + error = dlm_callback_start(ls); + if (error) { + log_error(ls, "can't start dlm_callback %d", error); + goto out_delist; + } + } + + init_waitqueue_head(&ls->ls_recover_lock_wait); + + /* + * Once started, dlm_recoverd first looks for ls in lslist, then + * initializes ls_in_recovery as locked in "down" mode. We need + * to wait for the wakeup from dlm_recoverd because in_recovery + * has to start out in down mode. + */ + + error = dlm_recoverd_start(ls); + if (error) { + log_error(ls, "can't start dlm_recoverd %d", error); + goto out_callback; + } + + wait_event(ls->ls_recover_lock_wait, + test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); + + ls->ls_kobj.kset = dlm_kset; + error = kobject_init_and_add(&ls->ls_kobj, &dlm_ktype, NULL, + "%s", ls->ls_name); + if (error) + goto out_recoverd; + kobject_uevent(&ls->ls_kobj, KOBJ_ADD); + + /* let kobject handle freeing of ls if there's an error */ + do_unreg = 1; + + /* This uevent triggers dlm_controld in userspace to add us to the + group of nodes that are members of this lockspace (managed by the + cluster infrastructure.) Once it's done that, it tells us who the + current lockspace members are (via configfs) and then tells the + lockspace to start running (via sysfs) in dlm_ls_start(). */ + + error = do_uevent(ls, 1); + if (error) + goto out_recoverd; + + wait_for_completion(&ls->ls_members_done); + error = ls->ls_members_result; + if (error) + goto out_members; + + dlm_create_debug_file(ls); + + log_debug(ls, "join complete"); + *lockspace = ls; + return 0; + + out_members: + do_uevent(ls, 0); + dlm_clear_members(ls); + kfree(ls->ls_node_array); + out_recoverd: + dlm_recoverd_stop(ls); + out_callback: + dlm_callback_stop(ls); + out_delist: + spin_lock(&lslist_lock); + list_del(&ls->ls_list); + spin_unlock(&lslist_lock); + idr_destroy(&ls->ls_recover_idr); + kfree(ls->ls_recover_buf); + out_lkbidr: + idr_destroy(&ls->ls_lkbidr); + for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) { + if (ls->ls_remove_names[i]) + kfree(ls->ls_remove_names[i]); + } + out_rsbtbl: + vfree(ls->ls_rsbtbl); + out_lsfree: + if (do_unreg) + kobject_put(&ls->ls_kobj); + else + kfree(ls); + out: + module_put(THIS_MODULE); + return error; +} + +int dlm_new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, void *ops_arg, + int *ops_result, dlm_lockspace_t **lockspace) +{ + int error = 0; + + mutex_lock(&ls_lock); + if (!ls_count) + error = threads_start(); + if (error) + goto out; + + error = new_lockspace(name, cluster, flags, lvblen, ops, ops_arg, + ops_result, lockspace); + if (!error) + ls_count++; + if (error > 0) + error = 0; + if (!ls_count) + threads_stop(); + out: + mutex_unlock(&ls_lock); + return error; +} + +static int lkb_idr_is_local(int id, void *p, void *data) +{ + struct dlm_lkb *lkb = p; + + if (!lkb->lkb_nodeid) + return 1; + return 0; +} + +static int lkb_idr_is_any(int id, void *p, void *data) +{ + return 1; +} + +static int lkb_idr_free(int id, void *p, void *data) +{ + struct dlm_lkb *lkb = p; + + if (lkb->lkb_lvbptr && lkb->lkb_flags & DLM_IFL_MSTCPY) + dlm_free_lvb(lkb->lkb_lvbptr); + + dlm_free_lkb(lkb); + return 0; +} + +/* NOTE: We check the lkbidr here rather than the resource table. + This is because there may be LKBs queued as ASTs that have been unlinked + from their RSBs and are pending deletion once the AST has been delivered */ + +static int lockspace_busy(struct dlm_ls *ls, int force) +{ + int rv; + + spin_lock(&ls->ls_lkbidr_spin); + if (force == 0) { + rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_any, ls); + } else if (force == 1) { + rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_local, ls); + } else { + rv = 0; + } + spin_unlock(&ls->ls_lkbidr_spin); + return rv; +} + +static int release_lockspace(struct dlm_ls *ls, int force) +{ + struct dlm_rsb *rsb; + struct rb_node *n; + int i, busy, rv; + + busy = lockspace_busy(ls, force); + + spin_lock(&lslist_lock); + if (ls->ls_create_count == 1) { + if (busy) { + rv = -EBUSY; + } else { + /* remove_lockspace takes ls off lslist */ + ls->ls_create_count = 0; + rv = 0; + } + } else if (ls->ls_create_count > 1) { + rv = --ls->ls_create_count; + } else { + rv = -EINVAL; + } + spin_unlock(&lslist_lock); + + if (rv) { + log_debug(ls, "release_lockspace no remove %d", rv); + return rv; + } + + dlm_device_deregister(ls); + + if (force < 3 && dlm_user_daemon_available()) + do_uevent(ls, 0); + + dlm_recoverd_stop(ls); + + dlm_callback_stop(ls); + + remove_lockspace(ls); + + dlm_delete_debug_file(ls); + + kfree(ls->ls_recover_buf); + + /* + * Free all lkb's in idr + */ + + idr_for_each(&ls->ls_lkbidr, lkb_idr_free, ls); + idr_destroy(&ls->ls_lkbidr); + + /* + * Free all rsb's on rsbtbl[] lists + */ + + for (i = 0; i < ls->ls_rsbtbl_size; i++) { + while ((n = rb_first(&ls->ls_rsbtbl[i].keep))) { + rsb = rb_entry(n, struct dlm_rsb, res_hashnode); + rb_erase(n, &ls->ls_rsbtbl[i].keep); + dlm_free_rsb(rsb); + } + + while ((n = rb_first(&ls->ls_rsbtbl[i].toss))) { + rsb = rb_entry(n, struct dlm_rsb, res_hashnode); + rb_erase(n, &ls->ls_rsbtbl[i].toss); + dlm_free_rsb(rsb); + } + } + + vfree(ls->ls_rsbtbl); + + for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) + kfree(ls->ls_remove_names[i]); + + while (!list_empty(&ls->ls_new_rsb)) { + rsb = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, + res_hashchain); + list_del(&rsb->res_hashchain); + dlm_free_rsb(rsb); + } + + /* + * Free structures on any other lists + */ + + dlm_purge_requestqueue(ls); + kfree(ls->ls_recover_args); + dlm_clear_members(ls); + dlm_clear_members_gone(ls); + kfree(ls->ls_node_array); + log_debug(ls, "release_lockspace final free"); + kobject_put(&ls->ls_kobj); + /* The ls structure will be freed when the kobject is done with */ + + module_put(THIS_MODULE); + return 0; +} + +/* + * Called when a system has released all its locks and is not going to use the + * lockspace any longer. We free everything we're managing for this lockspace. + * Remaining nodes will go through the recovery process as if we'd died. The + * lockspace must continue to function as usual, participating in recoveries, + * until this returns. + * + * Force has 4 possible values: + * 0 - don't destroy locksapce if it has any LKBs + * 1 - destroy lockspace if it has remote LKBs but not if it has local LKBs + * 2 - destroy lockspace regardless of LKBs + * 3 - destroy lockspace as part of a forced shutdown + */ + +int dlm_release_lockspace(void *lockspace, int force) +{ + struct dlm_ls *ls; + int error; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + dlm_put_lockspace(ls); + + mutex_lock(&ls_lock); + error = release_lockspace(ls, force); + if (!error) + ls_count--; + if (!ls_count) + threads_stop(); + mutex_unlock(&ls_lock); + + return error; +} + +void dlm_stop_lockspaces(void) +{ + struct dlm_ls *ls; + int count; + + restart: + count = 0; + spin_lock(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (!test_bit(LSFL_RUNNING, &ls->ls_flags)) { + count++; + continue; + } + spin_unlock(&lslist_lock); + log_error(ls, "no userland control daemon, stopping lockspace"); + dlm_ls_stop(ls); + goto restart; + } + spin_unlock(&lslist_lock); + + if (count) + log_print("dlm user daemon left %d lockspaces", count); +} + diff --git a/kmod/dlm/lockspace.h b/kmod/dlm/lockspace.h new file mode 100644 index 00000000..f879f879 --- /dev/null +++ b/kmod/dlm/lockspace.h @@ -0,0 +1,26 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOCKSPACE_DOT_H__ +#define __LOCKSPACE_DOT_H__ + +int dlm_lockspace_init(void); +void dlm_lockspace_exit(void); +struct dlm_ls *dlm_find_lockspace_global(uint32_t id); +struct dlm_ls *dlm_find_lockspace_local(void *id); +struct dlm_ls *dlm_find_lockspace_device(int minor); +void dlm_put_lockspace(struct dlm_ls *ls); +void dlm_stop_lockspaces(void); + +#endif /* __LOCKSPACE_DOT_H__ */ + diff --git a/kmod/dlm/lowcomms.c b/kmod/dlm/lowcomms.c new file mode 100644 index 00000000..d0ccd2fd --- /dev/null +++ b/kmod/dlm/lowcomms.c @@ -0,0 +1,1726 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +/* + * lowcomms.c + * + * This is the "low-level" comms layer. + * + * It is responsible for sending/receiving messages + * from other nodes in the cluster. + * + * Cluster nodes are referred to by their nodeids. nodeids are + * simply 32 bit numbers to the locking module - if they need to + * be expanded for the cluster infrastructure then that is its + * responsibility. It is this layer's + * responsibility to resolve these into IP address or + * whatever it needs for inter-node communication. + * + * The comms level is two kernel threads that deal mainly with + * the receiving of messages from other nodes and passing them + * up to the mid-level comms layer (which understands the + * message format) for execution by the locking core, and + * a send thread which does all the setting up of connections + * to remote nodes and the sending of data. Threads are not allowed + * to send their own data because it may cause them to wait in times + * of high load. Also, this way, the sending thread can collect together + * messages bound for one node and send them in one block. + * + * lowcomms will choose to use either TCP or SCTP as its transport layer + * depending on the configuration variable 'protocol'. This should be set + * to 0 (default) for TCP or 1 for SCTP. It should be configured using a + * cluster-wide mechanism as it must be the same on all nodes of the cluster + * for the DLM to function. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dlm_internal.h" +#include "lowcomms.h" +#include "midcomms.h" +#include "config.h" + +#define NEEDED_RMEM (4*1024*1024) +#define CONN_HASH_SIZE 32 + +/* Number of messages to send before rescheduling */ +#define MAX_SEND_MSG_COUNT 25 + +struct cbuf { + unsigned int base; + unsigned int len; + unsigned int mask; +}; + +static void cbuf_add(struct cbuf *cb, int n) +{ + cb->len += n; +} + +static int cbuf_data(struct cbuf *cb) +{ + return ((cb->base + cb->len) & cb->mask); +} + +static void cbuf_init(struct cbuf *cb, int size) +{ + cb->base = cb->len = 0; + cb->mask = size-1; +} + +static void cbuf_eat(struct cbuf *cb, int n) +{ + cb->len -= n; + cb->base += n; + cb->base &= cb->mask; +} + +static bool cbuf_empty(struct cbuf *cb) +{ + return cb->len == 0; +} + +struct connection { + struct socket *sock; /* NULL if not connected */ + uint32_t nodeid; /* So we know who we are in the list */ + struct mutex sock_mutex; + unsigned long flags; +#define CF_READ_PENDING 1 +#define CF_WRITE_PENDING 2 +#define CF_CONNECT_PENDING 3 +#define CF_INIT_PENDING 4 +#define CF_IS_OTHERCON 5 +#define CF_CLOSE 6 +#define CF_APP_LIMITED 7 + struct list_head writequeue; /* List of outgoing writequeue_entries */ + spinlock_t writequeue_lock; + int (*rx_action) (struct connection *); /* What to do when active */ + void (*connect_action) (struct connection *); /* What to do to connect */ + struct page *rx_page; + struct cbuf cb; + int retries; +#define MAX_CONNECT_RETRIES 3 + int sctp_assoc; + struct hlist_node list; + struct connection *othercon; + struct work_struct rwork; /* Receive workqueue */ + struct work_struct swork; /* Send workqueue */ +}; +#define sock2con(x) ((struct connection *)(x)->sk_user_data) + +/* An entry waiting to be sent */ +struct writequeue_entry { + struct list_head list; + struct page *page; + int offset; + int len; + int end; + int users; + struct connection *con; +}; + +struct dlm_node_addr { + struct list_head list; + int nodeid; + int addr_count; + struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; +}; + +static LIST_HEAD(dlm_node_addrs); +static DEFINE_SPINLOCK(dlm_node_addrs_spin); + +static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT]; +static int dlm_local_count; +static int dlm_allow_conn; + +/* Work queues */ +static struct workqueue_struct *recv_workqueue; +static struct workqueue_struct *send_workqueue; + +static struct hlist_head connection_hash[CONN_HASH_SIZE]; +static DEFINE_MUTEX(connections_lock); +static struct kmem_cache *con_cache; + +static void process_recv_sockets(struct work_struct *work); +static void process_send_sockets(struct work_struct *work); + + +/* This is deliberately very simple because most clusters have simple + sequential nodeids, so we should be able to go straight to a connection + struct in the array */ +static inline int nodeid_hash(int nodeid) +{ + return nodeid & (CONN_HASH_SIZE-1); +} + +static struct connection *__find_con(int nodeid) +{ + int r; + struct connection *con; + + r = nodeid_hash(nodeid); + + hlist_for_each_entry(con, &connection_hash[r], list) { + if (con->nodeid == nodeid) + return con; + } + return NULL; +} + +/* + * If 'allocation' is zero then we don't attempt to create a new + * connection structure for this node. + */ +static struct connection *__nodeid2con(int nodeid, gfp_t alloc) +{ + struct connection *con = NULL; + int r; + + con = __find_con(nodeid); + if (con || !alloc) + return con; + + con = kmem_cache_zalloc(con_cache, alloc); + if (!con) + return NULL; + + r = nodeid_hash(nodeid); + hlist_add_head(&con->list, &connection_hash[r]); + + con->nodeid = nodeid; + mutex_init(&con->sock_mutex); + INIT_LIST_HEAD(&con->writequeue); + spin_lock_init(&con->writequeue_lock); + INIT_WORK(&con->swork, process_send_sockets); + INIT_WORK(&con->rwork, process_recv_sockets); + + /* Setup action pointers for child sockets */ + if (con->nodeid) { + struct connection *zerocon = __find_con(0); + + con->connect_action = zerocon->connect_action; + if (!con->rx_action) + con->rx_action = zerocon->rx_action; + } + + return con; +} + +/* Loop round all connections */ +static void foreach_conn(void (*conn_func)(struct connection *c)) +{ + int i; + struct hlist_node *n; + struct connection *con; + + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_safe(con, n, &connection_hash[i], list) + conn_func(con); + } +} + +static struct connection *nodeid2con(int nodeid, gfp_t allocation) +{ + struct connection *con; + + mutex_lock(&connections_lock); + con = __nodeid2con(nodeid, allocation); + mutex_unlock(&connections_lock); + + return con; +} + +/* This is a bit drastic, but only called when things go wrong */ +static struct connection *assoc2con(int assoc_id) +{ + int i; + struct connection *con; + + mutex_lock(&connections_lock); + + for (i = 0 ; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry(con, &connection_hash[i], list) { + if (con->sctp_assoc == assoc_id) { + mutex_unlock(&connections_lock); + return con; + } + } + } + mutex_unlock(&connections_lock); + return NULL; +} + +static struct dlm_node_addr *find_node_addr(int nodeid) +{ + struct dlm_node_addr *na; + + list_for_each_entry(na, &dlm_node_addrs, list) { + if (na->nodeid == nodeid) + return na; + } + return NULL; +} + +static int addr_compare(struct sockaddr_storage *x, struct sockaddr_storage *y) +{ + switch (x->ss_family) { + case AF_INET: { + struct sockaddr_in *sinx = (struct sockaddr_in *)x; + struct sockaddr_in *siny = (struct sockaddr_in *)y; + if (sinx->sin_addr.s_addr != siny->sin_addr.s_addr) + return 0; + if (sinx->sin_port != siny->sin_port) + return 0; + break; + } + case AF_INET6: { + struct sockaddr_in6 *sinx = (struct sockaddr_in6 *)x; + struct sockaddr_in6 *siny = (struct sockaddr_in6 *)y; + if (!ipv6_addr_equal(&sinx->sin6_addr, &siny->sin6_addr)) + return 0; + if (sinx->sin6_port != siny->sin6_port) + return 0; + break; + } + default: + return 0; + } + return 1; +} + +static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, + struct sockaddr *sa_out) +{ + struct sockaddr_storage sas; + struct dlm_node_addr *na; + + if (!dlm_local_count) + return -1; + + spin_lock(&dlm_node_addrs_spin); + na = find_node_addr(nodeid); + if (na && na->addr_count) + memcpy(&sas, na->addr[0], sizeof(struct sockaddr_storage)); + spin_unlock(&dlm_node_addrs_spin); + + if (!na) + return -EEXIST; + + if (!na->addr_count) + return -ENOENT; + + if (sas_out) + memcpy(sas_out, &sas, sizeof(struct sockaddr_storage)); + + if (!sa_out) + return 0; + + if (dlm_local_addr[0]->ss_family == AF_INET) { + struct sockaddr_in *in4 = (struct sockaddr_in *) &sas; + struct sockaddr_in *ret4 = (struct sockaddr_in *) sa_out; + ret4->sin_addr.s_addr = in4->sin_addr.s_addr; + } else { + struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) &sas; + struct sockaddr_in6 *ret6 = (struct sockaddr_in6 *) sa_out; + ret6->sin6_addr = in6->sin6_addr; + } + + return 0; +} + +static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid) +{ + struct dlm_node_addr *na; + int rv = -EEXIST; + + spin_lock(&dlm_node_addrs_spin); + list_for_each_entry(na, &dlm_node_addrs, list) { + if (!na->addr_count) + continue; + + if (!addr_compare(na->addr[0], addr)) + continue; + + *nodeid = na->nodeid; + rv = 0; + break; + } + spin_unlock(&dlm_node_addrs_spin); + return rv; +} + +int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len) +{ + struct sockaddr_storage *new_addr; + struct dlm_node_addr *new_node, *na; + + new_node = kzalloc(sizeof(struct dlm_node_addr), GFP_NOFS); + if (!new_node) + return -ENOMEM; + + new_addr = kzalloc(sizeof(struct sockaddr_storage), GFP_NOFS); + if (!new_addr) { + kfree(new_node); + return -ENOMEM; + } + + memcpy(new_addr, addr, len); + + spin_lock(&dlm_node_addrs_spin); + na = find_node_addr(nodeid); + if (!na) { + new_node->nodeid = nodeid; + new_node->addr[0] = new_addr; + new_node->addr_count = 1; + list_add(&new_node->list, &dlm_node_addrs); + spin_unlock(&dlm_node_addrs_spin); + return 0; + } + + if (na->addr_count >= DLM_MAX_ADDR_COUNT) { + spin_unlock(&dlm_node_addrs_spin); + kfree(new_addr); + kfree(new_node); + return -ENOSPC; + } + + na->addr[na->addr_count++] = new_addr; + spin_unlock(&dlm_node_addrs_spin); + kfree(new_node); + return 0; +} + +/* Data available on socket or listen socket received a connect */ +static void lowcomms_data_ready(struct sock *sk, int count_unused) +{ + struct connection *con = sock2con(sk); + if (con && !test_and_set_bit(CF_READ_PENDING, &con->flags)) + queue_work(recv_workqueue, &con->rwork); +} + +static void lowcomms_write_space(struct sock *sk) +{ + struct connection *con = sock2con(sk); + + if (!con) + return; + + clear_bit(SOCK_NOSPACE, &con->sock->flags); + + if (test_and_clear_bit(CF_APP_LIMITED, &con->flags)) { + con->sock->sk->sk_write_pending--; + clear_bit(SOCK_ASYNC_NOSPACE, &con->sock->flags); + } + + if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) + queue_work(send_workqueue, &con->swork); +} + +static inline void lowcomms_connect_sock(struct connection *con) +{ + if (test_bit(CF_CLOSE, &con->flags)) + return; + if (!test_and_set_bit(CF_CONNECT_PENDING, &con->flags)) + queue_work(send_workqueue, &con->swork); +} + +static void lowcomms_state_change(struct sock *sk) +{ + if (sk->sk_state == TCP_ESTABLISHED) + lowcomms_write_space(sk); +} + +int dlm_lowcomms_connect_node(int nodeid) +{ + struct connection *con; + + /* with sctp there's no connecting without sending */ + if (dlm_config.ci_protocol != 0) + return 0; + + if (nodeid == dlm_our_nodeid()) + return 0; + + con = nodeid2con(nodeid, GFP_NOFS); + if (!con) + return -ENOMEM; + lowcomms_connect_sock(con); + return 0; +} + +/* Make a socket active */ +static void add_sock(struct socket *sock, struct connection *con) +{ + con->sock = sock; + + /* Install a data_ready callback */ + con->sock->sk->sk_data_ready = lowcomms_data_ready; + con->sock->sk->sk_write_space = lowcomms_write_space; + con->sock->sk->sk_state_change = lowcomms_state_change; + con->sock->sk->sk_user_data = con; + con->sock->sk->sk_allocation = GFP_NOFS; +} + +/* Add the port number to an IPv6 or 4 sockaddr and return the address + length */ +static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port, + int *addr_len) +{ + saddr->ss_family = dlm_local_addr[0]->ss_family; + if (saddr->ss_family == AF_INET) { + struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr; + in4_addr->sin_port = cpu_to_be16(port); + *addr_len = sizeof(struct sockaddr_in); + memset(&in4_addr->sin_zero, 0, sizeof(in4_addr->sin_zero)); + } else { + struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)saddr; + in6_addr->sin6_port = cpu_to_be16(port); + *addr_len = sizeof(struct sockaddr_in6); + } + memset((char *)saddr + *addr_len, 0, sizeof(struct sockaddr_storage) - *addr_len); +} + +/* Close a remote connection and tidy up */ +static void close_connection(struct connection *con, bool and_other) +{ + mutex_lock(&con->sock_mutex); + + if (con->sock) { + sock_release(con->sock); + con->sock = NULL; + } + if (con->othercon && and_other) { + /* Will only re-enter once. */ + close_connection(con->othercon, false); + } + if (con->rx_page) { + __free_page(con->rx_page); + con->rx_page = NULL; + } + + con->retries = 0; + mutex_unlock(&con->sock_mutex); +} + +/* We only send shutdown messages to nodes that are not part of the cluster */ +static void sctp_send_shutdown(sctp_assoc_t associd) +{ + static char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; + struct msghdr outmessage; + struct cmsghdr *cmsg; + struct sctp_sndrcvinfo *sinfo; + int ret; + struct connection *con; + + con = nodeid2con(0,0); + BUG_ON(con == NULL); + + outmessage.msg_name = NULL; + outmessage.msg_namelen = 0; + outmessage.msg_control = outcmsg; + outmessage.msg_controllen = sizeof(outcmsg); + outmessage.msg_flags = MSG_EOR; + + cmsg = CMSG_FIRSTHDR(&outmessage); + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_SNDRCV; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); + outmessage.msg_controllen = cmsg->cmsg_len; + sinfo = CMSG_DATA(cmsg); + memset(sinfo, 0x00, sizeof(struct sctp_sndrcvinfo)); + + sinfo->sinfo_flags |= MSG_EOF; + sinfo->sinfo_assoc_id = associd; + + ret = kernel_sendmsg(con->sock, &outmessage, NULL, 0, 0); + + if (ret != 0) + log_print("send EOF to node failed: %d", ret); +} + +static void sctp_init_failed_foreach(struct connection *con) +{ + con->sctp_assoc = 0; + if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) { + if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) + queue_work(send_workqueue, &con->swork); + } +} + +/* INIT failed but we don't know which node... + restart INIT on all pending nodes */ +static void sctp_init_failed(void) +{ + mutex_lock(&connections_lock); + + foreach_conn(sctp_init_failed_foreach); + + mutex_unlock(&connections_lock); +} + +/* Something happened to an association */ +static void process_sctp_notification(struct connection *con, + struct msghdr *msg, char *buf) +{ + union sctp_notification *sn = (union sctp_notification *)buf; + + if (sn->sn_header.sn_type == SCTP_ASSOC_CHANGE) { + switch (sn->sn_assoc_change.sac_state) { + + case SCTP_COMM_UP: + case SCTP_RESTART: + { + /* Check that the new node is in the lockspace */ + struct sctp_prim prim; + int nodeid; + int prim_len, ret; + int addr_len; + struct connection *new_con; + + /* + * We get this before any data for an association. + * We verify that the node is in the cluster and + * then peel off a socket for it. + */ + if ((int)sn->sn_assoc_change.sac_assoc_id <= 0) { + log_print("COMM_UP for invalid assoc ID %d", + (int)sn->sn_assoc_change.sac_assoc_id); + sctp_init_failed(); + return; + } + memset(&prim, 0, sizeof(struct sctp_prim)); + prim_len = sizeof(struct sctp_prim); + prim.ssp_assoc_id = sn->sn_assoc_change.sac_assoc_id; + + ret = kernel_getsockopt(con->sock, + IPPROTO_SCTP, + SCTP_PRIMARY_ADDR, + (char*)&prim, + &prim_len); + if (ret < 0) { + log_print("getsockopt/sctp_primary_addr on " + "new assoc %d failed : %d", + (int)sn->sn_assoc_change.sac_assoc_id, + ret); + + /* Retry INIT later */ + new_con = assoc2con(sn->sn_assoc_change.sac_assoc_id); + if (new_con) + clear_bit(CF_CONNECT_PENDING, &con->flags); + return; + } + make_sockaddr(&prim.ssp_addr, 0, &addr_len); + if (addr_to_nodeid(&prim.ssp_addr, &nodeid)) { + unsigned char *b=(unsigned char *)&prim.ssp_addr; + log_print("reject connect from unknown addr"); + print_hex_dump_bytes("ss: ", DUMP_PREFIX_NONE, + b, sizeof(struct sockaddr_storage)); + sctp_send_shutdown(prim.ssp_assoc_id); + return; + } + + new_con = nodeid2con(nodeid, GFP_NOFS); + if (!new_con) + return; + + /* Peel off a new sock */ + sctp_lock_sock(con->sock->sk); + ret = sctp_do_peeloff(con->sock->sk, + sn->sn_assoc_change.sac_assoc_id, + &new_con->sock); + sctp_release_sock(con->sock->sk); + if (ret < 0) { + log_print("Can't peel off a socket for " + "connection %d to node %d: err=%d", + (int)sn->sn_assoc_change.sac_assoc_id, + nodeid, ret); + return; + } + add_sock(new_con->sock, new_con); + + log_print("connecting to %d sctp association %d", + nodeid, (int)sn->sn_assoc_change.sac_assoc_id); + + /* Send any pending writes */ + clear_bit(CF_CONNECT_PENDING, &new_con->flags); + clear_bit(CF_INIT_PENDING, &con->flags); + if (!test_and_set_bit(CF_WRITE_PENDING, &new_con->flags)) { + queue_work(send_workqueue, &new_con->swork); + } + if (!test_and_set_bit(CF_READ_PENDING, &new_con->flags)) + queue_work(recv_workqueue, &new_con->rwork); + } + break; + + case SCTP_COMM_LOST: + case SCTP_SHUTDOWN_COMP: + { + con = assoc2con(sn->sn_assoc_change.sac_assoc_id); + if (con) { + con->sctp_assoc = 0; + } + } + break; + + /* We don't know which INIT failed, so clear the PENDING flags + * on them all. if assoc_id is zero then it will then try + * again */ + + case SCTP_CANT_STR_ASSOC: + { + log_print("Can't start SCTP association - retrying"); + sctp_init_failed(); + } + break; + + default: + log_print("unexpected SCTP assoc change id=%d state=%d", + (int)sn->sn_assoc_change.sac_assoc_id, + sn->sn_assoc_change.sac_state); + } + } +} + +/* Data received from remote end */ +static int receive_from_sock(struct connection *con) +{ + int ret = 0; + struct msghdr msg = {}; + struct kvec iov[2]; + unsigned len; + int r; + int call_again_soon = 0; + int nvec; + char incmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; + + mutex_lock(&con->sock_mutex); + + if (con->sock == NULL) { + ret = -EAGAIN; + goto out_close; + } + + if (con->rx_page == NULL) { + /* + * This doesn't need to be atomic, but I think it should + * improve performance if it is. + */ + con->rx_page = alloc_page(GFP_ATOMIC); + if (con->rx_page == NULL) + goto out_resched; + cbuf_init(&con->cb, PAGE_CACHE_SIZE); + } + + /* Only SCTP needs these really */ + memset(&incmsg, 0, sizeof(incmsg)); + msg.msg_control = incmsg; + msg.msg_controllen = sizeof(incmsg); + + /* + * iov[0] is the bit of the circular buffer between the current end + * point (cb.base + cb.len) and the end of the buffer. + */ + iov[0].iov_len = con->cb.base - cbuf_data(&con->cb); + iov[0].iov_base = page_address(con->rx_page) + cbuf_data(&con->cb); + iov[1].iov_len = 0; + nvec = 1; + + /* + * iov[1] is the bit of the circular buffer between the start of the + * buffer and the start of the currently used section (cb.base) + */ + if (cbuf_data(&con->cb) >= con->cb.base) { + iov[0].iov_len = PAGE_CACHE_SIZE - cbuf_data(&con->cb); + iov[1].iov_len = con->cb.base; + iov[1].iov_base = page_address(con->rx_page); + nvec = 2; + } + len = iov[0].iov_len + iov[1].iov_len; + + r = ret = kernel_recvmsg(con->sock, &msg, iov, nvec, len, + MSG_DONTWAIT | MSG_NOSIGNAL); + if (ret <= 0) + goto out_close; + + /* Process SCTP notifications */ + if (msg.msg_flags & MSG_NOTIFICATION) { + msg.msg_control = incmsg; + msg.msg_controllen = sizeof(incmsg); + + process_sctp_notification(con, &msg, + page_address(con->rx_page) + con->cb.base); + mutex_unlock(&con->sock_mutex); + return 0; + } + BUG_ON(con->nodeid == 0); + + if (ret == len) + call_again_soon = 1; + cbuf_add(&con->cb, ret); + ret = dlm_process_incoming_buffer(con->nodeid, + page_address(con->rx_page), + con->cb.base, con->cb.len, + PAGE_CACHE_SIZE); + if (ret == -EBADMSG) { + log_print("lowcomms: addr=%p, base=%u, len=%u, " + "iov_len=%u, iov_base[0]=%p, read=%d", + page_address(con->rx_page), con->cb.base, con->cb.len, + len, iov[0].iov_base, r); + } + if (ret < 0) + goto out_close; + cbuf_eat(&con->cb, ret); + + if (cbuf_empty(&con->cb) && !call_again_soon) { + __free_page(con->rx_page); + con->rx_page = NULL; + } + + if (call_again_soon) + goto out_resched; + mutex_unlock(&con->sock_mutex); + return 0; + +out_resched: + if (!test_and_set_bit(CF_READ_PENDING, &con->flags)) + queue_work(recv_workqueue, &con->rwork); + mutex_unlock(&con->sock_mutex); + return -EAGAIN; + +out_close: + mutex_unlock(&con->sock_mutex); + if (ret != -EAGAIN) { + close_connection(con, false); + /* Reconnect when there is something to send */ + } + /* Don't return success if we really got EOF */ + if (ret == 0) + ret = -EAGAIN; + + return ret; +} + +/* Listening socket is busy, accept a connection */ +static int tcp_accept_from_sock(struct connection *con) +{ + int result; + struct sockaddr_storage peeraddr; + struct socket *newsock; + int len; + int nodeid; + struct connection *newcon; + struct connection *addcon; + + mutex_lock(&connections_lock); + if (!dlm_allow_conn) { + mutex_unlock(&connections_lock); + return -1; + } + mutex_unlock(&connections_lock); + + memset(&peeraddr, 0, sizeof(peeraddr)); + result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, + IPPROTO_TCP, &newsock); + if (result < 0) + return -ENOMEM; + + mutex_lock_nested(&con->sock_mutex, 0); + + result = -ENOTCONN; + if (con->sock == NULL) + goto accept_err; + + newsock->type = con->sock->type; + newsock->ops = con->sock->ops; + + result = con->sock->ops->accept(con->sock, newsock, O_NONBLOCK); + if (result < 0) + goto accept_err; + + /* Get the connected socket's peer */ + memset(&peeraddr, 0, sizeof(peeraddr)); + if (newsock->ops->getname(newsock, (struct sockaddr *)&peeraddr, + &len, 2)) { + result = -ECONNABORTED; + goto accept_err; + } + + /* Get the new node's NODEID */ + make_sockaddr(&peeraddr, 0, &len); + if (addr_to_nodeid(&peeraddr, &nodeid)) { + unsigned char *b=(unsigned char *)&peeraddr; + log_print("connect from non cluster node"); + print_hex_dump_bytes("ss: ", DUMP_PREFIX_NONE, + b, sizeof(struct sockaddr_storage)); + sock_release(newsock); + mutex_unlock(&con->sock_mutex); + return -1; + } + + log_print("got connection from %d", nodeid); + + /* Check to see if we already have a connection to this node. This + * could happen if the two nodes initiate a connection at roughly + * the same time and the connections cross on the wire. + * In this case we store the incoming one in "othercon" + */ + newcon = nodeid2con(nodeid, GFP_NOFS); + if (!newcon) { + result = -ENOMEM; + goto accept_err; + } + mutex_lock_nested(&newcon->sock_mutex, 1); + if (newcon->sock) { + struct connection *othercon = newcon->othercon; + + if (!othercon) { + othercon = kmem_cache_zalloc(con_cache, GFP_NOFS); + if (!othercon) { + log_print("failed to allocate incoming socket"); + mutex_unlock(&newcon->sock_mutex); + result = -ENOMEM; + goto accept_err; + } + othercon->nodeid = nodeid; + othercon->rx_action = receive_from_sock; + mutex_init(&othercon->sock_mutex); + INIT_WORK(&othercon->swork, process_send_sockets); + INIT_WORK(&othercon->rwork, process_recv_sockets); + set_bit(CF_IS_OTHERCON, &othercon->flags); + } + if (!othercon->sock) { + newcon->othercon = othercon; + othercon->sock = newsock; + newsock->sk->sk_user_data = othercon; + add_sock(newsock, othercon); + addcon = othercon; + } + else { + printk("Extra connection from node %d attempted\n", nodeid); + result = -EAGAIN; + mutex_unlock(&newcon->sock_mutex); + goto accept_err; + } + } + else { + newsock->sk->sk_user_data = newcon; + newcon->rx_action = receive_from_sock; + add_sock(newsock, newcon); + addcon = newcon; + } + + mutex_unlock(&newcon->sock_mutex); + + /* + * Add it to the active queue in case we got data + * between processing the accept adding the socket + * to the read_sockets list + */ + if (!test_and_set_bit(CF_READ_PENDING, &addcon->flags)) + queue_work(recv_workqueue, &addcon->rwork); + mutex_unlock(&con->sock_mutex); + + return 0; + +accept_err: + mutex_unlock(&con->sock_mutex); + sock_release(newsock); + + if (result != -EAGAIN) + log_print("error accepting connection from node: %d", result); + return result; +} + +static void free_entry(struct writequeue_entry *e) +{ + __free_page(e->page); + kfree(e); +} + +/* Initiate an SCTP association. + This is a special case of send_to_sock() in that we don't yet have a + peeled-off socket for this association, so we use the listening socket + and add the primary IP address of the remote node. + */ +static void sctp_init_assoc(struct connection *con) +{ + struct sockaddr_storage rem_addr; + char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; + struct msghdr outmessage; + struct cmsghdr *cmsg; + struct sctp_sndrcvinfo *sinfo; + struct connection *base_con; + struct writequeue_entry *e; + int len, offset; + int ret; + int addrlen; + struct kvec iov[1]; + + if (test_and_set_bit(CF_INIT_PENDING, &con->flags)) + return; + + if (con->retries++ > MAX_CONNECT_RETRIES) + return; + + if (nodeid_to_addr(con->nodeid, NULL, (struct sockaddr *)&rem_addr)) { + log_print("no address for nodeid %d", con->nodeid); + return; + } + base_con = nodeid2con(0, 0); + BUG_ON(base_con == NULL); + + make_sockaddr(&rem_addr, dlm_config.ci_tcp_port, &addrlen); + + outmessage.msg_name = &rem_addr; + outmessage.msg_namelen = addrlen; + outmessage.msg_control = outcmsg; + outmessage.msg_controllen = sizeof(outcmsg); + outmessage.msg_flags = MSG_EOR; + + spin_lock(&con->writequeue_lock); + + if (list_empty(&con->writequeue)) { + spin_unlock(&con->writequeue_lock); + log_print("writequeue empty for nodeid %d", con->nodeid); + return; + } + + e = list_first_entry(&con->writequeue, struct writequeue_entry, list); + len = e->len; + offset = e->offset; + spin_unlock(&con->writequeue_lock); + + /* Send the first block off the write queue */ + iov[0].iov_base = page_address(e->page)+offset; + iov[0].iov_len = len; + + cmsg = CMSG_FIRSTHDR(&outmessage); + cmsg->cmsg_level = IPPROTO_SCTP; + cmsg->cmsg_type = SCTP_SNDRCV; + cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); + sinfo = CMSG_DATA(cmsg); + memset(sinfo, 0x00, sizeof(struct sctp_sndrcvinfo)); + sinfo->sinfo_ppid = cpu_to_le32(dlm_our_nodeid()); + outmessage.msg_controllen = cmsg->cmsg_len; + + ret = kernel_sendmsg(base_con->sock, &outmessage, iov, 1, len); + if (ret < 0) { + log_print("Send first packet to node %d failed: %d", + con->nodeid, ret); + + /* Try again later */ + clear_bit(CF_CONNECT_PENDING, &con->flags); + clear_bit(CF_INIT_PENDING, &con->flags); + } + else { + spin_lock(&con->writequeue_lock); + e->offset += ret; + e->len -= ret; + + if (e->len == 0 && e->users == 0) { + list_del(&e->list); + free_entry(e); + } + spin_unlock(&con->writequeue_lock); + } +} + +/* Connect a new socket to its peer */ +static void tcp_connect_to_sock(struct connection *con) +{ + struct sockaddr_storage saddr, src_addr; + int addr_len; + struct socket *sock = NULL; + int one = 1; + int result; + + if (con->nodeid == 0) { + log_print("attempt to connect sock 0 foiled"); + return; + } + + mutex_lock(&con->sock_mutex); + if (con->retries++ > MAX_CONNECT_RETRIES) + goto out; + + /* Some odd races can cause double-connects, ignore them */ + if (con->sock) + goto out; + + /* Create a socket to communicate with */ + result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, + IPPROTO_TCP, &sock); + if (result < 0) + goto out_err; + + memset(&saddr, 0, sizeof(saddr)); + result = nodeid_to_addr(con->nodeid, &saddr, NULL); + if (result < 0) { + log_print("no address for nodeid %d", con->nodeid); + goto out_err; + } + + sock->sk->sk_user_data = con; + con->rx_action = receive_from_sock; + con->connect_action = tcp_connect_to_sock; + add_sock(sock, con); + + /* Bind to our cluster-known address connecting to avoid + routing problems */ + memcpy(&src_addr, dlm_local_addr[0], sizeof(src_addr)); + make_sockaddr(&src_addr, 0, &addr_len); + result = sock->ops->bind(sock, (struct sockaddr *) &src_addr, + addr_len); + if (result < 0) { + log_print("could not bind for connect: %d", result); + /* This *may* not indicate a critical error */ + } + + make_sockaddr(&saddr, dlm_config.ci_tcp_port, &addr_len); + + log_print("connecting to %d", con->nodeid); + + /* Turn off Nagle's algorithm */ + kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&one, + sizeof(one)); + + result = sock->ops->connect(sock, (struct sockaddr *)&saddr, addr_len, + O_NONBLOCK); + if (result == -EINPROGRESS) + result = 0; + if (result == 0) + goto out; + +out_err: + if (con->sock) { + sock_release(con->sock); + con->sock = NULL; + } else if (sock) { + sock_release(sock); + } + /* + * Some errors are fatal and this list might need adjusting. For other + * errors we try again until the max number of retries is reached. + */ + if (result != -EHOSTUNREACH && + result != -ENETUNREACH && + result != -ENETDOWN && + result != -EINVAL && + result != -EPROTONOSUPPORT) { + log_print("connect %d try %d error %d", con->nodeid, + con->retries, result); + mutex_unlock(&con->sock_mutex); + msleep(1000); + lowcomms_connect_sock(con); + return; + } +out: + mutex_unlock(&con->sock_mutex); + return; +} + +static struct socket *tcp_create_listen_sock(struct connection *con, + struct sockaddr_storage *saddr) +{ + struct socket *sock = NULL; + int result = 0; + int one = 1; + int addr_len; + + if (dlm_local_addr[0]->ss_family == AF_INET) + addr_len = sizeof(struct sockaddr_in); + else + addr_len = sizeof(struct sockaddr_in6); + + /* Create a socket to communicate with */ + result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, + IPPROTO_TCP, &sock); + if (result < 0) { + log_print("Can't create listening comms socket"); + goto create_out; + } + + /* Turn off Nagle's algorithm */ + kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&one, + sizeof(one)); + + result = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&one, sizeof(one)); + + if (result < 0) { + log_print("Failed to set SO_REUSEADDR on socket: %d", result); + } + con->rx_action = tcp_accept_from_sock; + con->connect_action = tcp_connect_to_sock; + + /* Bind to our port */ + make_sockaddr(saddr, dlm_config.ci_tcp_port, &addr_len); + result = sock->ops->bind(sock, (struct sockaddr *) saddr, addr_len); + if (result < 0) { + log_print("Can't bind to port %d", dlm_config.ci_tcp_port); + sock_release(sock); + sock = NULL; + con->sock = NULL; + goto create_out; + } + result = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, + (char *)&one, sizeof(one)); + if (result < 0) { + log_print("Set keepalive failed: %d", result); + } + + result = sock->ops->listen(sock, 5); + if (result < 0) { + log_print("Can't listen on port %d", dlm_config.ci_tcp_port); + sock_release(sock); + sock = NULL; + goto create_out; + } + +create_out: + return sock; +} + +/* Get local addresses */ +static void init_local(void) +{ + struct sockaddr_storage sas, *addr; + int i; + + dlm_local_count = 0; + for (i = 0; i < DLM_MAX_ADDR_COUNT; i++) { + if (dlm_our_addr(&sas, i)) + break; + + addr = kmalloc(sizeof(*addr), GFP_NOFS); + if (!addr) + break; + memcpy(addr, &sas, sizeof(*addr)); + dlm_local_addr[dlm_local_count++] = addr; + } +} + +/* Bind to an IP address. SCTP allows multiple address so it can do + multi-homing */ +static int add_sctp_bind_addr(struct connection *sctp_con, + struct sockaddr_storage *addr, + int addr_len, int num) +{ + int result = 0; + + if (num == 1) + result = kernel_bind(sctp_con->sock, + (struct sockaddr *) addr, + addr_len); + else + result = kernel_setsockopt(sctp_con->sock, SOL_SCTP, + SCTP_SOCKOPT_BINDX_ADD, + (char *)addr, addr_len); + + if (result < 0) + log_print("Can't bind to port %d addr number %d", + dlm_config.ci_tcp_port, num); + + return result; +} + +/* Initialise SCTP socket and bind to all interfaces */ +static int sctp_listen_for_all(void) +{ + struct socket *sock = NULL; + struct sockaddr_storage localaddr; + struct sctp_event_subscribe subscribe; + int result = -EINVAL, num = 1, i, addr_len; + struct connection *con = nodeid2con(0, GFP_NOFS); + int bufsize = NEEDED_RMEM; + + if (!con) + return -ENOMEM; + + log_print("Using SCTP for communications"); + + result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_SEQPACKET, + IPPROTO_SCTP, &sock); + if (result < 0) { + log_print("Can't create comms socket, check SCTP is loaded"); + goto out; + } + + /* Listen for events */ + memset(&subscribe, 0, sizeof(subscribe)); + subscribe.sctp_data_io_event = 1; + subscribe.sctp_association_event = 1; + subscribe.sctp_send_failure_event = 1; + subscribe.sctp_shutdown_event = 1; + subscribe.sctp_partial_delivery_event = 1; + + result = kernel_setsockopt(sock, SOL_SOCKET, SO_RCVBUFFORCE, + (char *)&bufsize, sizeof(bufsize)); + if (result) + log_print("Error increasing buffer space on socket %d", result); + + result = kernel_setsockopt(sock, SOL_SCTP, SCTP_EVENTS, + (char *)&subscribe, sizeof(subscribe)); + if (result < 0) { + log_print("Failed to set SCTP_EVENTS on socket: result=%d", + result); + goto create_delsock; + } + + /* Init con struct */ + sock->sk->sk_user_data = con; + con->sock = sock; + con->sock->sk->sk_data_ready = lowcomms_data_ready; + con->rx_action = receive_from_sock; + con->connect_action = sctp_init_assoc; + + /* Bind to all interfaces. */ + for (i = 0; i < dlm_local_count; i++) { + memcpy(&localaddr, dlm_local_addr[i], sizeof(localaddr)); + make_sockaddr(&localaddr, dlm_config.ci_tcp_port, &addr_len); + + result = add_sctp_bind_addr(con, &localaddr, addr_len, num); + if (result) + goto create_delsock; + ++num; + } + + result = sock->ops->listen(sock, 5); + if (result < 0) { + log_print("Can't set socket listening"); + goto create_delsock; + } + + return 0; + +create_delsock: + sock_release(sock); + con->sock = NULL; +out: + return result; +} + +static int tcp_listen_for_all(void) +{ + struct socket *sock = NULL; + struct connection *con = nodeid2con(0, GFP_NOFS); + int result = -EINVAL; + + if (!con) + return -ENOMEM; + + /* We don't support multi-homed hosts */ + if (dlm_local_addr[1] != NULL) { + log_print("TCP protocol can't handle multi-homed hosts, " + "try SCTP"); + return -EINVAL; + } + + log_print("Using TCP for communications"); + + sock = tcp_create_listen_sock(con, dlm_local_addr[0]); + if (sock) { + add_sock(sock, con); + result = 0; + } + else { + result = -EADDRINUSE; + } + + return result; +} + + + +static struct writequeue_entry *new_writequeue_entry(struct connection *con, + gfp_t allocation) +{ + struct writequeue_entry *entry; + + entry = kmalloc(sizeof(struct writequeue_entry), allocation); + if (!entry) + return NULL; + + entry->page = alloc_page(allocation); + if (!entry->page) { + kfree(entry); + return NULL; + } + + entry->offset = 0; + entry->len = 0; + entry->end = 0; + entry->users = 0; + entry->con = con; + + return entry; +} + +void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc) +{ + struct connection *con; + struct writequeue_entry *e; + int offset = 0; + + con = nodeid2con(nodeid, allocation); + if (!con) + return NULL; + + spin_lock(&con->writequeue_lock); + e = list_entry(con->writequeue.prev, struct writequeue_entry, list); + if ((&e->list == &con->writequeue) || + (PAGE_CACHE_SIZE - e->end < len)) { + e = NULL; + } else { + offset = e->end; + e->end += len; + e->users++; + } + spin_unlock(&con->writequeue_lock); + + if (e) { + got_one: + *ppc = page_address(e->page) + offset; + return e; + } + + e = new_writequeue_entry(con, allocation); + if (e) { + spin_lock(&con->writequeue_lock); + offset = e->end; + e->end += len; + e->users++; + list_add_tail(&e->list, &con->writequeue); + spin_unlock(&con->writequeue_lock); + goto got_one; + } + return NULL; +} + +void dlm_lowcomms_commit_buffer(void *mh) +{ + struct writequeue_entry *e = (struct writequeue_entry *)mh; + struct connection *con = e->con; + int users; + + spin_lock(&con->writequeue_lock); + users = --e->users; + if (users) + goto out; + e->len = e->end - e->offset; + spin_unlock(&con->writequeue_lock); + + if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) { + queue_work(send_workqueue, &con->swork); + } + return; + +out: + spin_unlock(&con->writequeue_lock); + return; +} + +/* Send a message */ +static void send_to_sock(struct connection *con) +{ + int ret = 0; + const int msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; + struct writequeue_entry *e; + int len, offset; + int count = 0; + + mutex_lock(&con->sock_mutex); + if (con->sock == NULL) + goto out_connect; + + spin_lock(&con->writequeue_lock); + for (;;) { + e = list_entry(con->writequeue.next, struct writequeue_entry, + list); + if ((struct list_head *) e == &con->writequeue) + break; + + len = e->len; + offset = e->offset; + BUG_ON(len == 0 && e->users == 0); + spin_unlock(&con->writequeue_lock); + + ret = 0; + if (len) { + ret = kernel_sendpage(con->sock, e->page, offset, len, + msg_flags); + if (ret == -EAGAIN || ret == 0) { + if (ret == -EAGAIN && + test_bit(SOCK_ASYNC_NOSPACE, &con->sock->flags) && + !test_and_set_bit(CF_APP_LIMITED, &con->flags)) { + /* Notify TCP that we're limited by the + * application window size. + */ + set_bit(SOCK_NOSPACE, &con->sock->flags); + con->sock->sk->sk_write_pending++; + } + cond_resched(); + goto out; + } else if (ret < 0) + goto send_error; + } + + /* Don't starve people filling buffers */ + if (++count >= MAX_SEND_MSG_COUNT) { + cond_resched(); + count = 0; + } + + spin_lock(&con->writequeue_lock); + e->offset += ret; + e->len -= ret; + + if (e->len == 0 && e->users == 0) { + list_del(&e->list); + free_entry(e); + } + } + spin_unlock(&con->writequeue_lock); +out: + mutex_unlock(&con->sock_mutex); + return; + +send_error: + mutex_unlock(&con->sock_mutex); + close_connection(con, false); + lowcomms_connect_sock(con); + return; + +out_connect: + mutex_unlock(&con->sock_mutex); + if (!test_bit(CF_INIT_PENDING, &con->flags)) + lowcomms_connect_sock(con); +} + +static void clean_one_writequeue(struct connection *con) +{ + struct writequeue_entry *e, *safe; + + spin_lock(&con->writequeue_lock); + list_for_each_entry_safe(e, safe, &con->writequeue, list) { + list_del(&e->list); + free_entry(e); + } + spin_unlock(&con->writequeue_lock); +} + +/* Called from recovery when it knows that a node has + left the cluster */ +int dlm_lowcomms_close(int nodeid) +{ + struct connection *con; + struct dlm_node_addr *na; + + log_print("closing connection to node %d", nodeid); + con = nodeid2con(nodeid, 0); + if (con) { + clear_bit(CF_CONNECT_PENDING, &con->flags); + clear_bit(CF_WRITE_PENDING, &con->flags); + set_bit(CF_CLOSE, &con->flags); + if (cancel_work_sync(&con->swork)) + log_print("canceled swork for node %d", nodeid); + if (cancel_work_sync(&con->rwork)) + log_print("canceled rwork for node %d", nodeid); + clean_one_writequeue(con); + close_connection(con, true); + } + + spin_lock(&dlm_node_addrs_spin); + na = find_node_addr(nodeid); + if (na) { + list_del(&na->list); + while (na->addr_count--) + kfree(na->addr[na->addr_count]); + kfree(na); + } + spin_unlock(&dlm_node_addrs_spin); + + return 0; +} + +/* Receive workqueue function */ +static void process_recv_sockets(struct work_struct *work) +{ + struct connection *con = container_of(work, struct connection, rwork); + int err; + + clear_bit(CF_READ_PENDING, &con->flags); + do { + err = con->rx_action(con); + } while (!err); +} + +/* Send workqueue function */ +static void process_send_sockets(struct work_struct *work) +{ + struct connection *con = container_of(work, struct connection, swork); + + if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) { + con->connect_action(con); + set_bit(CF_WRITE_PENDING, &con->flags); + } + if (test_and_clear_bit(CF_WRITE_PENDING, &con->flags)) + send_to_sock(con); +} + + +/* Discard all entries on the write queues */ +static void clean_writequeues(void) +{ + foreach_conn(clean_one_writequeue); +} + +static void work_stop(void) +{ + destroy_workqueue(recv_workqueue); + destroy_workqueue(send_workqueue); +} + +static int work_start(void) +{ + recv_workqueue = alloc_workqueue("dlm_recv", + WQ_UNBOUND | WQ_MEM_RECLAIM, 1); + if (!recv_workqueue) { + log_print("can't start dlm_recv"); + return -ENOMEM; + } + + send_workqueue = alloc_workqueue("dlm_send", + WQ_UNBOUND | WQ_MEM_RECLAIM, 1); + if (!send_workqueue) { + log_print("can't start dlm_send"); + destroy_workqueue(recv_workqueue); + return -ENOMEM; + } + + return 0; +} + +static void stop_conn(struct connection *con) +{ + con->flags |= 0x0F; + if (con->sock && con->sock->sk) + con->sock->sk->sk_user_data = NULL; +} + +static void free_conn(struct connection *con) +{ + close_connection(con, true); + if (con->othercon) + kmem_cache_free(con_cache, con->othercon); + hlist_del(&con->list); + kmem_cache_free(con_cache, con); +} + +void dlm_lowcomms_stop(void) +{ + /* Set all the flags to prevent any + socket activity. + */ + mutex_lock(&connections_lock); + dlm_allow_conn = 0; + foreach_conn(stop_conn); + mutex_unlock(&connections_lock); + + work_stop(); + + mutex_lock(&connections_lock); + clean_writequeues(); + + foreach_conn(free_conn); + + mutex_unlock(&connections_lock); + kmem_cache_destroy(con_cache); +} + +int dlm_lowcomms_start(void) +{ + int error = -EINVAL; + struct connection *con; + int i; + + for (i = 0; i < CONN_HASH_SIZE; i++) + INIT_HLIST_HEAD(&connection_hash[i]); + + init_local(); + if (!dlm_local_count) { + error = -ENOTCONN; + log_print("no local IP address has been set"); + goto fail; + } + + error = -ENOMEM; + con_cache = kmem_cache_create("dlm_conn", sizeof(struct connection), + __alignof__(struct connection), 0, + NULL); + if (!con_cache) + goto fail; + + error = work_start(); + if (error) + goto fail_destroy; + + dlm_allow_conn = 1; + + /* Start listening */ + if (dlm_config.ci_protocol == 0) + error = tcp_listen_for_all(); + else + error = sctp_listen_for_all(); + if (error) + goto fail_unlisten; + + return 0; + +fail_unlisten: + dlm_allow_conn = 0; + con = nodeid2con(0,0); + if (con) { + close_connection(con, false); + kmem_cache_free(con_cache, con); + } +fail_destroy: + kmem_cache_destroy(con_cache); +fail: + return error; +} + +void dlm_lowcomms_exit(void) +{ + struct dlm_node_addr *na, *safe; + + spin_lock(&dlm_node_addrs_spin); + list_for_each_entry_safe(na, safe, &dlm_node_addrs, list) { + list_del(&na->list); + while (na->addr_count--) + kfree(na->addr[na->addr_count]); + kfree(na); + } + spin_unlock(&dlm_node_addrs_spin); +} diff --git a/kmod/dlm/lowcomms.h b/kmod/dlm/lowcomms.h new file mode 100644 index 00000000..67462e54 --- /dev/null +++ b/kmod/dlm/lowcomms.h @@ -0,0 +1,27 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOWCOMMS_DOT_H__ +#define __LOWCOMMS_DOT_H__ + +int dlm_lowcomms_start(void); +void dlm_lowcomms_stop(void); +void dlm_lowcomms_exit(void); +int dlm_lowcomms_close(int nodeid); +void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc); +void dlm_lowcomms_commit_buffer(void *mh); +int dlm_lowcomms_connect_node(int nodeid); +int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len); + +#endif /* __LOWCOMMS_DOT_H__ */ + diff --git a/kmod/dlm/lvb_table.h b/kmod/dlm/lvb_table.h new file mode 100644 index 00000000..cc3e92f3 --- /dev/null +++ b/kmod/dlm/lvb_table.h @@ -0,0 +1,18 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LVB_TABLE_DOT_H__ +#define __LVB_TABLE_DOT_H__ + +extern const int dlm_lvb_operations[8][8]; + +#endif diff --git a/kmod/dlm/main.c b/kmod/dlm/main.c new file mode 100644 index 00000000..079c0bd7 --- /dev/null +++ b/kmod/dlm/main.c @@ -0,0 +1,97 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "lock.h" +#include "user.h" +#include "memory.h" +#include "config.h" +#include "lowcomms.h" + +static int __init init_dlm(void) +{ + int error; + + error = dlm_memory_init(); + if (error) + goto out; + + error = dlm_lockspace_init(); + if (error) + goto out_mem; + + error = dlm_config_init(); + if (error) + goto out_lockspace; + + error = dlm_register_debugfs(); + if (error) + goto out_config; + + error = dlm_user_init(); + if (error) + goto out_debug; + + error = dlm_netlink_init(); + if (error) + goto out_user; + + error = dlm_plock_init(); + if (error) + goto out_netlink; + + printk("DLM installed\n"); + + return 0; + + out_netlink: + dlm_netlink_exit(); + out_user: + dlm_user_exit(); + out_debug: + dlm_unregister_debugfs(); + out_config: + dlm_config_exit(); + out_lockspace: + dlm_lockspace_exit(); + out_mem: + dlm_memory_exit(); + out: + return error; +} + +static void __exit exit_dlm(void) +{ + dlm_plock_exit(); + dlm_netlink_exit(); + dlm_user_exit(); + dlm_config_exit(); + dlm_memory_exit(); + dlm_lockspace_exit(); + dlm_lowcomms_exit(); + dlm_unregister_debugfs(); +} + +module_init(init_dlm); +module_exit(exit_dlm); + +MODULE_DESCRIPTION("Distributed Lock Manager"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +EXPORT_SYMBOL_GPL(dlm_new_lockspace); +EXPORT_SYMBOL_GPL(dlm_release_lockspace); +EXPORT_SYMBOL_GPL(dlm_lock); +EXPORT_SYMBOL_GPL(dlm_unlock); + diff --git a/kmod/dlm/member.c b/kmod/dlm/member.c new file mode 100644 index 00000000..476557b5 --- /dev/null +++ b/kmod/dlm/member.c @@ -0,0 +1,725 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "recoverd.h" +#include "recover.h" +#include "rcom.h" +#include "config.h" +#include "lowcomms.h" + +int dlm_slots_version(struct dlm_header *h) +{ + if ((h->h_version & 0x0000FFFF) < DLM_HEADER_SLOTS) + return 0; + return 1; +} + +void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, + struct dlm_member *memb) +{ + struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; + + if (!dlm_slots_version(&rc->rc_header)) + return; + + memb->slot = le16_to_cpu(rf->rf_our_slot); + memb->generation = le32_to_cpu(rf->rf_generation); +} + +void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc) +{ + struct dlm_slot *slot; + struct rcom_slot *ro; + int i; + + ro = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); + + /* ls_slots array is sparse, but not rcom_slots */ + + for (i = 0; i < ls->ls_slots_size; i++) { + slot = &ls->ls_slots[i]; + if (!slot->nodeid) + continue; + ro->ro_nodeid = cpu_to_le32(slot->nodeid); + ro->ro_slot = cpu_to_le16(slot->slot); + ro++; + } +} + +#define SLOT_DEBUG_LINE 128 + +static void log_debug_slots(struct dlm_ls *ls, uint32_t gen, int num_slots, + struct rcom_slot *ro0, struct dlm_slot *array, + int array_size) +{ + char line[SLOT_DEBUG_LINE]; + int len = SLOT_DEBUG_LINE - 1; + int pos = 0; + int ret, i; + + if (!dlm_config.ci_log_debug) + return; + + memset(line, 0, sizeof(line)); + + if (array) { + for (i = 0; i < array_size; i++) { + if (!array[i].nodeid) + continue; + + ret = snprintf(line + pos, len - pos, " %d:%d", + array[i].slot, array[i].nodeid); + if (ret >= len - pos) + break; + pos += ret; + } + } else if (ro0) { + for (i = 0; i < num_slots; i++) { + ret = snprintf(line + pos, len - pos, " %d:%d", + ro0[i].ro_slot, ro0[i].ro_nodeid); + if (ret >= len - pos) + break; + pos += ret; + } + } + + log_debug(ls, "generation %u slots %d%s", gen, num_slots, line); +} + +int dlm_slots_copy_in(struct dlm_ls *ls) +{ + struct dlm_member *memb; + struct dlm_rcom *rc = ls->ls_recover_buf; + struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; + struct rcom_slot *ro0, *ro; + int our_nodeid = dlm_our_nodeid(); + int i, num_slots; + uint32_t gen; + + if (!dlm_slots_version(&rc->rc_header)) + return -1; + + gen = le32_to_cpu(rf->rf_generation); + if (gen <= ls->ls_generation) { + log_error(ls, "dlm_slots_copy_in gen %u old %u", + gen, ls->ls_generation); + } + ls->ls_generation = gen; + + num_slots = le16_to_cpu(rf->rf_num_slots); + if (!num_slots) + return -1; + + ro0 = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); + + for (i = 0, ro = ro0; i < num_slots; i++, ro++) { + ro->ro_nodeid = le32_to_cpu(ro->ro_nodeid); + ro->ro_slot = le16_to_cpu(ro->ro_slot); + } + + log_debug_slots(ls, gen, num_slots, ro0, NULL, 0); + + list_for_each_entry(memb, &ls->ls_nodes, list) { + for (i = 0, ro = ro0; i < num_slots; i++, ro++) { + if (ro->ro_nodeid != memb->nodeid) + continue; + memb->slot = ro->ro_slot; + memb->slot_prev = memb->slot; + break; + } + + if (memb->nodeid == our_nodeid) { + if (ls->ls_slot && ls->ls_slot != memb->slot) { + log_error(ls, "dlm_slots_copy_in our slot " + "changed %d %d", ls->ls_slot, + memb->slot); + return -1; + } + + if (!ls->ls_slot) + ls->ls_slot = memb->slot; + } + + if (!memb->slot) { + log_error(ls, "dlm_slots_copy_in nodeid %d no slot", + memb->nodeid); + return -1; + } + } + + return 0; +} + +/* for any nodes that do not support slots, we will not have set memb->slot + in wait_status_all(), so memb->slot will remain -1, and we will not + assign slots or set ls_num_slots here */ + +int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, + struct dlm_slot **slots_out, uint32_t *gen_out) +{ + struct dlm_member *memb; + struct dlm_slot *array; + int our_nodeid = dlm_our_nodeid(); + int array_size, max_slots, i; + int need = 0; + int max = 0; + int num = 0; + uint32_t gen = 0; + + /* our own memb struct will have slot -1 gen 0 */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->nodeid == our_nodeid) { + memb->slot = ls->ls_slot; + memb->generation = ls->ls_generation; + break; + } + } + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->generation > gen) + gen = memb->generation; + + /* node doesn't support slots */ + + if (memb->slot == -1) + return -1; + + /* node needs a slot assigned */ + + if (!memb->slot) + need++; + + /* node has a slot assigned */ + + num++; + + if (!max || max < memb->slot) + max = memb->slot; + + /* sanity check, once slot is assigned it shouldn't change */ + + if (memb->slot_prev && memb->slot && memb->slot_prev != memb->slot) { + log_error(ls, "nodeid %d slot changed %d %d", + memb->nodeid, memb->slot_prev, memb->slot); + return -1; + } + memb->slot_prev = memb->slot; + } + + array_size = max + need; + + array = kzalloc(array_size * sizeof(struct dlm_slot), GFP_NOFS); + if (!array) + return -ENOMEM; + + num = 0; + + /* fill in slots (offsets) that are used */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (!memb->slot) + continue; + + if (memb->slot > array_size) { + log_error(ls, "invalid slot number %d", memb->slot); + kfree(array); + return -1; + } + + array[memb->slot - 1].nodeid = memb->nodeid; + array[memb->slot - 1].slot = memb->slot; + num++; + } + + /* assign new slots from unused offsets */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->slot) + continue; + + for (i = 0; i < array_size; i++) { + if (array[i].nodeid) + continue; + + memb->slot = i + 1; + memb->slot_prev = memb->slot; + array[i].nodeid = memb->nodeid; + array[i].slot = memb->slot; + num++; + + if (!ls->ls_slot && memb->nodeid == our_nodeid) + ls->ls_slot = memb->slot; + break; + } + + if (!memb->slot) { + log_error(ls, "no free slot found"); + kfree(array); + return -1; + } + } + + gen++; + + log_debug_slots(ls, gen, num, NULL, array, array_size); + + max_slots = (dlm_config.ci_buffer_size - sizeof(struct dlm_rcom) - + sizeof(struct rcom_config)) / sizeof(struct rcom_slot); + + if (num > max_slots) { + log_error(ls, "num_slots %d exceeds max_slots %d", + num, max_slots); + kfree(array); + return -1; + } + + *gen_out = gen; + *slots_out = array; + *slots_size = array_size; + *num_slots = num; + return 0; +} + +static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new) +{ + struct dlm_member *memb = NULL; + struct list_head *tmp; + struct list_head *newlist = &new->list; + struct list_head *head = &ls->ls_nodes; + + list_for_each(tmp, head) { + memb = list_entry(tmp, struct dlm_member, list); + if (new->nodeid < memb->nodeid) + break; + } + + if (!memb) + list_add_tail(newlist, head); + else { + /* FIXME: can use list macro here */ + newlist->prev = tmp->prev; + newlist->next = tmp; + tmp->prev->next = newlist; + tmp->prev = newlist; + } +} + +static int dlm_add_member(struct dlm_ls *ls, struct dlm_config_node *node) +{ + struct dlm_member *memb; + int error; + + memb = kzalloc(sizeof(struct dlm_member), GFP_NOFS); + if (!memb) + return -ENOMEM; + + error = dlm_lowcomms_connect_node(node->nodeid); + if (error < 0) { + kfree(memb); + return error; + } + + memb->nodeid = node->nodeid; + memb->weight = node->weight; + memb->comm_seq = node->comm_seq; + add_ordered_member(ls, memb); + ls->ls_num_nodes++; + return 0; +} + +static struct dlm_member *find_memb(struct list_head *head, int nodeid) +{ + struct dlm_member *memb; + + list_for_each_entry(memb, head, list) { + if (memb->nodeid == nodeid) + return memb; + } + return NULL; +} + +int dlm_is_member(struct dlm_ls *ls, int nodeid) +{ + if (find_memb(&ls->ls_nodes, nodeid)) + return 1; + return 0; +} + +int dlm_is_removed(struct dlm_ls *ls, int nodeid) +{ + if (find_memb(&ls->ls_nodes_gone, nodeid)) + return 1; + return 0; +} + +static void clear_memb_list(struct list_head *head) +{ + struct dlm_member *memb; + + while (!list_empty(head)) { + memb = list_entry(head->next, struct dlm_member, list); + list_del(&memb->list); + kfree(memb); + } +} + +void dlm_clear_members(struct dlm_ls *ls) +{ + clear_memb_list(&ls->ls_nodes); + ls->ls_num_nodes = 0; +} + +void dlm_clear_members_gone(struct dlm_ls *ls) +{ + clear_memb_list(&ls->ls_nodes_gone); +} + +static void make_member_array(struct dlm_ls *ls) +{ + struct dlm_member *memb; + int i, w, x = 0, total = 0, all_zero = 0, *array; + + kfree(ls->ls_node_array); + ls->ls_node_array = NULL; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->weight) + total += memb->weight; + } + + /* all nodes revert to weight of 1 if all have weight 0 */ + + if (!total) { + total = ls->ls_num_nodes; + all_zero = 1; + } + + ls->ls_total_weight = total; + + array = kmalloc(sizeof(int) * total, GFP_NOFS); + if (!array) + return; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (!all_zero && !memb->weight) + continue; + + if (all_zero) + w = 1; + else + w = memb->weight; + + DLM_ASSERT(x < total, printk("total %d x %d\n", total, x);); + + for (i = 0; i < w; i++) + array[x++] = memb->nodeid; + } + + ls->ls_node_array = array; +} + +/* send a status request to all members just to establish comms connections */ + +static int ping_members(struct dlm_ls *ls) +{ + struct dlm_member *memb; + int error = 0; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + error = dlm_recovery_stopped(ls); + if (error) + break; + error = dlm_rcom_status(ls, memb->nodeid, 0); + if (error) + break; + } + if (error) + log_debug(ls, "ping_members aborted %d last nodeid %d", + error, ls->ls_recover_nodeid); + return error; +} + +static void dlm_lsop_recover_prep(struct dlm_ls *ls) +{ + if (!ls->ls_ops || !ls->ls_ops->recover_prep) + return; + ls->ls_ops->recover_prep(ls->ls_ops_arg); +} + +static void dlm_lsop_recover_slot(struct dlm_ls *ls, struct dlm_member *memb) +{ + struct dlm_slot slot; + uint32_t seq; + int error; + + if (!ls->ls_ops || !ls->ls_ops->recover_slot) + return; + + /* if there is no comms connection with this node + or the present comms connection is newer + than the one when this member was added, then + we consider the node to have failed (versus + being removed due to dlm_release_lockspace) */ + + error = dlm_comm_seq(memb->nodeid, &seq); + + if (!error && seq == memb->comm_seq) + return; + + slot.nodeid = memb->nodeid; + slot.slot = memb->slot; + + ls->ls_ops->recover_slot(ls->ls_ops_arg, &slot); +} + +void dlm_lsop_recover_done(struct dlm_ls *ls) +{ + struct dlm_member *memb; + struct dlm_slot *slots; + int i, num; + + if (!ls->ls_ops || !ls->ls_ops->recover_done) + return; + + num = ls->ls_num_nodes; + + slots = kzalloc(num * sizeof(struct dlm_slot), GFP_KERNEL); + if (!slots) + return; + + i = 0; + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (i == num) { + log_error(ls, "dlm_lsop_recover_done bad num %d", num); + goto out; + } + slots[i].nodeid = memb->nodeid; + slots[i].slot = memb->slot; + i++; + } + + ls->ls_ops->recover_done(ls->ls_ops_arg, slots, num, + ls->ls_slot, ls->ls_generation); + out: + kfree(slots); +} + +static struct dlm_config_node *find_config_node(struct dlm_recover *rv, + int nodeid) +{ + int i; + + for (i = 0; i < rv->nodes_count; i++) { + if (rv->nodes[i].nodeid == nodeid) + return &rv->nodes[i]; + } + return NULL; +} + +int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) +{ + struct dlm_member *memb, *safe; + struct dlm_config_node *node; + int i, error, neg = 0, low = -1; + + /* previously removed members that we've not finished removing need to + count as a negative change so the "neg" recovery steps will happen */ + + list_for_each_entry(memb, &ls->ls_nodes_gone, list) { + log_debug(ls, "prev removed member %d", memb->nodeid); + neg++; + } + + /* move departed members from ls_nodes to ls_nodes_gone */ + + list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) { + node = find_config_node(rv, memb->nodeid); + if (node && !node->new) + continue; + + if (!node) { + log_debug(ls, "remove member %d", memb->nodeid); + } else { + /* removed and re-added */ + log_debug(ls, "remove member %d comm_seq %u %u", + memb->nodeid, memb->comm_seq, node->comm_seq); + } + + neg++; + list_move(&memb->list, &ls->ls_nodes_gone); + ls->ls_num_nodes--; + dlm_lsop_recover_slot(ls, memb); + } + + /* add new members to ls_nodes */ + + for (i = 0; i < rv->nodes_count; i++) { + node = &rv->nodes[i]; + if (dlm_is_member(ls, node->nodeid)) + continue; + dlm_add_member(ls, node); + log_debug(ls, "add member %d", node->nodeid); + } + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (low == -1 || memb->nodeid < low) + low = memb->nodeid; + } + ls->ls_low_nodeid = low; + + make_member_array(ls); + *neg_out = neg; + + error = ping_members(ls); + if (!error || error == -EPROTO) { + /* new_lockspace() may be waiting to know if the config + is good or bad */ + ls->ls_members_result = error; + complete(&ls->ls_members_done); + } + + log_debug(ls, "dlm_recover_members %d nodes", ls->ls_num_nodes); + return error; +} + +/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before + dlm_ls_start() is called on any of them to start the new recovery. */ + +int dlm_ls_stop(struct dlm_ls *ls) +{ + int new; + + /* + * Prevent dlm_recv from being in the middle of something when we do + * the stop. This includes ensuring dlm_recv isn't processing a + * recovery message (rcom), while dlm_recoverd is aborting and + * resetting things from an in-progress recovery. i.e. we want + * dlm_recoverd to abort its recovery without worrying about dlm_recv + * processing an rcom at the same time. Stopping dlm_recv also makes + * it easy for dlm_receive_message() to check locking stopped and add a + * message to the requestqueue without races. + */ + + down_write(&ls->ls_recv_active); + + /* + * Abort any recovery that's in progress (see RECOVER_STOP, + * dlm_recovery_stopped()) and tell any other threads running in the + * dlm to quit any processing (see RUNNING, dlm_locking_stopped()). + */ + + spin_lock(&ls->ls_recover_lock); + set_bit(LSFL_RECOVER_STOP, &ls->ls_flags); + new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags); + ls->ls_recover_seq++; + spin_unlock(&ls->ls_recover_lock); + + /* + * Let dlm_recv run again, now any normal messages will be saved on the + * requestqueue for later. + */ + + up_write(&ls->ls_recv_active); + + /* + * This in_recovery lock does two things: + * 1) Keeps this function from returning until all threads are out + * of locking routines and locking is truly stopped. + * 2) Keeps any new requests from being processed until it's unlocked + * when recovery is complete. + */ + + if (new) { + set_bit(LSFL_RECOVER_DOWN, &ls->ls_flags); + wake_up_process(ls->ls_recoverd_task); + wait_event(ls->ls_recover_lock_wait, + test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); + } + + /* + * The recoverd suspend/resume makes sure that dlm_recoverd (if + * running) has noticed RECOVER_STOP above and quit processing the + * previous recovery. + */ + + dlm_recoverd_suspend(ls); + + spin_lock(&ls->ls_recover_lock); + kfree(ls->ls_slots); + ls->ls_slots = NULL; + ls->ls_num_slots = 0; + ls->ls_slots_size = 0; + ls->ls_recover_status = 0; + spin_unlock(&ls->ls_recover_lock); + + dlm_recoverd_resume(ls); + + if (!ls->ls_recover_begin) + ls->ls_recover_begin = jiffies; + + dlm_lsop_recover_prep(ls); + return 0; +} + +int dlm_ls_start(struct dlm_ls *ls) +{ + struct dlm_recover *rv = NULL, *rv_old; + struct dlm_config_node *nodes; + int error, count; + + rv = kzalloc(sizeof(struct dlm_recover), GFP_NOFS); + if (!rv) + return -ENOMEM; + + error = dlm_config_nodes(ls->ls_name, &nodes, &count); + if (error < 0) + goto fail; + + spin_lock(&ls->ls_recover_lock); + + /* the lockspace needs to be stopped before it can be started */ + + if (!dlm_locking_stopped(ls)) { + spin_unlock(&ls->ls_recover_lock); + log_error(ls, "start ignored: lockspace running"); + error = -EINVAL; + goto fail; + } + + rv->nodes = nodes; + rv->nodes_count = count; + rv->seq = ++ls->ls_recover_seq; + rv_old = ls->ls_recover_args; + ls->ls_recover_args = rv; + spin_unlock(&ls->ls_recover_lock); + + if (rv_old) { + log_error(ls, "unused recovery %llx %d", + (unsigned long long)rv_old->seq, rv_old->nodes_count); + kfree(rv_old->nodes); + kfree(rv_old); + } + + set_bit(LSFL_RECOVER_WORK, &ls->ls_flags); + wake_up_process(ls->ls_recoverd_task); + return 0; + + fail: + kfree(rv); + kfree(nodes); + return error; +} + diff --git a/kmod/dlm/member.h b/kmod/dlm/member.h new file mode 100644 index 00000000..3deb7066 --- /dev/null +++ b/kmod/dlm/member.h @@ -0,0 +1,33 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MEMBER_DOT_H__ +#define __MEMBER_DOT_H__ + +int dlm_ls_stop(struct dlm_ls *ls); +int dlm_ls_start(struct dlm_ls *ls); +void dlm_clear_members(struct dlm_ls *ls); +void dlm_clear_members_gone(struct dlm_ls *ls); +int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv,int *neg_out); +int dlm_is_removed(struct dlm_ls *ls, int nodeid); +int dlm_is_member(struct dlm_ls *ls, int nodeid); +int dlm_slots_version(struct dlm_header *h); +void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, + struct dlm_member *memb); +void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc); +int dlm_slots_copy_in(struct dlm_ls *ls); +int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, + struct dlm_slot **slots_out, uint32_t *gen_out); +void dlm_lsop_recover_done(struct dlm_ls *ls); + +#endif /* __MEMBER_DOT_H__ */ + diff --git a/kmod/dlm/memory.c b/kmod/dlm/memory.c new file mode 100644 index 00000000..7cd24bcc --- /dev/null +++ b/kmod/dlm/memory.c @@ -0,0 +1,96 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "config.h" +#include "memory.h" + +static struct kmem_cache *lkb_cache; +static struct kmem_cache *rsb_cache; + + +int __init dlm_memory_init(void) +{ + lkb_cache = kmem_cache_create("dlm_lkb", sizeof(struct dlm_lkb), + __alignof__(struct dlm_lkb), 0, NULL); + if (!lkb_cache) + return -ENOMEM; + + rsb_cache = kmem_cache_create("dlm_rsb", sizeof(struct dlm_rsb), + __alignof__(struct dlm_rsb), 0, NULL); + if (!rsb_cache) { + kmem_cache_destroy(lkb_cache); + return -ENOMEM; + } + + return 0; +} + +void dlm_memory_exit(void) +{ + if (lkb_cache) + kmem_cache_destroy(lkb_cache); + if (rsb_cache) + kmem_cache_destroy(rsb_cache); +} + +char *dlm_allocate_lvb(struct dlm_ls *ls) +{ + char *p; + + p = kzalloc(ls->ls_lvblen, GFP_NOFS); + return p; +} + +void dlm_free_lvb(char *p) +{ + kfree(p); +} + +struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + + r = kmem_cache_zalloc(rsb_cache, GFP_NOFS); + return r; +} + +void dlm_free_rsb(struct dlm_rsb *r) +{ + if (r->res_lvbptr) + dlm_free_lvb(r->res_lvbptr); + kmem_cache_free(rsb_cache, r); +} + +struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + + lkb = kmem_cache_zalloc(lkb_cache, GFP_NOFS); + return lkb; +} + +void dlm_free_lkb(struct dlm_lkb *lkb) +{ + if (lkb->lkb_flags & DLM_IFL_USER) { + struct dlm_user_args *ua; + ua = lkb->lkb_ua; + if (ua) { + if (ua->lksb.sb_lvbptr) + kfree(ua->lksb.sb_lvbptr); + kfree(ua); + } + } + kmem_cache_free(lkb_cache, lkb); +} + diff --git a/kmod/dlm/memory.h b/kmod/dlm/memory.h new file mode 100644 index 00000000..177c11cb --- /dev/null +++ b/kmod/dlm/memory.h @@ -0,0 +1,27 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MEMORY_DOT_H__ +#define __MEMORY_DOT_H__ + +int dlm_memory_init(void); +void dlm_memory_exit(void); +struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls); +void dlm_free_rsb(struct dlm_rsb *r); +struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls); +void dlm_free_lkb(struct dlm_lkb *l); +char *dlm_allocate_lvb(struct dlm_ls *ls); +void dlm_free_lvb(char *l); + +#endif /* __MEMORY_DOT_H__ */ + diff --git a/kmod/dlm/midcomms.c b/kmod/dlm/midcomms.c new file mode 100644 index 00000000..f3396c62 --- /dev/null +++ b/kmod/dlm/midcomms.c @@ -0,0 +1,137 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +/* + * midcomms.c + * + * This is the appallingly named "mid-level" comms layer. + * + * Its purpose is to take packets from the "real" comms layer, + * split them up into packets and pass them to the interested + * part of the locking mechanism. + * + * It also takes messages from the locking layer, formats them + * into packets and sends them to the comms layer. + */ + +#include "dlm_internal.h" +#include "lowcomms.h" +#include "config.h" +#include "lock.h" +#include "midcomms.h" + + +static void copy_from_cb(void *dst, const void *base, unsigned offset, + unsigned len, unsigned limit) +{ + unsigned copy = len; + + if ((copy + offset) > limit) + copy = limit - offset; + memcpy(dst, base + offset, copy); + len -= copy; + if (len) + memcpy(dst + copy, base, len); +} + +/* + * Called from the low-level comms layer to process a buffer of + * commands. + * + * Only complete messages are processed here, any "spare" bytes from + * the end of a buffer are saved and tacked onto the front of the next + * message that comes in. I doubt this will happen very often but we + * need to be able to cope with it and I don't want the task to be waiting + * for packets to come in when there is useful work to be done. + */ + +int dlm_process_incoming_buffer(int nodeid, const void *base, + unsigned offset, unsigned len, unsigned limit) +{ + union { + unsigned char __buf[DLM_INBUF_LEN]; + /* this is to force proper alignment on some arches */ + union dlm_packet p; + } __tmp; + union dlm_packet *p = &__tmp.p; + int ret = 0; + int err = 0; + uint16_t msglen; + uint32_t lockspace; + + while (len > sizeof(struct dlm_header)) { + + /* Copy just the header to check the total length. The + message may wrap around the end of the buffer back to the + start, so we need to use a temp buffer and copy_from_cb. */ + + copy_from_cb(p, base, offset, sizeof(struct dlm_header), + limit); + + msglen = le16_to_cpu(p->header.h_length); + lockspace = p->header.h_lockspace; + + err = -EINVAL; + if (msglen < sizeof(struct dlm_header)) + break; + if (p->header.h_cmd == DLM_MSG) { + if (msglen < sizeof(struct dlm_message)) + break; + } else { + if (msglen < sizeof(struct dlm_rcom)) + break; + } + err = -E2BIG; + if (msglen > dlm_config.ci_buffer_size) { + log_print("message size %d from %d too big, buf len %d", + msglen, nodeid, len); + break; + } + err = 0; + + /* If only part of the full message is contained in this + buffer, then do nothing and wait for lowcomms to call + us again later with more data. We return 0 meaning + we've consumed none of the input buffer. */ + + if (msglen > len) + break; + + /* Allocate a larger temp buffer if the full message won't fit + in the buffer on the stack (which should work for most + ordinary messages). */ + + if (msglen > sizeof(__tmp) && p == &__tmp.p) { + p = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS); + if (p == NULL) + return ret; + } + + copy_from_cb(p, base, offset, msglen, limit); + + BUG_ON(lockspace != p->header.h_lockspace); + + ret += msglen; + offset += msglen; + offset &= (limit - 1); + len -= msglen; + + dlm_receive_buffer(p, nodeid); + } + + if (p != &__tmp.p) + kfree(p); + + return err ? err : ret; +} + diff --git a/kmod/dlm/midcomms.h b/kmod/dlm/midcomms.h new file mode 100644 index 00000000..95852a5f --- /dev/null +++ b/kmod/dlm/midcomms.h @@ -0,0 +1,21 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MIDCOMMS_DOT_H__ +#define __MIDCOMMS_DOT_H__ + +int dlm_process_incoming_buffer(int nodeid, const void *base, unsigned offset, + unsigned len, unsigned limit); + +#endif /* __MIDCOMMS_DOT_H__ */ + diff --git a/kmod/dlm/netlink.c b/kmod/dlm/netlink.c new file mode 100644 index 00000000..e7cfbaf8 --- /dev/null +++ b/kmod/dlm/netlink.c @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2007 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ + +#include +#include +#include +#include + +#include "dlm_internal.h" + +static uint32_t dlm_nl_seqnum; +static uint32_t listener_nlportid; + +static struct genl_family family = { + .id = GENL_ID_GENERATE, + .name = DLM_GENL_NAME, + .version = DLM_GENL_VERSION, +}; + +static int prepare_data(u8 cmd, struct sk_buff **skbp, size_t size) +{ + struct sk_buff *skb; + void *data; + + skb = genlmsg_new(size, GFP_NOFS); + if (!skb) + return -ENOMEM; + + /* add the message headers */ + data = genlmsg_put(skb, 0, dlm_nl_seqnum++, &family, 0, cmd); + if (!data) { + nlmsg_free(skb); + return -EINVAL; + } + + *skbp = skb; + return 0; +} + +static struct dlm_lock_data *mk_data(struct sk_buff *skb) +{ + struct nlattr *ret; + + ret = nla_reserve(skb, DLM_TYPE_LOCK, sizeof(struct dlm_lock_data)); + if (!ret) + return NULL; + return nla_data(ret); +} + +static int send_data(struct sk_buff *skb) +{ + struct genlmsghdr *genlhdr = nlmsg_data((struct nlmsghdr *)skb->data); + void *data = genlmsg_data(genlhdr); + int rv; + + rv = genlmsg_end(skb, data); + if (rv < 0) { + nlmsg_free(skb); + return rv; + } + + return genlmsg_unicast(&init_net, skb, listener_nlportid); +} + +static int user_cmd(struct sk_buff *skb, struct genl_info *info) +{ + listener_nlportid = info->snd_portid; + printk("user_cmd nlpid %u\n", listener_nlportid); + return 0; +} + +static struct genl_ops dlm_nl_ops[] = { + { + .cmd = DLM_CMD_HELLO, + .doit = user_cmd, + }, +}; + +int __init dlm_netlink_init(void) +{ + return genl_register_family_with_ops(&family, dlm_nl_ops); +} + +void dlm_netlink_exit(void) +{ + genl_unregister_family(&family); +} + +static void fill_data(struct dlm_lock_data *data, struct dlm_lkb *lkb) +{ + struct dlm_rsb *r = lkb->lkb_resource; + + memset(data, 0, sizeof(struct dlm_lock_data)); + + data->version = DLM_LOCK_DATA_VERSION; + data->nodeid = lkb->lkb_nodeid; + data->ownpid = lkb->lkb_ownpid; + data->id = lkb->lkb_id; + data->remid = lkb->lkb_remid; + data->status = lkb->lkb_status; + data->grmode = lkb->lkb_grmode; + data->rqmode = lkb->lkb_rqmode; + if (lkb->lkb_ua) + data->xid = lkb->lkb_ua->xid; + if (r) { + data->lockspace_id = r->res_ls->ls_global_id; + data->resource_namelen = r->res_length; + memcpy(data->resource_name, r->res_name, r->res_length); + } +} + +void dlm_timeout_warn(struct dlm_lkb *lkb) +{ + struct sk_buff *uninitialized_var(send_skb); + struct dlm_lock_data *data; + size_t size; + int rv; + + size = nla_total_size(sizeof(struct dlm_lock_data)) + + nla_total_size(0); /* why this? */ + + rv = prepare_data(DLM_CMD_TIMEOUT, &send_skb, size); + if (rv < 0) + return; + + data = mk_data(send_skb); + if (!data) { + nlmsg_free(send_skb); + return; + } + + fill_data(data, lkb); + + send_data(send_skb); +} + diff --git a/kmod/dlm/plock.c b/kmod/dlm/plock.c new file mode 100644 index 00000000..f704458e --- /dev/null +++ b/kmod/dlm/plock.c @@ -0,0 +1,515 @@ +/* + * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License version 2. + */ + +#include +#include +#include +#include +#include +#include + +#include "dlm_internal.h" +#include "lockspace.h" + +static spinlock_t ops_lock; +static struct list_head send_list; +static struct list_head recv_list; +static wait_queue_head_t send_wq; +static wait_queue_head_t recv_wq; + +struct plock_op { + struct list_head list; + int done; + struct dlm_plock_info info; +}; + +struct plock_xop { + struct plock_op xop; + void *callback; + void *fl; + void *file; + struct file_lock flc; +}; + + +static inline void set_version(struct dlm_plock_info *info) +{ + info->version[0] = DLM_PLOCK_VERSION_MAJOR; + info->version[1] = DLM_PLOCK_VERSION_MINOR; + info->version[2] = DLM_PLOCK_VERSION_PATCH; +} + +static int check_version(struct dlm_plock_info *info) +{ + if ((DLM_PLOCK_VERSION_MAJOR != info->version[0]) || + (DLM_PLOCK_VERSION_MINOR < info->version[1])) { + log_print("plock device version mismatch: " + "kernel (%u.%u.%u), user (%u.%u.%u)", + DLM_PLOCK_VERSION_MAJOR, + DLM_PLOCK_VERSION_MINOR, + DLM_PLOCK_VERSION_PATCH, + info->version[0], + info->version[1], + info->version[2]); + return -EINVAL; + } + return 0; +} + +static void send_op(struct plock_op *op) +{ + set_version(&op->info); + INIT_LIST_HEAD(&op->list); + spin_lock(&ops_lock); + list_add_tail(&op->list, &send_list); + spin_unlock(&ops_lock); + wake_up(&send_wq); +} + +/* If a process was killed while waiting for the only plock on a file, + locks_remove_posix will not see any lock on the file so it won't + send an unlock-close to us to pass on to userspace to clean up the + abandoned waiter. So, we have to insert the unlock-close when the + lock call is interrupted. */ + +static void do_unlock_close(struct dlm_ls *ls, u64 number, + struct file *file, struct file_lock *fl) +{ + struct plock_op *op; + + op = kzalloc(sizeof(*op), GFP_NOFS); + if (!op) + return; + + op->info.optype = DLM_PLOCK_OP_UNLOCK; + op->info.pid = fl->fl_pid; + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = 0; + op->info.end = OFFSET_MAX; + if (fl->fl_lmops && fl->fl_lmops->lm_grant) + op->info.owner = (__u64) fl->fl_pid; + else + op->info.owner = (__u64)(long) fl->fl_owner; + + op->info.flags |= DLM_PLOCK_FL_CLOSE; + send_op(op); +} + +int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + int cmd, struct file_lock *fl) +{ + struct dlm_ls *ls; + struct plock_op *op; + struct plock_xop *xop; + int rv; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + xop = kzalloc(sizeof(*xop), GFP_NOFS); + if (!xop) { + rv = -ENOMEM; + goto out; + } + + op = &xop->xop; + op->info.optype = DLM_PLOCK_OP_LOCK; + op->info.pid = fl->fl_pid; + op->info.ex = (fl->fl_type == F_WRLCK); + op->info.wait = IS_SETLKW(cmd); + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + if (fl->fl_lmops && fl->fl_lmops->lm_grant) { + /* fl_owner is lockd which doesn't distinguish + processes on the nfs client */ + op->info.owner = (__u64) fl->fl_pid; + xop->callback = fl->fl_lmops->lm_grant; + locks_init_lock(&xop->flc); + locks_copy_lock(&xop->flc, fl); + xop->fl = fl; + xop->file = file; + } else { + op->info.owner = (__u64)(long) fl->fl_owner; + xop->callback = NULL; + } + + send_op(op); + + if (xop->callback == NULL) { + rv = wait_event_killable(recv_wq, (op->done != 0)); + if (rv == -ERESTARTSYS) { + log_debug(ls, "dlm_posix_lock: wait killed %llx", + (unsigned long long)number); + spin_lock(&ops_lock); + list_del(&op->list); + spin_unlock(&ops_lock); + kfree(xop); + do_unlock_close(ls, number, file, fl); + goto out; + } + } else { + rv = FILE_LOCK_DEFERRED; + goto out; + } + + spin_lock(&ops_lock); + if (!list_empty(&op->list)) { + log_error(ls, "dlm_posix_lock: op on list %llx", + (unsigned long long)number); + list_del(&op->list); + } + spin_unlock(&ops_lock); + + rv = op->info.rv; + + if (!rv) { + if (posix_lock_file_wait(file, fl) < 0) + log_error(ls, "dlm_posix_lock: vfs lock error %llx", + (unsigned long long)number); + } + + kfree(xop); +out: + dlm_put_lockspace(ls); + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_lock); + +/* Returns failure iff a successful lock operation should be canceled */ +static int dlm_plock_callback(struct plock_op *op) +{ + struct file *file; + struct file_lock *fl; + struct file_lock *flc; + int (*notify)(void *, void *, int) = NULL; + struct plock_xop *xop = (struct plock_xop *)op; + int rv = 0; + + spin_lock(&ops_lock); + if (!list_empty(&op->list)) { + log_print("dlm_plock_callback: op on list %llx", + (unsigned long long)op->info.number); + list_del(&op->list); + } + spin_unlock(&ops_lock); + + /* check if the following 2 are still valid or make a copy */ + file = xop->file; + flc = &xop->flc; + fl = xop->fl; + notify = xop->callback; + + if (op->info.rv) { + notify(fl, NULL, op->info.rv); + goto out; + } + + /* got fs lock; bookkeep locally as well: */ + flc->fl_flags &= ~FL_SLEEP; + if (posix_lock_file(file, flc, NULL)) { + /* + * This can only happen in the case of kmalloc() failure. + * The filesystem's own lock is the authoritative lock, + * so a failure to get the lock locally is not a disaster. + * As long as the fs cannot reliably cancel locks (especially + * in a low-memory situation), we're better off ignoring + * this failure than trying to recover. + */ + log_print("dlm_plock_callback: vfs lock error %llx file %p fl %p", + (unsigned long long)op->info.number, file, fl); + } + + rv = notify(fl, NULL, 0); + if (rv) { + /* XXX: We need to cancel the fs lock here: */ + log_print("dlm_plock_callback: lock granted after lock request " + "failed; dangling lock!\n"); + goto out; + } + +out: + kfree(xop); + return rv; +} + +int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl) +{ + struct dlm_ls *ls; + struct plock_op *op; + int rv; + unsigned char fl_flags = fl->fl_flags; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + op = kzalloc(sizeof(*op), GFP_NOFS); + if (!op) { + rv = -ENOMEM; + goto out; + } + + /* cause the vfs unlock to return ENOENT if lock is not found */ + fl->fl_flags |= FL_EXISTS; + + rv = posix_lock_file_wait(file, fl); + if (rv == -ENOENT) { + rv = 0; + goto out_free; + } + if (rv < 0) { + log_error(ls, "dlm_posix_unlock: vfs unlock error %d %llx", + rv, (unsigned long long)number); + } + + op->info.optype = DLM_PLOCK_OP_UNLOCK; + op->info.pid = fl->fl_pid; + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + if (fl->fl_lmops && fl->fl_lmops->lm_grant) + op->info.owner = (__u64) fl->fl_pid; + else + op->info.owner = (__u64)(long) fl->fl_owner; + + if (fl->fl_flags & FL_CLOSE) { + op->info.flags |= DLM_PLOCK_FL_CLOSE; + send_op(op); + rv = 0; + goto out; + } + + send_op(op); + wait_event(recv_wq, (op->done != 0)); + + spin_lock(&ops_lock); + if (!list_empty(&op->list)) { + log_error(ls, "dlm_posix_unlock: op on list %llx", + (unsigned long long)number); + list_del(&op->list); + } + spin_unlock(&ops_lock); + + rv = op->info.rv; + + if (rv == -ENOENT) + rv = 0; + +out_free: + kfree(op); +out: + dlm_put_lockspace(ls); + fl->fl_flags = fl_flags; + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_unlock); + +int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl) +{ + struct dlm_ls *ls; + struct plock_op *op; + int rv; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + op = kzalloc(sizeof(*op), GFP_NOFS); + if (!op) { + rv = -ENOMEM; + goto out; + } + + op->info.optype = DLM_PLOCK_OP_GET; + op->info.pid = fl->fl_pid; + op->info.ex = (fl->fl_type == F_WRLCK); + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + if (fl->fl_lmops && fl->fl_lmops->lm_grant) + op->info.owner = (__u64) fl->fl_pid; + else + op->info.owner = (__u64)(long) fl->fl_owner; + + send_op(op); + wait_event(recv_wq, (op->done != 0)); + + spin_lock(&ops_lock); + if (!list_empty(&op->list)) { + log_error(ls, "dlm_posix_get: op on list %llx", + (unsigned long long)number); + list_del(&op->list); + } + spin_unlock(&ops_lock); + + /* info.rv from userspace is 1 for conflict, 0 for no-conflict, + -ENOENT if there are no locks on the file */ + + rv = op->info.rv; + + fl->fl_type = F_UNLCK; + if (rv == -ENOENT) + rv = 0; + else if (rv > 0) { + locks_init_lock(fl); + fl->fl_type = (op->info.ex) ? F_WRLCK : F_RDLCK; + fl->fl_flags = FL_POSIX; + fl->fl_pid = op->info.pid; + fl->fl_start = op->info.start; + fl->fl_end = op->info.end; + rv = 0; + } + + kfree(op); +out: + dlm_put_lockspace(ls); + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_get); + +/* a read copies out one plock request from the send list */ +static ssize_t dev_read(struct file *file, char __user *u, size_t count, + loff_t *ppos) +{ + struct dlm_plock_info info; + struct plock_op *op = NULL; + + if (count < sizeof(info)) + return -EINVAL; + + spin_lock(&ops_lock); + if (!list_empty(&send_list)) { + op = list_entry(send_list.next, struct plock_op, list); + if (op->info.flags & DLM_PLOCK_FL_CLOSE) + list_del(&op->list); + else + list_move(&op->list, &recv_list); + memcpy(&info, &op->info, sizeof(info)); + } + spin_unlock(&ops_lock); + + if (!op) + return -EAGAIN; + + /* there is no need to get a reply from userspace for unlocks + that were generated by the vfs cleaning up for a close + (the process did not make an unlock call). */ + + if (op->info.flags & DLM_PLOCK_FL_CLOSE) + kfree(op); + + if (copy_to_user(u, &info, sizeof(info))) + return -EFAULT; + return sizeof(info); +} + +/* a write copies in one plock result that should match a plock_op + on the recv list */ +static ssize_t dev_write(struct file *file, const char __user *u, size_t count, + loff_t *ppos) +{ + struct dlm_plock_info info; + struct plock_op *op; + int found = 0, do_callback = 0; + + if (count != sizeof(info)) + return -EINVAL; + + if (copy_from_user(&info, u, sizeof(info))) + return -EFAULT; + + if (check_version(&info)) + return -EINVAL; + + spin_lock(&ops_lock); + list_for_each_entry(op, &recv_list, list) { + if (op->info.fsid == info.fsid && + op->info.number == info.number && + op->info.owner == info.owner) { + struct plock_xop *xop = (struct plock_xop *)op; + list_del_init(&op->list); + memcpy(&op->info, &info, sizeof(info)); + if (xop->callback) + do_callback = 1; + else + op->done = 1; + found = 1; + break; + } + } + spin_unlock(&ops_lock); + + if (found) { + if (do_callback) + dlm_plock_callback(op); + else + wake_up(&recv_wq); + } else + log_print("dev_write no op %x %llx", info.fsid, + (unsigned long long)info.number); + return count; +} + +static unsigned int dev_poll(struct file *file, poll_table *wait) +{ + unsigned int mask = 0; + + poll_wait(file, &send_wq, wait); + + spin_lock(&ops_lock); + if (!list_empty(&send_list)) + mask = POLLIN | POLLRDNORM; + spin_unlock(&ops_lock); + + return mask; +} + +static const struct file_operations dev_fops = { + .read = dev_read, + .write = dev_write, + .poll = dev_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice plock_dev_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = DLM_PLOCK_MISC_NAME, + .fops = &dev_fops +}; + +int dlm_plock_init(void) +{ + int rv; + + spin_lock_init(&ops_lock); + INIT_LIST_HEAD(&send_list); + INIT_LIST_HEAD(&recv_list); + init_waitqueue_head(&send_wq); + init_waitqueue_head(&recv_wq); + + rv = misc_register(&plock_dev_misc); + if (rv) + log_print("dlm_plock_init: misc_register failed %d", rv); + return rv; +} + +void dlm_plock_exit(void) +{ + if (misc_deregister(&plock_dev_misc) < 0) + log_print("dlm_plock_exit: misc_deregister failed"); +} + diff --git a/kmod/dlm/rcom.c b/kmod/dlm/rcom.c new file mode 100644 index 00000000..f3f5e72a --- /dev/null +++ b/kmod/dlm/rcom.c @@ -0,0 +1,656 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "lowcomms.h" +#include "midcomms.h" +#include "rcom.h" +#include "recover.h" +#include "dir.h" +#include "config.h" +#include "memory.h" +#include "lock.h" +#include "util.h" + +static int rcom_response(struct dlm_ls *ls) +{ + return test_bit(LSFL_RCOM_READY, &ls->ls_flags); +} + +static int create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len, + struct dlm_rcom **rc_ret, struct dlm_mhandle **mh_ret) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + char *mb; + int mb_len = sizeof(struct dlm_rcom) + len; + + mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb); + if (!mh) { + log_print("create_rcom to %d type %d len %d ENOBUFS", + to_nodeid, type, len); + return -ENOBUFS; + } + memset(mb, 0, mb_len); + + rc = (struct dlm_rcom *) mb; + + rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + rc->rc_header.h_lockspace = ls->ls_global_id; + rc->rc_header.h_nodeid = dlm_our_nodeid(); + rc->rc_header.h_length = mb_len; + rc->rc_header.h_cmd = DLM_RCOM; + + rc->rc_type = type; + + spin_lock(&ls->ls_recover_lock); + rc->rc_seq = ls->ls_recover_seq; + spin_unlock(&ls->ls_recover_lock); + + *mh_ret = mh; + *rc_ret = rc; + return 0; +} + +static void send_rcom(struct dlm_ls *ls, struct dlm_mhandle *mh, + struct dlm_rcom *rc) +{ + dlm_rcom_out(rc); + dlm_lowcomms_commit_buffer(mh); +} + +static void set_rcom_status(struct dlm_ls *ls, struct rcom_status *rs, + uint32_t flags) +{ + rs->rs_flags = cpu_to_le32(flags); +} + +/* When replying to a status request, a node also sends back its + configuration values. The requesting node then checks that the remote + node is configured the same way as itself. */ + +static void set_rcom_config(struct dlm_ls *ls, struct rcom_config *rf, + uint32_t num_slots) +{ + rf->rf_lvblen = cpu_to_le32(ls->ls_lvblen); + rf->rf_lsflags = cpu_to_le32(ls->ls_exflags); + + rf->rf_our_slot = cpu_to_le16(ls->ls_slot); + rf->rf_num_slots = cpu_to_le16(num_slots); + rf->rf_generation = cpu_to_le32(ls->ls_generation); +} + +static int check_rcom_config(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid) +{ + struct rcom_config *rf = (struct rcom_config *) rc->rc_buf; + + if ((rc->rc_header.h_version & 0xFFFF0000) != DLM_HEADER_MAJOR) { + log_error(ls, "version mismatch: %x nodeid %d: %x", + DLM_HEADER_MAJOR | DLM_HEADER_MINOR, nodeid, + rc->rc_header.h_version); + return -EPROTO; + } + + if (le32_to_cpu(rf->rf_lvblen) != ls->ls_lvblen || + le32_to_cpu(rf->rf_lsflags) != ls->ls_exflags) { + log_error(ls, "config mismatch: %d,%x nodeid %d: %d,%x", + ls->ls_lvblen, ls->ls_exflags, nodeid, + le32_to_cpu(rf->rf_lvblen), + le32_to_cpu(rf->rf_lsflags)); + return -EPROTO; + } + return 0; +} + +static void allow_sync_reply(struct dlm_ls *ls, uint64_t *new_seq) +{ + spin_lock(&ls->ls_rcom_spin); + *new_seq = ++ls->ls_rcom_seq; + set_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + spin_unlock(&ls->ls_rcom_spin); +} + +static void disallow_sync_reply(struct dlm_ls *ls) +{ + spin_lock(&ls->ls_rcom_spin); + clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + clear_bit(LSFL_RCOM_READY, &ls->ls_flags); + spin_unlock(&ls->ls_rcom_spin); +} + +/* + * low nodeid gathers one slot value at a time from each node. + * it sets need_slots=0, and saves rf_our_slot returned from each + * rcom_config. + * + * other nodes gather all slot values at once from the low nodeid. + * they set need_slots=1, and ignore the rf_our_slot returned from each + * rcom_config. they use the rf_num_slots returned from the low + * node's rcom_config. + */ + +int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error = 0; + + ls->ls_recover_nodeid = nodeid; + + if (nodeid == dlm_our_nodeid()) { + rc = ls->ls_recover_buf; + rc->rc_result = dlm_recover_status(ls); + goto out; + } + + error = create_rcom(ls, nodeid, DLM_RCOM_STATUS, + sizeof(struct rcom_status), &rc, &mh); + if (error) + goto out; + + set_rcom_status(ls, (struct rcom_status *)rc->rc_buf, status_flags); + + allow_sync_reply(ls, &rc->rc_id); + memset(ls->ls_recover_buf, 0, dlm_config.ci_buffer_size); + + send_rcom(ls, mh, rc); + + error = dlm_wait_function(ls, &rcom_response); + disallow_sync_reply(ls); + if (error) + goto out; + + rc = ls->ls_recover_buf; + + if (rc->rc_result == -ESRCH) { + /* we pretend the remote lockspace exists with 0 status */ + log_debug(ls, "remote node %d not ready", nodeid); + rc->rc_result = 0; + error = 0; + } else { + error = check_rcom_config(ls, rc, nodeid); + } + + /* the caller looks at rc_result for the remote recovery status */ + out: + return error; +} + +static void receive_rcom_status(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct rcom_status *rs; + uint32_t status; + int nodeid = rc_in->rc_header.h_nodeid; + int len = sizeof(struct rcom_config); + int num_slots = 0; + int error; + + if (!dlm_slots_version(&rc_in->rc_header)) { + status = dlm_recover_status(ls); + goto do_create; + } + + rs = (struct rcom_status *)rc_in->rc_buf; + + if (!(le32_to_cpu(rs->rs_flags) & DLM_RSF_NEED_SLOTS)) { + status = dlm_recover_status(ls); + goto do_create; + } + + spin_lock(&ls->ls_recover_lock); + status = ls->ls_recover_status; + num_slots = ls->ls_num_slots; + spin_unlock(&ls->ls_recover_lock); + len += num_slots * sizeof(struct rcom_slot); + + do_create: + error = create_rcom(ls, nodeid, DLM_RCOM_STATUS_REPLY, + len, &rc, &mh); + if (error) + return; + + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + rc->rc_result = status; + + set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, num_slots); + + if (!num_slots) + goto do_send; + + spin_lock(&ls->ls_recover_lock); + if (ls->ls_num_slots != num_slots) { + spin_unlock(&ls->ls_recover_lock); + log_debug(ls, "receive_rcom_status num_slots %d to %d", + num_slots, ls->ls_num_slots); + rc->rc_result = 0; + set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, 0); + goto do_send; + } + + dlm_slots_copy_out(ls, rc); + spin_unlock(&ls->ls_recover_lock); + + do_send: + send_rcom(ls, mh, rc); +} + +static void receive_sync_reply(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + spin_lock(&ls->ls_rcom_spin); + if (!test_bit(LSFL_RCOM_WAIT, &ls->ls_flags) || + rc_in->rc_id != ls->ls_rcom_seq) { + log_debug(ls, "reject reply %d from %d seq %llx expect %llx", + rc_in->rc_type, rc_in->rc_header.h_nodeid, + (unsigned long long)rc_in->rc_id, + (unsigned long long)ls->ls_rcom_seq); + goto out; + } + memcpy(ls->ls_recover_buf, rc_in, rc_in->rc_header.h_length); + set_bit(LSFL_RCOM_READY, &ls->ls_flags); + clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + wake_up(&ls->ls_wait_general); + out: + spin_unlock(&ls->ls_rcom_spin); +} + +int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name, int last_len) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error = 0; + + ls->ls_recover_nodeid = nodeid; + + error = create_rcom(ls, nodeid, DLM_RCOM_NAMES, last_len, &rc, &mh); + if (error) + goto out; + memcpy(rc->rc_buf, last_name, last_len); + + allow_sync_reply(ls, &rc->rc_id); + memset(ls->ls_recover_buf, 0, dlm_config.ci_buffer_size); + + send_rcom(ls, mh, rc); + + error = dlm_wait_function(ls, &rcom_response); + disallow_sync_reply(ls); + out: + return error; +} + +static void receive_rcom_names(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error, inlen, outlen, nodeid; + + nodeid = rc_in->rc_header.h_nodeid; + inlen = rc_in->rc_header.h_length - sizeof(struct dlm_rcom); + outlen = dlm_config.ci_buffer_size - sizeof(struct dlm_rcom); + + error = create_rcom(ls, nodeid, DLM_RCOM_NAMES_REPLY, outlen, &rc, &mh); + if (error) + return; + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + dlm_copy_master_names(ls, rc_in->rc_buf, inlen, rc->rc_buf, outlen, + nodeid); + send_rcom(ls, mh, rc); +} + +int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct dlm_ls *ls = r->res_ls; + int error; + + error = create_rcom(ls, dir_nodeid, DLM_RCOM_LOOKUP, r->res_length, + &rc, &mh); + if (error) + goto out; + memcpy(rc->rc_buf, r->res_name, r->res_length); + rc->rc_id = (unsigned long) r->res_id; + + send_rcom(ls, mh, rc); + out: + return error; +} + +int dlm_send_rcom_lookup_dump(struct dlm_rsb *r, int to_nodeid) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct dlm_ls *ls = r->res_ls; + int error; + + error = create_rcom(ls, to_nodeid, DLM_RCOM_LOOKUP, r->res_length, + &rc, &mh); + if (error) + goto out; + memcpy(rc->rc_buf, r->res_name, r->res_length); + rc->rc_id = 0xFFFFFFFF; + + send_rcom(ls, mh, rc); + out: + return error; +} + +static void receive_rcom_lookup(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error, ret_nodeid, nodeid = rc_in->rc_header.h_nodeid; + int len = rc_in->rc_header.h_length - sizeof(struct dlm_rcom); + + error = create_rcom(ls, nodeid, DLM_RCOM_LOOKUP_REPLY, 0, &rc, &mh); + if (error) + return; + + if (rc_in->rc_id == 0xFFFFFFFF) { + log_error(ls, "receive_rcom_lookup dump from %d", nodeid); + dlm_dump_rsb_name(ls, rc_in->rc_buf, len); + return; + } + + error = dlm_master_lookup(ls, nodeid, rc_in->rc_buf, len, + DLM_LU_RECOVER_MASTER, &ret_nodeid, NULL); + if (error) + ret_nodeid = error; + rc->rc_result = ret_nodeid; + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + send_rcom(ls, mh, rc); +} + +static void receive_rcom_lookup_reply(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + dlm_recover_master_reply(ls, rc_in); +} + +static void pack_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct rcom_lock *rl) +{ + memset(rl, 0, sizeof(*rl)); + + rl->rl_ownpid = cpu_to_le32(lkb->lkb_ownpid); + rl->rl_lkid = cpu_to_le32(lkb->lkb_id); + rl->rl_exflags = cpu_to_le32(lkb->lkb_exflags); + rl->rl_flags = cpu_to_le32(lkb->lkb_flags); + rl->rl_lvbseq = cpu_to_le32(lkb->lkb_lvbseq); + rl->rl_rqmode = lkb->lkb_rqmode; + rl->rl_grmode = lkb->lkb_grmode; + rl->rl_status = lkb->lkb_status; + rl->rl_wait_type = cpu_to_le16(lkb->lkb_wait_type); + + if (lkb->lkb_bastfn) + rl->rl_asts |= DLM_CB_BAST; + if (lkb->lkb_astfn) + rl->rl_asts |= DLM_CB_CAST; + + rl->rl_namelen = cpu_to_le16(r->res_length); + memcpy(rl->rl_name, r->res_name, r->res_length); + + /* FIXME: might we have an lvb without DLM_LKF_VALBLK set ? + If so, receive_rcom_lock_args() won't take this copy. */ + + if (lkb->lkb_lvbptr) + memcpy(rl->rl_lvb, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); +} + +int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct dlm_ls *ls = r->res_ls; + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct rcom_lock *rl; + int error, len = sizeof(struct rcom_lock); + + if (lkb->lkb_lvbptr) + len += ls->ls_lvblen; + + error = create_rcom(ls, r->res_nodeid, DLM_RCOM_LOCK, len, &rc, &mh); + if (error) + goto out; + + rl = (struct rcom_lock *) rc->rc_buf; + pack_rcom_lock(r, lkb, rl); + rc->rc_id = (unsigned long) r; + + send_rcom(ls, mh, rc); + out: + return error; +} + +/* needs at least dlm_rcom + rcom_lock */ +static void receive_rcom_lock(struct dlm_ls *ls, struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error, nodeid = rc_in->rc_header.h_nodeid; + + dlm_recover_master_copy(ls, rc_in); + + error = create_rcom(ls, nodeid, DLM_RCOM_LOCK_REPLY, + sizeof(struct rcom_lock), &rc, &mh); + if (error) + return; + + /* We send back the same rcom_lock struct we received, but + dlm_recover_master_copy() has filled in rl_remid and rl_result */ + + memcpy(rc->rc_buf, rc_in->rc_buf, sizeof(struct rcom_lock)); + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + send_rcom(ls, mh, rc); +} + +/* If the lockspace doesn't exist then still send a status message + back; it's possible that it just doesn't have its global_id yet. */ + +int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct rcom_config *rf; + struct dlm_mhandle *mh; + char *mb; + int mb_len = sizeof(struct dlm_rcom) + sizeof(struct rcom_config); + + mh = dlm_lowcomms_get_buffer(nodeid, mb_len, GFP_NOFS, &mb); + if (!mh) + return -ENOBUFS; + memset(mb, 0, mb_len); + + rc = (struct dlm_rcom *) mb; + + rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + rc->rc_header.h_lockspace = rc_in->rc_header.h_lockspace; + rc->rc_header.h_nodeid = dlm_our_nodeid(); + rc->rc_header.h_length = mb_len; + rc->rc_header.h_cmd = DLM_RCOM; + + rc->rc_type = DLM_RCOM_STATUS_REPLY; + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + rc->rc_result = -ESRCH; + + rf = (struct rcom_config *) rc->rc_buf; + rf->rf_lvblen = cpu_to_le32(~0U); + + dlm_rcom_out(rc); + dlm_lowcomms_commit_buffer(mh); + + return 0; +} + +/* + * Ignore messages for stage Y before we set + * recover_status bit for stage X: + * + * recover_status = 0 + * + * dlm_recover_members() + * - send nothing + * - recv nothing + * - ignore NAMES, NAMES_REPLY + * - ignore LOOKUP, LOOKUP_REPLY + * - ignore LOCK, LOCK_REPLY + * + * recover_status |= NODES + * + * dlm_recover_members_wait() + * + * dlm_recover_directory() + * - send NAMES + * - recv NAMES_REPLY + * - ignore LOOKUP, LOOKUP_REPLY + * - ignore LOCK, LOCK_REPLY + * + * recover_status |= DIR + * + * dlm_recover_directory_wait() + * + * dlm_recover_masters() + * - send LOOKUP + * - recv LOOKUP_REPLY + * + * dlm_recover_locks() + * - send LOCKS + * - recv LOCKS_REPLY + * + * recover_status |= LOCKS + * + * dlm_recover_locks_wait() + * + * recover_status |= DONE + */ + +/* Called by dlm_recv; corresponds to dlm_receive_message() but special + recovery-only comms are sent through here. */ + +void dlm_receive_rcom(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid) +{ + int lock_size = sizeof(struct dlm_rcom) + sizeof(struct rcom_lock); + int stop, reply = 0, names = 0, lookup = 0, lock = 0; + uint32_t status; + uint64_t seq; + + switch (rc->rc_type) { + case DLM_RCOM_STATUS_REPLY: + reply = 1; + break; + case DLM_RCOM_NAMES: + names = 1; + break; + case DLM_RCOM_NAMES_REPLY: + names = 1; + reply = 1; + break; + case DLM_RCOM_LOOKUP: + lookup = 1; + break; + case DLM_RCOM_LOOKUP_REPLY: + lookup = 1; + reply = 1; + break; + case DLM_RCOM_LOCK: + lock = 1; + break; + case DLM_RCOM_LOCK_REPLY: + lock = 1; + reply = 1; + break; + }; + + spin_lock(&ls->ls_recover_lock); + status = ls->ls_recover_status; + stop = test_bit(LSFL_RECOVER_STOP, &ls->ls_flags); + seq = ls->ls_recover_seq; + spin_unlock(&ls->ls_recover_lock); + + if (stop && (rc->rc_type != DLM_RCOM_STATUS)) + goto ignore; + + if (reply && (rc->rc_seq_reply != seq)) + goto ignore; + + if (!(status & DLM_RS_NODES) && (names || lookup || lock)) + goto ignore; + + if (!(status & DLM_RS_DIR) && (lookup || lock)) + goto ignore; + + switch (rc->rc_type) { + case DLM_RCOM_STATUS: + receive_rcom_status(ls, rc); + break; + + case DLM_RCOM_NAMES: + receive_rcom_names(ls, rc); + break; + + case DLM_RCOM_LOOKUP: + receive_rcom_lookup(ls, rc); + break; + + case DLM_RCOM_LOCK: + if (rc->rc_header.h_length < lock_size) + goto Eshort; + receive_rcom_lock(ls, rc); + break; + + case DLM_RCOM_STATUS_REPLY: + receive_sync_reply(ls, rc); + break; + + case DLM_RCOM_NAMES_REPLY: + receive_sync_reply(ls, rc); + break; + + case DLM_RCOM_LOOKUP_REPLY: + receive_rcom_lookup_reply(ls, rc); + break; + + case DLM_RCOM_LOCK_REPLY: + if (rc->rc_header.h_length < lock_size) + goto Eshort; + dlm_recover_process_copy(ls, rc); + break; + + default: + log_error(ls, "receive_rcom bad type %d", rc->rc_type); + } + return; + +ignore: + log_limit(ls, "dlm_receive_rcom ignore msg %d " + "from %d %llu %llu recover seq %llu sts %x gen %u", + rc->rc_type, + nodeid, + (unsigned long long)rc->rc_seq, + (unsigned long long)rc->rc_seq_reply, + (unsigned long long)seq, + status, ls->ls_generation); + return; +Eshort: + log_error(ls, "recovery message %d from %d is too short", + rc->rc_type, nodeid); +} + diff --git a/kmod/dlm/rcom.h b/kmod/dlm/rcom.h new file mode 100644 index 00000000..f8e24346 --- /dev/null +++ b/kmod/dlm/rcom.h @@ -0,0 +1,26 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RCOM_DOT_H__ +#define __RCOM_DOT_H__ + +int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags); +int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name,int last_len); +int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid); +int dlm_send_rcom_lookup_dump(struct dlm_rsb *r, int to_nodeid); +int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); +void dlm_receive_rcom(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid); +int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in); + +#endif + diff --git a/kmod/dlm/recover.c b/kmod/dlm/recover.c new file mode 100644 index 00000000..a6bc63f6 --- /dev/null +++ b/kmod/dlm/recover.c @@ -0,0 +1,955 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "dir.h" +#include "config.h" +#include "ast.h" +#include "memory.h" +#include "rcom.h" +#include "lock.h" +#include "lowcomms.h" +#include "member.h" +#include "recover.h" + + +/* + * Recovery waiting routines: these functions wait for a particular reply from + * a remote node, or for the remote node to report a certain status. They need + * to abort if the lockspace is stopped indicating a node has failed (perhaps + * the one being waited for). + */ + +/* + * Wait until given function returns non-zero or lockspace is stopped + * (LS_RECOVERY_STOP set due to failure of a node in ls_nodes). When another + * function thinks it could have completed the waited-on task, they should wake + * up ls_wait_general to get an immediate response rather than waiting for the + * timeout. This uses a timeout so it can check periodically if the wait + * should abort due to node failure (which doesn't cause a wake_up). + * This should only be called by the dlm_recoverd thread. + */ + +int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)) +{ + int error = 0; + int rv; + + while (1) { + rv = wait_event_timeout(ls->ls_wait_general, + testfn(ls) || dlm_recovery_stopped(ls), + dlm_config.ci_recover_timer * HZ); + if (rv) + break; + } + + if (dlm_recovery_stopped(ls)) { + log_debug(ls, "dlm_wait_function aborted"); + error = -EINTR; + } + return error; +} + +/* + * An efficient way for all nodes to wait for all others to have a certain + * status. The node with the lowest nodeid polls all the others for their + * status (wait_status_all) and all the others poll the node with the low id + * for its accumulated result (wait_status_low). When all nodes have set + * status flag X, then status flag X_ALL will be set on the low nodeid. + */ + +uint32_t dlm_recover_status(struct dlm_ls *ls) +{ + uint32_t status; + spin_lock(&ls->ls_recover_lock); + status = ls->ls_recover_status; + spin_unlock(&ls->ls_recover_lock); + return status; +} + +static void _set_recover_status(struct dlm_ls *ls, uint32_t status) +{ + ls->ls_recover_status |= status; +} + +void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status) +{ + spin_lock(&ls->ls_recover_lock); + _set_recover_status(ls, status); + spin_unlock(&ls->ls_recover_lock); +} + +static int wait_status_all(struct dlm_ls *ls, uint32_t wait_status, + int save_slots) +{ + struct dlm_rcom *rc = ls->ls_recover_buf; + struct dlm_member *memb; + int error = 0, delay; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + delay = 0; + for (;;) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + error = dlm_rcom_status(ls, memb->nodeid, 0); + if (error) + goto out; + + if (save_slots) + dlm_slot_save(ls, rc, memb); + + if (rc->rc_result & wait_status) + break; + if (delay < 1000) + delay += 20; + msleep(delay); + } + } + out: + return error; +} + +static int wait_status_low(struct dlm_ls *ls, uint32_t wait_status, + uint32_t status_flags) +{ + struct dlm_rcom *rc = ls->ls_recover_buf; + int error = 0, delay = 0, nodeid = ls->ls_low_nodeid; + + for (;;) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + error = dlm_rcom_status(ls, nodeid, status_flags); + if (error) + break; + + if (rc->rc_result & wait_status) + break; + if (delay < 1000) + delay += 20; + msleep(delay); + } + out: + return error; +} + +static int wait_status(struct dlm_ls *ls, uint32_t status) +{ + uint32_t status_all = status << 1; + int error; + + if (ls->ls_low_nodeid == dlm_our_nodeid()) { + error = wait_status_all(ls, status, 0); + if (!error) + dlm_set_recover_status(ls, status_all); + } else + error = wait_status_low(ls, status_all, 0); + + return error; +} + +int dlm_recover_members_wait(struct dlm_ls *ls) +{ + struct dlm_member *memb; + struct dlm_slot *slots; + int num_slots, slots_size; + int error, rv; + uint32_t gen; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + memb->slot = -1; + memb->generation = 0; + } + + if (ls->ls_low_nodeid == dlm_our_nodeid()) { + error = wait_status_all(ls, DLM_RS_NODES, 1); + if (error) + goto out; + + /* slots array is sparse, slots_size may be > num_slots */ + + rv = dlm_slots_assign(ls, &num_slots, &slots_size, &slots, &gen); + if (!rv) { + spin_lock(&ls->ls_recover_lock); + _set_recover_status(ls, DLM_RS_NODES_ALL); + ls->ls_num_slots = num_slots; + ls->ls_slots_size = slots_size; + ls->ls_slots = slots; + ls->ls_generation = gen; + spin_unlock(&ls->ls_recover_lock); + } else { + dlm_set_recover_status(ls, DLM_RS_NODES_ALL); + } + } else { + error = wait_status_low(ls, DLM_RS_NODES_ALL, DLM_RSF_NEED_SLOTS); + if (error) + goto out; + + dlm_slots_copy_in(ls); + } + out: + return error; +} + +int dlm_recover_directory_wait(struct dlm_ls *ls) +{ + return wait_status(ls, DLM_RS_DIR); +} + +int dlm_recover_locks_wait(struct dlm_ls *ls) +{ + return wait_status(ls, DLM_RS_LOCKS); +} + +int dlm_recover_done_wait(struct dlm_ls *ls) +{ + return wait_status(ls, DLM_RS_DONE); +} + +/* + * The recover_list contains all the rsb's for which we've requested the new + * master nodeid. As replies are returned from the resource directories the + * rsb's are removed from the list. When the list is empty we're done. + * + * The recover_list is later similarly used for all rsb's for which we've sent + * new lkb's and need to receive new corresponding lkid's. + * + * We use the address of the rsb struct as a simple local identifier for the + * rsb so we can match an rcom reply with the rsb it was sent for. + */ + +static int recover_list_empty(struct dlm_ls *ls) +{ + int empty; + + spin_lock(&ls->ls_recover_list_lock); + empty = list_empty(&ls->ls_recover_list); + spin_unlock(&ls->ls_recover_list_lock); + + return empty; +} + +static void recover_list_add(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock(&ls->ls_recover_list_lock); + if (list_empty(&r->res_recover_list)) { + list_add_tail(&r->res_recover_list, &ls->ls_recover_list); + ls->ls_recover_list_count++; + dlm_hold_rsb(r); + } + spin_unlock(&ls->ls_recover_list_lock); +} + +static void recover_list_del(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock(&ls->ls_recover_list_lock); + list_del_init(&r->res_recover_list); + ls->ls_recover_list_count--; + spin_unlock(&ls->ls_recover_list_lock); + + dlm_put_rsb(r); +} + +static void recover_list_clear(struct dlm_ls *ls) +{ + struct dlm_rsb *r, *s; + + spin_lock(&ls->ls_recover_list_lock); + list_for_each_entry_safe(r, s, &ls->ls_recover_list, res_recover_list) { + list_del_init(&r->res_recover_list); + r->res_recover_locks_count = 0; + dlm_put_rsb(r); + ls->ls_recover_list_count--; + } + + if (ls->ls_recover_list_count != 0) { + log_error(ls, "warning: recover_list_count %d", + ls->ls_recover_list_count); + ls->ls_recover_list_count = 0; + } + spin_unlock(&ls->ls_recover_list_lock); +} + +static int recover_idr_empty(struct dlm_ls *ls) +{ + int empty = 1; + + spin_lock(&ls->ls_recover_idr_lock); + if (ls->ls_recover_list_count) + empty = 0; + spin_unlock(&ls->ls_recover_idr_lock); + + return empty; +} + +static int recover_idr_add(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + int rv; + + idr_preload(GFP_NOFS); + spin_lock(&ls->ls_recover_idr_lock); + if (r->res_id) { + rv = -1; + goto out_unlock; + } + rv = idr_alloc(&ls->ls_recover_idr, r, 1, 0, GFP_NOWAIT); + if (rv < 0) + goto out_unlock; + + r->res_id = rv; + ls->ls_recover_list_count++; + dlm_hold_rsb(r); + rv = 0; +out_unlock: + spin_unlock(&ls->ls_recover_idr_lock); + idr_preload_end(); + return rv; +} + +static void recover_idr_del(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock(&ls->ls_recover_idr_lock); + idr_remove(&ls->ls_recover_idr, r->res_id); + r->res_id = 0; + ls->ls_recover_list_count--; + spin_unlock(&ls->ls_recover_idr_lock); + + dlm_put_rsb(r); +} + +static struct dlm_rsb *recover_idr_find(struct dlm_ls *ls, uint64_t id) +{ + struct dlm_rsb *r; + + spin_lock(&ls->ls_recover_idr_lock); + r = idr_find(&ls->ls_recover_idr, (int)id); + spin_unlock(&ls->ls_recover_idr_lock); + return r; +} + +static void recover_idr_clear(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + int id; + + spin_lock(&ls->ls_recover_idr_lock); + + idr_for_each_entry(&ls->ls_recover_idr, r, id) { + idr_remove(&ls->ls_recover_idr, id); + r->res_id = 0; + r->res_recover_locks_count = 0; + ls->ls_recover_list_count--; + + dlm_put_rsb(r); + } + + if (ls->ls_recover_list_count != 0) { + log_error(ls, "warning: recover_list_count %d", + ls->ls_recover_list_count); + ls->ls_recover_list_count = 0; + } + spin_unlock(&ls->ls_recover_idr_lock); +} + + +/* Master recovery: find new master node for rsb's that were + mastered on nodes that have been removed. + + dlm_recover_masters + recover_master + dlm_send_rcom_lookup -> receive_rcom_lookup + dlm_dir_lookup + receive_rcom_lookup_reply <- + dlm_recover_master_reply + set_new_master + set_master_lkbs + set_lock_master +*/ + +/* + * Set the lock master for all LKBs in a lock queue + * If we are the new master of the rsb, we may have received new + * MSTCPY locks from other nodes already which we need to ignore + * when setting the new nodeid. + */ + +static void set_lock_master(struct list_head *queue, int nodeid) +{ + struct dlm_lkb *lkb; + + list_for_each_entry(lkb, queue, lkb_statequeue) { + if (!(lkb->lkb_flags & DLM_IFL_MSTCPY)) { + lkb->lkb_nodeid = nodeid; + lkb->lkb_remid = 0; + } + } +} + +static void set_master_lkbs(struct dlm_rsb *r) +{ + set_lock_master(&r->res_grantqueue, r->res_nodeid); + set_lock_master(&r->res_convertqueue, r->res_nodeid); + set_lock_master(&r->res_waitqueue, r->res_nodeid); +} + +/* + * Propagate the new master nodeid to locks + * The NEW_MASTER flag tells dlm_recover_locks() which rsb's to consider. + * The NEW_MASTER2 flag tells recover_lvb() and recover_grant() which + * rsb's to consider. + */ + +static void set_new_master(struct dlm_rsb *r) +{ + set_master_lkbs(r); + rsb_set_flag(r, RSB_NEW_MASTER); + rsb_set_flag(r, RSB_NEW_MASTER2); +} + +/* + * We do async lookups on rsb's that need new masters. The rsb's + * waiting for a lookup reply are kept on the recover_list. + * + * Another node recovering the master may have sent us a rcom lookup, + * and our dlm_master_lookup() set it as the new master, along with + * NEW_MASTER so that we'll recover it here (this implies dir_nodeid + * equals our_nodeid below). + */ + +static int recover_master(struct dlm_rsb *r, unsigned int *count) +{ + struct dlm_ls *ls = r->res_ls; + int our_nodeid, dir_nodeid; + int is_removed = 0; + int error; + + if (is_master(r)) + return 0; + + is_removed = dlm_is_removed(ls, r->res_nodeid); + + if (!is_removed && !rsb_flag(r, RSB_NEW_MASTER)) + return 0; + + our_nodeid = dlm_our_nodeid(); + dir_nodeid = dlm_dir_nodeid(r); + + if (dir_nodeid == our_nodeid) { + if (is_removed) { + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } + + /* set master of lkbs to ourself when is_removed, or to + another new master which we set along with NEW_MASTER + in dlm_master_lookup */ + set_new_master(r); + error = 0; + } else { + recover_idr_add(r); + error = dlm_send_rcom_lookup(r, dir_nodeid); + } + + (*count)++; + return error; +} + +/* + * All MSTCPY locks are purged and rebuilt, even if the master stayed the same. + * This is necessary because recovery can be started, aborted and restarted, + * causing the master nodeid to briefly change during the aborted recovery, and + * change back to the original value in the second recovery. The MSTCPY locks + * may or may not have been purged during the aborted recovery. Another node + * with an outstanding request in waiters list and a request reply saved in the + * requestqueue, cannot know whether it should ignore the reply and resend the + * request, or accept the reply and complete the request. It must do the + * former if the remote node purged MSTCPY locks, and it must do the later if + * the remote node did not. This is solved by always purging MSTCPY locks, in + * which case, the request reply would always be ignored and the request + * resent. + */ + +static int recover_master_static(struct dlm_rsb *r, unsigned int *count) +{ + int dir_nodeid = dlm_dir_nodeid(r); + int new_master = dir_nodeid; + + if (dir_nodeid == dlm_our_nodeid()) + new_master = 0; + + dlm_purge_mstcpy_locks(r); + r->res_master_nodeid = dir_nodeid; + r->res_nodeid = new_master; + set_new_master(r); + (*count)++; + return 0; +} + +/* + * Go through local root resources and for each rsb which has a master which + * has departed, get the new master nodeid from the directory. The dir will + * assign mastery to the first node to look up the new master. That means + * we'll discover in this lookup if we're the new master of any rsb's. + * + * We fire off all the dir lookup requests individually and asynchronously to + * the correct dir node. + */ + +int dlm_recover_masters(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + unsigned int total = 0; + unsigned int count = 0; + int nodir = dlm_no_directory(ls); + int error; + + log_debug(ls, "dlm_recover_masters"); + + down_read(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + if (dlm_recovery_stopped(ls)) { + up_read(&ls->ls_root_sem); + error = -EINTR; + goto out; + } + + lock_rsb(r); + if (nodir) + error = recover_master_static(r, &count); + else + error = recover_master(r, &count); + unlock_rsb(r); + cond_resched(); + total++; + + if (error) { + up_read(&ls->ls_root_sem); + goto out; + } + } + up_read(&ls->ls_root_sem); + + log_debug(ls, "dlm_recover_masters %u of %u", count, total); + + error = dlm_wait_function(ls, &recover_idr_empty); + out: + if (error) + recover_idr_clear(ls); + return error; +} + +int dlm_recover_master_reply(struct dlm_ls *ls, struct dlm_rcom *rc) +{ + struct dlm_rsb *r; + int ret_nodeid, new_master; + + r = recover_idr_find(ls, rc->rc_id); + if (!r) { + log_error(ls, "dlm_recover_master_reply no id %llx", + (unsigned long long)rc->rc_id); + goto out; + } + + ret_nodeid = rc->rc_result; + + if (ret_nodeid == dlm_our_nodeid()) + new_master = 0; + else + new_master = ret_nodeid; + + lock_rsb(r); + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = new_master; + set_new_master(r); + unlock_rsb(r); + recover_idr_del(r); + + if (recover_idr_empty(ls)) + wake_up(&ls->ls_wait_general); + out: + return 0; +} + + +/* Lock recovery: rebuild the process-copy locks we hold on a + remastered rsb on the new rsb master. + + dlm_recover_locks + recover_locks + recover_locks_queue + dlm_send_rcom_lock -> receive_rcom_lock + dlm_recover_master_copy + receive_rcom_lock_reply <- + dlm_recover_process_copy +*/ + + +/* + * keep a count of the number of lkb's we send to the new master; when we get + * an equal number of replies then recovery for the rsb is done + */ + +static int recover_locks_queue(struct dlm_rsb *r, struct list_head *head) +{ + struct dlm_lkb *lkb; + int error = 0; + + list_for_each_entry(lkb, head, lkb_statequeue) { + error = dlm_send_rcom_lock(r, lkb); + if (error) + break; + r->res_recover_locks_count++; + } + + return error; +} + +static int recover_locks(struct dlm_rsb *r) +{ + int error = 0; + + lock_rsb(r); + + DLM_ASSERT(!r->res_recover_locks_count, dlm_dump_rsb(r);); + + error = recover_locks_queue(r, &r->res_grantqueue); + if (error) + goto out; + error = recover_locks_queue(r, &r->res_convertqueue); + if (error) + goto out; + error = recover_locks_queue(r, &r->res_waitqueue); + if (error) + goto out; + + if (r->res_recover_locks_count) + recover_list_add(r); + else + rsb_clear_flag(r, RSB_NEW_MASTER); + out: + unlock_rsb(r); + return error; +} + +int dlm_recover_locks(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + int error, count = 0; + + down_read(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + if (is_master(r)) { + rsb_clear_flag(r, RSB_NEW_MASTER); + continue; + } + + if (!rsb_flag(r, RSB_NEW_MASTER)) + continue; + + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + up_read(&ls->ls_root_sem); + goto out; + } + + error = recover_locks(r); + if (error) { + up_read(&ls->ls_root_sem); + goto out; + } + + count += r->res_recover_locks_count; + } + up_read(&ls->ls_root_sem); + + log_debug(ls, "dlm_recover_locks %d out", count); + + error = dlm_wait_function(ls, &recover_list_empty); + out: + if (error) + recover_list_clear(ls); + return error; +} + +void dlm_recovered_lock(struct dlm_rsb *r) +{ + DLM_ASSERT(rsb_flag(r, RSB_NEW_MASTER), dlm_dump_rsb(r);); + + r->res_recover_locks_count--; + if (!r->res_recover_locks_count) { + rsb_clear_flag(r, RSB_NEW_MASTER); + recover_list_del(r); + } + + if (recover_list_empty(r->res_ls)) + wake_up(&r->res_ls->ls_wait_general); +} + +/* + * The lvb needs to be recovered on all master rsb's. This includes setting + * the VALNOTVALID flag if necessary, and determining the correct lvb contents + * based on the lvb's of the locks held on the rsb. + * + * RSB_VALNOTVALID is set in two cases: + * + * 1. we are master, but not new, and we purged an EX/PW lock held by a + * failed node (in dlm_recover_purge which set RSB_RECOVER_LVB_INVAL) + * + * 2. we are a new master, and there are only NL/CR locks left. + * (We could probably improve this by only invaliding in this way when + * the previous master left uncleanly. VMS docs mention that.) + * + * The LVB contents are only considered for changing when this is a new master + * of the rsb (NEW_MASTER2). Then, the rsb's lvb is taken from any lkb with + * mode > CR. If no lkb's exist with mode above CR, the lvb contents are taken + * from the lkb with the largest lvb sequence number. + */ + +static void recover_lvb(struct dlm_rsb *r) +{ + struct dlm_lkb *lkb, *high_lkb = NULL; + uint32_t high_seq = 0; + int lock_lvb_exists = 0; + int big_lock_exists = 0; + int lvblen = r->res_ls->ls_lvblen; + + if (!rsb_flag(r, RSB_NEW_MASTER2) && + rsb_flag(r, RSB_RECOVER_LVB_INVAL)) { + /* case 1 above */ + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!rsb_flag(r, RSB_NEW_MASTER2)) + return; + + /* we are the new master, so figure out if VALNOTVALID should + be set, and set the rsb lvb from the best lkb available. */ + + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + continue; + + lock_lvb_exists = 1; + + if (lkb->lkb_grmode > DLM_LOCK_CR) { + big_lock_exists = 1; + goto setflag; + } + + if (((int)lkb->lkb_lvbseq - (int)high_seq) >= 0) { + high_lkb = lkb; + high_seq = lkb->lkb_lvbseq; + } + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + continue; + + lock_lvb_exists = 1; + + if (lkb->lkb_grmode > DLM_LOCK_CR) { + big_lock_exists = 1; + goto setflag; + } + + if (((int)lkb->lkb_lvbseq - (int)high_seq) >= 0) { + high_lkb = lkb; + high_seq = lkb->lkb_lvbseq; + } + } + + setflag: + if (!lock_lvb_exists) + goto out; + + /* lvb is invalidated if only NL/CR locks remain */ + if (!big_lock_exists) + rsb_set_flag(r, RSB_VALNOTVALID); + + if (!r->res_lvbptr) { + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + if (!r->res_lvbptr) + goto out; + } + + if (big_lock_exists) { + r->res_lvbseq = lkb->lkb_lvbseq; + memcpy(r->res_lvbptr, lkb->lkb_lvbptr, lvblen); + } else if (high_lkb) { + r->res_lvbseq = high_lkb->lkb_lvbseq; + memcpy(r->res_lvbptr, high_lkb->lkb_lvbptr, lvblen); + } else { + r->res_lvbseq = 0; + memset(r->res_lvbptr, 0, lvblen); + } + out: + return; +} + +/* All master rsb's flagged RECOVER_CONVERT need to be looked at. The locks + converting PR->CW or CW->PR need to have their lkb_grmode set. */ + +static void recover_conversion(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + struct dlm_lkb *lkb; + int grmode = -1; + + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + if (lkb->lkb_grmode == DLM_LOCK_PR || + lkb->lkb_grmode == DLM_LOCK_CW) { + grmode = lkb->lkb_grmode; + break; + } + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + if (lkb->lkb_grmode != DLM_LOCK_IV) + continue; + if (grmode == -1) { + log_debug(ls, "recover_conversion %x set gr to rq %d", + lkb->lkb_id, lkb->lkb_rqmode); + lkb->lkb_grmode = lkb->lkb_rqmode; + } else { + log_debug(ls, "recover_conversion %x set gr %d", + lkb->lkb_id, grmode); + lkb->lkb_grmode = grmode; + } + } +} + +/* We've become the new master for this rsb and waiting/converting locks may + need to be granted in dlm_recover_grant() due to locks that may have + existed from a removed node. */ + +static void recover_grant(struct dlm_rsb *r) +{ + if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) + rsb_set_flag(r, RSB_RECOVER_GRANT); +} + +void dlm_recover_rsbs(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + unsigned int count = 0; + + down_read(&ls->ls_root_sem); + list_for_each_entry(r, &ls->ls_root_list, res_root_list) { + lock_rsb(r); + if (is_master(r)) { + if (rsb_flag(r, RSB_RECOVER_CONVERT)) + recover_conversion(r); + + /* recover lvb before granting locks so the updated + lvb/VALNOTVALID is presented in the completion */ + recover_lvb(r); + + if (rsb_flag(r, RSB_NEW_MASTER2)) + recover_grant(r); + count++; + } else { + rsb_clear_flag(r, RSB_VALNOTVALID); + } + rsb_clear_flag(r, RSB_RECOVER_CONVERT); + rsb_clear_flag(r, RSB_RECOVER_LVB_INVAL); + rsb_clear_flag(r, RSB_NEW_MASTER2); + unlock_rsb(r); + } + up_read(&ls->ls_root_sem); + + if (count) + log_debug(ls, "dlm_recover_rsbs %d done", count); +} + +/* Create a single list of all root rsb's to be used during recovery */ + +int dlm_create_root_list(struct dlm_ls *ls) +{ + struct rb_node *n; + struct dlm_rsb *r; + int i, error = 0; + + down_write(&ls->ls_root_sem); + if (!list_empty(&ls->ls_root_list)) { + log_error(ls, "root list not empty"); + error = -EINVAL; + goto out; + } + + for (i = 0; i < ls->ls_rsbtbl_size; i++) { + spin_lock(&ls->ls_rsbtbl[i].lock); + for (n = rb_first(&ls->ls_rsbtbl[i].keep); n; n = rb_next(n)) { + r = rb_entry(n, struct dlm_rsb, res_hashnode); + list_add(&r->res_root_list, &ls->ls_root_list); + dlm_hold_rsb(r); + } + + if (!RB_EMPTY_ROOT(&ls->ls_rsbtbl[i].toss)) + log_error(ls, "dlm_create_root_list toss not empty"); + spin_unlock(&ls->ls_rsbtbl[i].lock); + } + out: + up_write(&ls->ls_root_sem); + return error; +} + +void dlm_release_root_list(struct dlm_ls *ls) +{ + struct dlm_rsb *r, *safe; + + down_write(&ls->ls_root_sem); + list_for_each_entry_safe(r, safe, &ls->ls_root_list, res_root_list) { + list_del_init(&r->res_root_list); + dlm_put_rsb(r); + } + up_write(&ls->ls_root_sem); +} + +void dlm_clear_toss(struct dlm_ls *ls) +{ + struct rb_node *n, *next; + struct dlm_rsb *r; + unsigned int count = 0; + int i; + + for (i = 0; i < ls->ls_rsbtbl_size; i++) { + spin_lock(&ls->ls_rsbtbl[i].lock); + for (n = rb_first(&ls->ls_rsbtbl[i].toss); n; n = next) { + next = rb_next(n); + r = rb_entry(n, struct dlm_rsb, res_hashnode); + rb_erase(n, &ls->ls_rsbtbl[i].toss); + dlm_free_rsb(r); + count++; + } + spin_unlock(&ls->ls_rsbtbl[i].lock); + } + + if (count) + log_debug(ls, "dlm_clear_toss %u done", count); +} + diff --git a/kmod/dlm/recover.h b/kmod/dlm/recover.h new file mode 100644 index 00000000..d8c8738c --- /dev/null +++ b/kmod/dlm/recover.h @@ -0,0 +1,34 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RECOVER_DOT_H__ +#define __RECOVER_DOT_H__ + +int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)); +uint32_t dlm_recover_status(struct dlm_ls *ls); +void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status); +int dlm_recover_members_wait(struct dlm_ls *ls); +int dlm_recover_directory_wait(struct dlm_ls *ls); +int dlm_recover_locks_wait(struct dlm_ls *ls); +int dlm_recover_done_wait(struct dlm_ls *ls); +int dlm_recover_masters(struct dlm_ls *ls); +int dlm_recover_master_reply(struct dlm_ls *ls, struct dlm_rcom *rc); +int dlm_recover_locks(struct dlm_ls *ls); +void dlm_recovered_lock(struct dlm_rsb *r); +int dlm_create_root_list(struct dlm_ls *ls); +void dlm_release_root_list(struct dlm_ls *ls); +void dlm_clear_toss(struct dlm_ls *ls); +void dlm_recover_rsbs(struct dlm_ls *ls); + +#endif /* __RECOVER_DOT_H__ */ + diff --git a/kmod/dlm/recoverd.c b/kmod/dlm/recoverd.c new file mode 100644 index 00000000..32f9f892 --- /dev/null +++ b/kmod/dlm/recoverd.c @@ -0,0 +1,342 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "dir.h" +#include "ast.h" +#include "recover.h" +#include "lowcomms.h" +#include "lock.h" +#include "requestqueue.h" +#include "recoverd.h" + + +/* If the start for which we're re-enabling locking (seq) has been superseded + by a newer stop (ls_recover_seq), we need to leave locking disabled. + + We suspend dlm_recv threads here to avoid the race where dlm_recv a) sees + locking stopped and b) adds a message to the requestqueue, but dlm_recoverd + enables locking and clears the requestqueue between a and b. */ + +static int enable_locking(struct dlm_ls *ls, uint64_t seq) +{ + int error = -EINTR; + + down_write(&ls->ls_recv_active); + + spin_lock(&ls->ls_recover_lock); + if (ls->ls_recover_seq == seq) { + set_bit(LSFL_RUNNING, &ls->ls_flags); + /* unblocks processes waiting to enter the dlm */ + up_write(&ls->ls_in_recovery); + clear_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + error = 0; + } + spin_unlock(&ls->ls_recover_lock); + + up_write(&ls->ls_recv_active); + return error; +} + +static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) +{ + unsigned long start; + int error, neg = 0; + + log_debug(ls, "dlm_recover %llu", (unsigned long long)rv->seq); + + mutex_lock(&ls->ls_recoverd_active); + + dlm_callback_suspend(ls); + + dlm_clear_toss(ls); + + /* + * This list of root rsb's will be the basis of most of the recovery + * routines. + */ + + dlm_create_root_list(ls); + + /* + * Add or remove nodes from the lockspace's ls_nodes list. + */ + + error = dlm_recover_members(ls, rv, &neg); + if (error) { + log_debug(ls, "dlm_recover_members error %d", error); + goto fail; + } + + dlm_recover_dir_nodeid(ls); + + ls->ls_recover_dir_sent_res = 0; + ls->ls_recover_dir_sent_msg = 0; + ls->ls_recover_locks_in = 0; + + dlm_set_recover_status(ls, DLM_RS_NODES); + + error = dlm_recover_members_wait(ls); + if (error) { + log_debug(ls, "dlm_recover_members_wait error %d", error); + goto fail; + } + + start = jiffies; + + /* + * Rebuild our own share of the directory by collecting from all other + * nodes their master rsb names that hash to us. + */ + + error = dlm_recover_directory(ls); + if (error) { + log_debug(ls, "dlm_recover_directory error %d", error); + goto fail; + } + + dlm_set_recover_status(ls, DLM_RS_DIR); + + error = dlm_recover_directory_wait(ls); + if (error) { + log_debug(ls, "dlm_recover_directory_wait error %d", error); + goto fail; + } + + log_debug(ls, "dlm_recover_directory %u out %u messages", + ls->ls_recover_dir_sent_res, ls->ls_recover_dir_sent_msg); + + /* + * We may have outstanding operations that are waiting for a reply from + * a failed node. Mark these to be resent after recovery. Unlock and + * cancel ops can just be completed. + */ + + dlm_recover_waiters_pre(ls); + + error = dlm_recovery_stopped(ls); + if (error) + goto fail; + + if (neg || dlm_no_directory(ls)) { + /* + * Clear lkb's for departed nodes. + */ + + dlm_recover_purge(ls); + + /* + * Get new master nodeid's for rsb's that were mastered on + * departed nodes. + */ + + error = dlm_recover_masters(ls); + if (error) { + log_debug(ls, "dlm_recover_masters error %d", error); + goto fail; + } + + /* + * Send our locks on remastered rsb's to the new masters. + */ + + error = dlm_recover_locks(ls); + if (error) { + log_debug(ls, "dlm_recover_locks error %d", error); + goto fail; + } + + dlm_set_recover_status(ls, DLM_RS_LOCKS); + + error = dlm_recover_locks_wait(ls); + if (error) { + log_debug(ls, "dlm_recover_locks_wait error %d", error); + goto fail; + } + + log_debug(ls, "dlm_recover_locks %u in", + ls->ls_recover_locks_in); + + /* + * Finalize state in master rsb's now that all locks can be + * checked. This includes conversion resolution and lvb + * settings. + */ + + dlm_recover_rsbs(ls); + } else { + /* + * Other lockspace members may be going through the "neg" steps + * while also adding us to the lockspace, in which case they'll + * be doing the recover_locks (RS_LOCKS) barrier. + */ + dlm_set_recover_status(ls, DLM_RS_LOCKS); + + error = dlm_recover_locks_wait(ls); + if (error) { + log_debug(ls, "dlm_recover_locks_wait error %d", error); + goto fail; + } + } + + dlm_release_root_list(ls); + + /* + * Purge directory-related requests that are saved in requestqueue. + * All dir requests from before recovery are invalid now due to the dir + * rebuild and will be resent by the requesting nodes. + */ + + dlm_purge_requestqueue(ls); + + dlm_set_recover_status(ls, DLM_RS_DONE); + + error = dlm_recover_done_wait(ls); + if (error) { + log_debug(ls, "dlm_recover_done_wait error %d", error); + goto fail; + } + + dlm_clear_members_gone(ls); + + dlm_adjust_timeouts(ls); + + dlm_callback_resume(ls); + + error = enable_locking(ls, rv->seq); + if (error) { + log_debug(ls, "enable_locking error %d", error); + goto fail; + } + + error = dlm_process_requestqueue(ls); + if (error) { + log_debug(ls, "dlm_process_requestqueue error %d", error); + goto fail; + } + + error = dlm_recover_waiters_post(ls); + if (error) { + log_debug(ls, "dlm_recover_waiters_post error %d", error); + goto fail; + } + + dlm_recover_grant(ls); + + log_debug(ls, "dlm_recover %llu generation %u done: %u ms", + (unsigned long long)rv->seq, ls->ls_generation, + jiffies_to_msecs(jiffies - start)); + mutex_unlock(&ls->ls_recoverd_active); + + dlm_lsop_recover_done(ls); + return 0; + + fail: + dlm_release_root_list(ls); + log_debug(ls, "dlm_recover %llu error %d", + (unsigned long long)rv->seq, error); + mutex_unlock(&ls->ls_recoverd_active); + return error; +} + +/* The dlm_ls_start() that created the rv we take here may already have been + stopped via dlm_ls_stop(); in that case we need to leave the RECOVERY_STOP + flag set. */ + +static void do_ls_recovery(struct dlm_ls *ls) +{ + struct dlm_recover *rv = NULL; + + spin_lock(&ls->ls_recover_lock); + rv = ls->ls_recover_args; + ls->ls_recover_args = NULL; + if (rv && ls->ls_recover_seq == rv->seq) + clear_bit(LSFL_RECOVER_STOP, &ls->ls_flags); + spin_unlock(&ls->ls_recover_lock); + + if (rv) { + ls_recover(ls, rv); + kfree(rv->nodes); + kfree(rv); + } +} + +static int dlm_recoverd(void *arg) +{ + struct dlm_ls *ls; + + ls = dlm_find_lockspace_local(arg); + if (!ls) { + log_print("dlm_recoverd: no lockspace %p", arg); + return -1; + } + + down_write(&ls->ls_in_recovery); + set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + wake_up(&ls->ls_recover_lock_wait); + + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!test_bit(LSFL_RECOVER_WORK, &ls->ls_flags) && + !test_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) + schedule(); + set_current_state(TASK_RUNNING); + + if (test_and_clear_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) { + down_write(&ls->ls_in_recovery); + set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + wake_up(&ls->ls_recover_lock_wait); + } + + if (test_and_clear_bit(LSFL_RECOVER_WORK, &ls->ls_flags)) + do_ls_recovery(ls); + } + + if (test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)) + up_write(&ls->ls_in_recovery); + + dlm_put_lockspace(ls); + return 0; +} + +int dlm_recoverd_start(struct dlm_ls *ls) +{ + struct task_struct *p; + int error = 0; + + p = kthread_run(dlm_recoverd, ls, "dlm_recoverd"); + if (IS_ERR(p)) + error = PTR_ERR(p); + else + ls->ls_recoverd_task = p; + return error; +} + +void dlm_recoverd_stop(struct dlm_ls *ls) +{ + kthread_stop(ls->ls_recoverd_task); +} + +void dlm_recoverd_suspend(struct dlm_ls *ls) +{ + wake_up(&ls->ls_wait_general); + mutex_lock(&ls->ls_recoverd_active); +} + +void dlm_recoverd_resume(struct dlm_ls *ls) +{ + mutex_unlock(&ls->ls_recoverd_active); +} + diff --git a/kmod/dlm/recoverd.h b/kmod/dlm/recoverd.h new file mode 100644 index 00000000..88560797 --- /dev/null +++ b/kmod/dlm/recoverd.h @@ -0,0 +1,23 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RECOVERD_DOT_H__ +#define __RECOVERD_DOT_H__ + +void dlm_recoverd_stop(struct dlm_ls *ls); +int dlm_recoverd_start(struct dlm_ls *ls); +void dlm_recoverd_suspend(struct dlm_ls *ls); +void dlm_recoverd_resume(struct dlm_ls *ls); + +#endif /* __RECOVERD_DOT_H__ */ + diff --git a/kmod/dlm/requestqueue.c b/kmod/dlm/requestqueue.c new file mode 100644 index 00000000..1695f1b0 --- /dev/null +++ b/kmod/dlm/requestqueue.c @@ -0,0 +1,171 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "member.h" +#include "lock.h" +#include "dir.h" +#include "config.h" +#include "requestqueue.h" + +struct rq_entry { + struct list_head list; + uint32_t recover_seq; + int nodeid; + struct dlm_message request; +}; + +/* + * Requests received while the lockspace is in recovery get added to the + * request queue and processed when recovery is complete. This happens when + * the lockspace is suspended on some nodes before it is on others, or the + * lockspace is enabled on some while still suspended on others. + */ + +void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, struct dlm_message *ms) +{ + struct rq_entry *e; + int length = ms->m_header.h_length - sizeof(struct dlm_message); + + e = kmalloc(sizeof(struct rq_entry) + length, GFP_NOFS); + if (!e) { + log_print("dlm_add_requestqueue: out of memory len %d", length); + return; + } + + e->recover_seq = ls->ls_recover_seq & 0xFFFFFFFF; + e->nodeid = nodeid; + memcpy(&e->request, ms, ms->m_header.h_length); + + mutex_lock(&ls->ls_requestqueue_mutex); + list_add_tail(&e->list, &ls->ls_requestqueue); + mutex_unlock(&ls->ls_requestqueue_mutex); +} + +/* + * Called by dlm_recoverd to process normal messages saved while recovery was + * happening. Normal locking has been enabled before this is called. dlm_recv + * upon receiving a message, will wait for all saved messages to be drained + * here before processing the message it got. If a new dlm_ls_stop() arrives + * while we're processing these saved messages, it may block trying to suspend + * dlm_recv if dlm_recv is waiting for us in dlm_wait_requestqueue. In that + * case, we don't abort since locking_stopped is still 0. If dlm_recv is not + * waiting for us, then this processing may be aborted due to locking_stopped. + */ + +int dlm_process_requestqueue(struct dlm_ls *ls) +{ + struct rq_entry *e; + struct dlm_message *ms; + int error = 0; + + mutex_lock(&ls->ls_requestqueue_mutex); + + for (;;) { + if (list_empty(&ls->ls_requestqueue)) { + mutex_unlock(&ls->ls_requestqueue_mutex); + error = 0; + break; + } + e = list_entry(ls->ls_requestqueue.next, struct rq_entry, list); + mutex_unlock(&ls->ls_requestqueue_mutex); + + ms = &e->request; + + log_limit(ls, "dlm_process_requestqueue msg %d from %d " + "lkid %x remid %x result %d seq %u", + ms->m_type, ms->m_header.h_nodeid, + ms->m_lkid, ms->m_remid, ms->m_result, + e->recover_seq); + + dlm_receive_message_saved(ls, &e->request, e->recover_seq); + + mutex_lock(&ls->ls_requestqueue_mutex); + list_del(&e->list); + kfree(e); + + if (dlm_locking_stopped(ls)) { + log_debug(ls, "process_requestqueue abort running"); + mutex_unlock(&ls->ls_requestqueue_mutex); + error = -EINTR; + break; + } + schedule(); + } + + return error; +} + +/* + * After recovery is done, locking is resumed and dlm_recoverd takes all the + * saved requests and processes them as they would have been by dlm_recv. At + * the same time, dlm_recv will start receiving new requests from remote nodes. + * We want to delay dlm_recv processing new requests until dlm_recoverd has + * finished processing the old saved requests. We don't check for locking + * stopped here because dlm_ls_stop won't stop locking until it's suspended us + * (dlm_recv). + */ + +void dlm_wait_requestqueue(struct dlm_ls *ls) +{ + for (;;) { + mutex_lock(&ls->ls_requestqueue_mutex); + if (list_empty(&ls->ls_requestqueue)) + break; + mutex_unlock(&ls->ls_requestqueue_mutex); + schedule(); + } + mutex_unlock(&ls->ls_requestqueue_mutex); +} + +static int purge_request(struct dlm_ls *ls, struct dlm_message *ms, int nodeid) +{ + uint32_t type = ms->m_type; + + /* the ls is being cleaned up and freed by release_lockspace */ + if (!ls->ls_count) + return 1; + + if (dlm_is_removed(ls, nodeid)) + return 1; + + /* directory operations are always purged because the directory is + always rebuilt during recovery and the lookups resent */ + + if (type == DLM_MSG_REMOVE || + type == DLM_MSG_LOOKUP || + type == DLM_MSG_LOOKUP_REPLY) + return 1; + + if (!dlm_no_directory(ls)) + return 0; + + return 1; +} + +void dlm_purge_requestqueue(struct dlm_ls *ls) +{ + struct dlm_message *ms; + struct rq_entry *e, *safe; + + mutex_lock(&ls->ls_requestqueue_mutex); + list_for_each_entry_safe(e, safe, &ls->ls_requestqueue, list) { + ms = &e->request; + + if (purge_request(ls, ms, e->nodeid)) { + list_del(&e->list); + kfree(e); + } + } + mutex_unlock(&ls->ls_requestqueue_mutex); +} + diff --git a/kmod/dlm/requestqueue.h b/kmod/dlm/requestqueue.h new file mode 100644 index 00000000..10ce449b --- /dev/null +++ b/kmod/dlm/requestqueue.h @@ -0,0 +1,22 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __REQUESTQUEUE_DOT_H__ +#define __REQUESTQUEUE_DOT_H__ + +void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, struct dlm_message *ms); +int dlm_process_requestqueue(struct dlm_ls *ls); +void dlm_wait_requestqueue(struct dlm_ls *ls); +void dlm_purge_requestqueue(struct dlm_ls *ls); + +#endif + diff --git a/kmod/dlm/user.c b/kmod/dlm/user.c new file mode 100644 index 00000000..16a96f6f --- /dev/null +++ b/kmod/dlm/user.c @@ -0,0 +1,1028 @@ +/* + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dlm_internal.h" +#include "lockspace.h" +#include "lock.h" +#include "lvb_table.h" +#include "user.h" +#include "ast.h" + +static const char name_prefix[] = "dlm"; +static const struct file_operations device_fops; +static atomic_t dlm_monitor_opened; +static int dlm_monitor_unused = 1; + +#ifdef CONFIG_COMPAT + +struct dlm_lock_params32 { + __u8 mode; + __u8 namelen; + __u16 unused; + __u32 flags; + __u32 lkid; + __u32 parent; + __u64 xid; + __u64 timeout; + __u32 castparam; + __u32 castaddr; + __u32 bastparam; + __u32 bastaddr; + __u32 lksb; + char lvb[DLM_USER_LVB_LEN]; + char name[0]; +}; + +struct dlm_write_request32 { + __u32 version[3]; + __u8 cmd; + __u8 is64bit; + __u8 unused[2]; + + union { + struct dlm_lock_params32 lock; + struct dlm_lspace_params lspace; + struct dlm_purge_params purge; + } i; +}; + +struct dlm_lksb32 { + __u32 sb_status; + __u32 sb_lkid; + __u8 sb_flags; + __u32 sb_lvbptr; +}; + +struct dlm_lock_result32 { + __u32 version[3]; + __u32 length; + __u32 user_astaddr; + __u32 user_astparam; + __u32 user_lksb; + struct dlm_lksb32 lksb; + __u8 bast_mode; + __u8 unused[3]; + /* Offsets may be zero if no data is present */ + __u32 lvb_offset; +}; + +static void compat_input(struct dlm_write_request *kb, + struct dlm_write_request32 *kb32, + int namelen) +{ + kb->version[0] = kb32->version[0]; + kb->version[1] = kb32->version[1]; + kb->version[2] = kb32->version[2]; + + kb->cmd = kb32->cmd; + kb->is64bit = kb32->is64bit; + if (kb->cmd == DLM_USER_CREATE_LOCKSPACE || + kb->cmd == DLM_USER_REMOVE_LOCKSPACE) { + kb->i.lspace.flags = kb32->i.lspace.flags; + kb->i.lspace.minor = kb32->i.lspace.minor; + memcpy(kb->i.lspace.name, kb32->i.lspace.name, namelen); + } else if (kb->cmd == DLM_USER_PURGE) { + kb->i.purge.nodeid = kb32->i.purge.nodeid; + kb->i.purge.pid = kb32->i.purge.pid; + } else { + kb->i.lock.mode = kb32->i.lock.mode; + kb->i.lock.namelen = kb32->i.lock.namelen; + kb->i.lock.flags = kb32->i.lock.flags; + kb->i.lock.lkid = kb32->i.lock.lkid; + kb->i.lock.parent = kb32->i.lock.parent; + kb->i.lock.xid = kb32->i.lock.xid; + kb->i.lock.timeout = kb32->i.lock.timeout; + kb->i.lock.castparam = (void *)(long)kb32->i.lock.castparam; + kb->i.lock.castaddr = (void *)(long)kb32->i.lock.castaddr; + kb->i.lock.bastparam = (void *)(long)kb32->i.lock.bastparam; + kb->i.lock.bastaddr = (void *)(long)kb32->i.lock.bastaddr; + kb->i.lock.lksb = (void *)(long)kb32->i.lock.lksb; + memcpy(kb->i.lock.lvb, kb32->i.lock.lvb, DLM_USER_LVB_LEN); + memcpy(kb->i.lock.name, kb32->i.lock.name, namelen); + } +} + +static void compat_output(struct dlm_lock_result *res, + struct dlm_lock_result32 *res32) +{ + res32->version[0] = res->version[0]; + res32->version[1] = res->version[1]; + res32->version[2] = res->version[2]; + + res32->user_astaddr = (__u32)(long)res->user_astaddr; + res32->user_astparam = (__u32)(long)res->user_astparam; + res32->user_lksb = (__u32)(long)res->user_lksb; + res32->bast_mode = res->bast_mode; + + res32->lvb_offset = res->lvb_offset; + res32->length = res->length; + + res32->lksb.sb_status = res->lksb.sb_status; + res32->lksb.sb_flags = res->lksb.sb_flags; + res32->lksb.sb_lkid = res->lksb.sb_lkid; + res32->lksb.sb_lvbptr = (__u32)(long)res->lksb.sb_lvbptr; +} +#endif + +/* Figure out if this lock is at the end of its life and no longer + available for the application to use. The lkb still exists until + the final ast is read. A lock becomes EOL in three situations: + 1. a noqueue request fails with EAGAIN + 2. an unlock completes with EUNLOCK + 3. a cancel of a waiting request completes with ECANCEL/EDEADLK + An EOL lock needs to be removed from the process's list of locks. + And we can't allow any new operation on an EOL lock. This is + not related to the lifetime of the lkb struct which is managed + entirely by refcount. */ + +static int lkb_is_endoflife(int mode, int status) +{ + switch (status) { + case -DLM_EUNLOCK: + return 1; + case -DLM_ECANCEL: + case -ETIMEDOUT: + case -EDEADLK: + case -EAGAIN: + if (mode == DLM_LOCK_IV) + return 1; + break; + } + return 0; +} + +/* we could possibly check if the cancel of an orphan has resulted in the lkb + being removed and then remove that lkb from the orphans list and free it */ + +void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, uint64_t seq) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + struct dlm_user_proc *proc; + int rv; + + if (lkb->lkb_flags & (DLM_IFL_ORPHAN | DLM_IFL_DEAD)) + return; + + ls = lkb->lkb_resource->res_ls; + mutex_lock(&ls->ls_clear_proc_locks); + + /* If ORPHAN/DEAD flag is set, it means the process is dead so an ast + can't be delivered. For ORPHAN's, dlm_clear_proc_locks() freed + lkb->ua so we can't try to use it. This second check is necessary + for cases where a completion ast is received for an operation that + began before clear_proc_locks did its cancel/unlock. */ + + if (lkb->lkb_flags & (DLM_IFL_ORPHAN | DLM_IFL_DEAD)) + goto out; + + DLM_ASSERT(lkb->lkb_ua, dlm_print_lkb(lkb);); + ua = lkb->lkb_ua; + proc = ua->proc; + + if ((flags & DLM_CB_BAST) && ua->bastaddr == NULL) + goto out; + + if ((flags & DLM_CB_CAST) && lkb_is_endoflife(mode, status)) + lkb->lkb_flags |= DLM_IFL_ENDOFLIFE; + + spin_lock(&proc->asts_spin); + + rv = dlm_add_lkb_callback(lkb, flags, mode, status, sbflags, seq); + if (rv < 0) { + spin_unlock(&proc->asts_spin); + goto out; + } + + if (list_empty(&lkb->lkb_cb_list)) { + kref_get(&lkb->lkb_ref); + list_add_tail(&lkb->lkb_cb_list, &proc->asts); + wake_up_interruptible(&proc->wait); + } + spin_unlock(&proc->asts_spin); + + if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { + /* N.B. spin_lock locks_spin, not asts_spin */ + spin_lock(&proc->locks_spin); + if (!list_empty(&lkb->lkb_ownqueue)) { + list_del_init(&lkb->lkb_ownqueue); + dlm_put_lkb(lkb); + } + spin_unlock(&proc->locks_spin); + } + out: + mutex_unlock(&ls->ls_clear_proc_locks); +} + +static int device_user_lock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + uint32_t lkid; + int error = -ENOMEM; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + if (!params->castaddr || !params->lksb) { + error = -EINVAL; + goto out; + } + + ua = kzalloc(sizeof(struct dlm_user_args), GFP_NOFS); + if (!ua) + goto out; + ua->proc = proc; + ua->user_lksb = params->lksb; + ua->castparam = params->castparam; + ua->castaddr = params->castaddr; + ua->bastparam = params->bastparam; + ua->bastaddr = params->bastaddr; + ua->xid = params->xid; + + if (params->flags & DLM_LKF_CONVERT) { + error = dlm_user_convert(ls, ua, + params->mode, params->flags, + params->lkid, params->lvb, + (unsigned long) params->timeout); + } else if (params->flags & DLM_LKF_ORPHAN) { + error = dlm_user_adopt_orphan(ls, ua, + params->mode, params->flags, + params->name, params->namelen, + (unsigned long) params->timeout, + &lkid); + if (!error) + error = lkid; + } else { + error = dlm_user_request(ls, ua, + params->mode, params->flags, + params->name, params->namelen, + (unsigned long) params->timeout); + if (!error) + error = ua->lksb.sb_lkid; + } + out: + dlm_put_lockspace(ls); + return error; +} + +static int device_user_unlock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + int error = -ENOMEM; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + ua = kzalloc(sizeof(struct dlm_user_args), GFP_NOFS); + if (!ua) + goto out; + ua->proc = proc; + ua->user_lksb = params->lksb; + ua->castparam = params->castparam; + ua->castaddr = params->castaddr; + + if (params->flags & DLM_LKF_CANCEL) + error = dlm_user_cancel(ls, ua, params->flags, params->lkid); + else + error = dlm_user_unlock(ls, ua, params->flags, params->lkid, + params->lvb); + out: + dlm_put_lockspace(ls); + return error; +} + +static int device_user_deadlock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + int error; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + error = dlm_user_deadlock(ls, params->flags, params->lkid); + + dlm_put_lockspace(ls); + return error; +} + +static int dlm_device_register(struct dlm_ls *ls, char *name) +{ + int error, len; + + /* The device is already registered. This happens when the + lockspace is created multiple times from userspace. */ + if (ls->ls_device.name) + return 0; + + error = -ENOMEM; + len = strlen(name) + strlen(name_prefix) + 2; + ls->ls_device.name = kzalloc(len, GFP_NOFS); + if (!ls->ls_device.name) + goto fail; + + snprintf((char *)ls->ls_device.name, len, "%s_%s", name_prefix, + name); + ls->ls_device.fops = &device_fops; + ls->ls_device.minor = MISC_DYNAMIC_MINOR; + + error = misc_register(&ls->ls_device); + if (error) { + kfree(ls->ls_device.name); + } +fail: + return error; +} + +int dlm_device_deregister(struct dlm_ls *ls) +{ + int error; + + /* The device is not registered. This happens when the lockspace + was never used from userspace, or when device_create_lockspace() + calls dlm_release_lockspace() after the register fails. */ + if (!ls->ls_device.name) + return 0; + + error = misc_deregister(&ls->ls_device); + if (!error) + kfree(ls->ls_device.name); + return error; +} + +static int device_user_purge(struct dlm_user_proc *proc, + struct dlm_purge_params *params) +{ + struct dlm_ls *ls; + int error; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + error = dlm_user_purge(ls, proc, params->nodeid, params->pid); + + dlm_put_lockspace(ls); + return error; +} + +static int device_create_lockspace(struct dlm_lspace_params *params) +{ + dlm_lockspace_t *lockspace; + struct dlm_ls *ls; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = dlm_new_lockspace(params->name, NULL, params->flags, + DLM_USER_LVB_LEN, NULL, NULL, NULL, + &lockspace); + if (error) + return error; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -ENOENT; + + error = dlm_device_register(ls, params->name); + dlm_put_lockspace(ls); + + if (error) + dlm_release_lockspace(lockspace, 0); + else + error = ls->ls_device.minor; + + return error; +} + +static int device_remove_lockspace(struct dlm_lspace_params *params) +{ + dlm_lockspace_t *lockspace; + struct dlm_ls *ls; + int error, force = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ls = dlm_find_lockspace_device(params->minor); + if (!ls) + return -ENOENT; + + if (params->flags & DLM_USER_LSFLG_FORCEFREE) + force = 2; + + lockspace = ls->ls_local_handle; + dlm_put_lockspace(ls); + + /* The final dlm_release_lockspace waits for references to go to + zero, so all processes will need to close their device for the + ls before the release will proceed. release also calls the + device_deregister above. Converting a positive return value + from release to zero means that userspace won't know when its + release was the final one, but it shouldn't need to know. */ + + error = dlm_release_lockspace(lockspace, force); + if (error > 0) + error = 0; + return error; +} + +/* Check the user's version matches ours */ +static int check_version(struct dlm_write_request *req) +{ + if (req->version[0] != DLM_DEVICE_VERSION_MAJOR || + (req->version[0] == DLM_DEVICE_VERSION_MAJOR && + req->version[1] > DLM_DEVICE_VERSION_MINOR)) { + + printk(KERN_DEBUG "dlm: process %s (%d) version mismatch " + "user (%d.%d.%d) kernel (%d.%d.%d)\n", + current->comm, + task_pid_nr(current), + req->version[0], + req->version[1], + req->version[2], + DLM_DEVICE_VERSION_MAJOR, + DLM_DEVICE_VERSION_MINOR, + DLM_DEVICE_VERSION_PATCH); + return -EINVAL; + } + return 0; +} + +/* + * device_write + * + * device_user_lock + * dlm_user_request -> request_lock + * dlm_user_convert -> convert_lock + * + * device_user_unlock + * dlm_user_unlock -> unlock_lock + * dlm_user_cancel -> cancel_lock + * + * device_create_lockspace + * dlm_new_lockspace + * + * device_remove_lockspace + * dlm_release_lockspace + */ + +/* a write to a lockspace device is a lock or unlock request, a write + to the control device is to create/remove a lockspace */ + +static ssize_t device_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dlm_user_proc *proc = file->private_data; + struct dlm_write_request *kbuf; + sigset_t tmpsig, allsigs; + int error; + +#ifdef CONFIG_COMPAT + if (count < sizeof(struct dlm_write_request32)) +#else + if (count < sizeof(struct dlm_write_request)) +#endif + return -EINVAL; + + /* + * can't compare against COMPAT/dlm_write_request32 because + * we don't yet know if is64bit is zero + */ + if (count > sizeof(struct dlm_write_request) + DLM_RESNAME_MAXLEN) + return -EINVAL; + + kbuf = kzalloc(count + 1, GFP_NOFS); + if (!kbuf) + return -ENOMEM; + + if (copy_from_user(kbuf, buf, count)) { + error = -EFAULT; + goto out_free; + } + + if (check_version(kbuf)) { + error = -EBADE; + goto out_free; + } + +#ifdef CONFIG_COMPAT + if (!kbuf->is64bit) { + struct dlm_write_request32 *k32buf; + int namelen = 0; + + if (count > sizeof(struct dlm_write_request32)) + namelen = count - sizeof(struct dlm_write_request32); + + k32buf = (struct dlm_write_request32 *)kbuf; + + /* add 1 after namelen so that the name string is terminated */ + kbuf = kzalloc(sizeof(struct dlm_write_request) + namelen + 1, + GFP_NOFS); + if (!kbuf) { + kfree(k32buf); + return -ENOMEM; + } + + if (proc) + set_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags); + + compat_input(kbuf, k32buf, namelen); + kfree(k32buf); + } +#endif + + /* do we really need this? can a write happen after a close? */ + if ((kbuf->cmd == DLM_USER_LOCK || kbuf->cmd == DLM_USER_UNLOCK) && + (proc && test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags))) { + error = -EINVAL; + goto out_free; + } + + sigfillset(&allsigs); + sigprocmask(SIG_BLOCK, &allsigs, &tmpsig); + + error = -EINVAL; + + switch (kbuf->cmd) + { + case DLM_USER_LOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_sig; + } + error = device_user_lock(proc, &kbuf->i.lock); + break; + + case DLM_USER_UNLOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_sig; + } + error = device_user_unlock(proc, &kbuf->i.lock); + break; + + case DLM_USER_DEADLOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_sig; + } + error = device_user_deadlock(proc, &kbuf->i.lock); + break; + + case DLM_USER_CREATE_LOCKSPACE: + if (proc) { + log_print("create/remove only on control device"); + goto out_sig; + } + error = device_create_lockspace(&kbuf->i.lspace); + break; + + case DLM_USER_REMOVE_LOCKSPACE: + if (proc) { + log_print("create/remove only on control device"); + goto out_sig; + } + error = device_remove_lockspace(&kbuf->i.lspace); + break; + + case DLM_USER_PURGE: + if (!proc) { + log_print("no locking on control device"); + goto out_sig; + } + error = device_user_purge(proc, &kbuf->i.purge); + break; + + default: + log_print("Unknown command passed to DLM device : %d\n", + kbuf->cmd); + } + + out_sig: + sigprocmask(SIG_SETMASK, &tmpsig, NULL); + out_free: + kfree(kbuf); + return error; +} + +/* Every process that opens the lockspace device has its own "proc" structure + hanging off the open file that's used to keep track of locks owned by the + process and asts that need to be delivered to the process. */ + +static int device_open(struct inode *inode, struct file *file) +{ + struct dlm_user_proc *proc; + struct dlm_ls *ls; + + ls = dlm_find_lockspace_device(iminor(inode)); + if (!ls) + return -ENOENT; + + proc = kzalloc(sizeof(struct dlm_user_proc), GFP_NOFS); + if (!proc) { + dlm_put_lockspace(ls); + return -ENOMEM; + } + + proc->lockspace = ls->ls_local_handle; + INIT_LIST_HEAD(&proc->asts); + INIT_LIST_HEAD(&proc->locks); + INIT_LIST_HEAD(&proc->unlocking); + spin_lock_init(&proc->asts_spin); + spin_lock_init(&proc->locks_spin); + init_waitqueue_head(&proc->wait); + file->private_data = proc; + + return 0; +} + +static int device_close(struct inode *inode, struct file *file) +{ + struct dlm_user_proc *proc = file->private_data; + struct dlm_ls *ls; + sigset_t tmpsig, allsigs; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + sigfillset(&allsigs); + sigprocmask(SIG_BLOCK, &allsigs, &tmpsig); + + set_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags); + + dlm_clear_proc_locks(ls, proc); + + /* at this point no more lkb's should exist for this lockspace, + so there's no chance of dlm_user_add_ast() being called and + looking for lkb->ua->proc */ + + kfree(proc); + file->private_data = NULL; + + dlm_put_lockspace(ls); + dlm_put_lockspace(ls); /* for the find in device_open() */ + + /* FIXME: AUTOFREE: if this ls is no longer used do + device_remove_lockspace() */ + + sigprocmask(SIG_SETMASK, &tmpsig, NULL); + recalc_sigpending(); + + return 0; +} + +static int copy_result_to_user(struct dlm_user_args *ua, int compat, + uint32_t flags, int mode, int copy_lvb, + char __user *buf, size_t count) +{ +#ifdef CONFIG_COMPAT + struct dlm_lock_result32 result32; +#endif + struct dlm_lock_result result; + void *resultptr; + int error=0; + int len; + int struct_len; + + memset(&result, 0, sizeof(struct dlm_lock_result)); + result.version[0] = DLM_DEVICE_VERSION_MAJOR; + result.version[1] = DLM_DEVICE_VERSION_MINOR; + result.version[2] = DLM_DEVICE_VERSION_PATCH; + memcpy(&result.lksb, &ua->lksb, sizeof(struct dlm_lksb)); + result.user_lksb = ua->user_lksb; + + /* FIXME: dlm1 provides for the user's bastparam/addr to not be updated + in a conversion unless the conversion is successful. See code + in dlm_user_convert() for updating ua from ua_tmp. OpenVMS, though, + notes that a new blocking AST address and parameter are set even if + the conversion fails, so maybe we should just do that. */ + + if (flags & DLM_CB_BAST) { + result.user_astaddr = ua->bastaddr; + result.user_astparam = ua->bastparam; + result.bast_mode = mode; + } else { + result.user_astaddr = ua->castaddr; + result.user_astparam = ua->castparam; + } + +#ifdef CONFIG_COMPAT + if (compat) + len = sizeof(struct dlm_lock_result32); + else +#endif + len = sizeof(struct dlm_lock_result); + struct_len = len; + + /* copy lvb to userspace if there is one, it's been updated, and + the user buffer has space for it */ + + if (copy_lvb && ua->lksb.sb_lvbptr && count >= len + DLM_USER_LVB_LEN) { + if (copy_to_user(buf+len, ua->lksb.sb_lvbptr, + DLM_USER_LVB_LEN)) { + error = -EFAULT; + goto out; + } + + result.lvb_offset = len; + len += DLM_USER_LVB_LEN; + } + + result.length = len; + resultptr = &result; +#ifdef CONFIG_COMPAT + if (compat) { + compat_output(&result, &result32); + resultptr = &result32; + } +#endif + + if (copy_to_user(buf, resultptr, struct_len)) + error = -EFAULT; + else + error = len; + out: + return error; +} + +static int copy_version_to_user(char __user *buf, size_t count) +{ + struct dlm_device_version ver; + + memset(&ver, 0, sizeof(struct dlm_device_version)); + ver.version[0] = DLM_DEVICE_VERSION_MAJOR; + ver.version[1] = DLM_DEVICE_VERSION_MINOR; + ver.version[2] = DLM_DEVICE_VERSION_PATCH; + + if (copy_to_user(buf, &ver, sizeof(struct dlm_device_version))) + return -EFAULT; + return sizeof(struct dlm_device_version); +} + +/* a read returns a single ast described in a struct dlm_lock_result */ + +static ssize_t device_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + struct dlm_user_proc *proc = file->private_data; + struct dlm_lkb *lkb; + DECLARE_WAITQUEUE(wait, current); + struct dlm_callback cb; + int rv, resid, copy_lvb = 0; + + if (count == sizeof(struct dlm_device_version)) { + rv = copy_version_to_user(buf, count); + return rv; + } + + if (!proc) { + log_print("non-version read from control device %zu", count); + return -EINVAL; + } + +#ifdef CONFIG_COMPAT + if (count < sizeof(struct dlm_lock_result32)) +#else + if (count < sizeof(struct dlm_lock_result)) +#endif + return -EINVAL; + + try_another: + + /* do we really need this? can a read happen after a close? */ + if (test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags)) + return -EINVAL; + + spin_lock(&proc->asts_spin); + if (list_empty(&proc->asts)) { + if (file->f_flags & O_NONBLOCK) { + spin_unlock(&proc->asts_spin); + return -EAGAIN; + } + + add_wait_queue(&proc->wait, &wait); + + repeat: + set_current_state(TASK_INTERRUPTIBLE); + if (list_empty(&proc->asts) && !signal_pending(current)) { + spin_unlock(&proc->asts_spin); + schedule(); + spin_lock(&proc->asts_spin); + goto repeat; + } + set_current_state(TASK_RUNNING); + remove_wait_queue(&proc->wait, &wait); + + if (signal_pending(current)) { + spin_unlock(&proc->asts_spin); + return -ERESTARTSYS; + } + } + + /* if we empty lkb_callbacks, we don't want to unlock the spinlock + without removing lkb_cb_list; so empty lkb_cb_list is always + consistent with empty lkb_callbacks */ + + lkb = list_entry(proc->asts.next, struct dlm_lkb, lkb_cb_list); + + rv = dlm_rem_lkb_callback(lkb->lkb_resource->res_ls, lkb, &cb, &resid); + if (rv < 0) { + /* this shouldn't happen; lkb should have been removed from + list when resid was zero */ + log_print("dlm_rem_lkb_callback empty %x", lkb->lkb_id); + list_del_init(&lkb->lkb_cb_list); + spin_unlock(&proc->asts_spin); + /* removes ref for proc->asts, may cause lkb to be freed */ + dlm_put_lkb(lkb); + goto try_another; + } + if (!resid) + list_del_init(&lkb->lkb_cb_list); + spin_unlock(&proc->asts_spin); + + if (cb.flags & DLM_CB_SKIP) { + /* removes ref for proc->asts, may cause lkb to be freed */ + if (!resid) + dlm_put_lkb(lkb); + goto try_another; + } + + if (cb.flags & DLM_CB_CAST) { + int old_mode, new_mode; + + old_mode = lkb->lkb_last_cast.mode; + new_mode = cb.mode; + + if (!cb.sb_status && lkb->lkb_lksb->sb_lvbptr && + dlm_lvb_operations[old_mode + 1][new_mode + 1]) + copy_lvb = 1; + + lkb->lkb_lksb->sb_status = cb.sb_status; + lkb->lkb_lksb->sb_flags = cb.sb_flags; + } + + rv = copy_result_to_user(lkb->lkb_ua, + test_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags), + cb.flags, cb.mode, copy_lvb, buf, count); + + /* removes ref for proc->asts, may cause lkb to be freed */ + if (!resid) + dlm_put_lkb(lkb); + + return rv; +} + +static unsigned int device_poll(struct file *file, poll_table *wait) +{ + struct dlm_user_proc *proc = file->private_data; + + poll_wait(file, &proc->wait, wait); + + spin_lock(&proc->asts_spin); + if (!list_empty(&proc->asts)) { + spin_unlock(&proc->asts_spin); + return POLLIN | POLLRDNORM; + } + spin_unlock(&proc->asts_spin); + return 0; +} + +int dlm_user_daemon_available(void) +{ + /* dlm_controld hasn't started (or, has started, but not + properly populated configfs) */ + + if (!dlm_our_nodeid()) + return 0; + + /* This is to deal with versions of dlm_controld that don't + know about the monitor device. We assume that if the + dlm_controld was started (above), but the monitor device + was never opened, that it's an old version. dlm_controld + should open the monitor device before populating configfs. */ + + if (dlm_monitor_unused) + return 1; + + return atomic_read(&dlm_monitor_opened) ? 1 : 0; +} + +static int ctl_device_open(struct inode *inode, struct file *file) +{ + file->private_data = NULL; + return 0; +} + +static int ctl_device_close(struct inode *inode, struct file *file) +{ + return 0; +} + +static int monitor_device_open(struct inode *inode, struct file *file) +{ + atomic_inc(&dlm_monitor_opened); + dlm_monitor_unused = 0; + return 0; +} + +static int monitor_device_close(struct inode *inode, struct file *file) +{ + if (atomic_dec_and_test(&dlm_monitor_opened)) + dlm_stop_lockspaces(); + return 0; +} + +static const struct file_operations device_fops = { + .open = device_open, + .release = device_close, + .read = device_read, + .write = device_write, + .poll = device_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static const struct file_operations ctl_device_fops = { + .open = ctl_device_open, + .release = ctl_device_close, + .read = device_read, + .write = device_write, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice ctl_device = { + .name = "dlm-control", + .fops = &ctl_device_fops, + .minor = MISC_DYNAMIC_MINOR, +}; + +static const struct file_operations monitor_device_fops = { + .open = monitor_device_open, + .release = monitor_device_close, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice monitor_device = { + .name = "dlm-monitor", + .fops = &monitor_device_fops, + .minor = MISC_DYNAMIC_MINOR, +}; + +int __init dlm_user_init(void) +{ + int error; + + atomic_set(&dlm_monitor_opened, 0); + + error = misc_register(&ctl_device); + if (error) { + log_print("misc_register failed for control device"); + goto out; + } + + error = misc_register(&monitor_device); + if (error) { + log_print("misc_register failed for monitor device"); + misc_deregister(&ctl_device); + } + out: + return error; +} + +void dlm_user_exit(void) +{ + misc_deregister(&ctl_device); + misc_deregister(&monitor_device); +} + diff --git a/kmod/dlm/user.h b/kmod/dlm/user.h new file mode 100644 index 00000000..00499ab8 --- /dev/null +++ b/kmod/dlm/user.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + */ + +#ifndef __USER_DOT_H__ +#define __USER_DOT_H__ + +void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, uint64_t seq); +int dlm_user_init(void); +void dlm_user_exit(void); +int dlm_device_deregister(struct dlm_ls *ls); +int dlm_user_daemon_available(void); + +#endif diff --git a/kmod/dlm/util.c b/kmod/dlm/util.c new file mode 100644 index 00000000..e36520af --- /dev/null +++ b/kmod/dlm/util.c @@ -0,0 +1,154 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "rcom.h" +#include "util.h" + +#define DLM_ERRNO_EDEADLK 35 +#define DLM_ERRNO_EBADR 53 +#define DLM_ERRNO_EBADSLT 57 +#define DLM_ERRNO_EPROTO 71 +#define DLM_ERRNO_EOPNOTSUPP 95 +#define DLM_ERRNO_ETIMEDOUT 110 +#define DLM_ERRNO_EINPROGRESS 115 + +static void header_out(struct dlm_header *hd) +{ + hd->h_version = cpu_to_le32(hd->h_version); + hd->h_lockspace = cpu_to_le32(hd->h_lockspace); + hd->h_nodeid = cpu_to_le32(hd->h_nodeid); + hd->h_length = cpu_to_le16(hd->h_length); +} + +static void header_in(struct dlm_header *hd) +{ + hd->h_version = le32_to_cpu(hd->h_version); + hd->h_lockspace = le32_to_cpu(hd->h_lockspace); + hd->h_nodeid = le32_to_cpu(hd->h_nodeid); + hd->h_length = le16_to_cpu(hd->h_length); +} + +/* higher errno values are inconsistent across architectures, so select + one set of values for on the wire */ + +static int to_dlm_errno(int err) +{ + switch (err) { + case -EDEADLK: + return -DLM_ERRNO_EDEADLK; + case -EBADR: + return -DLM_ERRNO_EBADR; + case -EBADSLT: + return -DLM_ERRNO_EBADSLT; + case -EPROTO: + return -DLM_ERRNO_EPROTO; + case -EOPNOTSUPP: + return -DLM_ERRNO_EOPNOTSUPP; + case -ETIMEDOUT: + return -DLM_ERRNO_ETIMEDOUT; + case -EINPROGRESS: + return -DLM_ERRNO_EINPROGRESS; + } + return err; +} + +static int from_dlm_errno(int err) +{ + switch (err) { + case -DLM_ERRNO_EDEADLK: + return -EDEADLK; + case -DLM_ERRNO_EBADR: + return -EBADR; + case -DLM_ERRNO_EBADSLT: + return -EBADSLT; + case -DLM_ERRNO_EPROTO: + return -EPROTO; + case -DLM_ERRNO_EOPNOTSUPP: + return -EOPNOTSUPP; + case -DLM_ERRNO_ETIMEDOUT: + return -ETIMEDOUT; + case -DLM_ERRNO_EINPROGRESS: + return -EINPROGRESS; + } + return err; +} + +void dlm_message_out(struct dlm_message *ms) +{ + header_out(&ms->m_header); + + ms->m_type = cpu_to_le32(ms->m_type); + ms->m_nodeid = cpu_to_le32(ms->m_nodeid); + ms->m_pid = cpu_to_le32(ms->m_pid); + ms->m_lkid = cpu_to_le32(ms->m_lkid); + ms->m_remid = cpu_to_le32(ms->m_remid); + ms->m_parent_lkid = cpu_to_le32(ms->m_parent_lkid); + ms->m_parent_remid = cpu_to_le32(ms->m_parent_remid); + ms->m_exflags = cpu_to_le32(ms->m_exflags); + ms->m_sbflags = cpu_to_le32(ms->m_sbflags); + ms->m_flags = cpu_to_le32(ms->m_flags); + ms->m_lvbseq = cpu_to_le32(ms->m_lvbseq); + ms->m_hash = cpu_to_le32(ms->m_hash); + ms->m_status = cpu_to_le32(ms->m_status); + ms->m_grmode = cpu_to_le32(ms->m_grmode); + ms->m_rqmode = cpu_to_le32(ms->m_rqmode); + ms->m_bastmode = cpu_to_le32(ms->m_bastmode); + ms->m_asts = cpu_to_le32(ms->m_asts); + ms->m_result = cpu_to_le32(to_dlm_errno(ms->m_result)); +} + +void dlm_message_in(struct dlm_message *ms) +{ + header_in(&ms->m_header); + + ms->m_type = le32_to_cpu(ms->m_type); + ms->m_nodeid = le32_to_cpu(ms->m_nodeid); + ms->m_pid = le32_to_cpu(ms->m_pid); + ms->m_lkid = le32_to_cpu(ms->m_lkid); + ms->m_remid = le32_to_cpu(ms->m_remid); + ms->m_parent_lkid = le32_to_cpu(ms->m_parent_lkid); + ms->m_parent_remid = le32_to_cpu(ms->m_parent_remid); + ms->m_exflags = le32_to_cpu(ms->m_exflags); + ms->m_sbflags = le32_to_cpu(ms->m_sbflags); + ms->m_flags = le32_to_cpu(ms->m_flags); + ms->m_lvbseq = le32_to_cpu(ms->m_lvbseq); + ms->m_hash = le32_to_cpu(ms->m_hash); + ms->m_status = le32_to_cpu(ms->m_status); + ms->m_grmode = le32_to_cpu(ms->m_grmode); + ms->m_rqmode = le32_to_cpu(ms->m_rqmode); + ms->m_bastmode = le32_to_cpu(ms->m_bastmode); + ms->m_asts = le32_to_cpu(ms->m_asts); + ms->m_result = from_dlm_errno(le32_to_cpu(ms->m_result)); +} + +void dlm_rcom_out(struct dlm_rcom *rc) +{ + header_out(&rc->rc_header); + + rc->rc_type = cpu_to_le32(rc->rc_type); + rc->rc_result = cpu_to_le32(rc->rc_result); + rc->rc_id = cpu_to_le64(rc->rc_id); + rc->rc_seq = cpu_to_le64(rc->rc_seq); + rc->rc_seq_reply = cpu_to_le64(rc->rc_seq_reply); +} + +void dlm_rcom_in(struct dlm_rcom *rc) +{ + header_in(&rc->rc_header); + + rc->rc_type = le32_to_cpu(rc->rc_type); + rc->rc_result = le32_to_cpu(rc->rc_result); + rc->rc_id = le64_to_cpu(rc->rc_id); + rc->rc_seq = le64_to_cpu(rc->rc_seq); + rc->rc_seq_reply = le64_to_cpu(rc->rc_seq_reply); +} diff --git a/kmod/dlm/util.h b/kmod/dlm/util.h new file mode 100644 index 00000000..2b259151 --- /dev/null +++ b/kmod/dlm/util.h @@ -0,0 +1,22 @@ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005 Red Hat, Inc. All rights reserved. +** +** This copyrighted material is made available to anyone wishing to use, +** modify, copy, or redistribute it subject to the terms and conditions +** of the GNU General Public License v.2. +** +******************************************************************************* +******************************************************************************/ + +#ifndef __UTIL_DOT_H__ +#define __UTIL_DOT_H__ + +void dlm_message_out(struct dlm_message *ms); +void dlm_message_in(struct dlm_message *ms); +void dlm_rcom_out(struct dlm_rcom *rc); +void dlm_rcom_in(struct dlm_rcom *rc); + +#endif + From 0c1c2691e0e28ff0f31bc68f9894aa056cad31f7 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 8 Jun 2017 16:58:43 -0500 Subject: [PATCH 296/920] interval-tree: Allow user defined objects as endpoints Users pass in a comparison function which is used when endpoints need to be checked against each other. We also put each ITTYPE local definition on it's own line to facilitate the use of pointers. An upcoming dlm patch will make use of this to allow for keyed, ranged locking. Signed-off-by: Mark Fasheh --- kmod/dlm/interval_tree_generic.h | 216 +++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 kmod/dlm/interval_tree_generic.h diff --git a/kmod/dlm/interval_tree_generic.h b/kmod/dlm/interval_tree_generic.h new file mode 100644 index 00000000..d70e20c6 --- /dev/null +++ b/kmod/dlm/interval_tree_generic.h @@ -0,0 +1,216 @@ +/* + Interval Trees + (C) 2012 Michel Lespinasse + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + include/linux/interval_tree_generic.h +*/ + +#include + +#include + +/* + * Template for implementing interval trees + * + * ITSTRUCT: struct type of the interval tree nodes + * ITRB: name of struct rb_node field within ITSTRUCT + * ITTYPE: type of the interval endpoints + * ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree + * ITSTART(n): start endpoint of ITSTRUCT node n + * ITLAST(n): last endpoint of ITSTRUCT node n + * ITSTATIC: 'static' or empty + * ITPREFIX: prefix to use for the inline tree definitions + * + * Note - before using this, please consider if non-generic version + * (interval_tree.h) would work for you... + */ +#define INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \ + ITSTART, ITLAST, ITSTATIC, ITPREFIX) \ + \ +static inline int ITPREFIX ## _cmp(ITTYPE a, ITTYPE b) \ +{ \ + if (a < b) \ + return -1; \ + else if (a > b) \ + return 1; \ + else \ + return 0; \ +} \ +KEYED_INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, ITSTART, ITLAST,\ + ITPREFIX ## _cmp, ITSTATIC, ITPREFIX) + +/* + * int iTCMP(ITTYPE endpoint1, ITTYPE endpoint2); + * Returns: + * < 0 if endpoint1 < endpoint2 + * 0 if endpoint1 == endpoint2 + * > 0 if endpoint1 > endpoint2 + */ +#define KEYED_INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \ + ITSTART, ITLAST, ITCMP, ITSTATIC, ITPREFIX)\ +/* Callbacks for augmented rbtree insert and remove */ \ + \ +static inline ITTYPE ITPREFIX ## _compute_subtree_last(ITSTRUCT *node) \ +{ \ + ITTYPE max = ITLAST(node); \ + ITTYPE subtree_last; \ + \ + if (node->ITRB.rb_left) { \ + subtree_last = rb_entry(node->ITRB.rb_left, \ + ITSTRUCT, ITRB)->ITSUBTREE; \ + if (ITCMP(max, subtree_last) < 0) \ + max = subtree_last; \ + } \ + if (node->ITRB.rb_right) { \ + subtree_last = rb_entry(node->ITRB.rb_right, \ + ITSTRUCT, ITRB)->ITSUBTREE; \ + if (ITCMP(max, subtree_last) < 0) \ + max = subtree_last; \ + } \ + return max; \ +} \ + \ +RB_DECLARE_CALLBACKS(static, ITPREFIX ## _augment, ITSTRUCT, ITRB, \ + ITTYPE, ITSUBTREE, ITPREFIX ## _compute_subtree_last) \ + \ +/* Insert / remove interval nodes from the tree */ \ + \ +ITSTATIC void ITPREFIX ## _insert(ITSTRUCT *node, struct rb_root *root) \ +{ \ + struct rb_node **link = &root->rb_node, *rb_parent = NULL; \ + ITTYPE start = ITSTART(node); \ + ITTYPE last = ITLAST(node); \ + ITSTRUCT *parent; \ + \ + while (*link) { \ + rb_parent = *link; \ + parent = rb_entry(rb_parent, ITSTRUCT, ITRB); \ + if (ITCMP(parent->ITSUBTREE, last) < 0) \ + parent->ITSUBTREE = last; \ + if (ITCMP(start, ITSTART(parent)) < 0) \ + link = &parent->ITRB.rb_left; \ + else \ + link = &parent->ITRB.rb_right; \ + } \ + \ + node->ITSUBTREE = last; \ + rb_link_node(&node->ITRB, rb_parent, link); \ + rb_insert_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \ +} \ + \ +ITSTATIC void ITPREFIX ## _remove(ITSTRUCT *node, struct rb_root *root) \ +{ \ + rb_erase_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \ +} \ + \ +/* \ + * Iterate over intervals intersecting [start;last] \ + * \ + * Note that a node's interval intersects [start;last] iff: \ + * Cond1: ITSTART(node) <= last \ + * and \ + * Cond2: start <= ITLAST(node) \ + */ \ + \ +static ITSTRUCT * \ +ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ +{ \ + while (true) { \ + /* \ + * Loop invariant: start <= node->ITSUBTREE \ + * (Cond2 is satisfied by one of the subtree nodes) \ + */ \ + if (node->ITRB.rb_left) { \ + ITSTRUCT *left = rb_entry(node->ITRB.rb_left, \ + ITSTRUCT, ITRB); \ + if (ITCMP(start, left->ITSUBTREE) <= 0) { \ + /* \ + * Some nodes in left subtree satisfy Cond2. \ + * Iterate to find the leftmost such node N. \ + * If it also satisfies Cond1, that's the \ + * match we are looking for. Otherwise, there \ + * is no matching interval as nodes to the \ + * right of N can't satisfy Cond1 either. \ + */ \ + node = left; \ + continue; \ + } \ + } \ + if (ITCMP(ITSTART(node), last) <= 0) { /* Cond1 */ \ + if (ITCMP(start, ITLAST(node)) <= 0) /* Cond2 */ \ + return node; /* node is leftmost match */ \ + if (node->ITRB.rb_right) { \ + node = rb_entry(node->ITRB.rb_right, \ + ITSTRUCT, ITRB); \ + if (ITCMP(start, node->ITSUBTREE) <= 0) \ + continue; \ + } \ + } \ + return NULL; /* No match */ \ + } \ +} \ + \ +ITSTATIC ITSTRUCT * \ +ITPREFIX ## _iter_first(struct rb_root *root, ITTYPE start, ITTYPE last) \ +{ \ + ITSTRUCT *node; \ + \ + if (!root->rb_node) \ + return NULL; \ + node = rb_entry(root->rb_node, ITSTRUCT, ITRB); \ + if (ITCMP(node->ITSUBTREE, start) < 0) \ + return NULL; \ + return ITPREFIX ## _subtree_search(node, start, last); \ +} \ + \ +ITSTATIC ITSTRUCT * \ +ITPREFIX ## _iter_next(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ +{ \ + struct rb_node *rb = node->ITRB.rb_right, *prev; \ + \ + while (true) { \ + /* \ + * Loop invariants: \ + * Cond1: ITSTART(node) <= last \ + * rb == node->ITRB.rb_right \ + * \ + * First, search right subtree if suitable \ + */ \ + if (rb) { \ + ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB); \ + if (ITCMP(start, right->ITSUBTREE) <= 0) \ + return ITPREFIX ## _subtree_search(right, \ + start, last); \ + } \ + \ + /* Move up the tree until we come from a node's left child */ \ + do { \ + rb = rb_parent(&node->ITRB); \ + if (!rb) \ + return NULL; \ + prev = &node->ITRB; \ + node = rb_entry(rb, ITSTRUCT, ITRB); \ + rb = node->ITRB.rb_right; \ + } while (prev == rb); \ + \ + /* Check if the node intersects [start;last] */ \ + if (ITCMP(last, ITSTART(node)) < 0) /* !Cond1 */ \ + return NULL; \ + else if (ITCMP(start, ITLAST(node)) <= 0) /* Cond2 */ \ + return node; \ + } \ +} From 08bf1fea7956de68255468864e38d57d1ec1b393 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 10 Mar 2017 15:15:09 -0600 Subject: [PATCH 297/920] dlm: Give fs/dlm the notion of ranges Using the new interval tree code we add a tree for each lock status list to efficiently track ranged requests. Internally, most operations on a resources lock status list (granted, waiting, converting) then are turned into operations within a given range. There is no API change other than a new call, dlm_lock_range() and a new structure, 'struct dlm_key' to define our range endpoints. Keys can have arbitrary lengths and are compared via memcmp. A ranged blocking ast type is defined so that users of dlm_lock_range() can know which range they are blocking. A rudimentary test, dlmtest.ko is included. TODO: - Update userspace entry points, need to add one for new lock call - Manage backwards compatibility with network protocol Signed-off-by: Mark Fasheh --- kmod/dlm/Makefile | 4 +- kmod/dlm/ast.c | 77 ++++++- kmod/dlm/ast.h | 7 +- kmod/dlm/config.c | 2 +- kmod/dlm/dlm_internal.h | 51 ++++- kmod/dlm/dlmtest.c | 307 ++++++++++++++++++++++++++ kmod/dlm/include/linux/dlm.h | 21 ++ kmod/dlm/lock.c | 405 ++++++++++++++++++++++++++++++----- kmod/dlm/lock.h | 2 + kmod/dlm/lockspace.c | 2 +- kmod/dlm/main.c | 1 + kmod/dlm/netlink.c | 2 +- kmod/dlm/plock.c | 2 +- kmod/dlm/rcom.c | 2 +- kmod/dlm/user.c | 4 +- kmod/dlm/util.c | 18 ++ 16 files changed, 831 insertions(+), 76 deletions(-) create mode 100644 kmod/dlm/dlmtest.c diff --git a/kmod/dlm/Makefile b/kmod/dlm/Makefile index c43e89cd..5710f265 100644 --- a/kmod/dlm/Makefile +++ b/kmod/dlm/Makefile @@ -1,4 +1,4 @@ -obj-$(CONFIG_DLM) += dlm.o +obj-$(CONFIG_DLM) += dlm.o dlmtest.o dlm-y := ast.o \ config.o \ dir.o \ @@ -16,5 +16,5 @@ dlm-y := ast.o \ recoverd.o \ requestqueue.o \ user.o \ - util.o + util.o dlm-$(CONFIG_DLM_DEBUG) += debug_fs.o diff --git a/kmod/dlm/ast.c b/kmod/dlm/ast.c index 27a6ba9a..19630d43 100644 --- a/kmod/dlm/ast.c +++ b/kmod/dlm/ast.c @@ -49,13 +49,35 @@ static void dlm_dump_lkb_callbacks(struct dlm_lkb *lkb) } } +static void fixup_cb_pointers(struct dlm_callback *cb) +{ + struct dlm_key *start = &cb->start; + struct dlm_key *end = &cb->end; + + cb->range.start = start; + cb->range.end = end; + start->val = &cb->startval; + end->val = &cb->endval; +} + +/* + * Range must not be NULL for DLM_CB_BAST. + */ int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, - int status, uint32_t sbflags, uint64_t seq) + struct dlm_range *range, int status, uint32_t sbflags, + uint64_t seq) { struct dlm_ls *ls = lkb->lkb_resource->res_ls; uint64_t prev_seq; int prev_mode; int i, rv; + struct dlm_range *prev_range; + + if ((flags & DLM_CB_BAST) && !range) { + /* XXX: user.c doesn't handle this yet, fail for now */ + WARN_ON_ONCE(1); + return -EINVAL; + } for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { if (lkb->lkb_callbacks[i].seq) @@ -73,9 +95,12 @@ int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, prev_seq = lkb->lkb_callbacks[i-1].seq; prev_mode = lkb->lkb_callbacks[i-1].mode; + prev_range = &lkb->lkb_callbacks[i-1].range; - if ((prev_mode == mode) || - (prev_mode > mode && prev_mode > DLM_LOCK_PR)) { + /* Below check needs to look at range */ + if (ranges_overlap(prev_range, range) && + ((prev_mode == mode) || + (prev_mode > mode && prev_mode > DLM_LOCK_PR))) { log_debug(ls, "skip %x add bast %llu mode %d " "for bast %llu mode %d", @@ -94,6 +119,21 @@ int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, lkb->lkb_callbacks[i].mode = mode; lkb->lkb_callbacks[i].sb_status = status; lkb->lkb_callbacks[i].sb_flags = (sbflags & 0x000000FF); + + if (range) { + struct dlm_key *start = &lkb->lkb_callbacks[i].start; + struct dlm_key *end = &lkb->lkb_callbacks[i].end; + + lkb->lkb_callbacks[i].range.start = start; + lkb->lkb_callbacks[i].range.end = end; + + start->len = range->start->len; + start->val = &lkb->lkb_callbacks[i].startval; + end->len = range->end->len; + end->val = &lkb->lkb_callbacks[i].endval; + memcpy(start->val, range->start->val, range->start->len); + memcpy(end->val, range->end->val, range->end->len); + } rv = 0; break; } @@ -126,6 +166,7 @@ int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, memcpy(cb, &lkb->lkb_callbacks[0], sizeof(struct dlm_callback)); memset(&lkb->lkb_callbacks[0], 0, sizeof(struct dlm_callback)); + fixup_cb_pointers(cb); /* shift others down */ @@ -171,8 +212,8 @@ int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, return rv; } -void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, - uint32_t sbflags) +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, + struct dlm_range *range, int status, uint32_t sbflags) { struct dlm_ls *ls = lkb->lkb_resource->res_ls; uint64_t new_seq, prev_seq; @@ -190,7 +231,8 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, mutex_lock(&lkb->lkb_cb_mutex); prev_seq = lkb->lkb_callbacks[0].seq; - rv = dlm_add_lkb_callback(lkb, flags, mode, status, sbflags, new_seq); + rv = dlm_add_lkb_callback(lkb, flags, mode, range, status, sbflags, + new_seq); if (rv < 0) goto out; @@ -215,10 +257,21 @@ void dlm_callback_work(struct work_struct *work) struct dlm_ls *ls = lkb->lkb_resource->res_ls; void (*castfn) (void *astparam); void (*bastfn) (void *astparam, int mode); - struct dlm_callback callbacks[DLM_CALLBACKS_SIZE]; + void (*rbastfn) (void *astarg, int mode, struct dlm_key *start, + struct dlm_key *end); + /* + * XXX: This used to be on the stack, but the inline buffers + * added for range support blow out our stack. + * + * struct dlm_callback callbacks[DLM_CALLBACKS_SIZE]; + */ + struct dlm_callback *callbacks; int i, rv, resid; - memset(&callbacks, 0, sizeof(callbacks)); + callbacks = kcalloc(DLM_CALLBACKS_SIZE, sizeof(*callbacks), GFP_NOFS); + WARN_ON_ONCE(!callbacks); + if (!callbacks) + return; mutex_lock(&lkb->lkb_cb_mutex); if (!lkb->lkb_callbacks[0].seq) { @@ -245,6 +298,7 @@ void dlm_callback_work(struct work_struct *work) castfn = lkb->lkb_astfn; bastfn = lkb->lkb_bastfn; + rbastfn = lkb->lkb_rbastfn; for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { if (!callbacks[i].seq) @@ -252,7 +306,11 @@ void dlm_callback_work(struct work_struct *work) if (callbacks[i].flags & DLM_CB_SKIP) { continue; } else if (callbacks[i].flags & DLM_CB_BAST) { - bastfn(lkb->lkb_astparam, callbacks[i].mode); + if (rbastfn) + rbastfn(lkb->lkb_astparam, callbacks[i].mode, + &callbacks[i].start, &callbacks[i].end); + else + bastfn(lkb->lkb_astparam, callbacks[i].mode); } else if (callbacks[i].flags & DLM_CB_CAST) { lkb->lkb_lksb->sb_status = callbacks[i].sb_status; lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags; @@ -262,6 +320,7 @@ void dlm_callback_work(struct work_struct *work) /* undo kref_get from dlm_add_callback, may cause lkb to be freed */ dlm_put_lkb(lkb); + kfree(callbacks); } int dlm_callback_start(struct dlm_ls *ls) diff --git a/kmod/dlm/ast.h b/kmod/dlm/ast.h index 757b551c..09fc3934 100644 --- a/kmod/dlm/ast.h +++ b/kmod/dlm/ast.h @@ -15,11 +15,12 @@ void dlm_del_ast(struct dlm_lkb *lkb); int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, - int status, uint32_t sbflags, uint64_t seq); + struct dlm_range *range, int status, uint32_t sbflags, + uint64_t seq); int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, struct dlm_callback *cb, int *resid); -void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, - uint32_t sbflags); +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, + struct dlm_range *range, int status, uint32_t sbflags); void dlm_callback_work(struct work_struct *work); int dlm_callback_start(struct dlm_ls *ls); diff --git a/kmod/dlm/config.c b/kmod/dlm/config.c index 7d58d5b1..a6662120 100644 --- a/kmod/dlm/config.c +++ b/kmod/dlm/config.c @@ -999,7 +999,7 @@ int dlm_our_addr(struct sockaddr_storage *addr, int num) #define DEFAULT_RECOVER_TIMER 5 #define DEFAULT_TOSS_SECS 10 #define DEFAULT_SCAN_SECS 5 -#define DEFAULT_LOG_DEBUG 0 +#define DEFAULT_LOG_DEBUG 1 #define DEFAULT_PROTOCOL 0 #define DEFAULT_TIMEWARN_CS 500 /* 5 sec = 500 centiseconds */ #define DEFAULT_WAITWARN_US 0 diff --git a/kmod/dlm/dlm_internal.h b/kmod/dlm/dlm_internal.h index e7665c31..b4a91e4c 100644 --- a/kmod/dlm/dlm_internal.h +++ b/kmod/dlm/dlm_internal.h @@ -40,8 +40,9 @@ #include #include #include +#include -#include +#include "include/linux/dlm.h" #include "config.h" /* Size of the temp buffer midcomms allocates on the stack. @@ -95,6 +96,10 @@ do { \ } \ } +struct dlm_range { + struct dlm_key *start; + struct dlm_key *end; +}; #define DLM_RTF_SHRINK 0x00000001 @@ -140,7 +145,11 @@ struct dlm_args { void (*astfn) (void *astparam); void *astparam; void (*bastfn) (void *astparam, int mode); + void (*rbastfn) (void *astarg, int mode, + struct dlm_key *start, + struct dlm_key *end); int mode; + struct dlm_range range; struct dlm_lksb *lksb; unsigned long timeout; }; @@ -213,12 +222,22 @@ struct dlm_args { #define DLM_CB_BAST 0x00000002 #define DLM_CB_SKIP 0x00000004 + +#define DLM_KEY_LEN 296 + struct dlm_callback { uint64_t seq; uint32_t flags; /* DLM_CBF_ */ int sb_status; /* copy to lksb status */ uint8_t sb_flags; /* copy to lksb flags */ int8_t mode; /* rq mode of bast, gr mode of cast */ + struct dlm_range range; + + /* XXX: This should be dynamically allocated */ + struct dlm_key start; + struct dlm_key end; + char startval[DLM_KEY_LEN]; + char endval[DLM_KEY_LEN]; }; struct dlm_lkb { @@ -237,12 +256,18 @@ struct dlm_lkb { int8_t lkb_rqmode; /* requested lock mode */ int8_t lkb_grmode; /* granted lock mode */ int8_t lkb_highbast; /* highest mode bast sent for */ + /* XXX: Keep some history of bast ranges here? */ + + struct dlm_range lkb_rqrange; + struct dlm_range lkb_grrange; int8_t lkb_wait_type; /* type of reply waiting for */ int8_t lkb_wait_count; int lkb_wait_nodeid; /* for debugging */ struct list_head lkb_statequeue; /* rsb g/c/w list */ + struct rb_node lkb_statenode; /* rsb g/c/w interval tree */ + struct dlm_key *lkb_subtree_last; /* rsb g/c/w interval tree */ struct list_head lkb_rsb_lookup; /* waiting for rsb lookup */ struct list_head lkb_wait_reply; /* waiting for remote reply */ struct list_head lkb_ownqueue; /* list of locks for a process */ @@ -266,6 +291,9 @@ struct dlm_lkb { struct dlm_lksb *lkb_lksb; /* caller's status block */ void (*lkb_astfn) (void *astparam); void (*lkb_bastfn) (void *astparam, int mode); + void (*lkb_rbastfn) (void *astparam, int mode, + struct dlm_key *start, + struct dlm_key *end); union { void *lkb_astparam; /* caller's ast arg */ struct dlm_user_args *lkb_ua; @@ -303,7 +331,9 @@ struct dlm_rsb { struct rb_node res_hashnode; /* rsbtbl */ }; struct list_head res_grantqueue; + struct rb_root res_grantroot; struct list_head res_convertqueue; + struct rb_root res_convertroot; struct list_head res_waitqueue; struct list_head res_root_list; /* used for recovery */ @@ -414,6 +444,25 @@ struct dlm_message { int m_bastmode; int m_asts; int m_result; /* 0 or -EXXX */ + /* + * XXX: These should start *after* m_extra to preserve + * compatibility with the old message format + */ + char m_grstart[DLM_KEY_LEN]; + char m_grend[DLM_KEY_LEN]; + uint16_t m_grstart_len; + uint16_t m_grend_len; + + char m_rqstart[DLM_KEY_LEN]; + char m_rqend[DLM_KEY_LEN]; + uint16_t m_rqstart_len; + uint16_t m_rqend_len; + + char m_baststart[DLM_KEY_LEN]; + char m_bastend[DLM_KEY_LEN]; + uint16_t m_baststart_len; + uint16_t m_bastend_len; + char m_extra[0]; /* name or lvb */ }; diff --git a/kmod/dlm/dlmtest.c b/kmod/dlm/dlmtest.c new file mode 100644 index 00000000..4c7d72fc --- /dev/null +++ b/kmod/dlm/dlmtest.c @@ -0,0 +1,307 @@ +#include +#include +#include +#include + +#include "include/linux/dlm.h" + +static atomic_t granted; +static atomic_t blocking; +static int val = 1; +static dlm_lockspace_t *ls; +static char *lockres_name = "test_resource"; + +struct lockinfo { + char *lockname; + int unlocking; + u64 start; + u64 end; + struct dlm_key startkey; + struct dlm_key endkey; + struct dlm_lksb lksb; +}; + +static inline void set_lock_endpoints(struct lockinfo *lock, u64 start, u64 end) +{ + lock->start = cpu_to_be64(start); + lock->startkey.val = &lock->start; + lock->startkey.len = sizeof(lock->start); + lock->end = cpu_to_be64(end); + lock->endkey.val = &lock->end; + lock->endkey.len = sizeof(lock->end); +} + +#define NUM_LOCKS 3 +static struct lockinfo locks[NUM_LOCKS] = { + { "lock0", }, + { "lock1", }, + { "lock2", }, +}; + +static int glbl_exmode = 0; +module_param(glbl_exmode, int, 0); +MODULE_PARM_DESC(glbl_exmode, "Take global lock exclusively."); + +static void init_counters(void) +{ + atomic_set(&granted, 0); + atomic_set(&blocking, 0); + val = 1; +} + +static void wait_for_blocking_asts(int count) +{ + printk("wait for %d blocking asts\n", count); + while (atomic_read(&blocking) != count) { + printk("blocking: %d\n", atomic_read(&blocking)); + msleep_interruptible(2000); + } +} + +static void wait_for_lock_grants(int count) +{ + printk("wait for %d grants\n", count); + while (atomic_read(&granted) != count) { + printk("granted: %d\n", atomic_read(&granted)); + msleep_interruptible(2000); + } +} + +static void grant_function(void *arg) +{ + char *name = arg; + printk("lock %s granted\n", name); + atomic_add(val, &granted); +} + +static void blocking_function(void *arg, int mode, struct dlm_key *start, + struct dlm_key *end) +{ + char *name = arg; + BUG_ON(!start); + BUG_ON(!end); + printk("lock %s blocking mode %d, range (%llu, %llu)\n", name, mode, + be64_to_cpu(*((u64 *)start->val)), be64_to_cpu(*((u64 *)end->val))); + atomic_inc(&blocking); +} + +static int _test_lock(unsigned int lockidx, unsigned int mode, + unsigned long long start, unsigned long long end, + unsigned int flags) +{ + struct lockinfo *lock = &locks[lockidx]; + + BUG_ON(lockidx > NUM_LOCKS); + + set_lock_endpoints(lock, start, end); + + printk("lock %s (%u, %llu, %llu)\n", lock->lockname, mode, start, end); + return dlm_lock_range(ls, mode, &lock->startkey, &lock->endkey, + &lock->lksb, flags, lockres_name, + strlen(lockres_name), 0, grant_function, + lock->lockname, blocking_function); +} + +static inline int test_lock(unsigned int lockidx, unsigned int mode, + unsigned long long start, unsigned long long end) +{ + return _test_lock(lockidx, mode, start, end, 0); +} + +static inline int test_convert(unsigned int lockidx, unsigned int mode, + unsigned long long start, + unsigned long long end) +{ + return _test_lock(lockidx, mode, start, end, DLM_LKF_CONVERT); +} + +static int test_unlock(int lockidx) +{ + struct lockinfo *lock = &locks[lockidx]; + + printk("unlock %s (%llu, %llu)\n", lock->lockname, + be64_to_cpu(lock->start), be64_to_cpu(lock->end)); + return dlm_unlock(ls, lock->lksb.sb_lkid, 0, &lock->lksb, lock->lockname); +} + +static int test_locking(void) +{ + int ret; + + printk("Test basic lock/unlock.\n"); + + init_counters(); + + ret = test_lock(0, DLM_LOCK_EX, 0, 16384); + if (ret) + goto out; + + ret = test_lock(1, DLM_LOCK_EX, 16385, 32768); + if (ret) + goto out; + + wait_for_lock_grants(2); + + ret = test_lock(2, DLM_LOCK_EX, 0, 32768); + if (ret) + goto out; + + wait_for_blocking_asts(2); + + val = -1; + + ret = test_unlock(0); + if (ret) + goto out; + ret = test_unlock(1); + if (ret) + goto out; + + wait_for_lock_grants(-1); + + ret = test_unlock(2); + if (ret) + goto out; + + wait_for_lock_grants(-2); + +out: + return ret; +} + +static int test_lock_conversions(void) +{ + int ret; + + printk("Test lock conversions\n"); + + init_counters(); + + ret = test_lock(0, DLM_LOCK_PR, 0, 16384); + if (ret) + goto out; + + ret = test_lock(1, DLM_LOCK_PR, 16385, 32768); + if (ret) + goto out; + + ret = test_lock(2, DLM_LOCK_PR, 0, 32768); + if (ret) + goto out; + + wait_for_lock_grants(3); + + ret = test_convert(0, DLM_LOCK_EX, 0, 16384); + if (ret) + goto out; + + wait_for_blocking_asts(2); + + init_counters(); + + ret = test_convert(1, DLM_LOCK_NL, 16385, 32768); + if (ret) + goto out; + ret = test_convert(2, DLM_LOCK_NL, 0, 32768); + if (ret) + goto out; + + wait_for_lock_grants(3); + + init_counters(); + + ret = test_unlock(1); + if (ret) + goto out; + ret = test_unlock(2); + if (ret) + goto out; + + ret = test_unlock(0); + if (ret) + goto out; + + wait_for_lock_grants(3); + +out: + return ret; +} + +#define glbl_res "global" +#define glbl_res_len strlen(glbl_res) +static atomic_t glbl_grants; +static struct dlm_lksb glbl_lksb; + +static void glbl_granted(void *arg) +{ + printk("Got global lock at %d mode\n", *((int *) arg)); + atomic_set(&glbl_grants, 1); +} + +static void glbl_blocking(void *arg, int mode) +{ + printk("Global lock at %d mode blocking %d lock\n", *((int *)arg), mode); +} + +static int test_multinode(void) +{ + int ret; + int mode = DLM_LOCK_EX; + + printk("Test a global lock\n"); + + ret = dlm_lock(ls, mode, &glbl_lksb, 0, glbl_res, glbl_res_len, 0, + glbl_granted, &mode, glbl_blocking); + if (ret) + return ret; + + while (!atomic_read(&glbl_grants)) + msleep_interruptible(5000); + + mode = 0; + ret = dlm_unlock(ls, glbl_lksb.sb_lkid, 0, &glbl_lksb, &mode); + return ret; +} + +static int __init init_dlm_test(void) +{ + int ret; + + printk("dlmtest loaded!\n"); + + ret = dlm_new_lockspace("lockspace", "scoutfs", + DLM_LSFL_FS|DLM_LSFL_NEWEXCL, 8, NULL, NULL, + NULL, &ls); + if (ret) { + printk("new_lockspace returns %d\n", ret); + return ret; + } + + ret = test_multinode(); + if (!ret) + ret = test_locking(); + if (!ret) + ret = test_lock_conversions(); + + if (ret) + printk("FAILURE: Locking test returns %d\n", ret); + else + printk("Locking test completed with no errors.\n"); + + return 0; +} + +static void __exit exit_dlm_test(void) +{ + int ret; + + ret = dlm_release_lockspace(ls, 1); + printk("dlmtest unloaded (ret=%d)!\n", ret); +} + +module_init(init_dlm_test); +module_exit(exit_dlm_test); + +MODULE_DESCRIPTION("dlmtest"); +MODULE_AUTHOR("Mark Fasheh"); +MODULE_LICENSE("GPL"); diff --git a/kmod/dlm/include/linux/dlm.h b/kmod/dlm/include/linux/dlm.h index d02da2c6..cbf05678 100644 --- a/kmod/dlm/include/linux/dlm.h +++ b/kmod/dlm/include/linux/dlm.h @@ -169,4 +169,25 @@ int dlm_unlock(dlm_lockspace_t *lockspace, struct dlm_lksb *lksb, void *astarg); +struct dlm_key { + void *val; + int len; +}; + +int dlm_lock_range(dlm_lockspace_t *lockspace, + int mode, + struct dlm_key *start, + struct dlm_key *end, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*lockast) (void *astarg), + void *astarg, + void (*rbast) (void *astarg, int mode, + struct dlm_key *start, struct dlm_key *end)); + +#define dlm_unlock_range dlm_unlock + #endif /* __DLM_DOT_H__ */ diff --git a/kmod/dlm/lock.c b/kmod/dlm/lock.c index 275e171c..d62127fe 100644 --- a/kmod/dlm/lock.c +++ b/kmod/dlm/lock.c @@ -60,6 +60,7 @@ #include #include "dlm_internal.h" #include +#include "interval_tree_generic.h" #include "memory.h" #include "lowcomms.h" #include "requestqueue.h" @@ -92,6 +93,7 @@ static void do_purge(struct dlm_ls *ls, int nodeid, int pid); static void del_timeout(struct dlm_lkb *lkb); static void toss_rsb(struct kref *kref); + /* * Lock compatibilty matrix - thanks Steve * UN = Unlocked state. Not really a state, used as a flag @@ -133,11 +135,106 @@ const int dlm_lvb_operations[8][8] = { { -1, 0, 0, 0, 0, 0, 0, 0 } /* PD */ }; -#define modes_compat(gr, rq) \ +#define _modes_compat(gr, rq) \ __dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1] +/* + * Define start and end for a range that covers all possible + * values. Use these as defaults (instead of NULL pointers) for lkbs + * created by non ranged lock requests. Without these we'd have to + * implement switches or alternative algorithms each time a NULL key + * was encountered. + */ +static unsigned long long default_start = 0ULL; +#define default_start_len sizeof(default_start) +static struct dlm_key default_start_key = { &default_start, + default_start_len }; +static char default_end[DLM_KEY_LEN] = { [ 0 ... (DLM_KEY_LEN-1) ] = -1 }; +#define default_end_len DLM_KEY_LEN +static struct dlm_key default_end_key = { default_end, default_end_len }; +static struct dlm_range default_range = { .start = &default_start_key, + .end = &default_end_key }; + +/* Fast debug printing */ +#define debug_range_to_ull(ENDPOINT) \ +static inline u64 ENDPOINT ## _to_ull(struct dlm_range *range) \ +{ \ + u64 val = 0ULL; \ + \ + if (range->ENDPOINT && range->ENDPOINT->len > sizeof(u64)) { \ + memcpy(&val, range->ENDPOINT->val, min((int)sizeof(val),\ + range->ENDPOINT->len)); \ + return val; \ + } \ + return 0; \ +} +debug_range_to_ull(start); +debug_range_to_ull(end); + +static struct dlm_key *alloc_key(char *val, int len, gfp_t gfp) +{ + struct dlm_key *ret = kmalloc(sizeof(*ret), gfp); + if (ret) { + ret->len = len; + ret->val = kmalloc(len, gfp); + if (!ret->val) { + kfree(ret); + return NULL; + } + memcpy(ret->val, val, len); + } + return ret; +} + +static int cmp_range_keys(char *a, int a_len, char *b, int b_len) +{ + return memcmp(a, b, min(a_len, b_len)) ?: + a_len < b_len ? -1 : a_len > b_len ? 1 : 0; +} + +static inline int cmp_dlm_keys(struct dlm_key *a, struct dlm_key *b) +{ + return cmp_range_keys(a->val, a->len, b->val, b->len); +} + +/* + * Define our interval tree nodes to index by granted start/end + * values. We might want a tree sorted by requested start/end in the + * future if walking the converting list winds up being costly. + */ +#define START(lkb) ((lkb)->lkb_grrange.start) +#define LAST(lkb) ((lkb)->lkb_grrange.end) +KEYED_INTERVAL_TREE_DEFINE(struct dlm_lkb, lkb_statenode, struct dlm_key *, + lkb_subtree_last, START, LAST, cmp_dlm_keys, + static, rsb_interval); + +int ranges_overlap(struct dlm_range *range1, struct dlm_range *range2) +{ + int ret1, ret2; + + ret1 = cmp_range_keys(range1->start->val, range1->start->len, + range2->end->val, range2->end->len); + + ret2 = cmp_range_keys(range1->end->val, range1->end->len, + range2->start->val, range2->start->len); + + if (ret1 <= 0 && ret2 >= 0) + return 1; + + return 0; +} + +static int modes_compat(struct dlm_lkb *gr, struct dlm_lkb *rq) +{ + if (cmp_dlm_keys(rq->lkb_rqrange.start, gr->lkb_grrange.end) <= 0 && + cmp_dlm_keys(rq->lkb_rqrange.end, gr->lkb_grrange.start) >= 0) + return _modes_compat(gr, rq); + return 0; +} + int dlm_modes_compat(int mode1, int mode2) { + /* XXX: This needs to be fixed up to take ranges into account */ return __dlm_compat_matrix[mode1 + 1][mode2 + 1]; } @@ -308,7 +405,7 @@ static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) rv = -EDEADLK; } - dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, rv, lkb->lkb_sbflags); + dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, NULL, rv, lkb->lkb_sbflags); } static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) @@ -317,12 +414,13 @@ static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL); } -static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode) +static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode, + struct dlm_range *rqrange) { if (is_master_copy(lkb)) { send_bast(r, lkb, rqmode); } else { - dlm_add_cb(lkb, DLM_CB_BAST, rqmode, 0, 0); + dlm_add_cb(lkb, DLM_CB_BAST, rqmode, rqrange, 0, 0); } } @@ -1198,6 +1296,7 @@ static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret) INIT_LIST_HEAD(&lkb->lkb_cb_list); mutex_init(&lkb->lkb_cb_mutex); INIT_WORK(&lkb->lkb_cb_work, dlm_callback_work); + RB_CLEAR_NODE(&lkb->lkb_statenode); idr_preload(GFP_NOFS); spin_lock(&ls->ls_lkbidr_spin); @@ -1240,6 +1339,11 @@ static void kill_lkb(struct kref *kref) DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); } +static void free_range_keys(struct dlm_range *range) +{ + kfree(range->start); + kfree(range->end); +} /* __put_lkb() is used when an lkb may not have an rsb attached to it so we need to provide the lockspace explicitly */ @@ -1254,6 +1358,9 @@ static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb) detach_lkb(lkb); + free_range_keys(&lkb->lkb_rqrange); + free_range_keys(&lkb->lkb_grrange); + /* for local/process lkbs, lvbptr points to caller's lksb */ if (lkb->lkb_lvbptr && is_master_copy(lkb)) dlm_free_lvb(lkb->lkb_lvbptr); @@ -1315,6 +1422,8 @@ static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) kref_get(&lkb->lkb_ref); DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); + BUG_ON(status != DLM_LKSTS_WAITING && !START(lkb)); + BUG_ON(status != DLM_LKSTS_WAITING && !LAST(lkb)); lkb->lkb_timestamp = ktime_get(); @@ -1331,6 +1440,7 @@ static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) /* convention says granted locks kept in order of grmode */ lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue, lkb->lkb_grmode); + rsb_interval_insert(lkb, &r->res_grantroot); break; case DLM_LKSTS_CONVERT: if (lkb->lkb_exflags & DLM_LKF_HEADQUE) @@ -1338,16 +1448,45 @@ static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) else list_add_tail(&lkb->lkb_statequeue, &r->res_convertqueue); + rsb_interval_insert(lkb, &r->res_convertroot); break; default: DLM_ASSERT(0, dlm_print_lkb(lkb); printk("sts=%d\n", status);); } } +static struct rb_root *lkb_res_root(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct rb_root *ret = NULL; + + switch (lkb->lkb_status) { + case DLM_LKSTS_GRANTED: + ret = &r->res_grantroot; + break; + case DLM_LKSTS_CONVERT: + ret = &r->res_convertroot; + break; + default: + DLM_ASSERT(0, dlm_print_lkb(lkb); + printk("sts=%d\n", lkb->lkb_status);); + } + return ret; +} + static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) { + struct rb_root *root = NULL; + + if (lkb->lkb_status && lkb->lkb_status != DLM_LKSTS_WAITING) + root = lkb_res_root(r, lkb); + lkb->lkb_status = 0; list_del(&lkb->lkb_statequeue); + if (root) { + rsb_interval_remove(lkb, root); + RB_CLEAR_NODE(&lkb->lkb_statenode);/* To aid in debugging */ + } + WARN_ON(!RB_EMPTY_NODE(&lkb->lkb_statenode)); unhold_lkb(lkb); } @@ -2113,6 +2252,11 @@ static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) { + /* Set ranges now so move/add lkb has something to insert */ + lkb->lkb_grrange.start = lkb->lkb_rqrange.start; + lkb->lkb_grrange.end = lkb->lkb_rqrange.end; + lkb->lkb_rqrange.start = lkb->lkb_rqrange.end = NULL; + if (lkb->lkb_grmode != lkb->lkb_rqmode) { lkb->lkb_grmode = lkb->lkb_rqmode; if (lkb->lkb_status) @@ -2189,27 +2333,34 @@ static void munge_altmode(struct dlm_lkb *lkb, struct dlm_message *ms) } } -static inline int first_in_list(struct dlm_lkb *lkb, struct list_head *head) +static inline int first_in_list_range(struct dlm_lkb *lkb, struct list_head *head) { - struct dlm_lkb *first = list_entry(head->next, struct dlm_lkb, - lkb_statequeue); - if (lkb->lkb_id == first->lkb_id) - return 1; + struct dlm_lkb *first; + + list_for_each_entry(first, head, lkb_statequeue) { + if (ranges_overlap(&lkb->lkb_rqrange, &first->lkb_rqrange)) { + if (lkb->lkb_id == first->lkb_id) + return 1; + break; + } + } return 0; } /* Check if the given lkb conflicts with another lkb on the queue. */ - -static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb) +static int queue_conflict(struct rb_root *root, struct dlm_lkb *lkb) { struct dlm_lkb *this; + struct dlm_key *start, *end; - list_for_each_entry(this, head, lkb_statequeue) { - if (this == lkb) - continue; - if (!modes_compat(this, lkb)) + start = lkb->lkb_rqrange.start; + end = lkb->lkb_rqrange.end; + this = rsb_interval_iter_first(root, start, end); + while (this) { + if (this != lkb && !modes_compat(this, lkb)) return 1; + this = rsb_interval_iter_next(this, start, end); } return 0; } @@ -2266,6 +2417,9 @@ static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2) continue; } + if (!ranges_overlap(&lkb1->lkb_rqrange, &lkb2->lkb_grrange)) + continue; + if (!lkb_is_ahead) { if (!modes_compat(lkb2, lkb1)) return 1; @@ -2326,7 +2480,7 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, * added to the remaining conditions. */ - if (queue_conflict(&r->res_grantqueue, lkb)) + if (queue_conflict(&r->res_grantroot, lkb)) return 0; /* @@ -2335,7 +2489,7 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, * locks */ - if (queue_conflict(&r->res_convertqueue, lkb)) + if (queue_conflict(&r->res_convertroot, lkb)) return 0; /* @@ -2406,8 +2560,7 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, * granted until all other conversion requests ahead of it are granted * and/or canceled. */ - - if (!now && conv && first_in_list(lkb, &r->res_convertqueue)) + if (!now && conv && first_in_list_range(lkb, &r->res_convertqueue)) return 1; /* @@ -2434,7 +2587,7 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, */ if (!now && !conv && list_empty(&r->res_convertqueue) && - first_in_list(lkb, &r->res_waitqueue)) + first_in_list_range(lkb, &r->res_waitqueue)) return 1; return 0; @@ -2512,7 +2665,7 @@ static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, cw if there's a blocked conversion to DLM_LOCK_CW. */ static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, - unsigned int *count) + unsigned int *count, struct dlm_range **range) { struct dlm_lkb *lkb, *s; int recover = rsb_flag(r, RSB_RECOVER_GRANT); @@ -2551,7 +2704,11 @@ static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, continue; } - hi = max_t(int, lkb->lkb_rqmode, hi); + if (lkb->lkb_rqmode > hi) { + hi = lkb->lkb_rqmode; + if (range) + *range = &lkb->lkb_rqrange; + } if (cw && lkb->lkb_rqmode == DLM_LOCK_CW) *cw = 1; @@ -2568,7 +2725,7 @@ static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, } static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw, - unsigned int *count) + unsigned int *count, struct dlm_range **range) { struct dlm_lkb *lkb, *s; @@ -2578,7 +2735,11 @@ static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw, if (count) (*count)++; } else { - high = max_t(int, lkb->lkb_rqmode, high); + if (lkb->lkb_rqmode > high) { + high = lkb->lkb_rqmode; + *range = &lkb->lkb_rqrange; + } + if (lkb->lkb_rqmode == DLM_LOCK_CW) *cw = 1; } @@ -2611,15 +2772,15 @@ static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count) struct dlm_lkb *lkb, *s; int high = DLM_LOCK_IV; int cw = 0; + struct dlm_range *highrange = NULL; if (!is_master(r)) { - log_print("grant_pending_locks r nodeid %d", r->res_nodeid); dlm_dump_rsb(r); return; } - high = grant_pending_convert(r, high, &cw, count); - high = grant_pending_wait(r, high, &cw, count); + high = grant_pending_convert(r, high, &cw, count, &highrange); + high = grant_pending_wait(r, high, &cw, count, &highrange); if (high == DLM_LOCK_IV) return; @@ -2631,12 +2792,13 @@ static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count) */ list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) { - if (lkb->lkb_bastfn && lock_requires_bast(lkb, high, cw)) { + if ((lkb->lkb_bastfn || lkb->lkb_rbastfn) && + lock_requires_bast(lkb, high, cw)) { if (cw && high == DLM_LOCK_PR && lkb->lkb_grmode == DLM_LOCK_PR) - queue_bast(r, lkb, DLM_LOCK_CW); + queue_bast(r, lkb, DLM_LOCK_CW, highrange); else - queue_bast(r, lkb, high); + queue_bast(r, lkb, high, highrange); lkb->lkb_highbast = high; } } @@ -2665,8 +2827,9 @@ static void send_bast_queue(struct dlm_rsb *r, struct list_head *head, /* skip self when sending basts to convertqueue */ if (gr == lkb) continue; - if (gr->lkb_bastfn && modes_require_bast(gr, lkb)) { - queue_bast(r, gr, lkb->lkb_rqmode); + if ((gr->lkb_rbastfn || gr->lkb_bastfn) && + modes_require_bast(gr, lkb)) { + queue_bast(r, gr, lkb->lkb_rqmode, &lkb->lkb_rqrange); gr->lkb_highbast = lkb->lkb_rqmode; } } @@ -2801,11 +2964,15 @@ static void confirm_master(struct dlm_rsb *r, int error) } } -static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, +static int set_lock_args(int mode, struct dlm_range *range, + struct dlm_lksb *lksb, uint32_t flags, int namelen, unsigned long timeout_cs, void (*ast) (void *astparam), void *astparam, void (*bast) (void *astparam, int mode), + void (*rbast) (void *astarg, int mode, + struct dlm_key *start, + struct dlm_key *end), struct dlm_args *args) { int rv = -EINVAL; @@ -2815,6 +2982,9 @@ static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, if (mode < 0 || mode > DLM_LOCK_EX) goto out; + if (range && cmp_dlm_keys(range->start, range->end) > 0) + goto out; + if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN)) goto out; @@ -2851,6 +3021,10 @@ static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid) goto out; + /* XXX: The caller could pass default_range for us */ + if (!range) + range = &default_range; + /* these args will be copied to the lkb in validate_lock_args, it cannot be done now because when converting locks, fields in an active lkb cannot be modified before locking the rsb */ @@ -2859,8 +3033,10 @@ static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, args->astfn = ast; args->astparam = astparam; args->bastfn = bast; + args->rbastfn = rbast; args->timeout = timeout_cs; args->mode = mode; + args->range = *range; args->lksb = lksb; rv = 0; out: @@ -2910,7 +3086,21 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, lkb->lkb_astfn = args->astfn; lkb->lkb_astparam = args->astparam; lkb->lkb_bastfn = args->bastfn; + lkb->lkb_rbastfn = args->rbastfn; lkb->lkb_rqmode = args->mode; + if (args->range.start && args->range.end) { + rv = -ENOMEM; + lkb->lkb_rqrange.start = alloc_key(args->range.start->val, + args->range.start->len, + GFP_NOFS); + if (!lkb->lkb_rqrange.start) + goto out; + lkb->lkb_rqrange.end = alloc_key(args->range.end->val, + args->range.end->len, + GFP_NOFS); + if (!lkb->lkb_rqrange.end) + goto out; + } lkb->lkb_lksb = args->lksb; lkb->lkb_lvbptr = args->lksb->sb_lvbptr; lkb->lkb_ownpid = (int) current->pid; @@ -2970,6 +3160,11 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) goto out; } +#if 0 + /* XXX: Shouldn't CANCEL check against rqstart/rqend? */ + if (args->start != lkb->lkb_grstart || args->end != lkb->lkb_grend) + goto out; +#endif /* cancel not allowed with another cancel/unlock in progress */ if (args->flags & DLM_LKF_CANCEL) { @@ -3055,8 +3250,8 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) rv = 0; out: if (rv) - log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", rv, - lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, + log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", + rv, lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, args->flags, lkb->lkb_wait_type, lkb->lkb_resource->res_name); return rv; @@ -3138,7 +3333,7 @@ static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) before we try again to grant this one. */ if (is_demoted(lkb)) { - grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL); + grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL, NULL); if (_can_be_granted(r, lkb, 1, 0)) { grant_lock(r, lkb); queue_cast(r, lkb, 0); @@ -3405,19 +3600,24 @@ static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, * Two stage 1 varieties: dlm_lock() and dlm_unlock() */ -int dlm_lock(dlm_lockspace_t *lockspace, - int mode, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*ast) (void *astarg), - void *astarg, - void (*bast) (void *astarg, int mode)) +static int _dlm_lock(dlm_lockspace_t *lockspace, + int mode, + struct dlm_key *start, + struct dlm_key *end, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*ast) (void *astarg), + void *astarg, + void (*bast) (void *astarg, int mode), + void (*rbast) (void *astarg, int mode, + struct dlm_key *start, struct dlm_key *end)) { struct dlm_ls *ls; struct dlm_lkb *lkb; + struct dlm_range range = { start, end }; struct dlm_args args; int error, convert = flags & DLM_LKF_CONVERT; @@ -3435,8 +3635,8 @@ int dlm_lock(dlm_lockspace_t *lockspace, if (error) goto out; - error = set_lock_args(mode, lksb, flags, namelen, 0, ast, - astarg, bast, &args); + error = set_lock_args(mode, start ? &range : NULL, lksb, flags, namelen, + 0, ast, astarg, bast, rbast, &args); if (error) goto out_put; @@ -3458,6 +3658,47 @@ int dlm_lock(dlm_lockspace_t *lockspace, return error; } +int dlm_lock_range(dlm_lockspace_t *lockspace, + int mode, + struct dlm_key *start, + struct dlm_key *end, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*ast) (void *astarg), + void *astarg, + void (*rbast) (void *astarg, int mode, + struct dlm_key *start, struct dlm_key *end)) +{ + if (!start || !end) + return -EINVAL; + + if (start->len > DLM_KEY_LEN || end->len > DLM_KEY_LEN) { + WARN_ON_ONCE(1); + return -EINVAL; + } + + return _dlm_lock(lockspace, mode, start, end, lksb, flags, name, + namelen, parent_lkid, ast, astarg, NULL, rbast); +} + +int dlm_lock(dlm_lockspace_t *lockspace, + int mode, + struct dlm_lksb *lksb, + uint32_t flags, + void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*ast) (void *astarg), + void *astarg, + void (*bast) (void *astarg, int mode)) +{ + return _dlm_lock(lockspace, mode, NULL, NULL, lksb, flags, name, + namelen, parent_lkid, ast, astarg, bast, NULL); +} + int dlm_unlock(dlm_lockspace_t *lockspace, uint32_t lkid, uint32_t flags, @@ -3492,6 +3733,7 @@ int dlm_unlock(dlm_lockspace_t *lockspace, error = 0; if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK))) error = 0; + out_put: dlm_put_lkb(lkb); out: @@ -3609,10 +3851,29 @@ static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb, ms->m_rqmode = lkb->lkb_rqmode; ms->m_hash = r->res_hash; + ms->m_grstart_len = ms->m_grend_len = ms->m_rqstart_len = + ms->m_rqend_len = 0; + if (lkb->lkb_grrange.start) { + ms->m_grstart_len = lkb->lkb_grrange.start->len; + memcpy(ms->m_grstart, lkb->lkb_grrange.start->val, ms->m_grstart_len); + } + if (lkb->lkb_grrange.end) { + ms->m_grend_len = lkb->lkb_grrange.end->len; + memcpy(ms->m_grend, lkb->lkb_grrange.end->val, ms->m_grend_len); + } + if (lkb->lkb_rqrange.start) { + ms->m_rqstart_len = lkb->lkb_rqrange.start->len; + memcpy(ms->m_rqstart, lkb->lkb_rqrange.start->val, ms->m_rqstart_len); + } + if (lkb->lkb_rqrange.end) { + ms->m_rqend_len = lkb->lkb_rqrange.end->len; + memcpy(ms->m_rqend, lkb->lkb_rqrange.end->val, ms->m_rqend_len); + } + /* m_result and m_bastmode are set from function args, not from lkb fields */ - if (lkb->lkb_bastfn) + if (lkb->lkb_bastfn || lkb->lkb_rbastfn) ms->m_asts |= DLM_CB_BAST; if (lkb->lkb_astfn) ms->m_asts |= DLM_CB_CAST; @@ -3738,6 +3999,7 @@ static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) send_args(r, lkb, ms); ms->m_bastmode = mode; + /* XXX: Fill bastrange here */ error = send_message(mh, ms); out: @@ -3929,6 +4191,31 @@ static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb, return -ENOMEM; } + if (ms->m_grstart_len) { + lkb->lkb_grrange.start = alloc_key(ms->m_grstart, + ms->m_grstart_len, GFP_NOFS); + if (!lkb->lkb_grrange.start) + return -ENOMEM; + } + if (ms->m_grend_len) { + lkb->lkb_grrange.end = alloc_key(ms->m_grend, ms->m_grend_len, + GFP_NOFS); + if (!lkb->lkb_grrange.end) + return -ENOMEM; + } + + if (ms->m_rqstart_len) { + lkb->lkb_rqrange.start = alloc_key(ms->m_rqstart, + ms->m_rqstart_len, GFP_NOFS); + if (!lkb->lkb_rqrange.start) + return -ENOMEM; + } + if (ms->m_rqend_len) { + lkb->lkb_rqrange.end = alloc_key(ms->m_rqend, ms->m_rqend_len, + GFP_NOFS); + if (!lkb->lkb_rqrange.end) + return -ENOMEM; + } return 0; } @@ -4335,6 +4622,8 @@ static int receive_bast(struct dlm_ls *ls, struct dlm_message *ms) { struct dlm_lkb *lkb; struct dlm_rsb *r; + struct dlm_range range; + struct dlm_key start, end; int error; error = find_lkb(ls, ms->m_remid, &lkb); @@ -4350,7 +4639,14 @@ static int receive_bast(struct dlm_ls *ls, struct dlm_message *ms) if (error) goto out; - queue_bast(r, lkb, ms->m_bastmode); + start.val = ms->m_baststart; + start.len = ms->m_baststart_len; + end.val = ms->m_bastend; + end.len = ms->m_bastend_len; + range.start = &start; + range.end = &end; + + queue_bast(r, lkb, ms->m_bastmode, &range); lkb->lkb_highbast = ms->m_bastmode; out: unlock_rsb(r); @@ -5796,8 +6092,9 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, When DLM_IFL_USER is set, the dlm knows that this is a userspace lock and that lkb_astparam is the dlm_user_args structure. */ - error = set_lock_args(mode, &ua->lksb, flags, namelen, timeout_cs, - fake_astfn, ua, fake_bastfn, &args); + error = set_lock_args(mode, NULL, &ua->lksb, flags, namelen, + timeout_cs, fake_astfn, ua, fake_bastfn, NULL, + &args); lkb->lkb_flags |= DLM_IFL_USER; if (error) { @@ -5868,8 +6165,8 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, ua->bastaddr = ua_tmp->bastaddr; ua->user_lksb = ua_tmp->user_lksb; - error = set_lock_args(mode, &ua->lksb, flags, 0, timeout_cs, - fake_astfn, ua, fake_bastfn, &args); + error = set_lock_args(mode, NULL, &ua->lksb, flags, 0, timeout_cs, + fake_astfn, ua, fake_bastfn, NULL, &args); if (error) goto out_put; diff --git a/kmod/dlm/lock.h b/kmod/dlm/lock.h index ed8ebd3a..271736a4 100644 --- a/kmod/dlm/lock.h +++ b/kmod/dlm/lock.h @@ -76,5 +76,7 @@ static inline void unlock_rsb(struct dlm_rsb *r) mutex_unlock(&r->res_mutex); } +int ranges_overlap(struct dlm_range *range1, struct dlm_range *range2); + #endif diff --git a/kmod/dlm/lockspace.c b/kmod/dlm/lockspace.c index 88556dc0..84cb7210 100644 --- a/kmod/dlm/lockspace.c +++ b/kmod/dlm/lockspace.c @@ -632,9 +632,9 @@ static int new_lockspace(const char *name, const char *cluster, error = do_uevent(ls, 1); if (error) goto out_recoverd; - wait_for_completion(&ls->ls_members_done); error = ls->ls_members_result; + if (error) goto out_members; diff --git a/kmod/dlm/main.c b/kmod/dlm/main.c index 079c0bd7..d880842d 100644 --- a/kmod/dlm/main.c +++ b/kmod/dlm/main.c @@ -94,4 +94,5 @@ EXPORT_SYMBOL_GPL(dlm_new_lockspace); EXPORT_SYMBOL_GPL(dlm_release_lockspace); EXPORT_SYMBOL_GPL(dlm_lock); EXPORT_SYMBOL_GPL(dlm_unlock); +EXPORT_SYMBOL_GPL(dlm_lock_range); diff --git a/kmod/dlm/netlink.c b/kmod/dlm/netlink.c index e7cfbaf8..71275218 100644 --- a/kmod/dlm/netlink.c +++ b/kmod/dlm/netlink.c @@ -7,7 +7,7 @@ */ #include -#include +#include "include/linux/dlm.h" #include #include diff --git a/kmod/dlm/plock.c b/kmod/dlm/plock.c index f704458e..a33b4f27 100644 --- a/kmod/dlm/plock.c +++ b/kmod/dlm/plock.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include "include/linux/dlm.h" #include #include diff --git a/kmod/dlm/rcom.c b/kmod/dlm/rcom.c index f3f5e72a..7af563a2 100644 --- a/kmod/dlm/rcom.c +++ b/kmod/dlm/rcom.c @@ -399,7 +399,7 @@ static void pack_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, rl->rl_status = lkb->lkb_status; rl->rl_wait_type = cpu_to_le16(lkb->lkb_wait_type); - if (lkb->lkb_bastfn) + if (lkb->lkb_bastfn || lkb->lkb_rbastfn) rl->rl_asts |= DLM_CB_BAST; if (lkb->lkb_astfn) rl->rl_asts |= DLM_CB_CAST; diff --git a/kmod/dlm/user.c b/kmod/dlm/user.c index 16a96f6f..826437e3 100644 --- a/kmod/dlm/user.c +++ b/kmod/dlm/user.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include "include/linux/dlm.h" #include #include @@ -207,7 +207,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, spin_lock(&proc->asts_spin); - rv = dlm_add_lkb_callback(lkb, flags, mode, status, sbflags, seq); + rv = dlm_add_lkb_callback(lkb, flags, mode, NULL, status, sbflags, seq); if (rv < 0) { spin_unlock(&proc->asts_spin); goto out; diff --git a/kmod/dlm/util.c b/kmod/dlm/util.c index e36520af..b18eded3 100644 --- a/kmod/dlm/util.c +++ b/kmod/dlm/util.c @@ -105,6 +105,15 @@ void dlm_message_out(struct dlm_message *ms) ms->m_bastmode = cpu_to_le32(ms->m_bastmode); ms->m_asts = cpu_to_le32(ms->m_asts); ms->m_result = cpu_to_le32(to_dlm_errno(ms->m_result)); + + ms->m_grstart_len = cpu_to_le16(ms->m_grstart_len); + ms->m_grend_len = cpu_to_le16(ms->m_grend_len); + + ms->m_rqstart_len = cpu_to_le16(ms->m_rqstart_len); + ms->m_rqend_len = cpu_to_le16(ms->m_rqend_len); + + ms->m_baststart_len = cpu_to_le16(ms->m_baststart_len); + ms->m_bastend_len = cpu_to_le16(ms->m_bastend_len); } void dlm_message_in(struct dlm_message *ms) @@ -129,6 +138,15 @@ void dlm_message_in(struct dlm_message *ms) ms->m_bastmode = le32_to_cpu(ms->m_bastmode); ms->m_asts = le32_to_cpu(ms->m_asts); ms->m_result = from_dlm_errno(le32_to_cpu(ms->m_result)); + + ms->m_grstart_len = le16_to_cpu(ms->m_grstart_len); + ms->m_grend_len = le16_to_cpu(ms->m_grend_len); + + ms->m_rqstart_len = le16_to_cpu(ms->m_rqstart_len); + ms->m_rqend_len = le16_to_cpu(ms->m_rqend_len); + + ms->m_baststart_len = le16_to_cpu(ms->m_baststart_len); + ms->m_bastend_len = le16_to_cpu(ms->m_bastend_len); } void dlm_rcom_out(struct dlm_rcom *rc) From e711c15acf6e39dc37c0ac106d6867b5bc4c13b5 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 8 Jun 2017 17:40:39 -0500 Subject: [PATCH 298/920] scoutfs: use dlm for locking To actually use it, we first have to copy symbols over from the dlm build into the scoutfs source directory. Make that happen automatically for us in the Makefile. The only users of locking at the moment are mount, unmount and xattr read/write. Adding more locking calls should be a straight-forward endeavor. The LVB based server ip communication didn't work out, and LVBS as they are written don't make sense in a range locking world. So instead, we record the server ip address in the superblock. This is protected by the listen lock, which also arbitrates which node will be the manifest server. We take and drop the dlm lock on each lock/unlock call. Lock caching will come in a future patch. Signed-off-by: Mark Fasheh --- kmod/Makefile | 5 +- kmod/dlm/lock.c | 5 + kmod/src/Makefile | 4 +- kmod/src/format.h | 14 +- kmod/src/lock.c | 405 +++++++++++++++++++-------------------- kmod/src/lock.h | 22 ++- kmod/src/net.c | 65 +++++-- kmod/src/options.c | 81 ++++++++ kmod/src/options.h | 17 ++ kmod/src/scoutfs_trace.h | 14 +- kmod/src/super.c | 21 +- kmod/src/super.h | 6 +- 12 files changed, 402 insertions(+), 257 deletions(-) create mode 100644 kmod/src/options.c create mode 100644 kmod/src/options.h diff --git a/kmod/Makefile b/kmod/Makefile index bc37906e..ada76969 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -12,14 +12,15 @@ else SP = @: endif -SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src +SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) -I $(CURDIR)/dlm/include M=$(CURDIR)/src DLM_ARGS := CONFIG_DLM=m CONFIG_DLM_DEBUG=y -C $(SK_KSRC) M=$(CURDIR)/dlm all: module module: - make $(SCOUTFS_ARGS) make $(DLM_ARGS) + cp $(CURDIR)/dlm/Module.symvers $(CURDIR)/src/ + make $(SCOUTFS_ARGS) $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) # Do not enable until we can clean up some warnings # $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(DLM_ARGS) diff --git a/kmod/dlm/lock.c b/kmod/dlm/lock.c index d62127fe..d7464204 100644 --- a/kmod/dlm/lock.c +++ b/kmod/dlm/lock.c @@ -2552,6 +2552,11 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, * order. */ + /* + * XXX: Right now scoutfs uses NOORDER but if that changes + * we'll have to replace the list_empty() checks below with + * tree searches. + */ if (lkb->lkb_exflags & DLM_LKF_NOORDER) return 1; diff --git a/kmod/src/Makefile b/kmod/src/Makefile index ecb5f39b..8370da64 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -3,5 +3,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ - ioctl.o item.o key.o lock.o manifest.o msg.o net.o ring.o seg.o \ - scoutfs_trace.o super.o trans.o xattr.o + ioctl.o item.o key.o lock.o manifest.o msg.o net.o options.o \ + ring.o seg.o scoutfs_trace.o super.o trans.o xattr.o diff --git a/kmod/src/format.h b/kmod/src/format.h index 58adf6ba..26c2bbb8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -276,6 +276,13 @@ struct scoutfs_inode_index_key { #define SCOUTFS_UUID_BYTES 16 +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; +} __packed; + +#define SCOUTFS_DEFAULT_PORT 12345 /* * The ring fields describe the statically allocated ring log. The @@ -298,6 +305,7 @@ struct scoutfs_super_block { __le64 next_seg_seq; struct scoutfs_ring_descriptor alloc_ring; struct scoutfs_manifest manifest; + struct scoutfs_inet_addr server_addr; } __packed; #define SCOUTFS_ROOT_INO 1 @@ -397,12 +405,6 @@ enum { * messages over the wire. */ -/* XXX ipv6 */ -struct scoutfs_inet_addr { - __le32 addr; - __le16 port; -} __packed; - /* * This header precedes and describes all network messages sent over * sockets. The id is set by the request and sent in the reply. The diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 6078b024..379983be 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -19,13 +19,9 @@ #include "lock.h" #include "item.h" #include "scoutfs_trace.h" +#include "msg.h" -/* - * This is meant to be simple and correct, not performant. - */ - -static DECLARE_RWSEM(global_rwsem); -static LIST_HEAD(global_super_list); +#include "linux/dlm.h" /* * Allocated once and pointed to by the lock info of all the supers with @@ -34,13 +30,8 @@ static LIST_HEAD(global_super_list); struct held_locks { spinlock_t lock; struct list_head list; + unsigned int seq_cnt; wait_queue_head_t waitq; - - /* super hacky fake lvb that only allows one specific key */ - char fake_lvb[sizeof(struct scoutfs_inet_addr)]; - struct scoutfs_key_buf fake_lvb_key; - char fake_lvb_key_data[SCOUTFS_MAX_KEY_SIZE]; - }; /* @@ -50,62 +41,25 @@ struct held_locks { */ struct lock_info { struct super_block *sb; + dlm_lockspace_t *ls; + char ls_name[DLM_LOCKSPACE_LEN]; bool shutdown; struct held_locks *held; struct list_head id_head; - struct list_head global_head; }; +#define RANGE_LOCK_RESOURCE "fs_range" +#define RANGE_LOCK_RESOURCE_LEN (strlen(RANGE_LOCK_RESOURCE)) + + #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info /* - * locks are compatible if they're from the same super, or are both reads, - * or don't overlap. - */ -static bool compatible_locks(struct scoutfs_lock *a, struct scoutfs_lock *b) -{ - return a->sb == b->sb || - (a->mode == SCOUTFS_LOCK_MODE_READ && - b->mode == SCOUTFS_LOCK_MODE_READ) || - scoutfs_key_compare_ranges(a->start, a->end, b->start, b->end); -} - -/* also returns true if we're shutting down, caller tests after waiting */ -static bool lock_added(struct lock_info *linf, struct scoutfs_lock *add) -{ - struct held_locks *held = linf->held; - struct scoutfs_lock *lck; - bool added = true; - - spin_lock(&held->lock); - - if (linf->shutdown) { - added = true; - goto out; - } - - list_for_each_entry(lck, &held->list, head) { - if (!compatible_locks(lck, add)) { - added = false; - break; - } - } - - if (added) - list_add(&add->head, &held->list); - -out: - spin_unlock(&held->lock); - - return added; -} - -/* - * Invalidate caches on this super because another super has acquired - * a lock with the given mode and range. We always have to write out - * dirty overlapping items. If they're writing then we need to also - * invalidate all cached overlapping structures. + * Invalidate caches on this because another node wants a lock + * with the a lock with the given mode and range. We always have to + * write out dirty overlapping items. If they're writing then we need + * to also invalidate all cached overlapping structures. */ static int invalidate_caches(struct super_block *sb, int mode, struct scoutfs_key_buf *start, @@ -119,120 +73,81 @@ static int invalidate_caches(struct super_block *sb, int mode, if (ret) return ret; - if (mode == SCOUTFS_LOCK_MODE_WRITE) { + if (mode == SCOUTFS_LOCK_MODE_WRITE) ret = scoutfs_item_invalidate(sb, start, end); -#if 0 - scoutfs_dir_invalidate(sb, start, end) ?: - scoutfs_inode_invalidate(sb, start, end) ?: - scoutfs_data_invalidate(sb, start, end); -#endif - } return ret; } -#define for_each_other_linf(linf, from_linf) \ - for (linf = list_entry(from_linf->id_head.next, struct lock_info, \ - id_head); \ - linf != from_linf; \ - linf = list_entry(linf->id_head.next, struct lock_info, \ - id_head)) - -static int invalidate_others(struct super_block *from, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) -{ - DECLARE_LOCK_INFO(from, from_linf); - struct lock_info *linf; - int ret = 0; - - down_read(&global_rwsem); - - for_each_other_linf(linf, from_linf) { - ret = invalidate_caches(linf->sb, mode, start, end); - if (ret) - break; - } - - up_read(&global_rwsem); - - return ret; -} - -static void unlock(struct held_locks *held, struct scoutfs_lock *lck) +static void uninit_scoutfs_lock(struct held_locks *held, + struct scoutfs_lock *lck) { spin_lock(&held->lock); + lck->rqmode = SCOUTFS_LOCK_MODE_IV; list_del_init(&lck->head); spin_unlock(&held->lock); + lck->sequence = 0; +} + +static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + + memset(lck, 0, sizeof(*lck)); + INIT_LIST_HEAD(&lck->head); + lck->sb = sb; + lck->mode = SCOUTFS_LOCK_MODE_IV; + + if (start) { + lck->start = start; + lck->dlm_start.val = start->data; + lck->dlm_start.len = start->key_len; + } + if (end) { + lck->end = end; + lck->dlm_end.val = end->data; + lck->dlm_end.len = end->key_len; + } + + spin_lock(&held->lock); + lck->sequence = ++held->seq_cnt; + spin_unlock(&held->lock); +} + +static void scoutfs_ast(void *astarg) +{ + struct scoutfs_lock *lck = astarg; + DECLARE_LOCK_INFO(lck->sb, linfo); + struct held_locks *held = linfo->held; + + trace_scoutfs_ast(lck->sb, lck); + + spin_lock(&held->lock); + lck->mode = lck->rqmode; + lck->rqmode = SCOUTFS_LOCK_MODE_IV; + spin_unlock(&held->lock); wake_up(&held->waitq); } -static void assert_fake_lvb(struct held_locks *held, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, unsigned lvb_len) +static void scoutfs_rbast(void *astarg, int mode, + struct dlm_key *start, struct dlm_key *end) { - BUG_ON(scoutfs_key_compare(start, end)); - BUG_ON(lvb_len != sizeof(held->fake_lvb)); - BUG_ON(held->fake_lvb_key.key_len && - scoutfs_key_compare(&held->fake_lvb_key, start)); } -/* - * Acquire a coherent lock on the given range of keys. While the lock - * is held other lockers are serialized. Cache coherency is maintained - * by the locking infrastructure. Lock acquisition causes writeout from - * or invalidation of other caches. - * - * The caller provides the opaque lock structure used for storage and - * their start and end pointers will be accessed while the lock is held. - */ -int scoutfs_lock_range_lvb(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - void *caller_lvb, unsigned lvb_len, - struct scoutfs_lock *lck) +static int lock_granted(struct held_locks *held, struct scoutfs_lock *lck, + int mode) { - DECLARE_LOCK_INFO(sb, linf); - struct held_locks *held = linf->held; int ret; - INIT_LIST_HEAD(&lck->head); - lck->sb = sb; - lck->start = start; - lck->end = end; - lck->mode = mode; + spin_lock(&held->lock); + ret = !!(mode == lck->mode); + spin_unlock(&held->lock); - trace_scoutfs_lock_range(sb, lck); - - ret = wait_event_interruptible(held->waitq, lock_added(linf, lck)); - if (ret) - goto out; - - if (linf->shutdown) { - ret = -ESHUTDOWN; - goto out; - } - - ret = invalidate_others(sb, mode, start, end); - if (ret) - goto out; - - if (caller_lvb) { - assert_fake_lvb(held, start, end, lvb_len); - if (mode == SCOUTFS_LOCK_MODE_WRITE) { - memcpy(held->fake_lvb, caller_lvb, lvb_len); - scoutfs_key_copy(&held->fake_lvb_key, start); - } else { - memcpy(caller_lvb, held->fake_lvb, lvb_len); - } - } - ret = 0; - -out: - if (ret < 0 && !list_empty(&lck->head)) - unlock(held, lck); return ret; } @@ -250,81 +165,124 @@ int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *end, struct scoutfs_lock *lck) { - return scoutfs_lock_range_lvb(sb, mode, start, end, NULL, 0, lck); + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + int ret; + + init_scoutfs_lock(sb, lck, start, end); + + trace_scoutfs_lock_range(sb, lck); + + spin_lock(&held->lock); + if (linfo->shutdown) { + spin_unlock(&held->lock); + return -ESHUTDOWN; + } + + list_add(&lck->head, &held->list); + spin_unlock(&held->lock); + + lck->rqmode = mode; + ret = dlm_lock_range(linfo->ls, mode, &lck->dlm_start, &lck->dlm_end, + &lck->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, + RANGE_LOCK_RESOURCE_LEN, 0, scoutfs_ast, lck, + scoutfs_rbast); + if (ret) { + scoutfs_err(sb, "Error %d locking %s\n", ret, + RANGE_LOCK_RESOURCE); + uninit_scoutfs_lock(held, lck); + return ret; + } + + wait_event(held->waitq, lock_granted(held, lck, mode)); + + return 0; } void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) { - DECLARE_LOCK_INFO(sb, linf); - struct held_locks *held = linf->held; + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + int ret; trace_scoutfs_unlock_range(sb, lck); - unlock(held, lck); + BUG_ON(!lck->sequence); + + /* + * Use write mode to invalidate all since we are completely + * dropping the lock. Once we keep the locks around then we + * can invalidate based on what level we're downconverting to + * (PR, NL). + */ + invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lck->start, lck->end); + + lck->rqmode = DLM_LOCK_IV; + ret = dlm_unlock(linfo->ls, lck->lksb.sb_lkid, 0, &lck->lksb, lck); + if (ret) { + scoutfs_err(sb, "Error %d unlocking %s\n", ret, + RANGE_LOCK_RESOURCE); + goto out; + } + + wait_event(held->waitq, lock_granted(held, lck, DLM_LOCK_IV)); +out: + uninit_scoutfs_lock(held, lck); + /* lock was removed from held list, wake up umount process */ + wake_up(&held->waitq); } /* * The moment this is done we can have other mounts start asking * us to write back and invalidate, so do this very very late. */ -int scoutfs_lock_setup(struct super_block *sb) +static int init_lock_info(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_sb_info *other_sbi; - struct lock_info *other_linf; struct held_locks *held; - struct lock_info *linf; + struct lock_info *linfo; - linf = kmalloc(sizeof(struct lock_info), GFP_KERNEL); - if (!linf) + linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); + if (!linfo) return -ENOMEM; held = kzalloc(sizeof(struct held_locks), GFP_KERNEL); if (!held) { - kfree(linf); + kfree(linfo); return -ENOMEM; } spin_lock_init(&held->lock); INIT_LIST_HEAD(&held->list); init_waitqueue_head(&held->waitq); - scoutfs_key_init_buf_len(&held->fake_lvb_key, &held->fake_lvb_key_data, - 0, sizeof(held->fake_lvb_key_data)); - linf->sb = sb; - linf->shutdown = false; - linf->held = held; - INIT_LIST_HEAD(&linf->id_head); - INIT_LIST_HEAD(&linf->global_head); + linfo->sb = sb; + linfo->shutdown = false; + linfo->held = held; + INIT_LIST_HEAD(&linfo->id_head); + linfo->ls = NULL; - sbi->lock_info = linf; + snprintf(linfo->ls_name, DLM_LOCKSPACE_LEN, "%llx", + le64_to_cpu(sbi->super.hdr.fsid)); - trace_printk("sb %p id %016llx allocated linf %p held %p\n", - sb, le64_to_cpu(sbi->super.id), linf, held); + sbi->lock_info = linfo; - down_write(&global_rwsem); - - list_for_each_entry(other_linf, &global_super_list, global_head) { - other_sbi = SCOUTFS_SB(other_linf->sb); - if (other_sbi->super.id == sbi->super.id) { - list_add(&linf->id_head, &other_linf->id_head); - linf->held = other_linf->held; - trace_printk("sharing held %p\n", linf->held); - break; - } - } - - /* add to global list after walking so we don't see ourselves */ - list_add(&linf->global_head, &global_super_list); - - up_write(&global_rwsem); - - if (linf->held != held) - kfree(held); + trace_printk("sb %p id %016llx allocated linfo %p held %p\n", + sb, le64_to_cpu(sbi->super.id), linfo, held); return 0; } +static int can_complete_shutdown(struct held_locks *held) +{ + int ret; + + spin_lock(&held->lock); + ret = !!list_empty(&held->list); + spin_unlock(&held->lock); + return ret; +} + /* * Cause all lock attempts from our super to fail, waking anyone who is * currently blocked attempting to lock. Now that locks can't block we @@ -333,13 +291,13 @@ int scoutfs_lock_setup(struct super_block *sb) */ void scoutfs_lock_shutdown(struct super_block *sb) { - DECLARE_LOCK_INFO(sb, linf); - struct held_locks *held = linf->held; + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; - if (linf) { - held = linf->held; + if (linfo) { + held = linfo->held; spin_lock(&held->lock); - linf->shutdown = true; + linfo->shutdown = true; spin_unlock(&held->lock); wake_up(&held->waitq); @@ -349,27 +307,50 @@ void scoutfs_lock_shutdown(struct super_block *sb) void scoutfs_lock_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_LOCK_INFO(sb, linf); + DECLARE_LOCK_INFO(sb, linfo); struct held_locks *held; + int ret; - if (linf) { - down_write(&global_rwsem); + if (linfo) { + held = linfo->held; + wait_event(held->waitq, can_complete_shutdown(held)); - list_del_init(&linf->global_head); + ret = dlm_release_lockspace(linfo->ls, 2); + if (ret) + scoutfs_info(sb, "Error %d releasing lockspace %s\n", + ret, linfo->ls_name); - if (!list_empty(&linf->id_head)) { - list_del_init(&linf->id_head); - held = NULL; - } else { - held = linf->held; - } + sbi->lock_info = NULL; - up_write(&global_rwsem); - - trace_printk("sb %p id %016llx freeing linf %p held %p\n", - sb, le64_to_cpu(sbi->super.id), linf, held); + trace_printk("sb %p id %016llx freeing linfo %p held %p\n", + sb, le64_to_cpu(sbi->super.id), linfo, held); kfree(held); - kfree(linf); + kfree(linfo); } } + +int scoutfs_lock_setup(struct super_block *sb) +{ + struct lock_info *linfo; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret; + + ret = init_lock_info(sb); + if (ret) + return ret; + + linfo = sbi->lock_info; + /* + * Open coded '64' here is for lvb_len. We never use the LVB + * flag so this doesn't matter, but the dlm needs a non-zero + * multiple of 8 + */ + ret = dlm_new_lockspace(linfo->ls_name, sbi->opts.cluster_name, + DLM_LSFL_FS|DLM_LSFL_NEWEXCL, 64, NULL, + NULL, NULL, &linfo->ls); + if (ret) + scoutfs_lock_destroy(sb); + + return ret; +} diff --git a/kmod/src/lock.h b/kmod/src/lock.h index fe011988..89e820e9 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -1,30 +1,38 @@ #ifndef _SCOUTFS_LOCK_H_ #define _SCOUTFS_LOCK_H_ +#include "../dlm/include/linux/dlm.h" + struct scoutfs_lock { struct list_head head; struct super_block *sb; struct scoutfs_key_buf *start; struct scoutfs_key_buf *end; int mode; + int rqmode; + struct dlm_lksb lksb; + struct dlm_key dlm_start; + struct dlm_key dlm_end; + unsigned int sequence; /* for debugging and sanity checks */ }; enum { - SCOUTFS_LOCK_MODE_READ, - SCOUTFS_LOCK_MODE_WRITE, + SCOUTFS_LOCK_MODE_IV = DLM_LOCK_IV, + SCOUTFS_LOCK_MODE_READ = DLM_LOCK_PR, + SCOUTFS_LOCK_MODE_WRITE = DLM_LOCK_EX, }; int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, struct scoutfs_lock *lck); -int scoutfs_lock_range_lvb(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - void *caller_lvb, unsigned lvb_len, - struct scoutfs_lock *lck); void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck); +int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, + void *caller_lvb, unsigned lvb_len); +void scoutfs_unlock_addr(struct super_block *sb, void *caller_lvb, + unsigned lvb_len); + int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index f915e24a..f50c1fa3 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "seg.h" #include "compact.h" #include "scoutfs_trace.h" +#include "msg.h" /* * scoutfs mounts use a simple client-server model to send and process @@ -889,7 +891,7 @@ static void scoutfs_net_proc_func(struct work_struct *work) while (!nti->server_loaded) { mutex_lock(&nti->mutex); if (!nti->server_loaded) { - ret = scoutfs_read_supers(sb) ?: + ret = scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_manifest_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_compact_setup(sb); @@ -1108,18 +1110,29 @@ static void set_sock_callbacks(struct sock_info *sinf) } -/* get or set the address of the listening server depending on mode */ -static int lock_addr_lvb(struct super_block *sb, int mode, - struct scoutfs_inet_addr *addr) +static int write_server_addr(struct super_block *sb, + struct scoutfs_inet_addr *addr) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + + super->server_addr.addr = addr->addr; + super->server_addr.port = addr->port; + + return scoutfs_write_dirty_super(sb); +} + +static int read_server_addr(struct super_block *sb, + struct scoutfs_inet_addr *addr) { - struct scoutfs_lock lck; int ret; + struct scoutfs_super_block stack; - ret = scoutfs_lock_range_lvb(sb, mode, &addr_key, &addr_key, - addr, sizeof(*addr), &lck); - if (ret == 0) - scoutfs_unlock_range(sb, &lck); - + ret = scoutfs_read_supers(sb, &stack); + if (ret == 0) { + addr->addr = stack.server_addr.addr; + addr->port = stack.server_addr.port; + } return ret; } @@ -1177,6 +1190,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) struct super_block *sb = sinf->sb; DECLARE_NET_INFO(sb, nti); struct socket *sock = sinf->sock; + int ret; trace_printk("sinf %p sock %p shutting_down %d\n", sinf, sock, sinf->shutting_down); @@ -1198,9 +1212,13 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) destroy_server_state(sb); nti->server_loaded = false; - /* clear addr lvb and try to reacquire lock and listen */ + /* clear addr, try to reacquire lock and listen */ memset(&sinf->addr, 0, sizeof(sinf->addr)); - lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_WRITE, &sinf->addr); + ret = write_server_addr(sb, &sinf->addr); + if (ret) + scoutfs_err(sb, + "Non-fatal error %d while writing server " + "address\n", ret); scoutfs_unlock_range(sb, &sinf->listen_lck); queue_delayed_work(nti->proc_wq, &nti->server_work, 0); @@ -1883,6 +1901,7 @@ static void scoutfs_net_listen_func(struct work_struct *work) struct sock_info *sinf = container_of(work, struct sock_info, listen_work); struct super_block *sb = sinf->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_inet_addr addr; struct sockaddr_in sin; struct socket *sock; @@ -1890,10 +1909,9 @@ static void scoutfs_net_listen_func(struct work_struct *work) int optval; int ret; - /* XXX option to set listening address */ sin.sin_family = AF_INET; - sin.sin_addr.s_addr = cpu_to_be32(INADDR_LOOPBACK); - sin.sin_port = 0; + sin.sin_addr.s_addr = cpu_to_be32(le32_to_cpu(sbi->opts.listen_addr.addr)); + sin.sin_port = cpu_to_be16(le16_to_cpu(sbi->opts.listen_addr.port)); trace_printk("binding to %pIS:%u\n", &sin, be16_to_cpu(sin.sin_port)); @@ -1931,10 +1949,17 @@ static void scoutfs_net_listen_func(struct work_struct *work) set_sock_callbacks(sinf); - ret = kernel_listen(sock, 255) ?: - lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_WRITE, &addr); - if (ret == 0) - queue_sock_work(sinf, &sinf->accept_work); + ret = kernel_listen(sock, 255); + if (ret) + goto out; + + scoutfs_advance_dirty_super(sb); + ret = write_server_addr(sb, &addr); + if (ret) + goto out; + scoutfs_advance_dirty_super(sb); + + queue_sock_work(sinf, &sinf->accept_work); out: if (ret) { @@ -2012,7 +2037,7 @@ static void scoutfs_net_client_func(struct work_struct *work) INIT_WORK(&sinf->send_work, scoutfs_net_send_func); INIT_WORK(&sinf->recv_work, scoutfs_net_recv_func); - ret = lock_addr_lvb(sb, SCOUTFS_LOCK_MODE_READ, &sinf->addr); + ret = read_server_addr(sb, &sinf->addr); if (ret == 0 && sinf->addr.addr == cpu_to_le32(INADDR_ANY)) ret = -ENOENT; if (ret < 0) { diff --git a/kmod/src/options.c b/kmod/src/options.c new file mode 100644 index 00000000..9f909e62 --- /dev/null +++ b/kmod/src/options.c @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "msg.h" +#include "options.h" + +enum { + Opt_listen = 0, + Opt_cluster, + Opt_err, +}; + +static const match_table_t tokens = { + {Opt_listen, "listen=%s"}, + {Opt_cluster, "cluster=%s"}, + {Opt_err, NULL} +}; + +int scoutfs_parse_options(struct super_block *sb, char *options, + struct mount_options *parsed) +{ + char ipstr[INET_ADDRSTRLEN + 1]; + substring_t args[MAX_OPT_ARGS]; + int token, len; + __be32 addr; + char *p; + + /* Set defaults */ + memset(parsed, 0, sizeof(*parsed)); + strcpy(parsed->cluster_name, "scoutfs"); + + while ((p = strsep(&options, ",")) != NULL) { + if (!*p) + continue; + + token = match_token(p, tokens, args); + switch (token) { + case Opt_listen: + match_strlcpy(ipstr, args, ARRAY_SIZE(ipstr)); + addr = in_aton(ipstr); + if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) || + ipv4_is_zeronet(addr) || ipv4_is_local_multicast(addr)) + return -EINVAL; + parsed->listen_addr.addr = + cpu_to_le32(be32_to_cpu(addr)); + break; + case Opt_cluster: + len = args[0].to - args[0].from; + if (len == 0 || len > (MAX_CLUSTER_NAME_LEN - 1)) + return -EINVAL; + match_strlcpy(parsed->cluster_name, args, + MAX_CLUSTER_NAME_LEN); + break; + default: + scoutfs_err(sb, "Unknown or malformed option, \"%s\"\n", + p); + break; + } + } + + return 0; +} diff --git a/kmod/src/options.h b/kmod/src/options.h new file mode 100644 index 00000000..30009faf --- /dev/null +++ b/kmod/src/options.h @@ -0,0 +1,17 @@ +#ifndef _SCOUTFS_OPTIONS_H_ +#define _SCOUTFS_OPTIONS_H_ + +#include +#include "format.h" + +#define MAX_CLUSTER_NAME_LEN 17 +struct mount_options +{ + struct scoutfs_inet_addr listen_addr; + char cluster_name[MAX_CLUSTER_NAME_LEN]; +}; + +int scoutfs_parse_options(struct super_block *sb, char *options, + struct mount_options *parsed); + +#endif /* _SCOUTFS_OPTIONS_H_ */ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 24348d6b..98bb8f25 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -267,16 +267,21 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_ARGS(sb, lck), TP_STRUCT__entry( __field(int, mode) + __field(int, rqmode) + __field(unsigned int, seq) __dynamic_array(char, start, scoutfs_key_str(NULL, lck->start)) __dynamic_array(char, end, scoutfs_key_str(NULL, lck->end)) ), TP_fast_assign( __entry->mode = lck->mode; + __entry->rqmode = lck->rqmode; + __entry->seq = lck->sequence; scoutfs_key_str(__get_dynamic_array(start), lck->start); scoutfs_key_str(__get_dynamic_array(end), lck->end); ), - TP_printk("mode %s start %s end %s", - lock_mode(__entry->mode), __get_str(start), __get_str(end)) + TP_printk("seq %u mode %s rqmode %s start %s end %s", + __entry->seq, lock_mode(__entry->mode), + lock_mode(__entry->rqmode), __get_str(start), __get_str(end)) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_range, @@ -289,6 +294,11 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock_range, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_ast, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + TRACE_EVENT(scoutfs_lock_invalidate_sb, TP_PROTO(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), diff --git a/kmod/src/super.c b/kmod/src/super.c index 90367575..4c0ba6a3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -36,6 +37,7 @@ #include "data.h" #include "lock.h" #include "net.h" +#include "options.h" #include "scoutfs_trace.h" static struct kset *scoutfs_kset; @@ -139,7 +141,8 @@ int scoutfs_write_dirty_super(struct super_block *sb) * to re-read the super every time it comes up so that it can work from * the most recent persistent state. */ -int scoutfs_read_supers(struct super_block *sb) +int scoutfs_read_supers(struct super_block *sb, + struct scoutfs_super_block *local) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; @@ -147,6 +150,7 @@ int scoutfs_read_supers(struct super_block *sb) int found = -1; int ret; int i; + u64 seq = 0; page = alloc_page(GFP_KERNEL); if (!page) @@ -168,9 +172,9 @@ int scoutfs_read_supers(struct super_block *sb) continue; } - if (found < 0 || (le64_to_cpu(super->hdr.seq) > - le64_to_cpu(sbi->super.hdr.seq))) { - sbi->super = *super; + if (found < 0 || (le64_to_cpu(super->hdr.seq) > seq)) { + *local = *super; + seq = le64_to_cpu((*local).hdr.seq); found = i; } } @@ -191,6 +195,7 @@ int scoutfs_read_supers(struct super_block *sb) static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; + struct mount_options opts; struct inode *inode; int ret; @@ -221,8 +226,14 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi->kset) return -ENOMEM; + ret = scoutfs_parse_options(sb, data, &opts); + if (ret) + return ret; + + sbi->opts = opts; + ret = scoutfs_setup_counters(sb) ?: - scoutfs_read_supers(sb) ?: + scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_inode_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 350dbca9..fc278b69 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,6 +5,7 @@ #include #include "format.h" +#include "options.h" struct scoutfs_counters; struct item_cache; @@ -54,6 +55,8 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; + + struct mount_options opts; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) @@ -61,7 +64,8 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } -int scoutfs_read_supers(struct super_block *sb); +int scoutfs_read_supers(struct super_block *sb, + struct scoutfs_super_block *local); void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); From 94e78414f94419ea6b08c13a96118d2065b02566 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 22 Jun 2017 13:00:33 -0700 Subject: [PATCH 299/920] scoutfs: add key trace class Some item tracing functions were really just tracing a key. Refactor it into a trace class with event users. Later patches can then use the key trace class. Signed-off-by: Zach Brown --- kmod/src/item.c | 4 ++-- kmod/src/scoutfs_trace.h | 26 +++++++++++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index b3e9d8f5..b0ab001d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -408,7 +408,7 @@ restart: } } - trace_scoutfs_item_insertion(sb, ins->key, ins->val); + trace_scoutfs_item_insertion(sb, ins->key); rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); @@ -632,7 +632,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; - trace_scoutfs_item_lookup(sb, key, val); + trace_scoutfs_item_lookup(sb, key); end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); if (!end) { diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 98bb8f25..d31c0415 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -204,10 +204,9 @@ TRACE_EVENT(scoutfs_manifest_add, __entry->seq, __entry->level) ); -TRACE_EVENT(scoutfs_item_lookup, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val), - TP_ARGS(sb, key, val), +DECLARE_EVENT_CLASS(scoutfs_key_class, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_ARGS(sb, key), TP_STRUCT__entry( __dynamic_array(char, key, scoutfs_key_str(NULL, key)) ), @@ -217,17 +216,14 @@ TRACE_EVENT(scoutfs_item_lookup, TP_printk("key %s", __get_str(key)) ); -TRACE_EVENT(scoutfs_item_insertion, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val), - TP_ARGS(sb, key, val), - TP_STRUCT__entry( - __dynamic_array(char, key, scoutfs_key_str(NULL, key)) - ), - TP_fast_assign( - scoutfs_key_str(__get_dynamic_array(key), key); - ), - TP_printk("key %s", __get_str(key)) +DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_ARGS(sb, key) +); + +DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_ARGS(sb, key) ); DECLARE_EVENT_CLASS(scoutfs_range_class, From d52f09449d5f1e726544b2f741504b1c8db69cfe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Jun 2017 08:57:22 -0700 Subject: [PATCH 300/920] scoutfs: reclaim item cache Add a LRU and shrinker to reclaim old cached items under memory pressure. This is pretty awful today because of the separate cached range structs and rbtree. We do our best to blow away enough of the cache and range to try and make progress. Signed-off-by: Zach Brown --- kmod/src/counters.h | 10 +- kmod/src/item.c | 294 +++++++++++++++++++++++++++++++++++---- kmod/src/scoutfs_trace.h | 11 ++ 3 files changed, 283 insertions(+), 32 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e5d31e5a..e0b92c44 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -34,10 +34,16 @@ EXPAND_COUNTER(item_delete) \ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ - EXPAND_COUNTER(item_range_insert) + EXPAND_COUNTER(item_range_insert) \ + EXPAND_COUNTER(item_shrink_no_items) \ + EXPAND_COUNTER(item_shrink_outside) \ + EXPAND_COUNTER(item_shrink_dirty_abort) \ + EXPAND_COUNTER(item_shrink_skip_inced) \ + EXPAND_COUNTER(item_shrink_range) \ + EXPAND_COUNTER(item_shrink) #define FIRST_COUNTER alloc_alloc -#define LAST_COUNTER item_range_insert +#define LAST_COUNTER item_shrink #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/item.c b/kmod/src/item.c index b0ab001d..c9c13906 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -47,6 +47,8 @@ static bool invalid_flags(int sif) } struct item_cache { + struct super_block *sb; + spinlock_t lock; struct rb_root items; struct rb_root ranges; @@ -54,20 +56,23 @@ struct item_cache { long nr_dirty_items; long dirty_key_bytes; long dirty_val_bytes; + + struct shrinker shrinker; + struct list_head lru_list; + unsigned long lru_nr; }; /* * The dirty bits track if the given item is dirty and if its child * subtrees contain any dirty items. * - * The entry is only used when the items are in a private batch list - * before insertion. + * The entry list_head typically stores clean items on an lru for shrinking. + * It's also briefly used to track items in a batch after they're + * allocated but before they're inserted for the first time. */ struct cached_item { - union { - struct rb_node node; - struct list_head entry; - }; + struct rb_node node; + struct list_head entry; long dirty; unsigned deletion:1; @@ -92,6 +97,8 @@ static u8 item_flags(struct cached_item *item) static void free_item(struct super_block *sb, struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { + WARN_ON_ONCE(!list_empty(&item->entry)); + WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); scoutfs_key_free(sb, item->key); scoutfs_kvec_kfree(item->val); kfree(item); @@ -106,6 +113,9 @@ static struct cached_item *alloc_item(struct super_block *sb, item = kzalloc(sizeof(struct cached_item), GFP_NOFS); if (item) { + RB_CLEAR_NODE(&item->node); + INIT_LIST_HEAD(&item->entry); + if (!val) scoutfs_kvec_init_null(item->val); @@ -301,6 +311,9 @@ static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, return; item->dirty |= ITEM_DIRTY; + list_del_init(&item->entry); + cac->lru_nr--; + cac->nr_dirty_items++; cac->dirty_key_bytes += item->key->key_len; cac->dirty_val_bytes += scoutfs_kvec_length(item->val); @@ -321,6 +334,9 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, return; item->dirty &= ~ITEM_DIRTY; + list_add_tail(&item->entry, &cac->lru_list); + cac->lru_nr++; + cac->nr_dirty_items--; cac->dirty_key_bytes -= item->key->key_len; cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); @@ -334,6 +350,12 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, update_dirty_parents(item); } +static void item_referenced(struct item_cache *cac, struct cached_item *item) +{ + if (!item->dirty) + list_move_tail(&item->entry, &cac->lru_list); +} + /* * Safely erase an item from the tree. Make sure to remove its dirty * accounting, use the augmented erase, and free it. @@ -345,6 +367,11 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, clear_item_dirty(sb, cac, item); rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); + RB_CLEAR_NODE(&item->node); + if (!list_empty(&item->entry)) { + list_del_init(&item->entry); + cac->lru_nr--; + } free_item(sb, item); } @@ -413,9 +440,58 @@ restart: rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); + BUG_ON(ins->dirty & ITEM_DIRTY); + list_add_tail(&ins->entry, &cac->lru_list); + cac->lru_nr++; + return 0; } +static struct cached_range *rb_first_rng(struct rb_root *root) +{ + struct rb_node *node; + + if ((node = rb_first(root))) + return container_of(node, struct cached_range, node); + + return NULL; +} + +static struct cached_range *walk_ranges(struct rb_root *root, + struct scoutfs_key_buf *key, + struct cached_range **prev, + struct cached_range **next) +{ + struct rb_node *node = root->rb_node; + struct cached_range *rng; + int cmp; + + if (prev) + *prev = NULL; + if (next) + *next = NULL; + + while (node) { + rng = container_of(node, struct cached_range, node); + + cmp = scoutfs_key_compare_ranges(key, key, + rng->start, rng->end); + if (cmp < 0) { + if (next) + *next = rng; + node = node->rb_left; + } else if (cmp > 0) { + if (prev) + *prev = rng; + node = node->rb_right; + } else { + return rng; + } + } + + return NULL; +} + /* * Return true if the given key is covered by a cached range. end is * set to the end of the cached range. @@ -428,26 +504,16 @@ static bool check_range(struct super_block *sb, struct rb_root *root, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end) { - struct rb_node *node = root->rb_node; - struct cached_range *next = NULL; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_range *next; struct cached_range *rng; - int cmp; - while (node) { - rng = container_of(node, struct cached_range, node); - - cmp = scoutfs_key_compare_ranges(key, key, - rng->start, rng->end); - if (cmp < 0) { - next = rng; - node = node->rb_left; - } else if (cmp > 0) { - node = node->rb_right; - } else { - scoutfs_key_copy(end, rng->end); - scoutfs_inc_counter(sb, item_range_hit); - return true; - } + rng = walk_ranges(&cac->ranges, key, NULL, &next); + if (rng) { + scoutfs_key_copy(end, rng->end); + scoutfs_inc_counter(sb, item_range_hit); + return true; } if (next) @@ -644,12 +710,14 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); - if (item) + if (item) { + item_referenced(cac, item); ret = scoutfs_kvec_memcpy(val, item->val); - else if (check_range(sb, &cac->ranges, key, end)) + } else if (check_range(sb, &cac->ranges, key, end)) { ret = -ENOENT; - else + } else { ret = -ENODATA; + } spin_unlock_irqrestore(&cac->lock, flags); @@ -795,10 +863,12 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, if (cached && (item = item_for_next(&cac->items, key, range_end, last))) { scoutfs_key_copy(key, item->key); - if (val) + if (val) { + item_referenced(cac, item); ret = scoutfs_kvec_memcpy(val, item->val); - else + } else { ret = 0; + } break; } @@ -990,7 +1060,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, insert_range(sb, &cac->ranges, rng); list_for_each_entry_safe(item, tmp, list, entry) { - list_del(&item->entry); + list_del_init(&item->entry); if (insert_item(sb, cac, item, false)) list_add(&item->entry, list); } @@ -1625,6 +1695,160 @@ out: return ret; } +static struct cached_item *rb_next_item(struct cached_item *item) +{ + struct rb_node *node; + + if (item && (node = rb_next(&item->node))) + return container_of(node, struct cached_item, node); + + return NULL; +} + +/* + * Shrink the item cache. + * + * Unfortunately this is complicated by the rbtree of ranges that track + * the validity of the cache. If we free items we have to make sure + * they're not covered by ranges or else they'd be considered a valid + * negative cache hit. We don't want to allocate more memory for new + * range entries that would be required to poke holes int he cached + * range. + * + * So instead of just freeing the oldest item we shrink the range that + * contains the oldest item. We bias towards freeing the lesser side of + * the range. + * + * Instead of allocating a new range start key we use the key of the + * item we're removing. We have to increment it past the removed key + * value. That increment can move it past the next key in the range if + * the next key is of higher precision. This will be rare and can't go + * on indefinitely so we keep searching until we can inc a key and not + * extend past the next item. Eventually we have a range of items to + * free. + * + * During all of this, we chose to abort if we see dirty items. They + * won't be dirty forever and the mm can call back in. + * + * We can also hit items in the lru which aren't covered by ranges. We + * just free them straight away. And finally if we're completely out of + * items we walk and free the ranges. + */ +static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) +{ + struct item_cache *cac = container_of(shrink, struct item_cache, + shrinker); + struct super_block *sb = cac->sb; + struct cached_range *rng; + struct cached_item *item; + struct cached_item *next; + struct cached_item *begin; + struct cached_item *end; + unsigned long flags; + unsigned long nr; + + nr = sc->nr_to_scan; + if (nr == 0) + goto out; + + spin_lock_irqsave(&cac->lock, flags); + + while (nr > 0) { + item = list_first_entry_or_null(&cac->lru_list, + struct cached_item, entry); + + /* no lru items, if no items at all then free ranges */ + if (!item) { + if (!RB_EMPTY_ROOT(&cac->items)) { + scoutfs_inc_counter(sb, item_shrink_dirty_abort); + goto abort; + } + rng = rb_first_rng(&cac->ranges); + if (!rng) + break; + scoutfs_inc_counter(sb, item_shrink_no_items); + begin = NULL; + end = NULL; + goto free; + } + + /* can't have dirty items on the lru */ + BUG_ON(item->dirty & ITEM_DIRTY); + + /* if we're not in a range just shrink the item */ + rng = walk_ranges(&cac->ranges, item->key, NULL, NULL); + if (!rng) { + begin = item; + end = item; + scoutfs_inc_counter(sb, item_shrink_outside); + goto free; + } + + /* find the string of items to free, ending with range start */ + item = next_item(&cac->items, rng->start); + begin = item; + end = item; + + while (item) { + /* can't if it's dirty :( */ + if (item->dirty & ITEM_DIRTY) { + scoutfs_inc_counter(sb, item_shrink_dirty_abort); + goto abort; + } + + /* we're going to free this item now */ + end = item; + + /* free items and range if we exhausted the range */ + next = rb_next_item(item); + if (!next || scoutfs_key_compare(next->key, rng->end) > 0) + break; + + /* truncate range using after our key as start, if safe */ + scoutfs_key_inc_cur_len(item->key); + if (scoutfs_key_compare(item->key, next->key) <= 0) { + trace_scoutfs_item_shrink(sb, item->key); + scoutfs_key_free(sb, rng->start); + rng->start = item->key; + item->key = NULL; + rng = NULL; + break; + } + scoutfs_key_dec_cur_len(item->key); + + /* keep searching for valid range start key */ + scoutfs_inc_counter(sb, item_shrink_skip_inced); + item = next; + } + +free: + if (rng) { + trace_scoutfs_item_shrink_range(sb, rng->start, rng->end); + scoutfs_inc_counter(sb, item_shrink_range); + rb_erase(&rng->node, &cac->ranges); + free_range(sb, rng); + } + + /* free items from begin to end */ + for (item = begin; + item && (next = item == end ? NULL : rb_next_item(item), 1); + item = next) { + if (item->key) + trace_scoutfs_item_shrink(sb, item->key); + scoutfs_inc_counter(sb, item_shrink); + erase_item(sb, cac, item); + } + + nr--; + } + +abort: + spin_unlock_irqrestore(&cac->lock, flags); + +out: + return min_t(unsigned long, cac->lru_nr, INT_MAX); +} + int scoutfs_item_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1635,9 +1859,14 @@ int scoutfs_item_setup(struct super_block *sb) return -ENOMEM; sbi->item_cache = cac; + cac->sb = sb; spin_lock_init(&cac->lock); cac->items = RB_ROOT; cac->ranges = RB_ROOT; + cac->shrinker.shrink = item_lru_shrink; + cac->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&cac->shrinker); + INIT_LIST_HEAD(&cac->lru_list); return 0; } @@ -1656,8 +1885,13 @@ void scoutfs_item_destroy(struct super_block *sb) struct cached_range *pos_rng; if (cac) { + if (cac->shrinker.shrink == item_lru_shrink) + unregister_shrinker(&cac->shrinker); + rbtree_postorder_for_each_entry_safe(item, pos_item, &cac->items, node) { + RB_CLEAR_NODE(&item->node); + INIT_LIST_HEAD(&item->entry); free_item(sb, item); } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index d31c0415..b7b87809 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -226,6 +226,11 @@ DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, TP_ARGS(sb, key) ); +DEFINE_EVENT(scoutfs_key_class, scoutfs_item_shrink, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_ARGS(sb, key) +); + DECLARE_EVENT_CLASS(scoutfs_range_class, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), @@ -253,6 +258,12 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, TP_ARGS(sb, start, end) ); +DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), + TP_ARGS(sb, start, end) +); + #define lock_mode(mode) \ __print_symbolic(mode, \ { SCOUTFS_LOCK_MODE_READ, "READ" }, \ From 5f5729b2a431fadbdaef214789e0d4cc853df254 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Jun 2017 09:28:48 -0700 Subject: [PATCH 301/920] scoutfs: add sticky compaction As we write segments we're not limiting the number of segments they intersect at the next level. Compactions are limited to a fanout's worth of overlapping segments. This means that we can get a compaction where the upper level segment overlapps more than the segments that are part of the compaction. In this case we can't write the remaining upper level items at the lower level because now we can have a level with segments whose keys intersect. Instead we detect this compaction case. We call it sticky because after merging with the lower level segments the remaining items in the upper level need to stick to the upper level. The next time compaction comes around it'll compact the remaining items with the additional lower overlaping segments. Signed-off-by: Zach Brown --- kmod/src/compact.c | 97 ++++++++++++++++++++++++++++++--------------- kmod/src/compact.h | 2 +- kmod/src/counters.h | 3 +- kmod/src/format.h | 10 +++++ kmod/src/manifest.c | 13 +++++- kmod/src/net.c | 3 +- 6 files changed, 90 insertions(+), 38 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index f02234e0..9c83eca6 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -83,7 +83,7 @@ struct compact_cursor { struct list_head csegs; /* buffer holds allocations and our returning them */ - u64 segnos[2 * (1 + SCOUTFS_MANIFEST_FANOUT)]; + u64 segnos[SCOUTFS_COMPACTION_MAX_UPDATE]; unsigned nr_segnos; u8 lower_level; @@ -93,6 +93,9 @@ struct compact_cursor { struct compact_seg *saved_upper; struct compact_seg *lower; struct compact_seg *saved_lower; + + bool sticky; + struct compact_seg *last_lower; }; static void free_cseg(struct super_block *sb, struct compact_seg *cseg) @@ -256,6 +259,19 @@ retry: *item_flags = lower_flags; } + /* + * If we have a sticky compaction then we can't mix items from + * the upper level past the last lower key into the lower level. + * The caller will notice when they're emptying the final upper + * level in a sticky merge and leave it at the upper level. + */ + if (curs->sticky && curs->lower && + (!lower || lower == curs->last_lower) && + scoutfs_key_compare(item_key, curs->last_lower->last) > 0) { + ret = 0; + goto out; + } + if (cmp <= 0) upper->pos++; if (cmp >= 0) @@ -346,7 +362,6 @@ static int compact_segments(struct super_block *sb, struct scoutfs_bio_completion *comp, struct list_head *results) { - struct scoutfs_key_buf upper_next; struct scoutfs_segment *seg; struct compact_seg *cseg; struct compact_seg *upper; @@ -357,24 +372,25 @@ static int compact_segments(struct super_block *sb, int ret; scoutfs_inc_counter(sb, compact_operations); + if (curs->sticky) + scoutfs_inc_counter(sb, compact_sticky_upper); for (;;) { upper = curs->upper; lower = curs->lower; /* - * We can just move the upper segment down a level if it - * doesn't intersect any lower segments. + * If we're at the start of the upper segment and + * there's no lower segment then we might as well just + * move the segment in the manifest. We can't do this + * if we're moving to the last level because we might + * need to drop any deletion items. * - * XXX we can't do this if the segment we're moving has - * deletion items. We need to copy the non-deletion items - * and drop the deletion items in that case. To do that - * we'll need the manifest to count the number of deletion - * and non-deletion items. + * XXX We should have metadata in the manifest to tell + * us that there's no deletion items in the segment. */ - if (upper && upper->pos == 0 && - (!lower || - scoutfs_key_compare(upper->last, lower->first) < 0)) { + if (upper && upper->pos == 0 && !lower && !curs->sticky && + ((upper->level + 1) < curs->last_level)) { /* * XXX blah! these csegs are getting @@ -412,26 +428,17 @@ static int compact_segments(struct super_block *sb, break; /* - * We can skip a lower segment if there's no upper segment - * or the next upper item is past the last in the lower. + * XXX we could intelligently skip reading and merging + * lower segments here. The lower segment won't change + * if: + * - the lower segment is entirely before the upper + * - the lower segment is full * - * XXX this will need to test for intersection with range - * deletion items. + * We don't have the metadata to determine that it's + * full today so we want to read lower segments that don't + * overlap so that we can merge partial lowers with + * its neighbours. */ - if (lower && lower->pos == 0 && - (!upper || - (!scoutfs_seg_item_ptrs(upper->seg, upper->pos, - &upper_next, NULL, NULL) && - scoutfs_key_compare(&upper_next, lower->last) > 0))) { - - curs->lower = next_spos(curs, lower); - - list_del_init(&lower->entry); - free_cseg(sb, lower); - - scoutfs_inc_counter(sb, compact_segment_skipped); - continue; - } ret = read_segment(sb, lower); if (ret) @@ -467,8 +474,18 @@ static int compact_segments(struct super_block *sb, break; } + /* + * The remaining upper items in a sticky merge have to + * be written into the upper level. + */ + if (curs->sticky && !lower) { + cseg->level = curs->lower_level - 1; + scoutfs_inc_counter(sb, compact_sticky_written); + } else { + cseg->level = curs->lower_level; + } + /* csegs will be claned up once they're on the list */ - cseg->level = curs->lower_level; cseg->seg = seg; list_add_tail(&cseg->entry, results); @@ -476,6 +493,17 @@ static int compact_segments(struct super_block *sb, if (ret < 0) break; + /* + * Clear lower after we've consumed it so that sticky + * compaction can decide to write the rest of the items + * into the upper level. We decide that it's done by + * testing the pos that next_item() is going to try. + */ + if (curs->sticky && curs->lower == curs->last_lower && + scoutfs_seg_item_ptrs(curs->lower->seg, curs->lower->pos, + NULL, NULL, NULL) < 0) + curs->lower = NULL; + /* start a complete segment write now, we'll wait later */ ret = scoutfs_seg_submit_write(sb, seg, comp); if (ret) @@ -489,15 +517,16 @@ static int compact_segments(struct super_block *sb, /* * Manifest walking is providing the details of the overall compaction - * operation. It'll then add all the segments involved. + * operation. */ void scoutfs_compact_describe(struct super_block *sb, void *data, - u8 upper_level, u8 last_level) + u8 upper_level, u8 last_level, bool sticky) { struct compact_cursor *curs = data; curs->lower_level = upper_level + 1; curs->last_level = last_level; + curs->sticky = sticky; } /* @@ -531,6 +560,8 @@ int scoutfs_compact_add(struct super_block *sb, void *data, curs->upper = cseg; else if (!curs->lower) curs->lower = cseg; + if (curs->lower) + curs->last_lower = cseg; ret = 0; out: diff --git a/kmod/src/compact.h b/kmod/src/compact.h index e017dd87..f6f4bb60 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -4,7 +4,7 @@ void scoutfs_compact_kick(struct super_block *sb); void scoutfs_compact_describe(struct super_block *sb, void *data, - u8 upper_level, u8 last_level); + u8 upper_level, u8 last_level, bool sticky); int scoutfs_compact_add(struct super_block *sb, void *data, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, u64 segno, u64 seq, diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e0b92c44..a41340e2 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -19,9 +19,10 @@ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(compact_operations) \ EXPAND_COUNTER(compact_segment_moved) \ - EXPAND_COUNTER(compact_segment_skipped) \ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_written) \ + EXPAND_COUNTER(compact_sticky_upper) \ + EXPAND_COUNTER(compact_sticky_written) \ EXPAND_COUNTER(data_readpage) \ EXPAND_COUNTER(data_write_begin) \ EXPAND_COUNTER(data_write_end) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 26c2bbb8..3451a04b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -448,6 +448,16 @@ struct scoutfs_net_segnos { __le64 segnos[0]; } __packed; +/* XXX eventually we'll have net compaction and will need agents to agree */ + +/* one upper segment and fanout lower segments */ +#define SCOUTFS_COMPACTION_MAX_INPUT (1 + SCOUTFS_MANIFEST_FANOUT) +/* sticky can add one, and so can item page alignment */ +#define SCOUTFS_COMPACTION_SLOP 2 +/* delete all inputs and insert all outputs (same goes for alloc|free segnos) */ +#define SCOUTFS_COMPACTION_MAX_UPDATE \ + (2 * (SCOUTFS_COMPACTION_MAX_INPUT + SCOUTFS_COMPACTION_SLOP)) + enum { SCOUTFS_NET_ALLOC_INODES = 0, SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index a04fbe50..4cec4184 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -719,6 +719,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) struct scoutfs_key_buf ment_last; struct scoutfs_key_buf over_first; struct scoutfs_key_buf over_last; + bool sticky; int level; int ret; int nr = 0; @@ -739,7 +740,6 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } - scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1); /* find the oldest level 0 or the next higher order level by key */ if (level == 0) { @@ -779,7 +779,8 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) over = scoutfs_ring_lookup_next(&mani->ring, &skey); /* and add a fanout's worth of lower overlapping segments */ - for (i = 0; i < SCOUTFS_MANIFEST_FANOUT; i++) { + sticky = false; + for (i = 0; i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { if (!over || over->level != (ment->level + 1)) break; @@ -789,6 +790,12 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) &over_first, &over_last) != 0) break; + /* upper level has to stay around when more than fanout */ + if (i == SCOUTFS_MANIFEST_FANOUT) { + sticky = true; + break; + } + ret = scoutfs_compact_add(sb, data, &over_first, &over_last, le64_to_cpu(over->segno), le64_to_cpu(over->seq), level + 1); @@ -799,6 +806,8 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) over = scoutfs_ring_next(&mani->ring, over); } + scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1, sticky); + /* record the next key to start from */ scoutfs_key_copy(mani->compact_keys[level], &ment_last); scoutfs_key_inc(mani->compact_keys[level]); diff --git a/kmod/src/net.c b/kmod/src/net.c index f50c1fa3..76d9ca3a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1420,7 +1420,8 @@ int scoutfs_net_get_compaction(struct super_block *sb, void *curs) return nr; } - for (i = 0; i < nr; i++) { + /* allow for expansion slop from sticky and alignment */ + for (i = 0; i < nr + SCOUTFS_COMPACTION_SLOP; i++) { ret = scoutfs_alloc_segno(sb, &segno); if (ret < 0) break; From 2eecbbe78a6862d99c335077f0965871c221521a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Jun 2017 22:44:51 -0700 Subject: [PATCH 302/920] scoutfs: add item cache key ioctls These ioctls let userspace see the items and ranges that are cached. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 69 ++++++++++++++++++++++++++ kmod/src/ioctl.h | 16 ++++++ kmod/src/item.c | 124 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/item.h | 6 +++ 4 files changed, 215 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index e4cdeb7d..588dcf3e 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -433,6 +433,73 @@ static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) return 0; } +static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_item_cache_keys ick; + struct scoutfs_key_buf *key; + struct page *page; + unsigned bytes; + void *buf; + int total; + int ret; + + if (copy_from_user(&ick, (void __user *)arg, sizeof(ick))) + return -EFAULT; + + if ((!!ick.key_ptr != !!ick.key_len) || + ick.key_len > SCOUTFS_MAX_KEY_SIZE || + ick.which > SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES) + return -EINVAL; + + /* don't overflow signed 32bit syscall return longs */ + ick.buf_len = min_t(u64, ick.buf_len, S32_MAX); + + key = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + page = alloc_page(GFP_KERNEL); + if (!key || !page) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(key->data, (void __user *)ick.key_ptr, ick.key_len)) { + ret = -EFAULT; + goto out; + } + scoutfs_key_init_buf_len(key, key->data, ick.key_len, + SCOUTFS_MAX_KEY_SIZE); + scoutfs_key_inc(key); + + buf = page_address(page); + total = 0; + ret = 0; + while (ick.buf_len) { + bytes = min_t(u64, ick.buf_len, PAGE_SIZE); + + if (ick.which == SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS) + ret = scoutfs_item_copy_keys(sb, key, buf, bytes); + else + ret = scoutfs_item_copy_range_keys(sb, key, buf, bytes); + + if (ret > 0 && copy_to_user((void __user *)ick.buf_ptr, buf, ret)) + ret = -EFAULT; + if (ret <= 0) + break; + + ick.buf_len -= ret; + ick.buf_ptr += ret; + total += ret; + ret = 0; + } + +out: + scoutfs_key_free(sb, key); + if (page) + __free_page(page); + + return ret ?: total; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -446,6 +513,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_stage(file, arg); case SCOUTFS_IOC_STAT_MORE: return scoutfs_ioc_stat_more(file, arg); + case SCOUTFS_IOC_ITEM_CACHE_KEYS: + return scoutfs_ioc_item_cache_keys(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index d1814b8c..79241e9f 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -166,4 +166,20 @@ struct scoutfs_ioctl_stat_more { #define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_stat_more) +struct scoutfs_ioctl_item_cache_keys { + __u64 key_ptr; + __u64 key_len; + __u64 buf_ptr; + __u64 buf_len; + __u8 which; +} __packed; + +enum { + SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS = 0, + SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES, +}; + +#define SCOUTFS_IOC_ITEM_CACHE_KEYS _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ + struct scoutfs_ioctl_item_cache_keys) + #endif diff --git a/kmod/src/item.c b/kmod/src/item.c index c9c13906..5cfb7ab3 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -457,6 +457,16 @@ static struct cached_range *rb_first_rng(struct rb_root *root) return NULL; } +static struct cached_range *rb_next_rng(struct cached_range *rng) +{ + struct rb_node *node; + + if (rng && (node = rb_next(&rng->node))) + return container_of(node, struct cached_range, node); + + return NULL; +} + static struct cached_range *walk_ranges(struct rb_root *root, struct scoutfs_key_buf *key, struct cached_range **prev, @@ -1849,6 +1859,120 @@ out: return min_t(unsigned long, cac->lru_nr, INT_MAX); } +static void *copy_key_with_len(void *data, struct scoutfs_key_buf *key) +{ + u16 len = key->key_len; + + memcpy(data, &len, sizeof(len)); + data += sizeof(len); + memcpy(data, key->data, len); + + return data + len; +} + +/* + * Copy the next cached ranges starting with the key into the caller's + * buffer. Each range copied by storing each keys size in a u16 + * followed by the binary key data. The number of bytes of full copied + * ranges is returned. The caller's key is incremented past the last + * key returned so that they can iterate without worrying about + * examining the returned keys. + */ +int scoutfs_item_copy_range_keys(struct super_block *sb, + struct scoutfs_key_buf *key, void *data, + unsigned len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct rb_node *node = cac->ranges.rb_node; + struct cached_range *next = NULL; + struct scoutfs_key_buf *last = NULL; + struct cached_range *rng; + unsigned long flags; + unsigned bytes; + int ret = 0; + int cmp; + + spin_lock_irqsave(&cac->lock, flags); + + while (node) { + rng = container_of(node, struct cached_range, node); + + cmp = scoutfs_key_compare_ranges(key, key, + rng->start, rng->end); + if (cmp < 0) { + next = rng; + node = node->rb_left; + } else if (cmp > 0) { + node = node->rb_right; + } else { + next = rng; + break; + } + } + + for (rng = next; rng; rng = rb_next_rng(rng)) { + bytes = 2 + rng->start->key_len + 2 + rng->end->key_len; + if (len < bytes) + break; + + data = copy_key_with_len(data, rng->start); + data = copy_key_with_len(data, rng->end); + len -= bytes; + ret += bytes; + + last = rng->end; + } + + if (last) { + scoutfs_key_copy(key, last); + scoutfs_key_inc(key); + } + + spin_unlock_irqrestore(&cac->lock, flags); + + return ret; +} + +/* like copy_range_keys, but for present items */ +int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key_buf *key, + void *data, unsigned len) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct scoutfs_key_buf *last = NULL; + struct cached_item *item = NULL; + unsigned long flags; + unsigned bytes; + int ret = 0; + + spin_lock_irqsave(&cac->lock, flags); + + for (item = next_item(&cac->items, key); item; item = rb_next_item(item)) { + if (item->deletion) + continue; + + bytes = 2 + item->key->key_len; + if (len < bytes) + break; + + data = copy_key_with_len(data, item->key); + len -= bytes; + ret += bytes; + + last = item->key; + } + + if (last) { + scoutfs_key_copy(key, last); + scoutfs_key_inc(key); + } + + spin_unlock_irqrestore(&cac->lock, flags); + + return ret; +} + int scoutfs_item_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/item.h b/kmod/src/item.h index 68f14f8c..fb9c1df4 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -57,6 +57,12 @@ int scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); +int scoutfs_item_copy_range_keys(struct super_block *sb, + struct scoutfs_key_buf *key, void *data, + unsigned len); +int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key_buf *key, + void *data, unsigned len); + int scoutfs_item_setup(struct super_block *sb); void scoutfs_item_destroy(struct super_block *sb); From bf7b3ac5064c32febe63af6a6be678fa67a4d879 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Jun 2017 23:22:28 -0700 Subject: [PATCH 303/920] scoutfs: fix ring first_seq calculation As we write ring blocks we need to update the first_seq to point at the first live block in the ring. The existing calculation gets it wrong and stores the seq of the first block that we wrote in this commit, not the first ring block that is still live and would need to be read. Fix the calculation to so that we set first_seq to the first live block in the ring. This fixes the bug where a mount can spin printing the super it's using. This is the server trying to constantly startup as each server start fails as it can't read the ring. Signed-off-by: Zach Brown --- kmod/src/ring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 47a21bc9..657bfe82 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -731,7 +731,7 @@ int scoutfs_ring_submit_write(struct super_block *sb, nr = last + le64_to_cpu(rdesc->total_blocks) - first; rdesc->first_block = cpu_to_le64(first); - rdesc->first_seq = cpu_to_le64(ring->first_dirty_seq); + rdesc->first_seq = cpu_to_le64(ring->first_dirty_seq + nr_blocks - nr); rdesc->nr_blocks = cpu_to_le64(nr); /* the contig dirty blocks in pages might wrap around ring */ From ef551776aea9d641d034d49e65fc7f52547ae775 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 17 Jun 2017 09:51:51 -0700 Subject: [PATCH 304/920] scoutfs: item cache reads shouldn't clobber dirty We read into the item cache when we hit a region that isn't cached. Inode index items are created without cached range coverage. It's easy to trigger read attempts that overlap with dirty inode index items. insert_item() had a bug where it's notion of overwriting only applied to logical presence. It always let an insertion overwrite an existing item if it was a deletion. But that only makes sense for new item creation. Item cache population can't do this. In this inode index case it can replace a correct dirty inode index item with its old pre-deletion item from the read. This clobbers the deletions and leaks the old inode index item versions. So teach the item insertion for caching to never, ever, replace an existing item. This fixes assertion failures from trying to immediately walk meta seq items after creating a few thousand dirty entries. Signed-off-by: Zach Brown --- kmod/src/item.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 5cfb7ab3..79b18f60 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -394,14 +394,24 @@ static void become_deletion_item(struct super_block *sb, } /* - * Try to insert the given item. If there's already a non-deletion item - * with the insertion key then return -EEXIST. An existing deletion - * item is replaced and freed. + * Try to add an item to the cache. The caller is responsible for + * marking the newly inserted item dirty. * - * The caller is responsible for marking the newly inserted item dirty. + * We distinguish between callers seeing trying to insert a new logical + * item and others trying to populate the cache. + * + * New logical item creaters have made sure the items are participating + * in consistent locking. It's safe for them to clobber dirty deletion + * items with a new version of the item. + * + * Cache readers can only populate items that weren't present already. + * In particular, they absolutely cannot replace dirty old inode index items + * with the old version that was just deleted (outside of range caching and + * locking consistency). */ static int insert_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *ins, bool overwrite) + struct cached_item *ins, bool logical_overwrite, + bool cache_populate) { struct rb_root *root = &cac->items; struct cached_item *item; @@ -426,7 +436,8 @@ restart: item->dirty |= RIGHT_DIRTY; node = &(*node)->rb_right; } else { - if (!item->deletion && !overwrite) + if (cache_populate || + (!item->deletion && !logical_overwrite)) return -EEXIST; /* sadly there's no augmented replace */ @@ -986,7 +997,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return -ENOMEM; spin_lock_irqsave(&cac->lock, flags); - ret = insert_item(sb, cac, item, false); + ret = insert_item(sb, cac, item, false, false); if (!ret) { scoutfs_inc_counter(sb, item_create); mark_item_dirty(sb, cac, item); @@ -1071,7 +1082,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); - if (insert_item(sb, cac, item, false)) + if (insert_item(sb, cac, item, false, true)) list_add(&item->entry, list); } @@ -1188,7 +1199,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, /* insert the caller's items, overwriting any existing */ list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); - insert_item(sb, cac, item, true); + insert_item(sb, cac, item, true, false); mark_item_dirty(sb, cac, item); } From 71711c8b56b8cf0417f9a9e9614b9156e587cd73 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 17 Jun 2017 10:08:19 -0700 Subject: [PATCH 305/920] scoutfs: add manifest and item tracing Add some tracing to get visibility into compaction and item reading. Signed-off-by: Zach Brown --- kmod/src/compact.c | 5 ++-- kmod/src/manifest.c | 18 ++++++++++--- kmod/src/scoutfs_trace.h | 56 ++++++++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 9c83eca6..b129b931 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -666,9 +666,8 @@ static void scoutfs_compact_func(struct work_struct *work) /* trace compaction ranges */ list_for_each_entry(cseg, &curs.csegs, entry) { - SK_TRACE_PRINTK("level %u segno %llu first "SK_FMT" last "SK_FMT"\n", - cseg->level, cseg->segno, SK_ARG(cseg->first), - SK_ARG(cseg->last)); + trace_scoutfs_compact_input(sb, cseg->level, cseg->segno, + cseg->seq, cseg->first, cseg->last); } if (ret == 0 && !list_empty(&curs.csegs)) { diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 4cec4184..ed6736c4 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -130,7 +130,7 @@ int scoutfs_manifest_add(struct super_block *sb, unsigned key_bytes; unsigned bytes; - trace_scoutfs_manifest_add(sb, first, last, segno, seq, level); + trace_scoutfs_manifest_add(sb, level, segno, seq, first, last); key_bytes = first->key_len + last->key_len; bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); @@ -173,11 +173,14 @@ int scoutfs_manifest_add_ment(struct super_block *sb, struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; struct scoutfs_key_buf first; + struct scoutfs_key_buf last; unsigned bytes; lockdep_assert_held(&mani->rwsem); - init_ment_keys(add, &first, NULL); + init_ment_keys(add, &first, &last); + trace_scoutfs_manifest_add(sb, add->level, le64_to_cpu(add->segno), + le64_to_cpu(add->seq), &first, &last); skey.key = &first; skey.level = add->level; @@ -230,6 +233,7 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, struct scoutfs_super_block *super = &sbi->super; struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; + struct scoutfs_key_buf last; skey.key = first; skey.level = level; @@ -239,6 +243,10 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, if (!ment) return -ENOENT; + init_ment_keys(ment, NULL, &last); + trace_scoutfs_manifest_delete(sb, ment->level, le64_to_cpu(ment->segno), + le64_to_cpu(ment->seq), first, &last); + scoutfs_ring_delete(&mani->ring, ment); le64_add_cpu(&super->manifest.level_counts[level], -1ULL); return 0; @@ -500,7 +508,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, int cmp; int n; - trace_printk("reading items\n"); + trace_scoutfs_read_items(sb, key, end); /* get refs on all the segments */ ret = scoutfs_net_manifest_range_entries(sb, key, end, &ref_list); @@ -509,6 +517,10 @@ int scoutfs_manifest_read_items(struct super_block *sb, /* submit reads for all the segments */ list_for_each_entry(ref, &ref_list, entry) { + + trace_scoutfs_read_item_segment(sb, ref->level, ref->segno, + ref->seq, ref->first, ref->last); + seg = scoutfs_seg_submit_read(sb, ref->segno); if (IS_ERR(seg)) { ret = PTR_ERR(seg); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b7b87809..e03c2c6d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -181,27 +181,51 @@ TRACE_EVENT(scoutfs_scan_orphans, TP_printk("dev %d,%d", MAJOR(__entry->dev), MINOR(__entry->dev)) ); -TRACE_EVENT(scoutfs_manifest_add, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, u8 level), - TP_ARGS(sb, first, last, segno, seq, level), +DECLARE_EVENT_CLASS(scoutfs_manifest_class, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + TP_ARGS(sb, level, segno, seq, first, last), TP_STRUCT__entry( - __dynamic_array(char, first, scoutfs_key_str(NULL, first)) - __dynamic_array(char, last, scoutfs_key_str(NULL, last)) + __field(u8, level) __field(u64, segno) __field(u64, seq) - __field(u8, level) + __dynamic_array(char, first, scoutfs_key_str(NULL, first)) + __dynamic_array(char, last, scoutfs_key_str(NULL, last)) ), TP_fast_assign( - scoutfs_key_str(__get_dynamic_array(first), first); - scoutfs_key_str(__get_dynamic_array(last), last); + __entry->level = level; __entry->segno = segno; __entry->seq = seq; - __entry->level = level; + scoutfs_key_str(__get_dynamic_array(first), first); + scoutfs_key_str(__get_dynamic_array(last), last); ), - TP_printk("first %s last %s segno %llu seq %llu level %u", - __get_str(first), __get_str(last), __entry->segno, - __entry->seq, __entry->level) + TP_printk("level %u segno %llu seq %llu first %s last %s", + __entry->level, __entry->segno, __entry->seq, + __get_str(first), __get_str(last)) +); + +DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_add, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + TP_ARGS(sb, level, segno, seq, first, last) +); + +DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_delete, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + TP_ARGS(sb, level, segno, seq, first, last) +); + +DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_input, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + TP_ARGS(sb, level, segno, seq, first, last) +); + +DEFINE_EVENT(scoutfs_manifest_class, scoutfs_read_item_segment, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + TP_ARGS(sb, level, segno, seq, first, last) ); DECLARE_EVENT_CLASS(scoutfs_key_class, @@ -264,6 +288,12 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, TP_ARGS(sb, start, end) ); +DEFINE_EVENT(scoutfs_range_class, scoutfs_read_items, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), + TP_ARGS(sb, start, end) +); + #define lock_mode(mode) \ __print_symbolic(mode, \ { SCOUTFS_LOCK_MODE_READ, "READ" }, \ From 3b56161ed364c2d00ad764bc9380bf8c274274c6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 17 Jun 2017 11:39:39 -0700 Subject: [PATCH 306/920] scoutfs: fix item read seg walk limit When read items into the cache we have the range of keys that were missing from the cache. The item walk was stopping when it hit the end of the missing cache range, not when it hit the end of the keys that were covered by all the segments. This would manifest as huge regions of missing items. The read would walk off the relatively closed end of the highest level segment. It would keep reading while there were items in the upper levels but all those keys that would have been found in additional lower level segments are missing. Eventually it'd hit the end of the higher level sgement and mark that region as cached. With it fixed it now stops the read appropriately and will come around next time to read the range that coveres the next lowest level segment. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index ed6736c4..faa7ef96 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -547,11 +547,12 @@ int scoutfs_manifest_read_items(struct super_block *sb, ref->pos = scoutfs_seg_find_pos(ref->seg, key); /* - * Find the greatest range we can cover if we walk all the - * segments. We only have level 0 segments for the missing - * range so that's the greatest. Then we shrink the range by - * the limit of each higher level segment that intersected with - * our starting key. + * Find the limit of the range we can safely walk. We have all + * the level 0 segments that intersect with the caller's range. + * But we only have the level > 0 segments that intersected with + * the starting key. We have to stop at the nearest end of + * those segments because other segments might overlap after + * that. */ scoutfs_key_clone(&seg_end, end); list_for_each_entry(ref, &ref_list, entry) { @@ -576,13 +577,14 @@ int scoutfs_manifest_read_items(struct super_block *sb, /* * Check the next item in the segment. We're * done with the segment if there are no more - * items or if the next item is past the - * caller's end. + * items or if the next item is past the keys + * that our segments can see. */ ret = scoutfs_seg_item_ptrs(ref->seg, ref->pos, &item_key, item_val, &item_flags); - if (ret < 0 || scoutfs_key_compare(&item_key, end) > 0){ + if (ret < 0 || + scoutfs_key_compare(&item_key, &seg_end) > 0){ ref->pos = -1; continue; } @@ -636,7 +638,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, scoutfs_key_clone(&batch_end, &found_key); /* if we just saw the end key then we're done */ - if (scoutfs_key_compare(&found_key, end) == 0) { + if (scoutfs_key_compare(&found_key, &seg_end) == 0) { ret = 0; break; } From 793f84b86b7567082c0d56584d7f800309f28de0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 17 Jun 2017 20:43:09 -0700 Subject: [PATCH 307/920] scoutfs: remove item reading limit The item reading limit was intended to minimize latency when we were directly reading cached manifests. We're now asking the server to walk the manifest for us and that's a lot more expensive than querying local cached blocks. Let's gulp in an entire segment's worth of items if we can. We'll have plenty of opportunity to tune this down later. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index faa7ef96..51e20802 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -482,8 +482,6 @@ out: * The segments are immutable at this point so we can use their contents * as long as we hold refs. */ -#define MAX_ITEMS_READ 32 - int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end) @@ -503,10 +501,10 @@ int scoutfs_manifest_read_items(struct super_block *sb, u8 item_flags; int found_ctr; bool found; + bool added; int ret = 0; int err; int cmp; - int n; trace_scoutfs_read_items(sb, key, end); @@ -564,8 +562,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, found_ctr = 0; - for (n = 0; n < MAX_ITEMS_READ; n++) { - + added = false; + for (;;) { found = false; found_ctr++; @@ -628,10 +626,11 @@ int scoutfs_manifest_read_items(struct super_block *sb, ret = scoutfs_item_add_batch(sb, &batch, &found_key, found_val); if (ret) { - if (n > 0) + if (added) ret = 0; break; } + added = true; } /* the last successful key determines range end until run out */ From 8d59e6d0719f3265d4306e83480f6cd23f6ad977 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 17 Jun 2017 23:34:47 -0700 Subject: [PATCH 308/920] scoutfs: fix alloc eio for free region It's possible for the next segno to fall at the end of an allocation region that doesn't have any bits set. The code shouldn't return -EIO in that case, it should carry on to the next region. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 75aa408e..6fef77c5 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -131,31 +131,35 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) ind = sal->next_segno >> SCOUTFS_ALLOC_REGION_SHIFT; nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; - do { + for (;;) { reg = scoutfs_ring_lookup_next(&sal->ring, &ind); - } while (reg == NULL && ind && (ind = 0, nr = 0, 1)); + if (reg == NULL && ind != 0) { + ind = 0; + nr = 0; + continue; + } + if (IS_ERR_OR_NULL(reg)) { + if (IS_ERR(reg)) + ret = PTR_ERR(reg); + else + ret = -ENOSPC; + goto out; + } - if (IS_ERR_OR_NULL(reg)) { - if (IS_ERR(reg)) - ret = PTR_ERR(reg); - else - ret = -ENOSPC; - goto out; + nr = find_next_bit_le(reg->bits, SCOUTFS_ALLOC_REGION_BITS, nr); + if (nr < SCOUTFS_ALLOC_REGION_BITS) + break; + + /* possible for nr to be after all free bits, keep going */ + ind++; + nr = 0; } scoutfs_ring_dirty(&sal->ring, reg); - nr = find_next_bit_le(reg->bits, SCOUTFS_ALLOC_REGION_BITS, nr); - if (nr >= SCOUTFS_ALLOC_REGION_BITS) { - /* XXX corruption? shouldn't find empty regions */ - ret = -EIO; - goto out; - } - ind = le64_to_cpu(reg->index); clear_bit_le(nr, reg->bits); - if (empty_region(reg)) scoutfs_ring_delete(&sal->ring, reg); From 1724bab8ea7b8b1402b2775ea0281b64590c1106 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 22 Jun 2017 11:35:22 -0700 Subject: [PATCH 309/920] scoutfs: store large symlinks in multiple items We're shrinking the max item value size so we need to store symlinks with large target paths in multiple items. The arbitrary max value size defined here will be replaced in the future with the new global maximum value size. Signed-off-by: Zach Brown --- kmod/src/count.h | 5 ++- kmod/src/dir.c | 96 +++++++++++++++++++++++++++++++++-------------- kmod/src/dir.h | 2 +- kmod/src/format.h | 5 ++- kmod/src/inode.c | 2 +- 5 files changed, 77 insertions(+), 33 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 921dea89..b763cc2b 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -54,9 +54,10 @@ static inline void scoutfs_count_dirents(struct scoutfs_item_count *cnt, static inline void scoutfs_count_sym_target(struct scoutfs_item_count *cnt, unsigned size) { + unsigned nr = DIV_ROUND_UP(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); - cnt->items += 1; - cnt->keys += sizeof(struct scoutfs_symlink_key); + cnt->items += nr; + cnt->keys += nr * sizeof(struct scoutfs_symlink_key); cnt->vals += size; } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0d77d95a..04461e75 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -634,14 +634,70 @@ out: } static void init_symlink_key(struct scoutfs_key_buf *key, - struct scoutfs_symlink_key *skey, u64 ino) + struct scoutfs_symlink_key *skey, u64 ino, u8 nr) { skey->type = SCOUTFS_SYMLINK_KEY; skey->ino = cpu_to_be64(ino); + skey->nr = nr; scoutfs_key_init(key, skey, sizeof(struct scoutfs_symlink_key)); } +/* + * Operate on all the items that make up a symlink whose target might + * have to be split up into multiple items each with a maximally sized + * value. + * + * returns 0 or -errno from the item calls, particularly including + * EEXIST, EIO, or ENOENT if the item population doesn't match what was + * expected given the op. + * + * The target name can be null for deletion when val isn't used. Size + * still has to be provided to determine the number of items. + */ +enum { + SYM_CREATE = 0, + SYM_LOOKUP, + SYM_DELETE, +}; +static int symlink_item_ops(struct super_block *sb, int op, u64 ino, + const char *target, int size) +{ + struct scoutfs_symlink_key skey; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned bytes; + unsigned nr; + int ret; + int i; + + if (WARN_ON_ONCE(size <= 0 || size > SCOUTFS_SYMLINK_MAX_SIZE || + op > SYM_DELETE)) + return -EINVAL; + + nr = DIV_ROUND_UP(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); + for (i = 0; i < nr; i++) { + + init_symlink_key(&key, &skey, ino, i); + bytes = min(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); + scoutfs_kvec_init(val, (void *)target, bytes); + + if (op == SYM_CREATE) + ret = scoutfs_item_create(sb, &key, val); + else if (op == SYM_LOOKUP) + ret = scoutfs_item_lookup_exact(sb, &key, val, bytes); + else if (op == SYM_DELETE) + ret = scoutfs_item_delete(sb, &key); + if (ret) + break; + + target += SCOUTFS_SYMLINK_MAX_VAL_SIZE; + size -= bytes; + } + + return ret; +} + /* * Full a buffer with the null terminated symlink, point nd at it, and * return it so put_link can free it once the vfs is done. @@ -655,9 +711,6 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; loff_t size = i_size_read(inode); - struct scoutfs_symlink_key skey; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); char *path; int ret; @@ -673,14 +726,10 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (!path) return ERR_PTR(-ENOMEM); - init_symlink_key(&key, &skey, scoutfs_ino(inode)); - scoutfs_kvec_init(val, path, size); + ret = symlink_item_ops(sb, SYM_LOOKUP, scoutfs_ino(inode), path, size); - ret = scoutfs_item_lookup(sb, &key, val); - - /* XXX corruption: missing item, wrong size, not null term */ - if (ret == -ENOENT || - (ret >= 0 && (ret != size || path[size - 1] != '\0'))) + /* XXX corruption: missing items or not null term */ + if (ret == -ENOENT || (ret == 0 && path[size - 1])) ret = -EIO; if (ret < 0) { @@ -711,19 +760,15 @@ const struct inode_operations scoutfs_symlink_iops = { }; /* - * Symlink target paths can be annoyingly huge. We don't want large - * items gumming up the btree so we store relatively rare large paths in - * multiple items. + * Symlink target paths can be annoyingly large. We store relatively + * rare large paths in multiple items. */ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) { struct super_block *sb = dir->i_sb; const int name_len = strlen(symname) + 1; - struct scoutfs_symlink_key skey; - struct scoutfs_key_buf key; struct inode *inode = NULL; - SCOUTFS_DECLARE_KVEC(val); DECLARE_ITEM_COUNT(cnt); int ret; @@ -746,10 +791,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, goto out; } - init_symlink_key(&key, &skey, scoutfs_ino(inode)); - scoutfs_kvec_init(val, (void *)symname, name_len); - - ret = scoutfs_item_create(sb, &key, val); + ret = symlink_item_ops(sb, SYM_CREATE, scoutfs_ino(inode), + symname, name_len); if (ret) goto out; @@ -774,22 +817,19 @@ out: if (!IS_ERR_OR_NULL(inode)) iput(inode); - scoutfs_item_delete(sb, &key); + symlink_item_ops(sb, SYM_DELETE, scoutfs_ino(inode), + NULL, name_len); } scoutfs_release_trans(sb); return ret; } -int scoutfs_symlink_drop(struct super_block *sb, u64 ino) +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, u64 i_size) { - struct scoutfs_symlink_key skey; - struct scoutfs_key_buf key; int ret; - init_symlink_key(&key, &skey, ino); - - ret = scoutfs_item_delete(sb, &key); + ret = symlink_item_ops(sb, SYM_DELETE, ino, NULL, i_size); if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 273d1f54..81b5de4e 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -19,7 +19,7 @@ int scoutfs_dir_get_backref_path(struct super_block *sb, u64 target_ino, void scoutfs_dir_free_backref_path(struct super_block *sb, struct list_head *list); -int scoutfs_symlink_drop(struct super_block *sb, u64 ino); +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, u64 i_size); int scoutfs_dir_init(void); void scoutfs_dir_exit(void); diff --git a/kmod/src/format.h b/kmod/src/format.h index 3451a04b..fc36d50e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -253,12 +253,15 @@ struct scoutfs_xattr_val_header { __u8 last_part; } __packed; -/* value is the null terminated target path */ +/* size determines nr needed to store full target path in their values */ struct scoutfs_symlink_key { __u8 type; __be64 ino; + __u8 nr; } __packed; +#define SCOUTFS_SYMLINK_MAX_VAL_SIZE 200 + struct scoutfs_betimespec { __be64 sec; __be32 nsec; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index b5774682..c54e21dc 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -798,7 +798,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, goto out; if (S_ISLNK(mode)) - ret = scoutfs_symlink_drop(sb, ino); + ret = scoutfs_symlink_drop(sb, ino, i_size); else if (S_ISREG(mode)) ret = scoutfs_truncate_extent_items(sb, ino, 0, ~0ULL, false); if (ret) From 463a69657505ad5db58a9a5ccaa96ffb8939187a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Jun 2017 13:56:53 -0700 Subject: [PATCH 310/920] scoutfs: add value length limit Add a relatively small universal value size limit. This will be needed by more dense item packing to predict the worst case padding to avoid full items crossing block boundaries. We refactor the existing symlink and xattr item value limit to use this new limit. Signed-off-by: Zach Brown --- kmod/src/count.h | 2 +- kmod/src/dir.c | 10 +++++----- kmod/src/format.h | 21 ++++++++++++--------- kmod/src/item.c | 20 ++++++++++++++++++++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index b763cc2b..4198dbf5 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -54,7 +54,7 @@ static inline void scoutfs_count_dirents(struct scoutfs_item_count *cnt, static inline void scoutfs_count_sym_target(struct scoutfs_item_count *cnt, unsigned size) { - unsigned nr = DIV_ROUND_UP(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); + unsigned nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); cnt->items += nr; cnt->keys += nr * sizeof(struct scoutfs_symlink_key); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 04461e75..52448cd5 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -661,7 +661,7 @@ enum { SYM_DELETE, }; static int symlink_item_ops(struct super_block *sb, int op, u64 ino, - const char *target, int size) + const char *target, size_t size) { struct scoutfs_symlink_key skey; struct scoutfs_key_buf key; @@ -671,15 +671,15 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, int ret; int i; - if (WARN_ON_ONCE(size <= 0 || size > SCOUTFS_SYMLINK_MAX_SIZE || + if (WARN_ON_ONCE(size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE || op > SYM_DELETE)) return -EINVAL; - nr = DIV_ROUND_UP(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); + nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); for (i = 0; i < nr; i++) { init_symlink_key(&key, &skey, ino, i); - bytes = min(size, SCOUTFS_SYMLINK_MAX_VAL_SIZE); + bytes = min(size, SCOUTFS_MAX_VAL_SIZE); scoutfs_kvec_init(val, (void *)target, bytes); if (op == SYM_CREATE) @@ -691,7 +691,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, if (ret) break; - target += SCOUTFS_SYMLINK_MAX_VAL_SIZE; + target += SCOUTFS_MAX_VAL_SIZE; size -= bytes; } diff --git a/kmod/src/format.h b/kmod/src/format.h index fc36d50e..c511d51a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -260,8 +260,6 @@ struct scoutfs_symlink_key { __u8 nr; } __packed; -#define SCOUTFS_SYMLINK_MAX_VAL_SIZE 200 - struct scoutfs_betimespec { __be64 sec; __be32 nsec; @@ -377,13 +375,6 @@ struct scoutfs_dirent { /* S32_MAX avoids the (int) sign bit and might avoid sloppy bugs */ #define SCOUTFS_LINK_MAX S32_MAX -#define SCOUTFS_XATTR_MAX_NAME_LEN 255 -#define SCOUTFS_XATTR_MAX_SIZE 65536 -#define SCOUTFS_XATTR_PART_SIZE \ - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_xattr_val_header)) -#define SCOUTFS_XATTR_MAX_PARTS \ - DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) - /* entries begin after . and .. */ #define SCOUTFS_DIRENT_FIRST_POS 2 /* getdents returns next pos with an entry, no entry at (f_pos)~0 */ @@ -404,6 +395,18 @@ enum { #define SCOUTFS_MAX_KEY_SIZE \ offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) +/* largest single val are dirents, larger broken up into units of this */ +#define SCOUTFS_MAX_VAL_SIZE \ + offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]) + +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_SIZE 65536 +#define SCOUTFS_XATTR_PART_SIZE \ + (SCOUTFS_MAX_VAL_SIZE - sizeof(struct scoutfs_xattr_val_header)) +#define SCOUTFS_XATTR_MAX_PARTS \ + DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) + + /* * messages over the wire. */ diff --git a/kmod/src/item.c b/kmod/src/item.c index 79b18f60..6135b0eb 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -41,6 +41,12 @@ * clobber them in creation and skip them in lookups. */ +static bool invalid_key_val(struct scoutfs_key_buf *key, struct kvec *val) +{ + return WARN_ON_ONCE(key->key_len > SCOUTFS_MAX_KEY_SIZE || + (val && (scoutfs_kvec_length(val) > SCOUTFS_MAX_VAL_SIZE))); +} + static bool invalid_flags(int sif) { return (sif & SIF_EXCLUSIVE) && (sif & SIF_REPLACE); @@ -992,6 +998,9 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; + if (invalid_key_val(key, val)) + return -EINVAL; + item = alloc_item(sb, key, val); if (!item) return -ENOMEM; @@ -1021,6 +1030,9 @@ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct cached_item *item; int ret; + if (invalid_key_val(key, val)) + return -EINVAL; + item = alloc_item(sb, key, val); if (item) { list_add_tail(&item->entry, list); @@ -1128,6 +1140,11 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, if (WARN_ON_ONCE(invalid_flags(sif))) return -EINVAL; + list_for_each_entry(item, list, entry) { + if (invalid_key_val(item->key, item->val)) + return -EINVAL; + } + trace_scoutfs_item_set_batch(sb, start, end); if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) @@ -1283,6 +1300,9 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; + if (invalid_key_val(key, val)) + return -EINVAL; + end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); if (!end) { ret = -ENOMEM; From 70c7178e6a58a7df639253087236a351909035fa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Jun 2017 14:39:00 -0700 Subject: [PATCH 311/920] scoutfs: index segment items with skip list We want to be able to read a region of items from a segment by searching for the key that starts the item. In the first version of the segment format we find a key by performing a binary search across an array of offsets that point to the items. Unfortunately the current format requires that we know the number of items before we start writing. With thousands of items per segment it's a little bonkers to ask compaction to walk through all the items twice. Worse still, we didn't want the item offset array entries to span pages so they're rounded up to a power of two after having seqs and offsets and lengths. This makes them surprisingly large and sometimes they can consume up to 60% (!) of a segment. We know that we're inserting in sort order so it's very easy to build an index as we insert. Skip lists give us a nice simple way to ensure o(log n) lookups with only an average of two links per node. CPU use is greatly reduced by removing a full redundant item walk and we know use up almost all of the space in segments. There's still little gaps at the ends of blocks as item's still won't cross block boundaries. Most of this change is safely mechanical. The big difference is in how the compaction loop is built. It used to count the items before hand. It would never try to append when out of segments and writing would stop after the exact number of items. Now it discovers its out of items by allocating and trying to append and finding that there's no more work to do. It required rethinking the loop exit and segment allocation and stopping conditions. Signed-off-by: Zach Brown --- kmod/src/compact.c | 152 +++++------------- kmod/src/format.h | 34 ++-- kmod/src/item.c | 64 +------- kmod/src/manifest.c | 17 +- kmod/src/seg.c | 382 +++++++++++++++++++++++++------------------- kmod/src/seg.h | 15 +- 6 files changed, 308 insertions(+), 356 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index b129b931..9bcefa30 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -70,8 +70,7 @@ struct compact_seg { struct scoutfs_key_buf *first; struct scoutfs_key_buf *last; struct scoutfs_segment *seg; - int pos; - int saved_pos; + int off; bool part_of_move; }; @@ -90,12 +89,12 @@ struct compact_cursor { u8 last_level; struct compact_seg *upper; - struct compact_seg *saved_upper; struct compact_seg *lower; - struct compact_seg *saved_lower; bool sticky; struct compact_seg *last_lower; + + __le32 *links[SCOUTFS_MAX_SKIP_LINKS]; }; static void free_cseg(struct super_block *sb, struct compact_seg *cseg) @@ -140,28 +139,6 @@ static void free_cseg_list(struct super_block *sb, struct list_head *list) } } -static void save_pos(struct compact_cursor *curs) -{ - struct compact_seg *cseg; - - list_for_each_entry(cseg, &curs->csegs, entry) - cseg->saved_pos = cseg->pos; - - curs->saved_upper = curs->upper; - curs->saved_lower = curs->lower; -} - -static void restore_pos(struct compact_cursor *curs) -{ - struct compact_seg *cseg; - - list_for_each_entry(cseg, &curs->csegs, entry) - cseg->pos = cseg->saved_pos; - - curs->upper = curs->saved_upper; - curs->lower = curs->saved_lower; -} - static int read_segment(struct super_block *sb, struct compact_seg *cseg) { struct scoutfs_segment *seg; @@ -216,7 +193,7 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, retry: if (upper) { - ret = scoutfs_seg_item_ptrs(upper->seg, upper->pos, + ret = scoutfs_seg_item_ptrs(upper->seg, upper->off, item_key, item_val, item_flags); if (ret < 0) upper = NULL; @@ -227,7 +204,7 @@ retry: if (ret) goto out; - ret = scoutfs_seg_item_ptrs(lower->seg, lower->pos, + ret = scoutfs_seg_item_ptrs(lower->seg, lower->off, &lower_key, lower_val, &lower_flags); if (ret == 0) @@ -273,9 +250,9 @@ retry: } if (cmp <= 0) - upper->pos++; + upper->off = scoutfs_seg_next_off(upper->seg, upper->off); if (cmp >= 0) - lower->pos++; + lower->off = scoutfs_seg_next_off(lower->seg, lower->off); /* * Deletion items make their way down all the levels, replacing @@ -296,64 +273,38 @@ out: } /* - * Figure out how many items and bytes of keys we're going to try and - * compact into the next segment. + * Walk the input segments for items and append them to the output segment. + * Items can exist in the input segments but not be written to the output + * segment, for example if they're deletions. The output segment can be + * full. + * + * Return -errno if something went wrong, then 1 or 0 indicating items written. */ -static int count_items(struct super_block *sb, struct compact_cursor *curs, - u32 *nr_items, u32 *key_bytes) -{ - struct scoutfs_key_buf item_key; - SCOUTFS_DECLARE_KVEC(item_val); - u32 items = 0; - u32 keys = 0; - u32 vals = 0; - u8 flags; - int ret; - - *nr_items = 0; - *key_bytes = 0; - - while ((ret = next_item(sb, curs, &item_key, item_val, &flags)) > 0) { - - items++; - keys += item_key.key_len; - vals += scoutfs_kvec_length(item_val); - - if (!scoutfs_seg_fits_single(items, keys, vals)) - break; - - *nr_items = items; - *key_bytes = keys; - } - - return ret; -} - static int compact_items(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_segment *seg, u32 nr_items, - u32 key_bytes) + struct scoutfs_segment *seg) { struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); + int has_next; + int ret = 0; u8 flags; - int ret; - ret = next_item(sb, curs, &item_key, item_val, &flags); - if (ret <= 0) - goto out; - - scoutfs_seg_first_item(sb, seg, &item_key, item_val, flags, - nr_items, key_bytes); - - while (--nr_items) { - ret = next_item(sb, curs, &item_key, item_val, &flags); - if (ret <= 0) + for (;;) { + has_next = next_item(sb, curs, &item_key, item_val, &flags); + if (has_next <= 0) { + if (has_next < 0) + ret = has_next; break; - scoutfs_seg_append_item(sb, seg, &item_key, item_val, flags); + } + + if (scoutfs_seg_append_item(sb, seg, &item_key, item_val, flags, + curs->links)) + ret = 1; + else + break; } -out: return ret; } @@ -367,15 +318,14 @@ static int compact_segments(struct super_block *sb, struct compact_seg *upper; struct compact_seg *lower; unsigned next_segno = 0; - u32 key_bytes; - u32 nr_items; - int ret; + int ret = 0; scoutfs_inc_counter(sb, compact_operations); if (curs->sticky) scoutfs_inc_counter(sb, compact_sticky_upper); - for (;;) { + while (curs->upper || curs->lower) { + upper = curs->upper; lower = curs->lower; @@ -389,7 +339,7 @@ static int compact_segments(struct super_block *sb, * XXX We should have metadata in the manifest to tell * us that there's no deletion items in the segment. */ - if (upper && upper->pos == 0 && !lower && !curs->sticky && + if (upper && upper->off == 0 && !lower && !curs->sticky && ((upper->level + 1) < curs->last_level)) { /* @@ -417,9 +367,9 @@ static int compact_segments(struct super_block *sb, cseg->part_of_move = true; curs->upper = NULL; - upper = NULL; scoutfs_inc_counter(sb, compact_segment_moved); + break; } /* we're going to need its next key */ @@ -444,17 +394,6 @@ static int compact_segments(struct super_block *sb, if (ret) break; - save_pos(curs); - ret = count_items(sb, curs, &nr_items, &key_bytes); - restore_pos(curs); - if (ret < 0) - break; - - if (nr_items == 0) { - ret = 0; - break; - } - /* no cseg keys, manifest update uses seg item keys */ cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); if (!cseg) { @@ -466,11 +405,19 @@ static int compact_segments(struct super_block *sb, curs->segnos[next_segno] = 0; next_segno++; + /* + * Compaction can free all the remaining items resulting + * in an empty output segment. We just free it in that + * case. + */ ret = scoutfs_seg_alloc(sb, cseg->segno, &seg); - if (ret) { + if (ret == 0) + ret = compact_items(sb, curs, seg); + if (ret < 1) { next_segno--; curs->segnos[next_segno] = cseg->segno; kfree(cseg); + scoutfs_seg_put(seg); break; } @@ -489,21 +436,6 @@ static int compact_segments(struct super_block *sb, cseg->seg = seg; list_add_tail(&cseg->entry, results); - ret = compact_items(sb, curs, seg, nr_items, key_bytes); - if (ret < 0) - break; - - /* - * Clear lower after we've consumed it so that sticky - * compaction can decide to write the rest of the items - * into the upper level. We decide that it's done by - * testing the pos that next_item() is going to try. - */ - if (curs->sticky && curs->lower == curs->last_lower && - scoutfs_seg_item_ptrs(curs->lower->seg, curs->lower->pos, - NULL, NULL, NULL) < 0) - curs->lower = NULL; - /* start a complete segment write now, we'll wait later */ ret = scoutfs_seg_submit_write(sb, seg, comp); if (ret) diff --git a/kmod/src/format.h b/kmod/src/format.h index c511d51a..6ef16073 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -112,19 +112,29 @@ struct scoutfs_alloc_region { } __packed; /* - * We really want these to be a power of two size so that they're naturally - * aligned. This ensures that they won't cross page boundaries and we - * can use pointers to them in the page vecs that make up segments without - * funny business. + * The max number of links defines the max number of entries that we can + * index in o(log n) and the static list head storage size in the + * segment block. We always pay the static storage cost, which is tiny, + * and we can look at the number of items to know the greatest number of + * links and skip most of the initial 0 links. + */ +#define SCOUTFS_MAX_SKIP_LINKS 32 + +/* + * Items are packed into segments and linked together in a skip list. + * Each item's header, links, key, and value are stored contiguously. + * They're not allowed to cross a block boundary. */ struct scoutfs_segment_item { - __le64 seq; - __le32 key_off; - __le32 val_off; __le16 key_len; __le16 val_len; - __u8 padding[11]; __u8 flags; + __u8 nr_links; + __le32 skip_links[0]; + /* + * u8 key_bytes[key_len] + * u8 val_bytes[val_len] + */ } __packed; #define SCOUTFS_ITEM_FLAG_DELETION (1 << 0) @@ -138,11 +148,11 @@ struct scoutfs_segment_block { __le32 _padding; __le64 segno; __le64 seq; + __le32 last_item_off; + __le32 total_bytes; __le32 nr_items; - __le32 _moar_pads; - struct scoutfs_segment_item items[0]; - /* packed keys */ - /* packed vals */ + __le32 skip_links[SCOUTFS_MAX_SKIP_LINKS]; + /* packed items */ } __packed; /* diff --git a/kmod/src/item.c b/kmod/src/item.c index 6135b0eb..977ae63d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1550,39 +1550,6 @@ bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, return fits; } -/* - * Find the initial sorted dirty items that will fit in a segment. Give - * the caller the number of items and the total bytes of their keys. - */ -static void count_seg_items(struct item_cache *cac, u32 *nr_items, - u32 *key_bytes) -{ - struct cached_item *item; - u32 items = 0; - u32 keys = 0; - u32 vals = 0; - - *nr_items = 0; - *key_bytes = 0; - - for (item = first_dirty(cac->items.rb_node); item; - item = next_dirty(item)) { - - items++; - keys += item->key->key_len; - vals += scoutfs_kvec_length(item->val); - - if (!scoutfs_seg_fits_single(items, keys, vals)) - break; - - *nr_items = items; - *key_bytes = keys; - - trace_printk("counted item %p nr %u keys %u\n", - item, items, keys); - } -} - /* * Fill the given segment with sorted dirty items. * @@ -1597,33 +1564,20 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; + __le32 *links[SCOUTFS_MAX_SKIP_LINKS]; struct cached_item *item = NULL; struct cached_item *del; unsigned long flags; - u32 key_bytes; - u32 nr_items; + bool appended; spin_lock_irqsave(&cac->lock, flags); - count_seg_items(cac, &nr_items, &key_bytes); - - /* remember nr_items is passed to _first_item */ - while (nr_items) { - - trace_printk("copying item %p nr %u keys %u\n", - item, nr_items, key_bytes); - - if (!item) { - item = first_dirty(cac->items.rb_node); - scoutfs_seg_first_item(sb, seg, item->key, item->val, - item_flags(item), nr_items, - key_bytes); - } else { - scoutfs_seg_append_item(sb, seg, item->key, item->val, - item_flags(item)); - } - - key_bytes -= item->key->key_len; + item = first_dirty(cac->items.rb_node); + while (item) { + appended = scoutfs_seg_append_item(sb, seg, item->key, item->val, + item_flags(item), links); + /* trans reservation should have limited dirty */ + BUG_ON(!appended); clear_item_dirty(sb, cac, item); @@ -1632,8 +1586,6 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) if (del->deletion) erase_item(sb, cac, del); - - nr_items--; } spin_unlock_irqrestore(&cac->lock, flags); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 51e20802..df6ba43f 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -67,7 +67,7 @@ struct manifest_ref { u64 seq; struct scoutfs_segment *seg; int found_ctr; - int pos; + int off; u8 level; struct scoutfs_key_buf *first; @@ -542,7 +542,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, /* start from the next item from the key in each segment */ list_for_each_entry(ref, &ref_list, entry) - ref->pos = scoutfs_seg_find_pos(ref->seg, key); + ref->off = scoutfs_seg_find_off(ref->seg, key); /* * Find the limit of the range we can safely walk. We have all @@ -567,9 +567,9 @@ int scoutfs_manifest_read_items(struct super_block *sb, found = false; found_ctr++; - /* find the next least key from the pos in each segment */ + /* find the next least key from the off in each segment */ list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - if (ref->pos == -1) + if (ref->off < 0) continue; /* @@ -578,12 +578,12 @@ int scoutfs_manifest_read_items(struct super_block *sb, * items or if the next item is past the keys * that our segments can see. */ - ret = scoutfs_seg_item_ptrs(ref->seg, ref->pos, + ret = scoutfs_seg_item_ptrs(ref->seg, ref->off, &item_key, item_val, &item_flags); if (ret < 0 || - scoutfs_key_compare(&item_key, &seg_end) > 0){ - ref->pos = -1; + scoutfs_key_compare(&item_key, &seg_end) > 0) { + ref->off = -1; continue; } @@ -645,7 +645,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, /* advance all the positions that had the found key */ list_for_each_entry(ref, &ref_list, entry) { if (ref->found_ctr == found_ctr) - ref->pos++; + ref->off = scoutfs_seg_next_off(ref->seg, + ref->off); } ret = 0; diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 1441d152..8437d98b 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -265,6 +265,10 @@ int scoutfs_seg_alloc(struct super_block *sb, u64 segno, /* reads shouldn't wait for this */ set_bit(SF_END_IO, &seg->flags); + /* zero the block header so the caller knows to initialize */ + memset(page_address(seg->pages[0]), 0, + sizeof(struct scoutfs_segment_block)); + /* XXX always remove existing segs, is that necessary? */ spin_lock_irqsave(&cac->lock, flags); @@ -371,22 +375,6 @@ static void *off_ptr(struct scoutfs_segment *seg, u32 off) return page_address(seg->pages[pg]) + pg_off; } -static u32 pos_off(u32 pos) -{ - /* items need of be a power of two */ - BUILD_BUG_ON(!is_power_of_2(sizeof(struct scoutfs_segment_item))); - /* and the first item has to be naturally aligned */ - BUILD_BUG_ON(offsetof(struct scoutfs_segment_block, items) % - sizeof(struct scoutfs_segment_item)); - - return offsetof(struct scoutfs_segment_block, items[pos]); -} - -static void *pos_ptr(struct scoutfs_segment *seg, u32 pos) -{ - return off_ptr(seg, pos_off(pos)); -} - static void kvec_from_pages(struct scoutfs_segment *seg, struct kvec *kvec, u32 off, u16 len) { @@ -401,118 +389,225 @@ static void kvec_from_pages(struct scoutfs_segment *seg, off_ptr(seg, off + first), len - first); } -int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, +static u32 item_bytes(u8 nr_links, u16 key_len, u16 val_len) +{ + return offsetof(struct scoutfs_segment_item, skip_links[nr_links]) + + key_len + val_len; +} + +static inline int item_key_off(struct scoutfs_segment_item *item, int item_off) +{ + return item_off + item_bytes(item->nr_links, 0, 0); +} + +static inline void *item_key_ptr(struct scoutfs_segment_item *item) +{ + return (void *)item + item_bytes(item->nr_links, 0, 0); +} + +static inline int item_val_off(struct scoutfs_segment_item *item, int item_off) +{ + return item_key_off(item, item_off) + le16_to_cpu(item->key_len); +} + +static void item_ptrs(struct scoutfs_segment *seg, int off, + struct scoutfs_key_buf *key, struct kvec *val) +{ + struct scoutfs_segment_item *item = off_ptr(seg, off); + + if (key) + scoutfs_key_init(key, item_key_ptr(item), + le16_to_cpu(item->key_len)); + if (val) + kvec_from_pages(seg, val, item_val_off(item, off), + le16_to_cpu(item->val_len)); +} + +static void first_last_keys(struct scoutfs_segment *seg, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + + item_ptrs(seg, sizeof(struct scoutfs_segment_block), first, NULL); + item_ptrs(seg, le32_to_cpu(sblk->last_item_off), last, NULL); +} + +static int check_caller_off(struct scoutfs_segment_block *sblk, int off) +{ + if (off >= 0 && off < sizeof(struct scoutfs_segment_block)) + off = sizeof(struct scoutfs_segment_block); + + if (off > le32_to_cpu(sblk->last_item_off)) + off = -ENOENT; + + return off; +} + +/* + * Give the caller the key and value of the item at the given offset. + * + * Negative offsets are sticky errors and offsets outside the used bytes + * in the segment return -ENOENT; + * + * All other offsets must be initial values less than the segment header + * size, notably including 0, or returned from _next_off(). + */ +int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, struct scoutfs_key_buf *key, struct kvec *val, u8 *flags) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; - if (pos < 0 || pos >= le32_to_cpu(sblk->nr_items)) - return -ENOENT; + off = check_caller_off(sblk, off); + if (off < 0) + return off; - item = pos_ptr(seg, pos); + item_ptrs(seg, off, key, val); - if (key) - scoutfs_key_init(key, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); - if (val) - kvec_from_pages(seg, val, le32_to_cpu(item->val_off), - le16_to_cpu(item->val_len)); - if (flags) + if (flags) { + item = off_ptr(seg, off); *flags = item->flags; + } return 0; } /* - * Find the first item array position whose key is >= the search key. - * This can return the number of positions if the key is greater than - * all the keys. + * Return the number of links that the *next* added node should have. + * We're appending in order so we can use the low bits of the node count + * to get an ideal distribution of the number of links to enable (log n) + * searching: of links in each node. Half of the nodes will have 1 + * links, a quarter will have 2, an eighth will have 3, and so on. */ -static int find_key_pos(struct scoutfs_segment *seg, - struct scoutfs_key_buf *search) +static u8 skip_next_nr(u32 nr_items) { - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_key_buf key; - unsigned int start = 0; - unsigned int end = le32_to_cpu(sblk->nr_items); - unsigned int pos = 0; - int cmp; - - while (start < end) { - pos = start + (end - start) / 2; - scoutfs_seg_item_ptrs(seg, pos, &key, NULL, NULL); - - cmp = scoutfs_key_compare(search, &key); - if (cmp < 0) - end = pos; - else if (cmp > 0) - start = ++pos; - else - break; - } - - return pos; + return ffs(nr_items + 1); } -int scoutfs_seg_find_pos(struct scoutfs_segment *seg, +/* The highest 1-based set bit is the max number of links any node can have */ +static u8 skip_most_nr(u32 nr_items) +{ + return fls(nr_items); +} + +/* + * Find offset of the first item in the segment whose key is greater + * than or equal to the search key. -ENOENT is returned if there's no + * item that matches. + * + * This is a standard skip list search from the segment block through + * the items. Follow high less frequent links while the key is greater + * than the items and descend down to lower more frequent links when the + * search key is less. + */ +int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key_buf *key) { - return find_key_pos(seg, key); + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct scoutfs_segment_item *item; + struct scoutfs_key_buf item_key; + __le32 *links; + int cmp; + int ret; + int i; + int off; + + links = sblk->skip_links; + ret = -ENOENT; + for (i = skip_most_nr(le32_to_cpu(sblk->nr_items)) - 1; i >= 0; i--) { + if (links[i] == 0) + continue; + + off = le32_to_cpu(links[i]); + item = off_ptr(seg, off); + scoutfs_key_init(&item_key, item_key_ptr(item), + le16_to_cpu(item->key_len)); + + cmp = scoutfs_key_compare(key, &item_key); + if (cmp == 0) { + ret = off; + break; + } + + if (cmp > 0) { + links = item->skip_links; + i++; + } else { + ret = off; + } + } + + return ret; } /* - * Keys are aligned to the next block boundary if they'd cross a block - * boundary. To find the first value offset we have to assume that - * there will be a worst case key alignment at every block boundary. + * Return the offset of the next item after the current item. The input offset + * must be a valid offset from _find_off(). */ -static u32 first_val_off(u32 nr_items, u32 key_bytes) +int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off) { - u32 key_padding = SCOUTFS_MAX_KEY_SIZE - 1; - u32 partial_block = SCOUTFS_BLOCK_SIZE - key_padding; - u32 first_key_off = pos_off(nr_items); - u32 block_off = first_key_off & SCOUTFS_BLOCK_MASK; - u32 total_padding = ((block_off + key_bytes) / partial_block) * - key_padding; + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + struct scoutfs_segment_item *item; - return first_key_off + key_bytes + total_padding; + off = check_caller_off(sblk, off); + if (off > 0) { + item = off_ptr(seg, off); + off = le32_to_cpu(item->skip_links[0]); + if (off == 0) + off = -ENOENT; + } + return off; } /* - * Returns true if the given number of items with the given total byte - * counts of keys and values fits inside a single segment. + * Returns true if the given item population will fit in a single + * segment. + * + * We don't have items cross block boundaries. It would be too + * expensive to maintain packing of sorted dirty items in bins. Instead + * we assume that we'll lose the worst case largest possible item on every + * block transition. This will almost never be the case. This causes us + * to lose around 15% of space for level 0 segment writes. + * + * Our pattern of item link counts ensures that there will always be fewer + * than two links per item. We assume the worst case items have the + * max number of links. */ bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes) { - return (first_val_off(nr_items, key_bytes) + val_bytes) + u32 header = sizeof(struct scoutfs_segment_block); + u32 items = nr_items * item_bytes(2, 0, 0); + u32 item_pad = item_bytes(skip_most_nr(nr_items), SCOUTFS_MAX_KEY_SIZE, + SCOUTFS_MAX_VAL_SIZE) - 1; + u32 padding = (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) * item_pad; + + return (header + items + key_bytes + val_bytes + padding) <= SCOUTFS_SEGMENT_SIZE; } -static u32 align_key_off(struct scoutfs_segment *seg, u32 key_off, u32 len) +static u32 align_item_off(struct scoutfs_segment *seg, u32 item_off, u32 bytes) { - u32 space = SCOUTFS_BLOCK_SIZE - (key_off & SCOUTFS_BLOCK_MASK); + u32 space = SCOUTFS_BLOCK_SIZE - (item_off & SCOUTFS_BLOCK_MASK); - if (len > space) { - memset(off_ptr(seg, key_off), 0, space); - return key_off + space; + if (bytes > space) { + memset(off_ptr(seg, item_off), 0, space); + return item_off + space; } - return key_off; + return item_off; } + /* - * Store the first item in the segment. The caller knows the number - * of items and bytes of keys that determine where the keys and values - * start. Future items are appended by looking at the last item. - * - * This should never fail because any item must always fit in a segment. + * Append an item to the segment. The caller always appends items that + * have been sorted by their keys. They may not know how many will fit. + * We return true if we appended and false if the segment was full. */ -void scoutfs_seg_first_item(struct super_block *sb, - struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val, - u8 flags, unsigned int nr_items, - unsigned int key_bytes) +bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, + struct scoutfs_key_buf *key, struct kvec *val, + u8 flags, __le32 **links) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; @@ -520,82 +615,66 @@ void scoutfs_seg_first_item(struct super_block *sb, struct scoutfs_segment_item *item; struct scoutfs_key_buf item_key; SCOUTFS_DECLARE_KVEC(item_val); - u32 key_off; - u32 val_off; + u8 nr_links; + u32 val_len; + u32 bytes; + u32 off; + int i; - /* XXX the segment block header is a mess, be better */ - sblk->segno = cpu_to_le64(seg->segno); - sblk->seq = super->next_seg_seq; - le64_add_cpu(&super->next_seg_seq, 1); + val_len = scoutfs_kvec_length(val); - key_off = align_key_off(seg, pos_off(nr_items), key->key_len); - val_off = first_val_off(nr_items, key_bytes); + /* initialize the segment and skip links as the first item is appended */ + if (sblk->nr_items == 0) { + /* XXX the segment block header is a mess, be better */ + sblk->segno = cpu_to_le64(seg->segno); + sblk->seq = super->next_seg_seq; + le64_add_cpu(&super->next_seg_seq, 1); + sblk->total_bytes = cpu_to_le32(sizeof(*sblk)); - sblk->nr_items = cpu_to_le32(1); - - trace_printk("first item offs key %u val %u\n", key_off, val_off); - - item = pos_ptr(seg, 0); - item->seq = cpu_to_le64(1); - item->key_off = cpu_to_le32(key_off); - item->val_off = cpu_to_le32(val_off); - item->key_len = cpu_to_le16(key->key_len); - item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); - item->flags = flags; - - scoutfs_seg_item_ptrs(seg, 0, &item_key, item_val, NULL); - scoutfs_key_copy(&item_key, key); - scoutfs_kvec_memcpy(item_val, val); -} - -void scoutfs_seg_append_item(struct super_block *sb, - struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val, - u8 flags) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; - struct scoutfs_segment_item *prev; - struct scoutfs_key_buf item_key; - SCOUTFS_DECLARE_KVEC(item_val); - u32 key_off; - u32 val_off; - u32 pos; - - pos = le32_to_cpu(sblk->nr_items); - sblk->nr_items = cpu_to_le32(pos + 1); + for (i = 0; i < SCOUTFS_MAX_SKIP_LINKS; i++) + links[i] = &sblk->skip_links[i]; + } /* * It's very bad data corruption if we write out of order items * to a segment. It'll mislead the key search during read and * stop it from finding its items. */ - if (pos) { - scoutfs_seg_item_ptrs(seg, pos - 1, &item_key, NULL, NULL); + off = le32_to_cpu(sblk->last_item_off); + if (off) { + item_ptrs(seg, off, &item_key, NULL); BUG_ON(scoutfs_key_compare(key, &item_key) <= 0); } - prev = pos_ptr(seg, pos - 1); - item = pos_ptr(seg, pos); + nr_links = skip_next_nr(le32_to_cpu(sblk->nr_items)); + bytes = item_bytes(nr_links, key->key_len, val_len); + off = align_item_off(seg, le32_to_cpu(sblk->total_bytes), bytes); - key_off = le32_to_cpu(prev->key_off) + le16_to_cpu(prev->key_len); - val_off = le32_to_cpu(prev->val_off) + le16_to_cpu(prev->val_len); + if ((off + bytes) > SCOUTFS_SEGMENT_SIZE) + return false; - key_off = align_key_off(seg, key_off, key->key_len); + sblk->last_item_off = cpu_to_le32(off); + sblk->total_bytes = cpu_to_le32(off + bytes); + le32_add_cpu(&sblk->nr_items, 1); - item->seq = cpu_to_le64(1); - item->key_off = cpu_to_le32(key_off); - item->val_off = cpu_to_le32(val_off); + item = off_ptr(seg, off); item->key_len = cpu_to_le16(key->key_len); - item->val_len = cpu_to_le16(scoutfs_kvec_length(val)); + item->val_len = cpu_to_le16(val_len); item->flags = flags; - trace_printk("item %u offs key %u val %u\n", - pos, key_off, val_off); + /* point the previous skip links at our appended item */ + item->nr_links = nr_links; + for (i = 0; i < nr_links; i++) { + item->skip_links[i] = 0; + *links[i] = cpu_to_le32(off); + links[i] = &item->skip_links[i]; + } - scoutfs_seg_item_ptrs(seg, pos, &item_key, item_val, NULL); + item_ptrs(seg, off, &item_key, item_val); scoutfs_key_copy(&item_key, key); scoutfs_kvec_memcpy(item_val, val); + + return true; } /* @@ -605,17 +684,10 @@ int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; struct scoutfs_key_buf first; struct scoutfs_key_buf last; - item = pos_ptr(seg, 0); - scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); - - item = pos_ptr(seg, le32_to_cpu(sblk->nr_items) - 1); - scoutfs_key_init(&last, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); + first_last_keys(seg, &first, &last); return scoutfs_manifest_add(sb, &first, &last, le64_to_cpu(sblk->segno), le64_to_cpu(sblk->seq), level); @@ -625,12 +697,9 @@ int scoutfs_seg_manifest_del(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; struct scoutfs_key_buf first; - item = pos_ptr(seg, 0); - scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); + first_last_keys(seg, &first, NULL); return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); } @@ -644,17 +713,10 @@ scoutfs_seg_manifest_entry(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; struct scoutfs_key_buf first; struct scoutfs_key_buf last; - item = pos_ptr(seg, 0); - scoutfs_key_init(&first, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); - - item = pos_ptr(seg, le32_to_cpu(sblk->nr_items) - 1); - scoutfs_key_init(&last, off_ptr(seg, le32_to_cpu(item->key_off)), - le16_to_cpu(item->key_len)); + first_last_keys(seg, &first, &last); return scoutfs_manifest_alloc_entry(sb, &first, &last, le64_to_cpu(sblk->segno), diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 5ce0c076..fbe11d88 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -10,9 +10,10 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, u64 segno); int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); -int scoutfs_seg_find_pos(struct scoutfs_segment *seg, +int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key_buf *key); -int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int pos, +int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off); +int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, struct scoutfs_key_buf *key, struct kvec *val, u8 *flags); @@ -24,15 +25,9 @@ int scoutfs_seg_alloc(struct super_block *sb, u64 segno, int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); -void scoutfs_seg_first_item(struct super_block *sb, - struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val, - u8 flags, unsigned int nr_items, - unsigned int key_bytes); -void scoutfs_seg_append_item(struct super_block *sb, - struct scoutfs_segment *seg, +bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_key_buf *key, struct kvec *val, - u8 flags); + u8 flags, __le32 **links); int scoutfs_seg_manifest_add(struct super_block *sb, struct scoutfs_segment *seg, u8 level); int scoutfs_seg_manifest_del(struct super_block *sb, From 823a5bed3429401ab73bee9187317451332a81cb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Jun 2017 16:15:53 -0700 Subject: [PATCH 312/920] scoutfs: add some segment cache life cycle tracing Signed-off-by: Zach Brown --- kmod/src/counters.h | 4 +++- kmod/src/scoutfs_trace.h | 42 ++++++++++++++++++++++++++++++++++++++++ kmod/src/seg.c | 27 ++++++++++++-------------- kmod/src/seg.h | 13 ++++++++++++- 4 files changed, 69 insertions(+), 17 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a41340e2..9fe291c0 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -14,7 +14,9 @@ #define EXPAND_EACH_COUNTER \ EXPAND_COUNTER(alloc_alloc) \ EXPAND_COUNTER(alloc_free) \ - EXPAND_COUNTER(seg_lru_shrink) \ + EXPAND_COUNTER(seg_alloc) \ + EXPAND_COUNTER(seg_shrink) \ + EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(trans_level0_seg_write) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(compact_operations) \ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index e03c2c6d..175fe3e6 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -29,6 +29,7 @@ #include "format.h" #include "kvec.h" #include "lock.h" +#include "seg.h" struct scoutfs_sb_info; @@ -357,6 +358,47 @@ TRACE_EVENT(scoutfs_lock_invalidate_sb, __get_str(start), __get_str(end)) ); +DECLARE_EVENT_CLASS(scoutfs_seg_class, + TP_PROTO(struct scoutfs_segment *seg), + TP_ARGS(seg), + TP_STRUCT__entry( + __field(unsigned int, major) + __field(unsigned int, minor) + __field(struct scoutfs_segment *, seg) + __field(int, refcount) + __field(u64, segno) + __field(unsigned long, flags) + __field(int, err) + ), + TP_fast_assign( + __entry->major = MAJOR(seg->sb->s_bdev->bd_dev); + __entry->minor = MINOR(seg->sb->s_bdev->bd_dev); + __entry->seg = seg; + __entry->refcount = atomic_read(&seg->refcount); + __entry->segno = seg->segno; + __entry->flags = seg->flags; + __entry->err = seg->err; + ), + TP_printk("dev %u:%u seg %p refcount %d segno %llu flags %lx err %d", + __entry->major, __entry->minor, __entry->seg, __entry->refcount, + __entry->segno, __entry->flags, __entry->err) +); + +DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_alloc, + TP_PROTO(struct scoutfs_segment *seg), + TP_ARGS(seg) +); + +DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_shrink, + TP_PROTO(struct scoutfs_segment *seg), + TP_ARGS(seg) +); + +DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_free, + TP_PROTO(struct scoutfs_segment *seg), + TP_ARGS(seg) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 8437d98b..fffbbf34 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -26,6 +26,7 @@ #include "alloc.h" #include "key.h" #include "counters.h" +#include "scoutfs_trace.h" /* * seg.c should just be about the cache and io, and maybe @@ -49,21 +50,12 @@ struct segment_cache { unsigned long lru_nr; }; -struct scoutfs_segment { - struct rb_node node; - struct list_head lru_entry; - atomic_t refcount; - u64 segno; - unsigned long flags; - int err; - struct page *pages[SCOUTFS_SEGMENT_PAGES]; -}; enum { SF_END_IO = 0, }; -static struct scoutfs_segment *alloc_seg(u64 segno) +static struct scoutfs_segment *alloc_seg(struct super_block *sb, u64 segno) { struct scoutfs_segment *seg; struct page *page; @@ -76,6 +68,7 @@ static struct scoutfs_segment *alloc_seg(u64 segno) if (!seg) return seg; + seg->sb = sb; RB_CLEAR_NODE(&seg->node); INIT_LIST_HEAD(&seg->lru_entry); atomic_set(&seg->refcount, 1); @@ -83,8 +76,6 @@ static struct scoutfs_segment *alloc_seg(u64 segno) for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) { page = alloc_page(GFP_NOFS); - trace_printk("seg %p segno %llu page %u %p\n", - seg, segno, i, page); if (!page) { scoutfs_seg_put(seg); return ERR_PTR(-ENOMEM); @@ -93,6 +84,9 @@ static struct scoutfs_segment *alloc_seg(u64 segno) seg->pages[i] = page; } + trace_scoutfs_seg_alloc(seg); + scoutfs_inc_counter(sb, seg_alloc); + return seg; } @@ -106,6 +100,8 @@ void scoutfs_seg_put(struct scoutfs_segment *seg) int i; if (!IS_ERR_OR_NULL(seg) && atomic_dec_and_test(&seg->refcount)) { + trace_scoutfs_seg_free(seg); + scoutfs_inc_counter(seg->sb, seg_free); WARN_ON_ONCE(!RB_EMPTY_NODE(&seg->node)); WARN_ON_ONCE(!list_empty(&seg->lru_entry)); for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) @@ -256,7 +252,7 @@ int scoutfs_seg_alloc(struct super_block *sb, u64 segno, unsigned long flags; int ret; - seg = alloc_seg(segno); + seg = alloc_seg(sb, segno); if (!seg) { ret = -ENOMEM; goto out; @@ -321,7 +317,7 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, if (seg) return seg; - seg = alloc_seg(segno); + seg = alloc_seg(sb, segno); if (IS_ERR(seg)) return seg; @@ -776,7 +772,8 @@ static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) spin_unlock_irqrestore(&cac->lock, flags); list_for_each_entry_safe(seg, tmp, &list, lru_entry) { - scoutfs_inc_counter(sb, seg_lru_shrink); + trace_scoutfs_seg_shrink(seg); + scoutfs_inc_counter(sb, seg_shrink); list_del_init(&seg->lru_entry); scoutfs_seg_put(seg); } diff --git a/kmod/src/seg.h b/kmod/src/seg.h index fbe11d88..9d1cd4c9 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -2,10 +2,21 @@ #define _SCOUTFS_SEG_H_ struct scoutfs_bio_completion; -struct scoutfs_segment; struct scoutfs_key_buf; struct kvec; +/* this is only visible for trace events */ +struct scoutfs_segment { + struct super_block *sb; + struct rb_node node; + struct list_head lru_entry; + atomic_t refcount; + u64 segno; + unsigned long flags; + int err; + struct page *pages[SCOUTFS_SEGMENT_PAGES]; +}; + struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, u64 segno); int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); From 9f545782fb32b14e310a20129b80364ac786ddff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Jun 2017 16:40:47 -0700 Subject: [PATCH 313/920] scoutfs: add missing segment put Back when we changed the transaction commit to ask the server to update the commit we accidentally lost the put of the level0 segment that was just written. This leaked refcount would pin segments over time and eventually drag the box into crippling oom. Signed-off-by: Zach Brown --- kmod/src/trans.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 4c41de88..bfa82f57 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -112,7 +112,7 @@ void scoutfs_trans_write_func(struct work_struct *work) struct super_block *sb = sbi->sb; DECLARE_TRANS_INFO(sb, tri); struct scoutfs_bio_completion comp; - struct scoutfs_segment *seg; + struct scoutfs_segment *seg = NULL; u64 segno; int ret = 0; @@ -138,6 +138,7 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_net_record_segment(sb, seg, 0) ?: scoutfs_net_advance_seq(sb, &sbi->trans_seq); + scoutfs_seg_put(seg); if (ret) goto out; From f7701177d2c6c686a2ffd552470a1de95e1b7bee Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Jun 2017 20:47:11 -0700 Subject: [PATCH 314/920] scoutfs: throttle addition of level 0 segments Writers can add level 0 segments much faster (~20x) than compaction can compact them down into the lower levels. Without a limit on the number of level 0 segments item readind can try to read an extraordinary number of level 0 segments and wedge the box nonreclaimable page allocations. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 56 +++++++++++++++++++++++++++++++++++++-------- kmod/src/manifest.h | 2 ++ kmod/src/net.c | 42 +++++++++++++++++++++++++++++++--- 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index df6ba43f..564b7259 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -45,9 +45,13 @@ struct manifest { /* calculated on mount, const thereafter */ u64 level_limits[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + unsigned long flags; + struct scoutfs_key_buf *compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; }; +#define MANI_FLAG_LEVEL0_FULL (1 << 0) + #define DECLARE_MANIFEST(sb, name) \ struct manifest *name = SCOUTFS_SB(sb)->manifest @@ -109,6 +113,46 @@ static bool cmp_range_ment(struct scoutfs_key_buf *key, return scoutfs_key_compare_ranges(key, end, &first, &last); } +/* + * Change the level count under the manifest lock. We then maintain a + * bit that can be tested outside the lock to determine if the caller + * should wait for level 0 segments to drain. + */ +static void add_level_count(struct super_block *sb, int level, s64 val) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + __le64 count; + int full; + + le64_add_cpu(&super->manifest.level_counts[level], val); + + if (level == 0) { + count = super->manifest.level_counts[level]; + full = test_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); + if (count && !full) + set_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); + else if (!count && full) + clear_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); + } +} + +/* + * Return whether or not level 0 segments are full. It's safe to use + * this as a wait_event condition because it doesn't block. + * + * Callers rely on on the spin locks in wait queues to synchronize + * testing this as a sleeping condition with addition to the wait queue + * and waking of the waitqueue. + */ +bool scoutfs_manifest_level0_full(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + + return test_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); +} + /* * Insert a new manifest entry in the ring. The ring allocates a new * node for us and we fill it. @@ -121,8 +165,6 @@ int scoutfs_manifest_add(struct super_block *sb, u8 level) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct scoutfs_manifest_entry *ment; struct scoutfs_key_buf ment_first; struct scoutfs_key_buf ment_last; @@ -154,7 +196,7 @@ int scoutfs_manifest_add(struct super_block *sb, scoutfs_key_copy(&ment_last, last); mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - le64_add_cpu(&super->manifest.level_counts[level], 1); + add_level_count(sb, level, 1); return 0; } @@ -168,8 +210,6 @@ int scoutfs_manifest_add_ment(struct super_block *sb, struct scoutfs_manifest_entry *add) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; struct scoutfs_key_buf first; @@ -195,7 +235,7 @@ int scoutfs_manifest_add_ment(struct super_block *sb, memcpy(ment, add, bytes); mani->nr_levels = max_t(u8, mani->nr_levels, add->level + 1); - le64_add_cpu(&super->manifest.level_counts[add->level], 1); + add_level_count(sb, add->level, 1); return 0; } @@ -229,8 +269,6 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, u64 seq, u8 level) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct scoutfs_manifest_entry *ment; struct manifest_search_key skey; struct scoutfs_key_buf last; @@ -248,7 +286,7 @@ int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, le64_to_cpu(ment->seq), first, &last); scoutfs_ring_delete(&mani->ring, ment); - le64_add_cpu(&super->manifest.level_counts[level], -1ULL); + add_level_count(sb, level, -1ULL); return 0; } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index b65860a2..46aef7d1 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -45,6 +45,8 @@ int scoutfs_manifest_add_ment_ref(struct super_block *sb, int scoutfs_manifest_next_compact(struct super_block *sb, void *data); +bool scoutfs_manifest_level0_full(struct super_block *sb); + int scoutfs_manifest_setup(struct super_block *sb); void scoutfs_manifest_destroy(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index 76d9ca3a..91189304 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -84,6 +84,9 @@ struct net_info { struct llist_head ring_commit_waiters; struct work_struct ring_commit_work; + /* level 0 segment addition waits for it to clear */ + wait_queue_head_t waitq; + /* server tracks seq use */ spinlock_t seq_lock; struct list_head pending_seqs; @@ -422,6 +425,20 @@ static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, return sbuf; } +/* + * This is new segments arriving. It needs to wait for level 0 to be + * free. It has relatively little visibility into the manifest, though. + * We don't want it to block holding commits because that'll stop + * manifest updates from emptying level 0. + * + * Maybe the easiest way is to protect the level counts with a seqlock, + * or whatever. + */ + +/* + * The sender has written their level 0 segment and has given us its + * details. We wait for there to be room in level 0 before adding it. + */ static struct send_buf *process_record_segment(struct super_block *sb, void *req, int req_len) { @@ -443,9 +460,18 @@ static struct send_buf *process_record_segment(struct super_block *sb, goto out; } +retry: down_read(&nti->ring_commit_rwsem); - scoutfs_manifest_lock(sb); + + if (scoutfs_manifest_level0_full(sb)) { + scoutfs_manifest_unlock(sb); + up_read(&nti->ring_commit_rwsem); + /* XXX waits indefinitely? io errors? */ + wait_event(nti->waitq, !scoutfs_manifest_level0_full(sb)); + goto retry; + } + ret = scoutfs_manifest_add_ment(sb, ment); scoutfs_manifest_unlock(sb); @@ -1446,20 +1472,29 @@ int scoutfs_net_get_compaction(struct super_block *sb, void *curs) * In the future we'd encode the manifest and segnos in requests sent to * the server who'd update the manifest and allocator in request * processing. + * + * As we finish a compaction we wait level0 writers if it opened up + * space in level 0. */ int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, void *list) { DECLARE_NET_INFO(sb, nti); struct commit_waiter cw; + bool level0_was_full; int ret; down_read(&nti->ring_commit_rwsem); - ret = scoutfs_compact_commit(sb, curs, list); + level0_was_full = scoutfs_manifest_level0_full(sb); - if (ret == 0) + ret = scoutfs_compact_commit(sb, curs, list); + if (ret == 0) { queue_commit_work(nti, &cw); + if (level0_was_full && !scoutfs_manifest_level0_full(sb)) + wake_up(&nti->waitq); + } + up_read(&nti->ring_commit_rwsem); if (ret == 0) @@ -2150,6 +2185,7 @@ int scoutfs_net_setup(struct super_block *sb) init_rwsem(&nti->ring_commit_rwsem); init_llist_head(&nti->ring_commit_waiters); INIT_WORK(&nti->ring_commit_work, scoutfs_net_ring_commit_func); + init_waitqueue_head(&nti->waitq); spin_lock_init(&nti->seq_lock); INIT_LIST_HEAD(&nti->pending_seqs); INIT_LIST_HEAD(&nti->active_socks); From e7655f00ee7d660700bfa67e703a8ee063b79972 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 22 Jun 2017 17:08:02 -0700 Subject: [PATCH 315/920] scoutfs: read items from next segment in level If the starting key for a segment read doesn't fall in a segment then we have to include the next segment from that level in the read. If we don't then the read can think that there are no more items at that level and assume that all the items in the upper level are all that exist. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 564b7259..f757c1b9 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -423,7 +423,9 @@ int scoutfs_manifest_add_ment_ref(struct super_block *sb, * * We only need to search for the starting key in all the higher levels. * They do not overlap so we can iterate through the key space in each - * segment starting with the key. + * segment starting with the key. In each level we need the first + * existing segment that intersects with the range, even if it doesn't + * contain the key. The key might fall between segments at that level. * * This is called by the server who is processing manifest search * messages from mounts. The server locks down the manifest while it @@ -481,7 +483,7 @@ scoutfs_manifest_find_range_entries(struct super_block *sb, /* XXX should use level counts to skip searches */ - ment = scoutfs_ring_lookup(&mani->ring, &skey); + ment = scoutfs_ring_lookup_next(&mani->ring, &skey); if (ment) { found[nr++] = ment; *found_bytes += scoutfs_manifest_bytes(ment); From 76a73baefd28e15925ac5f12cd0c9a9cb63b2bb9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 22 Jun 2017 22:44:43 -0700 Subject: [PATCH 316/920] scoutfs: don't lose items between segments The refactoring of compaction to use the skip lists changed the nature of item insertion. Previously it would precisely count the number of items to insert. Now it discovers that the current output segment is full by having _append_item() return false. In this case the cursors currently point to the item that would have been inserted but failed. compact_items() caller loops around to allocate the next segment. Then it calls compact_items() again and it mistakenly advances *past* the current item that still needed to be inserted. Hiding next_item() away from the segment loop made it hard to see this mechanism. Let's drop the compact_items() function and bring item iteration and item appending into the main loop so we can more carefully advance or not as we write and allocate new segments. This stops losing items at segment boundaries. Signed-off-by: Zach Brown --- kmod/src/compact.c | 67 +++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 9bcefa30..56b248a8 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -272,53 +272,21 @@ out: return ret; } -/* - * Walk the input segments for items and append them to the output segment. - * Items can exist in the input segments but not be written to the output - * segment, for example if they're deletions. The output segment can be - * full. - * - * Return -errno if something went wrong, then 1 or 0 indicating items written. - */ -static int compact_items(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_segment *seg) -{ - struct scoutfs_key_buf item_key; - SCOUTFS_DECLARE_KVEC(item_val); - int has_next; - int ret = 0; - u8 flags; - - for (;;) { - has_next = next_item(sb, curs, &item_key, item_val, &flags); - if (has_next <= 0) { - if (has_next < 0) - ret = has_next; - break; - - } - - if (scoutfs_seg_append_item(sb, seg, &item_key, item_val, flags, - curs->links)) - ret = 1; - else - break; - } - - return ret; -} - static int compact_segments(struct super_block *sb, struct compact_cursor *curs, struct scoutfs_bio_completion *comp, struct list_head *results) { + struct scoutfs_key_buf item_key; + SCOUTFS_DECLARE_KVEC(item_val); struct scoutfs_segment *seg; struct compact_seg *cseg; struct compact_seg *upper; struct compact_seg *lower; unsigned next_segno = 0; + bool append_filled = false; int ret = 0; + u8 flags; scoutfs_inc_counter(sb, compact_operations); if (curs->sticky) @@ -394,6 +362,13 @@ static int compact_segments(struct super_block *sb, if (ret) break; + if (!append_filled) + ret = next_item(sb, curs, &item_key, item_val, &flags); + else + ret = 1; + if (ret <= 0) + break; + /* no cseg keys, manifest update uses seg item keys */ cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); if (!cseg) { @@ -411,9 +386,7 @@ static int compact_segments(struct super_block *sb, * case. */ ret = scoutfs_seg_alloc(sb, cseg->segno, &seg); - if (ret == 0) - ret = compact_items(sb, curs, seg); - if (ret < 1) { + if (ret < 0) { next_segno--; curs->segnos[next_segno] = cseg->segno; kfree(cseg); @@ -436,6 +409,22 @@ static int compact_segments(struct super_block *sb, cseg->seg = seg; list_add_tail(&cseg->entry, results); + for (;;) { + if (!scoutfs_seg_append_item(sb, seg, &item_key, item_val, + flags, curs->links)) { + append_filled = true; + ret = 0; + break; + } + ret = next_item(sb, curs, &item_key, item_val, &flags); + if (ret <= 0) { + append_filled = false; + break; + } + } + if (ret < 0) + break; + /* start a complete segment write now, we'll wait later */ ret = scoutfs_seg_submit_write(sb, seg, comp); if (ret) From e6f3b3ca8fbb10530357f37021b2507bb04ff72e Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 19 Jun 2017 15:26:57 -0500 Subject: [PATCH 317/920] scoutfs: add lock caching We refcount our locks and hold them across system calls. If another node wants access to a given lock we'll mark it as blocking in the bast and queue a work item so that the lock can later be released. Otherwise locks are free'd under memory pressure, unmount or after a timer fires. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 366 +++++++++++++++++++++++++++++++++++---- kmod/src/lock.h | 17 +- kmod/src/net.c | 4 +- kmod/src/scoutfs_trace.h | 32 +++- kmod/src/xattr.c | 16 +- 5 files changed, 386 insertions(+), 49 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 379983be..ea915df8 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -21,6 +21,8 @@ #include "scoutfs_trace.h" #include "msg.h" +#include "../dlm/interval_tree_generic.h" + #include "linux/dlm.h" /* @@ -28,10 +30,15 @@ * the same fsid. Freed as the last super unmounts. */ struct held_locks { + struct super_block *sb; spinlock_t lock; - struct list_head list; unsigned int seq_cnt; wait_queue_head_t waitq; + struct rb_root lock_tree; + struct workqueue_struct *downconvert_wq; + struct shrinker shrinker; + struct list_head lru_list; + unsigned long long lru_nr; }; /* @@ -51,10 +58,18 @@ struct lock_info { #define RANGE_LOCK_RESOURCE "fs_range" #define RANGE_LOCK_RESOURCE_LEN (strlen(RANGE_LOCK_RESOURCE)) - #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info +static void scoutfs_downconvert_func(struct work_struct *work); + +#define START(lck) ((lck)->start) +#define LAST(lck) ((lck)->end) +KEYED_INTERVAL_TREE_DEFINE(struct scoutfs_lock, interval_node, + struct scoutfs_key_buf *, subtree_last, START, LAST, + scoutfs_key_compare, static, scoutfs_lock); + + /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -79,14 +94,34 @@ static int invalidate_caches(struct super_block *sb, int mode, return ret; } -static void uninit_scoutfs_lock(struct held_locks *held, - struct scoutfs_lock *lck) +static void free_scoutfs_lock(struct scoutfs_lock *lck) { - spin_lock(&held->lock); - lck->rqmode = SCOUTFS_LOCK_MODE_IV; - list_del_init(&lck->head); - spin_unlock(&held->lock); - lck->sequence = 0; + kfree(lck->start); + kfree(lck->end); + kfree(lck); +} + +static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + unsigned int refs; + + if (lck) { + spin_lock(&held->lock); + BUG_ON(!lck->refcnt); + refs = --lck->refcnt; + if (!refs) { + BUG_ON(lck->holders); + BUG_ON(delayed_work_pending(&lck->dc_work)); + scoutfs_lock_remove(lck, &held->lock_tree); + list_del(&lck->lru_entry); + spin_unlock(&held->lock); + free_scoutfs_lock(lck); + return; + } + spin_unlock(&held->lock); + } } static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, @@ -96,10 +131,11 @@ static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, DECLARE_LOCK_INFO(sb, linfo); struct held_locks *held = linfo->held; - memset(lck, 0, sizeof(*lck)); - INIT_LIST_HEAD(&lck->head); + RB_CLEAR_NODE(&lck->interval_node); lck->sb = sb; lck->mode = SCOUTFS_LOCK_MODE_IV; + INIT_DELAYED_WORK(&lck->dc_work, scoutfs_downconvert_func); + INIT_LIST_HEAD(&lck->lru_entry); if (start) { lck->start = start; @@ -117,6 +153,124 @@ static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, spin_unlock(&held->lock); } +static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) + +{ + struct scoutfs_key_buf *s, *e; + struct scoutfs_lock *lck; + + s = scoutfs_key_dup(sb, start); + if (!s) + return NULL; + e = scoutfs_key_dup(sb, end); + if (!e) { + kfree(s); + return NULL; + } + lck = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); + if (!lck) { + kfree(e); + kfree(s); + } + + init_scoutfs_lock(sb, lck, s, e); + return lck; +} + +static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + struct scoutfs_lock *found, *new; + + new = NULL; + spin_lock(&held->lock); +search: + found = scoutfs_lock_iter_first(&held->lock_tree, start, end); + if (!found) { + if (!new) { + spin_unlock(&held->lock); + new = alloc_scoutfs_lock(sb, start, end); + if (!new) + return NULL; + + spin_lock(&held->lock); + goto search; + } + new->refcnt = 1; /* Freed by shrinker or on umount */ + scoutfs_lock_insert(new, &held->lock_tree); + found = new; + new = NULL; + } + found->refcnt++; + if (!list_empty(&found->lru_entry)) { + list_del_init(&found->lru_entry); + held->lru_nr--; + } + spin_unlock(&held->lock); + + kfree(new); + return found; +} + +static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) +{ + struct held_locks *held = container_of(shrink, struct held_locks, + shrinker); + struct scoutfs_lock *lck; + struct scoutfs_lock *tmp; + unsigned long flags; + unsigned long nr; + LIST_HEAD(list); + + nr = sc->nr_to_scan; + if (!nr) + goto out; + + spin_lock_irqsave(&held->lock, flags); + list_for_each_entry_safe(lck, tmp, &held->lru_list, lru_entry) { + if (nr-- == 0) + break; + + WARN_ON(lck->holders); + WARN_ON(lck->refcnt != 1); + WARN_ON(lck->flags & SCOUTFS_LOCK_QUEUED); + + scoutfs_lock_remove(lck, &held->lock_tree); + list_del(&lck->lru_entry); + list_add_tail(&lck->lru_entry, &list); + held->lru_nr--; + } + spin_unlock_irqrestore(&held->lock, flags); + + list_for_each_entry_safe(lck, tmp, &list, lru_entry) { + trace_shrink_lock_tree(held->sb, lck); + list_del(&lck->lru_entry); + free_scoutfs_lock(lck); + } +out: + return min_t(unsigned long, held->lru_nr, INT_MAX); +} + +static void free_lock_tree(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + struct rb_node *node = rb_first(&held->lock_tree); + + while (node) { + struct scoutfs_lock *lck; + + lck = rb_entry(node, struct scoutfs_lock, interval_node); + node = rb_next(node); + put_scoutfs_lock(sb, lck); + } +} + static void scoutfs_ast(void *astarg) { struct scoutfs_lock *lck = astarg; @@ -127,16 +281,48 @@ static void scoutfs_ast(void *astarg) spin_lock(&held->lock); lck->mode = lck->rqmode; - lck->rqmode = SCOUTFS_LOCK_MODE_IV; + /* Clear blocking flag when we are granted an unlock request */ + if (lck->rqmode == DLM_LOCK_IV) + lck->flags &= ~SCOUTFS_LOCK_BLOCKING; + lck->rqmode = DLM_LOCK_IV; spin_unlock(&held->lock); wake_up(&held->waitq); } +static void queue_blocking_work(struct held_locks *held, + struct scoutfs_lock *lck, unsigned int seconds) +{ + assert_spin_locked(&held->lock); + if (!(lck->flags & SCOUTFS_LOCK_QUEUED)) { + /* Take a ref for the workqueue */ + lck->flags |= SCOUTFS_LOCK_QUEUED; + lck->refcnt++; + } + mod_delayed_work(held->downconvert_wq, &lck->dc_work, seconds * HZ); +} + +static void set_lock_blocking(struct held_locks *held, + struct scoutfs_lock *lck, unsigned int seconds) +{ + assert_spin_locked(&held->lock); + lck->flags |= SCOUTFS_LOCK_BLOCKING; + if (lck->holders == 0) + queue_blocking_work(held, lck, seconds); +} + static void scoutfs_rbast(void *astarg, int mode, struct dlm_key *start, struct dlm_key *end) { + struct scoutfs_lock *lck = astarg; + struct lock_info *linfo = SCOUTFS_SB(lck->sb)->lock_info; + struct held_locks *held = linfo->held; + trace_scoutfs_rbast(lck->sb, lck); + + spin_lock(&held->lock); + set_lock_blocking(held, lck, 0); + spin_unlock(&held->lock); } static int lock_granted(struct held_locks *held, struct scoutfs_lock *lck, @@ -151,6 +337,17 @@ static int lock_granted(struct held_locks *held, struct scoutfs_lock *lck, return ret; } +static int lock_blocking(struct held_locks *held, struct scoutfs_lock *lck) +{ + int ret; + + spin_lock(&held->lock); + ret = !!(lck->flags & SCOUTFS_LOCK_BLOCKING); + spin_unlock(&held->lock); + + return ret; +} + /* * Acquire a coherent lock on the given range of keys. While the lock * is held other lockers are serialized. Cache coherency is maintained @@ -163,26 +360,54 @@ static int lock_granted(struct held_locks *held, struct scoutfs_lock *lck, int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, - struct scoutfs_lock *lck) + struct scoutfs_lock **ret_lck) { DECLARE_LOCK_INFO(sb, linfo); struct held_locks *held = linfo->held; + struct scoutfs_lock *lck; int ret; - init_scoutfs_lock(sb, lck, start, end); + lck = find_alloc_scoutfs_lock(sb, start, end); + if (!lck) + return -ENOMEM; trace_scoutfs_lock_range(sb, lck); +check_lock_state: spin_lock(&held->lock); if (linfo->shutdown) { spin_unlock(&held->lock); + put_scoutfs_lock(sb, lck); return -ESHUTDOWN; } - list_add(&lck->head, &held->list); - spin_unlock(&held->lock); + if (lck->flags & SCOUTFS_LOCK_BLOCKING) { + spin_unlock(&held->lock); + wait_event(held->waitq, !lock_blocking(held, lck)); + goto check_lock_state; + } + + if (lck->mode > DLM_LOCK_IV) { + if (lck->mode < mode) { + /* + * We already have the lock but at a mode which is not + * compatible with what the caller wants. Set the lock + * blocking to let the downconvert thread do it's work + * so we can reacquire at the correct mode. + */ + set_lock_blocking(held, lck, 0); + spin_unlock(&held->lock); + goto check_lock_state; + } + lck->holders++; + spin_unlock(&held->lock); + goto out; + } lck->rqmode = mode; + lck->holders++; + spin_unlock(&held->lock); + ret = dlm_lock_range(linfo->ls, mode, &lck->dlm_start, &lck->dlm_end, &lck->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, RANGE_LOCK_RESOURCE_LEN, 0, scoutfs_ast, lck, @@ -190,16 +415,37 @@ int scoutfs_lock_range(struct super_block *sb, int mode, if (ret) { scoutfs_err(sb, "Error %d locking %s\n", ret, RANGE_LOCK_RESOURCE); - uninit_scoutfs_lock(held, lck); + put_scoutfs_lock(sb, lck); return ret; } wait_event(held->waitq, lock_granted(held, lck, mode)); - +out: + *ret_lck = lck; return 0; } void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + unsigned int seconds = 60; + + trace_scoutfs_unlock_range(sb, lck); + + spin_lock(&held->lock); + lck->holders--; + if (lck->holders == 0) { + if (lck->flags & SCOUTFS_LOCK_BLOCKING) + seconds = 0; + queue_blocking_work(held, lck, seconds); + } + spin_unlock(&held->lock); + + put_scoutfs_lock(sb, lck); +} + +static void unlock_range(struct super_block *sb, struct scoutfs_lock *lck) { DECLARE_LOCK_INFO(sb, linfo); struct held_locks *held = linfo->held; @@ -209,15 +455,9 @@ void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) BUG_ON(!lck->sequence); - /* - * Use write mode to invalidate all since we are completely - * dropping the lock. Once we keep the locks around then we - * can invalidate based on what level we're downconverting to - * (PR, NL). - */ - invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lck->start, lck->end); - + spin_lock(&held->lock); lck->rqmode = DLM_LOCK_IV; + spin_unlock(&held->lock); ret = dlm_unlock(linfo->ls, lck->lksb.sb_lkid, 0, &lck->lksb, lck); if (ret) { scoutfs_err(sb, "Error %d unlocking %s\n", ret, @@ -227,11 +467,58 @@ void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) wait_event(held->waitq, lock_granted(held, lck, DLM_LOCK_IV)); out: - uninit_scoutfs_lock(held, lck); - /* lock was removed from held list, wake up umount process */ + /* lock was removed from tree, wake up umount process */ wake_up(&held->waitq); } +static void scoutfs_downconvert_func(struct work_struct *work) +{ + struct scoutfs_lock *lck = container_of(work, struct scoutfs_lock, + dc_work.work); + struct super_block *sb = lck->sb; + DECLARE_LOCK_INFO(sb, linfo); + struct held_locks *held = linfo->held; + + trace_scoutfs_downconvert_func(sb, lck); + + spin_lock(&held->lock); + lck->flags &= ~SCOUTFS_LOCK_QUEUED; + if (lck->holders) + goto out; /* scoutfs_unlock_range will requeue for us */ + + spin_unlock(&held->lock); + + WARN_ON_ONCE(lck->holders); + WARN_ON_ONCE(lck->refcnt == 0); + /* + * Use write mode to invalidate all since we are completely + * dropping the lock. Once we are dowconverting, we can + * invalidate based on what level we're downconverting to (PR, + * NL). + */ + invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lck->start, lck->end); + unlock_range(sb, lck); + + spin_lock(&held->lock); + /* Check whether we can add the lock to the LRU list: + * + * First, check mode to be sure that the lock wasn't reacquired + * while we slept in unlock_range(). + * + * Next, check refs. refcnt == 1 means the only holder is the + * lock tree so in particular we have nobody in + * scoutfs_lock_range concurrently trying to acquire a lock. + */ + if (lck->mode == SCOUTFS_LOCK_MODE_IV && lck->refcnt == 1 && + list_empty(&lck->lru_entry)) { + list_add_tail(&lck->lru_entry, &held->lru_list); + held->lru_nr++; + } +out: + spin_unlock(&held->lock); + put_scoutfs_lock(sb, lck); +} + /* * The moment this is done we can have other mounts start asking * us to write back and invalidate, so do this very very late. @@ -253,8 +540,12 @@ static int init_lock_info(struct super_block *sb) } spin_lock_init(&held->lock); - INIT_LIST_HEAD(&held->list); init_waitqueue_head(&held->waitq); + INIT_LIST_HEAD(&held->lru_list); + held->shrinker.shrink = shrink_lock_tree; + held->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&held->shrinker); + held->sb = sb; linfo->sb = sb; linfo->shutdown = false; @@ -278,7 +569,7 @@ static int can_complete_shutdown(struct held_locks *held) int ret; spin_lock(&held->lock); - ret = !!list_empty(&held->list); + ret = !!RB_EMPTY_ROOT(&held->lock_tree); spin_unlock(&held->lock); return ret; } @@ -313,13 +604,16 @@ void scoutfs_lock_destroy(struct super_block *sb) if (linfo) { held = linfo->held; - wait_event(held->waitq, can_complete_shutdown(held)); + destroy_workqueue(held->downconvert_wq); + unregister_shrinker(&held->shrinker); ret = dlm_release_lockspace(linfo->ls, 2); if (ret) scoutfs_info(sb, "Error %d releasing lockspace %s\n", ret, linfo->ls_name); + free_lock_tree(sb); + sbi->lock_info = NULL; trace_printk("sb %p id %016llx freeing linfo %p held %p\n", @@ -332,6 +626,7 @@ void scoutfs_lock_destroy(struct super_block *sb) int scoutfs_lock_setup(struct super_block *sb) { + struct held_locks *held; struct lock_info *linfo; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); int ret; @@ -341,6 +636,15 @@ int scoutfs_lock_setup(struct super_block *sb) return ret; linfo = sbi->lock_info; + held = linfo->held; + held->downconvert_wq = alloc_workqueue("scoutfs_dc", + WQ_UNBOUND|WQ_HIGHPRI, 0); + if (!held->downconvert_wq) { + kfree(held); + kfree(linfo); + return -ENOMEM; + } + /* * Open coded '64' here is for lvb_len. We never use the LVB * flag so this doesn't matter, but the dlm needs a non-zero diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 89e820e9..08459e07 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -3,8 +3,10 @@ #include "../dlm/include/linux/dlm.h" +#define SCOUTFS_LOCK_BLOCKING 0x01 /* Blocking another lock request */ +#define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ + struct scoutfs_lock { - struct list_head head; struct super_block *sb; struct scoutfs_key_buf *start; struct scoutfs_key_buf *end; @@ -14,6 +16,13 @@ struct scoutfs_lock { struct dlm_key dlm_start; struct dlm_key dlm_end; unsigned int sequence; /* for debugging and sanity checks */ + struct rb_node interval_node; + struct scoutfs_key_buf *subtree_last; + struct list_head lru_entry; + unsigned int refcnt; + unsigned int holders; /* Tracks active users of this lock */ + unsigned int flags; + struct delayed_work dc_work; }; enum { @@ -23,9 +32,9 @@ enum { }; int scoutfs_lock_range(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct scoutfs_lock *lck); + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_lock **ret_lck); void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck); int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, diff --git a/kmod/src/net.c b/kmod/src/net.c index 91189304..d971bb36 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -144,7 +144,7 @@ struct sock_info { struct list_head have_sent; struct list_head active_rbufs; - struct scoutfs_lock listen_lck; + struct scoutfs_lock *listen_lck; struct scoutfs_inet_addr addr; struct work_struct listen_work; @@ -1245,7 +1245,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) scoutfs_err(sb, "Non-fatal error %d while writing server " "address\n", ret); - scoutfs_unlock_range(sb, &sinf->listen_lck); + scoutfs_unlock_range(sb, sinf->listen_lck); queue_delayed_work(nti->proc_wq, &nti->server_work, 0); } if (sinf == nti->connected_sinf) { diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 175fe3e6..7b1ecff4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -309,17 +309,26 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(unsigned int, seq) __dynamic_array(char, start, scoutfs_key_str(NULL, lck->start)) __dynamic_array(char, end, scoutfs_key_str(NULL, lck->end)) - ), + __field(unsigned int, flags) + __field(unsigned int, refcnt) + __field(unsigned int, holders) + ), TP_fast_assign( __entry->mode = lck->mode; __entry->rqmode = lck->rqmode; __entry->seq = lck->sequence; + __entry->flags = lck->flags; + __entry->refcnt = lck->refcnt; + __entry->holders = lck->holders; scoutfs_key_str(__get_dynamic_array(start), lck->start); scoutfs_key_str(__get_dynamic_array(end), lck->end); ), - TP_printk("seq %u mode %s rqmode %s start %s end %s", - __entry->seq, lock_mode(__entry->mode), - lock_mode(__entry->rqmode), __get_str(start), __get_str(end)) + TP_printk("seq %u refs %d holders %d mode %s rqmode %s flags 0x%x " + "start %s end %s", + __entry->seq, __entry->refcnt, __entry->holders, + lock_mode(__entry->mode), lock_mode(__entry->rqmode), + __entry->flags, __get_str(start), + __get_str(end)) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_range, @@ -337,6 +346,21 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_ast, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_rbast, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +DEFINE_EVENT(scoutfs_lock_class, scoutfs_downconvert_func, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +DEFINE_EVENT(scoutfs_lock_class, shrink_lock_tree, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + TRACE_EVENT(scoutfs_lock_invalidate_sb, TP_PROTO(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 2bd066d4..fc8a4af7 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -151,7 +151,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, struct scoutfs_key_buf *key = NULL; struct scoutfs_key_buf *last = NULL; SCOUTFS_DECLARE_KVEC(val); - struct scoutfs_lock lck; + struct scoutfs_lock *lck; unsigned int total; unsigned int bytes; unsigned int off; @@ -228,7 +228,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = -ERANGE; up_read(&si->xattr_rwsem); - scoutfs_unlock_range(sb, &lck); + scoutfs_unlock_range(sb, lck); out: scoutfs_key_free(sb, key); @@ -263,7 +263,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); DECLARE_ITEM_COUNT(cnt); - struct scoutfs_lock lck; + struct scoutfs_lock *lck; unsigned int bytes; unsigned int off; LIST_HEAD(list); @@ -335,7 +335,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, scoutfs_release_trans(sb); unlock: - scoutfs_unlock_range(sb, &lck); + scoutfs_unlock_range(sb, lck); out: scoutfs_item_free_batch(sb, &list); @@ -368,7 +368,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct scoutfs_xattr_key *xkey; struct scoutfs_key_buf *key; struct scoutfs_key_buf *last; - struct scoutfs_lock lck; + struct scoutfs_lock *lck; ssize_t total; int name_len; int ret; @@ -435,7 +435,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) } up_read(&si->xattr_rwsem); - scoutfs_unlock_range(sb, &lck); + scoutfs_unlock_range(sb, lck); out: scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); @@ -456,7 +456,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { struct scoutfs_key_buf *key; struct scoutfs_key_buf *last; - struct scoutfs_lock lck; + struct scoutfs_lock *lck; int ret; key = alloc_xattr_key(sb, ino, NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); @@ -489,7 +489,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* don't need to increment past deleted key */ } - scoutfs_unlock_range(sb, &lck); + scoutfs_unlock_range(sb, lck); out: scoutfs_key_free(sb, key); From 250e9d2701ca01a4270773bea0e3ef1cfeb319d8 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 27 Jun 2017 16:29:31 -0500 Subject: [PATCH 318/920] scoutfs: remove unused function, can_complete_shutdown Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ea915df8..88b1661c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -564,16 +564,6 @@ static int init_lock_info(struct super_block *sb) return 0; } -static int can_complete_shutdown(struct held_locks *held) -{ - int ret; - - spin_lock(&held->lock); - ret = !!RB_EMPTY_ROOT(&held->lock_tree); - spin_unlock(&held->lock); - return ret; -} - /* * Cause all lock attempts from our super to fail, waking anyone who is * currently blocked attempting to lock. Now that locks can't block we From 19f6f40fee592398e1250760506abc4ec08c287e Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 27 Jun 2017 16:38:23 -0500 Subject: [PATCH 319/920] scoutfs: get rid of held_locks construct Now that we have a dlm, this is a needless redirection. Merge all fields back into the lock_info struct. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 213 ++++++++++++++++++++---------------------------- 1 file changed, 87 insertions(+), 126 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 88b1661c..0e610ad6 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -26,11 +26,15 @@ #include "linux/dlm.h" /* - * Allocated once and pointed to by the lock info of all the supers with - * the same fsid. Freed as the last super unmounts. + * allocated per-super, freed on unmount. */ -struct held_locks { +struct lock_info { struct super_block *sb; + dlm_lockspace_t *ls; + char ls_name[DLM_LOCKSPACE_LEN]; + bool shutdown; + struct list_head id_head; + spinlock_t lock; unsigned int seq_cnt; wait_queue_head_t waitq; @@ -41,20 +45,6 @@ struct held_locks { unsigned long long lru_nr; }; -/* - * allocated per-super. Stored in the global list for finding supers - * with fsids and stored in a list with others with the same fsid for - * invalidation. Freed on unmount. - */ -struct lock_info { - struct super_block *sb; - dlm_lockspace_t *ls; - char ls_name[DLM_LOCKSPACE_LEN]; - bool shutdown; - struct held_locks *held; - struct list_head id_head; -}; - #define RANGE_LOCK_RESOURCE "fs_range" #define RANGE_LOCK_RESOURCE_LEN (strlen(RANGE_LOCK_RESOURCE)) @@ -104,23 +94,22 @@ static void free_scoutfs_lock(struct scoutfs_lock *lck) static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; unsigned int refs; if (lck) { - spin_lock(&held->lock); + spin_lock(&linfo->lock); BUG_ON(!lck->refcnt); refs = --lck->refcnt; if (!refs) { BUG_ON(lck->holders); BUG_ON(delayed_work_pending(&lck->dc_work)); - scoutfs_lock_remove(lck, &held->lock_tree); + scoutfs_lock_remove(lck, &linfo->lock_tree); list_del(&lck->lru_entry); - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); free_scoutfs_lock(lck); return; } - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); } } @@ -129,7 +118,6 @@ static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, struct scoutfs_key_buf *end) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; RB_CLEAR_NODE(&lck->interval_node); lck->sb = sb; @@ -148,9 +136,9 @@ static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, lck->dlm_end.len = end->key_len; } - spin_lock(&held->lock); - lck->sequence = ++held->seq_cnt; - spin_unlock(&held->lock); + spin_lock(&linfo->lock); + lck->sequence = ++linfo->seq_cnt; + spin_unlock(&linfo->lock); } static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, @@ -184,34 +172,33 @@ static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_key_buf *end) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; struct scoutfs_lock *found, *new; new = NULL; - spin_lock(&held->lock); + spin_lock(&linfo->lock); search: - found = scoutfs_lock_iter_first(&held->lock_tree, start, end); + found = scoutfs_lock_iter_first(&linfo->lock_tree, start, end); if (!found) { if (!new) { - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); new = alloc_scoutfs_lock(sb, start, end); if (!new) return NULL; - spin_lock(&held->lock); + spin_lock(&linfo->lock); goto search; } new->refcnt = 1; /* Freed by shrinker or on umount */ - scoutfs_lock_insert(new, &held->lock_tree); + scoutfs_lock_insert(new, &linfo->lock_tree); found = new; new = NULL; } found->refcnt++; if (!list_empty(&found->lru_entry)) { list_del_init(&found->lru_entry); - held->lru_nr--; + linfo->lru_nr--; } - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); kfree(new); return found; @@ -219,7 +206,7 @@ search: static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) { - struct held_locks *held = container_of(shrink, struct held_locks, + struct lock_info *linfo = container_of(shrink, struct lock_info, shrinker); struct scoutfs_lock *lck; struct scoutfs_lock *tmp; @@ -231,8 +218,8 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) if (!nr) goto out; - spin_lock_irqsave(&held->lock, flags); - list_for_each_entry_safe(lck, tmp, &held->lru_list, lru_entry) { + spin_lock_irqsave(&linfo->lock, flags); + list_for_each_entry_safe(lck, tmp, &linfo->lru_list, lru_entry) { if (nr-- == 0) break; @@ -240,27 +227,26 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) WARN_ON(lck->refcnt != 1); WARN_ON(lck->flags & SCOUTFS_LOCK_QUEUED); - scoutfs_lock_remove(lck, &held->lock_tree); + scoutfs_lock_remove(lck, &linfo->lock_tree); list_del(&lck->lru_entry); list_add_tail(&lck->lru_entry, &list); - held->lru_nr--; + linfo->lru_nr--; } - spin_unlock_irqrestore(&held->lock, flags); + spin_unlock_irqrestore(&linfo->lock, flags); list_for_each_entry_safe(lck, tmp, &list, lru_entry) { - trace_shrink_lock_tree(held->sb, lck); + trace_shrink_lock_tree(linfo->sb, lck); list_del(&lck->lru_entry); free_scoutfs_lock(lck); } out: - return min_t(unsigned long, held->lru_nr, INT_MAX); + return min_t(unsigned long, linfo->lru_nr, INT_MAX); } static void free_lock_tree(struct super_block *sb) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; - struct rb_node *node = rb_first(&held->lock_tree); + struct rb_node *node = rb_first(&linfo->lock_tree); while (node) { struct scoutfs_lock *lck; @@ -275,40 +261,39 @@ static void scoutfs_ast(void *astarg) { struct scoutfs_lock *lck = astarg; DECLARE_LOCK_INFO(lck->sb, linfo); - struct held_locks *held = linfo->held; trace_scoutfs_ast(lck->sb, lck); - spin_lock(&held->lock); + spin_lock(&linfo->lock); lck->mode = lck->rqmode; /* Clear blocking flag when we are granted an unlock request */ if (lck->rqmode == DLM_LOCK_IV) lck->flags &= ~SCOUTFS_LOCK_BLOCKING; lck->rqmode = DLM_LOCK_IV; - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); - wake_up(&held->waitq); + wake_up(&linfo->waitq); } -static void queue_blocking_work(struct held_locks *held, +static void queue_blocking_work(struct lock_info *linfo, struct scoutfs_lock *lck, unsigned int seconds) { - assert_spin_locked(&held->lock); + assert_spin_locked(&linfo->lock); if (!(lck->flags & SCOUTFS_LOCK_QUEUED)) { /* Take a ref for the workqueue */ lck->flags |= SCOUTFS_LOCK_QUEUED; lck->refcnt++; } - mod_delayed_work(held->downconvert_wq, &lck->dc_work, seconds * HZ); + mod_delayed_work(linfo->downconvert_wq, &lck->dc_work, seconds * HZ); } -static void set_lock_blocking(struct held_locks *held, - struct scoutfs_lock *lck, unsigned int seconds) +static void set_lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lck, + unsigned int seconds) { - assert_spin_locked(&held->lock); + assert_spin_locked(&linfo->lock); lck->flags |= SCOUTFS_LOCK_BLOCKING; if (lck->holders == 0) - queue_blocking_work(held, lck, seconds); + queue_blocking_work(linfo, lck, seconds); } static void scoutfs_rbast(void *astarg, int mode, @@ -316,34 +301,33 @@ static void scoutfs_rbast(void *astarg, int mode, { struct scoutfs_lock *lck = astarg; struct lock_info *linfo = SCOUTFS_SB(lck->sb)->lock_info; - struct held_locks *held = linfo->held; trace_scoutfs_rbast(lck->sb, lck); - spin_lock(&held->lock); - set_lock_blocking(held, lck, 0); - spin_unlock(&held->lock); + spin_lock(&linfo->lock); + set_lock_blocking(linfo, lck, 0); + spin_unlock(&linfo->lock); } -static int lock_granted(struct held_locks *held, struct scoutfs_lock *lck, +static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lck, int mode) { int ret; - spin_lock(&held->lock); + spin_lock(&linfo->lock); ret = !!(mode == lck->mode); - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); return ret; } -static int lock_blocking(struct held_locks *held, struct scoutfs_lock *lck) +static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lck) { int ret; - spin_lock(&held->lock); + spin_lock(&linfo->lock); ret = !!(lck->flags & SCOUTFS_LOCK_BLOCKING); - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); return ret; } @@ -363,7 +347,6 @@ int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_lock **ret_lck) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; struct scoutfs_lock *lck; int ret; @@ -374,16 +357,16 @@ int scoutfs_lock_range(struct super_block *sb, int mode, trace_scoutfs_lock_range(sb, lck); check_lock_state: - spin_lock(&held->lock); + spin_lock(&linfo->lock); if (linfo->shutdown) { - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); put_scoutfs_lock(sb, lck); return -ESHUTDOWN; } if (lck->flags & SCOUTFS_LOCK_BLOCKING) { - spin_unlock(&held->lock); - wait_event(held->waitq, !lock_blocking(held, lck)); + spin_unlock(&linfo->lock); + wait_event(linfo->waitq, !lock_blocking(linfo, lck)); goto check_lock_state; } @@ -395,18 +378,18 @@ check_lock_state: * blocking to let the downconvert thread do it's work * so we can reacquire at the correct mode. */ - set_lock_blocking(held, lck, 0); - spin_unlock(&held->lock); + set_lock_blocking(linfo, lck, 0); + spin_unlock(&linfo->lock); goto check_lock_state; } lck->holders++; - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); goto out; } lck->rqmode = mode; lck->holders++; - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); ret = dlm_lock_range(linfo->ls, mode, &lck->dlm_start, &lck->dlm_end, &lck->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, @@ -419,7 +402,7 @@ check_lock_state: return ret; } - wait_event(held->waitq, lock_granted(held, lck, mode)); + wait_event(linfo->waitq, lock_granted(linfo, lck, mode)); out: *ret_lck = lck; return 0; @@ -428,19 +411,18 @@ out: void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; unsigned int seconds = 60; trace_scoutfs_unlock_range(sb, lck); - spin_lock(&held->lock); + spin_lock(&linfo->lock); lck->holders--; if (lck->holders == 0) { if (lck->flags & SCOUTFS_LOCK_BLOCKING) seconds = 0; - queue_blocking_work(held, lck, seconds); + queue_blocking_work(linfo, lck, seconds); } - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); put_scoutfs_lock(sb, lck); } @@ -448,16 +430,15 @@ void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) static void unlock_range(struct super_block *sb, struct scoutfs_lock *lck) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; int ret; trace_scoutfs_unlock_range(sb, lck); BUG_ON(!lck->sequence); - spin_lock(&held->lock); + spin_lock(&linfo->lock); lck->rqmode = DLM_LOCK_IV; - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); ret = dlm_unlock(linfo->ls, lck->lksb.sb_lkid, 0, &lck->lksb, lck); if (ret) { scoutfs_err(sb, "Error %d unlocking %s\n", ret, @@ -465,10 +446,10 @@ static void unlock_range(struct super_block *sb, struct scoutfs_lock *lck) goto out; } - wait_event(held->waitq, lock_granted(held, lck, DLM_LOCK_IV)); + wait_event(linfo->waitq, lock_granted(linfo, lck, DLM_LOCK_IV)); out: /* lock was removed from tree, wake up umount process */ - wake_up(&held->waitq); + wake_up(&linfo->waitq); } static void scoutfs_downconvert_func(struct work_struct *work) @@ -477,16 +458,15 @@ static void scoutfs_downconvert_func(struct work_struct *work) dc_work.work); struct super_block *sb = lck->sb; DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; trace_scoutfs_downconvert_func(sb, lck); - spin_lock(&held->lock); + spin_lock(&linfo->lock); lck->flags &= ~SCOUTFS_LOCK_QUEUED; if (lck->holders) goto out; /* scoutfs_unlock_range will requeue for us */ - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); WARN_ON_ONCE(lck->holders); WARN_ON_ONCE(lck->refcnt == 0); @@ -499,7 +479,7 @@ static void scoutfs_downconvert_func(struct work_struct *work) invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lck->start, lck->end); unlock_range(sb, lck); - spin_lock(&held->lock); + spin_lock(&linfo->lock); /* Check whether we can add the lock to the LRU list: * * First, check mode to be sure that the lock wasn't reacquired @@ -511,11 +491,11 @@ static void scoutfs_downconvert_func(struct work_struct *work) */ if (lck->mode == SCOUTFS_LOCK_MODE_IV && lck->refcnt == 1 && list_empty(&lck->lru_entry)) { - list_add_tail(&lck->lru_entry, &held->lru_list); - held->lru_nr++; + list_add_tail(&lck->lru_entry, &linfo->lru_list); + linfo->lru_nr++; } out: - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); put_scoutfs_lock(sb, lck); } @@ -526,30 +506,20 @@ out: static int init_lock_info(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct held_locks *held; struct lock_info *linfo; linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); if (!linfo) return -ENOMEM; - held = kzalloc(sizeof(struct held_locks), GFP_KERNEL); - if (!held) { - kfree(linfo); - return -ENOMEM; - } - - spin_lock_init(&held->lock); - init_waitqueue_head(&held->waitq); - INIT_LIST_HEAD(&held->lru_list); - held->shrinker.shrink = shrink_lock_tree; - held->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&held->shrinker); - held->sb = sb; - + spin_lock_init(&linfo->lock); + init_waitqueue_head(&linfo->waitq); + INIT_LIST_HEAD(&linfo->lru_list); + linfo->shrinker.shrink = shrink_lock_tree; + linfo->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&linfo->shrinker); linfo->sb = sb; linfo->shutdown = false; - linfo->held = held; INIT_LIST_HEAD(&linfo->id_head); linfo->ls = NULL; @@ -559,7 +529,7 @@ static int init_lock_info(struct super_block *sb) sbi->lock_info = linfo; trace_printk("sb %p id %016llx allocated linfo %p held %p\n", - sb, le64_to_cpu(sbi->super.id), linfo, held); + sb, le64_to_cpu(sbi->super.id), linfo, linfo); return 0; } @@ -573,15 +543,13 @@ static int init_lock_info(struct super_block *sb) void scoutfs_lock_shutdown(struct super_block *sb) { DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held = linfo->held; if (linfo) { - held = linfo->held; - spin_lock(&held->lock); + spin_lock(&linfo->lock); linfo->shutdown = true; - spin_unlock(&held->lock); + spin_unlock(&linfo->lock); - wake_up(&held->waitq); + wake_up(&linfo->waitq); } } @@ -589,14 +557,11 @@ void scoutfs_lock_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_LOCK_INFO(sb, linfo); - struct held_locks *held; int ret; if (linfo) { - held = linfo->held; - - destroy_workqueue(held->downconvert_wq); - unregister_shrinker(&held->shrinker); + destroy_workqueue(linfo->downconvert_wq); + unregister_shrinker(&linfo->shrinker); ret = dlm_release_lockspace(linfo->ls, 2); if (ret) scoutfs_info(sb, "Error %d releasing lockspace %s\n", @@ -606,17 +571,15 @@ void scoutfs_lock_destroy(struct super_block *sb) sbi->lock_info = NULL; - trace_printk("sb %p id %016llx freeing linfo %p held %p\n", - sb, le64_to_cpu(sbi->super.id), linfo, held); + trace_printk("sb %p id %016llx freeing linfo %p linfo %p\n", + sb, le64_to_cpu(sbi->super.id), linfo, linfo); - kfree(held); kfree(linfo); } } int scoutfs_lock_setup(struct super_block *sb) { - struct held_locks *held; struct lock_info *linfo; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); int ret; @@ -626,11 +589,9 @@ int scoutfs_lock_setup(struct super_block *sb) return ret; linfo = sbi->lock_info; - held = linfo->held; - held->downconvert_wq = alloc_workqueue("scoutfs_dc", + linfo->downconvert_wq = alloc_workqueue("scoutfs_dc", WQ_UNBOUND|WQ_HIGHPRI, 0); - if (!held->downconvert_wq) { - kfree(held); + if (!linfo->downconvert_wq) { kfree(linfo); return -ENOMEM; } From 136cbbed2994dea7b12c0586e2b0ba76b962f10d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 28 Jun 2017 13:28:40 -0500 Subject: [PATCH 320/920] scoutfs: only release lockspace/workqueues in lock_destroy if they exist Mount failure means these might be NULL. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 0e610ad6..0bac51d8 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -560,12 +560,15 @@ void scoutfs_lock_destroy(struct super_block *sb) int ret; if (linfo) { - destroy_workqueue(linfo->downconvert_wq); + if (linfo->downconvert_wq) + destroy_workqueue(linfo->downconvert_wq); unregister_shrinker(&linfo->shrinker); - ret = dlm_release_lockspace(linfo->ls, 2); - if (ret) - scoutfs_info(sb, "Error %d releasing lockspace %s\n", - ret, linfo->ls_name); + if (linfo->ls) { + ret = dlm_release_lockspace(linfo->ls, 2); + if (ret) + scoutfs_info(sb, "Error %d releasing lockspace %s\n", + ret, linfo->ls_name); + } free_lock_tree(sb); From eb439ccc01ccaaabc29c0a7f80a9376e6a11304a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 29 Jun 2017 15:54:08 -0500 Subject: [PATCH 321/920] scoutfs: s/lck/lock/ lock.[ch] Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 236 ++++++++++++++++++++++++------------------------ kmod/src/lock.h | 4 +- 2 files changed, 120 insertions(+), 120 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 0bac51d8..3e368879 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -53,8 +53,8 @@ struct lock_info { static void scoutfs_downconvert_func(struct work_struct *work); -#define START(lck) ((lck)->start) -#define LAST(lck) ((lck)->end) +#define START(lock) ((lock)->start) +#define LAST(lock) ((lock)->end) KEYED_INTERVAL_TREE_DEFINE(struct scoutfs_lock, interval_node, struct scoutfs_key_buf *, subtree_last, START, LAST, scoutfs_key_compare, static, scoutfs_lock); @@ -84,60 +84,60 @@ static int invalidate_caches(struct super_block *sb, int mode, return ret; } -static void free_scoutfs_lock(struct scoutfs_lock *lck) +static void free_scoutfs_lock(struct scoutfs_lock *lock) { - kfree(lck->start); - kfree(lck->end); - kfree(lck); + kfree(lock->start); + kfree(lock->end); + kfree(lock); } -static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck) +static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); unsigned int refs; - if (lck) { + if (lock) { spin_lock(&linfo->lock); - BUG_ON(!lck->refcnt); - refs = --lck->refcnt; + BUG_ON(!lock->refcnt); + refs = --lock->refcnt; if (!refs) { - BUG_ON(lck->holders); - BUG_ON(delayed_work_pending(&lck->dc_work)); - scoutfs_lock_remove(lck, &linfo->lock_tree); - list_del(&lck->lru_entry); + BUG_ON(lock->holders); + BUG_ON(delayed_work_pending(&lock->dc_work)); + scoutfs_lock_remove(lock, &linfo->lock_tree); + list_del(&lock->lru_entry); spin_unlock(&linfo->lock); - free_scoutfs_lock(lck); + free_scoutfs_lock(lock); return; } spin_unlock(&linfo->lock); } } -static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lck, +static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end) { DECLARE_LOCK_INFO(sb, linfo); - RB_CLEAR_NODE(&lck->interval_node); - lck->sb = sb; - lck->mode = SCOUTFS_LOCK_MODE_IV; - INIT_DELAYED_WORK(&lck->dc_work, scoutfs_downconvert_func); - INIT_LIST_HEAD(&lck->lru_entry); + RB_CLEAR_NODE(&lock->interval_node); + lock->sb = sb; + lock->mode = SCOUTFS_LOCK_MODE_IV; + INIT_DELAYED_WORK(&lock->dc_work, scoutfs_downconvert_func); + INIT_LIST_HEAD(&lock->lru_entry); if (start) { - lck->start = start; - lck->dlm_start.val = start->data; - lck->dlm_start.len = start->key_len; + lock->start = start; + lock->dlm_start.val = start->data; + lock->dlm_start.len = start->key_len; } if (end) { - lck->end = end; - lck->dlm_end.val = end->data; - lck->dlm_end.len = end->key_len; + lock->end = end; + lock->dlm_end.val = end->data; + lock->dlm_end.len = end->key_len; } spin_lock(&linfo->lock); - lck->sequence = ++linfo->seq_cnt; + lock->sequence = ++linfo->seq_cnt; spin_unlock(&linfo->lock); } @@ -147,7 +147,7 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, { struct scoutfs_key_buf *s, *e; - struct scoutfs_lock *lck; + struct scoutfs_lock *lock; s = scoutfs_key_dup(sb, start); if (!s) @@ -157,14 +157,14 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, kfree(s); return NULL; } - lck = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); - if (!lck) { + lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); + if (!lock) { kfree(e); kfree(s); } - init_scoutfs_lock(sb, lck, s, e); - return lck; + init_scoutfs_lock(sb, lock, s, e); + return lock; } static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, @@ -208,7 +208,7 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) { struct lock_info *linfo = container_of(shrink, struct lock_info, shrinker); - struct scoutfs_lock *lck; + struct scoutfs_lock *lock; struct scoutfs_lock *tmp; unsigned long flags; unsigned long nr; @@ -219,25 +219,25 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) goto out; spin_lock_irqsave(&linfo->lock, flags); - list_for_each_entry_safe(lck, tmp, &linfo->lru_list, lru_entry) { + list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_entry) { if (nr-- == 0) break; - WARN_ON(lck->holders); - WARN_ON(lck->refcnt != 1); - WARN_ON(lck->flags & SCOUTFS_LOCK_QUEUED); + WARN_ON(lock->holders); + WARN_ON(lock->refcnt != 1); + WARN_ON(lock->flags & SCOUTFS_LOCK_QUEUED); - scoutfs_lock_remove(lck, &linfo->lock_tree); - list_del(&lck->lru_entry); - list_add_tail(&lck->lru_entry, &list); + scoutfs_lock_remove(lock, &linfo->lock_tree); + list_del(&lock->lru_entry); + list_add_tail(&lock->lru_entry, &list); linfo->lru_nr--; } spin_unlock_irqrestore(&linfo->lock, flags); - list_for_each_entry_safe(lck, tmp, &list, lru_entry) { - trace_shrink_lock_tree(linfo->sb, lck); - list_del(&lck->lru_entry); - free_scoutfs_lock(lck); + list_for_each_entry_safe(lock, tmp, &list, lru_entry) { + trace_shrink_lock_tree(linfo->sb, lock); + list_del(&lock->lru_entry); + free_scoutfs_lock(lock); } out: return min_t(unsigned long, linfo->lru_nr, INT_MAX); @@ -249,84 +249,84 @@ static void free_lock_tree(struct super_block *sb) struct rb_node *node = rb_first(&linfo->lock_tree); while (node) { - struct scoutfs_lock *lck; + struct scoutfs_lock *lock; - lck = rb_entry(node, struct scoutfs_lock, interval_node); + lock = rb_entry(node, struct scoutfs_lock, interval_node); node = rb_next(node); - put_scoutfs_lock(sb, lck); + put_scoutfs_lock(sb, lock); } } static void scoutfs_ast(void *astarg) { - struct scoutfs_lock *lck = astarg; - DECLARE_LOCK_INFO(lck->sb, linfo); + struct scoutfs_lock *lock = astarg; + DECLARE_LOCK_INFO(lock->sb, linfo); - trace_scoutfs_ast(lck->sb, lck); + trace_scoutfs_ast(lock->sb, lock); spin_lock(&linfo->lock); - lck->mode = lck->rqmode; + lock->mode = lock->rqmode; /* Clear blocking flag when we are granted an unlock request */ - if (lck->rqmode == DLM_LOCK_IV) - lck->flags &= ~SCOUTFS_LOCK_BLOCKING; - lck->rqmode = DLM_LOCK_IV; + if (lock->rqmode == DLM_LOCK_IV) + lock->flags &= ~SCOUTFS_LOCK_BLOCKING; + lock->rqmode = DLM_LOCK_IV; spin_unlock(&linfo->lock); wake_up(&linfo->waitq); } static void queue_blocking_work(struct lock_info *linfo, - struct scoutfs_lock *lck, unsigned int seconds) + struct scoutfs_lock *lock, unsigned int seconds) { assert_spin_locked(&linfo->lock); - if (!(lck->flags & SCOUTFS_LOCK_QUEUED)) { + if (!(lock->flags & SCOUTFS_LOCK_QUEUED)) { /* Take a ref for the workqueue */ - lck->flags |= SCOUTFS_LOCK_QUEUED; - lck->refcnt++; + lock->flags |= SCOUTFS_LOCK_QUEUED; + lock->refcnt++; } - mod_delayed_work(linfo->downconvert_wq, &lck->dc_work, seconds * HZ); + mod_delayed_work(linfo->downconvert_wq, &lock->dc_work, seconds * HZ); } -static void set_lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lck, +static void set_lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock, unsigned int seconds) { assert_spin_locked(&linfo->lock); - lck->flags |= SCOUTFS_LOCK_BLOCKING; - if (lck->holders == 0) - queue_blocking_work(linfo, lck, seconds); + lock->flags |= SCOUTFS_LOCK_BLOCKING; + if (lock->holders == 0) + queue_blocking_work(linfo, lock, seconds); } static void scoutfs_rbast(void *astarg, int mode, struct dlm_key *start, struct dlm_key *end) { - struct scoutfs_lock *lck = astarg; - struct lock_info *linfo = SCOUTFS_SB(lck->sb)->lock_info; + struct scoutfs_lock *lock = astarg; + struct lock_info *linfo = SCOUTFS_SB(lock->sb)->lock_info; - trace_scoutfs_rbast(lck->sb, lck); + trace_scoutfs_rbast(lock->sb, lock); spin_lock(&linfo->lock); - set_lock_blocking(linfo, lck, 0); + set_lock_blocking(linfo, lock, 0); spin_unlock(&linfo->lock); } -static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lck, +static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lock, int mode) { int ret; spin_lock(&linfo->lock); - ret = !!(mode == lck->mode); + ret = !!(mode == lock->mode); spin_unlock(&linfo->lock); return ret; } -static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lck) +static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock) { int ret; spin_lock(&linfo->lock); - ret = !!(lck->flags & SCOUTFS_LOCK_BLOCKING); + ret = !!(lock->flags & SCOUTFS_LOCK_BLOCKING); spin_unlock(&linfo->lock); return ret; @@ -344,109 +344,109 @@ static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lck) int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, - struct scoutfs_lock **ret_lck) + struct scoutfs_lock **ret_lock) { DECLARE_LOCK_INFO(sb, linfo); - struct scoutfs_lock *lck; + struct scoutfs_lock *lock; int ret; - lck = find_alloc_scoutfs_lock(sb, start, end); - if (!lck) + lock = find_alloc_scoutfs_lock(sb, start, end); + if (!lock) return -ENOMEM; - trace_scoutfs_lock_range(sb, lck); + trace_scoutfs_lock_range(sb, lock); check_lock_state: spin_lock(&linfo->lock); if (linfo->shutdown) { spin_unlock(&linfo->lock); - put_scoutfs_lock(sb, lck); + put_scoutfs_lock(sb, lock); return -ESHUTDOWN; } - if (lck->flags & SCOUTFS_LOCK_BLOCKING) { + if (lock->flags & SCOUTFS_LOCK_BLOCKING) { spin_unlock(&linfo->lock); - wait_event(linfo->waitq, !lock_blocking(linfo, lck)); + wait_event(linfo->waitq, !lock_blocking(linfo, lock)); goto check_lock_state; } - if (lck->mode > DLM_LOCK_IV) { - if (lck->mode < mode) { + if (lock->mode > DLM_LOCK_IV) { + if (lock->mode < mode) { /* * We already have the lock but at a mode which is not * compatible with what the caller wants. Set the lock * blocking to let the downconvert thread do it's work * so we can reacquire at the correct mode. */ - set_lock_blocking(linfo, lck, 0); + set_lock_blocking(linfo, lock, 0); spin_unlock(&linfo->lock); goto check_lock_state; } - lck->holders++; + lock->holders++; spin_unlock(&linfo->lock); goto out; } - lck->rqmode = mode; - lck->holders++; + lock->rqmode = mode; + lock->holders++; spin_unlock(&linfo->lock); - ret = dlm_lock_range(linfo->ls, mode, &lck->dlm_start, &lck->dlm_end, - &lck->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, - RANGE_LOCK_RESOURCE_LEN, 0, scoutfs_ast, lck, + ret = dlm_lock_range(linfo->ls, mode, &lock->dlm_start, &lock->dlm_end, + &lock->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, + RANGE_LOCK_RESOURCE_LEN, 0, scoutfs_ast, lock, scoutfs_rbast); if (ret) { scoutfs_err(sb, "Error %d locking %s\n", ret, RANGE_LOCK_RESOURCE); - put_scoutfs_lock(sb, lck); + put_scoutfs_lock(sb, lock); return ret; } - wait_event(linfo->waitq, lock_granted(linfo, lck, mode)); + wait_event(linfo->waitq, lock_granted(linfo, lock, mode)); out: - *ret_lck = lck; + *ret_lock = lock; return 0; } -void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck) +void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); unsigned int seconds = 60; - trace_scoutfs_unlock_range(sb, lck); + trace_scoutfs_unlock_range(sb, lock); spin_lock(&linfo->lock); - lck->holders--; - if (lck->holders == 0) { - if (lck->flags & SCOUTFS_LOCK_BLOCKING) + lock->holders--; + if (lock->holders == 0) { + if (lock->flags & SCOUTFS_LOCK_BLOCKING) seconds = 0; - queue_blocking_work(linfo, lck, seconds); + queue_blocking_work(linfo, lock, seconds); } spin_unlock(&linfo->lock); - put_scoutfs_lock(sb, lck); + put_scoutfs_lock(sb, lock); } -static void unlock_range(struct super_block *sb, struct scoutfs_lock *lck) +static void unlock_range(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); int ret; - trace_scoutfs_unlock_range(sb, lck); + trace_scoutfs_unlock_range(sb, lock); - BUG_ON(!lck->sequence); + BUG_ON(!lock->sequence); spin_lock(&linfo->lock); - lck->rqmode = DLM_LOCK_IV; + lock->rqmode = DLM_LOCK_IV; spin_unlock(&linfo->lock); - ret = dlm_unlock(linfo->ls, lck->lksb.sb_lkid, 0, &lck->lksb, lck); + ret = dlm_unlock(linfo->ls, lock->lksb.sb_lkid, 0, &lock->lksb, lock); if (ret) { scoutfs_err(sb, "Error %d unlocking %s\n", ret, RANGE_LOCK_RESOURCE); goto out; } - wait_event(linfo->waitq, lock_granted(linfo, lck, DLM_LOCK_IV)); + wait_event(linfo->waitq, lock_granted(linfo, lock, DLM_LOCK_IV)); out: /* lock was removed from tree, wake up umount process */ wake_up(&linfo->waitq); @@ -454,30 +454,30 @@ out: static void scoutfs_downconvert_func(struct work_struct *work) { - struct scoutfs_lock *lck = container_of(work, struct scoutfs_lock, + struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, dc_work.work); - struct super_block *sb = lck->sb; + struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - trace_scoutfs_downconvert_func(sb, lck); + trace_scoutfs_downconvert_func(sb, lock); spin_lock(&linfo->lock); - lck->flags &= ~SCOUTFS_LOCK_QUEUED; - if (lck->holders) + lock->flags &= ~SCOUTFS_LOCK_QUEUED; + if (lock->holders) goto out; /* scoutfs_unlock_range will requeue for us */ spin_unlock(&linfo->lock); - WARN_ON_ONCE(lck->holders); - WARN_ON_ONCE(lck->refcnt == 0); + WARN_ON_ONCE(lock->holders); + WARN_ON_ONCE(lock->refcnt == 0); /* * Use write mode to invalidate all since we are completely * dropping the lock. Once we are dowconverting, we can * invalidate based on what level we're downconverting to (PR, * NL). */ - invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lck->start, lck->end); - unlock_range(sb, lck); + invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lock->start, lock->end); + unlock_range(sb, lock); spin_lock(&linfo->lock); /* Check whether we can add the lock to the LRU list: @@ -489,14 +489,14 @@ static void scoutfs_downconvert_func(struct work_struct *work) * lock tree so in particular we have nobody in * scoutfs_lock_range concurrently trying to acquire a lock. */ - if (lck->mode == SCOUTFS_LOCK_MODE_IV && lck->refcnt == 1 && - list_empty(&lck->lru_entry)) { - list_add_tail(&lck->lru_entry, &linfo->lru_list); + if (lock->mode == SCOUTFS_LOCK_MODE_IV && lock->refcnt == 1 && + list_empty(&lock->lru_entry)) { + list_add_tail(&lock->lru_entry, &linfo->lru_list); linfo->lru_nr++; } out: spin_unlock(&linfo->lock); - put_scoutfs_lock(sb, lck); + put_scoutfs_lock(sb, lock); } /* diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 08459e07..2dcdb648 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -34,8 +34,8 @@ enum { int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, - struct scoutfs_lock **ret_lck); -void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lck); + struct scoutfs_lock **ret_lock); +void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lock); int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, void *caller_lvb, unsigned lvb_len); From 3eaabe81de90518af57e01ae57fd5bdba7021a42 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 26 Jun 2017 13:56:21 -0700 Subject: [PATCH 322/920] scoutfs: add btree stored in persistent ring Add a cow btree whose blocks are stored in a persistently allocated ring. This will let us incrementally index very large data sets efficiently. This is an adaptation of the previous btree code which now uses the ring, stores variable length keys, and augments the items with bits that ored up through parents. Signed-off-by: Zach Brown --- kmod/src/Makefile | 7 +- kmod/src/btree.c | 1878 ++++++++++++++++++++++++++++++++++++++++++ kmod/src/btree.h | 51 ++ kmod/src/format.h | 85 ++ kmod/src/sort_priv.c | 71 ++ kmod/src/sort_priv.h | 8 + kmod/src/super.c | 3 + kmod/src/super.h | 2 + 8 files changed, 2102 insertions(+), 3 deletions(-) create mode 100644 kmod/src/btree.c create mode 100644 kmod/src/btree.h create mode 100644 kmod/src/sort_priv.c create mode 100644 kmod/src/sort_priv.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 8370da64..8eefa971 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += alloc.o bio.o compact.o counters.o data.o dir.o kvec.o inode.o \ - ioctl.o item.o key.o lock.o manifest.o msg.o net.o options.o \ - ring.o seg.o scoutfs_trace.o super.o trans.o xattr.o +scoutfs-y += alloc.o bio.o btree.o compact.o counters.o data.o dir.o kvec.o \ + inode.o ioctl.o item.o key.o lock.o manifest.o msg.o net.o \ + options.o ring.o seg.o scoutfs_trace.o sort_priv.o super.o trans.o \ + xattr.o diff --git a/kmod/src/btree.c b/kmod/src/btree.c new file mode 100644 index 00000000..e30f5af0 --- /dev/null +++ b/kmod/src/btree.c @@ -0,0 +1,1878 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "key.h" +#include "btree.h" +#include "sort_priv.h" + +#include "scoutfs_trace.h" + +/* + * scoutfs uses a cow btree in a ring of preallocated blocks to index + * the manifest (and allocator, but mostly the manifest). + * + * Using a cow btree lets nodes determine the validity of cached blocks + * based on a single root ref (blkno, seq) that is communicated through + * locking and messaging. As long as their cached blocks aren't + * overwritten in the ring they can continue to use those cached blocks + * as the newer cowed blocks continue to reference them. + * + * New blocks written to the btree are allocated from the tail of the + * preallocated ring. This avoids a fine grained persistent record of + * free btree blocks. It also gathers all dirty btree blocks into one + * contiguous write. + * + * To ensure that newly written blocks don't overwrite previously valid + * existing blocks in the ring we take two preventative measures. First + * we ensure that there are 4x the number of preallocated blocks that + * would be needed to store the btrees. Then, second, for every set of + * blocks written to the current half of the ring we ensure that at + * least half of the written blocks are cow copies of valid blocks that + * were stored in the old half of the ring. This ensures that the + * current half of the ring will contain all the valid referenced btree + * blocks by the time it fills up and wraps around to start overwriting + * the old half of the ring. + * + * To find the blocks in the old half of the ring we augment the btree + * items to store bits that are or-ed in parent items up to the root. + * Parent items have bits set for the half of the ring that their child + * block is stored in. + * + * Blocks are of a fixed size and are set to 4k to avoid multi-page + * blocks. This means they can be smaller than the page size and we can + * need to pin dirty blocks and invalidate and re-read stable blocks + * that could fall in the same page. We use buffer heads to track + * sub-page block state for us. We abuse knowledge of the page cache + * and buffer heads to cast between pointers to the blocks and the + * buffer heads that contain reference counts of the block contents. + * + * We store modified blocks in a list on b_private instead of marking + * the blocks dirty. We don't want them written out (and possibly + * reclaimed and re-read) before we have a chance to update their + * checksums. We hold an elevated bh count to avoid the buffers from + * being removed from the pages while we have them in the list. + * + * Today callers provide all the locking. They serialize readers and + * writers and writers and committing all the dirty blocks. + * + * Btree items are stored in each block as a small header with the key + * followed by the value. New items are allocated from the back of the + * block towards the front. Deleted items can be reclaimed by packing + * items towards the back of the block by walking them in reverse offset + * order. + * + * A dense array of item headers after the btree block header stores the + * offsets and bits of the items and is kept sorted by the item's keys. + * The array is small enough that keeping it sorted with memmove() + * involves a few cache lines at most. + * + * Parent blocks in the btree have the same format as leaf blocks. + * There's one key for every child reference instead of having separator + * keys between child references. The key in a child reference contains + * the largest key that may be found in the child subtree. The right + * spine of the tree has maximal keys so that they don't have to be + * updated if we insert an item with a key greater than everything in + * the tree. + */ + +/* + * XXX: + * - counters and tracing + * - could issue read-ahead around reads up to dirty blkno + * - have barrier as we cross to prevent refreshing clobbering stale reads + * - audit split and merge for bit updating + * - audit/comment that dirty blknos can wrap around ring + * - figure out some max transaction size so ring won't wrap in one + * - update the world of comments + * - validate structures on read? + */ + +/* + * There's one physical ring that stores the blocks for all btrees. We + * track the state of the ring and all its dirty blocks in this one + * btree_info per mount/super. + */ +struct btree_info { + struct mutex mutex; + + unsigned long cur_dirtied; + unsigned long old_dirtied; + struct buffer_head *first_dirty_bh; + struct buffer_head *last_dirty_bh; + u64 first_dirty_blkno; + u64 first_dirty_seq; +}; + +#define DECLARE_BTREE_INFO(sb, name) \ + struct btree_info *name = SCOUTFS_SB(sb)->btree_info + +/* btree walking has a bunch of behavioural bit flags */ +enum { + BTW_NEXT = (1 << 0), /* return >= key */ + BTW_AFTER = (1 << 1), /* return > key */ + BTW_PREV = (1 << 2), /* return <= key */ + BTW_BEFORE = (1 << 3), /* return < key */ + BTW_DIRTY = (1 << 4), /* cow stable blocks */ + BTW_BIT = (1 << 5), /* search for the first set bit, not key */ + BTW_DIRTY_OLD = (1 << 6), /* dirty old leaf blocks to balance ring */ + BTW_ALLOC = (1 << 7), /* allocate a new block for 0 ref */ + BTW_INSERT = (1 << 8), /* walking to insert, try splitting */ + BTW_DELETE = (1 << 9), /* walking to delete, try merging */ +}; + +/* + * This greatest key value is stored down the right spine of the tree + * and has to be sorted by memcmp() greater than all possible keys in + * all btrees. We give it room for a decent number of big-endian + * primary sort values. + */ +static char max_key[SCOUTFS_BTREE_GREATEST_KEY_LEN] = { + [0 ... (SCOUTFS_BTREE_GREATEST_KEY_LEN - 1)] = 0xff, +}; + +/* number of contiguous bytes used by the item header, key, and value */ +static inline unsigned len_bytes(unsigned key_len, unsigned val_len) +{ + return sizeof(struct scoutfs_btree_item) + key_len + val_len; +} + +/* number of contiguous bytes used an existing item */ +static inline unsigned int item_bytes(struct scoutfs_btree_item *item) +{ + return len_bytes(le16_to_cpu(item->key_len), le16_to_cpu(item->val_len)); +} + +/* total block bytes used by an item: header, item, key, value */ +static inline unsigned int all_len_bytes(unsigned key_len, unsigned val_len) +{ + return sizeof(struct scoutfs_btree_item_header) + + len_bytes(key_len, val_len); +} + +/* total block bytes used by an existing item */ +static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) +{ + return all_len_bytes(le16_to_cpu(item->key_len), + le16_to_cpu(item->val_len)); +} + +/* number of contig free bytes between last item header and first item */ +static inline unsigned int contig_free(struct scoutfs_btree_block *bt) +{ + unsigned int nr = le16_to_cpu(bt->nr_items); + + return le16_to_cpu(bt->free_end) - + offsetof(struct scoutfs_btree_block, item_hdrs[nr]); +} + +/* number of contig bytes free after reclaiming free amongst items */ +static inline unsigned int reclaimable_free(struct scoutfs_btree_block *bt) +{ + return contig_free(bt) + le16_to_cpu(bt->free_reclaim); +} + +/* all bytes used by item offsets, headers, and values */ +static inline unsigned int used_total(struct scoutfs_btree_block *bt) +{ + return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - + reclaimable_free(bt); +} + +static inline struct scoutfs_btree_item * +off_item(struct scoutfs_btree_block *bt, __le16 off) +{ + return (void *)bt + le16_to_cpu(off); +} + +static inline struct scoutfs_btree_item * +pos_item(struct scoutfs_btree_block *bt, unsigned int pos) +{ + return off_item(bt, bt->item_hdrs[pos].off); +} + +static inline struct scoutfs_btree_item * +last_item(struct scoutfs_btree_block *bt) +{ + return pos_item(bt, le16_to_cpu(bt->nr_items) - 1); +} + +static inline void *item_key(struct scoutfs_btree_item *item) +{ + return item->data; +} + +static inline unsigned item_key_len(struct scoutfs_btree_item *item) +{ + return le16_to_cpu(item->key_len); +} + +static inline void *item_val(struct scoutfs_btree_item *item) +{ + return item_key(item) + le16_to_cpu(item->key_len); +} + +static inline unsigned item_val_len(struct scoutfs_btree_item *item) +{ + return le16_to_cpu(item->val_len); +} + +static inline int cmp_keys(void *a, unsigned a_len, void *b, unsigned b_len) +{ + return memcmp(a, b, min(a_len, b_len)) ?: + a_len < b_len ? -1 : a_len > b_len ? 1 : 0; +} + +/* + * Returns the sorted item position that an item with the given key + * should occupy. + * + * It sets *cmp to the final comparison of the given key and the + * position's item key. This can only be -1 or 0 because we bias + * towards returning the pos that a key should occupy. + * + * If the given key is greater then all items' keys then the number of + * items can be returned. + */ +static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, + int *cmp) +{ + struct scoutfs_btree_item *item; + unsigned int start = 0; + unsigned int end = le16_to_cpu(bt->nr_items); + unsigned int pos = 0; + + *cmp = -1; + + while (start < end) { + pos = start + (end - start) / 2; + + item = pos_item(bt, pos); + *cmp = cmp_keys(key, key_len, item_key(item), item_key_len(item)); + if (*cmp < 0) { + end = pos; + } else if (*cmp > 0) { + start = ++pos; + *cmp = -1; + } else { + break; + } + } + + return pos; +} + +static inline u8 pos_bits(struct scoutfs_btree_block *bt, unsigned int pos) +{ + return bt->item_hdrs[pos].bits; +} + +static inline bool pos_bit_set(struct scoutfs_btree_block *bt, unsigned int pos, + u8 bit) +{ + return bt->item_hdrs[pos].bits & bit; +} + +static inline u16 bit_count(struct scoutfs_btree_block *bt, u8 bit) +{ + int ind; + + BUG_ON(hweight8(bit) != 1); + + ind = ffs(bit) - 1; + return le16_to_cpu(bt->bit_counts[ind]); +} + +/* find the first item pos with the given bit set */ +static int find_pos_bit(struct scoutfs_btree_block *bt, int pos, u8 bit) +{ + unsigned int nr = le16_to_cpu(bt->nr_items); + + while (pos < nr && !pos_bit_set(bt, pos, bit)) + pos++; + + return pos; +} + +/* + * Record the path we took through parent blocks. Used to set the bits + * in parent reference items that lead to bits in leaves. + */ +struct btree_path { + unsigned nr; + struct scoutfs_btree_block *bt[SCOUTFS_BTREE_MAX_HEIGHT]; + u16 pos[SCOUTFS_BTREE_MAX_HEIGHT]; +}; + +#define DECLARE_BTREE_PATH(name) \ + struct btree_path name = {0, } + +/* + * Add a block to the path for later traversal for updating bits. Only dirty + * blocks are put in the path and they have an extra ref to keep them pinned + * until we write them out. + */ +static void path_push(struct btree_path *path, + struct scoutfs_btree_block *bt, unsigned pos) +{ + if (path) { + BUG_ON(path->nr >= SCOUTFS_BTREE_MAX_HEIGHT); + + path->bt[path->nr] = bt; + path->pos[path->nr++] = pos; + } +} + +static struct scoutfs_btree_block *path_pop(struct btree_path *path, unsigned *pos) +{ + if (!path || path->nr == 0) + return NULL; + + *pos = path->pos[--path->nr]; + return path->bt[path->nr]; +} + +static u8 half_bit(struct scoutfs_btree_ring *bring, u64 blkno) +{ + u64 half_blkno = le64_to_cpu(bring->first_blkno) + + (le64_to_cpu(bring->nr_blocks) / 2); + + return blkno < half_blkno ? SCOUTFS_BTREE_BIT_HALF1 : + SCOUTFS_BTREE_BIT_HALF2; +} + +static u8 other_half_bit(struct scoutfs_btree_ring *bring, u64 blkno) +{ + return half_bit(bring, blkno) ^ (SCOUTFS_BTREE_BIT_HALF1 | + SCOUTFS_BTREE_BIT_HALF2); +} + +static u8 bits_from_counts(struct scoutfs_btree_block *bt) +{ + u8 bits = 0; + int i; + + for (i = 0; i < SCOUTFS_BTREE_BITS; i++) { + if (bt->bit_counts[i]) + bits |= 1 << i; + } + + return bits; +} + +/* + * Iterate through 0-based bit numbers set in 'bits' from least to + * greatest. It modifies 'bits' as it goes! + */ +#define for_each_bit(i, bits) \ + for (i = bits ? ffs(bits) : 0; i-- > 0; bits &= ~(1 < i)) + +/* + * Store the new bits and update the counts to match the difference from + * the previously set bits. Callers use this to keep item bits in sync + * with the counts of bits in the block headers. + */ +static void store_pos_bits(struct scoutfs_btree_block *bt, int pos, u8 bits) +{ + u8 diff = bits ^ pos_bits(bt, pos); + int b; + + if (!diff) + return; + + for_each_bit(b, diff) { + if (bits & (1 << b)) + le16_add_cpu(&bt->bit_counts[b], 1); + else + le16_add_cpu(&bt->bit_counts[b], -1); + } + + bt->item_hdrs[pos].bits = bits; +} + +/* + * The caller has descended through parents to a final block. Each + * block may have had item bits modified and counts updated but they + * didn't keep parent item bits in sync with modifications to all the + * children. Our job is to ascend back through parents and set their + * bits to the union of all the bits down through the path to the final + * block. + */ +static void path_repair_reset(struct btree_path *path) +{ + struct scoutfs_btree_block *parent; + struct scoutfs_btree_block *bt; + u8 bits; + int pos; + + bt = path_pop(path, &pos); + + while ((parent = path_pop(path, &pos))) { + bits = bits_from_counts(bt); + store_pos_bits(parent, pos, bits); + bt = parent; + } +} + +static int cmp_hdr_item_key(void *priv, const void *a_ptr, const void *b_ptr) +{ + struct scoutfs_btree_block *bt = priv; + const struct scoutfs_btree_item_header *a_hdr = a_ptr; + const struct scoutfs_btree_item_header *b_hdr = b_ptr; + struct scoutfs_btree_item *a_item = off_item(bt, a_hdr->off); + struct scoutfs_btree_item *b_item = off_item(bt, b_hdr->off); + + return cmp_keys(item_key(a_item), item_key_len(a_item), + item_key(b_item), item_key_len(b_item)); +} + +static int cmp_hdr_off(void *priv, const void *a_ptr, const void *b_ptr) +{ + const struct scoutfs_btree_item_header *a_hdr = a_ptr; + const struct scoutfs_btree_item_header *b_hdr = b_ptr; + + return (int)le16_to_cpu(a_hdr->off) - (int)le16_to_cpu(b_hdr->off); +} + +static void swap_hdr(void *priv, void *a_ptr, void *b_ptr, int size) +{ + struct scoutfs_btree_item_header *a_hdr = a_ptr; + struct scoutfs_btree_item_header *b_hdr = b_ptr; + + swap(*a_hdr, *b_hdr); +} + +/* + * As items are deleted they create fragmented free space. Even if we + * indexed free space in the block it could still get sufficiently + * fragmented to force a split on insertion even though the two + * resulting blocks would have less than the minimum space consumed by + * items. + * + * We don't bother implementing free space indexing and addressing that + * corner case. Instead we track the number of bytes that could be + * reclaimed if we compacted the item space after the free_end offset. + * If this additional free space would satisfy an insertion then we + * compact the items instead of splitting the block. + * + * We move the free space to the center of the block by walking + * backwards through the items in offset order and packing them towards + * the end of the block. + * + * We don't have specific metadata to either walk the items in offset + * order or to update the item offsets as we move items. We sort the + * item offset array to achieve both ends. First we sort it by offset + * so we can walk in reverse order. As we move items we update their + * offset and then sort by keys once we're done. + */ +static void compact_items(struct scoutfs_btree_block *bt) +{ + unsigned int nr = le16_to_cpu(bt->nr_items); + struct scoutfs_btree_item *from; + struct scoutfs_btree_item *to; + unsigned int bytes; + __le16 end; + int i; + + sort_priv(bt, bt->item_hdrs, nr, sizeof(bt->item_hdrs[0]), + cmp_hdr_off, swap_hdr); + + end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); + + for (i = nr - 1; i >= 0; i--) { + from = pos_item(bt, i); + + bytes = item_bytes(from); + le16_add_cpu(&end, -bytes); + to = off_item(bt, end); + bt->item_hdrs[i].off = end; + + if (from != to) + memmove(to, from, bytes); + } + + bt->free_end = end; + bt->free_reclaim = 0; + + sort_priv(bt, bt->item_hdrs, nr, sizeof(bt->item_hdrs[0]), + cmp_hdr_item_key, swap_hdr); +} + +/* move a number of contigous elements from the src index to the dst index */ +#define memmove_arr(arr, dst, src, nr) \ + memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) + +/* + * Insert a new item into the block. The caller has made sure that + * there's space for the item and its metadata but we might have to + * compact the block to make that space contiguous. + * + * The possibility of compaction means that callers *can not* hold item, + * key, or value pointers across item creation. An easy way to verify + * this is to audit pos_item() callers. + */ +static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, u8 bits, + void *key, unsigned key_len, void *val, unsigned val_len) +{ + unsigned nr = le16_to_cpu(bt->nr_items); + struct scoutfs_btree_item *item; + unsigned all_bytes; + + all_bytes = all_len_bytes(key_len, val_len); + if (contig_free(bt) < all_bytes) { + BUG_ON(reclaimable_free(bt) < all_bytes); + compact_items(bt); + } + + if (pos < nr) + memmove_arr(bt->item_hdrs, pos + 1, pos, nr - pos); + + le16_add_cpu(&bt->free_end, -len_bytes(key_len, val_len)); + bt->item_hdrs[pos].off = bt->free_end; + nr++; + bt->nr_items = cpu_to_le16(nr); + + BUG_ON(le16_to_cpu(bt->free_end) < + offsetof(struct scoutfs_btree_block, item_hdrs[nr])); + + bt->item_hdrs[pos].bits = 0; + store_pos_bits(bt, pos, bits); + + item = pos_item(bt, pos); + item->key_len = cpu_to_le16(key_len); + item->val_len = cpu_to_le16(val_len); + + memcpy(item_key(item), key, key_len); + if (val_len) + memcpy(item_val(item), val, val_len); +} + +/* + * Delete an item from a btree block. We record the amount of space it + * frees to later decide if we can satisfy an insertion by compaction + * instead of splitting. + */ +static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) +{ + struct scoutfs_btree_item *item = pos_item(bt, pos); + unsigned int nr = le16_to_cpu(bt->nr_items); + + store_pos_bits(bt, pos, 0); + + if (pos < (nr - 1)) + memmove_arr(bt->item_hdrs, pos, pos + 1, nr - 1 - pos); + + le16_add_cpu(&bt->free_reclaim, item_bytes(item)); + nr--; + bt->nr_items = cpu_to_le16(nr); + + /* wipe deleted items to avoid leaking data */ + memset(item, 0, item_bytes(item)); +} + +/* + * Move items from a source block to a destination block. The caller + * tells us if we're moving from the tail of the source block right to + * the head of the destination block, or vice versa. We stop moving + * once we've moved enough bytes of items. + */ +static void move_items(struct scoutfs_btree_block *dst, + struct scoutfs_btree_block *src, bool move_right, + int to_move) +{ + struct scoutfs_btree_item *from; + unsigned int t; + unsigned int f; + + if (move_right) { + f = le16_to_cpu(src->nr_items) - 1; + t = 0; + } else { + f = 0; + t = le16_to_cpu(dst->nr_items); + } + + while (f < le16_to_cpu(src->nr_items) && to_move > 0) { + from = pos_item(src, f); + + create_item(dst, t, pos_bits(src, f), item_key(from), + item_key_len(from), item_val(from), + item_val_len(from)); + + to_move -= all_item_bytes(from); + + delete_item(src, f); + if (move_right) + f--; + else + t++; + } +} + +/* + * This is only used after we've elevated bh reference counts. Until we + * drop the counts the bhs won't be removed from the page. This lets us + * use pointers to the block contents in the api and not have to litter + * it with redundant containers. + */ +static struct buffer_head *virt_to_bh(void *kaddr) +{ + struct buffer_head *bh; + struct page *page; + long off; + + page = virt_to_page((unsigned long)kaddr); + BUG_ON(!page_has_buffers(page)); + bh = page_buffers(page); + BUG_ON((unsigned long)bh->b_data != + ((unsigned long)kaddr & PAGE_CACHE_MASK)); + + off = (unsigned long)kaddr & ~PAGE_CACHE_MASK; + while (off >= SCOUTFS_BLOCK_SIZE) { + bh = bh->b_this_page; + off -= SCOUTFS_BLOCK_SIZE; + } + + return bh; +} + +static void put_btree_block(void *ptr) +{ + if (!IS_ERR_OR_NULL(ptr)) + put_bh(virt_to_bh(ptr)); +} + +enum { + BH_ScoutfsChecked = BH_PrivateStart, + BH_ScoutfsValidCrc, +}; + +BUFFER_FNS(ScoutfsChecked, scoutfs_checked) /* has had crc checked */ +BUFFER_FNS(ScoutfsValidCrc, scoutfs_valid_crc) /* crc matched */ + + +/* + * Make sure that we've found a valid block and that it's the block that + * we're looking for. + */ +static bool valid_referenced_block(struct scoutfs_super_block *super, + struct scoutfs_btree_ref *ref, + struct scoutfs_btree_block *bt, + struct buffer_head *bh) +{ + __le32 existing; + u32 calc; + + if (!buffer_scoutfs_checked(bh)) { + lock_buffer(bh); + if (!buffer_scoutfs_checked(bh)) { + existing = bt->crc; + bt->crc = 0; + calc = crc32c(~0, bt, SCOUTFS_BLOCK_SIZE); + bt->crc = existing; + + set_buffer_scoutfs_checked(bh); + if (calc == le32_to_cpu(existing)) + set_buffer_scoutfs_valid_crc(bh); + else + clear_buffer_scoutfs_valid_crc(bh); + } + unlock_buffer(bh); + } + + return buffer_scoutfs_valid_crc(bh) && super->hdr.fsid == bt->fsid && + ref->blkno == bt->blkno && ref->seq == bt->seq; +} + +/* + * This is used to lookup cached blocks, read blocks, cow blocks for + * dirtying, and allocate new blocks. + * + * Btree blocks don't have rigid cache consistency. We can be following + * a new root to read refs into previously stale cached blocks. If we + * see that the block metadata doesn't match we first assume that we + * just have a stale block and try and re-read it. If it still doesn't + * match we assume that we're an reader racing with a writer overwriting + * old blocks in the ring. We return an error that tells the caller to + * deal with this error: either find a new root or return a hard error + * if the block is really corrupt. + * + * This only sets the caller's reference. It doesn't know if the + * caller's ref is in a parent item and would need to update bits and + * counts based on the blkno. It's up to the callers to take care of + * that. + * + * btree callers serialize concurrent writers in a btree but not between + * btrees. We have to lock around the shared btree_info. Callers do + * lock between all btree writers and writing dirty blocks. We don't + * have to lock around the bti fields that are only changed by commits. + */ +static int get_ref_block(struct super_block *sb, int flags, + struct scoutfs_btree_ref *ref, + struct scoutfs_btree_block **bt_ret) +{ + DECLARE_BTREE_INFO(sb, bti); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_btree_ring *bring = &super->bring; + struct scoutfs_btree_block *bt = NULL; + struct scoutfs_btree_block *new; + struct buffer_head *bh; + int retries = 1; + u64 blkno; + u64 seq; + int ret; + +retry: + /* always get the current block, either to return or cow from */ + if (ref && ref->blkno) { + bh = sb_bread(sb, le64_to_cpu(ref->blkno)); + if (!bh) { + ret = -EIO; + goto out; + } + bt = (void *)bh->b_data; + + if (!valid_referenced_block(super, ref, bt, bh)) { + if (retries-- > 0) { + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + put_bh(bh); + bt = NULL; + goto retry; + } + /* XXX let us know when we eventually hit this */ + ret = WARN_ON_ONCE(-ESTALE); + goto out; + } + + /* done if not dirtying or already dirty */ + if (!(flags & BTW_DIRTY) || + (le64_to_cpu(bt->seq) >= bti->first_dirty_seq)) { + ret = 0; + goto out; + } + + } else if (!(flags & BTW_ALLOC)) { + ret = -ENOENT; + goto out; + } + + mutex_lock(&bti->mutex); + + blkno = le64_to_cpu(bring->first_blkno) + le64_to_cpu(bring->next_block); + seq = le64_to_cpu(bring->next_seq); + + bh = sb_getblk(sb, blkno); + if (!bh) { + ret = -ENOMEM; + mutex_unlock(&bti->mutex); + goto out; + } + new = (void *)bh->b_data; + + set_buffer_uptodate(bh); + set_buffer_scoutfs_checked(bh); + set_buffer_scoutfs_valid_crc(bh); + + /* + * Track our contiguous dirty blocks by holding a ref and putting + * them in a list. We don't want them marked dirty or else they + * can be written out before we're ready. + */ + get_bh(bh); + bh->b_private = NULL; + if (bti->last_dirty_bh) + bti->last_dirty_bh->b_private = bh; + bti->last_dirty_bh = bh; + if (!bti->first_dirty_bh) + bti->first_dirty_bh = bh; + + /* wrap next block and increase next seq */ + if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) + bring->next_block = 0; + else + le64_add_cpu(&bring->next_block, 1); + + le64_add_cpu(&bring->next_seq, 1); + + if (half_bit(bring, blkno) == half_bit(bring, bti->first_dirty_blkno)) + bti->cur_dirtied++; + else + bti->old_dirtied++; + + mutex_unlock(&bti->mutex); + + if (bt) { + /* returning a cow of an existing block */ + memcpy(new, bt, SCOUTFS_BLOCK_SIZE); + put_btree_block(bt); + bt = new; + } else { + /* returning a newly allocated block */ + bt = new; + new = NULL; + memset(bt, 0, SCOUTFS_BLOCK_SIZE); + bt->fsid = super->hdr.fsid; + bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); + } + + bt->blkno = cpu_to_le64(blkno); + bt->seq = cpu_to_le64(seq); + if (ref) { + ref->blkno = bt->blkno; + ref->seq = bt->seq; + } + ret = 0; + +out: + if (ret) { + put_btree_block(bt); + bt = NULL; + } + + *bt_ret = bt; + return ret; +} + +/* + * Get the block referenced by the given parent item. The parent item + * and its bits are updated. + */ +static int get_parent_ref_block(struct super_block *sb, int flags, + struct scoutfs_btree_block *parent, unsigned pos, + struct scoutfs_btree_block **bt_ret) +{ + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; + struct scoutfs_btree_item *item; + struct scoutfs_btree_ref *ref; + u8 bits; + int ret; + + /* ref can only be updated, no insertion or compaction */ + item = pos_item(parent, pos); + ref = item_val(item); + + ret = get_ref_block(sb, flags, ref, bt_ret); + if (ret == 0) { + bits = bits_from_counts(*bt_ret) | + half_bit(bring, le64_to_cpu(ref->blkno)); + store_pos_bits(parent, pos, bits); + } + + return ret; +} + +/* + * Create a new item in the parent which references the child. The caller + * specifies the key in the item that describes the items in the child. + */ +static void create_parent_item(struct scoutfs_btree_ring *bring, + struct scoutfs_btree_block *parent, + unsigned pos, struct scoutfs_btree_block *child, + void *key, unsigned key_len) +{ + struct scoutfs_btree_ref ref = { + .blkno = child->blkno, + .seq = child->seq, + }; + u8 bits = bits_from_counts(child) | + half_bit(bring, le64_to_cpu(ref.blkno)); + + create_item(parent, pos, bits, key, key_len, &ref, sizeof(ref)); +} + +/* + * Update the parent item that refers to a child by deleting and + * recreating it. Descent should have ensured that there was always + * room for a maximal key in parents. + */ +static void update_parent_item(struct scoutfs_btree_ring *bring, + struct scoutfs_btree_block *parent, + unsigned pos, struct scoutfs_btree_block *child) +{ + struct scoutfs_btree_item *item = last_item(child); + + delete_item(parent, pos); + create_parent_item(bring, parent, pos, child, + item_key(item), item_key_len(item)); +} + +/* the parent item key and value are fine, but child items have changed */ +static void update_parent_bits(struct scoutfs_btree_ring *bring, + struct scoutfs_btree_block *parent, + unsigned pos, struct scoutfs_btree_block *child) +{ + u8 bits = bits_from_counts(child) | + half_bit(bring, le64_to_cpu(child->blkno)); + + store_pos_bits(parent, pos, bits); +} + +/* + * See if we need to split this block while descending for insertion so + * that we have enough space to insert. Parent blocks need enough space + * for a new item and child ref if a child block splits. Leaf blocks + * need enough space to insert the new item with its value. + * + * We split to the left so that the greatest key in the existing block + * doesn't change so we don't have to update the key in its parent item. + * We still have to update its bits. + * + * Returns -errno, 0 if nothing done, or 1 if we split. + */ +static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, unsigned val_len, + struct scoutfs_btree_block *parent, unsigned pos, + struct scoutfs_btree_block *right) +{ + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; + struct scoutfs_btree_block *left = NULL; + struct scoutfs_btree_item *item; + unsigned int all_bytes; + bool put_parent = false; + int ret; + + if (right->level) + all_bytes = all_len_bytes(SCOUTFS_BTREE_MAX_KEY_LEN, + sizeof(struct scoutfs_btree_ref)); + else + all_bytes = all_len_bytes(key_len, val_len); + + if (reclaimable_free(right) >= all_bytes) + return 0; + + /* alloc split neighbour first to avoid unwinding tree growth */ + ret = get_ref_block(sb, BTW_ALLOC, NULL, &left); + if (ret) + return ret; + left->level = right->level; + + if (!parent) { + ret = get_ref_block(sb, BTW_ALLOC, NULL, &parent); + if (ret) { + put_btree_block(left); + return ret; + } + put_parent = true; + + parent->level = root->height; + root->height++; + root->ref.blkno = parent->blkno; + root->ref.seq = parent->seq; + + pos = 0; + create_parent_item(bring, parent, pos, right, + &max_key, sizeof(max_key)); + } + + move_items(left, right, false, used_total(right) / 2); + update_parent_bits(bring, parent, pos, right); + + item = last_item(left); + create_parent_item(bring, parent, pos, left, + item_key(item), item_key_len(item)); + + put_btree_block(left); + if (put_parent) + put_btree_block(parent); + + return 1; +} + +/* + * This is called during descent for deletion when we have a parent and + * might need to merge items from a sibling block if this block has too + * much free space. Eventually we'll be able to fit all of the + * sibling's items in our free space which lets us delete the sibling + * block. + * + * XXX this could more cleverly chose a merge candidate sibling + */ +static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_btree_block *parent, unsigned pos, + struct scoutfs_btree_block *bt) +{ + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; + struct scoutfs_btree_block *sib; + unsigned int sib_pos; + bool move_right; + int to_move; + int ret; + + if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) + return 0; + + /* move items right into our block if we have a left sibling */ + if (pos) { + sib_pos = pos - 1; + move_right = true; + } else { + sib_pos = pos + 1; + move_right = false; + } + + ret = get_parent_ref_block(sb, BTW_DIRTY, parent, sib_pos, &sib); + if (ret) + return ret; + + if (used_total(sib) <= reclaimable_free(bt)) + to_move = used_total(sib); + else + to_move = reclaimable_free(bt) - SCOUTFS_BTREE_FREE_LIMIT; + + move_items(bt, sib, move_right, to_move); + + /* update our parent's item */ + if (!move_right) + update_parent_item(bring, parent, pos, bt); + else + update_parent_bits(bring, parent, pos, bt); + + /* update or delete sibling's parent item */ + if (le16_to_cpu(sib->nr_items) == 0) + delete_item(parent, sib_pos); + else if (move_right) + update_parent_item(bring, parent, sib_pos, sib); + else + update_parent_bits(bring, parent, sib_pos, sib); + + /* and finally shrink the tree if our parent is the root with 1 */ + if (le16_to_cpu(parent->nr_items) == 1) { + root->height--; + root->ref.blkno = bt->blkno; + root->ref.seq = bt->seq; + } + + put_btree_block(sib); + + return 1; +} + +/* + * This is called before writing dirty blocks to ensure that each batch + * of dirty blocks migrates half as many blocks from the old half of the + * ring as it dirties from the current half. This ensures that by the + * time we fill the current half of the ring it will no longer reference + * the old half. + * + * We've walked to the parent of the leaf level which might have dirtied + * more blocks. Our job is to dirty as many leaves as we need to bring + * the old count back up to equal the current count. The caller will + * keep trying to walk down different paths of each of the btrees. + */ +static int try_dirty_old(struct super_block *sb, struct scoutfs_btree_block *bt, + u8 old_bit) +{ + DECLARE_BTREE_INFO(sb, bti); + struct scoutfs_btree_block *dirtied; + struct scoutfs_btree_item *item; + struct scoutfs_btree_ref *ref; + struct blk_plug plug; + int ret = 0; + int pos = 0; + int nr; + int i; + + if (bti->old_dirtied >= bti->cur_dirtied) + return 0; + + /* called when first parent level is highest level, can have nothing */ + nr = min_t(int, bti->cur_dirtied - bti->old_dirtied, + bit_count(bt, old_bit)); + if (nr == 0) + return -ENOENT; + + blk_start_plug(&plug); + + /* read 'em all */ + for (i = 0, pos = 0; i < nr; i++, pos++) { + pos = find_pos_bit(bt, pos, old_bit); + if (pos >= le16_to_cpu(bt->nr_items)) { + /* XXX bits in headers didn't match count */ + ret = -EIO; + blk_finish_plug(&plug); + goto out; + } + + item = pos_item(bt, pos); + ref = item_val(item); + + sb_breadahead(sb, le64_to_cpu(ref->blkno)); + } + + blk_finish_plug(&plug); + + /* then actually try and dirty the blocks */ + for (i = 0, pos = 0; i < nr; i++, pos++) { + pos = find_pos_bit(bt, pos, old_bit); + + ret = get_parent_ref_block(sb, BTW_DIRTY, bt, pos, &dirtied); + if (ret) + break; + put_btree_block(dirtied); + } + +out: + return ret; +} + +/* + * A quick and dirty verification of the btree block. We could add a + * lot more checks and make it only verified on read or after + * significant events like splitting and merging. + */ +static int verify_btree_block(struct scoutfs_btree_block *bt, int level) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_item *prev; + unsigned int bytes = 0; + unsigned int after_off = sizeof(struct scoutfs_btree_block); + unsigned int first_off; + unsigned int off; + unsigned int nr; + unsigned int i = 0; + int bad = 1; + + nr = le16_to_cpu(bt->nr_items); + if (nr == 0) + goto out; + + after_off = offsetof(struct scoutfs_btree_block, item_hdrs[nr]); + first_off = SCOUTFS_BLOCK_SIZE; + + if (after_off > SCOUTFS_BLOCK_SIZE) { + nr = 0; + goto out; + } + + for (i = 0; i < nr; i++) { + off = le16_to_cpu(bt->item_hdrs[i].off); + if (off >= SCOUTFS_BLOCK_SIZE || off < after_off) + goto out; + + first_off = min(first_off, off); + + item = pos_item(bt, i); + bytes += item_bytes(item); + + if (i > 0 && cmp_keys(item_key(item), item_key_len(item), + item_key(prev), item_key_len(prev)) <= 0) + goto out; + + prev = item; + } + + if (first_off < le16_to_cpu(bt->free_end)) + goto out; + + if ((le16_to_cpu(bt->free_end) + bytes + + le16_to_cpu(bt->free_reclaim)) != SCOUTFS_BLOCK_SIZE) + goto out; + + bad = 0; +out: + if (bad) { + printk("bt %p blkno %llu level %d end %u reclaim %u nr %u (after %u bytes %u)\n", + bt, le64_to_cpu(bt->blkno), level, + le16_to_cpu(bt->free_end), + le16_to_cpu(bt->free_reclaim), le16_to_cpu(bt->nr_items), + after_off, bytes); + for (i = 0; i < nr; i++) { + item = pos_item(bt, i); + printk(" [%u] off %u key_len %u val_len %u\n", + i, le16_to_cpu(bt->item_hdrs[i].off), + item_key_len(item), item_val_len(item)); + } + BUG_ON(bad); + } + + return 0; +} + +/* XXX bleh, this should probably share code with the key_buf equivalent */ +static void inc_key(u8 *bytes, unsigned *len) +{ + int i; + + if (*len < SCOUTFS_BTREE_MAX_KEY_LEN) { + memset(bytes + *len, 0, SCOUTFS_BTREE_MAX_KEY_LEN - *len); + *len = SCOUTFS_BTREE_MAX_KEY_LEN; + } + + for (i = *len - 1; i >= 0; i--) { + if (++bytes[i] != 0) + break; + } +} + +/* + * Return the leaf block that should contain the given key. The caller + * is responsible for searching the leaf block and performing their + * operation. + * + * Iteration starting from a key can end up in a leaf that doesn't + * contain the next item in the direction iteration. As we descend we + * give the caller the nearest key in the direction of iteration that + * will land in a different leaf. + * + * The caller provides the path to record the parent blocks and items + * used to reach the leaf. We let them repair the path once they've + * potentially updated bits in the leaf. They must always repair the + * path because we can modify parent bits during descent before + * returning an error. + */ +static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, + struct btree_path *path, int flags, + void *key, unsigned key_len, unsigned int val_len, u8 bit, + struct scoutfs_btree_block **bt_ret, + void *iter_key, unsigned *iter_len) +{ + struct scoutfs_btree_block *parent = NULL; + struct scoutfs_btree_block *bt = NULL; + struct scoutfs_btree_item *item; + unsigned level; + unsigned pos; + unsigned nr; + int cmp; + int ret; + + if (WARN_ON_ONCE((flags & BTW_DIRTY) && path == NULL) || + WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL)) + return -EINVAL; + +restart: + path_repair_reset(path); + put_btree_block(parent); + parent = NULL; + put_btree_block(bt); + bt = NULL; + level = root->height; + if (iter_len) + *iter_len = 0; + pos = 0; + ret = 0; + + if (!root->height) { + if (!(flags & BTW_INSERT)) { + ret = -ENOENT; + } else { + ret = get_ref_block(sb, BTW_ALLOC, &root->ref, &bt); + if (ret == 0) { + bt->level = 0; + root->height = 1; + } + } + goto out; + } + + while(level-- > 0) { + if (parent) + ret = get_parent_ref_block(sb, flags, parent, pos, &bt); + else + ret = get_ref_block(sb, flags, &root->ref, &bt); + if (ret) + break; + + /* push the parent once we could have updated its bits */ + if (parent) + path_push(path, parent, pos); + + /* XXX it'd be nice to make this tunable */ + ret = 0 && verify_btree_block(bt, level); + if (ret) + break; + + /* XXX more aggressive block verification, before ref updates? */ + if (bt->level != level) { + ret = -EIO; + break; + } + + /* + * Splitting and merging can add or remove parents or + * change the pos we take through parents to reach the + * block with the search key|bit. In the rare case that + * we split or merge we simply restart the walk rather + * than try and special case modifying the path to + * reflect the tree changes. + */ + if (flags & BTW_INSERT) + ret = try_split(sb, root, key, key_len, val_len, + parent, pos, bt); + else if ((flags & BTW_DELETE) && parent) + ret = try_merge(sb, root, parent, pos, bt); + else + ret = 0; + if (ret > 0) + goto restart; + else if (ret < 0) + break; + + /* dirtying old stops at the last parent level */ + if ((flags & BTW_DIRTY_OLD) && (level < 2)) { + if (level == 1) { + path_push(path, bt, 0); + ret = try_dirty_old(sb, bt, bit); + } else { + ret = -ENOENT; + } + break; + } + + /* done at the leaf */ + if (level == 0) { + path_push(path, bt, 0); + break; + } + + nr = le16_to_cpu(bt->nr_items); + + /* + * Find the next child block for the search key or bit. + * Key searches should always find a child, bit searches + * can find that the bit isn't set in the first block. + */ + if (flags & BTW_BIT) { + pos = find_pos_bit(bt, 0, bit); + if (pos >= nr) + ret = -ENOENT; + } else { + pos = find_pos(bt, key, key_len, &cmp); + if (pos >= nr) + ret = -EIO; + } + if (ret) + break; + + /* give the caller the next key to iterate towards */ + if (iter_key && (flags & BTW_NEXT) && (pos < (nr - 1))) { + item = pos_item(bt, pos); + *iter_len = item_key_len(item); + memcpy(iter_key, item_key(item), *iter_len); + inc_key(iter_key, iter_len); + + } else if (iter_key && (flags & BTW_PREV) && (pos > 0)) { + item = pos_item(bt, pos - 1); + *iter_len = item_key_len(item); + memcpy(iter_key, item_key(item), *iter_len); + } + + put_btree_block(parent); + parent = bt; + bt = NULL; + } + +out: + put_btree_block(parent); + if (ret) { + put_btree_block(bt); + bt = NULL; + } + + if (bt_ret) + *bt_ret = bt; + else + put_btree_block(bt); + + return ret; +} + +static void init_item_ref(struct scoutfs_btree_item_ref *iref, + struct scoutfs_btree_item *item) +{ + iref->key = item_key(item); + iref->key_len = le16_to_cpu(item->key_len); + iref->val = item_val(item); + iref->val_len = le16_to_cpu(item->val_len); +} + +void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref) +{ + if (!IS_ERR_OR_NULL(iref) && !IS_ERR_OR_NULL(iref->key)) { + put_btree_block(iref->key); + memset(iref, 0, sizeof(struct scoutfs_btree_item_ref)); + } +} + +/* + * Find the item with the given key and point to it from the caller's + * item ref. They're given a reference to the block that they'll drop + * when they're done. + */ +int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + unsigned int pos; + int cmp; + int ret; + + if (WARN_ON_ONCE(iref->key)) + return -EINVAL; + + ret = btree_walk(sb, root, NULL, 0, key, key_len, 0, 0, &bt, NULL, NULL); + if (ret == 0) { + pos = find_pos(bt, key, key_len, &cmp); + if (cmp == 0) { + item = pos_item(bt, pos); + init_item_ref(iref, item); + ret = 0; + } else { + put_btree_block(bt); + ret = -ENOENT; + } + + } + + return ret; +} + +static bool invalid_item(void *key, unsigned key_len, unsigned val_len) +{ + return WARN_ON_ONCE(key_len == 0) || + WARN_ON_ONCE(key_len > SCOUTFS_BTREE_MAX_KEY_LEN) || + WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN) || + WARN_ON_ONCE(key_len > SCOUTFS_BTREE_GREATEST_KEY_LEN && + cmp_keys(key, key_len, max_key, sizeof(max_key)) > 0); +} + +/* + * Insert a new item in the tree. + * + * 0 is returned on success. -EEXIST is returned if the key is already + * present in the tree. + * + * If no value pointer is given then the item is created with a zero + * length value. + */ +int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len) +{ + struct scoutfs_btree_block *bt; + DECLARE_BTREE_PATH(path); + int pos; + int cmp; + int ret; + + if (invalid_item(key, key_len, val_len)) + return -EINVAL; + + ret = btree_walk(sb, root, &path, BTW_DIRTY | BTW_INSERT, key, key_len, + val_len, 0, &bt, NULL, NULL); + if (ret == 0) { + pos = find_pos(bt, key, key_len, &cmp); + if (cmp) { + create_item(bt, pos, 0, key, key_len, val, val_len); + ret = 0; + } else { + ret = -EEXIST; + } + + put_btree_block(bt); + } + + path_repair_reset(&path); + return ret; +} + +/* + * Update a btree item. The key and value must be of the same length (though + * it would be easy enough for us to change that if a caller cared). + */ +int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + DECLARE_BTREE_PATH(path); + int pos; + int cmp; + int ret; + + if (invalid_item(key, key_len, val_len)) + return -EINVAL; + + ret = btree_walk(sb, root, &path, BTW_DIRTY, key, key_len, 0, 0, &bt, + NULL, NULL); + if (ret == 0) { + pos = find_pos(bt, key, key_len, &cmp); + if (cmp == 0) { + item = pos_item(bt, pos); + if (item_key_len(item) != key_len || + item_val_len(item) != val_len) { + ret = -EINVAL; + } else { + memcpy(item_key(item), key, key_len); + memcpy(item_val(item), val, val_len); + ret = 0; + } + ret = 0; + } else { + ret = -ENOENT; + } + + put_btree_block(bt); + } + + path_repair_reset(&path); + return ret; +} + +/* + * Delete an item from the tree. -ENOENT is returned if the key isn't + * found. + */ +int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len) +{ + struct scoutfs_btree_block *bt; + DECLARE_BTREE_PATH(path); + int pos; + int cmp; + int ret; + + ret = btree_walk(sb, root, &path, BTW_DELETE | BTW_DIRTY, key, key_len, + 0, 0, &bt, NULL, NULL); + if (ret == 0) { + pos = find_pos(bt, key, key_len, &cmp); + if (cmp == 0) { + delete_item(bt, pos); + ret = 0; + + /* delete the final block in the tree */ + if (bt->nr_items == 0) { + root->height = 0; + root->ref.blkno = 0; + root->ref.seq = 0; + } + } else { + ret = -ENOENT; + } + + put_btree_block(bt); + } + + path_repair_reset(&path); + return ret; +} + +/* + * Iterate from a key value to the next item in the direction of + * iteration. Callers set flags to tell which way to iterate and + * whether the search key is inclusive, or not. + * + * Walking can land in a leaf that doesn't contain any items in the + * direction of the iteration. Walking gives us the next key to walk + * towards in this case. We keep trying until we run out of blocks or + * find the next item. This method is aggressively permissive because + * it lets the tree shape change between each walk and allows empty + * blocks. + */ +static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, + int flags, void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + unsigned iter_len; + unsigned walk_len; + void *iter_key; + void *walk_key; + int pos; + int cmp; + int ret; + + if (WARN_ON_ONCE(flags & BTW_DIRTY) || + WARN_ON_ONCE(iref->key)) + return -EINVAL; + + walk_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); + iter_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); + if (!walk_key || !iter_key) + return -ENOMEM; + + memcpy(walk_key, key, key_len); + walk_len = key_len; + + for (;;) { + ret = btree_walk(sb, root, NULL, flags, walk_key, walk_len, + 0, 0, &bt, iter_key, &iter_len); + if (ret < 0) + break; + + pos = find_pos(bt, key, key_len, &cmp); + + /* point pos towards iteration, find_pos already for _NEXT */ + if ((flags & BTW_AFTER) && cmp == 0) + pos++; + else if ((flags & BTW_PREV) && cmp < 0) + pos--; + else if ((flags & BTW_BEFORE) && cmp == 0) + pos--; + + /* found the next item in this leaf */ + if (pos >= 0 && pos < le16_to_cpu(bt->nr_items)) { + item = pos_item(bt, pos); + init_item_ref(iref, item); + ret = 0; + break; + } + + put_btree_block(bt); + + /* nothing in this leaf, walk gave us a key */ + if (iter_len > 0) { + memcpy(walk_key, iter_key, iter_len); + walk_len = iter_len; + continue; + } + + ret = -ENOENT; + break; + } + + kfree(walk_key); + kfree(iter_key); + + return ret; +} + +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + return btree_iter(sb, root, BTW_NEXT, key, key_len, iref); +} + +int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + return btree_iter(sb, root, BTW_NEXT | BTW_AFTER, key, key_len, iref); +} + +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + return btree_iter(sb, root, BTW_PREV, key, key_len, iref); +} + +int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref) +{ + return btree_iter(sb, root, BTW_PREV | BTW_BEFORE, key, key_len, iref); +} + +/* + * Ensure that the blocks that lead to the item with the given key are + * dirty. caller can hold a transaction to pin the dirty blocks and + * guarantee that later updates of the item will succeed. + * + * <0 is returned on error, including -ENOENT if the key isn't present. + */ +int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len) +{ + struct scoutfs_btree_block *bt; + DECLARE_BTREE_PATH(path); + int cmp; + int ret; + + ret = btree_walk(sb, root, &path, BTW_DIRTY, key, key_len, 0, 0, &bt, + NULL, NULL); + if (ret == 0) { + find_pos(bt, key, key_len, &cmp); + if (cmp == 0) + ret = 0; + else + ret = -ENOENT; + put_btree_block(bt); + } + + path_repair_reset(&path); + return ret; +} + +/* + * This initializes all our tracking info based on the super. Called + * before dirtying anything after having read the super or finished + * writing dirty blocks. + */ +static int btree_prepare_write(struct super_block *sb) +{ + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; + DECLARE_BTREE_INFO(sb, bti); + + bti->cur_dirtied = 0; + bti->old_dirtied = 0; + bti->first_dirty_bh = NULL; + bti->last_dirty_bh = NULL; + bti->first_dirty_blkno = le64_to_cpu(bring->first_blkno) + + le64_to_cpu(bring->next_block); + bti->first_dirty_seq = le64_to_cpu(bring->next_seq); + + return 0; +} + +/* + * The caller is serializing btree item dirtying and dirty block writing. + */ +bool scoutfs_btree_has_dirty(struct super_block *sb) +{ + DECLARE_BTREE_INFO(sb, bti); + + return bti->first_dirty_bh != NULL; +} + +/* dirty block allocation built this list */ +#define for_each_dirty_bh(bti, bh, tmp) \ + for (bh = bti->first_dirty_bh; bh && (tmp = bh->b_private, 1); bh = tmp) + +/* + * Write the dirty region of blocks to the ring. The caller still has + * to write the super after we're done. That could fail and we could + * be asked to write the blocks all over again. + * + * We're the only writer. + */ +int scoutfs_btree_write_dirty(struct super_block *sb) +{ + DECLARE_BTREE_INFO(sb, bti); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_btree_ring *bring = &super->bring; + struct scoutfs_btree_root *roots[] = { + /* XXX super roots go here */ + NULL, + }; + struct scoutfs_btree_root *root; + struct scoutfs_btree_block *bt; + DECLARE_BTREE_PATH(path); + struct buffer_head *tmp; + struct buffer_head *bh; + struct blk_plug plug; + unsigned next_root; + u8 bit; + int ret; + + if (bti->first_dirty_bh == NULL) + return 0; + + /* cow old dirty blocks to balance ring */ + bit = other_half_bit(bring, bti->first_dirty_blkno); + next_root = 0; + root = roots[next_root]; + while (root && bti->old_dirtied < bti->cur_dirtied) { + ret = btree_walk(sb, root, &path, + BTW_DIRTY | BTW_BIT | BTW_DIRTY_OLD, + NULL, 0, 0, bit, NULL, NULL, NULL); + path_repair_reset(&path); + if (ret == -ENOENT) { + root = roots[next_root++]; + continue; + } + if (ret < 0) + goto out; + } + + /* checksum everything to reduce time between io submission merging */ + for_each_dirty_bh(bti, bh, tmp) { + bt = (void *)bh->b_data; + bt->crc = 0; + bt->crc = cpu_to_le32(crc32c(~0, bt, SCOUTFS_BLOCK_SIZE)); + } + + blk_start_plug(&plug); + + for_each_dirty_bh(bti, bh, tmp) { + lock_buffer(bh); + set_buffer_dirty(bh); + set_buffer_mapped(bh); + bh->b_end_io = end_buffer_write_sync; + get_bh(bh); + /* XXX should be more careful with flags */ + submit_bh(WRITE_SYNC | REQ_META | REQ_PRIO, bh); + } + + blk_finish_plug(&plug); + + ret = 0; + for_each_dirty_bh(bti, bh, tmp) { + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + } +out: + return ret; +} + +/* + * The dirty blocks and their super reference have been successfully written. + * Remove them from the dirty list and drop their references and prepare + * for the next write. + */ +void scoutfs_btree_write_complete(struct super_block *sb) +{ + DECLARE_BTREE_INFO(sb, bti); + struct buffer_head *bh; + struct buffer_head *tmp; + + for_each_dirty_bh(bti, bh, tmp) { + bh->b_private = NULL; + put_bh(bh); + } + + btree_prepare_write(sb); +} + +int scoutfs_btree_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct btree_info *bti; + + bti = kzalloc(sizeof(struct btree_info), GFP_KERNEL); + if (!bti) + return -ENOMEM; + + mutex_init(&bti->mutex); + + sbi->btree_info = bti; + + btree_prepare_write(sb); + + return 0; +} + +void scoutfs_btree_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + kfree(sbi->btree_info); + sbi->btree_info = NULL; +} diff --git a/kmod/src/btree.h b/kmod/src/btree.h new file mode 100644 index 00000000..860923a2 --- /dev/null +++ b/kmod/src/btree.h @@ -0,0 +1,51 @@ +#ifndef _SCOUTFS_BTREE_H_ +#define _SCOUTFS_BTREE_H_ + +#include + +struct scoutfs_btree_item_ref { + void *key; + unsigned key_len; + void *val; + unsigned val_len; +}; + +#define SCOUTFS_BTREE_ITEM_REF(name) \ + struct scoutfs_btree_item_ref name = {NULL,} + +int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len); +int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len); +int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len); +int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref); +int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, + void *key, unsigned key_len); + +void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref); + +bool scoutfs_btree_has_dirty(struct super_block *sb); +int scoutfs_btree_write_dirty(struct super_block *sb); +void scoutfs_btree_write_complete(struct super_block *sb); + +int scoutfs_btree_setup(struct super_block *sb); +void scoutfs_btree_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 6ef16073..8a7a2df5 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -76,6 +76,90 @@ struct scoutfs_ring_descriptor { __le64 nr_blocks; } __packed; +/* + * Assert that we'll be able to represent all possible keys with 8 64bit + * primary sort values. + */ +#define SCOUTFS_BTREE_GREATEST_KEY_LEN 32 +/* level >0 segments can have a full key and some metadata */ +#define SCOUTFS_BTREE_MAX_KEY_LEN 320 +/* level 0 segments can have two full keys in the value :/ */ +#define SCOUTFS_BTREE_MAX_VAL_LEN 768 + +/* + * A 4EB test image measured a worst case height of 17. This is plenty + * generous. + */ +#define SCOUTFS_BTREE_MAX_HEIGHT 20 + +/* btree blocks (beyond the first) need to be at least half full */ +#define SCOUTFS_BTREE_FREE_LIMIT \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) + +#define SCOUTFS_BTREE_BITS 8 + +/* + * Btree items can have bits associated with them. Their parent items + * reflect all the bits that their child block contain. Thus searches + * can find items with bits set. + * + * @SCOUTFS_BTREE_BIT_HALF1: Tracks blocks found in the first half of + * the ring. It's used to migrate blocks from the old half of the ring + * into the current half as blocks are dirtied. It's not found in leaf + * items but is calculated based on the block number of referenced + * blocks. _HALF2 is identical but for the second half of the ring. + */ +enum { + SCOUTFS_BTREE_BIT_HALF1 = (1 << 0), + SCOUTFS_BTREE_BIT_HALF2 = (1 << 1), +}; + +struct scoutfs_btree_ref { + __le64 blkno; + __le64 seq; +} __packed; + +/* + * A height of X means that the first block read will have level X-1 and + * the leaves will have level 0. + */ +struct scoutfs_btree_root { + struct scoutfs_btree_ref ref; + __u8 height; +} __packed; + +struct scoutfs_btree_item_header { + __le16 off; + __u8 bits; +} __packed; + +struct scoutfs_btree_item { + __le16 key_len; + __le16 val_len; + __u8 data[0]; +} __packed; + +struct scoutfs_btree_block { + __le64 fsid; + __le64 blkno; + __le64 seq; + __le32 crc; + __le32 _pad; + __le16 free_end; + __le16 free_reclaim; + __le16 nr_items; + __le16 bit_counts[SCOUTFS_BTREE_BITS]; + __u8 level; + struct scoutfs_btree_item_header item_hdrs[0]; +} __packed; + +struct scoutfs_btree_ring { + __le64 first_blkno; + __le64 nr_blocks; + __le64 next_block; + __le64 next_seq; +} __packed; + /* * This is absurdly huge. If there was only ever 1 item per segment and * 2^64 items the tree could get this deep. @@ -313,6 +397,7 @@ struct scoutfs_super_block { __le64 ring_blocks; __le64 ring_tail_block; __le64 ring_gen; + struct scoutfs_btree_ring bring; __le64 next_seg_seq; struct scoutfs_ring_descriptor alloc_ring; struct scoutfs_manifest manifest; diff --git a/kmod/src/sort_priv.c b/kmod/src/sort_priv.c new file mode 100644 index 00000000..2acc0802 --- /dev/null +++ b/kmod/src/sort_priv.c @@ -0,0 +1,71 @@ +/* + * A copy of sort() from upstream with a priv argument that's passed + * to comparison, like list_sort(). + */ + +/* ------------------------ */ + +/* + * A fast, small, non-recursive O(nlog n) sort for the Linux kernel + * + * Jan 23 2005 Matt Mackall + */ + +#include +#include +#include +#include +#include "sort_priv.h" + +/** + * sort - sort an array of elements + * @priv: caller's pointer to pass to comparison and swap functions + * @base: pointer to data to sort + * @num: number of elements + * @size: size of each element + * @cmp_func: pointer to comparison function + * @swap_func: pointer to swap function or NULL + * + * This function does a heapsort on the given array. You may provide a + * swap_func function optimized to your element type. + * + * Sorting time is O(n log n) both on average and worst-case. While + * qsort is about 20% faster on average, it suffers from exploitable + * O(n*n) worst-case behavior and extra memory requirements that make + * it less suitable for kernel use. + */ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)) +{ + /* pre-scale counters for performance */ + int i = (num/2 - 1) * size, n = num * size, c, r; + + /* heapify */ + for ( ; i >= 0; i -= size) { + for (r = i; r * 2 + size < n; r = c) { + c = r * 2 + size; + if (c < n - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } + + /* sort */ + for (i = n - size; i > 0; i -= size) { + swap_func(priv, base, base + i, size); + for (r = 0; r * 2 + size < i; r = c) { + c = r * 2 + size; + if (c < i - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } +} diff --git a/kmod/src/sort_priv.h b/kmod/src/sort_priv.h new file mode 100644 index 00000000..c5fde547 --- /dev/null +++ b/kmod/src/sort_priv.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_SORT_PRIV_H_ +#define _SCOUTFS_SORT_PRIV_H_ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 4c0ba6a3..d7321786 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -203,6 +203,9 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sb->s_maxbytes = MAX_LFS_FILESIZE; sb->s_op = &scoutfs_super_ops; + /* btree blocks use long lived bh->b_data refs */ + mapping_set_gfp_mask(sb->s_bdev->bd_inode->i_mapping, GFP_NOFS); + sbi = kzalloc(sizeof(struct scoutfs_sb_info), GFP_KERNEL); sb->s_fs_info = sbi; sbi->sb = sb; diff --git a/kmod/src/super.h b/kmod/src/super.h index fc278b69..95285080 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -17,6 +17,7 @@ struct trans_info; struct lock_info; struct net_info; struct inode_sb_info; +struct btree_info; struct scoutfs_sb_info { struct super_block *sb; @@ -34,6 +35,7 @@ struct scoutfs_sb_info { struct compact_info *compact_info; struct data_info *data_info; struct inode_sb_info *inode_sb_info; + struct btree_info *btree_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From fc50072cf97537d3cb9dd4c3ef8de1397c74fa38 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 29 Jun 2017 14:22:10 -0700 Subject: [PATCH 323/920] scoutfs: store manifest entries in the btree Convert the manifest to store entries in persistent btree keys and values instead of using the rbtree in memory from the ring. The btree doesn't have a sort function. It just compares variable length keys. The most complicated part of this transformation is dealing with the fallout of this. The compare function can't compare different search keys and item keys so searches need to construct full synthetic btree keys to search. It also can't return different comparisons, like overlaping, so the caller needs to do a bit more work to use key comparisons to find overlapping segments. And it can't compare differently depending on the level of the manifest so we store the manifest in keys differently depending on whether its in level 0 or not. All mount clients can now see the manifest blocks. They can query the manifest directly when trying to find segments to read. We can get rid of all the networking calls that were finding the segments for readers. We change the manifest functions that relied on the ring that the to make changes in the manifest persistent. We don't touch the allocator or the rest of the manifest server, though, so this commit breaks the world. It'll be restored in future patches as we update the segment allocator and server to work with the btree. Signed-off-by: Zach Brown --- kmod/src/btree.c | 2 +- kmod/src/compact.c | 32 +- kmod/src/compact.h | 4 +- kmod/src/format.h | 39 ++- kmod/src/manifest.c | 818 ++++++++++++++++++++------------------------ kmod/src/manifest.h | 54 ++- kmod/src/net.c | 283 +++++---------- kmod/src/net.h | 4 - kmod/src/ring.c | 6 +- kmod/src/seg.c | 41 +-- kmod/src/seg.h | 11 +- 11 files changed, 548 insertions(+), 746 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index e30f5af0..55a2a0a8 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1768,7 +1768,7 @@ int scoutfs_btree_write_dirty(struct super_block *sb) struct scoutfs_super_block *super = &sbi->super; struct scoutfs_btree_ring *bring = &super->bring; struct scoutfs_btree_root *roots[] = { - /* XXX super roots go here */ + &super->manifest.root, NULL, }; struct scoutfs_btree_root *root; diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 56b248a8..12f8d902 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -457,15 +457,13 @@ void scoutfs_compact_describe(struct super_block *sb, void *data, * and is then possibly adding all the lower overlapping segments. */ int scoutfs_compact_add(struct super_block *sb, void *data, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level) + struct scoutfs_manifest_entry *ment) { struct compact_cursor *curs = data; struct compact_seg *cseg; int ret; - cseg = alloc_cseg(sb, first, last); + cseg = alloc_cseg(sb, &ment->first, &ment->last); if (!cseg) { ret = -ENOMEM; goto out; @@ -473,9 +471,9 @@ int scoutfs_compact_add(struct super_block *sb, void *data, list_add_tail(&cseg->entry, &curs->csegs); - cseg->segno = segno; - cseg->seq = seq; - cseg->level = level; + cseg->segno = ment->segno; + cseg->seq = ment->seq; + cseg->level = ment->level; if (!curs->upper) curs->upper = cseg; @@ -501,8 +499,8 @@ void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno) /* * Commit the result of a compaction based on the state of the cursor. - * The net caller stops the rings from being written while we're making - * changes. We lock the manifest to atomically make our changes. + * The net caller stops the manifest from being written while we're + * making changes. We lock the manifest to atomically make our changes. * * The erorr handling is sketchy here because calling the manifest from * here is temporary. We should be sending a message to the server @@ -510,6 +508,7 @@ void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno) */ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) { + struct scoutfs_manifest_entry ment; struct compact_cursor *curs = c; struct list_head *results = r; struct compact_seg *cseg; @@ -533,8 +532,9 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) BUG_ON(ret); } - ret = scoutfs_manifest_del(sb, cseg->first, - cseg->seq, cseg->level); + scoutfs_manifest_init_entry(&ment, cseg->level, 0, cseg->seq, + cseg->first, NULL); + ret = scoutfs_manifest_del(sb, &ment); BUG_ON(ret); } @@ -542,12 +542,12 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) list_for_each_entry(cseg, results, entry) { /* XXX moved upper segments won't have read the segment :P */ if (cseg->seg) - ret = scoutfs_seg_manifest_add(sb, cseg->seg, - cseg->level); + scoutfs_seg_init_ment(&ment, cseg->level, cseg->seg); else - ret = scoutfs_manifest_add(sb, cseg->first, - cseg->last, cseg->segno, - cseg->seq, cseg->level); + scoutfs_manifest_init_entry(&ment, cseg->level, + cseg->segno, cseg->seq, + cseg->first, cseg->last); + ret = scoutfs_manifest_add(sb, &ment); BUG_ON(ret); } diff --git a/kmod/src/compact.h b/kmod/src/compact.h index f6f4bb60..c163ce56 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -6,9 +6,7 @@ void scoutfs_compact_kick(struct super_block *sb); void scoutfs_compact_describe(struct super_block *sb, void *data, u8 upper_level, u8 last_level, bool sticky); int scoutfs_compact_add(struct super_block *sb, void *data, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level); + struct scoutfs_manifest_entry *ment); void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno); int scoutfs_compact_commit(struct super_block *sb, void *c, void *r); diff --git a/kmod/src/format.h b/kmod/src/format.h index 8a7a2df5..7ca2f2e3 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -169,16 +169,38 @@ struct scoutfs_btree_ring { #define SCOUTFS_MANIFEST_FANOUT 10 struct scoutfs_manifest { - struct scoutfs_ring_descriptor ring; + struct scoutfs_btree_root root; __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; } __packed; -struct scoutfs_manifest_entry { +/* + * Manifest entries are packed into btree keys and values in a very + * fiddly way so that we can sort them with memcmp first by level then + * by their position in the level. First comes the level. + * + * Level 0 segments are sorted by their seq so they don't have the first + * segment key in the manifest btree key. Both of their keys are in the + * value. + * + * Level 1 segments are sorted by their first key so their last key is + * in the value. + * + * We go to all this trouble so that we can communicate a version of the + * manifest with one btree root, have dense btree keys which are used as + * seperators in parent blocks, and don't duplicate the large keys in + * the manifest btree key and value. + */ + +struct scoutfs_manifest_btree_key { + __u8 level; + __u8 bkey[0]; +} __packed; + +struct scoutfs_manifest_btree_val { __le64 segno; __le64 seq; __le16 first_key_len; __le16 last_key_len; - __u8 level; __u8 keys[0]; } __packed; @@ -536,9 +558,13 @@ struct scoutfs_net_key_range { __u8 key_bytes[0]; } __packed; -struct scoutfs_net_manifest_entries { - __le16 nr; - struct scoutfs_manifest_entry ments[0]; +struct scoutfs_net_manifest_entry { + __le64 segno; + __le64 seq; + __le16 first_key_len; + __le16 last_key_len; + __u8 level; + __u8 keys[0]; } __packed; /* XXX I dunno, totally made up */ @@ -561,7 +587,6 @@ struct scoutfs_net_segnos { enum { SCOUTFS_NET_ALLOC_INODES = 0, - SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_BULK_ALLOC, diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index f757c1b9..31c55283 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -20,7 +20,7 @@ #include "kvec.h" #include "seg.h" #include "item.h" -#include "ring.h" +#include "btree.h" #include "cmp.h" #include "compact.h" #include "manifest.h" @@ -30,16 +30,17 @@ #include "scoutfs_trace.h" /* - * Manifest entries are stored in ring nodes. + * Manifest entries are stored in the cow btrees in the persistently + * allocated ring of blocks in the shared device. This lets clients + * read consistent old versions of the manifest when it's safe to do so. * - * They're sorted first by level then by their first key. This enables - * the primary searches based on key value for looking up items in - * segments via the manifest. + * Manifest entries are sorted first by level then by their first key. + * This enables the primary searches based on key value for looking up + * items in segments via the manifest. */ struct manifest { struct rw_semaphore rwsem; - struct scoutfs_ring_info ring; u8 nr_levels; /* calculated on mount, const thereafter */ @@ -78,41 +79,6 @@ struct manifest_ref { struct scoutfs_key_buf *last; }; -/* - * Seq is only specified for operations that differentiate between - * segments with identical items by their sequence number. - */ -struct manifest_search_key { - u64 seq; - struct scoutfs_key_buf *key; - u8 level; -}; - -static void init_ment_keys(struct scoutfs_manifest_entry *ment, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) -{ - if (first) - scoutfs_key_init(first, ment->keys, - le16_to_cpu(ment->first_key_len)); - if (last) - scoutfs_key_init(last, ment->keys + - le16_to_cpu(ment->first_key_len), - le16_to_cpu(ment->last_key_len)); -} - -static bool cmp_range_ment(struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - init_ment_keys(ment, &first, &last); - - return scoutfs_key_compare_ranges(key, end, &first, &last); -} - /* * Change the level count under the manifest lock. We then maintain a * bit that can be tested outside the lock to determine if the caller @@ -153,6 +119,152 @@ bool scoutfs_manifest_level0_full(struct super_block *sb) return test_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); } +void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, + u64 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) +{ + ment->level = level; + ment->segno = segno; + ment->seq = seq; + + if (first) + scoutfs_key_clone(&ment->first, first); + else + scoutfs_key_init(&ment->first, NULL, 0); + + if (last) + scoutfs_key_clone(&ment->last, last); + else + scoutfs_key_init(&ment->last, NULL, 0); +} + +/* + * level 0 segments have the extra seq up in the btree key. + */ +static struct scoutfs_manifest_btree_key * +alloc_btree_key_val_lens(unsigned first_len, unsigned last_len) +{ + return kmalloc(sizeof(struct scoutfs_manifest_btree_key) + + sizeof(u64) + + sizeof(struct scoutfs_manifest_btree_val) + + first_len + last_len, GFP_NOFS); +} + +/* + * Initialize the btree key and value for a manifest entry in one contiguous + * allocation. + */ +static struct scoutfs_manifest_btree_key * +alloc_btree_key_val(struct scoutfs_manifest_entry *ment, unsigned *mkey_len, + struct scoutfs_manifest_btree_val **mval_ret, + unsigned *mval_len_ret) +{ + struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_btree_val *mval; + struct scoutfs_key_buf b_first; + struct scoutfs_key_buf b_last; + unsigned bkey_len; + unsigned mval_len; + __be64 seq; + + mkey = alloc_btree_key_val_lens(ment->first.key_len, ment->last.key_len); + if (!mkey) + return NULL; + + if (ment->level == 0) { + seq = cpu_to_be64(ment->seq); + bkey_len = sizeof(seq); + memcpy(mkey->bkey, &seq, bkey_len); + } else { + bkey_len = ment->first.key_len; + } + + *mkey_len = offsetof(struct scoutfs_manifest_btree_key, bkey[bkey_len]); + mval = (void *)mkey + *mkey_len; + + if (ment->level == 0) { + scoutfs_key_init(&b_first, mval->keys, ment->first.key_len); + scoutfs_key_init(&b_last, mval->keys + ment->first.key_len, + ment->last.key_len); + mval_len = sizeof(struct scoutfs_manifest_btree_val) + + ment->first.key_len + ment->last.key_len; + } else { + scoutfs_key_init(&b_first, mkey->bkey, ment->first.key_len); + scoutfs_key_init(&b_last, mval->keys, ment->last.key_len); + mval_len = sizeof(struct scoutfs_manifest_btree_val) + + ment->last.key_len; + } + + mkey->level = ment->level; + mval->segno = cpu_to_le64(ment->segno); + mval->seq = cpu_to_le64(ment->seq); + mval->first_key_len = cpu_to_le16(ment->first.key_len); + mval->last_key_len = cpu_to_le16(ment->last.key_len); + + scoutfs_key_copy(&b_first, &ment->first); + scoutfs_key_copy(&b_last, &ment->last); + + if (mval_ret) { + *mval_ret = mval; + *mval_len_ret = mval_len; + } + return mkey; +} + +/* initialize a native manifest entry to point to the btree key and value */ +static void init_ment_iref(struct scoutfs_manifest_entry *ment, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_manifest_btree_key *mkey = iref->key; + struct scoutfs_manifest_btree_val *mval = iref->val; + + ment->level = mkey->level; + ment->segno = le64_to_cpu(mval->segno); + ment->seq = le64_to_cpu(mval->seq); + + if (ment->level == 0) { + scoutfs_key_init(&ment->first, mval->keys, + le16_to_cpu(mval->first_key_len)); + scoutfs_key_init(&ment->last, mval->keys + + le16_to_cpu(mval->first_key_len), + le16_to_cpu(mval->last_key_len)); + } else { + scoutfs_key_init(&ment->first, mkey->bkey, + le16_to_cpu(mval->first_key_len)); + scoutfs_key_init(&ment->last, mval->keys, + le16_to_cpu(mval->last_key_len)); + } +} + +/* + * Fill the callers max-size btree key with the given values and return + * its length. + */ +static unsigned init_btree_key(struct scoutfs_manifest_btree_key *mkey, + u8 level, u64 seq, struct scoutfs_key_buf *first) +{ + struct scoutfs_key_buf b_first; + unsigned bkey_len; + __be64 bseq; + + mkey->level = level; + + if (level == 0) { + bseq = cpu_to_be64(seq); + bkey_len = sizeof(bseq); + memcpy(mkey->bkey, &bseq, bkey_len); + } else if (first) { + scoutfs_key_init(&b_first, mkey->bkey, first->key_len); + scoutfs_key_copy(&b_first, first); + bkey_len = first->key_len; + } else { + bkey_len = 0; + } + + return offsetof(struct scoutfs_manifest_btree_key, bkey[bkey_len]); +} + /* * Insert a new manifest entry in the ring. The ring allocates a new * node for us and we fill it. @@ -160,180 +272,68 @@ bool scoutfs_manifest_level0_full(struct super_block *sb) * This must be called with the manifest lock held. */ int scoutfs_manifest_add(struct super_block *sb, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level) + struct scoutfs_manifest_entry *ment) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_manifest_entry *ment; - struct scoutfs_key_buf ment_first; - struct scoutfs_key_buf ment_last; - struct manifest_search_key skey; - unsigned key_bytes; - unsigned bytes; - - trace_scoutfs_manifest_add(sb, level, segno, seq, first, last); - - key_bytes = first->key_len + last->key_len; - bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); - - skey.key = first; - skey.level = level; - skey.seq = seq; - - ment = scoutfs_ring_insert(&mani->ring, &skey, bytes); - if (!ment) - return -ENOMEM; - - ment->segno = cpu_to_le64(segno); - ment->seq = cpu_to_le64(seq); - ment->first_key_len = cpu_to_le16(first->key_len); - ment->last_key_len = cpu_to_le16(last->key_len); - ment->level = level; - - init_ment_keys(ment, &ment_first, &ment_last); - scoutfs_key_copy(&ment_first, first); - scoutfs_key_copy(&ment_last, last); - - mani->nr_levels = max_t(u8, mani->nr_levels, level + 1); - add_level_count(sb, level, 1); - return 0; -} - -/* - * Add a manifest entry as provided by the caller instead of exploded - * out into arguments. - * - * This must be called with the manifest lock held. - */ -int scoutfs_manifest_add_ment(struct super_block *sb, - struct scoutfs_manifest_entry *add) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_manifest_entry *ment; - struct manifest_search_key skey; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - unsigned bytes; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_btree_val *mval; + unsigned mkey_len; + unsigned mval_len; + int ret; lockdep_assert_held(&mani->rwsem); - init_ment_keys(add, &first, &last); - trace_scoutfs_manifest_add(sb, add->level, le64_to_cpu(add->segno), - le64_to_cpu(add->seq), &first, &last); - - skey.key = &first; - skey.level = add->level; - skey.seq = le64_to_cpu(add->seq); - - bytes = scoutfs_manifest_bytes(add); - - ment = scoutfs_ring_insert(&mani->ring, &skey, bytes); - if (!ment) + mkey = alloc_btree_key_val(ment, &mkey_len, &mval, &mval_len); + if (!mkey) return -ENOMEM; - memcpy(ment, add, bytes); + trace_scoutfs_manifest_add(sb, ment->level, ment->segno, ment->seq, + &ment->first, &ment->last); - mani->nr_levels = max_t(u8, mani->nr_levels, add->level + 1); - add_level_count(sb, add->level, 1); + ret = scoutfs_btree_insert(sb, &super->manifest.root, mkey, mkey_len, + mval, mval_len); + if (ret == 0) { + mani->nr_levels = max_t(u8, mani->nr_levels, ment->level + 1); + add_level_count(sb, ment->level, 1); + } - return 0; + kfree(mkey); + return ret; } /* * This must be called with the manifest lock held. + * + * When this is called from the network we can take the keys directly as + * they were sent from the clients. */ -int scoutfs_manifest_dirty(struct super_block *sb, - struct scoutfs_key_buf *first, u64 seq, u8 level) +int scoutfs_manifest_del(struct super_block *sb, + struct scoutfs_manifest_entry *ment) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_manifest_entry *ment; - struct manifest_search_key skey; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_btree_key *mkey; + unsigned mkey_len; + int ret; - skey.key = first; - skey.level = level; - skey.seq = seq; + trace_scoutfs_manifest_delete(sb, ment->level, ment->segno, ment->seq, + &ment->first, &ment->last); - ment = scoutfs_ring_lookup(&mani->ring, &skey); - if (!ment) - return -ENOENT; + lockdep_assert_held(&mani->rwsem); - scoutfs_ring_dirty(&mani->ring, ment); - return 0; -} + mkey = alloc_btree_key_val(ment, &mkey_len, NULL, NULL); + if (!mkey) + return -ENOMEM; -/* - * This must be called with the manifest lock held. - */ -int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, - u64 seq, u8 level) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_manifest_entry *ment; - struct manifest_search_key skey; - struct scoutfs_key_buf last; + ret = scoutfs_btree_delete(sb, &super->manifest.root, mkey, mkey_len); + if (ret == 0) + add_level_count(sb, ment->level, -1ULL); - skey.key = first; - skey.level = level; - skey.seq = seq; - - ment = scoutfs_ring_lookup(&mani->ring, &skey); - if (!ment) - return -ENOENT; - - init_ment_keys(ment, NULL, &last); - trace_scoutfs_manifest_delete(sb, ment->level, le64_to_cpu(ment->segno), - le64_to_cpu(ment->seq), first, &last); - - scoutfs_ring_delete(&mani->ring, ment); - add_level_count(sb, level, -1ULL); - return 0; -} - -/* - * Return the total number of bytes used by the given manifest entry, - * including its struct. - */ -int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment) -{ - return sizeof(struct scoutfs_manifest_entry) + - le16_to_cpu(ment->first_key_len) + - le16_to_cpu(ment->last_key_len); -} - -/* - * Return an allocated and filled in manifest entry. - */ -struct scoutfs_manifest_entry * -scoutfs_manifest_alloc_entry(struct super_block *sb, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level) -{ - struct scoutfs_manifest_entry *ment; - struct scoutfs_key_buf ment_first; - struct scoutfs_key_buf ment_last; - unsigned key_bytes; - unsigned bytes; - - key_bytes = first->key_len + last->key_len; - bytes = offsetof(struct scoutfs_manifest_entry, keys[key_bytes]); - - ment = kmalloc(bytes, GFP_NOFS); - if (!ment) - return NULL; - - ment->segno = cpu_to_le64(segno); - ment->seq = cpu_to_le64(seq); - ment->first_key_len = cpu_to_le16(first->key_len); - ment->last_key_len = cpu_to_le16(last->key_len); - ment->level = level; - - init_ment_keys(ment, &ment_first, &ment_last); - scoutfs_key_copy(&ment_first, first); - scoutfs_key_copy(&ment_last, last); - - return ment; + kfree(mkey); + return ret; } /* @@ -372,50 +372,70 @@ static void free_ref(struct super_block *sb, struct manifest_ref *ref) } /* - * Allocate a native manifest ref so that we can work with segments described - * by the callers manifest entry. - (* - * This frees all the elements on the list if it returns an error. + * Allocate a reading manifest ref so that we can work with segments + * described by the callers manifest entry. */ -int scoutfs_manifest_add_ment_ref(struct super_block *sb, - struct list_head *list, - struct scoutfs_manifest_entry *ment) +static int alloc_manifest_ref(struct super_block *sb, struct list_head *ref_list, + struct scoutfs_manifest_entry *ment) { - struct scoutfs_key_buf ment_first; - struct scoutfs_key_buf ment_last; struct manifest_ref *ref; - struct manifest_ref *tmp; - - init_ment_keys(ment, &ment_first, &ment_last); ref = kzalloc(sizeof(struct manifest_ref), GFP_NOFS); if (ref) { - ref->first = scoutfs_key_dup(sb, &ment_first); - ref->last = scoutfs_key_dup(sb, &ment_last); + ref->first = scoutfs_key_dup(sb, &ment->first); + ref->last = scoutfs_key_dup(sb, &ment->last); } if (!ref || !ref->first || !ref->last) { free_ref(sb, ref); - list_for_each_entry_safe(ref, tmp, list, entry) { - list_del_init(&ref->entry); - free_ref(sb, ref); - } return -ENOMEM; } - ref->segno = le64_to_cpu(ment->segno); - ref->seq = le64_to_cpu(ment->seq); ref->level = ment->level; + ref->segno = ment->segno; + ref->seq = ment->seq; - list_add_tail(&ref->entry, list); + list_add_tail(&ref->entry, ref_list); return 0; } /* - * Return an array of pointers to the entries in the manifest that - * intersect with the given key range. The entries will be ordered by - * the order that they should be read: level 0 from newest to oldest - * then increasing higher order levels. + * Return the previous entry if it's in the right level and it overlaps + * with the start key by having a last key that's >=. If no such entry + * exists it just returns the next entry after the key and doesn't test + * it at all. If this returns 0 then the caller has to put the iref. + */ +static int btree_prev_overlap_or_next(struct super_block *sb, + struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_key_buf *start, u8 level, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_manifest_entry ment; + int ret; + + ret = scoutfs_btree_prev(sb, root, key, key_len, iref); + if (ret < 0 && ret != -ENOENT) + return ret; + + if (ret == 0) { + init_ment_iref(&ment, iref); + if (ment.level != level || + scoutfs_key_compare(&ment.last, start) < 0) + ret = -ENOENT; + } + if (ret == -ENOENT) { + scoutfs_btree_put_iref(iref); + ret = scoutfs_btree_next(sb, root, key, key_len, iref); + } + + return ret; +} + +/* + * starting with the caller's key. The entries will be ordered by the + * order that they should be read: level 0 from newest to oldest then + * increasing higher order levels. * * We have to get all the level 0 segments that intersect with the range * of items that we want to search because the level 0 segments can @@ -427,74 +447,96 @@ int scoutfs_manifest_add_ment_ref(struct super_block *sb, * existing segment that intersects with the range, even if it doesn't * contain the key. The key might fall between segments at that level. * - * This is called by the server who is processing manifest search - * messages from mounts. The server locks down the manifest while it - * gets these pointers and then uses them to allocate and fill a reply - * message. + * XXX Today this using the roots from the mount-wide super. This is + * super wrong. Doing so lets it use the dirty btree that could be + * modified by the manifest server running on this node so it has to + * lock. It should be using a specific root communicated by lock lvbs + * (or read from the super on mount). Then the btrees it traverses will + * be stable and read-only. (But can still get -ESTALE if they're + * re-written under us, would need to re-sample roots from the super in + * that case, I imagine.) */ -struct scoutfs_manifest_entry ** -scoutfs_manifest_find_range_entries(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - unsigned *found_bytes) +static int get_manifest_refs(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, + struct list_head *ref_list) { DECLARE_MANIFEST(sb, mani); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_manifest_entry **found; - struct scoutfs_manifest_entry *ment; - struct manifest_search_key skey; - unsigned nr; + struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_entry ment; + SCOUTFS_BTREE_ITEM_REF(iref); + SCOUTFS_BTREE_ITEM_REF(prev); + unsigned mkey_len; + int ret; int i; - lockdep_assert_held(&mani->rwsem); + scoutfs_manifest_init_entry(&ment, 0, 0, 0, key, NULL); + mkey = alloc_btree_key_val(&ment, &mkey_len, NULL, NULL); + if (!mkey) + return -ENOMEM; - *found_bytes = 0; - - /* at most we get all level 0, one from other levels, and null term */ - nr = le64_to_cpu(super->manifest.level_counts[0]) + mani->nr_levels + 1; - - found = kcalloc(nr, sizeof(struct scoutfs_manifest_entry *), GFP_NOFS); - if (!found) { - found = ERR_PTR(-ENOMEM); - goto out; - } - - nr = 0; + scoutfs_manifest_lock(sb); /* get level 0 segments that overlap with the missing range */ - skey.key = NULL; - skey.level = 0; - skey.seq = ~0ULL; - ment = scoutfs_ring_lookup_prev(&mani->ring, &skey); - while (ment) { - if (cmp_range_ment(key, end, ment) == 0) { - found[nr++] = ment; - *found_bytes += scoutfs_manifest_bytes(ment); + mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); + ret = scoutfs_btree_prev(sb, &super->manifest.root, + mkey, mkey_len, &iref); + while (ret == 0) { + init_ment_iref(&ment, &iref); + + if (scoutfs_key_compare_ranges(key, end, &ment.first, + &ment.last) == 0) { + ret = alloc_manifest_ref(sb, ref_list, &ment); + if (ret) + goto out; } - ment = scoutfs_ring_prev(&mani->ring, ment); + swap(prev, iref); + ret = scoutfs_btree_before(sb, &super->manifest.root, + prev.key, prev.key_len, &iref); + scoutfs_btree_put_iref(&prev); } + if (ret != -ENOENT) + goto out; - /* get higher level segments that overlap with the starting key */ + /* + * XXX Today we need to read the next segment if our starting key + * falls between segments. That won't be the case once we tie + * cached items to their locks. + */ + mkey_len = init_btree_key(mkey, 1, 0, key); for (i = 1; i < mani->nr_levels; i++) { - skey.key = key; - skey.level = i; - skey.seq = 0; + mkey->level = i; /* XXX should use level counts to skip searches */ - ment = scoutfs_ring_lookup_next(&mani->ring, &skey); - if (ment) { - found[nr++] = ment; - *found_bytes += scoutfs_manifest_bytes(ment); + scoutfs_btree_put_iref(&iref); + ret = btree_prev_overlap_or_next(sb, &super->manifest.root, + mkey, mkey_len, key, i, + &iref); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; } - } - /* null terminate */ - found[nr++] = NULL; + init_ment_iref(&ment, &iref); + + if (ment.level != i) + continue; + + ret = alloc_manifest_ref(sb, ref_list, &ment); + if (ret) + goto out; + } + ret = 0; out: - return found; + scoutfs_btree_put_iref(&iref); + scoutfs_btree_put_iref(&prev); + scoutfs_manifest_unlock(sb); + kfree(mkey); + return ret; } /* @@ -549,7 +591,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, trace_scoutfs_read_items(sb, key, end); /* get refs on all the segments */ - ret = scoutfs_net_manifest_range_entries(sb, key, end, &ref_list); + ret = get_manifest_refs(sb, key, end, &ref_list); if (ret) goto out; @@ -705,40 +747,6 @@ out: return ret; } -int scoutfs_manifest_has_dirty(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - int ret; - - down_write(&mani->rwsem); - ret = scoutfs_ring_has_dirty(&mani->ring); - up_write(&mani->rwsem); - - return ret; -} - -int scoutfs_manifest_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp) -{ - DECLARE_MANIFEST(sb, mani); - int ret; - - down_write(&mani->rwsem); - ret = scoutfs_ring_submit_write(sb, &mani->ring, comp); - up_write(&mani->rwsem); - - return ret; -} - -void scoutfs_manifest_write_complete(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - - down_write(&mani->rwsem); - scoutfs_ring_write_complete(&mani->ring); - up_write(&mani->rwsem); -} - /* * Give the caller the segments that will be involved in the next * compaction. @@ -766,13 +774,13 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_entry *ment; - struct scoutfs_manifest_entry *over; - struct manifest_search_key skey; - struct scoutfs_key_buf ment_first; - struct scoutfs_key_buf ment_last; - struct scoutfs_key_buf over_first; - struct scoutfs_key_buf over_last; + struct scoutfs_manifest_entry ment; + struct scoutfs_manifest_entry over; + struct scoutfs_manifest_btree_key *mkey = NULL; + SCOUTFS_BTREE_ITEM_REF(iref); + SCOUTFS_BTREE_ITEM_REF(over_iref); + SCOUTFS_BTREE_ITEM_REF(prev); + unsigned mkey_len; bool sticky; int level; int ret; @@ -794,54 +802,70 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } + /* alloc a full size mkey, fill it with whatever search key */ - /* find the oldest level 0 or the next higher order level by key */ - if (level == 0) { - ment = scoutfs_ring_first(&mani->ring); - if (ment && ment->level) - ment = NULL; - } else { - skey.key = mani->compact_keys[level]; - skey.level = level; - skey.seq = 0; - ment = scoutfs_ring_lookup_next(&mani->ring, &skey); - if (ment == NULL || ment->level != level) { - scoutfs_key_set_min(skey.key); - ment = scoutfs_ring_lookup_next(&mani->ring, &skey); - } - } - if (ment == NULL || ment->level != level) { - /* XXX shouldn't be possible */ - ret = 0; + mkey = alloc_btree_key_val_lens(SCOUTFS_MAX_KEY_SIZE, 0); + if (!mkey) { + ret = -ENOMEM; goto out; } - init_ment_keys(ment, &ment_first, &ment_last); + /* find the oldest level 0 or the next higher order level by key */ + if (level == 0) { + /* find the oldest level 0 */ + mkey_len = init_btree_key(mkey, 0, 0, NULL); + ret = scoutfs_btree_next(sb, &super->manifest.root, + mkey, mkey_len, &iref); + } else { + /* find the next segment after the compaction at this level */ + mkey_len = init_btree_key(mkey, level, 0, + mani->compact_keys[level]); + + ret = scoutfs_btree_next(sb, &super->manifest.root, + mkey, mkey_len, &iref); + if (ret == 0) { + init_ment_iref(&ment, &iref); + if (ment.level != level) + ret = -ENOENT; + } + if (ret == -ENOENT) { + /* .. possibly wrapping to the first key in level */ + mkey_len = init_btree_key(mkey, level, 0, NULL); + scoutfs_btree_put_iref(&iref); + ret = scoutfs_btree_next(sb, &super->manifest.root, + mkey, mkey_len, &iref); + } + } + if (ret == 0) { + init_ment_iref(&ment, &iref); + if (ment.level != level) + goto out; + } + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } /* add the upper input segment */ - ret = scoutfs_compact_add(sb, data, &ment_first, &ment_last, - le64_to_cpu(ment->segno), - le64_to_cpu(ment->seq), level); + ret = scoutfs_compact_add(sb, data, &ment); if (ret) goto out; nr++; - /* start with the first overlapping at the next level */ - skey.key = &ment_first; - skey.level = level + 1; - skey.seq = 0; - over = scoutfs_ring_lookup_next(&mani->ring, &skey); - /* and add a fanout's worth of lower overlapping segments */ + mkey_len = init_btree_key(mkey, level + 1, 0, &ment.first); + ret = btree_prev_overlap_or_next(sb, &super->manifest.root, + mkey, mkey_len, + &ment.first, level + 1, &over_iref); sticky = false; - for (i = 0; i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { - if (!over || over->level != (ment->level + 1)) + for (i = 0; ret == 0 && i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { + init_ment_iref(&over, &over_iref); + if (over.level != level + 1) break; - init_ment_keys(over, &over_first, &over_last); - - if (scoutfs_key_compare_ranges(&ment_first, &ment_last, - &over_first, &over_last) != 0) + if (scoutfs_key_compare_ranges(&ment.first, &ment.last, + &over.first, &over.last) != 0) break; /* upper level has to stay around when more than fanout */ @@ -850,114 +874,42 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) break; } - ret = scoutfs_compact_add(sb, data, &over_first, &over_last, - le64_to_cpu(over->segno), - le64_to_cpu(over->seq), level + 1); + ret = scoutfs_compact_add(sb, data, &over); if (ret) goto out; nr++; - over = scoutfs_ring_next(&mani->ring, over); + swap(prev, over_iref); + ret = scoutfs_btree_after(sb, &super->manifest.root, + prev.key, prev.key_len, &over_iref); + scoutfs_btree_put_iref(&prev); } + if (ret < 0 && ret != -ENOENT) + goto out; scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1, sticky); /* record the next key to start from */ - scoutfs_key_copy(mani->compact_keys[level], &ment_last); + scoutfs_key_copy(mani->compact_keys[level], &ment.last); scoutfs_key_inc(mani->compact_keys[level]); ret = 0; out: up_write(&mani->rwsem); + + kfree(mkey); + scoutfs_btree_put_iref(&iref); + scoutfs_btree_put_iref(&over_iref); + scoutfs_btree_put_iref(&prev); + return ret ?: nr; } -/* - * Manifest entries for all levels are stored in a single ring. - * - * First they're sorted by their level. - * - * Level 0 segments can contain any items which overlap so they are - * sorted by their sequence number. Compaction can find the first node - * and reading walks backwards through level 0 to get them from newest - * to oldest to resolve matching items. - * - * Higher level segments don't overlap. They are sorted by their first - * key. - * - * Searching comparisons are different than insertion and deletion - * comparisons for higher level segments. Searches want to find the - * segment that intersects with a given key. Insertions and deletions - * want to operate on the segment with a specific first key and sequence - * number. We tell the difference by the presence of a sequence number. - * A segment will never have a seq of 0. - */ -static int manifest_ring_compare_key(void *key, void *data) -{ - struct manifest_search_key *skey = key; - struct scoutfs_manifest_entry *ment = data; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - int cmp; - - scoutfs_key_init(&first, NULL, 0); - - if (skey->level < ment->level) { - cmp = -1; - goto out; - } - if (skey->level > ment->level) { - cmp = 1; - goto out; - } - - if (skey->level == 0) { - cmp = scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); - goto out; - } - - init_ment_keys(ment, &first, &last); - - if (skey->seq == 0) { - cmp = scoutfs_key_compare_ranges(skey->key, skey->key, - &first, &last); - } else { - cmp = scoutfs_key_compare(skey->key, &first) ?: - scoutfs_cmp_u64s(skey->seq, le64_to_cpu(ment->seq)); - } - -out: -#if 0 - /* pretty expensive to be on by default */ - SK_TRACE_PRINTK("%u,%llu,"SK_FMT" %c %u,%llu,"SK_FMT"\n", - skey->level, skey->seq, SK_ARG(skey->key), - cmp < 0 ? '<' : cmp == 0 ? '=' : '>', - ment->level, le64_to_cpu(ment->seq), SK_ARG(&first)); -#endif - return cmp; -} - -static int manifest_ring_compare_data(void *a, void *b) -{ - struct manifest_search_key skey; - struct scoutfs_manifest_entry *ment = a; - struct scoutfs_key_buf key; - - init_ment_keys(ment, &key, NULL); - - skey.seq = le64_to_cpu(ment->seq); - skey.key = &key; - skey.level = ment->level; - - return manifest_ring_compare_key(&skey, b); -} - int scoutfs_manifest_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct manifest *mani; - int ret; int i; mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); @@ -965,14 +917,6 @@ int scoutfs_manifest_setup(struct super_block *sb) return -ENOMEM; init_rwsem(&mani->rwsem); - scoutfs_ring_init(&mani->ring, &super->manifest.ring, - manifest_ring_compare_key, - manifest_ring_compare_data); - ret = scoutfs_ring_load(sb, &mani->ring); - if (ret) { - kfree(mani); - return ret; - } for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) { mani->compact_keys[i] = scoutfs_key_alloc(sb, @@ -980,7 +924,6 @@ int scoutfs_manifest_setup(struct super_block *sb) if (!mani->compact_keys[i]) { while (--i >= 0) scoutfs_key_free(sb, mani->compact_keys[i]); - scoutfs_ring_destroy(&mani->ring); kfree(mani); return -ENOMEM; } @@ -1015,7 +958,6 @@ void scoutfs_manifest_destroy(struct super_block *sb) int i; if (mani) { - scoutfs_ring_destroy(&mani->ring); for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) scoutfs_key_free(sb, mani->compact_keys[i]); kfree(mani); diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 46aef7d1..39e0e134 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -1,47 +1,39 @@ #ifndef _SCOUTFS_MANIFEST_H_ #define _SCOUTFS_MANIFEST_H_ -struct scoutfs_key_buf; +#include "key.h" + struct scoutfs_bio_completion; +/* + * This native manifest entry references the physical storage of a + * manifest entry which can exist in a segment header and its edge keys, + * a network transmission of a packed entry and its keys, or in btree + * blocks spread between an item key and value. + */ +struct scoutfs_manifest_entry { + u8 level; + u64 segno; + u64 seq; + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; +}; + +void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, + u64 level, u64 segno, u64 seq, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last); int scoutfs_manifest_add(struct super_block *sb, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level); -int scoutfs_manifest_add_ment(struct super_block *sb, - struct scoutfs_manifest_entry *add); -int scoutfs_manifest_dirty(struct super_block *sb, - struct scoutfs_key_buf *first, u64 seq, u8 level); -int scoutfs_manifest_del(struct super_block *sb, struct scoutfs_key_buf *first, - u64 seq, u8 level); -int scoutfs_manifest_has_dirty(struct super_block *sb); -int scoutfs_manifest_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp); -void scoutfs_manifest_write_complete(struct super_block *sb); - -int scoutfs_manifest_bytes(struct scoutfs_manifest_entry *ment); - -struct scoutfs_manifest_entry * -scoutfs_manifest_alloc_entry(struct super_block *sb, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, u64 segno, u64 seq, - u8 level); + struct scoutfs_manifest_entry *ment); +int scoutfs_manifest_del(struct super_block *sb, + struct scoutfs_manifest_entry *ment); int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); -struct scoutfs_manifest_entry ** -scoutfs_manifest_find_range_entries(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - unsigned *found_bytes); - int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); -int scoutfs_manifest_add_ment_ref(struct super_block *sb, - struct list_head *list, - struct scoutfs_manifest_entry *ment); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); diff --git a/kmod/src/net.c b/kmod/src/net.c index d971bb36..09a4d8b1 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -25,6 +25,7 @@ #include "net.h" #include "counters.h" #include "inode.h" +#include "btree.h" #include "manifest.h" #include "bio.h" #include "alloc.h" @@ -331,8 +332,8 @@ static void scoutfs_net_ring_commit_func(struct work_struct *work) down_write(&nti->ring_commit_rwsem); - if (scoutfs_manifest_has_dirty(sb) || scoutfs_alloc_has_dirty(sb)) { - ret = scoutfs_manifest_submit_write(sb, &comp) ?: + if (scoutfs_btree_has_dirty(sb)) { + ret = scoutfs_btree_write_dirty(sb) ?: scoutfs_alloc_submit_write(sb, &comp) ?: scoutfs_bio_wait_comp(sb, &comp) ?: scoutfs_write_dirty_super(sb); @@ -340,7 +341,7 @@ static void scoutfs_net_ring_commit_func(struct work_struct *work) /* we'd need to loop or something */ BUG_ON(ret); - scoutfs_manifest_write_complete(sb); + scoutfs_btree_write_complete(sb); scoutfs_alloc_write_complete(sb); scoutfs_advance_dirty_super(sb); @@ -425,6 +426,69 @@ static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, return sbuf; } +static void init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) +{ + scoutfs_key_init(first, net_ment->keys, + le16_to_cpu(net_ment->first_key_len)); + scoutfs_key_init(last, net_ment->keys + + le16_to_cpu(net_ment->first_key_len), + le16_to_cpu(net_ment->last_key_len)); +} + +/* + * Allocate a contiguous manifest entry for communication over the network. + */ +static struct scoutfs_net_manifest_entry * +alloc_net_ment(struct scoutfs_manifest_entry *ment) +{ + struct scoutfs_net_manifest_entry *net_ment; + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; + + net_ment = kmalloc(offsetof(struct scoutfs_net_manifest_entry, + keys[ment->first.key_len + + ment->last.key_len]), GFP_NOFS); + if (!net_ment) + return NULL; + + net_ment->segno = cpu_to_le64(ment->segno); + net_ment->seq = cpu_to_le64(ment->seq); + net_ment->first_key_len = cpu_to_le16(ment->first.key_len); + net_ment->last_key_len = cpu_to_le16(ment->last.key_len); + net_ment->level = ment->level; + + init_net_ment_keys(net_ment, &first, &last); + scoutfs_key_copy(&first, &ment->first); + scoutfs_key_copy(&last, &ment->last); + + return net_ment; +} + +/* point a native manifest entry at a contiguous net manifest */ +static void init_ment_net_ment(struct scoutfs_manifest_entry *ment, + struct scoutfs_net_manifest_entry *net_ment) +{ + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; + + init_net_ment_keys(net_ment, &first, &last); + scoutfs_key_clone(&ment->first, &first); + scoutfs_key_clone(&ment->last, &last); + + ment->segno = le64_to_cpu(net_ment->segno); + ment->seq = le64_to_cpu(net_ment->seq); + ment->level = net_ment->level; +} + +static unsigned net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment) +{ + return offsetof(struct scoutfs_net_manifest_entry, + keys[le16_to_cpu(net_ment->first_key_len) + + le16_to_cpu(net_ment->last_key_len)]); +} + /* * This is new segments arriving. It needs to wait for level 0 to be * free. It has relatively little visibility into the manifest, though. @@ -443,19 +507,20 @@ static struct send_buf *process_record_segment(struct super_block *sb, void *req, int req_len) { DECLARE_NET_INFO(sb, nti); - struct scoutfs_manifest_entry *ment; + struct scoutfs_manifest_entry ment; + struct scoutfs_net_manifest_entry *net_ment; struct commit_waiter cw; struct send_buf *sbuf; int ret; - if (req_len < sizeof(struct scoutfs_manifest_entry)) { + if (req_len < sizeof(struct scoutfs_net_manifest_entry)) { sbuf = ERR_PTR(-EINVAL); goto out; } - ment = req; + net_ment = req; - if (req_len != scoutfs_manifest_bytes(ment)) { + if (req_len != net_ment_bytes(net_ment)) { sbuf = ERR_PTR(-EINVAL); goto out; } @@ -472,7 +537,9 @@ retry: goto retry; } - ret = scoutfs_manifest_add_ment(sb, ment); + init_ment_net_ment(&ment, net_ment); + + ret = scoutfs_manifest_add(sb, &ment); scoutfs_manifest_unlock(sb); if (ret == 0) @@ -542,73 +609,6 @@ out: return sbuf; } -/* - * Find the manifest entries that intersect with the request's key - * range. We lock the manifest and get pointers to the manifest entries - * that intersect. We then allocate a reply buffer and copy them over. - */ -static struct send_buf *process_manifest_range_entries(struct super_block *sb, - void *req, int req_len) -{ - struct scoutfs_net_key_range *kr = req; - struct scoutfs_net_manifest_entries *ments; - struct scoutfs_manifest_entry **found = NULL; - struct scoutfs_manifest_entry *ment; - struct scoutfs_key_buf start; - struct scoutfs_key_buf end; - struct send_buf *sbuf; - unsigned total; - unsigned bytes; - int i; - - /* XXX this is a write lock and should be a read lock */ - scoutfs_manifest_lock(sb); - - if (req_len < sizeof(struct scoutfs_net_key_range) || - req_len < offsetof(struct scoutfs_net_key_range, - key_bytes[le16_to_cpu(kr->start_len) + - le16_to_cpu(kr->end_len)])) { - sbuf = ERR_PTR(-EINVAL); - goto out; - } - - scoutfs_key_init(&start, kr->key_bytes, le16_to_cpu(kr->start_len)); - scoutfs_key_init(&end, kr->key_bytes + le16_to_cpu(kr->start_len), - le16_to_cpu(kr->end_len)); - - found = scoutfs_manifest_find_range_entries(sb, &start, &end, &total); - if (IS_ERR(found)) { - sbuf = ERR_CAST(found); - goto out; - } - - total += sizeof(struct scoutfs_net_manifest_entries); - - sbuf = alloc_sbuf(total); - if (!sbuf) { - sbuf = ERR_PTR(-ENOMEM); - goto out; - } - - ments = (void *)sbuf->nh->data; - ment = ments->ments; - - for (i = 0; found[i]; i++) { - bytes = scoutfs_manifest_bytes(found[i]); - memcpy(ment, found[i], bytes); - ment = (void *)((char *)ment + bytes); - } - - ments->nr = cpu_to_le16(i); - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - -out: - scoutfs_manifest_unlock(sb); - if (!IS_ERR_OR_NULL(found)) - kfree(found); - return sbuf; -} - /* * XXX should this call into inodes? not sure about the layering here. */ @@ -790,8 +790,6 @@ static proc_func_t type_proc_func(u8 type) { static proc_func_t funcs[] = { [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, - [SCOUTFS_NET_MANIFEST_RANGE_ENTRIES] = - process_manifest_range_entries, [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, @@ -889,7 +887,8 @@ static void destroy_server_state(struct super_block *sb) scoutfs_compact_destroy(sb); scoutfs_alloc_destroy(sb); - scoutfs_manifest_destroy(sb); + /* XXX this drops dirty data on the floor.. has it committed? */ + scoutfs_btree_write_complete(sb); /* XXX these should be persistent and reclaimed during recovery */ list_for_each_entry_safe(ps, tmp, &nti->pending_seqs, head) { @@ -918,6 +917,7 @@ static void scoutfs_net_proc_func(struct work_struct *work) mutex_lock(&nti->mutex); if (!nti->server_loaded) { ret = scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: + scoutfs_btree_prepare_write(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_compact_setup(sb); @@ -1526,22 +1526,24 @@ static int record_segment_reply(struct super_block *sb, void *reply, int ret, int scoutfs_net_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { - struct scoutfs_manifest_entry *ment; + struct scoutfs_net_manifest_entry *net_ment; struct record_segment_args args; + struct scoutfs_manifest_entry ment; int ret; - ment = scoutfs_seg_manifest_entry(sb, seg, level); - if (!ment) { + scoutfs_seg_init_ment(&ment, level, seg); + net_ment = alloc_net_ment(&ment); + if (!net_ment) { ret = -ENOMEM; goto out; } init_completion(&args.comp); - ret = add_send_buf(sb, SCOUTFS_NET_RECORD_SEGMENT, ment, - scoutfs_manifest_bytes(ment), + ret = add_send_buf(sb, SCOUTFS_NET_RECORD_SEGMENT, net_ment, + net_ment_bytes(net_ment), record_segment_reply, &args); - kfree(ment); + kfree(net_ment); if (ret == 0) { wait_for_completion(&args.comp); ret = args.ret; @@ -1592,119 +1594,6 @@ int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno) return ret; } -struct manifest_range_entries_args { - struct list_head *list; - struct completion comp; - int ret; -}; - -/* - * The server has given us entries that intersect with our request's - * key range. Our caller is still blocked waiting for our completion. - * We walk the manifest entries and add native manifest refs to their - * list and wake them. - */ -static int manifest_range_entries_reply(struct super_block *sb, void *reply, - int reply_bytes, void *arg) -{ - struct manifest_range_entries_args *args = arg; - struct scoutfs_net_manifest_entries *ments = reply; - struct scoutfs_manifest_entry *ment; - unsigned bytes; - int ret = 0; - int i; - - if (reply_bytes < 0) { - ret = reply_bytes; - goto out; - } - - reply_bytes -= sizeof(struct scoutfs_net_manifest_entries); - if (reply_bytes < 0) { - ret = -EINVAL; - goto out; - } - - ment = ments->ments; - for (i = 0; i < le16_to_cpu(ments->nr); i++) { - - - if (reply_bytes < sizeof(struct scoutfs_manifest_entry)) { - ret = -EINVAL; - goto out; - } - - bytes = scoutfs_manifest_bytes(ment); - reply_bytes -= bytes; - if (reply_bytes < 0) { - ret = -EINVAL; - goto out; - } - - ret = scoutfs_manifest_add_ment_ref(sb, args->list, ment); - if (ret) - break; - - ment = (void *)((char *)ment + bytes); - } - -out: - args->ret = ret; - complete(&args->comp); /* args can be freed from this point */ - return ret; -} - -/* - * Ask the manifest server for the manifest entries whose key range - * intersects with the callers key range. The reply func will fill the - * caller's list with the reply's entries. - * - * XXX for now this can't be interrupted. The reply func which is off - * in work in a worker thread is blocking to allocate and put things on - * a list in our stack. We'd need better lifetime support to let it - * find out that we've returned and that it should stop processing the - * reply. - */ -int scoutfs_net_manifest_range_entries(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct list_head *list) -{ - struct manifest_range_entries_args args; - struct scoutfs_net_key_range *kr; - struct scoutfs_key_buf start_key; - struct scoutfs_key_buf end_key; - unsigned len; - int ret; - - len = sizeof(struct scoutfs_net_key_range) + - start->key_len + end->key_len; - kr = kmalloc(len, GFP_NOFS); - if (!kr) - return -ENOMEM; - - kr->start_len = cpu_to_le16(start->key_len); - kr->end_len = cpu_to_le16(end->key_len); - - scoutfs_key_init(&start_key, kr->key_bytes, start->key_len); - scoutfs_key_init(&end_key, kr->key_bytes + start->key_len, - end->key_len); - scoutfs_key_copy(&start_key, start); - scoutfs_key_copy(&end_key, end); - - args.list = list; - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_MANIFEST_RANGE_ENTRIES, kr, len, - manifest_range_entries_reply, &args); - kfree(kr); - if (ret) - return ret; - - wait_for_completion(&args.comp); - return args.ret; -} - static int alloc_inodes_reply(struct super_block *sb, void *reply, int ret, void *arg) { diff --git a/kmod/src/net.h b/kmod/src/net.h index ea131144..bcfa34f9 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -5,10 +5,6 @@ struct scoutfs_key_buf; struct scoutfs_segment; int scoutfs_net_alloc_inodes(struct super_block *sb); -int scoutfs_net_manifest_range_entries(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct list_head *list); int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_net_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 657bfe82..26e256f6 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -350,8 +350,7 @@ void *scoutfs_ring_prev(struct scoutfs_ring_info *ring, void *data) /* * Calculate the most blocks we could have to use to store a given number - * of bytes of entries. At worst each block has a header and leaves one - * less than the max manifest entry unused. + * of bytes of entries. */ static unsigned most_blocks(unsigned long bytes) { @@ -359,8 +358,7 @@ static unsigned most_blocks(unsigned long bytes) space = SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_ring_block) - - (sizeof(struct scoutfs_manifest_entry) + - (2 * SCOUTFS_MAX_KEY_SIZE) - 1); + sizeof(struct scoutfs_alloc_region); return DIV_ROUND_UP(bytes, space); } diff --git a/kmod/src/seg.c b/kmod/src/seg.c index fffbbf34..1cda3462 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -673,11 +673,8 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg return true; } -/* - * Add a dirty manifest entry for the given segment at the given level. - */ -int scoutfs_seg_manifest_add(struct super_block *sb, - struct scoutfs_segment *seg, u8 level) +void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, + struct scoutfs_segment *seg) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_key_buf first; @@ -685,38 +682,8 @@ int scoutfs_seg_manifest_add(struct super_block *sb, first_last_keys(seg, &first, &last); - return scoutfs_manifest_add(sb, &first, &last, le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->seq), level); -} - -int scoutfs_seg_manifest_del(struct super_block *sb, - struct scoutfs_segment *seg, u8 level) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_key_buf first; - - first_last_keys(seg, &first, NULL); - - return scoutfs_manifest_del(sb, &first, le64_to_cpu(sblk->seq), level); -} - -/* - * Return an allocated manifest entry that describes the segment, returns - * NULL if it couldn't allocate. - */ -struct scoutfs_manifest_entry * -scoutfs_seg_manifest_entry(struct super_block *sb, - struct scoutfs_segment *seg, u8 level) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - first_last_keys(seg, &first, &last); - - return scoutfs_manifest_alloc_entry(sb, &first, &last, - le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->seq), level); + scoutfs_manifest_init_entry(ment, level, le64_to_cpu(sblk->segno), + le64_to_cpu(sblk->seq), &first, &last); } /* diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 9d1cd4c9..5a2909d4 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -3,6 +3,7 @@ struct scoutfs_bio_completion; struct scoutfs_key_buf; +struct scoutfs_manifest_entry; struct kvec; /* this is only visible for trace events */ @@ -39,19 +40,13 @@ bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_key_buf *key, struct kvec *val, u8 flags, __le32 **links); -int scoutfs_seg_manifest_add(struct super_block *sb, - struct scoutfs_segment *seg, u8 level); -int scoutfs_seg_manifest_del(struct super_block *sb, - struct scoutfs_segment *seg, u8 level); +void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, + struct scoutfs_segment *seg); int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_bio_completion *comp); -struct scoutfs_manifest_entry * -scoutfs_seg_manifest_entry(struct super_block *sb, - struct scoutfs_segment *seg, u8 level); - int scoutfs_seg_setup(struct super_block *sb); void scoutfs_seg_destroy(struct super_block *sb); From ff5a0948339cb1738f78a63710fe427b587191da Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 29 Jun 2017 15:57:44 -0700 Subject: [PATCH 324/920] scoutfs: store allocator regions in btree Convert the segment allocator to store its free region bitmaps in the btree. This is a very straight forward mechanical transformation. We split the allocator region into a big-endian index key and the bitmap value payload. We're careful to operate on aligned copies of the bitmaps so that they're long aligned. We can remove all the funky functions that were needed when writing the ring. All we're left with is a call to apply the pending allocations to dirty btree blocks before writing the btree. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 216 ++++++++++++++++++++-------------------------- kmod/src/alloc.h | 6 +- kmod/src/btree.c | 1 + kmod/src/format.h | 14 +-- kmod/src/net.c | 9 +- kmod/src/ring.c | 3 +- 6 files changed, 105 insertions(+), 144 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 6fef77c5..0a938174 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -17,14 +17,14 @@ #include "super.h" #include "format.h" -#include "ring.h" +#include "btree.h" #include "cmp.h" #include "alloc.h" #include "counters.h" /* - * scoutfs allocates segments by storing regions of a bitmap in ring - * nodes. + * scoutfs allocates segments using regions of an allocation bitmap + * stored in btree items. * * Freed segments are recorded in nodes in an rbtree. The frees can't * satisfy allocation until they're committed to prevent overwriting @@ -40,7 +40,6 @@ struct seg_alloc { struct rw_semaphore rwsem; struct rb_root pending_root; - struct scoutfs_ring_info ring; u64 next_segno; }; @@ -49,7 +48,8 @@ struct seg_alloc { struct pending_region { struct rb_node node; - struct scoutfs_alloc_region reg; + u64 ind; + struct scoutfs_alloc_region_btree_val reg_val; }; static struct pending_region *find_pending(struct rb_root *root, u64 ind) @@ -60,9 +60,9 @@ static struct pending_region *find_pending(struct rb_root *root, u64 ind) while (node) { pend = container_of(node, struct pending_region, node); - if (ind < le64_to_cpu(pend->reg.index)) + if (ind < pend->ind) node = node->rb_left; - else if (ind > le64_to_cpu(pend->reg.index)) + else if (ind > pend->ind) node = node->rb_right; else return pend; @@ -76,15 +76,14 @@ static void insert_pending(struct rb_root *root, struct pending_region *ins) struct rb_node **node = &root->rb_node; struct rb_node *parent = NULL; struct pending_region *pend; - u64 ind = le64_to_cpu(ins->reg.index); while (*node) { parent = *node; pend = container_of(*node, struct pending_region, node); - if (ind < le64_to_cpu(pend->reg.index)) + if (ins->ind < pend->ind) node = &(*node)->rb_left; - else if (ind > le64_to_cpu(pend->reg.index)) + else if (ins->ind > pend->ind) node = &(*node)->rb_right; else BUG(); @@ -94,23 +93,29 @@ static void insert_pending(struct rb_root *root, struct pending_region *ins) rb_insert_color(&ins->node, root); } -static bool empty_region(struct scoutfs_alloc_region *reg) +static int copy_region_item(struct scoutfs_alloc_region_btree_key *reg_key, + struct scoutfs_alloc_region_btree_val *reg_val, + struct scoutfs_btree_item_ref *iref) { - int i; + if (iref->key_len != sizeof(struct scoutfs_alloc_region_btree_key) || + iref->val_len != sizeof(struct scoutfs_alloc_region_btree_val)) + return -EIO; - for (i = 0; i < ARRAY_SIZE(reg->bits); i++) { - if (reg->bits[i]) - return false; - } - - return true; + memcpy(reg_key, iref->key, iref->key_len); + memcpy(reg_val, iref->val, iref->val_len); + return 0; } +/* + * We're careful to copy the bitmaps out to aligned versions so that + * we can use native bitops that require aligned longs. + */ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_alloc_region *reg; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_alloc_region_btree_key reg_key; + struct scoutfs_alloc_region_btree_val __aligned(sizeof(long)) reg_val; + SCOUTFS_BTREE_ITEM_REF(iref); DECLARE_SEG_ALLOC(sb, sal); u64 ind; int ret; @@ -132,36 +137,47 @@ int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; for (;;) { - reg = scoutfs_ring_lookup_next(&sal->ring, &ind); - if (reg == NULL && ind != 0) { + reg_key.index = cpu_to_be64(ind); + ret = scoutfs_btree_next(sb, &super->alloc_root, + ®_key, sizeof(reg_key), &iref); + if (ret == -ENOENT && ind != 0) { ind = 0; nr = 0; continue; } - if (IS_ERR_OR_NULL(reg)) { - if (IS_ERR(reg)) - ret = PTR_ERR(reg); - else + if (ret < 0) { + if (ret == -ENOENT) ret = -ENOSPC; goto out; } - nr = find_next_bit_le(reg->bits, SCOUTFS_ALLOC_REGION_BITS, nr); - if (nr < SCOUTFS_ALLOC_REGION_BITS) + ret = copy_region_item(®_key, ®_val, &iref); + scoutfs_btree_put_iref(&iref); + if (ret) + goto out; + + ind = be64_to_cpu(reg_key.index); + nr = find_next_bit_le(reg_val.bits, SCOUTFS_ALLOC_REGION_BITS, nr); + if (nr < SCOUTFS_ALLOC_REGION_BITS) { break; + } /* possible for nr to be after all free bits, keep going */ ind++; nr = 0; } - scoutfs_ring_dirty(&sal->ring, reg); + clear_bit_le(nr, reg_val.bits); - ind = le64_to_cpu(reg->index); - - clear_bit_le(nr, reg->bits); - if (empty_region(reg)) - scoutfs_ring_delete(&sal->ring, reg); + if (bitmap_empty((long *)reg_val.bits, SCOUTFS_ALLOC_REGION_BITS)) + ret = scoutfs_btree_delete(sb, &super->alloc_root, + ®_key, sizeof(reg_key)); + else + ret = scoutfs_btree_update(sb, &super->alloc_root, + ®_key, sizeof(reg_key), + ®_val, sizeof(reg_val)); + if (ret) + goto out; *segno = (ind << SCOUTFS_ALLOC_REGION_SHIFT) + nr; sal->next_segno = *segno + 1; @@ -180,12 +196,11 @@ out: /* * Record newly freed sgements in pending regions. These are applied to - * ring nodes as the transaction commits. + * persistent regions in btree items as the transaction commits. */ int scoutfs_alloc_free(struct super_block *sb, u64 segno) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct pending_region *pend; DECLARE_SEG_ALLOC(sb, sal); u64 ind; @@ -205,11 +220,11 @@ int scoutfs_alloc_free(struct super_block *sb, u64 segno) goto out; } - pend->reg.index = cpu_to_le64(ind); + pend->ind = ind; insert_pending(&sal->pending_root, pend); } - set_bit_le(nr, pend->reg.bits); + set_bit_le(nr, pend->reg_val.bits); scoutfs_inc_counter(sb, alloc_free); le64_add_cpu(&super->free_segs, 1); ret = 0; @@ -221,91 +236,73 @@ out: return ret; } -static void or_region_bits(struct scoutfs_alloc_region *dst, - struct scoutfs_alloc_region *src) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(dst->bits); i++) - dst->bits[i] |= src->bits[i]; -} - -int scoutfs_alloc_has_dirty(struct super_block *sb) -{ - DECLARE_SEG_ALLOC(sb, sal); - int ret; - - down_write(&sal->rwsem); - ret = !!(scoutfs_ring_has_dirty(&sal->ring) || - !RB_EMPTY_ROOT(&sal->pending_root)); - up_write(&sal->rwsem); - - return ret; -} - /* - * First we apply the pending frees to create the final set of dirty - * region nodes and then ask the ring to write them to the ring. + * Apply the pending frees to create the final set of dirty btree + * blocks. The caller will write the btree blocks. We're destroying + * the pending free record here so from this point on the pending free + * blocks could be visible to allocation. The caller can't finish with + * the transaction until the btree is written successfully. */ -int scoutfs_alloc_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp) +int scoutfs_alloc_apply_pending(struct super_block *sb) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SEG_ALLOC(sb, sal); - struct scoutfs_alloc_region *reg; struct pending_region *pend; struct rb_node *node; - u64 ind; + struct scoutfs_alloc_region_btree_key reg_key; + struct scoutfs_alloc_region_btree_val __aligned(sizeof(long)) reg_val; + SCOUTFS_BTREE_ITEM_REF(iref); int ret; down_write(&sal->rwsem); + ret = 0; while ((node = rb_first(&sal->pending_root))) { pend = container_of(node, struct pending_region, node); - ind = le64_to_cpu(pend->reg.index); + /* see if we have a region for this index */ + reg_key.index = cpu_to_be64(pend->ind); + ret = scoutfs_btree_lookup(sb, &super->alloc_root, + ®_key, sizeof(reg_key), &iref); + if (ret == -ENOENT) { + /* create a new item if we don't */ + ret = scoutfs_btree_insert(sb, &super->alloc_root, + ®_key, sizeof(reg_key), + &pend->reg_val, + sizeof(pend->reg_val)); + } else if (ret == 0) { + /* and update the existing item if we do */ + ret = copy_region_item(®_key, ®_val, &iref); + scoutfs_btree_put_iref(&iref); + if (ret) + break; - reg = scoutfs_ring_lookup(&sal->ring, &ind); - if (!reg) { - reg = scoutfs_ring_insert(&sal->ring, &ind, - sizeof(struct scoutfs_alloc_region)); - if (!reg) { - ret = -ENOMEM; - goto out; - } + bitmap_or((long *)reg_val.bits, (long *)reg_val.bits, + (long *)pend->reg_val.bits, + SCOUTFS_ALLOC_REGION_BITS); - memset(reg, 0, sizeof(struct scoutfs_alloc_region)); - reg->index = cpu_to_le64(ind); + ret = scoutfs_btree_update(sb, &super->alloc_root, + ®_key, sizeof(reg_key), + ®_val, sizeof(reg_val)); } - - or_region_bits(reg, &pend->reg); - scoutfs_ring_dirty(&sal->ring, reg); + if (ret < 0) + break; rb_erase(&pend->node, &sal->pending_root); kfree(pend); } - ret = scoutfs_ring_submit_write(sb, &sal->ring, comp); -out: up_write(&sal->rwsem); + return ret; } -void scoutfs_alloc_write_complete(struct super_block *sb) -{ - DECLARE_SEG_ALLOC(sb, sal); - - down_write(&sal->rwsem); - scoutfs_ring_write_complete(&sal->ring); - up_write(&sal->rwsem); -} - /* * Return the number of blocks free for statfs. */ u64 scoutfs_alloc_bfree(struct super_block *sb) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SEG_ALLOC(sb, sal); u64 bfree; @@ -316,31 +313,13 @@ u64 scoutfs_alloc_bfree(struct super_block *sb) return bfree; } -static int alloc_ring_compare_key(void *key, void *data) -{ - u64 *ind = key; - struct scoutfs_alloc_region *reg = data; - - return scoutfs_cmp_u64s(*ind, le64_to_cpu(reg->index)); -} - -static int alloc_ring_compare_data(void *A, void *B) -{ - struct scoutfs_alloc_region *a = A; - struct scoutfs_alloc_region *b = B; - - return scoutfs_cmp_u64s(le64_to_cpu(a->index), le64_to_cpu(b->index)); -} - int scoutfs_alloc_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; struct seg_alloc *sal; - int ret; /* bits need to be aligned so hosts can use native bitops */ - BUILD_BUG_ON(offsetof(struct scoutfs_alloc_region, bits) & + BUILD_BUG_ON(offsetof(struct scoutfs_alloc_region_btree_val, bits) & (sizeof(long) - 1)); sal = kzalloc(sizeof(struct seg_alloc), GFP_KERNEL); @@ -349,14 +328,6 @@ int scoutfs_alloc_setup(struct super_block *sb) init_rwsem(&sal->rwsem); sal->pending_root = RB_ROOT; - scoutfs_ring_init(&sal->ring, &super->alloc_ring, - alloc_ring_compare_key, alloc_ring_compare_data); - - ret = scoutfs_ring_load(sb, &sal->ring); - if (ret) { - kfree(sal); - return ret; - } /* XXX read next_segno from super? */ @@ -373,7 +344,6 @@ void scoutfs_alloc_destroy(struct super_block *sb) struct rb_node *node; if (sal) { - scoutfs_ring_destroy(&sal->ring); while ((node = rb_first(&sal->pending_root))) { pend = container_of(node, struct pending_region, node); rb_erase(&pend->node, &sal->pending_root); diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index bb185d90..d6c8a5b0 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -2,15 +2,11 @@ #define _SCOUTFS_ALLOC_H_ struct scoutfs_alloc_region; -struct scoutfs_bio_completion; int scoutfs_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_alloc_free(struct super_block *sb, u64 segno); -int scoutfs_alloc_has_dirty(struct super_block *sb); -int scoutfs_alloc_submit_write(struct super_block *sb, - struct scoutfs_bio_completion *comp); -void scoutfs_alloc_write_complete(struct super_block *sb); +int scoutfs_alloc_apply_pending(struct super_block *sb); u64 scoutfs_alloc_bfree(struct super_block *sb); int scoutfs_alloc_setup(struct super_block *sb); diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 55a2a0a8..e6ad7f71 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1769,6 +1769,7 @@ int scoutfs_btree_write_dirty(struct super_block *sb) struct scoutfs_btree_ring *bring = &super->bring; struct scoutfs_btree_root *roots[] = { &super->manifest.root, + &super->alloc_root, NULL, }; struct scoutfs_btree_root *root; diff --git a/kmod/src/format.h b/kmod/src/format.h index 7ca2f2e3..0f5b8638 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -208,12 +208,12 @@ struct scoutfs_manifest_btree_val { #define SCOUTFS_ALLOC_REGION_BITS (1 << SCOUTFS_ALLOC_REGION_SHIFT) #define SCOUTFS_ALLOC_REGION_MASK (SCOUTFS_ALLOC_REGION_BITS - 1) -/* - * The bits need to be aligned so that the host can use native long - * bitops on the bits in memory. - */ -struct scoutfs_alloc_region { - __le64 index; +struct scoutfs_alloc_region_btree_key { + __be64 index; +} __packed; + +/* The bits need to be aligned so that the hosts can use native long bit ops */ +struct scoutfs_alloc_region_btree_val { __le64 bits[SCOUTFS_ALLOC_REGION_BITS / 64]; } __packed; @@ -421,7 +421,7 @@ struct scoutfs_super_block { __le64 ring_gen; struct scoutfs_btree_ring bring; __le64 next_seg_seq; - struct scoutfs_ring_descriptor alloc_ring; + struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; } __packed; diff --git a/kmod/src/net.c b/kmod/src/net.c index 09a4d8b1..53a5e6f3 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -322,27 +322,22 @@ static void scoutfs_net_ring_commit_func(struct work_struct *work) struct net_info *nti = container_of(work, struct net_info, ring_commit_work); struct super_block *sb = nti->sb; - struct scoutfs_bio_completion comp; struct commit_waiter *cw; struct commit_waiter *pos; struct llist_node *node; int ret; - scoutfs_bio_init_comp(&comp); - down_write(&nti->ring_commit_rwsem); if (scoutfs_btree_has_dirty(sb)) { - ret = scoutfs_btree_write_dirty(sb) ?: - scoutfs_alloc_submit_write(sb, &comp) ?: - scoutfs_bio_wait_comp(sb, &comp) ?: + ret = scoutfs_alloc_apply_pending(sb) ?: + scoutfs_btree_write_dirty(sb) ?: scoutfs_write_dirty_super(sb); /* we'd need to loop or something */ BUG_ON(ret); scoutfs_btree_write_complete(sb); - scoutfs_alloc_write_complete(sb); scoutfs_advance_dirty_super(sb); } else { diff --git a/kmod/src/ring.c b/kmod/src/ring.c index 26e256f6..1e5f8e78 100644 --- a/kmod/src/ring.c +++ b/kmod/src/ring.c @@ -357,8 +357,7 @@ static unsigned most_blocks(unsigned long bytes) unsigned long space; space = SCOUTFS_BLOCK_SIZE - - sizeof(struct scoutfs_ring_block) - - sizeof(struct scoutfs_alloc_region); + sizeof(struct scoutfs_ring_block); return DIV_ROUND_UP(bytes, space); } From c2f13ccf2415d6c16cfdf7872608a445e0ea3c26 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 30 Jun 2017 13:18:33 -0700 Subject: [PATCH 325/920] scoutfs: have net.c commit btree blocks Convert the net server metadata dirtying and committing code to use the btree instead of the ring. It has to be careful to setup and teardown the btree info as it starts up and shuts down the server. This fixes up some questionable setup/teardown changes made in the previous patches to convert the manifest and allocator. We could rebase the patches to merge those together. But given that the previous patches don't work at all without the net updates it might not be worth the trouble. Signed-off-by: Zach Brown --- kmod/src/net.c | 83 +++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 53a5e6f3..932e599d 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -80,10 +80,10 @@ struct net_info { struct sock_info *listening_sinf; bool server_loaded; - /* server commits ring changes while processing requests */ - struct rw_semaphore ring_commit_rwsem; - struct llist_head ring_commit_waiters; - struct work_struct ring_commit_work; + /* server commits metadata while processing requests */ + struct rw_semaphore commit_rwsem; + struct llist_head commit_waiters; + struct work_struct commit_work; /* level 0 segment addition waits for it to clear */ wait_queue_head_t waitq; @@ -283,12 +283,12 @@ struct commit_waiter { */ static void queue_commit_work(struct net_info *nti, struct commit_waiter *cw) { - lockdep_assert_held(&nti->ring_commit_rwsem); + lockdep_assert_held(&nti->commit_rwsem); cw->ret = 0; init_completion(&cw->comp); - llist_add(&cw->node, &nti->ring_commit_waiters); - queue_work(nti->proc_wq, &nti->ring_commit_work); + llist_add(&cw->node, &nti->commit_waiters); + queue_work(nti->proc_wq, &nti->commit_work); } static int wait_for_commit(struct commit_waiter *cw) @@ -306,9 +306,9 @@ static int wait_for_commit(struct commit_waiter *cw) * * Getting that batching right is bound up in the concurrency of request * processing so a clear way to implement the batched commits is to - * implement commits with work funcs like the processing. This ring - * commit work is queued on the non-reentrant proc_wq so there will only - * ever be one commit executing at a time. + * implement commits with work funcs like the processing. This commit + * work is queued on the non-reentrant proc_wq so there will only ever + * be one commit executing at a time. * * Processing paths acquire the rwsem for reading while they're making * multiple dependent changes. When they're done and want it persistent @@ -317,17 +317,16 @@ static int wait_for_commit(struct commit_waiter *cw) * performs the commit. Readers can run concurrently with these * commits. */ -static void scoutfs_net_ring_commit_func(struct work_struct *work) +static void scoutfs_net_commit_func(struct work_struct *work) { - struct net_info *nti = container_of(work, struct net_info, - ring_commit_work); + struct net_info *nti = container_of(work, struct net_info, commit_work); struct super_block *sb = nti->sb; struct commit_waiter *cw; struct commit_waiter *pos; struct llist_node *node; int ret; - down_write(&nti->ring_commit_rwsem); + down_write(&nti->commit_rwsem); if (scoutfs_btree_has_dirty(sb)) { ret = scoutfs_alloc_apply_pending(sb) ?: @@ -344,7 +343,7 @@ static void scoutfs_net_ring_commit_func(struct work_struct *work) ret = 0; } - node = llist_del_all(&nti->ring_commit_waiters); + node = llist_del_all(&nti->commit_waiters); /* waiters always wait on completion, cw could be free after complete */ llist_for_each_entry_safe(cw, pos, node, node) { @@ -352,7 +351,7 @@ static void scoutfs_net_ring_commit_func(struct work_struct *work) complete(&cw->comp); } - up_write(&nti->ring_commit_rwsem); + up_write(&nti->commit_rwsem); } static struct send_buf *alloc_sbuf(unsigned data_len) @@ -391,7 +390,7 @@ static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, ns = (void *)sbuf->nh->data; ns->nr = cpu_to_le16(SCOUTFS_BULK_ALLOC_COUNT); - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); for (i = 0; i < SCOUTFS_BULK_ALLOC_COUNT; i++) { ret = scoutfs_alloc_segno(sb, &segno); @@ -408,7 +407,7 @@ static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, if (ret == 0) queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -521,12 +520,12 @@ static struct send_buf *process_record_segment(struct super_block *sb, } retry: - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); scoutfs_manifest_lock(sb); if (scoutfs_manifest_level0_full(sb)) { scoutfs_manifest_unlock(sb); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); /* XXX waits indefinitely? io errors? */ wait_event(nti->waitq, !scoutfs_manifest_level0_full(sb)); goto retry; @@ -539,7 +538,7 @@ retry: if (ret == 0) queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -575,13 +574,13 @@ static struct send_buf *process_alloc_segno(struct super_block *sb, goto out; } - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); ret = scoutfs_alloc_segno(sb, &segno); if (ret == 0) queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -627,7 +626,7 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, if (!sbuf) return ERR_PTR(-ENOMEM); - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); spin_lock(&sbi->next_ino_lock); ino = le64_to_cpu(super->next_ino); @@ -636,7 +635,7 @@ static struct send_buf *process_alloc_inodes(struct super_block *sb, spin_unlock(&sbi->next_ino_lock); queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); ret = wait_for_commit(&cw); @@ -700,7 +699,7 @@ static struct send_buf *process_advance_seq(struct super_block *sb, goto out; } - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); spin_lock(&nti->seq_lock); @@ -725,7 +724,7 @@ static struct send_buf *process_advance_seq(struct super_block *sb, spin_unlock(&nti->seq_lock); queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); ret = wait_for_commit(&cw); out: @@ -882,8 +881,8 @@ static void destroy_server_state(struct super_block *sb) scoutfs_compact_destroy(sb); scoutfs_alloc_destroy(sb); - /* XXX this drops dirty data on the floor.. has it committed? */ - scoutfs_btree_write_complete(sb); + scoutfs_manifest_destroy(sb); + scoutfs_btree_destroy(sb); /* XXX these should be persistent and reclaimed during recovery */ list_for_each_entry_safe(ps, tmp, &nti->pending_seqs, head) { @@ -912,7 +911,7 @@ static void scoutfs_net_proc_func(struct work_struct *work) mutex_lock(&nti->mutex); if (!nti->server_loaded) { ret = scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: - scoutfs_btree_prepare_write(sb) ?: + scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_alloc_setup(sb) ?: scoutfs_compact_setup(sb); @@ -1229,7 +1228,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) if (sinf == nti->listening_sinf) { nti->listening_sinf = NULL; - /* shutdown the server, processing won't leave rings dirty */ + /* shutdown the server, processing won't leave dirty metadata */ destroy_server_state(sb); nti->server_loaded = false; @@ -1418,9 +1417,9 @@ u64 *scoutfs_net_bulk_alloc(struct super_block *sb) * * This is a short circuit that's called directly by a work function * that's only queued on the server. It makes compaction work inside - * the ring update consistency mechanics inside net message processing - * and demonstrates the moving pieces that we'd need to cut up into a - * series of messages and replies. + * the commit consistency mechanics inside net message processing and + * demonstrates the moving pieces that we'd need to cut up into a series + * of messages and replies. * * The compaction work caller cleans up everything on errors. */ @@ -1433,11 +1432,11 @@ int scoutfs_net_get_compaction(struct super_block *sb, void *curs) int nr; int i; - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); nr = scoutfs_manifest_next_compact(sb, curs); if (nr <= 0) { - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); return nr; } @@ -1451,7 +1450,7 @@ int scoutfs_net_get_compaction(struct super_block *sb, void *curs) if (ret == 0) queue_commit_work(nti, &cw); - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -1479,7 +1478,7 @@ int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, bool level0_was_full; int ret; - down_read(&nti->ring_commit_rwsem); + down_read(&nti->commit_rwsem); level0_was_full = scoutfs_manifest_level0_full(sb); @@ -1490,7 +1489,7 @@ int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, wake_up(&nti->waitq); } - up_read(&nti->ring_commit_rwsem); + up_read(&nti->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -2066,9 +2065,9 @@ int scoutfs_net_setup(struct super_block *sb) INIT_LIST_HEAD(&nti->to_send); nti->next_id = 1; INIT_DELAYED_WORK(&nti->server_work, scoutfs_net_server_func); - init_rwsem(&nti->ring_commit_rwsem); - init_llist_head(&nti->ring_commit_waiters); - INIT_WORK(&nti->ring_commit_work, scoutfs_net_ring_commit_func); + init_rwsem(&nti->commit_rwsem); + init_llist_head(&nti->commit_waiters); + INIT_WORK(&nti->commit_work, scoutfs_net_commit_func); init_waitqueue_head(&nti->waitq); spin_lock_init(&nti->seq_lock); INIT_LIST_HEAD(&nti->pending_seqs); From 412e7a7e3bf09f24e26b35706d19ef15653d865a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 30 Jun 2017 13:24:43 -0700 Subject: [PATCH 326/920] scoutfs: remove unused ring log storage Remove the old unused ring now all of its previous callers now use the btree. Signed-off-by: Zach Brown --- kmod/src/Makefile | 3 +- kmod/src/format.h | 37 +-- kmod/src/ring.c | 817 ---------------------------------------------- kmod/src/ring.h | 55 ---- 4 files changed, 2 insertions(+), 910 deletions(-) delete mode 100644 kmod/src/ring.c delete mode 100644 kmod/src/ring.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 8eefa971..6f0d66d8 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -4,5 +4,4 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o compact.o counters.o data.o dir.o kvec.o \ inode.o ioctl.o item.o key.o lock.o manifest.o msg.o net.o \ - options.o ring.o seg.o scoutfs_trace.o sort_priv.o super.o trans.o \ - xattr.o + options.o seg.o scoutfs_trace.o sort_priv.o super.o trans.o xattr.o diff --git a/kmod/src/format.h b/kmod/src/format.h index 0f5b8638..1ccac901 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -7,7 +7,7 @@ #define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ /* - * The super block and ring blocks are fixed 4k. + * The super block and btree blocks are fixed 4k. */ #define SCOUTFS_BLOCK_SHIFT 12 #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) @@ -50,32 +50,6 @@ struct scoutfs_block_header { __le64 blkno; } __packed; -struct scoutfs_ring_entry { - __le16 data_len; - __u8 flags; - __u8 data[0]; -} __packed; - -#define SCOUTFS_RING_ENTRY_FLAG_DELETION (1 << 0) - -struct scoutfs_ring_block { - __le32 crc; - __le32 pad; - __le64 fsid; - __le64 seq; - __le64 block; - __le32 nr_entries; - struct scoutfs_ring_entry entries[0]; -} __packed; - -struct scoutfs_ring_descriptor { - __le64 blkno; - __le64 total_blocks; - __le64 first_block; - __le64 first_seq; - __le64 nr_blocks; -} __packed; - /* * Assert that we'll be able to represent all possible keys with 8 64bit * primary sort values. @@ -401,11 +375,6 @@ struct scoutfs_inet_addr { #define SCOUTFS_DEFAULT_PORT 12345 -/* - * The ring fields describe the statically allocated ring log. The - * head and tail indexes are logical 4k blocks offsets inside the ring. - * The head block should contain the seq. - */ struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -415,10 +384,6 @@ struct scoutfs_super_block { __le64 alloc_uninit; __le64 total_segs; __le64 free_segs; - __le64 ring_blkno; - __le64 ring_blocks; - __le64 ring_tail_block; - __le64 ring_gen; struct scoutfs_btree_ring bring; __le64 next_seg_seq; struct scoutfs_btree_root alloc_root; diff --git a/kmod/src/ring.c b/kmod/src/ring.c deleted file mode 100644 index 1e5f8e78..00000000 --- a/kmod/src/ring.c +++ /dev/null @@ -1,817 +0,0 @@ -/* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "bio.h" -#include "ring.h" - -/* - * scoutfs stores the persistent indexes for the server in a simple log - * entries in a preallocated ring of blocks. - * - * The index is read from the log and loaded in to an rbtree in memory. - * Callers then lock around operations that work on the rbtrees. Dirty - * and deleted nodes are tracked and are eventually copied to pages that - * are written to the tail of the log. - * - * This has the great benefit of updating an index with very few (often - * one) contiguous block writes with low write amplification. - * - * This has the significant cost of requiring reading the indexes in to - * memory before doing any work and then having to hold them resident. - * This is fine for now but we'll have to address these latency and - * capacity limitations before too long. - * - * Callers are entirely responsible for locking. - */ - -/* - * XXX - * - deletion entries could be smaller if we understood keys - * - shouldn't be too hard to compress - */ - -/* - * @block records the logical ring index of the block that contained the - * node. As we commit a ring update we can look at the clean list to - * find the first block that we have to read out of the ring. This - * helps minimize the active region of the ring. - * - * @in_ring is used to mark nodes that were present in the ring and - * which need deletion entries written to the ring before they can be - * freed. - */ -struct ring_node { - struct rb_node rb_node; - struct list_head head; - u64 block; - - u16 data_len; - - u8 dirty:1, - deleted:1, - in_ring:1; - - /* data is packed but callers perform native long bitops */ - u8 data[0] __aligned(__alignof__(long)); -}; - -static struct ring_node *data_rnode(void *data) -{ - return data ? container_of(data, struct ring_node, data) : NULL; -} - -static void *rnode_data(struct ring_node *rnode) -{ - return rnode ? rnode->data : NULL; -} - -static unsigned total_entry_bytes(unsigned data_len) -{ - return offsetof(struct scoutfs_ring_entry, data[data_len]); -} - -/* - * Each time we mark a node dirty we also dirty the oldest clean entry. - * This ensures that we never overwrite stable data. - * - * Picture a ring of blocks where the first half of the ring is full of - * existing entries. Imagine that we continuously update a set of - * entries that make up a single block. Each new update block - * invalidates the previous update block but it advances through the - * ring while the old entries are sitting idle in the first half. - * Eventually the new update blocks wrap around and clobber the old - * blocks. - * - * Now instead imagine that each time we dirty an entry in this set of - * constantly changing entries that we also go and dirty the earliest - * existing entry in the ring. Now each update is a block of the - * useless updating entries and a block of old entries that have been - * migrated. Each time we write two blocks to the ring we migrate one - * block from the start of the ring. Now by the time we fill the second - * half of the ring we've reclaimed half of the first half of the ring. - * - * So we size the ring to fit 4x the largest possible index. Now we're - * sure that we'll be able to fully migrate the index from the first - * half of the ring into the second half before it wraps around and - * starts overwriting the first. - */ -static void mark_node_dirty(struct scoutfs_ring_info *ring, - struct ring_node *rnode, bool migrate) -{ - struct ring_node *pos; - long total; - - if (!rnode || rnode->dirty) - return; - - list_move_tail(&rnode->head, &ring->dirty_list); - rnode->dirty = 1; - ring->dirty_bytes += total_entry_bytes(rnode->data_len); - - if (migrate) { - total = total_entry_bytes(rnode->data_len); - - list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { - mark_node_dirty(ring, rnode, false); - total -= total_entry_bytes(rnode->data_len); - if (total < 0) - break; - } - } -} - -static void mark_node_clean(struct scoutfs_ring_info *ring, - struct ring_node *rnode) -{ - if (!rnode || !rnode->dirty) - return; - - list_move_tail(&rnode->head, &ring->clean_list); - rnode->dirty = 0; - ring->dirty_bytes -= total_entry_bytes(rnode->data_len); -} - -static void free_node(struct scoutfs_ring_info *ring, - struct ring_node *rnode) -{ - if (rnode) { - mark_node_clean(ring, rnode); - - if (!list_empty(&rnode->head)) - list_del_init(&rnode->head); - if (!RB_EMPTY_NODE(&rnode->rb_node)) - rb_erase(&rnode->rb_node, &ring->rb_root); - - kfree(rnode); - } -} - -/* - * Walk the tree and return the last node traversed. cmp gives the - * caller the comparison between their key and the returned node. The - * caller can provide either their key or another nodes data to compare - * with during descent. If we're asked to insert we replace any node we - * find in the key's place. - */ -static struct ring_node *ring_rb_walk(struct scoutfs_ring_info *ring, - void *key, void *data, - struct ring_node *ins, - int *cmp) -{ - struct rb_node **node = &ring->rb_root.rb_node; - struct rb_node *parent = NULL; - struct ring_node *found = NULL; - struct ring_node *rnode = NULL; - - /* only provide one or the other */ - BUG_ON(!!key == !!data); - - while (*node) { - parent = *node; - rnode = container_of(*node, struct ring_node, rb_node); - - if (key) - *cmp = ring->compare_key(key, &rnode->data); - else - *cmp = ring->compare_data(data, &rnode->data); - - if (*cmp < 0) { - node = &(*node)->rb_left; - } else if (*cmp > 0) { - node = &(*node)->rb_right; - } else { - found = rnode; - break; - } - } - - if (ins) { - if (found) { - rb_replace_node(&found->rb_node, &ins->rb_node, - &ring->rb_root); - RB_CLEAR_NODE(&found->rb_node); - free_node(ring, found); - } else { - rb_link_node(&ins->rb_node, parent, node); - rb_insert_color(&ins->rb_node, &ring->rb_root); - } - found = ins; - *cmp = 0; - } - - return rnode; -} - -static struct ring_node *ring_rb_entry(struct rb_node *node) -{ - return node ? rb_entry(node, struct ring_node, rb_node) : NULL; -} - -/* return the next node, skipping deleted */ -static struct ring_node *ring_rb_next(struct ring_node *rnode) -{ - do { - if (rnode) - rnode = ring_rb_entry(rb_next(&rnode->rb_node)); - } while (rnode && rnode->deleted); - - return rnode; -} - -/* return the prev node, skipping deleted */ -static struct ring_node *ring_rb_prev(struct ring_node *rnode) -{ - do { - if (rnode) - rnode = ring_rb_entry(rb_prev(&rnode->rb_node)); - } while (rnode && rnode->deleted); - - return rnode; -} - -/* return the first node, skipping deleted */ -static struct ring_node *ring_rb_first(struct scoutfs_ring_info *ring) -{ - struct ring_node *rnode; - - rnode = ring_rb_entry(rb_first(&ring->rb_root)); - if (rnode && rnode->deleted) - rnode = ring_rb_next(rnode); - return rnode; -} - -static struct ring_node *alloc_node(unsigned data_len) -{ - struct ring_node *rnode; - - rnode = kzalloc(offsetof(struct ring_node, data[data_len]), GFP_NOFS); - if (rnode) { - RB_CLEAR_NODE(&rnode->rb_node); - INIT_LIST_HEAD(&rnode->head); - rnode->data_len = data_len; - } - - return rnode; -} - -/* - * Insert a new node. This will replace any existing node which could - * be in any state. - */ -void *scoutfs_ring_insert(struct scoutfs_ring_info *ring, void *key, - unsigned data_len) -{ - struct ring_node *rnode; - int cmp; - - rnode = alloc_node(data_len); - if (!rnode) - return NULL; - - ring_rb_walk(ring, key, NULL, rnode, &cmp); - /* just put it on a list, dirtying moves it to dirty */ - list_add_tail(&rnode->head, &ring->dirty_list); - mark_node_dirty(ring, rnode, true); - - trace_printk("inserted rnode %p in %u deleted %u dirty %u\n", - rnode, rnode->in_ring, rnode->deleted, - rnode->dirty); - - return rnode->data; -} - -void *scoutfs_ring_first(struct scoutfs_ring_info *ring) -{ - return rnode_data(ring_rb_first(ring)); -} - -void *scoutfs_ring_lookup(struct scoutfs_ring_info *ring, void *key) -{ - struct ring_node *rnode; - int cmp; - - rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); - if (rnode && (cmp || rnode->deleted)) - rnode = NULL; - - return rnode_data(rnode); -} - -void *scoutfs_ring_lookup_next(struct scoutfs_ring_info *ring, void *key) -{ - struct ring_node *rnode; - int cmp; - - rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); - if (rnode && (cmp > 0 || rnode->deleted)) - rnode = ring_rb_next(rnode); - - return rnode_data(rnode); -} - -void *scoutfs_ring_lookup_prev(struct scoutfs_ring_info *ring, void *key) -{ - struct ring_node *rnode; - int cmp; - - rnode = ring_rb_walk(ring, key, NULL, NULL, &cmp); - if (rnode && (cmp < 0 || rnode->deleted)) - rnode = ring_rb_prev(rnode); - - return rnode_data(rnode); -} - -void *scoutfs_ring_next(struct scoutfs_ring_info *ring, void *data) -{ - return rnode_data(ring_rb_next(data_rnode(data))); -} - -void *scoutfs_ring_prev(struct scoutfs_ring_info *ring, void *data) -{ - return rnode_data(ring_rb_prev(data_rnode(data))); -} - -/* - * Calculate the most blocks we could have to use to store a given number - * of bytes of entries. - */ -static unsigned most_blocks(unsigned long bytes) -{ - unsigned long space; - - space = SCOUTFS_BLOCK_SIZE - - sizeof(struct scoutfs_ring_block); - - return DIV_ROUND_UP(bytes, space); -} - -static u64 wrap_ring_block(struct scoutfs_ring_descriptor *rdesc, u64 block) -{ - if (block >= le64_to_cpu(rdesc->total_blocks)) - block -= le64_to_cpu(rdesc->total_blocks); - - /* XXX callers should have verified on load */ - BUG_ON(block >= le64_to_cpu(rdesc->total_blocks)); - - return block; -} - -static u64 calc_first_dirty_block(struct scoutfs_ring_descriptor *rdesc) -{ - return wrap_ring_block(rdesc, le64_to_cpu(rdesc->first_block) + - le64_to_cpu(rdesc->nr_blocks)); -} - -static __le32 rblk_crc(struct scoutfs_ring_block *rblk) -{ - unsigned long skip = (char *)(&rblk->crc + 1) - (char *)rblk; - - return cpu_to_le32(crc32c(~0, (char *)rblk + skip, - SCOUTFS_BLOCK_SIZE - skip)); -} - -/* - * This is called after the caller has copied all the dirty nodes into - * blocks in pages for writing. We might be able to dirty a few more - * clean nodes to fill up the end of the last dirty block to keep the - * ring blocks densely populated. - */ -static void fill_last_dirty_block(struct scoutfs_ring_info *ring, - unsigned space) -{ - struct ring_node *rnode; - struct ring_node *pos; - unsigned tot; - - list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { - - tot = total_entry_bytes(rnode->data_len); - if (tot > space) - break; - - mark_node_dirty(ring, rnode, false); - space -= tot; - } -} - -void scoutfs_ring_dirty(struct scoutfs_ring_info *ring, void *data) -{ - struct ring_node *rnode; - - rnode = data_rnode(data); - if (rnode) - mark_node_dirty(ring, rnode, true); -} - -/* - * Delete the given node. This can free the node so the caller cannot - * use the data after calling this. - * - * If the node previously existed in the ring then we have to save it and - * write a deletion entry before freeing it. - */ -void scoutfs_ring_delete(struct scoutfs_ring_info *ring, void *data) -{ - struct ring_node *rnode = data_rnode(data); - - trace_printk("deleting rnode %p in %u deleted %u dirty %u\n", - rnode, rnode->in_ring, rnode->deleted, rnode->dirty); - - BUG_ON(rnode->deleted); - - if (rnode->in_ring) { - rnode->deleted = 1; - mark_node_dirty(ring, rnode, true); - } else { - free_node(ring, rnode); - } -} - -static struct scoutfs_ring_block *block_in_pages(struct page **pages, - unsigned i) -{ - return page_address(pages[i / SCOUTFS_BLOCKS_PER_PAGE]) + - ((i % SCOUTFS_BLOCKS_PER_PAGE) << SCOUTFS_BLOCK_SHIFT); -} - -static int load_ring_block(struct scoutfs_ring_info *ring, - struct scoutfs_ring_block *rblk) -{ - struct scoutfs_ring_entry *rent; - struct ring_node *rnode; - unsigned data_len; - unsigned i; - int ret = 0; - int cmp; - - trace_printk("block %llu\n", le64_to_cpu(rblk->block)); - - rent = rblk->entries; - for (i = 0; i < le32_to_cpu(rblk->nr_entries); i++) { - - /* XXX verify fields? */ - data_len = le16_to_cpu(rent->data_len); - - trace_printk("rent %u data_len %u\n", i, data_len); - - if (rent->flags & SCOUTFS_RING_ENTRY_FLAG_DELETION) { - rnode = ring_rb_walk(ring, NULL, rent->data, NULL, - &cmp); - if (rnode && cmp == 0) - free_node(ring, rnode); - } else { - rnode = alloc_node(data_len); - if (!rnode) { - ret = -ENOMEM; - break; - } - - rnode->block = le64_to_cpu(rblk->block); - rnode->in_ring = 1; - memcpy(rnode->data, rent->data, data_len); - - ring_rb_walk(ring, NULL, rnode->data, rnode, &cmp); - list_add_tail(&rnode->head, &ring->clean_list); - } - - rent = (void *)&rent->data[data_len]; - } - - return ret; -} - -/* - * Read the ring entries into rb nodes with nice large synchronous reads. - */ -#define LOAD_BYTES (4 * 1024 * 1024) -#define LOAD_BLOCKS DIV_ROUND_UP(LOAD_BYTES, SCOUTFS_BLOCK_SIZE) -#define LOAD_PAGES DIV_ROUND_UP(LOAD_BYTES, PAGE_SIZE) -int scoutfs_ring_load(struct super_block *sb, struct scoutfs_ring_info *ring) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_ring_descriptor *rdesc = ring->rdesc; - struct scoutfs_ring_block *rblk; - struct page **pages; - unsigned read_nr; - unsigned i; - __le32 crc; - u64 block; - u64 total; - u64 seq; - u64 nr; - int ret; - - pages = kcalloc(LOAD_PAGES, sizeof(struct page *), GFP_NOFS); - if (!pages) - return -ENOMEM; - - for (i = 0; i < LOAD_PAGES; i++) { - pages[i] = alloc_page(GFP_NOFS); - if (!pages[i]) { - ret = -ENOMEM; - goto out; - } - } - - block = le64_to_cpu(rdesc->first_block); - seq = le64_to_cpu(rdesc->first_seq); - total = le64_to_cpu(rdesc->total_blocks); - nr = le64_to_cpu(rdesc->nr_blocks); - - while (nr) { - read_nr = min3(nr, (u64)LOAD_BLOCKS, total - block); - - ret = scoutfs_bio_read(sb, pages, le64_to_cpu(rdesc->blkno) + - block, read_nr); - if (ret) - goto out; - - for (i = 0; i < read_nr; i++) { - rblk = block_in_pages(pages, i); - crc = rblk_crc(rblk); - - if (rblk->fsid != super->hdr.fsid || - le64_to_cpu(rblk->block) != (block + i) || - le64_to_cpu(rblk->seq) != (seq + i) || - rblk->crc != crc) { - ret = -EIO; - goto out; - } - - ret = load_ring_block(ring, rblk); - if (ret) - goto out; - } - - block = wrap_ring_block(rdesc, block + read_nr); - seq += read_nr; - nr -= read_nr; - } - ret = 0; - -out: - for (i = 0; pages && i < LOAD_PAGES && pages[i]; i++) - __free_page(pages[i]); - kfree(pages); - - if (ret) - scoutfs_ring_destroy(ring); - - return ret; -} - -static struct ring_node *first_dirty_node(struct scoutfs_ring_info *ring) -{ - return list_first_entry_or_null(&ring->dirty_list, struct ring_node, - head); -} - -static struct ring_node *next_dirty_node(struct scoutfs_ring_info *ring, - struct ring_node *rnode) -{ - if (rnode->head.next == &ring->dirty_list) - return NULL; - - return list_next_entry(rnode, head); -} - -static void ring_free_pages(struct scoutfs_ring_info *ring) -{ - unsigned i; - - if (!ring->pages) - return; - - for (i = 0; i < ring->nr_pages; i++) { - if (ring->pages[i]) - __free_page(ring->pages[i]); - } - - kfree(ring->pages); - - ring->pages = NULL; - ring->nr_pages = 0; -} - -int scoutfs_ring_has_dirty(struct scoutfs_ring_info *ring) -{ - return !!ring->dirty_bytes; -} - -int scoutfs_ring_submit_write(struct super_block *sb, - struct scoutfs_ring_info *ring, - struct scoutfs_bio_completion *comp) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_ring_descriptor *rdesc = ring->rdesc; - struct scoutfs_ring_block *rblk; - struct scoutfs_ring_entry *rent; - struct ring_node *rnode; - struct ring_node *next; - struct page **pages; - unsigned nr_blocks; - unsigned nr_pages; - unsigned i; - u64 blkno; - u64 block; - u64 first; - u64 last; - u64 seq; - u64 nr; - u8 *end; - int ret; - - if (ring->dirty_bytes == 0) - return 0; - - nr_blocks = most_blocks(ring->dirty_bytes); - nr_pages = DIV_ROUND_UP(nr_blocks, SCOUTFS_BLOCKS_PER_PAGE); - - ring_free_pages(ring); - - pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); - if (!pages) - return -ENOMEM; - - ring->pages = pages; - ring->nr_pages = nr_pages; - - for (i = 0; i < nr_pages; i++) { - pages[i] = alloc_page(GFP_NOFS | __GFP_ZERO); - if (!pages[i]) { - ret = -ENOMEM; - goto out; - } - } - - block = ring->first_dirty_block; - seq = ring->first_dirty_seq; - rnode = first_dirty_node(ring); - - for (i = 0; rnode && i < nr_blocks; i++) { - - rblk = block_in_pages(pages, i); - end = (u8 *)rblk + SCOUTFS_BLOCK_SIZE; - - rblk->fsid = super->hdr.fsid; - rblk->seq = cpu_to_le64(seq); - rblk->block = cpu_to_le64(block); - - rent = rblk->entries; - - while (rnode && &rent->data[rnode->data_len] <= end) { - - trace_printk("writing ent %u rnode %p in %u deleted %u dirty %u\n", - le32_to_cpu(rblk->nr_entries), - rnode, rnode->in_ring, rnode->deleted, - rnode->dirty); - - rent->data_len = cpu_to_le16(rnode->data_len); - if (rnode->deleted) - rent->flags = SCOUTFS_RING_ENTRY_FLAG_DELETION; - memcpy(rent->data, rnode->data, rnode->data_len); - - le32_add_cpu(&rblk->nr_entries, 1); - - rnode->block = block; - - rent = (void *)&rent->data[le16_to_cpu(rent->data_len)]; - - next = next_dirty_node(ring, rnode); - if (!next) { - fill_last_dirty_block(ring, (char *)end - - (char *)rent); - next = next_dirty_node(ring, rnode); - } - rnode = next; - } - - rblk->crc = rblk_crc(rblk); - - block = wrap_ring_block(rdesc, block + 1); - seq++; - } - - /* update the number of blocks we actually filled */ - nr_blocks = i; - - /* point the descriptor at the new active region of the ring */ - rnode = list_first_entry_or_null(&ring->clean_list, struct ring_node, - head); - if (rnode) - first = rnode->block; - else - first = ring->first_dirty_block; - - last = wrap_ring_block(rdesc, ring->first_dirty_block + nr_blocks); - - if (first < last) - nr = last - first; - else - nr = last + le64_to_cpu(rdesc->total_blocks) - first; - - rdesc->first_block = cpu_to_le64(first); - rdesc->first_seq = cpu_to_le64(ring->first_dirty_seq + nr_blocks - nr); - rdesc->nr_blocks = cpu_to_le64(nr); - - /* the contig dirty blocks in pages might wrap around ring */ - blkno = le64_to_cpu(rdesc->blkno) + ring->first_dirty_block; - nr = min_t(u64, nr_blocks, - le64_to_cpu(rdesc->total_blocks) - ring->first_dirty_block); - - scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr, comp); - - if (nr != nr_blocks) { - pages += nr / SCOUTFS_BLOCKS_PER_PAGE; - blkno = le64_to_cpu(rdesc->blkno); - nr = nr_blocks - nr; - - scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr, comp); - } - - ret = 0; - -out: - if (ret) - ring_free_pages(ring); - - return ret; -} - -void scoutfs_ring_write_complete(struct scoutfs_ring_info *ring) -{ - struct ring_node *rnode; - struct ring_node *pos; - - list_for_each_entry_safe(rnode, pos, &ring->dirty_list, head) { - if (rnode->deleted) { - free_node(ring, rnode); - } else { - mark_node_clean(ring, rnode); - rnode->in_ring = 1; - } - } - - ring_free_pages(ring); - - ring->dirty_bytes = 0; - ring->first_dirty_block = calc_first_dirty_block(ring->rdesc); - ring->first_dirty_seq = le64_to_cpu(ring->rdesc->first_seq) + - le64_to_cpu(ring->rdesc->nr_blocks); -} - -void scoutfs_ring_init(struct scoutfs_ring_info *ring, - struct scoutfs_ring_descriptor *rdesc, - scoutfs_ring_cmp_t compare_key, - scoutfs_ring_cmp_t compare_data) -{ - ring->rdesc = rdesc; - ring->compare_key = compare_key; - ring->compare_data = compare_data; - ring->rb_root = RB_ROOT; - INIT_LIST_HEAD(&ring->clean_list); - INIT_LIST_HEAD(&ring->dirty_list); - ring->dirty_bytes = 0; - ring->first_dirty_block = calc_first_dirty_block(rdesc); - ring->first_dirty_seq = le64_to_cpu(rdesc->first_seq) + - le64_to_cpu(rdesc->nr_blocks); - ring->pages = NULL; - ring->nr_pages = 0; -} - -void scoutfs_ring_destroy(struct scoutfs_ring_info *ring) -{ - struct ring_node *rnode; - struct ring_node *pos; - - /* XXX we don't really have a coherent forced dirty unmount story */ - WARN_ON_ONCE(!list_empty(&ring->dirty_list)); - - list_splice_init(&ring->dirty_list, &ring->clean_list); - - list_for_each_entry_safe(rnode, pos, &ring->clean_list, head) { - list_del_init(&rnode->head); - kfree(rnode); - } - - ring_free_pages(ring); - scoutfs_ring_init(ring, ring->rdesc, ring->compare_key, - ring->compare_data); -} diff --git a/kmod/src/ring.h b/kmod/src/ring.h deleted file mode 100644 index a9341092..00000000 --- a/kmod/src/ring.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef _SCOUTFS_RING_H_ -#define _SCOUTFS_RING_H_ - -struct scoutfs_bio_completion; - -typedef int (*scoutfs_ring_cmp_t)(void *a, void *b); - -struct scoutfs_ring_info { - struct scoutfs_ring_descriptor *rdesc; - - scoutfs_ring_cmp_t compare_key; - scoutfs_ring_cmp_t compare_data; - - struct rb_root rb_root; - - struct list_head clean_list; - struct list_head dirty_list; - - unsigned long dirty_bytes; - u64 first_dirty_block; - u64 first_dirty_seq; - - struct page **pages; - unsigned long nr_pages; -}; - -void scoutfs_ring_init(struct scoutfs_ring_info *ring, - struct scoutfs_ring_descriptor *rdesc, - scoutfs_ring_cmp_t compare_key, - scoutfs_ring_cmp_t compare_data); - -int scoutfs_ring_load(struct super_block *sb, struct scoutfs_ring_info *ring); - -void *scoutfs_ring_insert(struct scoutfs_ring_info *ring, void *key, - unsigned data_len); - -void *scoutfs_ring_first(struct scoutfs_ring_info *ring); -void *scoutfs_ring_lookup(struct scoutfs_ring_info *ring, void *key); -void *scoutfs_ring_lookup_next(struct scoutfs_ring_info *ring, void *key); -void *scoutfs_ring_lookup_prev(struct scoutfs_ring_info *ring, void *key); - -void *scoutfs_ring_next(struct scoutfs_ring_info *ring, void *rdata); -void *scoutfs_ring_prev(struct scoutfs_ring_info *ring, void *rdata); -void scoutfs_ring_dirty(struct scoutfs_ring_info *ring, void *rdata); -void scoutfs_ring_delete(struct scoutfs_ring_info *ring, void *rdata); - -int scoutfs_ring_has_dirty(struct scoutfs_ring_info *ring); -int scoutfs_ring_submit_write(struct super_block *sb, - struct scoutfs_ring_info *ring, - struct scoutfs_bio_completion *comp); -void scoutfs_ring_write_complete(struct scoutfs_ring_info *ring); - -void scoutfs_ring_destroy(struct scoutfs_ring_info *ring); - -#endif From 690049c293b05a25d217efbd36625b54eeb351e5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 6 Jul 2017 10:31:41 -0700 Subject: [PATCH 327/920] scoutfs: add GET_MANIFEST_ROOT network op We're going to need to be able to sample the current stable manifest root occasionally. We're adding it now because we don't yet have the lock plumbing that would provide the lvb. Eventually this call will bubble up into the locking and the root will be stored in the lock instead of always requested. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/manifest.c | 34 ++++++++++++-------- kmod/src/net.c | 75 +++++++++++++++++++++++++++++++++++++++++++++ kmod/src/net.h | 3 ++ 4 files changed, 100 insertions(+), 13 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 1ccac901..fbd59c41 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -557,6 +557,7 @@ enum { SCOUTFS_NET_BULK_ALLOC, SCOUTFS_NET_ADVANCE_SEQ, SCOUTFS_NET_GET_LAST_SEQ, + SCOUTFS_NET_GET_MANIFEST_ROOT, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 31c55283..7cad79cd 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -447,16 +447,14 @@ static int btree_prev_overlap_or_next(struct super_block *sb, * existing segment that intersects with the range, even if it doesn't * contain the key. The key might fall between segments at that level. * - * XXX Today this using the roots from the mount-wide super. This is - * super wrong. Doing so lets it use the dirty btree that could be - * modified by the manifest server running on this node so it has to - * lock. It should be using a specific root communicated by lock lvbs - * (or read from the super on mount). Then the btrees it traverses will - * be stable and read-only. (But can still get -ESTALE if they're - * re-written under us, would need to re-sample roots from the super in - * that case, I imagine.) + * This is walking stable btree roots. The blocks won't be changed as + * long as we read valid blocks. They can be overwritten in which case + * we'll return -ESTALE and the caller can retry with a newer root or + * return hard errors. */ -static int get_manifest_refs(struct super_block *sb, struct scoutfs_key_buf *key, +static int get_manifest_refs(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key_buf *key, struct scoutfs_key_buf *end, struct list_head *ref_list) { @@ -475,8 +473,6 @@ static int get_manifest_refs(struct super_block *sb, struct scoutfs_key_buf *key if (!mkey) return -ENOMEM; - scoutfs_manifest_lock(sb); - /* get level 0 segments that overlap with the missing range */ mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); ret = scoutfs_btree_prev(sb, &super->manifest.root, @@ -534,8 +530,8 @@ static int get_manifest_refs(struct super_block *sb, struct scoutfs_key_buf *key out: scoutfs_btree_put_iref(&iref); scoutfs_btree_put_iref(&prev); - scoutfs_manifest_unlock(sb); kfree(mkey); + BUG_ON(ret == -ESTALE); /* XXX caller needs to retry or return error */ return ret; } @@ -572,6 +568,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf found_key; struct scoutfs_key_buf batch_end; struct scoutfs_key_buf seg_end; + struct scoutfs_btree_root root; SCOUTFS_DECLARE_KVEC(item_val); SCOUTFS_DECLARE_KVEC(found_val); struct scoutfs_segment *seg; @@ -590,8 +587,19 @@ int scoutfs_manifest_read_items(struct super_block *sb, trace_scoutfs_read_items(sb, key, end); + + /* + * Ask the manifest server which manifest root to read from. Lock + * holding callers will be responsible for this in the future. They'll + * either get a manifest ref in the lvb of their lock or they'll + * ask the server the first time the system sees the lock. + */ + ret = scoutfs_net_get_manifest_root(sb, &root); + if (ret) + goto out; + /* get refs on all the segments */ - ret = get_manifest_refs(sb, key, end, &ref_list); + ret = get_manifest_refs(sb, &root, key, end, &ref_list); if (ret) goto out; diff --git a/kmod/src/net.c b/kmod/src/net.c index 932e599d..d542bd5d 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -85,6 +85,9 @@ struct net_info { struct llist_head commit_waiters; struct work_struct commit_work; + /* server remembers the stable manifest root for clients */ + struct scoutfs_btree_root stable_manifest_root; + /* level 0 segment addition waits for it to clear */ wait_queue_head_t waitq; @@ -338,6 +341,7 @@ static void scoutfs_net_commit_func(struct work_struct *work) scoutfs_btree_write_complete(sb); + nti->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; scoutfs_advance_dirty_super(sb); } else { ret = 0; @@ -777,6 +781,32 @@ static struct send_buf *process_get_last_seq(struct super_block *sb, return sbuf; } +static struct send_buf *process_get_manifest_root(struct super_block *sb, + void *req, int req_len) +{ + DECLARE_NET_INFO(sb, nti); + struct scoutfs_btree_root *root; + struct send_buf *sbuf; + + if (req_len != 0) + return ERR_PTR(-EINVAL); + + sbuf = alloc_sbuf(sizeof(struct scoutfs_btree_root)); + if (!sbuf) + return ERR_PTR(-ENOMEM); + + root = (void *)sbuf->nh->data; + + scoutfs_manifest_lock(sb); + memcpy(root, &nti->stable_manifest_root, + sizeof(struct scoutfs_btree_root)); + scoutfs_manifest_unlock(sb); + + sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; + + return sbuf; +} + typedef struct send_buf *(*proc_func_t)(struct super_block *sb, void *req, int req_len); @@ -789,6 +819,7 @@ static proc_func_t type_proc_func(u8 type) [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, + [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, }; return type < SCOUTFS_NET_UNKNOWN ? funcs[type] : NULL; @@ -918,6 +949,8 @@ static void scoutfs_net_proc_func(struct work_struct *work) if (ret == 0) { scoutfs_advance_dirty_super(sb); nti->server_loaded = true; + nti->stable_manifest_root = + SCOUTFS_SB(sb)->super.manifest.root; } else { destroy_server_state(sb); } @@ -1707,6 +1740,48 @@ int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq) return ret; } +struct get_manifest_root_args { + struct scoutfs_btree_root *root; + struct completion comp; + int ret; +}; + +static int get_manifest_root_reply(struct super_block *sb, void *reply, int ret, + void *arg) +{ + struct get_manifest_root_args *args = arg; + struct scoutfs_btree_root *root = reply; + + if (ret == sizeof(struct scoutfs_btree_root)) { + memcpy(args->root, root, sizeof(struct scoutfs_btree_root)); + args->ret = 0; + } else { + args->ret = -EINVAL; + } + + complete(&args->comp); /* args can be freed from this point */ + return args->ret; +} + +int scoutfs_net_get_manifest_root(struct super_block *sb, + struct scoutfs_btree_root *root) +{ + struct get_manifest_root_args args; + int ret; + + args.root = root; + init_completion(&args.comp); + + ret = add_send_buf(sb, SCOUTFS_NET_GET_MANIFEST_ROOT, NULL, 0, + get_manifest_root_reply, &args); + if (ret == 0) { + wait_for_completion(&args.comp); + ret = args.ret; + } + return ret; +} + + static struct sock_info *alloc_sinf(struct super_block *sb) { struct sock_info *sinf; diff --git a/kmod/src/net.h b/kmod/src/net.h index bcfa34f9..b20a6cb8 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -16,6 +16,9 @@ int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq); int scoutfs_net_advance_seq(struct super_block *sb, u64 *seq); +int scoutfs_net_get_manifest_root(struct super_block *sb, + struct scoutfs_btree_root *root); + int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); From 8d29c82306f992e6c4fa52ba2871843d76bb905e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Jul 2017 11:50:32 -0700 Subject: [PATCH 328/920] scoutfs: sort keys by zone, then inode, then type Holding a DLM lock protects a range of the key space. The DLM locks span inodes or regions of inodes. We need the sort order in LSM items to match the DLM range keys so that we can read all the items covered by a lock into the cache from a region of LSM segments. If their orders differered then we'd have to jump around segments to find all the items covered by a given DLM lock. Previously we were sorting by type then, within types, by inode. Now we want to sort by inode then by type. But there are structures which previously had a type but weren't then sorted by inode. We introduce zones as the primary sort key. Inode index and node zones are sorted by the inode fields and node ids respectively. Then comes the fs zone first sorted by inode then the type of the key. The bulk of this is the mechanical introduction of the zone field to the keys, moving the type field down, and a bulk rename of _KEY to _TYPE. But there are some more substantial changes. The orphan keys needed to be put in a zone. They fit in the NODE zone which is all about resources that nodes hold and would need to be cleaned up if the node went away. The key formatting is significantly changed to match the new formatting. Formatted keys are now generally of the form "zone.primary.type..." And finally with the keys now properly sorted by inodes we can correctly construct a single range of item cache keys to invalidate when unlocking the inode group locks. Signed-off-by: Zach Brown --- kmod/src/data.c | 60 ++++++++-------- kmod/src/dir.c | 12 ++-- kmod/src/format.h | 98 +++++++++++++++----------- kmod/src/inode.c | 35 ++++++---- kmod/src/ioctl.c | 18 +++-- kmod/src/key.c | 145 ++++++++++++++++++++++++--------------- kmod/src/kvec.c | 2 +- kmod/src/net.c | 4 +- kmod/src/scoutfs_trace.h | 11 --- kmod/src/xattr.c | 3 +- 10 files changed, 224 insertions(+), 164 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 77dc5f98..bc573bfc 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -139,8 +139,9 @@ static void init_file_extent_key(struct scoutfs_key_buf *key, void *key_bytes, { struct scoutfs_file_extent_key *fkey = key_bytes; - fkey->type = SCOUTFS_FILE_EXTENT_KEY; + fkey->zone = SCOUTFS_FS_ZONE; fkey->ino = cpu_to_be64(arg); + fkey->type = SCOUTFS_FILE_EXTENT_TYPE; fkey->last_blk_off = cpu_to_be64(ext->blk_off + ext->blocks - 1); fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); fkey->blocks = cpu_to_be64(ext->blocks); @@ -153,8 +154,9 @@ static void init_file_extent_key(struct scoutfs_key_buf *key, void *key_bytes, do { \ struct which_type *fkey = key_bytes; \ \ - fkey->type = type; \ + fkey->zone = SCOUTFS_NODE_ZONE; \ fkey->node_id = cpu_to_be64(arg); \ + fkey->type = type; \ fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); \ fkey->blocks = cpu_to_be64(ext->blocks); \ \ @@ -164,9 +166,9 @@ do { \ static void init_extent_key(struct scoutfs_key_buf *key, void *key_bytes, struct native_extent *ext, u64 arg, u8 type) { - if (type == SCOUTFS_FILE_EXTENT_KEY) + if (type == SCOUTFS_FILE_EXTENT_TYPE) init_file_extent_key(key, key_bytes, ext, arg); - else if(type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) + else if(type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) INIT_FREE_EXTENT_KEY(scoutfs_free_extent_blkno_key, key, key_bytes, ext, arg, type); else @@ -206,9 +208,9 @@ static void load_extent(struct native_extent *ext, struct scoutfs_key_buf *key) offsetof(struct scoutfs_file_extent_key, type) != offsetof(struct scoutfs_free_extent_blocks_key, type)); - if (fkey->type == SCOUTFS_FILE_EXTENT_KEY) + if (fkey->type == SCOUTFS_FILE_EXTENT_TYPE) load_file_extent(ext, key); - else if (fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) + else if (fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) LOAD_FREE_EXTENT(scoutfs_free_extent_blkno_key, ext, key); else LOAD_FREE_EXTENT(scoutfs_free_extent_blocks_key, ext, key); @@ -344,16 +346,16 @@ static int modify_items(struct super_block *sb, struct native_extent *ext, trace_printk("mod cre %u "EXTF"\n", create, EXTA(ext)); - BUG_ON(type != SCOUTFS_FILE_EXTENT_KEY && - type != SCOUTFS_FREE_EXTENT_BLKNO_KEY); + BUG_ON(type != SCOUTFS_FILE_EXTENT_TYPE && + type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE); init_extent_key(&key, key_bytes, ext, arg, type); ret = create ? scoutfs_item_create(sb, &key, NULL) : scoutfs_item_delete(sb, &key); - if (ret == 0 && type == SCOUTFS_FREE_EXTENT_BLKNO_KEY) { + if (ret == 0 && type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { init_extent_key(&key, key_bytes, ext, arg, - SCOUTFS_FREE_EXTENT_BLOCKS_KEY); + SCOUTFS_FREE_EXTENT_BLOCKS_TYPE); ret = create ? scoutfs_item_create(sb, &key, NULL) : scoutfs_item_delete(sb, &key); if (ret) { @@ -538,7 +540,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, iblock, len, offline); memset(&ext, ~0, sizeof(ext)); - init_extent_key(&last, last_bytes, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); + init_extent_key(&last, last_bytes, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); rng.blk_off = iblock; rng.blocks = len; @@ -548,7 +550,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, while (rng.blocks) { /* find the next extent that could include our first block */ init_extent_key(&key, key_bytes, &rng, ino, - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); ret = scoutfs_item_next_same(sb, &key, &last, NULL); if (ret < 0) { @@ -602,14 +604,14 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, fr = ext; fr.blk_off = fr.blkno; ret = insert_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY); + SCOUTFS_FREE_EXTENT_BLKNO_TYPE); if (ret) break; rem_fr = true; } /* always remove the overlapping file extent */ - ret = remove_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); + ret = remove_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); if (ret) break; ins_ext = true; @@ -620,7 +622,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, ofl.blkno = 0; ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; ret = insert_extent(sb, &ofl, sbi->node_id, - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); if (ret) break; } @@ -637,12 +639,12 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, if (ret) { if (ins_ext) { err = insert_extent(sb, &ext, ino, - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); BUG_ON(err); } if (rem_fr) { err = remove_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY); + SCOUTFS_FREE_EXTENT_BLKNO_TYPE); BUG_ON(err); } } @@ -720,7 +722,7 @@ static int bulk_alloc(struct super_block *sb) ext.blk_off = ext.blkno; ext.flags = 0; ret = insert_extent(sb, &ext, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY); + SCOUTFS_FREE_EXTENT_BLKNO_TYPE); if (ret) break; } @@ -778,11 +780,11 @@ reset_cursor: if (curs->blocks) { ext.blkno = curs->blkno; ext.blocks = 0; - type = SCOUTFS_FREE_EXTENT_BLKNO_KEY; + type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; } else { ext.blkno = datinf->next_large_blkno; ext.blocks = LARGE_EXTENT_BLOCKS; - type = SCOUTFS_FREE_EXTENT_BLOCKS_KEY; + type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; } ext.flags = 0; @@ -825,7 +827,7 @@ retry: if (ext.blocks) { ext.blkno = 0; ext.blocks = 0; - type = SCOUTFS_FREE_EXTENT_BLKNO_KEY; + type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; goto retry; } @@ -876,7 +878,7 @@ retry: ofl.blkno = 0; ofl.blocks = 1; ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; - ret = remove_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_KEY); + ret = remove_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_TYPE); if (ret < 0) goto out; ins_ofl = true; @@ -888,7 +890,7 @@ retry: ext.blkno = found.blkno; ext.blocks = 1; ext.flags = 0; - ret = insert_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_KEY); + ret = insert_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); if (ret < 0) goto out; rem_ext = true; @@ -897,7 +899,7 @@ retry: fr = ext; fr.blk_off = ext.blkno; ret = remove_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_KEY); + SCOUTFS_FREE_EXTENT_BLKNO_TYPE); if (ret) goto out; @@ -914,12 +916,12 @@ out: if (ret) { if (rem_ext) { err = remove_extent(sb, &ext, ino, - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); BUG_ON(err); } if (ins_ofl) { err = insert_extent(sb, &ofl, ino, - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); BUG_ON(err); } } @@ -953,11 +955,11 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, ext.blkno = 0; ext.flags = 0; init_extent_key(&key, key_bytes, &ext, scoutfs_ino(inode), - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); memset(&ext, ~0, sizeof(ext)); init_extent_key(&last, last_bytes, &ext, scoutfs_ino(inode), - SCOUTFS_FILE_EXTENT_KEY); + SCOUTFS_FILE_EXTENT_TYPE); /* * XXX think about how far this next can go, given locking and @@ -1107,7 +1109,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct super_block *sb = inode->i_sb; - const u8 type = SCOUTFS_FILE_EXTENT_KEY; + const u8 type = SCOUTFS_FILE_EXTENT_TYPE; const u64 ino = scoutfs_ino(inode); u8 last_bytes[MAX_KEY_BYTES]; u8 key_bytes[MAX_KEY_BYTES]; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 52448cd5..7bb21fe0 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -178,8 +178,9 @@ static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, name[dentry->d_name.len])); if (key) { dkey = key->data; - dkey->type = SCOUTFS_DIRENT_KEY; + dkey->zone = SCOUTFS_FS_ZONE; dkey->ino = cpu_to_be64(scoutfs_ino(dir)); + dkey->type = SCOUTFS_DIRENT_TYPE; memcpy(dkey->name, (void *)dentry->d_name.name, dentry->d_name.len); } @@ -192,8 +193,9 @@ static void init_link_backref_key(struct scoutfs_key_buf *key, u64 ino, u64 dir_ino, char *name, unsigned name_len) { - lbrkey->type = SCOUTFS_LINK_BACKREF_KEY; + lbrkey->zone = SCOUTFS_FS_ZONE; lbrkey->ino = cpu_to_be64(ino); + lbrkey->type = SCOUTFS_LINK_BACKREF_TYPE; lbrkey->dir_ino = cpu_to_be64(dir_ino); if (name_len) memcpy(lbrkey->name, name, name_len); @@ -297,8 +299,9 @@ static void init_readdir_key(struct scoutfs_key_buf *key, struct scoutfs_readdir_key *rkey, struct inode *inode, loff_t pos) { - rkey->type = SCOUTFS_READDIR_KEY; + rkey->zone = SCOUTFS_FS_ZONE; rkey->ino = cpu_to_be64(scoutfs_ino(inode)); + rkey->type = SCOUTFS_READDIR_TYPE; rkey->pos = cpu_to_be64(pos); scoutfs_key_init(key, rkey, sizeof(struct scoutfs_readdir_key)); @@ -636,8 +639,9 @@ out: static void init_symlink_key(struct scoutfs_key_buf *key, struct scoutfs_symlink_key *skey, u64 ino, u8 nr) { - skey->type = SCOUTFS_SYMLINK_KEY; + skey->zone = SCOUTFS_FS_ZONE; skey->ino = cpu_to_be64(ino); + skey->type = SCOUTFS_SYMLINK_TYPE; skey->nr = nr; scoutfs_key_init(key, skey, sizeof(struct scoutfs_symlink_key)); diff --git a/kmod/src/format.h b/kmod/src/format.h index fbd59c41..23bd959c 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -236,73 +236,78 @@ struct scoutfs_segment_block { } __packed; /* - * Currently we sort keys by the numeric value of the types, but that - * isn't necessary. We could have an arbitrary sort order. So we don't - * have to stress about cleverly allocating the types. + * Keys are first sorted by major key zones. */ -#define SCOUTFS_INODE_KEY 1 -#define SCOUTFS_XATTR_KEY 3 -#define SCOUTFS_DIRENT_KEY 5 -#define SCOUTFS_READDIR_KEY 6 -#define SCOUTFS_LINK_BACKREF_KEY 7 -#define SCOUTFS_SYMLINK_KEY 8 -#define SCOUTFS_FILE_EXTENT_KEY 9 -#define SCOUTFS_ORPHAN_KEY 10 -#define SCOUTFS_FREE_EXTENT_BLKNO_KEY 11 -#define SCOUTFS_FREE_EXTENT_BLOCKS_KEY 12 -#define SCOUTFS_INODE_INDEX_CTIME_KEY 13 /* don't forget first and last */ -#define SCOUTFS_INODE_INDEX_MTIME_KEY 14 -#define SCOUTFS_INODE_INDEX_SIZE_KEY 15 -#define SCOUTFS_INODE_INDEX_META_SEQ_KEY 16 -#define SCOUTFS_INODE_INDEX_DATA_SEQ_KEY 17 -/* not found in the fs */ -#define SCOUTFS_MAX_UNUSED_KEY 253 -#define SCOUTFS_NET_ADDR_KEY 254 -#define SCOUTFS_NET_LISTEN_KEY 255 +#define SCOUTFS_INODE_INDEX_ZONE 1 +#define SCOUTFS_NODE_ZONE 2 +#define SCOUTFS_FS_ZONE 3 + +/* inode index zone */ +#define SCOUTFS_INODE_INDEX_CTIME_TYPE 1 +#define SCOUTFS_INODE_INDEX_MTIME_TYPE 2 +#define SCOUTFS_INODE_INDEX_SIZE_TYPE 3 +#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 4 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 5 -#define SCOUTFS_INODE_INDEX_FIRST SCOUTFS_INODE_INDEX_CTIME_KEY -#define SCOUTFS_INODE_INDEX_LAST SCOUTFS_INODE_INDEX_DATA_SEQ_KEY #define SCOUTFS_INODE_INDEX_NR \ - (SCOUTFS_INODE_INDEX_LAST - SCOUTFS_INODE_INDEX_FIRST + 1) + (SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE - SCOUTFS_INODE_INDEX_CTIME_TYPE + 1) + +/* node zone */ +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 11 +#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 12 + +/* fs zone */ +#define SCOUTFS_INODE_TYPE 1 +#define SCOUTFS_XATTR_TYPE 2 +#define SCOUTFS_DIRENT_TYPE 3 +#define SCOUTFS_READDIR_TYPE 4 +#define SCOUTFS_LINK_BACKREF_TYPE 5 +#define SCOUTFS_SYMLINK_TYPE 6 +#define SCOUTFS_FILE_EXTENT_TYPE 7 +#define SCOUTFS_ORPHAN_TYPE 8 + +/* XXX don't need these now that we have dlm lock spaces and resources */ +#define SCOUTFS_NET_ADDR_TYPE 254 +#define SCOUTFS_NET_LISTEN_TYPE 255 /* value is struct scoutfs_inode */ struct scoutfs_inode_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; } __packed; /* value is struct scoutfs_dirent without the name */ struct scoutfs_dirent_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; __u8 name[0]; } __packed; /* value is struct scoutfs_dirent with the name */ struct scoutfs_readdir_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; __be64 pos; } __packed; /* value is empty */ struct scoutfs_link_backref_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; __be64 dir_ino; __u8 name[0]; } __packed; -/* no value */ -struct scoutfs_orphan_key { - __u8 type; - __be64 ino; -} __packed; /* no value */ struct scoutfs_file_extent_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; __be64 last_blk_off; __be64 last_blkno; __be64 blocks; @@ -313,23 +318,34 @@ struct scoutfs_file_extent_key { /* no value */ struct scoutfs_free_extent_blkno_key { - __u8 type; + __u8 zone; __be64 node_id; + __u8 type; __be64 last_blkno; __be64 blocks; } __packed; struct scoutfs_free_extent_blocks_key { - __u8 type; + __u8 zone; __be64 node_id; + __u8 type; __be64 blocks; __be64 last_blkno; } __packed; -/* value is each item's part of the full xattr value for the off/len */ -struct scoutfs_xattr_key { +/* no value */ +struct scoutfs_orphan_key { + __u8 zone; + __be64 node_id; __u8 type; __be64 ino; +} __packed; + +/* value is each item's part of the full xattr value for the off/len */ +struct scoutfs_xattr_key { + __u8 zone; + __be64 ino; + __u8 type; __u8 name[0]; } __packed; @@ -345,8 +361,9 @@ struct scoutfs_xattr_val_header { /* size determines nr needed to store full target path in their values */ struct scoutfs_symlink_key { - __u8 type; + __u8 zone; __be64 ino; + __u8 type; __u8 nr; } __packed; @@ -356,6 +373,7 @@ struct scoutfs_betimespec { } __packed; struct scoutfs_inode_index_key { + __u8 zone; __u8 type; __be64 major; __be32 minor; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index c54e21dc..cf3a1714 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -224,8 +224,9 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) void scoutfs_inode_init_key(struct scoutfs_key_buf *key, struct scoutfs_inode_key *ikey, u64 ino) { - ikey->type = SCOUTFS_INODE_KEY; + ikey->zone = SCOUTFS_FS_ZONE; ikey->ino = cpu_to_be64(ino); + ikey->type = SCOUTFS_INODE_TYPE; scoutfs_key_init(key, ikey, sizeof(struct scoutfs_inode_key)); } @@ -479,6 +480,7 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, if (si->have_item && now_major == then_major && now_minor == then_minor) return 0; + ins_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; ins_ikey.type = type; ins_ikey.major = cpu_to_be64(now_major); ins_ikey.minor = cpu_to_be32(now_minor); @@ -489,6 +491,7 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, if (ret || !si->have_item) return ret; + del_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; del_ikey.type = type; del_ikey.major = cpu_to_be64(then_major); del_ikey.minor = cpu_to_be32(then_minor); @@ -527,18 +530,18 @@ void scoutfs_update_inode_item(struct inode *inode) /* set the meta version once per trans for any inode updates */ scoutfs_inode_set_meta_seq(inode); - ret = update_index(inode, SCOUTFS_INODE_INDEX_CTIME_KEY, + ret = update_index(inode, SCOUTFS_INODE_INDEX_CTIME_TYPE, inode->i_ctime.tv_sec, inode->i_ctime.tv_nsec, si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: - update_index(inode, SCOUTFS_INODE_INDEX_MTIME_KEY, + update_index(inode, SCOUTFS_INODE_INDEX_MTIME_TYPE, inode->i_mtime.tv_sec, inode->i_mtime.tv_nsec, si->item_mtime.tv_sec, si->item_mtime.tv_nsec) ?: - update_index(inode, SCOUTFS_INODE_INDEX_SIZE_KEY, + update_index(inode, SCOUTFS_INODE_INDEX_SIZE_TYPE, i_size_read(inode), 0, si->item_size, 0) ?: - update_index(inode, SCOUTFS_INODE_INDEX_META_SEQ_KEY, + update_index(inode, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, scoutfs_inode_meta_seq(inode), 0, si->item_meta_seq, 0) ?: - update_index(inode, SCOUTFS_INODE_INDEX_DATA_SEQ_KEY, + update_index(inode, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, scoutfs_inode_data_seq(inode), 0, si->item_data_seq, 0); BUG_ON(ret); @@ -753,9 +756,11 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, } static void init_orphan_key(struct scoutfs_key_buf *key, - struct scoutfs_orphan_key *okey, u64 ino) + struct scoutfs_orphan_key *okey, u64 node_id, u64 ino) { - okey->type = SCOUTFS_ORPHAN_KEY; + okey->zone = SCOUTFS_NODE_ZONE; + okey->node_id = cpu_to_be64(node_id); + okey->type = SCOUTFS_ORPHAN_TYPE; okey->ino = cpu_to_be64(ino); scoutfs_key_init(key, okey, sizeof(struct scoutfs_orphan_key)); @@ -763,11 +768,12 @@ static void init_orphan_key(struct scoutfs_key_buf *key, static int remove_orphan_item(struct super_block *sb, u64 ino) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_orphan_key okey; struct scoutfs_key_buf key; int ret; - init_orphan_key(&key, &okey, ino); + init_orphan_key(&key, &okey, sbi->node_id, ino); ret = scoutfs_item_delete(sb, &key); if (ret == -ENOENT) @@ -907,9 +913,13 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) * Runtime of this will be bounded by the number of orphans, which could * theoretically be very large. If that becomes a problem we might want to push * this work off to a thread. + * + * This only scans orphans for this node. This will need to be covered by + * the rest of node zone cleanup. */ int scoutfs_scan_orphans(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_orphan_key okey; struct scoutfs_orphan_key last_okey; struct scoutfs_key_buf key; @@ -919,8 +929,8 @@ int scoutfs_scan_orphans(struct super_block *sb) trace_scoutfs_scan_orphans(sb); - init_orphan_key(&key, &okey, 0); - init_orphan_key(&last, &last_okey, ~0ULL); + init_orphan_key(&key, &okey, sbi->node_id, 0); + init_orphan_key(&last, &last_okey, sbi->node_id, ~0ULL); while (1) { ret = scoutfs_item_next_same(sb, &key, &last, NULL); @@ -944,13 +954,14 @@ out: int scoutfs_orphan_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_orphan_key okey; struct scoutfs_key_buf key; int ret; trace_scoutfs_orphan_inode(sb, inode); - init_orphan_key(&key, &okey, scoutfs_ino(inode)); + init_orphan_key(&key, &okey, sbi->node_id, scoutfs_ino(inode)); ret = scoutfs_item_create(sb, &key, NULL); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 588dcf3e..ad477782 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -47,6 +47,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) struct scoutfs_key_buf key; u64 last_seq; int ret = 0; + u8 type; u32 nr; if (copy_from_user(&walk, uwalk, sizeof(walk))) @@ -58,21 +59,21 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) walk.last.ino); if (walk.index == SCOUTFS_IOC_WALK_INODES_CTIME) - ikey.type = SCOUTFS_INODE_INDEX_CTIME_KEY; + type = SCOUTFS_INODE_INDEX_CTIME_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_MTIME) - ikey.type = SCOUTFS_INODE_INDEX_MTIME_KEY; + type = SCOUTFS_INODE_INDEX_MTIME_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) - ikey.type = SCOUTFS_INODE_INDEX_SIZE_KEY; + type = SCOUTFS_INODE_INDEX_SIZE_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) - ikey.type = SCOUTFS_INODE_INDEX_META_SEQ_KEY; + type = SCOUTFS_INODE_INDEX_META_SEQ_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_DATA_SEQ) - ikey.type = SCOUTFS_INODE_INDEX_DATA_SEQ_KEY; + type = SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE; else return -EINVAL; /* clamp results to the inodes in the farthest stable seq */ - if (ikey.type == SCOUTFS_INODE_INDEX_META_SEQ_KEY || - ikey.type == SCOUTFS_INODE_INDEX_DATA_SEQ_KEY) { + if (type == SCOUTFS_INODE_INDEX_META_SEQ_TYPE || + type == SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE) { ret = scoutfs_net_get_last_seq(sb, &last_seq); if (ret) @@ -85,11 +86,14 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) } } + ikey.zone = SCOUTFS_INODE_INDEX_ZONE; + ikey.type = type; ikey.major = cpu_to_be64(walk.first.major); ikey.minor = cpu_to_be32(walk.first.minor); ikey.ino = cpu_to_be64(walk.first.ino); scoutfs_key_init(&key, &ikey, sizeof(ikey)); + last_ikey.zone = ikey.zone; last_ikey.type = ikey.type; last_ikey.major = cpu_to_be64(walk.last.major); last_ikey.minor = cpu_to_be32(walk.last.minor); diff --git a/kmod/src/key.c b/kmod/src/key.c index 6135b983..85091bbc 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -118,8 +118,10 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) */ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) { + struct scoutfs_inode_key *ikey; + u8 zone = 0; + u8 type = 0; int len; - u8 type; if (key == NULL || key->data == NULL) return snprintf_null(buf, size, "[NULL]"); @@ -127,21 +129,88 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) if (key->key_len == 0) return snprintf_null(buf, size, "[0 len]"); - type = *(u8 *)key->data; + zone = *(u8 *)key->data; - switch(type) { + /* handle smaller and unknown zones, fall through to fs types */ + switch(zone) { + case SCOUTFS_INODE_INDEX_ZONE: { + struct scoutfs_inode_index_key *ikey = key->data; + static char *type_strings[] = { + [SCOUTFS_INODE_INDEX_CTIME_TYPE] = "ctm", + [SCOUTFS_INODE_INDEX_MTIME_TYPE] = "mtm", + [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", + [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", + [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", + }; - case SCOUTFS_INODE_KEY: { + if (key->key_len < sizeof(struct scoutfs_inode_index_key)) + break; + + if (type_strings[ikey->type]) + return snprintf_null(buf, size, "iin.%s.%llu.%u.%llu", + type_strings[ikey->type], + be64_to_cpu(ikey->major), + be32_to_cpu(ikey->minor), + be64_to_cpu(ikey->ino)); + else + return snprintf_null(buf, size, "[iin type %u?]", + ikey->type); + } + + /* node zone keys start with zone, node, type */ + case SCOUTFS_NODE_ZONE: { + struct scoutfs_free_extent_blkno_key *fkey = key->data; + + static char *type_strings[] = { + [SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fno", + [SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fks", + }; + + switch(fkey->type) { + case SCOUTFS_ORPHAN_TYPE: { + struct scoutfs_orphan_key *okey = key->data; + + if (key->key_len < sizeof(struct scoutfs_orphan_key)) + break; + return snprintf_null(buf, size, "nod.%llu.orp.%llu", + be64_to_cpu(okey->node_id), + be64_to_cpu(okey->ino)); + } + + case SCOUTFS_FREE_EXTENT_BLKNO_TYPE: + case SCOUTFS_FREE_EXTENT_BLOCKS_TYPE: + return snprintf_null(buf, size, "nod.%llu.%s.%llu.%llu", + be64_to_cpu(fkey->node_id), + type_strings[fkey->type], + be64_to_cpu(fkey->last_blkno), + be64_to_cpu(fkey->blocks)); + default: + return snprintf_null(buf, size, "[nod type %u?]", + fkey->type); + } + } + + case SCOUTFS_FS_ZONE: + break; + + default: + return snprintf_null(buf, size, "[zone %u?]", zone); + } + + /* everything in the fs tree starts with zone, ino, type */ + ikey = key->data; + switch(ikey->type) { + case SCOUTFS_INODE_TYPE: { struct scoutfs_inode_key *ikey = key->data; if (key->key_len < sizeof(struct scoutfs_inode_key)) break; - return snprintf_null(buf, size, "ino.%llu", + return snprintf_null(buf, size, "fs.%llu.ino", be64_to_cpu(ikey->ino)); } - case SCOUTFS_XATTR_KEY: { + case SCOUTFS_XATTR_TYPE: { struct scoutfs_xattr_key *xkey = key->data; len = (int)key->key_len - offsetof(struct scoutfs_xattr_key, @@ -149,53 +218,53 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) if (len <= 0) break; - return snprintf_null(buf, size, "xat.%llu.%.*s", + return snprintf_null(buf, size, "fs.%llu.xat.%.*s", be64_to_cpu(xkey->ino), len, xkey->name); } - case SCOUTFS_DIRENT_KEY: { + case SCOUTFS_DIRENT_TYPE: { struct scoutfs_dirent_key *dkey = key->data; len = (int)key->key_len - sizeof(struct scoutfs_dirent_key); if (len <= 0) break; - return snprintf_null(buf, size, "dnt.%llu.%.*s", + return snprintf_null(buf, size, "fs.%llu.dnt.%.*s", be64_to_cpu(dkey->ino), len, dkey->name); } - case SCOUTFS_READDIR_KEY: { + case SCOUTFS_READDIR_TYPE: { struct scoutfs_readdir_key *rkey = key->data; - return snprintf_null(buf, size, "rdr.%llu.%llu", + return snprintf_null(buf, size, "fs.%llu.rdr.%llu", be64_to_cpu(rkey->ino), be64_to_cpu(rkey->pos)); } - case SCOUTFS_LINK_BACKREF_KEY: { + case SCOUTFS_LINK_BACKREF_TYPE: { struct scoutfs_link_backref_key *lkey = key->data; len = (int)key->key_len - sizeof(*lkey); if (len <= 0) break; - return snprintf_null(buf, size, "lbr.%llu.%llu.%.*s", + return snprintf_null(buf, size, "fs.%llu.lbr.%llu.%.*s", be64_to_cpu(lkey->ino), be64_to_cpu(lkey->dir_ino), len, lkey->name); } - case SCOUTFS_SYMLINK_KEY: { + case SCOUTFS_SYMLINK_TYPE: { struct scoutfs_symlink_key *skey = key->data; - return snprintf_null(buf, size, "sym.%llu", + return snprintf_null(buf, size, "fs.%llu.sym", be64_to_cpu(skey->ino)); } - case SCOUTFS_FILE_EXTENT_KEY: { + case SCOUTFS_FILE_EXTENT_TYPE: { struct scoutfs_file_extent_key *ekey = key->data; - return snprintf_null(buf, size, "ext.%llu.%llu.%llu.%llu.%x", + return snprintf_null(buf, size, "fs.%llu.ext.%llu.%llu.%llu.%x", be64_to_cpu(ekey->ino), be64_to_cpu(ekey->last_blk_off), be64_to_cpu(ekey->last_blkno), @@ -203,49 +272,11 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) ekey->flags); } - case SCOUTFS_ORPHAN_KEY: { - struct scoutfs_orphan_key *okey = key->data; - - return snprintf_null(buf, size, "orp.%llu", - be64_to_cpu(okey->ino)); - } - - case SCOUTFS_FREE_EXTENT_BLKNO_KEY: - case SCOUTFS_FREE_EXTENT_BLOCKS_KEY: { - struct scoutfs_free_extent_blkno_key *fkey = key->data; - - return snprintf_null(buf, size, "%s.%llu.%llu.%llu", - fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_KEY ? "fel" : - "fes", - be64_to_cpu(fkey->node_id), - be64_to_cpu(fkey->last_blkno), - be64_to_cpu(fkey->blocks)); - } - - case SCOUTFS_INODE_INDEX_CTIME_KEY: - case SCOUTFS_INODE_INDEX_MTIME_KEY: - case SCOUTFS_INODE_INDEX_SIZE_KEY: - case SCOUTFS_INODE_INDEX_META_SEQ_KEY: - case SCOUTFS_INODE_INDEX_DATA_SEQ_KEY: { - struct scoutfs_inode_index_key *ikey = key->data; - - return snprintf_null(buf, size, "%s.%llu.%u.%llu", - ikey->type == SCOUTFS_INODE_INDEX_CTIME_KEY ? "ctm" : - ikey->type == SCOUTFS_INODE_INDEX_MTIME_KEY ? "mtm" : - ikey->type == SCOUTFS_INODE_INDEX_SIZE_KEY ? "siz" : - ikey->type == SCOUTFS_INODE_INDEX_META_SEQ_KEY ? "msq" : - ikey->type == SCOUTFS_INODE_INDEX_DATA_SEQ_KEY ? "dsq" : - "uii", be64_to_cpu(ikey->major), - be32_to_cpu(ikey->minor), - be64_to_cpu(ikey->ino)); - } - default: - return snprintf_null(buf, size, "[unknown type %u len %u]", - type, key->key_len); + return snprintf_null(buf, size, "[fs type %u?]", type); } - return snprintf_null(buf, size, "[truncated type %u len %u]", + return snprintf_null(buf, size, "[fs type %u trunc len %u]", type, key->key_len); } diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c index ca4bcf22..21d7d68e 100644 --- a/kmod/src/kvec.c +++ b/kmod/src/kvec.c @@ -246,7 +246,7 @@ void scoutfs_kvec_set_max_key(struct kvec *kvec) { __u8 *type = kvec[0].iov_base; - *type = SCOUTFS_MAX_UNUSED_KEY; + *type = 255; scoutfs_kvec_init(kvec, type, 1); } diff --git a/kmod/src/net.c b/kmod/src/net.c index d542bd5d..f6323d8b 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -165,9 +165,9 @@ struct sock_info { * XXX instead of magic keys in the main fs resource we could have * another resource that contains the server locks. */ -static u8 listen_type = SCOUTFS_NET_LISTEN_KEY; +static u8 listen_type = SCOUTFS_NET_LISTEN_TYPE; static struct scoutfs_key_buf listen_key; -static u8 addr_type = SCOUTFS_NET_ADDR_KEY; +static u8 addr_type = SCOUTFS_NET_ADDR_TYPE; static struct scoutfs_key_buf addr_key; static int send_msg(struct socket *sock, void *buf, unsigned len) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7b1ecff4..4f12afa1 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -33,17 +33,6 @@ struct scoutfs_sb_info; -#define show_key_type(type) \ - __print_symbolic(type, \ - { SCOUTFS_INODE_KEY, "INODE" }, \ - { SCOUTFS_XATTR_KEY, "XATTR" }, \ - { SCOUTFS_DIRENT_KEY, "DIRENT" }, \ - { SCOUTFS_LINK_BACKREF_KEY, "LINK_BACKREF"}, \ - { SCOUTFS_SYMLINK_KEY, "SYMLINK" }, \ - { SCOUTFS_EXTENT_KEY, "EXTENT" }) - -#define TRACE_KEYF "%llu.%s.%llu" - TRACE_EVENT(scoutfs_write_begin, TP_PROTO(u64 ino, loff_t pos, unsigned len), diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index fc8a4af7..92ccce63 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -87,8 +87,9 @@ static struct scoutfs_key_buf *alloc_xattr_key(struct super_block *sb, xkey = key->data; foot = xattr_key_footer(key); - xkey->type = SCOUTFS_XATTR_KEY; + xkey->zone = SCOUTFS_FS_ZONE; xkey->ino = cpu_to_be64(ino); + xkey->type = SCOUTFS_XATTR_TYPE; if (name && name_len) memcpy(xkey->name, name, name_len); From 6de2bfc1c5877fcde3d5474a2495a7070d5e3923 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 8 Jul 2017 11:56:05 -0700 Subject: [PATCH 329/920] scoutfs: use the dlm mode/levels directly We intend to use more of the dlm lock levels. Let's use its modes directly so we don't have to maintain a mental map from differently named modes. Signed-off-by: Zach Brown --- kmod/src/lock.c | 8 ++++---- kmod/src/lock.h | 6 ------ kmod/src/net.c | 2 +- kmod/src/scoutfs_trace.h | 13 +++++++++---- kmod/src/xattr.c | 8 ++++---- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 3e368879..b6ee02fc 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -78,7 +78,7 @@ static int invalidate_caches(struct super_block *sb, int mode, if (ret) return ret; - if (mode == SCOUTFS_LOCK_MODE_WRITE) + if (mode == DLM_LOCK_EX) ret = scoutfs_item_invalidate(sb, start, end); return ret; @@ -121,7 +121,7 @@ static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock, RB_CLEAR_NODE(&lock->interval_node); lock->sb = sb; - lock->mode = SCOUTFS_LOCK_MODE_IV; + lock->mode = DLM_LOCK_IV; INIT_DELAYED_WORK(&lock->dc_work, scoutfs_downconvert_func); INIT_LIST_HEAD(&lock->lru_entry); @@ -476,7 +476,7 @@ static void scoutfs_downconvert_func(struct work_struct *work) * invalidate based on what level we're downconverting to (PR, * NL). */ - invalidate_caches(sb, SCOUTFS_LOCK_MODE_WRITE, lock->start, lock->end); + invalidate_caches(sb, DLM_LOCK_EX, lock->start, lock->end); unlock_range(sb, lock); spin_lock(&linfo->lock); @@ -489,7 +489,7 @@ static void scoutfs_downconvert_func(struct work_struct *work) * lock tree so in particular we have nobody in * scoutfs_lock_range concurrently trying to acquire a lock. */ - if (lock->mode == SCOUTFS_LOCK_MODE_IV && lock->refcnt == 1 && + if (lock->mode == DLM_LOCK_IV && lock->refcnt == 1 && list_empty(&lock->lru_entry)) { list_add_tail(&lock->lru_entry, &linfo->lru_list); linfo->lru_nr++; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 2dcdb648..38eba1e7 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -25,12 +25,6 @@ struct scoutfs_lock { struct delayed_work dc_work; }; -enum { - SCOUTFS_LOCK_MODE_IV = DLM_LOCK_IV, - SCOUTFS_LOCK_MODE_READ = DLM_LOCK_PR, - SCOUTFS_LOCK_MODE_WRITE = DLM_LOCK_EX, -}; - int scoutfs_lock_range(struct super_block *sb, int mode, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, diff --git a/kmod/src/net.c b/kmod/src/net.c index f6323d8b..0a741631 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -2083,7 +2083,7 @@ static void scoutfs_net_server_func(struct work_struct *work) INIT_WORK(&sinf->listen_work, scoutfs_net_listen_func); INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); - ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, &listen_key, + ret = scoutfs_lock_range(sb, DLM_LOCK_EX, &listen_key, &listen_key, &sinf->listen_lck); if (ret) { kfree(sinf); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 4f12afa1..4ab822d4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -284,10 +284,15 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_read_items, TP_ARGS(sb, start, end) ); -#define lock_mode(mode) \ - __print_symbolic(mode, \ - { SCOUTFS_LOCK_MODE_READ, "READ" }, \ - { SCOUTFS_LOCK_MODE_WRITE, "WRITE" }) +#define lock_mode(mode) \ + __print_symbolic(mode, \ + { DLM_LOCK_IV, "IV" }, \ + { DLM_LOCK_NL, "NL" }, \ + { DLM_LOCK_CR, "CR" }, \ + { DLM_LOCK_CW, "CW" }, \ + { DLM_LOCK_PR, "PR" }, \ + { DLM_LOCK_PW, "PW" }, \ + { DLM_LOCK_EX, "EX" }) DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 92ccce63..df9923ac 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -178,7 +178,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, goto out; } - ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_READ, key, last, &lck); + ret = scoutfs_lock_range(sb, DLM_LOCK_PR, key, last, &lck); if (ret) goto out; @@ -289,7 +289,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } - ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, key, last, &lck); + ret = scoutfs_lock_range(sb, DLM_LOCK_EX, key, last, &lck); if (ret) goto out; @@ -386,7 +386,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) xkey = key->data; xkey->name[0] = '\0'; - ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_READ, key, last, &lck); + ret = scoutfs_lock_range(sb, DLM_LOCK_PR, key, last, &lck); if (ret) goto out; @@ -469,7 +469,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) } /* while we read to delete we need to writeback others */ - ret = scoutfs_lock_range(sb, SCOUTFS_LOCK_MODE_WRITE, key, last, &lck); + ret = scoutfs_lock_range(sb, DLM_LOCK_EX, key, last, &lck); if (ret) goto out; From 8a42a4d75abd36ec21780b723e1a8d0f1e17c4cf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 11 Jul 2017 16:03:51 -0700 Subject: [PATCH 330/920] scoutfs: introduce lock names Instead of locking one resource with ranges we'll have callers map their logical resources to a tuple name that we'll store in lock resources. The names still map to ranges for cache reading and cache invalidation but the ranges aren't exposed to the DLM. This lets us use the stock DLM and distribute resources across masters. Signed-off-by: Zach Brown --- kmod/src/format.h | 13 +++ kmod/src/lock.c | 211 ++++++++++++++++++++++----------------- kmod/src/lock.h | 17 ++-- kmod/src/net.c | 5 +- kmod/src/scoutfs_trace.h | 29 +++--- kmod/src/xattr.c | 16 +-- 6 files changed, 167 insertions(+), 124 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 23bd959c..6251c8f8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -506,6 +506,19 @@ enum { #define SCOUTFS_XATTR_MAX_PARTS \ DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) +/* + * structures used by dlm + */ +struct scoutfs_lock_name { + __u8 zone; + __u8 type; + __le64 first; + __le64 second; +} __packed; + +#define SCOUTFS_LOCK_INODE_GROUP_NR 1024 +#define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) +#define SCOUTFS_LOCK_INODE_GROUP_OFFSET (~0ULL) /* * messages over the wire. diff --git a/kmod/src/lock.c b/kmod/src/lock.c index b6ee02fc..77336cbc 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -14,16 +14,19 @@ #include #include #include +#include #include "super.h" #include "lock.h" #include "item.h" #include "scoutfs_trace.h" #include "msg.h" +#include "cmp.h" -#include "../dlm/interval_tree_generic.h" - -#include "linux/dlm.h" +#define LN_FMT "%u.%u.%llu.%llu" +#define LN_ARG(name) \ + (name)->zone, (name)->type, le64_to_cpu((name)->first), \ + le64_to_cpu((name)->second) /* * allocated per-super, freed on unmount. @@ -45,21 +48,11 @@ struct lock_info { unsigned long long lru_nr; }; -#define RANGE_LOCK_RESOURCE "fs_range" -#define RANGE_LOCK_RESOURCE_LEN (strlen(RANGE_LOCK_RESOURCE)) - #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info static void scoutfs_downconvert_func(struct work_struct *work); -#define START(lock) ((lock)->start) -#define LAST(lock) ((lock)->end) -KEYED_INTERVAL_TREE_DEFINE(struct scoutfs_lock, interval_node, - struct scoutfs_key_buf *, subtree_last, START, LAST, - scoutfs_key_compare, static, scoutfs_lock); - - /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -86,9 +79,11 @@ static int invalidate_caches(struct super_block *sb, int mode, static void free_scoutfs_lock(struct scoutfs_lock *lock) { - kfree(lock->start); - kfree(lock->end); - kfree(lock); + if (lock) { + scoutfs_key_free(lock->sb, lock->start); + scoutfs_key_free(lock->sb, lock->end); + kfree(lock); + } } static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) @@ -103,7 +98,7 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) if (!refs) { BUG_ON(lock->holders); BUG_ON(delayed_work_pending(&lock->dc_work)); - scoutfs_lock_remove(lock, &linfo->lock_tree); + rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); spin_unlock(&linfo->lock); free_scoutfs_lock(lock); @@ -113,85 +108,93 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) } } -static void init_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) -{ - DECLARE_LOCK_INFO(sb, linfo); - - RB_CLEAR_NODE(&lock->interval_node); - lock->sb = sb; - lock->mode = DLM_LOCK_IV; - INIT_DELAYED_WORK(&lock->dc_work, scoutfs_downconvert_func); - INIT_LIST_HEAD(&lock->lru_entry); - - if (start) { - lock->start = start; - lock->dlm_start.val = start->data; - lock->dlm_start.len = start->key_len; - } - if (end) { - lock->end = end; - lock->dlm_end.val = end->data; - lock->dlm_end.len = end->key_len; - } - - spin_lock(&linfo->lock); - lock->sequence = ++linfo->seq_cnt; - spin_unlock(&linfo->lock); -} - static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, + struct scoutfs_lock_name *lock_name, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end) { - struct scoutfs_key_buf *s, *e; struct scoutfs_lock *lock; - s = scoutfs_key_dup(sb, start); - if (!s) - return NULL; - e = scoutfs_key_dup(sb, end); - if (!e) { - kfree(s); - return NULL; - } lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); - if (!lock) { - kfree(e); - kfree(s); + if (lock) { + lock->start = scoutfs_key_dup(sb, start); + lock->end = scoutfs_key_dup(sb, end); + if (!lock->start || !lock->end) { + free_scoutfs_lock(lock); + lock = NULL; + } else { + RB_CLEAR_NODE(&lock->node); + lock->sb = sb; + lock->lock_name = *lock_name; + lock->mode = DLM_LOCK_IV; + INIT_DELAYED_WORK(&lock->dc_work, + scoutfs_downconvert_func); + INIT_LIST_HEAD(&lock->lru_entry); + } } - init_scoutfs_lock(sb, lock, s, e); return lock; } +static int cmp_lock_names(struct scoutfs_lock_name *a, + struct scoutfs_lock_name *b) +{ + return (int)a->zone - (int)b->zone ?: + (int)a->type - (int)b->type ?: + scoutfs_cmp_u64s(le64_to_cpu(a->first), le64_to_cpu(b->first)) ?: + scoutfs_cmp_u64s(le64_to_cpu(b->second), le64_to_cpu(b->second)); +} + static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_lock_name *lock_name, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) { DECLARE_LOCK_INFO(sb, linfo); - struct scoutfs_lock *found, *new; + struct scoutfs_lock *new = NULL; + struct scoutfs_lock *found; + struct scoutfs_lock *lock; + struct rb_node *parent; + struct rb_node **node; + int cmp; - new = NULL; - spin_lock(&linfo->lock); search: - found = scoutfs_lock_iter_first(&linfo->lock_tree, start, end); + spin_lock(&linfo->lock); + node = &linfo->lock_tree.rb_node; + parent = NULL; + found = NULL; + while (*node) { + parent = *node; + lock = container_of(*node, struct scoutfs_lock, node); + + cmp = cmp_lock_names(lock_name, &lock->lock_name); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + found = lock; + break; + } + lock = NULL; + } + if (!found) { if (!new) { spin_unlock(&linfo->lock); - new = alloc_scoutfs_lock(sb, start, end); + new = alloc_scoutfs_lock(sb, lock_name, start, end); if (!new) return NULL; - spin_lock(&linfo->lock); goto search; } - new->refcnt = 1; /* Freed by shrinker or on umount */ - scoutfs_lock_insert(new, &linfo->lock_tree); found = new; new = NULL; + found->refcnt = 1; /* Freed by shrinker or on umount */ + found->sequence = ++linfo->seq_cnt; + rb_link_node(&found->node, parent, node); + rb_insert_color(&found->node, &linfo->lock_tree); } found->refcnt++; if (!list_empty(&found->lru_entry)) { @@ -227,7 +230,7 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) WARN_ON(lock->refcnt != 1); WARN_ON(lock->flags & SCOUTFS_LOCK_QUEUED); - scoutfs_lock_remove(lock, &linfo->lock_tree); + rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); list_add_tail(&lock->lru_entry, &list); linfo->lru_nr--; @@ -251,7 +254,7 @@ static void free_lock_tree(struct super_block *sb) while (node) { struct scoutfs_lock *lock; - lock = rb_entry(node, struct scoutfs_lock, interval_node); + lock = rb_entry(node, struct scoutfs_lock, node); node = rb_next(node); put_scoutfs_lock(sb, lock); } @@ -296,13 +299,12 @@ static void set_lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock queue_blocking_work(linfo, lock, seconds); } -static void scoutfs_rbast(void *astarg, int mode, - struct dlm_key *start, struct dlm_key *end) +static void scoutfs_bast(void *astarg, int mode) { struct scoutfs_lock *lock = astarg; struct lock_info *linfo = SCOUTFS_SB(lock->sb)->lock_info; - trace_scoutfs_rbast(lock->sb, lock); + trace_scoutfs_bast(lock->sb, lock); spin_lock(&linfo->lock); set_lock_blocking(linfo, lock, 0); @@ -341,20 +343,21 @@ static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock) * The caller provides the opaque lock structure used for storage and * their start and end pointers will be accessed while the lock is held. */ -int scoutfs_lock_range(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct scoutfs_lock **ret_lock) +static int lock_name_keys(struct super_block *sb, int mode, + struct scoutfs_lock_name *lock_name, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_lock **ret_lock) { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; int ret; - lock = find_alloc_scoutfs_lock(sb, start, end); + lock = find_alloc_scoutfs_lock(sb, lock_name, start, end); if (!lock) return -ENOMEM; - trace_scoutfs_lock_range(sb, lock); + trace_scoutfs_lock_resource(sb, lock); check_lock_state: spin_lock(&linfo->lock); @@ -391,13 +394,12 @@ check_lock_state: lock->holders++; spin_unlock(&linfo->lock); - ret = dlm_lock_range(linfo->ls, mode, &lock->dlm_start, &lock->dlm_end, - &lock->lksb, DLM_LKF_NOORDER, RANGE_LOCK_RESOURCE, - RANGE_LOCK_RESOURCE_LEN, 0, scoutfs_ast, lock, - scoutfs_rbast); + ret = dlm_lock(linfo->ls, mode, &lock->lksb, DLM_LKF_NOORDER, + &lock->lock_name, sizeof(struct scoutfs_lock_name), + 0, scoutfs_ast, lock, scoutfs_bast); if (ret) { - scoutfs_err(sb, "Error %d locking %s\n", ret, - RANGE_LOCK_RESOURCE); + scoutfs_err(sb, "Error %d locking "LN_FMT, ret, + LN_ARG(&lock->lock_name)); put_scoutfs_lock(sb, lock); return ret; } @@ -408,12 +410,41 @@ out: return 0; } -void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lock) +int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, + struct scoutfs_lock **ret_lock) +{ + struct scoutfs_lock_name lock_name; + struct scoutfs_inode_key start_ikey; + struct scoutfs_inode_key end_ikey; + struct scoutfs_key_buf start; + struct scoutfs_key_buf end; + + ino &= ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK; + + lock_name.zone = SCOUTFS_FS_ZONE; + lock_name.type = SCOUTFS_INODE_TYPE; + lock_name.first = cpu_to_le64(ino); + lock_name.second = 0; + + start_ikey.zone = SCOUTFS_FS_ZONE; + start_ikey.ino = cpu_to_be64(ino); + start_ikey.type = 0; + scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); + + end_ikey.zone = SCOUTFS_FS_ZONE; + end_ikey.ino = cpu_to_be64(ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1); + end_ikey.type = ~0; + scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); + + return lock_name_keys(sb, mode, &lock_name, &start, &end, ret_lock); +} + +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); unsigned int seconds = 60; - trace_scoutfs_unlock_range(sb, lock); + trace_scoutfs_unlock(sb, lock); spin_lock(&linfo->lock); lock->holders--; @@ -432,7 +463,7 @@ static void unlock_range(struct super_block *sb, struct scoutfs_lock *lock) DECLARE_LOCK_INFO(sb, linfo); int ret; - trace_scoutfs_unlock_range(sb, lock); + trace_scoutfs_unlock(sb, lock); BUG_ON(!lock->sequence); @@ -441,8 +472,8 @@ static void unlock_range(struct super_block *sb, struct scoutfs_lock *lock) spin_unlock(&linfo->lock); ret = dlm_unlock(linfo->ls, lock->lksb.sb_lkid, 0, &lock->lksb, lock); if (ret) { - scoutfs_err(sb, "Error %d unlocking %s\n", ret, - RANGE_LOCK_RESOURCE); + scoutfs_err(sb, "Error %d unlocking "LN_FMT, ret, + LN_ARG(&lock->lock_name)); goto out; } diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 38eba1e7..e08bcfd2 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -1,23 +1,22 @@ #ifndef _SCOUTFS_LOCK_H_ #define _SCOUTFS_LOCK_H_ -#include "../dlm/include/linux/dlm.h" +#include +#include "key.h" #define SCOUTFS_LOCK_BLOCKING 0x01 /* Blocking another lock request */ #define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ struct scoutfs_lock { struct super_block *sb; + struct scoutfs_lock_name lock_name; struct scoutfs_key_buf *start; struct scoutfs_key_buf *end; int mode; int rqmode; struct dlm_lksb lksb; - struct dlm_key dlm_start; - struct dlm_key dlm_end; unsigned int sequence; /* for debugging and sanity checks */ - struct rb_node interval_node; - struct scoutfs_key_buf *subtree_last; + struct rb_node node; struct list_head lru_entry; unsigned int refcnt; unsigned int holders; /* Tracks active users of this lock */ @@ -25,11 +24,9 @@ struct scoutfs_lock { struct delayed_work dc_work; }; -int scoutfs_lock_range(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct scoutfs_lock **ret_lock); -void scoutfs_unlock_range(struct super_block *sb, struct scoutfs_lock *lock); +int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, + struct scoutfs_lock **ret_lock); +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock); int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, void *caller_lvb, unsigned lvb_len); diff --git a/kmod/src/net.c b/kmod/src/net.c index 0a741631..e35cea27 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1272,7 +1272,7 @@ static void scoutfs_net_shutdown_func(struct work_struct *work) scoutfs_err(sb, "Non-fatal error %d while writing server " "address\n", ret); - scoutfs_unlock_range(sb, sinf->listen_lck); + scoutfs_unlock(sb, sinf->listen_lck); queue_delayed_work(nti->proc_wq, &nti->server_work, 0); } if (sinf == nti->connected_sinf) { @@ -2083,8 +2083,7 @@ static void scoutfs_net_server_func(struct work_struct *work) INIT_WORK(&sinf->listen_work, scoutfs_net_listen_func); INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); - ret = scoutfs_lock_range(sb, DLM_LOCK_EX, &listen_key, - &listen_key, &sinf->listen_lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, 0, &sinf->listen_lck); if (ret) { kfree(sinf); goto out; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 4ab822d4..6f36eada 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -298,39 +298,42 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck), TP_STRUCT__entry( + __field(u8, name_zone) + __field(u8, name_type) + __field(u64, name_first) + __field(u64, name_second) __field(int, mode) __field(int, rqmode) __field(unsigned int, seq) - __dynamic_array(char, start, scoutfs_key_str(NULL, lck->start)) - __dynamic_array(char, end, scoutfs_key_str(NULL, lck->end)) __field(unsigned int, flags) __field(unsigned int, refcnt) __field(unsigned int, holders) ), TP_fast_assign( + __entry->name_zone = lck->lock_name.zone; + __entry->name_type = lck->lock_name.type; + __entry->name_first = le64_to_cpu(lck->lock_name.first); + __entry->name_second = le64_to_cpu(lck->lock_name.second); __entry->mode = lck->mode; __entry->rqmode = lck->rqmode; __entry->seq = lck->sequence; __entry->flags = lck->flags; __entry->refcnt = lck->refcnt; __entry->holders = lck->holders; - scoutfs_key_str(__get_dynamic_array(start), lck->start); - scoutfs_key_str(__get_dynamic_array(end), lck->end); ), - TP_printk("seq %u refs %d holders %d mode %s rqmode %s flags 0x%x " - "start %s end %s", - __entry->seq, __entry->refcnt, __entry->holders, - lock_mode(__entry->mode), lock_mode(__entry->rqmode), - __entry->flags, __get_str(start), - __get_str(end)) + TP_printk("name %u.%u.%llu.%llu seq %u refs %d holders %d mode %s rqmode %s flags 0x%x", + __entry->name_zone, __entry->name_type, __entry->name_first, + __entry->name_second, __entry->seq, + __entry->refcnt, __entry->holders, lock_mode(__entry->mode), + lock_mode(__entry->rqmode), __entry->flags) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_range, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock_range, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); @@ -340,7 +343,7 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_ast, TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_rbast, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_bast, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index df9923ac..29aba33d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -178,7 +178,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, goto out; } - ret = scoutfs_lock_range(sb, DLM_LOCK_PR, key, last, &lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), &lck); if (ret) goto out; @@ -229,7 +229,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = -ERANGE; up_read(&si->xattr_rwsem); - scoutfs_unlock_range(sb, lck); + scoutfs_unlock(sb, lck); out: scoutfs_key_free(sb, key); @@ -289,7 +289,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } - ret = scoutfs_lock_range(sb, DLM_LOCK_EX, key, last, &lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), &lck); if (ret) goto out; @@ -336,7 +336,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, scoutfs_release_trans(sb); unlock: - scoutfs_unlock_range(sb, lck); + scoutfs_unlock(sb, lck); out: scoutfs_item_free_batch(sb, &list); @@ -386,7 +386,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) xkey = key->data; xkey->name[0] = '\0'; - ret = scoutfs_lock_range(sb, DLM_LOCK_PR, key, last, &lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), &lck); if (ret) goto out; @@ -436,7 +436,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) } up_read(&si->xattr_rwsem); - scoutfs_unlock_range(sb, lck); + scoutfs_unlock(sb, lck); out: scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); @@ -469,7 +469,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) } /* while we read to delete we need to writeback others */ - ret = scoutfs_lock_range(sb, DLM_LOCK_EX, key, last, &lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ino, &lck); if (ret) goto out; @@ -490,7 +490,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* don't need to increment past deleted key */ } - scoutfs_unlock_range(sb, lck); + scoutfs_unlock(sb, lck); out: scoutfs_key_free(sb, key); From 11a857011787770ab165311acaa1e46dfb855ccf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 8 Jul 2017 11:51:15 -0700 Subject: [PATCH 331/920] scoutfs: remove our copy of the dlm We don't need the dlm to track key ranges if we implement ranges by mapping keys to resources which represent ranges of the key space. Signed-off-by: Zach Brown --- kmod/.gitignore | 3 - kmod/Makefile | 6 - kmod/dlm/Kconfig | 16 - kmod/dlm/Makefile | 20 - kmod/dlm/ast.c | 375 -- kmod/dlm/ast.h | 33 - kmod/dlm/config.c | 1025 --- kmod/dlm/config.h | 53 - kmod/dlm/debug_fs.c | 815 --- kmod/dlm/dir.c | 308 - kmod/dlm/dir.h | 25 - kmod/dlm/dlm_internal.h | 776 --- kmod/dlm/dlmtest.c | 307 - kmod/dlm/include/linux/dlm.h | 193 - kmod/dlm/include/linux/dlm_plock.h | 19 - kmod/dlm/include/uapi/linux/dlm.h | 75 - kmod/dlm/include/uapi/linux/dlm_device.h | 108 - kmod/dlm/include/uapi/linux/dlm_netlink.h | 58 - kmod/dlm/include/uapi/linux/dlm_plock.h | 45 - kmod/dlm/include/uapi/linux/dlmconstants.h | 163 - kmod/dlm/interval_tree_generic.h | 216 - kmod/dlm/lock.c | 6605 -------------------- kmod/dlm/lock.h | 82 - kmod/dlm/lockspace.c | 906 --- kmod/dlm/lockspace.h | 26 - kmod/dlm/lowcomms.c | 1726 ----- kmod/dlm/lowcomms.h | 27 - kmod/dlm/lvb_table.h | 18 - kmod/dlm/main.c | 98 - kmod/dlm/member.c | 725 --- kmod/dlm/member.h | 33 - kmod/dlm/memory.c | 96 - kmod/dlm/memory.h | 27 - kmod/dlm/midcomms.c | 137 - kmod/dlm/midcomms.h | 21 - kmod/dlm/netlink.c | 141 - kmod/dlm/plock.c | 515 -- kmod/dlm/rcom.c | 656 -- kmod/dlm/rcom.h | 26 - kmod/dlm/recover.c | 955 --- kmod/dlm/recover.h | 34 - kmod/dlm/recoverd.c | 342 - kmod/dlm/recoverd.h | 23 - kmod/dlm/requestqueue.c | 171 - kmod/dlm/requestqueue.h | 22 - kmod/dlm/user.c | 1028 --- kmod/dlm/user.h | 19 - kmod/dlm/util.c | 172 - kmod/dlm/util.h | 22 - 49 files changed, 19292 deletions(-) delete mode 100644 kmod/dlm/Kconfig delete mode 100644 kmod/dlm/Makefile delete mode 100644 kmod/dlm/ast.c delete mode 100644 kmod/dlm/ast.h delete mode 100644 kmod/dlm/config.c delete mode 100644 kmod/dlm/config.h delete mode 100644 kmod/dlm/debug_fs.c delete mode 100644 kmod/dlm/dir.c delete mode 100644 kmod/dlm/dir.h delete mode 100644 kmod/dlm/dlm_internal.h delete mode 100644 kmod/dlm/dlmtest.c delete mode 100644 kmod/dlm/include/linux/dlm.h delete mode 100644 kmod/dlm/include/linux/dlm_plock.h delete mode 100644 kmod/dlm/include/uapi/linux/dlm.h delete mode 100644 kmod/dlm/include/uapi/linux/dlm_device.h delete mode 100644 kmod/dlm/include/uapi/linux/dlm_netlink.h delete mode 100644 kmod/dlm/include/uapi/linux/dlm_plock.h delete mode 100644 kmod/dlm/include/uapi/linux/dlmconstants.h delete mode 100644 kmod/dlm/interval_tree_generic.h delete mode 100644 kmod/dlm/lock.c delete mode 100644 kmod/dlm/lock.h delete mode 100644 kmod/dlm/lockspace.c delete mode 100644 kmod/dlm/lockspace.h delete mode 100644 kmod/dlm/lowcomms.c delete mode 100644 kmod/dlm/lowcomms.h delete mode 100644 kmod/dlm/lvb_table.h delete mode 100644 kmod/dlm/main.c delete mode 100644 kmod/dlm/member.c delete mode 100644 kmod/dlm/member.h delete mode 100644 kmod/dlm/memory.c delete mode 100644 kmod/dlm/memory.h delete mode 100644 kmod/dlm/midcomms.c delete mode 100644 kmod/dlm/midcomms.h delete mode 100644 kmod/dlm/netlink.c delete mode 100644 kmod/dlm/plock.c delete mode 100644 kmod/dlm/rcom.c delete mode 100644 kmod/dlm/rcom.h delete mode 100644 kmod/dlm/recover.c delete mode 100644 kmod/dlm/recover.h delete mode 100644 kmod/dlm/recoverd.c delete mode 100644 kmod/dlm/recoverd.h delete mode 100644 kmod/dlm/requestqueue.c delete mode 100644 kmod/dlm/requestqueue.h delete mode 100644 kmod/dlm/user.c delete mode 100644 kmod/dlm/user.h delete mode 100644 kmod/dlm/util.c delete mode 100644 kmod/dlm/util.h diff --git a/kmod/.gitignore b/kmod/.gitignore index 03621d2d..6117d0be 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -4,9 +4,6 @@ *.cmd *~ src/.tmp_versions/ -dlm/.tmp_versions/ src/Module.symvers -dlm/Module.symvers src/modules.order -dlm/modules.order cscope.* diff --git a/kmod/Makefile b/kmod/Makefile index ada76969..11bc4d4a 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -13,18 +13,12 @@ SP = @: endif SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) -I $(CURDIR)/dlm/include M=$(CURDIR)/src -DLM_ARGS := CONFIG_DLM=m CONFIG_DLM_DEBUG=y -C $(SK_KSRC) M=$(CURDIR)/dlm all: module module: - make $(DLM_ARGS) - cp $(CURDIR)/dlm/Module.symvers $(CURDIR)/src/ make $(SCOUTFS_ARGS) $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) -# Do not enable until we can clean up some warnings -# $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(DLM_ARGS) clean: make $(SCOUTFS_ARGS) clean - make $(DLM_ARGS) clean diff --git a/kmod/dlm/Kconfig b/kmod/dlm/Kconfig deleted file mode 100644 index e4242c3f..00000000 --- a/kmod/dlm/Kconfig +++ /dev/null @@ -1,16 +0,0 @@ -menuconfig DLM - tristate "Distributed Lock Manager (DLM)" - depends on INET - depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n) - select IP_SCTP - help - A general purpose distributed lock manager for kernel or userspace - applications. - -config DLM_DEBUG - bool "DLM debugging" - depends on DLM - help - Under the debugfs mount point, the name of each lockspace will - appear as a file in the "dlm" directory. The output is the - list of resource and locks the local node knows about. diff --git a/kmod/dlm/Makefile b/kmod/dlm/Makefile deleted file mode 100644 index 5710f265..00000000 --- a/kmod/dlm/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -obj-$(CONFIG_DLM) += dlm.o dlmtest.o -dlm-y := ast.o \ - config.o \ - dir.o \ - lock.o \ - lockspace.o \ - main.o \ - member.o \ - memory.o \ - midcomms.o \ - netlink.o \ - lowcomms.o \ - plock.o \ - rcom.o \ - recover.o \ - recoverd.o \ - requestqueue.o \ - user.o \ - util.o -dlm-$(CONFIG_DLM_DEBUG) += debug_fs.o diff --git a/kmod/dlm/ast.c b/kmod/dlm/ast.c deleted file mode 100644 index 19630d43..00000000 --- a/kmod/dlm/ast.c +++ /dev/null @@ -1,375 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lock.h" -#include "user.h" - -static uint64_t dlm_cb_seq; -static DEFINE_SPINLOCK(dlm_cb_seq_spin); - -static void dlm_dump_lkb_callbacks(struct dlm_lkb *lkb) -{ - int i; - - log_print("last_bast %x %llu flags %x mode %d sb %d %x", - lkb->lkb_id, - (unsigned long long)lkb->lkb_last_bast.seq, - lkb->lkb_last_bast.flags, - lkb->lkb_last_bast.mode, - lkb->lkb_last_bast.sb_status, - lkb->lkb_last_bast.sb_flags); - - log_print("last_cast %x %llu flags %x mode %d sb %d %x", - lkb->lkb_id, - (unsigned long long)lkb->lkb_last_cast.seq, - lkb->lkb_last_cast.flags, - lkb->lkb_last_cast.mode, - lkb->lkb_last_cast.sb_status, - lkb->lkb_last_cast.sb_flags); - - for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { - log_print("cb %x %llu flags %x mode %d sb %d %x", - lkb->lkb_id, - (unsigned long long)lkb->lkb_callbacks[i].seq, - lkb->lkb_callbacks[i].flags, - lkb->lkb_callbacks[i].mode, - lkb->lkb_callbacks[i].sb_status, - lkb->lkb_callbacks[i].sb_flags); - } -} - -static void fixup_cb_pointers(struct dlm_callback *cb) -{ - struct dlm_key *start = &cb->start; - struct dlm_key *end = &cb->end; - - cb->range.start = start; - cb->range.end = end; - start->val = &cb->startval; - end->val = &cb->endval; -} - -/* - * Range must not be NULL for DLM_CB_BAST. - */ -int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, - struct dlm_range *range, int status, uint32_t sbflags, - uint64_t seq) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - uint64_t prev_seq; - int prev_mode; - int i, rv; - struct dlm_range *prev_range; - - if ((flags & DLM_CB_BAST) && !range) { - /* XXX: user.c doesn't handle this yet, fail for now */ - WARN_ON_ONCE(1); - return -EINVAL; - } - - for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { - if (lkb->lkb_callbacks[i].seq) - continue; - - /* - * Suppress some redundant basts here, do more on removal. - * Don't even add a bast if the callback just before it - * is a bast for the same mode or a more restrictive mode. - * (the addional > PR check is needed for PR/CW inversion) - */ - - if ((i > 0) && (flags & DLM_CB_BAST) && - (lkb->lkb_callbacks[i-1].flags & DLM_CB_BAST)) { - - prev_seq = lkb->lkb_callbacks[i-1].seq; - prev_mode = lkb->lkb_callbacks[i-1].mode; - prev_range = &lkb->lkb_callbacks[i-1].range; - - /* Below check needs to look at range */ - if (ranges_overlap(prev_range, range) && - ((prev_mode == mode) || - (prev_mode > mode && prev_mode > DLM_LOCK_PR))) { - - log_debug(ls, "skip %x add bast %llu mode %d " - "for bast %llu mode %d", - lkb->lkb_id, - (unsigned long long)seq, - mode, - (unsigned long long)prev_seq, - prev_mode); - rv = 0; - goto out; - } - } - - lkb->lkb_callbacks[i].seq = seq; - lkb->lkb_callbacks[i].flags = flags; - lkb->lkb_callbacks[i].mode = mode; - lkb->lkb_callbacks[i].sb_status = status; - lkb->lkb_callbacks[i].sb_flags = (sbflags & 0x000000FF); - - if (range) { - struct dlm_key *start = &lkb->lkb_callbacks[i].start; - struct dlm_key *end = &lkb->lkb_callbacks[i].end; - - lkb->lkb_callbacks[i].range.start = start; - lkb->lkb_callbacks[i].range.end = end; - - start->len = range->start->len; - start->val = &lkb->lkb_callbacks[i].startval; - end->len = range->end->len; - end->val = &lkb->lkb_callbacks[i].endval; - memcpy(start->val, range->start->val, range->start->len); - memcpy(end->val, range->end->val, range->end->len); - } - rv = 0; - break; - } - - if (i == DLM_CALLBACKS_SIZE) { - log_error(ls, "no callbacks %x %llu flags %x mode %d sb %d %x", - lkb->lkb_id, (unsigned long long)seq, - flags, mode, status, sbflags); - dlm_dump_lkb_callbacks(lkb); - rv = -1; - goto out; - } - out: - return rv; -} - -int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_callback *cb, int *resid) -{ - int i, rv; - - *resid = 0; - - if (!lkb->lkb_callbacks[0].seq) { - rv = -ENOENT; - goto out; - } - - /* oldest undelivered cb is callbacks[0] */ - - memcpy(cb, &lkb->lkb_callbacks[0], sizeof(struct dlm_callback)); - memset(&lkb->lkb_callbacks[0], 0, sizeof(struct dlm_callback)); - fixup_cb_pointers(cb); - - /* shift others down */ - - for (i = 1; i < DLM_CALLBACKS_SIZE; i++) { - if (!lkb->lkb_callbacks[i].seq) - break; - memcpy(&lkb->lkb_callbacks[i-1], &lkb->lkb_callbacks[i], - sizeof(struct dlm_callback)); - memset(&lkb->lkb_callbacks[i], 0, sizeof(struct dlm_callback)); - (*resid)++; - } - - /* if cb is a bast, it should be skipped if the blocking mode is - compatible with the last granted mode */ - - if ((cb->flags & DLM_CB_BAST) && lkb->lkb_last_cast.seq) { - if (dlm_modes_compat(cb->mode, lkb->lkb_last_cast.mode)) { - cb->flags |= DLM_CB_SKIP; - - log_debug(ls, "skip %x bast %llu mode %d " - "for cast %llu mode %d", - lkb->lkb_id, - (unsigned long long)cb->seq, - cb->mode, - (unsigned long long)lkb->lkb_last_cast.seq, - lkb->lkb_last_cast.mode); - rv = 0; - goto out; - } - } - - if (cb->flags & DLM_CB_CAST) { - memcpy(&lkb->lkb_last_cast, cb, sizeof(struct dlm_callback)); - lkb->lkb_last_cast_time = ktime_get(); - } - - if (cb->flags & DLM_CB_BAST) { - memcpy(&lkb->lkb_last_bast, cb, sizeof(struct dlm_callback)); - lkb->lkb_last_bast_time = ktime_get(); - } - rv = 0; - out: - return rv; -} - -void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, - struct dlm_range *range, int status, uint32_t sbflags) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - uint64_t new_seq, prev_seq; - int rv; - - spin_lock(&dlm_cb_seq_spin); - new_seq = ++dlm_cb_seq; - spin_unlock(&dlm_cb_seq_spin); - - if (lkb->lkb_flags & DLM_IFL_USER) { - dlm_user_add_ast(lkb, flags, mode, status, sbflags, new_seq); - return; - } - - mutex_lock(&lkb->lkb_cb_mutex); - prev_seq = lkb->lkb_callbacks[0].seq; - - rv = dlm_add_lkb_callback(lkb, flags, mode, range, status, sbflags, - new_seq); - if (rv < 0) - goto out; - - if (!prev_seq) { - kref_get(&lkb->lkb_ref); - - if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) { - mutex_lock(&ls->ls_cb_mutex); - list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay); - mutex_unlock(&ls->ls_cb_mutex); - } else { - queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work); - } - } - out: - mutex_unlock(&lkb->lkb_cb_mutex); -} - -void dlm_callback_work(struct work_struct *work) -{ - struct dlm_lkb *lkb = container_of(work, struct dlm_lkb, lkb_cb_work); - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - void (*castfn) (void *astparam); - void (*bastfn) (void *astparam, int mode); - void (*rbastfn) (void *astarg, int mode, struct dlm_key *start, - struct dlm_key *end); - /* - * XXX: This used to be on the stack, but the inline buffers - * added for range support blow out our stack. - * - * struct dlm_callback callbacks[DLM_CALLBACKS_SIZE]; - */ - struct dlm_callback *callbacks; - int i, rv, resid; - - callbacks = kcalloc(DLM_CALLBACKS_SIZE, sizeof(*callbacks), GFP_NOFS); - WARN_ON_ONCE(!callbacks); - if (!callbacks) - return; - - mutex_lock(&lkb->lkb_cb_mutex); - if (!lkb->lkb_callbacks[0].seq) { - /* no callback work exists, shouldn't happen */ - log_error(ls, "dlm_callback_work %x no work", lkb->lkb_id); - dlm_print_lkb(lkb); - dlm_dump_lkb_callbacks(lkb); - } - - for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { - rv = dlm_rem_lkb_callback(ls, lkb, &callbacks[i], &resid); - if (rv < 0) - break; - } - - if (resid) { - /* cbs remain, loop should have removed all, shouldn't happen */ - log_error(ls, "dlm_callback_work %x resid %d", lkb->lkb_id, - resid); - dlm_print_lkb(lkb); - dlm_dump_lkb_callbacks(lkb); - } - mutex_unlock(&lkb->lkb_cb_mutex); - - castfn = lkb->lkb_astfn; - bastfn = lkb->lkb_bastfn; - rbastfn = lkb->lkb_rbastfn; - - for (i = 0; i < DLM_CALLBACKS_SIZE; i++) { - if (!callbacks[i].seq) - break; - if (callbacks[i].flags & DLM_CB_SKIP) { - continue; - } else if (callbacks[i].flags & DLM_CB_BAST) { - if (rbastfn) - rbastfn(lkb->lkb_astparam, callbacks[i].mode, - &callbacks[i].start, &callbacks[i].end); - else - bastfn(lkb->lkb_astparam, callbacks[i].mode); - } else if (callbacks[i].flags & DLM_CB_CAST) { - lkb->lkb_lksb->sb_status = callbacks[i].sb_status; - lkb->lkb_lksb->sb_flags = callbacks[i].sb_flags; - castfn(lkb->lkb_astparam); - } - } - - /* undo kref_get from dlm_add_callback, may cause lkb to be freed */ - dlm_put_lkb(lkb); - kfree(callbacks); -} - -int dlm_callback_start(struct dlm_ls *ls) -{ - ls->ls_callback_wq = alloc_workqueue("dlm_callback", - WQ_UNBOUND | - WQ_MEM_RECLAIM | - WQ_NON_REENTRANT, - 0); - if (!ls->ls_callback_wq) { - log_print("can't start dlm_callback workqueue"); - return -ENOMEM; - } - return 0; -} - -void dlm_callback_stop(struct dlm_ls *ls) -{ - if (ls->ls_callback_wq) - destroy_workqueue(ls->ls_callback_wq); -} - -void dlm_callback_suspend(struct dlm_ls *ls) -{ - set_bit(LSFL_CB_DELAY, &ls->ls_flags); - - if (ls->ls_callback_wq) - flush_workqueue(ls->ls_callback_wq); -} - -void dlm_callback_resume(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb, *safe; - int count = 0; - - clear_bit(LSFL_CB_DELAY, &ls->ls_flags); - - if (!ls->ls_callback_wq) - return; - - mutex_lock(&ls->ls_cb_mutex); - list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) { - list_del_init(&lkb->lkb_cb_list); - queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work); - count++; - } - mutex_unlock(&ls->ls_cb_mutex); - - if (count) - log_debug(ls, "dlm_callback_resume %d", count); -} - diff --git a/kmod/dlm/ast.h b/kmod/dlm/ast.h deleted file mode 100644 index 09fc3934..00000000 --- a/kmod/dlm/ast.h +++ /dev/null @@ -1,33 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __ASTD_DOT_H__ -#define __ASTD_DOT_H__ - -void dlm_del_ast(struct dlm_lkb *lkb); -int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, - struct dlm_range *range, int status, uint32_t sbflags, - uint64_t seq); -int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_callback *cb, int *resid); -void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, - struct dlm_range *range, int status, uint32_t sbflags); - -void dlm_callback_work(struct work_struct *work); -int dlm_callback_start(struct dlm_ls *ls); -void dlm_callback_stop(struct dlm_ls *ls); -void dlm_callback_suspend(struct dlm_ls *ls); -void dlm_callback_resume(struct dlm_ls *ls); - -#endif - - diff --git a/kmod/dlm/config.c b/kmod/dlm/config.c deleted file mode 100644 index a6662120..00000000 --- a/kmod/dlm/config.c +++ /dev/null @@ -1,1025 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "config.h" -#include "lowcomms.h" - -/* - * /config/dlm//spaces//nodes//nodeid - * /config/dlm//spaces//nodes//weight - * /config/dlm//comms//nodeid - * /config/dlm//comms//local - * /config/dlm//comms//addr (write only) - * /config/dlm//comms//addr_list (read only) - * The level is useless, but I haven't figured out how to avoid it. - */ - -static struct config_group *space_list; -static struct config_group *comm_list; -static struct dlm_comm *local_comm; -static uint32_t dlm_comm_count; - -struct dlm_clusters; -struct dlm_cluster; -struct dlm_spaces; -struct dlm_space; -struct dlm_comms; -struct dlm_comm; -struct dlm_nodes; -struct dlm_node; - -static struct config_group *make_cluster(struct config_group *, const char *); -static void drop_cluster(struct config_group *, struct config_item *); -static void release_cluster(struct config_item *); -static struct config_group *make_space(struct config_group *, const char *); -static void drop_space(struct config_group *, struct config_item *); -static void release_space(struct config_item *); -static struct config_item *make_comm(struct config_group *, const char *); -static void drop_comm(struct config_group *, struct config_item *); -static void release_comm(struct config_item *); -static struct config_item *make_node(struct config_group *, const char *); -static void drop_node(struct config_group *, struct config_item *); -static void release_node(struct config_item *); - -static ssize_t show_cluster(struct config_item *i, struct configfs_attribute *a, - char *buf); -static ssize_t store_cluster(struct config_item *i, - struct configfs_attribute *a, - const char *buf, size_t len); -static ssize_t show_comm(struct config_item *i, struct configfs_attribute *a, - char *buf); -static ssize_t store_comm(struct config_item *i, struct configfs_attribute *a, - const char *buf, size_t len); -static ssize_t show_node(struct config_item *i, struct configfs_attribute *a, - char *buf); -static ssize_t store_node(struct config_item *i, struct configfs_attribute *a, - const char *buf, size_t len); - -static ssize_t comm_nodeid_read(struct dlm_comm *cm, char *buf); -static ssize_t comm_nodeid_write(struct dlm_comm *cm, const char *buf, - size_t len); -static ssize_t comm_local_read(struct dlm_comm *cm, char *buf); -static ssize_t comm_local_write(struct dlm_comm *cm, const char *buf, - size_t len); -static ssize_t comm_addr_write(struct dlm_comm *cm, const char *buf, - size_t len); -static ssize_t comm_addr_list_read(struct dlm_comm *cm, char *buf); -static ssize_t node_nodeid_read(struct dlm_node *nd, char *buf); -static ssize_t node_nodeid_write(struct dlm_node *nd, const char *buf, - size_t len); -static ssize_t node_weight_read(struct dlm_node *nd, char *buf); -static ssize_t node_weight_write(struct dlm_node *nd, const char *buf, - size_t len); - -struct dlm_cluster { - struct config_group group; - unsigned int cl_tcp_port; - unsigned int cl_buffer_size; - unsigned int cl_rsbtbl_size; - unsigned int cl_recover_timer; - unsigned int cl_toss_secs; - unsigned int cl_scan_secs; - unsigned int cl_log_debug; - unsigned int cl_protocol; - unsigned int cl_timewarn_cs; - unsigned int cl_waitwarn_us; - unsigned int cl_new_rsb_count; - unsigned int cl_recover_callbacks; - char cl_cluster_name[DLM_LOCKSPACE_LEN]; -}; - -enum { - CLUSTER_ATTR_TCP_PORT = 0, - CLUSTER_ATTR_BUFFER_SIZE, - CLUSTER_ATTR_RSBTBL_SIZE, - CLUSTER_ATTR_RECOVER_TIMER, - CLUSTER_ATTR_TOSS_SECS, - CLUSTER_ATTR_SCAN_SECS, - CLUSTER_ATTR_LOG_DEBUG, - CLUSTER_ATTR_PROTOCOL, - CLUSTER_ATTR_TIMEWARN_CS, - CLUSTER_ATTR_WAITWARN_US, - CLUSTER_ATTR_NEW_RSB_COUNT, - CLUSTER_ATTR_RECOVER_CALLBACKS, - CLUSTER_ATTR_CLUSTER_NAME, -}; - -struct cluster_attribute { - struct configfs_attribute attr; - ssize_t (*show)(struct dlm_cluster *, char *); - ssize_t (*store)(struct dlm_cluster *, const char *, size_t); -}; - -static ssize_t cluster_cluster_name_read(struct dlm_cluster *cl, char *buf) -{ - return sprintf(buf, "%s\n", cl->cl_cluster_name); -} - -static ssize_t cluster_cluster_name_write(struct dlm_cluster *cl, - const char *buf, size_t len) -{ - strncpy(dlm_config.ci_cluster_name, buf, DLM_LOCKSPACE_LEN); - strncpy(cl->cl_cluster_name, buf, DLM_LOCKSPACE_LEN); - return len; -} - -static struct cluster_attribute cluster_attr_cluster_name = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "cluster_name", - .ca_mode = S_IRUGO | S_IWUSR }, - .show = cluster_cluster_name_read, - .store = cluster_cluster_name_write, -}; - -static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field, - int *info_field, int check_zero, - const char *buf, size_t len) -{ - unsigned int x; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - x = simple_strtoul(buf, NULL, 0); - - if (check_zero && !x) - return -EINVAL; - - *cl_field = x; - *info_field = x; - - return len; -} - -#define CLUSTER_ATTR(name, check_zero) \ -static ssize_t name##_write(struct dlm_cluster *cl, const char *buf, size_t len) \ -{ \ - return cluster_set(cl, &cl->cl_##name, &dlm_config.ci_##name, \ - check_zero, buf, len); \ -} \ -static ssize_t name##_read(struct dlm_cluster *cl, char *buf) \ -{ \ - return snprintf(buf, PAGE_SIZE, "%u\n", cl->cl_##name); \ -} \ -static struct cluster_attribute cluster_attr_##name = \ -__CONFIGFS_ATTR(name, 0644, name##_read, name##_write) - -CLUSTER_ATTR(tcp_port, 1); -CLUSTER_ATTR(buffer_size, 1); -CLUSTER_ATTR(rsbtbl_size, 1); -CLUSTER_ATTR(recover_timer, 1); -CLUSTER_ATTR(toss_secs, 1); -CLUSTER_ATTR(scan_secs, 1); -CLUSTER_ATTR(log_debug, 0); -CLUSTER_ATTR(protocol, 0); -CLUSTER_ATTR(timewarn_cs, 1); -CLUSTER_ATTR(waitwarn_us, 0); -CLUSTER_ATTR(new_rsb_count, 0); -CLUSTER_ATTR(recover_callbacks, 0); - -static struct configfs_attribute *cluster_attrs[] = { - [CLUSTER_ATTR_TCP_PORT] = &cluster_attr_tcp_port.attr, - [CLUSTER_ATTR_BUFFER_SIZE] = &cluster_attr_buffer_size.attr, - [CLUSTER_ATTR_RSBTBL_SIZE] = &cluster_attr_rsbtbl_size.attr, - [CLUSTER_ATTR_RECOVER_TIMER] = &cluster_attr_recover_timer.attr, - [CLUSTER_ATTR_TOSS_SECS] = &cluster_attr_toss_secs.attr, - [CLUSTER_ATTR_SCAN_SECS] = &cluster_attr_scan_secs.attr, - [CLUSTER_ATTR_LOG_DEBUG] = &cluster_attr_log_debug.attr, - [CLUSTER_ATTR_PROTOCOL] = &cluster_attr_protocol.attr, - [CLUSTER_ATTR_TIMEWARN_CS] = &cluster_attr_timewarn_cs.attr, - [CLUSTER_ATTR_WAITWARN_US] = &cluster_attr_waitwarn_us.attr, - [CLUSTER_ATTR_NEW_RSB_COUNT] = &cluster_attr_new_rsb_count.attr, - [CLUSTER_ATTR_RECOVER_CALLBACKS] = &cluster_attr_recover_callbacks.attr, - [CLUSTER_ATTR_CLUSTER_NAME] = &cluster_attr_cluster_name.attr, - NULL, -}; - -enum { - COMM_ATTR_NODEID = 0, - COMM_ATTR_LOCAL, - COMM_ATTR_ADDR, - COMM_ATTR_ADDR_LIST, -}; - -struct comm_attribute { - struct configfs_attribute attr; - ssize_t (*show)(struct dlm_comm *, char *); - ssize_t (*store)(struct dlm_comm *, const char *, size_t); -}; - -static struct comm_attribute comm_attr_nodeid = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "nodeid", - .ca_mode = S_IRUGO | S_IWUSR }, - .show = comm_nodeid_read, - .store = comm_nodeid_write, -}; - -static struct comm_attribute comm_attr_local = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "local", - .ca_mode = S_IRUGO | S_IWUSR }, - .show = comm_local_read, - .store = comm_local_write, -}; - -static struct comm_attribute comm_attr_addr = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "addr", - .ca_mode = S_IWUSR }, - .store = comm_addr_write, -}; - -static struct comm_attribute comm_attr_addr_list = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "addr_list", - .ca_mode = S_IRUGO }, - .show = comm_addr_list_read, -}; - -static struct configfs_attribute *comm_attrs[] = { - [COMM_ATTR_NODEID] = &comm_attr_nodeid.attr, - [COMM_ATTR_LOCAL] = &comm_attr_local.attr, - [COMM_ATTR_ADDR] = &comm_attr_addr.attr, - [COMM_ATTR_ADDR_LIST] = &comm_attr_addr_list.attr, - NULL, -}; - -enum { - NODE_ATTR_NODEID = 0, - NODE_ATTR_WEIGHT, -}; - -struct node_attribute { - struct configfs_attribute attr; - ssize_t (*show)(struct dlm_node *, char *); - ssize_t (*store)(struct dlm_node *, const char *, size_t); -}; - -static struct node_attribute node_attr_nodeid = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "nodeid", - .ca_mode = S_IRUGO | S_IWUSR }, - .show = node_nodeid_read, - .store = node_nodeid_write, -}; - -static struct node_attribute node_attr_weight = { - .attr = { .ca_owner = THIS_MODULE, - .ca_name = "weight", - .ca_mode = S_IRUGO | S_IWUSR }, - .show = node_weight_read, - .store = node_weight_write, -}; - -static struct configfs_attribute *node_attrs[] = { - [NODE_ATTR_NODEID] = &node_attr_nodeid.attr, - [NODE_ATTR_WEIGHT] = &node_attr_weight.attr, - NULL, -}; - -struct dlm_clusters { - struct configfs_subsystem subsys; -}; - -struct dlm_spaces { - struct config_group ss_group; -}; - -struct dlm_space { - struct config_group group; - struct list_head members; - struct mutex members_lock; - int members_count; -}; - -struct dlm_comms { - struct config_group cs_group; -}; - -struct dlm_comm { - struct config_item item; - int seq; - int nodeid; - int local; - int addr_count; - struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; -}; - -struct dlm_nodes { - struct config_group ns_group; -}; - -struct dlm_node { - struct config_item item; - struct list_head list; /* space->members */ - int nodeid; - int weight; - int new; - int comm_seq; /* copy of cm->seq when nd->nodeid is set */ -}; - -static struct configfs_group_operations clusters_ops = { - .make_group = make_cluster, - .drop_item = drop_cluster, -}; - -static struct configfs_item_operations cluster_ops = { - .release = release_cluster, - .show_attribute = show_cluster, - .store_attribute = store_cluster, -}; - -static struct configfs_group_operations spaces_ops = { - .make_group = make_space, - .drop_item = drop_space, -}; - -static struct configfs_item_operations space_ops = { - .release = release_space, -}; - -static struct configfs_group_operations comms_ops = { - .make_item = make_comm, - .drop_item = drop_comm, -}; - -static struct configfs_item_operations comm_ops = { - .release = release_comm, - .show_attribute = show_comm, - .store_attribute = store_comm, -}; - -static struct configfs_group_operations nodes_ops = { - .make_item = make_node, - .drop_item = drop_node, -}; - -static struct configfs_item_operations node_ops = { - .release = release_node, - .show_attribute = show_node, - .store_attribute = store_node, -}; - -static struct config_item_type clusters_type = { - .ct_group_ops = &clusters_ops, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type cluster_type = { - .ct_item_ops = &cluster_ops, - .ct_attrs = cluster_attrs, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type spaces_type = { - .ct_group_ops = &spaces_ops, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type space_type = { - .ct_item_ops = &space_ops, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type comms_type = { - .ct_group_ops = &comms_ops, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type comm_type = { - .ct_item_ops = &comm_ops, - .ct_attrs = comm_attrs, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type nodes_type = { - .ct_group_ops = &nodes_ops, - .ct_owner = THIS_MODULE, -}; - -static struct config_item_type node_type = { - .ct_item_ops = &node_ops, - .ct_attrs = node_attrs, - .ct_owner = THIS_MODULE, -}; - -static struct dlm_cluster *config_item_to_cluster(struct config_item *i) -{ - return i ? container_of(to_config_group(i), struct dlm_cluster, group) : - NULL; -} - -static struct dlm_space *config_item_to_space(struct config_item *i) -{ - return i ? container_of(to_config_group(i), struct dlm_space, group) : - NULL; -} - -static struct dlm_comm *config_item_to_comm(struct config_item *i) -{ - return i ? container_of(i, struct dlm_comm, item) : NULL; -} - -static struct dlm_node *config_item_to_node(struct config_item *i) -{ - return i ? container_of(i, struct dlm_node, item) : NULL; -} - -static struct config_group *make_cluster(struct config_group *g, - const char *name) -{ - struct dlm_cluster *cl = NULL; - struct dlm_spaces *sps = NULL; - struct dlm_comms *cms = NULL; - void *gps = NULL; - - cl = kzalloc(sizeof(struct dlm_cluster), GFP_NOFS); - gps = kcalloc(3, sizeof(struct config_group *), GFP_NOFS); - sps = kzalloc(sizeof(struct dlm_spaces), GFP_NOFS); - cms = kzalloc(sizeof(struct dlm_comms), GFP_NOFS); - - if (!cl || !gps || !sps || !cms) - goto fail; - - config_group_init_type_name(&cl->group, name, &cluster_type); - config_group_init_type_name(&sps->ss_group, "spaces", &spaces_type); - config_group_init_type_name(&cms->cs_group, "comms", &comms_type); - - cl->group.default_groups = gps; - cl->group.default_groups[0] = &sps->ss_group; - cl->group.default_groups[1] = &cms->cs_group; - cl->group.default_groups[2] = NULL; - - cl->cl_tcp_port = dlm_config.ci_tcp_port; - cl->cl_buffer_size = dlm_config.ci_buffer_size; - cl->cl_rsbtbl_size = dlm_config.ci_rsbtbl_size; - cl->cl_recover_timer = dlm_config.ci_recover_timer; - cl->cl_toss_secs = dlm_config.ci_toss_secs; - cl->cl_scan_secs = dlm_config.ci_scan_secs; - cl->cl_log_debug = dlm_config.ci_log_debug; - cl->cl_protocol = dlm_config.ci_protocol; - cl->cl_timewarn_cs = dlm_config.ci_timewarn_cs; - cl->cl_waitwarn_us = dlm_config.ci_waitwarn_us; - cl->cl_new_rsb_count = dlm_config.ci_new_rsb_count; - cl->cl_recover_callbacks = dlm_config.ci_recover_callbacks; - memcpy(cl->cl_cluster_name, dlm_config.ci_cluster_name, - DLM_LOCKSPACE_LEN); - - space_list = &sps->ss_group; - comm_list = &cms->cs_group; - return &cl->group; - - fail: - kfree(cl); - kfree(gps); - kfree(sps); - kfree(cms); - return ERR_PTR(-ENOMEM); -} - -static void drop_cluster(struct config_group *g, struct config_item *i) -{ - struct dlm_cluster *cl = config_item_to_cluster(i); - struct config_item *tmp; - int j; - - for (j = 0; cl->group.default_groups[j]; j++) { - tmp = &cl->group.default_groups[j]->cg_item; - cl->group.default_groups[j] = NULL; - config_item_put(tmp); - } - - space_list = NULL; - comm_list = NULL; - - config_item_put(i); -} - -static void release_cluster(struct config_item *i) -{ - struct dlm_cluster *cl = config_item_to_cluster(i); - kfree(cl->group.default_groups); - kfree(cl); -} - -static struct config_group *make_space(struct config_group *g, const char *name) -{ - struct dlm_space *sp = NULL; - struct dlm_nodes *nds = NULL; - void *gps = NULL; - - sp = kzalloc(sizeof(struct dlm_space), GFP_NOFS); - gps = kcalloc(2, sizeof(struct config_group *), GFP_NOFS); - nds = kzalloc(sizeof(struct dlm_nodes), GFP_NOFS); - - if (!sp || !gps || !nds) - goto fail; - - config_group_init_type_name(&sp->group, name, &space_type); - config_group_init_type_name(&nds->ns_group, "nodes", &nodes_type); - - sp->group.default_groups = gps; - sp->group.default_groups[0] = &nds->ns_group; - sp->group.default_groups[1] = NULL; - - INIT_LIST_HEAD(&sp->members); - mutex_init(&sp->members_lock); - sp->members_count = 0; - return &sp->group; - - fail: - kfree(sp); - kfree(gps); - kfree(nds); - return ERR_PTR(-ENOMEM); -} - -static void drop_space(struct config_group *g, struct config_item *i) -{ - struct dlm_space *sp = config_item_to_space(i); - struct config_item *tmp; - int j; - - /* assert list_empty(&sp->members) */ - - for (j = 0; sp->group.default_groups[j]; j++) { - tmp = &sp->group.default_groups[j]->cg_item; - sp->group.default_groups[j] = NULL; - config_item_put(tmp); - } - - config_item_put(i); -} - -static void release_space(struct config_item *i) -{ - struct dlm_space *sp = config_item_to_space(i); - kfree(sp->group.default_groups); - kfree(sp); -} - -static struct config_item *make_comm(struct config_group *g, const char *name) -{ - struct dlm_comm *cm; - - cm = kzalloc(sizeof(struct dlm_comm), GFP_NOFS); - if (!cm) - return ERR_PTR(-ENOMEM); - - config_item_init_type_name(&cm->item, name, &comm_type); - - cm->seq = dlm_comm_count++; - if (!cm->seq) - cm->seq = dlm_comm_count++; - - cm->nodeid = -1; - cm->local = 0; - cm->addr_count = 0; - return &cm->item; -} - -static void drop_comm(struct config_group *g, struct config_item *i) -{ - struct dlm_comm *cm = config_item_to_comm(i); - if (local_comm == cm) - local_comm = NULL; - dlm_lowcomms_close(cm->nodeid); - while (cm->addr_count--) - kfree(cm->addr[cm->addr_count]); - config_item_put(i); -} - -static void release_comm(struct config_item *i) -{ - struct dlm_comm *cm = config_item_to_comm(i); - kfree(cm); -} - -static struct config_item *make_node(struct config_group *g, const char *name) -{ - struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); - struct dlm_node *nd; - - nd = kzalloc(sizeof(struct dlm_node), GFP_NOFS); - if (!nd) - return ERR_PTR(-ENOMEM); - - config_item_init_type_name(&nd->item, name, &node_type); - nd->nodeid = -1; - nd->weight = 1; /* default weight of 1 if none is set */ - nd->new = 1; /* set to 0 once it's been read by dlm_nodeid_list() */ - - mutex_lock(&sp->members_lock); - list_add(&nd->list, &sp->members); - sp->members_count++; - mutex_unlock(&sp->members_lock); - - return &nd->item; -} - -static void drop_node(struct config_group *g, struct config_item *i) -{ - struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); - struct dlm_node *nd = config_item_to_node(i); - - mutex_lock(&sp->members_lock); - list_del(&nd->list); - sp->members_count--; - mutex_unlock(&sp->members_lock); - - config_item_put(i); -} - -static void release_node(struct config_item *i) -{ - struct dlm_node *nd = config_item_to_node(i); - kfree(nd); -} - -static struct dlm_clusters clusters_root = { - .subsys = { - .su_group = { - .cg_item = { - .ci_namebuf = "dlm", - .ci_type = &clusters_type, - }, - }, - }, -}; - -int __init dlm_config_init(void) -{ - config_group_init(&clusters_root.subsys.su_group); - mutex_init(&clusters_root.subsys.su_mutex); - return configfs_register_subsystem(&clusters_root.subsys); -} - -void dlm_config_exit(void) -{ - configfs_unregister_subsystem(&clusters_root.subsys); -} - -/* - * Functions for user space to read/write attributes - */ - -static ssize_t show_cluster(struct config_item *i, struct configfs_attribute *a, - char *buf) -{ - struct dlm_cluster *cl = config_item_to_cluster(i); - struct cluster_attribute *cla = - container_of(a, struct cluster_attribute, attr); - return cla->show ? cla->show(cl, buf) : 0; -} - -static ssize_t store_cluster(struct config_item *i, - struct configfs_attribute *a, - const char *buf, size_t len) -{ - struct dlm_cluster *cl = config_item_to_cluster(i); - struct cluster_attribute *cla = - container_of(a, struct cluster_attribute, attr); - return cla->store ? cla->store(cl, buf, len) : -EINVAL; -} - -static ssize_t show_comm(struct config_item *i, struct configfs_attribute *a, - char *buf) -{ - struct dlm_comm *cm = config_item_to_comm(i); - struct comm_attribute *cma = - container_of(a, struct comm_attribute, attr); - return cma->show ? cma->show(cm, buf) : 0; -} - -static ssize_t store_comm(struct config_item *i, struct configfs_attribute *a, - const char *buf, size_t len) -{ - struct dlm_comm *cm = config_item_to_comm(i); - struct comm_attribute *cma = - container_of(a, struct comm_attribute, attr); - return cma->store ? cma->store(cm, buf, len) : -EINVAL; -} - -static ssize_t comm_nodeid_read(struct dlm_comm *cm, char *buf) -{ - return sprintf(buf, "%d\n", cm->nodeid); -} - -static ssize_t comm_nodeid_write(struct dlm_comm *cm, const char *buf, - size_t len) -{ - cm->nodeid = simple_strtol(buf, NULL, 0); - return len; -} - -static ssize_t comm_local_read(struct dlm_comm *cm, char *buf) -{ - return sprintf(buf, "%d\n", cm->local); -} - -static ssize_t comm_local_write(struct dlm_comm *cm, const char *buf, - size_t len) -{ - cm->local= simple_strtol(buf, NULL, 0); - if (cm->local && !local_comm) - local_comm = cm; - return len; -} - -static ssize_t comm_addr_write(struct dlm_comm *cm, const char *buf, size_t len) -{ - struct sockaddr_storage *addr; - int rv; - - if (len != sizeof(struct sockaddr_storage)) - return -EINVAL; - - if (cm->addr_count >= DLM_MAX_ADDR_COUNT) - return -ENOSPC; - - addr = kzalloc(sizeof(*addr), GFP_NOFS); - if (!addr) - return -ENOMEM; - - memcpy(addr, buf, len); - - rv = dlm_lowcomms_addr(cm->nodeid, addr, len); - if (rv) { - kfree(addr); - return rv; - } - - cm->addr[cm->addr_count++] = addr; - return len; -} - -static ssize_t comm_addr_list_read(struct dlm_comm *cm, char *buf) -{ - ssize_t s; - ssize_t allowance; - int i; - struct sockaddr_storage *addr; - struct sockaddr_in *addr_in; - struct sockaddr_in6 *addr_in6; - - /* Taken from ip6_addr_string() defined in lib/vsprintf.c */ - char buf0[sizeof("AF_INET6 xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255\n")]; - - - /* Derived from SIMPLE_ATTR_SIZE of fs/configfs/file.c */ - allowance = 4096; - buf[0] = '\0'; - - for (i = 0; i < cm->addr_count; i++) { - addr = cm->addr[i]; - - switch(addr->ss_family) { - case AF_INET: - addr_in = (struct sockaddr_in *)addr; - s = sprintf(buf0, "AF_INET %pI4\n", &addr_in->sin_addr.s_addr); - break; - case AF_INET6: - addr_in6 = (struct sockaddr_in6 *)addr; - s = sprintf(buf0, "AF_INET6 %pI6\n", &addr_in6->sin6_addr); - break; - default: - s = sprintf(buf0, "%s\n", ""); - break; - } - allowance -= s; - if (allowance >= 0) - strcat(buf, buf0); - else { - allowance += s; - break; - } - } - return 4096 - allowance; -} - -static ssize_t show_node(struct config_item *i, struct configfs_attribute *a, - char *buf) -{ - struct dlm_node *nd = config_item_to_node(i); - struct node_attribute *nda = - container_of(a, struct node_attribute, attr); - return nda->show ? nda->show(nd, buf) : 0; -} - -static ssize_t store_node(struct config_item *i, struct configfs_attribute *a, - const char *buf, size_t len) -{ - struct dlm_node *nd = config_item_to_node(i); - struct node_attribute *nda = - container_of(a, struct node_attribute, attr); - return nda->store ? nda->store(nd, buf, len) : -EINVAL; -} - -static ssize_t node_nodeid_read(struct dlm_node *nd, char *buf) -{ - return sprintf(buf, "%d\n", nd->nodeid); -} - -static ssize_t node_nodeid_write(struct dlm_node *nd, const char *buf, - size_t len) -{ - uint32_t seq = 0; - nd->nodeid = simple_strtol(buf, NULL, 0); - dlm_comm_seq(nd->nodeid, &seq); - nd->comm_seq = seq; - return len; -} - -static ssize_t node_weight_read(struct dlm_node *nd, char *buf) -{ - return sprintf(buf, "%d\n", nd->weight); -} - -static ssize_t node_weight_write(struct dlm_node *nd, const char *buf, - size_t len) -{ - nd->weight = simple_strtol(buf, NULL, 0); - return len; -} - -/* - * Functions for the dlm to get the info that's been configured - */ - -static struct dlm_space *get_space(char *name) -{ - struct config_item *i; - - if (!space_list) - return NULL; - - mutex_lock(&space_list->cg_subsys->su_mutex); - i = config_group_find_item(space_list, name); - mutex_unlock(&space_list->cg_subsys->su_mutex); - - return config_item_to_space(i); -} - -static void put_space(struct dlm_space *sp) -{ - config_item_put(&sp->group.cg_item); -} - -static struct dlm_comm *get_comm(int nodeid) -{ - struct config_item *i; - struct dlm_comm *cm = NULL; - int found = 0; - - if (!comm_list) - return NULL; - - mutex_lock(&clusters_root.subsys.su_mutex); - - list_for_each_entry(i, &comm_list->cg_children, ci_entry) { - cm = config_item_to_comm(i); - - if (cm->nodeid != nodeid) - continue; - found = 1; - config_item_get(i); - break; - } - mutex_unlock(&clusters_root.subsys.su_mutex); - - if (!found) - cm = NULL; - return cm; -} - -static void put_comm(struct dlm_comm *cm) -{ - config_item_put(&cm->item); -} - -/* caller must free mem */ -int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, - int *count_out) -{ - struct dlm_space *sp; - struct dlm_node *nd; - struct dlm_config_node *nodes, *node; - int rv, count; - - sp = get_space(lsname); - if (!sp) - return -EEXIST; - - mutex_lock(&sp->members_lock); - if (!sp->members_count) { - rv = -EINVAL; - printk(KERN_ERR "dlm: zero members_count\n"); - goto out; - } - - count = sp->members_count; - - nodes = kcalloc(count, sizeof(struct dlm_config_node), GFP_NOFS); - if (!nodes) { - rv = -ENOMEM; - goto out; - } - - node = nodes; - list_for_each_entry(nd, &sp->members, list) { - node->nodeid = nd->nodeid; - node->weight = nd->weight; - node->new = nd->new; - node->comm_seq = nd->comm_seq; - node++; - - nd->new = 0; - } - - *count_out = count; - *nodes_out = nodes; - rv = 0; - out: - mutex_unlock(&sp->members_lock); - put_space(sp); - return rv; -} - -int dlm_comm_seq(int nodeid, uint32_t *seq) -{ - struct dlm_comm *cm = get_comm(nodeid); - if (!cm) - return -EEXIST; - *seq = cm->seq; - put_comm(cm); - return 0; -} - -int dlm_our_nodeid(void) -{ - return local_comm ? local_comm->nodeid : 0; -} - -/* num 0 is first addr, num 1 is second addr */ -int dlm_our_addr(struct sockaddr_storage *addr, int num) -{ - if (!local_comm) - return -1; - if (num + 1 > local_comm->addr_count) - return -1; - memcpy(addr, local_comm->addr[num], sizeof(*addr)); - return 0; -} - -/* Config file defaults */ -#define DEFAULT_TCP_PORT 21064 -#define DEFAULT_BUFFER_SIZE 4096 -#define DEFAULT_RSBTBL_SIZE 1024 -#define DEFAULT_RECOVER_TIMER 5 -#define DEFAULT_TOSS_SECS 10 -#define DEFAULT_SCAN_SECS 5 -#define DEFAULT_LOG_DEBUG 1 -#define DEFAULT_PROTOCOL 0 -#define DEFAULT_TIMEWARN_CS 500 /* 5 sec = 500 centiseconds */ -#define DEFAULT_WAITWARN_US 0 -#define DEFAULT_NEW_RSB_COUNT 128 -#define DEFAULT_RECOVER_CALLBACKS 0 -#define DEFAULT_CLUSTER_NAME "" - -struct dlm_config_info dlm_config = { - .ci_tcp_port = DEFAULT_TCP_PORT, - .ci_buffer_size = DEFAULT_BUFFER_SIZE, - .ci_rsbtbl_size = DEFAULT_RSBTBL_SIZE, - .ci_recover_timer = DEFAULT_RECOVER_TIMER, - .ci_toss_secs = DEFAULT_TOSS_SECS, - .ci_scan_secs = DEFAULT_SCAN_SECS, - .ci_log_debug = DEFAULT_LOG_DEBUG, - .ci_protocol = DEFAULT_PROTOCOL, - .ci_timewarn_cs = DEFAULT_TIMEWARN_CS, - .ci_waitwarn_us = DEFAULT_WAITWARN_US, - .ci_new_rsb_count = DEFAULT_NEW_RSB_COUNT, - .ci_recover_callbacks = DEFAULT_RECOVER_CALLBACKS, - .ci_cluster_name = DEFAULT_CLUSTER_NAME -}; - diff --git a/kmod/dlm/config.h b/kmod/dlm/config.h deleted file mode 100644 index f30697bc..00000000 --- a/kmod/dlm/config.h +++ /dev/null @@ -1,53 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __CONFIG_DOT_H__ -#define __CONFIG_DOT_H__ - -struct dlm_config_node { - int nodeid; - int weight; - int new; - uint32_t comm_seq; -}; - -#define DLM_MAX_ADDR_COUNT 3 - -struct dlm_config_info { - int ci_tcp_port; - int ci_buffer_size; - int ci_rsbtbl_size; - int ci_recover_timer; - int ci_toss_secs; - int ci_scan_secs; - int ci_log_debug; - int ci_protocol; - int ci_timewarn_cs; - int ci_waitwarn_us; - int ci_new_rsb_count; - int ci_recover_callbacks; - char ci_cluster_name[DLM_LOCKSPACE_LEN]; -}; - -extern struct dlm_config_info dlm_config; - -int dlm_config_init(void); -void dlm_config_exit(void); -int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, - int *count_out); -int dlm_comm_seq(int nodeid, uint32_t *seq); -int dlm_our_nodeid(void); -int dlm_our_addr(struct sockaddr_storage *addr, int num); - -#endif /* __CONFIG_DOT_H__ */ - diff --git a/kmod/dlm/debug_fs.c b/kmod/dlm/debug_fs.c deleted file mode 100644 index b969deef..00000000 --- a/kmod/dlm/debug_fs.c +++ /dev/null @@ -1,815 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include -#include -#include -#include -#include -#include - -#include "dlm_internal.h" -#include "lock.h" - -#define DLM_DEBUG_BUF_LEN 4096 -static char debug_buf[DLM_DEBUG_BUF_LEN]; -static struct mutex debug_buf_lock; - -static struct dentry *dlm_root; - -static char *print_lockmode(int mode) -{ - switch (mode) { - case DLM_LOCK_IV: - return "--"; - case DLM_LOCK_NL: - return "NL"; - case DLM_LOCK_CR: - return "CR"; - case DLM_LOCK_CW: - return "CW"; - case DLM_LOCK_PR: - return "PR"; - case DLM_LOCK_PW: - return "PW"; - case DLM_LOCK_EX: - return "EX"; - default: - return "??"; - } -} - -static int print_format1_lock(struct seq_file *s, struct dlm_lkb *lkb, - struct dlm_rsb *res) -{ - seq_printf(s, "%08x %s", lkb->lkb_id, print_lockmode(lkb->lkb_grmode)); - - if (lkb->lkb_status == DLM_LKSTS_CONVERT || - lkb->lkb_status == DLM_LKSTS_WAITING) - seq_printf(s, " (%s)", print_lockmode(lkb->lkb_rqmode)); - - if (lkb->lkb_nodeid) { - if (lkb->lkb_nodeid != res->res_nodeid) - seq_printf(s, " Remote: %3d %08x", lkb->lkb_nodeid, - lkb->lkb_remid); - else - seq_printf(s, " Master: %08x", lkb->lkb_remid); - } - - if (lkb->lkb_wait_type) - seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); - - return seq_printf(s, "\n"); -} - -static int print_format1(struct dlm_rsb *res, struct seq_file *s) -{ - struct dlm_lkb *lkb; - int i, lvblen = res->res_ls->ls_lvblen, recover_list, root_list; - int rv; - - lock_rsb(res); - - rv = seq_printf(s, "\nResource %p Name (len=%d) \"", - res, res->res_length); - if (rv) - goto out; - - for (i = 0; i < res->res_length; i++) { - if (isprint(res->res_name[i])) - seq_printf(s, "%c", res->res_name[i]); - else - seq_printf(s, "%c", '.'); - } - - if (res->res_nodeid > 0) - rv = seq_printf(s, "\" \nLocal Copy, Master is node %d\n", - res->res_nodeid); - else if (res->res_nodeid == 0) - rv = seq_printf(s, "\" \nMaster Copy\n"); - else if (res->res_nodeid == -1) - rv = seq_printf(s, "\" \nLooking up master (lkid %x)\n", - res->res_first_lkid); - else - rv = seq_printf(s, "\" \nInvalid master %d\n", - res->res_nodeid); - if (rv) - goto out; - - /* Print the LVB: */ - if (res->res_lvbptr) { - seq_printf(s, "LVB: "); - for (i = 0; i < lvblen; i++) { - if (i == lvblen / 2) - seq_printf(s, "\n "); - seq_printf(s, "%02x ", - (unsigned char) res->res_lvbptr[i]); - } - if (rsb_flag(res, RSB_VALNOTVALID)) - seq_printf(s, " (INVALID)"); - rv = seq_printf(s, "\n"); - if (rv) - goto out; - } - - root_list = !list_empty(&res->res_root_list); - recover_list = !list_empty(&res->res_recover_list); - - if (root_list || recover_list) { - rv = seq_printf(s, "Recovery: root %d recover %d flags %lx " - "count %d\n", root_list, recover_list, - res->res_flags, res->res_recover_locks_count); - if (rv) - goto out; - } - - /* Print the locks attached to this resource */ - seq_printf(s, "Granted Queue\n"); - list_for_each_entry(lkb, &res->res_grantqueue, lkb_statequeue) { - rv = print_format1_lock(s, lkb, res); - if (rv) - goto out; - } - - seq_printf(s, "Conversion Queue\n"); - list_for_each_entry(lkb, &res->res_convertqueue, lkb_statequeue) { - rv = print_format1_lock(s, lkb, res); - if (rv) - goto out; - } - - seq_printf(s, "Waiting Queue\n"); - list_for_each_entry(lkb, &res->res_waitqueue, lkb_statequeue) { - rv = print_format1_lock(s, lkb, res); - if (rv) - goto out; - } - - if (list_empty(&res->res_lookup)) - goto out; - - seq_printf(s, "Lookup Queue\n"); - list_for_each_entry(lkb, &res->res_lookup, lkb_rsb_lookup) { - rv = seq_printf(s, "%08x %s", lkb->lkb_id, - print_lockmode(lkb->lkb_rqmode)); - if (lkb->lkb_wait_type) - seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); - rv = seq_printf(s, "\n"); - } - out: - unlock_rsb(res); - return rv; -} - -static int print_format2_lock(struct seq_file *s, struct dlm_lkb *lkb, - struct dlm_rsb *r) -{ - u64 xid = 0; - u64 us; - int rv; - - if (lkb->lkb_flags & DLM_IFL_USER) { - if (lkb->lkb_ua) - xid = lkb->lkb_ua->xid; - } - - /* microseconds since lkb was added to current queue */ - us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_timestamp)); - - /* id nodeid remid pid xid exflags flags sts grmode rqmode time_us - r_nodeid r_len r_name */ - - rv = seq_printf(s, "%x %d %x %u %llu %x %x %d %d %d %llu %u %d \"%s\"\n", - lkb->lkb_id, - lkb->lkb_nodeid, - lkb->lkb_remid, - lkb->lkb_ownpid, - (unsigned long long)xid, - lkb->lkb_exflags, - lkb->lkb_flags, - lkb->lkb_status, - lkb->lkb_grmode, - lkb->lkb_rqmode, - (unsigned long long)us, - r->res_nodeid, - r->res_length, - r->res_name); - return rv; -} - -static int print_format2(struct dlm_rsb *r, struct seq_file *s) -{ - struct dlm_lkb *lkb; - int rv = 0; - - lock_rsb(r); - - list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { - rv = print_format2_lock(s, lkb, r); - if (rv) - goto out; - } - - list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { - rv = print_format2_lock(s, lkb, r); - if (rv) - goto out; - } - - list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { - rv = print_format2_lock(s, lkb, r); - if (rv) - goto out; - } - out: - unlock_rsb(r); - return rv; -} - -static int print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb, - int rsb_lookup) -{ - u64 xid = 0; - int rv; - - if (lkb->lkb_flags & DLM_IFL_USER) { - if (lkb->lkb_ua) - xid = lkb->lkb_ua->xid; - } - - rv = seq_printf(s, "lkb %x %d %x %u %llu %x %x %d %d %d %d %d %d %u %llu %llu\n", - lkb->lkb_id, - lkb->lkb_nodeid, - lkb->lkb_remid, - lkb->lkb_ownpid, - (unsigned long long)xid, - lkb->lkb_exflags, - lkb->lkb_flags, - lkb->lkb_status, - lkb->lkb_grmode, - lkb->lkb_rqmode, - lkb->lkb_last_bast.mode, - rsb_lookup, - lkb->lkb_wait_type, - lkb->lkb_lvbseq, - (unsigned long long)ktime_to_ns(lkb->lkb_timestamp), - (unsigned long long)ktime_to_ns(lkb->lkb_last_bast_time)); - return rv; -} - -static int print_format3(struct dlm_rsb *r, struct seq_file *s) -{ - struct dlm_lkb *lkb; - int i, lvblen = r->res_ls->ls_lvblen; - int print_name = 1; - int rv; - - lock_rsb(r); - - rv = seq_printf(s, "rsb %p %d %x %lx %d %d %u %d ", - r, - r->res_nodeid, - r->res_first_lkid, - r->res_flags, - !list_empty(&r->res_root_list), - !list_empty(&r->res_recover_list), - r->res_recover_locks_count, - r->res_length); - if (rv) - goto out; - - for (i = 0; i < r->res_length; i++) { - if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) - print_name = 0; - } - - seq_printf(s, "%s", print_name ? "str " : "hex"); - - for (i = 0; i < r->res_length; i++) { - if (print_name) - seq_printf(s, "%c", r->res_name[i]); - else - seq_printf(s, " %02x", (unsigned char)r->res_name[i]); - } - rv = seq_printf(s, "\n"); - if (rv) - goto out; - - if (!r->res_lvbptr) - goto do_locks; - - seq_printf(s, "lvb %u %d", r->res_lvbseq, lvblen); - - for (i = 0; i < lvblen; i++) - seq_printf(s, " %02x", (unsigned char)r->res_lvbptr[i]); - rv = seq_printf(s, "\n"); - if (rv) - goto out; - - do_locks: - list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { - rv = print_format3_lock(s, lkb, 0); - if (rv) - goto out; - } - - list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { - rv = print_format3_lock(s, lkb, 0); - if (rv) - goto out; - } - - list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { - rv = print_format3_lock(s, lkb, 0); - if (rv) - goto out; - } - - list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) { - rv = print_format3_lock(s, lkb, 1); - if (rv) - goto out; - } - out: - unlock_rsb(r); - return rv; -} - -static int print_format4(struct dlm_rsb *r, struct seq_file *s) -{ - int our_nodeid = dlm_our_nodeid(); - int print_name = 1; - int i, rv; - - lock_rsb(r); - - rv = seq_printf(s, "rsb %p %d %d %d %d %lu %lx %d ", - r, - r->res_nodeid, - r->res_master_nodeid, - r->res_dir_nodeid, - our_nodeid, - r->res_toss_time, - r->res_flags, - r->res_length); - if (rv) - goto out; - - for (i = 0; i < r->res_length; i++) { - if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) - print_name = 0; - } - - seq_printf(s, "%s", print_name ? "str " : "hex"); - - for (i = 0; i < r->res_length; i++) { - if (print_name) - seq_printf(s, "%c", r->res_name[i]); - else - seq_printf(s, " %02x", (unsigned char)r->res_name[i]); - } - rv = seq_printf(s, "\n"); - out: - unlock_rsb(r); - return rv; -} - -struct rsbtbl_iter { - struct dlm_rsb *rsb; - unsigned bucket; - int format; - int header; -}; - -/* seq_printf returns -1 if the buffer is full, and 0 otherwise. - If the buffer is full, seq_printf can be called again, but it - does nothing and just returns -1. So, the these printing routines - periodically check the return value to avoid wasting too much time - trying to print to a full buffer. */ - -static int table_seq_show(struct seq_file *seq, void *iter_ptr) -{ - struct rsbtbl_iter *ri = iter_ptr; - int rv = 0; - - switch (ri->format) { - case 1: - rv = print_format1(ri->rsb, seq); - break; - case 2: - if (ri->header) { - seq_printf(seq, "id nodeid remid pid xid exflags " - "flags sts grmode rqmode time_ms " - "r_nodeid r_len r_name\n"); - ri->header = 0; - } - rv = print_format2(ri->rsb, seq); - break; - case 3: - if (ri->header) { - seq_printf(seq, "version rsb 1.1 lvb 1.1 lkb 1.1\n"); - ri->header = 0; - } - rv = print_format3(ri->rsb, seq); - break; - case 4: - if (ri->header) { - seq_printf(seq, "version 4 rsb 2\n"); - ri->header = 0; - } - rv = print_format4(ri->rsb, seq); - break; - } - - return rv; -} - -static const struct seq_operations format1_seq_ops; -static const struct seq_operations format2_seq_ops; -static const struct seq_operations format3_seq_ops; -static const struct seq_operations format4_seq_ops; - -static void *table_seq_start(struct seq_file *seq, loff_t *pos) -{ - struct rb_root *tree; - struct rb_node *node; - struct dlm_ls *ls = seq->private; - struct rsbtbl_iter *ri; - struct dlm_rsb *r; - loff_t n = *pos; - unsigned bucket, entry; - int toss = (seq->op == &format4_seq_ops); - - bucket = n >> 32; - entry = n & ((1LL << 32) - 1); - - if (bucket >= ls->ls_rsbtbl_size) - return NULL; - - ri = kzalloc(sizeof(struct rsbtbl_iter), GFP_NOFS); - if (!ri) - return NULL; - if (n == 0) - ri->header = 1; - if (seq->op == &format1_seq_ops) - ri->format = 1; - if (seq->op == &format2_seq_ops) - ri->format = 2; - if (seq->op == &format3_seq_ops) - ri->format = 3; - if (seq->op == &format4_seq_ops) - ri->format = 4; - - tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - if (!RB_EMPTY_ROOT(tree)) { - for (node = rb_first(tree); node; node = rb_next(node)) { - r = rb_entry(node, struct dlm_rsb, res_hashnode); - if (!entry--) { - dlm_hold_rsb(r); - ri->rsb = r; - ri->bucket = bucket; - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - return ri; - } - } - } - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - - /* - * move to the first rsb in the next non-empty bucket - */ - - /* zero the entry */ - n &= ~((1LL << 32) - 1); - - while (1) { - bucket++; - n += 1LL << 32; - - if (bucket >= ls->ls_rsbtbl_size) { - kfree(ri); - return NULL; - } - tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - if (!RB_EMPTY_ROOT(tree)) { - node = rb_first(tree); - r = rb_entry(node, struct dlm_rsb, res_hashnode); - dlm_hold_rsb(r); - ri->rsb = r; - ri->bucket = bucket; - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - *pos = n; - return ri; - } - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - } -} - -static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos) -{ - struct dlm_ls *ls = seq->private; - struct rsbtbl_iter *ri = iter_ptr; - struct rb_root *tree; - struct rb_node *next; - struct dlm_rsb *r, *rp; - loff_t n = *pos; - unsigned bucket; - int toss = (seq->op == &format4_seq_ops); - - bucket = n >> 32; - - /* - * move to the next rsb in the same bucket - */ - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - rp = ri->rsb; - next = rb_next(&rp->res_hashnode); - - if (next) { - r = rb_entry(next, struct dlm_rsb, res_hashnode); - dlm_hold_rsb(r); - ri->rsb = r; - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - dlm_put_rsb(rp); - ++*pos; - return ri; - } - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - dlm_put_rsb(rp); - - /* - * move to the first rsb in the next non-empty bucket - */ - - /* zero the entry */ - n &= ~((1LL << 32) - 1); - - while (1) { - bucket++; - n += 1LL << 32; - - if (bucket >= ls->ls_rsbtbl_size) { - kfree(ri); - return NULL; - } - tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep; - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - if (!RB_EMPTY_ROOT(tree)) { - next = rb_first(tree); - r = rb_entry(next, struct dlm_rsb, res_hashnode); - dlm_hold_rsb(r); - ri->rsb = r; - ri->bucket = bucket; - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - *pos = n; - return ri; - } - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - } -} - -static void table_seq_stop(struct seq_file *seq, void *iter_ptr) -{ - struct rsbtbl_iter *ri = iter_ptr; - - if (ri) { - dlm_put_rsb(ri->rsb); - kfree(ri); - } -} - -static const struct seq_operations format1_seq_ops = { - .start = table_seq_start, - .next = table_seq_next, - .stop = table_seq_stop, - .show = table_seq_show, -}; - -static const struct seq_operations format2_seq_ops = { - .start = table_seq_start, - .next = table_seq_next, - .stop = table_seq_stop, - .show = table_seq_show, -}; - -static const struct seq_operations format3_seq_ops = { - .start = table_seq_start, - .next = table_seq_next, - .stop = table_seq_stop, - .show = table_seq_show, -}; - -static const struct seq_operations format4_seq_ops = { - .start = table_seq_start, - .next = table_seq_next, - .stop = table_seq_stop, - .show = table_seq_show, -}; - -static const struct file_operations format1_fops; -static const struct file_operations format2_fops; -static const struct file_operations format3_fops; -static const struct file_operations format4_fops; - -static int table_open(struct inode *inode, struct file *file) -{ - struct seq_file *seq; - int ret = -1; - - if (file->f_op == &format1_fops) - ret = seq_open(file, &format1_seq_ops); - else if (file->f_op == &format2_fops) - ret = seq_open(file, &format2_seq_ops); - else if (file->f_op == &format3_fops) - ret = seq_open(file, &format3_seq_ops); - else if (file->f_op == &format4_fops) - ret = seq_open(file, &format4_seq_ops); - - if (ret) - return ret; - - seq = file->private_data; - seq->private = inode->i_private; /* the dlm_ls */ - return 0; -} - -static const struct file_operations format1_fops = { - .owner = THIS_MODULE, - .open = table_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; - -static const struct file_operations format2_fops = { - .owner = THIS_MODULE, - .open = table_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; - -static const struct file_operations format3_fops = { - .owner = THIS_MODULE, - .open = table_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; - -static const struct file_operations format4_fops = { - .owner = THIS_MODULE, - .open = table_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release -}; - -/* - * dump lkb's on the ls_waiters list - */ -static ssize_t waiters_read(struct file *file, char __user *userbuf, - size_t count, loff_t *ppos) -{ - struct dlm_ls *ls = file->private_data; - struct dlm_lkb *lkb; - size_t len = DLM_DEBUG_BUF_LEN, pos = 0, ret, rv; - - mutex_lock(&debug_buf_lock); - mutex_lock(&ls->ls_waiters_mutex); - memset(debug_buf, 0, sizeof(debug_buf)); - - list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { - ret = snprintf(debug_buf + pos, len - pos, "%x %d %d %s\n", - lkb->lkb_id, lkb->lkb_wait_type, - lkb->lkb_nodeid, lkb->lkb_resource->res_name); - if (ret >= len - pos) - break; - pos += ret; - } - mutex_unlock(&ls->ls_waiters_mutex); - - rv = simple_read_from_buffer(userbuf, count, ppos, debug_buf, pos); - mutex_unlock(&debug_buf_lock); - return rv; -} - -static const struct file_operations waiters_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = waiters_read, - .llseek = default_llseek, -}; - -void dlm_delete_debug_file(struct dlm_ls *ls) -{ - if (ls->ls_debug_rsb_dentry) - debugfs_remove(ls->ls_debug_rsb_dentry); - if (ls->ls_debug_waiters_dentry) - debugfs_remove(ls->ls_debug_waiters_dentry); - if (ls->ls_debug_locks_dentry) - debugfs_remove(ls->ls_debug_locks_dentry); - if (ls->ls_debug_all_dentry) - debugfs_remove(ls->ls_debug_all_dentry); - if (ls->ls_debug_toss_dentry) - debugfs_remove(ls->ls_debug_toss_dentry); -} - -int dlm_create_debug_file(struct dlm_ls *ls) -{ - char name[DLM_LOCKSPACE_LEN+8]; - - /* format 1 */ - - ls->ls_debug_rsb_dentry = debugfs_create_file(ls->ls_name, - S_IFREG | S_IRUGO, - dlm_root, - ls, - &format1_fops); - if (!ls->ls_debug_rsb_dentry) - goto fail; - - /* format 2 */ - - memset(name, 0, sizeof(name)); - snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_locks", ls->ls_name); - - ls->ls_debug_locks_dentry = debugfs_create_file(name, - S_IFREG | S_IRUGO, - dlm_root, - ls, - &format2_fops); - if (!ls->ls_debug_locks_dentry) - goto fail; - - /* format 3 */ - - memset(name, 0, sizeof(name)); - snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_all", ls->ls_name); - - ls->ls_debug_all_dentry = debugfs_create_file(name, - S_IFREG | S_IRUGO, - dlm_root, - ls, - &format3_fops); - if (!ls->ls_debug_all_dentry) - goto fail; - - /* format 4 */ - - memset(name, 0, sizeof(name)); - snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_toss", ls->ls_name); - - ls->ls_debug_toss_dentry = debugfs_create_file(name, - S_IFREG | S_IRUGO, - dlm_root, - ls, - &format4_fops); - if (!ls->ls_debug_toss_dentry) - goto fail; - - memset(name, 0, sizeof(name)); - snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_waiters", ls->ls_name); - - ls->ls_debug_waiters_dentry = debugfs_create_file(name, - S_IFREG | S_IRUGO, - dlm_root, - ls, - &waiters_fops); - if (!ls->ls_debug_waiters_dentry) - goto fail; - - return 0; - - fail: - dlm_delete_debug_file(ls); - return -ENOMEM; -} - -int __init dlm_register_debugfs(void) -{ - mutex_init(&debug_buf_lock); - dlm_root = debugfs_create_dir("dlm", NULL); - return dlm_root ? 0 : -ENOMEM; -} - -void dlm_unregister_debugfs(void) -{ - debugfs_remove(dlm_root); -} - diff --git a/kmod/dlm/dir.c b/kmod/dlm/dir.c deleted file mode 100644 index 278a75cd..00000000 --- a/kmod/dlm/dir.c +++ /dev/null @@ -1,308 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "member.h" -#include "lowcomms.h" -#include "rcom.h" -#include "config.h" -#include "memory.h" -#include "recover.h" -#include "util.h" -#include "lock.h" -#include "dir.h" - -/* - * We use the upper 16 bits of the hash value to select the directory node. - * Low bits are used for distribution of rsb's among hash buckets on each node. - * - * To give the exact range wanted (0 to num_nodes-1), we apply a modulus of - * num_nodes to the hash value. This value in the desired range is used as an - * offset into the sorted list of nodeid's to give the particular nodeid. - */ - -int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash) -{ - uint32_t node; - - if (ls->ls_num_nodes == 1) - return dlm_our_nodeid(); - else { - node = (hash >> 16) % ls->ls_total_weight; - return ls->ls_node_array[node]; - } -} - -int dlm_dir_nodeid(struct dlm_rsb *r) -{ - return r->res_dir_nodeid; -} - -void dlm_recover_dir_nodeid(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - - down_read(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - r->res_dir_nodeid = dlm_hash2nodeid(ls, r->res_hash); - } - up_read(&ls->ls_root_sem); -} - -int dlm_recover_directory(struct dlm_ls *ls) -{ - struct dlm_member *memb; - char *b, *last_name = NULL; - int error = -ENOMEM, last_len, nodeid, result; - uint16_t namelen; - unsigned int count = 0, count_match = 0, count_bad = 0, count_add = 0; - - log_debug(ls, "dlm_recover_directory"); - - if (dlm_no_directory(ls)) - goto out_status; - - last_name = kmalloc(DLM_RESNAME_MAXLEN, GFP_NOFS); - if (!last_name) - goto out; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (memb->nodeid == dlm_our_nodeid()) - continue; - - memset(last_name, 0, DLM_RESNAME_MAXLEN); - last_len = 0; - - for (;;) { - int left; - error = dlm_recovery_stopped(ls); - if (error) - goto out_free; - - error = dlm_rcom_names(ls, memb->nodeid, - last_name, last_len); - if (error) - goto out_free; - - cond_resched(); - - /* - * pick namelen/name pairs out of received buffer - */ - - b = ls->ls_recover_buf->rc_buf; - left = ls->ls_recover_buf->rc_header.h_length; - left -= sizeof(struct dlm_rcom); - - for (;;) { - __be16 v; - - error = -EINVAL; - if (left < sizeof(__be16)) - goto out_free; - - memcpy(&v, b, sizeof(__be16)); - namelen = be16_to_cpu(v); - b += sizeof(__be16); - left -= sizeof(__be16); - - /* namelen of 0xFFFFF marks end of names for - this node; namelen of 0 marks end of the - buffer */ - - if (namelen == 0xFFFF) - goto done; - if (!namelen) - break; - - if (namelen > left) - goto out_free; - - if (namelen > DLM_RESNAME_MAXLEN) - goto out_free; - - error = dlm_master_lookup(ls, memb->nodeid, - b, namelen, - DLM_LU_RECOVER_DIR, - &nodeid, &result); - if (error) { - log_error(ls, "recover_dir lookup %d", - error); - goto out_free; - } - - /* The name was found in rsbtbl, but the - * master nodeid is different from - * memb->nodeid which says it is the master. - * This should not happen. */ - - if (result == DLM_LU_MATCH && - nodeid != memb->nodeid) { - count_bad++; - log_error(ls, "recover_dir lookup %d " - "nodeid %d memb %d bad %u", - result, nodeid, memb->nodeid, - count_bad); - print_hex_dump_bytes("dlm_recover_dir ", - DUMP_PREFIX_NONE, - b, namelen); - } - - /* The name was found in rsbtbl, and the - * master nodeid matches memb->nodeid. */ - - if (result == DLM_LU_MATCH && - nodeid == memb->nodeid) { - count_match++; - } - - /* The name was not found in rsbtbl and was - * added with memb->nodeid as the master. */ - - if (result == DLM_LU_ADD) { - count_add++; - } - - last_len = namelen; - memcpy(last_name, b, namelen); - b += namelen; - left -= namelen; - count++; - } - } - done: - ; - } - - out_status: - error = 0; - dlm_set_recover_status(ls, DLM_RS_DIR); - - log_debug(ls, "dlm_recover_directory %u in %u new", - count, count_add); - out_free: - kfree(last_name); - out: - return error; -} - -static struct dlm_rsb *find_rsb_root(struct dlm_ls *ls, char *name, int len) -{ - struct dlm_rsb *r; - uint32_t hash, bucket; - int rv; - - hash = jhash(name, len, 0); - bucket = hash & (ls->ls_rsbtbl_size - 1); - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[bucket].keep, name, len, &r); - if (rv) - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[bucket].toss, - name, len, &r); - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - - if (!rv) - return r; - - down_read(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - if (len == r->res_length && !memcmp(name, r->res_name, len)) { - up_read(&ls->ls_root_sem); - log_debug(ls, "find_rsb_root revert to root_list %s", - r->res_name); - return r; - } - } - up_read(&ls->ls_root_sem); - return NULL; -} - -/* Find the rsb where we left off (or start again), then send rsb names - for rsb's we're master of and whose directory node matches the requesting - node. inbuf is the rsb name last sent, inlen is the name's length */ - -void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen, - char *outbuf, int outlen, int nodeid) -{ - struct list_head *list; - struct dlm_rsb *r; - int offset = 0, dir_nodeid; - __be16 be_namelen; - - down_read(&ls->ls_root_sem); - - if (inlen > 1) { - r = find_rsb_root(ls, inbuf, inlen); - if (!r) { - inbuf[inlen - 1] = '\0'; - log_error(ls, "copy_master_names from %d start %d %s", - nodeid, inlen, inbuf); - goto out; - } - list = r->res_root_list.next; - } else { - list = ls->ls_root_list.next; - } - - for (offset = 0; list != &ls->ls_root_list; list = list->next) { - r = list_entry(list, struct dlm_rsb, res_root_list); - if (r->res_nodeid) - continue; - - dir_nodeid = dlm_dir_nodeid(r); - if (dir_nodeid != nodeid) - continue; - - /* - * The block ends when we can't fit the following in the - * remaining buffer space: - * namelen (uint16_t) + - * name (r->res_length) + - * end-of-block record 0x0000 (uint16_t) - */ - - if (offset + sizeof(uint16_t)*2 + r->res_length > outlen) { - /* Write end-of-block record */ - be_namelen = cpu_to_be16(0); - memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); - offset += sizeof(__be16); - ls->ls_recover_dir_sent_msg++; - goto out; - } - - be_namelen = cpu_to_be16(r->res_length); - memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); - offset += sizeof(__be16); - memcpy(outbuf + offset, r->res_name, r->res_length); - offset += r->res_length; - ls->ls_recover_dir_sent_res++; - } - - /* - * If we've reached the end of the list (and there's room) write a - * terminating record. - */ - - if ((list == &ls->ls_root_list) && - (offset + sizeof(uint16_t) <= outlen)) { - be_namelen = cpu_to_be16(0xFFFF); - memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); - offset += sizeof(__be16); - ls->ls_recover_dir_sent_msg++; - } - out: - up_read(&ls->ls_root_sem); -} - diff --git a/kmod/dlm/dir.h b/kmod/dlm/dir.h deleted file mode 100644 index 41750634..00000000 --- a/kmod/dlm/dir.h +++ /dev/null @@ -1,25 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __DIR_DOT_H__ -#define __DIR_DOT_H__ - -int dlm_dir_nodeid(struct dlm_rsb *rsb); -int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash); -void dlm_recover_dir_nodeid(struct dlm_ls *ls); -int dlm_recover_directory(struct dlm_ls *ls); -void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen, - char *outbuf, int outlen, int nodeid); - -#endif /* __DIR_DOT_H__ */ - diff --git a/kmod/dlm/dlm_internal.h b/kmod/dlm/dlm_internal.h deleted file mode 100644 index b4a91e4c..00000000 --- a/kmod/dlm/dlm_internal.h +++ /dev/null @@ -1,776 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __DLM_INTERNAL_DOT_H__ -#define __DLM_INTERNAL_DOT_H__ - -/* - * This is the main header file to be included in each DLM source file. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "include/linux/dlm.h" -#include "config.h" - -/* Size of the temp buffer midcomms allocates on the stack. - We try to make this large enough so most messages fit. - FIXME: should sctp make this unnecessary? */ - -#define DLM_INBUF_LEN 148 - -struct dlm_ls; -struct dlm_lkb; -struct dlm_rsb; -struct dlm_member; -struct dlm_rsbtable; -struct dlm_recover; -struct dlm_header; -struct dlm_message; -struct dlm_rcom; -struct dlm_mhandle; - -#define log_print(fmt, args...) \ - printk(KERN_ERR "dlm: "fmt"\n" , ##args) -#define log_error(ls, fmt, args...) \ - printk(KERN_ERR "dlm: %s: " fmt "\n", (ls)->ls_name , ##args) - -#define log_debug(ls, fmt, args...) \ -do { \ - if (dlm_config.ci_log_debug) \ - printk(KERN_DEBUG "dlm: %s: " fmt "\n", \ - (ls)->ls_name , ##args); \ -} while (0) - -#define log_limit(ls, fmt, args...) \ -do { \ - if (dlm_config.ci_log_debug) \ - printk_ratelimited(KERN_DEBUG "dlm: %s: " fmt "\n", \ - (ls)->ls_name , ##args); \ -} while (0) - -#define DLM_ASSERT(x, do) \ -{ \ - if (!(x)) \ - { \ - printk(KERN_ERR "\nDLM: Assertion failed on line %d of file %s\n" \ - "DLM: assertion: \"%s\"\n" \ - "DLM: time = %lu\n", \ - __LINE__, __FILE__, #x, jiffies); \ - {do} \ - printk("\n"); \ - BUG(); \ - panic("DLM: Record message above and reboot.\n"); \ - } \ -} - -struct dlm_range { - struct dlm_key *start; - struct dlm_key *end; -}; - -#define DLM_RTF_SHRINK 0x00000001 - -struct dlm_rsbtable { - struct rb_root keep; - struct rb_root toss; - spinlock_t lock; - uint32_t flags; -}; - - -/* - * Lockspace member (per node in a ls) - */ - -struct dlm_member { - struct list_head list; - int nodeid; - int weight; - int slot; - int slot_prev; - int comm_seq; - uint32_t generation; -}; - -/* - * Save and manage recovery state for a lockspace. - */ - -struct dlm_recover { - struct list_head list; - struct dlm_config_node *nodes; - int nodes_count; - uint64_t seq; -}; - -/* - * Pass input args to second stage locking function. - */ - -struct dlm_args { - uint32_t flags; - void (*astfn) (void *astparam); - void *astparam; - void (*bastfn) (void *astparam, int mode); - void (*rbastfn) (void *astarg, int mode, - struct dlm_key *start, - struct dlm_key *end); - int mode; - struct dlm_range range; - struct dlm_lksb *lksb; - unsigned long timeout; -}; - - -/* - * Lock block - * - * A lock can be one of three types: - * - * local copy lock is mastered locally - * (lkb_nodeid is zero and DLM_LKF_MSTCPY is not set) - * process copy lock is mastered on a remote node - * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is not set) - * master copy master node's copy of a lock owned by remote node - * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is set) - * - * lkb_exflags: a copy of the most recent flags arg provided to dlm_lock or - * dlm_unlock. The dlm does not modify these or use any private flags in - * this field; it only contains DLM_LKF_ flags from dlm.h. These flags - * are sent as-is to the remote master when the lock is remote. - * - * lkb_flags: internal dlm flags (DLM_IFL_ prefix) from dlm_internal.h. - * Some internal flags are shared between the master and process nodes; - * these shared flags are kept in the lower two bytes. One of these - * flags set on the master copy will be propagated to the process copy - * and v.v. Other internal flags are private to the master or process - * node (e.g. DLM_IFL_MSTCPY). These are kept in the high two bytes. - * - * lkb_sbflags: status block flags. These flags are copied directly into - * the caller's lksb.sb_flags prior to the dlm_lock/dlm_unlock completion - * ast. All defined in dlm.h with DLM_SBF_ prefix. - * - * lkb_status: the lock status indicates which rsb queue the lock is - * on, grant, convert, or wait. DLM_LKSTS_ WAITING/GRANTED/CONVERT - * - * lkb_wait_type: the dlm message type (DLM_MSG_ prefix) for which a - * reply is needed. Only set when the lkb is on the lockspace waiters - * list awaiting a reply from a remote node. - * - * lkb_nodeid: when the lkb is a local copy, nodeid is 0; when the lkb - * is a master copy, nodeid specifies the remote lock holder, when the - * lkb is a process copy, the nodeid specifies the lock master. - */ - -/* lkb_status */ - -#define DLM_LKSTS_WAITING 1 -#define DLM_LKSTS_GRANTED 2 -#define DLM_LKSTS_CONVERT 3 - -/* lkb_flags */ - -#define DLM_IFL_MSTCPY 0x00010000 -#define DLM_IFL_RESEND 0x00020000 -#define DLM_IFL_DEAD 0x00040000 -#define DLM_IFL_OVERLAP_UNLOCK 0x00080000 -#define DLM_IFL_OVERLAP_CANCEL 0x00100000 -#define DLM_IFL_ENDOFLIFE 0x00200000 -#define DLM_IFL_WATCH_TIMEWARN 0x00400000 -#define DLM_IFL_TIMEOUT_CANCEL 0x00800000 -#define DLM_IFL_DEADLOCK_CANCEL 0x01000000 -#define DLM_IFL_STUB_MS 0x02000000 /* magic number for m_flags */ -#define DLM_IFL_USER 0x00000001 -#define DLM_IFL_ORPHAN 0x00000002 - -#define DLM_CALLBACKS_SIZE 6 - -#define DLM_CB_CAST 0x00000001 -#define DLM_CB_BAST 0x00000002 -#define DLM_CB_SKIP 0x00000004 - - -#define DLM_KEY_LEN 296 - -struct dlm_callback { - uint64_t seq; - uint32_t flags; /* DLM_CBF_ */ - int sb_status; /* copy to lksb status */ - uint8_t sb_flags; /* copy to lksb flags */ - int8_t mode; /* rq mode of bast, gr mode of cast */ - struct dlm_range range; - - /* XXX: This should be dynamically allocated */ - struct dlm_key start; - struct dlm_key end; - char startval[DLM_KEY_LEN]; - char endval[DLM_KEY_LEN]; -}; - -struct dlm_lkb { - struct dlm_rsb *lkb_resource; /* the rsb */ - struct kref lkb_ref; - int lkb_nodeid; /* copied from rsb */ - int lkb_ownpid; /* pid of lock owner */ - uint32_t lkb_id; /* our lock ID */ - uint32_t lkb_remid; /* lock ID on remote partner */ - uint32_t lkb_exflags; /* external flags from caller */ - uint32_t lkb_sbflags; /* lksb flags */ - uint32_t lkb_flags; /* internal flags */ - uint32_t lkb_lvbseq; /* lvb sequence number */ - - int8_t lkb_status; /* granted, waiting, convert */ - int8_t lkb_rqmode; /* requested lock mode */ - int8_t lkb_grmode; /* granted lock mode */ - int8_t lkb_highbast; /* highest mode bast sent for */ - /* XXX: Keep some history of bast ranges here? */ - - struct dlm_range lkb_rqrange; - struct dlm_range lkb_grrange; - - int8_t lkb_wait_type; /* type of reply waiting for */ - int8_t lkb_wait_count; - int lkb_wait_nodeid; /* for debugging */ - - struct list_head lkb_statequeue; /* rsb g/c/w list */ - struct rb_node lkb_statenode; /* rsb g/c/w interval tree */ - struct dlm_key *lkb_subtree_last; /* rsb g/c/w interval tree */ - struct list_head lkb_rsb_lookup; /* waiting for rsb lookup */ - struct list_head lkb_wait_reply; /* waiting for remote reply */ - struct list_head lkb_ownqueue; /* list of locks for a process */ - struct list_head lkb_time_list; - ktime_t lkb_timestamp; - ktime_t lkb_wait_time; - unsigned long lkb_timeout_cs; - - struct mutex lkb_cb_mutex; - struct work_struct lkb_cb_work; - struct list_head lkb_cb_list; /* for ls_cb_delay or proc->asts */ - struct dlm_callback lkb_callbacks[DLM_CALLBACKS_SIZE]; - struct dlm_callback lkb_last_cast; - struct dlm_callback lkb_last_bast; - ktime_t lkb_last_cast_time; /* for debugging */ - ktime_t lkb_last_bast_time; /* for debugging */ - - uint64_t lkb_recover_seq; /* from ls_recover_seq */ - - char *lkb_lvbptr; - struct dlm_lksb *lkb_lksb; /* caller's status block */ - void (*lkb_astfn) (void *astparam); - void (*lkb_bastfn) (void *astparam, int mode); - void (*lkb_rbastfn) (void *astparam, int mode, - struct dlm_key *start, - struct dlm_key *end); - union { - void *lkb_astparam; /* caller's ast arg */ - struct dlm_user_args *lkb_ua; - }; -}; - -/* - * res_master_nodeid is "normal": 0 is unset/invalid, non-zero is the real - * nodeid, even when nodeid is our_nodeid. - * - * res_nodeid is "odd": -1 is unset/invalid, zero means our_nodeid, - * greater than zero when another nodeid. - * - * (TODO: remove res_nodeid and only use res_master_nodeid) - */ - -struct dlm_rsb { - struct dlm_ls *res_ls; /* the lockspace */ - struct kref res_ref; - struct mutex res_mutex; - unsigned long res_flags; - int res_length; /* length of rsb name */ - int res_nodeid; - int res_master_nodeid; - int res_dir_nodeid; - int res_id; /* for ls_recover_idr */ - uint32_t res_lvbseq; - uint32_t res_hash; - uint32_t res_bucket; /* rsbtbl */ - unsigned long res_toss_time; - uint32_t res_first_lkid; - struct list_head res_lookup; /* lkbs waiting on first */ - union { - struct list_head res_hashchain; - struct rb_node res_hashnode; /* rsbtbl */ - }; - struct list_head res_grantqueue; - struct rb_root res_grantroot; - struct list_head res_convertqueue; - struct rb_root res_convertroot; - struct list_head res_waitqueue; - - struct list_head res_root_list; /* used for recovery */ - struct list_head res_recover_list; /* used for recovery */ - int res_recover_locks_count; - - char *res_lvbptr; - char res_name[DLM_RESNAME_MAXLEN+1]; -}; - -/* dlm_master_lookup() flags */ - -#define DLM_LU_RECOVER_DIR 1 -#define DLM_LU_RECOVER_MASTER 2 - -/* dlm_master_lookup() results */ - -#define DLM_LU_MATCH 1 -#define DLM_LU_ADD 2 - -/* find_rsb() flags */ - -#define R_REQUEST 0x00000001 -#define R_RECEIVE_REQUEST 0x00000002 -#define R_RECEIVE_RECOVER 0x00000004 - -/* rsb_flags */ - -enum rsb_flags { - RSB_MASTER_UNCERTAIN, - RSB_VALNOTVALID, - RSB_VALNOTVALID_PREV, - RSB_NEW_MASTER, - RSB_NEW_MASTER2, - RSB_RECOVER_CONVERT, - RSB_RECOVER_GRANT, - RSB_RECOVER_LVB_INVAL, -}; - -static inline void rsb_set_flag(struct dlm_rsb *r, enum rsb_flags flag) -{ - __set_bit(flag, &r->res_flags); -} - -static inline void rsb_clear_flag(struct dlm_rsb *r, enum rsb_flags flag) -{ - __clear_bit(flag, &r->res_flags); -} - -static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag) -{ - return test_bit(flag, &r->res_flags); -} - - -/* dlm_header is first element of all structs sent between nodes */ - -#define DLM_HEADER_MAJOR 0x00030000 -#define DLM_HEADER_MINOR 0x00000001 - -#define DLM_HEADER_SLOTS 0x00000001 - -#define DLM_MSG 1 -#define DLM_RCOM 2 - -struct dlm_header { - uint32_t h_version; - uint32_t h_lockspace; - uint32_t h_nodeid; /* nodeid of sender */ - uint16_t h_length; - uint8_t h_cmd; /* DLM_MSG, DLM_RCOM */ - uint8_t h_pad; -}; - - -#define DLM_MSG_REQUEST 1 -#define DLM_MSG_CONVERT 2 -#define DLM_MSG_UNLOCK 3 -#define DLM_MSG_CANCEL 4 -#define DLM_MSG_REQUEST_REPLY 5 -#define DLM_MSG_CONVERT_REPLY 6 -#define DLM_MSG_UNLOCK_REPLY 7 -#define DLM_MSG_CANCEL_REPLY 8 -#define DLM_MSG_GRANT 9 -#define DLM_MSG_BAST 10 -#define DLM_MSG_LOOKUP 11 -#define DLM_MSG_REMOVE 12 -#define DLM_MSG_LOOKUP_REPLY 13 -#define DLM_MSG_PURGE 14 - -struct dlm_message { - struct dlm_header m_header; - uint32_t m_type; /* DLM_MSG_ */ - uint32_t m_nodeid; - uint32_t m_pid; - uint32_t m_lkid; /* lkid on sender */ - uint32_t m_remid; /* lkid on receiver */ - uint32_t m_parent_lkid; - uint32_t m_parent_remid; - uint32_t m_exflags; - uint32_t m_sbflags; - uint32_t m_flags; - uint32_t m_lvbseq; - uint32_t m_hash; - int m_status; - int m_grmode; - int m_rqmode; - int m_bastmode; - int m_asts; - int m_result; /* 0 or -EXXX */ - /* - * XXX: These should start *after* m_extra to preserve - * compatibility with the old message format - */ - char m_grstart[DLM_KEY_LEN]; - char m_grend[DLM_KEY_LEN]; - uint16_t m_grstart_len; - uint16_t m_grend_len; - - char m_rqstart[DLM_KEY_LEN]; - char m_rqend[DLM_KEY_LEN]; - uint16_t m_rqstart_len; - uint16_t m_rqend_len; - - char m_baststart[DLM_KEY_LEN]; - char m_bastend[DLM_KEY_LEN]; - uint16_t m_baststart_len; - uint16_t m_bastend_len; - - char m_extra[0]; /* name or lvb */ -}; - - -#define DLM_RS_NODES 0x00000001 -#define DLM_RS_NODES_ALL 0x00000002 -#define DLM_RS_DIR 0x00000004 -#define DLM_RS_DIR_ALL 0x00000008 -#define DLM_RS_LOCKS 0x00000010 -#define DLM_RS_LOCKS_ALL 0x00000020 -#define DLM_RS_DONE 0x00000040 -#define DLM_RS_DONE_ALL 0x00000080 - -#define DLM_RCOM_STATUS 1 -#define DLM_RCOM_NAMES 2 -#define DLM_RCOM_LOOKUP 3 -#define DLM_RCOM_LOCK 4 -#define DLM_RCOM_STATUS_REPLY 5 -#define DLM_RCOM_NAMES_REPLY 6 -#define DLM_RCOM_LOOKUP_REPLY 7 -#define DLM_RCOM_LOCK_REPLY 8 - -struct dlm_rcom { - struct dlm_header rc_header; - uint32_t rc_type; /* DLM_RCOM_ */ - int rc_result; /* multi-purpose */ - uint64_t rc_id; /* match reply with request */ - uint64_t rc_seq; /* sender's ls_recover_seq */ - uint64_t rc_seq_reply; /* remote ls_recover_seq */ - char rc_buf[0]; -}; - -union dlm_packet { - struct dlm_header header; /* common to other two */ - struct dlm_message message; - struct dlm_rcom rcom; -}; - -#define DLM_RSF_NEED_SLOTS 0x00000001 - -/* RCOM_STATUS data */ -struct rcom_status { - __le32 rs_flags; - __le32 rs_unused1; - __le64 rs_unused2; -}; - -/* RCOM_STATUS_REPLY data */ -struct rcom_config { - __le32 rf_lvblen; - __le32 rf_lsflags; - - /* DLM_HEADER_SLOTS adds: */ - __le32 rf_flags; - __le16 rf_our_slot; - __le16 rf_num_slots; - __le32 rf_generation; - __le32 rf_unused1; - __le64 rf_unused2; -}; - -struct rcom_slot { - __le32 ro_nodeid; - __le16 ro_slot; - __le16 ro_unused1; - __le64 ro_unused2; -}; - -struct rcom_lock { - __le32 rl_ownpid; - __le32 rl_lkid; - __le32 rl_remid; - __le32 rl_parent_lkid; - __le32 rl_parent_remid; - __le32 rl_exflags; - __le32 rl_flags; - __le32 rl_lvbseq; - __le32 rl_result; - int8_t rl_rqmode; - int8_t rl_grmode; - int8_t rl_status; - int8_t rl_asts; - __le16 rl_wait_type; - __le16 rl_namelen; - char rl_name[DLM_RESNAME_MAXLEN]; - char rl_lvb[0]; -}; - -/* - * The max number of resources per rsbtbl bucket that shrink will attempt - * to remove in each iteration. - */ - -#define DLM_REMOVE_NAMES_MAX 8 - -struct dlm_ls { - struct list_head ls_list; /* list of lockspaces */ - dlm_lockspace_t *ls_local_handle; - uint32_t ls_global_id; /* global unique lockspace ID */ - uint32_t ls_generation; - uint32_t ls_exflags; - int ls_lvblen; - int ls_count; /* refcount of processes in - the dlm using this ls */ - int ls_create_count; /* create/release refcount */ - unsigned long ls_flags; /* LSFL_ */ - unsigned long ls_scan_time; - struct kobject ls_kobj; - - struct idr ls_lkbidr; - spinlock_t ls_lkbidr_spin; - - struct dlm_rsbtable *ls_rsbtbl; - uint32_t ls_rsbtbl_size; - - struct mutex ls_waiters_mutex; - struct list_head ls_waiters; /* lkbs needing a reply */ - - struct mutex ls_orphans_mutex; - struct list_head ls_orphans; - - struct mutex ls_timeout_mutex; - struct list_head ls_timeout; - - spinlock_t ls_new_rsb_spin; - int ls_new_rsb_count; - struct list_head ls_new_rsb; /* new rsb structs */ - - spinlock_t ls_remove_spin; - char ls_remove_name[DLM_RESNAME_MAXLEN+1]; - char *ls_remove_names[DLM_REMOVE_NAMES_MAX]; - int ls_remove_len; - int ls_remove_lens[DLM_REMOVE_NAMES_MAX]; - - struct list_head ls_nodes; /* current nodes in ls */ - struct list_head ls_nodes_gone; /* dead node list, recovery */ - int ls_num_nodes; /* number of nodes in ls */ - int ls_low_nodeid; - int ls_total_weight; - int *ls_node_array; - - int ls_slot; - int ls_num_slots; - int ls_slots_size; - struct dlm_slot *ls_slots; - - struct dlm_rsb ls_stub_rsb; /* for returning errors */ - struct dlm_lkb ls_stub_lkb; /* for returning errors */ - struct dlm_message ls_stub_ms; /* for faking a reply */ - - struct dentry *ls_debug_rsb_dentry; /* debugfs */ - struct dentry *ls_debug_waiters_dentry; /* debugfs */ - struct dentry *ls_debug_locks_dentry; /* debugfs */ - struct dentry *ls_debug_all_dentry; /* debugfs */ - struct dentry *ls_debug_toss_dentry; /* debugfs */ - - wait_queue_head_t ls_uevent_wait; /* user part of join/leave */ - int ls_uevent_result; - struct completion ls_members_done; - int ls_members_result; - - struct miscdevice ls_device; - - struct workqueue_struct *ls_callback_wq; - - /* recovery related */ - - struct mutex ls_cb_mutex; - struct list_head ls_cb_delay; /* save for queue_work later */ - struct timer_list ls_timer; - struct task_struct *ls_recoverd_task; - struct mutex ls_recoverd_active; - spinlock_t ls_recover_lock; - unsigned long ls_recover_begin; /* jiffies timestamp */ - uint32_t ls_recover_status; /* DLM_RS_ */ - uint64_t ls_recover_seq; - struct dlm_recover *ls_recover_args; - struct rw_semaphore ls_in_recovery; /* block local requests */ - struct rw_semaphore ls_recv_active; /* block dlm_recv */ - struct list_head ls_requestqueue;/* queue remote requests */ - struct mutex ls_requestqueue_mutex; - struct dlm_rcom *ls_recover_buf; - int ls_recover_nodeid; /* for debugging */ - unsigned int ls_recover_dir_sent_res; /* for log info */ - unsigned int ls_recover_dir_sent_msg; /* for log info */ - unsigned int ls_recover_locks_in; /* for log info */ - uint64_t ls_rcom_seq; - spinlock_t ls_rcom_spin; - struct list_head ls_recover_list; - spinlock_t ls_recover_list_lock; - int ls_recover_list_count; - struct idr ls_recover_idr; - spinlock_t ls_recover_idr_lock; - wait_queue_head_t ls_wait_general; - wait_queue_head_t ls_recover_lock_wait; - struct mutex ls_clear_proc_locks; - - struct list_head ls_root_list; /* root resources */ - struct rw_semaphore ls_root_sem; /* protect root_list */ - - const struct dlm_lockspace_ops *ls_ops; - void *ls_ops_arg; - - int ls_namelen; - char ls_name[1]; -}; - -/* - * LSFL_RECOVER_STOP - dlm_ls_stop() sets this to tell dlm recovery routines - * that they should abort what they're doing so new recovery can be started. - * - * LSFL_RECOVER_DOWN - dlm_ls_stop() sets this to tell dlm_recoverd that it - * should do down_write() on the in_recovery rw_semaphore. (doing down_write - * within dlm_ls_stop causes complaints about the lock acquired/released - * in different contexts.) - * - * LSFL_RECOVER_LOCK - dlm_recoverd holds the in_recovery rw_semaphore. - * It sets this after it is done with down_write() on the in_recovery - * rw_semaphore and clears it after it has released the rw_semaphore. - * - * LSFL_RECOVER_WORK - dlm_ls_start() sets this to tell dlm_recoverd that it - * should begin recovery of the lockspace. - * - * LSFL_RUNNING - set when normal locking activity is enabled. - * dlm_ls_stop() clears this to tell dlm locking routines that they should - * quit what they are doing so recovery can run. dlm_recoverd sets - * this after recovery is finished. - */ - -#define LSFL_RECOVER_STOP 0 -#define LSFL_RECOVER_DOWN 1 -#define LSFL_RECOVER_LOCK 2 -#define LSFL_RECOVER_WORK 3 -#define LSFL_RUNNING 4 - -#define LSFL_RCOM_READY 5 -#define LSFL_RCOM_WAIT 6 -#define LSFL_UEVENT_WAIT 7 -#define LSFL_TIMEWARN 8 -#define LSFL_CB_DELAY 9 -#define LSFL_NODIR 10 - -/* much of this is just saving user space pointers associated with the - lock that we pass back to the user lib with an ast */ - -struct dlm_user_args { - struct dlm_user_proc *proc; /* each process that opens the lockspace - device has private data - (dlm_user_proc) on the struct file, - the process's locks point back to it*/ - struct dlm_lksb lksb; - struct dlm_lksb __user *user_lksb; - void __user *castparam; - void __user *castaddr; - void __user *bastparam; - void __user *bastaddr; - uint64_t xid; -}; - -#define DLM_PROC_FLAGS_CLOSING 1 -#define DLM_PROC_FLAGS_COMPAT 2 - -/* locks list is kept so we can remove all a process's locks when it - exits (or orphan those that are persistent) */ - -struct dlm_user_proc { - dlm_lockspace_t *lockspace; - unsigned long flags; /* DLM_PROC_FLAGS */ - struct list_head asts; - spinlock_t asts_spin; - struct list_head locks; - spinlock_t locks_spin; - struct list_head unlocking; - wait_queue_head_t wait; -}; - -static inline int dlm_locking_stopped(struct dlm_ls *ls) -{ - return !test_bit(LSFL_RUNNING, &ls->ls_flags); -} - -static inline int dlm_recovery_stopped(struct dlm_ls *ls) -{ - return test_bit(LSFL_RECOVER_STOP, &ls->ls_flags); -} - -static inline int dlm_no_directory(struct dlm_ls *ls) -{ - return test_bit(LSFL_NODIR, &ls->ls_flags); -} - -int dlm_netlink_init(void); -void dlm_netlink_exit(void); -void dlm_timeout_warn(struct dlm_lkb *lkb); -int dlm_plock_init(void); -void dlm_plock_exit(void); - -#ifdef CONFIG_DLM_DEBUG -int dlm_register_debugfs(void); -void dlm_unregister_debugfs(void); -int dlm_create_debug_file(struct dlm_ls *ls); -void dlm_delete_debug_file(struct dlm_ls *ls); -#else -static inline int dlm_register_debugfs(void) { return 0; } -static inline void dlm_unregister_debugfs(void) { } -static inline int dlm_create_debug_file(struct dlm_ls *ls) { return 0; } -static inline void dlm_delete_debug_file(struct dlm_ls *ls) { } -#endif - -#endif /* __DLM_INTERNAL_DOT_H__ */ - diff --git a/kmod/dlm/dlmtest.c b/kmod/dlm/dlmtest.c deleted file mode 100644 index 4c7d72fc..00000000 --- a/kmod/dlm/dlmtest.c +++ /dev/null @@ -1,307 +0,0 @@ -#include -#include -#include -#include - -#include "include/linux/dlm.h" - -static atomic_t granted; -static atomic_t blocking; -static int val = 1; -static dlm_lockspace_t *ls; -static char *lockres_name = "test_resource"; - -struct lockinfo { - char *lockname; - int unlocking; - u64 start; - u64 end; - struct dlm_key startkey; - struct dlm_key endkey; - struct dlm_lksb lksb; -}; - -static inline void set_lock_endpoints(struct lockinfo *lock, u64 start, u64 end) -{ - lock->start = cpu_to_be64(start); - lock->startkey.val = &lock->start; - lock->startkey.len = sizeof(lock->start); - lock->end = cpu_to_be64(end); - lock->endkey.val = &lock->end; - lock->endkey.len = sizeof(lock->end); -} - -#define NUM_LOCKS 3 -static struct lockinfo locks[NUM_LOCKS] = { - { "lock0", }, - { "lock1", }, - { "lock2", }, -}; - -static int glbl_exmode = 0; -module_param(glbl_exmode, int, 0); -MODULE_PARM_DESC(glbl_exmode, "Take global lock exclusively."); - -static void init_counters(void) -{ - atomic_set(&granted, 0); - atomic_set(&blocking, 0); - val = 1; -} - -static void wait_for_blocking_asts(int count) -{ - printk("wait for %d blocking asts\n", count); - while (atomic_read(&blocking) != count) { - printk("blocking: %d\n", atomic_read(&blocking)); - msleep_interruptible(2000); - } -} - -static void wait_for_lock_grants(int count) -{ - printk("wait for %d grants\n", count); - while (atomic_read(&granted) != count) { - printk("granted: %d\n", atomic_read(&granted)); - msleep_interruptible(2000); - } -} - -static void grant_function(void *arg) -{ - char *name = arg; - printk("lock %s granted\n", name); - atomic_add(val, &granted); -} - -static void blocking_function(void *arg, int mode, struct dlm_key *start, - struct dlm_key *end) -{ - char *name = arg; - BUG_ON(!start); - BUG_ON(!end); - printk("lock %s blocking mode %d, range (%llu, %llu)\n", name, mode, - be64_to_cpu(*((u64 *)start->val)), be64_to_cpu(*((u64 *)end->val))); - atomic_inc(&blocking); -} - -static int _test_lock(unsigned int lockidx, unsigned int mode, - unsigned long long start, unsigned long long end, - unsigned int flags) -{ - struct lockinfo *lock = &locks[lockidx]; - - BUG_ON(lockidx > NUM_LOCKS); - - set_lock_endpoints(lock, start, end); - - printk("lock %s (%u, %llu, %llu)\n", lock->lockname, mode, start, end); - return dlm_lock_range(ls, mode, &lock->startkey, &lock->endkey, - &lock->lksb, flags, lockres_name, - strlen(lockres_name), 0, grant_function, - lock->lockname, blocking_function); -} - -static inline int test_lock(unsigned int lockidx, unsigned int mode, - unsigned long long start, unsigned long long end) -{ - return _test_lock(lockidx, mode, start, end, 0); -} - -static inline int test_convert(unsigned int lockidx, unsigned int mode, - unsigned long long start, - unsigned long long end) -{ - return _test_lock(lockidx, mode, start, end, DLM_LKF_CONVERT); -} - -static int test_unlock(int lockidx) -{ - struct lockinfo *lock = &locks[lockidx]; - - printk("unlock %s (%llu, %llu)\n", lock->lockname, - be64_to_cpu(lock->start), be64_to_cpu(lock->end)); - return dlm_unlock(ls, lock->lksb.sb_lkid, 0, &lock->lksb, lock->lockname); -} - -static int test_locking(void) -{ - int ret; - - printk("Test basic lock/unlock.\n"); - - init_counters(); - - ret = test_lock(0, DLM_LOCK_EX, 0, 16384); - if (ret) - goto out; - - ret = test_lock(1, DLM_LOCK_EX, 16385, 32768); - if (ret) - goto out; - - wait_for_lock_grants(2); - - ret = test_lock(2, DLM_LOCK_EX, 0, 32768); - if (ret) - goto out; - - wait_for_blocking_asts(2); - - val = -1; - - ret = test_unlock(0); - if (ret) - goto out; - ret = test_unlock(1); - if (ret) - goto out; - - wait_for_lock_grants(-1); - - ret = test_unlock(2); - if (ret) - goto out; - - wait_for_lock_grants(-2); - -out: - return ret; -} - -static int test_lock_conversions(void) -{ - int ret; - - printk("Test lock conversions\n"); - - init_counters(); - - ret = test_lock(0, DLM_LOCK_PR, 0, 16384); - if (ret) - goto out; - - ret = test_lock(1, DLM_LOCK_PR, 16385, 32768); - if (ret) - goto out; - - ret = test_lock(2, DLM_LOCK_PR, 0, 32768); - if (ret) - goto out; - - wait_for_lock_grants(3); - - ret = test_convert(0, DLM_LOCK_EX, 0, 16384); - if (ret) - goto out; - - wait_for_blocking_asts(2); - - init_counters(); - - ret = test_convert(1, DLM_LOCK_NL, 16385, 32768); - if (ret) - goto out; - ret = test_convert(2, DLM_LOCK_NL, 0, 32768); - if (ret) - goto out; - - wait_for_lock_grants(3); - - init_counters(); - - ret = test_unlock(1); - if (ret) - goto out; - ret = test_unlock(2); - if (ret) - goto out; - - ret = test_unlock(0); - if (ret) - goto out; - - wait_for_lock_grants(3); - -out: - return ret; -} - -#define glbl_res "global" -#define glbl_res_len strlen(glbl_res) -static atomic_t glbl_grants; -static struct dlm_lksb glbl_lksb; - -static void glbl_granted(void *arg) -{ - printk("Got global lock at %d mode\n", *((int *) arg)); - atomic_set(&glbl_grants, 1); -} - -static void glbl_blocking(void *arg, int mode) -{ - printk("Global lock at %d mode blocking %d lock\n", *((int *)arg), mode); -} - -static int test_multinode(void) -{ - int ret; - int mode = DLM_LOCK_EX; - - printk("Test a global lock\n"); - - ret = dlm_lock(ls, mode, &glbl_lksb, 0, glbl_res, glbl_res_len, 0, - glbl_granted, &mode, glbl_blocking); - if (ret) - return ret; - - while (!atomic_read(&glbl_grants)) - msleep_interruptible(5000); - - mode = 0; - ret = dlm_unlock(ls, glbl_lksb.sb_lkid, 0, &glbl_lksb, &mode); - return ret; -} - -static int __init init_dlm_test(void) -{ - int ret; - - printk("dlmtest loaded!\n"); - - ret = dlm_new_lockspace("lockspace", "scoutfs", - DLM_LSFL_FS|DLM_LSFL_NEWEXCL, 8, NULL, NULL, - NULL, &ls); - if (ret) { - printk("new_lockspace returns %d\n", ret); - return ret; - } - - ret = test_multinode(); - if (!ret) - ret = test_locking(); - if (!ret) - ret = test_lock_conversions(); - - if (ret) - printk("FAILURE: Locking test returns %d\n", ret); - else - printk("Locking test completed with no errors.\n"); - - return 0; -} - -static void __exit exit_dlm_test(void) -{ - int ret; - - ret = dlm_release_lockspace(ls, 1); - printk("dlmtest unloaded (ret=%d)!\n", ret); -} - -module_init(init_dlm_test); -module_exit(exit_dlm_test); - -MODULE_DESCRIPTION("dlmtest"); -MODULE_AUTHOR("Mark Fasheh"); -MODULE_LICENSE("GPL"); diff --git a/kmod/dlm/include/linux/dlm.h b/kmod/dlm/include/linux/dlm.h deleted file mode 100644 index cbf05678..00000000 --- a/kmod/dlm/include/linux/dlm.h +++ /dev/null @@ -1,193 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ -#ifndef __DLM_DOT_H__ -#define __DLM_DOT_H__ - -#include - - -struct dlm_slot { - int nodeid; /* 1 to MAX_INT */ - int slot; /* 1 to MAX_INT */ -}; - -/* - * recover_prep: called before the dlm begins lock recovery. - * Notfies lockspace user that locks from failed members will be granted. - * recover_slot: called after recover_prep and before recover_done. - * Identifies a failed lockspace member. - * recover_done: called after the dlm completes lock recovery. - * Identifies lockspace members and lockspace generation number. - */ - -struct dlm_lockspace_ops { - void (*recover_prep) (void *ops_arg); - void (*recover_slot) (void *ops_arg, struct dlm_slot *slot); - void (*recover_done) (void *ops_arg, struct dlm_slot *slots, - int num_slots, int our_slot, uint32_t generation); -}; - -/* - * dlm_new_lockspace - * - * Create/join a lockspace. - * - * name: lockspace name, null terminated, up to DLM_LOCKSPACE_LEN (not - * including terminating null). - * - * cluster: cluster name, null terminated, up to DLM_LOCKSPACE_LEN (not - * including terminating null). Optional. When cluster is null, it - * is not used. When set, dlm_new_lockspace() returns -EBADR if cluster - * is not equal to the dlm cluster name. - * - * flags: - * DLM_LSFL_NODIR - * The dlm should not use a resource directory, but statically assign - * resource mastery to nodes based on the name hash that is otherwise - * used to select the directory node. Must be the same on all nodes. - * DLM_LSFL_TIMEWARN - * The dlm should emit netlink messages if locks have been waiting - * for a configurable amount of time. (Unused.) - * DLM_LSFL_FS - * The lockspace user is in the kernel (i.e. filesystem). Enables - * direct bast/cast callbacks. - * DLM_LSFL_NEWEXCL - * dlm_new_lockspace() should return -EEXIST if the lockspace exists. - * - * lvblen: length of lvb in bytes. Must be multiple of 8. - * dlm_new_lockspace() returns an error if this does not match - * what other nodes are using. - * - * ops: callbacks that indicate lockspace recovery points so the - * caller can coordinate its recovery and know lockspace members. - * This is only used by the initial dlm_new_lockspace() call. - * Optional. - * - * ops_arg: arg for ops callbacks. - * - * ops_result: tells caller if the ops callbacks (if provided) will - * be used or not. 0: will be used, -EXXX will not be used. - * -EOPNOTSUPP: the dlm does not have recovery_callbacks enabled. - * - * lockspace: handle for dlm functions - */ - -int dlm_new_lockspace(const char *name, const char *cluster, - uint32_t flags, int lvblen, - const struct dlm_lockspace_ops *ops, void *ops_arg, - int *ops_result, dlm_lockspace_t **lockspace); - -/* - * dlm_release_lockspace - * - * Stop a lockspace. - */ - -int dlm_release_lockspace(dlm_lockspace_t *lockspace, int force); - -/* - * dlm_lock - * - * Make an asynchronous request to acquire or convert a lock on a named - * resource. - * - * lockspace: context for the request - * mode: the requested mode of the lock (DLM_LOCK_) - * lksb: lock status block for input and async return values - * flags: input flags (DLM_LKF_) - * name: name of the resource to lock, can be binary - * namelen: the length in bytes of the resource name (MAX_RESNAME_LEN) - * parent: the lock ID of a parent lock or 0 if none - * lockast: function DLM executes when it completes processing the request - * astarg: argument passed to lockast and bast functions - * bast: function DLM executes when this lock later blocks another request - * - * Returns: - * 0 if request is successfully queued for processing - * -EINVAL if any input parameters are invalid - * -EAGAIN if request would block and is flagged DLM_LKF_NOQUEUE - * -ENOMEM if there is no memory to process request - * -ENOTCONN if there is a communication error - * - * If the call to dlm_lock returns an error then the operation has failed and - * the AST routine will not be called. If dlm_lock returns 0 it is still - * possible that the lock operation will fail. The AST routine will be called - * when the locking is complete and the status is returned in the lksb. - * - * If the AST routines or parameter are passed to a conversion operation then - * they will overwrite those values that were passed to a previous dlm_lock - * call. - * - * AST routines should not block (at least not for long), but may make - * any locking calls they please. - */ - -int dlm_lock(dlm_lockspace_t *lockspace, - int mode, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*lockast) (void *astarg), - void *astarg, - void (*bast) (void *astarg, int mode)); - -/* - * dlm_unlock - * - * Asynchronously release a lock on a resource. The AST routine is called - * when the resource is successfully unlocked. - * - * lockspace: context for the request - * lkid: the lock ID as returned in the lksb - * flags: input flags (DLM_LKF_) - * lksb: if NULL the lksb parameter passed to last lock request is used - * astarg: the arg used with the completion ast for the unlock - * - * Returns: - * 0 if request is successfully queued for processing - * -EINVAL if any input parameters are invalid - * -ENOTEMPTY if the lock still has sublocks - * -EBUSY if the lock is waiting for a remote lock operation - * -ENOTCONN if there is a communication error - */ - -int dlm_unlock(dlm_lockspace_t *lockspace, - uint32_t lkid, - uint32_t flags, - struct dlm_lksb *lksb, - void *astarg); - -struct dlm_key { - void *val; - int len; -}; - -int dlm_lock_range(dlm_lockspace_t *lockspace, - int mode, - struct dlm_key *start, - struct dlm_key *end, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*lockast) (void *astarg), - void *astarg, - void (*rbast) (void *astarg, int mode, - struct dlm_key *start, struct dlm_key *end)); - -#define dlm_unlock_range dlm_unlock - -#endif /* __DLM_DOT_H__ */ diff --git a/kmod/dlm/include/linux/dlm_plock.h b/kmod/dlm/include/linux/dlm_plock.h deleted file mode 100644 index 95ad387a..00000000 --- a/kmod/dlm/include/linux/dlm_plock.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ -#ifndef __DLM_PLOCK_DOT_H__ -#define __DLM_PLOCK_DOT_H__ - -#include - -int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, - int cmd, struct file_lock *fl); -int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file, - struct file_lock *fl); -int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file, - struct file_lock *fl); -#endif diff --git a/kmod/dlm/include/uapi/linux/dlm.h b/kmod/dlm/include/uapi/linux/dlm.h deleted file mode 100644 index 1f73cc06..00000000 --- a/kmod/dlm/include/uapi/linux/dlm.h +++ /dev/null @@ -1,75 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef _UAPI__DLM_DOT_H__ -#define _UAPI__DLM_DOT_H__ - -/* - * Interface to Distributed Lock Manager (DLM) - * routines and structures to use DLM lockspaces - */ - -/* Lock levels and flags are here */ -#include -#include - -typedef void dlm_lockspace_t; - -/* - * Lock status block - * - * Use this structure to specify the contents of the lock value block. For a - * conversion request, this structure is used to specify the lock ID of the - * lock. DLM writes the status of the lock request and the lock ID assigned - * to the request in the lock status block. - * - * sb_lkid: the returned lock ID. It is set on new (non-conversion) requests. - * It is available when dlm_lock returns. - * - * sb_lvbptr: saves or returns the contents of the lock's LVB according to rules - * shown for the DLM_LKF_VALBLK flag. - * - * sb_flags: DLM_SBF_DEMOTED is returned if in the process of promoting a lock, - * it was first demoted to NL to avoid conversion deadlock. - * DLM_SBF_VALNOTVALID is returned if the resource's LVB is marked invalid. - * - * sb_status: the returned status of the lock request set prior to AST - * execution. Possible return values: - * - * 0 if lock request was successful - * -EAGAIN if request would block and is flagged DLM_LKF_NOQUEUE - * -DLM_EUNLOCK if unlock request was successful - * -DLM_ECANCEL if a cancel completed successfully - * -EDEADLK if a deadlock was detected - * -ETIMEDOUT if the lock request was canceled due to a timeout - */ - -#define DLM_SBF_DEMOTED 0x01 -#define DLM_SBF_VALNOTVALID 0x02 -#define DLM_SBF_ALTMODE 0x04 - -struct dlm_lksb { - int sb_status; - __u32 sb_lkid; - char sb_flags; - char * sb_lvbptr; -}; - -/* dlm_new_lockspace() flags */ - -#define DLM_LSFL_TIMEWARN 0x00000002 -#define DLM_LSFL_FS 0x00000004 -#define DLM_LSFL_NEWEXCL 0x00000008 - - -#endif /* _UAPI__DLM_DOT_H__ */ diff --git a/kmod/dlm/include/uapi/linux/dlm_device.h b/kmod/dlm/include/uapi/linux/dlm_device.h deleted file mode 100644 index 3060783c..00000000 --- a/kmod/dlm/include/uapi/linux/dlm_device.h +++ /dev/null @@ -1,108 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef _LINUX_DLM_DEVICE_H -#define _LINUX_DLM_DEVICE_H - -/* This is the device interface for dlm, most users will use a library - * interface. - */ - -#include -#include - -#define DLM_USER_LVB_LEN 32 - -/* Version of the device interface */ -#define DLM_DEVICE_VERSION_MAJOR 6 -#define DLM_DEVICE_VERSION_MINOR 0 -#define DLM_DEVICE_VERSION_PATCH 1 - -/* struct passed to the lock write */ -struct dlm_lock_params { - __u8 mode; - __u8 namelen; - __u16 unused; - __u32 flags; - __u32 lkid; - __u32 parent; - __u64 xid; - __u64 timeout; - void __user *castparam; - void __user *castaddr; - void __user *bastparam; - void __user *bastaddr; - struct dlm_lksb __user *lksb; - char lvb[DLM_USER_LVB_LEN]; - char name[0]; -}; - -struct dlm_lspace_params { - __u32 flags; - __u32 minor; - char name[0]; -}; - -struct dlm_purge_params { - __u32 nodeid; - __u32 pid; -}; - -struct dlm_write_request { - __u32 version[3]; - __u8 cmd; - __u8 is64bit; - __u8 unused[2]; - - union { - struct dlm_lock_params lock; - struct dlm_lspace_params lspace; - struct dlm_purge_params purge; - } i; -}; - -struct dlm_device_version { - __u32 version[3]; -}; - -/* struct read from the "device" fd, - consists mainly of userspace pointers for the library to use */ - -struct dlm_lock_result { - __u32 version[3]; - __u32 length; - void __user * user_astaddr; - void __user * user_astparam; - struct dlm_lksb __user * user_lksb; - struct dlm_lksb lksb; - __u8 bast_mode; - __u8 unused[3]; - /* Offsets may be zero if no data is present */ - __u32 lvb_offset; -}; - -/* Commands passed to the device */ -#define DLM_USER_LOCK 1 -#define DLM_USER_UNLOCK 2 -#define DLM_USER_QUERY 3 -#define DLM_USER_CREATE_LOCKSPACE 4 -#define DLM_USER_REMOVE_LOCKSPACE 5 -#define DLM_USER_PURGE 6 -#define DLM_USER_DEADLOCK 7 - -/* Lockspace flags */ -#define DLM_USER_LSFLG_AUTOFREE 1 -#define DLM_USER_LSFLG_FORCEFREE 2 - -#endif - diff --git a/kmod/dlm/include/uapi/linux/dlm_netlink.h b/kmod/dlm/include/uapi/linux/dlm_netlink.h deleted file mode 100644 index 647c8ef2..00000000 --- a/kmod/dlm/include/uapi/linux/dlm_netlink.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2007 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ - -#ifndef _DLM_NETLINK_H -#define _DLM_NETLINK_H - -#include - -enum { - DLM_STATUS_WAITING = 1, - DLM_STATUS_GRANTED = 2, - DLM_STATUS_CONVERT = 3, -}; - -#define DLM_LOCK_DATA_VERSION 1 - -struct dlm_lock_data { - __u16 version; - __u32 lockspace_id; - int nodeid; - int ownpid; - __u32 id; - __u32 remid; - __u64 xid; - __s8 status; - __s8 grmode; - __s8 rqmode; - unsigned long timestamp; - int resource_namelen; - char resource_name[DLM_RESNAME_MAXLEN]; -}; - -enum { - DLM_CMD_UNSPEC = 0, - DLM_CMD_HELLO, /* user->kernel */ - DLM_CMD_TIMEOUT, /* kernel->user */ - __DLM_CMD_MAX, -}; - -#define DLM_CMD_MAX (__DLM_CMD_MAX - 1) - -enum { - DLM_TYPE_UNSPEC = 0, - DLM_TYPE_LOCK, - __DLM_TYPE_MAX, -}; - -#define DLM_TYPE_MAX (__DLM_TYPE_MAX - 1) - -#define DLM_GENL_VERSION 0x1 -#define DLM_GENL_NAME "DLM" - -#endif /* _DLM_NETLINK_H */ diff --git a/kmod/dlm/include/uapi/linux/dlm_plock.h b/kmod/dlm/include/uapi/linux/dlm_plock.h deleted file mode 100644 index 6ae692c9..00000000 --- a/kmod/dlm/include/uapi/linux/dlm_plock.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ - -#ifndef _UAPI__DLM_PLOCK_DOT_H__ -#define _UAPI__DLM_PLOCK_DOT_H__ - -#include - -#define DLM_PLOCK_MISC_NAME "dlm_plock" - -#define DLM_PLOCK_VERSION_MAJOR 1 -#define DLM_PLOCK_VERSION_MINOR 2 -#define DLM_PLOCK_VERSION_PATCH 0 - -enum { - DLM_PLOCK_OP_LOCK = 1, - DLM_PLOCK_OP_UNLOCK, - DLM_PLOCK_OP_GET, -}; - -#define DLM_PLOCK_FL_CLOSE 1 - -struct dlm_plock_info { - __u32 version[3]; - __u8 optype; - __u8 ex; - __u8 wait; - __u8 flags; - __u32 pid; - __s32 nodeid; - __s32 rv; - __u32 fsid; - __u64 number; - __u64 start; - __u64 end; - __u64 owner; -}; - - -#endif /* _UAPI__DLM_PLOCK_DOT_H__ */ diff --git a/kmod/dlm/include/uapi/linux/dlmconstants.h b/kmod/dlm/include/uapi/linux/dlmconstants.h deleted file mode 100644 index 2857bdc5..00000000 --- a/kmod/dlm/include/uapi/linux/dlmconstants.h +++ /dev/null @@ -1,163 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __DLMCONSTANTS_DOT_H__ -#define __DLMCONSTANTS_DOT_H__ - -/* - * Constants used by DLM interface. - */ - -#define DLM_LOCKSPACE_LEN 64 -#define DLM_RESNAME_MAXLEN 64 - - -/* - * Lock Modes - */ - -#define DLM_LOCK_IV (-1) /* invalid */ -#define DLM_LOCK_NL 0 /* null */ -#define DLM_LOCK_CR 1 /* concurrent read */ -#define DLM_LOCK_CW 2 /* concurrent write */ -#define DLM_LOCK_PR 3 /* protected read */ -#define DLM_LOCK_PW 4 /* protected write */ -#define DLM_LOCK_EX 5 /* exclusive */ - - -/* - * Flags to dlm_lock - * - * DLM_LKF_NOQUEUE - * - * Do not queue the lock request on the wait queue if it cannot be granted - * immediately. If the lock cannot be granted because of this flag, DLM will - * either return -EAGAIN from the dlm_lock call or will return 0 from - * dlm_lock and -EAGAIN in the lock status block when the AST is executed. - * - * DLM_LKF_CANCEL - * - * Used to cancel a pending lock request or conversion. A converting lock is - * returned to its previously granted mode. - * - * DLM_LKF_CONVERT - * - * Indicates a lock conversion request. For conversions the name and namelen - * are ignored and the lock ID in the LKSB is used to identify the lock. - * - * DLM_LKF_VALBLK - * - * Requests DLM to return the current contents of the lock value block in the - * lock status block. When this flag is set in a lock conversion from PW or EX - * modes, DLM assigns the value specified in the lock status block to the lock - * value block of the lock resource. The LVB is a DLM_LVB_LEN size array - * containing application-specific information. - * - * DLM_LKF_QUECVT - * - * Force a conversion request to be queued, even if it is compatible with - * the granted modes of other locks on the same resource. - * - * DLM_LKF_IVVALBLK - * - * Invalidate the lock value block. - * - * DLM_LKF_CONVDEADLK - * - * Allows the dlm to resolve conversion deadlocks internally by demoting the - * granted mode of a converting lock to NL. The DLM_SBF_DEMOTED flag is - * returned for a conversion that's been effected by this. - * - * DLM_LKF_PERSISTENT - * - * Only relevant to locks originating in userspace. A persistent lock will not - * be removed if the process holding the lock exits. - * - * DLM_LKF_NODLCKWT - * - * Do not cancel the lock if it gets into conversion deadlock. - * Exclude this lock from being monitored due to DLM_LSFL_TIMEWARN. - * - * DLM_LKF_NODLCKBLK - * - * net yet implemented - * - * DLM_LKF_EXPEDITE - * - * Used only with new requests for NL mode locks. Tells the lock manager - * to grant the lock, ignoring other locks in convert and wait queues. - * - * DLM_LKF_NOQUEUEBAST - * - * Send blocking AST's before returning -EAGAIN to the caller. It is only - * used along with the NOQUEUE flag. Blocking AST's are not sent for failed - * NOQUEUE requests otherwise. - * - * DLM_LKF_HEADQUE - * - * Add a lock to the head of the convert or wait queue rather than the tail. - * - * DLM_LKF_NOORDER - * - * Disregard the standard grant order rules and grant a lock as soon as it - * is compatible with other granted locks. - * - * DLM_LKF_ORPHAN - * - * Acquire an orphan lock. - * - * DLM_LKF_ALTPR - * - * If the requested mode cannot be granted immediately, try to grant the lock - * in PR mode instead. If this alternate mode is granted instead of the - * requested mode, DLM_SBF_ALTMODE is returned in the lksb. - * - * DLM_LKF_ALTCW - * - * The same as ALTPR, but the alternate mode is CW. - * - * DLM_LKF_FORCEUNLOCK - * - * Unlock the lock even if it is converting or waiting or has sublocks. - * Only really for use by the userland device.c code. - * - */ - -#define DLM_LKF_NOQUEUE 0x00000001 -#define DLM_LKF_CANCEL 0x00000002 -#define DLM_LKF_CONVERT 0x00000004 -#define DLM_LKF_VALBLK 0x00000008 -#define DLM_LKF_QUECVT 0x00000010 -#define DLM_LKF_IVVALBLK 0x00000020 -#define DLM_LKF_CONVDEADLK 0x00000040 -#define DLM_LKF_PERSISTENT 0x00000080 -#define DLM_LKF_NODLCKWT 0x00000100 -#define DLM_LKF_NODLCKBLK 0x00000200 -#define DLM_LKF_EXPEDITE 0x00000400 -#define DLM_LKF_NOQUEUEBAST 0x00000800 -#define DLM_LKF_HEADQUE 0x00001000 -#define DLM_LKF_NOORDER 0x00002000 -#define DLM_LKF_ORPHAN 0x00004000 -#define DLM_LKF_ALTPR 0x00008000 -#define DLM_LKF_ALTCW 0x00010000 -#define DLM_LKF_FORCEUNLOCK 0x00020000 -#define DLM_LKF_TIMEOUT 0x00040000 - -/* - * Some return codes that are not in errno.h - */ - -#define DLM_ECANCEL 0x10001 -#define DLM_EUNLOCK 0x10002 - -#endif /* __DLMCONSTANTS_DOT_H__ */ diff --git a/kmod/dlm/interval_tree_generic.h b/kmod/dlm/interval_tree_generic.h deleted file mode 100644 index d70e20c6..00000000 --- a/kmod/dlm/interval_tree_generic.h +++ /dev/null @@ -1,216 +0,0 @@ -/* - Interval Trees - (C) 2012 Michel Lespinasse - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - include/linux/interval_tree_generic.h -*/ - -#include - -#include - -/* - * Template for implementing interval trees - * - * ITSTRUCT: struct type of the interval tree nodes - * ITRB: name of struct rb_node field within ITSTRUCT - * ITTYPE: type of the interval endpoints - * ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree - * ITSTART(n): start endpoint of ITSTRUCT node n - * ITLAST(n): last endpoint of ITSTRUCT node n - * ITSTATIC: 'static' or empty - * ITPREFIX: prefix to use for the inline tree definitions - * - * Note - before using this, please consider if non-generic version - * (interval_tree.h) would work for you... - */ -#define INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \ - ITSTART, ITLAST, ITSTATIC, ITPREFIX) \ - \ -static inline int ITPREFIX ## _cmp(ITTYPE a, ITTYPE b) \ -{ \ - if (a < b) \ - return -1; \ - else if (a > b) \ - return 1; \ - else \ - return 0; \ -} \ -KEYED_INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, ITSTART, ITLAST,\ - ITPREFIX ## _cmp, ITSTATIC, ITPREFIX) - -/* - * int iTCMP(ITTYPE endpoint1, ITTYPE endpoint2); - * Returns: - * < 0 if endpoint1 < endpoint2 - * 0 if endpoint1 == endpoint2 - * > 0 if endpoint1 > endpoint2 - */ -#define KEYED_INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \ - ITSTART, ITLAST, ITCMP, ITSTATIC, ITPREFIX)\ -/* Callbacks for augmented rbtree insert and remove */ \ - \ -static inline ITTYPE ITPREFIX ## _compute_subtree_last(ITSTRUCT *node) \ -{ \ - ITTYPE max = ITLAST(node); \ - ITTYPE subtree_last; \ - \ - if (node->ITRB.rb_left) { \ - subtree_last = rb_entry(node->ITRB.rb_left, \ - ITSTRUCT, ITRB)->ITSUBTREE; \ - if (ITCMP(max, subtree_last) < 0) \ - max = subtree_last; \ - } \ - if (node->ITRB.rb_right) { \ - subtree_last = rb_entry(node->ITRB.rb_right, \ - ITSTRUCT, ITRB)->ITSUBTREE; \ - if (ITCMP(max, subtree_last) < 0) \ - max = subtree_last; \ - } \ - return max; \ -} \ - \ -RB_DECLARE_CALLBACKS(static, ITPREFIX ## _augment, ITSTRUCT, ITRB, \ - ITTYPE, ITSUBTREE, ITPREFIX ## _compute_subtree_last) \ - \ -/* Insert / remove interval nodes from the tree */ \ - \ -ITSTATIC void ITPREFIX ## _insert(ITSTRUCT *node, struct rb_root *root) \ -{ \ - struct rb_node **link = &root->rb_node, *rb_parent = NULL; \ - ITTYPE start = ITSTART(node); \ - ITTYPE last = ITLAST(node); \ - ITSTRUCT *parent; \ - \ - while (*link) { \ - rb_parent = *link; \ - parent = rb_entry(rb_parent, ITSTRUCT, ITRB); \ - if (ITCMP(parent->ITSUBTREE, last) < 0) \ - parent->ITSUBTREE = last; \ - if (ITCMP(start, ITSTART(parent)) < 0) \ - link = &parent->ITRB.rb_left; \ - else \ - link = &parent->ITRB.rb_right; \ - } \ - \ - node->ITSUBTREE = last; \ - rb_link_node(&node->ITRB, rb_parent, link); \ - rb_insert_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \ -} \ - \ -ITSTATIC void ITPREFIX ## _remove(ITSTRUCT *node, struct rb_root *root) \ -{ \ - rb_erase_augmented(&node->ITRB, root, &ITPREFIX ## _augment); \ -} \ - \ -/* \ - * Iterate over intervals intersecting [start;last] \ - * \ - * Note that a node's interval intersects [start;last] iff: \ - * Cond1: ITSTART(node) <= last \ - * and \ - * Cond2: start <= ITLAST(node) \ - */ \ - \ -static ITSTRUCT * \ -ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ -{ \ - while (true) { \ - /* \ - * Loop invariant: start <= node->ITSUBTREE \ - * (Cond2 is satisfied by one of the subtree nodes) \ - */ \ - if (node->ITRB.rb_left) { \ - ITSTRUCT *left = rb_entry(node->ITRB.rb_left, \ - ITSTRUCT, ITRB); \ - if (ITCMP(start, left->ITSUBTREE) <= 0) { \ - /* \ - * Some nodes in left subtree satisfy Cond2. \ - * Iterate to find the leftmost such node N. \ - * If it also satisfies Cond1, that's the \ - * match we are looking for. Otherwise, there \ - * is no matching interval as nodes to the \ - * right of N can't satisfy Cond1 either. \ - */ \ - node = left; \ - continue; \ - } \ - } \ - if (ITCMP(ITSTART(node), last) <= 0) { /* Cond1 */ \ - if (ITCMP(start, ITLAST(node)) <= 0) /* Cond2 */ \ - return node; /* node is leftmost match */ \ - if (node->ITRB.rb_right) { \ - node = rb_entry(node->ITRB.rb_right, \ - ITSTRUCT, ITRB); \ - if (ITCMP(start, node->ITSUBTREE) <= 0) \ - continue; \ - } \ - } \ - return NULL; /* No match */ \ - } \ -} \ - \ -ITSTATIC ITSTRUCT * \ -ITPREFIX ## _iter_first(struct rb_root *root, ITTYPE start, ITTYPE last) \ -{ \ - ITSTRUCT *node; \ - \ - if (!root->rb_node) \ - return NULL; \ - node = rb_entry(root->rb_node, ITSTRUCT, ITRB); \ - if (ITCMP(node->ITSUBTREE, start) < 0) \ - return NULL; \ - return ITPREFIX ## _subtree_search(node, start, last); \ -} \ - \ -ITSTATIC ITSTRUCT * \ -ITPREFIX ## _iter_next(ITSTRUCT *node, ITTYPE start, ITTYPE last) \ -{ \ - struct rb_node *rb = node->ITRB.rb_right, *prev; \ - \ - while (true) { \ - /* \ - * Loop invariants: \ - * Cond1: ITSTART(node) <= last \ - * rb == node->ITRB.rb_right \ - * \ - * First, search right subtree if suitable \ - */ \ - if (rb) { \ - ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB); \ - if (ITCMP(start, right->ITSUBTREE) <= 0) \ - return ITPREFIX ## _subtree_search(right, \ - start, last); \ - } \ - \ - /* Move up the tree until we come from a node's left child */ \ - do { \ - rb = rb_parent(&node->ITRB); \ - if (!rb) \ - return NULL; \ - prev = &node->ITRB; \ - node = rb_entry(rb, ITSTRUCT, ITRB); \ - rb = node->ITRB.rb_right; \ - } while (prev == rb); \ - \ - /* Check if the node intersects [start;last] */ \ - if (ITCMP(last, ITSTART(node)) < 0) /* !Cond1 */ \ - return NULL; \ - else if (ITCMP(start, ITLAST(node)) <= 0) /* Cond2 */ \ - return node; \ - } \ -} diff --git a/kmod/dlm/lock.c b/kmod/dlm/lock.c deleted file mode 100644 index d7464204..00000000 --- a/kmod/dlm/lock.c +++ /dev/null @@ -1,6605 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -/* Central locking logic has four stages: - - dlm_lock() - dlm_unlock() - - request_lock(ls, lkb) - convert_lock(ls, lkb) - unlock_lock(ls, lkb) - cancel_lock(ls, lkb) - - _request_lock(r, lkb) - _convert_lock(r, lkb) - _unlock_lock(r, lkb) - _cancel_lock(r, lkb) - - do_request(r, lkb) - do_convert(r, lkb) - do_unlock(r, lkb) - do_cancel(r, lkb) - - Stage 1 (lock, unlock) is mainly about checking input args and - splitting into one of the four main operations: - - dlm_lock = request_lock - dlm_lock+CONVERT = convert_lock - dlm_unlock = unlock_lock - dlm_unlock+CANCEL = cancel_lock - - Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is - provided to the next stage. - - Stage 3, _xxxx_lock(), determines if the operation is local or remote. - When remote, it calls send_xxxx(), when local it calls do_xxxx(). - - Stage 4, do_xxxx(), is the guts of the operation. It manipulates the - given rsb and lkb and queues callbacks. - - For remote operations, send_xxxx() results in the corresponding do_xxxx() - function being executed on the remote node. The connecting send/receive - calls on local (L) and remote (R) nodes: - - L: send_xxxx() -> R: receive_xxxx() - R: do_xxxx() - L: receive_xxxx_reply() <- R: send_xxxx_reply() -*/ -#include -#include -#include -#include "dlm_internal.h" -#include -#include "interval_tree_generic.h" -#include "memory.h" -#include "lowcomms.h" -#include "requestqueue.h" -#include "util.h" -#include "dir.h" -#include "member.h" -#include "lockspace.h" -#include "ast.h" -#include "lock.h" -#include "rcom.h" -#include "recover.h" -#include "lvb_table.h" -#include "user.h" -#include "config.h" - -static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode); -static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int send_remove(struct dlm_rsb *r); -static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); -static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); -static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct dlm_message *ms); -static int receive_extralen(struct dlm_message *ms); -static void do_purge(struct dlm_ls *ls, int nodeid, int pid); -static void del_timeout(struct dlm_lkb *lkb); -static void toss_rsb(struct kref *kref); - - -/* - * Lock compatibilty matrix - thanks Steve - * UN = Unlocked state. Not really a state, used as a flag - * PD = Padding. Used to make the matrix a nice power of two in size - * Other states are the same as the VMS DLM. - * Usage: matrix[grmode+1][rqmode+1] (although m[rq+1][gr+1] is the same) - */ - -static const int __dlm_compat_matrix[8][8] = { - /* UN NL CR CW PR PW EX PD */ - {1, 1, 1, 1, 1, 1, 1, 0}, /* UN */ - {1, 1, 1, 1, 1, 1, 1, 0}, /* NL */ - {1, 1, 1, 1, 1, 1, 0, 0}, /* CR */ - {1, 1, 1, 1, 0, 0, 0, 0}, /* CW */ - {1, 1, 1, 0, 1, 0, 0, 0}, /* PR */ - {1, 1, 1, 0, 0, 0, 0, 0}, /* PW */ - {1, 1, 0, 0, 0, 0, 0, 0}, /* EX */ - {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ -}; - -/* - * This defines the direction of transfer of LVB data. - * Granted mode is the row; requested mode is the column. - * Usage: matrix[grmode+1][rqmode+1] - * 1 = LVB is returned to the caller - * 0 = LVB is written to the resource - * -1 = nothing happens to the LVB - */ - -const int dlm_lvb_operations[8][8] = { - /* UN NL CR CW PR PW EX PD*/ - { -1, 1, 1, 1, 1, 1, 1, -1 }, /* UN */ - { -1, 1, 1, 1, 1, 1, 1, 0 }, /* NL */ - { -1, -1, 1, 1, 1, 1, 1, 0 }, /* CR */ - { -1, -1, -1, 1, 1, 1, 1, 0 }, /* CW */ - { -1, -1, -1, -1, 1, 1, 1, 0 }, /* PR */ - { -1, 0, 0, 0, 0, 0, 1, 0 }, /* PW */ - { -1, 0, 0, 0, 0, 0, 0, 0 }, /* EX */ - { -1, 0, 0, 0, 0, 0, 0, 0 } /* PD */ -}; - -#define _modes_compat(gr, rq) \ - __dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1] - -/* - * Define start and end for a range that covers all possible - * values. Use these as defaults (instead of NULL pointers) for lkbs - * created by non ranged lock requests. Without these we'd have to - * implement switches or alternative algorithms each time a NULL key - * was encountered. - */ -static unsigned long long default_start = 0ULL; -#define default_start_len sizeof(default_start) -static struct dlm_key default_start_key = { &default_start, - default_start_len }; -static char default_end[DLM_KEY_LEN] = { [ 0 ... (DLM_KEY_LEN-1) ] = -1 }; -#define default_end_len DLM_KEY_LEN -static struct dlm_key default_end_key = { default_end, default_end_len }; -static struct dlm_range default_range = { .start = &default_start_key, - .end = &default_end_key }; - -/* Fast debug printing */ -#define debug_range_to_ull(ENDPOINT) \ -static inline u64 ENDPOINT ## _to_ull(struct dlm_range *range) \ -{ \ - u64 val = 0ULL; \ - \ - if (range->ENDPOINT && range->ENDPOINT->len > sizeof(u64)) { \ - memcpy(&val, range->ENDPOINT->val, min((int)sizeof(val),\ - range->ENDPOINT->len)); \ - return val; \ - } \ - return 0; \ -} -debug_range_to_ull(start); -debug_range_to_ull(end); - -static struct dlm_key *alloc_key(char *val, int len, gfp_t gfp) -{ - struct dlm_key *ret = kmalloc(sizeof(*ret), gfp); - if (ret) { - ret->len = len; - ret->val = kmalloc(len, gfp); - if (!ret->val) { - kfree(ret); - return NULL; - } - memcpy(ret->val, val, len); - } - return ret; -} - -static int cmp_range_keys(char *a, int a_len, char *b, int b_len) -{ - return memcmp(a, b, min(a_len, b_len)) ?: - a_len < b_len ? -1 : a_len > b_len ? 1 : 0; -} - -static inline int cmp_dlm_keys(struct dlm_key *a, struct dlm_key *b) -{ - return cmp_range_keys(a->val, a->len, b->val, b->len); -} - -/* - * Define our interval tree nodes to index by granted start/end - * values. We might want a tree sorted by requested start/end in the - * future if walking the converting list winds up being costly. - */ -#define START(lkb) ((lkb)->lkb_grrange.start) -#define LAST(lkb) ((lkb)->lkb_grrange.end) -KEYED_INTERVAL_TREE_DEFINE(struct dlm_lkb, lkb_statenode, struct dlm_key *, - lkb_subtree_last, START, LAST, cmp_dlm_keys, - static, rsb_interval); - -int ranges_overlap(struct dlm_range *range1, struct dlm_range *range2) -{ - int ret1, ret2; - - ret1 = cmp_range_keys(range1->start->val, range1->start->len, - range2->end->val, range2->end->len); - - ret2 = cmp_range_keys(range1->end->val, range1->end->len, - range2->start->val, range2->start->len); - - if (ret1 <= 0 && ret2 >= 0) - return 1; - - return 0; -} - -static int modes_compat(struct dlm_lkb *gr, struct dlm_lkb *rq) -{ - if (cmp_dlm_keys(rq->lkb_rqrange.start, gr->lkb_grrange.end) <= 0 && - cmp_dlm_keys(rq->lkb_rqrange.end, gr->lkb_grrange.start) >= 0) - return _modes_compat(gr, rq); - return 0; -} - -int dlm_modes_compat(int mode1, int mode2) -{ - /* XXX: This needs to be fixed up to take ranges into account */ - return __dlm_compat_matrix[mode1 + 1][mode2 + 1]; -} - -/* - * Compatibility matrix for conversions with QUECVT set. - * Granted mode is the row; requested mode is the column. - * Usage: matrix[grmode+1][rqmode+1] - */ - -static const int __quecvt_compat_matrix[8][8] = { - /* UN NL CR CW PR PW EX PD */ - {0, 0, 0, 0, 0, 0, 0, 0}, /* UN */ - {0, 0, 1, 1, 1, 1, 1, 0}, /* NL */ - {0, 0, 0, 1, 1, 1, 1, 0}, /* CR */ - {0, 0, 0, 0, 1, 1, 1, 0}, /* CW */ - {0, 0, 0, 1, 0, 1, 1, 0}, /* PR */ - {0, 0, 0, 0, 0, 0, 1, 0}, /* PW */ - {0, 0, 0, 0, 0, 0, 0, 0}, /* EX */ - {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ -}; - -void dlm_print_lkb(struct dlm_lkb *lkb) -{ - printk(KERN_ERR "lkb: nodeid %d id %x remid %x exflags %x flags %x " - "sts %d rq %d gr %d wait_type %d wait_nodeid %d seq %llu\n", - lkb->lkb_nodeid, lkb->lkb_id, lkb->lkb_remid, lkb->lkb_exflags, - lkb->lkb_flags, lkb->lkb_status, lkb->lkb_rqmode, - lkb->lkb_grmode, lkb->lkb_wait_type, lkb->lkb_wait_nodeid, - (unsigned long long)lkb->lkb_recover_seq); -} - -static void dlm_print_rsb(struct dlm_rsb *r) -{ - printk(KERN_ERR "rsb: nodeid %d master %d dir %d flags %lx first %x " - "rlc %d name %s\n", - r->res_nodeid, r->res_master_nodeid, r->res_dir_nodeid, - r->res_flags, r->res_first_lkid, r->res_recover_locks_count, - r->res_name); -} - -void dlm_dump_rsb(struct dlm_rsb *r) -{ - struct dlm_lkb *lkb; - - dlm_print_rsb(r); - - printk(KERN_ERR "rsb: root_list empty %d recover_list empty %d\n", - list_empty(&r->res_root_list), list_empty(&r->res_recover_list)); - printk(KERN_ERR "rsb lookup list\n"); - list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) - dlm_print_lkb(lkb); - printk(KERN_ERR "rsb grant queue:\n"); - list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) - dlm_print_lkb(lkb); - printk(KERN_ERR "rsb convert queue:\n"); - list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) - dlm_print_lkb(lkb); - printk(KERN_ERR "rsb wait queue:\n"); - list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) - dlm_print_lkb(lkb); -} - -/* Threads cannot use the lockspace while it's being recovered */ - -static inline void dlm_lock_recovery(struct dlm_ls *ls) -{ - down_read(&ls->ls_in_recovery); -} - -void dlm_unlock_recovery(struct dlm_ls *ls) -{ - up_read(&ls->ls_in_recovery); -} - -int dlm_lock_recovery_try(struct dlm_ls *ls) -{ - return down_read_trylock(&ls->ls_in_recovery); -} - -static inline int can_be_queued(struct dlm_lkb *lkb) -{ - return !(lkb->lkb_exflags & DLM_LKF_NOQUEUE); -} - -static inline int force_blocking_asts(struct dlm_lkb *lkb) -{ - return (lkb->lkb_exflags & DLM_LKF_NOQUEUEBAST); -} - -static inline int is_demoted(struct dlm_lkb *lkb) -{ - return (lkb->lkb_sbflags & DLM_SBF_DEMOTED); -} - -static inline int is_altmode(struct dlm_lkb *lkb) -{ - return (lkb->lkb_sbflags & DLM_SBF_ALTMODE); -} - -static inline int is_granted(struct dlm_lkb *lkb) -{ - return (lkb->lkb_status == DLM_LKSTS_GRANTED); -} - -static inline int is_remote(struct dlm_rsb *r) -{ - DLM_ASSERT(r->res_nodeid >= 0, dlm_print_rsb(r);); - return !!r->res_nodeid; -} - -static inline int is_process_copy(struct dlm_lkb *lkb) -{ - return (lkb->lkb_nodeid && !(lkb->lkb_flags & DLM_IFL_MSTCPY)); -} - -static inline int is_master_copy(struct dlm_lkb *lkb) -{ - return (lkb->lkb_flags & DLM_IFL_MSTCPY) ? 1 : 0; -} - -static inline int middle_conversion(struct dlm_lkb *lkb) -{ - if ((lkb->lkb_grmode==DLM_LOCK_PR && lkb->lkb_rqmode==DLM_LOCK_CW) || - (lkb->lkb_rqmode==DLM_LOCK_PR && lkb->lkb_grmode==DLM_LOCK_CW)) - return 1; - return 0; -} - -static inline int down_conversion(struct dlm_lkb *lkb) -{ - return (!middle_conversion(lkb) && lkb->lkb_rqmode < lkb->lkb_grmode); -} - -static inline int is_overlap_unlock(struct dlm_lkb *lkb) -{ - return lkb->lkb_flags & DLM_IFL_OVERLAP_UNLOCK; -} - -static inline int is_overlap_cancel(struct dlm_lkb *lkb) -{ - return lkb->lkb_flags & DLM_IFL_OVERLAP_CANCEL; -} - -static inline int is_overlap(struct dlm_lkb *lkb) -{ - return (lkb->lkb_flags & (DLM_IFL_OVERLAP_UNLOCK | - DLM_IFL_OVERLAP_CANCEL)); -} - -static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) -{ - if (is_master_copy(lkb)) - return; - - del_timeout(lkb); - - DLM_ASSERT(lkb->lkb_lksb, dlm_print_lkb(lkb);); - - /* if the operation was a cancel, then return -DLM_ECANCEL, if a - timeout caused the cancel then return -ETIMEDOUT */ - if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_TIMEOUT_CANCEL)) { - lkb->lkb_flags &= ~DLM_IFL_TIMEOUT_CANCEL; - rv = -ETIMEDOUT; - } - - if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_DEADLOCK_CANCEL)) { - lkb->lkb_flags &= ~DLM_IFL_DEADLOCK_CANCEL; - rv = -EDEADLK; - } - - dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, NULL, rv, lkb->lkb_sbflags); -} - -static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - queue_cast(r, lkb, - is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL); -} - -static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode, - struct dlm_range *rqrange) -{ - if (is_master_copy(lkb)) { - send_bast(r, lkb, rqmode); - } else { - dlm_add_cb(lkb, DLM_CB_BAST, rqmode, rqrange, 0, 0); - } -} - -/* - * Basic operations on rsb's and lkb's - */ - -/* This is only called to add a reference when the code already holds - a valid reference to the rsb, so there's no need for locking. */ - -static inline void hold_rsb(struct dlm_rsb *r) -{ - kref_get(&r->res_ref); -} - -void dlm_hold_rsb(struct dlm_rsb *r) -{ - hold_rsb(r); -} - -/* When all references to the rsb are gone it's transferred to - the tossed list for later disposal. */ - -static void put_rsb(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - uint32_t bucket = r->res_bucket; - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - kref_put(&r->res_ref, toss_rsb); - spin_unlock(&ls->ls_rsbtbl[bucket].lock); -} - -void dlm_put_rsb(struct dlm_rsb *r) -{ - put_rsb(r); -} - -static int pre_rsb_struct(struct dlm_ls *ls) -{ - struct dlm_rsb *r1, *r2; - int count = 0; - - spin_lock(&ls->ls_new_rsb_spin); - if (ls->ls_new_rsb_count > dlm_config.ci_new_rsb_count / 2) { - spin_unlock(&ls->ls_new_rsb_spin); - return 0; - } - spin_unlock(&ls->ls_new_rsb_spin); - - r1 = dlm_allocate_rsb(ls); - r2 = dlm_allocate_rsb(ls); - - spin_lock(&ls->ls_new_rsb_spin); - if (r1) { - list_add(&r1->res_hashchain, &ls->ls_new_rsb); - ls->ls_new_rsb_count++; - } - if (r2) { - list_add(&r2->res_hashchain, &ls->ls_new_rsb); - ls->ls_new_rsb_count++; - } - count = ls->ls_new_rsb_count; - spin_unlock(&ls->ls_new_rsb_spin); - - if (!count) - return -ENOMEM; - return 0; -} - -/* If ls->ls_new_rsb is empty, return -EAGAIN, so the caller can - unlock any spinlocks, go back and call pre_rsb_struct again. - Otherwise, take an rsb off the list and return it. */ - -static int get_rsb_struct(struct dlm_ls *ls, char *name, int len, - struct dlm_rsb **r_ret) -{ - struct dlm_rsb *r; - int count; - - spin_lock(&ls->ls_new_rsb_spin); - if (list_empty(&ls->ls_new_rsb)) { - count = ls->ls_new_rsb_count; - spin_unlock(&ls->ls_new_rsb_spin); - log_debug(ls, "find_rsb retry %d %d %s", - count, dlm_config.ci_new_rsb_count, name); - return -EAGAIN; - } - - r = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, res_hashchain); - list_del(&r->res_hashchain); - /* Convert the empty list_head to a NULL rb_node for tree usage: */ - memset(&r->res_hashnode, 0, sizeof(struct rb_node)); - ls->ls_new_rsb_count--; - spin_unlock(&ls->ls_new_rsb_spin); - - r->res_ls = ls; - r->res_length = len; - memcpy(r->res_name, name, len); - mutex_init(&r->res_mutex); - - INIT_LIST_HEAD(&r->res_lookup); - INIT_LIST_HEAD(&r->res_grantqueue); - INIT_LIST_HEAD(&r->res_convertqueue); - INIT_LIST_HEAD(&r->res_waitqueue); - INIT_LIST_HEAD(&r->res_root_list); - INIT_LIST_HEAD(&r->res_recover_list); - - *r_ret = r; - return 0; -} - -static int rsb_cmp(struct dlm_rsb *r, const char *name, int nlen) -{ - char maxname[DLM_RESNAME_MAXLEN]; - - memset(maxname, 0, DLM_RESNAME_MAXLEN); - memcpy(maxname, name, nlen); - return memcmp(r->res_name, maxname, DLM_RESNAME_MAXLEN); -} - -int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, - struct dlm_rsb **r_ret) -{ - struct rb_node *node = tree->rb_node; - struct dlm_rsb *r; - int rc; - - while (node) { - r = rb_entry(node, struct dlm_rsb, res_hashnode); - rc = rsb_cmp(r, name, len); - if (rc < 0) - node = node->rb_left; - else if (rc > 0) - node = node->rb_right; - else - goto found; - } - *r_ret = NULL; - return -EBADR; - - found: - *r_ret = r; - return 0; -} - -static int rsb_insert(struct dlm_rsb *rsb, struct rb_root *tree) -{ - struct rb_node **newn = &tree->rb_node; - struct rb_node *parent = NULL; - int rc; - - while (*newn) { - struct dlm_rsb *cur = rb_entry(*newn, struct dlm_rsb, - res_hashnode); - - parent = *newn; - rc = rsb_cmp(cur, rsb->res_name, rsb->res_length); - if (rc < 0) - newn = &parent->rb_left; - else if (rc > 0) - newn = &parent->rb_right; - else { - log_print("rsb_insert match"); - dlm_dump_rsb(rsb); - dlm_dump_rsb(cur); - return -EEXIST; - } - } - - rb_link_node(&rsb->res_hashnode, parent, newn); - rb_insert_color(&rsb->res_hashnode, tree); - return 0; -} - -/* - * Find rsb in rsbtbl and potentially create/add one - * - * Delaying the release of rsb's has a similar benefit to applications keeping - * NL locks on an rsb, but without the guarantee that the cached master value - * will still be valid when the rsb is reused. Apps aren't always smart enough - * to keep NL locks on an rsb that they may lock again shortly; this can lead - * to excessive master lookups and removals if we don't delay the release. - * - * Searching for an rsb means looking through both the normal list and toss - * list. When found on the toss list the rsb is moved to the normal list with - * ref count of 1; when found on normal list the ref count is incremented. - * - * rsb's on the keep list are being used locally and refcounted. - * rsb's on the toss list are not being used locally, and are not refcounted. - * - * The toss list rsb's were either - * - previously used locally but not any more (were on keep list, then - * moved to toss list when last refcount dropped) - * - created and put on toss list as a directory record for a lookup - * (we are the dir node for the res, but are not using the res right now, - * but some other node is) - * - * The purpose of find_rsb() is to return a refcounted rsb for local use. - * So, if the given rsb is on the toss list, it is moved to the keep list - * before being returned. - * - * toss_rsb() happens when all local usage of the rsb is done, i.e. no - * more refcounts exist, so the rsb is moved from the keep list to the - * toss list. - * - * rsb's on both keep and toss lists are used for doing a name to master - * lookups. rsb's that are in use locally (and being refcounted) are on - * the keep list, rsb's that are not in use locally (not refcounted) and - * only exist for name/master lookups are on the toss list. - * - * rsb's on the toss list who's dir_nodeid is not local can have stale - * name/master mappings. So, remote requests on such rsb's can potentially - * return with an error, which means the mapping is stale and needs to - * be updated with a new lookup. (The idea behind MASTER UNCERTAIN and - * first_lkid is to keep only a single outstanding request on an rsb - * while that rsb has a potentially stale master.) - */ - -static int find_rsb_dir(struct dlm_ls *ls, char *name, int len, - uint32_t hash, uint32_t b, - int dir_nodeid, int from_nodeid, - unsigned int flags, struct dlm_rsb **r_ret) -{ - struct dlm_rsb *r = NULL; - int our_nodeid = dlm_our_nodeid(); - int from_local = 0; - int from_other = 0; - int from_dir = 0; - int create = 0; - int error; - - if (flags & R_RECEIVE_REQUEST) { - if (from_nodeid == dir_nodeid) - from_dir = 1; - else - from_other = 1; - } else if (flags & R_REQUEST) { - from_local = 1; - } - - /* - * flags & R_RECEIVE_RECOVER is from dlm_recover_master_copy, so - * from_nodeid has sent us a lock in dlm_recover_locks, believing - * we're the new master. Our local recovery may not have set - * res_master_nodeid to our_nodeid yet, so allow either. Don't - * create the rsb; dlm_recover_process_copy() will handle EBADR - * by resending. - * - * If someone sends us a request, we are the dir node, and we do - * not find the rsb anywhere, then recreate it. This happens if - * someone sends us a request after we have removed/freed an rsb - * from our toss list. (They sent a request instead of lookup - * because they are using an rsb from their toss list.) - */ - - if (from_local || from_dir || - (from_other && (dir_nodeid == our_nodeid))) { - create = 1; - } - - retry: - if (create) { - error = pre_rsb_struct(ls); - if (error < 0) - goto out; - } - - spin_lock(&ls->ls_rsbtbl[b].lock); - - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (error) - goto do_toss; - - /* - * rsb is active, so we can't check master_nodeid without lock_rsb. - */ - - kref_get(&r->res_ref); - error = 0; - goto out_unlock; - - - do_toss: - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (error) - goto do_new; - - /* - * rsb found inactive (master_nodeid may be out of date unless - * we are the dir_nodeid or were the master) No other thread - * is using this rsb because it's on the toss list, so we can - * look at or update res_master_nodeid without lock_rsb. - */ - - if ((r->res_master_nodeid != our_nodeid) && from_other) { - /* our rsb was not master, and another node (not the dir node) - has sent us a request */ - log_debug(ls, "find_rsb toss from_other %d master %d dir %d %s", - from_nodeid, r->res_master_nodeid, dir_nodeid, - r->res_name); - error = -ENOTBLK; - goto out_unlock; - } - - if ((r->res_master_nodeid != our_nodeid) && from_dir) { - /* don't think this should ever happen */ - log_error(ls, "find_rsb toss from_dir %d master %d", - from_nodeid, r->res_master_nodeid); - dlm_print_rsb(r); - /* fix it and go on */ - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); - r->res_first_lkid = 0; - } - - if (from_local && (r->res_master_nodeid != our_nodeid)) { - /* Because we have held no locks on this rsb, - res_master_nodeid could have become stale. */ - rsb_set_flag(r, RSB_MASTER_UNCERTAIN); - r->res_first_lkid = 0; - } - - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); - error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); - goto out_unlock; - - - do_new: - /* - * rsb not found - */ - - if (error == -EBADR && !create) - goto out_unlock; - - error = get_rsb_struct(ls, name, len, &r); - if (error == -EAGAIN) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - goto retry; - } - if (error) - goto out_unlock; - - r->res_hash = hash; - r->res_bucket = b; - r->res_dir_nodeid = dir_nodeid; - kref_init(&r->res_ref); - - if (from_dir) { - /* want to see how often this happens */ - log_debug(ls, "find_rsb new from_dir %d recreate %s", - from_nodeid, r->res_name); - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - goto out_add; - } - - if (from_other && (dir_nodeid != our_nodeid)) { - /* should never happen */ - log_error(ls, "find_rsb new from_other %d dir %d our %d %s", - from_nodeid, dir_nodeid, our_nodeid, r->res_name); - dlm_free_rsb(r); - error = -ENOTBLK; - goto out_unlock; - } - - if (from_other) { - log_debug(ls, "find_rsb new from_other %d dir %d %s", - from_nodeid, dir_nodeid, r->res_name); - } - - if (dir_nodeid == our_nodeid) { - /* When we are the dir nodeid, we can set the master - node immediately */ - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - } else { - /* set_master will send_lookup to dir_nodeid */ - r->res_master_nodeid = 0; - r->res_nodeid = -1; - } - - out_add: - error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); - out_unlock: - spin_unlock(&ls->ls_rsbtbl[b].lock); - out: - *r_ret = r; - return error; -} - -/* During recovery, other nodes can send us new MSTCPY locks (from - dlm_recover_locks) before we've made ourself master (in - dlm_recover_masters). */ - -static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len, - uint32_t hash, uint32_t b, - int dir_nodeid, int from_nodeid, - unsigned int flags, struct dlm_rsb **r_ret) -{ - struct dlm_rsb *r = NULL; - int our_nodeid = dlm_our_nodeid(); - int recover = (flags & R_RECEIVE_RECOVER); - int error; - - retry: - error = pre_rsb_struct(ls); - if (error < 0) - goto out; - - spin_lock(&ls->ls_rsbtbl[b].lock); - - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (error) - goto do_toss; - - /* - * rsb is active, so we can't check master_nodeid without lock_rsb. - */ - - kref_get(&r->res_ref); - goto out_unlock; - - - do_toss: - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (error) - goto do_new; - - /* - * rsb found inactive. No other thread is using this rsb because - * it's on the toss list, so we can look at or update - * res_master_nodeid without lock_rsb. - */ - - if (!recover && (r->res_master_nodeid != our_nodeid) && from_nodeid) { - /* our rsb is not master, and another node has sent us a - request; this should never happen */ - log_error(ls, "find_rsb toss from_nodeid %d master %d dir %d", - from_nodeid, r->res_master_nodeid, dir_nodeid); - dlm_print_rsb(r); - error = -ENOTBLK; - goto out_unlock; - } - - if (!recover && (r->res_master_nodeid != our_nodeid) && - (dir_nodeid == our_nodeid)) { - /* our rsb is not master, and we are dir; may as well fix it; - this should never happen */ - log_error(ls, "find_rsb toss our %d master %d dir %d", - our_nodeid, r->res_master_nodeid, dir_nodeid); - dlm_print_rsb(r); - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - } - - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); - error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); - goto out_unlock; - - - do_new: - /* - * rsb not found - */ - - error = get_rsb_struct(ls, name, len, &r); - if (error == -EAGAIN) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - goto retry; - } - if (error) - goto out_unlock; - - r->res_hash = hash; - r->res_bucket = b; - r->res_dir_nodeid = dir_nodeid; - r->res_master_nodeid = dir_nodeid; - r->res_nodeid = (dir_nodeid == our_nodeid) ? 0 : dir_nodeid; - kref_init(&r->res_ref); - - error = rsb_insert(r, &ls->ls_rsbtbl[b].keep); - out_unlock: - spin_unlock(&ls->ls_rsbtbl[b].lock); - out: - *r_ret = r; - return error; -} - -static int find_rsb(struct dlm_ls *ls, char *name, int len, int from_nodeid, - unsigned int flags, struct dlm_rsb **r_ret) -{ - uint32_t hash, b; - int dir_nodeid; - - if (len > DLM_RESNAME_MAXLEN) - return -EINVAL; - - hash = jhash(name, len, 0); - b = hash & (ls->ls_rsbtbl_size - 1); - - dir_nodeid = dlm_hash2nodeid(ls, hash); - - if (dlm_no_directory(ls)) - return find_rsb_nodir(ls, name, len, hash, b, dir_nodeid, - from_nodeid, flags, r_ret); - else - return find_rsb_dir(ls, name, len, hash, b, dir_nodeid, - from_nodeid, flags, r_ret); -} - -/* we have received a request and found that res_master_nodeid != our_nodeid, - so we need to return an error or make ourself the master */ - -static int validate_master_nodeid(struct dlm_ls *ls, struct dlm_rsb *r, - int from_nodeid) -{ - if (dlm_no_directory(ls)) { - log_error(ls, "find_rsb keep from_nodeid %d master %d dir %d", - from_nodeid, r->res_master_nodeid, - r->res_dir_nodeid); - dlm_print_rsb(r); - return -ENOTBLK; - } - - if (from_nodeid != r->res_dir_nodeid) { - /* our rsb is not master, and another node (not the dir node) - has sent us a request. this is much more common when our - master_nodeid is zero, so limit debug to non-zero. */ - - if (r->res_master_nodeid) { - log_debug(ls, "validate master from_other %d master %d " - "dir %d first %x %s", from_nodeid, - r->res_master_nodeid, r->res_dir_nodeid, - r->res_first_lkid, r->res_name); - } - return -ENOTBLK; - } else { - /* our rsb is not master, but the dir nodeid has sent us a - request; this could happen with master 0 / res_nodeid -1 */ - - if (r->res_master_nodeid) { - log_error(ls, "validate master from_dir %d master %d " - "first %x %s", - from_nodeid, r->res_master_nodeid, - r->res_first_lkid, r->res_name); - } - - r->res_master_nodeid = dlm_our_nodeid(); - r->res_nodeid = 0; - return 0; - } -} - -/* - * We're the dir node for this res and another node wants to know the - * master nodeid. During normal operation (non recovery) this is only - * called from receive_lookup(); master lookups when the local node is - * the dir node are done by find_rsb(). - * - * normal operation, we are the dir node for a resource - * . _request_lock - * . set_master - * . send_lookup - * . receive_lookup - * . dlm_master_lookup flags 0 - * - * recover directory, we are rebuilding dir for all resources - * . dlm_recover_directory - * . dlm_rcom_names - * remote node sends back the rsb names it is master of and we are dir of - * . dlm_master_lookup RECOVER_DIR (fix_master 0, from_master 1) - * we either create new rsb setting remote node as master, or find existing - * rsb and set master to be the remote node. - * - * recover masters, we are finding the new master for resources - * . dlm_recover_masters - * . recover_master - * . dlm_send_rcom_lookup - * . receive_rcom_lookup - * . dlm_master_lookup RECOVER_MASTER (fix_master 1, from_master 0) - */ - -int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, char *name, int len, - unsigned int flags, int *r_nodeid, int *result) -{ - struct dlm_rsb *r = NULL; - uint32_t hash, b; - int from_master = (flags & DLM_LU_RECOVER_DIR); - int fix_master = (flags & DLM_LU_RECOVER_MASTER); - int our_nodeid = dlm_our_nodeid(); - int dir_nodeid, error, toss_list = 0; - - if (len > DLM_RESNAME_MAXLEN) - return -EINVAL; - - if (from_nodeid == our_nodeid) { - log_error(ls, "dlm_master_lookup from our_nodeid %d flags %x", - our_nodeid, flags); - return -EINVAL; - } - - hash = jhash(name, len, 0); - b = hash & (ls->ls_rsbtbl_size - 1); - - dir_nodeid = dlm_hash2nodeid(ls, hash); - if (dir_nodeid != our_nodeid) { - log_error(ls, "dlm_master_lookup from %d dir %d our %d h %x %d", - from_nodeid, dir_nodeid, our_nodeid, hash, - ls->ls_num_nodes); - *r_nodeid = -1; - return -EINVAL; - } - - retry: - error = pre_rsb_struct(ls); - if (error < 0) - return error; - - spin_lock(&ls->ls_rsbtbl[b].lock); - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (!error) { - /* because the rsb is active, we need to lock_rsb before - checking/changing re_master_nodeid */ - - hold_rsb(r); - spin_unlock(&ls->ls_rsbtbl[b].lock); - lock_rsb(r); - goto found; - } - - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (error) - goto not_found; - - /* because the rsb is inactive (on toss list), it's not refcounted - and lock_rsb is not used, but is protected by the rsbtbl lock */ - - toss_list = 1; - found: - if (r->res_dir_nodeid != our_nodeid) { - /* should not happen, but may as well fix it and carry on */ - log_error(ls, "dlm_master_lookup res_dir %d our %d %s", - r->res_dir_nodeid, our_nodeid, r->res_name); - r->res_dir_nodeid = our_nodeid; - } - - if (fix_master && dlm_is_removed(ls, r->res_master_nodeid)) { - /* Recovery uses this function to set a new master when - the previous master failed. Setting NEW_MASTER will - force dlm_recover_masters to call recover_master on this - rsb even though the res_nodeid is no longer removed. */ - - r->res_master_nodeid = from_nodeid; - r->res_nodeid = from_nodeid; - rsb_set_flag(r, RSB_NEW_MASTER); - - if (toss_list) { - /* I don't think we should ever find it on toss list. */ - log_error(ls, "dlm_master_lookup fix_master on toss"); - dlm_dump_rsb(r); - } - } - - if (from_master && (r->res_master_nodeid != from_nodeid)) { - /* this will happen if from_nodeid became master during - a previous recovery cycle, and we aborted the previous - cycle before recovering this master value */ - - log_limit(ls, "dlm_master_lookup from_master %d " - "master_nodeid %d res_nodeid %d first %x %s", - from_nodeid, r->res_master_nodeid, r->res_nodeid, - r->res_first_lkid, r->res_name); - - if (r->res_master_nodeid == our_nodeid) { - log_error(ls, "from_master %d our_master", from_nodeid); - dlm_dump_rsb(r); - dlm_send_rcom_lookup_dump(r, from_nodeid); - goto out_found; - } - - r->res_master_nodeid = from_nodeid; - r->res_nodeid = from_nodeid; - rsb_set_flag(r, RSB_NEW_MASTER); - } - - if (!r->res_master_nodeid) { - /* this will happen if recovery happens while we're looking - up the master for this rsb */ - - log_debug(ls, "dlm_master_lookup master 0 to %d first %x %s", - from_nodeid, r->res_first_lkid, r->res_name); - r->res_master_nodeid = from_nodeid; - r->res_nodeid = from_nodeid; - } - - if (!from_master && !fix_master && - (r->res_master_nodeid == from_nodeid)) { - /* this can happen when the master sends remove, the dir node - finds the rsb on the keep list and ignores the remove, - and the former master sends a lookup */ - - log_limit(ls, "dlm_master_lookup from master %d flags %x " - "first %x %s", from_nodeid, flags, - r->res_first_lkid, r->res_name); - } - - out_found: - *r_nodeid = r->res_master_nodeid; - if (result) - *result = DLM_LU_MATCH; - - if (toss_list) { - r->res_toss_time = jiffies; - /* the rsb was inactive (on toss list) */ - spin_unlock(&ls->ls_rsbtbl[b].lock); - } else { - /* the rsb was active */ - unlock_rsb(r); - put_rsb(r); - } - return 0; - - not_found: - error = get_rsb_struct(ls, name, len, &r); - if (error == -EAGAIN) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - goto retry; - } - if (error) - goto out_unlock; - - r->res_hash = hash; - r->res_bucket = b; - r->res_dir_nodeid = our_nodeid; - r->res_master_nodeid = from_nodeid; - r->res_nodeid = from_nodeid; - kref_init(&r->res_ref); - r->res_toss_time = jiffies; - - error = rsb_insert(r, &ls->ls_rsbtbl[b].toss); - if (error) { - /* should never happen */ - dlm_free_rsb(r); - spin_unlock(&ls->ls_rsbtbl[b].lock); - goto retry; - } - - if (result) - *result = DLM_LU_ADD; - *r_nodeid = from_nodeid; - error = 0; - out_unlock: - spin_unlock(&ls->ls_rsbtbl[b].lock); - return error; -} - -static void dlm_dump_rsb_hash(struct dlm_ls *ls, uint32_t hash) -{ - struct rb_node *n; - struct dlm_rsb *r; - int i; - - for (i = 0; i < ls->ls_rsbtbl_size; i++) { - spin_lock(&ls->ls_rsbtbl[i].lock); - for (n = rb_first(&ls->ls_rsbtbl[i].keep); n; n = rb_next(n)) { - r = rb_entry(n, struct dlm_rsb, res_hashnode); - if (r->res_hash == hash) - dlm_dump_rsb(r); - } - spin_unlock(&ls->ls_rsbtbl[i].lock); - } -} - -void dlm_dump_rsb_name(struct dlm_ls *ls, char *name, int len) -{ - struct dlm_rsb *r = NULL; - uint32_t hash, b; - int error; - - hash = jhash(name, len, 0); - b = hash & (ls->ls_rsbtbl_size - 1); - - spin_lock(&ls->ls_rsbtbl[b].lock); - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (!error) - goto out_dump; - - error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (error) - goto out; - out_dump: - dlm_dump_rsb(r); - out: - spin_unlock(&ls->ls_rsbtbl[b].lock); -} - -static void toss_rsb(struct kref *kref) -{ - struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref); - struct dlm_ls *ls = r->res_ls; - - DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r);); - kref_init(&r->res_ref); - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[r->res_bucket].keep); - rsb_insert(r, &ls->ls_rsbtbl[r->res_bucket].toss); - r->res_toss_time = jiffies; - ls->ls_rsbtbl[r->res_bucket].flags |= DLM_RTF_SHRINK; - if (r->res_lvbptr) { - dlm_free_lvb(r->res_lvbptr); - r->res_lvbptr = NULL; - } -} - -/* See comment for unhold_lkb */ - -static void unhold_rsb(struct dlm_rsb *r) -{ - int rv; - rv = kref_put(&r->res_ref, toss_rsb); - DLM_ASSERT(!rv, dlm_dump_rsb(r);); -} - -static void kill_rsb(struct kref *kref) -{ - struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref); - - /* All work is done after the return from kref_put() so we - can release the write_lock before the remove and free. */ - - DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r);); - DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r);); - DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r);); - DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r);); - DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r);); - DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r);); -} - -/* Attaching/detaching lkb's from rsb's is for rsb reference counting. - The rsb must exist as long as any lkb's for it do. */ - -static void attach_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - hold_rsb(r); - lkb->lkb_resource = r; -} - -static void detach_lkb(struct dlm_lkb *lkb) -{ - if (lkb->lkb_resource) { - put_rsb(lkb->lkb_resource); - lkb->lkb_resource = NULL; - } -} - -static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret) -{ - struct dlm_lkb *lkb; - int rv; - - lkb = dlm_allocate_lkb(ls); - if (!lkb) - return -ENOMEM; - - lkb->lkb_nodeid = -1; - lkb->lkb_grmode = DLM_LOCK_IV; - kref_init(&lkb->lkb_ref); - INIT_LIST_HEAD(&lkb->lkb_ownqueue); - INIT_LIST_HEAD(&lkb->lkb_rsb_lookup); - INIT_LIST_HEAD(&lkb->lkb_time_list); - INIT_LIST_HEAD(&lkb->lkb_cb_list); - mutex_init(&lkb->lkb_cb_mutex); - INIT_WORK(&lkb->lkb_cb_work, dlm_callback_work); - RB_CLEAR_NODE(&lkb->lkb_statenode); - - idr_preload(GFP_NOFS); - spin_lock(&ls->ls_lkbidr_spin); - rv = idr_alloc(&ls->ls_lkbidr, lkb, 1, 0, GFP_NOWAIT); - if (rv >= 0) - lkb->lkb_id = rv; - spin_unlock(&ls->ls_lkbidr_spin); - idr_preload_end(); - - if (rv < 0) { - log_error(ls, "create_lkb idr error %d", rv); - return rv; - } - - *lkb_ret = lkb; - return 0; -} - -static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret) -{ - struct dlm_lkb *lkb; - - spin_lock(&ls->ls_lkbidr_spin); - lkb = idr_find(&ls->ls_lkbidr, lkid); - if (lkb) - kref_get(&lkb->lkb_ref); - spin_unlock(&ls->ls_lkbidr_spin); - - *lkb_ret = lkb; - return lkb ? 0 : -ENOENT; -} - -static void kill_lkb(struct kref *kref) -{ - struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref); - - /* All work is done after the return from kref_put() so we - can release the write_lock before the detach_lkb */ - - DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); -} - -static void free_range_keys(struct dlm_range *range) -{ - kfree(range->start); - kfree(range->end); -} -/* __put_lkb() is used when an lkb may not have an rsb attached to - it so we need to provide the lockspace explicitly */ - -static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb) -{ - uint32_t lkid = lkb->lkb_id; - - spin_lock(&ls->ls_lkbidr_spin); - if (kref_put(&lkb->lkb_ref, kill_lkb)) { - idr_remove(&ls->ls_lkbidr, lkid); - spin_unlock(&ls->ls_lkbidr_spin); - - detach_lkb(lkb); - - free_range_keys(&lkb->lkb_rqrange); - free_range_keys(&lkb->lkb_grrange); - - /* for local/process lkbs, lvbptr points to caller's lksb */ - if (lkb->lkb_lvbptr && is_master_copy(lkb)) - dlm_free_lvb(lkb->lkb_lvbptr); - dlm_free_lkb(lkb); - return 1; - } else { - spin_unlock(&ls->ls_lkbidr_spin); - return 0; - } -} - -int dlm_put_lkb(struct dlm_lkb *lkb) -{ - struct dlm_ls *ls; - - DLM_ASSERT(lkb->lkb_resource, dlm_print_lkb(lkb);); - DLM_ASSERT(lkb->lkb_resource->res_ls, dlm_print_lkb(lkb);); - - ls = lkb->lkb_resource->res_ls; - return __put_lkb(ls, lkb); -} - -/* This is only called to add a reference when the code already holds - a valid reference to the lkb, so there's no need for locking. */ - -static inline void hold_lkb(struct dlm_lkb *lkb) -{ - kref_get(&lkb->lkb_ref); -} - -/* This is called when we need to remove a reference and are certain - it's not the last ref. e.g. del_lkb is always called between a - find_lkb/put_lkb and is always the inverse of a previous add_lkb. - put_lkb would work fine, but would involve unnecessary locking */ - -static inline void unhold_lkb(struct dlm_lkb *lkb) -{ - int rv; - rv = kref_put(&lkb->lkb_ref, kill_lkb); - DLM_ASSERT(!rv, dlm_print_lkb(lkb);); -} - -static void lkb_add_ordered(struct list_head *new, struct list_head *head, - int mode) -{ - struct dlm_lkb *lkb = NULL; - - list_for_each_entry(lkb, head, lkb_statequeue) - if (lkb->lkb_rqmode < mode) - break; - - __list_add(new, lkb->lkb_statequeue.prev, &lkb->lkb_statequeue); -} - -/* add/remove lkb to rsb's grant/convert/wait queue */ - -static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) -{ - kref_get(&lkb->lkb_ref); - - DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); - BUG_ON(status != DLM_LKSTS_WAITING && !START(lkb)); - BUG_ON(status != DLM_LKSTS_WAITING && !LAST(lkb)); - - lkb->lkb_timestamp = ktime_get(); - - lkb->lkb_status = status; - - switch (status) { - case DLM_LKSTS_WAITING: - if (lkb->lkb_exflags & DLM_LKF_HEADQUE) - list_add(&lkb->lkb_statequeue, &r->res_waitqueue); - else - list_add_tail(&lkb->lkb_statequeue, &r->res_waitqueue); - break; - case DLM_LKSTS_GRANTED: - /* convention says granted locks kept in order of grmode */ - lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue, - lkb->lkb_grmode); - rsb_interval_insert(lkb, &r->res_grantroot); - break; - case DLM_LKSTS_CONVERT: - if (lkb->lkb_exflags & DLM_LKF_HEADQUE) - list_add(&lkb->lkb_statequeue, &r->res_convertqueue); - else - list_add_tail(&lkb->lkb_statequeue, - &r->res_convertqueue); - rsb_interval_insert(lkb, &r->res_convertroot); - break; - default: - DLM_ASSERT(0, dlm_print_lkb(lkb); printk("sts=%d\n", status);); - } -} - -static struct rb_root *lkb_res_root(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - struct rb_root *ret = NULL; - - switch (lkb->lkb_status) { - case DLM_LKSTS_GRANTED: - ret = &r->res_grantroot; - break; - case DLM_LKSTS_CONVERT: - ret = &r->res_convertroot; - break; - default: - DLM_ASSERT(0, dlm_print_lkb(lkb); - printk("sts=%d\n", lkb->lkb_status);); - } - return ret; -} - -static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - struct rb_root *root = NULL; - - if (lkb->lkb_status && lkb->lkb_status != DLM_LKSTS_WAITING) - root = lkb_res_root(r, lkb); - - lkb->lkb_status = 0; - list_del(&lkb->lkb_statequeue); - if (root) { - rsb_interval_remove(lkb, root); - RB_CLEAR_NODE(&lkb->lkb_statenode);/* To aid in debugging */ - } - WARN_ON(!RB_EMPTY_NODE(&lkb->lkb_statenode)); - unhold_lkb(lkb); -} - -static void move_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int sts) -{ - hold_lkb(lkb); - del_lkb(r, lkb); - add_lkb(r, lkb, sts); - unhold_lkb(lkb); -} - -static int msg_reply_type(int mstype) -{ - switch (mstype) { - case DLM_MSG_REQUEST: - return DLM_MSG_REQUEST_REPLY; - case DLM_MSG_CONVERT: - return DLM_MSG_CONVERT_REPLY; - case DLM_MSG_UNLOCK: - return DLM_MSG_UNLOCK_REPLY; - case DLM_MSG_CANCEL: - return DLM_MSG_CANCEL_REPLY; - case DLM_MSG_LOOKUP: - return DLM_MSG_LOOKUP_REPLY; - } - return -1; -} - -static int nodeid_warned(int nodeid, int num_nodes, int *warned) -{ - int i; - - for (i = 0; i < num_nodes; i++) { - if (!warned[i]) { - warned[i] = nodeid; - return 0; - } - if (warned[i] == nodeid) - return 1; - } - return 0; -} - -void dlm_scan_waiters(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb; - ktime_t zero = ktime_set(0, 0); - s64 us; - s64 debug_maxus = 0; - u32 debug_scanned = 0; - u32 debug_expired = 0; - int num_nodes = 0; - int *warned = NULL; - - if (!dlm_config.ci_waitwarn_us) - return; - - mutex_lock(&ls->ls_waiters_mutex); - - list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { - if (ktime_equal(lkb->lkb_wait_time, zero)) - continue; - - debug_scanned++; - - us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_wait_time)); - - if (us < dlm_config.ci_waitwarn_us) - continue; - - lkb->lkb_wait_time = zero; - - debug_expired++; - if (us > debug_maxus) - debug_maxus = us; - - if (!num_nodes) { - num_nodes = ls->ls_num_nodes; - warned = kzalloc(num_nodes * sizeof(int), GFP_KERNEL); - } - if (!warned) - continue; - if (nodeid_warned(lkb->lkb_wait_nodeid, num_nodes, warned)) - continue; - - log_error(ls, "waitwarn %x %lld %d us check connection to " - "node %d", lkb->lkb_id, (long long)us, - dlm_config.ci_waitwarn_us, lkb->lkb_wait_nodeid); - } - mutex_unlock(&ls->ls_waiters_mutex); - kfree(warned); - - if (debug_expired) - log_debug(ls, "scan_waiters %u warn %u over %d us max %lld us", - debug_scanned, debug_expired, - dlm_config.ci_waitwarn_us, (long long)debug_maxus); -} - -/* add/remove lkb from global waiters list of lkb's waiting for - a reply from a remote node */ - -static int add_to_waiters(struct dlm_lkb *lkb, int mstype, int to_nodeid) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int error = 0; - - mutex_lock(&ls->ls_waiters_mutex); - - if (is_overlap_unlock(lkb) || - (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL))) { - error = -EINVAL; - goto out; - } - - if (lkb->lkb_wait_type || is_overlap_cancel(lkb)) { - switch (mstype) { - case DLM_MSG_UNLOCK: - lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; - break; - case DLM_MSG_CANCEL: - lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; - break; - default: - error = -EBUSY; - goto out; - } - lkb->lkb_wait_count++; - hold_lkb(lkb); - - log_debug(ls, "addwait %x cur %d overlap %d count %d f %x", - lkb->lkb_id, lkb->lkb_wait_type, mstype, - lkb->lkb_wait_count, lkb->lkb_flags); - goto out; - } - - DLM_ASSERT(!lkb->lkb_wait_count, - dlm_print_lkb(lkb); - printk("wait_count %d\n", lkb->lkb_wait_count);); - - lkb->lkb_wait_count++; - lkb->lkb_wait_type = mstype; - lkb->lkb_wait_time = ktime_get(); - lkb->lkb_wait_nodeid = to_nodeid; /* for debugging */ - hold_lkb(lkb); - list_add(&lkb->lkb_wait_reply, &ls->ls_waiters); - out: - if (error) - log_error(ls, "addwait error %x %d flags %x %d %d %s", - lkb->lkb_id, error, lkb->lkb_flags, mstype, - lkb->lkb_wait_type, lkb->lkb_resource->res_name); - mutex_unlock(&ls->ls_waiters_mutex); - return error; -} - -/* We clear the RESEND flag because we might be taking an lkb off the waiters - list as part of process_requestqueue (e.g. a lookup that has an optimized - request reply on the requestqueue) between dlm_recover_waiters_pre() which - set RESEND and dlm_recover_waiters_post() */ - -static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype, - struct dlm_message *ms) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int overlap_done = 0; - - if (is_overlap_unlock(lkb) && (mstype == DLM_MSG_UNLOCK_REPLY)) { - log_debug(ls, "remwait %x unlock_reply overlap", lkb->lkb_id); - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; - overlap_done = 1; - goto out_del; - } - - if (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL_REPLY)) { - log_debug(ls, "remwait %x cancel_reply overlap", lkb->lkb_id); - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - overlap_done = 1; - goto out_del; - } - - /* Cancel state was preemptively cleared by a successful convert, - see next comment, nothing to do. */ - - if ((mstype == DLM_MSG_CANCEL_REPLY) && - (lkb->lkb_wait_type != DLM_MSG_CANCEL)) { - log_debug(ls, "remwait %x cancel_reply wait_type %d", - lkb->lkb_id, lkb->lkb_wait_type); - return -1; - } - - /* Remove for the convert reply, and premptively remove for the - cancel reply. A convert has been granted while there's still - an outstanding cancel on it (the cancel is moot and the result - in the cancel reply should be 0). We preempt the cancel reply - because the app gets the convert result and then can follow up - with another op, like convert. This subsequent op would see the - lingering state of the cancel and fail with -EBUSY. */ - - if ((mstype == DLM_MSG_CONVERT_REPLY) && - (lkb->lkb_wait_type == DLM_MSG_CONVERT) && - is_overlap_cancel(lkb) && ms && !ms->m_result) { - log_debug(ls, "remwait %x convert_reply zap overlap_cancel", - lkb->lkb_id); - lkb->lkb_wait_type = 0; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - lkb->lkb_wait_count--; - goto out_del; - } - - /* N.B. type of reply may not always correspond to type of original - msg due to lookup->request optimization, verify others? */ - - if (lkb->lkb_wait_type) { - lkb->lkb_wait_type = 0; - goto out_del; - } - - log_error(ls, "remwait error %x remote %d %x msg %d flags %x no wait", - lkb->lkb_id, ms ? ms->m_header.h_nodeid : 0, lkb->lkb_remid, - mstype, lkb->lkb_flags); - return -1; - - out_del: - /* the force-unlock/cancel has completed and we haven't recvd a reply - to the op that was in progress prior to the unlock/cancel; we - give up on any reply to the earlier op. FIXME: not sure when/how - this would happen */ - - if (overlap_done && lkb->lkb_wait_type) { - log_error(ls, "remwait error %x reply %d wait_type %d overlap", - lkb->lkb_id, mstype, lkb->lkb_wait_type); - lkb->lkb_wait_count--; - lkb->lkb_wait_type = 0; - } - - DLM_ASSERT(lkb->lkb_wait_count, dlm_print_lkb(lkb);); - - lkb->lkb_flags &= ~DLM_IFL_RESEND; - lkb->lkb_wait_count--; - if (!lkb->lkb_wait_count) - list_del_init(&lkb->lkb_wait_reply); - unhold_lkb(lkb); - return 0; -} - -static int remove_from_waiters(struct dlm_lkb *lkb, int mstype) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int error; - - mutex_lock(&ls->ls_waiters_mutex); - error = _remove_from_waiters(lkb, mstype, NULL); - mutex_unlock(&ls->ls_waiters_mutex); - return error; -} - -/* Handles situations where we might be processing a "fake" or "stub" reply in - which we can't try to take waiters_mutex again. */ - -static int remove_from_waiters_ms(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int error; - - if (ms->m_flags != DLM_IFL_STUB_MS) - mutex_lock(&ls->ls_waiters_mutex); - error = _remove_from_waiters(lkb, ms->m_type, ms); - if (ms->m_flags != DLM_IFL_STUB_MS) - mutex_unlock(&ls->ls_waiters_mutex); - return error; -} - -/* If there's an rsb for the same resource being removed, ensure - that the remove message is sent before the new lookup message. - It should be rare to need a delay here, but if not, then it may - be worthwhile to add a proper wait mechanism rather than a delay. */ - -static void wait_pending_remove(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - restart: - spin_lock(&ls->ls_remove_spin); - if (ls->ls_remove_len && - !rsb_cmp(r, ls->ls_remove_name, ls->ls_remove_len)) { - log_debug(ls, "delay lookup for remove dir %d %s", - r->res_dir_nodeid, r->res_name); - spin_unlock(&ls->ls_remove_spin); - msleep(1); - goto restart; - } - spin_unlock(&ls->ls_remove_spin); -} - -/* - * ls_remove_spin protects ls_remove_name and ls_remove_len which are - * read by other threads in wait_pending_remove. ls_remove_names - * and ls_remove_lens are only used by the scan thread, so they do - * not need protection. - */ - -static void shrink_bucket(struct dlm_ls *ls, int b) -{ - struct rb_node *n, *next; - struct dlm_rsb *r; - char *name; - int our_nodeid = dlm_our_nodeid(); - int remote_count = 0; - int need_shrink = 0; - int i, len, rv; - - memset(&ls->ls_remove_lens, 0, sizeof(int) * DLM_REMOVE_NAMES_MAX); - - spin_lock(&ls->ls_rsbtbl[b].lock); - - if (!(ls->ls_rsbtbl[b].flags & DLM_RTF_SHRINK)) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - return; - } - - for (n = rb_first(&ls->ls_rsbtbl[b].toss); n; n = next) { - next = rb_next(n); - r = rb_entry(n, struct dlm_rsb, res_hashnode); - - /* If we're the directory record for this rsb, and - we're not the master of it, then we need to wait - for the master node to send us a dir remove for - before removing the dir record. */ - - if (!dlm_no_directory(ls) && - (r->res_master_nodeid != our_nodeid) && - (dlm_dir_nodeid(r) == our_nodeid)) { - continue; - } - - need_shrink = 1; - - if (!time_after_eq(jiffies, r->res_toss_time + - dlm_config.ci_toss_secs * HZ)) { - continue; - } - - if (!dlm_no_directory(ls) && - (r->res_master_nodeid == our_nodeid) && - (dlm_dir_nodeid(r) != our_nodeid)) { - - /* We're the master of this rsb but we're not - the directory record, so we need to tell the - dir node to remove the dir record. */ - - ls->ls_remove_lens[remote_count] = r->res_length; - memcpy(ls->ls_remove_names[remote_count], r->res_name, - DLM_RESNAME_MAXLEN); - remote_count++; - - if (remote_count >= DLM_REMOVE_NAMES_MAX) - break; - continue; - } - - if (!kref_put(&r->res_ref, kill_rsb)) { - log_error(ls, "tossed rsb in use %s", r->res_name); - continue; - } - - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); - dlm_free_rsb(r); - } - - if (need_shrink) - ls->ls_rsbtbl[b].flags |= DLM_RTF_SHRINK; - else - ls->ls_rsbtbl[b].flags &= ~DLM_RTF_SHRINK; - spin_unlock(&ls->ls_rsbtbl[b].lock); - - /* - * While searching for rsb's to free, we found some that require - * remote removal. We leave them in place and find them again here - * so there is a very small gap between removing them from the toss - * list and sending the removal. Keeping this gap small is - * important to keep us (the master node) from being out of sync - * with the remote dir node for very long. - * - * From the time the rsb is removed from toss until just after - * send_remove, the rsb name is saved in ls_remove_name. A new - * lookup checks this to ensure that a new lookup message for the - * same resource name is not sent just before the remove message. - */ - - for (i = 0; i < remote_count; i++) { - name = ls->ls_remove_names[i]; - len = ls->ls_remove_lens[i]; - - spin_lock(&ls->ls_rsbtbl[b].lock); - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (rv) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_debug(ls, "remove_name not toss %s", name); - continue; - } - - if (r->res_master_nodeid != our_nodeid) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_debug(ls, "remove_name master %d dir %d our %d %s", - r->res_master_nodeid, r->res_dir_nodeid, - our_nodeid, name); - continue; - } - - if (r->res_dir_nodeid == our_nodeid) { - /* should never happen */ - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_error(ls, "remove_name dir %d master %d our %d %s", - r->res_dir_nodeid, r->res_master_nodeid, - our_nodeid, name); - continue; - } - - if (!time_after_eq(jiffies, r->res_toss_time + - dlm_config.ci_toss_secs * HZ)) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_debug(ls, "remove_name toss_time %lu now %lu %s", - r->res_toss_time, jiffies, name); - continue; - } - - if (!kref_put(&r->res_ref, kill_rsb)) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_error(ls, "remove_name in use %s", name); - continue; - } - - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); - - /* block lookup of same name until we've sent remove */ - spin_lock(&ls->ls_remove_spin); - ls->ls_remove_len = len; - memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN); - spin_unlock(&ls->ls_remove_spin); - spin_unlock(&ls->ls_rsbtbl[b].lock); - - send_remove(r); - - /* allow lookup of name again */ - spin_lock(&ls->ls_remove_spin); - ls->ls_remove_len = 0; - memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN); - spin_unlock(&ls->ls_remove_spin); - - dlm_free_rsb(r); - } -} - -void dlm_scan_rsbs(struct dlm_ls *ls) -{ - int i; - - for (i = 0; i < ls->ls_rsbtbl_size; i++) { - shrink_bucket(ls, i); - if (dlm_locking_stopped(ls)) - break; - cond_resched(); - } -} - -static void add_timeout(struct dlm_lkb *lkb) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - - if (is_master_copy(lkb)) - return; - - if (test_bit(LSFL_TIMEWARN, &ls->ls_flags) && - !(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) { - lkb->lkb_flags |= DLM_IFL_WATCH_TIMEWARN; - goto add_it; - } - if (lkb->lkb_exflags & DLM_LKF_TIMEOUT) - goto add_it; - return; - - add_it: - DLM_ASSERT(list_empty(&lkb->lkb_time_list), dlm_print_lkb(lkb);); - mutex_lock(&ls->ls_timeout_mutex); - hold_lkb(lkb); - list_add_tail(&lkb->lkb_time_list, &ls->ls_timeout); - mutex_unlock(&ls->ls_timeout_mutex); -} - -static void del_timeout(struct dlm_lkb *lkb) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - - mutex_lock(&ls->ls_timeout_mutex); - if (!list_empty(&lkb->lkb_time_list)) { - list_del_init(&lkb->lkb_time_list); - unhold_lkb(lkb); - } - mutex_unlock(&ls->ls_timeout_mutex); -} - -/* FIXME: is it safe to look at lkb_exflags, lkb_flags, lkb_timestamp, and - lkb_lksb_timeout without lock_rsb? Note: we can't lock timeout_mutex - and then lock rsb because of lock ordering in add_timeout. We may need - to specify some special timeout-related bits in the lkb that are just to - be accessed under the timeout_mutex. */ - -void dlm_scan_timeout(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - struct dlm_lkb *lkb; - int do_cancel, do_warn; - s64 wait_us; - - for (;;) { - if (dlm_locking_stopped(ls)) - break; - - do_cancel = 0; - do_warn = 0; - mutex_lock(&ls->ls_timeout_mutex); - list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) { - - wait_us = ktime_to_us(ktime_sub(ktime_get(), - lkb->lkb_timestamp)); - - if ((lkb->lkb_exflags & DLM_LKF_TIMEOUT) && - wait_us >= (lkb->lkb_timeout_cs * 10000)) - do_cancel = 1; - - if ((lkb->lkb_flags & DLM_IFL_WATCH_TIMEWARN) && - wait_us >= dlm_config.ci_timewarn_cs * 10000) - do_warn = 1; - - if (!do_cancel && !do_warn) - continue; - hold_lkb(lkb); - break; - } - mutex_unlock(&ls->ls_timeout_mutex); - - if (!do_cancel && !do_warn) - break; - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - if (do_warn) { - /* clear flag so we only warn once */ - lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN; - if (!(lkb->lkb_exflags & DLM_LKF_TIMEOUT)) - del_timeout(lkb); - dlm_timeout_warn(lkb); - } - - if (do_cancel) { - log_debug(ls, "timeout cancel %x node %d %s", - lkb->lkb_id, lkb->lkb_nodeid, r->res_name); - lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN; - lkb->lkb_flags |= DLM_IFL_TIMEOUT_CANCEL; - del_timeout(lkb); - _cancel_lock(r, lkb); - } - - unlock_rsb(r); - unhold_rsb(r); - dlm_put_lkb(lkb); - } -} - -/* This is only called by dlm_recoverd, and we rely on dlm_ls_stop() stopping - dlm_recoverd before checking/setting ls_recover_begin. */ - -void dlm_adjust_timeouts(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb; - u64 adj_us = jiffies_to_usecs(jiffies - ls->ls_recover_begin); - - ls->ls_recover_begin = 0; - mutex_lock(&ls->ls_timeout_mutex); - list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) - lkb->lkb_timestamp = ktime_add_us(lkb->lkb_timestamp, adj_us); - mutex_unlock(&ls->ls_timeout_mutex); - - if (!dlm_config.ci_waitwarn_us) - return; - - mutex_lock(&ls->ls_waiters_mutex); - list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { - if (ktime_to_us(lkb->lkb_wait_time)) - lkb->lkb_wait_time = ktime_get(); - } - mutex_unlock(&ls->ls_waiters_mutex); -} - -/* lkb is master or local copy */ - -static void set_lvb_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int b, len = r->res_ls->ls_lvblen; - - /* b=1 lvb returned to caller - b=0 lvb written to rsb or invalidated - b=-1 do nothing */ - - b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; - - if (b == 1) { - if (!lkb->lkb_lvbptr) - return; - - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - return; - - if (!r->res_lvbptr) - return; - - memcpy(lkb->lkb_lvbptr, r->res_lvbptr, len); - lkb->lkb_lvbseq = r->res_lvbseq; - - } else if (b == 0) { - if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { - rsb_set_flag(r, RSB_VALNOTVALID); - return; - } - - if (!lkb->lkb_lvbptr) - return; - - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - return; - - if (!r->res_lvbptr) - r->res_lvbptr = dlm_allocate_lvb(r->res_ls); - - if (!r->res_lvbptr) - return; - - memcpy(r->res_lvbptr, lkb->lkb_lvbptr, len); - r->res_lvbseq++; - lkb->lkb_lvbseq = r->res_lvbseq; - rsb_clear_flag(r, RSB_VALNOTVALID); - } - - if (rsb_flag(r, RSB_VALNOTVALID)) - lkb->lkb_sbflags |= DLM_SBF_VALNOTVALID; -} - -static void set_lvb_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - if (lkb->lkb_grmode < DLM_LOCK_PW) - return; - - if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { - rsb_set_flag(r, RSB_VALNOTVALID); - return; - } - - if (!lkb->lkb_lvbptr) - return; - - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - return; - - if (!r->res_lvbptr) - r->res_lvbptr = dlm_allocate_lvb(r->res_ls); - - if (!r->res_lvbptr) - return; - - memcpy(r->res_lvbptr, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); - r->res_lvbseq++; - rsb_clear_flag(r, RSB_VALNOTVALID); -} - -/* lkb is process copy (pc) */ - -static void set_lvb_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - int b; - - if (!lkb->lkb_lvbptr) - return; - - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - return; - - b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; - if (b == 1) { - int len = receive_extralen(ms); - if (len > DLM_RESNAME_MAXLEN) - len = DLM_RESNAME_MAXLEN; - memcpy(lkb->lkb_lvbptr, ms->m_extra, len); - lkb->lkb_lvbseq = ms->m_lvbseq; - } -} - -/* Manipulate lkb's on rsb's convert/granted/waiting queues - remove_lock -- used for unlock, removes lkb from granted - revert_lock -- used for cancel, moves lkb from convert to granted - grant_lock -- used for request and convert, adds lkb to granted or - moves lkb from convert or waiting to granted - - Each of these is used for master or local copy lkb's. There is - also a _pc() variation used to make the corresponding change on - a process copy (pc) lkb. */ - -static void _remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - del_lkb(r, lkb); - lkb->lkb_grmode = DLM_LOCK_IV; - /* this unhold undoes the original ref from create_lkb() - so this leads to the lkb being freed */ - unhold_lkb(lkb); -} - -static void remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - set_lvb_unlock(r, lkb); - _remove_lock(r, lkb); -} - -static void remove_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - _remove_lock(r, lkb); -} - -/* returns: 0 did nothing - 1 moved lock to granted - -1 removed lock */ - -static int revert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int rv = 0; - - lkb->lkb_rqmode = DLM_LOCK_IV; - - switch (lkb->lkb_status) { - case DLM_LKSTS_GRANTED: - break; - case DLM_LKSTS_CONVERT: - move_lkb(r, lkb, DLM_LKSTS_GRANTED); - rv = 1; - break; - case DLM_LKSTS_WAITING: - del_lkb(r, lkb); - lkb->lkb_grmode = DLM_LOCK_IV; - /* this unhold undoes the original ref from create_lkb() - so this leads to the lkb being freed */ - unhold_lkb(lkb); - rv = -1; - break; - default: - log_print("invalid status for revert %d", lkb->lkb_status); - } - return rv; -} - -static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - return revert_lock(r, lkb); -} - -static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - /* Set ranges now so move/add lkb has something to insert */ - lkb->lkb_grrange.start = lkb->lkb_rqrange.start; - lkb->lkb_grrange.end = lkb->lkb_rqrange.end; - lkb->lkb_rqrange.start = lkb->lkb_rqrange.end = NULL; - - if (lkb->lkb_grmode != lkb->lkb_rqmode) { - lkb->lkb_grmode = lkb->lkb_rqmode; - if (lkb->lkb_status) - move_lkb(r, lkb, DLM_LKSTS_GRANTED); - else - add_lkb(r, lkb, DLM_LKSTS_GRANTED); - } - - lkb->lkb_rqmode = DLM_LOCK_IV; - lkb->lkb_highbast = 0; -} - -static void grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - set_lvb_lock(r, lkb); - _grant_lock(r, lkb); -} - -static void grant_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - set_lvb_lock_pc(r, lkb, ms); - _grant_lock(r, lkb); -} - -/* called by grant_pending_locks() which means an async grant message must - be sent to the requesting node in addition to granting the lock if the - lkb belongs to a remote node. */ - -static void grant_lock_pending(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - grant_lock(r, lkb); - if (is_master_copy(lkb)) - send_grant(r, lkb); - else - queue_cast(r, lkb, 0); -} - -/* The special CONVDEADLK, ALTPR and ALTCW flags allow the master to - change the granted/requested modes. We're munging things accordingly in - the process copy. - CONVDEADLK: our grmode may have been forced down to NL to resolve a - conversion deadlock - ALTPR/ALTCW: our rqmode may have been changed to PR or CW to become - compatible with other granted locks */ - -static void munge_demoted(struct dlm_lkb *lkb) -{ - if (lkb->lkb_rqmode == DLM_LOCK_IV || lkb->lkb_grmode == DLM_LOCK_IV) { - log_print("munge_demoted %x invalid modes gr %d rq %d", - lkb->lkb_id, lkb->lkb_grmode, lkb->lkb_rqmode); - return; - } - - lkb->lkb_grmode = DLM_LOCK_NL; -} - -static void munge_altmode(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - if (ms->m_type != DLM_MSG_REQUEST_REPLY && - ms->m_type != DLM_MSG_GRANT) { - log_print("munge_altmode %x invalid reply type %d", - lkb->lkb_id, ms->m_type); - return; - } - - if (lkb->lkb_exflags & DLM_LKF_ALTPR) - lkb->lkb_rqmode = DLM_LOCK_PR; - else if (lkb->lkb_exflags & DLM_LKF_ALTCW) - lkb->lkb_rqmode = DLM_LOCK_CW; - else { - log_print("munge_altmode invalid exflags %x", lkb->lkb_exflags); - dlm_print_lkb(lkb); - } -} - -static inline int first_in_list_range(struct dlm_lkb *lkb, struct list_head *head) -{ - struct dlm_lkb *first; - - list_for_each_entry(first, head, lkb_statequeue) { - if (ranges_overlap(&lkb->lkb_rqrange, &first->lkb_rqrange)) { - if (lkb->lkb_id == first->lkb_id) - return 1; - break; - } - } - - return 0; -} - -/* Check if the given lkb conflicts with another lkb on the queue. */ -static int queue_conflict(struct rb_root *root, struct dlm_lkb *lkb) -{ - struct dlm_lkb *this; - struct dlm_key *start, *end; - - start = lkb->lkb_rqrange.start; - end = lkb->lkb_rqrange.end; - this = rsb_interval_iter_first(root, start, end); - while (this) { - if (this != lkb && !modes_compat(this, lkb)) - return 1; - this = rsb_interval_iter_next(this, start, end); - } - return 0; -} - -/* - * "A conversion deadlock arises with a pair of lock requests in the converting - * queue for one resource. The granted mode of each lock blocks the requested - * mode of the other lock." - * - * Part 2: if the granted mode of lkb is preventing an earlier lkb in the - * convert queue from being granted, then deadlk/demote lkb. - * - * Example: - * Granted Queue: empty - * Convert Queue: NL->EX (first lock) - * PR->EX (second lock) - * - * The first lock can't be granted because of the granted mode of the second - * lock and the second lock can't be granted because it's not first in the - * list. We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we - * demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK - * flag set and return DEMOTED in the lksb flags. - * - * Originally, this function detected conv-deadlk in a more limited scope: - * - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or - * - if lkb1 was the first entry in the queue (not just earlier), and was - * blocked by the granted mode of lkb2, and there was nothing on the - * granted queue preventing lkb1 from being granted immediately, i.e. - * lkb2 was the only thing preventing lkb1 from being granted. - * - * That second condition meant we'd only say there was conv-deadlk if - * resolving it (by demotion) would lead to the first lock on the convert - * queue being granted right away. It allowed conversion deadlocks to exist - * between locks on the convert queue while they couldn't be granted anyway. - * - * Now, we detect and take action on conversion deadlocks immediately when - * they're created, even if they may not be immediately consequential. If - * lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted - * mode that would prevent lkb1's conversion from being granted, we do a - * deadlk/demote on lkb2 right away and don't let it onto the convert queue. - * I think this means that the lkb_is_ahead condition below should always - * be zero, i.e. there will never be conv-deadlk between two locks that are - * both already on the convert queue. - */ - -static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2) -{ - struct dlm_lkb *lkb1; - int lkb_is_ahead = 0; - - list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) { - if (lkb1 == lkb2) { - lkb_is_ahead = 1; - continue; - } - - if (!ranges_overlap(&lkb1->lkb_rqrange, &lkb2->lkb_grrange)) - continue; - - if (!lkb_is_ahead) { - if (!modes_compat(lkb2, lkb1)) - return 1; - } else { - if (!modes_compat(lkb2, lkb1) && - !modes_compat(lkb1, lkb2)) - return 1; - } - } - return 0; -} - -/* - * Return 1 if the lock can be granted, 0 otherwise. - * Also detect and resolve conversion deadlocks. - * - * lkb is the lock to be granted - * - * now is 1 if the function is being called in the context of the - * immediate request, it is 0 if called later, after the lock has been - * queued. - * - * recover is 1 if dlm_recover_grant() is trying to grant conversions - * after recovery. - * - * References are from chapter 6 of "VAXcluster Principles" by Roy Davis - */ - -static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, - int recover) -{ - int8_t conv = (lkb->lkb_grmode != DLM_LOCK_IV); - - /* - * 6-10: Version 5.4 introduced an option to address the phenomenon of - * a new request for a NL mode lock being blocked. - * - * 6-11: If the optional EXPEDITE flag is used with the new NL mode - * request, then it would be granted. In essence, the use of this flag - * tells the Lock Manager to expedite theis request by not considering - * what may be in the CONVERTING or WAITING queues... As of this - * writing, the EXPEDITE flag can be used only with new requests for NL - * mode locks. This flag is not valid for conversion requests. - * - * A shortcut. Earlier checks return an error if EXPEDITE is used in a - * conversion or used with a non-NL requested mode. We also know an - * EXPEDITE request is always granted immediately, so now must always - * be 1. The full condition to grant an expedite request: (now && - * !conv && lkb->rqmode == DLM_LOCK_NL && (flags & EXPEDITE)) can - * therefore be shortened to just checking the flag. - */ - - if (lkb->lkb_exflags & DLM_LKF_EXPEDITE) - return 1; - - /* - * A shortcut. Without this, !queue_conflict(grantqueue, lkb) would be - * added to the remaining conditions. - */ - - if (queue_conflict(&r->res_grantroot, lkb)) - return 0; - - /* - * 6-3: By default, a conversion request is immediately granted if the - * requested mode is compatible with the modes of all other granted - * locks - */ - - if (queue_conflict(&r->res_convertroot, lkb)) - return 0; - - /* - * The RECOVER_GRANT flag means dlm_recover_grant() is granting - * locks for a recovered rsb, on which lkb's have been rebuilt. - * The lkb's may have been rebuilt on the queues in a different - * order than they were in on the previous master. So, granting - * queued conversions in order after recovery doesn't make sense - * since the order hasn't been preserved anyway. The new order - * could also have created a new "in place" conversion deadlock. - * (e.g. old, failed master held granted EX, with PR->EX, NL->EX. - * After recovery, there would be no granted locks, and possibly - * NL->EX, PR->EX, an in-place conversion deadlock.) So, after - * recovery, grant conversions without considering order. - */ - - if (conv && recover) - return 1; - - /* - * 6-5: But the default algorithm for deciding whether to grant or - * queue conversion requests does not by itself guarantee that such - * requests are serviced on a "first come first serve" basis. This, in - * turn, can lead to a phenomenon known as "indefinate postponement". - * - * 6-7: This issue is dealt with by using the optional QUECVT flag with - * the system service employed to request a lock conversion. This flag - * forces certain conversion requests to be queued, even if they are - * compatible with the granted modes of other locks on the same - * resource. Thus, the use of this flag results in conversion requests - * being ordered on a "first come first servce" basis. - * - * DCT: This condition is all about new conversions being able to occur - * "in place" while the lock remains on the granted queue (assuming - * nothing else conflicts.) IOW if QUECVT isn't set, a conversion - * doesn't _have_ to go onto the convert queue where it's processed in - * order. The "now" variable is necessary to distinguish converts - * being received and processed for the first time now, because once a - * convert is moved to the conversion queue the condition below applies - * requiring fifo granting. - */ - - if (now && conv && !(lkb->lkb_exflags & DLM_LKF_QUECVT)) - return 1; - - /* - * Even if the convert is compat with all granted locks, - * QUECVT forces it behind other locks on the convert queue. - */ - - if (now && conv && (lkb->lkb_exflags & DLM_LKF_QUECVT)) { - if (list_empty(&r->res_convertqueue)) - return 1; - else - return 0; - } - - /* - * The NOORDER flag is set to avoid the standard vms rules on grant - * order. - */ - - /* - * XXX: Right now scoutfs uses NOORDER but if that changes - * we'll have to replace the list_empty() checks below with - * tree searches. - */ - if (lkb->lkb_exflags & DLM_LKF_NOORDER) - return 1; - - /* - * 6-3: Once in that queue [CONVERTING], a conversion request cannot be - * granted until all other conversion requests ahead of it are granted - * and/or canceled. - */ - if (!now && conv && first_in_list_range(lkb, &r->res_convertqueue)) - return 1; - - /* - * 6-4: By default, a new request is immediately granted only if all - * three of the following conditions are satisfied when the request is - * issued: - * - The queue of ungranted conversion requests for the resource is - * empty. - * - The queue of ungranted new requests for the resource is empty. - * - The mode of the new request is compatible with the most - * restrictive mode of all granted locks on the resource. - */ - - if (now && !conv && list_empty(&r->res_convertqueue) && - list_empty(&r->res_waitqueue)) - return 1; - - /* - * 6-4: Once a lock request is in the queue of ungranted new requests, - * it cannot be granted until the queue of ungranted conversion - * requests is empty, all ungranted new requests ahead of it are - * granted and/or canceled, and it is compatible with the granted mode - * of the most restrictive lock granted on the resource. - */ - - if (!now && !conv && list_empty(&r->res_convertqueue) && - first_in_list_range(lkb, &r->res_waitqueue)) - return 1; - - return 0; -} - -static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, - int recover, int *err) -{ - int rv; - int8_t alt = 0, rqmode = lkb->lkb_rqmode; - int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV); - - if (err) - *err = 0; - - rv = _can_be_granted(r, lkb, now, recover); - if (rv) - goto out; - - /* - * The CONVDEADLK flag is non-standard and tells the dlm to resolve - * conversion deadlocks by demoting grmode to NL, otherwise the dlm - * cancels one of the locks. - */ - - if (is_convert && can_be_queued(lkb) && - conversion_deadlock_detect(r, lkb)) { - if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) { - lkb->lkb_grmode = DLM_LOCK_NL; - lkb->lkb_sbflags |= DLM_SBF_DEMOTED; - } else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) { - if (err) - *err = -EDEADLK; - else { - log_print("can_be_granted deadlock %x now %d", - lkb->lkb_id, now); - dlm_dump_rsb(r); - } - } - goto out; - } - - /* - * The ALTPR and ALTCW flags are non-standard and tell the dlm to try - * to grant a request in a mode other than the normal rqmode. It's a - * simple way to provide a big optimization to applications that can - * use them. - */ - - if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR)) - alt = DLM_LOCK_PR; - else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW)) - alt = DLM_LOCK_CW; - - if (alt) { - lkb->lkb_rqmode = alt; - rv = _can_be_granted(r, lkb, now, 0); - if (rv) - lkb->lkb_sbflags |= DLM_SBF_ALTMODE; - else - lkb->lkb_rqmode = rqmode; - } - out: - return rv; -} - -/* FIXME: I don't think that can_be_granted() can/will demote or find deadlock - for locks pending on the convert list. Once verified (watch for these - log_prints), we should be able to just call _can_be_granted() and not - bother with the demote/deadlk cases here (and there's no easy way to deal - with a deadlk here, we'd have to generate something like grant_lock with - the deadlk error.) */ - -/* Returns the highest requested mode of all blocked conversions; sets - cw if there's a blocked conversion to DLM_LOCK_CW. */ - -static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, - unsigned int *count, struct dlm_range **range) -{ - struct dlm_lkb *lkb, *s; - int recover = rsb_flag(r, RSB_RECOVER_GRANT); - int hi, demoted, quit, grant_restart, demote_restart; - int deadlk; - - quit = 0; - restart: - grant_restart = 0; - demote_restart = 0; - hi = DLM_LOCK_IV; - - list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) { - demoted = is_demoted(lkb); - deadlk = 0; - - if (can_be_granted(r, lkb, 0, recover, &deadlk)) { - grant_lock_pending(r, lkb); - grant_restart = 1; - if (count) - (*count)++; - continue; - } - - if (!demoted && is_demoted(lkb)) { - log_print("WARN: pending demoted %x node %d %s", - lkb->lkb_id, lkb->lkb_nodeid, r->res_name); - demote_restart = 1; - continue; - } - - if (deadlk) { - log_print("WARN: pending deadlock %x node %d %s", - lkb->lkb_id, lkb->lkb_nodeid, r->res_name); - dlm_dump_rsb(r); - continue; - } - - if (lkb->lkb_rqmode > hi) { - hi = lkb->lkb_rqmode; - if (range) - *range = &lkb->lkb_rqrange; - } - - if (cw && lkb->lkb_rqmode == DLM_LOCK_CW) - *cw = 1; - } - - if (grant_restart) - goto restart; - if (demote_restart && !quit) { - quit = 1; - goto restart; - } - - return max_t(int, high, hi); -} - -static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw, - unsigned int *count, struct dlm_range **range) -{ - struct dlm_lkb *lkb, *s; - - list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) { - if (can_be_granted(r, lkb, 0, 0, NULL)) { - grant_lock_pending(r, lkb); - if (count) - (*count)++; - } else { - if (lkb->lkb_rqmode > high) { - high = lkb->lkb_rqmode; - *range = &lkb->lkb_rqrange; - } - - if (lkb->lkb_rqmode == DLM_LOCK_CW) - *cw = 1; - } - } - - return high; -} - -/* cw of 1 means there's a lock with a rqmode of DLM_LOCK_CW that's blocked - on either the convert or waiting queue. - high is the largest rqmode of all locks blocked on the convert or - waiting queue. */ - -static int lock_requires_bast(struct dlm_lkb *gr, int high, int cw) -{ - if (gr->lkb_grmode == DLM_LOCK_PR && cw) { - if (gr->lkb_highbast < DLM_LOCK_EX) - return 1; - return 0; - } - - if (gr->lkb_highbast < high && - !__dlm_compat_matrix[gr->lkb_grmode+1][high+1]) - return 1; - return 0; -} - -static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count) -{ - struct dlm_lkb *lkb, *s; - int high = DLM_LOCK_IV; - int cw = 0; - struct dlm_range *highrange = NULL; - - if (!is_master(r)) { - dlm_dump_rsb(r); - return; - } - - high = grant_pending_convert(r, high, &cw, count, &highrange); - high = grant_pending_wait(r, high, &cw, count, &highrange); - - if (high == DLM_LOCK_IV) - return; - - /* - * If there are locks left on the wait/convert queue then send blocking - * ASTs to granted locks based on the largest requested mode (high) - * found above. - */ - - list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) { - if ((lkb->lkb_bastfn || lkb->lkb_rbastfn) && - lock_requires_bast(lkb, high, cw)) { - if (cw && high == DLM_LOCK_PR && - lkb->lkb_grmode == DLM_LOCK_PR) - queue_bast(r, lkb, DLM_LOCK_CW, highrange); - else - queue_bast(r, lkb, high, highrange); - lkb->lkb_highbast = high; - } - } -} - -static int modes_require_bast(struct dlm_lkb *gr, struct dlm_lkb *rq) -{ - if ((gr->lkb_grmode == DLM_LOCK_PR && rq->lkb_rqmode == DLM_LOCK_CW) || - (gr->lkb_grmode == DLM_LOCK_CW && rq->lkb_rqmode == DLM_LOCK_PR)) { - if (gr->lkb_highbast < DLM_LOCK_EX) - return 1; - return 0; - } - - if (gr->lkb_highbast < rq->lkb_rqmode && !modes_compat(gr, rq)) - return 1; - return 0; -} - -static void send_bast_queue(struct dlm_rsb *r, struct list_head *head, - struct dlm_lkb *lkb) -{ - struct dlm_lkb *gr; - - list_for_each_entry(gr, head, lkb_statequeue) { - /* skip self when sending basts to convertqueue */ - if (gr == lkb) - continue; - if ((gr->lkb_rbastfn || gr->lkb_bastfn) && - modes_require_bast(gr, lkb)) { - queue_bast(r, gr, lkb->lkb_rqmode, &lkb->lkb_rqrange); - gr->lkb_highbast = lkb->lkb_rqmode; - } - } -} - -static void send_blocking_asts(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - send_bast_queue(r, &r->res_grantqueue, lkb); -} - -static void send_blocking_asts_all(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - send_bast_queue(r, &r->res_grantqueue, lkb); - send_bast_queue(r, &r->res_convertqueue, lkb); -} - -/* set_master(r, lkb) -- set the master nodeid of a resource - - The purpose of this function is to set the nodeid field in the given - lkb using the nodeid field in the given rsb. If the rsb's nodeid is - known, it can just be copied to the lkb and the function will return - 0. If the rsb's nodeid is _not_ known, it needs to be looked up - before it can be copied to the lkb. - - When the rsb nodeid is being looked up remotely, the initial lkb - causing the lookup is kept on the ls_waiters list waiting for the - lookup reply. Other lkb's waiting for the same rsb lookup are kept - on the rsb's res_lookup list until the master is verified. - - Return values: - 0: nodeid is set in rsb/lkb and the caller should go ahead and use it - 1: the rsb master is not available and the lkb has been placed on - a wait queue -*/ - -static int set_master(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int our_nodeid = dlm_our_nodeid(); - - if (rsb_flag(r, RSB_MASTER_UNCERTAIN)) { - rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); - r->res_first_lkid = lkb->lkb_id; - lkb->lkb_nodeid = r->res_nodeid; - return 0; - } - - if (r->res_first_lkid && r->res_first_lkid != lkb->lkb_id) { - list_add_tail(&lkb->lkb_rsb_lookup, &r->res_lookup); - return 1; - } - - if (r->res_master_nodeid == our_nodeid) { - lkb->lkb_nodeid = 0; - return 0; - } - - if (r->res_master_nodeid) { - lkb->lkb_nodeid = r->res_master_nodeid; - return 0; - } - - if (dlm_dir_nodeid(r) == our_nodeid) { - /* This is a somewhat unusual case; find_rsb will usually - have set res_master_nodeid when dir nodeid is local, but - there are cases where we become the dir node after we've - past find_rsb and go through _request_lock again. - confirm_master() or process_lookup_list() needs to be - called after this. */ - log_debug(r->res_ls, "set_master %x self master %d dir %d %s", - lkb->lkb_id, r->res_master_nodeid, r->res_dir_nodeid, - r->res_name); - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - lkb->lkb_nodeid = 0; - return 0; - } - - wait_pending_remove(r); - - r->res_first_lkid = lkb->lkb_id; - send_lookup(r, lkb); - return 1; -} - -static void process_lookup_list(struct dlm_rsb *r) -{ - struct dlm_lkb *lkb, *safe; - - list_for_each_entry_safe(lkb, safe, &r->res_lookup, lkb_rsb_lookup) { - list_del_init(&lkb->lkb_rsb_lookup); - _request_lock(r, lkb); - schedule(); - } -} - -/* confirm_master -- confirm (or deny) an rsb's master nodeid */ - -static void confirm_master(struct dlm_rsb *r, int error) -{ - struct dlm_lkb *lkb; - - if (!r->res_first_lkid) - return; - - switch (error) { - case 0: - case -EINPROGRESS: - r->res_first_lkid = 0; - process_lookup_list(r); - break; - - case -EAGAIN: - case -EBADR: - case -ENOTBLK: - /* the remote request failed and won't be retried (it was - a NOQUEUE, or has been canceled/unlocked); make a waiting - lkb the first_lkid */ - - r->res_first_lkid = 0; - - if (!list_empty(&r->res_lookup)) { - lkb = list_entry(r->res_lookup.next, struct dlm_lkb, - lkb_rsb_lookup); - list_del_init(&lkb->lkb_rsb_lookup); - r->res_first_lkid = lkb->lkb_id; - _request_lock(r, lkb); - } - break; - - default: - log_error(r->res_ls, "confirm_master unknown error %d", error); - } -} - -static int set_lock_args(int mode, struct dlm_range *range, - struct dlm_lksb *lksb, uint32_t flags, - int namelen, unsigned long timeout_cs, - void (*ast) (void *astparam), - void *astparam, - void (*bast) (void *astparam, int mode), - void (*rbast) (void *astarg, int mode, - struct dlm_key *start, - struct dlm_key *end), - struct dlm_args *args) -{ - int rv = -EINVAL; - - /* check for invalid arg usage */ - - if (mode < 0 || mode > DLM_LOCK_EX) - goto out; - - if (range && cmp_dlm_keys(range->start, range->end) > 0) - goto out; - - if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN)) - goto out; - - if (flags & DLM_LKF_CANCEL) - goto out; - - if (flags & DLM_LKF_QUECVT && !(flags & DLM_LKF_CONVERT)) - goto out; - - if (flags & DLM_LKF_CONVDEADLK && !(flags & DLM_LKF_CONVERT)) - goto out; - - if (flags & DLM_LKF_CONVDEADLK && flags & DLM_LKF_NOQUEUE) - goto out; - - if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_CONVERT) - goto out; - - if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_QUECVT) - goto out; - - if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_NOQUEUE) - goto out; - - if (flags & DLM_LKF_EXPEDITE && mode != DLM_LOCK_NL) - goto out; - - if (!ast || !lksb) - goto out; - - if (flags & DLM_LKF_VALBLK && !lksb->sb_lvbptr) - goto out; - - if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid) - goto out; - - /* XXX: The caller could pass default_range for us */ - if (!range) - range = &default_range; - - /* these args will be copied to the lkb in validate_lock_args, - it cannot be done now because when converting locks, fields in - an active lkb cannot be modified before locking the rsb */ - - args->flags = flags; - args->astfn = ast; - args->astparam = astparam; - args->bastfn = bast; - args->rbastfn = rbast; - args->timeout = timeout_cs; - args->mode = mode; - args->range = *range; - args->lksb = lksb; - rv = 0; - out: - return rv; -} - -static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args) -{ - if (flags & ~(DLM_LKF_CANCEL | DLM_LKF_VALBLK | DLM_LKF_IVVALBLK | - DLM_LKF_FORCEUNLOCK)) - return -EINVAL; - - if (flags & DLM_LKF_CANCEL && flags & DLM_LKF_FORCEUNLOCK) - return -EINVAL; - - args->flags = flags; - args->astparam = astarg; - return 0; -} - -static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_args *args) -{ - int rv = -EINVAL; - - if (args->flags & DLM_LKF_CONVERT) { - if (lkb->lkb_flags & DLM_IFL_MSTCPY) - goto out; - - if (args->flags & DLM_LKF_QUECVT && - !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1]) - goto out; - - rv = -EBUSY; - if (lkb->lkb_status != DLM_LKSTS_GRANTED) - goto out; - - if (lkb->lkb_wait_type) - goto out; - - if (is_overlap(lkb)) - goto out; - } - - lkb->lkb_exflags = args->flags; - lkb->lkb_sbflags = 0; - lkb->lkb_astfn = args->astfn; - lkb->lkb_astparam = args->astparam; - lkb->lkb_bastfn = args->bastfn; - lkb->lkb_rbastfn = args->rbastfn; - lkb->lkb_rqmode = args->mode; - if (args->range.start && args->range.end) { - rv = -ENOMEM; - lkb->lkb_rqrange.start = alloc_key(args->range.start->val, - args->range.start->len, - GFP_NOFS); - if (!lkb->lkb_rqrange.start) - goto out; - lkb->lkb_rqrange.end = alloc_key(args->range.end->val, - args->range.end->len, - GFP_NOFS); - if (!lkb->lkb_rqrange.end) - goto out; - } - lkb->lkb_lksb = args->lksb; - lkb->lkb_lvbptr = args->lksb->sb_lvbptr; - lkb->lkb_ownpid = (int) current->pid; - lkb->lkb_timeout_cs = args->timeout; - rv = 0; - out: - if (rv) - log_debug(ls, "validate_lock_args %d %x %x %x %d %d %s", - rv, lkb->lkb_id, lkb->lkb_flags, args->flags, - lkb->lkb_status, lkb->lkb_wait_type, - lkb->lkb_resource->res_name); - return rv; -} - -/* when dlm_unlock() sees -EBUSY with CANCEL/FORCEUNLOCK it returns 0 - for success */ - -/* note: it's valid for lkb_nodeid/res_nodeid to be -1 when we get here - because there may be a lookup in progress and it's valid to do - cancel/unlockf on it */ - -static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) -{ - struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int rv = -EINVAL; - - if (lkb->lkb_flags & DLM_IFL_MSTCPY) { - log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id); - dlm_print_lkb(lkb); - goto out; - } - - /* an lkb may still exist even though the lock is EOL'ed due to a - cancel, unlock or failed noqueue request; an app can't use these - locks; return same error as if the lkid had not been found at all */ - - if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { - log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id); - rv = -ENOENT; - goto out; - } - - /* an lkb may be waiting for an rsb lookup to complete where the - lookup was initiated by another lock */ - - if (!list_empty(&lkb->lkb_rsb_lookup)) { - if (args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) { - log_debug(ls, "unlock on rsb_lookup %x", lkb->lkb_id); - list_del_init(&lkb->lkb_rsb_lookup); - queue_cast(lkb->lkb_resource, lkb, - args->flags & DLM_LKF_CANCEL ? - -DLM_ECANCEL : -DLM_EUNLOCK); - unhold_lkb(lkb); /* undoes create_lkb() */ - } - /* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */ - rv = -EBUSY; - goto out; - } - -#if 0 - /* XXX: Shouldn't CANCEL check against rqstart/rqend? */ - if (args->start != lkb->lkb_grstart || args->end != lkb->lkb_grend) - goto out; -#endif - /* cancel not allowed with another cancel/unlock in progress */ - - if (args->flags & DLM_LKF_CANCEL) { - if (lkb->lkb_exflags & DLM_LKF_CANCEL) - goto out; - - if (is_overlap(lkb)) - goto out; - - /* don't let scand try to do a cancel */ - del_timeout(lkb); - - if (lkb->lkb_flags & DLM_IFL_RESEND) { - lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; - rv = -EBUSY; - goto out; - } - - /* there's nothing to cancel */ - if (lkb->lkb_status == DLM_LKSTS_GRANTED && - !lkb->lkb_wait_type) { - rv = -EBUSY; - goto out; - } - - switch (lkb->lkb_wait_type) { - case DLM_MSG_LOOKUP: - case DLM_MSG_REQUEST: - lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL; - rv = -EBUSY; - goto out; - case DLM_MSG_UNLOCK: - case DLM_MSG_CANCEL: - goto out; - } - /* add_to_waiters() will set OVERLAP_CANCEL */ - goto out_ok; - } - - /* do we need to allow a force-unlock if there's a normal unlock - already in progress? in what conditions could the normal unlock - fail such that we'd want to send a force-unlock to be sure? */ - - if (args->flags & DLM_LKF_FORCEUNLOCK) { - if (lkb->lkb_exflags & DLM_LKF_FORCEUNLOCK) - goto out; - - if (is_overlap_unlock(lkb)) - goto out; - - /* don't let scand try to do a cancel */ - del_timeout(lkb); - - if (lkb->lkb_flags & DLM_IFL_RESEND) { - lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; - rv = -EBUSY; - goto out; - } - - switch (lkb->lkb_wait_type) { - case DLM_MSG_LOOKUP: - case DLM_MSG_REQUEST: - lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK; - rv = -EBUSY; - goto out; - case DLM_MSG_UNLOCK: - goto out; - } - /* add_to_waiters() will set OVERLAP_UNLOCK */ - goto out_ok; - } - - /* normal unlock not allowed if there's any op in progress */ - rv = -EBUSY; - if (lkb->lkb_wait_type || lkb->lkb_wait_count) - goto out; - - out_ok: - /* an overlapping op shouldn't blow away exflags from other op */ - lkb->lkb_exflags |= args->flags; - lkb->lkb_sbflags = 0; - lkb->lkb_astparam = args->astparam; - rv = 0; - out: - if (rv) - log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", - rv, lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, - args->flags, lkb->lkb_wait_type, - lkb->lkb_resource->res_name); - return rv; -} - -/* - * Four stage 4 varieties: - * do_request(), do_convert(), do_unlock(), do_cancel() - * These are called on the master node for the given lock and - * from the central locking logic. - */ - -static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error = 0; - - if (can_be_granted(r, lkb, 1, 0, NULL)) { - grant_lock(r, lkb); - queue_cast(r, lkb, 0); - goto out; - } - - if (can_be_queued(lkb)) { - error = -EINPROGRESS; - add_lkb(r, lkb, DLM_LKSTS_WAITING); - add_timeout(lkb); - goto out; - } - - error = -EAGAIN; - queue_cast(r, lkb, -EAGAIN); - out: - return error; -} - -static void do_request_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, - int error) -{ - switch (error) { - case -EAGAIN: - if (force_blocking_asts(lkb)) - send_blocking_asts_all(r, lkb); - break; - case -EINPROGRESS: - send_blocking_asts(r, lkb); - break; - } -} - -static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error = 0; - int deadlk = 0; - - /* changing an existing lock may allow others to be granted */ - - if (can_be_granted(r, lkb, 1, 0, &deadlk)) { - grant_lock(r, lkb); - queue_cast(r, lkb, 0); - goto out; - } - - /* can_be_granted() detected that this lock would block in a conversion - deadlock, so we leave it on the granted queue and return EDEADLK in - the ast for the convert. */ - - if (deadlk) { - /* it's left on the granted queue */ - revert_lock(r, lkb); - queue_cast(r, lkb, -EDEADLK); - error = -EDEADLK; - goto out; - } - - /* is_demoted() means the can_be_granted() above set the grmode - to NL, and left us on the granted queue. This auto-demotion - (due to CONVDEADLK) might mean other locks, and/or this lock, are - now grantable. We have to try to grant other converting locks - before we try again to grant this one. */ - - if (is_demoted(lkb)) { - grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL, NULL); - if (_can_be_granted(r, lkb, 1, 0)) { - grant_lock(r, lkb); - queue_cast(r, lkb, 0); - goto out; - } - /* else fall through and move to convert queue */ - } - - if (can_be_queued(lkb)) { - error = -EINPROGRESS; - del_lkb(r, lkb); - add_lkb(r, lkb, DLM_LKSTS_CONVERT); - add_timeout(lkb); - goto out; - } - - error = -EAGAIN; - queue_cast(r, lkb, -EAGAIN); - out: - return error; -} - -static void do_convert_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, - int error) -{ - switch (error) { - case 0: - grant_pending_locks(r, NULL); - /* grant_pending_locks also sends basts */ - break; - case -EAGAIN: - if (force_blocking_asts(lkb)) - send_blocking_asts_all(r, lkb); - break; - case -EINPROGRESS: - send_blocking_asts(r, lkb); - break; - } -} - -static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - remove_lock(r, lkb); - queue_cast(r, lkb, -DLM_EUNLOCK); - return -DLM_EUNLOCK; -} - -static void do_unlock_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, - int error) -{ - grant_pending_locks(r, NULL); -} - -/* returns: 0 did nothing, -DLM_ECANCEL canceled lock */ - -static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - error = revert_lock(r, lkb); - if (error) { - queue_cast(r, lkb, -DLM_ECANCEL); - return -DLM_ECANCEL; - } - return 0; -} - -static void do_cancel_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, - int error) -{ - if (error) - grant_pending_locks(r, NULL); -} - -/* - * Four stage 3 varieties: - * _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock() - */ - -/* add a new lkb to a possibly new rsb, called by requesting process */ - -static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - /* set_master: sets lkb nodeid from r */ - - error = set_master(r, lkb); - if (error < 0) - goto out; - if (error) { - error = 0; - goto out; - } - - if (is_remote(r)) { - /* receive_request() calls do_request() on remote node */ - error = send_request(r, lkb); - } else { - error = do_request(r, lkb); - /* for remote locks the request_reply is sent - between do_request and do_request_effects */ - do_request_effects(r, lkb, error); - } - out: - return error; -} - -/* change some property of an existing lkb, e.g. mode */ - -static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - if (is_remote(r)) { - /* receive_convert() calls do_convert() on remote node */ - error = send_convert(r, lkb); - } else { - error = do_convert(r, lkb); - /* for remote locks the convert_reply is sent - between do_convert and do_convert_effects */ - do_convert_effects(r, lkb, error); - } - - return error; -} - -/* remove an existing lkb from the granted queue */ - -static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - if (is_remote(r)) { - /* receive_unlock() calls do_unlock() on remote node */ - error = send_unlock(r, lkb); - } else { - error = do_unlock(r, lkb); - /* for remote locks the unlock_reply is sent - between do_unlock and do_unlock_effects */ - do_unlock_effects(r, lkb, error); - } - - return error; -} - -/* remove an existing lkb from the convert or wait queue */ - -static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - if (is_remote(r)) { - /* receive_cancel() calls do_cancel() on remote node */ - error = send_cancel(r, lkb); - } else { - error = do_cancel(r, lkb); - /* for remote locks the cancel_reply is sent - between do_cancel and do_cancel_effects */ - do_cancel_effects(r, lkb, error); - } - - return error; -} - -/* - * Four stage 2 varieties: - * request_lock(), convert_lock(), unlock_lock(), cancel_lock() - */ - -static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name, - int len, struct dlm_args *args) -{ - struct dlm_rsb *r; - int error; - - error = validate_lock_args(ls, lkb, args); - if (error) - return error; - - error = find_rsb(ls, name, len, 0, R_REQUEST, &r); - if (error) - return error; - - lock_rsb(r); - - attach_lkb(r, lkb); - lkb->lkb_lksb->sb_lkid = lkb->lkb_id; - - error = _request_lock(r, lkb); - - unlock_rsb(r); - put_rsb(r); - return error; -} - -static int convert_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_args *args) -{ - struct dlm_rsb *r; - int error; - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_lock_args(ls, lkb, args); - if (error) - goto out; - - error = _convert_lock(r, lkb); - out: - unlock_rsb(r); - put_rsb(r); - return error; -} - -static int unlock_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_args *args) -{ - struct dlm_rsb *r; - int error; - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_unlock_args(lkb, args); - if (error) - goto out; - - error = _unlock_lock(r, lkb); - out: - unlock_rsb(r); - put_rsb(r); - return error; -} - -static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_args *args) -{ - struct dlm_rsb *r; - int error; - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_unlock_args(lkb, args); - if (error) - goto out; - - error = _cancel_lock(r, lkb); - out: - unlock_rsb(r); - put_rsb(r); - return error; -} - -/* - * Two stage 1 varieties: dlm_lock() and dlm_unlock() - */ - -static int _dlm_lock(dlm_lockspace_t *lockspace, - int mode, - struct dlm_key *start, - struct dlm_key *end, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*ast) (void *astarg), - void *astarg, - void (*bast) (void *astarg, int mode), - void (*rbast) (void *astarg, int mode, - struct dlm_key *start, struct dlm_key *end)) -{ - struct dlm_ls *ls; - struct dlm_lkb *lkb; - struct dlm_range range = { start, end }; - struct dlm_args args; - int error, convert = flags & DLM_LKF_CONVERT; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - - dlm_lock_recovery(ls); - - if (convert) - error = find_lkb(ls, lksb->sb_lkid, &lkb); - else - error = create_lkb(ls, &lkb); - - if (error) - goto out; - - error = set_lock_args(mode, start ? &range : NULL, lksb, flags, namelen, - 0, ast, astarg, bast, rbast, &args); - if (error) - goto out_put; - - if (convert) - error = convert_lock(ls, lkb, &args); - else - error = request_lock(ls, lkb, name, namelen, &args); - - if (error == -EINPROGRESS) - error = 0; - out_put: - if (convert || error) - __put_lkb(ls, lkb); - if (error == -EAGAIN || error == -EDEADLK) - error = 0; - out: - dlm_unlock_recovery(ls); - dlm_put_lockspace(ls); - return error; -} - -int dlm_lock_range(dlm_lockspace_t *lockspace, - int mode, - struct dlm_key *start, - struct dlm_key *end, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*ast) (void *astarg), - void *astarg, - void (*rbast) (void *astarg, int mode, - struct dlm_key *start, struct dlm_key *end)) -{ - if (!start || !end) - return -EINVAL; - - if (start->len > DLM_KEY_LEN || end->len > DLM_KEY_LEN) { - WARN_ON_ONCE(1); - return -EINVAL; - } - - return _dlm_lock(lockspace, mode, start, end, lksb, flags, name, - namelen, parent_lkid, ast, astarg, NULL, rbast); -} - -int dlm_lock(dlm_lockspace_t *lockspace, - int mode, - struct dlm_lksb *lksb, - uint32_t flags, - void *name, - unsigned int namelen, - uint32_t parent_lkid, - void (*ast) (void *astarg), - void *astarg, - void (*bast) (void *astarg, int mode)) -{ - return _dlm_lock(lockspace, mode, NULL, NULL, lksb, flags, name, - namelen, parent_lkid, ast, astarg, bast, NULL); -} - -int dlm_unlock(dlm_lockspace_t *lockspace, - uint32_t lkid, - uint32_t flags, - struct dlm_lksb *lksb, - void *astarg) -{ - struct dlm_ls *ls; - struct dlm_lkb *lkb; - struct dlm_args args; - int error; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - - dlm_lock_recovery(ls); - - error = find_lkb(ls, lkid, &lkb); - if (error) - goto out; - - error = set_unlock_args(flags, astarg, &args); - if (error) - goto out_put; - - if (flags & DLM_LKF_CANCEL) - error = cancel_lock(ls, lkb, &args); - else - error = unlock_lock(ls, lkb, &args); - - if (error == -DLM_EUNLOCK || error == -DLM_ECANCEL) - error = 0; - if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK))) - error = 0; - - out_put: - dlm_put_lkb(lkb); - out: - dlm_unlock_recovery(ls); - dlm_put_lockspace(ls); - return error; -} - -/* - * send/receive routines for remote operations and replies - * - * send_args - * send_common - * send_request receive_request - * send_convert receive_convert - * send_unlock receive_unlock - * send_cancel receive_cancel - * send_grant receive_grant - * send_bast receive_bast - * send_lookup receive_lookup - * send_remove receive_remove - * - * send_common_reply - * receive_request_reply send_request_reply - * receive_convert_reply send_convert_reply - * receive_unlock_reply send_unlock_reply - * receive_cancel_reply send_cancel_reply - * receive_lookup_reply send_lookup_reply - */ - -static int _create_message(struct dlm_ls *ls, int mb_len, - int to_nodeid, int mstype, - struct dlm_message **ms_ret, - struct dlm_mhandle **mh_ret) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - char *mb; - - /* get_buffer gives us a message handle (mh) that we need to - pass into lowcomms_commit and a message buffer (mb) that we - write our data into */ - - mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb); - if (!mh) - return -ENOBUFS; - - memset(mb, 0, mb_len); - - ms = (struct dlm_message *) mb; - - ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); - ms->m_header.h_lockspace = ls->ls_global_id; - ms->m_header.h_nodeid = dlm_our_nodeid(); - ms->m_header.h_length = mb_len; - ms->m_header.h_cmd = DLM_MSG; - - ms->m_type = mstype; - - *mh_ret = mh; - *ms_ret = ms; - return 0; -} - -static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb, - int to_nodeid, int mstype, - struct dlm_message **ms_ret, - struct dlm_mhandle **mh_ret) -{ - int mb_len = sizeof(struct dlm_message); - - switch (mstype) { - case DLM_MSG_REQUEST: - case DLM_MSG_LOOKUP: - case DLM_MSG_REMOVE: - mb_len += r->res_length; - break; - case DLM_MSG_CONVERT: - case DLM_MSG_UNLOCK: - case DLM_MSG_REQUEST_REPLY: - case DLM_MSG_CONVERT_REPLY: - case DLM_MSG_GRANT: - if (lkb && lkb->lkb_lvbptr) - mb_len += r->res_ls->ls_lvblen; - break; - } - - return _create_message(r->res_ls, mb_len, to_nodeid, mstype, - ms_ret, mh_ret); -} - -/* further lowcomms enhancements or alternate implementations may make - the return value from this function useful at some point */ - -static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms) -{ - dlm_message_out(ms); - dlm_lowcomms_commit_buffer(mh); - return 0; -} - -static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - ms->m_nodeid = lkb->lkb_nodeid; - ms->m_pid = lkb->lkb_ownpid; - ms->m_lkid = lkb->lkb_id; - ms->m_remid = lkb->lkb_remid; - ms->m_exflags = lkb->lkb_exflags; - ms->m_sbflags = lkb->lkb_sbflags; - ms->m_flags = lkb->lkb_flags; - ms->m_lvbseq = lkb->lkb_lvbseq; - ms->m_status = lkb->lkb_status; - ms->m_grmode = lkb->lkb_grmode; - ms->m_rqmode = lkb->lkb_rqmode; - ms->m_hash = r->res_hash; - - ms->m_grstart_len = ms->m_grend_len = ms->m_rqstart_len = - ms->m_rqend_len = 0; - if (lkb->lkb_grrange.start) { - ms->m_grstart_len = lkb->lkb_grrange.start->len; - memcpy(ms->m_grstart, lkb->lkb_grrange.start->val, ms->m_grstart_len); - } - if (lkb->lkb_grrange.end) { - ms->m_grend_len = lkb->lkb_grrange.end->len; - memcpy(ms->m_grend, lkb->lkb_grrange.end->val, ms->m_grend_len); - } - if (lkb->lkb_rqrange.start) { - ms->m_rqstart_len = lkb->lkb_rqrange.start->len; - memcpy(ms->m_rqstart, lkb->lkb_rqrange.start->val, ms->m_rqstart_len); - } - if (lkb->lkb_rqrange.end) { - ms->m_rqend_len = lkb->lkb_rqrange.end->len; - memcpy(ms->m_rqend, lkb->lkb_rqrange.end->val, ms->m_rqend_len); - } - - /* m_result and m_bastmode are set from function args, - not from lkb fields */ - - if (lkb->lkb_bastfn || lkb->lkb_rbastfn) - ms->m_asts |= DLM_CB_BAST; - if (lkb->lkb_astfn) - ms->m_asts |= DLM_CB_CAST; - - /* compare with switch in create_message; send_remove() doesn't - use send_args() */ - - switch (ms->m_type) { - case DLM_MSG_REQUEST: - case DLM_MSG_LOOKUP: - memcpy(ms->m_extra, r->res_name, r->res_length); - break; - case DLM_MSG_CONVERT: - case DLM_MSG_UNLOCK: - case DLM_MSG_REQUEST_REPLY: - case DLM_MSG_CONVERT_REPLY: - case DLM_MSG_GRANT: - if (!lkb->lkb_lvbptr) - break; - memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); - break; - } -} - -static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = r->res_nodeid; - - error = add_to_waiters(lkb, mstype, to_nodeid); - if (error) - return error; - - error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); - if (error) - goto fail; - - send_args(r, lkb, ms); - - error = send_message(mh, ms); - if (error) - goto fail; - return 0; - - fail: - remove_from_waiters(lkb, msg_reply_type(mstype)); - return error; -} - -static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - return send_common(r, lkb, DLM_MSG_REQUEST); -} - -static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - int error; - - error = send_common(r, lkb, DLM_MSG_CONVERT); - - /* down conversions go without a reply from the master */ - if (!error && down_conversion(lkb)) { - remove_from_waiters(lkb, DLM_MSG_CONVERT_REPLY); - r->res_ls->ls_stub_ms.m_flags = DLM_IFL_STUB_MS; - r->res_ls->ls_stub_ms.m_type = DLM_MSG_CONVERT_REPLY; - r->res_ls->ls_stub_ms.m_result = 0; - __receive_convert_reply(r, lkb, &r->res_ls->ls_stub_ms); - } - - return error; -} - -/* FIXME: if this lkb is the only lock we hold on the rsb, then set - MASTER_UNCERTAIN to force the next request on the rsb to confirm - that the master is still correct. */ - -static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - return send_common(r, lkb, DLM_MSG_UNLOCK); -} - -static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - return send_common(r, lkb, DLM_MSG_CANCEL); -} - -static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = lkb->lkb_nodeid; - - error = create_message(r, lkb, to_nodeid, DLM_MSG_GRANT, &ms, &mh); - if (error) - goto out; - - send_args(r, lkb, ms); - - ms->m_result = 0; - - error = send_message(mh, ms); - out: - return error; -} - -static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = lkb->lkb_nodeid; - - error = create_message(r, NULL, to_nodeid, DLM_MSG_BAST, &ms, &mh); - if (error) - goto out; - - send_args(r, lkb, ms); - - ms->m_bastmode = mode; - /* XXX: Fill bastrange here */ - - error = send_message(mh, ms); - out: - return error; -} - -static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = dlm_dir_nodeid(r); - - error = add_to_waiters(lkb, DLM_MSG_LOOKUP, to_nodeid); - if (error) - return error; - - error = create_message(r, NULL, to_nodeid, DLM_MSG_LOOKUP, &ms, &mh); - if (error) - goto fail; - - send_args(r, lkb, ms); - - error = send_message(mh, ms); - if (error) - goto fail; - return 0; - - fail: - remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); - return error; -} - -static int send_remove(struct dlm_rsb *r) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = dlm_dir_nodeid(r); - - error = create_message(r, NULL, to_nodeid, DLM_MSG_REMOVE, &ms, &mh); - if (error) - goto out; - - memcpy(ms->m_extra, r->res_name, r->res_length); - ms->m_hash = r->res_hash; - - error = send_message(mh, ms); - out: - return error; -} - -static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, - int mstype, int rv) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int to_nodeid, error; - - to_nodeid = lkb->lkb_nodeid; - - error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); - if (error) - goto out; - - send_args(r, lkb, ms); - - ms->m_result = rv; - - error = send_message(mh, ms); - out: - return error; -} - -static int send_request_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) -{ - return send_common_reply(r, lkb, DLM_MSG_REQUEST_REPLY, rv); -} - -static int send_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) -{ - return send_common_reply(r, lkb, DLM_MSG_CONVERT_REPLY, rv); -} - -static int send_unlock_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) -{ - return send_common_reply(r, lkb, DLM_MSG_UNLOCK_REPLY, rv); -} - -static int send_cancel_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) -{ - return send_common_reply(r, lkb, DLM_MSG_CANCEL_REPLY, rv); -} - -static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in, - int ret_nodeid, int rv) -{ - struct dlm_rsb *r = &ls->ls_stub_rsb; - struct dlm_message *ms; - struct dlm_mhandle *mh; - int error, nodeid = ms_in->m_header.h_nodeid; - - error = create_message(r, NULL, nodeid, DLM_MSG_LOOKUP_REPLY, &ms, &mh); - if (error) - goto out; - - ms->m_lkid = ms_in->m_lkid; - ms->m_result = rv; - ms->m_nodeid = ret_nodeid; - - error = send_message(mh, ms); - out: - return error; -} - -/* which args we save from a received message depends heavily on the type - of message, unlike the send side where we can safely send everything about - the lkb for any type of message */ - -static void receive_flags(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - lkb->lkb_exflags = ms->m_exflags; - lkb->lkb_sbflags = ms->m_sbflags; - lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) | - (ms->m_flags & 0x0000FFFF); -} - -static void receive_flags_reply(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - if (ms->m_flags == DLM_IFL_STUB_MS) - return; - - lkb->lkb_sbflags = ms->m_sbflags; - lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) | - (ms->m_flags & 0x0000FFFF); -} - -static int receive_extralen(struct dlm_message *ms) -{ - return (ms->m_header.h_length - sizeof(struct dlm_message)); -} - -static int receive_lvb(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - int len; - - if (lkb->lkb_exflags & DLM_LKF_VALBLK) { - if (!lkb->lkb_lvbptr) - lkb->lkb_lvbptr = dlm_allocate_lvb(ls); - if (!lkb->lkb_lvbptr) - return -ENOMEM; - len = receive_extralen(ms); - if (len > DLM_RESNAME_MAXLEN) - len = DLM_RESNAME_MAXLEN; - memcpy(lkb->lkb_lvbptr, ms->m_extra, len); - } - return 0; -} - -static void fake_bastfn(void *astparam, int mode) -{ - log_print("fake_bastfn should not be called"); -} - -static void fake_astfn(void *astparam) -{ - log_print("fake_astfn should not be called"); -} - -static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - lkb->lkb_nodeid = ms->m_header.h_nodeid; - lkb->lkb_ownpid = ms->m_pid; - lkb->lkb_remid = ms->m_lkid; - lkb->lkb_grmode = DLM_LOCK_IV; - lkb->lkb_rqmode = ms->m_rqmode; - - lkb->lkb_bastfn = (ms->m_asts & DLM_CB_BAST) ? &fake_bastfn : NULL; - lkb->lkb_astfn = (ms->m_asts & DLM_CB_CAST) ? &fake_astfn : NULL; - - if (lkb->lkb_exflags & DLM_LKF_VALBLK) { - /* lkb was just created so there won't be an lvb yet */ - lkb->lkb_lvbptr = dlm_allocate_lvb(ls); - if (!lkb->lkb_lvbptr) - return -ENOMEM; - } - - if (ms->m_grstart_len) { - lkb->lkb_grrange.start = alloc_key(ms->m_grstart, - ms->m_grstart_len, GFP_NOFS); - if (!lkb->lkb_grrange.start) - return -ENOMEM; - } - if (ms->m_grend_len) { - lkb->lkb_grrange.end = alloc_key(ms->m_grend, ms->m_grend_len, - GFP_NOFS); - if (!lkb->lkb_grrange.end) - return -ENOMEM; - } - - if (ms->m_rqstart_len) { - lkb->lkb_rqrange.start = alloc_key(ms->m_rqstart, - ms->m_rqstart_len, GFP_NOFS); - if (!lkb->lkb_rqrange.start) - return -ENOMEM; - } - if (ms->m_rqend_len) { - lkb->lkb_rqrange.end = alloc_key(ms->m_rqend, ms->m_rqend_len, - GFP_NOFS); - if (!lkb->lkb_rqrange.end) - return -ENOMEM; - } - return 0; -} - -static int receive_convert_args(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - if (lkb->lkb_status != DLM_LKSTS_GRANTED) - return -EBUSY; - - if (receive_lvb(ls, lkb, ms)) - return -ENOMEM; - - lkb->lkb_rqmode = ms->m_rqmode; - lkb->lkb_lvbseq = ms->m_lvbseq; - - return 0; -} - -static int receive_unlock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - if (receive_lvb(ls, lkb, ms)) - return -ENOMEM; - return 0; -} - -/* We fill in the stub-lkb fields with the info that send_xxxx_reply() - uses to send a reply and that the remote end uses to process the reply. */ - -static void setup_stub_lkb(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb = &ls->ls_stub_lkb; - lkb->lkb_nodeid = ms->m_header.h_nodeid; - lkb->lkb_remid = ms->m_lkid; -} - -/* This is called after the rsb is locked so that we can safely inspect - fields in the lkb. */ - -static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - int from = ms->m_header.h_nodeid; - int error = 0; - - switch (ms->m_type) { - case DLM_MSG_CONVERT: - case DLM_MSG_UNLOCK: - case DLM_MSG_CANCEL: - if (!is_master_copy(lkb) || lkb->lkb_nodeid != from) - error = -EINVAL; - break; - - case DLM_MSG_CONVERT_REPLY: - case DLM_MSG_UNLOCK_REPLY: - case DLM_MSG_CANCEL_REPLY: - case DLM_MSG_GRANT: - case DLM_MSG_BAST: - if (!is_process_copy(lkb) || lkb->lkb_nodeid != from) - error = -EINVAL; - break; - - case DLM_MSG_REQUEST_REPLY: - if (!is_process_copy(lkb)) - error = -EINVAL; - else if (lkb->lkb_nodeid != -1 && lkb->lkb_nodeid != from) - error = -EINVAL; - break; - - default: - error = -EINVAL; - } - - if (error) - log_error(lkb->lkb_resource->res_ls, - "ignore invalid message %d from %d %x %x %x %d", - ms->m_type, from, lkb->lkb_id, lkb->lkb_remid, - lkb->lkb_flags, lkb->lkb_nodeid); - return error; -} - -static void send_repeat_remove(struct dlm_ls *ls, char *ms_name, int len) -{ - char name[DLM_RESNAME_MAXLEN + 1]; - struct dlm_message *ms; - struct dlm_mhandle *mh; - struct dlm_rsb *r; - uint32_t hash, b; - int rv, dir_nodeid; - - memset(name, 0, sizeof(name)); - memcpy(name, ms_name, len); - - hash = jhash(name, len, 0); - b = hash & (ls->ls_rsbtbl_size - 1); - - dir_nodeid = dlm_hash2nodeid(ls, hash); - - log_error(ls, "send_repeat_remove dir %d %s", dir_nodeid, name); - - spin_lock(&ls->ls_rsbtbl[b].lock); - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (!rv) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_error(ls, "repeat_remove on keep %s", name); - return; - } - - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (!rv) { - spin_unlock(&ls->ls_rsbtbl[b].lock); - log_error(ls, "repeat_remove on toss %s", name); - return; - } - - /* use ls->remove_name2 to avoid conflict with shrink? */ - - spin_lock(&ls->ls_remove_spin); - ls->ls_remove_len = len; - memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN); - spin_unlock(&ls->ls_remove_spin); - spin_unlock(&ls->ls_rsbtbl[b].lock); - - rv = _create_message(ls, sizeof(struct dlm_message) + len, - dir_nodeid, DLM_MSG_REMOVE, &ms, &mh); - if (rv) - return; - - memcpy(ms->m_extra, name, len); - ms->m_hash = hash; - - send_message(mh, ms); - - spin_lock(&ls->ls_remove_spin); - ls->ls_remove_len = 0; - memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN); - spin_unlock(&ls->ls_remove_spin); -} - -static int receive_request(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int from_nodeid; - int error, namelen = 0; - - from_nodeid = ms->m_header.h_nodeid; - - error = create_lkb(ls, &lkb); - if (error) - goto fail; - - receive_flags(lkb, ms); - lkb->lkb_flags |= DLM_IFL_MSTCPY; - error = receive_request_args(ls, lkb, ms); - if (error) { - __put_lkb(ls, lkb); - goto fail; - } - - /* The dir node is the authority on whether we are the master - for this rsb or not, so if the master sends us a request, we should - recreate the rsb if we've destroyed it. This race happens when we - send a remove message to the dir node at the same time that the dir - node sends us a request for the rsb. */ - - namelen = receive_extralen(ms); - - error = find_rsb(ls, ms->m_extra, namelen, from_nodeid, - R_RECEIVE_REQUEST, &r); - if (error) { - __put_lkb(ls, lkb); - goto fail; - } - - lock_rsb(r); - - if (r->res_master_nodeid != dlm_our_nodeid()) { - error = validate_master_nodeid(ls, r, from_nodeid); - if (error) { - unlock_rsb(r); - put_rsb(r); - __put_lkb(ls, lkb); - goto fail; - } - } - - attach_lkb(r, lkb); - error = do_request(r, lkb); - send_request_reply(r, lkb, error); - do_request_effects(r, lkb, error); - - unlock_rsb(r); - put_rsb(r); - - if (error == -EINPROGRESS) - error = 0; - if (error) - dlm_put_lkb(lkb); - return 0; - - fail: - /* TODO: instead of returning ENOTBLK, add the lkb to res_lookup - and do this receive_request again from process_lookup_list once - we get the lookup reply. This would avoid a many repeated - ENOTBLK request failures when the lookup reply designating us - as master is delayed. */ - - /* We could repeatedly return -EBADR here if our send_remove() is - delayed in being sent/arriving/being processed on the dir node. - Another node would repeatedly lookup up the master, and the dir - node would continue returning our nodeid until our send_remove - took effect. - - We send another remove message in case our previous send_remove - was lost/ignored/missed somehow. */ - - if (error != -ENOTBLK) { - log_limit(ls, "receive_request %x from %d %d", - ms->m_lkid, from_nodeid, error); - } - - if (namelen && error == -EBADR) { - send_repeat_remove(ls, ms->m_extra, namelen); - msleep(1000); - } - - setup_stub_lkb(ls, ms); - send_request_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); - return error; -} - -static int receive_convert(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error, reply = 1; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - goto fail; - - if (lkb->lkb_remid != ms->m_lkid) { - log_error(ls, "receive_convert %x remid %x recover_seq %llu " - "remote %d %x", lkb->lkb_id, lkb->lkb_remid, - (unsigned long long)lkb->lkb_recover_seq, - ms->m_header.h_nodeid, ms->m_lkid); - error = -ENOENT; - goto fail; - } - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - receive_flags(lkb, ms); - - error = receive_convert_args(ls, lkb, ms); - if (error) { - send_convert_reply(r, lkb, error); - goto out; - } - - reply = !down_conversion(lkb); - - error = do_convert(r, lkb); - if (reply) - send_convert_reply(r, lkb, error); - do_convert_effects(r, lkb, error); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; - - fail: - setup_stub_lkb(ls, ms); - send_convert_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); - return error; -} - -static int receive_unlock(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - goto fail; - - if (lkb->lkb_remid != ms->m_lkid) { - log_error(ls, "receive_unlock %x remid %x remote %d %x", - lkb->lkb_id, lkb->lkb_remid, - ms->m_header.h_nodeid, ms->m_lkid); - error = -ENOENT; - goto fail; - } - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - receive_flags(lkb, ms); - - error = receive_unlock_args(ls, lkb, ms); - if (error) { - send_unlock_reply(r, lkb, error); - goto out; - } - - error = do_unlock(r, lkb); - send_unlock_reply(r, lkb, error); - do_unlock_effects(r, lkb, error); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; - - fail: - setup_stub_lkb(ls, ms); - send_unlock_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); - return error; -} - -static int receive_cancel(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - goto fail; - - receive_flags(lkb, ms); - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - error = do_cancel(r, lkb); - send_cancel_reply(r, lkb, error); - do_cancel_effects(r, lkb, error); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; - - fail: - setup_stub_lkb(ls, ms); - send_cancel_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error); - return error; -} - -static int receive_grant(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - receive_flags_reply(lkb, ms); - if (is_altmode(lkb)) - munge_altmode(lkb, ms); - grant_lock_pc(r, lkb, ms); - queue_cast(r, lkb, 0); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; -} - -static int receive_bast(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - struct dlm_range range; - struct dlm_key start, end; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - r = lkb->lkb_resource; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - start.val = ms->m_baststart; - start.len = ms->m_baststart_len; - end.val = ms->m_bastend; - end.len = ms->m_bastend_len; - range.start = &start; - range.end = &end; - - queue_bast(r, lkb, ms->m_bastmode, &range); - lkb->lkb_highbast = ms->m_bastmode; - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; -} - -static void receive_lookup(struct dlm_ls *ls, struct dlm_message *ms) -{ - int len, error, ret_nodeid, from_nodeid, our_nodeid; - - from_nodeid = ms->m_header.h_nodeid; - our_nodeid = dlm_our_nodeid(); - - len = receive_extralen(ms); - - error = dlm_master_lookup(ls, from_nodeid, ms->m_extra, len, 0, - &ret_nodeid, NULL); - - /* Optimization: we're master so treat lookup as a request */ - if (!error && ret_nodeid == our_nodeid) { - receive_request(ls, ms); - return; - } - send_lookup_reply(ls, ms, ret_nodeid, error); -} - -static void receive_remove(struct dlm_ls *ls, struct dlm_message *ms) -{ - char name[DLM_RESNAME_MAXLEN+1]; - struct dlm_rsb *r; - uint32_t hash, b; - int rv, len, dir_nodeid, from_nodeid; - - from_nodeid = ms->m_header.h_nodeid; - - len = receive_extralen(ms); - - if (len > DLM_RESNAME_MAXLEN) { - log_error(ls, "receive_remove from %d bad len %d", - from_nodeid, len); - return; - } - - dir_nodeid = dlm_hash2nodeid(ls, ms->m_hash); - if (dir_nodeid != dlm_our_nodeid()) { - log_error(ls, "receive_remove from %d bad nodeid %d", - from_nodeid, dir_nodeid); - return; - } - - /* Look for name on rsbtbl.toss, if it's there, kill it. - If it's on rsbtbl.keep, it's being used, and we should ignore this - message. This is an expected race between the dir node sending a - request to the master node at the same time as the master node sends - a remove to the dir node. The resolution to that race is for the - dir node to ignore the remove message, and the master node to - recreate the master rsb when it gets a request from the dir node for - an rsb it doesn't have. */ - - memset(name, 0, sizeof(name)); - memcpy(name, ms->m_extra, len); - - hash = jhash(name, len, 0); - b = hash & (ls->ls_rsbtbl_size - 1); - - spin_lock(&ls->ls_rsbtbl[b].lock); - - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r); - if (rv) { - /* verify the rsb is on keep list per comment above */ - rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r); - if (rv) { - /* should not happen */ - log_error(ls, "receive_remove from %d not found %s", - from_nodeid, name); - spin_unlock(&ls->ls_rsbtbl[b].lock); - return; - } - if (r->res_master_nodeid != from_nodeid) { - /* should not happen */ - log_error(ls, "receive_remove keep from %d master %d", - from_nodeid, r->res_master_nodeid); - dlm_print_rsb(r); - spin_unlock(&ls->ls_rsbtbl[b].lock); - return; - } - - log_debug(ls, "receive_remove from %d master %d first %x %s", - from_nodeid, r->res_master_nodeid, r->res_first_lkid, - name); - spin_unlock(&ls->ls_rsbtbl[b].lock); - return; - } - - if (r->res_master_nodeid != from_nodeid) { - log_error(ls, "receive_remove toss from %d master %d", - from_nodeid, r->res_master_nodeid); - dlm_print_rsb(r); - spin_unlock(&ls->ls_rsbtbl[b].lock); - return; - } - - if (kref_put(&r->res_ref, kill_rsb)) { - rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss); - spin_unlock(&ls->ls_rsbtbl[b].lock); - dlm_free_rsb(r); - } else { - log_error(ls, "receive_remove from %d rsb ref error", - from_nodeid); - dlm_print_rsb(r); - spin_unlock(&ls->ls_rsbtbl[b].lock); - } -} - -static void receive_purge(struct dlm_ls *ls, struct dlm_message *ms) -{ - do_purge(ls, ms->m_nodeid, ms->m_pid); -} - -static int receive_request_reply(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error, mstype, result; - int from_nodeid = ms->m_header.h_nodeid; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - mstype = lkb->lkb_wait_type; - error = remove_from_waiters(lkb, DLM_MSG_REQUEST_REPLY); - if (error) { - log_error(ls, "receive_request_reply %x remote %d %x result %d", - lkb->lkb_id, from_nodeid, ms->m_lkid, ms->m_result); - dlm_dump_rsb(r); - goto out; - } - - /* Optimization: the dir node was also the master, so it took our - lookup as a request and sent request reply instead of lookup reply */ - if (mstype == DLM_MSG_LOOKUP) { - r->res_master_nodeid = from_nodeid; - r->res_nodeid = from_nodeid; - lkb->lkb_nodeid = from_nodeid; - } - - /* this is the value returned from do_request() on the master */ - result = ms->m_result; - - switch (result) { - case -EAGAIN: - /* request would block (be queued) on remote master */ - queue_cast(r, lkb, -EAGAIN); - confirm_master(r, -EAGAIN); - unhold_lkb(lkb); /* undoes create_lkb() */ - break; - - case -EINPROGRESS: - case 0: - /* request was queued or granted on remote master */ - receive_flags_reply(lkb, ms); - lkb->lkb_remid = ms->m_lkid; - if (is_altmode(lkb)) - munge_altmode(lkb, ms); - if (result) { - add_lkb(r, lkb, DLM_LKSTS_WAITING); - add_timeout(lkb); - } else { - grant_lock_pc(r, lkb, ms); - queue_cast(r, lkb, 0); - } - confirm_master(r, result); - break; - - case -EBADR: - case -ENOTBLK: - /* find_rsb failed to find rsb or rsb wasn't master */ - log_limit(ls, "receive_request_reply %x from %d %d " - "master %d dir %d first %x %s", lkb->lkb_id, - from_nodeid, result, r->res_master_nodeid, - r->res_dir_nodeid, r->res_first_lkid, r->res_name); - - if (r->res_dir_nodeid != dlm_our_nodeid() && - r->res_master_nodeid != dlm_our_nodeid()) { - /* cause _request_lock->set_master->send_lookup */ - r->res_master_nodeid = 0; - r->res_nodeid = -1; - lkb->lkb_nodeid = -1; - } - - if (is_overlap(lkb)) { - /* we'll ignore error in cancel/unlock reply */ - queue_cast_overlap(r, lkb); - confirm_master(r, result); - unhold_lkb(lkb); /* undoes create_lkb() */ - } else { - _request_lock(r, lkb); - - if (r->res_master_nodeid == dlm_our_nodeid()) - confirm_master(r, 0); - } - break; - - default: - log_error(ls, "receive_request_reply %x error %d", - lkb->lkb_id, result); - } - - if (is_overlap_unlock(lkb) && (result == 0 || result == -EINPROGRESS)) { - log_debug(ls, "receive_request_reply %x result %d unlock", - lkb->lkb_id, result); - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - send_unlock(r, lkb); - } else if (is_overlap_cancel(lkb) && (result == -EINPROGRESS)) { - log_debug(ls, "receive_request_reply %x cancel", lkb->lkb_id); - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - send_cancel(r, lkb); - } else { - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; - } - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return 0; -} - -static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct dlm_message *ms) -{ - /* this is the value returned from do_convert() on the master */ - switch (ms->m_result) { - case -EAGAIN: - /* convert would block (be queued) on remote master */ - queue_cast(r, lkb, -EAGAIN); - break; - - case -EDEADLK: - receive_flags_reply(lkb, ms); - revert_lock_pc(r, lkb); - queue_cast(r, lkb, -EDEADLK); - break; - - case -EINPROGRESS: - /* convert was queued on remote master */ - receive_flags_reply(lkb, ms); - if (is_demoted(lkb)) - munge_demoted(lkb); - del_lkb(r, lkb); - add_lkb(r, lkb, DLM_LKSTS_CONVERT); - add_timeout(lkb); - break; - - case 0: - /* convert was granted on remote master */ - receive_flags_reply(lkb, ms); - if (is_demoted(lkb)) - munge_demoted(lkb); - grant_lock_pc(r, lkb, ms); - queue_cast(r, lkb, 0); - break; - - default: - log_error(r->res_ls, "receive_convert_reply %x remote %d %x %d", - lkb->lkb_id, ms->m_header.h_nodeid, ms->m_lkid, - ms->m_result); - dlm_print_rsb(r); - dlm_print_lkb(lkb); - } -} - -static void _receive_convert_reply(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - struct dlm_rsb *r = lkb->lkb_resource; - int error; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - /* stub reply can happen with waiters_mutex held */ - error = remove_from_waiters_ms(lkb, ms); - if (error) - goto out; - - __receive_convert_reply(r, lkb, ms); - out: - unlock_rsb(r); - put_rsb(r); -} - -static int receive_convert_reply(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - _receive_convert_reply(lkb, ms); - dlm_put_lkb(lkb); - return 0; -} - -static void _receive_unlock_reply(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - struct dlm_rsb *r = lkb->lkb_resource; - int error; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - /* stub reply can happen with waiters_mutex held */ - error = remove_from_waiters_ms(lkb, ms); - if (error) - goto out; - - /* this is the value returned from do_unlock() on the master */ - - switch (ms->m_result) { - case -DLM_EUNLOCK: - receive_flags_reply(lkb, ms); - remove_lock_pc(r, lkb); - queue_cast(r, lkb, -DLM_EUNLOCK); - break; - case -ENOENT: - break; - default: - log_error(r->res_ls, "receive_unlock_reply %x error %d", - lkb->lkb_id, ms->m_result); - } - out: - unlock_rsb(r); - put_rsb(r); -} - -static int receive_unlock_reply(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - _receive_unlock_reply(lkb, ms); - dlm_put_lkb(lkb); - return 0; -} - -static void _receive_cancel_reply(struct dlm_lkb *lkb, struct dlm_message *ms) -{ - struct dlm_rsb *r = lkb->lkb_resource; - int error; - - hold_rsb(r); - lock_rsb(r); - - error = validate_message(lkb, ms); - if (error) - goto out; - - /* stub reply can happen with waiters_mutex held */ - error = remove_from_waiters_ms(lkb, ms); - if (error) - goto out; - - /* this is the value returned from do_cancel() on the master */ - - switch (ms->m_result) { - case -DLM_ECANCEL: - receive_flags_reply(lkb, ms); - revert_lock_pc(r, lkb); - queue_cast(r, lkb, -DLM_ECANCEL); - break; - case 0: - break; - default: - log_error(r->res_ls, "receive_cancel_reply %x error %d", - lkb->lkb_id, ms->m_result); - } - out: - unlock_rsb(r); - put_rsb(r); -} - -static int receive_cancel_reply(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - int error; - - error = find_lkb(ls, ms->m_remid, &lkb); - if (error) - return error; - - _receive_cancel_reply(lkb, ms); - dlm_put_lkb(lkb); - return 0; -} - -static void receive_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error, ret_nodeid; - int do_lookup_list = 0; - - error = find_lkb(ls, ms->m_lkid, &lkb); - if (error) { - log_error(ls, "receive_lookup_reply no lkid %x", ms->m_lkid); - return; - } - - /* ms->m_result is the value returned by dlm_master_lookup on dir node - FIXME: will a non-zero error ever be returned? */ - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - error = remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); - if (error) - goto out; - - ret_nodeid = ms->m_nodeid; - - /* We sometimes receive a request from the dir node for this - rsb before we've received the dir node's loookup_reply for it. - The request from the dir node implies we're the master, so we set - ourself as master in receive_request_reply, and verify here that - we are indeed the master. */ - - if (r->res_master_nodeid && (r->res_master_nodeid != ret_nodeid)) { - /* This should never happen */ - log_error(ls, "receive_lookup_reply %x from %d ret %d " - "master %d dir %d our %d first %x %s", - lkb->lkb_id, ms->m_header.h_nodeid, ret_nodeid, - r->res_master_nodeid, r->res_dir_nodeid, - dlm_our_nodeid(), r->res_first_lkid, r->res_name); - } - - if (ret_nodeid == dlm_our_nodeid()) { - r->res_master_nodeid = ret_nodeid; - r->res_nodeid = 0; - do_lookup_list = 1; - r->res_first_lkid = 0; - } else if (ret_nodeid == -1) { - /* the remote node doesn't believe it's the dir node */ - log_error(ls, "receive_lookup_reply %x from %d bad ret_nodeid", - lkb->lkb_id, ms->m_header.h_nodeid); - r->res_master_nodeid = 0; - r->res_nodeid = -1; - lkb->lkb_nodeid = -1; - } else { - /* set_master() will set lkb_nodeid from r */ - r->res_master_nodeid = ret_nodeid; - r->res_nodeid = ret_nodeid; - } - - if (is_overlap(lkb)) { - log_debug(ls, "receive_lookup_reply %x unlock %x", - lkb->lkb_id, lkb->lkb_flags); - queue_cast_overlap(r, lkb); - unhold_lkb(lkb); /* undoes create_lkb() */ - goto out_list; - } - - _request_lock(r, lkb); - - out_list: - if (do_lookup_list) - process_lookup_list(r); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); -} - -static void _receive_message(struct dlm_ls *ls, struct dlm_message *ms, - uint32_t saved_seq) -{ - int error = 0, noent = 0; - - if (!dlm_is_member(ls, ms->m_header.h_nodeid)) { - log_limit(ls, "receive %d from non-member %d %x %x %d", - ms->m_type, ms->m_header.h_nodeid, ms->m_lkid, - ms->m_remid, ms->m_result); - return; - } - - switch (ms->m_type) { - - /* messages sent to a master node */ - - case DLM_MSG_REQUEST: - error = receive_request(ls, ms); - break; - - case DLM_MSG_CONVERT: - error = receive_convert(ls, ms); - break; - - case DLM_MSG_UNLOCK: - error = receive_unlock(ls, ms); - break; - - case DLM_MSG_CANCEL: - noent = 1; - error = receive_cancel(ls, ms); - break; - - /* messages sent from a master node (replies to above) */ - - case DLM_MSG_REQUEST_REPLY: - error = receive_request_reply(ls, ms); - break; - - case DLM_MSG_CONVERT_REPLY: - error = receive_convert_reply(ls, ms); - break; - - case DLM_MSG_UNLOCK_REPLY: - error = receive_unlock_reply(ls, ms); - break; - - case DLM_MSG_CANCEL_REPLY: - error = receive_cancel_reply(ls, ms); - break; - - /* messages sent from a master node (only two types of async msg) */ - - case DLM_MSG_GRANT: - noent = 1; - error = receive_grant(ls, ms); - break; - - case DLM_MSG_BAST: - noent = 1; - error = receive_bast(ls, ms); - break; - - /* messages sent to a dir node */ - - case DLM_MSG_LOOKUP: - receive_lookup(ls, ms); - break; - - case DLM_MSG_REMOVE: - receive_remove(ls, ms); - break; - - /* messages sent from a dir node (remove has no reply) */ - - case DLM_MSG_LOOKUP_REPLY: - receive_lookup_reply(ls, ms); - break; - - /* other messages */ - - case DLM_MSG_PURGE: - receive_purge(ls, ms); - break; - - default: - log_error(ls, "unknown message type %d", ms->m_type); - } - - /* - * When checking for ENOENT, we're checking the result of - * find_lkb(m_remid): - * - * The lock id referenced in the message wasn't found. This may - * happen in normal usage for the async messages and cancel, so - * only use log_debug for them. - * - * Some errors are expected and normal. - */ - - if (error == -ENOENT && noent) { - log_debug(ls, "receive %d no %x remote %d %x saved_seq %u", - ms->m_type, ms->m_remid, ms->m_header.h_nodeid, - ms->m_lkid, saved_seq); - } else if (error == -ENOENT) { - log_error(ls, "receive %d no %x remote %d %x saved_seq %u", - ms->m_type, ms->m_remid, ms->m_header.h_nodeid, - ms->m_lkid, saved_seq); - - if (ms->m_type == DLM_MSG_CONVERT) - dlm_dump_rsb_hash(ls, ms->m_hash); - } - - if (error == -EINVAL) { - log_error(ls, "receive %d inval from %d lkid %x remid %x " - "saved_seq %u", - ms->m_type, ms->m_header.h_nodeid, - ms->m_lkid, ms->m_remid, saved_seq); - } -} - -/* If the lockspace is in recovery mode (locking stopped), then normal - messages are saved on the requestqueue for processing after recovery is - done. When not in recovery mode, we wait for dlm_recoverd to drain saved - messages off the requestqueue before we process new ones. This occurs right - after recovery completes when we transition from saving all messages on - requestqueue, to processing all the saved messages, to processing new - messages as they arrive. */ - -static void dlm_receive_message(struct dlm_ls *ls, struct dlm_message *ms, - int nodeid) -{ - if (dlm_locking_stopped(ls)) { - /* If we were a member of this lockspace, left, and rejoined, - other nodes may still be sending us messages from the - lockspace generation before we left. */ - if (!ls->ls_generation) { - log_limit(ls, "receive %d from %d ignore old gen", - ms->m_type, nodeid); - return; - } - - dlm_add_requestqueue(ls, nodeid, ms); - } else { - dlm_wait_requestqueue(ls); - _receive_message(ls, ms, 0); - } -} - -/* This is called by dlm_recoverd to process messages that were saved on - the requestqueue. */ - -void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms, - uint32_t saved_seq) -{ - _receive_message(ls, ms, saved_seq); -} - -/* This is called by the midcomms layer when something is received for - the lockspace. It could be either a MSG (normal message sent as part of - standard locking activity) or an RCOM (recovery message sent as part of - lockspace recovery). */ - -void dlm_receive_buffer(union dlm_packet *p, int nodeid) -{ - struct dlm_header *hd = &p->header; - struct dlm_ls *ls; - int type = 0; - - switch (hd->h_cmd) { - case DLM_MSG: - dlm_message_in(&p->message); - type = p->message.m_type; - break; - case DLM_RCOM: - dlm_rcom_in(&p->rcom); - type = p->rcom.rc_type; - break; - default: - log_print("invalid h_cmd %d from %u", hd->h_cmd, nodeid); - return; - } - - if (hd->h_nodeid != nodeid) { - log_print("invalid h_nodeid %d from %d lockspace %x", - hd->h_nodeid, nodeid, hd->h_lockspace); - return; - } - - ls = dlm_find_lockspace_global(hd->h_lockspace); - if (!ls) { - if (dlm_config.ci_log_debug) { - printk_ratelimited(KERN_DEBUG "dlm: invalid lockspace " - "%u from %d cmd %d type %d\n", - hd->h_lockspace, nodeid, hd->h_cmd, type); - } - - if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS) - dlm_send_ls_not_ready(nodeid, &p->rcom); - return; - } - - /* this rwsem allows dlm_ls_stop() to wait for all dlm_recv threads to - be inactive (in this ls) before transitioning to recovery mode */ - - down_read(&ls->ls_recv_active); - if (hd->h_cmd == DLM_MSG) - dlm_receive_message(ls, &p->message, nodeid); - else - dlm_receive_rcom(ls, &p->rcom, nodeid); - up_read(&ls->ls_recv_active); - - dlm_put_lockspace(ls); -} - -static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_message *ms_stub) -{ - if (middle_conversion(lkb)) { - hold_lkb(lkb); - memset(ms_stub, 0, sizeof(struct dlm_message)); - ms_stub->m_flags = DLM_IFL_STUB_MS; - ms_stub->m_type = DLM_MSG_CONVERT_REPLY; - ms_stub->m_result = -EINPROGRESS; - ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; - _receive_convert_reply(lkb, ms_stub); - - /* Same special case as in receive_rcom_lock_args() */ - lkb->lkb_grmode = DLM_LOCK_IV; - rsb_set_flag(lkb->lkb_resource, RSB_RECOVER_CONVERT); - unhold_lkb(lkb); - - } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) { - lkb->lkb_flags |= DLM_IFL_RESEND; - } - - /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down - conversions are async; there's no reply from the remote master */ -} - -/* A waiting lkb needs recovery if the master node has failed, or - the master node is changing (only when no directory is used) */ - -static int waiter_needs_recovery(struct dlm_ls *ls, struct dlm_lkb *lkb, - int dir_nodeid) -{ - if (dlm_no_directory(ls)) - return 1; - - if (dlm_is_removed(ls, lkb->lkb_wait_nodeid)) - return 1; - - return 0; -} - -/* Recovery for locks that are waiting for replies from nodes that are now - gone. We can just complete unlocks and cancels by faking a reply from the - dead node. Requests and up-conversions we flag to be resent after - recovery. Down-conversions can just be completed with a fake reply like - unlocks. Conversions between PR and CW need special attention. */ - -void dlm_recover_waiters_pre(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb, *safe; - struct dlm_message *ms_stub; - int wait_type, stub_unlock_result, stub_cancel_result; - int dir_nodeid; - - ms_stub = kmalloc(sizeof(struct dlm_message), GFP_KERNEL); - if (!ms_stub) { - log_error(ls, "dlm_recover_waiters_pre no mem"); - return; - } - - mutex_lock(&ls->ls_waiters_mutex); - - list_for_each_entry_safe(lkb, safe, &ls->ls_waiters, lkb_wait_reply) { - - dir_nodeid = dlm_dir_nodeid(lkb->lkb_resource); - - /* exclude debug messages about unlocks because there can be so - many and they aren't very interesting */ - - if (lkb->lkb_wait_type != DLM_MSG_UNLOCK) { - log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " - "lkb_nodeid %d wait_nodeid %d dir_nodeid %d", - lkb->lkb_id, - lkb->lkb_remid, - lkb->lkb_wait_type, - lkb->lkb_resource->res_nodeid, - lkb->lkb_nodeid, - lkb->lkb_wait_nodeid, - dir_nodeid); - } - - /* all outstanding lookups, regardless of destination will be - resent after recovery is done */ - - if (lkb->lkb_wait_type == DLM_MSG_LOOKUP) { - lkb->lkb_flags |= DLM_IFL_RESEND; - continue; - } - - if (!waiter_needs_recovery(ls, lkb, dir_nodeid)) - continue; - - wait_type = lkb->lkb_wait_type; - stub_unlock_result = -DLM_EUNLOCK; - stub_cancel_result = -DLM_ECANCEL; - - /* Main reply may have been received leaving a zero wait_type, - but a reply for the overlapping op may not have been - received. In that case we need to fake the appropriate - reply for the overlap op. */ - - if (!wait_type) { - if (is_overlap_cancel(lkb)) { - wait_type = DLM_MSG_CANCEL; - if (lkb->lkb_grmode == DLM_LOCK_IV) - stub_cancel_result = 0; - } - if (is_overlap_unlock(lkb)) { - wait_type = DLM_MSG_UNLOCK; - if (lkb->lkb_grmode == DLM_LOCK_IV) - stub_unlock_result = -ENOENT; - } - - log_debug(ls, "rwpre overlap %x %x %d %d %d", - lkb->lkb_id, lkb->lkb_flags, wait_type, - stub_cancel_result, stub_unlock_result); - } - - switch (wait_type) { - - case DLM_MSG_REQUEST: - lkb->lkb_flags |= DLM_IFL_RESEND; - break; - - case DLM_MSG_CONVERT: - recover_convert_waiter(ls, lkb, ms_stub); - break; - - case DLM_MSG_UNLOCK: - hold_lkb(lkb); - memset(ms_stub, 0, sizeof(struct dlm_message)); - ms_stub->m_flags = DLM_IFL_STUB_MS; - ms_stub->m_type = DLM_MSG_UNLOCK_REPLY; - ms_stub->m_result = stub_unlock_result; - ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; - _receive_unlock_reply(lkb, ms_stub); - dlm_put_lkb(lkb); - break; - - case DLM_MSG_CANCEL: - hold_lkb(lkb); - memset(ms_stub, 0, sizeof(struct dlm_message)); - ms_stub->m_flags = DLM_IFL_STUB_MS; - ms_stub->m_type = DLM_MSG_CANCEL_REPLY; - ms_stub->m_result = stub_cancel_result; - ms_stub->m_header.h_nodeid = lkb->lkb_nodeid; - _receive_cancel_reply(lkb, ms_stub); - dlm_put_lkb(lkb); - break; - - default: - log_error(ls, "invalid lkb wait_type %d %d", - lkb->lkb_wait_type, wait_type); - } - schedule(); - } - mutex_unlock(&ls->ls_waiters_mutex); - kfree(ms_stub); -} - -static struct dlm_lkb *find_resend_waiter(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb; - int found = 0; - - mutex_lock(&ls->ls_waiters_mutex); - list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { - if (lkb->lkb_flags & DLM_IFL_RESEND) { - hold_lkb(lkb); - found = 1; - break; - } - } - mutex_unlock(&ls->ls_waiters_mutex); - - if (!found) - lkb = NULL; - return lkb; -} - -/* Deal with lookups and lkb's marked RESEND from _pre. We may now be the - master or dir-node for r. Processing the lkb may result in it being placed - back on waiters. */ - -/* We do this after normal locking has been enabled and any saved messages - (in requestqueue) have been processed. We should be confident that at - this point we won't get or process a reply to any of these waiting - operations. But, new ops may be coming in on the rsbs/locks here from - userspace or remotely. */ - -/* there may have been an overlap unlock/cancel prior to recovery or after - recovery. if before, the lkb may still have a pos wait_count; if after, the - overlap flag would just have been set and nothing new sent. we can be - confident here than any replies to either the initial op or overlap ops - prior to recovery have been received. */ - -int dlm_recover_waiters_post(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb; - struct dlm_rsb *r; - int error = 0, mstype, err, oc, ou; - - while (1) { - if (dlm_locking_stopped(ls)) { - log_debug(ls, "recover_waiters_post aborted"); - error = -EINTR; - break; - } - - lkb = find_resend_waiter(ls); - if (!lkb) - break; - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - mstype = lkb->lkb_wait_type; - oc = is_overlap_cancel(lkb); - ou = is_overlap_unlock(lkb); - err = 0; - - log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " - "lkb_nodeid %d wait_nodeid %d dir_nodeid %d " - "overlap %d %d", lkb->lkb_id, lkb->lkb_remid, mstype, - r->res_nodeid, lkb->lkb_nodeid, lkb->lkb_wait_nodeid, - dlm_dir_nodeid(r), oc, ou); - - /* At this point we assume that we won't get a reply to any - previous op or overlap op on this lock. First, do a big - remove_from_waiters() for all previous ops. */ - - lkb->lkb_flags &= ~DLM_IFL_RESEND; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK; - lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL; - lkb->lkb_wait_type = 0; - lkb->lkb_wait_count = 0; - mutex_lock(&ls->ls_waiters_mutex); - list_del_init(&lkb->lkb_wait_reply); - mutex_unlock(&ls->ls_waiters_mutex); - unhold_lkb(lkb); /* for waiters list */ - - if (oc || ou) { - /* do an unlock or cancel instead of resending */ - switch (mstype) { - case DLM_MSG_LOOKUP: - case DLM_MSG_REQUEST: - queue_cast(r, lkb, ou ? -DLM_EUNLOCK : - -DLM_ECANCEL); - unhold_lkb(lkb); /* undoes create_lkb() */ - break; - case DLM_MSG_CONVERT: - if (oc) { - queue_cast(r, lkb, -DLM_ECANCEL); - } else { - lkb->lkb_exflags |= DLM_LKF_FORCEUNLOCK; - _unlock_lock(r, lkb); - } - break; - default: - err = 1; - } - } else { - switch (mstype) { - case DLM_MSG_LOOKUP: - case DLM_MSG_REQUEST: - _request_lock(r, lkb); - if (is_master(r)) - confirm_master(r, 0); - break; - case DLM_MSG_CONVERT: - _convert_lock(r, lkb); - break; - default: - err = 1; - } - } - - if (err) { - log_error(ls, "waiter %x msg %d r_nodeid %d " - "dir_nodeid %d overlap %d %d", - lkb->lkb_id, mstype, r->res_nodeid, - dlm_dir_nodeid(r), oc, ou); - } - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - } - - return error; -} - -static void purge_mstcpy_list(struct dlm_ls *ls, struct dlm_rsb *r, - struct list_head *list) -{ - struct dlm_lkb *lkb, *safe; - - list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { - if (!is_master_copy(lkb)) - continue; - - /* don't purge lkbs we've added in recover_master_copy for - the current recovery seq */ - - if (lkb->lkb_recover_seq == ls->ls_recover_seq) - continue; - - del_lkb(r, lkb); - - /* this put should free the lkb */ - if (!dlm_put_lkb(lkb)) - log_error(ls, "purged mstcpy lkb not released"); - } -} - -void dlm_purge_mstcpy_locks(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - - purge_mstcpy_list(ls, r, &r->res_grantqueue); - purge_mstcpy_list(ls, r, &r->res_convertqueue); - purge_mstcpy_list(ls, r, &r->res_waitqueue); -} - -static void purge_dead_list(struct dlm_ls *ls, struct dlm_rsb *r, - struct list_head *list, - int nodeid_gone, unsigned int *count) -{ - struct dlm_lkb *lkb, *safe; - - list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { - if (!is_master_copy(lkb)) - continue; - - if ((lkb->lkb_nodeid == nodeid_gone) || - dlm_is_removed(ls, lkb->lkb_nodeid)) { - - /* tell recover_lvb to invalidate the lvb - because a node holding EX/PW failed */ - if ((lkb->lkb_exflags & DLM_LKF_VALBLK) && - (lkb->lkb_grmode >= DLM_LOCK_PW)) { - rsb_set_flag(r, RSB_RECOVER_LVB_INVAL); - } - - del_lkb(r, lkb); - - /* this put should free the lkb */ - if (!dlm_put_lkb(lkb)) - log_error(ls, "purged dead lkb not released"); - - rsb_set_flag(r, RSB_RECOVER_GRANT); - - (*count)++; - } - } -} - -/* Get rid of locks held by nodes that are gone. */ - -void dlm_recover_purge(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - struct dlm_member *memb; - int nodes_count = 0; - int nodeid_gone = 0; - unsigned int lkb_count = 0; - - /* cache one removed nodeid to optimize the common - case of a single node removed */ - - list_for_each_entry(memb, &ls->ls_nodes_gone, list) { - nodes_count++; - nodeid_gone = memb->nodeid; - } - - if (!nodes_count) - return; - - down_write(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - hold_rsb(r); - lock_rsb(r); - if (is_master(r)) { - purge_dead_list(ls, r, &r->res_grantqueue, - nodeid_gone, &lkb_count); - purge_dead_list(ls, r, &r->res_convertqueue, - nodeid_gone, &lkb_count); - purge_dead_list(ls, r, &r->res_waitqueue, - nodeid_gone, &lkb_count); - } - unlock_rsb(r); - unhold_rsb(r); - cond_resched(); - } - up_write(&ls->ls_root_sem); - - if (lkb_count) - log_debug(ls, "dlm_recover_purge %u locks for %u nodes", - lkb_count, nodes_count); -} - -static struct dlm_rsb *find_grant_rsb(struct dlm_ls *ls, int bucket) -{ - struct rb_node *n; - struct dlm_rsb *r; - - spin_lock(&ls->ls_rsbtbl[bucket].lock); - for (n = rb_first(&ls->ls_rsbtbl[bucket].keep); n; n = rb_next(n)) { - r = rb_entry(n, struct dlm_rsb, res_hashnode); - - if (!rsb_flag(r, RSB_RECOVER_GRANT)) - continue; - if (!is_master(r)) { - rsb_clear_flag(r, RSB_RECOVER_GRANT); - continue; - } - hold_rsb(r); - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - return r; - } - spin_unlock(&ls->ls_rsbtbl[bucket].lock); - return NULL; -} - -/* - * Attempt to grant locks on resources that we are the master of. - * Locks may have become grantable during recovery because locks - * from departed nodes have been purged (or not rebuilt), allowing - * previously blocked locks to now be granted. The subset of rsb's - * we are interested in are those with lkb's on either the convert or - * waiting queues. - * - * Simplest would be to go through each master rsb and check for non-empty - * convert or waiting queues, and attempt to grant on those rsbs. - * Checking the queues requires lock_rsb, though, for which we'd need - * to release the rsbtbl lock. This would make iterating through all - * rsb's very inefficient. So, we rely on earlier recovery routines - * to set RECOVER_GRANT on any rsb's that we should attempt to grant - * locks for. - */ - -void dlm_recover_grant(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - int bucket = 0; - unsigned int count = 0; - unsigned int rsb_count = 0; - unsigned int lkb_count = 0; - - while (1) { - r = find_grant_rsb(ls, bucket); - if (!r) { - if (bucket == ls->ls_rsbtbl_size - 1) - break; - bucket++; - continue; - } - rsb_count++; - count = 0; - lock_rsb(r); - /* the RECOVER_GRANT flag is checked in the grant path */ - grant_pending_locks(r, &count); - rsb_clear_flag(r, RSB_RECOVER_GRANT); - lkb_count += count; - confirm_master(r, 0); - unlock_rsb(r); - put_rsb(r); - cond_resched(); - } - - if (lkb_count) - log_debug(ls, "dlm_recover_grant %u locks on %u resources", - lkb_count, rsb_count); -} - -static struct dlm_lkb *search_remid_list(struct list_head *head, int nodeid, - uint32_t remid) -{ - struct dlm_lkb *lkb; - - list_for_each_entry(lkb, head, lkb_statequeue) { - if (lkb->lkb_nodeid == nodeid && lkb->lkb_remid == remid) - return lkb; - } - return NULL; -} - -static struct dlm_lkb *search_remid(struct dlm_rsb *r, int nodeid, - uint32_t remid) -{ - struct dlm_lkb *lkb; - - lkb = search_remid_list(&r->res_grantqueue, nodeid, remid); - if (lkb) - return lkb; - lkb = search_remid_list(&r->res_convertqueue, nodeid, remid); - if (lkb) - return lkb; - lkb = search_remid_list(&r->res_waitqueue, nodeid, remid); - if (lkb) - return lkb; - return NULL; -} - -/* needs at least dlm_rcom + rcom_lock */ -static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, - struct dlm_rsb *r, struct dlm_rcom *rc) -{ - struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; - - lkb->lkb_nodeid = rc->rc_header.h_nodeid; - lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid); - lkb->lkb_remid = le32_to_cpu(rl->rl_lkid); - lkb->lkb_exflags = le32_to_cpu(rl->rl_exflags); - lkb->lkb_flags = le32_to_cpu(rl->rl_flags) & 0x0000FFFF; - lkb->lkb_flags |= DLM_IFL_MSTCPY; - lkb->lkb_lvbseq = le32_to_cpu(rl->rl_lvbseq); - lkb->lkb_rqmode = rl->rl_rqmode; - lkb->lkb_grmode = rl->rl_grmode; - /* don't set lkb_status because add_lkb wants to itself */ - - lkb->lkb_bastfn = (rl->rl_asts & DLM_CB_BAST) ? &fake_bastfn : NULL; - lkb->lkb_astfn = (rl->rl_asts & DLM_CB_CAST) ? &fake_astfn : NULL; - - if (lkb->lkb_exflags & DLM_LKF_VALBLK) { - int lvblen = rc->rc_header.h_length - sizeof(struct dlm_rcom) - - sizeof(struct rcom_lock); - if (lvblen > ls->ls_lvblen) - return -EINVAL; - lkb->lkb_lvbptr = dlm_allocate_lvb(ls); - if (!lkb->lkb_lvbptr) - return -ENOMEM; - memcpy(lkb->lkb_lvbptr, rl->rl_lvb, lvblen); - } - - /* Conversions between PR and CW (middle modes) need special handling. - The real granted mode of these converting locks cannot be determined - until all locks have been rebuilt on the rsb (recover_conversion) */ - - if (rl->rl_wait_type == cpu_to_le16(DLM_MSG_CONVERT) && - middle_conversion(lkb)) { - rl->rl_status = DLM_LKSTS_CONVERT; - lkb->lkb_grmode = DLM_LOCK_IV; - rsb_set_flag(r, RSB_RECOVER_CONVERT); - } - - return 0; -} - -/* This lkb may have been recovered in a previous aborted recovery so we need - to check if the rsb already has an lkb with the given remote nodeid/lkid. - If so we just send back a standard reply. If not, we create a new lkb with - the given values and send back our lkid. We send back our lkid by sending - back the rcom_lock struct we got but with the remid field filled in. */ - -/* needs at least dlm_rcom + rcom_lock */ -int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc) -{ - struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; - struct dlm_rsb *r; - struct dlm_lkb *lkb; - uint32_t remid = 0; - int from_nodeid = rc->rc_header.h_nodeid; - int error; - - if (rl->rl_parent_lkid) { - error = -EOPNOTSUPP; - goto out; - } - - remid = le32_to_cpu(rl->rl_lkid); - - /* In general we expect the rsb returned to be R_MASTER, but we don't - have to require it. Recovery of masters on one node can overlap - recovery of locks on another node, so one node can send us MSTCPY - locks before we've made ourselves master of this rsb. We can still - add new MSTCPY locks that we receive here without any harm; when - we make ourselves master, dlm_recover_masters() won't touch the - MSTCPY locks we've received early. */ - - error = find_rsb(ls, rl->rl_name, le16_to_cpu(rl->rl_namelen), - from_nodeid, R_RECEIVE_RECOVER, &r); - if (error) - goto out; - - lock_rsb(r); - - if (dlm_no_directory(ls) && (dlm_dir_nodeid(r) != dlm_our_nodeid())) { - log_error(ls, "dlm_recover_master_copy remote %d %x not dir", - from_nodeid, remid); - error = -EBADR; - goto out_unlock; - } - - lkb = search_remid(r, from_nodeid, remid); - if (lkb) { - error = -EEXIST; - goto out_remid; - } - - error = create_lkb(ls, &lkb); - if (error) - goto out_unlock; - - error = receive_rcom_lock_args(ls, lkb, r, rc); - if (error) { - __put_lkb(ls, lkb); - goto out_unlock; - } - - attach_lkb(r, lkb); - add_lkb(r, lkb, rl->rl_status); - error = 0; - ls->ls_recover_locks_in++; - - if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) - rsb_set_flag(r, RSB_RECOVER_GRANT); - - out_remid: - /* this is the new value returned to the lock holder for - saving in its process-copy lkb */ - rl->rl_remid = cpu_to_le32(lkb->lkb_id); - - lkb->lkb_recover_seq = ls->ls_recover_seq; - - out_unlock: - unlock_rsb(r); - put_rsb(r); - out: - if (error && error != -EEXIST) - log_debug(ls, "dlm_recover_master_copy remote %d %x error %d", - from_nodeid, remid, error); - rl->rl_result = cpu_to_le32(error); - return error; -} - -/* needs at least dlm_rcom + rcom_lock */ -int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc) -{ - struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; - struct dlm_rsb *r; - struct dlm_lkb *lkb; - uint32_t lkid, remid; - int error, result; - - lkid = le32_to_cpu(rl->rl_lkid); - remid = le32_to_cpu(rl->rl_remid); - result = le32_to_cpu(rl->rl_result); - - error = find_lkb(ls, lkid, &lkb); - if (error) { - log_error(ls, "dlm_recover_process_copy no %x remote %d %x %d", - lkid, rc->rc_header.h_nodeid, remid, result); - return error; - } - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - if (!is_process_copy(lkb)) { - log_error(ls, "dlm_recover_process_copy bad %x remote %d %x %d", - lkid, rc->rc_header.h_nodeid, remid, result); - dlm_dump_rsb(r); - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - return -EINVAL; - } - - switch (result) { - case -EBADR: - /* There's a chance the new master received our lock before - dlm_recover_master_reply(), this wouldn't happen if we did - a barrier between recover_masters and recover_locks. */ - - log_debug(ls, "dlm_recover_process_copy %x remote %d %x %d", - lkid, rc->rc_header.h_nodeid, remid, result); - - dlm_send_rcom_lock(r, lkb); - goto out; - case -EEXIST: - case 0: - lkb->lkb_remid = remid; - break; - default: - log_error(ls, "dlm_recover_process_copy %x remote %d %x %d unk", - lkid, rc->rc_header.h_nodeid, remid, result); - } - - /* an ack for dlm_recover_locks() which waits for replies from - all the locks it sends to new masters */ - dlm_recovered_lock(r); - out: - unlock_rsb(r); - put_rsb(r); - dlm_put_lkb(lkb); - - return 0; -} - -int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, - int mode, uint32_t flags, void *name, unsigned int namelen, - unsigned long timeout_cs) -{ - struct dlm_lkb *lkb; - struct dlm_args args; - int error; - - dlm_lock_recovery(ls); - - error = create_lkb(ls, &lkb); - if (error) { - kfree(ua); - goto out; - } - - if (flags & DLM_LKF_VALBLK) { - ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); - if (!ua->lksb.sb_lvbptr) { - kfree(ua); - __put_lkb(ls, lkb); - error = -ENOMEM; - goto out; - } - } - - /* After ua is attached to lkb it will be freed by dlm_free_lkb(). - When DLM_IFL_USER is set, the dlm knows that this is a userspace - lock and that lkb_astparam is the dlm_user_args structure. */ - - error = set_lock_args(mode, NULL, &ua->lksb, flags, namelen, - timeout_cs, fake_astfn, ua, fake_bastfn, NULL, - &args); - lkb->lkb_flags |= DLM_IFL_USER; - - if (error) { - __put_lkb(ls, lkb); - goto out; - } - - error = request_lock(ls, lkb, name, namelen, &args); - - switch (error) { - case 0: - break; - case -EINPROGRESS: - error = 0; - break; - case -EAGAIN: - error = 0; - /* fall through */ - default: - __put_lkb(ls, lkb); - goto out; - } - - /* add this new lkb to the per-process list of locks */ - spin_lock(&ua->proc->locks_spin); - hold_lkb(lkb); - list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); - spin_unlock(&ua->proc->locks_spin); - out: - dlm_unlock_recovery(ls); - return error; -} - -int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - int mode, uint32_t flags, uint32_t lkid, char *lvb_in, - unsigned long timeout_cs) -{ - struct dlm_lkb *lkb; - struct dlm_args args; - struct dlm_user_args *ua; - int error; - - dlm_lock_recovery(ls); - - error = find_lkb(ls, lkid, &lkb); - if (error) - goto out; - - /* user can change the params on its lock when it converts it, or - add an lvb that didn't exist before */ - - ua = lkb->lkb_ua; - - if (flags & DLM_LKF_VALBLK && !ua->lksb.sb_lvbptr) { - ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); - if (!ua->lksb.sb_lvbptr) { - error = -ENOMEM; - goto out_put; - } - } - if (lvb_in && ua->lksb.sb_lvbptr) - memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); - - ua->xid = ua_tmp->xid; - ua->castparam = ua_tmp->castparam; - ua->castaddr = ua_tmp->castaddr; - ua->bastparam = ua_tmp->bastparam; - ua->bastaddr = ua_tmp->bastaddr; - ua->user_lksb = ua_tmp->user_lksb; - - error = set_lock_args(mode, NULL, &ua->lksb, flags, 0, timeout_cs, - fake_astfn, ua, fake_bastfn, NULL, &args); - if (error) - goto out_put; - - error = convert_lock(ls, lkb, &args); - - if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK) - error = 0; - out_put: - dlm_put_lkb(lkb); - out: - dlm_unlock_recovery(ls); - kfree(ua_tmp); - return error; -} - -/* - * The caller asks for an orphan lock on a given resource with a given mode. - * If a matching lock exists, it's moved to the owner's list of locks and - * the lkid is returned. - */ - -int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - int mode, uint32_t flags, void *name, unsigned int namelen, - unsigned long timeout_cs, uint32_t *lkid) -{ - struct dlm_lkb *lkb; - struct dlm_user_args *ua; - int found_other_mode = 0; - int found = 0; - int rv = 0; - - mutex_lock(&ls->ls_orphans_mutex); - list_for_each_entry(lkb, &ls->ls_orphans, lkb_ownqueue) { - if (lkb->lkb_resource->res_length != namelen) - continue; - if (memcmp(lkb->lkb_resource->res_name, name, namelen)) - continue; - if (lkb->lkb_grmode != mode) { - found_other_mode = 1; - continue; - } - - found = 1; - list_del_init(&lkb->lkb_ownqueue); - lkb->lkb_flags &= ~DLM_IFL_ORPHAN; - *lkid = lkb->lkb_id; - break; - } - mutex_unlock(&ls->ls_orphans_mutex); - - if (!found && found_other_mode) { - rv = -EAGAIN; - goto out; - } - - if (!found) { - rv = -ENOENT; - goto out; - } - - lkb->lkb_exflags = flags; - lkb->lkb_ownpid = (int) current->pid; - - ua = lkb->lkb_ua; - - ua->proc = ua_tmp->proc; - ua->xid = ua_tmp->xid; - ua->castparam = ua_tmp->castparam; - ua->castaddr = ua_tmp->castaddr; - ua->bastparam = ua_tmp->bastparam; - ua->bastaddr = ua_tmp->bastaddr; - ua->user_lksb = ua_tmp->user_lksb; - - /* - * The lkb reference from the ls_orphans list was not - * removed above, and is now considered the reference - * for the proc locks list. - */ - - spin_lock(&ua->proc->locks_spin); - list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); - spin_unlock(&ua->proc->locks_spin); - out: - kfree(ua_tmp); - return rv; -} - -int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - uint32_t flags, uint32_t lkid, char *lvb_in) -{ - struct dlm_lkb *lkb; - struct dlm_args args; - struct dlm_user_args *ua; - int error; - - dlm_lock_recovery(ls); - - error = find_lkb(ls, lkid, &lkb); - if (error) - goto out; - - ua = lkb->lkb_ua; - - if (lvb_in && ua->lksb.sb_lvbptr) - memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); - if (ua_tmp->castparam) - ua->castparam = ua_tmp->castparam; - ua->user_lksb = ua_tmp->user_lksb; - - error = set_unlock_args(flags, ua, &args); - if (error) - goto out_put; - - error = unlock_lock(ls, lkb, &args); - - if (error == -DLM_EUNLOCK) - error = 0; - /* from validate_unlock_args() */ - if (error == -EBUSY && (flags & DLM_LKF_FORCEUNLOCK)) - error = 0; - if (error) - goto out_put; - - spin_lock(&ua->proc->locks_spin); - /* dlm_user_add_cb() may have already taken lkb off the proc list */ - if (!list_empty(&lkb->lkb_ownqueue)) - list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking); - spin_unlock(&ua->proc->locks_spin); - out_put: - dlm_put_lkb(lkb); - out: - dlm_unlock_recovery(ls); - kfree(ua_tmp); - return error; -} - -int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - uint32_t flags, uint32_t lkid) -{ - struct dlm_lkb *lkb; - struct dlm_args args; - struct dlm_user_args *ua; - int error; - - dlm_lock_recovery(ls); - - error = find_lkb(ls, lkid, &lkb); - if (error) - goto out; - - ua = lkb->lkb_ua; - if (ua_tmp->castparam) - ua->castparam = ua_tmp->castparam; - ua->user_lksb = ua_tmp->user_lksb; - - error = set_unlock_args(flags, ua, &args); - if (error) - goto out_put; - - error = cancel_lock(ls, lkb, &args); - - if (error == -DLM_ECANCEL) - error = 0; - /* from validate_unlock_args() */ - if (error == -EBUSY) - error = 0; - out_put: - dlm_put_lkb(lkb); - out: - dlm_unlock_recovery(ls); - kfree(ua_tmp); - return error; -} - -int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid) -{ - struct dlm_lkb *lkb; - struct dlm_args args; - struct dlm_user_args *ua; - struct dlm_rsb *r; - int error; - - dlm_lock_recovery(ls); - - error = find_lkb(ls, lkid, &lkb); - if (error) - goto out; - - ua = lkb->lkb_ua; - - error = set_unlock_args(flags, ua, &args); - if (error) - goto out_put; - - /* same as cancel_lock(), but set DEADLOCK_CANCEL after lock_rsb */ - - r = lkb->lkb_resource; - hold_rsb(r); - lock_rsb(r); - - error = validate_unlock_args(lkb, &args); - if (error) - goto out_r; - lkb->lkb_flags |= DLM_IFL_DEADLOCK_CANCEL; - - error = _cancel_lock(r, lkb); - out_r: - unlock_rsb(r); - put_rsb(r); - - if (error == -DLM_ECANCEL) - error = 0; - /* from validate_unlock_args() */ - if (error == -EBUSY) - error = 0; - out_put: - dlm_put_lkb(lkb); - out: - dlm_unlock_recovery(ls); - return error; -} - -/* lkb's that are removed from the waiters list by revert are just left on the - orphans list with the granted orphan locks, to be freed by purge */ - -static int orphan_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) -{ - struct dlm_args args; - int error; - - hold_lkb(lkb); /* reference for the ls_orphans list */ - mutex_lock(&ls->ls_orphans_mutex); - list_add_tail(&lkb->lkb_ownqueue, &ls->ls_orphans); - mutex_unlock(&ls->ls_orphans_mutex); - - set_unlock_args(0, lkb->lkb_ua, &args); - - error = cancel_lock(ls, lkb, &args); - if (error == -DLM_ECANCEL) - error = 0; - return error; -} - -/* The FORCEUNLOCK flag allows the unlock to go ahead even if the lkb isn't - granted. Regardless of what rsb queue the lock is on, it's removed and - freed. The IVVALBLK flag causes the lvb on the resource to be invalidated - if our lock is PW/EX (it's ignored if our granted mode is smaller.) */ - -static int unlock_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) -{ - struct dlm_args args; - int error; - - set_unlock_args(DLM_LKF_FORCEUNLOCK | DLM_LKF_IVVALBLK, - lkb->lkb_ua, &args); - - error = unlock_lock(ls, lkb, &args); - if (error == -DLM_EUNLOCK) - error = 0; - return error; -} - -/* We have to release clear_proc_locks mutex before calling unlock_proc_lock() - (which does lock_rsb) due to deadlock with receiving a message that does - lock_rsb followed by dlm_user_add_cb() */ - -static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls, - struct dlm_user_proc *proc) -{ - struct dlm_lkb *lkb = NULL; - - mutex_lock(&ls->ls_clear_proc_locks); - if (list_empty(&proc->locks)) - goto out; - - lkb = list_entry(proc->locks.next, struct dlm_lkb, lkb_ownqueue); - list_del_init(&lkb->lkb_ownqueue); - - if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) - lkb->lkb_flags |= DLM_IFL_ORPHAN; - else - lkb->lkb_flags |= DLM_IFL_DEAD; - out: - mutex_unlock(&ls->ls_clear_proc_locks); - return lkb; -} - -/* The ls_clear_proc_locks mutex protects against dlm_user_add_cb() which - 1) references lkb->ua which we free here and 2) adds lkbs to proc->asts, - which we clear here. */ - -/* proc CLOSING flag is set so no more device_reads should look at proc->asts - list, and no more device_writes should add lkb's to proc->locks list; so we - shouldn't need to take asts_spin or locks_spin here. this assumes that - device reads/writes/closes are serialized -- FIXME: we may need to serialize - them ourself. */ - -void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) -{ - struct dlm_lkb *lkb, *safe; - - dlm_lock_recovery(ls); - - while (1) { - lkb = del_proc_lock(ls, proc); - if (!lkb) - break; - del_timeout(lkb); - if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) - orphan_proc_lock(ls, lkb); - else - unlock_proc_lock(ls, lkb); - - /* this removes the reference for the proc->locks list - added by dlm_user_request, it may result in the lkb - being freed */ - - dlm_put_lkb(lkb); - } - - mutex_lock(&ls->ls_clear_proc_locks); - - /* in-progress unlocks */ - list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { - list_del_init(&lkb->lkb_ownqueue); - lkb->lkb_flags |= DLM_IFL_DEAD; - dlm_put_lkb(lkb); - } - - list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_cb_list) { - memset(&lkb->lkb_callbacks, 0, - sizeof(struct dlm_callback) * DLM_CALLBACKS_SIZE); - list_del_init(&lkb->lkb_cb_list); - dlm_put_lkb(lkb); - } - - mutex_unlock(&ls->ls_clear_proc_locks); - dlm_unlock_recovery(ls); -} - -static void purge_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) -{ - struct dlm_lkb *lkb, *safe; - - while (1) { - lkb = NULL; - spin_lock(&proc->locks_spin); - if (!list_empty(&proc->locks)) { - lkb = list_entry(proc->locks.next, struct dlm_lkb, - lkb_ownqueue); - list_del_init(&lkb->lkb_ownqueue); - } - spin_unlock(&proc->locks_spin); - - if (!lkb) - break; - - lkb->lkb_flags |= DLM_IFL_DEAD; - unlock_proc_lock(ls, lkb); - dlm_put_lkb(lkb); /* ref from proc->locks list */ - } - - spin_lock(&proc->locks_spin); - list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { - list_del_init(&lkb->lkb_ownqueue); - lkb->lkb_flags |= DLM_IFL_DEAD; - dlm_put_lkb(lkb); - } - spin_unlock(&proc->locks_spin); - - spin_lock(&proc->asts_spin); - list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_cb_list) { - memset(&lkb->lkb_callbacks, 0, - sizeof(struct dlm_callback) * DLM_CALLBACKS_SIZE); - list_del_init(&lkb->lkb_cb_list); - dlm_put_lkb(lkb); - } - spin_unlock(&proc->asts_spin); -} - -/* pid of 0 means purge all orphans */ - -static void do_purge(struct dlm_ls *ls, int nodeid, int pid) -{ - struct dlm_lkb *lkb, *safe; - - mutex_lock(&ls->ls_orphans_mutex); - list_for_each_entry_safe(lkb, safe, &ls->ls_orphans, lkb_ownqueue) { - if (pid && lkb->lkb_ownpid != pid) - continue; - unlock_proc_lock(ls, lkb); - list_del_init(&lkb->lkb_ownqueue); - dlm_put_lkb(lkb); - } - mutex_unlock(&ls->ls_orphans_mutex); -} - -static int send_purge(struct dlm_ls *ls, int nodeid, int pid) -{ - struct dlm_message *ms; - struct dlm_mhandle *mh; - int error; - - error = _create_message(ls, sizeof(struct dlm_message), nodeid, - DLM_MSG_PURGE, &ms, &mh); - if (error) - return error; - ms->m_nodeid = nodeid; - ms->m_pid = pid; - - return send_message(mh, ms); -} - -int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, - int nodeid, int pid) -{ - int error = 0; - - if (nodeid && (nodeid != dlm_our_nodeid())) { - error = send_purge(ls, nodeid, pid); - } else { - dlm_lock_recovery(ls); - if (pid == current->pid) - purge_proc_locks(ls, proc); - else - do_purge(ls, nodeid, pid); - dlm_unlock_recovery(ls); - } - return error; -} - diff --git a/kmod/dlm/lock.h b/kmod/dlm/lock.h deleted file mode 100644 index 271736a4..00000000 --- a/kmod/dlm/lock.h +++ /dev/null @@ -1,82 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __LOCK_DOT_H__ -#define __LOCK_DOT_H__ - -void dlm_dump_rsb(struct dlm_rsb *r); -void dlm_dump_rsb_name(struct dlm_ls *ls, char *name, int len); -void dlm_print_lkb(struct dlm_lkb *lkb); -void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms, - uint32_t saved_seq); -void dlm_receive_buffer(union dlm_packet *p, int nodeid); -int dlm_modes_compat(int mode1, int mode2); -void dlm_put_rsb(struct dlm_rsb *r); -void dlm_hold_rsb(struct dlm_rsb *r); -int dlm_put_lkb(struct dlm_lkb *lkb); -void dlm_scan_rsbs(struct dlm_ls *ls); -int dlm_lock_recovery_try(struct dlm_ls *ls); -void dlm_unlock_recovery(struct dlm_ls *ls); -void dlm_scan_waiters(struct dlm_ls *ls); -void dlm_scan_timeout(struct dlm_ls *ls); -void dlm_adjust_timeouts(struct dlm_ls *ls); -int dlm_master_lookup(struct dlm_ls *ls, int nodeid, char *name, int len, - unsigned int flags, int *r_nodeid, int *result); - -int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, - struct dlm_rsb **r_ret); - -void dlm_recover_purge(struct dlm_ls *ls); -void dlm_purge_mstcpy_locks(struct dlm_rsb *r); -void dlm_recover_grant(struct dlm_ls *ls); -int dlm_recover_waiters_post(struct dlm_ls *ls); -void dlm_recover_waiters_pre(struct dlm_ls *ls); -int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc); -int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc); - -int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, int mode, - uint32_t flags, void *name, unsigned int namelen, - unsigned long timeout_cs); -int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - int mode, uint32_t flags, uint32_t lkid, char *lvb_in, - unsigned long timeout_cs); -int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - int mode, uint32_t flags, void *name, unsigned int namelen, - unsigned long timeout_cs, uint32_t *lkid); -int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - uint32_t flags, uint32_t lkid, char *lvb_in); -int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, - uint32_t flags, uint32_t lkid); -int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, - int nodeid, int pid); -int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid); -void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc); - -static inline int is_master(struct dlm_rsb *r) -{ - return !r->res_nodeid; -} - -static inline void lock_rsb(struct dlm_rsb *r) -{ - mutex_lock(&r->res_mutex); -} - -static inline void unlock_rsb(struct dlm_rsb *r) -{ - mutex_unlock(&r->res_mutex); -} - -int ranges_overlap(struct dlm_range *range1, struct dlm_range *range2); - -#endif - diff --git a/kmod/dlm/lockspace.c b/kmod/dlm/lockspace.c deleted file mode 100644 index 84cb7210..00000000 --- a/kmod/dlm/lockspace.c +++ /dev/null @@ -1,906 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "member.h" -#include "recoverd.h" -#include "dir.h" -#include "lowcomms.h" -#include "config.h" -#include "memory.h" -#include "lock.h" -#include "recover.h" -#include "requestqueue.h" -#include "user.h" -#include "ast.h" - -static int ls_count; -static struct mutex ls_lock; -static struct list_head lslist; -static spinlock_t lslist_lock; -static struct task_struct * scand_task; - - -static ssize_t dlm_control_store(struct dlm_ls *ls, const char *buf, size_t len) -{ - ssize_t ret = len; - int n = simple_strtol(buf, NULL, 0); - - ls = dlm_find_lockspace_local(ls->ls_local_handle); - if (!ls) - return -EINVAL; - - switch (n) { - case 0: - dlm_ls_stop(ls); - break; - case 1: - dlm_ls_start(ls); - break; - default: - ret = -EINVAL; - } - dlm_put_lockspace(ls); - return ret; -} - -static ssize_t dlm_event_store(struct dlm_ls *ls, const char *buf, size_t len) -{ - ls->ls_uevent_result = simple_strtol(buf, NULL, 0); - set_bit(LSFL_UEVENT_WAIT, &ls->ls_flags); - wake_up(&ls->ls_uevent_wait); - return len; -} - -static ssize_t dlm_id_show(struct dlm_ls *ls, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%u\n", ls->ls_global_id); -} - -static ssize_t dlm_id_store(struct dlm_ls *ls, const char *buf, size_t len) -{ - ls->ls_global_id = simple_strtoul(buf, NULL, 0); - return len; -} - -static ssize_t dlm_nodir_show(struct dlm_ls *ls, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%u\n", dlm_no_directory(ls)); -} - -static ssize_t dlm_nodir_store(struct dlm_ls *ls, const char *buf, size_t len) -{ - int val = simple_strtoul(buf, NULL, 0); - if (val == 1) - set_bit(LSFL_NODIR, &ls->ls_flags); - return len; -} - -static ssize_t dlm_recover_status_show(struct dlm_ls *ls, char *buf) -{ - uint32_t status = dlm_recover_status(ls); - return snprintf(buf, PAGE_SIZE, "%x\n", status); -} - -static ssize_t dlm_recover_nodeid_show(struct dlm_ls *ls, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%d\n", ls->ls_recover_nodeid); -} - -struct dlm_attr { - struct attribute attr; - ssize_t (*show)(struct dlm_ls *, char *); - ssize_t (*store)(struct dlm_ls *, const char *, size_t); -}; - -static struct dlm_attr dlm_attr_control = { - .attr = {.name = "control", .mode = S_IWUSR}, - .store = dlm_control_store -}; - -static struct dlm_attr dlm_attr_event = { - .attr = {.name = "event_done", .mode = S_IWUSR}, - .store = dlm_event_store -}; - -static struct dlm_attr dlm_attr_id = { - .attr = {.name = "id", .mode = S_IRUGO | S_IWUSR}, - .show = dlm_id_show, - .store = dlm_id_store -}; - -static struct dlm_attr dlm_attr_nodir = { - .attr = {.name = "nodir", .mode = S_IRUGO | S_IWUSR}, - .show = dlm_nodir_show, - .store = dlm_nodir_store -}; - -static struct dlm_attr dlm_attr_recover_status = { - .attr = {.name = "recover_status", .mode = S_IRUGO}, - .show = dlm_recover_status_show -}; - -static struct dlm_attr dlm_attr_recover_nodeid = { - .attr = {.name = "recover_nodeid", .mode = S_IRUGO}, - .show = dlm_recover_nodeid_show -}; - -static struct attribute *dlm_attrs[] = { - &dlm_attr_control.attr, - &dlm_attr_event.attr, - &dlm_attr_id.attr, - &dlm_attr_nodir.attr, - &dlm_attr_recover_status.attr, - &dlm_attr_recover_nodeid.attr, - NULL, -}; - -static ssize_t dlm_attr_show(struct kobject *kobj, struct attribute *attr, - char *buf) -{ - struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); - struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); - return a->show ? a->show(ls, buf) : 0; -} - -static ssize_t dlm_attr_store(struct kobject *kobj, struct attribute *attr, - const char *buf, size_t len) -{ - struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); - struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); - return a->store ? a->store(ls, buf, len) : len; -} - -static void lockspace_kobj_release(struct kobject *k) -{ - struct dlm_ls *ls = container_of(k, struct dlm_ls, ls_kobj); - kfree(ls); -} - -static const struct sysfs_ops dlm_attr_ops = { - .show = dlm_attr_show, - .store = dlm_attr_store, -}; - -static struct kobj_type dlm_ktype = { - .default_attrs = dlm_attrs, - .sysfs_ops = &dlm_attr_ops, - .release = lockspace_kobj_release, -}; - -static struct kset *dlm_kset; - -static int do_uevent(struct dlm_ls *ls, int in) -{ - int error; - - if (in) - kobject_uevent(&ls->ls_kobj, KOBJ_ONLINE); - else - kobject_uevent(&ls->ls_kobj, KOBJ_OFFLINE); - - log_debug(ls, "%s the lockspace group...", in ? "joining" : "leaving"); - - /* dlm_controld will see the uevent, do the necessary group management - and then write to sysfs to wake us */ - - error = wait_event_interruptible(ls->ls_uevent_wait, - test_and_clear_bit(LSFL_UEVENT_WAIT, &ls->ls_flags)); - - log_debug(ls, "group event done %d %d", error, ls->ls_uevent_result); - - if (error) - goto out; - - error = ls->ls_uevent_result; - out: - if (error) - log_error(ls, "group %s failed %d %d", in ? "join" : "leave", - error, ls->ls_uevent_result); - return error; -} - -static int dlm_uevent(struct kset *kset, struct kobject *kobj, - struct kobj_uevent_env *env) -{ - struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); - - add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name); - return 0; -} - -static struct kset_uevent_ops dlm_uevent_ops = { - .uevent = dlm_uevent, -}; - -int __init dlm_lockspace_init(void) -{ - ls_count = 0; - mutex_init(&ls_lock); - INIT_LIST_HEAD(&lslist); - spin_lock_init(&lslist_lock); - - dlm_kset = kset_create_and_add("dlm", &dlm_uevent_ops, kernel_kobj); - if (!dlm_kset) { - printk(KERN_WARNING "%s: can not create kset\n", __func__); - return -ENOMEM; - } - return 0; -} - -void dlm_lockspace_exit(void) -{ - kset_unregister(dlm_kset); -} - -static struct dlm_ls *find_ls_to_scan(void) -{ - struct dlm_ls *ls; - - spin_lock(&lslist_lock); - list_for_each_entry(ls, &lslist, ls_list) { - if (time_after_eq(jiffies, ls->ls_scan_time + - dlm_config.ci_scan_secs * HZ)) { - spin_unlock(&lslist_lock); - return ls; - } - } - spin_unlock(&lslist_lock); - return NULL; -} - -static int dlm_scand(void *data) -{ - struct dlm_ls *ls; - - while (!kthread_should_stop()) { - ls = find_ls_to_scan(); - if (ls) { - if (dlm_lock_recovery_try(ls)) { - ls->ls_scan_time = jiffies; - dlm_scan_rsbs(ls); - dlm_scan_timeout(ls); - dlm_scan_waiters(ls); - dlm_unlock_recovery(ls); - } else { - ls->ls_scan_time += HZ; - } - continue; - } - schedule_timeout_interruptible(dlm_config.ci_scan_secs * HZ); - } - return 0; -} - -static int dlm_scand_start(void) -{ - struct task_struct *p; - int error = 0; - - p = kthread_run(dlm_scand, NULL, "dlm_scand"); - if (IS_ERR(p)) - error = PTR_ERR(p); - else - scand_task = p; - return error; -} - -static void dlm_scand_stop(void) -{ - kthread_stop(scand_task); -} - -struct dlm_ls *dlm_find_lockspace_global(uint32_t id) -{ - struct dlm_ls *ls; - - spin_lock(&lslist_lock); - - list_for_each_entry(ls, &lslist, ls_list) { - if (ls->ls_global_id == id) { - ls->ls_count++; - goto out; - } - } - ls = NULL; - out: - spin_unlock(&lslist_lock); - return ls; -} - -struct dlm_ls *dlm_find_lockspace_local(dlm_lockspace_t *lockspace) -{ - struct dlm_ls *ls; - - spin_lock(&lslist_lock); - list_for_each_entry(ls, &lslist, ls_list) { - if (ls->ls_local_handle == lockspace) { - ls->ls_count++; - goto out; - } - } - ls = NULL; - out: - spin_unlock(&lslist_lock); - return ls; -} - -struct dlm_ls *dlm_find_lockspace_device(int minor) -{ - struct dlm_ls *ls; - - spin_lock(&lslist_lock); - list_for_each_entry(ls, &lslist, ls_list) { - if (ls->ls_device.minor == minor) { - ls->ls_count++; - goto out; - } - } - ls = NULL; - out: - spin_unlock(&lslist_lock); - return ls; -} - -void dlm_put_lockspace(struct dlm_ls *ls) -{ - spin_lock(&lslist_lock); - ls->ls_count--; - spin_unlock(&lslist_lock); -} - -static void remove_lockspace(struct dlm_ls *ls) -{ - for (;;) { - spin_lock(&lslist_lock); - if (ls->ls_count == 0) { - WARN_ON(ls->ls_create_count != 0); - list_del(&ls->ls_list); - spin_unlock(&lslist_lock); - return; - } - spin_unlock(&lslist_lock); - ssleep(1); - } -} - -static int threads_start(void) -{ - int error; - - error = dlm_scand_start(); - if (error) { - log_print("cannot start dlm_scand thread %d", error); - goto fail; - } - - /* Thread for sending/receiving messages for all lockspace's */ - error = dlm_lowcomms_start(); - if (error) { - log_print("cannot start dlm lowcomms %d", error); - goto scand_fail; - } - - return 0; - - scand_fail: - dlm_scand_stop(); - fail: - return error; -} - -static void threads_stop(void) -{ - dlm_scand_stop(); - dlm_lowcomms_stop(); -} - -static int new_lockspace(const char *name, const char *cluster, - uint32_t flags, int lvblen, - const struct dlm_lockspace_ops *ops, void *ops_arg, - int *ops_result, dlm_lockspace_t **lockspace) -{ - struct dlm_ls *ls; - int i, size, error; - int do_unreg = 0; - int namelen = strlen(name); - - if (namelen > DLM_LOCKSPACE_LEN) - return -EINVAL; - - if (!lvblen || (lvblen % 8)) - return -EINVAL; - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - - if (!dlm_user_daemon_available()) { - log_print("dlm user daemon not available"); - error = -EUNATCH; - goto out; - } - - if (ops && ops_result) { - if (!dlm_config.ci_recover_callbacks) - *ops_result = -EOPNOTSUPP; - else - *ops_result = 0; - } - - if (dlm_config.ci_recover_callbacks && cluster && - strncmp(cluster, dlm_config.ci_cluster_name, DLM_LOCKSPACE_LEN)) { - log_print("dlm cluster name %s mismatch %s", - dlm_config.ci_cluster_name, cluster); - error = -EBADR; - goto out; - } - - error = 0; - - spin_lock(&lslist_lock); - list_for_each_entry(ls, &lslist, ls_list) { - WARN_ON(ls->ls_create_count <= 0); - if (ls->ls_namelen != namelen) - continue; - if (memcmp(ls->ls_name, name, namelen)) - continue; - if (flags & DLM_LSFL_NEWEXCL) { - error = -EEXIST; - break; - } - ls->ls_create_count++; - *lockspace = ls; - error = 1; - break; - } - spin_unlock(&lslist_lock); - - if (error) - goto out; - - error = -ENOMEM; - - ls = kzalloc(sizeof(struct dlm_ls) + namelen, GFP_NOFS); - if (!ls) - goto out; - memcpy(ls->ls_name, name, namelen); - ls->ls_namelen = namelen; - ls->ls_lvblen = lvblen; - ls->ls_count = 0; - ls->ls_flags = 0; - ls->ls_scan_time = jiffies; - - if (ops && dlm_config.ci_recover_callbacks) { - ls->ls_ops = ops; - ls->ls_ops_arg = ops_arg; - } - - if (flags & DLM_LSFL_TIMEWARN) - set_bit(LSFL_TIMEWARN, &ls->ls_flags); - - /* ls_exflags are forced to match among nodes, and we don't - need to require all nodes to have some flags set */ - ls->ls_exflags = (flags & ~(DLM_LSFL_TIMEWARN | DLM_LSFL_FS | - DLM_LSFL_NEWEXCL)); - - size = dlm_config.ci_rsbtbl_size; - ls->ls_rsbtbl_size = size; - - ls->ls_rsbtbl = vmalloc(sizeof(struct dlm_rsbtable) * size); - if (!ls->ls_rsbtbl) - goto out_lsfree; - for (i = 0; i < size; i++) { - ls->ls_rsbtbl[i].keep.rb_node = NULL; - ls->ls_rsbtbl[i].toss.rb_node = NULL; - spin_lock_init(&ls->ls_rsbtbl[i].lock); - } - - spin_lock_init(&ls->ls_remove_spin); - - for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) { - ls->ls_remove_names[i] = kzalloc(DLM_RESNAME_MAXLEN+1, - GFP_KERNEL); - if (!ls->ls_remove_names[i]) - goto out_rsbtbl; - } - - idr_init(&ls->ls_lkbidr); - spin_lock_init(&ls->ls_lkbidr_spin); - - INIT_LIST_HEAD(&ls->ls_waiters); - mutex_init(&ls->ls_waiters_mutex); - INIT_LIST_HEAD(&ls->ls_orphans); - mutex_init(&ls->ls_orphans_mutex); - INIT_LIST_HEAD(&ls->ls_timeout); - mutex_init(&ls->ls_timeout_mutex); - - INIT_LIST_HEAD(&ls->ls_new_rsb); - spin_lock_init(&ls->ls_new_rsb_spin); - - INIT_LIST_HEAD(&ls->ls_nodes); - INIT_LIST_HEAD(&ls->ls_nodes_gone); - ls->ls_num_nodes = 0; - ls->ls_low_nodeid = 0; - ls->ls_total_weight = 0; - ls->ls_node_array = NULL; - - memset(&ls->ls_stub_rsb, 0, sizeof(struct dlm_rsb)); - ls->ls_stub_rsb.res_ls = ls; - - ls->ls_debug_rsb_dentry = NULL; - ls->ls_debug_waiters_dentry = NULL; - - init_waitqueue_head(&ls->ls_uevent_wait); - ls->ls_uevent_result = 0; - init_completion(&ls->ls_members_done); - ls->ls_members_result = -1; - - mutex_init(&ls->ls_cb_mutex); - INIT_LIST_HEAD(&ls->ls_cb_delay); - - ls->ls_recoverd_task = NULL; - mutex_init(&ls->ls_recoverd_active); - spin_lock_init(&ls->ls_recover_lock); - spin_lock_init(&ls->ls_rcom_spin); - get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t)); - ls->ls_recover_status = 0; - ls->ls_recover_seq = 0; - ls->ls_recover_args = NULL; - init_rwsem(&ls->ls_in_recovery); - init_rwsem(&ls->ls_recv_active); - INIT_LIST_HEAD(&ls->ls_requestqueue); - mutex_init(&ls->ls_requestqueue_mutex); - mutex_init(&ls->ls_clear_proc_locks); - - ls->ls_recover_buf = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS); - if (!ls->ls_recover_buf) - goto out_lkbidr; - - ls->ls_slot = 0; - ls->ls_num_slots = 0; - ls->ls_slots_size = 0; - ls->ls_slots = NULL; - - INIT_LIST_HEAD(&ls->ls_recover_list); - spin_lock_init(&ls->ls_recover_list_lock); - idr_init(&ls->ls_recover_idr); - spin_lock_init(&ls->ls_recover_idr_lock); - ls->ls_recover_list_count = 0; - ls->ls_local_handle = ls; - init_waitqueue_head(&ls->ls_wait_general); - INIT_LIST_HEAD(&ls->ls_root_list); - init_rwsem(&ls->ls_root_sem); - - spin_lock(&lslist_lock); - ls->ls_create_count = 1; - list_add(&ls->ls_list, &lslist); - spin_unlock(&lslist_lock); - - if (flags & DLM_LSFL_FS) { - error = dlm_callback_start(ls); - if (error) { - log_error(ls, "can't start dlm_callback %d", error); - goto out_delist; - } - } - - init_waitqueue_head(&ls->ls_recover_lock_wait); - - /* - * Once started, dlm_recoverd first looks for ls in lslist, then - * initializes ls_in_recovery as locked in "down" mode. We need - * to wait for the wakeup from dlm_recoverd because in_recovery - * has to start out in down mode. - */ - - error = dlm_recoverd_start(ls); - if (error) { - log_error(ls, "can't start dlm_recoverd %d", error); - goto out_callback; - } - - wait_event(ls->ls_recover_lock_wait, - test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); - - ls->ls_kobj.kset = dlm_kset; - error = kobject_init_and_add(&ls->ls_kobj, &dlm_ktype, NULL, - "%s", ls->ls_name); - if (error) - goto out_recoverd; - kobject_uevent(&ls->ls_kobj, KOBJ_ADD); - - /* let kobject handle freeing of ls if there's an error */ - do_unreg = 1; - - /* This uevent triggers dlm_controld in userspace to add us to the - group of nodes that are members of this lockspace (managed by the - cluster infrastructure.) Once it's done that, it tells us who the - current lockspace members are (via configfs) and then tells the - lockspace to start running (via sysfs) in dlm_ls_start(). */ - - error = do_uevent(ls, 1); - if (error) - goto out_recoverd; - wait_for_completion(&ls->ls_members_done); - error = ls->ls_members_result; - - if (error) - goto out_members; - - dlm_create_debug_file(ls); - - log_debug(ls, "join complete"); - *lockspace = ls; - return 0; - - out_members: - do_uevent(ls, 0); - dlm_clear_members(ls); - kfree(ls->ls_node_array); - out_recoverd: - dlm_recoverd_stop(ls); - out_callback: - dlm_callback_stop(ls); - out_delist: - spin_lock(&lslist_lock); - list_del(&ls->ls_list); - spin_unlock(&lslist_lock); - idr_destroy(&ls->ls_recover_idr); - kfree(ls->ls_recover_buf); - out_lkbidr: - idr_destroy(&ls->ls_lkbidr); - for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) { - if (ls->ls_remove_names[i]) - kfree(ls->ls_remove_names[i]); - } - out_rsbtbl: - vfree(ls->ls_rsbtbl); - out_lsfree: - if (do_unreg) - kobject_put(&ls->ls_kobj); - else - kfree(ls); - out: - module_put(THIS_MODULE); - return error; -} - -int dlm_new_lockspace(const char *name, const char *cluster, - uint32_t flags, int lvblen, - const struct dlm_lockspace_ops *ops, void *ops_arg, - int *ops_result, dlm_lockspace_t **lockspace) -{ - int error = 0; - - mutex_lock(&ls_lock); - if (!ls_count) - error = threads_start(); - if (error) - goto out; - - error = new_lockspace(name, cluster, flags, lvblen, ops, ops_arg, - ops_result, lockspace); - if (!error) - ls_count++; - if (error > 0) - error = 0; - if (!ls_count) - threads_stop(); - out: - mutex_unlock(&ls_lock); - return error; -} - -static int lkb_idr_is_local(int id, void *p, void *data) -{ - struct dlm_lkb *lkb = p; - - if (!lkb->lkb_nodeid) - return 1; - return 0; -} - -static int lkb_idr_is_any(int id, void *p, void *data) -{ - return 1; -} - -static int lkb_idr_free(int id, void *p, void *data) -{ - struct dlm_lkb *lkb = p; - - if (lkb->lkb_lvbptr && lkb->lkb_flags & DLM_IFL_MSTCPY) - dlm_free_lvb(lkb->lkb_lvbptr); - - dlm_free_lkb(lkb); - return 0; -} - -/* NOTE: We check the lkbidr here rather than the resource table. - This is because there may be LKBs queued as ASTs that have been unlinked - from their RSBs and are pending deletion once the AST has been delivered */ - -static int lockspace_busy(struct dlm_ls *ls, int force) -{ - int rv; - - spin_lock(&ls->ls_lkbidr_spin); - if (force == 0) { - rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_any, ls); - } else if (force == 1) { - rv = idr_for_each(&ls->ls_lkbidr, lkb_idr_is_local, ls); - } else { - rv = 0; - } - spin_unlock(&ls->ls_lkbidr_spin); - return rv; -} - -static int release_lockspace(struct dlm_ls *ls, int force) -{ - struct dlm_rsb *rsb; - struct rb_node *n; - int i, busy, rv; - - busy = lockspace_busy(ls, force); - - spin_lock(&lslist_lock); - if (ls->ls_create_count == 1) { - if (busy) { - rv = -EBUSY; - } else { - /* remove_lockspace takes ls off lslist */ - ls->ls_create_count = 0; - rv = 0; - } - } else if (ls->ls_create_count > 1) { - rv = --ls->ls_create_count; - } else { - rv = -EINVAL; - } - spin_unlock(&lslist_lock); - - if (rv) { - log_debug(ls, "release_lockspace no remove %d", rv); - return rv; - } - - dlm_device_deregister(ls); - - if (force < 3 && dlm_user_daemon_available()) - do_uevent(ls, 0); - - dlm_recoverd_stop(ls); - - dlm_callback_stop(ls); - - remove_lockspace(ls); - - dlm_delete_debug_file(ls); - - kfree(ls->ls_recover_buf); - - /* - * Free all lkb's in idr - */ - - idr_for_each(&ls->ls_lkbidr, lkb_idr_free, ls); - idr_destroy(&ls->ls_lkbidr); - - /* - * Free all rsb's on rsbtbl[] lists - */ - - for (i = 0; i < ls->ls_rsbtbl_size; i++) { - while ((n = rb_first(&ls->ls_rsbtbl[i].keep))) { - rsb = rb_entry(n, struct dlm_rsb, res_hashnode); - rb_erase(n, &ls->ls_rsbtbl[i].keep); - dlm_free_rsb(rsb); - } - - while ((n = rb_first(&ls->ls_rsbtbl[i].toss))) { - rsb = rb_entry(n, struct dlm_rsb, res_hashnode); - rb_erase(n, &ls->ls_rsbtbl[i].toss); - dlm_free_rsb(rsb); - } - } - - vfree(ls->ls_rsbtbl); - - for (i = 0; i < DLM_REMOVE_NAMES_MAX; i++) - kfree(ls->ls_remove_names[i]); - - while (!list_empty(&ls->ls_new_rsb)) { - rsb = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, - res_hashchain); - list_del(&rsb->res_hashchain); - dlm_free_rsb(rsb); - } - - /* - * Free structures on any other lists - */ - - dlm_purge_requestqueue(ls); - kfree(ls->ls_recover_args); - dlm_clear_members(ls); - dlm_clear_members_gone(ls); - kfree(ls->ls_node_array); - log_debug(ls, "release_lockspace final free"); - kobject_put(&ls->ls_kobj); - /* The ls structure will be freed when the kobject is done with */ - - module_put(THIS_MODULE); - return 0; -} - -/* - * Called when a system has released all its locks and is not going to use the - * lockspace any longer. We free everything we're managing for this lockspace. - * Remaining nodes will go through the recovery process as if we'd died. The - * lockspace must continue to function as usual, participating in recoveries, - * until this returns. - * - * Force has 4 possible values: - * 0 - don't destroy locksapce if it has any LKBs - * 1 - destroy lockspace if it has remote LKBs but not if it has local LKBs - * 2 - destroy lockspace regardless of LKBs - * 3 - destroy lockspace as part of a forced shutdown - */ - -int dlm_release_lockspace(void *lockspace, int force) -{ - struct dlm_ls *ls; - int error; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - dlm_put_lockspace(ls); - - mutex_lock(&ls_lock); - error = release_lockspace(ls, force); - if (!error) - ls_count--; - if (!ls_count) - threads_stop(); - mutex_unlock(&ls_lock); - - return error; -} - -void dlm_stop_lockspaces(void) -{ - struct dlm_ls *ls; - int count; - - restart: - count = 0; - spin_lock(&lslist_lock); - list_for_each_entry(ls, &lslist, ls_list) { - if (!test_bit(LSFL_RUNNING, &ls->ls_flags)) { - count++; - continue; - } - spin_unlock(&lslist_lock); - log_error(ls, "no userland control daemon, stopping lockspace"); - dlm_ls_stop(ls); - goto restart; - } - spin_unlock(&lslist_lock); - - if (count) - log_print("dlm user daemon left %d lockspaces", count); -} - diff --git a/kmod/dlm/lockspace.h b/kmod/dlm/lockspace.h deleted file mode 100644 index f879f879..00000000 --- a/kmod/dlm/lockspace.h +++ /dev/null @@ -1,26 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __LOCKSPACE_DOT_H__ -#define __LOCKSPACE_DOT_H__ - -int dlm_lockspace_init(void); -void dlm_lockspace_exit(void); -struct dlm_ls *dlm_find_lockspace_global(uint32_t id); -struct dlm_ls *dlm_find_lockspace_local(void *id); -struct dlm_ls *dlm_find_lockspace_device(int minor); -void dlm_put_lockspace(struct dlm_ls *ls); -void dlm_stop_lockspaces(void); - -#endif /* __LOCKSPACE_DOT_H__ */ - diff --git a/kmod/dlm/lowcomms.c b/kmod/dlm/lowcomms.c deleted file mode 100644 index d0ccd2fd..00000000 --- a/kmod/dlm/lowcomms.c +++ /dev/null @@ -1,1726 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -/* - * lowcomms.c - * - * This is the "low-level" comms layer. - * - * It is responsible for sending/receiving messages - * from other nodes in the cluster. - * - * Cluster nodes are referred to by their nodeids. nodeids are - * simply 32 bit numbers to the locking module - if they need to - * be expanded for the cluster infrastructure then that is its - * responsibility. It is this layer's - * responsibility to resolve these into IP address or - * whatever it needs for inter-node communication. - * - * The comms level is two kernel threads that deal mainly with - * the receiving of messages from other nodes and passing them - * up to the mid-level comms layer (which understands the - * message format) for execution by the locking core, and - * a send thread which does all the setting up of connections - * to remote nodes and the sending of data. Threads are not allowed - * to send their own data because it may cause them to wait in times - * of high load. Also, this way, the sending thread can collect together - * messages bound for one node and send them in one block. - * - * lowcomms will choose to use either TCP or SCTP as its transport layer - * depending on the configuration variable 'protocol'. This should be set - * to 0 (default) for TCP or 1 for SCTP. It should be configured using a - * cluster-wide mechanism as it must be the same on all nodes of the cluster - * for the DLM to function. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dlm_internal.h" -#include "lowcomms.h" -#include "midcomms.h" -#include "config.h" - -#define NEEDED_RMEM (4*1024*1024) -#define CONN_HASH_SIZE 32 - -/* Number of messages to send before rescheduling */ -#define MAX_SEND_MSG_COUNT 25 - -struct cbuf { - unsigned int base; - unsigned int len; - unsigned int mask; -}; - -static void cbuf_add(struct cbuf *cb, int n) -{ - cb->len += n; -} - -static int cbuf_data(struct cbuf *cb) -{ - return ((cb->base + cb->len) & cb->mask); -} - -static void cbuf_init(struct cbuf *cb, int size) -{ - cb->base = cb->len = 0; - cb->mask = size-1; -} - -static void cbuf_eat(struct cbuf *cb, int n) -{ - cb->len -= n; - cb->base += n; - cb->base &= cb->mask; -} - -static bool cbuf_empty(struct cbuf *cb) -{ - return cb->len == 0; -} - -struct connection { - struct socket *sock; /* NULL if not connected */ - uint32_t nodeid; /* So we know who we are in the list */ - struct mutex sock_mutex; - unsigned long flags; -#define CF_READ_PENDING 1 -#define CF_WRITE_PENDING 2 -#define CF_CONNECT_PENDING 3 -#define CF_INIT_PENDING 4 -#define CF_IS_OTHERCON 5 -#define CF_CLOSE 6 -#define CF_APP_LIMITED 7 - struct list_head writequeue; /* List of outgoing writequeue_entries */ - spinlock_t writequeue_lock; - int (*rx_action) (struct connection *); /* What to do when active */ - void (*connect_action) (struct connection *); /* What to do to connect */ - struct page *rx_page; - struct cbuf cb; - int retries; -#define MAX_CONNECT_RETRIES 3 - int sctp_assoc; - struct hlist_node list; - struct connection *othercon; - struct work_struct rwork; /* Receive workqueue */ - struct work_struct swork; /* Send workqueue */ -}; -#define sock2con(x) ((struct connection *)(x)->sk_user_data) - -/* An entry waiting to be sent */ -struct writequeue_entry { - struct list_head list; - struct page *page; - int offset; - int len; - int end; - int users; - struct connection *con; -}; - -struct dlm_node_addr { - struct list_head list; - int nodeid; - int addr_count; - struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; -}; - -static LIST_HEAD(dlm_node_addrs); -static DEFINE_SPINLOCK(dlm_node_addrs_spin); - -static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT]; -static int dlm_local_count; -static int dlm_allow_conn; - -/* Work queues */ -static struct workqueue_struct *recv_workqueue; -static struct workqueue_struct *send_workqueue; - -static struct hlist_head connection_hash[CONN_HASH_SIZE]; -static DEFINE_MUTEX(connections_lock); -static struct kmem_cache *con_cache; - -static void process_recv_sockets(struct work_struct *work); -static void process_send_sockets(struct work_struct *work); - - -/* This is deliberately very simple because most clusters have simple - sequential nodeids, so we should be able to go straight to a connection - struct in the array */ -static inline int nodeid_hash(int nodeid) -{ - return nodeid & (CONN_HASH_SIZE-1); -} - -static struct connection *__find_con(int nodeid) -{ - int r; - struct connection *con; - - r = nodeid_hash(nodeid); - - hlist_for_each_entry(con, &connection_hash[r], list) { - if (con->nodeid == nodeid) - return con; - } - return NULL; -} - -/* - * If 'allocation' is zero then we don't attempt to create a new - * connection structure for this node. - */ -static struct connection *__nodeid2con(int nodeid, gfp_t alloc) -{ - struct connection *con = NULL; - int r; - - con = __find_con(nodeid); - if (con || !alloc) - return con; - - con = kmem_cache_zalloc(con_cache, alloc); - if (!con) - return NULL; - - r = nodeid_hash(nodeid); - hlist_add_head(&con->list, &connection_hash[r]); - - con->nodeid = nodeid; - mutex_init(&con->sock_mutex); - INIT_LIST_HEAD(&con->writequeue); - spin_lock_init(&con->writequeue_lock); - INIT_WORK(&con->swork, process_send_sockets); - INIT_WORK(&con->rwork, process_recv_sockets); - - /* Setup action pointers for child sockets */ - if (con->nodeid) { - struct connection *zerocon = __find_con(0); - - con->connect_action = zerocon->connect_action; - if (!con->rx_action) - con->rx_action = zerocon->rx_action; - } - - return con; -} - -/* Loop round all connections */ -static void foreach_conn(void (*conn_func)(struct connection *c)) -{ - int i; - struct hlist_node *n; - struct connection *con; - - for (i = 0; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry_safe(con, n, &connection_hash[i], list) - conn_func(con); - } -} - -static struct connection *nodeid2con(int nodeid, gfp_t allocation) -{ - struct connection *con; - - mutex_lock(&connections_lock); - con = __nodeid2con(nodeid, allocation); - mutex_unlock(&connections_lock); - - return con; -} - -/* This is a bit drastic, but only called when things go wrong */ -static struct connection *assoc2con(int assoc_id) -{ - int i; - struct connection *con; - - mutex_lock(&connections_lock); - - for (i = 0 ; i < CONN_HASH_SIZE; i++) { - hlist_for_each_entry(con, &connection_hash[i], list) { - if (con->sctp_assoc == assoc_id) { - mutex_unlock(&connections_lock); - return con; - } - } - } - mutex_unlock(&connections_lock); - return NULL; -} - -static struct dlm_node_addr *find_node_addr(int nodeid) -{ - struct dlm_node_addr *na; - - list_for_each_entry(na, &dlm_node_addrs, list) { - if (na->nodeid == nodeid) - return na; - } - return NULL; -} - -static int addr_compare(struct sockaddr_storage *x, struct sockaddr_storage *y) -{ - switch (x->ss_family) { - case AF_INET: { - struct sockaddr_in *sinx = (struct sockaddr_in *)x; - struct sockaddr_in *siny = (struct sockaddr_in *)y; - if (sinx->sin_addr.s_addr != siny->sin_addr.s_addr) - return 0; - if (sinx->sin_port != siny->sin_port) - return 0; - break; - } - case AF_INET6: { - struct sockaddr_in6 *sinx = (struct sockaddr_in6 *)x; - struct sockaddr_in6 *siny = (struct sockaddr_in6 *)y; - if (!ipv6_addr_equal(&sinx->sin6_addr, &siny->sin6_addr)) - return 0; - if (sinx->sin6_port != siny->sin6_port) - return 0; - break; - } - default: - return 0; - } - return 1; -} - -static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, - struct sockaddr *sa_out) -{ - struct sockaddr_storage sas; - struct dlm_node_addr *na; - - if (!dlm_local_count) - return -1; - - spin_lock(&dlm_node_addrs_spin); - na = find_node_addr(nodeid); - if (na && na->addr_count) - memcpy(&sas, na->addr[0], sizeof(struct sockaddr_storage)); - spin_unlock(&dlm_node_addrs_spin); - - if (!na) - return -EEXIST; - - if (!na->addr_count) - return -ENOENT; - - if (sas_out) - memcpy(sas_out, &sas, sizeof(struct sockaddr_storage)); - - if (!sa_out) - return 0; - - if (dlm_local_addr[0]->ss_family == AF_INET) { - struct sockaddr_in *in4 = (struct sockaddr_in *) &sas; - struct sockaddr_in *ret4 = (struct sockaddr_in *) sa_out; - ret4->sin_addr.s_addr = in4->sin_addr.s_addr; - } else { - struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) &sas; - struct sockaddr_in6 *ret6 = (struct sockaddr_in6 *) sa_out; - ret6->sin6_addr = in6->sin6_addr; - } - - return 0; -} - -static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid) -{ - struct dlm_node_addr *na; - int rv = -EEXIST; - - spin_lock(&dlm_node_addrs_spin); - list_for_each_entry(na, &dlm_node_addrs, list) { - if (!na->addr_count) - continue; - - if (!addr_compare(na->addr[0], addr)) - continue; - - *nodeid = na->nodeid; - rv = 0; - break; - } - spin_unlock(&dlm_node_addrs_spin); - return rv; -} - -int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len) -{ - struct sockaddr_storage *new_addr; - struct dlm_node_addr *new_node, *na; - - new_node = kzalloc(sizeof(struct dlm_node_addr), GFP_NOFS); - if (!new_node) - return -ENOMEM; - - new_addr = kzalloc(sizeof(struct sockaddr_storage), GFP_NOFS); - if (!new_addr) { - kfree(new_node); - return -ENOMEM; - } - - memcpy(new_addr, addr, len); - - spin_lock(&dlm_node_addrs_spin); - na = find_node_addr(nodeid); - if (!na) { - new_node->nodeid = nodeid; - new_node->addr[0] = new_addr; - new_node->addr_count = 1; - list_add(&new_node->list, &dlm_node_addrs); - spin_unlock(&dlm_node_addrs_spin); - return 0; - } - - if (na->addr_count >= DLM_MAX_ADDR_COUNT) { - spin_unlock(&dlm_node_addrs_spin); - kfree(new_addr); - kfree(new_node); - return -ENOSPC; - } - - na->addr[na->addr_count++] = new_addr; - spin_unlock(&dlm_node_addrs_spin); - kfree(new_node); - return 0; -} - -/* Data available on socket or listen socket received a connect */ -static void lowcomms_data_ready(struct sock *sk, int count_unused) -{ - struct connection *con = sock2con(sk); - if (con && !test_and_set_bit(CF_READ_PENDING, &con->flags)) - queue_work(recv_workqueue, &con->rwork); -} - -static void lowcomms_write_space(struct sock *sk) -{ - struct connection *con = sock2con(sk); - - if (!con) - return; - - clear_bit(SOCK_NOSPACE, &con->sock->flags); - - if (test_and_clear_bit(CF_APP_LIMITED, &con->flags)) { - con->sock->sk->sk_write_pending--; - clear_bit(SOCK_ASYNC_NOSPACE, &con->sock->flags); - } - - if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) - queue_work(send_workqueue, &con->swork); -} - -static inline void lowcomms_connect_sock(struct connection *con) -{ - if (test_bit(CF_CLOSE, &con->flags)) - return; - if (!test_and_set_bit(CF_CONNECT_PENDING, &con->flags)) - queue_work(send_workqueue, &con->swork); -} - -static void lowcomms_state_change(struct sock *sk) -{ - if (sk->sk_state == TCP_ESTABLISHED) - lowcomms_write_space(sk); -} - -int dlm_lowcomms_connect_node(int nodeid) -{ - struct connection *con; - - /* with sctp there's no connecting without sending */ - if (dlm_config.ci_protocol != 0) - return 0; - - if (nodeid == dlm_our_nodeid()) - return 0; - - con = nodeid2con(nodeid, GFP_NOFS); - if (!con) - return -ENOMEM; - lowcomms_connect_sock(con); - return 0; -} - -/* Make a socket active */ -static void add_sock(struct socket *sock, struct connection *con) -{ - con->sock = sock; - - /* Install a data_ready callback */ - con->sock->sk->sk_data_ready = lowcomms_data_ready; - con->sock->sk->sk_write_space = lowcomms_write_space; - con->sock->sk->sk_state_change = lowcomms_state_change; - con->sock->sk->sk_user_data = con; - con->sock->sk->sk_allocation = GFP_NOFS; -} - -/* Add the port number to an IPv6 or 4 sockaddr and return the address - length */ -static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port, - int *addr_len) -{ - saddr->ss_family = dlm_local_addr[0]->ss_family; - if (saddr->ss_family == AF_INET) { - struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr; - in4_addr->sin_port = cpu_to_be16(port); - *addr_len = sizeof(struct sockaddr_in); - memset(&in4_addr->sin_zero, 0, sizeof(in4_addr->sin_zero)); - } else { - struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)saddr; - in6_addr->sin6_port = cpu_to_be16(port); - *addr_len = sizeof(struct sockaddr_in6); - } - memset((char *)saddr + *addr_len, 0, sizeof(struct sockaddr_storage) - *addr_len); -} - -/* Close a remote connection and tidy up */ -static void close_connection(struct connection *con, bool and_other) -{ - mutex_lock(&con->sock_mutex); - - if (con->sock) { - sock_release(con->sock); - con->sock = NULL; - } - if (con->othercon && and_other) { - /* Will only re-enter once. */ - close_connection(con->othercon, false); - } - if (con->rx_page) { - __free_page(con->rx_page); - con->rx_page = NULL; - } - - con->retries = 0; - mutex_unlock(&con->sock_mutex); -} - -/* We only send shutdown messages to nodes that are not part of the cluster */ -static void sctp_send_shutdown(sctp_assoc_t associd) -{ - static char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; - struct msghdr outmessage; - struct cmsghdr *cmsg; - struct sctp_sndrcvinfo *sinfo; - int ret; - struct connection *con; - - con = nodeid2con(0,0); - BUG_ON(con == NULL); - - outmessage.msg_name = NULL; - outmessage.msg_namelen = 0; - outmessage.msg_control = outcmsg; - outmessage.msg_controllen = sizeof(outcmsg); - outmessage.msg_flags = MSG_EOR; - - cmsg = CMSG_FIRSTHDR(&outmessage); - cmsg->cmsg_level = IPPROTO_SCTP; - cmsg->cmsg_type = SCTP_SNDRCV; - cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); - outmessage.msg_controllen = cmsg->cmsg_len; - sinfo = CMSG_DATA(cmsg); - memset(sinfo, 0x00, sizeof(struct sctp_sndrcvinfo)); - - sinfo->sinfo_flags |= MSG_EOF; - sinfo->sinfo_assoc_id = associd; - - ret = kernel_sendmsg(con->sock, &outmessage, NULL, 0, 0); - - if (ret != 0) - log_print("send EOF to node failed: %d", ret); -} - -static void sctp_init_failed_foreach(struct connection *con) -{ - con->sctp_assoc = 0; - if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) { - if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) - queue_work(send_workqueue, &con->swork); - } -} - -/* INIT failed but we don't know which node... - restart INIT on all pending nodes */ -static void sctp_init_failed(void) -{ - mutex_lock(&connections_lock); - - foreach_conn(sctp_init_failed_foreach); - - mutex_unlock(&connections_lock); -} - -/* Something happened to an association */ -static void process_sctp_notification(struct connection *con, - struct msghdr *msg, char *buf) -{ - union sctp_notification *sn = (union sctp_notification *)buf; - - if (sn->sn_header.sn_type == SCTP_ASSOC_CHANGE) { - switch (sn->sn_assoc_change.sac_state) { - - case SCTP_COMM_UP: - case SCTP_RESTART: - { - /* Check that the new node is in the lockspace */ - struct sctp_prim prim; - int nodeid; - int prim_len, ret; - int addr_len; - struct connection *new_con; - - /* - * We get this before any data for an association. - * We verify that the node is in the cluster and - * then peel off a socket for it. - */ - if ((int)sn->sn_assoc_change.sac_assoc_id <= 0) { - log_print("COMM_UP for invalid assoc ID %d", - (int)sn->sn_assoc_change.sac_assoc_id); - sctp_init_failed(); - return; - } - memset(&prim, 0, sizeof(struct sctp_prim)); - prim_len = sizeof(struct sctp_prim); - prim.ssp_assoc_id = sn->sn_assoc_change.sac_assoc_id; - - ret = kernel_getsockopt(con->sock, - IPPROTO_SCTP, - SCTP_PRIMARY_ADDR, - (char*)&prim, - &prim_len); - if (ret < 0) { - log_print("getsockopt/sctp_primary_addr on " - "new assoc %d failed : %d", - (int)sn->sn_assoc_change.sac_assoc_id, - ret); - - /* Retry INIT later */ - new_con = assoc2con(sn->sn_assoc_change.sac_assoc_id); - if (new_con) - clear_bit(CF_CONNECT_PENDING, &con->flags); - return; - } - make_sockaddr(&prim.ssp_addr, 0, &addr_len); - if (addr_to_nodeid(&prim.ssp_addr, &nodeid)) { - unsigned char *b=(unsigned char *)&prim.ssp_addr; - log_print("reject connect from unknown addr"); - print_hex_dump_bytes("ss: ", DUMP_PREFIX_NONE, - b, sizeof(struct sockaddr_storage)); - sctp_send_shutdown(prim.ssp_assoc_id); - return; - } - - new_con = nodeid2con(nodeid, GFP_NOFS); - if (!new_con) - return; - - /* Peel off a new sock */ - sctp_lock_sock(con->sock->sk); - ret = sctp_do_peeloff(con->sock->sk, - sn->sn_assoc_change.sac_assoc_id, - &new_con->sock); - sctp_release_sock(con->sock->sk); - if (ret < 0) { - log_print("Can't peel off a socket for " - "connection %d to node %d: err=%d", - (int)sn->sn_assoc_change.sac_assoc_id, - nodeid, ret); - return; - } - add_sock(new_con->sock, new_con); - - log_print("connecting to %d sctp association %d", - nodeid, (int)sn->sn_assoc_change.sac_assoc_id); - - /* Send any pending writes */ - clear_bit(CF_CONNECT_PENDING, &new_con->flags); - clear_bit(CF_INIT_PENDING, &con->flags); - if (!test_and_set_bit(CF_WRITE_PENDING, &new_con->flags)) { - queue_work(send_workqueue, &new_con->swork); - } - if (!test_and_set_bit(CF_READ_PENDING, &new_con->flags)) - queue_work(recv_workqueue, &new_con->rwork); - } - break; - - case SCTP_COMM_LOST: - case SCTP_SHUTDOWN_COMP: - { - con = assoc2con(sn->sn_assoc_change.sac_assoc_id); - if (con) { - con->sctp_assoc = 0; - } - } - break; - - /* We don't know which INIT failed, so clear the PENDING flags - * on them all. if assoc_id is zero then it will then try - * again */ - - case SCTP_CANT_STR_ASSOC: - { - log_print("Can't start SCTP association - retrying"); - sctp_init_failed(); - } - break; - - default: - log_print("unexpected SCTP assoc change id=%d state=%d", - (int)sn->sn_assoc_change.sac_assoc_id, - sn->sn_assoc_change.sac_state); - } - } -} - -/* Data received from remote end */ -static int receive_from_sock(struct connection *con) -{ - int ret = 0; - struct msghdr msg = {}; - struct kvec iov[2]; - unsigned len; - int r; - int call_again_soon = 0; - int nvec; - char incmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; - - mutex_lock(&con->sock_mutex); - - if (con->sock == NULL) { - ret = -EAGAIN; - goto out_close; - } - - if (con->rx_page == NULL) { - /* - * This doesn't need to be atomic, but I think it should - * improve performance if it is. - */ - con->rx_page = alloc_page(GFP_ATOMIC); - if (con->rx_page == NULL) - goto out_resched; - cbuf_init(&con->cb, PAGE_CACHE_SIZE); - } - - /* Only SCTP needs these really */ - memset(&incmsg, 0, sizeof(incmsg)); - msg.msg_control = incmsg; - msg.msg_controllen = sizeof(incmsg); - - /* - * iov[0] is the bit of the circular buffer between the current end - * point (cb.base + cb.len) and the end of the buffer. - */ - iov[0].iov_len = con->cb.base - cbuf_data(&con->cb); - iov[0].iov_base = page_address(con->rx_page) + cbuf_data(&con->cb); - iov[1].iov_len = 0; - nvec = 1; - - /* - * iov[1] is the bit of the circular buffer between the start of the - * buffer and the start of the currently used section (cb.base) - */ - if (cbuf_data(&con->cb) >= con->cb.base) { - iov[0].iov_len = PAGE_CACHE_SIZE - cbuf_data(&con->cb); - iov[1].iov_len = con->cb.base; - iov[1].iov_base = page_address(con->rx_page); - nvec = 2; - } - len = iov[0].iov_len + iov[1].iov_len; - - r = ret = kernel_recvmsg(con->sock, &msg, iov, nvec, len, - MSG_DONTWAIT | MSG_NOSIGNAL); - if (ret <= 0) - goto out_close; - - /* Process SCTP notifications */ - if (msg.msg_flags & MSG_NOTIFICATION) { - msg.msg_control = incmsg; - msg.msg_controllen = sizeof(incmsg); - - process_sctp_notification(con, &msg, - page_address(con->rx_page) + con->cb.base); - mutex_unlock(&con->sock_mutex); - return 0; - } - BUG_ON(con->nodeid == 0); - - if (ret == len) - call_again_soon = 1; - cbuf_add(&con->cb, ret); - ret = dlm_process_incoming_buffer(con->nodeid, - page_address(con->rx_page), - con->cb.base, con->cb.len, - PAGE_CACHE_SIZE); - if (ret == -EBADMSG) { - log_print("lowcomms: addr=%p, base=%u, len=%u, " - "iov_len=%u, iov_base[0]=%p, read=%d", - page_address(con->rx_page), con->cb.base, con->cb.len, - len, iov[0].iov_base, r); - } - if (ret < 0) - goto out_close; - cbuf_eat(&con->cb, ret); - - if (cbuf_empty(&con->cb) && !call_again_soon) { - __free_page(con->rx_page); - con->rx_page = NULL; - } - - if (call_again_soon) - goto out_resched; - mutex_unlock(&con->sock_mutex); - return 0; - -out_resched: - if (!test_and_set_bit(CF_READ_PENDING, &con->flags)) - queue_work(recv_workqueue, &con->rwork); - mutex_unlock(&con->sock_mutex); - return -EAGAIN; - -out_close: - mutex_unlock(&con->sock_mutex); - if (ret != -EAGAIN) { - close_connection(con, false); - /* Reconnect when there is something to send */ - } - /* Don't return success if we really got EOF */ - if (ret == 0) - ret = -EAGAIN; - - return ret; -} - -/* Listening socket is busy, accept a connection */ -static int tcp_accept_from_sock(struct connection *con) -{ - int result; - struct sockaddr_storage peeraddr; - struct socket *newsock; - int len; - int nodeid; - struct connection *newcon; - struct connection *addcon; - - mutex_lock(&connections_lock); - if (!dlm_allow_conn) { - mutex_unlock(&connections_lock); - return -1; - } - mutex_unlock(&connections_lock); - - memset(&peeraddr, 0, sizeof(peeraddr)); - result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, - IPPROTO_TCP, &newsock); - if (result < 0) - return -ENOMEM; - - mutex_lock_nested(&con->sock_mutex, 0); - - result = -ENOTCONN; - if (con->sock == NULL) - goto accept_err; - - newsock->type = con->sock->type; - newsock->ops = con->sock->ops; - - result = con->sock->ops->accept(con->sock, newsock, O_NONBLOCK); - if (result < 0) - goto accept_err; - - /* Get the connected socket's peer */ - memset(&peeraddr, 0, sizeof(peeraddr)); - if (newsock->ops->getname(newsock, (struct sockaddr *)&peeraddr, - &len, 2)) { - result = -ECONNABORTED; - goto accept_err; - } - - /* Get the new node's NODEID */ - make_sockaddr(&peeraddr, 0, &len); - if (addr_to_nodeid(&peeraddr, &nodeid)) { - unsigned char *b=(unsigned char *)&peeraddr; - log_print("connect from non cluster node"); - print_hex_dump_bytes("ss: ", DUMP_PREFIX_NONE, - b, sizeof(struct sockaddr_storage)); - sock_release(newsock); - mutex_unlock(&con->sock_mutex); - return -1; - } - - log_print("got connection from %d", nodeid); - - /* Check to see if we already have a connection to this node. This - * could happen if the two nodes initiate a connection at roughly - * the same time and the connections cross on the wire. - * In this case we store the incoming one in "othercon" - */ - newcon = nodeid2con(nodeid, GFP_NOFS); - if (!newcon) { - result = -ENOMEM; - goto accept_err; - } - mutex_lock_nested(&newcon->sock_mutex, 1); - if (newcon->sock) { - struct connection *othercon = newcon->othercon; - - if (!othercon) { - othercon = kmem_cache_zalloc(con_cache, GFP_NOFS); - if (!othercon) { - log_print("failed to allocate incoming socket"); - mutex_unlock(&newcon->sock_mutex); - result = -ENOMEM; - goto accept_err; - } - othercon->nodeid = nodeid; - othercon->rx_action = receive_from_sock; - mutex_init(&othercon->sock_mutex); - INIT_WORK(&othercon->swork, process_send_sockets); - INIT_WORK(&othercon->rwork, process_recv_sockets); - set_bit(CF_IS_OTHERCON, &othercon->flags); - } - if (!othercon->sock) { - newcon->othercon = othercon; - othercon->sock = newsock; - newsock->sk->sk_user_data = othercon; - add_sock(newsock, othercon); - addcon = othercon; - } - else { - printk("Extra connection from node %d attempted\n", nodeid); - result = -EAGAIN; - mutex_unlock(&newcon->sock_mutex); - goto accept_err; - } - } - else { - newsock->sk->sk_user_data = newcon; - newcon->rx_action = receive_from_sock; - add_sock(newsock, newcon); - addcon = newcon; - } - - mutex_unlock(&newcon->sock_mutex); - - /* - * Add it to the active queue in case we got data - * between processing the accept adding the socket - * to the read_sockets list - */ - if (!test_and_set_bit(CF_READ_PENDING, &addcon->flags)) - queue_work(recv_workqueue, &addcon->rwork); - mutex_unlock(&con->sock_mutex); - - return 0; - -accept_err: - mutex_unlock(&con->sock_mutex); - sock_release(newsock); - - if (result != -EAGAIN) - log_print("error accepting connection from node: %d", result); - return result; -} - -static void free_entry(struct writequeue_entry *e) -{ - __free_page(e->page); - kfree(e); -} - -/* Initiate an SCTP association. - This is a special case of send_to_sock() in that we don't yet have a - peeled-off socket for this association, so we use the listening socket - and add the primary IP address of the remote node. - */ -static void sctp_init_assoc(struct connection *con) -{ - struct sockaddr_storage rem_addr; - char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; - struct msghdr outmessage; - struct cmsghdr *cmsg; - struct sctp_sndrcvinfo *sinfo; - struct connection *base_con; - struct writequeue_entry *e; - int len, offset; - int ret; - int addrlen; - struct kvec iov[1]; - - if (test_and_set_bit(CF_INIT_PENDING, &con->flags)) - return; - - if (con->retries++ > MAX_CONNECT_RETRIES) - return; - - if (nodeid_to_addr(con->nodeid, NULL, (struct sockaddr *)&rem_addr)) { - log_print("no address for nodeid %d", con->nodeid); - return; - } - base_con = nodeid2con(0, 0); - BUG_ON(base_con == NULL); - - make_sockaddr(&rem_addr, dlm_config.ci_tcp_port, &addrlen); - - outmessage.msg_name = &rem_addr; - outmessage.msg_namelen = addrlen; - outmessage.msg_control = outcmsg; - outmessage.msg_controllen = sizeof(outcmsg); - outmessage.msg_flags = MSG_EOR; - - spin_lock(&con->writequeue_lock); - - if (list_empty(&con->writequeue)) { - spin_unlock(&con->writequeue_lock); - log_print("writequeue empty for nodeid %d", con->nodeid); - return; - } - - e = list_first_entry(&con->writequeue, struct writequeue_entry, list); - len = e->len; - offset = e->offset; - spin_unlock(&con->writequeue_lock); - - /* Send the first block off the write queue */ - iov[0].iov_base = page_address(e->page)+offset; - iov[0].iov_len = len; - - cmsg = CMSG_FIRSTHDR(&outmessage); - cmsg->cmsg_level = IPPROTO_SCTP; - cmsg->cmsg_type = SCTP_SNDRCV; - cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); - sinfo = CMSG_DATA(cmsg); - memset(sinfo, 0x00, sizeof(struct sctp_sndrcvinfo)); - sinfo->sinfo_ppid = cpu_to_le32(dlm_our_nodeid()); - outmessage.msg_controllen = cmsg->cmsg_len; - - ret = kernel_sendmsg(base_con->sock, &outmessage, iov, 1, len); - if (ret < 0) { - log_print("Send first packet to node %d failed: %d", - con->nodeid, ret); - - /* Try again later */ - clear_bit(CF_CONNECT_PENDING, &con->flags); - clear_bit(CF_INIT_PENDING, &con->flags); - } - else { - spin_lock(&con->writequeue_lock); - e->offset += ret; - e->len -= ret; - - if (e->len == 0 && e->users == 0) { - list_del(&e->list); - free_entry(e); - } - spin_unlock(&con->writequeue_lock); - } -} - -/* Connect a new socket to its peer */ -static void tcp_connect_to_sock(struct connection *con) -{ - struct sockaddr_storage saddr, src_addr; - int addr_len; - struct socket *sock = NULL; - int one = 1; - int result; - - if (con->nodeid == 0) { - log_print("attempt to connect sock 0 foiled"); - return; - } - - mutex_lock(&con->sock_mutex); - if (con->retries++ > MAX_CONNECT_RETRIES) - goto out; - - /* Some odd races can cause double-connects, ignore them */ - if (con->sock) - goto out; - - /* Create a socket to communicate with */ - result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, - IPPROTO_TCP, &sock); - if (result < 0) - goto out_err; - - memset(&saddr, 0, sizeof(saddr)); - result = nodeid_to_addr(con->nodeid, &saddr, NULL); - if (result < 0) { - log_print("no address for nodeid %d", con->nodeid); - goto out_err; - } - - sock->sk->sk_user_data = con; - con->rx_action = receive_from_sock; - con->connect_action = tcp_connect_to_sock; - add_sock(sock, con); - - /* Bind to our cluster-known address connecting to avoid - routing problems */ - memcpy(&src_addr, dlm_local_addr[0], sizeof(src_addr)); - make_sockaddr(&src_addr, 0, &addr_len); - result = sock->ops->bind(sock, (struct sockaddr *) &src_addr, - addr_len); - if (result < 0) { - log_print("could not bind for connect: %d", result); - /* This *may* not indicate a critical error */ - } - - make_sockaddr(&saddr, dlm_config.ci_tcp_port, &addr_len); - - log_print("connecting to %d", con->nodeid); - - /* Turn off Nagle's algorithm */ - kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&one, - sizeof(one)); - - result = sock->ops->connect(sock, (struct sockaddr *)&saddr, addr_len, - O_NONBLOCK); - if (result == -EINPROGRESS) - result = 0; - if (result == 0) - goto out; - -out_err: - if (con->sock) { - sock_release(con->sock); - con->sock = NULL; - } else if (sock) { - sock_release(sock); - } - /* - * Some errors are fatal and this list might need adjusting. For other - * errors we try again until the max number of retries is reached. - */ - if (result != -EHOSTUNREACH && - result != -ENETUNREACH && - result != -ENETDOWN && - result != -EINVAL && - result != -EPROTONOSUPPORT) { - log_print("connect %d try %d error %d", con->nodeid, - con->retries, result); - mutex_unlock(&con->sock_mutex); - msleep(1000); - lowcomms_connect_sock(con); - return; - } -out: - mutex_unlock(&con->sock_mutex); - return; -} - -static struct socket *tcp_create_listen_sock(struct connection *con, - struct sockaddr_storage *saddr) -{ - struct socket *sock = NULL; - int result = 0; - int one = 1; - int addr_len; - - if (dlm_local_addr[0]->ss_family == AF_INET) - addr_len = sizeof(struct sockaddr_in); - else - addr_len = sizeof(struct sockaddr_in6); - - /* Create a socket to communicate with */ - result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM, - IPPROTO_TCP, &sock); - if (result < 0) { - log_print("Can't create listening comms socket"); - goto create_out; - } - - /* Turn off Nagle's algorithm */ - kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&one, - sizeof(one)); - - result = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&one, sizeof(one)); - - if (result < 0) { - log_print("Failed to set SO_REUSEADDR on socket: %d", result); - } - con->rx_action = tcp_accept_from_sock; - con->connect_action = tcp_connect_to_sock; - - /* Bind to our port */ - make_sockaddr(saddr, dlm_config.ci_tcp_port, &addr_len); - result = sock->ops->bind(sock, (struct sockaddr *) saddr, addr_len); - if (result < 0) { - log_print("Can't bind to port %d", dlm_config.ci_tcp_port); - sock_release(sock); - sock = NULL; - con->sock = NULL; - goto create_out; - } - result = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, - (char *)&one, sizeof(one)); - if (result < 0) { - log_print("Set keepalive failed: %d", result); - } - - result = sock->ops->listen(sock, 5); - if (result < 0) { - log_print("Can't listen on port %d", dlm_config.ci_tcp_port); - sock_release(sock); - sock = NULL; - goto create_out; - } - -create_out: - return sock; -} - -/* Get local addresses */ -static void init_local(void) -{ - struct sockaddr_storage sas, *addr; - int i; - - dlm_local_count = 0; - for (i = 0; i < DLM_MAX_ADDR_COUNT; i++) { - if (dlm_our_addr(&sas, i)) - break; - - addr = kmalloc(sizeof(*addr), GFP_NOFS); - if (!addr) - break; - memcpy(addr, &sas, sizeof(*addr)); - dlm_local_addr[dlm_local_count++] = addr; - } -} - -/* Bind to an IP address. SCTP allows multiple address so it can do - multi-homing */ -static int add_sctp_bind_addr(struct connection *sctp_con, - struct sockaddr_storage *addr, - int addr_len, int num) -{ - int result = 0; - - if (num == 1) - result = kernel_bind(sctp_con->sock, - (struct sockaddr *) addr, - addr_len); - else - result = kernel_setsockopt(sctp_con->sock, SOL_SCTP, - SCTP_SOCKOPT_BINDX_ADD, - (char *)addr, addr_len); - - if (result < 0) - log_print("Can't bind to port %d addr number %d", - dlm_config.ci_tcp_port, num); - - return result; -} - -/* Initialise SCTP socket and bind to all interfaces */ -static int sctp_listen_for_all(void) -{ - struct socket *sock = NULL; - struct sockaddr_storage localaddr; - struct sctp_event_subscribe subscribe; - int result = -EINVAL, num = 1, i, addr_len; - struct connection *con = nodeid2con(0, GFP_NOFS); - int bufsize = NEEDED_RMEM; - - if (!con) - return -ENOMEM; - - log_print("Using SCTP for communications"); - - result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_SEQPACKET, - IPPROTO_SCTP, &sock); - if (result < 0) { - log_print("Can't create comms socket, check SCTP is loaded"); - goto out; - } - - /* Listen for events */ - memset(&subscribe, 0, sizeof(subscribe)); - subscribe.sctp_data_io_event = 1; - subscribe.sctp_association_event = 1; - subscribe.sctp_send_failure_event = 1; - subscribe.sctp_shutdown_event = 1; - subscribe.sctp_partial_delivery_event = 1; - - result = kernel_setsockopt(sock, SOL_SOCKET, SO_RCVBUFFORCE, - (char *)&bufsize, sizeof(bufsize)); - if (result) - log_print("Error increasing buffer space on socket %d", result); - - result = kernel_setsockopt(sock, SOL_SCTP, SCTP_EVENTS, - (char *)&subscribe, sizeof(subscribe)); - if (result < 0) { - log_print("Failed to set SCTP_EVENTS on socket: result=%d", - result); - goto create_delsock; - } - - /* Init con struct */ - sock->sk->sk_user_data = con; - con->sock = sock; - con->sock->sk->sk_data_ready = lowcomms_data_ready; - con->rx_action = receive_from_sock; - con->connect_action = sctp_init_assoc; - - /* Bind to all interfaces. */ - for (i = 0; i < dlm_local_count; i++) { - memcpy(&localaddr, dlm_local_addr[i], sizeof(localaddr)); - make_sockaddr(&localaddr, dlm_config.ci_tcp_port, &addr_len); - - result = add_sctp_bind_addr(con, &localaddr, addr_len, num); - if (result) - goto create_delsock; - ++num; - } - - result = sock->ops->listen(sock, 5); - if (result < 0) { - log_print("Can't set socket listening"); - goto create_delsock; - } - - return 0; - -create_delsock: - sock_release(sock); - con->sock = NULL; -out: - return result; -} - -static int tcp_listen_for_all(void) -{ - struct socket *sock = NULL; - struct connection *con = nodeid2con(0, GFP_NOFS); - int result = -EINVAL; - - if (!con) - return -ENOMEM; - - /* We don't support multi-homed hosts */ - if (dlm_local_addr[1] != NULL) { - log_print("TCP protocol can't handle multi-homed hosts, " - "try SCTP"); - return -EINVAL; - } - - log_print("Using TCP for communications"); - - sock = tcp_create_listen_sock(con, dlm_local_addr[0]); - if (sock) { - add_sock(sock, con); - result = 0; - } - else { - result = -EADDRINUSE; - } - - return result; -} - - - -static struct writequeue_entry *new_writequeue_entry(struct connection *con, - gfp_t allocation) -{ - struct writequeue_entry *entry; - - entry = kmalloc(sizeof(struct writequeue_entry), allocation); - if (!entry) - return NULL; - - entry->page = alloc_page(allocation); - if (!entry->page) { - kfree(entry); - return NULL; - } - - entry->offset = 0; - entry->len = 0; - entry->end = 0; - entry->users = 0; - entry->con = con; - - return entry; -} - -void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc) -{ - struct connection *con; - struct writequeue_entry *e; - int offset = 0; - - con = nodeid2con(nodeid, allocation); - if (!con) - return NULL; - - spin_lock(&con->writequeue_lock); - e = list_entry(con->writequeue.prev, struct writequeue_entry, list); - if ((&e->list == &con->writequeue) || - (PAGE_CACHE_SIZE - e->end < len)) { - e = NULL; - } else { - offset = e->end; - e->end += len; - e->users++; - } - spin_unlock(&con->writequeue_lock); - - if (e) { - got_one: - *ppc = page_address(e->page) + offset; - return e; - } - - e = new_writequeue_entry(con, allocation); - if (e) { - spin_lock(&con->writequeue_lock); - offset = e->end; - e->end += len; - e->users++; - list_add_tail(&e->list, &con->writequeue); - spin_unlock(&con->writequeue_lock); - goto got_one; - } - return NULL; -} - -void dlm_lowcomms_commit_buffer(void *mh) -{ - struct writequeue_entry *e = (struct writequeue_entry *)mh; - struct connection *con = e->con; - int users; - - spin_lock(&con->writequeue_lock); - users = --e->users; - if (users) - goto out; - e->len = e->end - e->offset; - spin_unlock(&con->writequeue_lock); - - if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) { - queue_work(send_workqueue, &con->swork); - } - return; - -out: - spin_unlock(&con->writequeue_lock); - return; -} - -/* Send a message */ -static void send_to_sock(struct connection *con) -{ - int ret = 0; - const int msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; - struct writequeue_entry *e; - int len, offset; - int count = 0; - - mutex_lock(&con->sock_mutex); - if (con->sock == NULL) - goto out_connect; - - spin_lock(&con->writequeue_lock); - for (;;) { - e = list_entry(con->writequeue.next, struct writequeue_entry, - list); - if ((struct list_head *) e == &con->writequeue) - break; - - len = e->len; - offset = e->offset; - BUG_ON(len == 0 && e->users == 0); - spin_unlock(&con->writequeue_lock); - - ret = 0; - if (len) { - ret = kernel_sendpage(con->sock, e->page, offset, len, - msg_flags); - if (ret == -EAGAIN || ret == 0) { - if (ret == -EAGAIN && - test_bit(SOCK_ASYNC_NOSPACE, &con->sock->flags) && - !test_and_set_bit(CF_APP_LIMITED, &con->flags)) { - /* Notify TCP that we're limited by the - * application window size. - */ - set_bit(SOCK_NOSPACE, &con->sock->flags); - con->sock->sk->sk_write_pending++; - } - cond_resched(); - goto out; - } else if (ret < 0) - goto send_error; - } - - /* Don't starve people filling buffers */ - if (++count >= MAX_SEND_MSG_COUNT) { - cond_resched(); - count = 0; - } - - spin_lock(&con->writequeue_lock); - e->offset += ret; - e->len -= ret; - - if (e->len == 0 && e->users == 0) { - list_del(&e->list); - free_entry(e); - } - } - spin_unlock(&con->writequeue_lock); -out: - mutex_unlock(&con->sock_mutex); - return; - -send_error: - mutex_unlock(&con->sock_mutex); - close_connection(con, false); - lowcomms_connect_sock(con); - return; - -out_connect: - mutex_unlock(&con->sock_mutex); - if (!test_bit(CF_INIT_PENDING, &con->flags)) - lowcomms_connect_sock(con); -} - -static void clean_one_writequeue(struct connection *con) -{ - struct writequeue_entry *e, *safe; - - spin_lock(&con->writequeue_lock); - list_for_each_entry_safe(e, safe, &con->writequeue, list) { - list_del(&e->list); - free_entry(e); - } - spin_unlock(&con->writequeue_lock); -} - -/* Called from recovery when it knows that a node has - left the cluster */ -int dlm_lowcomms_close(int nodeid) -{ - struct connection *con; - struct dlm_node_addr *na; - - log_print("closing connection to node %d", nodeid); - con = nodeid2con(nodeid, 0); - if (con) { - clear_bit(CF_CONNECT_PENDING, &con->flags); - clear_bit(CF_WRITE_PENDING, &con->flags); - set_bit(CF_CLOSE, &con->flags); - if (cancel_work_sync(&con->swork)) - log_print("canceled swork for node %d", nodeid); - if (cancel_work_sync(&con->rwork)) - log_print("canceled rwork for node %d", nodeid); - clean_one_writequeue(con); - close_connection(con, true); - } - - spin_lock(&dlm_node_addrs_spin); - na = find_node_addr(nodeid); - if (na) { - list_del(&na->list); - while (na->addr_count--) - kfree(na->addr[na->addr_count]); - kfree(na); - } - spin_unlock(&dlm_node_addrs_spin); - - return 0; -} - -/* Receive workqueue function */ -static void process_recv_sockets(struct work_struct *work) -{ - struct connection *con = container_of(work, struct connection, rwork); - int err; - - clear_bit(CF_READ_PENDING, &con->flags); - do { - err = con->rx_action(con); - } while (!err); -} - -/* Send workqueue function */ -static void process_send_sockets(struct work_struct *work) -{ - struct connection *con = container_of(work, struct connection, swork); - - if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) { - con->connect_action(con); - set_bit(CF_WRITE_PENDING, &con->flags); - } - if (test_and_clear_bit(CF_WRITE_PENDING, &con->flags)) - send_to_sock(con); -} - - -/* Discard all entries on the write queues */ -static void clean_writequeues(void) -{ - foreach_conn(clean_one_writequeue); -} - -static void work_stop(void) -{ - destroy_workqueue(recv_workqueue); - destroy_workqueue(send_workqueue); -} - -static int work_start(void) -{ - recv_workqueue = alloc_workqueue("dlm_recv", - WQ_UNBOUND | WQ_MEM_RECLAIM, 1); - if (!recv_workqueue) { - log_print("can't start dlm_recv"); - return -ENOMEM; - } - - send_workqueue = alloc_workqueue("dlm_send", - WQ_UNBOUND | WQ_MEM_RECLAIM, 1); - if (!send_workqueue) { - log_print("can't start dlm_send"); - destroy_workqueue(recv_workqueue); - return -ENOMEM; - } - - return 0; -} - -static void stop_conn(struct connection *con) -{ - con->flags |= 0x0F; - if (con->sock && con->sock->sk) - con->sock->sk->sk_user_data = NULL; -} - -static void free_conn(struct connection *con) -{ - close_connection(con, true); - if (con->othercon) - kmem_cache_free(con_cache, con->othercon); - hlist_del(&con->list); - kmem_cache_free(con_cache, con); -} - -void dlm_lowcomms_stop(void) -{ - /* Set all the flags to prevent any - socket activity. - */ - mutex_lock(&connections_lock); - dlm_allow_conn = 0; - foreach_conn(stop_conn); - mutex_unlock(&connections_lock); - - work_stop(); - - mutex_lock(&connections_lock); - clean_writequeues(); - - foreach_conn(free_conn); - - mutex_unlock(&connections_lock); - kmem_cache_destroy(con_cache); -} - -int dlm_lowcomms_start(void) -{ - int error = -EINVAL; - struct connection *con; - int i; - - for (i = 0; i < CONN_HASH_SIZE; i++) - INIT_HLIST_HEAD(&connection_hash[i]); - - init_local(); - if (!dlm_local_count) { - error = -ENOTCONN; - log_print("no local IP address has been set"); - goto fail; - } - - error = -ENOMEM; - con_cache = kmem_cache_create("dlm_conn", sizeof(struct connection), - __alignof__(struct connection), 0, - NULL); - if (!con_cache) - goto fail; - - error = work_start(); - if (error) - goto fail_destroy; - - dlm_allow_conn = 1; - - /* Start listening */ - if (dlm_config.ci_protocol == 0) - error = tcp_listen_for_all(); - else - error = sctp_listen_for_all(); - if (error) - goto fail_unlisten; - - return 0; - -fail_unlisten: - dlm_allow_conn = 0; - con = nodeid2con(0,0); - if (con) { - close_connection(con, false); - kmem_cache_free(con_cache, con); - } -fail_destroy: - kmem_cache_destroy(con_cache); -fail: - return error; -} - -void dlm_lowcomms_exit(void) -{ - struct dlm_node_addr *na, *safe; - - spin_lock(&dlm_node_addrs_spin); - list_for_each_entry_safe(na, safe, &dlm_node_addrs, list) { - list_del(&na->list); - while (na->addr_count--) - kfree(na->addr[na->addr_count]); - kfree(na); - } - spin_unlock(&dlm_node_addrs_spin); -} diff --git a/kmod/dlm/lowcomms.h b/kmod/dlm/lowcomms.h deleted file mode 100644 index 67462e54..00000000 --- a/kmod/dlm/lowcomms.h +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __LOWCOMMS_DOT_H__ -#define __LOWCOMMS_DOT_H__ - -int dlm_lowcomms_start(void); -void dlm_lowcomms_stop(void); -void dlm_lowcomms_exit(void); -int dlm_lowcomms_close(int nodeid); -void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc); -void dlm_lowcomms_commit_buffer(void *mh); -int dlm_lowcomms_connect_node(int nodeid); -int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr, int len); - -#endif /* __LOWCOMMS_DOT_H__ */ - diff --git a/kmod/dlm/lvb_table.h b/kmod/dlm/lvb_table.h deleted file mode 100644 index cc3e92f3..00000000 --- a/kmod/dlm/lvb_table.h +++ /dev/null @@ -1,18 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __LVB_TABLE_DOT_H__ -#define __LVB_TABLE_DOT_H__ - -extern const int dlm_lvb_operations[8][8]; - -#endif diff --git a/kmod/dlm/main.c b/kmod/dlm/main.c deleted file mode 100644 index d880842d..00000000 --- a/kmod/dlm/main.c +++ /dev/null @@ -1,98 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "lock.h" -#include "user.h" -#include "memory.h" -#include "config.h" -#include "lowcomms.h" - -static int __init init_dlm(void) -{ - int error; - - error = dlm_memory_init(); - if (error) - goto out; - - error = dlm_lockspace_init(); - if (error) - goto out_mem; - - error = dlm_config_init(); - if (error) - goto out_lockspace; - - error = dlm_register_debugfs(); - if (error) - goto out_config; - - error = dlm_user_init(); - if (error) - goto out_debug; - - error = dlm_netlink_init(); - if (error) - goto out_user; - - error = dlm_plock_init(); - if (error) - goto out_netlink; - - printk("DLM installed\n"); - - return 0; - - out_netlink: - dlm_netlink_exit(); - out_user: - dlm_user_exit(); - out_debug: - dlm_unregister_debugfs(); - out_config: - dlm_config_exit(); - out_lockspace: - dlm_lockspace_exit(); - out_mem: - dlm_memory_exit(); - out: - return error; -} - -static void __exit exit_dlm(void) -{ - dlm_plock_exit(); - dlm_netlink_exit(); - dlm_user_exit(); - dlm_config_exit(); - dlm_memory_exit(); - dlm_lockspace_exit(); - dlm_lowcomms_exit(); - dlm_unregister_debugfs(); -} - -module_init(init_dlm); -module_exit(exit_dlm); - -MODULE_DESCRIPTION("Distributed Lock Manager"); -MODULE_AUTHOR("Red Hat, Inc."); -MODULE_LICENSE("GPL"); - -EXPORT_SYMBOL_GPL(dlm_new_lockspace); -EXPORT_SYMBOL_GPL(dlm_release_lockspace); -EXPORT_SYMBOL_GPL(dlm_lock); -EXPORT_SYMBOL_GPL(dlm_unlock); -EXPORT_SYMBOL_GPL(dlm_lock_range); - diff --git a/kmod/dlm/member.c b/kmod/dlm/member.c deleted file mode 100644 index 476557b5..00000000 --- a/kmod/dlm/member.c +++ /dev/null @@ -1,725 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "member.h" -#include "recoverd.h" -#include "recover.h" -#include "rcom.h" -#include "config.h" -#include "lowcomms.h" - -int dlm_slots_version(struct dlm_header *h) -{ - if ((h->h_version & 0x0000FFFF) < DLM_HEADER_SLOTS) - return 0; - return 1; -} - -void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, - struct dlm_member *memb) -{ - struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; - - if (!dlm_slots_version(&rc->rc_header)) - return; - - memb->slot = le16_to_cpu(rf->rf_our_slot); - memb->generation = le32_to_cpu(rf->rf_generation); -} - -void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc) -{ - struct dlm_slot *slot; - struct rcom_slot *ro; - int i; - - ro = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); - - /* ls_slots array is sparse, but not rcom_slots */ - - for (i = 0; i < ls->ls_slots_size; i++) { - slot = &ls->ls_slots[i]; - if (!slot->nodeid) - continue; - ro->ro_nodeid = cpu_to_le32(slot->nodeid); - ro->ro_slot = cpu_to_le16(slot->slot); - ro++; - } -} - -#define SLOT_DEBUG_LINE 128 - -static void log_debug_slots(struct dlm_ls *ls, uint32_t gen, int num_slots, - struct rcom_slot *ro0, struct dlm_slot *array, - int array_size) -{ - char line[SLOT_DEBUG_LINE]; - int len = SLOT_DEBUG_LINE - 1; - int pos = 0; - int ret, i; - - if (!dlm_config.ci_log_debug) - return; - - memset(line, 0, sizeof(line)); - - if (array) { - for (i = 0; i < array_size; i++) { - if (!array[i].nodeid) - continue; - - ret = snprintf(line + pos, len - pos, " %d:%d", - array[i].slot, array[i].nodeid); - if (ret >= len - pos) - break; - pos += ret; - } - } else if (ro0) { - for (i = 0; i < num_slots; i++) { - ret = snprintf(line + pos, len - pos, " %d:%d", - ro0[i].ro_slot, ro0[i].ro_nodeid); - if (ret >= len - pos) - break; - pos += ret; - } - } - - log_debug(ls, "generation %u slots %d%s", gen, num_slots, line); -} - -int dlm_slots_copy_in(struct dlm_ls *ls) -{ - struct dlm_member *memb; - struct dlm_rcom *rc = ls->ls_recover_buf; - struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; - struct rcom_slot *ro0, *ro; - int our_nodeid = dlm_our_nodeid(); - int i, num_slots; - uint32_t gen; - - if (!dlm_slots_version(&rc->rc_header)) - return -1; - - gen = le32_to_cpu(rf->rf_generation); - if (gen <= ls->ls_generation) { - log_error(ls, "dlm_slots_copy_in gen %u old %u", - gen, ls->ls_generation); - } - ls->ls_generation = gen; - - num_slots = le16_to_cpu(rf->rf_num_slots); - if (!num_slots) - return -1; - - ro0 = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); - - for (i = 0, ro = ro0; i < num_slots; i++, ro++) { - ro->ro_nodeid = le32_to_cpu(ro->ro_nodeid); - ro->ro_slot = le16_to_cpu(ro->ro_slot); - } - - log_debug_slots(ls, gen, num_slots, ro0, NULL, 0); - - list_for_each_entry(memb, &ls->ls_nodes, list) { - for (i = 0, ro = ro0; i < num_slots; i++, ro++) { - if (ro->ro_nodeid != memb->nodeid) - continue; - memb->slot = ro->ro_slot; - memb->slot_prev = memb->slot; - break; - } - - if (memb->nodeid == our_nodeid) { - if (ls->ls_slot && ls->ls_slot != memb->slot) { - log_error(ls, "dlm_slots_copy_in our slot " - "changed %d %d", ls->ls_slot, - memb->slot); - return -1; - } - - if (!ls->ls_slot) - ls->ls_slot = memb->slot; - } - - if (!memb->slot) { - log_error(ls, "dlm_slots_copy_in nodeid %d no slot", - memb->nodeid); - return -1; - } - } - - return 0; -} - -/* for any nodes that do not support slots, we will not have set memb->slot - in wait_status_all(), so memb->slot will remain -1, and we will not - assign slots or set ls_num_slots here */ - -int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, - struct dlm_slot **slots_out, uint32_t *gen_out) -{ - struct dlm_member *memb; - struct dlm_slot *array; - int our_nodeid = dlm_our_nodeid(); - int array_size, max_slots, i; - int need = 0; - int max = 0; - int num = 0; - uint32_t gen = 0; - - /* our own memb struct will have slot -1 gen 0 */ - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (memb->nodeid == our_nodeid) { - memb->slot = ls->ls_slot; - memb->generation = ls->ls_generation; - break; - } - } - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (memb->generation > gen) - gen = memb->generation; - - /* node doesn't support slots */ - - if (memb->slot == -1) - return -1; - - /* node needs a slot assigned */ - - if (!memb->slot) - need++; - - /* node has a slot assigned */ - - num++; - - if (!max || max < memb->slot) - max = memb->slot; - - /* sanity check, once slot is assigned it shouldn't change */ - - if (memb->slot_prev && memb->slot && memb->slot_prev != memb->slot) { - log_error(ls, "nodeid %d slot changed %d %d", - memb->nodeid, memb->slot_prev, memb->slot); - return -1; - } - memb->slot_prev = memb->slot; - } - - array_size = max + need; - - array = kzalloc(array_size * sizeof(struct dlm_slot), GFP_NOFS); - if (!array) - return -ENOMEM; - - num = 0; - - /* fill in slots (offsets) that are used */ - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (!memb->slot) - continue; - - if (memb->slot > array_size) { - log_error(ls, "invalid slot number %d", memb->slot); - kfree(array); - return -1; - } - - array[memb->slot - 1].nodeid = memb->nodeid; - array[memb->slot - 1].slot = memb->slot; - num++; - } - - /* assign new slots from unused offsets */ - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (memb->slot) - continue; - - for (i = 0; i < array_size; i++) { - if (array[i].nodeid) - continue; - - memb->slot = i + 1; - memb->slot_prev = memb->slot; - array[i].nodeid = memb->nodeid; - array[i].slot = memb->slot; - num++; - - if (!ls->ls_slot && memb->nodeid == our_nodeid) - ls->ls_slot = memb->slot; - break; - } - - if (!memb->slot) { - log_error(ls, "no free slot found"); - kfree(array); - return -1; - } - } - - gen++; - - log_debug_slots(ls, gen, num, NULL, array, array_size); - - max_slots = (dlm_config.ci_buffer_size - sizeof(struct dlm_rcom) - - sizeof(struct rcom_config)) / sizeof(struct rcom_slot); - - if (num > max_slots) { - log_error(ls, "num_slots %d exceeds max_slots %d", - num, max_slots); - kfree(array); - return -1; - } - - *gen_out = gen; - *slots_out = array; - *slots_size = array_size; - *num_slots = num; - return 0; -} - -static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new) -{ - struct dlm_member *memb = NULL; - struct list_head *tmp; - struct list_head *newlist = &new->list; - struct list_head *head = &ls->ls_nodes; - - list_for_each(tmp, head) { - memb = list_entry(tmp, struct dlm_member, list); - if (new->nodeid < memb->nodeid) - break; - } - - if (!memb) - list_add_tail(newlist, head); - else { - /* FIXME: can use list macro here */ - newlist->prev = tmp->prev; - newlist->next = tmp; - tmp->prev->next = newlist; - tmp->prev = newlist; - } -} - -static int dlm_add_member(struct dlm_ls *ls, struct dlm_config_node *node) -{ - struct dlm_member *memb; - int error; - - memb = kzalloc(sizeof(struct dlm_member), GFP_NOFS); - if (!memb) - return -ENOMEM; - - error = dlm_lowcomms_connect_node(node->nodeid); - if (error < 0) { - kfree(memb); - return error; - } - - memb->nodeid = node->nodeid; - memb->weight = node->weight; - memb->comm_seq = node->comm_seq; - add_ordered_member(ls, memb); - ls->ls_num_nodes++; - return 0; -} - -static struct dlm_member *find_memb(struct list_head *head, int nodeid) -{ - struct dlm_member *memb; - - list_for_each_entry(memb, head, list) { - if (memb->nodeid == nodeid) - return memb; - } - return NULL; -} - -int dlm_is_member(struct dlm_ls *ls, int nodeid) -{ - if (find_memb(&ls->ls_nodes, nodeid)) - return 1; - return 0; -} - -int dlm_is_removed(struct dlm_ls *ls, int nodeid) -{ - if (find_memb(&ls->ls_nodes_gone, nodeid)) - return 1; - return 0; -} - -static void clear_memb_list(struct list_head *head) -{ - struct dlm_member *memb; - - while (!list_empty(head)) { - memb = list_entry(head->next, struct dlm_member, list); - list_del(&memb->list); - kfree(memb); - } -} - -void dlm_clear_members(struct dlm_ls *ls) -{ - clear_memb_list(&ls->ls_nodes); - ls->ls_num_nodes = 0; -} - -void dlm_clear_members_gone(struct dlm_ls *ls) -{ - clear_memb_list(&ls->ls_nodes_gone); -} - -static void make_member_array(struct dlm_ls *ls) -{ - struct dlm_member *memb; - int i, w, x = 0, total = 0, all_zero = 0, *array; - - kfree(ls->ls_node_array); - ls->ls_node_array = NULL; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (memb->weight) - total += memb->weight; - } - - /* all nodes revert to weight of 1 if all have weight 0 */ - - if (!total) { - total = ls->ls_num_nodes; - all_zero = 1; - } - - ls->ls_total_weight = total; - - array = kmalloc(sizeof(int) * total, GFP_NOFS); - if (!array) - return; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (!all_zero && !memb->weight) - continue; - - if (all_zero) - w = 1; - else - w = memb->weight; - - DLM_ASSERT(x < total, printk("total %d x %d\n", total, x);); - - for (i = 0; i < w; i++) - array[x++] = memb->nodeid; - } - - ls->ls_node_array = array; -} - -/* send a status request to all members just to establish comms connections */ - -static int ping_members(struct dlm_ls *ls) -{ - struct dlm_member *memb; - int error = 0; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - error = dlm_recovery_stopped(ls); - if (error) - break; - error = dlm_rcom_status(ls, memb->nodeid, 0); - if (error) - break; - } - if (error) - log_debug(ls, "ping_members aborted %d last nodeid %d", - error, ls->ls_recover_nodeid); - return error; -} - -static void dlm_lsop_recover_prep(struct dlm_ls *ls) -{ - if (!ls->ls_ops || !ls->ls_ops->recover_prep) - return; - ls->ls_ops->recover_prep(ls->ls_ops_arg); -} - -static void dlm_lsop_recover_slot(struct dlm_ls *ls, struct dlm_member *memb) -{ - struct dlm_slot slot; - uint32_t seq; - int error; - - if (!ls->ls_ops || !ls->ls_ops->recover_slot) - return; - - /* if there is no comms connection with this node - or the present comms connection is newer - than the one when this member was added, then - we consider the node to have failed (versus - being removed due to dlm_release_lockspace) */ - - error = dlm_comm_seq(memb->nodeid, &seq); - - if (!error && seq == memb->comm_seq) - return; - - slot.nodeid = memb->nodeid; - slot.slot = memb->slot; - - ls->ls_ops->recover_slot(ls->ls_ops_arg, &slot); -} - -void dlm_lsop_recover_done(struct dlm_ls *ls) -{ - struct dlm_member *memb; - struct dlm_slot *slots; - int i, num; - - if (!ls->ls_ops || !ls->ls_ops->recover_done) - return; - - num = ls->ls_num_nodes; - - slots = kzalloc(num * sizeof(struct dlm_slot), GFP_KERNEL); - if (!slots) - return; - - i = 0; - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (i == num) { - log_error(ls, "dlm_lsop_recover_done bad num %d", num); - goto out; - } - slots[i].nodeid = memb->nodeid; - slots[i].slot = memb->slot; - i++; - } - - ls->ls_ops->recover_done(ls->ls_ops_arg, slots, num, - ls->ls_slot, ls->ls_generation); - out: - kfree(slots); -} - -static struct dlm_config_node *find_config_node(struct dlm_recover *rv, - int nodeid) -{ - int i; - - for (i = 0; i < rv->nodes_count; i++) { - if (rv->nodes[i].nodeid == nodeid) - return &rv->nodes[i]; - } - return NULL; -} - -int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) -{ - struct dlm_member *memb, *safe; - struct dlm_config_node *node; - int i, error, neg = 0, low = -1; - - /* previously removed members that we've not finished removing need to - count as a negative change so the "neg" recovery steps will happen */ - - list_for_each_entry(memb, &ls->ls_nodes_gone, list) { - log_debug(ls, "prev removed member %d", memb->nodeid); - neg++; - } - - /* move departed members from ls_nodes to ls_nodes_gone */ - - list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) { - node = find_config_node(rv, memb->nodeid); - if (node && !node->new) - continue; - - if (!node) { - log_debug(ls, "remove member %d", memb->nodeid); - } else { - /* removed and re-added */ - log_debug(ls, "remove member %d comm_seq %u %u", - memb->nodeid, memb->comm_seq, node->comm_seq); - } - - neg++; - list_move(&memb->list, &ls->ls_nodes_gone); - ls->ls_num_nodes--; - dlm_lsop_recover_slot(ls, memb); - } - - /* add new members to ls_nodes */ - - for (i = 0; i < rv->nodes_count; i++) { - node = &rv->nodes[i]; - if (dlm_is_member(ls, node->nodeid)) - continue; - dlm_add_member(ls, node); - log_debug(ls, "add member %d", node->nodeid); - } - - list_for_each_entry(memb, &ls->ls_nodes, list) { - if (low == -1 || memb->nodeid < low) - low = memb->nodeid; - } - ls->ls_low_nodeid = low; - - make_member_array(ls); - *neg_out = neg; - - error = ping_members(ls); - if (!error || error == -EPROTO) { - /* new_lockspace() may be waiting to know if the config - is good or bad */ - ls->ls_members_result = error; - complete(&ls->ls_members_done); - } - - log_debug(ls, "dlm_recover_members %d nodes", ls->ls_num_nodes); - return error; -} - -/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before - dlm_ls_start() is called on any of them to start the new recovery. */ - -int dlm_ls_stop(struct dlm_ls *ls) -{ - int new; - - /* - * Prevent dlm_recv from being in the middle of something when we do - * the stop. This includes ensuring dlm_recv isn't processing a - * recovery message (rcom), while dlm_recoverd is aborting and - * resetting things from an in-progress recovery. i.e. we want - * dlm_recoverd to abort its recovery without worrying about dlm_recv - * processing an rcom at the same time. Stopping dlm_recv also makes - * it easy for dlm_receive_message() to check locking stopped and add a - * message to the requestqueue without races. - */ - - down_write(&ls->ls_recv_active); - - /* - * Abort any recovery that's in progress (see RECOVER_STOP, - * dlm_recovery_stopped()) and tell any other threads running in the - * dlm to quit any processing (see RUNNING, dlm_locking_stopped()). - */ - - spin_lock(&ls->ls_recover_lock); - set_bit(LSFL_RECOVER_STOP, &ls->ls_flags); - new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags); - ls->ls_recover_seq++; - spin_unlock(&ls->ls_recover_lock); - - /* - * Let dlm_recv run again, now any normal messages will be saved on the - * requestqueue for later. - */ - - up_write(&ls->ls_recv_active); - - /* - * This in_recovery lock does two things: - * 1) Keeps this function from returning until all threads are out - * of locking routines and locking is truly stopped. - * 2) Keeps any new requests from being processed until it's unlocked - * when recovery is complete. - */ - - if (new) { - set_bit(LSFL_RECOVER_DOWN, &ls->ls_flags); - wake_up_process(ls->ls_recoverd_task); - wait_event(ls->ls_recover_lock_wait, - test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); - } - - /* - * The recoverd suspend/resume makes sure that dlm_recoverd (if - * running) has noticed RECOVER_STOP above and quit processing the - * previous recovery. - */ - - dlm_recoverd_suspend(ls); - - spin_lock(&ls->ls_recover_lock); - kfree(ls->ls_slots); - ls->ls_slots = NULL; - ls->ls_num_slots = 0; - ls->ls_slots_size = 0; - ls->ls_recover_status = 0; - spin_unlock(&ls->ls_recover_lock); - - dlm_recoverd_resume(ls); - - if (!ls->ls_recover_begin) - ls->ls_recover_begin = jiffies; - - dlm_lsop_recover_prep(ls); - return 0; -} - -int dlm_ls_start(struct dlm_ls *ls) -{ - struct dlm_recover *rv = NULL, *rv_old; - struct dlm_config_node *nodes; - int error, count; - - rv = kzalloc(sizeof(struct dlm_recover), GFP_NOFS); - if (!rv) - return -ENOMEM; - - error = dlm_config_nodes(ls->ls_name, &nodes, &count); - if (error < 0) - goto fail; - - spin_lock(&ls->ls_recover_lock); - - /* the lockspace needs to be stopped before it can be started */ - - if (!dlm_locking_stopped(ls)) { - spin_unlock(&ls->ls_recover_lock); - log_error(ls, "start ignored: lockspace running"); - error = -EINVAL; - goto fail; - } - - rv->nodes = nodes; - rv->nodes_count = count; - rv->seq = ++ls->ls_recover_seq; - rv_old = ls->ls_recover_args; - ls->ls_recover_args = rv; - spin_unlock(&ls->ls_recover_lock); - - if (rv_old) { - log_error(ls, "unused recovery %llx %d", - (unsigned long long)rv_old->seq, rv_old->nodes_count); - kfree(rv_old->nodes); - kfree(rv_old); - } - - set_bit(LSFL_RECOVER_WORK, &ls->ls_flags); - wake_up_process(ls->ls_recoverd_task); - return 0; - - fail: - kfree(rv); - kfree(nodes); - return error; -} - diff --git a/kmod/dlm/member.h b/kmod/dlm/member.h deleted file mode 100644 index 3deb7066..00000000 --- a/kmod/dlm/member.h +++ /dev/null @@ -1,33 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __MEMBER_DOT_H__ -#define __MEMBER_DOT_H__ - -int dlm_ls_stop(struct dlm_ls *ls); -int dlm_ls_start(struct dlm_ls *ls); -void dlm_clear_members(struct dlm_ls *ls); -void dlm_clear_members_gone(struct dlm_ls *ls); -int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv,int *neg_out); -int dlm_is_removed(struct dlm_ls *ls, int nodeid); -int dlm_is_member(struct dlm_ls *ls, int nodeid); -int dlm_slots_version(struct dlm_header *h); -void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, - struct dlm_member *memb); -void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc); -int dlm_slots_copy_in(struct dlm_ls *ls); -int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, - struct dlm_slot **slots_out, uint32_t *gen_out); -void dlm_lsop_recover_done(struct dlm_ls *ls); - -#endif /* __MEMBER_DOT_H__ */ - diff --git a/kmod/dlm/memory.c b/kmod/dlm/memory.c deleted file mode 100644 index 7cd24bcc..00000000 --- a/kmod/dlm/memory.c +++ /dev/null @@ -1,96 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "config.h" -#include "memory.h" - -static struct kmem_cache *lkb_cache; -static struct kmem_cache *rsb_cache; - - -int __init dlm_memory_init(void) -{ - lkb_cache = kmem_cache_create("dlm_lkb", sizeof(struct dlm_lkb), - __alignof__(struct dlm_lkb), 0, NULL); - if (!lkb_cache) - return -ENOMEM; - - rsb_cache = kmem_cache_create("dlm_rsb", sizeof(struct dlm_rsb), - __alignof__(struct dlm_rsb), 0, NULL); - if (!rsb_cache) { - kmem_cache_destroy(lkb_cache); - return -ENOMEM; - } - - return 0; -} - -void dlm_memory_exit(void) -{ - if (lkb_cache) - kmem_cache_destroy(lkb_cache); - if (rsb_cache) - kmem_cache_destroy(rsb_cache); -} - -char *dlm_allocate_lvb(struct dlm_ls *ls) -{ - char *p; - - p = kzalloc(ls->ls_lvblen, GFP_NOFS); - return p; -} - -void dlm_free_lvb(char *p) -{ - kfree(p); -} - -struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - - r = kmem_cache_zalloc(rsb_cache, GFP_NOFS); - return r; -} - -void dlm_free_rsb(struct dlm_rsb *r) -{ - if (r->res_lvbptr) - dlm_free_lvb(r->res_lvbptr); - kmem_cache_free(rsb_cache, r); -} - -struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls) -{ - struct dlm_lkb *lkb; - - lkb = kmem_cache_zalloc(lkb_cache, GFP_NOFS); - return lkb; -} - -void dlm_free_lkb(struct dlm_lkb *lkb) -{ - if (lkb->lkb_flags & DLM_IFL_USER) { - struct dlm_user_args *ua; - ua = lkb->lkb_ua; - if (ua) { - if (ua->lksb.sb_lvbptr) - kfree(ua->lksb.sb_lvbptr); - kfree(ua); - } - } - kmem_cache_free(lkb_cache, lkb); -} - diff --git a/kmod/dlm/memory.h b/kmod/dlm/memory.h deleted file mode 100644 index 177c11cb..00000000 --- a/kmod/dlm/memory.h +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __MEMORY_DOT_H__ -#define __MEMORY_DOT_H__ - -int dlm_memory_init(void); -void dlm_memory_exit(void); -struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls); -void dlm_free_rsb(struct dlm_rsb *r); -struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls); -void dlm_free_lkb(struct dlm_lkb *l); -char *dlm_allocate_lvb(struct dlm_ls *ls); -void dlm_free_lvb(char *l); - -#endif /* __MEMORY_DOT_H__ */ - diff --git a/kmod/dlm/midcomms.c b/kmod/dlm/midcomms.c deleted file mode 100644 index f3396c62..00000000 --- a/kmod/dlm/midcomms.c +++ /dev/null @@ -1,137 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -/* - * midcomms.c - * - * This is the appallingly named "mid-level" comms layer. - * - * Its purpose is to take packets from the "real" comms layer, - * split them up into packets and pass them to the interested - * part of the locking mechanism. - * - * It also takes messages from the locking layer, formats them - * into packets and sends them to the comms layer. - */ - -#include "dlm_internal.h" -#include "lowcomms.h" -#include "config.h" -#include "lock.h" -#include "midcomms.h" - - -static void copy_from_cb(void *dst, const void *base, unsigned offset, - unsigned len, unsigned limit) -{ - unsigned copy = len; - - if ((copy + offset) > limit) - copy = limit - offset; - memcpy(dst, base + offset, copy); - len -= copy; - if (len) - memcpy(dst + copy, base, len); -} - -/* - * Called from the low-level comms layer to process a buffer of - * commands. - * - * Only complete messages are processed here, any "spare" bytes from - * the end of a buffer are saved and tacked onto the front of the next - * message that comes in. I doubt this will happen very often but we - * need to be able to cope with it and I don't want the task to be waiting - * for packets to come in when there is useful work to be done. - */ - -int dlm_process_incoming_buffer(int nodeid, const void *base, - unsigned offset, unsigned len, unsigned limit) -{ - union { - unsigned char __buf[DLM_INBUF_LEN]; - /* this is to force proper alignment on some arches */ - union dlm_packet p; - } __tmp; - union dlm_packet *p = &__tmp.p; - int ret = 0; - int err = 0; - uint16_t msglen; - uint32_t lockspace; - - while (len > sizeof(struct dlm_header)) { - - /* Copy just the header to check the total length. The - message may wrap around the end of the buffer back to the - start, so we need to use a temp buffer and copy_from_cb. */ - - copy_from_cb(p, base, offset, sizeof(struct dlm_header), - limit); - - msglen = le16_to_cpu(p->header.h_length); - lockspace = p->header.h_lockspace; - - err = -EINVAL; - if (msglen < sizeof(struct dlm_header)) - break; - if (p->header.h_cmd == DLM_MSG) { - if (msglen < sizeof(struct dlm_message)) - break; - } else { - if (msglen < sizeof(struct dlm_rcom)) - break; - } - err = -E2BIG; - if (msglen > dlm_config.ci_buffer_size) { - log_print("message size %d from %d too big, buf len %d", - msglen, nodeid, len); - break; - } - err = 0; - - /* If only part of the full message is contained in this - buffer, then do nothing and wait for lowcomms to call - us again later with more data. We return 0 meaning - we've consumed none of the input buffer. */ - - if (msglen > len) - break; - - /* Allocate a larger temp buffer if the full message won't fit - in the buffer on the stack (which should work for most - ordinary messages). */ - - if (msglen > sizeof(__tmp) && p == &__tmp.p) { - p = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS); - if (p == NULL) - return ret; - } - - copy_from_cb(p, base, offset, msglen, limit); - - BUG_ON(lockspace != p->header.h_lockspace); - - ret += msglen; - offset += msglen; - offset &= (limit - 1); - len -= msglen; - - dlm_receive_buffer(p, nodeid); - } - - if (p != &__tmp.p) - kfree(p); - - return err ? err : ret; -} - diff --git a/kmod/dlm/midcomms.h b/kmod/dlm/midcomms.h deleted file mode 100644 index 95852a5f..00000000 --- a/kmod/dlm/midcomms.h +++ /dev/null @@ -1,21 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __MIDCOMMS_DOT_H__ -#define __MIDCOMMS_DOT_H__ - -int dlm_process_incoming_buffer(int nodeid, const void *base, unsigned offset, - unsigned len, unsigned limit); - -#endif /* __MIDCOMMS_DOT_H__ */ - diff --git a/kmod/dlm/netlink.c b/kmod/dlm/netlink.c deleted file mode 100644 index 71275218..00000000 --- a/kmod/dlm/netlink.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2007 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ - -#include -#include "include/linux/dlm.h" -#include -#include - -#include "dlm_internal.h" - -static uint32_t dlm_nl_seqnum; -static uint32_t listener_nlportid; - -static struct genl_family family = { - .id = GENL_ID_GENERATE, - .name = DLM_GENL_NAME, - .version = DLM_GENL_VERSION, -}; - -static int prepare_data(u8 cmd, struct sk_buff **skbp, size_t size) -{ - struct sk_buff *skb; - void *data; - - skb = genlmsg_new(size, GFP_NOFS); - if (!skb) - return -ENOMEM; - - /* add the message headers */ - data = genlmsg_put(skb, 0, dlm_nl_seqnum++, &family, 0, cmd); - if (!data) { - nlmsg_free(skb); - return -EINVAL; - } - - *skbp = skb; - return 0; -} - -static struct dlm_lock_data *mk_data(struct sk_buff *skb) -{ - struct nlattr *ret; - - ret = nla_reserve(skb, DLM_TYPE_LOCK, sizeof(struct dlm_lock_data)); - if (!ret) - return NULL; - return nla_data(ret); -} - -static int send_data(struct sk_buff *skb) -{ - struct genlmsghdr *genlhdr = nlmsg_data((struct nlmsghdr *)skb->data); - void *data = genlmsg_data(genlhdr); - int rv; - - rv = genlmsg_end(skb, data); - if (rv < 0) { - nlmsg_free(skb); - return rv; - } - - return genlmsg_unicast(&init_net, skb, listener_nlportid); -} - -static int user_cmd(struct sk_buff *skb, struct genl_info *info) -{ - listener_nlportid = info->snd_portid; - printk("user_cmd nlpid %u\n", listener_nlportid); - return 0; -} - -static struct genl_ops dlm_nl_ops[] = { - { - .cmd = DLM_CMD_HELLO, - .doit = user_cmd, - }, -}; - -int __init dlm_netlink_init(void) -{ - return genl_register_family_with_ops(&family, dlm_nl_ops); -} - -void dlm_netlink_exit(void) -{ - genl_unregister_family(&family); -} - -static void fill_data(struct dlm_lock_data *data, struct dlm_lkb *lkb) -{ - struct dlm_rsb *r = lkb->lkb_resource; - - memset(data, 0, sizeof(struct dlm_lock_data)); - - data->version = DLM_LOCK_DATA_VERSION; - data->nodeid = lkb->lkb_nodeid; - data->ownpid = lkb->lkb_ownpid; - data->id = lkb->lkb_id; - data->remid = lkb->lkb_remid; - data->status = lkb->lkb_status; - data->grmode = lkb->lkb_grmode; - data->rqmode = lkb->lkb_rqmode; - if (lkb->lkb_ua) - data->xid = lkb->lkb_ua->xid; - if (r) { - data->lockspace_id = r->res_ls->ls_global_id; - data->resource_namelen = r->res_length; - memcpy(data->resource_name, r->res_name, r->res_length); - } -} - -void dlm_timeout_warn(struct dlm_lkb *lkb) -{ - struct sk_buff *uninitialized_var(send_skb); - struct dlm_lock_data *data; - size_t size; - int rv; - - size = nla_total_size(sizeof(struct dlm_lock_data)) + - nla_total_size(0); /* why this? */ - - rv = prepare_data(DLM_CMD_TIMEOUT, &send_skb, size); - if (rv < 0) - return; - - data = mk_data(send_skb); - if (!data) { - nlmsg_free(send_skb); - return; - } - - fill_data(data, lkb); - - send_data(send_skb); -} - diff --git a/kmod/dlm/plock.c b/kmod/dlm/plock.c deleted file mode 100644 index a33b4f27..00000000 --- a/kmod/dlm/plock.c +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License version 2. - */ - -#include -#include -#include -#include "include/linux/dlm.h" -#include -#include - -#include "dlm_internal.h" -#include "lockspace.h" - -static spinlock_t ops_lock; -static struct list_head send_list; -static struct list_head recv_list; -static wait_queue_head_t send_wq; -static wait_queue_head_t recv_wq; - -struct plock_op { - struct list_head list; - int done; - struct dlm_plock_info info; -}; - -struct plock_xop { - struct plock_op xop; - void *callback; - void *fl; - void *file; - struct file_lock flc; -}; - - -static inline void set_version(struct dlm_plock_info *info) -{ - info->version[0] = DLM_PLOCK_VERSION_MAJOR; - info->version[1] = DLM_PLOCK_VERSION_MINOR; - info->version[2] = DLM_PLOCK_VERSION_PATCH; -} - -static int check_version(struct dlm_plock_info *info) -{ - if ((DLM_PLOCK_VERSION_MAJOR != info->version[0]) || - (DLM_PLOCK_VERSION_MINOR < info->version[1])) { - log_print("plock device version mismatch: " - "kernel (%u.%u.%u), user (%u.%u.%u)", - DLM_PLOCK_VERSION_MAJOR, - DLM_PLOCK_VERSION_MINOR, - DLM_PLOCK_VERSION_PATCH, - info->version[0], - info->version[1], - info->version[2]); - return -EINVAL; - } - return 0; -} - -static void send_op(struct plock_op *op) -{ - set_version(&op->info); - INIT_LIST_HEAD(&op->list); - spin_lock(&ops_lock); - list_add_tail(&op->list, &send_list); - spin_unlock(&ops_lock); - wake_up(&send_wq); -} - -/* If a process was killed while waiting for the only plock on a file, - locks_remove_posix will not see any lock on the file so it won't - send an unlock-close to us to pass on to userspace to clean up the - abandoned waiter. So, we have to insert the unlock-close when the - lock call is interrupted. */ - -static void do_unlock_close(struct dlm_ls *ls, u64 number, - struct file *file, struct file_lock *fl) -{ - struct plock_op *op; - - op = kzalloc(sizeof(*op), GFP_NOFS); - if (!op) - return; - - op->info.optype = DLM_PLOCK_OP_UNLOCK; - op->info.pid = fl->fl_pid; - op->info.fsid = ls->ls_global_id; - op->info.number = number; - op->info.start = 0; - op->info.end = OFFSET_MAX; - if (fl->fl_lmops && fl->fl_lmops->lm_grant) - op->info.owner = (__u64) fl->fl_pid; - else - op->info.owner = (__u64)(long) fl->fl_owner; - - op->info.flags |= DLM_PLOCK_FL_CLOSE; - send_op(op); -} - -int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, - int cmd, struct file_lock *fl) -{ - struct dlm_ls *ls; - struct plock_op *op; - struct plock_xop *xop; - int rv; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - - xop = kzalloc(sizeof(*xop), GFP_NOFS); - if (!xop) { - rv = -ENOMEM; - goto out; - } - - op = &xop->xop; - op->info.optype = DLM_PLOCK_OP_LOCK; - op->info.pid = fl->fl_pid; - op->info.ex = (fl->fl_type == F_WRLCK); - op->info.wait = IS_SETLKW(cmd); - op->info.fsid = ls->ls_global_id; - op->info.number = number; - op->info.start = fl->fl_start; - op->info.end = fl->fl_end; - if (fl->fl_lmops && fl->fl_lmops->lm_grant) { - /* fl_owner is lockd which doesn't distinguish - processes on the nfs client */ - op->info.owner = (__u64) fl->fl_pid; - xop->callback = fl->fl_lmops->lm_grant; - locks_init_lock(&xop->flc); - locks_copy_lock(&xop->flc, fl); - xop->fl = fl; - xop->file = file; - } else { - op->info.owner = (__u64)(long) fl->fl_owner; - xop->callback = NULL; - } - - send_op(op); - - if (xop->callback == NULL) { - rv = wait_event_killable(recv_wq, (op->done != 0)); - if (rv == -ERESTARTSYS) { - log_debug(ls, "dlm_posix_lock: wait killed %llx", - (unsigned long long)number); - spin_lock(&ops_lock); - list_del(&op->list); - spin_unlock(&ops_lock); - kfree(xop); - do_unlock_close(ls, number, file, fl); - goto out; - } - } else { - rv = FILE_LOCK_DEFERRED; - goto out; - } - - spin_lock(&ops_lock); - if (!list_empty(&op->list)) { - log_error(ls, "dlm_posix_lock: op on list %llx", - (unsigned long long)number); - list_del(&op->list); - } - spin_unlock(&ops_lock); - - rv = op->info.rv; - - if (!rv) { - if (posix_lock_file_wait(file, fl) < 0) - log_error(ls, "dlm_posix_lock: vfs lock error %llx", - (unsigned long long)number); - } - - kfree(xop); -out: - dlm_put_lockspace(ls); - return rv; -} -EXPORT_SYMBOL_GPL(dlm_posix_lock); - -/* Returns failure iff a successful lock operation should be canceled */ -static int dlm_plock_callback(struct plock_op *op) -{ - struct file *file; - struct file_lock *fl; - struct file_lock *flc; - int (*notify)(void *, void *, int) = NULL; - struct plock_xop *xop = (struct plock_xop *)op; - int rv = 0; - - spin_lock(&ops_lock); - if (!list_empty(&op->list)) { - log_print("dlm_plock_callback: op on list %llx", - (unsigned long long)op->info.number); - list_del(&op->list); - } - spin_unlock(&ops_lock); - - /* check if the following 2 are still valid or make a copy */ - file = xop->file; - flc = &xop->flc; - fl = xop->fl; - notify = xop->callback; - - if (op->info.rv) { - notify(fl, NULL, op->info.rv); - goto out; - } - - /* got fs lock; bookkeep locally as well: */ - flc->fl_flags &= ~FL_SLEEP; - if (posix_lock_file(file, flc, NULL)) { - /* - * This can only happen in the case of kmalloc() failure. - * The filesystem's own lock is the authoritative lock, - * so a failure to get the lock locally is not a disaster. - * As long as the fs cannot reliably cancel locks (especially - * in a low-memory situation), we're better off ignoring - * this failure than trying to recover. - */ - log_print("dlm_plock_callback: vfs lock error %llx file %p fl %p", - (unsigned long long)op->info.number, file, fl); - } - - rv = notify(fl, NULL, 0); - if (rv) { - /* XXX: We need to cancel the fs lock here: */ - log_print("dlm_plock_callback: lock granted after lock request " - "failed; dangling lock!\n"); - goto out; - } - -out: - kfree(xop); - return rv; -} - -int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file, - struct file_lock *fl) -{ - struct dlm_ls *ls; - struct plock_op *op; - int rv; - unsigned char fl_flags = fl->fl_flags; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - - op = kzalloc(sizeof(*op), GFP_NOFS); - if (!op) { - rv = -ENOMEM; - goto out; - } - - /* cause the vfs unlock to return ENOENT if lock is not found */ - fl->fl_flags |= FL_EXISTS; - - rv = posix_lock_file_wait(file, fl); - if (rv == -ENOENT) { - rv = 0; - goto out_free; - } - if (rv < 0) { - log_error(ls, "dlm_posix_unlock: vfs unlock error %d %llx", - rv, (unsigned long long)number); - } - - op->info.optype = DLM_PLOCK_OP_UNLOCK; - op->info.pid = fl->fl_pid; - op->info.fsid = ls->ls_global_id; - op->info.number = number; - op->info.start = fl->fl_start; - op->info.end = fl->fl_end; - if (fl->fl_lmops && fl->fl_lmops->lm_grant) - op->info.owner = (__u64) fl->fl_pid; - else - op->info.owner = (__u64)(long) fl->fl_owner; - - if (fl->fl_flags & FL_CLOSE) { - op->info.flags |= DLM_PLOCK_FL_CLOSE; - send_op(op); - rv = 0; - goto out; - } - - send_op(op); - wait_event(recv_wq, (op->done != 0)); - - spin_lock(&ops_lock); - if (!list_empty(&op->list)) { - log_error(ls, "dlm_posix_unlock: op on list %llx", - (unsigned long long)number); - list_del(&op->list); - } - spin_unlock(&ops_lock); - - rv = op->info.rv; - - if (rv == -ENOENT) - rv = 0; - -out_free: - kfree(op); -out: - dlm_put_lockspace(ls); - fl->fl_flags = fl_flags; - return rv; -} -EXPORT_SYMBOL_GPL(dlm_posix_unlock); - -int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file, - struct file_lock *fl) -{ - struct dlm_ls *ls; - struct plock_op *op; - int rv; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -EINVAL; - - op = kzalloc(sizeof(*op), GFP_NOFS); - if (!op) { - rv = -ENOMEM; - goto out; - } - - op->info.optype = DLM_PLOCK_OP_GET; - op->info.pid = fl->fl_pid; - op->info.ex = (fl->fl_type == F_WRLCK); - op->info.fsid = ls->ls_global_id; - op->info.number = number; - op->info.start = fl->fl_start; - op->info.end = fl->fl_end; - if (fl->fl_lmops && fl->fl_lmops->lm_grant) - op->info.owner = (__u64) fl->fl_pid; - else - op->info.owner = (__u64)(long) fl->fl_owner; - - send_op(op); - wait_event(recv_wq, (op->done != 0)); - - spin_lock(&ops_lock); - if (!list_empty(&op->list)) { - log_error(ls, "dlm_posix_get: op on list %llx", - (unsigned long long)number); - list_del(&op->list); - } - spin_unlock(&ops_lock); - - /* info.rv from userspace is 1 for conflict, 0 for no-conflict, - -ENOENT if there are no locks on the file */ - - rv = op->info.rv; - - fl->fl_type = F_UNLCK; - if (rv == -ENOENT) - rv = 0; - else if (rv > 0) { - locks_init_lock(fl); - fl->fl_type = (op->info.ex) ? F_WRLCK : F_RDLCK; - fl->fl_flags = FL_POSIX; - fl->fl_pid = op->info.pid; - fl->fl_start = op->info.start; - fl->fl_end = op->info.end; - rv = 0; - } - - kfree(op); -out: - dlm_put_lockspace(ls); - return rv; -} -EXPORT_SYMBOL_GPL(dlm_posix_get); - -/* a read copies out one plock request from the send list */ -static ssize_t dev_read(struct file *file, char __user *u, size_t count, - loff_t *ppos) -{ - struct dlm_plock_info info; - struct plock_op *op = NULL; - - if (count < sizeof(info)) - return -EINVAL; - - spin_lock(&ops_lock); - if (!list_empty(&send_list)) { - op = list_entry(send_list.next, struct plock_op, list); - if (op->info.flags & DLM_PLOCK_FL_CLOSE) - list_del(&op->list); - else - list_move(&op->list, &recv_list); - memcpy(&info, &op->info, sizeof(info)); - } - spin_unlock(&ops_lock); - - if (!op) - return -EAGAIN; - - /* there is no need to get a reply from userspace for unlocks - that were generated by the vfs cleaning up for a close - (the process did not make an unlock call). */ - - if (op->info.flags & DLM_PLOCK_FL_CLOSE) - kfree(op); - - if (copy_to_user(u, &info, sizeof(info))) - return -EFAULT; - return sizeof(info); -} - -/* a write copies in one plock result that should match a plock_op - on the recv list */ -static ssize_t dev_write(struct file *file, const char __user *u, size_t count, - loff_t *ppos) -{ - struct dlm_plock_info info; - struct plock_op *op; - int found = 0, do_callback = 0; - - if (count != sizeof(info)) - return -EINVAL; - - if (copy_from_user(&info, u, sizeof(info))) - return -EFAULT; - - if (check_version(&info)) - return -EINVAL; - - spin_lock(&ops_lock); - list_for_each_entry(op, &recv_list, list) { - if (op->info.fsid == info.fsid && - op->info.number == info.number && - op->info.owner == info.owner) { - struct plock_xop *xop = (struct plock_xop *)op; - list_del_init(&op->list); - memcpy(&op->info, &info, sizeof(info)); - if (xop->callback) - do_callback = 1; - else - op->done = 1; - found = 1; - break; - } - } - spin_unlock(&ops_lock); - - if (found) { - if (do_callback) - dlm_plock_callback(op); - else - wake_up(&recv_wq); - } else - log_print("dev_write no op %x %llx", info.fsid, - (unsigned long long)info.number); - return count; -} - -static unsigned int dev_poll(struct file *file, poll_table *wait) -{ - unsigned int mask = 0; - - poll_wait(file, &send_wq, wait); - - spin_lock(&ops_lock); - if (!list_empty(&send_list)) - mask = POLLIN | POLLRDNORM; - spin_unlock(&ops_lock); - - return mask; -} - -static const struct file_operations dev_fops = { - .read = dev_read, - .write = dev_write, - .poll = dev_poll, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static struct miscdevice plock_dev_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = DLM_PLOCK_MISC_NAME, - .fops = &dev_fops -}; - -int dlm_plock_init(void) -{ - int rv; - - spin_lock_init(&ops_lock); - INIT_LIST_HEAD(&send_list); - INIT_LIST_HEAD(&recv_list); - init_waitqueue_head(&send_wq); - init_waitqueue_head(&recv_wq); - - rv = misc_register(&plock_dev_misc); - if (rv) - log_print("dlm_plock_init: misc_register failed %d", rv); - return rv; -} - -void dlm_plock_exit(void) -{ - if (misc_deregister(&plock_dev_misc) < 0) - log_print("dlm_plock_exit: misc_deregister failed"); -} - diff --git a/kmod/dlm/rcom.c b/kmod/dlm/rcom.c deleted file mode 100644 index 7af563a2..00000000 --- a/kmod/dlm/rcom.c +++ /dev/null @@ -1,656 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "member.h" -#include "lowcomms.h" -#include "midcomms.h" -#include "rcom.h" -#include "recover.h" -#include "dir.h" -#include "config.h" -#include "memory.h" -#include "lock.h" -#include "util.h" - -static int rcom_response(struct dlm_ls *ls) -{ - return test_bit(LSFL_RCOM_READY, &ls->ls_flags); -} - -static int create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len, - struct dlm_rcom **rc_ret, struct dlm_mhandle **mh_ret) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - char *mb; - int mb_len = sizeof(struct dlm_rcom) + len; - - mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb); - if (!mh) { - log_print("create_rcom to %d type %d len %d ENOBUFS", - to_nodeid, type, len); - return -ENOBUFS; - } - memset(mb, 0, mb_len); - - rc = (struct dlm_rcom *) mb; - - rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); - rc->rc_header.h_lockspace = ls->ls_global_id; - rc->rc_header.h_nodeid = dlm_our_nodeid(); - rc->rc_header.h_length = mb_len; - rc->rc_header.h_cmd = DLM_RCOM; - - rc->rc_type = type; - - spin_lock(&ls->ls_recover_lock); - rc->rc_seq = ls->ls_recover_seq; - spin_unlock(&ls->ls_recover_lock); - - *mh_ret = mh; - *rc_ret = rc; - return 0; -} - -static void send_rcom(struct dlm_ls *ls, struct dlm_mhandle *mh, - struct dlm_rcom *rc) -{ - dlm_rcom_out(rc); - dlm_lowcomms_commit_buffer(mh); -} - -static void set_rcom_status(struct dlm_ls *ls, struct rcom_status *rs, - uint32_t flags) -{ - rs->rs_flags = cpu_to_le32(flags); -} - -/* When replying to a status request, a node also sends back its - configuration values. The requesting node then checks that the remote - node is configured the same way as itself. */ - -static void set_rcom_config(struct dlm_ls *ls, struct rcom_config *rf, - uint32_t num_slots) -{ - rf->rf_lvblen = cpu_to_le32(ls->ls_lvblen); - rf->rf_lsflags = cpu_to_le32(ls->ls_exflags); - - rf->rf_our_slot = cpu_to_le16(ls->ls_slot); - rf->rf_num_slots = cpu_to_le16(num_slots); - rf->rf_generation = cpu_to_le32(ls->ls_generation); -} - -static int check_rcom_config(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid) -{ - struct rcom_config *rf = (struct rcom_config *) rc->rc_buf; - - if ((rc->rc_header.h_version & 0xFFFF0000) != DLM_HEADER_MAJOR) { - log_error(ls, "version mismatch: %x nodeid %d: %x", - DLM_HEADER_MAJOR | DLM_HEADER_MINOR, nodeid, - rc->rc_header.h_version); - return -EPROTO; - } - - if (le32_to_cpu(rf->rf_lvblen) != ls->ls_lvblen || - le32_to_cpu(rf->rf_lsflags) != ls->ls_exflags) { - log_error(ls, "config mismatch: %d,%x nodeid %d: %d,%x", - ls->ls_lvblen, ls->ls_exflags, nodeid, - le32_to_cpu(rf->rf_lvblen), - le32_to_cpu(rf->rf_lsflags)); - return -EPROTO; - } - return 0; -} - -static void allow_sync_reply(struct dlm_ls *ls, uint64_t *new_seq) -{ - spin_lock(&ls->ls_rcom_spin); - *new_seq = ++ls->ls_rcom_seq; - set_bit(LSFL_RCOM_WAIT, &ls->ls_flags); - spin_unlock(&ls->ls_rcom_spin); -} - -static void disallow_sync_reply(struct dlm_ls *ls) -{ - spin_lock(&ls->ls_rcom_spin); - clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); - clear_bit(LSFL_RCOM_READY, &ls->ls_flags); - spin_unlock(&ls->ls_rcom_spin); -} - -/* - * low nodeid gathers one slot value at a time from each node. - * it sets need_slots=0, and saves rf_our_slot returned from each - * rcom_config. - * - * other nodes gather all slot values at once from the low nodeid. - * they set need_slots=1, and ignore the rf_our_slot returned from each - * rcom_config. they use the rf_num_slots returned from the low - * node's rcom_config. - */ - -int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - int error = 0; - - ls->ls_recover_nodeid = nodeid; - - if (nodeid == dlm_our_nodeid()) { - rc = ls->ls_recover_buf; - rc->rc_result = dlm_recover_status(ls); - goto out; - } - - error = create_rcom(ls, nodeid, DLM_RCOM_STATUS, - sizeof(struct rcom_status), &rc, &mh); - if (error) - goto out; - - set_rcom_status(ls, (struct rcom_status *)rc->rc_buf, status_flags); - - allow_sync_reply(ls, &rc->rc_id); - memset(ls->ls_recover_buf, 0, dlm_config.ci_buffer_size); - - send_rcom(ls, mh, rc); - - error = dlm_wait_function(ls, &rcom_response); - disallow_sync_reply(ls); - if (error) - goto out; - - rc = ls->ls_recover_buf; - - if (rc->rc_result == -ESRCH) { - /* we pretend the remote lockspace exists with 0 status */ - log_debug(ls, "remote node %d not ready", nodeid); - rc->rc_result = 0; - error = 0; - } else { - error = check_rcom_config(ls, rc, nodeid); - } - - /* the caller looks at rc_result for the remote recovery status */ - out: - return error; -} - -static void receive_rcom_status(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - struct rcom_status *rs; - uint32_t status; - int nodeid = rc_in->rc_header.h_nodeid; - int len = sizeof(struct rcom_config); - int num_slots = 0; - int error; - - if (!dlm_slots_version(&rc_in->rc_header)) { - status = dlm_recover_status(ls); - goto do_create; - } - - rs = (struct rcom_status *)rc_in->rc_buf; - - if (!(le32_to_cpu(rs->rs_flags) & DLM_RSF_NEED_SLOTS)) { - status = dlm_recover_status(ls); - goto do_create; - } - - spin_lock(&ls->ls_recover_lock); - status = ls->ls_recover_status; - num_slots = ls->ls_num_slots; - spin_unlock(&ls->ls_recover_lock); - len += num_slots * sizeof(struct rcom_slot); - - do_create: - error = create_rcom(ls, nodeid, DLM_RCOM_STATUS_REPLY, - len, &rc, &mh); - if (error) - return; - - rc->rc_id = rc_in->rc_id; - rc->rc_seq_reply = rc_in->rc_seq; - rc->rc_result = status; - - set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, num_slots); - - if (!num_slots) - goto do_send; - - spin_lock(&ls->ls_recover_lock); - if (ls->ls_num_slots != num_slots) { - spin_unlock(&ls->ls_recover_lock); - log_debug(ls, "receive_rcom_status num_slots %d to %d", - num_slots, ls->ls_num_slots); - rc->rc_result = 0; - set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, 0); - goto do_send; - } - - dlm_slots_copy_out(ls, rc); - spin_unlock(&ls->ls_recover_lock); - - do_send: - send_rcom(ls, mh, rc); -} - -static void receive_sync_reply(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - spin_lock(&ls->ls_rcom_spin); - if (!test_bit(LSFL_RCOM_WAIT, &ls->ls_flags) || - rc_in->rc_id != ls->ls_rcom_seq) { - log_debug(ls, "reject reply %d from %d seq %llx expect %llx", - rc_in->rc_type, rc_in->rc_header.h_nodeid, - (unsigned long long)rc_in->rc_id, - (unsigned long long)ls->ls_rcom_seq); - goto out; - } - memcpy(ls->ls_recover_buf, rc_in, rc_in->rc_header.h_length); - set_bit(LSFL_RCOM_READY, &ls->ls_flags); - clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); - wake_up(&ls->ls_wait_general); - out: - spin_unlock(&ls->ls_rcom_spin); -} - -int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name, int last_len) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - int error = 0; - - ls->ls_recover_nodeid = nodeid; - - error = create_rcom(ls, nodeid, DLM_RCOM_NAMES, last_len, &rc, &mh); - if (error) - goto out; - memcpy(rc->rc_buf, last_name, last_len); - - allow_sync_reply(ls, &rc->rc_id); - memset(ls->ls_recover_buf, 0, dlm_config.ci_buffer_size); - - send_rcom(ls, mh, rc); - - error = dlm_wait_function(ls, &rcom_response); - disallow_sync_reply(ls); - out: - return error; -} - -static void receive_rcom_names(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - int error, inlen, outlen, nodeid; - - nodeid = rc_in->rc_header.h_nodeid; - inlen = rc_in->rc_header.h_length - sizeof(struct dlm_rcom); - outlen = dlm_config.ci_buffer_size - sizeof(struct dlm_rcom); - - error = create_rcom(ls, nodeid, DLM_RCOM_NAMES_REPLY, outlen, &rc, &mh); - if (error) - return; - rc->rc_id = rc_in->rc_id; - rc->rc_seq_reply = rc_in->rc_seq; - - dlm_copy_master_names(ls, rc_in->rc_buf, inlen, rc->rc_buf, outlen, - nodeid); - send_rcom(ls, mh, rc); -} - -int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - struct dlm_ls *ls = r->res_ls; - int error; - - error = create_rcom(ls, dir_nodeid, DLM_RCOM_LOOKUP, r->res_length, - &rc, &mh); - if (error) - goto out; - memcpy(rc->rc_buf, r->res_name, r->res_length); - rc->rc_id = (unsigned long) r->res_id; - - send_rcom(ls, mh, rc); - out: - return error; -} - -int dlm_send_rcom_lookup_dump(struct dlm_rsb *r, int to_nodeid) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - struct dlm_ls *ls = r->res_ls; - int error; - - error = create_rcom(ls, to_nodeid, DLM_RCOM_LOOKUP, r->res_length, - &rc, &mh); - if (error) - goto out; - memcpy(rc->rc_buf, r->res_name, r->res_length); - rc->rc_id = 0xFFFFFFFF; - - send_rcom(ls, mh, rc); - out: - return error; -} - -static void receive_rcom_lookup(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - int error, ret_nodeid, nodeid = rc_in->rc_header.h_nodeid; - int len = rc_in->rc_header.h_length - sizeof(struct dlm_rcom); - - error = create_rcom(ls, nodeid, DLM_RCOM_LOOKUP_REPLY, 0, &rc, &mh); - if (error) - return; - - if (rc_in->rc_id == 0xFFFFFFFF) { - log_error(ls, "receive_rcom_lookup dump from %d", nodeid); - dlm_dump_rsb_name(ls, rc_in->rc_buf, len); - return; - } - - error = dlm_master_lookup(ls, nodeid, rc_in->rc_buf, len, - DLM_LU_RECOVER_MASTER, &ret_nodeid, NULL); - if (error) - ret_nodeid = error; - rc->rc_result = ret_nodeid; - rc->rc_id = rc_in->rc_id; - rc->rc_seq_reply = rc_in->rc_seq; - - send_rcom(ls, mh, rc); -} - -static void receive_rcom_lookup_reply(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - dlm_recover_master_reply(ls, rc_in); -} - -static void pack_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, - struct rcom_lock *rl) -{ - memset(rl, 0, sizeof(*rl)); - - rl->rl_ownpid = cpu_to_le32(lkb->lkb_ownpid); - rl->rl_lkid = cpu_to_le32(lkb->lkb_id); - rl->rl_exflags = cpu_to_le32(lkb->lkb_exflags); - rl->rl_flags = cpu_to_le32(lkb->lkb_flags); - rl->rl_lvbseq = cpu_to_le32(lkb->lkb_lvbseq); - rl->rl_rqmode = lkb->lkb_rqmode; - rl->rl_grmode = lkb->lkb_grmode; - rl->rl_status = lkb->lkb_status; - rl->rl_wait_type = cpu_to_le16(lkb->lkb_wait_type); - - if (lkb->lkb_bastfn || lkb->lkb_rbastfn) - rl->rl_asts |= DLM_CB_BAST; - if (lkb->lkb_astfn) - rl->rl_asts |= DLM_CB_CAST; - - rl->rl_namelen = cpu_to_le16(r->res_length); - memcpy(rl->rl_name, r->res_name, r->res_length); - - /* FIXME: might we have an lvb without DLM_LKF_VALBLK set ? - If so, receive_rcom_lock_args() won't take this copy. */ - - if (lkb->lkb_lvbptr) - memcpy(rl->rl_lvb, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); -} - -int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) -{ - struct dlm_ls *ls = r->res_ls; - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - struct rcom_lock *rl; - int error, len = sizeof(struct rcom_lock); - - if (lkb->lkb_lvbptr) - len += ls->ls_lvblen; - - error = create_rcom(ls, r->res_nodeid, DLM_RCOM_LOCK, len, &rc, &mh); - if (error) - goto out; - - rl = (struct rcom_lock *) rc->rc_buf; - pack_rcom_lock(r, lkb, rl); - rc->rc_id = (unsigned long) r; - - send_rcom(ls, mh, rc); - out: - return error; -} - -/* needs at least dlm_rcom + rcom_lock */ -static void receive_rcom_lock(struct dlm_ls *ls, struct dlm_rcom *rc_in) -{ - struct dlm_rcom *rc; - struct dlm_mhandle *mh; - int error, nodeid = rc_in->rc_header.h_nodeid; - - dlm_recover_master_copy(ls, rc_in); - - error = create_rcom(ls, nodeid, DLM_RCOM_LOCK_REPLY, - sizeof(struct rcom_lock), &rc, &mh); - if (error) - return; - - /* We send back the same rcom_lock struct we received, but - dlm_recover_master_copy() has filled in rl_remid and rl_result */ - - memcpy(rc->rc_buf, rc_in->rc_buf, sizeof(struct rcom_lock)); - rc->rc_id = rc_in->rc_id; - rc->rc_seq_reply = rc_in->rc_seq; - - send_rcom(ls, mh, rc); -} - -/* If the lockspace doesn't exist then still send a status message - back; it's possible that it just doesn't have its global_id yet. */ - -int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in) -{ - struct dlm_rcom *rc; - struct rcom_config *rf; - struct dlm_mhandle *mh; - char *mb; - int mb_len = sizeof(struct dlm_rcom) + sizeof(struct rcom_config); - - mh = dlm_lowcomms_get_buffer(nodeid, mb_len, GFP_NOFS, &mb); - if (!mh) - return -ENOBUFS; - memset(mb, 0, mb_len); - - rc = (struct dlm_rcom *) mb; - - rc->rc_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR); - rc->rc_header.h_lockspace = rc_in->rc_header.h_lockspace; - rc->rc_header.h_nodeid = dlm_our_nodeid(); - rc->rc_header.h_length = mb_len; - rc->rc_header.h_cmd = DLM_RCOM; - - rc->rc_type = DLM_RCOM_STATUS_REPLY; - rc->rc_id = rc_in->rc_id; - rc->rc_seq_reply = rc_in->rc_seq; - rc->rc_result = -ESRCH; - - rf = (struct rcom_config *) rc->rc_buf; - rf->rf_lvblen = cpu_to_le32(~0U); - - dlm_rcom_out(rc); - dlm_lowcomms_commit_buffer(mh); - - return 0; -} - -/* - * Ignore messages for stage Y before we set - * recover_status bit for stage X: - * - * recover_status = 0 - * - * dlm_recover_members() - * - send nothing - * - recv nothing - * - ignore NAMES, NAMES_REPLY - * - ignore LOOKUP, LOOKUP_REPLY - * - ignore LOCK, LOCK_REPLY - * - * recover_status |= NODES - * - * dlm_recover_members_wait() - * - * dlm_recover_directory() - * - send NAMES - * - recv NAMES_REPLY - * - ignore LOOKUP, LOOKUP_REPLY - * - ignore LOCK, LOCK_REPLY - * - * recover_status |= DIR - * - * dlm_recover_directory_wait() - * - * dlm_recover_masters() - * - send LOOKUP - * - recv LOOKUP_REPLY - * - * dlm_recover_locks() - * - send LOCKS - * - recv LOCKS_REPLY - * - * recover_status |= LOCKS - * - * dlm_recover_locks_wait() - * - * recover_status |= DONE - */ - -/* Called by dlm_recv; corresponds to dlm_receive_message() but special - recovery-only comms are sent through here. */ - -void dlm_receive_rcom(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid) -{ - int lock_size = sizeof(struct dlm_rcom) + sizeof(struct rcom_lock); - int stop, reply = 0, names = 0, lookup = 0, lock = 0; - uint32_t status; - uint64_t seq; - - switch (rc->rc_type) { - case DLM_RCOM_STATUS_REPLY: - reply = 1; - break; - case DLM_RCOM_NAMES: - names = 1; - break; - case DLM_RCOM_NAMES_REPLY: - names = 1; - reply = 1; - break; - case DLM_RCOM_LOOKUP: - lookup = 1; - break; - case DLM_RCOM_LOOKUP_REPLY: - lookup = 1; - reply = 1; - break; - case DLM_RCOM_LOCK: - lock = 1; - break; - case DLM_RCOM_LOCK_REPLY: - lock = 1; - reply = 1; - break; - }; - - spin_lock(&ls->ls_recover_lock); - status = ls->ls_recover_status; - stop = test_bit(LSFL_RECOVER_STOP, &ls->ls_flags); - seq = ls->ls_recover_seq; - spin_unlock(&ls->ls_recover_lock); - - if (stop && (rc->rc_type != DLM_RCOM_STATUS)) - goto ignore; - - if (reply && (rc->rc_seq_reply != seq)) - goto ignore; - - if (!(status & DLM_RS_NODES) && (names || lookup || lock)) - goto ignore; - - if (!(status & DLM_RS_DIR) && (lookup || lock)) - goto ignore; - - switch (rc->rc_type) { - case DLM_RCOM_STATUS: - receive_rcom_status(ls, rc); - break; - - case DLM_RCOM_NAMES: - receive_rcom_names(ls, rc); - break; - - case DLM_RCOM_LOOKUP: - receive_rcom_lookup(ls, rc); - break; - - case DLM_RCOM_LOCK: - if (rc->rc_header.h_length < lock_size) - goto Eshort; - receive_rcom_lock(ls, rc); - break; - - case DLM_RCOM_STATUS_REPLY: - receive_sync_reply(ls, rc); - break; - - case DLM_RCOM_NAMES_REPLY: - receive_sync_reply(ls, rc); - break; - - case DLM_RCOM_LOOKUP_REPLY: - receive_rcom_lookup_reply(ls, rc); - break; - - case DLM_RCOM_LOCK_REPLY: - if (rc->rc_header.h_length < lock_size) - goto Eshort; - dlm_recover_process_copy(ls, rc); - break; - - default: - log_error(ls, "receive_rcom bad type %d", rc->rc_type); - } - return; - -ignore: - log_limit(ls, "dlm_receive_rcom ignore msg %d " - "from %d %llu %llu recover seq %llu sts %x gen %u", - rc->rc_type, - nodeid, - (unsigned long long)rc->rc_seq, - (unsigned long long)rc->rc_seq_reply, - (unsigned long long)seq, - status, ls->ls_generation); - return; -Eshort: - log_error(ls, "recovery message %d from %d is too short", - rc->rc_type, nodeid); -} - diff --git a/kmod/dlm/rcom.h b/kmod/dlm/rcom.h deleted file mode 100644 index f8e24346..00000000 --- a/kmod/dlm/rcom.h +++ /dev/null @@ -1,26 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __RCOM_DOT_H__ -#define __RCOM_DOT_H__ - -int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags); -int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name,int last_len); -int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid); -int dlm_send_rcom_lookup_dump(struct dlm_rsb *r, int to_nodeid); -int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); -void dlm_receive_rcom(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid); -int dlm_send_ls_not_ready(int nodeid, struct dlm_rcom *rc_in); - -#endif - diff --git a/kmod/dlm/recover.c b/kmod/dlm/recover.c deleted file mode 100644 index a6bc63f6..00000000 --- a/kmod/dlm/recover.c +++ /dev/null @@ -1,955 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "dir.h" -#include "config.h" -#include "ast.h" -#include "memory.h" -#include "rcom.h" -#include "lock.h" -#include "lowcomms.h" -#include "member.h" -#include "recover.h" - - -/* - * Recovery waiting routines: these functions wait for a particular reply from - * a remote node, or for the remote node to report a certain status. They need - * to abort if the lockspace is stopped indicating a node has failed (perhaps - * the one being waited for). - */ - -/* - * Wait until given function returns non-zero or lockspace is stopped - * (LS_RECOVERY_STOP set due to failure of a node in ls_nodes). When another - * function thinks it could have completed the waited-on task, they should wake - * up ls_wait_general to get an immediate response rather than waiting for the - * timeout. This uses a timeout so it can check periodically if the wait - * should abort due to node failure (which doesn't cause a wake_up). - * This should only be called by the dlm_recoverd thread. - */ - -int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)) -{ - int error = 0; - int rv; - - while (1) { - rv = wait_event_timeout(ls->ls_wait_general, - testfn(ls) || dlm_recovery_stopped(ls), - dlm_config.ci_recover_timer * HZ); - if (rv) - break; - } - - if (dlm_recovery_stopped(ls)) { - log_debug(ls, "dlm_wait_function aborted"); - error = -EINTR; - } - return error; -} - -/* - * An efficient way for all nodes to wait for all others to have a certain - * status. The node with the lowest nodeid polls all the others for their - * status (wait_status_all) and all the others poll the node with the low id - * for its accumulated result (wait_status_low). When all nodes have set - * status flag X, then status flag X_ALL will be set on the low nodeid. - */ - -uint32_t dlm_recover_status(struct dlm_ls *ls) -{ - uint32_t status; - spin_lock(&ls->ls_recover_lock); - status = ls->ls_recover_status; - spin_unlock(&ls->ls_recover_lock); - return status; -} - -static void _set_recover_status(struct dlm_ls *ls, uint32_t status) -{ - ls->ls_recover_status |= status; -} - -void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status) -{ - spin_lock(&ls->ls_recover_lock); - _set_recover_status(ls, status); - spin_unlock(&ls->ls_recover_lock); -} - -static int wait_status_all(struct dlm_ls *ls, uint32_t wait_status, - int save_slots) -{ - struct dlm_rcom *rc = ls->ls_recover_buf; - struct dlm_member *memb; - int error = 0, delay; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - delay = 0; - for (;;) { - if (dlm_recovery_stopped(ls)) { - error = -EINTR; - goto out; - } - - error = dlm_rcom_status(ls, memb->nodeid, 0); - if (error) - goto out; - - if (save_slots) - dlm_slot_save(ls, rc, memb); - - if (rc->rc_result & wait_status) - break; - if (delay < 1000) - delay += 20; - msleep(delay); - } - } - out: - return error; -} - -static int wait_status_low(struct dlm_ls *ls, uint32_t wait_status, - uint32_t status_flags) -{ - struct dlm_rcom *rc = ls->ls_recover_buf; - int error = 0, delay = 0, nodeid = ls->ls_low_nodeid; - - for (;;) { - if (dlm_recovery_stopped(ls)) { - error = -EINTR; - goto out; - } - - error = dlm_rcom_status(ls, nodeid, status_flags); - if (error) - break; - - if (rc->rc_result & wait_status) - break; - if (delay < 1000) - delay += 20; - msleep(delay); - } - out: - return error; -} - -static int wait_status(struct dlm_ls *ls, uint32_t status) -{ - uint32_t status_all = status << 1; - int error; - - if (ls->ls_low_nodeid == dlm_our_nodeid()) { - error = wait_status_all(ls, status, 0); - if (!error) - dlm_set_recover_status(ls, status_all); - } else - error = wait_status_low(ls, status_all, 0); - - return error; -} - -int dlm_recover_members_wait(struct dlm_ls *ls) -{ - struct dlm_member *memb; - struct dlm_slot *slots; - int num_slots, slots_size; - int error, rv; - uint32_t gen; - - list_for_each_entry(memb, &ls->ls_nodes, list) { - memb->slot = -1; - memb->generation = 0; - } - - if (ls->ls_low_nodeid == dlm_our_nodeid()) { - error = wait_status_all(ls, DLM_RS_NODES, 1); - if (error) - goto out; - - /* slots array is sparse, slots_size may be > num_slots */ - - rv = dlm_slots_assign(ls, &num_slots, &slots_size, &slots, &gen); - if (!rv) { - spin_lock(&ls->ls_recover_lock); - _set_recover_status(ls, DLM_RS_NODES_ALL); - ls->ls_num_slots = num_slots; - ls->ls_slots_size = slots_size; - ls->ls_slots = slots; - ls->ls_generation = gen; - spin_unlock(&ls->ls_recover_lock); - } else { - dlm_set_recover_status(ls, DLM_RS_NODES_ALL); - } - } else { - error = wait_status_low(ls, DLM_RS_NODES_ALL, DLM_RSF_NEED_SLOTS); - if (error) - goto out; - - dlm_slots_copy_in(ls); - } - out: - return error; -} - -int dlm_recover_directory_wait(struct dlm_ls *ls) -{ - return wait_status(ls, DLM_RS_DIR); -} - -int dlm_recover_locks_wait(struct dlm_ls *ls) -{ - return wait_status(ls, DLM_RS_LOCKS); -} - -int dlm_recover_done_wait(struct dlm_ls *ls) -{ - return wait_status(ls, DLM_RS_DONE); -} - -/* - * The recover_list contains all the rsb's for which we've requested the new - * master nodeid. As replies are returned from the resource directories the - * rsb's are removed from the list. When the list is empty we're done. - * - * The recover_list is later similarly used for all rsb's for which we've sent - * new lkb's and need to receive new corresponding lkid's. - * - * We use the address of the rsb struct as a simple local identifier for the - * rsb so we can match an rcom reply with the rsb it was sent for. - */ - -static int recover_list_empty(struct dlm_ls *ls) -{ - int empty; - - spin_lock(&ls->ls_recover_list_lock); - empty = list_empty(&ls->ls_recover_list); - spin_unlock(&ls->ls_recover_list_lock); - - return empty; -} - -static void recover_list_add(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - - spin_lock(&ls->ls_recover_list_lock); - if (list_empty(&r->res_recover_list)) { - list_add_tail(&r->res_recover_list, &ls->ls_recover_list); - ls->ls_recover_list_count++; - dlm_hold_rsb(r); - } - spin_unlock(&ls->ls_recover_list_lock); -} - -static void recover_list_del(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - - spin_lock(&ls->ls_recover_list_lock); - list_del_init(&r->res_recover_list); - ls->ls_recover_list_count--; - spin_unlock(&ls->ls_recover_list_lock); - - dlm_put_rsb(r); -} - -static void recover_list_clear(struct dlm_ls *ls) -{ - struct dlm_rsb *r, *s; - - spin_lock(&ls->ls_recover_list_lock); - list_for_each_entry_safe(r, s, &ls->ls_recover_list, res_recover_list) { - list_del_init(&r->res_recover_list); - r->res_recover_locks_count = 0; - dlm_put_rsb(r); - ls->ls_recover_list_count--; - } - - if (ls->ls_recover_list_count != 0) { - log_error(ls, "warning: recover_list_count %d", - ls->ls_recover_list_count); - ls->ls_recover_list_count = 0; - } - spin_unlock(&ls->ls_recover_list_lock); -} - -static int recover_idr_empty(struct dlm_ls *ls) -{ - int empty = 1; - - spin_lock(&ls->ls_recover_idr_lock); - if (ls->ls_recover_list_count) - empty = 0; - spin_unlock(&ls->ls_recover_idr_lock); - - return empty; -} - -static int recover_idr_add(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - int rv; - - idr_preload(GFP_NOFS); - spin_lock(&ls->ls_recover_idr_lock); - if (r->res_id) { - rv = -1; - goto out_unlock; - } - rv = idr_alloc(&ls->ls_recover_idr, r, 1, 0, GFP_NOWAIT); - if (rv < 0) - goto out_unlock; - - r->res_id = rv; - ls->ls_recover_list_count++; - dlm_hold_rsb(r); - rv = 0; -out_unlock: - spin_unlock(&ls->ls_recover_idr_lock); - idr_preload_end(); - return rv; -} - -static void recover_idr_del(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - - spin_lock(&ls->ls_recover_idr_lock); - idr_remove(&ls->ls_recover_idr, r->res_id); - r->res_id = 0; - ls->ls_recover_list_count--; - spin_unlock(&ls->ls_recover_idr_lock); - - dlm_put_rsb(r); -} - -static struct dlm_rsb *recover_idr_find(struct dlm_ls *ls, uint64_t id) -{ - struct dlm_rsb *r; - - spin_lock(&ls->ls_recover_idr_lock); - r = idr_find(&ls->ls_recover_idr, (int)id); - spin_unlock(&ls->ls_recover_idr_lock); - return r; -} - -static void recover_idr_clear(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - int id; - - spin_lock(&ls->ls_recover_idr_lock); - - idr_for_each_entry(&ls->ls_recover_idr, r, id) { - idr_remove(&ls->ls_recover_idr, id); - r->res_id = 0; - r->res_recover_locks_count = 0; - ls->ls_recover_list_count--; - - dlm_put_rsb(r); - } - - if (ls->ls_recover_list_count != 0) { - log_error(ls, "warning: recover_list_count %d", - ls->ls_recover_list_count); - ls->ls_recover_list_count = 0; - } - spin_unlock(&ls->ls_recover_idr_lock); -} - - -/* Master recovery: find new master node for rsb's that were - mastered on nodes that have been removed. - - dlm_recover_masters - recover_master - dlm_send_rcom_lookup -> receive_rcom_lookup - dlm_dir_lookup - receive_rcom_lookup_reply <- - dlm_recover_master_reply - set_new_master - set_master_lkbs - set_lock_master -*/ - -/* - * Set the lock master for all LKBs in a lock queue - * If we are the new master of the rsb, we may have received new - * MSTCPY locks from other nodes already which we need to ignore - * when setting the new nodeid. - */ - -static void set_lock_master(struct list_head *queue, int nodeid) -{ - struct dlm_lkb *lkb; - - list_for_each_entry(lkb, queue, lkb_statequeue) { - if (!(lkb->lkb_flags & DLM_IFL_MSTCPY)) { - lkb->lkb_nodeid = nodeid; - lkb->lkb_remid = 0; - } - } -} - -static void set_master_lkbs(struct dlm_rsb *r) -{ - set_lock_master(&r->res_grantqueue, r->res_nodeid); - set_lock_master(&r->res_convertqueue, r->res_nodeid); - set_lock_master(&r->res_waitqueue, r->res_nodeid); -} - -/* - * Propagate the new master nodeid to locks - * The NEW_MASTER flag tells dlm_recover_locks() which rsb's to consider. - * The NEW_MASTER2 flag tells recover_lvb() and recover_grant() which - * rsb's to consider. - */ - -static void set_new_master(struct dlm_rsb *r) -{ - set_master_lkbs(r); - rsb_set_flag(r, RSB_NEW_MASTER); - rsb_set_flag(r, RSB_NEW_MASTER2); -} - -/* - * We do async lookups on rsb's that need new masters. The rsb's - * waiting for a lookup reply are kept on the recover_list. - * - * Another node recovering the master may have sent us a rcom lookup, - * and our dlm_master_lookup() set it as the new master, along with - * NEW_MASTER so that we'll recover it here (this implies dir_nodeid - * equals our_nodeid below). - */ - -static int recover_master(struct dlm_rsb *r, unsigned int *count) -{ - struct dlm_ls *ls = r->res_ls; - int our_nodeid, dir_nodeid; - int is_removed = 0; - int error; - - if (is_master(r)) - return 0; - - is_removed = dlm_is_removed(ls, r->res_nodeid); - - if (!is_removed && !rsb_flag(r, RSB_NEW_MASTER)) - return 0; - - our_nodeid = dlm_our_nodeid(); - dir_nodeid = dlm_dir_nodeid(r); - - if (dir_nodeid == our_nodeid) { - if (is_removed) { - r->res_master_nodeid = our_nodeid; - r->res_nodeid = 0; - } - - /* set master of lkbs to ourself when is_removed, or to - another new master which we set along with NEW_MASTER - in dlm_master_lookup */ - set_new_master(r); - error = 0; - } else { - recover_idr_add(r); - error = dlm_send_rcom_lookup(r, dir_nodeid); - } - - (*count)++; - return error; -} - -/* - * All MSTCPY locks are purged and rebuilt, even if the master stayed the same. - * This is necessary because recovery can be started, aborted and restarted, - * causing the master nodeid to briefly change during the aborted recovery, and - * change back to the original value in the second recovery. The MSTCPY locks - * may or may not have been purged during the aborted recovery. Another node - * with an outstanding request in waiters list and a request reply saved in the - * requestqueue, cannot know whether it should ignore the reply and resend the - * request, or accept the reply and complete the request. It must do the - * former if the remote node purged MSTCPY locks, and it must do the later if - * the remote node did not. This is solved by always purging MSTCPY locks, in - * which case, the request reply would always be ignored and the request - * resent. - */ - -static int recover_master_static(struct dlm_rsb *r, unsigned int *count) -{ - int dir_nodeid = dlm_dir_nodeid(r); - int new_master = dir_nodeid; - - if (dir_nodeid == dlm_our_nodeid()) - new_master = 0; - - dlm_purge_mstcpy_locks(r); - r->res_master_nodeid = dir_nodeid; - r->res_nodeid = new_master; - set_new_master(r); - (*count)++; - return 0; -} - -/* - * Go through local root resources and for each rsb which has a master which - * has departed, get the new master nodeid from the directory. The dir will - * assign mastery to the first node to look up the new master. That means - * we'll discover in this lookup if we're the new master of any rsb's. - * - * We fire off all the dir lookup requests individually and asynchronously to - * the correct dir node. - */ - -int dlm_recover_masters(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - unsigned int total = 0; - unsigned int count = 0; - int nodir = dlm_no_directory(ls); - int error; - - log_debug(ls, "dlm_recover_masters"); - - down_read(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - if (dlm_recovery_stopped(ls)) { - up_read(&ls->ls_root_sem); - error = -EINTR; - goto out; - } - - lock_rsb(r); - if (nodir) - error = recover_master_static(r, &count); - else - error = recover_master(r, &count); - unlock_rsb(r); - cond_resched(); - total++; - - if (error) { - up_read(&ls->ls_root_sem); - goto out; - } - } - up_read(&ls->ls_root_sem); - - log_debug(ls, "dlm_recover_masters %u of %u", count, total); - - error = dlm_wait_function(ls, &recover_idr_empty); - out: - if (error) - recover_idr_clear(ls); - return error; -} - -int dlm_recover_master_reply(struct dlm_ls *ls, struct dlm_rcom *rc) -{ - struct dlm_rsb *r; - int ret_nodeid, new_master; - - r = recover_idr_find(ls, rc->rc_id); - if (!r) { - log_error(ls, "dlm_recover_master_reply no id %llx", - (unsigned long long)rc->rc_id); - goto out; - } - - ret_nodeid = rc->rc_result; - - if (ret_nodeid == dlm_our_nodeid()) - new_master = 0; - else - new_master = ret_nodeid; - - lock_rsb(r); - r->res_master_nodeid = ret_nodeid; - r->res_nodeid = new_master; - set_new_master(r); - unlock_rsb(r); - recover_idr_del(r); - - if (recover_idr_empty(ls)) - wake_up(&ls->ls_wait_general); - out: - return 0; -} - - -/* Lock recovery: rebuild the process-copy locks we hold on a - remastered rsb on the new rsb master. - - dlm_recover_locks - recover_locks - recover_locks_queue - dlm_send_rcom_lock -> receive_rcom_lock - dlm_recover_master_copy - receive_rcom_lock_reply <- - dlm_recover_process_copy -*/ - - -/* - * keep a count of the number of lkb's we send to the new master; when we get - * an equal number of replies then recovery for the rsb is done - */ - -static int recover_locks_queue(struct dlm_rsb *r, struct list_head *head) -{ - struct dlm_lkb *lkb; - int error = 0; - - list_for_each_entry(lkb, head, lkb_statequeue) { - error = dlm_send_rcom_lock(r, lkb); - if (error) - break; - r->res_recover_locks_count++; - } - - return error; -} - -static int recover_locks(struct dlm_rsb *r) -{ - int error = 0; - - lock_rsb(r); - - DLM_ASSERT(!r->res_recover_locks_count, dlm_dump_rsb(r);); - - error = recover_locks_queue(r, &r->res_grantqueue); - if (error) - goto out; - error = recover_locks_queue(r, &r->res_convertqueue); - if (error) - goto out; - error = recover_locks_queue(r, &r->res_waitqueue); - if (error) - goto out; - - if (r->res_recover_locks_count) - recover_list_add(r); - else - rsb_clear_flag(r, RSB_NEW_MASTER); - out: - unlock_rsb(r); - return error; -} - -int dlm_recover_locks(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - int error, count = 0; - - down_read(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - if (is_master(r)) { - rsb_clear_flag(r, RSB_NEW_MASTER); - continue; - } - - if (!rsb_flag(r, RSB_NEW_MASTER)) - continue; - - if (dlm_recovery_stopped(ls)) { - error = -EINTR; - up_read(&ls->ls_root_sem); - goto out; - } - - error = recover_locks(r); - if (error) { - up_read(&ls->ls_root_sem); - goto out; - } - - count += r->res_recover_locks_count; - } - up_read(&ls->ls_root_sem); - - log_debug(ls, "dlm_recover_locks %d out", count); - - error = dlm_wait_function(ls, &recover_list_empty); - out: - if (error) - recover_list_clear(ls); - return error; -} - -void dlm_recovered_lock(struct dlm_rsb *r) -{ - DLM_ASSERT(rsb_flag(r, RSB_NEW_MASTER), dlm_dump_rsb(r);); - - r->res_recover_locks_count--; - if (!r->res_recover_locks_count) { - rsb_clear_flag(r, RSB_NEW_MASTER); - recover_list_del(r); - } - - if (recover_list_empty(r->res_ls)) - wake_up(&r->res_ls->ls_wait_general); -} - -/* - * The lvb needs to be recovered on all master rsb's. This includes setting - * the VALNOTVALID flag if necessary, and determining the correct lvb contents - * based on the lvb's of the locks held on the rsb. - * - * RSB_VALNOTVALID is set in two cases: - * - * 1. we are master, but not new, and we purged an EX/PW lock held by a - * failed node (in dlm_recover_purge which set RSB_RECOVER_LVB_INVAL) - * - * 2. we are a new master, and there are only NL/CR locks left. - * (We could probably improve this by only invaliding in this way when - * the previous master left uncleanly. VMS docs mention that.) - * - * The LVB contents are only considered for changing when this is a new master - * of the rsb (NEW_MASTER2). Then, the rsb's lvb is taken from any lkb with - * mode > CR. If no lkb's exist with mode above CR, the lvb contents are taken - * from the lkb with the largest lvb sequence number. - */ - -static void recover_lvb(struct dlm_rsb *r) -{ - struct dlm_lkb *lkb, *high_lkb = NULL; - uint32_t high_seq = 0; - int lock_lvb_exists = 0; - int big_lock_exists = 0; - int lvblen = r->res_ls->ls_lvblen; - - if (!rsb_flag(r, RSB_NEW_MASTER2) && - rsb_flag(r, RSB_RECOVER_LVB_INVAL)) { - /* case 1 above */ - rsb_set_flag(r, RSB_VALNOTVALID); - return; - } - - if (!rsb_flag(r, RSB_NEW_MASTER2)) - return; - - /* we are the new master, so figure out if VALNOTVALID should - be set, and set the rsb lvb from the best lkb available. */ - - list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - continue; - - lock_lvb_exists = 1; - - if (lkb->lkb_grmode > DLM_LOCK_CR) { - big_lock_exists = 1; - goto setflag; - } - - if (((int)lkb->lkb_lvbseq - (int)high_seq) >= 0) { - high_lkb = lkb; - high_seq = lkb->lkb_lvbseq; - } - } - - list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { - if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) - continue; - - lock_lvb_exists = 1; - - if (lkb->lkb_grmode > DLM_LOCK_CR) { - big_lock_exists = 1; - goto setflag; - } - - if (((int)lkb->lkb_lvbseq - (int)high_seq) >= 0) { - high_lkb = lkb; - high_seq = lkb->lkb_lvbseq; - } - } - - setflag: - if (!lock_lvb_exists) - goto out; - - /* lvb is invalidated if only NL/CR locks remain */ - if (!big_lock_exists) - rsb_set_flag(r, RSB_VALNOTVALID); - - if (!r->res_lvbptr) { - r->res_lvbptr = dlm_allocate_lvb(r->res_ls); - if (!r->res_lvbptr) - goto out; - } - - if (big_lock_exists) { - r->res_lvbseq = lkb->lkb_lvbseq; - memcpy(r->res_lvbptr, lkb->lkb_lvbptr, lvblen); - } else if (high_lkb) { - r->res_lvbseq = high_lkb->lkb_lvbseq; - memcpy(r->res_lvbptr, high_lkb->lkb_lvbptr, lvblen); - } else { - r->res_lvbseq = 0; - memset(r->res_lvbptr, 0, lvblen); - } - out: - return; -} - -/* All master rsb's flagged RECOVER_CONVERT need to be looked at. The locks - converting PR->CW or CW->PR need to have their lkb_grmode set. */ - -static void recover_conversion(struct dlm_rsb *r) -{ - struct dlm_ls *ls = r->res_ls; - struct dlm_lkb *lkb; - int grmode = -1; - - list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { - if (lkb->lkb_grmode == DLM_LOCK_PR || - lkb->lkb_grmode == DLM_LOCK_CW) { - grmode = lkb->lkb_grmode; - break; - } - } - - list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { - if (lkb->lkb_grmode != DLM_LOCK_IV) - continue; - if (grmode == -1) { - log_debug(ls, "recover_conversion %x set gr to rq %d", - lkb->lkb_id, lkb->lkb_rqmode); - lkb->lkb_grmode = lkb->lkb_rqmode; - } else { - log_debug(ls, "recover_conversion %x set gr %d", - lkb->lkb_id, grmode); - lkb->lkb_grmode = grmode; - } - } -} - -/* We've become the new master for this rsb and waiting/converting locks may - need to be granted in dlm_recover_grant() due to locks that may have - existed from a removed node. */ - -static void recover_grant(struct dlm_rsb *r) -{ - if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) - rsb_set_flag(r, RSB_RECOVER_GRANT); -} - -void dlm_recover_rsbs(struct dlm_ls *ls) -{ - struct dlm_rsb *r; - unsigned int count = 0; - - down_read(&ls->ls_root_sem); - list_for_each_entry(r, &ls->ls_root_list, res_root_list) { - lock_rsb(r); - if (is_master(r)) { - if (rsb_flag(r, RSB_RECOVER_CONVERT)) - recover_conversion(r); - - /* recover lvb before granting locks so the updated - lvb/VALNOTVALID is presented in the completion */ - recover_lvb(r); - - if (rsb_flag(r, RSB_NEW_MASTER2)) - recover_grant(r); - count++; - } else { - rsb_clear_flag(r, RSB_VALNOTVALID); - } - rsb_clear_flag(r, RSB_RECOVER_CONVERT); - rsb_clear_flag(r, RSB_RECOVER_LVB_INVAL); - rsb_clear_flag(r, RSB_NEW_MASTER2); - unlock_rsb(r); - } - up_read(&ls->ls_root_sem); - - if (count) - log_debug(ls, "dlm_recover_rsbs %d done", count); -} - -/* Create a single list of all root rsb's to be used during recovery */ - -int dlm_create_root_list(struct dlm_ls *ls) -{ - struct rb_node *n; - struct dlm_rsb *r; - int i, error = 0; - - down_write(&ls->ls_root_sem); - if (!list_empty(&ls->ls_root_list)) { - log_error(ls, "root list not empty"); - error = -EINVAL; - goto out; - } - - for (i = 0; i < ls->ls_rsbtbl_size; i++) { - spin_lock(&ls->ls_rsbtbl[i].lock); - for (n = rb_first(&ls->ls_rsbtbl[i].keep); n; n = rb_next(n)) { - r = rb_entry(n, struct dlm_rsb, res_hashnode); - list_add(&r->res_root_list, &ls->ls_root_list); - dlm_hold_rsb(r); - } - - if (!RB_EMPTY_ROOT(&ls->ls_rsbtbl[i].toss)) - log_error(ls, "dlm_create_root_list toss not empty"); - spin_unlock(&ls->ls_rsbtbl[i].lock); - } - out: - up_write(&ls->ls_root_sem); - return error; -} - -void dlm_release_root_list(struct dlm_ls *ls) -{ - struct dlm_rsb *r, *safe; - - down_write(&ls->ls_root_sem); - list_for_each_entry_safe(r, safe, &ls->ls_root_list, res_root_list) { - list_del_init(&r->res_root_list); - dlm_put_rsb(r); - } - up_write(&ls->ls_root_sem); -} - -void dlm_clear_toss(struct dlm_ls *ls) -{ - struct rb_node *n, *next; - struct dlm_rsb *r; - unsigned int count = 0; - int i; - - for (i = 0; i < ls->ls_rsbtbl_size; i++) { - spin_lock(&ls->ls_rsbtbl[i].lock); - for (n = rb_first(&ls->ls_rsbtbl[i].toss); n; n = next) { - next = rb_next(n); - r = rb_entry(n, struct dlm_rsb, res_hashnode); - rb_erase(n, &ls->ls_rsbtbl[i].toss); - dlm_free_rsb(r); - count++; - } - spin_unlock(&ls->ls_rsbtbl[i].lock); - } - - if (count) - log_debug(ls, "dlm_clear_toss %u done", count); -} - diff --git a/kmod/dlm/recover.h b/kmod/dlm/recover.h deleted file mode 100644 index d8c8738c..00000000 --- a/kmod/dlm/recover.h +++ /dev/null @@ -1,34 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __RECOVER_DOT_H__ -#define __RECOVER_DOT_H__ - -int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)); -uint32_t dlm_recover_status(struct dlm_ls *ls); -void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status); -int dlm_recover_members_wait(struct dlm_ls *ls); -int dlm_recover_directory_wait(struct dlm_ls *ls); -int dlm_recover_locks_wait(struct dlm_ls *ls); -int dlm_recover_done_wait(struct dlm_ls *ls); -int dlm_recover_masters(struct dlm_ls *ls); -int dlm_recover_master_reply(struct dlm_ls *ls, struct dlm_rcom *rc); -int dlm_recover_locks(struct dlm_ls *ls); -void dlm_recovered_lock(struct dlm_rsb *r); -int dlm_create_root_list(struct dlm_ls *ls); -void dlm_release_root_list(struct dlm_ls *ls); -void dlm_clear_toss(struct dlm_ls *ls); -void dlm_recover_rsbs(struct dlm_ls *ls); - -#endif /* __RECOVER_DOT_H__ */ - diff --git a/kmod/dlm/recoverd.c b/kmod/dlm/recoverd.c deleted file mode 100644 index 32f9f892..00000000 --- a/kmod/dlm/recoverd.c +++ /dev/null @@ -1,342 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "lockspace.h" -#include "member.h" -#include "dir.h" -#include "ast.h" -#include "recover.h" -#include "lowcomms.h" -#include "lock.h" -#include "requestqueue.h" -#include "recoverd.h" - - -/* If the start for which we're re-enabling locking (seq) has been superseded - by a newer stop (ls_recover_seq), we need to leave locking disabled. - - We suspend dlm_recv threads here to avoid the race where dlm_recv a) sees - locking stopped and b) adds a message to the requestqueue, but dlm_recoverd - enables locking and clears the requestqueue between a and b. */ - -static int enable_locking(struct dlm_ls *ls, uint64_t seq) -{ - int error = -EINTR; - - down_write(&ls->ls_recv_active); - - spin_lock(&ls->ls_recover_lock); - if (ls->ls_recover_seq == seq) { - set_bit(LSFL_RUNNING, &ls->ls_flags); - /* unblocks processes waiting to enter the dlm */ - up_write(&ls->ls_in_recovery); - clear_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); - error = 0; - } - spin_unlock(&ls->ls_recover_lock); - - up_write(&ls->ls_recv_active); - return error; -} - -static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) -{ - unsigned long start; - int error, neg = 0; - - log_debug(ls, "dlm_recover %llu", (unsigned long long)rv->seq); - - mutex_lock(&ls->ls_recoverd_active); - - dlm_callback_suspend(ls); - - dlm_clear_toss(ls); - - /* - * This list of root rsb's will be the basis of most of the recovery - * routines. - */ - - dlm_create_root_list(ls); - - /* - * Add or remove nodes from the lockspace's ls_nodes list. - */ - - error = dlm_recover_members(ls, rv, &neg); - if (error) { - log_debug(ls, "dlm_recover_members error %d", error); - goto fail; - } - - dlm_recover_dir_nodeid(ls); - - ls->ls_recover_dir_sent_res = 0; - ls->ls_recover_dir_sent_msg = 0; - ls->ls_recover_locks_in = 0; - - dlm_set_recover_status(ls, DLM_RS_NODES); - - error = dlm_recover_members_wait(ls); - if (error) { - log_debug(ls, "dlm_recover_members_wait error %d", error); - goto fail; - } - - start = jiffies; - - /* - * Rebuild our own share of the directory by collecting from all other - * nodes their master rsb names that hash to us. - */ - - error = dlm_recover_directory(ls); - if (error) { - log_debug(ls, "dlm_recover_directory error %d", error); - goto fail; - } - - dlm_set_recover_status(ls, DLM_RS_DIR); - - error = dlm_recover_directory_wait(ls); - if (error) { - log_debug(ls, "dlm_recover_directory_wait error %d", error); - goto fail; - } - - log_debug(ls, "dlm_recover_directory %u out %u messages", - ls->ls_recover_dir_sent_res, ls->ls_recover_dir_sent_msg); - - /* - * We may have outstanding operations that are waiting for a reply from - * a failed node. Mark these to be resent after recovery. Unlock and - * cancel ops can just be completed. - */ - - dlm_recover_waiters_pre(ls); - - error = dlm_recovery_stopped(ls); - if (error) - goto fail; - - if (neg || dlm_no_directory(ls)) { - /* - * Clear lkb's for departed nodes. - */ - - dlm_recover_purge(ls); - - /* - * Get new master nodeid's for rsb's that were mastered on - * departed nodes. - */ - - error = dlm_recover_masters(ls); - if (error) { - log_debug(ls, "dlm_recover_masters error %d", error); - goto fail; - } - - /* - * Send our locks on remastered rsb's to the new masters. - */ - - error = dlm_recover_locks(ls); - if (error) { - log_debug(ls, "dlm_recover_locks error %d", error); - goto fail; - } - - dlm_set_recover_status(ls, DLM_RS_LOCKS); - - error = dlm_recover_locks_wait(ls); - if (error) { - log_debug(ls, "dlm_recover_locks_wait error %d", error); - goto fail; - } - - log_debug(ls, "dlm_recover_locks %u in", - ls->ls_recover_locks_in); - - /* - * Finalize state in master rsb's now that all locks can be - * checked. This includes conversion resolution and lvb - * settings. - */ - - dlm_recover_rsbs(ls); - } else { - /* - * Other lockspace members may be going through the "neg" steps - * while also adding us to the lockspace, in which case they'll - * be doing the recover_locks (RS_LOCKS) barrier. - */ - dlm_set_recover_status(ls, DLM_RS_LOCKS); - - error = dlm_recover_locks_wait(ls); - if (error) { - log_debug(ls, "dlm_recover_locks_wait error %d", error); - goto fail; - } - } - - dlm_release_root_list(ls); - - /* - * Purge directory-related requests that are saved in requestqueue. - * All dir requests from before recovery are invalid now due to the dir - * rebuild and will be resent by the requesting nodes. - */ - - dlm_purge_requestqueue(ls); - - dlm_set_recover_status(ls, DLM_RS_DONE); - - error = dlm_recover_done_wait(ls); - if (error) { - log_debug(ls, "dlm_recover_done_wait error %d", error); - goto fail; - } - - dlm_clear_members_gone(ls); - - dlm_adjust_timeouts(ls); - - dlm_callback_resume(ls); - - error = enable_locking(ls, rv->seq); - if (error) { - log_debug(ls, "enable_locking error %d", error); - goto fail; - } - - error = dlm_process_requestqueue(ls); - if (error) { - log_debug(ls, "dlm_process_requestqueue error %d", error); - goto fail; - } - - error = dlm_recover_waiters_post(ls); - if (error) { - log_debug(ls, "dlm_recover_waiters_post error %d", error); - goto fail; - } - - dlm_recover_grant(ls); - - log_debug(ls, "dlm_recover %llu generation %u done: %u ms", - (unsigned long long)rv->seq, ls->ls_generation, - jiffies_to_msecs(jiffies - start)); - mutex_unlock(&ls->ls_recoverd_active); - - dlm_lsop_recover_done(ls); - return 0; - - fail: - dlm_release_root_list(ls); - log_debug(ls, "dlm_recover %llu error %d", - (unsigned long long)rv->seq, error); - mutex_unlock(&ls->ls_recoverd_active); - return error; -} - -/* The dlm_ls_start() that created the rv we take here may already have been - stopped via dlm_ls_stop(); in that case we need to leave the RECOVERY_STOP - flag set. */ - -static void do_ls_recovery(struct dlm_ls *ls) -{ - struct dlm_recover *rv = NULL; - - spin_lock(&ls->ls_recover_lock); - rv = ls->ls_recover_args; - ls->ls_recover_args = NULL; - if (rv && ls->ls_recover_seq == rv->seq) - clear_bit(LSFL_RECOVER_STOP, &ls->ls_flags); - spin_unlock(&ls->ls_recover_lock); - - if (rv) { - ls_recover(ls, rv); - kfree(rv->nodes); - kfree(rv); - } -} - -static int dlm_recoverd(void *arg) -{ - struct dlm_ls *ls; - - ls = dlm_find_lockspace_local(arg); - if (!ls) { - log_print("dlm_recoverd: no lockspace %p", arg); - return -1; - } - - down_write(&ls->ls_in_recovery); - set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); - wake_up(&ls->ls_recover_lock_wait); - - while (!kthread_should_stop()) { - set_current_state(TASK_INTERRUPTIBLE); - if (!test_bit(LSFL_RECOVER_WORK, &ls->ls_flags) && - !test_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) - schedule(); - set_current_state(TASK_RUNNING); - - if (test_and_clear_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) { - down_write(&ls->ls_in_recovery); - set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); - wake_up(&ls->ls_recover_lock_wait); - } - - if (test_and_clear_bit(LSFL_RECOVER_WORK, &ls->ls_flags)) - do_ls_recovery(ls); - } - - if (test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)) - up_write(&ls->ls_in_recovery); - - dlm_put_lockspace(ls); - return 0; -} - -int dlm_recoverd_start(struct dlm_ls *ls) -{ - struct task_struct *p; - int error = 0; - - p = kthread_run(dlm_recoverd, ls, "dlm_recoverd"); - if (IS_ERR(p)) - error = PTR_ERR(p); - else - ls->ls_recoverd_task = p; - return error; -} - -void dlm_recoverd_stop(struct dlm_ls *ls) -{ - kthread_stop(ls->ls_recoverd_task); -} - -void dlm_recoverd_suspend(struct dlm_ls *ls) -{ - wake_up(&ls->ls_wait_general); - mutex_lock(&ls->ls_recoverd_active); -} - -void dlm_recoverd_resume(struct dlm_ls *ls) -{ - mutex_unlock(&ls->ls_recoverd_active); -} - diff --git a/kmod/dlm/recoverd.h b/kmod/dlm/recoverd.h deleted file mode 100644 index 88560797..00000000 --- a/kmod/dlm/recoverd.h +++ /dev/null @@ -1,23 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __RECOVERD_DOT_H__ -#define __RECOVERD_DOT_H__ - -void dlm_recoverd_stop(struct dlm_ls *ls); -int dlm_recoverd_start(struct dlm_ls *ls); -void dlm_recoverd_suspend(struct dlm_ls *ls); -void dlm_recoverd_resume(struct dlm_ls *ls); - -#endif /* __RECOVERD_DOT_H__ */ - diff --git a/kmod/dlm/requestqueue.c b/kmod/dlm/requestqueue.c deleted file mode 100644 index 1695f1b0..00000000 --- a/kmod/dlm/requestqueue.c +++ /dev/null @@ -1,171 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "member.h" -#include "lock.h" -#include "dir.h" -#include "config.h" -#include "requestqueue.h" - -struct rq_entry { - struct list_head list; - uint32_t recover_seq; - int nodeid; - struct dlm_message request; -}; - -/* - * Requests received while the lockspace is in recovery get added to the - * request queue and processed when recovery is complete. This happens when - * the lockspace is suspended on some nodes before it is on others, or the - * lockspace is enabled on some while still suspended on others. - */ - -void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, struct dlm_message *ms) -{ - struct rq_entry *e; - int length = ms->m_header.h_length - sizeof(struct dlm_message); - - e = kmalloc(sizeof(struct rq_entry) + length, GFP_NOFS); - if (!e) { - log_print("dlm_add_requestqueue: out of memory len %d", length); - return; - } - - e->recover_seq = ls->ls_recover_seq & 0xFFFFFFFF; - e->nodeid = nodeid; - memcpy(&e->request, ms, ms->m_header.h_length); - - mutex_lock(&ls->ls_requestqueue_mutex); - list_add_tail(&e->list, &ls->ls_requestqueue); - mutex_unlock(&ls->ls_requestqueue_mutex); -} - -/* - * Called by dlm_recoverd to process normal messages saved while recovery was - * happening. Normal locking has been enabled before this is called. dlm_recv - * upon receiving a message, will wait for all saved messages to be drained - * here before processing the message it got. If a new dlm_ls_stop() arrives - * while we're processing these saved messages, it may block trying to suspend - * dlm_recv if dlm_recv is waiting for us in dlm_wait_requestqueue. In that - * case, we don't abort since locking_stopped is still 0. If dlm_recv is not - * waiting for us, then this processing may be aborted due to locking_stopped. - */ - -int dlm_process_requestqueue(struct dlm_ls *ls) -{ - struct rq_entry *e; - struct dlm_message *ms; - int error = 0; - - mutex_lock(&ls->ls_requestqueue_mutex); - - for (;;) { - if (list_empty(&ls->ls_requestqueue)) { - mutex_unlock(&ls->ls_requestqueue_mutex); - error = 0; - break; - } - e = list_entry(ls->ls_requestqueue.next, struct rq_entry, list); - mutex_unlock(&ls->ls_requestqueue_mutex); - - ms = &e->request; - - log_limit(ls, "dlm_process_requestqueue msg %d from %d " - "lkid %x remid %x result %d seq %u", - ms->m_type, ms->m_header.h_nodeid, - ms->m_lkid, ms->m_remid, ms->m_result, - e->recover_seq); - - dlm_receive_message_saved(ls, &e->request, e->recover_seq); - - mutex_lock(&ls->ls_requestqueue_mutex); - list_del(&e->list); - kfree(e); - - if (dlm_locking_stopped(ls)) { - log_debug(ls, "process_requestqueue abort running"); - mutex_unlock(&ls->ls_requestqueue_mutex); - error = -EINTR; - break; - } - schedule(); - } - - return error; -} - -/* - * After recovery is done, locking is resumed and dlm_recoverd takes all the - * saved requests and processes them as they would have been by dlm_recv. At - * the same time, dlm_recv will start receiving new requests from remote nodes. - * We want to delay dlm_recv processing new requests until dlm_recoverd has - * finished processing the old saved requests. We don't check for locking - * stopped here because dlm_ls_stop won't stop locking until it's suspended us - * (dlm_recv). - */ - -void dlm_wait_requestqueue(struct dlm_ls *ls) -{ - for (;;) { - mutex_lock(&ls->ls_requestqueue_mutex); - if (list_empty(&ls->ls_requestqueue)) - break; - mutex_unlock(&ls->ls_requestqueue_mutex); - schedule(); - } - mutex_unlock(&ls->ls_requestqueue_mutex); -} - -static int purge_request(struct dlm_ls *ls, struct dlm_message *ms, int nodeid) -{ - uint32_t type = ms->m_type; - - /* the ls is being cleaned up and freed by release_lockspace */ - if (!ls->ls_count) - return 1; - - if (dlm_is_removed(ls, nodeid)) - return 1; - - /* directory operations are always purged because the directory is - always rebuilt during recovery and the lookups resent */ - - if (type == DLM_MSG_REMOVE || - type == DLM_MSG_LOOKUP || - type == DLM_MSG_LOOKUP_REPLY) - return 1; - - if (!dlm_no_directory(ls)) - return 0; - - return 1; -} - -void dlm_purge_requestqueue(struct dlm_ls *ls) -{ - struct dlm_message *ms; - struct rq_entry *e, *safe; - - mutex_lock(&ls->ls_requestqueue_mutex); - list_for_each_entry_safe(e, safe, &ls->ls_requestqueue, list) { - ms = &e->request; - - if (purge_request(ls, ms, e->nodeid)) { - list_del(&e->list); - kfree(e); - } - } - mutex_unlock(&ls->ls_requestqueue_mutex); -} - diff --git a/kmod/dlm/requestqueue.h b/kmod/dlm/requestqueue.h deleted file mode 100644 index 10ce449b..00000000 --- a/kmod/dlm/requestqueue.h +++ /dev/null @@ -1,22 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __REQUESTQUEUE_DOT_H__ -#define __REQUESTQUEUE_DOT_H__ - -void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, struct dlm_message *ms); -int dlm_process_requestqueue(struct dlm_ls *ls); -void dlm_wait_requestqueue(struct dlm_ls *ls); -void dlm_purge_requestqueue(struct dlm_ls *ls); - -#endif - diff --git a/kmod/dlm/user.c b/kmod/dlm/user.c deleted file mode 100644 index 826437e3..00000000 --- a/kmod/dlm/user.c +++ /dev/null @@ -1,1028 +0,0 @@ -/* - * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "include/linux/dlm.h" -#include -#include - -#include "dlm_internal.h" -#include "lockspace.h" -#include "lock.h" -#include "lvb_table.h" -#include "user.h" -#include "ast.h" - -static const char name_prefix[] = "dlm"; -static const struct file_operations device_fops; -static atomic_t dlm_monitor_opened; -static int dlm_monitor_unused = 1; - -#ifdef CONFIG_COMPAT - -struct dlm_lock_params32 { - __u8 mode; - __u8 namelen; - __u16 unused; - __u32 flags; - __u32 lkid; - __u32 parent; - __u64 xid; - __u64 timeout; - __u32 castparam; - __u32 castaddr; - __u32 bastparam; - __u32 bastaddr; - __u32 lksb; - char lvb[DLM_USER_LVB_LEN]; - char name[0]; -}; - -struct dlm_write_request32 { - __u32 version[3]; - __u8 cmd; - __u8 is64bit; - __u8 unused[2]; - - union { - struct dlm_lock_params32 lock; - struct dlm_lspace_params lspace; - struct dlm_purge_params purge; - } i; -}; - -struct dlm_lksb32 { - __u32 sb_status; - __u32 sb_lkid; - __u8 sb_flags; - __u32 sb_lvbptr; -}; - -struct dlm_lock_result32 { - __u32 version[3]; - __u32 length; - __u32 user_astaddr; - __u32 user_astparam; - __u32 user_lksb; - struct dlm_lksb32 lksb; - __u8 bast_mode; - __u8 unused[3]; - /* Offsets may be zero if no data is present */ - __u32 lvb_offset; -}; - -static void compat_input(struct dlm_write_request *kb, - struct dlm_write_request32 *kb32, - int namelen) -{ - kb->version[0] = kb32->version[0]; - kb->version[1] = kb32->version[1]; - kb->version[2] = kb32->version[2]; - - kb->cmd = kb32->cmd; - kb->is64bit = kb32->is64bit; - if (kb->cmd == DLM_USER_CREATE_LOCKSPACE || - kb->cmd == DLM_USER_REMOVE_LOCKSPACE) { - kb->i.lspace.flags = kb32->i.lspace.flags; - kb->i.lspace.minor = kb32->i.lspace.minor; - memcpy(kb->i.lspace.name, kb32->i.lspace.name, namelen); - } else if (kb->cmd == DLM_USER_PURGE) { - kb->i.purge.nodeid = kb32->i.purge.nodeid; - kb->i.purge.pid = kb32->i.purge.pid; - } else { - kb->i.lock.mode = kb32->i.lock.mode; - kb->i.lock.namelen = kb32->i.lock.namelen; - kb->i.lock.flags = kb32->i.lock.flags; - kb->i.lock.lkid = kb32->i.lock.lkid; - kb->i.lock.parent = kb32->i.lock.parent; - kb->i.lock.xid = kb32->i.lock.xid; - kb->i.lock.timeout = kb32->i.lock.timeout; - kb->i.lock.castparam = (void *)(long)kb32->i.lock.castparam; - kb->i.lock.castaddr = (void *)(long)kb32->i.lock.castaddr; - kb->i.lock.bastparam = (void *)(long)kb32->i.lock.bastparam; - kb->i.lock.bastaddr = (void *)(long)kb32->i.lock.bastaddr; - kb->i.lock.lksb = (void *)(long)kb32->i.lock.lksb; - memcpy(kb->i.lock.lvb, kb32->i.lock.lvb, DLM_USER_LVB_LEN); - memcpy(kb->i.lock.name, kb32->i.lock.name, namelen); - } -} - -static void compat_output(struct dlm_lock_result *res, - struct dlm_lock_result32 *res32) -{ - res32->version[0] = res->version[0]; - res32->version[1] = res->version[1]; - res32->version[2] = res->version[2]; - - res32->user_astaddr = (__u32)(long)res->user_astaddr; - res32->user_astparam = (__u32)(long)res->user_astparam; - res32->user_lksb = (__u32)(long)res->user_lksb; - res32->bast_mode = res->bast_mode; - - res32->lvb_offset = res->lvb_offset; - res32->length = res->length; - - res32->lksb.sb_status = res->lksb.sb_status; - res32->lksb.sb_flags = res->lksb.sb_flags; - res32->lksb.sb_lkid = res->lksb.sb_lkid; - res32->lksb.sb_lvbptr = (__u32)(long)res->lksb.sb_lvbptr; -} -#endif - -/* Figure out if this lock is at the end of its life and no longer - available for the application to use. The lkb still exists until - the final ast is read. A lock becomes EOL in three situations: - 1. a noqueue request fails with EAGAIN - 2. an unlock completes with EUNLOCK - 3. a cancel of a waiting request completes with ECANCEL/EDEADLK - An EOL lock needs to be removed from the process's list of locks. - And we can't allow any new operation on an EOL lock. This is - not related to the lifetime of the lkb struct which is managed - entirely by refcount. */ - -static int lkb_is_endoflife(int mode, int status) -{ - switch (status) { - case -DLM_EUNLOCK: - return 1; - case -DLM_ECANCEL: - case -ETIMEDOUT: - case -EDEADLK: - case -EAGAIN: - if (mode == DLM_LOCK_IV) - return 1; - break; - } - return 0; -} - -/* we could possibly check if the cancel of an orphan has resulted in the lkb - being removed and then remove that lkb from the orphans list and free it */ - -void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, - int status, uint32_t sbflags, uint64_t seq) -{ - struct dlm_ls *ls; - struct dlm_user_args *ua; - struct dlm_user_proc *proc; - int rv; - - if (lkb->lkb_flags & (DLM_IFL_ORPHAN | DLM_IFL_DEAD)) - return; - - ls = lkb->lkb_resource->res_ls; - mutex_lock(&ls->ls_clear_proc_locks); - - /* If ORPHAN/DEAD flag is set, it means the process is dead so an ast - can't be delivered. For ORPHAN's, dlm_clear_proc_locks() freed - lkb->ua so we can't try to use it. This second check is necessary - for cases where a completion ast is received for an operation that - began before clear_proc_locks did its cancel/unlock. */ - - if (lkb->lkb_flags & (DLM_IFL_ORPHAN | DLM_IFL_DEAD)) - goto out; - - DLM_ASSERT(lkb->lkb_ua, dlm_print_lkb(lkb);); - ua = lkb->lkb_ua; - proc = ua->proc; - - if ((flags & DLM_CB_BAST) && ua->bastaddr == NULL) - goto out; - - if ((flags & DLM_CB_CAST) && lkb_is_endoflife(mode, status)) - lkb->lkb_flags |= DLM_IFL_ENDOFLIFE; - - spin_lock(&proc->asts_spin); - - rv = dlm_add_lkb_callback(lkb, flags, mode, NULL, status, sbflags, seq); - if (rv < 0) { - spin_unlock(&proc->asts_spin); - goto out; - } - - if (list_empty(&lkb->lkb_cb_list)) { - kref_get(&lkb->lkb_ref); - list_add_tail(&lkb->lkb_cb_list, &proc->asts); - wake_up_interruptible(&proc->wait); - } - spin_unlock(&proc->asts_spin); - - if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { - /* N.B. spin_lock locks_spin, not asts_spin */ - spin_lock(&proc->locks_spin); - if (!list_empty(&lkb->lkb_ownqueue)) { - list_del_init(&lkb->lkb_ownqueue); - dlm_put_lkb(lkb); - } - spin_unlock(&proc->locks_spin); - } - out: - mutex_unlock(&ls->ls_clear_proc_locks); -} - -static int device_user_lock(struct dlm_user_proc *proc, - struct dlm_lock_params *params) -{ - struct dlm_ls *ls; - struct dlm_user_args *ua; - uint32_t lkid; - int error = -ENOMEM; - - ls = dlm_find_lockspace_local(proc->lockspace); - if (!ls) - return -ENOENT; - - if (!params->castaddr || !params->lksb) { - error = -EINVAL; - goto out; - } - - ua = kzalloc(sizeof(struct dlm_user_args), GFP_NOFS); - if (!ua) - goto out; - ua->proc = proc; - ua->user_lksb = params->lksb; - ua->castparam = params->castparam; - ua->castaddr = params->castaddr; - ua->bastparam = params->bastparam; - ua->bastaddr = params->bastaddr; - ua->xid = params->xid; - - if (params->flags & DLM_LKF_CONVERT) { - error = dlm_user_convert(ls, ua, - params->mode, params->flags, - params->lkid, params->lvb, - (unsigned long) params->timeout); - } else if (params->flags & DLM_LKF_ORPHAN) { - error = dlm_user_adopt_orphan(ls, ua, - params->mode, params->flags, - params->name, params->namelen, - (unsigned long) params->timeout, - &lkid); - if (!error) - error = lkid; - } else { - error = dlm_user_request(ls, ua, - params->mode, params->flags, - params->name, params->namelen, - (unsigned long) params->timeout); - if (!error) - error = ua->lksb.sb_lkid; - } - out: - dlm_put_lockspace(ls); - return error; -} - -static int device_user_unlock(struct dlm_user_proc *proc, - struct dlm_lock_params *params) -{ - struct dlm_ls *ls; - struct dlm_user_args *ua; - int error = -ENOMEM; - - ls = dlm_find_lockspace_local(proc->lockspace); - if (!ls) - return -ENOENT; - - ua = kzalloc(sizeof(struct dlm_user_args), GFP_NOFS); - if (!ua) - goto out; - ua->proc = proc; - ua->user_lksb = params->lksb; - ua->castparam = params->castparam; - ua->castaddr = params->castaddr; - - if (params->flags & DLM_LKF_CANCEL) - error = dlm_user_cancel(ls, ua, params->flags, params->lkid); - else - error = dlm_user_unlock(ls, ua, params->flags, params->lkid, - params->lvb); - out: - dlm_put_lockspace(ls); - return error; -} - -static int device_user_deadlock(struct dlm_user_proc *proc, - struct dlm_lock_params *params) -{ - struct dlm_ls *ls; - int error; - - ls = dlm_find_lockspace_local(proc->lockspace); - if (!ls) - return -ENOENT; - - error = dlm_user_deadlock(ls, params->flags, params->lkid); - - dlm_put_lockspace(ls); - return error; -} - -static int dlm_device_register(struct dlm_ls *ls, char *name) -{ - int error, len; - - /* The device is already registered. This happens when the - lockspace is created multiple times from userspace. */ - if (ls->ls_device.name) - return 0; - - error = -ENOMEM; - len = strlen(name) + strlen(name_prefix) + 2; - ls->ls_device.name = kzalloc(len, GFP_NOFS); - if (!ls->ls_device.name) - goto fail; - - snprintf((char *)ls->ls_device.name, len, "%s_%s", name_prefix, - name); - ls->ls_device.fops = &device_fops; - ls->ls_device.minor = MISC_DYNAMIC_MINOR; - - error = misc_register(&ls->ls_device); - if (error) { - kfree(ls->ls_device.name); - } -fail: - return error; -} - -int dlm_device_deregister(struct dlm_ls *ls) -{ - int error; - - /* The device is not registered. This happens when the lockspace - was never used from userspace, or when device_create_lockspace() - calls dlm_release_lockspace() after the register fails. */ - if (!ls->ls_device.name) - return 0; - - error = misc_deregister(&ls->ls_device); - if (!error) - kfree(ls->ls_device.name); - return error; -} - -static int device_user_purge(struct dlm_user_proc *proc, - struct dlm_purge_params *params) -{ - struct dlm_ls *ls; - int error; - - ls = dlm_find_lockspace_local(proc->lockspace); - if (!ls) - return -ENOENT; - - error = dlm_user_purge(ls, proc, params->nodeid, params->pid); - - dlm_put_lockspace(ls); - return error; -} - -static int device_create_lockspace(struct dlm_lspace_params *params) -{ - dlm_lockspace_t *lockspace; - struct dlm_ls *ls; - int error; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - error = dlm_new_lockspace(params->name, NULL, params->flags, - DLM_USER_LVB_LEN, NULL, NULL, NULL, - &lockspace); - if (error) - return error; - - ls = dlm_find_lockspace_local(lockspace); - if (!ls) - return -ENOENT; - - error = dlm_device_register(ls, params->name); - dlm_put_lockspace(ls); - - if (error) - dlm_release_lockspace(lockspace, 0); - else - error = ls->ls_device.minor; - - return error; -} - -static int device_remove_lockspace(struct dlm_lspace_params *params) -{ - dlm_lockspace_t *lockspace; - struct dlm_ls *ls; - int error, force = 0; - - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - ls = dlm_find_lockspace_device(params->minor); - if (!ls) - return -ENOENT; - - if (params->flags & DLM_USER_LSFLG_FORCEFREE) - force = 2; - - lockspace = ls->ls_local_handle; - dlm_put_lockspace(ls); - - /* The final dlm_release_lockspace waits for references to go to - zero, so all processes will need to close their device for the - ls before the release will proceed. release also calls the - device_deregister above. Converting a positive return value - from release to zero means that userspace won't know when its - release was the final one, but it shouldn't need to know. */ - - error = dlm_release_lockspace(lockspace, force); - if (error > 0) - error = 0; - return error; -} - -/* Check the user's version matches ours */ -static int check_version(struct dlm_write_request *req) -{ - if (req->version[0] != DLM_DEVICE_VERSION_MAJOR || - (req->version[0] == DLM_DEVICE_VERSION_MAJOR && - req->version[1] > DLM_DEVICE_VERSION_MINOR)) { - - printk(KERN_DEBUG "dlm: process %s (%d) version mismatch " - "user (%d.%d.%d) kernel (%d.%d.%d)\n", - current->comm, - task_pid_nr(current), - req->version[0], - req->version[1], - req->version[2], - DLM_DEVICE_VERSION_MAJOR, - DLM_DEVICE_VERSION_MINOR, - DLM_DEVICE_VERSION_PATCH); - return -EINVAL; - } - return 0; -} - -/* - * device_write - * - * device_user_lock - * dlm_user_request -> request_lock - * dlm_user_convert -> convert_lock - * - * device_user_unlock - * dlm_user_unlock -> unlock_lock - * dlm_user_cancel -> cancel_lock - * - * device_create_lockspace - * dlm_new_lockspace - * - * device_remove_lockspace - * dlm_release_lockspace - */ - -/* a write to a lockspace device is a lock or unlock request, a write - to the control device is to create/remove a lockspace */ - -static ssize_t device_write(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct dlm_user_proc *proc = file->private_data; - struct dlm_write_request *kbuf; - sigset_t tmpsig, allsigs; - int error; - -#ifdef CONFIG_COMPAT - if (count < sizeof(struct dlm_write_request32)) -#else - if (count < sizeof(struct dlm_write_request)) -#endif - return -EINVAL; - - /* - * can't compare against COMPAT/dlm_write_request32 because - * we don't yet know if is64bit is zero - */ - if (count > sizeof(struct dlm_write_request) + DLM_RESNAME_MAXLEN) - return -EINVAL; - - kbuf = kzalloc(count + 1, GFP_NOFS); - if (!kbuf) - return -ENOMEM; - - if (copy_from_user(kbuf, buf, count)) { - error = -EFAULT; - goto out_free; - } - - if (check_version(kbuf)) { - error = -EBADE; - goto out_free; - } - -#ifdef CONFIG_COMPAT - if (!kbuf->is64bit) { - struct dlm_write_request32 *k32buf; - int namelen = 0; - - if (count > sizeof(struct dlm_write_request32)) - namelen = count - sizeof(struct dlm_write_request32); - - k32buf = (struct dlm_write_request32 *)kbuf; - - /* add 1 after namelen so that the name string is terminated */ - kbuf = kzalloc(sizeof(struct dlm_write_request) + namelen + 1, - GFP_NOFS); - if (!kbuf) { - kfree(k32buf); - return -ENOMEM; - } - - if (proc) - set_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags); - - compat_input(kbuf, k32buf, namelen); - kfree(k32buf); - } -#endif - - /* do we really need this? can a write happen after a close? */ - if ((kbuf->cmd == DLM_USER_LOCK || kbuf->cmd == DLM_USER_UNLOCK) && - (proc && test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags))) { - error = -EINVAL; - goto out_free; - } - - sigfillset(&allsigs); - sigprocmask(SIG_BLOCK, &allsigs, &tmpsig); - - error = -EINVAL; - - switch (kbuf->cmd) - { - case DLM_USER_LOCK: - if (!proc) { - log_print("no locking on control device"); - goto out_sig; - } - error = device_user_lock(proc, &kbuf->i.lock); - break; - - case DLM_USER_UNLOCK: - if (!proc) { - log_print("no locking on control device"); - goto out_sig; - } - error = device_user_unlock(proc, &kbuf->i.lock); - break; - - case DLM_USER_DEADLOCK: - if (!proc) { - log_print("no locking on control device"); - goto out_sig; - } - error = device_user_deadlock(proc, &kbuf->i.lock); - break; - - case DLM_USER_CREATE_LOCKSPACE: - if (proc) { - log_print("create/remove only on control device"); - goto out_sig; - } - error = device_create_lockspace(&kbuf->i.lspace); - break; - - case DLM_USER_REMOVE_LOCKSPACE: - if (proc) { - log_print("create/remove only on control device"); - goto out_sig; - } - error = device_remove_lockspace(&kbuf->i.lspace); - break; - - case DLM_USER_PURGE: - if (!proc) { - log_print("no locking on control device"); - goto out_sig; - } - error = device_user_purge(proc, &kbuf->i.purge); - break; - - default: - log_print("Unknown command passed to DLM device : %d\n", - kbuf->cmd); - } - - out_sig: - sigprocmask(SIG_SETMASK, &tmpsig, NULL); - out_free: - kfree(kbuf); - return error; -} - -/* Every process that opens the lockspace device has its own "proc" structure - hanging off the open file that's used to keep track of locks owned by the - process and asts that need to be delivered to the process. */ - -static int device_open(struct inode *inode, struct file *file) -{ - struct dlm_user_proc *proc; - struct dlm_ls *ls; - - ls = dlm_find_lockspace_device(iminor(inode)); - if (!ls) - return -ENOENT; - - proc = kzalloc(sizeof(struct dlm_user_proc), GFP_NOFS); - if (!proc) { - dlm_put_lockspace(ls); - return -ENOMEM; - } - - proc->lockspace = ls->ls_local_handle; - INIT_LIST_HEAD(&proc->asts); - INIT_LIST_HEAD(&proc->locks); - INIT_LIST_HEAD(&proc->unlocking); - spin_lock_init(&proc->asts_spin); - spin_lock_init(&proc->locks_spin); - init_waitqueue_head(&proc->wait); - file->private_data = proc; - - return 0; -} - -static int device_close(struct inode *inode, struct file *file) -{ - struct dlm_user_proc *proc = file->private_data; - struct dlm_ls *ls; - sigset_t tmpsig, allsigs; - - ls = dlm_find_lockspace_local(proc->lockspace); - if (!ls) - return -ENOENT; - - sigfillset(&allsigs); - sigprocmask(SIG_BLOCK, &allsigs, &tmpsig); - - set_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags); - - dlm_clear_proc_locks(ls, proc); - - /* at this point no more lkb's should exist for this lockspace, - so there's no chance of dlm_user_add_ast() being called and - looking for lkb->ua->proc */ - - kfree(proc); - file->private_data = NULL; - - dlm_put_lockspace(ls); - dlm_put_lockspace(ls); /* for the find in device_open() */ - - /* FIXME: AUTOFREE: if this ls is no longer used do - device_remove_lockspace() */ - - sigprocmask(SIG_SETMASK, &tmpsig, NULL); - recalc_sigpending(); - - return 0; -} - -static int copy_result_to_user(struct dlm_user_args *ua, int compat, - uint32_t flags, int mode, int copy_lvb, - char __user *buf, size_t count) -{ -#ifdef CONFIG_COMPAT - struct dlm_lock_result32 result32; -#endif - struct dlm_lock_result result; - void *resultptr; - int error=0; - int len; - int struct_len; - - memset(&result, 0, sizeof(struct dlm_lock_result)); - result.version[0] = DLM_DEVICE_VERSION_MAJOR; - result.version[1] = DLM_DEVICE_VERSION_MINOR; - result.version[2] = DLM_DEVICE_VERSION_PATCH; - memcpy(&result.lksb, &ua->lksb, sizeof(struct dlm_lksb)); - result.user_lksb = ua->user_lksb; - - /* FIXME: dlm1 provides for the user's bastparam/addr to not be updated - in a conversion unless the conversion is successful. See code - in dlm_user_convert() for updating ua from ua_tmp. OpenVMS, though, - notes that a new blocking AST address and parameter are set even if - the conversion fails, so maybe we should just do that. */ - - if (flags & DLM_CB_BAST) { - result.user_astaddr = ua->bastaddr; - result.user_astparam = ua->bastparam; - result.bast_mode = mode; - } else { - result.user_astaddr = ua->castaddr; - result.user_astparam = ua->castparam; - } - -#ifdef CONFIG_COMPAT - if (compat) - len = sizeof(struct dlm_lock_result32); - else -#endif - len = sizeof(struct dlm_lock_result); - struct_len = len; - - /* copy lvb to userspace if there is one, it's been updated, and - the user buffer has space for it */ - - if (copy_lvb && ua->lksb.sb_lvbptr && count >= len + DLM_USER_LVB_LEN) { - if (copy_to_user(buf+len, ua->lksb.sb_lvbptr, - DLM_USER_LVB_LEN)) { - error = -EFAULT; - goto out; - } - - result.lvb_offset = len; - len += DLM_USER_LVB_LEN; - } - - result.length = len; - resultptr = &result; -#ifdef CONFIG_COMPAT - if (compat) { - compat_output(&result, &result32); - resultptr = &result32; - } -#endif - - if (copy_to_user(buf, resultptr, struct_len)) - error = -EFAULT; - else - error = len; - out: - return error; -} - -static int copy_version_to_user(char __user *buf, size_t count) -{ - struct dlm_device_version ver; - - memset(&ver, 0, sizeof(struct dlm_device_version)); - ver.version[0] = DLM_DEVICE_VERSION_MAJOR; - ver.version[1] = DLM_DEVICE_VERSION_MINOR; - ver.version[2] = DLM_DEVICE_VERSION_PATCH; - - if (copy_to_user(buf, &ver, sizeof(struct dlm_device_version))) - return -EFAULT; - return sizeof(struct dlm_device_version); -} - -/* a read returns a single ast described in a struct dlm_lock_result */ - -static ssize_t device_read(struct file *file, char __user *buf, size_t count, - loff_t *ppos) -{ - struct dlm_user_proc *proc = file->private_data; - struct dlm_lkb *lkb; - DECLARE_WAITQUEUE(wait, current); - struct dlm_callback cb; - int rv, resid, copy_lvb = 0; - - if (count == sizeof(struct dlm_device_version)) { - rv = copy_version_to_user(buf, count); - return rv; - } - - if (!proc) { - log_print("non-version read from control device %zu", count); - return -EINVAL; - } - -#ifdef CONFIG_COMPAT - if (count < sizeof(struct dlm_lock_result32)) -#else - if (count < sizeof(struct dlm_lock_result)) -#endif - return -EINVAL; - - try_another: - - /* do we really need this? can a read happen after a close? */ - if (test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags)) - return -EINVAL; - - spin_lock(&proc->asts_spin); - if (list_empty(&proc->asts)) { - if (file->f_flags & O_NONBLOCK) { - spin_unlock(&proc->asts_spin); - return -EAGAIN; - } - - add_wait_queue(&proc->wait, &wait); - - repeat: - set_current_state(TASK_INTERRUPTIBLE); - if (list_empty(&proc->asts) && !signal_pending(current)) { - spin_unlock(&proc->asts_spin); - schedule(); - spin_lock(&proc->asts_spin); - goto repeat; - } - set_current_state(TASK_RUNNING); - remove_wait_queue(&proc->wait, &wait); - - if (signal_pending(current)) { - spin_unlock(&proc->asts_spin); - return -ERESTARTSYS; - } - } - - /* if we empty lkb_callbacks, we don't want to unlock the spinlock - without removing lkb_cb_list; so empty lkb_cb_list is always - consistent with empty lkb_callbacks */ - - lkb = list_entry(proc->asts.next, struct dlm_lkb, lkb_cb_list); - - rv = dlm_rem_lkb_callback(lkb->lkb_resource->res_ls, lkb, &cb, &resid); - if (rv < 0) { - /* this shouldn't happen; lkb should have been removed from - list when resid was zero */ - log_print("dlm_rem_lkb_callback empty %x", lkb->lkb_id); - list_del_init(&lkb->lkb_cb_list); - spin_unlock(&proc->asts_spin); - /* removes ref for proc->asts, may cause lkb to be freed */ - dlm_put_lkb(lkb); - goto try_another; - } - if (!resid) - list_del_init(&lkb->lkb_cb_list); - spin_unlock(&proc->asts_spin); - - if (cb.flags & DLM_CB_SKIP) { - /* removes ref for proc->asts, may cause lkb to be freed */ - if (!resid) - dlm_put_lkb(lkb); - goto try_another; - } - - if (cb.flags & DLM_CB_CAST) { - int old_mode, new_mode; - - old_mode = lkb->lkb_last_cast.mode; - new_mode = cb.mode; - - if (!cb.sb_status && lkb->lkb_lksb->sb_lvbptr && - dlm_lvb_operations[old_mode + 1][new_mode + 1]) - copy_lvb = 1; - - lkb->lkb_lksb->sb_status = cb.sb_status; - lkb->lkb_lksb->sb_flags = cb.sb_flags; - } - - rv = copy_result_to_user(lkb->lkb_ua, - test_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags), - cb.flags, cb.mode, copy_lvb, buf, count); - - /* removes ref for proc->asts, may cause lkb to be freed */ - if (!resid) - dlm_put_lkb(lkb); - - return rv; -} - -static unsigned int device_poll(struct file *file, poll_table *wait) -{ - struct dlm_user_proc *proc = file->private_data; - - poll_wait(file, &proc->wait, wait); - - spin_lock(&proc->asts_spin); - if (!list_empty(&proc->asts)) { - spin_unlock(&proc->asts_spin); - return POLLIN | POLLRDNORM; - } - spin_unlock(&proc->asts_spin); - return 0; -} - -int dlm_user_daemon_available(void) -{ - /* dlm_controld hasn't started (or, has started, but not - properly populated configfs) */ - - if (!dlm_our_nodeid()) - return 0; - - /* This is to deal with versions of dlm_controld that don't - know about the monitor device. We assume that if the - dlm_controld was started (above), but the monitor device - was never opened, that it's an old version. dlm_controld - should open the monitor device before populating configfs. */ - - if (dlm_monitor_unused) - return 1; - - return atomic_read(&dlm_monitor_opened) ? 1 : 0; -} - -static int ctl_device_open(struct inode *inode, struct file *file) -{ - file->private_data = NULL; - return 0; -} - -static int ctl_device_close(struct inode *inode, struct file *file) -{ - return 0; -} - -static int monitor_device_open(struct inode *inode, struct file *file) -{ - atomic_inc(&dlm_monitor_opened); - dlm_monitor_unused = 0; - return 0; -} - -static int monitor_device_close(struct inode *inode, struct file *file) -{ - if (atomic_dec_and_test(&dlm_monitor_opened)) - dlm_stop_lockspaces(); - return 0; -} - -static const struct file_operations device_fops = { - .open = device_open, - .release = device_close, - .read = device_read, - .write = device_write, - .poll = device_poll, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static const struct file_operations ctl_device_fops = { - .open = ctl_device_open, - .release = ctl_device_close, - .read = device_read, - .write = device_write, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static struct miscdevice ctl_device = { - .name = "dlm-control", - .fops = &ctl_device_fops, - .minor = MISC_DYNAMIC_MINOR, -}; - -static const struct file_operations monitor_device_fops = { - .open = monitor_device_open, - .release = monitor_device_close, - .owner = THIS_MODULE, - .llseek = noop_llseek, -}; - -static struct miscdevice monitor_device = { - .name = "dlm-monitor", - .fops = &monitor_device_fops, - .minor = MISC_DYNAMIC_MINOR, -}; - -int __init dlm_user_init(void) -{ - int error; - - atomic_set(&dlm_monitor_opened, 0); - - error = misc_register(&ctl_device); - if (error) { - log_print("misc_register failed for control device"); - goto out; - } - - error = misc_register(&monitor_device); - if (error) { - log_print("misc_register failed for monitor device"); - misc_deregister(&ctl_device); - } - out: - return error; -} - -void dlm_user_exit(void) -{ - misc_deregister(&ctl_device); - misc_deregister(&monitor_device); -} - diff --git a/kmod/dlm/user.h b/kmod/dlm/user.h deleted file mode 100644 index 00499ab8..00000000 --- a/kmod/dlm/user.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. - * - * This copyrighted material is made available to anyone wishing to use, - * modify, copy, or redistribute it subject to the terms and conditions - * of the GNU General Public License v.2. - */ - -#ifndef __USER_DOT_H__ -#define __USER_DOT_H__ - -void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, - int status, uint32_t sbflags, uint64_t seq); -int dlm_user_init(void); -void dlm_user_exit(void); -int dlm_device_deregister(struct dlm_ls *ls); -int dlm_user_daemon_available(void); - -#endif diff --git a/kmod/dlm/util.c b/kmod/dlm/util.c deleted file mode 100644 index b18eded3..00000000 --- a/kmod/dlm/util.c +++ /dev/null @@ -1,172 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#include "dlm_internal.h" -#include "rcom.h" -#include "util.h" - -#define DLM_ERRNO_EDEADLK 35 -#define DLM_ERRNO_EBADR 53 -#define DLM_ERRNO_EBADSLT 57 -#define DLM_ERRNO_EPROTO 71 -#define DLM_ERRNO_EOPNOTSUPP 95 -#define DLM_ERRNO_ETIMEDOUT 110 -#define DLM_ERRNO_EINPROGRESS 115 - -static void header_out(struct dlm_header *hd) -{ - hd->h_version = cpu_to_le32(hd->h_version); - hd->h_lockspace = cpu_to_le32(hd->h_lockspace); - hd->h_nodeid = cpu_to_le32(hd->h_nodeid); - hd->h_length = cpu_to_le16(hd->h_length); -} - -static void header_in(struct dlm_header *hd) -{ - hd->h_version = le32_to_cpu(hd->h_version); - hd->h_lockspace = le32_to_cpu(hd->h_lockspace); - hd->h_nodeid = le32_to_cpu(hd->h_nodeid); - hd->h_length = le16_to_cpu(hd->h_length); -} - -/* higher errno values are inconsistent across architectures, so select - one set of values for on the wire */ - -static int to_dlm_errno(int err) -{ - switch (err) { - case -EDEADLK: - return -DLM_ERRNO_EDEADLK; - case -EBADR: - return -DLM_ERRNO_EBADR; - case -EBADSLT: - return -DLM_ERRNO_EBADSLT; - case -EPROTO: - return -DLM_ERRNO_EPROTO; - case -EOPNOTSUPP: - return -DLM_ERRNO_EOPNOTSUPP; - case -ETIMEDOUT: - return -DLM_ERRNO_ETIMEDOUT; - case -EINPROGRESS: - return -DLM_ERRNO_EINPROGRESS; - } - return err; -} - -static int from_dlm_errno(int err) -{ - switch (err) { - case -DLM_ERRNO_EDEADLK: - return -EDEADLK; - case -DLM_ERRNO_EBADR: - return -EBADR; - case -DLM_ERRNO_EBADSLT: - return -EBADSLT; - case -DLM_ERRNO_EPROTO: - return -EPROTO; - case -DLM_ERRNO_EOPNOTSUPP: - return -EOPNOTSUPP; - case -DLM_ERRNO_ETIMEDOUT: - return -ETIMEDOUT; - case -DLM_ERRNO_EINPROGRESS: - return -EINPROGRESS; - } - return err; -} - -void dlm_message_out(struct dlm_message *ms) -{ - header_out(&ms->m_header); - - ms->m_type = cpu_to_le32(ms->m_type); - ms->m_nodeid = cpu_to_le32(ms->m_nodeid); - ms->m_pid = cpu_to_le32(ms->m_pid); - ms->m_lkid = cpu_to_le32(ms->m_lkid); - ms->m_remid = cpu_to_le32(ms->m_remid); - ms->m_parent_lkid = cpu_to_le32(ms->m_parent_lkid); - ms->m_parent_remid = cpu_to_le32(ms->m_parent_remid); - ms->m_exflags = cpu_to_le32(ms->m_exflags); - ms->m_sbflags = cpu_to_le32(ms->m_sbflags); - ms->m_flags = cpu_to_le32(ms->m_flags); - ms->m_lvbseq = cpu_to_le32(ms->m_lvbseq); - ms->m_hash = cpu_to_le32(ms->m_hash); - ms->m_status = cpu_to_le32(ms->m_status); - ms->m_grmode = cpu_to_le32(ms->m_grmode); - ms->m_rqmode = cpu_to_le32(ms->m_rqmode); - ms->m_bastmode = cpu_to_le32(ms->m_bastmode); - ms->m_asts = cpu_to_le32(ms->m_asts); - ms->m_result = cpu_to_le32(to_dlm_errno(ms->m_result)); - - ms->m_grstart_len = cpu_to_le16(ms->m_grstart_len); - ms->m_grend_len = cpu_to_le16(ms->m_grend_len); - - ms->m_rqstart_len = cpu_to_le16(ms->m_rqstart_len); - ms->m_rqend_len = cpu_to_le16(ms->m_rqend_len); - - ms->m_baststart_len = cpu_to_le16(ms->m_baststart_len); - ms->m_bastend_len = cpu_to_le16(ms->m_bastend_len); -} - -void dlm_message_in(struct dlm_message *ms) -{ - header_in(&ms->m_header); - - ms->m_type = le32_to_cpu(ms->m_type); - ms->m_nodeid = le32_to_cpu(ms->m_nodeid); - ms->m_pid = le32_to_cpu(ms->m_pid); - ms->m_lkid = le32_to_cpu(ms->m_lkid); - ms->m_remid = le32_to_cpu(ms->m_remid); - ms->m_parent_lkid = le32_to_cpu(ms->m_parent_lkid); - ms->m_parent_remid = le32_to_cpu(ms->m_parent_remid); - ms->m_exflags = le32_to_cpu(ms->m_exflags); - ms->m_sbflags = le32_to_cpu(ms->m_sbflags); - ms->m_flags = le32_to_cpu(ms->m_flags); - ms->m_lvbseq = le32_to_cpu(ms->m_lvbseq); - ms->m_hash = le32_to_cpu(ms->m_hash); - ms->m_status = le32_to_cpu(ms->m_status); - ms->m_grmode = le32_to_cpu(ms->m_grmode); - ms->m_rqmode = le32_to_cpu(ms->m_rqmode); - ms->m_bastmode = le32_to_cpu(ms->m_bastmode); - ms->m_asts = le32_to_cpu(ms->m_asts); - ms->m_result = from_dlm_errno(le32_to_cpu(ms->m_result)); - - ms->m_grstart_len = le16_to_cpu(ms->m_grstart_len); - ms->m_grend_len = le16_to_cpu(ms->m_grend_len); - - ms->m_rqstart_len = le16_to_cpu(ms->m_rqstart_len); - ms->m_rqend_len = le16_to_cpu(ms->m_rqend_len); - - ms->m_baststart_len = le16_to_cpu(ms->m_baststart_len); - ms->m_bastend_len = le16_to_cpu(ms->m_bastend_len); -} - -void dlm_rcom_out(struct dlm_rcom *rc) -{ - header_out(&rc->rc_header); - - rc->rc_type = cpu_to_le32(rc->rc_type); - rc->rc_result = cpu_to_le32(rc->rc_result); - rc->rc_id = cpu_to_le64(rc->rc_id); - rc->rc_seq = cpu_to_le64(rc->rc_seq); - rc->rc_seq_reply = cpu_to_le64(rc->rc_seq_reply); -} - -void dlm_rcom_in(struct dlm_rcom *rc) -{ - header_in(&rc->rc_header); - - rc->rc_type = le32_to_cpu(rc->rc_type); - rc->rc_result = le32_to_cpu(rc->rc_result); - rc->rc_id = le64_to_cpu(rc->rc_id); - rc->rc_seq = le64_to_cpu(rc->rc_seq); - rc->rc_seq_reply = le64_to_cpu(rc->rc_seq_reply); -} diff --git a/kmod/dlm/util.h b/kmod/dlm/util.h deleted file mode 100644 index 2b259151..00000000 --- a/kmod/dlm/util.h +++ /dev/null @@ -1,22 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __UTIL_DOT_H__ -#define __UTIL_DOT_H__ - -void dlm_message_out(struct dlm_message *ms); -void dlm_message_in(struct dlm_message *ms); -void dlm_rcom_out(struct dlm_rcom *rc); -void dlm_rcom_in(struct dlm_rcom *rc); - -#endif - From 67cc4fb6971639de44d211b806e5d0f2cdb37455 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Jul 2017 14:13:23 -0700 Subject: [PATCH 332/920] scoutfs: allow NULL end around read_items Let both check_range and read_items take a NULL end. check_range just doesn't do anything with the end of the range. read_items defaults to trying to read as many items as it can but clamps to the extent of the segments that intersect with the key. This will let us incrementally add end arguments to the item functions that are intially passed in as NULL in callers as we add lock coverage. Signed-off-by: Zach Brown --- kmod/src/item.c | 13 ++++++++----- kmod/src/manifest.c | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 977ae63d..9dcc38c9 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -538,15 +538,18 @@ static bool check_range(struct super_block *sb, struct rb_root *root, rng = walk_ranges(&cac->ranges, key, NULL, &next); if (rng) { - scoutfs_key_copy(end, rng->end); scoutfs_inc_counter(sb, item_range_hit); + if (end) + scoutfs_key_copy(end, rng->end); return true; } - if (next) - scoutfs_key_copy(end, next->start); - else - scoutfs_key_set_max(end); + if (end) { + if (next) + scoutfs_key_copy(end, next->start); + else + scoutfs_key_set_max(end); + } scoutfs_inc_counter(sb, item_range_miss); return false; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 7cad79cd..e8065724 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -539,9 +539,8 @@ out: * The caller found a hole in the item cache that they'd like populated. * * We search the manifest for all the segments we'll need to iterate - * from the key to the end key. We walk the segments and insert as many - * items as we can from the segments, trying to amortize the per-item - * cost of segment searching. + * from the key to the end key. If the end key is null then we'll read + * as many items as the intersecting segments contain. * * As we insert the batch of items we give the item cache the range of * keys that contain these items. This lets the cache return negative @@ -569,6 +568,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf batch_end; struct scoutfs_key_buf seg_end; struct scoutfs_btree_root root; + struct scoutfs_inode_key junk; SCOUTFS_DECLARE_KVEC(item_val); SCOUTFS_DECLARE_KVEC(found_val); struct scoutfs_segment *seg; @@ -585,7 +585,14 @@ int scoutfs_manifest_read_items(struct super_block *sb, int err; int cmp; - trace_scoutfs_read_items(sb, key, end); + if (end) { + scoutfs_key_clone(&seg_end, end); + } else { + scoutfs_key_init(&seg_end, &junk, sizeof(junk)); + scoutfs_key_set_max(&seg_end); + } + + trace_scoutfs_read_items(sb, key, &seg_end); /* @@ -599,7 +606,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, goto out; /* get refs on all the segments */ - ret = get_manifest_refs(sb, &root, key, end, &ref_list); + ret = get_manifest_refs(sb, &root, key, &seg_end, &ref_list); if (ret) goto out; @@ -642,7 +649,6 @@ int scoutfs_manifest_read_items(struct super_block *sb, * those segments because other segments might overlap after * that. */ - scoutfs_key_clone(&seg_end, end); list_for_each_entry(ref, &ref_list, entry) { if (ref->level > 0 && scoutfs_key_compare(ref->last, &seg_end) < 0) { From 19171f7a251ee2ad68039c3818c2427f049dec9a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Jul 2017 14:20:18 -0700 Subject: [PATCH 333/920] scoutfs: add end to _item_lookup The item cache can only be populated with items that are covered by locks. Require callers to provide the farthest key that can be covered by the locks. Locks provide a key for exactly this purpose. Signed-off-by: Zach Brown --- kmod/src/dir.c | 5 +++-- kmod/src/inode.c | 6 +++--- kmod/src/item.c | 23 ++++++++++------------- kmod/src/item.h | 4 ++-- kmod/src/xattr.c | 2 +- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 7bb21fe0..4dc5b3da 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -251,7 +251,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent)); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), NULL); if (ret == -ENOENT) { ino = 0; ret = 0; @@ -689,7 +689,8 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, if (op == SYM_CREATE) ret = scoutfs_item_create(sb, &key, val); else if (op == SYM_LOOKUP) - ret = scoutfs_item_lookup_exact(sb, &key, val, bytes); + ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, + NULL); else if (op == SYM_DELETE) ret = scoutfs_item_delete(sb, &key); if (ret) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index cf3a1714..bd49c494 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -243,7 +243,7 @@ static int scoutfs_read_locked_inode(struct inode *inode) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); if (ret == 0) load_inode(inode, &sinode); @@ -838,7 +838,7 @@ static void delete_inode(struct super_block *sb, u64 ino) scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); if (ret < 0) goto out; @@ -892,7 +892,7 @@ static int process_orphaned_inode(struct super_block *sb, u64 ino) scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode)); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/item.c b/kmod/src/item.c index 9dcc38c9..2e9de37c 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -717,25 +717,21 @@ restart: * Find an item with the given key and copy its value into the caller's * value vector. The amount of bytes copied is returned which can be 0 * or truncated if the caller's buffer isn't big enough. + * + * The end key limits how many keys after the search key can be read + * and inserted into the cache. */ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val) + struct kvec *val, struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *end; struct cached_item *item; unsigned long flags; int ret; trace_scoutfs_item_lookup(sb, key); - end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!end) { - ret = -ENOMEM; - goto out; - } - do { spin_lock_irqsave(&cac->lock, flags); @@ -743,7 +739,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, if (item) { item_referenced(cac, item); ret = scoutfs_kvec_memcpy(val, item->val); - } else if (check_range(sb, &cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -754,8 +750,6 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - scoutfs_key_free(sb, end); -out: trace_printk("ret %d\n", ret); return ret; } @@ -768,15 +762,18 @@ out: * overhead that comes from only detecting the size mismatch after the * copy by reusing the more permissive _lookup(). * + * The end key limits how many keys after the search key can be read + * and inserted into the cache. + * * Returns 0 or -errno. */ int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size) + int size, struct scoutfs_key_buf *end) { int ret; - ret = scoutfs_item_lookup(sb, key, val); + ret = scoutfs_item_lookup(sb, key, val, end); if (ret == size) ret = 0; else if (ret >= 0) diff --git a/kmod/src/item.h b/kmod/src/item.h index fb9c1df4..774f0ab7 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -13,10 +13,10 @@ struct scoutfs_segment; struct scoutfs_key_buf; int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val); + struct kvec *val, struct scoutfs_key_buf *end); int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size); + int size, struct scoutfs_key_buf *end); int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val); int scoutfs_item_next_same_min(struct super_block *sb, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 29aba33d..19dbce6f 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -189,7 +189,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, for_each_xattr_item(key, val, &vh, buffer, size, part, off, bytes) { - ret = scoutfs_item_lookup(sb, key, val); + ret = scoutfs_item_lookup(sb, key, val, lck->end); if (ret < 0) { if (ret == -ENOENT) ret = -EIO; From c80dd579e1d9c82fe2605e7252f98fb062ea9761 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 08:56:06 -0700 Subject: [PATCH 334/920] scoutfs: add scoutfs_manifest_next_key Add an item reading variant that just returns the next key that it finds in segments after the given key. This will be used while iterating to find the next key to lock and then try to iterate towards. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 35 +++++++++++++++++++++++++++++++---- kmod/src/manifest.h | 3 +++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index e8065724..6781f44e 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -542,6 +542,9 @@ out: * from the key to the end key. If the end key is null then we'll read * as many items as the intersecting segments contain. * + * If next_key is provided then the segments are only walked to find the + * next key after the search key. If none is found -ENOENT is returned. + * * As we insert the batch of items we give the item cache the range of * keys that contain these items. This lets the cache return negative * cache lookups for missing items within the range. @@ -559,9 +562,9 @@ out: * The segments are immutable at this point so we can use their contents * as long as we hold refs. */ -int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end) +static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, + struct scoutfs_key_buf *next_key) { struct scoutfs_key_buf item_key; struct scoutfs_key_buf found_key; @@ -702,6 +705,16 @@ int scoutfs_manifest_read_items(struct super_block *sb, found = true; } + if (next_key) { + if (found) { + scoutfs_key_copy(next_key, &found_key); + ret = 0; + } else { + ret = -ENOENT; + } + break; + } + /* ran out of keys in segs, range extends to seg end */ if (!found) { scoutfs_key_clone(&batch_end, &seg_end); @@ -748,7 +761,7 @@ int scoutfs_manifest_read_items(struct super_block *sb, ret = 0; } - if (ret) + if (next_key || ret) scoutfs_item_free_batch(sb, &batch); else ret = scoutfs_item_insert_batch(sb, &batch, key, &batch_end); @@ -761,6 +774,20 @@ out: return ret; } +int scoutfs_manifest_read_items(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end) +{ + return read_items(sb, key, end, NULL); +} + +int scoutfs_manifest_next_key(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *next_key) +{ + return read_items(sb, key, NULL, next_key); +} + /* * Give the caller the segments that will be involved in the next * compaction. diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 39e0e134..f36a58fe 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -34,6 +34,9 @@ int scoutfs_manifest_unlock(struct super_block *sb); int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); +int scoutfs_manifest_next_key(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *next_key); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); From 4f6f842efaf5c7ecbc962fc23a0f1e186f0e5b5b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 09:00:03 -0700 Subject: [PATCH 335/920] scoutfs: add inode index item locking Add a locking wrapper for the inode index items. It maps the index fields to a lock name for each index type. Signed-off-by: Zach Brown --- kmod/src/lock.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 3 +++ 2 files changed, 73 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 77336cbc..4e47c87e 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -439,6 +439,76 @@ int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, return lock_name_keys(sb, mode, &lock_name, &start, &end, ret_lock); } +/* + * map inode index items to locks. The idea is to not have to + * constantly get locks over a reasonable distribution of items, but + * also not have an insane amount of items covered by locks. time and + * seq indexes have natural batching and limits on the number of keys + * per major value. Size keys are very different. For them we use a + * mix of a sort of linear-log distribution (top 4 bits of size), and + * then also a lot of inodes per size. + */ +int scoutfs_lock_inode_index(struct super_block *sb, int mode, + u8 type, u64 major, u64 ino, + struct scoutfs_lock **ret_lock) +{ + struct scoutfs_lock_name lock_name; + struct scoutfs_inode_index_key start_ikey; + struct scoutfs_inode_index_key end_ikey; + struct scoutfs_key_buf start; + struct scoutfs_key_buf end; + u64 major_mask; + u64 ino_mask; + int bit; + + switch(type) { + case SCOUTFS_INODE_INDEX_CTIME_TYPE: + case SCOUTFS_INODE_INDEX_MTIME_TYPE: + major_mask = (1 << 5) - 1; + ino_mask = ~0ULL; + break; + + case SCOUTFS_INODE_INDEX_SIZE_TYPE: + major_mask = 0; + if (major) { + bit = fls64(major); + if (bit > 4) + major_mask = (1 << (bit - 4)) - 1; + } + ino_mask = (1 << 12) - 1; + break; + + case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: + case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: + major_mask = (1 << 10) - 1; + ino_mask = ~0ULL; + break; + default: + BUG(); + } + + lock_name.zone = SCOUTFS_INODE_INDEX_ZONE; + lock_name.type = type; + lock_name.first = cpu_to_le64(major & ~major_mask); + lock_name.second = cpu_to_le64(ino & ~ino_mask); + + start_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; + start_ikey.type = type; + start_ikey.major = cpu_to_be64(major & ~major_mask); + start_ikey.minor = cpu_to_be32(0); + start_ikey.ino = cpu_to_be64(ino & ~ino_mask); + scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); + + end_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; + end_ikey.type = type; + end_ikey.major = cpu_to_be64(major | major_mask); + end_ikey.minor = cpu_to_be32(U32_MAX); + end_ikey.ino = cpu_to_be64(ino | ino_mask); + scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); + + return lock_name_keys(sb, mode, &lock_name, &start, &end, ret_lock); +} + void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index e08bcfd2..949efd14 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -26,6 +26,9 @@ struct scoutfs_lock { int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, struct scoutfs_lock **ret_lock); +int scoutfs_lock_inode_index(struct super_block *sb, int mode, + u8 type, u64 major, u64 ino, + struct scoutfs_lock **ret_lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock); int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, From f611c769e2a6d4b04e0113f3e28b6fe35ffe89a8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 10:02:36 -0700 Subject: [PATCH 336/920] scoutfs: add 'end' to item_next to limit reads Add an end key to the item_next calls to limit how many items will be read into the cache. Callers typically get this from the lock they hold that covers the iteration. We differentiate between iteration and caching so that a series of small iterations (listxattr on inodes, namespace walk in small dirs) can be satisfied by a single read of adjacent items from segments. Signed-off-by: Zach Brown --- kmod/src/data.c | 12 ++++++------ kmod/src/dir.c | 4 ++-- kmod/src/inode.c | 2 +- kmod/src/item.c | 26 ++++++++++++++------------ kmod/src/item.h | 9 ++++++--- kmod/src/xattr.c | 4 ++-- 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index bc573bfc..ca437490 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -309,7 +309,7 @@ static int try_merge(struct super_block *sb, struct native_extent *cur, ext.flags = 0; init_extent_key(&key, key_bytes, &ext, arg, type); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -455,7 +455,7 @@ static int remove_extent(struct super_block *sb, /* find outer existing extent that contains removal extent */ init_extent_key(&key, key_bytes, rem, arg, type); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret) goto out; @@ -552,7 +552,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, init_extent_key(&key, key_bytes, &rng, ino, SCOUTFS_FILE_EXTENT_TYPE); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -797,7 +797,7 @@ retry: init_extent_key(&key, key_bytes, &ext, sbi->node_id, type); init_extent_key(&last, last_bytes, &last_ext, sbi->node_id, type); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret < 0) { if (ret == -ENOENT) { /* if the cursor's empty fall back to next large */ @@ -966,7 +966,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, * item consistency. */ down_read(&datinf->alloc_rwsem); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); up_read(&datinf->alloc_rwsem); if (ret < 0) { if (ret == -ENOENT) @@ -1146,7 +1146,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, ext.flags = 0; init_extent_key(&key, key_bytes, &ext, ino, type); - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret < 0) { if (ret != -ENOENT) break; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 4dc5b3da..55909f8f 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -344,7 +344,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) scoutfs_kvec_init(val, dent, item_len); ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, - offsetof(struct scoutfs_dirent, name[1])); + offsetof(struct scoutfs_dirent, name[1]), NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -878,7 +878,7 @@ static int add_next_linkref(struct super_block *sb, u64 ino, init_link_backref_key(&last, &last_lbkey, ino, U64_MAX, NULL, 0); /* next backref key is now in ent */ - ret = scoutfs_item_next(sb, &key, &last, NULL); + ret = scoutfs_item_next(sb, &key, &last, NULL, NULL); trace_printk("ino %llu dir_ino %llu ret %d key_len %u\n", ino, dir_ino, ret, key.key_len); if (ret < 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index bd49c494..176bc306 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -933,7 +933,7 @@ int scoutfs_scan_orphans(struct super_block *sb) init_orphan_key(&last, &last_okey, sbi->node_id, ~0ULL); while (1) { - ret = scoutfs_item_next_same(sb, &key, &last, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) diff --git a/kmod/src/item.c b/kmod/src/item.c index 2e9de37c..ac990cf6 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -844,6 +844,10 @@ static struct cached_item *item_for_next(struct rb_root *root, * Return the next item starting with the given key, returning the last * key at the most. * + * While iteration stops the last key we can cache up to the end key so + * that a sequence of small iterations covered by one lock are satisfied + * with a large read of items from segments into the cache. + * * -ENOENT is returned if there are no items between the given and last * keys. * @@ -855,12 +859,12 @@ static struct cached_item *item_for_next(struct rb_root *root, * by the caller's value buffer length. */ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val) + struct scoutfs_key_buf *last, struct kvec *val, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct scoutfs_key_buf *read_start = NULL; - struct scoutfs_key_buf *read_end = NULL; struct scoutfs_key_buf *range_end = NULL; struct cached_item *item; unsigned long flags; @@ -874,9 +878,8 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, } read_start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - read_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!read_start || !read_end || !range_end) { + if (!read_start || !range_end) { ret = -ENOMEM; goto out; } @@ -902,12 +905,10 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, if (!cached) { /* missing cache starts at key */ scoutfs_key_copy(read_start, key); - scoutfs_key_copy(read_end, range_end); } else if (scoutfs_key_compare(range_end, last) < 0) { /* missing cache starts at range_end */ scoutfs_key_copy(read_start, range_end); - scoutfs_key_copy(read_end, last); } else { /* no items and we have cache between key and last */ @@ -917,7 +918,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, read_start, read_end); + ret = scoutfs_manifest_read_items(sb, read_start, end); spin_lock_irqsave(&cac->lock, flags); if (ret) @@ -927,7 +928,6 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); out: scoutfs_key_free(sb, read_start); - scoutfs_key_free(sb, read_end); scoutfs_key_free(sb, range_end); trace_printk("ret %d\n", ret); @@ -943,7 +943,8 @@ out: int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, - struct kvec *val, int len) + struct kvec *val, int len, + struct scoutfs_key_buf *end) { int key_len = key->key_len; int ret; @@ -953,7 +954,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, if (WARN_ON_ONCE(!val || scoutfs_kvec_length(val) < len)) return -EINVAL; - ret = scoutfs_item_next(sb, key, last, val); + ret = scoutfs_item_next(sb, key, last, val, end); if (ret >= 0 && (key->key_len != key_len || ret < len)) ret = -EIO; @@ -967,14 +968,15 @@ int scoutfs_item_next_same_min(struct super_block *sb, * search key. It treats size mismatches as a sign of corruption. */ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val) + struct scoutfs_key_buf *last, struct kvec *val, + struct scoutfs_key_buf *end) { int key_len = key->key_len; int ret; trace_printk("key len %u\n", key_len); - ret = scoutfs_item_next(sb, key, last, val); + ret = scoutfs_item_next(sb, key, last, val, end); if (ret >= 0 && (key->key_len != key_len)) ret = -EIO; diff --git a/kmod/src/item.h b/kmod/src/item.h index 774f0ab7..924f2fda 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -18,13 +18,16 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, int size, struct scoutfs_key_buf *end); int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val); + struct scoutfs_key_buf *last, struct kvec *val, + struct scoutfs_key_buf *end); int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, - struct kvec *val, int len); + struct kvec *val, int len, + struct scoutfs_key_buf *end); int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val); + struct scoutfs_key_buf *last, struct kvec *val, + struct scoutfs_key_buf *end); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 19dbce6f..0fac057d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -394,7 +394,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) total = 0; for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL); + ret = scoutfs_item_next(sb, key, last, NULL, lck->end); if (ret < 0) { if (ret == -ENOENT) ret = total; @@ -476,7 +476,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* the inode is dead so we don't need the xattr sem */ for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL); + ret = scoutfs_item_next(sb, key, last, NULL, lck->end); if (ret < 0) { if (ret == -ENOENT) ret = 0; From 0b64a4c83f92bceb5300e6207f3324641393f624 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 10:10:37 -0700 Subject: [PATCH 337/920] scoutfs: lock inode index item iteration Add locks around inode index item iteration. This is tricky because the inode index items are enormous and we can't default to coarse locks that let it read and iterate over the entire key space. We use the manifest to find the next small fixed size region to lock and iterate from. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 83 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index ad477782..911d696f 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -30,10 +30,23 @@ #include "item.h" #include "data.h" #include "net.h" +#include "lock.h" +#include "manifest.h" /* - * Walk one of the inode index items. This is a thin ioctl wrapper - * around the core item interface. + * We make inode index items coherent by locking fixed size regions of + * the key space. But the inode index item key space is vast and can + * have huge sparse regions. To avoid trying every possible lock in the + * sparse regions we use the manifest to find the next stable key in the + * key space after we find no items in a given lock region. This is + * relatively cheap because reading is going to check the segments + * anyway. + * + * This is copying to userspace while holding a DLM lock. This is safe + * because faulting can convert the lock to a higher level while we hold + * the lower level. DLM locks don't block tasks in a node, they match + * and the tasks fall back to local locking. In this case the spin + * locks around the item cache. */ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) { @@ -43,12 +56,14 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) struct scoutfs_ioctl_walk_inodes_entry ent; struct scoutfs_inode_index_key last_ikey; struct scoutfs_inode_index_key ikey; + struct scoutfs_key_buf *next_key; struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; + struct scoutfs_lock *lock; u64 last_seq; int ret = 0; + u32 nr = 0; u8 type; - u32 nr; if (copy_from_user(&walk, uwalk, sizeof(walk))) return -EFAULT; @@ -86,6 +101,10 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) } } + next_key = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!next_key) + return -ENOMEM; + ikey.zone = SCOUTFS_INODE_INDEX_ZONE; ikey.type = type; ikey.major = cpu_to_be64(walk.first.major); @@ -103,14 +122,54 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) /* cap nr to the max the ioctl can return to a compat task */ walk.nr_entries = min_t(u64, walk.nr_entries, INT_MAX); + ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, type, walk.first.major, + walk.first.ino, &lock); + if (ret < 0) + goto out; + for (nr = 0; nr < walk.nr_entries; nr++, walk.entries_ptr += sizeof(ent)) { - ret = scoutfs_item_next_same(sb, &key, &last_key, NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; + ret = scoutfs_item_next_same(sb, &key, &last_key, NULL, lock->end); + if (ret < 0 && ret != -ENOENT) break; + + if (ret == -ENOENT) { + + scoutfs_unlock(sb, lock); + /* + * XXX This will miss dirty items. We'd need to + * force writeouts of dirty items in our + * zone|type and get the manifest root for that. + * It'd mean adding a lock to the inode index + * items which isn't quite there yet. + */ + ret = scoutfs_manifest_next_key(sb, &key, next_key); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == -ENOENT || + scoutfs_key_compare(next_key, &last_key) > 0) { + ret = 0; + goto out; + } + + /* if it's within last it should be same size */ + if (next_key->key_len != key.key_len) { + ret = -EIO; + goto out; + } + + scoutfs_key_copy(&key, next_key); + + ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, ikey.type, + be64_to_cpu(ikey.major), + be64_to_cpu(ikey.ino), + &lock); + if (ret < 0) + goto out; + + continue; } ent.major = be64_to_cpu(ikey.major); @@ -126,7 +185,15 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) scoutfs_key_inc_cur_len(&key); } - return nr ?: ret; + scoutfs_unlock(sb, lock); + +out: + scoutfs_key_free(sb, next_key); + + if (nr > 0) + ret = nr; + + return ret; } struct ino_path_cursor { From d78ed098a768eb4371e2e13850a2f3b19f0e5863 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 10:27:35 -0700 Subject: [PATCH 338/920] scoutfs: add cache reading limit to _set_batch Add an end argument to _set_batch to specify the limit of items we'll read into the cache. And it turns out that the loop in _set_batch that meant to cache all the items covered by the batch didn't try hard enough. It would stop once the first key was covered but didn't make sure that the coverage extended to cover last. This can happen if segment boundaries happen to fall within the items that make up the batch. Fix it up while we're in here. Signed-off-by: Zach Brown --- kmod/src/item.c | 39 +++++++++++++++++++++++++-------------- kmod/src/item.h | 5 +++-- kmod/src/xattr.c | 2 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index ac990cf6..a1fa214e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1125,12 +1125,13 @@ out: * batch item does have an existing item. */ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, int sif) + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, int sif, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *missing; + struct scoutfs_key_buf *range_end; SCOUTFS_DECLARE_KVEC(del_val); struct cached_item *exist; struct cached_item *item; @@ -1147,21 +1148,31 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, return -EINVAL; } - trace_scoutfs_item_set_batch(sb, start, end); + trace_scoutfs_item_set_batch(sb, first, last); - if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) + if (WARN_ON_ONCE(scoutfs_key_compare(first, last) > 0) || + WARN_ON_ONCE(scoutfs_key_compare(end, last) < 0)) return -EINVAL; - missing = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!missing) + range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + if (!range_end) return -ENOMEM; spin_lock_irqsave(&cac->lock, flags); - while (!check_range(sb, &cac->ranges, start, missing)) { + /* make sure all of first through last are cached */ + scoutfs_key_copy(range_end, first); + for (;;) { + if (check_range(sb, &cac->ranges, range_end, range_end)) { + if (scoutfs_key_compare(range_end, last) >= 0) + break; + /* start reading from hole starting at range_end */ + } else { + scoutfs_key_copy(range_end, first); + } spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, start, missing); + ret = scoutfs_manifest_read_items(sb, range_end, end); spin_lock_irqsave(&cac->lock, flags); if (ret) @@ -1172,7 +1183,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, if (!list_empty(list) && (sif & (SIF_EXCLUSIVE | SIF_REPLACE))) { item = list_first_entry(list, struct cached_item, entry); - exist = item_for_next(&cac->items, start, NULL, end); + exist = item_for_next(&cac->items, first, NULL, last); while (item) { /* compare keys, with bias to finding _REPLACE err */ @@ -1193,7 +1204,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, item = NULL; } else if (cmp > 0) { - exist = next_item_node(&cac->items, exist, end); + exist = next_item_node(&cac->items, exist, last); } else { /* cmp == 0 */ @@ -1207,8 +1218,8 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, } /* delete everything in the range */ - for (exist = item_for_next(&cac->items, start, NULL, end); - exist; exist = next_item_node(&cac->items, exist, end)) { + for (exist = item_for_next(&cac->items, first, NULL, last); + exist; exist = next_item_node(&cac->items, exist, last)) { scoutfs_kvec_init_null(del_val); become_deletion_item(sb, cac, exist, del_val); @@ -1225,7 +1236,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, ret = 0; out: spin_unlock_irqrestore(&cac->lock, flags); - scoutfs_key_free(sb, missing); + scoutfs_key_free(sb, range_end); return ret; } diff --git a/kmod/src/item.h b/kmod/src/item.h index 924f2fda..ef751a95 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -45,8 +45,9 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, int sif); + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, int sif, + struct scoutfs_key_buf *end); void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 0fac057d..9c91da3c 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -324,7 +324,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, down_write(&si->xattr_rwsem); ret = scoutfs_dirty_inode_item(inode) ?: - scoutfs_item_set_batch(sb, &list, key, last, sif); + scoutfs_item_set_batch(sb, &list, key, last, sif, lck->end); if (ret == 0) { /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); From d5b4677e7f35bcefe03bae21b9989483346c6818 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 10:37:01 -0700 Subject: [PATCH 339/920] scoutfs: add end to _dirty, _delete_many, _update These transformations are mechanical and there aren't many callers of these so we combine them into one commit. Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 +- kmod/src/inode.c | 4 ++-- kmod/src/item.c | 31 ++++++++----------------------- kmod/src/item.h | 8 +++++--- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 55909f8f..eab2afe7 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -600,7 +600,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto out; } - ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys)); + ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys), NULL); if (ret) goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 176bc306..57d5e23d 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -428,7 +428,7 @@ int scoutfs_dirty_inode_item(struct inode *inode) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - ret = scoutfs_item_dirty(sb, &key); + ret = scoutfs_item_dirty(sb, &key, NULL); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -551,7 +551,7 @@ void scoutfs_update_inode_item(struct inode *inode) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, &key, val); + err = scoutfs_item_update(sb, &key, val, NULL); if (err) { scoutfs_err(sb, "inode %llu update err %d", scoutfs_ino(inode), err); diff --git a/kmod/src/item.c b/kmod/src/item.c index a1fa214e..8824c945 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1257,21 +1257,15 @@ void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) * If the item exists make sure it's dirty and pinned. It can be read * if it wasn't cached. -ENOENT is returned if the item doesn't exist. */ -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key) +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *end; struct cached_item *item; unsigned long flags; int ret; - end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!end) { - ret = -ENOMEM; - goto out; - } - do { spin_lock_irqsave(&cac->lock, flags); @@ -1279,7 +1273,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key) if (item) { mark_item_dirty(sb, cac, item); ret = 0; - } else if (check_range(sb, &cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1290,8 +1284,6 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key) } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - scoutfs_key_free(sb, end); -out: trace_printk("ret %d\n", ret); return ret; } @@ -1303,11 +1295,10 @@ out: * Returns -ENOENT if the item doesn't exist. */ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val) + struct kvec *val, struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *end; SCOUTFS_DECLARE_KVEC(up_val); struct cached_item *item; unsigned long flags; @@ -1316,12 +1307,6 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, if (invalid_key_val(key, val)) return -EINVAL; - end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!end) { - ret = -ENOMEM; - goto out; - } - if (val) { ret = scoutfs_kvec_dup_flatten(up_val, val); if (ret) @@ -1339,7 +1324,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, scoutfs_kvec_swap(up_val, item->val); mark_item_dirty(sb, cac, item); ret = 0; - } else if (check_range(sb, &cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1350,7 +1335,6 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); out: - scoutfs_key_free(sb, end); scoutfs_kvec_kfree(up_val); trace_printk("ret %d\n", ret); @@ -1449,13 +1433,14 @@ void scoutfs_item_delete_dirty(struct super_block *sb, * searches if we remembered the items we dirtied. */ int scoutfs_item_delete_many(struct super_block *sb, - struct scoutfs_key_buf **keys, unsigned nr) + struct scoutfs_key_buf **keys, unsigned nr, + struct scoutfs_key_buf *end) { int ret = 0; int i; for (i = 0; i < nr; i++) { - ret = scoutfs_item_dirty(sb, keys[i]); + ret = scoutfs_item_dirty(sb, keys[i], end); if (ret) goto out; } diff --git a/kmod/src/item.h b/kmod/src/item.h index ef751a95..33f13d35 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -30,13 +30,15 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key); +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val); + struct kvec *val, struct scoutfs_key_buf *end); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_delete_many(struct super_block *sb, - struct scoutfs_key_buf **keys, unsigned nr); + struct scoutfs_key_buf **keys, unsigned nr, + struct scoutfs_key_buf *end); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, From 47b26d78887f589dbc1d0618fd5c9d6d19a33c04 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Jul 2017 10:42:10 -0700 Subject: [PATCH 340/920] scoutfs: add end to _item_delete Add the end argument to scoutfs_item_delete() to limit how many items it will read into the cache. Signed-off-by: Zach Brown --- kmod/src/data.c | 6 +++--- kmod/src/dir.c | 4 ++-- kmod/src/inode.c | 8 ++++---- kmod/src/item.c | 15 ++++----------- kmod/src/item.h | 3 ++- kmod/src/xattr.c | 2 +- 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index ca437490..cc6ef98f 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -351,16 +351,16 @@ static int modify_items(struct super_block *sb, struct native_extent *ext, init_extent_key(&key, key_bytes, ext, arg, type); ret = create ? scoutfs_item_create(sb, &key, NULL) : - scoutfs_item_delete(sb, &key); + scoutfs_item_delete(sb, &key, NULL); if (ret == 0 && type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { init_extent_key(&key, key_bytes, ext, arg, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE); ret = create ? scoutfs_item_create(sb, &key, NULL) : - scoutfs_item_delete(sb, &key); + scoutfs_item_delete(sb, &key, NULL); if (ret) { init_extent_key(&key, key_bytes, ext, arg, type); - err = create ? scoutfs_item_delete(sb, &key) : + err = create ? scoutfs_item_delete(sb, &key, NULL) : scoutfs_item_create(sb, &key, NULL); BUG_ON(err); } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index eab2afe7..1266b67b 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -443,7 +443,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, ret = 0; out: while (ret < 0 && --del >= 0) { - err = scoutfs_item_delete(sb, del_keys[del]); + err = scoutfs_item_delete(sb, del_keys[del], NULL); /* can always delete dirty while holding */ BUG_ON(err); } @@ -692,7 +692,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, NULL); else if (op == SYM_DELETE) - ret = scoutfs_item_delete(sb, &key); + ret = scoutfs_item_delete(sb, &key, NULL); if (ret) break; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 57d5e23d..ac877938 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -498,9 +498,9 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, del_ikey.ino = cpu_to_be64(scoutfs_ino(inode)); scoutfs_key_init(&del, &del_ikey, sizeof(del_ikey)); - ret = scoutfs_item_delete(sb, &del); + ret = scoutfs_item_delete(sb, &del, NULL); if (ret) { - err = scoutfs_item_delete(sb, &ins); + err = scoutfs_item_delete(sb, &ins, NULL); BUG_ON(err); } @@ -775,7 +775,7 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) init_orphan_key(&key, &okey, sbi->node_id, ino); - ret = scoutfs_item_delete(sb, &key); + ret = scoutfs_item_delete(sb, &key, NULL); if (ret == -ENOENT) ret = 0; @@ -811,7 +811,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, goto out; #endif - ret = scoutfs_item_delete(sb, key); + ret = scoutfs_item_delete(sb, key, NULL); if (ret) goto out; diff --git a/kmod/src/item.c b/kmod/src/item.c index 8824c945..e91a6dd3 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1353,11 +1353,11 @@ out: * there are any ways for userspace to overwhelm the system with * deletion items for items that didn't exist in the first place. */ -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *end; struct cached_item *item; SCOUTFS_DECLARE_KVEC(del_val); unsigned long flags; @@ -1365,12 +1365,6 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) scoutfs_kvec_init_null(del_val); - end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!end) { - ret = -ENOMEM; - goto out; - } - do { spin_lock_irqsave(&cac->lock, flags); @@ -1378,7 +1372,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) if (item) { become_deletion_item(sb, cac, item, del_val); ret = 0; - } else if (check_range(sb, &cac->ranges, key, end)) { + } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1389,9 +1383,8 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key) } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - scoutfs_key_free(sb, end); scoutfs_kvec_kfree(del_val); -out: + trace_printk("ret %d\n", ret); return ret; } diff --git a/kmod/src/item.h b/kmod/src/item.h index 33f13d35..cb50e805 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -39,7 +39,8 @@ void scoutfs_item_delete_dirty(struct super_block *sb, int scoutfs_item_delete_many(struct super_block *sb, struct scoutfs_key_buf **keys, unsigned nr, struct scoutfs_key_buf *end); -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key); +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 9c91da3c..85d42557 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -483,7 +483,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) break; } - ret = scoutfs_item_delete(sb, key); + ret = scoutfs_item_delete(sb, key, lck->end); if (ret) break; From 13ebd8d18cd52b8c4140f3518ab15c91f59571b5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Jul 2017 10:19:21 -0700 Subject: [PATCH 341/920] scoutfs: don't use delayed downconvert work The delayed downconvert work wasn't being canceled on shutdown. 60s after unmount at least the net lock's timer would fire and crash trying to queue the delayed work on the destroyed workqueue. Proactively unlocking the locks isn't always beneficial to begin with. The relative costs of mispredicting the future are wildly different if we have to re-read item caches from segments or have to downconvert a blocking read lock. So we can just remove the delayed work to fix the bug and remove a moving piece that would need to be considered and tuned. There's still a race where we can get basts after destroying the workqueue but before we destroy the lockspace, we'll get there. Signed-off-by: Zach Brown --- kmod/src/lock.c | 30 +++++++++++++----------------- kmod/src/lock.h | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 4e47c87e..71a64952 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -97,7 +97,8 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) refs = --lock->refcnt; if (!refs) { BUG_ON(lock->holders); - BUG_ON(delayed_work_pending(&lock->dc_work)); + /* can't be (even racy) busy without refs */ + BUG_ON(work_busy(&lock->dc_work)); rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); spin_unlock(&linfo->lock); @@ -128,8 +129,7 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, lock->sb = sb; lock->lock_name = *lock_name; lock->mode = DLM_LOCK_IV; - INIT_DELAYED_WORK(&lock->dc_work, - scoutfs_downconvert_func); + INIT_WORK(&lock->dc_work, scoutfs_downconvert_func); INIT_LIST_HEAD(&lock->lru_entry); } } @@ -279,24 +279,24 @@ static void scoutfs_ast(void *astarg) } static void queue_blocking_work(struct lock_info *linfo, - struct scoutfs_lock *lock, unsigned int seconds) + struct scoutfs_lock *lock) { assert_spin_locked(&linfo->lock); if (!(lock->flags & SCOUTFS_LOCK_QUEUED)) { /* Take a ref for the workqueue */ lock->flags |= SCOUTFS_LOCK_QUEUED; lock->refcnt++; + queue_work(linfo->downconvert_wq, &lock->dc_work); } - mod_delayed_work(linfo->downconvert_wq, &lock->dc_work, seconds * HZ); } -static void set_lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock, - unsigned int seconds) +static void set_lock_blocking(struct lock_info *linfo, + struct scoutfs_lock *lock) { assert_spin_locked(&linfo->lock); lock->flags |= SCOUTFS_LOCK_BLOCKING; if (lock->holders == 0) - queue_blocking_work(linfo, lock, seconds); + queue_blocking_work(linfo, lock); } static void scoutfs_bast(void *astarg, int mode) @@ -307,7 +307,7 @@ static void scoutfs_bast(void *astarg, int mode) trace_scoutfs_bast(lock->sb, lock); spin_lock(&linfo->lock); - set_lock_blocking(linfo, lock, 0); + set_lock_blocking(linfo, lock); spin_unlock(&linfo->lock); } @@ -381,7 +381,7 @@ check_lock_state: * blocking to let the downconvert thread do it's work * so we can reacquire at the correct mode. */ - set_lock_blocking(linfo, lock, 0); + set_lock_blocking(linfo, lock); spin_unlock(&linfo->lock); goto check_lock_state; } @@ -512,17 +512,13 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); - unsigned int seconds = 60; trace_scoutfs_unlock(sb, lock); spin_lock(&linfo->lock); lock->holders--; - if (lock->holders == 0) { - if (lock->flags & SCOUTFS_LOCK_BLOCKING) - seconds = 0; - queue_blocking_work(linfo, lock, seconds); - } + if (lock->holders == 0 && (lock->flags & SCOUTFS_LOCK_BLOCKING)) + queue_blocking_work(linfo, lock); spin_unlock(&linfo->lock); put_scoutfs_lock(sb, lock); @@ -556,7 +552,7 @@ out: static void scoutfs_downconvert_func(struct work_struct *work) { struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - dc_work.work); + dc_work); struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 949efd14..74a5549d 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -21,7 +21,7 @@ struct scoutfs_lock { unsigned int refcnt; unsigned int holders; /* Tracks active users of this lock */ unsigned int flags; - struct delayed_work dc_work; + struct work_struct dc_work; }; int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, From 2d11f08f5e6164eada1c77b2dee69f557bfb724c Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 17 Jul 2017 19:07:29 -0500 Subject: [PATCH 342/920] scoutfs: Remove unused functions, scoutfs_[un]lock_addr Signed-off-by: Mark Fasheh --- kmod/src/lock.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 74a5549d..0b118e04 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -31,11 +31,6 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, struct scoutfs_lock **ret_lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock); -int scoutfs_lock_addr(struct super_block *sb, int wanted_mode, - void *caller_lvb, unsigned lvb_len); -void scoutfs_unlock_addr(struct super_block *sb, void *caller_lvb, - unsigned lvb_len); - int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); From a65b28d440494feb8d469cc95a286f66243f2c37 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 18 Jul 2017 17:40:03 -0500 Subject: [PATCH 343/920] scoutfs: lock impossible ino group for listen lock Otherwise we get into a problem where the listen lock is conflicting with regular inode group requests. Since we never drop the listen lock and it (by design) blocks progress on another node, those inode group requests may hang. Signed-off-by: Mark Fasheh --- kmod/src/net.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index e35cea27..f4a62de5 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -2083,7 +2083,7 @@ static void scoutfs_net_server_func(struct work_struct *work) INIT_WORK(&sinf->listen_work, scoutfs_net_listen_func); INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, 0, &sinf->listen_lck); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ~0ULL, &sinf->listen_lck); if (ret) { kfree(sinf); goto out; From 4ff2148f10e51a5d47de2fe645d5a19700bc481a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 19 Jul 2017 18:41:51 -0500 Subject: [PATCH 344/920] scoutfs: Don't use stale root in get_manifest_refs get_manifest_refs was using the btree root in its stale copy of the super block. It is supposed to use the btree root that it was given by its caller who went to the trouble of finding a sufficiently current btree root. Signed-off-by: Mark Fasheh [zab: added commit message and fixed formatting] Signed-off-by: Zach Brown --- kmod/src/manifest.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 6781f44e..ae7b4217 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -459,7 +459,6 @@ static int get_manifest_refs(struct super_block *sb, struct list_head *ref_list) { DECLARE_MANIFEST(sb, mani); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_manifest_btree_key *mkey; struct scoutfs_manifest_entry ment; SCOUTFS_BTREE_ITEM_REF(iref); @@ -475,8 +474,7 @@ static int get_manifest_refs(struct super_block *sb, /* get level 0 segments that overlap with the missing range */ mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); - ret = scoutfs_btree_prev(sb, &super->manifest.root, - mkey, mkey_len, &iref); + ret = scoutfs_btree_prev(sb, root, mkey, mkey_len, &iref); while (ret == 0) { init_ment_iref(&ment, &iref); @@ -488,8 +486,8 @@ static int get_manifest_refs(struct super_block *sb, } swap(prev, iref); - ret = scoutfs_btree_before(sb, &super->manifest.root, - prev.key, prev.key_len, &iref); + ret = scoutfs_btree_before(sb, root, prev.key, prev.key_len, + &iref); scoutfs_btree_put_iref(&prev); } if (ret != -ENOENT) @@ -507,9 +505,8 @@ static int get_manifest_refs(struct super_block *sb, /* XXX should use level counts to skip searches */ scoutfs_btree_put_iref(&iref); - ret = btree_prev_overlap_or_next(sb, &super->manifest.root, - mkey, mkey_len, key, i, - &iref); + ret = btree_prev_overlap_or_next(sb, root, mkey, mkey_len, key, + i, &iref); if (ret < 0) { if (ret == -ENOENT) ret = 0; From 325eadca9f7dbb9f5d83a4dfbc05eeeacceb02f3 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 19 Jul 2017 19:10:56 -0500 Subject: [PATCH 345/920] scoutfs: check for NULL lock in scoutfs_unlock This reduces the amount of duplicate code in callers and makes error handling easier. The alternative is to sprinkle the code with 'if (lock)' lines at the end of our functions. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 71a64952..0d5455fb 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -513,6 +513,9 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) { DECLARE_LOCK_INFO(sb, linfo); + if (!lock) + return; + trace_scoutfs_unlock(sb, lock); spin_lock(&linfo->lock); From 172cff553755b3f680ba4da93f39db8b483fd9e9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 27 Jul 2017 15:21:13 -0700 Subject: [PATCH 346/920] scoutfs: return -ENODATA from getxattr The conversion to the multi-item xattrs accidentally returned -EIO when an attribute wasn't found instead of -ENODATA. That broke a huge number of xfstests because ls can look up xattrs and return EIO. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 85d42557..d48a9d2e 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -192,7 +192,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = scoutfs_item_lookup(sb, key, val, lck->end); if (ret < 0) { if (ret == -ENOENT) - ret = -EIO; + ret = -ENODATA; break; } From 65c3ac50434e58f29f75af4cb373c365422397d4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 3 Aug 2017 11:13:31 -0700 Subject: [PATCH 347/920] scoutfs: Add cluster locking to node/file ops This gives us cluster locking for the overwhelming majority of metadata ops that scoutfs supports. In particular, we can create and modify file metadata from one node and immediately see the changes reflected on another node. In addition to synchonrization the cluster locks here are providing an I/O endpoint for our item cache, ensuring that it doesn't read stale items. Readdir and file read/write are notable exception - they require a more specific approach and will be implemented in a future patch. Signed-off-by: Mark Fasheh [fixed iget unlock and truncated commit message summary] Signed-off-by: Zach Brown --- kmod/src/data.c | 17 +++-- kmod/src/dir.c | 184 +++++++++++++++++++++++++++++++++++++---------- kmod/src/dir.h | 4 +- kmod/src/inode.c | 91 +++++++++++------------ kmod/src/inode.h | 3 +- kmod/src/item.c | 4 +- kmod/src/item.h | 2 +- kmod/src/super.c | 1 - kmod/src/xattr.c | 2 +- 9 files changed, 215 insertions(+), 93 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index cc6ef98f..2b45a406 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -31,6 +31,7 @@ #include "item.h" #include "ioctl.h" #include "net.h" +#include "lock.h" #define EXTF "[off %llu bno %llu bks %llu fl %x]" #define EXTA(ne) (ne)->blk_off, (ne)->blkno, (ne)->blocks, (ne)->flags @@ -1065,7 +1066,7 @@ static int scoutfs_write_begin(struct file *file, flags |= AOP_FLAG_NOFS; /* generic write_end updates i_size and calls dirty_inode */ - ret = scoutfs_dirty_inode_item(inode); + ret = scoutfs_dirty_inode_item(inode, NULL); if (ret == 0) ret = block_write_begin(mapping, pos, len, flags, pagep, scoutfs_get_block); @@ -1116,17 +1117,18 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct scoutfs_key_buf last; struct scoutfs_key_buf key; struct native_extent ext; + struct scoutfs_lock *inode_lock = NULL; u64 logical; u64 blk_off; u64 final; u64 phys; u64 size; u32 flags; - int ret = 0; + int ret; ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) - goto out; + return ret; memset(&ext, ~0, sizeof(ext)); init_extent_key(&last, last_bytes, &ext, ino, type); @@ -1139,6 +1141,11 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), + &inode_lock); + if (ret) + goto out; + for (;;) { ext.blk_off = blk_off; ext.blkno = 0; @@ -1181,8 +1188,10 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, blk_off = ext.blk_off + ext.blocks; } - mutex_unlock(&inode->i_mutex); + scoutfs_unlock(sb, inode_lock); out: + mutex_unlock(&inode->i_mutex); + return ret; } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 1266b67b..85730f25 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -28,6 +28,7 @@ #include "xattr.h" #include "kvec.h" #include "item.h" +#include "lock.h" /* * Directory entries are stored in entries with offsets calculated from @@ -117,8 +118,16 @@ static void scoutfs_d_release(struct dentry *dentry) } } +static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + return 0;/* Always revalidate for now */ +} + static const struct dentry_operations scoutfs_dentry_ops = { .d_release = scoutfs_d_release, + .d_revalidate = scoutfs_d_revalidate, }; static int alloc_dentry_info(struct dentry *dentry) @@ -229,6 +238,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, struct super_block *sb = dir->i_sb; struct scoutfs_key_buf *key = NULL; struct scoutfs_dirent dent; + struct scoutfs_lock *dir_lock = NULL; SCOUTFS_DECLARE_KVEC(val); struct inode *inode; u64 ino = 0; @@ -249,9 +259,15 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(dir), + &dir_lock); + if (ret) + goto out; + scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), NULL); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), + dir_lock->end); if (ret == -ENOENT) { ino = 0; ret = 0; @@ -259,7 +275,6 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, ino = le64_to_cpu(dent.ino); update_dentry_info(dentry, &dent); } - out: if (ret < 0) inode = ERR_PTR(ret); @@ -268,6 +283,8 @@ out: else inode = scoutfs_iget(sb, ino); + scoutfs_unlock(sb, dir_lock); + scoutfs_key_free(sb, key); return d_splice_alias(inode, dentry); @@ -323,6 +340,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct scoutfs_key_buf last_key; struct scoutfs_readdir_key rkey; struct scoutfs_readdir_key last_rkey; + struct scoutfs_lock *dir_lock; SCOUTFS_DECLARE_KVEC(val); unsigned int item_len; unsigned int name_len; @@ -332,19 +350,27 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (!dir_emit_dots(file, dirent, filldir)) return 0; + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), + &dir_lock); + if (ret) + return ret; + init_readdir_key(&last_key, &last_rkey, inode, SCOUTFS_DIRENT_LAST_POS); item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); dent = kmalloc(item_len, GFP_KERNEL); - if (!dent) - return -ENOMEM; + if (!dent) { + ret = -ENOMEM; + goto out; + } for (;;) { init_readdir_key(&key, &rkey, inode, file->f_pos); scoutfs_kvec_init(val, dent, item_len); ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, - offsetof(struct scoutfs_dirent, name[1]), NULL); + offsetof(struct scoutfs_dirent, name[1]), + dir_lock->end); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -363,12 +389,16 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) file->f_pos = pos + 1; } +out: + scoutfs_unlock(sb, dir_lock); + kfree(dent); return ret; } -static int add_entry_items(struct inode *dir, struct dentry *dentry, - struct inode *inode) +static int add_entry_items(struct inode *dir, struct scoutfs_lock *dir_lock, + struct dentry *dentry, struct inode *inode, + struct scoutfs_lock *inode_lock) { struct scoutfs_inode_info *si = SCOUTFS_I(dir); struct dentry_info *di = dentry->d_fsdata; @@ -376,6 +406,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, struct scoutfs_key_buf *ent_key = NULL; struct scoutfs_key_buf *lb_key = NULL; struct scoutfs_key_buf *del_keys[3]; + struct scoutfs_key_buf *end_keys[3]; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; struct scoutfs_dirent dent; @@ -392,7 +423,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_dirty_inode_item(dir); + ret = scoutfs_dirty_inode_item(dir, dir_lock->end); if (ret) return ret; @@ -413,6 +444,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) goto out; del_keys[del++] = ent_key; + end_keys[del] = dir_lock->end; /* readdir item for .. readdir */ init_readdir_key(&rdir_key, &rkey, dir, pos); @@ -423,6 +455,7 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) goto out; del_keys[del++] = &rdir_key; + end_keys[del] = dir_lock->end; /* link backref item for inode to path resolution */ lb_key = alloc_link_backref_key(sb, scoutfs_ino(inode), @@ -438,12 +471,13 @@ static int add_entry_items(struct inode *dir, struct dentry *dentry, if (ret) goto out; del_keys[del++] = lb_key; + end_keys[del] = inode_lock->end; update_dentry_info(dentry, &dent); ret = 0; out: while (ret < 0 && --del >= 0) { - err = scoutfs_item_delete(sb, del_keys[del], NULL); + err = scoutfs_item_delete(sb, del_keys[del], end_keys[del]); /* can always delete dirty while holding */ BUG_ON(err); } @@ -459,17 +493,24 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, { struct super_block *sb = dir->i_sb; DECLARE_ITEM_COUNT(cnt); - struct inode *inode; + struct inode *inode = NULL; + struct scoutfs_lock *dir_lock; + struct scoutfs_lock *inode_lock = NULL; int ret; ret = alloc_dentry_info(dentry); if (ret) return ret; + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), + &dir_lock); + if (ret) + return ret; + scoutfs_count_mknod(&cnt, dentry->d_name.len); ret = scoutfs_hold_trans(sb, &cnt); if (ret) - return ret; + goto out_unlock; inode = scoutfs_new_inode(sb, dir, mode, rdev); if (IS_ERR(inode)) { @@ -477,7 +518,13 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, goto out; } - ret = add_entry_items(dir, dentry, inode); + /* Now that we have ino from scoutfs_new_inode, allocate a lock */ + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), + &inode_lock); + if (ret) + goto out; + + ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); if (ret) goto out; @@ -496,10 +543,13 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, insert_inode_hash(inode); d_instantiate(dentry, inode); out: + scoutfs_release_trans(sb); +out_unlock: + scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, inode_lock); /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); - scoutfs_release_trans(sb); return ret; } @@ -520,22 +570,34 @@ static int scoutfs_link(struct dentry *old_dentry, { struct inode *inode = old_dentry->d_inode; struct super_block *sb = dir->i_sb; + struct scoutfs_lock *dir_lock; + struct scoutfs_lock *inode_lock = NULL; DECLARE_ITEM_COUNT(cnt); int ret; if (inode->i_nlink >= SCOUTFS_LINK_MAX) return -EMLINK; - ret = alloc_dentry_info(dentry); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), + &dir_lock); if (ret) return ret; + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), + &inode_lock); + if (ret) + goto out_unlock; + + ret = alloc_dentry_info(dentry); + if (ret) + goto out_unlock; + scoutfs_count_link(&cnt, dentry->d_name.len); ret = scoutfs_hold_trans(sb, &cnt); if (ret) - return ret; + goto out_unlock; - ret = add_entry_items(dir, dentry, inode); + ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); if (ret) goto out; @@ -551,6 +613,9 @@ static int scoutfs_link(struct dentry *old_dentry, d_instantiate(dentry, inode); out: scoutfs_release_trans(sb); +out_unlock: + scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, inode_lock); return ret; } @@ -564,21 +629,24 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); struct scoutfs_key_buf *keys[3] = {NULL,}; + struct scoutfs_key_buf *ends[3] = {NULL,}; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; DECLARE_ITEM_COUNT(cnt); + struct scoutfs_lock *dir_lock = NULL; + struct scoutfs_lock *inode_lock = NULL; int ret = 0; if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - scoutfs_count_unlink(&cnt, dentry->d_name.len); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), + &dir_lock); if (ret) return ret; - ret = scoutfs_dirty_inode_item(dir) ?: - scoutfs_dirty_inode_item(inode); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), + &inode_lock); if (ret) goto out; @@ -587,9 +655,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) ret = -ENOMEM; goto out; } + ends[0] = dir_lock->end; init_readdir_key(&rdir_key, &rkey, dir, dentry_info_pos(dentry)); keys[1] = &rdir_key; + ends[1] = dir_lock->end; keys[2] = alloc_link_backref_key(sb, scoutfs_ino(inode), scoutfs_ino(dir), @@ -600,10 +670,20 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto out; } - ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys), NULL); + scoutfs_count_unlink(&cnt, dentry->d_name.len); + ret = scoutfs_hold_trans(sb, &cnt); if (ret) goto out; + ret = scoutfs_dirty_inode_item(dir, dir_lock->end) ?: + scoutfs_dirty_inode_item(inode, inode_lock->end); + if (ret) + goto out_trans; + + ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys), ends); + if (ret) + goto out_trans; + if ((inode->i_nlink == 1) || (S_ISDIR(inode->i_mode) && inode->i_nlink == 2)) { /* @@ -613,7 +693,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) */ ret = scoutfs_orphan_inode(inode); if (ret) - goto out; + goto out_trans; } dir->i_ctime = ts; @@ -629,10 +709,13 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(inode); scoutfs_update_inode_item(dir); +out_trans: + scoutfs_release_trans(sb); out: scoutfs_key_free(sb, keys[0]); scoutfs_key_free(sb, keys[2]); - scoutfs_release_trans(sb); + scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, inode_lock); return ret; } @@ -665,7 +748,8 @@ enum { SYM_DELETE, }; static int symlink_item_ops(struct super_block *sb, int op, u64 ino, - const char *target, size_t size) + struct scoutfs_lock *lock, const char *target, + size_t size) { struct scoutfs_symlink_key skey; struct scoutfs_key_buf key; @@ -690,9 +774,9 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, ret = scoutfs_item_create(sb, &key, val); else if (op == SYM_LOOKUP) ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, - NULL); + lock->end); else if (op == SYM_DELETE) - ret = scoutfs_item_delete(sb, &key, NULL); + ret = scoutfs_item_delete(sb, &key, lock->end); if (ret) break; @@ -715,6 +799,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; loff_t size = i_size_read(inode); char *path; int ret; @@ -727,11 +812,19 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (size > PATH_MAX) return ERR_PTR(-ENAMETOOLONG); - path = kmalloc(size, GFP_NOFS); - if (!path) - return ERR_PTR(-ENOMEM); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), + &inode_lock); + if (ret) + return ERR_PTR(ret); - ret = symlink_item_ops(sb, SYM_LOOKUP, scoutfs_ino(inode), path, size); + path = kmalloc(size, GFP_NOFS); + if (!path) { + path = ERR_PTR(-ENOMEM); + goto out; + } + + ret = symlink_item_ops(sb, SYM_LOOKUP, scoutfs_ino(inode), inode_lock, + path, size); /* XXX corruption: missing items or not null term */ if (ret == -ENOENT || (ret == 0 && path[size - 1])) @@ -743,7 +836,8 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) } else { nd_set_link(nd, path); } - +out: + scoutfs_unlock(sb, inode_lock); return path; } @@ -774,6 +868,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct super_block *sb = dir->i_sb; const int name_len = strlen(symname) + 1; struct inode *inode = NULL; + struct scoutfs_lock *dir_lock; + struct scoutfs_lock *inode_lock = NULL; DECLARE_ITEM_COUNT(cnt); int ret; @@ -785,10 +881,15 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), + &dir_lock); + if (ret) + return ret; + scoutfs_count_symlink(&cnt, dentry->d_name.len, name_len); ret = scoutfs_hold_trans(sb, &cnt); if (ret) - return ret; + goto out_unlock; inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0); if (IS_ERR(inode)) { @@ -796,12 +897,17 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, goto out; } - ret = symlink_item_ops(sb, SYM_CREATE, scoutfs_ino(inode), + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), + &inode_lock); + if (ret) + goto out; + + ret = symlink_item_ops(sb, SYM_CREATE, scoutfs_ino(inode), inode_lock, symname, name_len); if (ret) goto out; - ret = add_entry_items(dir, dentry, inode); + ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); if (ret) goto out; @@ -822,19 +928,23 @@ out: if (!IS_ERR_OR_NULL(inode)) iput(inode); - symlink_item_ops(sb, SYM_DELETE, scoutfs_ino(inode), + symlink_item_ops(sb, SYM_DELETE, scoutfs_ino(inode), inode_lock, NULL, name_len); } scoutfs_release_trans(sb); +out_unlock: + scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, inode_lock); return ret; } -int scoutfs_symlink_drop(struct super_block *sb, u64 ino, u64 i_size) +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock, u64 i_size) { int ret; - ret = symlink_item_ops(sb, SYM_DELETE, ino, NULL, i_size); + ret = symlink_item_ops(sb, SYM_DELETE, ino, lock, NULL, i_size); if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 81b5de4e..1a17fd70 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -2,6 +2,7 @@ #define _SCOUTFS_DIR_H_ #include "format.h" +#include "lock.h" extern const struct file_operations scoutfs_dir_fops; extern const struct inode_operations scoutfs_dir_iops; @@ -19,7 +20,8 @@ int scoutfs_dir_get_backref_path(struct super_block *sb, u64 target_ino, void scoutfs_dir_free_backref_path(struct super_block *sb, struct list_head *list); -int scoutfs_symlink_drop(struct super_block *sb, u64 ino, u64 i_size); +int scoutfs_symlink_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock, u64 i_size); int scoutfs_dir_init(void); void scoutfs_dir_exit(void); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index ac877938..570d9cbf 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -58,6 +58,8 @@ struct inode_sb_info { struct inode_sb_info *name = SCOUTFS_SB(sb)->inode_sb_info static struct kmem_cache *scoutfs_inode_cachep; +static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat); /* * This is called once before all the allocations and frees of a inode @@ -141,6 +143,7 @@ void scoutfs_destroy_inode(struct inode *inode) } static const struct inode_operations scoutfs_file_iops = { + .getattr = scoutfs_getattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -221,6 +224,25 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) set_item_info(inode); } +static int refresh_inode(struct inode *inode, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key_buf key; + struct scoutfs_inode_key ikey; + struct scoutfs_inode sinode; + SCOUTFS_DECLARE_KVEC(val); + int ret; + + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), lock->end); + if (ret == 0) + load_inode(inode, &sinode); + + return ret; +} + void scoutfs_inode_init_key(struct scoutfs_key_buf *key, struct scoutfs_inode_key *ikey, u64 ino) { @@ -231,22 +253,24 @@ void scoutfs_inode_init_key(struct scoutfs_key_buf *key, scoutfs_key_init(key, ikey, sizeof(struct scoutfs_inode_key)); } -static int scoutfs_read_locked_inode(struct inode *inode) +static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat) { + struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf key; - struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(val); + struct scoutfs_lock *lock = NULL; int ret; - scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), + &lock); + if (ret) + return ret; - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); + ret = refresh_inode(inode, lock); if (ret == 0) - load_inode(inode, &sinode); + generic_fillattr(inode, stat); + scoutfs_unlock(sb, lock); return ret; } @@ -352,15 +376,22 @@ static int scoutfs_iget_set(struct inode *inode, void *arg) struct inode *scoutfs_iget(struct super_block *sb, u64 ino) { struct inode *inode; + struct scoutfs_lock *lock = NULL; int ret; + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, ino, &lock); + if (ret) + return ERR_PTR(ret); + inode = iget5_locked(sb, ino, scoutfs_iget_test, scoutfs_iget_set, &ino); - if (!inode) - return ERR_PTR(-ENOMEM); + if (!inode) { + inode = ERR_PTR(-ENOMEM); + goto out; + } if (inode->i_state & I_NEW) { - ret = scoutfs_read_locked_inode(inode); + ret = refresh_inode(inode, lock); if (ret) { iget_failed(inode); inode = ERR_PTR(ret); @@ -370,6 +401,8 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) } } +out: + scoutfs_unlock(sb, lock); return inode; } @@ -416,7 +449,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) * * XXX this will have to do something about variable length inodes */ -int scoutfs_dirty_inode_item(struct inode *inode) +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end) { struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; @@ -428,7 +461,7 @@ int scoutfs_dirty_inode_item(struct inode *inode) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - ret = scoutfs_item_dirty(sb, &key, NULL); + ret = scoutfs_item_dirty(sb, &key, end); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -562,36 +595,6 @@ void scoutfs_update_inode_item(struct inode *inode) trace_scoutfs_update_inode(inode); } -/* - * sop->dirty_inode() can't return failure. Our use of it has to be - * careful to pin the inode during a transaction. The generic write - * paths pin the inode in write_begin and get called to update the inode - * in write_end. - * - * The caller should have a trans but it's cheap for us to grab it - * ourselves to make sure. - * - * This will holler at us if a caller didn't pin the inode and we - * couldn't dirty the inode ourselves. - */ -void scoutfs_dirty_inode(struct inode *inode, int flags) -{ - struct super_block *sb = inode->i_sb; - DECLARE_ITEM_COUNT(cnt); - int ret; - - scoutfs_count_dirty_inode(&cnt); - ret = scoutfs_hold_trans(sb, &cnt); - if (ret == 0) { - ret = scoutfs_dirty_inode_item(inode); - if (ret == 0) - scoutfs_update_inode_item(inode); - scoutfs_release_trans(sb); - } - - WARN_ON_ONCE(ret); -} - /* * A quick atomic sample of the last inode number that's been allocated. */ diff --git a/kmod/src/inode.h b/kmod/src/inode.h index d95139c0..73ae1ce7 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -46,8 +46,7 @@ void scoutfs_evict_inode(struct inode *inode); int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); -int scoutfs_dirty_inode_item(struct inode *inode); -void scoutfs_dirty_inode(struct inode *inode, int flags); +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end); void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, diff --git a/kmod/src/item.c b/kmod/src/item.c index e91a6dd3..94c45c9b 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1427,13 +1427,13 @@ void scoutfs_item_delete_dirty(struct super_block *sb, */ int scoutfs_item_delete_many(struct super_block *sb, struct scoutfs_key_buf **keys, unsigned nr, - struct scoutfs_key_buf *end) + struct scoutfs_key_buf **ends) { int ret = 0; int i; for (i = 0; i < nr; i++) { - ret = scoutfs_item_dirty(sb, keys[i], end); + ret = scoutfs_item_dirty(sb, keys[i], ends[i]); if (ret) goto out; } diff --git a/kmod/src/item.h b/kmod/src/item.h index cb50e805..e7bae7da 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -38,7 +38,7 @@ void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); int scoutfs_item_delete_many(struct super_block *sb, struct scoutfs_key_buf **keys, unsigned nr, - struct scoutfs_key_buf *end); + struct scoutfs_key_buf **ends); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); diff --git a/kmod/src/super.c b/kmod/src/super.c index d7321786..8a52c35d 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -79,7 +79,6 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, - .dirty_inode = scoutfs_dirty_inode, .drop_inode = scoutfs_drop_inode, .evict_inode = scoutfs_evict_inode, .destroy_inode = scoutfs_destroy_inode, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index d48a9d2e..cd860bab 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -323,7 +323,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, down_write(&si->xattr_rwsem); - ret = scoutfs_dirty_inode_item(inode) ?: + ret = scoutfs_dirty_inode_item(inode, lck->end) ?: scoutfs_item_set_batch(sb, &list, key, last, sif, lck->end); if (ret == 0) { /* XXX do these want i_mutex or anything? */ From 6d16034112535cda9c2aee4c5db479b5599f8475 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 2 Aug 2017 14:04:05 -0700 Subject: [PATCH 348/920] scoutfs: remove old dlm make -I We don't need arguments for a dlm build. Signed-off-by: Zach Brown --- kmod/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/Makefile b/kmod/Makefile index 11bc4d4a..45ea3442 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -12,7 +12,7 @@ else SP = @: endif -SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) -I $(CURDIR)/dlm/include M=$(CURDIR)/src +SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src all: module From cefe06af61b7a6387df2d12be0dc2e4c21239c9f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 2 Aug 2017 14:12:12 -0700 Subject: [PATCH 349/920] scoutfs: add git describe to built module It's handy to quickly find the git commit that built a given module. We add a MOD_INFO() tag for it so we can see it in modinfo on the built module. We add a ELF note that the kernel tracks in /sys/modules/$m/notes/ when the module is loaded. Signed-off-by: Zach Brown --- kmod/Makefile | 7 ++++++- kmod/src/Makefile | 2 ++ kmod/src/super.c | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/kmod/Makefile b/kmod/Makefile index 45ea3442..bb162626 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -12,7 +12,12 @@ else SP = @: endif -SCOUTFS_ARGS := CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src +SCOUTFS_GIT_DESCRIBE := \ + $(shell git describe --all --abbrev=6 --long 2>/dev/null || \ + echo not-in-a-git-repository) + +SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ + CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src all: module diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 6f0d66d8..5b824081 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,5 +1,7 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o +CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" + CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o compact.o counters.o data.o dir.o kvec.o \ diff --git a/kmod/src/super.c b/kmod/src/super.c index 8a52c35d..3fd6de5e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -324,6 +324,15 @@ static int __init scoutfs_module_init(void) { int ret; + /* + * gcc only recently learned to let __attribute__(section) add + * SHT_NOTE notes. But the assembler always could. + */ + __asm__ __volatile__ ( + ".section .note.git_describe,\"a\"\n" + ".string \""SCOUTFS_GIT_DESCRIBE"\\n\"\n" + ".previous\n"); + scoutfs_init_counters(); scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); @@ -349,3 +358,4 @@ module_exit(scoutfs_module_exit) MODULE_AUTHOR("Zach Brown "); MODULE_LICENSE("GPL"); +MODULE_INFO(git_describe, SCOUTFS_GIT_DESCRIBE); From 9f4095bffb62ac22e34bc3cc0f4b4ba09566b9c2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 2 Aug 2017 13:19:48 -0700 Subject: [PATCH 350/920] scoutfs: break the build if we export raw types Raw [su]{8,16,32,64} types keep leaking into our exported headers where they break userspace builds. Make sure that we only use the exported __ types and add a check to break our build if we get it wrong. Signed-off-by: Zach Brown --- kmod/src/Makefile | 15 +++++++++++++++ kmod/src/format.h | 4 ++-- kmod/src/ioctl.h | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 5b824081..889f80d7 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -7,3 +7,18 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o compact.o counters.o data.o dir.o kvec.o \ inode.o ioctl.o item.o key.o lock.o manifest.o msg.o net.o \ options.o seg.o scoutfs_trace.o sort_priv.o super.o trans.o xattr.o + +# +# The raw types aren't available in userspace headers. Make sure all +# the types we use in the headers are the exported __ versions. +# +# XXX dunno how we're really supposed to do this in kbuild +# +.PHONY: $(src)/check_exported_types +$(src)/check_exported_types: + @if egrep '\<[us](8|16|32|64\>)' $(src)/format.h $(src)/ioctl.h; then \ + echo "no raw types in exported headers, preface with __"; \ + exit 1; \ + fi + +extra-y += check_exported_types diff --git a/kmod/src/format.h b/kmod/src/format.h index 6251c8f8..6c37816d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -212,8 +212,8 @@ struct scoutfs_segment_item { __u8 nr_links; __le32 skip_links[0]; /* - * u8 key_bytes[key_len] - * u8 val_bytes[val_len] + * __u8 key_bytes[key_len] + * __u8 val_bytes[val_len] */ } __packed; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 79241e9f..0dcb6bc7 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -116,7 +116,7 @@ struct scoutfs_ioctl_ino_path { } __packed; #define SCOUTFS_IOC_INO_PATH_CURSOR_BYTES \ - (sizeof(u64) + SCOUTFS_NAME_LEN + 1) + (sizeof(__u64) + SCOUTFS_NAME_LEN + 1) /* Get a single path from the root to the given inode number */ #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ From b98f97e143bae9ce19ee2a916178f97e0e5f9d90 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 20 Jul 2017 09:55:02 -0700 Subject: [PATCH 351/920] scoutfs: use hlist hash for data cursors The rhashtable API has changed over time. Continuing to use it means having to worry about maintaining different APIs in different kernel generations. We have a static pool of cursors so we don't need the flexibility of the resizable rhashtable. We can roll a simple array of hlist heads to use as a hash table. And finally, these cursors will probably disappear eventually anyway. Let's not invest too much in them. Signed-off-by: Zach Brown --- kmod/src/data.c | 123 ++++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 2b45a406..34ea7c45 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -80,18 +79,21 @@ * - direct IO */ +/* more than enough for a few tasks per core on moderate hardware */ +#define NR_CURSORS 4096 +#define CURSOR_HASH_HEADS (PAGE_SIZE / sizeof(void *) / 2) +#define CURSOR_HASH_BITS ilog2(CURSOR_HASH_HEADS) + struct data_info { struct rw_semaphore alloc_rwsem; u64 next_large_blkno; - struct rhashtable cursors; struct list_head cursor_lru; + struct hlist_head cursor_hash[CURSOR_HASH_HEADS]; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -/* more than enough for a few tasks per core on moderate hardware */ -#define NR_CURSORS 4096 /* * This is the size of extents that are tracked by a cursor and so end @@ -103,17 +105,13 @@ struct data_info { */ #define LARGE_EXTENT_BLOCKS SCOUTFS_SEGMENT_BLOCKS -struct cursor_id { - struct task_struct *task; - pid_t pid; -} __packed; /* rhashtable_lookup() always memcmp()s, avoid padding */ - struct task_cursor { u64 blkno; u64 blocks; - struct rhash_head hash_head; + struct hlist_node hnode; struct list_head list_head; - struct cursor_id id; + struct task_struct *task; + pid_t pid; }; /* @@ -653,6 +651,44 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, return ret; } +static inline struct hlist_head *cursor_head(struct data_info *datinf, + struct task_struct *task, + pid_t pid) +{ + unsigned h = hash_ptr(task, CURSOR_HASH_BITS) ^ + hash_long(pid, CURSOR_HASH_BITS); + + return &datinf->cursor_hash[h]; +} + +static struct task_cursor *search_head(struct hlist_head *head, + struct task_struct *task, pid_t pid) +{ + struct task_cursor *curs; + + hlist_for_each_entry(curs, head, hnode) { + if (curs->task == task && curs->pid == pid) + return curs; + } + + return NULL; +} + +static void destroy_cursors(struct data_info *datinf) +{ + struct task_cursor *curs; + struct hlist_node *tmp; + int i; + + for (i = 0; i < CURSOR_HASH_HEADS; i++) { + hlist_for_each_entry_safe(curs, tmp, &datinf->cursor_hash[i], + hnode) { + hlist_del_init(&curs->hnode); + kfree(curs); + } + } +} + /* * These cheesy cursors are only meant to encourage nice IO patterns for * concurrent tasks either streaming large file writes or creating lots @@ -662,21 +698,22 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, */ static struct task_cursor *get_cursor(struct data_info *datinf) { + struct task_struct *task = current; + pid_t pid = current->pid; + struct hlist_head *head; struct task_cursor *curs; - struct cursor_id id = { - .task = current, - .pid = current->pid, - }; - curs = rhashtable_lookup(&datinf->cursors, &id); + head = cursor_head(datinf, task, pid); + curs = search_head(head, task, pid); if (!curs) { curs = list_last_entry(&datinf->cursor_lru, struct task_cursor, list_head); trace_printk("resetting curs %p was task %p pid %u\n", - curs, curs->id.task, curs->id.pid); - rhashtable_remove(&datinf->cursors, &curs->hash_head, GFP_NOFS); - curs->id = id; - rhashtable_insert(&datinf->cursors, &curs->hash_head, GFP_NOFS); + curs, task, pid); + hlist_del_init(&curs->hnode); + curs->task = task; + curs->pid = pid; + hlist_add_head(&curs->hnode, head); curs->blkno = 0; curs->blocks = 0; } @@ -791,7 +828,7 @@ reset_cursor: retry: trace_printk("searching %llu,%llu curs %p task %p pid %u %llu,%llu\n", - ext.blkno, ext.blocks, curs, curs->id.task, curs->id.pid, + ext.blkno, ext.blocks, curs, curs->task, curs->pid, curs->blkno, curs->blocks); ext.blk_off = ext.blkno; @@ -1213,40 +1250,13 @@ const struct file_operations scoutfs_file_fops = { .fsync = scoutfs_file_fsync, }; -static int derpy_global_mutex_is_held(void) -{ - return 1; -} - -static struct rhashtable_params cursor_hash_params = { - .key_len = member_sizeof(struct task_cursor, id), - .key_offset = offsetof(struct task_cursor, id), - .head_offset = offsetof(struct task_cursor, hash_head), - .hashfn = arch_fast_hash, - .grow_decision = rht_grow_above_75, - .shrink_decision = rht_shrink_below_30, - - .mutex_is_held = derpy_global_mutex_is_held, -}; - -static void destroy_cursors(struct data_info *datinf) -{ - struct task_cursor *curs; - struct task_cursor *pos; - - list_for_each_entry_safe(curs, pos, &datinf->cursor_lru, list_head) { - list_del_init(&curs->list_head); - kfree(curs); - } - rhashtable_destroy(&datinf->cursors); -} int scoutfs_data_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct hlist_head *head; struct data_info *datinf; struct task_cursor *curs; - int ret; int i; datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); @@ -1258,11 +1268,8 @@ int scoutfs_data_setup(struct super_block *sb) /* always search for large aligned extents */ datinf->next_large_blkno = LARGE_EXTENT_BLOCKS; - ret = rhashtable_init(&datinf->cursors, &cursor_hash_params); - if (ret) { - kfree(datinf); - return -ENOMEM; - } + for (i = 0; i < CURSOR_HASH_HEADS; i++) + INIT_HLIST_HEAD(&datinf->cursor_hash[i]); /* just allocate all of these up front */ for (i = 0; i < NR_CURSORS; i++) { @@ -1273,9 +1280,11 @@ int scoutfs_data_setup(struct super_block *sb) return -ENOMEM; } - curs->id.pid = i; - rhashtable_insert(&datinf->cursors, &curs->hash_head, - GFP_KERNEL); + curs->pid = i; + + head = cursor_head(datinf, curs->task, curs->pid); + hlist_add_head(&curs->hnode, head); + list_add(&curs->list_head, &datinf->cursor_lru); } From 74a80b772efc8f2533a1e236b62c253d2df39d83 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 Jul 2017 13:56:20 -0700 Subject: [PATCH 352/920] scoutfs: add endian_swap.h Add a helper header for conversions between little and big endian. Signed-off-by: Zach Brown --- kmod/src/endian_swap.h | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 kmod/src/endian_swap.h diff --git a/kmod/src/endian_swap.h b/kmod/src/endian_swap.h new file mode 100644 index 00000000..5693584b --- /dev/null +++ b/kmod/src/endian_swap.h @@ -0,0 +1,10 @@ +#ifndef _SCOUTFS_ENDIAN_SWAP_H_ +#define _SCOUTFS_ENDIAN_SWAP_H_ + +#define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) +#define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) + +#define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) +#define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) + +#endif From c1b2ad942129d6ac5cd5a04e7963b103b629d5fc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 Jul 2017 13:59:40 -0700 Subject: [PATCH 353/920] scoutfs: separate client and server net processing The networking code was really suffering by trying to combine the client and server processing paths into one file. The code can be a lot simpler by giving the client and server their own processing paths that take their different socket lifecysles into account. The client maintains a single connection. Blocked senders work on the socket under a sending mutex. The recv path runs in work that can be canceled after first shutting down the socket. A long running server work function acquires the listener lock, manages the listening socket, and accepts new sockets. Each accepted socket has a single recv work blocked waiting for requests. That then spawns concurrent processing work which sends replies under a sending mutex. All of this is torn down by shutting down sockets and canceling work which frees its context. All this restructuring makes it a lot easier to track what is happening in mount and unmount between the client and server. This fixes bugs where unmount was failing because the monolithic socket shutdown function was queueing other work while running while draining. Signed-off-by: Zach Brown --- kmod/src/Makefile | 7 +- kmod/src/client.c | 729 +++++++++++++ kmod/src/client.h | 17 + kmod/src/compact.c | 6 +- kmod/src/data.c | 4 +- kmod/src/inode.c | 4 +- kmod/src/ioctl.c | 4 +- kmod/src/manifest.c | 4 +- kmod/src/net.c | 2210 -------------------------------------- kmod/src/net.h | 25 - kmod/src/scoutfs_trace.h | 62 ++ kmod/src/server.c | 1051 ++++++++++++++++++ kmod/src/server.h | 20 + kmod/src/sock.c | 96 ++ kmod/src/sock.h | 7 + kmod/src/super.c | 50 +- kmod/src/super.h | 6 +- kmod/src/trans.c | 10 +- kmod/src/trans.h | 1 - 19 files changed, 2043 insertions(+), 2270 deletions(-) create mode 100644 kmod/src/client.c create mode 100644 kmod/src/client.h delete mode 100644 kmod/src/net.c delete mode 100644 kmod/src/net.h create mode 100644 kmod/src/server.c create mode 100644 kmod/src/server.h create mode 100644 kmod/src/sock.c create mode 100644 kmod/src/sock.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 889f80d7..2200b81d 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -4,9 +4,10 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += alloc.o bio.o btree.o compact.o counters.o data.o dir.o kvec.o \ - inode.o ioctl.o item.o key.o lock.o manifest.o msg.o net.o \ - options.o seg.o scoutfs_trace.o sort_priv.o super.o trans.o xattr.o +scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ + kvec.o inode.o ioctl.o item.o key.o lock.o manifest.o msg.o \ + options.o seg.o server.o scoutfs_trace.o sock.o sort_priv.o \ + super.o trans.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/client.c b/kmod/src/client.c new file mode 100644 index 00000000..49293be1 --- /dev/null +++ b/kmod/src/client.c @@ -0,0 +1,729 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "btree.h" +#include "manifest.h" +#include "seg.h" +#include "compact.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "server.h" +#include "client.h" +#include "sock.h" +#include "endian_swap.h" + +/* + * Client callers block sending requests to the server. Senders connect + * and send down the socket in their blocked context under a mutex. + * Once a socket is connected recv work is fired up. Destroying a + * socket shuts down the socket and cancels the work. + * + * Clients are responsible for resending their requests after + * reconnecting to a new socket. These new socket connections might be + * connecting to the same server. The message sending and processing + * paths are responsible for dealing with duplicate requests. + */ + +#define SIN_FMT "%pIS:%u" +#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) + +/* + * Have a pretty aggressive keepalive timeout of around 10 seconds. The + * TCP keepalives are being processed out of task context so they should + * be responsive even when mounts are under load. We also derive the + * connect timeout from this. + */ +#define KEEPCNT 3 +#define KEEPIDLE 7 +#define KEEPINTVL 1 +#define KEEP_TIMEO_SECS (KEEPIDLE + (KEEPCNT * KEEPINTVL)) +#define CONNECT_TIMEO_SECS KEEP_TIMEO_SECS +#define CONNECT_TIMEO_MSECS (KEEP_TIMEO_SECS * MSEC_PER_SEC) + +struct client_info { + struct super_block *sb; + + /* spinlock protects quick critical sections between send,recv,umount */ + spinlock_t recv_lock; + struct rb_root sender_root; + + /* the sock mutex serializes connecting and sending */ + struct mutex send_mutex; + bool recv_shutdown; + u64 next_id; + u64 sock_gen; + struct socket *sock; + struct sockaddr_in peername; + struct sockaddr_in sockname; + + /* blocked senders sit on a waitq that's woken for resends */ + wait_queue_head_t waitq; + + struct workqueue_struct *recv_wq; + struct work_struct recv_work; +}; + +struct waiting_sender { + struct rb_node node; + struct task_struct *task; + + u64 id; + void *rx; + size_t rx_size; + int result; +}; + +static struct waiting_sender *walk_sender_tree(struct client_info *client, + u64 id, + struct waiting_sender *ins) +{ + struct rb_node **node = &client->sender_root.rb_node; + struct waiting_sender *found = NULL; + struct waiting_sender *sender; + struct rb_node *parent = NULL; + + assert_spin_locked(&client->recv_lock); + + while (*node) { + parent = *node; + sender = container_of(*node, struct waiting_sender, node); + + if (id < sender->id) { + node = &(*node)->rb_left; + } else if (id > sender->id) { + node = &(*node)->rb_right; + } else { + found = sender; + break; + } + } + + if (ins) { + /* ids are never reused and assigned under lock */ + BUG_ON(found); + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &client->sender_root); + found = ins; + } + + return found; +} + +/* + * This work is queued once the socket is created. It blocks trying to + * receive replies to sent messages. If the sender is still around it + * receives the reply data into their buffer. If the sender has left + * then it silently drops the reply. + * + * This exits once someone shuts down the socket. If this sees a fatal + * error it shuts down the socket which causes senders to reconnect. + */ +static void scoutfs_client_recv_func(struct work_struct *work) +{ + struct client_info *client = container_of(work, struct client_info, + recv_work); + struct waiting_sender *sender; + struct scoutfs_net_header nh; + void *rx_alloc = NULL; + int result = 0; + u16 data_len; + void *rx; + int ret; + + for (;;) { + /* receive the header */ + ret = scoutfs_sock_recvmsg(client->sock, &nh, sizeof(nh)); + if (ret) + break; + + data_len = le16_to_cpu(nh.data_len); + + trace_scoutfs_client_recv_reply(client->sb, + &client->sockname, + &client->peername, &nh); + + /* see if we have a waiting sender */ + spin_lock(&client->recv_lock); + sender = walk_sender_tree(client, le64_to_cpu(nh.id), NULL); + spin_unlock(&client->recv_lock); + + if (sender) { + if (sender->rx_size < data_len) { + /* protocol mismatch is fatal */ + rx = NULL; + result = -EIO; + } else { + rx = sender->rx; + result = 0; + } + } else { + rx = NULL; + } + + if (!rx) { + kfree(rx_alloc); + rx_alloc = kmalloc(data_len, GFP_NOFS); + if (!rx_alloc) { + ret = -ENOMEM; + break; + } + rx = rx_alloc; + } + + /* recv failure can be server crashing, not fatal */ + ret = scoutfs_sock_recvmsg(client->sock, rx, data_len); + if (ret) { + break; + } + + if (sender) { + /* lock to keep sender around until after we wake */ + spin_lock(&client->recv_lock); + sender->result = result; + smp_mb(); /* store result before waking */ + wake_up_process(sender->task); + spin_unlock(&client->recv_lock); + } + } + + /* make senders reconnect if we see an rx error */ + if (ret) { + /* XXX would need to break out send */ + kernel_sock_shutdown(client->sock, SHUT_RDWR); + client->recv_shutdown = true; + } + + kfree(rx_alloc); +} + + +/* + * Spin discovering the address of the server and trying to connect to + * it until either we connect or we're interrupted by a signal. + * + * A single mount coming up starts both the server and the client. The + * server takes a few IOs and network messages to get going and communicate + * its address. We want to aggressively retry getting the address so that + * these mounts can be quick. But we back off to avoid storms waiting for + * recovery after an existing server explodes. + */ +static int client_connect(struct client_info *client) +{ + struct super_block *sb = client->sb; + struct scoutfs_super_block super; + struct sockaddr_in *sin; + struct socket *sock = NULL; + struct timeval tv; + unsigned int msecs = MSEC_PER_SEC / 10; + int addrlen; + int optval; + int ret; + + BUG_ON(!mutex_is_locked(&client->send_mutex)); + + for(;;) { + if (sock) { + sock_release(sock); + sock = NULL; + } + + ret = scoutfs_read_supers(sb, &super); + if (ret) + continue; + + if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) { + msleep_interruptible(msecs); + if (msecs < CONNECT_TIMEO_MSECS) + msecs = max(msecs + MSEC_PER_SEC, + CONNECT_TIMEO_MSECS); + continue; + } + + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + sin = &client->peername; + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = le32_to_be32(super.server_addr.addr); + sin->sin_port = le16_to_be16(super.server_addr.port); + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, + &sock); + if (ret) + continue; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, + (char *)&optval, sizeof(optval)); + if (ret) + continue; + + /* start with a connect timeout */ + tv.tv_sec = CONNECT_TIMEO_SECS; + tv.tv_usec = 0; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) + continue; + + client->sock = sock; + + ret = kernel_connect(sock, (struct sockaddr *)sin, + sizeof(struct sockaddr_in), 0); + if (ret) + continue; + + /* but use a keepalive timeout instead of send timeout */ + tv.tv_sec = 0; + tv.tv_usec = 0; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) + continue; + + optval = KEEPCNT; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, + (char *)&optval, sizeof(optval)); + if (ret) + continue; + + optval = KEEPIDLE; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, + (char *)&optval, sizeof(optval)); + if (ret) + continue; + + optval = KEEPINTVL; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, + (char *)&optval, sizeof(optval)); + if (ret) + continue; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, + (char *)&optval, sizeof(optval)); + if (ret) + continue; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(sock, + (struct sockaddr *)&client->sockname, + &addrlen); + if (ret) + continue; + + scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&client->sockname), + SIN_ARG(&client->peername)); + + client->sock_gen++; + client->recv_shutdown = false; + queue_work(client->recv_wq, &client->recv_work); + wake_up(&client->waitq); + ret = 0; + break; + } + + if (ret && sock) + sock_release(sock); + + return ret; +} + +/* either a sender or unmount is destroying the socket */ +static void shutdown_sock_sync(struct client_info *client) +{ + struct super_block *sb = client->sb; + struct socket *sock = client->sock; + + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + cancel_work_sync(&client->recv_work); + sock_release(sock); + client->sock = NULL; + + scoutfs_info(sb, "client disconnected "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&client->sockname), + SIN_ARG(&client->peername)); + } +} + +/* + * Senders sleep waiting for a reply to come down the connection out + * which they just sent a request. They need to wake up when the recv + * work has given them a reply or when it's given up and the sender + * needs to reconnect and resend. + * + * This is a condition for wait_event. The barrier orders the task + * state store before loading the sender and client fields. + */ +static int sender_should_wake(struct client_info *client, + struct waiting_sender *sender) +{ + smp_mb(); + return sender->result != -EINPROGRESS || client->recv_shutdown; +} + +/* + * Block sending a request and then waiting for the reply. All senders + * are responsible for connecting sockets and sending their requests. + * recv work blocks receiving from the socket and waking senders if + * they're reply has been copied to their buffer. If the socket sees an + * error the recv work will shutdown and wake us to reconnect. + */ +static int client_request(struct client_info *client, int type, void *data, + unsigned data_len, void *rx, size_t rx_size) +{ + struct waiting_sender sender; + struct scoutfs_net_header nh; + struct kvec kv[2]; + unsigned kv_len; + u64 sent_to_gen = ~0ULL; + int ret = 0; + + if (WARN_ON_ONCE(!data && data_len)) + return -EINVAL; + + spin_lock(&client->recv_lock); + + sender.task = current; + sender.id = client->next_id++; + sender.rx = rx; + sender.rx_size = rx_size; + sender.result = -EINPROGRESS; + + nh.id = cpu_to_le64(sender.id); + nh.data_len = cpu_to_le16(data_len); + nh.type = type; + nh.status = SCOUTFS_NET_STATUS_REQUEST; + + walk_sender_tree(client, sender.id, &sender); + + spin_unlock(&client->recv_lock); + + mutex_lock(&client->send_mutex); + + while (sender.result == -EINPROGRESS) { + + if (!client->sock) { + ret = client_connect(client); + if (ret < 0) + break; + } + + if (sent_to_gen != client->sock_gen) { + kv[0].iov_base = &nh; + kv[0].iov_len = sizeof(nh); + kv[1].iov_base = data; + kv[1].iov_len = data_len; + kv_len = data ? 2 : 1; + + trace_scoutfs_client_send_request(client->sb, + &client->sockname, + &client->peername, + &nh); + + ret = scoutfs_sock_sendmsg(client->sock, kv, kv_len); + if (ret) { + shutdown_sock_sync(client); + continue; + } + + sent_to_gen = client->sock_gen; + } + + /* XXX would need to protect erase during rx if interruptible */ + mutex_unlock(&client->send_mutex); + + wait_event(client->waitq, sender_should_wake(client, &sender)); + + mutex_lock(&client->send_mutex); + + /* finish tearing down the socket if recv shutdown */ + if (client->sock && client->recv_shutdown) { + shutdown_sock_sync(client); + continue; + } + } + + mutex_unlock(&client->send_mutex); + + /* safe to remove, we only finish after canceling recv or we're woke */ + spin_lock(&client->recv_lock); + rb_erase(&sender.node, &client->sender_root); + spin_unlock(&client->recv_lock); + + if (ret == 0) + ret = sender.result; + + return ret; +} + +int scoutfs_client_alloc_inodes(struct super_block *sb) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_inode_alloc ial; + u64 ino = 0; + u64 nr = 0; + int ret; + + ret = client_request(client, SCOUTFS_NET_ALLOC_INODES, NULL, 0, + &ial, sizeof(ial)); + if (ret == 0) { + ino = le64_to_cpu(ial.ino); + nr = le64_to_cpu(ial.nr); + + /* catch wrapping */ + if (ino + nr < ino) + ret = -EINVAL; + } + + if (ret < 0) + scoutfs_inode_fill_pool(sb, 0, 0); + else + scoutfs_inode_fill_pool(sb, ino, nr); + + return ret; +} + +int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 lesegno; + int ret; + + ret = client_request(client, SCOUTFS_NET_ALLOC_SEGNO, NULL, 0, + &lesegno, sizeof(lesegno)); + if (ret == 0) { + if (lesegno == 0) + ret = -ENOSPC; + else + *segno = le64_to_cpu(lesegno); + } + + return ret; +} + +int scoutfs_client_record_segment(struct super_block *sb, + struct scoutfs_segment *seg, u8 level) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_manifest_entry *net_ment; + struct scoutfs_manifest_entry ment; + int ret; + + scoutfs_seg_init_ment(&ment, level, seg); + net_ment = scoutfs_alloc_net_ment(&ment); + if (net_ment) { + ret = client_request(client, SCOUTFS_NET_RECORD_SEGMENT, + net_ment, scoutfs_net_ment_bytes(net_ment), + NULL, 0); + kfree(net_ment); + } else { + ret = -ENOMEM; + } + + return ret; +} + +static int sort_cmp_u64s(const void *A, const void *B) +{ + const u64 *a = A; + const u64 *b = B; + + return *a < *b ? -1 : *a > *b ? 1 : 0; +} + +static void sort_swap_u64s(void *A, void *B, int size) +{ + u64 *a = A; + u64 *b = B; + + swap(*a, *b); +} + +/* + * Returns a 0-terminated allocated array of segnos, the caller is + * responsible for freeing it. + * + * This double alloc is silly. But the caller does have an easier time + * with native u64s. We'll probably clean this up. + */ +u64 *scoutfs_client_bulk_alloc(struct super_block *sb) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_segnos *ns = NULL; + u64 *segnos = NULL; + size_t size; + unsigned nr; + u64 prev; + int ret; + int i; + + size = offsetof(struct scoutfs_net_segnos, + segnos[SCOUTFS_BULK_ALLOC_COUNT]); + ns = kmalloc(size, GFP_NOFS); + if (!ns) { + ret = -ENOMEM; + goto out; + } + + ret = client_request(client, SCOUTFS_NET_BULK_ALLOC, NULL, 0, ns, size); + if (ret) + goto out; + + nr = le16_to_cpu(ns->nr); + if (nr == 0) { + ret = -ENOSPC; + goto out; + } + + if (nr > SCOUTFS_BULK_ALLOC_COUNT) { + ret = -EINVAL; + goto out; + } + + segnos = kmalloc_array(nr + 1, sizeof(*segnos), GFP_NOFS); + if (segnos == NULL) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < nr; i++) + segnos[i] = le64_to_cpu(ns->segnos[i]); + segnos[nr] = 0; + + /* sort segnos for the caller so they can merge easily */ + sort(segnos, nr, sizeof(segnos[0]), sort_cmp_u64s, sort_swap_u64s); + + /* make sure they're all non-zero and unique */ + prev = 0; + for (i = 0; i < nr; i++) { + if (segnos[i] == prev) { + ret = -EINVAL; + goto out; + } + prev = segnos[i]; + } + + ret = 0; +out: + kfree(ns); + if (ret) { + kfree(segnos); + segnos = ERR_PTR(ret); + } + + return segnos; +} + +int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 before = cpu_to_le64p(seq); + __le64 after; + int ret; + + ret = client_request(client, SCOUTFS_NET_ADVANCE_SEQ, + &before, sizeof(before), &after, sizeof(after)); + if (ret == 0) + *seq = le64_to_cpu(after); + + return ret; +} + +int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 last_seq; + int ret; + + ret = client_request(client, SCOUTFS_NET_GET_LAST_SEQ, + NULL, 0, &last_seq, sizeof(last_seq)); + if (ret == 0) + *seq = le64_to_cpu(last_seq); + + return ret; +} + +int scoutfs_client_get_manifest_root(struct super_block *sb, + struct scoutfs_btree_root *root) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return client_request(client, SCOUTFS_NET_GET_MANIFEST_ROOT, + NULL, 0, root, sizeof(struct scoutfs_btree_root)); +} + +int scoutfs_client_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct client_info *client; + + client = kzalloc(sizeof(struct client_info), GFP_KERNEL); + if (!client) + return -ENOMEM; + + client->sb = sb; + spin_lock_init(&client->recv_lock); + client->sender_root = RB_ROOT; + mutex_init(&client->send_mutex); + init_waitqueue_head(&client->waitq); + INIT_WORK(&client->recv_work, scoutfs_client_recv_func); + + client->recv_wq = alloc_workqueue("scoutfs_client_recv", WQ_UNBOUND, 1); + if (!client->recv_wq) { + kfree(client); + return -ENOMEM; + } + + sbi->client_info = client; + return 0; +} + +/* + * There must be no more callers to the client send functions by the + * time we get here. We just need to free the socket if it's + * still sitting around. + */ +void scoutfs_client_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + if (client) { + shutdown_sock_sync(client); + + cancel_work_sync(&client->recv_work); + destroy_workqueue(client->recv_wq); + + kfree(client); + sbi->client_info = NULL; + } +} diff --git a/kmod/src/client.h b/kmod/src/client.h new file mode 100644 index 00000000..59b7b151 --- /dev/null +++ b/kmod/src/client.h @@ -0,0 +1,17 @@ +#ifndef _SCOUTFS_CLIENT_H_ +#define _SCOUTFS_CLIENT_H_ + +int scoutfs_client_alloc_inodes(struct super_block *sb); +int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); +int scoutfs_client_record_segment(struct super_block *sb, + struct scoutfs_segment *seg, u8 level); +u64 *scoutfs_client_bulk_alloc(struct super_block *sb); +int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); +int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); +int scoutfs_client_get_manifest_root(struct super_block *sb, + struct scoutfs_btree_root *root); + +int scoutfs_client_setup(struct super_block *sb); +void scoutfs_client_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 12f8d902..e81cea5f 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -24,7 +24,7 @@ #include "manifest.h" #include "counters.h" #include "alloc.h" -#include "net.h" +#include "server.h" #include "scoutfs_trace.h" /* @@ -579,7 +579,7 @@ static void scoutfs_compact_func(struct work_struct *work) INIT_LIST_HEAD(&curs.csegs); scoutfs_bio_init_comp(&comp); - ret = scoutfs_net_get_compaction(sb, (void *)&curs); + ret = scoutfs_client_get_compaction(sb, (void *)&curs); /* short circuit no compaction work to do */ if (ret == 0 && list_empty(&curs.csegs)) @@ -610,7 +610,7 @@ static void scoutfs_compact_func(struct work_struct *work) free_cseg_list(sb, &results); } - err = scoutfs_net_finish_compaction(sb, &curs, &results); + err = scoutfs_client_finish_compaction(sb, &curs, &results); if (!ret && err) ret = err; diff --git a/kmod/src/data.c b/kmod/src/data.c index 34ea7c45..d1070f8f 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -29,7 +29,7 @@ #include "scoutfs_trace.h" #include "item.h" #include "ioctl.h" -#include "net.h" +#include "client.h" #include "lock.h" #define EXTF "[off %llu bno %llu bks %llu fl %x]" @@ -731,7 +731,7 @@ static int bulk_alloc(struct super_block *sb) int ret; int i; - segnos = scoutfs_net_bulk_alloc(sb); + segnos = scoutfs_client_bulk_alloc(sb); if (IS_ERR(segnos)) { ret = PTR_ERR(segnos); goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 570d9cbf..ad02f4d1 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -31,7 +31,7 @@ #include "msg.h" #include "kvec.h" #include "item.h" -#include "net.h" +#include "client.h" /* * XXX @@ -670,7 +670,7 @@ static int alloc_ino(struct super_block *sb, u64 *ino) spin_unlock(&pool->lock); if (request) { - ret = scoutfs_net_alloc_inodes(sb); + ret = scoutfs_client_alloc_inodes(sb); if (ret) { spin_lock(&pool->lock); pool->in_flight = false; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 911d696f..664e7f22 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -29,7 +29,7 @@ #include "inode.h" #include "item.h" #include "data.h" -#include "net.h" +#include "client.h" #include "lock.h" #include "manifest.h" @@ -90,7 +90,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (type == SCOUTFS_INODE_INDEX_META_SEQ_TYPE || type == SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE) { - ret = scoutfs_net_get_last_seq(sb, &last_seq); + ret = scoutfs_client_get_last_seq(sb, &last_seq); if (ret) return ret; diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index ae7b4217..1748dc5b 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -26,7 +26,7 @@ #include "manifest.h" #include "trans.h" #include "counters.h" -#include "net.h" +#include "client.h" #include "scoutfs_trace.h" /* @@ -601,7 +601,7 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, * either get a manifest ref in the lvb of their lock or they'll * ask the server the first time the system sees the lock. */ - ret = scoutfs_net_get_manifest_root(sb, &root); + ret = scoutfs_client_get_manifest_root(sb, &root); if (ret) goto out; diff --git a/kmod/src/net.c b/kmod/src/net.c deleted file mode 100644 index f4a62de5..00000000 --- a/kmod/src/net.c +++ /dev/null @@ -1,2210 +0,0 @@ -/* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "format.h" -#include "net.h" -#include "counters.h" -#include "inode.h" -#include "btree.h" -#include "manifest.h" -#include "bio.h" -#include "alloc.h" -#include "seg.h" -#include "compact.h" -#include "scoutfs_trace.h" -#include "msg.h" - -/* - * scoutfs mounts use a simple client-server model to send and process - * requests to maintain consistency with lighter overhead than full - * locking. - * - * All mounts try to establish themselves as a server. They try to - * acquire an exclusive lock that allows them to act as the server. - * While they hold that lock they broadcast their listening address with - * an address lock's lvb. The server only accepts client connections, - * processes requests, and sends replies. It never sends requests to - * clients. The client is responsible for reliability and forward - * progress. - * - * All mounts must connect to the server to function. They sample the - * address lock's lvb to find an address and try to connect to it. - * Callers enqueue reqeust messages with a reply function. The requests - * are sent down each re-established connection to the server. If the - * client receives a reply it frees the request and calls the reply - * function. - * - * All kernel socket calls are non-blocking and made from work functions - * in a single threaded workqueue. This makes it easy to stop all work - * on the socket before shutting it down. - * - * XXX: - * - include mount id in the workqueue names? - * - set recv buf size to multiple of largest message size - */ - -struct net_info { - struct super_block *sb; - - /* protects lists and sock info pointers */ - struct mutex mutex; - - /* client connects and sends requests */ - struct delayed_work client_work; - struct sock_info *connected_sinf; - struct list_head to_send; - u64 next_id; - - /* server listens and processes requests */ - struct delayed_work server_work; - struct sock_info *listening_sinf; - bool server_loaded; - - /* server commits metadata while processing requests */ - struct rw_semaphore commit_rwsem; - struct llist_head commit_waiters; - struct work_struct commit_work; - - /* server remembers the stable manifest root for clients */ - struct scoutfs_btree_root stable_manifest_root; - - /* level 0 segment addition waits for it to clear */ - wait_queue_head_t waitq; - - /* server tracks seq use */ - spinlock_t seq_lock; - struct list_head pending_seqs; - - /* both track active sockets for destruction */ - struct list_head active_socks; - - /* non-blocking sock work is serialized, one at a time */ - struct workqueue_struct *sock_wq; - /* processing is unlimited and concurrent but each is non-reentrant */ - struct workqueue_struct *proc_wq; -}; - -#define DECLARE_NET_INFO(sb, name) \ - struct net_info *name = SCOUTFS_SB(sb)->net_info - -typedef int (*reply_func_t)(struct super_block *sb, void *recv, int bytes, - void *arg); - -/* - * Send buffers are allocated either by clients who send requests or by - * the server who sends replies. Request sends are freed when they get - * a reply and reply sends are freed either after they're sent or when - * their accepted client socket is shut down. - */ -struct send_buf { - struct list_head head; - reply_func_t func; - void *arg; - struct scoutfs_net_header nh[0]; -}; - -/* - * Receive bufs hold messages from the socket while they're being - * processed. They have embedded work so we can have easy concurrent - * processing. Their processing can block for IO. Their sending socket - * can be torn down during their processing in which case no reply is - * sent. - */ -struct recv_buf { - struct net_info *nti; - struct sock_info *sinf; - struct list_head head; - struct work_struct proc_work; - struct scoutfs_net_header nh[0]; -}; - -struct sock_info { - struct super_block *sb; - struct list_head head; - bool shutting_down; - - unsigned send_pos; - struct list_head to_send; - struct list_head have_sent; - struct list_head active_rbufs; - - struct scoutfs_lock *listen_lck; - struct scoutfs_inet_addr addr; - - struct work_struct listen_work; - struct work_struct accept_work; - struct work_struct connect_work; - struct work_struct send_work; - struct work_struct recv_work; - struct work_struct shutdown_work; - - struct socket *sock; -}; - -/* - * XXX instead of magic keys in the main fs resource we could have - * another resource that contains the server locks. - */ -static u8 listen_type = SCOUTFS_NET_LISTEN_TYPE; -static struct scoutfs_key_buf listen_key; -static u8 addr_type = SCOUTFS_NET_ADDR_TYPE; -static struct scoutfs_key_buf addr_key; - -static int send_msg(struct socket *sock, void *buf, unsigned len) -{ - struct kvec kvec = { .iov_base = buf, .iov_len = len }; - struct msghdr msg = { - .msg_iov = (struct iovec *)&kvec, - .msg_iovlen = 1, - .msg_flags = MSG_NOSIGNAL | MSG_DONTWAIT, - }; - - return kernel_sendmsg(sock, &msg, &kvec, 1, len); -} - -static int recv_msg(struct socket *sock, void *buf, unsigned len, int flags) -{ - struct kvec kvec = { .iov_base = buf, .iov_len = len }; - struct msghdr msg = { - .msg_iov = (struct iovec *)&kvec, - .msg_iovlen = 1, - .msg_flags = MSG_NOSIGNAL | MSG_DONTWAIT | flags, - }; - - return kernel_recvmsg(sock, &msg, &kvec, 1, len, msg.msg_flags); -} - -/* - * Don't queue work on the socket if it's shutting down so that the - * shutdown work knows it can free the socket without work pending. - */ -static void queue_sock_work(struct sock_info *sinf, struct work_struct *work) -{ - DECLARE_NET_INFO(sinf->sb, nti); - - if (!sinf->shutting_down) - queue_work(nti->sock_wq, work); -} - -/* - * This non-blocking work consumes the send queue in the socket info as - * messages are sent out. If the messages have a reply function then - * they're requests that are resent until we receive a reply. If they - * don't then they're one-off replies that we free once they're sent. - */ -static void scoutfs_net_send_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - send_work); - DECLARE_NET_INFO(sinf->sb, nti); - struct send_buf *sbuf; - struct send_buf *pos; - char *buf; - int total; - int len; - int ret = 0; - - mutex_lock(&nti->mutex); - - list_for_each_entry_safe(sbuf, pos, &sinf->to_send, head) { - total = sizeof(struct scoutfs_net_header) + - le16_to_cpu(sbuf->nh->data_len); - - buf = (char *)sbuf->nh + sinf->send_pos; - len = total - sinf->send_pos; - - ret = send_msg(sinf->sock, buf, len); - trace_printk("sinf %p sock %p send len %d ret %d\n", - sinf, sinf->sock, len, ret); - if (ret < 0) { - if (ret == -EAGAIN) - ret = 0; - break; - } - if (ret == 0 || ret > len) { - ret = -EINVAL; - break; - } - - sinf->send_pos += ret; - - if (sinf->send_pos == total) { - sinf->send_pos = 0; - list_del_init(&sbuf->head); - - if (sbuf->func) - list_add_tail(&sbuf->head, &sinf->have_sent); - else - kfree(sbuf); - } - } - - if (ret < 0) { - trace_printk("ret %d\n", ret); - queue_sock_work(sinf, &sinf->shutdown_work); - } - - mutex_unlock(&nti->mutex); -} - -struct commit_waiter { - struct completion comp; - struct llist_node node; - int ret; -}; - -/* - * This is called while still holding the rwsem that prevents commits so - * that the caller can be sure to be woken by the next commit after they - * queue and release the lock. - * - * This could queue delayed work but we're first trying to have batching - * work by having concurrent modification line up behind a commit in - * flight. Once the commit finishes it'll unlock and hopefully everyone - * will race to make their changes and they'll all be applied by the - * next commit after that. - */ -static void queue_commit_work(struct net_info *nti, struct commit_waiter *cw) -{ - lockdep_assert_held(&nti->commit_rwsem); - - cw->ret = 0; - init_completion(&cw->comp); - llist_add(&cw->node, &nti->commit_waiters); - queue_work(nti->proc_wq, &nti->commit_work); -} - -static int wait_for_commit(struct commit_waiter *cw) -{ - wait_for_completion(&cw->comp); - return cw->ret; -} - -/* - * A core function of request processing is to modify the manifest and - * allocator. Often the processing needs to make the modifications - * persistent before replying. We'd like to batch these commits as much - * as is reasonable so that we don't degrade to a few IO round trips per - * request. - * - * Getting that batching right is bound up in the concurrency of request - * processing so a clear way to implement the batched commits is to - * implement commits with work funcs like the processing. This commit - * work is queued on the non-reentrant proc_wq so there will only ever - * be one commit executing at a time. - * - * Processing paths acquire the rwsem for reading while they're making - * multiple dependent changes. When they're done and want it persistent - * they add themselves to the list of waiters and queue the commit work. - * This work runs, acquires the lock to exclude other writers, and - * performs the commit. Readers can run concurrently with these - * commits. - */ -static void scoutfs_net_commit_func(struct work_struct *work) -{ - struct net_info *nti = container_of(work, struct net_info, commit_work); - struct super_block *sb = nti->sb; - struct commit_waiter *cw; - struct commit_waiter *pos; - struct llist_node *node; - int ret; - - down_write(&nti->commit_rwsem); - - if (scoutfs_btree_has_dirty(sb)) { - ret = scoutfs_alloc_apply_pending(sb) ?: - scoutfs_btree_write_dirty(sb) ?: - scoutfs_write_dirty_super(sb); - - /* we'd need to loop or something */ - BUG_ON(ret); - - scoutfs_btree_write_complete(sb); - - nti->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; - scoutfs_advance_dirty_super(sb); - } else { - ret = 0; - } - - node = llist_del_all(&nti->commit_waiters); - - /* waiters always wait on completion, cw could be free after complete */ - llist_for_each_entry_safe(cw, pos, node, node) { - cw->ret = ret; - complete(&cw->comp); - } - - up_write(&nti->commit_rwsem); -} - -static struct send_buf *alloc_sbuf(unsigned data_len) -{ - unsigned len = offsetof(struct send_buf, nh[0].data[data_len]); - struct send_buf *sbuf; - - sbuf = kmalloc(len, GFP_NOFS); - if (sbuf) { - INIT_LIST_HEAD(&sbuf->head); - sbuf->nh->data_len = cpu_to_le16(data_len); - } - - return sbuf; -} - -static struct send_buf *process_bulk_alloc(struct super_block *sb,void *req, - int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_net_segnos *ns; - struct commit_waiter cw; - struct send_buf *sbuf; - u64 segno; - int ret; - int i; - - if (req_len != 0) - return ERR_PTR(-EINVAL); - - sbuf = alloc_sbuf(offsetof(struct scoutfs_net_segnos, - segnos[SCOUTFS_BULK_ALLOC_COUNT])); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - ns = (void *)sbuf->nh->data; - ns->nr = cpu_to_le16(SCOUTFS_BULK_ALLOC_COUNT); - - down_read(&nti->commit_rwsem); - - for (i = 0; i < SCOUTFS_BULK_ALLOC_COUNT; i++) { - ret = scoutfs_alloc_segno(sb, &segno); - if (ret) { - while (i-- > 0) - scoutfs_alloc_free(sb, - le64_to_cpu(ns->segnos[i])); - break; - } - - ns->segnos[i] = cpu_to_le64(segno); - } - - - if (ret == 0) - queue_commit_work(nti, &cw); - up_read(&nti->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(&cw); - - if (ret) - sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; - else - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - -static void init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) -{ - scoutfs_key_init(first, net_ment->keys, - le16_to_cpu(net_ment->first_key_len)); - scoutfs_key_init(last, net_ment->keys + - le16_to_cpu(net_ment->first_key_len), - le16_to_cpu(net_ment->last_key_len)); -} - -/* - * Allocate a contiguous manifest entry for communication over the network. - */ -static struct scoutfs_net_manifest_entry * -alloc_net_ment(struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_net_manifest_entry *net_ment; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - net_ment = kmalloc(offsetof(struct scoutfs_net_manifest_entry, - keys[ment->first.key_len + - ment->last.key_len]), GFP_NOFS); - if (!net_ment) - return NULL; - - net_ment->segno = cpu_to_le64(ment->segno); - net_ment->seq = cpu_to_le64(ment->seq); - net_ment->first_key_len = cpu_to_le16(ment->first.key_len); - net_ment->last_key_len = cpu_to_le16(ment->last.key_len); - net_ment->level = ment->level; - - init_net_ment_keys(net_ment, &first, &last); - scoutfs_key_copy(&first, &ment->first); - scoutfs_key_copy(&last, &ment->last); - - return net_ment; -} - -/* point a native manifest entry at a contiguous net manifest */ -static void init_ment_net_ment(struct scoutfs_manifest_entry *ment, - struct scoutfs_net_manifest_entry *net_ment) -{ - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - init_net_ment_keys(net_ment, &first, &last); - scoutfs_key_clone(&ment->first, &first); - scoutfs_key_clone(&ment->last, &last); - - ment->segno = le64_to_cpu(net_ment->segno); - ment->seq = le64_to_cpu(net_ment->seq); - ment->level = net_ment->level; -} - -static unsigned net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment) -{ - return offsetof(struct scoutfs_net_manifest_entry, - keys[le16_to_cpu(net_ment->first_key_len) + - le16_to_cpu(net_ment->last_key_len)]); -} - -/* - * This is new segments arriving. It needs to wait for level 0 to be - * free. It has relatively little visibility into the manifest, though. - * We don't want it to block holding commits because that'll stop - * manifest updates from emptying level 0. - * - * Maybe the easiest way is to protect the level counts with a seqlock, - * or whatever. - */ - -/* - * The sender has written their level 0 segment and has given us its - * details. We wait for there to be room in level 0 before adding it. - */ -static struct send_buf *process_record_segment(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_manifest_entry ment; - struct scoutfs_net_manifest_entry *net_ment; - struct commit_waiter cw; - struct send_buf *sbuf; - int ret; - - if (req_len < sizeof(struct scoutfs_net_manifest_entry)) { - sbuf = ERR_PTR(-EINVAL); - goto out; - } - - net_ment = req; - - if (req_len != net_ment_bytes(net_ment)) { - sbuf = ERR_PTR(-EINVAL); - goto out; - } - -retry: - down_read(&nti->commit_rwsem); - scoutfs_manifest_lock(sb); - - if (scoutfs_manifest_level0_full(sb)) { - scoutfs_manifest_unlock(sb); - up_read(&nti->commit_rwsem); - /* XXX waits indefinitely? io errors? */ - wait_event(nti->waitq, !scoutfs_manifest_level0_full(sb)); - goto retry; - } - - init_ment_net_ment(&ment, net_ment); - - ret = scoutfs_manifest_add(sb, &ment); - scoutfs_manifest_unlock(sb); - - if (ret == 0) - queue_commit_work(nti, &cw); - up_read(&nti->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(&cw); - - scoutfs_compact_kick(sb); - - sbuf = alloc_sbuf(0); - if (!sbuf) { - sbuf = ERR_PTR(-ENOMEM); - goto out; - } - - if (ret) - sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; - else - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; -out: - return sbuf; -} - -static struct send_buf *process_alloc_segno(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - __le64 * __packed lesegno; - struct commit_waiter cw; - struct send_buf *sbuf; - u64 segno; - int ret; - - if (req_len != 0) { - sbuf = ERR_PTR(-EINVAL); - goto out; - } - - down_read(&nti->commit_rwsem); - - ret = scoutfs_alloc_segno(sb, &segno); - if (ret == 0) - queue_commit_work(nti, &cw); - - up_read(&nti->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(&cw); - - sbuf = alloc_sbuf(sizeof(__le64)); - if (!sbuf) { - sbuf = ERR_PTR(-ENOMEM); - goto out; - } - - if (ret) { - sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; - } else { - lesegno = (void *)sbuf->nh->data; - *lesegno = cpu_to_le64(segno); - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - } - -out: - return sbuf; -} - -/* - * XXX should this call into inodes? not sure about the layering here. - */ -static struct send_buf *process_alloc_inodes(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_net_inode_alloc *ial; - struct commit_waiter cw; - struct send_buf *sbuf; - int ret; - u64 ino; - u64 nr; - - if (req_len != 0) - return ERR_PTR(-EINVAL); - - sbuf = alloc_sbuf(sizeof(struct scoutfs_net_inode_alloc)); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - down_read(&nti->commit_rwsem); - - spin_lock(&sbi->next_ino_lock); - ino = le64_to_cpu(super->next_ino); - nr = min(100000ULL, ~0ULL - ino); - le64_add_cpu(&super->next_ino, nr); - spin_unlock(&sbi->next_ino_lock); - - queue_commit_work(nti, &cw); - up_read(&nti->commit_rwsem); - - ret = wait_for_commit(&cw); - - ial = (void *)sbuf->nh->data; - ial->ino = cpu_to_le64(ino); - ial->nr = cpu_to_le64(nr); - - if (ret < 0) - sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; - else - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - -struct pending_seq { - struct list_head head; - u64 seq; -}; - -/* - * Give the client the next seq for it to use in items in its - * transaction. They tell us the seq they just used so we can remove it - * from pending tracking and possibly include it in get_last_seq - * replies. - * - * The list walk is O(clients) and the message processing rate goes from - * every committed segment to every sync deadline interval. - * - * XXX The pending seq tracking should be persistent so that it survives - * server failover. - */ -static struct send_buf *process_advance_seq(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct pending_seq *next_ps; - struct pending_seq *ps; - struct commit_waiter cw; - __le64 * __packed prev; - __le64 * __packed next; - struct send_buf *sbuf; - int ret; - - if (req_len != sizeof(__le64)) - return ERR_PTR(-EINVAL); - - prev = req; - - sbuf = alloc_sbuf(sizeof(__le64)); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - next = (void *)sbuf->nh->data; - - next_ps = kmalloc(sizeof(struct pending_seq), GFP_NOFS); - if (!next_ps) { - ret = -ENOMEM; - goto out; - } - - down_read(&nti->commit_rwsem); - - spin_lock(&nti->seq_lock); - - list_for_each_entry(ps, &nti->pending_seqs, head) { - if (ps->seq == le64_to_cpu(*prev)) { - list_del_init(&ps->head); - kfree(ps); - break; - } - } - - *next = super->next_seq; - le64_add_cpu(&super->next_seq, 1); - - trace_printk("prev %llu next %llu, super next_seq %llu\n", - le64_to_cpup(prev), le64_to_cpup(next), - le64_to_cpu(super->next_seq)); - - next_ps->seq = le64_to_cpup(next); - list_add_tail(&next_ps->head, &nti->pending_seqs); - - spin_unlock(&nti->seq_lock); - - queue_commit_work(nti, &cw); - up_read(&nti->commit_rwsem); - - ret = wait_for_commit(&cw); -out: - if (ret < 0) - sbuf->nh->status = SCOUTFS_NET_STATUS_ERROR; - else - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - -/* - * Give the client the last seq that is stable before the lowest seq - * that is still dirty out at a client. - */ -static struct send_buf *process_get_last_seq(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct pending_seq *ps; - __le64 * __packed last; - struct send_buf *sbuf; - - if (req_len != 0) - return ERR_PTR(-EINVAL); - - sbuf = alloc_sbuf(sizeof(__le64)); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - last = (void *)sbuf->nh->data; - - spin_lock(&nti->seq_lock); - ps = list_first_entry_or_null(&nti->pending_seqs, - struct pending_seq, head); - if (ps) { - *last = cpu_to_le64(ps->seq - 1); - } else { - *last = super->next_seq; - le64_add_cpu(last, -1ULL); - } - spin_unlock(&nti->seq_lock); - - trace_printk("last %llu\n", le64_to_cpup(last)); - - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - -static struct send_buf *process_get_manifest_root(struct super_block *sb, - void *req, int req_len) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_btree_root *root; - struct send_buf *sbuf; - - if (req_len != 0) - return ERR_PTR(-EINVAL); - - sbuf = alloc_sbuf(sizeof(struct scoutfs_btree_root)); - if (!sbuf) - return ERR_PTR(-ENOMEM); - - root = (void *)sbuf->nh->data; - - scoutfs_manifest_lock(sb); - memcpy(root, &nti->stable_manifest_root, - sizeof(struct scoutfs_btree_root)); - scoutfs_manifest_unlock(sb); - - sbuf->nh->status = SCOUTFS_NET_STATUS_SUCCESS; - - return sbuf; -} - -typedef struct send_buf *(*proc_func_t)(struct super_block *sb, void *req, - int req_len); - -static proc_func_t type_proc_func(u8 type) -{ - static proc_func_t funcs[] = { - [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, - [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, - [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, - [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, - [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, - [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, - [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, - }; - - return type < SCOUTFS_NET_UNKNOWN ? funcs[type] : NULL; -} - -/* - * Process an incoming request and queue its reply to send if the socket - * is still open by the time we have the reply. - */ -static int process_request(struct net_info *nti, struct recv_buf *rbuf) -{ - struct super_block *sb = nti->sb; - struct send_buf *sbuf; - proc_func_t proc; - unsigned data_len; - - data_len = le16_to_cpu(rbuf->nh->data_len); - proc = type_proc_func(rbuf->nh->type); - if (proc) - sbuf = proc(sb, (void *)rbuf->nh->data, data_len); - else - sbuf = ERR_PTR(-EINVAL); - if (IS_ERR(sbuf)) - return PTR_ERR(sbuf); - - /* processing sets data_len and status */ - sbuf->func = NULL; - sbuf->nh->id = rbuf->nh->id; - sbuf->nh->type = rbuf->nh->type; - - mutex_lock(&nti->mutex); - if (rbuf->sinf) { - list_add(&sbuf->head, &rbuf->sinf->to_send); - queue_sock_work(rbuf->sinf, &rbuf->sinf->send_work); - sbuf = NULL; - } - mutex_unlock(&nti->mutex); - - kfree(sbuf); - - return 0; -} - -/* - * The server only sends replies down the socket on which it receives - * the request. If we receive a reply we must have sent the request - * down the socket and the send buf will be found on the have_sent list. - */ -static int process_reply(struct net_info *nti, struct recv_buf *rbuf) -{ - struct super_block *sb = nti->sb; - reply_func_t func = NULL; - struct send_buf *sbuf; - void *arg; - int ret; - - mutex_lock(&nti->mutex); - - if (rbuf->sinf) { - list_for_each_entry(sbuf, &rbuf->sinf->have_sent, head) { - if (sbuf->nh->id == rbuf->nh->id) { - list_del_init(&sbuf->head); - func = sbuf->func; - arg = sbuf->arg; - kfree(sbuf); - sbuf = NULL; - break; - } - } - } - - mutex_unlock(&nti->mutex); - - if (func == NULL) - return 0; - - if (rbuf->nh->status == SCOUTFS_NET_STATUS_SUCCESS) - ret = le16_to_cpu(rbuf->nh->data_len); - else - ret = -EIO; - - return func(sb, rbuf->nh->data, ret, arg); -} - -static void destroy_server_state(struct super_block *sb) -{ - DECLARE_NET_INFO(sb, nti); - struct pending_seq *ps; - struct pending_seq *tmp; - - scoutfs_compact_destroy(sb); - scoutfs_alloc_destroy(sb); - scoutfs_manifest_destroy(sb); - scoutfs_btree_destroy(sb); - - /* XXX these should be persistent and reclaimed during recovery */ - list_for_each_entry_safe(ps, tmp, &nti->pending_seqs, head) { - list_del_init(&ps->head); - kfree(ps); - } -} - -/* - * Process each received message in its own non-reentrant work so we get - * concurrent request processing. - */ -static void scoutfs_net_proc_func(struct work_struct *work) -{ - struct recv_buf *rbuf = container_of(work, struct recv_buf, proc_work); - struct net_info *nti = rbuf->nti; - struct super_block *sb = nti->sb; - int ret = 0; - - /* - * This is the first blocking context we have once all the - * server locking and networking is set up so we bring up the - * rest of the server state the first time we get here. - */ - while (!nti->server_loaded) { - mutex_lock(&nti->mutex); - if (!nti->server_loaded) { - ret = scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: - scoutfs_btree_setup(sb) ?: - scoutfs_manifest_setup(sb) ?: - scoutfs_alloc_setup(sb) ?: - scoutfs_compact_setup(sb); - if (ret == 0) { - scoutfs_advance_dirty_super(sb); - nti->server_loaded = true; - nti->stable_manifest_root = - SCOUTFS_SB(sb)->super.manifest.root; - } else { - destroy_server_state(sb); - } - } - mutex_unlock(&nti->mutex); - if (ret) { - trace_printk("server setup failed %d\n", ret); - queue_sock_work(rbuf->sinf, &rbuf->sinf->shutdown_work); - return; - } - } - - if (rbuf->nh->status == SCOUTFS_NET_STATUS_REQUEST) - ret = process_request(nti, rbuf); - else - ret = process_reply(nti, rbuf); - - if (ret) - trace_printk("type %u id %llu status %u ret %d\n", - rbuf->nh->type, le64_to_cpu(rbuf->nh->id), - rbuf->nh->status, ret); - - mutex_lock(&nti->mutex); - - if (ret < 0 && rbuf->sinf) - queue_sock_work(rbuf->sinf, &rbuf->sinf->shutdown_work); - - if (!list_empty(&rbuf->head)) - list_del_init(&rbuf->head); - - mutex_unlock(&nti->mutex); - - kfree(rbuf); -} - -/* - * only accepted (not listening or connected) sockets receive requests - * and only connected sockets receive replies. This is running in the - * single threaded socket workqueue so it isn't racing with the shutdown - * work that would null the sinf pointer if it matches this sinf. - */ -static bool inappropriate_message(struct net_info *nti, struct sock_info *sinf, - struct recv_buf *rbuf) -{ - if (rbuf->nh->status == SCOUTFS_NET_STATUS_REQUEST && - (sinf == nti->listening_sinf || sinf == nti->connected_sinf)) - return true; - - if (rbuf->nh->status != SCOUTFS_NET_STATUS_REQUEST && - sinf != nti->connected_sinf) - return true; - - return false; -} - -/* - * Parse an incoming message on a socket. We peek at the socket buffer - * until it has the whole message. Then we queue request or reply - * processing work and shut down the socket if anything weird happens. - */ -static void scoutfs_net_recv_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - recv_work); - DECLARE_NET_INFO(sinf->sb, nti); - struct scoutfs_net_header nh; - struct recv_buf *rbuf; - int len; - int inq; - int ret; - - for (;;) { - /* peek to see data_len in the header */ - ret = recv_msg(sinf->sock, &nh, sizeof(nh), MSG_PEEK); - trace_printk("sinf %p sock %p peek ret %d\n", - sinf, sinf->sock, ret); - if (ret != sizeof(nh)) { - if (ret > 0 || ret == -EAGAIN) - ret = 0; - else if (ret == 0) - ret = -EIO; - break; - } - - /* XXX verify data_len isn't insane */ - - len = sizeof(struct scoutfs_net_header) + - le16_to_cpu(nh.data_len); - - /* XXX rx buf has to be > max packet len */ - ret = kernel_sock_ioctl(sinf->sock, SIOCINQ, - (unsigned long)&inq); - trace_printk("sinf %p sock %p ioctl ret %d\n", - sinf, sinf->sock, ret); - if (ret < 0 || inq < len) - break; - - rbuf = kmalloc(sizeof(struct recv_buf) + len, GFP_NOFS); - if (!rbuf) { - ret = -ENOMEM; - break; - } - - ret = recv_msg(sinf->sock, rbuf->nh, len, 0); - trace_printk("sinf %p sock %p recv len %d ret %d\n", - sinf, sinf->sock, len, ret); - if (ret != len) { - if (ret >= 0) - ret = -EIO; - break; - } - - if (inappropriate_message(nti, sinf, rbuf)) { - ret = -EINVAL; - break; - } - - rbuf->nti = nti; - rbuf->sinf = sinf; - INIT_LIST_HEAD(&rbuf->head); - INIT_WORK(&rbuf->proc_work, scoutfs_net_proc_func); - - mutex_lock(&nti->mutex); - list_add(&rbuf->head, &sinf->active_rbufs); - mutex_unlock(&nti->mutex); - queue_work(nti->proc_wq, &rbuf->proc_work); - rbuf = NULL; - } - - if (ret < 0) { - kfree(rbuf); - trace_printk("ret %d\n", ret); - queue_sock_work(sinf, &sinf->shutdown_work); - } -} - - -/* - * Connecting sockets kick off send and recv work once the socket is - * connected and all sockets shutdown when closed. - */ -static void scoutfs_net_state_change(struct sock *sk) -{ - struct sock_info *sinf = sk->sk_user_data; - - trace_printk("sk %p state %u sinf %p\n", sk, sk->sk_state, sinf); - - if (sinf && sinf->sock->sk == sk) { - switch(sk->sk_state) { - case TCP_ESTABLISHED: - queue_sock_work(sinf, &sinf->send_work); - queue_sock_work(sinf, &sinf->recv_work); - break; - case TCP_CLOSE: - queue_sock_work(sinf, &sinf->shutdown_work); - break; - } - } -} - -/* - * Listening sockets accept incoming sockets and accepted and connected - * sockets recv data. - */ -static void scoutfs_net_data_ready(struct sock *sk, int bytes) -{ - struct sock_info *sinf = sk->sk_user_data; - - trace_printk("sk %p bytes %d sinf %p\n", sk, bytes, sinf); - - if (sinf && sinf->sock->sk == sk) { - if (sk->sk_state == TCP_LISTEN) - queue_sock_work(sinf, &sinf->accept_work); - else - queue_sock_work(sinf, &sinf->recv_work); - } -} - -/* - * Connected and accepted sockets send once there's space again in the - * tx buffer. - */ -static void scoutfs_net_write_space(struct sock *sk) -{ - struct sock_info *sinf = sk->sk_user_data; - - trace_printk("sk %p sinf %p\n", sk, sinf); - - if (sinf && sinf->sock->sk == sk) { - if (sk_stream_is_writeable(sk)) - clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); - queue_sock_work(sinf, &sinf->send_work); - } -} - -/* - * Accepted sockets inherit the sk fields from the listening socket so - * all the callbacks check that the sinf they're working on points to - * the socket executing the callback. This ensures that we'll only get - * callbacks doing work once we've initialized sinf for the socket. - */ -static void set_sock_callbacks(struct sock_info *sinf) -{ - struct sock *sk = sinf->sock->sk; - - sk->sk_state_change = scoutfs_net_state_change; - sk->sk_data_ready = scoutfs_net_data_ready; - sk->sk_write_space = scoutfs_net_write_space; - sk->sk_user_data = sinf; - -} - -static int write_server_addr(struct super_block *sb, - struct scoutfs_inet_addr *addr) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - super->server_addr.addr = addr->addr; - super->server_addr.port = addr->port; - - return scoutfs_write_dirty_super(sb); -} - -static int read_server_addr(struct super_block *sb, - struct scoutfs_inet_addr *addr) -{ - int ret; - struct scoutfs_super_block stack; - - ret = scoutfs_read_supers(sb, &stack); - if (ret == 0) { - addr->addr = stack.server_addr.addr; - addr->port = stack.server_addr.port; - } - return ret; -} - -/* - * The caller can provide an error to give to pending sends before - * freeing them. - */ -static void free_sbuf_list(struct super_block *sb, struct list_head *list, - int ret) -{ - struct send_buf *sbuf; - struct send_buf *pos; - - list_for_each_entry_safe(sbuf, pos, list, head) { - list_del_init(&sbuf->head); - if (ret && sbuf->func) - sbuf->func(sb, NULL, ret, sbuf->arg); - kfree(sbuf); - } -} - -/* - * Remove the rbufs from the list and clear their sinf pointers so that - * they can't reference a sinf that's being freed. - */ -static void empty_rbuf_list(struct list_head *list) -{ - struct recv_buf *rbuf; - struct recv_buf *pos; - - list_for_each_entry_safe(rbuf, pos, list, head) { - list_del_init(&rbuf->head); - rbuf->sinf = NULL; - } -} - -/* - * Shutdown and free a socket. This can be queued from most all socket - * work. It executes in the single socket workqueue context so we know - * that we're serialized with all other socket work. Listening, - * connecting, and accepting don't reference the socket once it's - * possible for this work to execute. - * - * Other work won't be executing but could be queued when we get here. - * We can't cancel other work from inside their workqueue (until later - * kernels when cancel_work() comes back). So we have a two phase - * shutdown where we first prevent additional work from being queued and - * then queue the work again. By the time the work executes again we - * know that none of our work will be pending. - */ -static void scoutfs_net_shutdown_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - shutdown_work); - struct super_block *sb = sinf->sb; - DECLARE_NET_INFO(sb, nti); - struct socket *sock = sinf->sock; - int ret; - - trace_printk("sinf %p sock %p shutting_down %d\n", - sinf, sock, sinf->shutting_down); - - if (!sinf->shutting_down) { - sinf->shutting_down = true; - queue_work(nti->sock_wq, &sinf->shutdown_work); - return; - } - - kernel_sock_shutdown(sock, SHUT_RDWR); - - mutex_lock(&nti->mutex); - - if (sinf == nti->listening_sinf) { - nti->listening_sinf = NULL; - - /* shutdown the server, processing won't leave dirty metadata */ - destroy_server_state(sb); - nti->server_loaded = false; - - /* clear addr, try to reacquire lock and listen */ - memset(&sinf->addr, 0, sizeof(sinf->addr)); - ret = write_server_addr(sb, &sinf->addr); - if (ret) - scoutfs_err(sb, - "Non-fatal error %d while writing server " - "address\n", ret); - scoutfs_unlock(sb, sinf->listen_lck); - queue_delayed_work(nti->proc_wq, &nti->server_work, 0); - - } if (sinf == nti->connected_sinf) { - /* save reliable sends and try to reconnect */ - nti->connected_sinf = NULL; - list_splice_init(&sinf->have_sent, &nti->to_send); - list_splice_init(&sinf->to_send, &nti->to_send); - queue_delayed_work(nti->proc_wq, &nti->client_work, 0); - - } else { - /* free reply sends and stop rbuf socket refs */ - free_sbuf_list(sb, &sinf->to_send, 0); - empty_rbuf_list(&sinf->active_rbufs); - } - - list_del_init(&sinf->head); - - mutex_unlock(&nti->mutex); - - sock_release(sock); - kfree(sinf); -} - -static int add_send_buf(struct super_block *sb, int type, void *data, - unsigned data_len, reply_func_t func, void *arg) -{ - DECLARE_NET_INFO(sb, nti); - struct scoutfs_net_header *nh; - struct sock_info *sinf; - struct send_buf *sbuf; - - sbuf = alloc_sbuf(data_len); - if (!sbuf) - return -ENOMEM; - - sbuf->func = func; - sbuf->arg = arg; - sbuf->nh->status = SCOUTFS_NET_STATUS_REQUEST; - - nh = sbuf->nh; - nh->type = type; - if (data_len) - memcpy(nh->data, data, data_len); - - mutex_lock(&nti->mutex); - - nh->id = cpu_to_le64(nti->next_id++); - - sinf = nti->connected_sinf; - if (sinf) { - list_add_tail(&sbuf->head, &sinf->to_send); - queue_sock_work(sinf, &sinf->send_work); - } else { - list_add_tail(&sbuf->head, &nti->to_send); - } - - mutex_unlock(&nti->mutex); - - return 0; -} - -struct bulk_alloc_args { - struct completion comp; - u64 *segnos; - int ret; -}; - -static int sort_cmp_u64s(const void *A, const void *B) -{ - const u64 *a = A; - const u64 *b = B; - - return *a < *b ? -1 : *a > *b ? 1 : 0; -} - -static void sort_swap_u64s(void *A, void *B, int size) -{ - u64 *a = A; - u64 *b = B; - - swap(*a, *b); -} - -static int bulk_alloc_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct bulk_alloc_args *args = arg; - struct scoutfs_net_segnos *ns = reply; - u16 nr; - int i; - - if (ret < sizeof(struct scoutfs_net_segnos) || - ret != offsetof(struct scoutfs_net_segnos, - segnos[le16_to_cpu(ns->nr)])) { - ret = -EINVAL; - goto out; - } - - nr = le16_to_cpu(ns->nr); - - args->segnos = kmalloc((nr + 1) * sizeof(args->segnos[0]), GFP_NOFS); - if (args->segnos == NULL) { - ret = -ENOMEM; /* XXX hmm. */ - goto out; - } - - for (i = 0; i < nr; i++) { - args->segnos[i] = le64_to_cpu(ns->segnos[i]); - - /* make sure they're all non-zero */ - if (args->segnos[i] == 0) { - ret = -EINVAL; - goto out; - } - } - - sort(args->segnos, nr, sizeof(args->segnos[0]), - sort_cmp_u64s, sort_swap_u64s); - - /* make sure they're all unique */ - for (i = 1; i < nr; i++) { - if (args->segnos[i] == args->segnos[i - 1]) { - ret = -EINVAL; - goto out; - } - } - - args->segnos[nr] = 0; - ret = 0; -out: - if (ret && args->segnos) { - kfree(args->segnos); - args->segnos = NULL; - } - args->ret = ret; - complete(&args->comp); - return args->ret; -} - -/* - * Returns a 0-terminated allocated array of segnos, the caller is - * responsible for freeing it. - */ -u64 *scoutfs_net_bulk_alloc(struct super_block *sb) -{ - struct bulk_alloc_args args; - int ret; - - args.segnos = NULL; - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_BULK_ALLOC, NULL, 0, - bulk_alloc_reply, &args); - if (ret == 0) { - wait_for_completion(&args.comp); - ret = args.ret; - if (ret == 0 && (args.segnos == NULL || args.segnos[0] == 0)) - ret = -ENOSPC; - } - - if (ret) { - kfree(args.segnos); - args.segnos = ERR_PTR(ret); - } - - return args.segnos; -} - -/* - * Eventually we're going to have messages that control compaction. - * Each client mount would have long-lived work that sends requests - * which are stuck in processing until there's work to do. They'd get - * their entries, perform the compaction, and send a reply. But we're - * not there yet. - * - * This is a short circuit that's called directly by a work function - * that's only queued on the server. It makes compaction work inside - * the commit consistency mechanics inside net message processing and - * demonstrates the moving pieces that we'd need to cut up into a series - * of messages and replies. - * - * The compaction work caller cleans up everything on errors. - */ -int scoutfs_net_get_compaction(struct super_block *sb, void *curs) -{ - DECLARE_NET_INFO(sb, nti); - struct commit_waiter cw; - u64 segno; - int ret = 0; - int nr; - int i; - - down_read(&nti->commit_rwsem); - - nr = scoutfs_manifest_next_compact(sb, curs); - if (nr <= 0) { - up_read(&nti->commit_rwsem); - return nr; - } - - /* allow for expansion slop from sticky and alignment */ - for (i = 0; i < nr + SCOUTFS_COMPACTION_SLOP; i++) { - ret = scoutfs_alloc_segno(sb, &segno); - if (ret < 0) - break; - scoutfs_compact_add_segno(sb, curs, segno); - } - - if (ret == 0) - queue_commit_work(nti, &cw); - up_read(&nti->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(&cw); - - return ret; -} - -/* - * This is a stub for recording the results of a compaction. We just - * call back into compaction to have it call the manifest and allocator - * updates. - * - * In the future we'd encode the manifest and segnos in requests sent to - * the server who'd update the manifest and allocator in request - * processing. - * - * As we finish a compaction we wait level0 writers if it opened up - * space in level 0. - */ -int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, - void *list) -{ - DECLARE_NET_INFO(sb, nti); - struct commit_waiter cw; - bool level0_was_full; - int ret; - - down_read(&nti->commit_rwsem); - - level0_was_full = scoutfs_manifest_level0_full(sb); - - ret = scoutfs_compact_commit(sb, curs, list); - if (ret == 0) { - queue_commit_work(nti, &cw); - if (level0_was_full && !scoutfs_manifest_level0_full(sb)) - wake_up(&nti->waitq); - } - - up_read(&nti->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(&cw); - - scoutfs_compact_kick(sb); - - return ret; -} - -struct record_segment_args { - struct completion comp; - int ret; -}; - -static int record_segment_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct record_segment_args *args = arg; - - if (ret > 0) - ret = -EINVAL; - - args->ret = ret; - complete(&args->comp); - return args->ret; -} - -int scoutfs_net_record_segment(struct super_block *sb, - struct scoutfs_segment *seg, u8 level) -{ - struct scoutfs_net_manifest_entry *net_ment; - struct record_segment_args args; - struct scoutfs_manifest_entry ment; - int ret; - - scoutfs_seg_init_ment(&ment, level, seg); - net_ment = alloc_net_ment(&ment); - if (!net_ment) { - ret = -ENOMEM; - goto out; - } - - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_RECORD_SEGMENT, net_ment, - net_ment_bytes(net_ment), - record_segment_reply, &args); - kfree(net_ment); - if (ret == 0) { - wait_for_completion(&args.comp); - ret = args.ret; - } -out: - return ret; -} - -struct alloc_segno_args { - u64 segno; - struct completion comp; - int ret; -}; - -static int alloc_segno_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct alloc_segno_args *args = arg; - __le64 * __packed segno = reply; - - if (ret == sizeof(__le64)) { - args->segno = le64_to_cpup(segno); - args->ret = 0; - } else { - args->ret = -EINVAL; - } - - complete(&args->comp); /* args can be freed from this point */ - return args->ret; -} - -int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno) -{ - struct alloc_segno_args args; - int ret; - - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_ALLOC_SEGNO, NULL, 0, - alloc_segno_reply, &args); - if (ret == 0) { - wait_for_completion(&args.comp); - *segno = args.segno; - ret = args.ret; - if (ret == 0 && *segno == 0) - ret = -ENOSPC; - } - return ret; -} - -static int alloc_inodes_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct scoutfs_net_inode_alloc *ial = reply; - u64 ino; - u64 nr; - - if (ret != sizeof(*ial)) { - ret = -EINVAL; - goto out; - } - - ino = le64_to_cpu(ial->ino); - nr = le64_to_cpu(ial->nr); - - /* catch wrapping */ - if (ino + nr < ino) { - ret = -EINVAL; - goto out; - } - - /* XXX compare to greatest inode we've seen? */ - - ret = 0; -out: - if (ret < 0) - scoutfs_inode_fill_pool(sb, 0, 0); - else - scoutfs_inode_fill_pool(sb, ino, nr); - return ret; -} - -int scoutfs_net_alloc_inodes(struct super_block *sb) -{ - return add_send_buf(sb, SCOUTFS_NET_ALLOC_INODES, NULL, 0, - alloc_inodes_reply, NULL); -} - -struct advance_seq_args { - u64 seq; - struct completion comp; - int ret; -}; - -static int advance_seq_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct advance_seq_args *args = arg; - __le64 * __packed seq = reply; - - if (ret == sizeof(__le64)) { - args->seq = le64_to_cpup(seq); - args->ret = 0; - } else { - args->ret = -EINVAL; - } - - complete(&args->comp); /* args can be freed from this point */ - return args->ret; -} - -int scoutfs_net_advance_seq(struct super_block *sb, u64 *seq) -{ - struct advance_seq_args args; - __le64 leseq = cpu_to_le64p(seq); - int ret; - - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_ADVANCE_SEQ, &leseq, - sizeof(leseq), advance_seq_reply, &args); - if (ret == 0) { - wait_for_completion(&args.comp); - *seq = args.seq; - ret = args.ret; - } - return ret; -} - -struct get_last_seq_args { - u64 seq; - struct completion comp; - int ret; -}; - -static int get_last_seq_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct get_last_seq_args *args = arg; - __le64 * __packed seq = reply; - - if (ret == sizeof(__le64)) { - args->seq = le64_to_cpup(seq); - args->ret = 0; - } else { - args->ret = -EINVAL; - } - - complete(&args->comp); /* args can be freed from this point */ - return args->ret; -} - -int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq) -{ - struct get_last_seq_args args; - int ret; - - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_GET_LAST_SEQ, NULL, 0, - get_last_seq_reply, &args); - if (ret == 0) { - wait_for_completion(&args.comp); - *seq = args.seq; - ret = args.ret; - } - return ret; -} - -struct get_manifest_root_args { - struct scoutfs_btree_root *root; - struct completion comp; - int ret; -}; - -static int get_manifest_root_reply(struct super_block *sb, void *reply, int ret, - void *arg) -{ - struct get_manifest_root_args *args = arg; - struct scoutfs_btree_root *root = reply; - - if (ret == sizeof(struct scoutfs_btree_root)) { - memcpy(args->root, root, sizeof(struct scoutfs_btree_root)); - args->ret = 0; - } else { - args->ret = -EINVAL; - } - - complete(&args->comp); /* args can be freed from this point */ - return args->ret; -} - -int scoutfs_net_get_manifest_root(struct super_block *sb, - struct scoutfs_btree_root *root) -{ - struct get_manifest_root_args args; - int ret; - - args.root = root; - init_completion(&args.comp); - - ret = add_send_buf(sb, SCOUTFS_NET_GET_MANIFEST_ROOT, NULL, 0, - get_manifest_root_reply, &args); - if (ret == 0) { - wait_for_completion(&args.comp); - ret = args.ret; - } - return ret; -} - - -static struct sock_info *alloc_sinf(struct super_block *sb) -{ - struct sock_info *sinf; - - sinf = kzalloc(sizeof(struct sock_info), GFP_NOFS); - if (sinf) { - sinf->sb = sb; - INIT_LIST_HEAD(&sinf->head); - INIT_LIST_HEAD(&sinf->to_send); - INIT_LIST_HEAD(&sinf->have_sent); - INIT_LIST_HEAD(&sinf->active_rbufs); - - /* callers set other role specific work as appropriate */ - INIT_WORK(&sinf->shutdown_work, scoutfs_net_shutdown_func); - } - - return sinf; -} - -static void scoutfs_net_accept_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - accept_work); - struct super_block *sb = sinf->sb; - DECLARE_NET_INFO(sb, nti); - struct sock_info *new_sinf; - struct socket *new_sock; - int ret; - - for (;;) { - ret = kernel_accept(sinf->sock, &new_sock, O_NONBLOCK); - trace_printk("nti %p accept sock %p ret %d\n", - nti, new_sock, ret); - if (ret < 0) { - if (ret == -EAGAIN) - ret = 0; - break; - } - - new_sinf = alloc_sinf(sb); - if (!new_sinf) { - ret = -ENOMEM; - sock_release(new_sock); - break; - } - - trace_printk("accepted sinf %p sock %p sk %p\n", - new_sinf, new_sock, new_sock->sk); - - new_sinf->sock = new_sock; - INIT_WORK(&new_sinf->send_work, scoutfs_net_send_func); - INIT_WORK(&new_sinf->recv_work, scoutfs_net_recv_func); - - mutex_lock(&nti->mutex); - list_add(&new_sinf->head, &nti->active_socks); - queue_sock_work(new_sinf, &new_sinf->recv_work); - mutex_unlock(&nti->mutex); - - set_sock_callbacks(new_sinf); - } - - if (ret) { - trace_printk("ret %d\n", ret); - queue_sock_work(sinf, &sinf->shutdown_work); - } -} - -/* - * Create a new TCP socket and set all the options that are used for - * both connecting and listening sockets. - */ -static int create_sock_setopts(struct socket **sock_ret) -{ - struct socket *sock; - int optval; - int ret; - - *sock_ret = NULL; - - ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); - if (ret) { - trace_printk("sock create ret %d\n", ret); - return ret; - } - - optval = 1; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (char *)&optval, - sizeof(optval)); - if (ret) { - trace_printk("nodelay ret %d\n", ret); - sock_release(sock); - return ret; - } - - *sock_ret = sock; - - return 0; -} - -/* - * The server work has acquired the listen lock. We create a socket and - * publish its bound address in the addr lock's lvb. - * - * This can block in the otherwise non-blocking socket workqueue while - * acquiring the addr lock but it should be brief and doesn't matter - * much given that we're bringing up a new server. This should happen - * rarely. - */ -static void scoutfs_net_listen_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - listen_work); - struct super_block *sb = sinf->sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_inet_addr addr; - struct sockaddr_in sin; - struct socket *sock; - int addrlen; - int optval; - int ret; - - sin.sin_family = AF_INET; - sin.sin_addr.s_addr = cpu_to_be32(le32_to_cpu(sbi->opts.listen_addr.addr)); - sin.sin_port = cpu_to_be16(le16_to_cpu(sbi->opts.listen_addr.port)); - - trace_printk("binding to %pIS:%u\n", - &sin, be16_to_cpu(sin.sin_port)); - - ret = create_sock_setopts(&sock); - if (ret) - goto out; - - trace_printk("listening sinf %p sock %p sk %p\n", - sinf, sock, sock->sk); - - optval = 1; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&optval, - sizeof(optval)); - if (ret) { - trace_printk("reuseaddr ret %d\n", ret); - sock_release(sock); - goto out; - } - - sinf->sock = sock; - INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); - - addrlen = sizeof(sin); - ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen) ?: - kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); - if (ret) - goto out; - - trace_printk("sock %p listening on %pIS:%u\n", - sock, &sin, be16_to_cpu(sin.sin_port)); - - addr.addr = cpu_to_le32(be32_to_cpu(sin.sin_addr.s_addr)); - addr.port = cpu_to_le16(be16_to_cpu(sin.sin_port)); - - set_sock_callbacks(sinf); - - ret = kernel_listen(sock, 255); - if (ret) - goto out; - - scoutfs_advance_dirty_super(sb); - ret = write_server_addr(sb, &addr); - if (ret) - goto out; - scoutfs_advance_dirty_super(sb); - - queue_sock_work(sinf, &sinf->accept_work); - -out: - if (ret) { - trace_printk("ret %d\n", ret); - queue_sock_work(sinf, &sinf->shutdown_work); - } -} - -/* - * The client work has found an address to try and connect to. Create a - * connecting socket and wire up its callbacks. - */ -static void scoutfs_net_connect_func(struct work_struct *work) -{ - struct sock_info *sinf = container_of(work, struct sock_info, - connect_work); - struct sockaddr_in sin; - struct socket *sock; - int addrlen; - int ret; - - ret = create_sock_setopts(&sock); - if (ret) - goto out; - - trace_printk("connecting sinf %p sock %p sk %p\n", - sinf, sock, sock->sk); - - sinf->sock = sock; - - sin.sin_family = AF_INET; - sin.sin_addr.s_addr = cpu_to_be32(le32_to_cpu(sinf->addr.addr)); - sin.sin_port = cpu_to_be16(le16_to_cpu(sinf->addr.port)); - - trace_printk("connecting to %pIS:%u\n", - &sin, be16_to_cpu(sin.sin_port)); - - /* callbacks can fire once inside connect that'll succeed */ - set_sock_callbacks(sinf); - - addrlen = sizeof(sin); - ret = kernel_connect(sock, (struct sockaddr *)&sin, addrlen, - O_NONBLOCK); - if (ret == -EINPROGRESS) - ret = 0; -out: - if (ret) { - trace_printk("ret %d\n", ret); - queue_sock_work(sinf, &sinf->shutdown_work); - } -} - -/* - * This work executes whenever there isn't a socket on the client connected - * to the server: on mount, after the connected socket is shut down, and - * when we can't find an address in the addr lock's lvb. - */ -static void scoutfs_net_client_func(struct work_struct *work) -{ - struct net_info *nti = container_of(work, struct net_info, - client_work.work); - struct super_block *sb = nti->sb; - struct sock_info *sinf = NULL; - int ret; - - BUG_ON(nti->connected_sinf); - - sinf = alloc_sinf(sb); - if (!sinf) { - ret = -ENOMEM; - goto out; - } - - INIT_WORK(&sinf->connect_work, scoutfs_net_connect_func); - INIT_WORK(&sinf->send_work, scoutfs_net_send_func); - INIT_WORK(&sinf->recv_work, scoutfs_net_recv_func); - - ret = read_server_addr(sb, &sinf->addr); - if (ret == 0 && sinf->addr.addr == cpu_to_le32(INADDR_ANY)) - ret = -ENOENT; - if (ret < 0) { - kfree(sinf); - goto out; - } - - mutex_lock(&nti->mutex); - nti->connected_sinf = sinf; - list_splice_init(&nti->to_send, &sinf->to_send); - list_add(&sinf->head, &nti->active_socks); - queue_sock_work(sinf, &sinf->connect_work); - mutex_unlock(&nti->mutex); - -out: - if (ret < 0 && ret != -ESHUTDOWN) { - trace_printk("ret %d\n", ret); - queue_delayed_work(nti->proc_wq, &nti->client_work, HZ / 2); - } -} - -/* - * This very long running blocking work just sits trying to acquire a - * lock on the listening key which marks it as the active server. When - * it does that it queues off work to build up the listening socket. - * The lock is associated with the listening socket and is unlocked when - * the socket is shut down. - * - * This work is queued by mount, shutdown of the listening socket, and - * errors. It stops re-arming itself if it sees that locking has been - * shut down. - */ -static void scoutfs_net_server_func(struct work_struct *work) -{ - struct net_info *nti = container_of(work, struct net_info, - server_work.work); - struct super_block *sb = nti->sb; - struct sock_info *sinf = NULL; - int ret; - - BUG_ON(nti->listening_sinf); - - sinf = alloc_sinf(sb); - if (!sinf) { - ret = -ENOMEM; - goto out; - } - - INIT_WORK(&sinf->listen_work, scoutfs_net_listen_func); - INIT_WORK(&sinf->accept_work, scoutfs_net_accept_func); - - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ~0ULL, &sinf->listen_lck); - if (ret) { - kfree(sinf); - goto out; - } - - mutex_lock(&nti->mutex); - nti->listening_sinf = sinf; - list_add(&sinf->head, &nti->active_socks); - queue_sock_work(sinf, &sinf->listen_work); - mutex_unlock(&nti->mutex); - -out: - if (ret < 0 && ret != -ESHUTDOWN) { - trace_printk("ret %d\n", ret); - queue_delayed_work(nti->proc_wq, &nti->server_work, HZ / 2); - } -} - -static void free_nti(struct net_info *nti) -{ - if (nti) { - if (nti->sock_wq) - destroy_workqueue(nti->sock_wq); - if (nti->proc_wq) - destroy_workqueue(nti->proc_wq); - kfree(nti); - } -} - -int scoutfs_net_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct net_info *nti; - - scoutfs_key_init(&listen_key, &listen_type, sizeof(listen_type)); - scoutfs_key_init(&addr_key, &addr_type, sizeof(addr_type)); - - nti = kzalloc(sizeof(struct net_info), GFP_KERNEL); - if (nti) { - nti->sock_wq = alloc_workqueue("scoutfs_net_sock", - WQ_UNBOUND, 1); - nti->proc_wq = alloc_workqueue("scoutfs_net_proc", - WQ_NON_REENTRANT, 0); - } - if (!nti || !nti->sock_wq || !nti->proc_wq) { - free_nti(nti); - return -ENOMEM; - } - - nti->sb = sb; - mutex_init(&nti->mutex); - INIT_DELAYED_WORK(&nti->client_work, scoutfs_net_client_func); - INIT_LIST_HEAD(&nti->to_send); - nti->next_id = 1; - INIT_DELAYED_WORK(&nti->server_work, scoutfs_net_server_func); - init_rwsem(&nti->commit_rwsem); - init_llist_head(&nti->commit_waiters); - INIT_WORK(&nti->commit_work, scoutfs_net_commit_func); - init_waitqueue_head(&nti->waitq); - spin_lock_init(&nti->seq_lock); - INIT_LIST_HEAD(&nti->pending_seqs); - INIT_LIST_HEAD(&nti->active_socks); - - sbi->net_info = nti; - - queue_delayed_work(nti->proc_wq, &nti->server_work, 0); - queue_delayed_work(nti->proc_wq, &nti->client_work, 0); - - return 0; -} - -/* - * Shutdown and destroy all our socket communications. - * - * This is called after locking has been shutdown. Client and server - * work that executes from this point on will fail with -ESHUTDOWN and - * won't rearm itself. That prevents new sockets from being created so - * our job is to shutdown all the existing sockets. - * - * We'll have to be careful to shut down any non-vfs callers of ours - * that might try to send requests during destruction. - */ -void scoutfs_net_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_NET_INFO(sb, nti); - struct sock_info *sinf; - struct sock_info *pos; - - if (nti) { - /* let any currently executing client/server work finish */ - flush_workqueue(nti->proc_wq); - - /* stop any additional incoming accepted sockets */ - mutex_lock(&nti->mutex); - sinf = nti->listening_sinf; - if (sinf) - queue_sock_work(sinf, &sinf->shutdown_work); - mutex_unlock(&nti->mutex); - drain_workqueue(nti->sock_wq); - - /* shutdown all the remaining sockets */ - mutex_lock(&nti->mutex); - list_for_each_entry_safe(sinf, pos, &nti->active_socks, head) - queue_sock_work(sinf, &sinf->shutdown_work); - mutex_unlock(&nti->mutex); - drain_workqueue(nti->sock_wq); - - /* wait for processing (and commits) to finish and free rbufs */ - drain_workqueue(nti->proc_wq); - - /* make sure client/server work isn't queued */ - cancel_delayed_work_sync(&nti->server_work); - cancel_delayed_work_sync(&nti->client_work); - - /* call all pending replies with errors */ - list_for_each_entry_safe(sinf, pos, &nti->active_socks, head) - - /* and free all resources */ - free_sbuf_list(sb, &nti->to_send, -ESHUTDOWN); - free_nti(nti); - sbi->net_info = NULL; - } -} diff --git a/kmod/src/net.h b/kmod/src/net.h deleted file mode 100644 index b20a6cb8..00000000 --- a/kmod/src/net.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef _SCOUTFS_NET_H_ -#define _SCOUTFS_NET_H_ - -struct scoutfs_key_buf; -struct scoutfs_segment; - -int scoutfs_net_alloc_inodes(struct super_block *sb); -int scoutfs_net_alloc_segno(struct super_block *sb, u64 *segno); -int scoutfs_net_record_segment(struct super_block *sb, - struct scoutfs_segment *seg, u8 level); -u64 *scoutfs_net_bulk_alloc(struct super_block *sb); - -int scoutfs_net_get_compaction(struct super_block *sb, void *curs); -int scoutfs_net_finish_compaction(struct super_block *sb, void *curs, - void *list); -int scoutfs_net_get_last_seq(struct super_block *sb, u64 *seq); -int scoutfs_net_advance_seq(struct super_block *sb, u64 *seq); - -int scoutfs_net_get_manifest_root(struct super_block *sb, - struct scoutfs_btree_root *root); - -int scoutfs_net_setup(struct super_block *sb); -void scoutfs_net_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 6f36eada..bd5b3855 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -23,6 +23,7 @@ #define _TRACE_SCOUTFS_H #include +#include #include #include "key.h" @@ -420,6 +421,67 @@ DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_free, TP_ARGS(seg) ); +DECLARE_EVENT_CLASS(scoutfs_net_class, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh), + TP_STRUCT__entry( + __field(unsigned int, major) + __field(unsigned int, minor) + __field(u32, name_addr) + __field(u16, name_port) + __field(u32, peer_addr) + __field(u16, peer_port) + __field(u64, id) + __field(u8, type) + __field(u8, status) + __field(u16, data_len) + ), + TP_fast_assign( + __entry->major = MAJOR(sb->s_bdev->bd_dev); + __entry->minor = MINOR(sb->s_bdev->bd_dev); + /* sparse can't handle this cpp nightmare */ + __entry->name_addr = (u32 __force)name->sin_addr.s_addr; + __entry->name_port = be16_to_cpu(name->sin_port); + __entry->peer_addr = (u32 __force)peer->sin_addr.s_addr; + __entry->peer_port = be16_to_cpu(peer->sin_port); + __entry->id = le64_to_cpu(nh->id); + __entry->type = nh->type; + __entry->status = nh->status; + __entry->data_len = le16_to_cpu(nh->data_len); + ), + TP_printk("dev %u:%u %pI4:%u -> %pI4:%u id %llu type %u status %u data_len %u", + __entry->major, __entry->minor, + &__entry->name_addr, __entry->name_port, + &__entry->peer_addr, __entry->peer_port, + __entry->id, __entry->type, __entry->status, + __entry->data_len) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_client_send_request, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_server_recv_request, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_server_send_reply, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + +DEFINE_EVENT(scoutfs_net_class, scoutfs_client_recv_reply, + TP_PROTO(struct super_block *sb, struct sockaddr_in *name, + struct sockaddr_in *peer, struct scoutfs_net_header *nh), + TP_ARGS(sb, name, peer, nh) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/server.c b/kmod/src/server.c new file mode 100644 index 00000000..c02a5d21 --- /dev/null +++ b/kmod/src/server.c @@ -0,0 +1,1051 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "btree.h" +#include "manifest.h" +#include "alloc.h" +#include "seg.h" +#include "compact.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "client.h" +#include "server.h" +#include "sock.h" +#include "endian_swap.h" + +#define SIN_FMT "%pIS:%u" +#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) + +struct server_info { + struct super_block *sb; + + struct workqueue_struct *wq; + struct delayed_work dwork; + + struct mutex mutex; + bool shutting_down; + struct task_struct *listen_task; + struct socket *listen_sock; + + /* request processing coordinates committing manifest and alloc */ + struct rw_semaphore commit_rwsem; + struct llist_head commit_waiters; + struct work_struct commit_work; + + + /* adding new segments can have to wait for compaction */ + wait_queue_head_t compaction_waitq; + + /* server remembers the stable manifest root for clients */ + struct scoutfs_btree_root stable_manifest_root; + + /* server tracks seq use */ + spinlock_t seq_lock; + struct list_head pending_seqs; +}; + +struct server_request { + struct server_connection *conn; + struct work_struct work; + + struct scoutfs_net_header nh; + /* data payload is allocated here, referenced as ->nh.data */ +}; + +struct server_connection { + struct server_info *server; + struct sockaddr_in sockname; + struct sockaddr_in peername; + struct list_head head; + struct socket *sock; + struct work_struct recv_work; + struct mutex send_mutex; +}; + +struct commit_waiter { + struct completion comp; + struct llist_node node; + int ret; +}; + +/* + * This is called while still holding the rwsem that prevents commits so + * that the caller can be sure to be woken by the next commit after they + * queue and release the lock. + * + * This could queue delayed work but we're first trying to have batching + * work by having concurrent modification line up behind a commit in + * flight. Once the commit finishes it'll unlock and hopefully everyone + * will race to make their changes and they'll all be applied by the + * next commit after that. + */ +static void queue_commit_work(struct server_info *server, + struct commit_waiter *cw) +{ + lockdep_assert_held(&server->commit_rwsem); + + cw->ret = 0; + init_completion(&cw->comp); + llist_add(&cw->node, &server->commit_waiters); + queue_work(server->wq, &server->commit_work); +} + +static int wait_for_commit(struct commit_waiter *cw) +{ + wait_for_completion(&cw->comp); + return cw->ret; +} + +/* + * A core function of request processing is to modify the manifest and + * allocator. Often the processing needs to make the modifications + * persistent before replying. We'd like to batch these commits as much + * as is reasonable so that we don't degrade to a few IO round trips per + * request. + * + * Getting that batching right is bound up in the concurrency of request + * processing so a clear way to implement the batched commits is to + * implement commits with a single pending work func like the + * processing. + * + * Processing paths acquire the rwsem for reading while they're making + * multiple dependent changes. When they're done and want it persistent + * they add themselves to the list of waiters and queue the commit work. + * This work runs, acquires the lock to exclude other writers, and + * performs the commit. Readers can run concurrently with these + * commits. + */ +static void scoutfs_server_commit_func(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + commit_work); + struct super_block *sb = server->sb; + struct commit_waiter *cw; + struct commit_waiter *pos; + struct llist_node *node; + int ret; + + down_write(&server->commit_rwsem); + + if (scoutfs_btree_has_dirty(sb)) { + ret = scoutfs_alloc_apply_pending(sb) ?: + scoutfs_btree_write_dirty(sb) ?: + scoutfs_write_dirty_super(sb); + + /* we'd need to loop or something */ + BUG_ON(ret); + + scoutfs_btree_write_complete(sb); + + server->stable_manifest_root = + SCOUTFS_SB(sb)->super.manifest.root; + scoutfs_advance_dirty_super(sb); + } else { + ret = 0; + } + + node = llist_del_all(&server->commit_waiters); + + /* waiters always wait on completion, cw could be free after complete */ + llist_for_each_entry_safe(cw, pos, node, node) { + cw->ret = ret; + complete(&cw->comp); + } + + up_write(&server->commit_rwsem); +} + +/* + * Request processing synchronously sends their reply from within their + * processing work. If this fails the socket is shutdown. + */ +static int send_reply(struct server_connection *conn, u64 id, + u8 type, int error, void *data, unsigned data_len) +{ + struct scoutfs_net_header nh; + struct kvec kv[2]; + unsigned kv_len; + u8 status; + int ret; + + if (WARN_ON_ONCE(error > 0) || WARN_ON_ONCE(data && data_len == 0)) + return -EINVAL; + + kv[0].iov_base = &nh; + kv[0].iov_len = sizeof(nh); + kv_len = 1; + + /* maybe we can have better error communication to clients */ + if (error < 0) { + status = SCOUTFS_NET_STATUS_ERROR; + data = NULL; + data_len = 0; + } else { + status = SCOUTFS_NET_STATUS_SUCCESS; + if (data) { + kv[1].iov_base = data; + kv[1].iov_len = data_len; + kv_len++; + } + } + + nh.id = cpu_to_le64(id); + nh.data_len = cpu_to_le16(data_len); + nh.type = type; + nh.status = status; + + trace_scoutfs_server_send_reply(conn->server->sb, &conn->sockname, + &conn->peername, &nh); + + mutex_lock(&conn->send_mutex); + ret = scoutfs_sock_sendmsg(conn->sock, kv, kv_len); + mutex_unlock(&conn->send_mutex); + + return ret; +} + +void scoutfs_init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last) +{ + scoutfs_key_init(first, net_ment->keys, + le16_to_cpu(net_ment->first_key_len)); + scoutfs_key_init(last, net_ment->keys + + le16_to_cpu(net_ment->first_key_len), + le16_to_cpu(net_ment->last_key_len)); +} + +/* + * Allocate a contiguous manifest entry for communication over the network. + */ +struct scoutfs_net_manifest_entry * +scoutfs_alloc_net_ment(struct scoutfs_manifest_entry *ment) +{ + struct scoutfs_net_manifest_entry *net_ment; + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; + + net_ment = kmalloc(offsetof(struct scoutfs_net_manifest_entry, + keys[ment->first.key_len + + ment->last.key_len]), GFP_NOFS); + if (!net_ment) + return NULL; + + net_ment->segno = cpu_to_le64(ment->segno); + net_ment->seq = cpu_to_le64(ment->seq); + net_ment->first_key_len = cpu_to_le16(ment->first.key_len); + net_ment->last_key_len = cpu_to_le16(ment->last.key_len); + net_ment->level = ment->level; + + scoutfs_init_net_ment_keys(net_ment, &first, &last); + scoutfs_key_copy(&first, &ment->first); + scoutfs_key_copy(&last, &ment->last); + + return net_ment; +} + +/* point a native manifest entry at a contiguous net manifest */ +void scoutfs_init_ment_net_ment(struct scoutfs_manifest_entry *ment, + struct scoutfs_net_manifest_entry *net_ment) +{ + struct scoutfs_key_buf first; + struct scoutfs_key_buf last; + + scoutfs_init_net_ment_keys(net_ment, &first, &last); + scoutfs_key_clone(&ment->first, &first); + scoutfs_key_clone(&ment->last, &last); + + ment->segno = le64_to_cpu(net_ment->segno); + ment->seq = le64_to_cpu(net_ment->seq); + ment->level = net_ment->level; +} + +unsigned scoutfs_net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment) +{ + return offsetof(struct scoutfs_net_manifest_entry, + keys[le16_to_cpu(net_ment->first_key_len) + + le16_to_cpu(net_ment->last_key_len)]); +} + +static int process_alloc_inodes(struct server_connection *conn, + u64 id, u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_inode_alloc ial; + struct commit_waiter cw; + u64 ino; + u64 nr; + int ret; + + if (data_len != 0) { + ret = -EINVAL; + goto out; + } + + down_read(&server->commit_rwsem); + + spin_lock(&sbi->next_ino_lock); + ino = le64_to_cpu(super->next_ino); + nr = min(100000ULL, ~0ULL - ino); + le64_add_cpu(&super->next_ino, nr); + spin_unlock(&sbi->next_ino_lock); + + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + + ial.ino = cpu_to_le64(ino); + ial.nr = cpu_to_le64(nr); + + ret = wait_for_commit(&cw); +out: + return send_reply(conn, id, type, ret, &ial, sizeof(ial)); +} + +static int process_alloc_segno(struct server_connection *conn, + u64 id, u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct commit_waiter cw; + __le64 lesegno; + u64 segno; + int ret; + + if (data_len != 0) { + ret = -EINVAL; + goto out; + } + + down_read(&server->commit_rwsem); + ret = scoutfs_alloc_segno(sb, &segno); + if (ret == 0) { + lesegno = cpu_to_le64(segno); + queue_commit_work(server, &cw); + } + up_read(&server->commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); +out: + return send_reply(conn, id, type, ret, &lesegno, sizeof(lesegno)); +} + +static int process_record_segment(struct server_connection *conn, u64 id, + u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_net_manifest_entry *net_ment; + struct scoutfs_manifest_entry ment; + struct commit_waiter cw; + int ret; + + if (data_len < sizeof(struct scoutfs_net_manifest_entry)) { + ret = -EINVAL; + goto out; + } + + net_ment = data; + + if (data_len != scoutfs_net_ment_bytes(net_ment)) { + ret = -EINVAL; + goto out; + } + +retry: + down_read(&server->commit_rwsem); + scoutfs_manifest_lock(sb); + + if (scoutfs_manifest_level0_full(sb)) { + scoutfs_manifest_unlock(sb); + up_read(&server->commit_rwsem); + /* XXX waits indefinitely? io errors? */ + wait_event(server->compaction_waitq, + !scoutfs_manifest_level0_full(sb)); + goto retry; + } + + scoutfs_init_ment_net_ment(&ment, net_ment); + + ret = scoutfs_manifest_add(sb, &ment); + scoutfs_manifest_unlock(sb); + + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + + if (ret == 0) { + ret = wait_for_commit(&cw); + if (ret == 0) + scoutfs_compact_kick(sb); + } +out: + return send_reply(conn, id, type, ret, NULL, 0); +} + +static int process_bulk_alloc(struct server_connection *conn, u64 id, u8 type, + void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_net_segnos *ns = NULL; + struct commit_waiter cw; + size_t size; + u64 segno; + int ret; + int i; + + if (data_len != 0) { + ret = -EINVAL; + goto out; + } + + size = offsetof(struct scoutfs_net_segnos, + segnos[SCOUTFS_BULK_ALLOC_COUNT]); + ns = kmalloc(size, GFP_NOFS); + if (!ns) { + ret = -ENOMEM; + goto out; + } + + down_read(&server->commit_rwsem); + + ns->nr = cpu_to_le16(SCOUTFS_BULK_ALLOC_COUNT); + for (i = 0; i < SCOUTFS_BULK_ALLOC_COUNT; i++) { + ret = scoutfs_alloc_segno(sb, &segno); + if (ret) { + while (i-- > 0) + scoutfs_alloc_free(sb, + le64_to_cpu(ns->segnos[i])); + break; + } + + ns->segnos[i] = cpu_to_le64(segno); + } + + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); +out: + ret = send_reply(conn, id, type, ret, ns, size); + kfree(ns); + return ret; +} + +struct pending_seq { + struct list_head head; + u64 seq; +}; + +/* + * Give the client the next seq for it to use in items in its + * transaction. They tell us the seq they just used so we can remove it + * from pending tracking and possibly include it in get_last_seq + * replies. + * + * The list walk is O(clients) and the message processing rate goes from + * every committed segment to every sync deadline interval. + * + * XXX The pending seq tracking should be persistent so that it survives + * server failover. + */ +static int process_advance_seq(struct server_connection *conn, u64 id, u8 type, + void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct pending_seq *next_ps; + struct pending_seq *ps; + struct commit_waiter cw; + __le64 * __packed their_seq = data; + __le64 next_seq; + int ret; + + if (data_len != sizeof(__le64)) { + ret = -EINVAL; + goto out; + } + + next_ps = kmalloc(sizeof(struct pending_seq), GFP_NOFS); + if (!next_ps) { + ret = -ENOMEM; + goto out; + } + + down_read(&server->commit_rwsem); + spin_lock(&server->seq_lock); + + list_for_each_entry(ps, &server->pending_seqs, head) { + if (ps->seq == le64_to_cpup(their_seq)) { + list_del_init(&ps->head); + kfree(ps); + break; + } + } + + next_seq = super->next_seq; + le64_add_cpu(&super->next_seq, 1); + + next_ps->seq = le64_to_cpu(next_seq); + list_add_tail(&next_ps->head, &server->pending_seqs); + + spin_unlock(&server->seq_lock); + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); + +out: + return send_reply(conn, id, type, ret, &next_seq, sizeof(next_seq)); +} + +static int process_get_last_seq(struct server_connection *conn, u64 id, + u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct pending_seq *ps; + __le64 last_seq; + int ret; + + if (data_len != 0) { + ret = -EINVAL; + goto out; + } + + spin_lock(&server->seq_lock); + ps = list_first_entry_or_null(&server->pending_seqs, + struct pending_seq, head); + if (ps) { + last_seq = cpu_to_le64(ps->seq - 1); + } else { + last_seq = super->next_seq; + le64_add_cpu(&last_seq, -1ULL); + } + spin_unlock(&server->seq_lock); + ret = 0; +out: + return send_reply(conn, id, type, ret, &last_seq, sizeof(last_seq)); +} + +static int process_get_manifest_root(struct server_connection *conn, u64 id, + u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_btree_root root; + int ret; + + if (data_len == 0) { + scoutfs_manifest_lock(sb); + memcpy(&root, &server->stable_manifest_root, + sizeof(struct scoutfs_btree_root)); + scoutfs_manifest_unlock(sb); + ret = 0; + } else { + ret = -EINVAL; + } + + return send_reply(conn, id, type, ret, &root, sizeof(root)); +} + +/* + * Eventually we're going to have messages that control compaction. + * Each client mount would have long-lived work that sends requests + * which are stuck in processing until there's work to do. They'd get + * their entries, perform the compaction, and send a reply. But we're + * not there yet. + * + * This is a short circuit that's called directly by a work function + * that's only queued on the server. It makes compaction work inside + * the commit consistency mechanics inside request processing and + * demonstrates the moving pieces that we'd need to cut up into a series + * of messages and replies. + * + * The compaction work caller cleans up everything on errors. + */ +int scoutfs_client_get_compaction(struct super_block *sb, void *curs) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct commit_waiter cw; + u64 segno; + int ret = 0; + int nr; + int i; + + down_read(&server->commit_rwsem); + + nr = scoutfs_manifest_next_compact(sb, curs); + if (nr <= 0) { + up_read(&server->commit_rwsem); + return nr; + } + + /* allow for expansion slop from sticky and alignment */ + for (i = 0; i < nr + SCOUTFS_COMPACTION_SLOP; i++) { + ret = scoutfs_alloc_segno(sb, &segno); + if (ret < 0) + break; + scoutfs_compact_add_segno(sb, curs, segno); + } + + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + return ret; +} + +/* + * This is a stub for recording the results of a compaction. We just + * call back into compaction to have it call the manifest and allocator + * updates. + * + * In the future we'd encode the manifest and segnos in requests sent to + * the server who'd update the manifest and allocator in request + * processing. + * + * As we finish a compaction we wait level0 writers if it opened up + * space in level 0. + */ +int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, + void *list) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct commit_waiter cw; + bool level0_was_full; + int ret; + + down_read(&server->commit_rwsem); + + level0_was_full = scoutfs_manifest_level0_full(sb); + + ret = scoutfs_compact_commit(sb, curs, list); + if (ret == 0) { + queue_commit_work(server, &cw); + if (level0_was_full && !scoutfs_manifest_level0_full(sb)) + wake_up(&server->compaction_waitq); + } + + up_read(&server->commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(&cw); + + scoutfs_compact_kick(sb); + + return ret; +} + +typedef int (*process_func_t)(struct server_connection *conn, u64 id, + u8 type, void *data, unsigned data_len); + +/* + * Each request message gets its own concurrent blocking request processing + * context. + */ +static void scoutfs_server_process_func(struct work_struct *work) +{ + struct server_request *req = container_of(work, struct server_request, + work); + struct server_connection *conn = req->conn; + static process_func_t process_funcs[] = { + [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, + [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, + [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, + [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, + [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, + [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, + [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, + }; + struct scoutfs_net_header *nh = &req->nh; + process_func_t func; + int ret; + + if (nh->type < ARRAY_SIZE(process_funcs)) + func = process_funcs[nh->type]; + else + func = NULL; + + if (func) + ret = func(conn, le64_to_cpu(nh->id), nh->type, nh->data, + le16_to_cpu(nh->data_len)); + else + ret = -EINVAL; + + if (ret) + kernel_sock_shutdown(conn->sock, SHUT_RDWR); + + /* process_one_work explicitly allows freeing work in its func */ + kfree(req); +} + +/* + * Always block receiving from the socket. This owns the socket. If + * receive fails this shuts down and frees the socket. + */ +static void scoutfs_server_recv_func(struct work_struct *work) +{ + struct server_connection *conn = container_of(work, + struct server_connection, + recv_work); + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct socket *sock = conn->sock; + struct workqueue_struct *req_wq; + struct scoutfs_net_header nh; + struct server_request *req; + unsigned data_len; + int ret; + + req_wq = alloc_workqueue("scoutfs_server_requests", + WQ_NON_REENTRANT, 0); + if (!req_wq) { + ret = -ENOMEM; + goto out; + } + + for (;;) { + + /* receive the header */ + ret = scoutfs_sock_recvmsg(sock, &nh, sizeof(nh)); + if (ret) + break; + + trace_scoutfs_server_recv_request(conn->server->sb, + &conn->sockname, + &conn->peername, &nh); + + /* XXX verify data_len isn't insane */ + /* XXX test for bad messages */ + data_len = le16_to_cpu(nh.data_len); + + req = kmalloc(sizeof(struct server_request) + data_len, + GFP_NOFS); + if (!req) { + ret = -ENOMEM; + break; + } + + ret = scoutfs_sock_recvmsg(sock, req->nh.data, data_len); + if (ret) + break; + + req->conn = conn; + INIT_WORK(&req->work, scoutfs_server_process_func); + req->nh = nh; + + queue_work(req_wq, &req->work); + /* req is freed by its work func */ + req = NULL; + } + +out: + scoutfs_info(sb, "server closing "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&conn->peername), SIN_ARG(&conn->sockname)); + + /* make sure reply sending returns */ + kernel_sock_shutdown(conn->sock, SHUT_RDWR); + + /* wait for processing work to drain */ + if (req_wq) { + drain_workqueue(req_wq); + destroy_workqueue(req_wq); + } + + sock_release(conn->sock); + + /* process_one_work explicitly allows freeing work in its func */ + mutex_lock(&server->mutex); + list_del_init(&conn->head); + kfree(conn); + smp_mb(); + wake_up_process(server->listen_task); + mutex_unlock(&server->mutex); +} + +/* + * This relies on the caller having read the current super and advanced + * its seq so that it's dirty. This will go away when we communicate + * the server address in a lock lvb. + */ +static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + super->server_addr.addr = be32_to_le32(sin->sin_addr.s_addr); + super->server_addr.port = be16_to_le16(sin->sin_port); + + return scoutfs_write_dirty_super(sb); +} + +static bool barrier_list_empty_careful(struct list_head *list) +{ + /* store caller's task state before loading wake condition */ + smp_mb(); + + return list_empty_careful(list); +} + +/* + * This work is always running or has a delayed timer set while a super + * is mounted. It tries to grab the lock to become the server. If it + * succeeds it publishes its address and accepts connections. If + * anything goes wrong it releases the lock and sets a timer to try to + * become the server all over again. + */ +static void scoutfs_server_func(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + dwork.work); + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + static struct sockaddr_in zeros = {0,}; + struct socket *new_sock; + struct socket *sock = NULL; + struct scoutfs_lock *lock; + struct server_connection *conn; + struct server_connection *conn_tmp; + struct pending_seq *ps; + struct pending_seq *ps_tmp; + DECLARE_WAIT_QUEUE_HEAD(waitq); + struct sockaddr_in sin; + LIST_HEAD(conn_list); + int addrlen; + int optval; + int ret; + + init_waitqueue_head(&waitq); + + /* lock attempt will return -ESHUTDOWN once we should not queue */ + ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ~0ULL, &lock); + if (ret) + goto out; + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = le32_to_be32(sbi->opts.listen_addr.addr); + sin.sin_port = le16_to_be16(sbi->opts.listen_addr.port); + + optval = 1; + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock) ?: + kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, + (char *)&optval, sizeof(optval)) ?: + kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + addrlen = sizeof(sin); + ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen) ?: + kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); + if (ret) + goto out; + + ret = kernel_listen(sock, 255); + if (ret) + goto out; + + /* publish the address for clients to connect to */ + ret = scoutfs_read_supers(sb, super); + if (ret) + goto out; + + scoutfs_advance_dirty_super(sb); + ret = write_server_addr(sb, &sin); + if (ret) + goto out; + + /* either see shutting down or they'll shutdown our sock */ + mutex_lock(&server->mutex); + server->listen_task = current; + server->listen_sock = sock; + if (server->shutting_down) + ret = -ESHUTDOWN; + mutex_unlock(&server->mutex); + if (ret) + goto out; + + /* finally start up the server subsystems before accepting */ + ret = scoutfs_btree_setup(sb) ?: + scoutfs_manifest_setup(sb) ?: + scoutfs_alloc_setup(sb) ?: + scoutfs_compact_setup(sb); + if (ret) + goto shutdown; + + scoutfs_advance_dirty_super(sb); + server->stable_manifest_root = super->manifest.root; + + scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); + + for (;;) { + ret = kernel_accept(sock, &new_sock, 0); + if (ret < 0) + break; + + conn = kmalloc(sizeof(struct server_connection), GFP_NOFS); + if (!conn) { + sock_release(new_sock); + ret = -ENOMEM; + continue; + } + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(new_sock, + (struct sockaddr *)&conn->sockname, + &addrlen) ?: + kernel_getpeername(new_sock, + (struct sockaddr *)&conn->peername, + &addrlen); + if (ret) { + sock_release(new_sock); + continue; + } + + /* + * XXX yeah, ok, killing the sock and accepting a new + * one is racey. think about that in all the code. Are + * we destroying a resource to shutdown that the thing + * we're canceling creates? + */ + + conn->server = server; + conn->sock = new_sock; + mutex_init(&conn->send_mutex); + + scoutfs_info(sb, "server accepted "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&conn->peername), + SIN_ARG(&conn->sockname)); + + /* recv work owns the conn once its in the list */ + mutex_lock(&server->mutex); + list_add(&conn->head, &conn_list); + mutex_unlock(&server->mutex); + + INIT_WORK(&conn->recv_work, scoutfs_server_recv_func); + queue_work(server->wq, &conn->recv_work); + } + + /* shutdown send and recv on all accepted sockets */ + mutex_lock(&server->mutex); + list_for_each_entry_safe(conn, conn_tmp, &conn_list, head) + kernel_sock_shutdown(conn->sock, SHUT_RDWR); + mutex_unlock(&server->mutex); + + /* wait for all recv work to finish and free connections */ + wait_event(waitq, barrier_list_empty_careful(&conn_list)); + + scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); + +shutdown: + + /* shut down all the server subsystems */ + scoutfs_compact_destroy(sb); + scoutfs_alloc_destroy(sb); + scoutfs_manifest_destroy(sb); + scoutfs_btree_destroy(sb); + + /* XXX these should be persistent and reclaimed during recovery */ + list_for_each_entry_safe(ps, ps_tmp, &server->pending_seqs, head) { + list_del_init(&ps->head); + kfree(ps); + } + + write_server_addr(sb, &zeros); + +out: + if (sock) + sock_release(sock); + + /* always requeues, cancel_delayed_work_sync cancels on shutdown */ + queue_delayed_work(server->wq, &server->dwork, HZ / 2); +} + +int scoutfs_server_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct server_info *server; + + server = kzalloc(sizeof(struct server_info), GFP_KERNEL); + if (!server) + return -ENOMEM; + + server->sb = sb; + INIT_DELAYED_WORK(&server->dwork, scoutfs_server_func); + mutex_init(&server->mutex); + init_rwsem(&server->commit_rwsem); + init_llist_head(&server->commit_waiters); + INIT_WORK(&server->commit_work, scoutfs_server_commit_func); + init_waitqueue_head(&server->compaction_waitq); + spin_lock_init(&server->seq_lock); + INIT_LIST_HEAD(&server->pending_seqs); + + server->wq = alloc_workqueue("scoutfs_server", WQ_NON_REENTRANT, 0); + if (!server->wq) { + kfree(server); + return -ENOMEM; + } + + queue_delayed_work(server->wq, &server->dwork, 0); + + sbi->server_info = server; + return 0; +} + +void scoutfs_server_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct server_info *server = sbi->server_info; + + if (server) { + /* break server thread out of blocking socket calls */ + mutex_lock(&server->mutex); + server->shutting_down = true; + if (server->listen_sock) + kernel_sock_shutdown(server->listen_sock, SHUT_RDWR); + mutex_unlock(&server->mutex); + + /* wait for server work to wait for everything to shut down */ + cancel_delayed_work_sync(&server->dwork); + destroy_workqueue(server->wq); + + kfree(server); + sbi->server_info = NULL; + } +} diff --git a/kmod/src/server.h b/kmod/src/server.h new file mode 100644 index 00000000..8cb7c05c --- /dev/null +++ b/kmod/src/server.h @@ -0,0 +1,20 @@ +#ifndef _SCOUTFS_SERVER_H_ +#define _SCOUTFS_SERVER_H_ + +void scoutfs_init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, + struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last); +struct scoutfs_net_manifest_entry * +scoutfs_alloc_net_ment(struct scoutfs_manifest_entry *ment); +void scoutfs_init_ment_net_ment(struct scoutfs_manifest_entry *ment, + struct scoutfs_net_manifest_entry *net_ment); +unsigned scoutfs_net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment); + +int scoutfs_client_get_compaction(struct super_block *sb, void *curs); +int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, + void *list); + +int scoutfs_server_setup(struct super_block *sb); +void scoutfs_server_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/sock.c b/kmod/src/sock.c new file mode 100644 index 00000000..4783310a --- /dev/null +++ b/kmod/src/sock.c @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sock.h" + +/* + * Some quick socket helper wrappers. + */ + +static struct kvec *kvec_advance(struct kvec *kv, unsigned *kv_len, + unsigned bytes) +{ + while (*kv_len && bytes) { + if (kv->iov_len <= bytes) { + bytes -= kv->iov_len; + kv++; + (*kv_len)--; + } else { + kv->iov_base += bytes; + kv->iov_len -= bytes; + bytes = 0; + } + } + + return kv; +} + +/* + * This can modify the kvec as it modifies the vec to continue after + * partial sends. + */ +int scoutfs_sock_sendmsg(struct socket *sock, struct kvec *kv, unsigned kv_len) +{ + struct msghdr msg; + int ret; + + while (kv_len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)kv; + msg.msg_iovlen = kv_len; + msg.msg_flags = MSG_NOSIGNAL; + + ret = kernel_sendmsg(sock, &msg, kv, kv_len, + iov_length((struct iovec *)kv, kv_len)); + if (ret <= 0) + return -ECONNABORTED; + + kv = kvec_advance(kv, &kv_len, ret); + } + + return 0; +} + +int scoutfs_sock_recvmsg(struct socket *sock, void *buf, unsigned len) +{ + struct msghdr msg; + struct kvec kv; + int ret; + + while (len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)&kv; + msg.msg_iovlen = 1; + msg.msg_flags = MSG_NOSIGNAL; + kv.iov_base = buf; + kv.iov_len = len; + + ret = kernel_recvmsg(sock, &msg, &kv, 1, len, msg.msg_flags); + if (ret <= 0) + return -ECONNABORTED; + + len -= ret; + buf += ret; + } + + return 0; +} diff --git a/kmod/src/sock.h b/kmod/src/sock.h new file mode 100644 index 00000000..5b61bea0 --- /dev/null +++ b/kmod/src/sock.h @@ -0,0 +1,7 @@ +#ifndef _SCOUTFS_SOCK_H_ +#define _SCOUTFS_SOCK_H_ + +int scoutfs_sock_recvmsg(struct socket *sock, void *buf, unsigned len); +int scoutfs_sock_sendmsg(struct socket *sock, struct kvec *kv, unsigned kv_len); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index 3fd6de5e..a3704ce4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -36,7 +36,8 @@ #include "compact.h" #include "data.h" #include "lock.h" -#include "net.h" +#include "client.h" +#include "server.h" #include "options.h" #include "scoutfs_trace.h" @@ -241,27 +242,49 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: - scoutfs_lock_setup(sb) ?: - scoutfs_net_setup(sb); + scoutfs_lock_setup(sb); if (ret) return ret; + /* + * The server is a bit magical because it can try to read the + * device in async work context. Once we return an error from + * here the kernel starts tearing down the mount and it isn't + * safe to do IO. So we shut the server down before returning + * an error. + * + * But we still want to start the server before the client to + * help single mounts come up without passing through connection + * timeouts. + */ + ret = scoutfs_server_setup(sb) ?: + scoutfs_client_setup(sb); + if (ret) + goto out; + inode = scoutfs_iget(sb, SCOUTFS_ROOT_INO); - if (IS_ERR(inode)) - return PTR_ERR(inode); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } sb->s_root = d_make_root(inode); - if (!sb->s_root) - return -ENOMEM; + if (!sb->s_root) { + ret = -ENOMEM; + goto out; + } - ret = scoutfs_net_advance_seq(sb, &sbi->trans_seq); + ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); if (ret) - return ret; + goto out; scoutfs_trans_restart_sync_deadline(sb); // scoutfs_scan_orphans(sb); - - return 0; + ret = 0; +out: + if (ret) + scoutfs_server_destroy(sb); + return ret; } static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, @@ -283,14 +306,15 @@ static void scoutfs_kill_sb(struct super_block *sb) sync_filesystem(sb); scoutfs_lock_shutdown(sb); - scoutfs_net_destroy(sb); + scoutfs_server_destroy(sb); } kill_block_super(sb); if (sbi) { scoutfs_lock_destroy(sb); - scoutfs_net_destroy(sb); + scoutfs_client_destroy(sb); + scoutfs_server_destroy(sb); scoutfs_shutdown_trans(sb); scoutfs_data_destroy(sb); scoutfs_inode_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 95285080..c8e8a9a6 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -15,7 +15,8 @@ struct compact_info; struct data_info; struct trans_info; struct lock_info; -struct net_info; +struct client_info; +struct server_info; struct inode_sb_info; struct btree_info; @@ -51,7 +52,8 @@ struct scoutfs_sb_info { struct trans_info *trans_info; struct lock_info *lock_info; - struct net_info *net_info; + struct client_info *client_info; + struct server_info *server_info; /* $sysfs/fs/scoutfs/$id/ */ struct kset *kset; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index bfa82f57..e630d0fc 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -26,7 +26,7 @@ #include "manifest.h" #include "seg.h" #include "counters.h" -#include "net.h" +#include "client.h" #include "inode.h" #include "scoutfs_trace.h" @@ -130,14 +130,14 @@ void scoutfs_trans_write_func(struct work_struct *work) * on crashes between us and the server. */ ret = scoutfs_inode_walk_writeback(sb, true) ?: - scoutfs_net_alloc_segno(sb, &segno) ?: + scoutfs_client_alloc_segno(sb, &segno) ?: scoutfs_seg_alloc(sb, segno, &seg) ?: scoutfs_item_dirty_seg(sb, seg) ?: scoutfs_seg_submit_write(sb, seg, &comp) ?: scoutfs_inode_walk_writeback(sb, false) ?: scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_net_record_segment(sb, seg, 0) ?: - scoutfs_net_advance_seq(sb, &sbi->trans_seq); + scoutfs_client_record_segment(sb, seg, 0) ?: + scoutfs_client_advance_seq(sb, &sbi->trans_seq); scoutfs_seg_put(seg); if (ret) goto out; @@ -152,7 +152,7 @@ void scoutfs_trans_write_func(struct work_struct *work) * seq indices but doesn't send a message for every sync * syscall. */ - ret = scoutfs_net_advance_seq(sb, &sbi->trans_seq); + ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); } out: diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 6f52553e..fcf0d376 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -1,7 +1,6 @@ #ifndef _SCOUTFS_TRANS_H_ #define _SCOUTFS_TRANS_H_ -#include "net.h" #include "count.h" void scoutfs_trans_write_func(struct work_struct *work); From cdb58a967a1224ddff64943fb245af69b0a25e06 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Aug 2017 13:41:59 -0700 Subject: [PATCH 354/920] scoutfs: give module fs scoutfs alias Use MODULE_ALIAS_FS() to register the "scoutfs" fs alias so that modprobe can find the module if it's installed and visible to depmod. We don't yet have clever enough xfstests to mess around with modules. I manually verified this by installing the module in /lib/modules and trying mount -t scoutfs before and after the change. Signed-off-by: Zach Brown --- kmod/src/super.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index a3704ce4..5e0604da 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -334,6 +334,7 @@ static struct file_system_type scoutfs_fs_type = { .kill_sb = scoutfs_kill_sb, .fs_flags = FS_REQUIRES_DEV, }; +MODULE_ALIAS_FS("scoutfs"); /* safe to call at any failure point in _init */ static void teardown_module(void) From 1398b2316d923915376b90f3f25a10066b67494a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 Aug 2017 15:52:18 -0700 Subject: [PATCH 355/920] scoutfs: clean up racey inode index updates The updating of the inode index items was racey. It loaded the inode values, updated the items, loaded the fields again, and then stored the fields in the inode info. All without locking. Concurrent attempts could get the fields scrambled and racing with other paths that update the fields could get the items and inode info out of sync. This fixes up the two races by only reading the inode fields once and performing the multi-stage update under a mutex. We add a new lock to avoid ordering problems with trying to add an existing lock at these points in the locking heirarchy. We specifically use a mutex because the item functions can block. Now the inode index field update just has to safely race with concurrent access to the fields. This was found by generic/037 once getattr started refreshing the inode. It now passes again. Signed-off-by: Zach Brown --- kmod/src/inode.c | 84 ++++++++++++++++++++++++++++++------------------ kmod/src/inode.h | 8 +++++ 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index ad02f4d1..614f3b87 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -70,6 +70,7 @@ static void scoutfs_inode_ctor(void *obj) { struct scoutfs_inode_info *ci = obj; + mutex_init(&ci->item_mutex); seqcount_init(&ci->seqcount); ci->staging = false; init_rwsem(&ci->xattr_rwsem); @@ -187,16 +188,26 @@ static void set_inode_ops(struct inode *inode) mapping_set_gfp_mask(inode->i_mapping, GFP_USER); } -static void set_item_info(struct inode *inode) +/* + * The caller has ensured that the fields in the incoming scoutfs inode + * reflect both the inode item and the inode index items. This happens + * when reading, refreshing, or updating the inodes. We set the inode + * info fields to match so that next time we try to update the inode we + * can tell which fields have changed. + */ +static void set_item_info(struct scoutfs_inode_info *si, + struct scoutfs_inode *sinode) { - struct scoutfs_inode_info *si = SCOUTFS_I(inode); + BUG_ON(!mutex_is_locked(&si->item_mutex)); si->have_item = true; - si->item_size = i_size_read(inode); - si->item_ctime = inode->i_ctime; - si->item_mtime = inode->i_mtime; - si->item_meta_seq = scoutfs_inode_meta_seq(inode); - si->item_data_seq = scoutfs_inode_data_seq(inode); + si->item_size = le64_to_cpu(sinode->size); + si->item_ctime.tv_sec = le64_to_cpu(sinode->ctime.sec); + si->item_ctime.tv_nsec = le32_to_cpu(sinode->ctime.nsec); + si->item_mtime.tv_sec = le64_to_cpu(sinode->mtime.sec); + si->item_mtime.tv_nsec = le32_to_cpu(sinode->mtime.nsec); + si->item_meta_seq = le64_to_cpu(sinode->meta_seq); + si->item_data_seq = le64_to_cpu(sinode->data_seq); } static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) @@ -221,11 +232,12 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->data_version = le64_to_cpu(cinode->data_version); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); - set_item_info(inode); + set_item_info(ci, cinode); } static int refresh_inode(struct inode *inode, struct scoutfs_lock *lock) { + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_key_buf key; struct scoutfs_inode_key ikey; @@ -236,9 +248,11 @@ static int refresh_inode(struct inode *inode, struct scoutfs_lock *lock) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + mutex_lock(&si->item_mutex); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), lock->end); if (ret == 0) load_inode(inode, &sinode); + mutex_unlock(&si->item_mutex); return ret; } @@ -494,11 +508,10 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end) * - make sure deletion items safely vanish w/o finding existing item * - ... error handling :( */ -static int update_index(struct inode *inode, u8 type, u64 now_major, - u32 now_minor, u64 then_major, u32 then_minor) +static int update_index(struct super_block *sb, struct scoutfs_inode_info *si, + u64 ino, u8 type, u64 now_major, u32 now_minor, + u64 then_major, u32 then_minor) { - struct scoutfs_inode_info *si = SCOUTFS_I(inode); - struct super_block *sb = inode->i_sb; struct scoutfs_inode_index_key ins_ikey; struct scoutfs_inode_index_key del_ikey; struct scoutfs_key_buf ins; @@ -507,8 +520,8 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, int err; trace_printk("ino %llu have %u now %llu.%u then %llu.%u \n", - scoutfs_ino(inode), si->have_item, - now_major, now_minor, then_major, then_minor); + ino, si->have_item, now_major, now_minor, then_major, + then_minor); if (si->have_item && now_major == then_major && now_minor == then_minor) return 0; @@ -517,7 +530,7 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, ins_ikey.type = type; ins_ikey.major = cpu_to_be64(now_major); ins_ikey.minor = cpu_to_be32(now_minor); - ins_ikey.ino = cpu_to_be64(scoutfs_ino(inode)); + ins_ikey.ino = cpu_to_be64(ino); scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); ret = scoutfs_item_create(sb, &ins, NULL); @@ -528,7 +541,7 @@ static int update_index(struct inode *inode, u8 type, u64 now_major, del_ikey.type = type; del_ikey.major = cpu_to_be64(then_major); del_ikey.minor = cpu_to_be32(then_minor); - del_ikey.ino = cpu_to_be64(scoutfs_ino(inode)); + del_ikey.ino = cpu_to_be64(ino); scoutfs_key_init(&del, &del_ikey, sizeof(del_ikey)); ret = scoutfs_item_delete(sb, &del, NULL); @@ -553,6 +566,7 @@ void scoutfs_update_inode_item(struct inode *inode) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); struct scoutfs_inode_key ikey; struct scoutfs_key_buf key; struct scoutfs_inode sinode; @@ -560,39 +574,45 @@ void scoutfs_update_inode_item(struct inode *inode) int ret; int err; + mutex_lock(&si->item_mutex); + /* set the meta version once per trans for any inode updates */ scoutfs_inode_set_meta_seq(inode); - ret = update_index(inode, SCOUTFS_INODE_INDEX_CTIME_TYPE, - inode->i_ctime.tv_sec, inode->i_ctime.tv_nsec, + /* only race with other inode field stores once */ + store_inode(&sinode, inode); + + ret = update_index(sb, si, ino, SCOUTFS_INODE_INDEX_CTIME_TYPE, + le64_to_cpu(sinode.ctime.sec), + le32_to_cpu(sinode.ctime.nsec), si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: - update_index(inode, SCOUTFS_INODE_INDEX_MTIME_TYPE, - inode->i_mtime.tv_sec, inode->i_mtime.tv_nsec, + update_index(sb, si, ino, SCOUTFS_INODE_INDEX_MTIME_TYPE, + le64_to_cpu(sinode.mtime.sec), + le32_to_cpu(sinode.mtime.nsec), si->item_mtime.tv_sec, si->item_mtime.tv_nsec) ?: - update_index(inode, SCOUTFS_INODE_INDEX_SIZE_TYPE, - i_size_read(inode), 0, si->item_size, 0) ?: - update_index(inode, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, - scoutfs_inode_meta_seq(inode), 0, + update_index(sb, si, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, + le64_to_cpu(sinode.size), 0, si->item_size, 0) ?: + update_index(sb, si, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode.meta_seq), 0, si->item_meta_seq, 0) ?: - update_index(inode, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, - scoutfs_inode_data_seq(inode), 0, + update_index(sb, si, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode.data_seq), 0, si->item_data_seq, 0); BUG_ON(ret); - store_inode(&sinode, inode); - - scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); + scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); err = scoutfs_item_update(sb, &key, val, NULL); if (err) { - scoutfs_err(sb, "inode %llu update err %d", - scoutfs_ino(inode), err); + scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); } - set_item_info(inode); + set_item_info(si, &sinode); trace_scoutfs_update_inode(inode); + + mutex_unlock(&si->item_mutex); } /* diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 73ae1ce7..182779ad 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -10,6 +10,14 @@ struct scoutfs_inode_info { u64 meta_seq; u64 data_seq; u64 data_version; + + /* + * The in-memory item info caches the current index item values + * so that we can decide to update them with comparisons instead + * of by maintaining state that tracks the inode differing from + * the item. The "item_" prefix is a bit clumsy :/. + */ + struct mutex item_mutex; bool have_item; u64 item_size; struct timespec item_ctime; From 3768e3c41cbead360d4a40ea514b2bbb296ee841 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 3 Aug 2017 15:56:31 -0700 Subject: [PATCH 356/920] scoutfs: don't add dirs to data_seq index Directories were getting added to the data_seq index. It might have looked like they weren't because their data_seqs were always 0 but when inodes are created they don't have 'have_item' set so all the fields are added regardless of their current value. We'd rather not have to wade their directories when looking for regular file data in the data_seq index so let's explicitly test for regular files when updating the data_seq index items. Signed-off-by: Zach Brown --- kmod/src/inode.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 614f3b87..ab4be9d0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -594,10 +594,12 @@ void scoutfs_update_inode_item(struct inode *inode) le64_to_cpu(sinode.size), 0, si->item_size, 0) ?: update_index(sb, si, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode.meta_seq), 0, - si->item_meta_seq, 0) ?: - update_index(sb, si, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, - le64_to_cpu(sinode.data_seq), 0, - si->item_data_seq, 0); + si->item_meta_seq, 0); + if (ret == 0 && S_ISREG(inode->i_mode)) + ret = update_index(sb, si, ino, + SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode.data_seq), 0, + si->item_data_seq, 0); BUG_ON(ret); scoutfs_inode_init_key(&key, &ikey, ino); From ba7bde30fcd10b78fc8134965d1a9e68a6f3e91a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Aug 2017 09:18:54 -0700 Subject: [PATCH 357/920] scoutfs: delete inode index items Delete inode index items when deleting all the items associated with an inode after its been unlinked and had all its references dropped. The index items should always match the fields in the inode item so we read it to determine the index items that should be deleted, regardless of if we have the vfs inode cached or not. We take the opportunity to collapse the two callers of item deletion which looked up the inode into item deletion so that it can use the inode fields. The deletion of index items is partially verified by an inode index test in xfstests which makes sure that unlinked files are no longer present in the index. Signed-off-by: Zach Brown --- kmod/src/inode.c | 151 +++++++++++++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 59 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index ab4be9d0..4a291e74 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -617,6 +617,59 @@ void scoutfs_update_inode_item(struct inode *inode) mutex_unlock(&si->item_mutex); } +/* this is called on final inode cleanup so enoent is fine */ +static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, + u32 minor) +{ + struct scoutfs_inode_index_key ikey; + struct scoutfs_key_buf key; + int ret; + + ikey.zone = SCOUTFS_INODE_INDEX_ZONE; + ikey.type = type; + ikey.major = cpu_to_be64(major); + ikey.minor = cpu_to_be32(minor); + ikey.ino = cpu_to_be64(ino); + scoutfs_key_init(&key, &ikey, sizeof(ikey)); + + /* XXX would be deletion under CW that doesn't need to read */ + ret = scoutfs_item_delete(sb, &key, NULL); + if (ret == -ENOENT) + ret = 0; + return ret; +} + +/* + * Remove all the inode's index items. The caller has ensured that + * there are no more active users of the inode. This can be racing with + * users of the inode index items. Once we can use them we'll get CW + * locks around the index items to invalidate remote caches. Racing + * users of the index items already have to deal with the possibility + * that the inodes returned by the index queries can go out of sync by + * the time they get to it, including being deleted. + */ +static int remove_index_items(struct super_block *sb, u64 ino, + struct scoutfs_inode *sinode) +{ + umode_t mode = le32_to_cpu(sinode->mode); + int ret; + + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_CTIME_TYPE, + le64_to_cpu(sinode->ctime.sec), + le32_to_cpu(sinode->ctime.nsec)) ?: + remove_index(sb, ino, SCOUTFS_INODE_INDEX_MTIME_TYPE, + le64_to_cpu(sinode->mtime.sec), + le32_to_cpu(sinode->mtime.nsec)) ?: + remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, + le64_to_cpu(sinode->size), 0) ?: + remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0); + if (ret == 0 && S_ISREG(mode)) + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0); + return ret; +} + /* * A quick atomic sample of the last inode number that's been allocated. */ @@ -807,13 +860,42 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) return ret; } -static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, - u64 ino, umode_t mode) +/* + * Remove all the items associated with a given inode. This is only + * called once nlink has dropped to zero so we don't have to worry about + * dirents referencing the inode or link backrefs. Dropping nlink to 0 + * also created an orphan item. That orphan item will continue + * triggering attempts to finish previous partial deletion until all + * deletion is complete and the orphan item is removed. + */ +static int delete_inode_items(struct super_block *sb, u64 ino) { + struct scoutfs_inode_key ikey; + struct scoutfs_inode sinode; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); DECLARE_ITEM_COUNT(cnt); bool release = false; + umode_t mode; int ret; + scoutfs_inode_init_key(&key, &ikey, ino); + scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + return ret; + } + + /* XXX corruption, inode probably won't be freed without repair */ + if (le32_to_cpu(sinode.nlink)) { + scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); + return -EIO; + } + + mode = le32_to_cpu(sinode.mode); trace_delete_inode(sb, ino, mode); /* XXX this is obviously not done yet :) */ @@ -823,6 +905,11 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, goto out; release = true; + /* first remove index items to try to avoid indexing partial deletion */ + ret = remove_index_items(sb, ino, &sinode); + if (ret) + goto out; + #if 0 ret = scoutfs_xattr_drop(sb, ino); if (ret) @@ -836,7 +923,7 @@ static int __delete_inode(struct super_block *sb, struct scoutfs_key_buf *key, goto out; #endif - ret = scoutfs_item_delete(sb, key, NULL); + ret = scoutfs_item_delete(sb, &key, NULL); if (ret) goto out; @@ -847,34 +934,6 @@ out: return ret; } -/* - * Remove all the items associated with a given inode. - */ -static void delete_inode(struct super_block *sb, u64 ino) -{ - struct scoutfs_inode sinode; - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); - umode_t mode; - int ret; - - /* sample the inode mode, XXX don't need to copy whole thing here */ - scoutfs_inode_init_key(&key, &ikey, ino); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); - if (ret < 0) - goto out; - - mode = le32_to_cpu(sinode.mode); - - ret = __delete_inode(sb, &key, ino, mode); -out: - if (ret) - trace_printk("drop items failed ret %d ino %llu\n", ret, ino); -} - /* * iput_final has already written out the dirty pages to the inode * before we get here. We're left with a clean inode that we have to @@ -892,7 +951,7 @@ void scoutfs_evict_inode(struct inode *inode) truncate_inode_pages_final(&inode->i_data); if (inode->i_nlink == 0) - delete_inode(inode->i_sb, scoutfs_ino(inode)); + delete_inode_items(inode->i_sb, scoutfs_ino(inode)); clear: clear_inode(inode); } @@ -906,32 +965,6 @@ int scoutfs_drop_inode(struct inode *inode) return ret; } -static int process_orphaned_inode(struct super_block *sb, u64 ino) -{ - struct scoutfs_inode_key ikey; - struct scoutfs_inode sinode; - struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); - int ret; - - scoutfs_inode_init_key(&key, &ikey, ino); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - return ret; - } - - if (le32_to_cpu(sinode.nlink) == 0) - __delete_inode(sb, &key, ino, le32_to_cpu(sinode.mode)); - else - scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); - - return ret; -} - /* * Find orphan items and process each one. * @@ -964,7 +997,7 @@ int scoutfs_scan_orphans(struct super_block *sb) if (ret < 0) goto out; - ret = process_orphaned_inode(sb, be64_to_cpu(okey.ino)); + ret = delete_inode_items(sb, be64_to_cpu(okey.ino)); if (ret && ret != -ENOENT && !err) err = ret; From 87ab27beb119d394766bd7c902ab71ec8c00f148 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Aug 2017 14:54:30 -0700 Subject: [PATCH 358/920] scoutfs: add statfs network message The ->statfs method was still using the super_block in the super_info that was read during mount. This will get progressively more out of date. We add a network message to ask the server for the current fields that impact statfs. This is always racy and the fields are mostly nonsense, but we try our best. Signed-off-by: Zach Brown --- kmod/src/client.c | 9 +++++++++ kmod/src/client.h | 2 ++ kmod/src/format.h | 8 ++++++++ kmod/src/server.c | 34 ++++++++++++++++++++++++++++++++++ kmod/src/super.c | 26 ++++++++++++++++++-------- 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 49293be1..628cca99 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -681,6 +681,15 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, NULL, 0, root, sizeof(struct scoutfs_btree_root)); } +int scoutfs_client_statfs(struct super_block *sb, + struct scoutfs_net_statfs *nstatfs) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return client_request(client, SCOUTFS_NET_STATFS, NULL, 0, nstatfs, + sizeof(struct scoutfs_net_statfs)); +} + int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/client.h b/kmod/src/client.h index 59b7b151..71c8c4eb 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -10,6 +10,8 @@ int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_manifest_root(struct super_block *sb, struct scoutfs_btree_root *root); +int scoutfs_client_statfs(struct super_block *sb, + struct scoutfs_net_statfs *nstatfs); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 6c37816d..b532cccc 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -571,6 +571,13 @@ struct scoutfs_net_segnos { __le64 segnos[0]; } __packed; +struct scoutfs_net_statfs { + __le64 total_segs; /* total segments in device */ + __le64 next_ino; /* next unused inode number */ + __le64 bfree; /* total free small blocks */ + __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ +} __packed; + /* XXX eventually we'll have net compaction and will need agents to agree */ /* one upper segment and fanout lower segments */ @@ -589,6 +596,7 @@ enum { SCOUTFS_NET_ADVANCE_SEQ, SCOUTFS_NET_GET_LAST_SEQ, SCOUTFS_NET_GET_MANIFEST_ROOT, + SCOUTFS_NET_STATFS, SCOUTFS_NET_UNKNOWN, }; diff --git a/kmod/src/server.c b/kmod/src/server.c index c02a5d21..427a6463 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -581,6 +581,39 @@ static int process_get_manifest_root(struct server_connection *conn, u64 id, return send_reply(conn, id, type, ret, &root, sizeof(root)); } +/* + * Sample the super stats that the client wants for statfs by serializing + * with each component. + */ +static int process_statfs(struct server_connection *conn, u64 id, u8 type, + void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_statfs nstatfs; + int ret; + + if (data_len == 0) { + /* uuid and total_segs are constant, so far */ + memcpy(nstatfs.uuid, super->uuid, sizeof(nstatfs.uuid)); + nstatfs.total_segs = super->total_segs; + + spin_lock(&sbi->next_ino_lock); + nstatfs.next_ino = super->next_ino; + spin_unlock(&sbi->next_ino_lock); + + /* alloc locks the bfree calculation */ + nstatfs.bfree = cpu_to_le64(scoutfs_alloc_bfree(sb)); + ret = 0; + } else { + ret = -EINVAL; + } + + return send_reply(conn, id, type, ret, &nstatfs, sizeof(nstatfs)); +} + /* * Eventually we're going to have messages that control compaction. * Each client mount would have long-lived work that sends requests @@ -692,6 +725,7 @@ static void scoutfs_server_process_func(struct work_struct *work) [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, + [SCOUTFS_NET_STATFS] = process_statfs, }; struct scoutfs_net_header *nh = &req->nh; process_func_t func; diff --git a/kmod/src/super.c b/kmod/src/super.c index 5e0604da..34c0835a 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -44,6 +44,11 @@ static struct kset *scoutfs_kset; /* + * Ask the server for the current statfs fields. The message is very + * cheap so we're not worrying about spinning in statfs flooding the + * server with requests. We can add a cache and stale results if that + * becomes a problem. + * * We fake the number of free inodes value by assuming that we can fill * free blocks with a certain number of inodes. We then the number of * current inodes to that free count to determine the total possible @@ -55,20 +60,25 @@ static struct kset *scoutfs_kset; static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) { struct super_block *sb = dentry->d_inode->i_sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - __le32 * __packed uuid = (void *)super->uuid; + struct scoutfs_net_statfs nstatfs; + __le32 * __packed uuid; + int ret; - kst->f_bfree = scoutfs_alloc_bfree(sb); + ret = scoutfs_client_statfs(sb, &nstatfs); + if (ret) + return ret; + + kst->f_bfree = le64_to_cpu(nstatfs.bfree); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; - kst->f_blocks = le64_to_cpu(super->total_segs) * SCOUTFS_SEGMENT_BLOCKS; + kst->f_blocks = le64_to_cpu(nstatfs.total_segs) * + SCOUTFS_SEGMENT_BLOCKS; kst->f_bavail = kst->f_bfree; - kst->f_ffree = kst->f_bfree * 17; - kst->f_files = kst->f_ffree + scoutfs_last_ino(sb); + kst->f_ffree = kst->f_bfree * 16; + kst->f_files = kst->f_ffree + le64_to_cpu(nstatfs.next_ino); - /* this fsid is constant.. the uuid is different */ + uuid = (void *)nstatfs.uuid; kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); kst->f_namelen = SCOUTFS_NAME_LEN; From c7ad9fe772803d22dc4e6b4b9f4e2a7e6fd1055c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Aug 2017 09:49:12 -0700 Subject: [PATCH 359/920] scoutfs: make release block granular The existing release interface specified byte regions to release but that didn't match what the underlying file data mapping structure is capable of. What happens if you specify a single byte to release? Does it release the whole block? Does it release nothing? Does it return an error? By making the interface match the capability of the operation we make the functioning of the system that much more predictable. Callers are forced to think about implementing their desires in terms of block granular releasing. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 23 +++++++---------------- kmod/src/ioctl.h | 24 +++++++++++++++++++++++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 664e7f22..a947bc6a 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -326,30 +326,19 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) struct scoutfs_ioctl_release args; loff_t start; loff_t end_inc; - u64 iblock; - u64 end_block; - u64 len; int ret; if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; - trace_printk("offset %llu count %llu vers %llu\n", - args.offset, args.count, args.data_version); + trace_printk("block %llu count %llu vers %llu\n", + args.block, args.count, args.data_version); if (args.count == 0) return 0; - if ((args.offset + args.count) < args.offset) + if ((args.block + args.count) < args.block) return -EINVAL; - start = round_up(args.offset, SCOUTFS_BLOCK_SIZE); - end_inc = round_down(args.offset + args.count, SCOUTFS_BLOCK_SIZE) - 1; - if (end_inc < start) - return 0; - - iblock = start >> SCOUTFS_BLOCK_SHIFT; - end_block = end_inc >> SCOUTFS_BLOCK_SHIFT; - len = end_block - iblock + 1; ret = mnt_want_write_file(file); if (ret) @@ -375,10 +364,12 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) inode_dio_wait(inode); /* drop all clean and dirty cached blocks in the range */ + start = args.block << SCOUTFS_BLOCK_SHIFT; + end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SHIFT) - 1; truncate_inode_pages_range(&inode->i_data, start, end_inc); - ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), iblock, len, - true); + ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), args.block, + args.count, true); out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 0dcb6bc7..c8a267b5 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -124,8 +124,30 @@ struct scoutfs_ioctl_ino_path { #define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, __u64) +/* + * "Release" a contiguous range of logical blocks of file data. + * Released blocks are removed from the file system like truncation, but + * an offline record is left behind to trigger demand staging if the + * file is read. + * + * The starting block offset and number of blocks to release are in + * units 4KB blocks. + * + * The specified range can extend past i_size and can straddle sparse + * regions or blocks that are already offline. The only change it makes + * is to free and mark offline any existing blocks that intersect with + * the region. + * + * Returns 0 if the operation succeeds. If an error is returned then + * some partial region of the blocks in the region may have been marked + * offline. + * + * If the operation succeeds then inode metadata that reflects file data + * contents are not updated. This is intended to be transparent to the + * presentation of the data in the file. + */ struct scoutfs_ioctl_release { - __u64 offset; + __u64 block; __u64 count; __u64 data_version; } __packed; From 7cc09761f54c3e561e5311c574a3c31cdc89d57f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Aug 2017 11:56:20 -0700 Subject: [PATCH 360/920] scoutfs: release item cleanup needs transaction Release tries to re-instate extents if it sees an error during release. Those item manipulations need to be covered by the transaction. Signed-off-by: Zach Brown --- kmod/src/data.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index d1070f8f..713a2d36 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -632,9 +632,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, holding = false; } - if (holding) - scoutfs_release_trans(sb); - if (ret) { if (ins_ext) { err = insert_extent(sb, &ext, ino, @@ -648,6 +645,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, } } + if (holding) + scoutfs_release_trans(sb); + return ret; } From 07bbc418c376b7f88a4e44437e948d299098433d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Aug 2017 14:40:18 -0700 Subject: [PATCH 361/920] scoutfs: merge offline extents Offline extents weren't being merged because they all had their physical blkno set to 0 and all the extent calculations didn't treat them specially. They would only merge if the physical blocks of two extent were contiguous. Instead of special casing offline extents everywhere we store them with a physical blkno set to the logical blk_off. This lets all the current extent calculations work as expected. Signed-off-by: Zach Brown --- kmod/src/data.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 713a2d36..e7753cde 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -59,6 +59,12 @@ * ensures that data is persistent before the metadata that references * it is visible. * + * Files can have offline extents. They have no allocated file data but + * the offline status represents file data that can be recalled through + * staging. While offline the extents have their physical blkno set to + * the logical blk_off so that all the usual block extent calculations + * still hold. It's mapped back to phys == 0 for fiemap. + * * Weirdly, the extents are indexed by the *final* logical block and * blkno of the extent. This lets us search for neighbouring previous * extents with a _next() call and avoids having to implement item @@ -388,7 +394,7 @@ static int insert_extent(struct super_block *sb, trace_printk("inserting "EXTF"\n", EXTA(caller_ins)); /* find previous that might be adjacent */ - ret = try_merge(sb, &ins, -1, &left, arg, type); + ret = try_merge(sb, &ins, -1, &left, arg, type) ?: try_merge(sb, &ins, 1, &right, arg, type); if (ret < 0) goto out; @@ -561,9 +567,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, load_extent(&found, &key); trace_printk("found "EXTF"\n", EXTA(&found)); - /* XXX corruption: offline and allocation are exclusive */ - if (!!found.blkno == - !!(found.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) { + /* XXX corruption: offline has phys == log */ + if ((found.flags & SCOUTFS_FILE_EXTENT_OFFLINE) && + found.blkno != found.blk_off) { ret = -EIO; break; } @@ -618,9 +624,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, /* maybe add new file extents with the offline flag set */ if (offline) { ofl = ext; - ofl.blkno = 0; + ofl.blkno = ofl.blk_off; ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; - ret = insert_extent(sb, &ofl, sbi->node_id, + ret = insert_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_TYPE); if (ret) break; @@ -913,7 +919,7 @@ retry: /* remove old offline block if we're staging */ if (was_offline) { ofl.blk_off = iblock; - ofl.blkno = 0; + ofl.blkno = iblock; ofl.blocks = 1; ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; ret = remove_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_TYPE); @@ -1219,8 +1225,12 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, logical = ext.blk_off << SCOUTFS_BLOCK_SHIFT; phys = ext.blkno << SCOUTFS_BLOCK_SHIFT; size = ext.blocks << SCOUTFS_BLOCK_SHIFT; - flags = ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE ? - FIEMAP_EXTENT_UNKNOWN : 0; + flags = 0; + + if (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) { + phys = 0; + flags = FIEMAP_EXTENT_UNKNOWN; + } blk_off = ext.blk_off + ext.blocks; } From d1ae486d83ad7f7ef00b5e5fca1342db3b700011 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 4 Aug 2017 16:42:46 -0500 Subject: [PATCH 362/920] scoutfs: provide ->llseek Without this we return -ESPIPE when a process tries to seek on a regular file. Signed-off-by: Mark Fasheh --- kmod/src/data.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index e7753cde..dad28b6c 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1258,6 +1258,7 @@ const struct file_operations scoutfs_file_fops = { .aio_write = generic_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, + .llseek = generic_file_llseek, }; From 8135b18c7636b46dc36ca370c2a858130e6cb3be Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Aug 2017 20:59:15 -0700 Subject: [PATCH 363/920] scoutfs: start truncate from first block Truncation updates extents that intersect with the input range. It starts with the first block in the range and iterates until it has searched for all the extents that could cover the range. Extents are stored in items at their final block location so that we can use _next to find intersections. Truncation was searching for the next extent after the full extent that it was still searching for. That means it was starting the search at the last block in the extent, not the first. It would miss all the extents that didn't overlap with the last block it was searching for. This fixed by searching from a temporary single block extent at the start of the search range. Signed-off-by: Zach Brown --- kmod/src/data.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index dad28b6c..661cff5b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -530,6 +530,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_key_buf last; struct scoutfs_key_buf key; struct native_extent found; + struct native_extent first; struct native_extent rng; struct native_extent ext; struct native_extent ofl; @@ -554,7 +555,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, while (rng.blocks) { /* find the next extent that could include our first block */ - init_extent_key(&key, key_bytes, &rng, ino, + first = rng; + first.blocks = 1; + init_extent_key(&key, key_bytes, &first, ino, SCOUTFS_FILE_EXTENT_TYPE); ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); From d59367262d585a67ae8e9d99ae8ebad5a7db0c3f Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 21 Aug 2017 18:44:35 -0500 Subject: [PATCH 364/920] scoutfs: remove inode mtime index This index is unused - we can gain some create performance by removing it. To verify this, I ran createmany for 10 million files: $ createmany -o '/scoutfs/file_%lu' 10000000 Before this patch: total: 10000000 creates in 776.54 seconds: 12877.56 creates/second real 12m56.557s user 0m7.861s sys 6m56.986s After this patch: total: 10000000 creates in 691.92 seconds: 14452.46 creates/second real 11m31.936s user 0m7.785s sys 6m19.328s So removing the index gained us about a minute and a half on the test or a 12% performance increase. Signed-off-by: Mark Fasheh --- kmod/src/format.h | 1 - kmod/src/inode.c | 7 ------- kmod/src/ioctl.c | 2 -- kmod/src/ioctl.h | 1 - kmod/src/key.c | 1 - kmod/src/lock.c | 1 - 6 files changed, 13 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index b532cccc..2d3c449d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -244,7 +244,6 @@ struct scoutfs_segment_block { /* inode index zone */ #define SCOUTFS_INODE_INDEX_CTIME_TYPE 1 -#define SCOUTFS_INODE_INDEX_MTIME_TYPE 2 #define SCOUTFS_INODE_INDEX_SIZE_TYPE 3 #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 4 #define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 5 diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 4a291e74..f7a81c13 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -586,10 +586,6 @@ void scoutfs_update_inode_item(struct inode *inode) le64_to_cpu(sinode.ctime.sec), le32_to_cpu(sinode.ctime.nsec), si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: - update_index(sb, si, ino, SCOUTFS_INODE_INDEX_MTIME_TYPE, - le64_to_cpu(sinode.mtime.sec), - le32_to_cpu(sinode.mtime.nsec), - si->item_mtime.tv_sec, si->item_mtime.tv_nsec) ?: update_index(sb, si, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, le64_to_cpu(sinode.size), 0, si->item_size, 0) ?: update_index(sb, si, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, @@ -657,9 +653,6 @@ static int remove_index_items(struct super_block *sb, u64 ino, ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_CTIME_TYPE, le64_to_cpu(sinode->ctime.sec), le32_to_cpu(sinode->ctime.nsec)) ?: - remove_index(sb, ino, SCOUTFS_INODE_INDEX_MTIME_TYPE, - le64_to_cpu(sinode->mtime.sec), - le32_to_cpu(sinode->mtime.nsec)) ?: remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, le64_to_cpu(sinode->size), 0) ?: remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index a947bc6a..4780c9b3 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -75,8 +75,6 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (walk.index == SCOUTFS_IOC_WALK_INODES_CTIME) type = SCOUTFS_INODE_INDEX_CTIME_TYPE; - else if (walk.index == SCOUTFS_IOC_WALK_INODES_MTIME) - type = SCOUTFS_INODE_INDEX_MTIME_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) type = SCOUTFS_INODE_INDEX_SIZE_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index c8a267b5..52b0b4d4 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -54,7 +54,6 @@ struct scoutfs_ioctl_walk_inodes { enum { SCOUTFS_IOC_WALK_INODES_CTIME = 0, - SCOUTFS_IOC_WALK_INODES_MTIME, SCOUTFS_IOC_WALK_INODES_SIZE, SCOUTFS_IOC_WALK_INODES_META_SEQ, SCOUTFS_IOC_WALK_INODES_DATA_SEQ, diff --git a/kmod/src/key.c b/kmod/src/key.c index 85091bbc..da2b4ea5 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -137,7 +137,6 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) struct scoutfs_inode_index_key *ikey = key->data; static char *type_strings[] = { [SCOUTFS_INODE_INDEX_CTIME_TYPE] = "ctm", - [SCOUTFS_INODE_INDEX_MTIME_TYPE] = "mtm", [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 0d5455fb..9ce58738 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -463,7 +463,6 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, switch(type) { case SCOUTFS_INODE_INDEX_CTIME_TYPE: - case SCOUTFS_INODE_INDEX_MTIME_TYPE: major_mask = (1 << 5) - 1; ino_mask = ~0ULL; break; From 021404bb6adb529ef0c1e1c631da357faf2591ba Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 22 Aug 2017 14:34:29 -0500 Subject: [PATCH 365/920] scoutfs: remove inode ctime index Like the mtime index, this index is unused. Removing it is a near identical task. Running the same createmany test from our last patch gives us the following: $ createmany -o '/scoutfs/file_%lu' 10000000 total: 10000000 creates in 598.28 seconds: 16714.59 creates/second real 9m58.292s user 0m7.420s sys 5m44.632s So after both indices are gone, we go from a 12m56 run time to 9m58s, saving almost 3 minutes which translates into a total performance increase of about 23%. Signed-off-by: Mark Fasheh --- kmod/src/format.h | 3 +-- kmod/src/inode.c | 11 ++--------- kmod/src/ioctl.c | 4 +--- kmod/src/ioctl.h | 3 +-- kmod/src/key.c | 1 - kmod/src/lock.c | 5 ----- 6 files changed, 5 insertions(+), 22 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 2d3c449d..5be9ed58 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -243,13 +243,12 @@ struct scoutfs_segment_block { #define SCOUTFS_FS_ZONE 3 /* inode index zone */ -#define SCOUTFS_INODE_INDEX_CTIME_TYPE 1 #define SCOUTFS_INODE_INDEX_SIZE_TYPE 3 #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 4 #define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 5 #define SCOUTFS_INODE_INDEX_NR \ - (SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE - SCOUTFS_INODE_INDEX_CTIME_TYPE + 1) + (SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE - SCOUTFS_INODE_INDEX_SIZE_TYPE + 1) /* node zone */ #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 11 diff --git a/kmod/src/inode.c b/kmod/src/inode.c index f7a81c13..2b7ddb0a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -582,11 +582,7 @@ void scoutfs_update_inode_item(struct inode *inode) /* only race with other inode field stores once */ store_inode(&sinode, inode); - ret = update_index(sb, si, ino, SCOUTFS_INODE_INDEX_CTIME_TYPE, - le64_to_cpu(sinode.ctime.sec), - le32_to_cpu(sinode.ctime.nsec), - si->item_ctime.tv_sec, si->item_ctime.tv_nsec) ?: - update_index(sb, si, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, + ret = update_index(sb, si, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, le64_to_cpu(sinode.size), 0, si->item_size, 0) ?: update_index(sb, si, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode.meta_seq), 0, @@ -650,10 +646,7 @@ static int remove_index_items(struct super_block *sb, u64 ino, umode_t mode = le32_to_cpu(sinode->mode); int ret; - ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_CTIME_TYPE, - le64_to_cpu(sinode->ctime.sec), - le32_to_cpu(sinode->ctime.nsec)) ?: - remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, le64_to_cpu(sinode->size), 0) ?: remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode->meta_seq), 0); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 4780c9b3..3fe08339 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -73,9 +73,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) walk.first.ino, walk.last.major, walk.last.minor, walk.last.ino); - if (walk.index == SCOUTFS_IOC_WALK_INODES_CTIME) - type = SCOUTFS_INODE_INDEX_CTIME_TYPE; - else if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) + if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) type = SCOUTFS_INODE_INDEX_SIZE_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) type = SCOUTFS_INODE_INDEX_META_SEQ_TYPE; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 52b0b4d4..2c980455 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -53,8 +53,7 @@ struct scoutfs_ioctl_walk_inodes { } __packed; enum { - SCOUTFS_IOC_WALK_INODES_CTIME = 0, - SCOUTFS_IOC_WALK_INODES_SIZE, + SCOUTFS_IOC_WALK_INODES_SIZE = 0, SCOUTFS_IOC_WALK_INODES_META_SEQ, SCOUTFS_IOC_WALK_INODES_DATA_SEQ, SCOUTFS_IOC_WALK_INODES_UNKNOWN, diff --git a/kmod/src/key.c b/kmod/src/key.c index da2b4ea5..40051d8a 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -136,7 +136,6 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) case SCOUTFS_INODE_INDEX_ZONE: { struct scoutfs_inode_index_key *ikey = key->data; static char *type_strings[] = { - [SCOUTFS_INODE_INDEX_CTIME_TYPE] = "ctm", [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 9ce58738..16855530 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -462,11 +462,6 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, int bit; switch(type) { - case SCOUTFS_INODE_INDEX_CTIME_TYPE: - major_mask = (1 << 5) - 1; - ino_mask = ~0ULL; - break; - case SCOUTFS_INODE_INDEX_SIZE_TYPE: major_mask = 0; if (major) { From f7e3f6f9e6f947da3f4e87fec5797036384e3118 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 22 Aug 2017 19:07:53 -0500 Subject: [PATCH 366/920] scoutfs: import fs/ocfs2/dlmglue.[ch] from Linux v4.13-rc6 Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 4213 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/dlmglue.h | 191 ++ 2 files changed, 4404 insertions(+) create mode 100644 kmod/src/dlmglue.c create mode 100644 kmod/src/dlmglue.h diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c new file mode 100644 index 00000000..4689940a --- /dev/null +++ b/kmod/src/dlmglue.c @@ -0,0 +1,4213 @@ +/* -*- mode: c; c-basic-offset: 8; -*- + * vim: noexpandtab sw=8 ts=8 sts=0: + * + * dlmglue.c + * + * Code which implements an OCFS2 specific interface to our DLM. + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MLOG_MASK_PREFIX ML_DLM_GLUE +#include + +#include "ocfs2.h" +#include "ocfs2_lockingver.h" + +#include "alloc.h" +#include "dcache.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "stackglue.h" +#include "slot_map.h" +#include "super.h" +#include "uptodate.h" +#include "quota.h" +#include "refcounttree.h" +#include "acl.h" + +#include "buffer_head_io.h" + +struct ocfs2_mask_waiter { + struct list_head mw_item; + int mw_status; + struct completion mw_complete; + unsigned long mw_mask; + unsigned long mw_goal; +#ifdef CONFIG_OCFS2_FS_STATS + ktime_t mw_lock_start; +#endif +}; + +static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres); + +/* + * Return value from ->downconvert_worker functions. + * + * These control the precise actions of ocfs2_unblock_lock() + * and ocfs2_process_blocked_lock() + * + */ +enum ocfs2_unblock_action { + UNBLOCK_CONTINUE = 0, /* Continue downconvert */ + UNBLOCK_CONTINUE_POST = 1, /* Continue downconvert, fire + * ->post_unlock callback */ + UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire + * ->post_unlock() callback. */ +}; + +struct ocfs2_unblock_ctl { + int requeue; + enum ocfs2_unblock_action unblock_action; +}; + +/* Lockdep class keys */ +struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; + +static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres); + +static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres); + +static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +#define mlog_meta_lvb(__level, __lockres) ocfs2_dump_meta_lvb_info(__level, __PRETTY_FUNCTION__, __LINE__, __lockres) + +/* This aids in debugging situations where a bad LVB might be involved. */ +static void ocfs2_dump_meta_lvb_info(u64 level, + const char *function, + unsigned int line, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + mlog(level, "LVB information for %s (called from %s:%u):\n", + lockres->l_name, function, line); + mlog(level, "version: %u, clusters: %u, generation: 0x%x\n", + lvb->lvb_version, be32_to_cpu(lvb->lvb_iclusters), + be32_to_cpu(lvb->lvb_igeneration)); + mlog(level, "size: %llu, uid %u, gid %u, mode 0x%x\n", + (unsigned long long)be64_to_cpu(lvb->lvb_isize), + be32_to_cpu(lvb->lvb_iuid), be32_to_cpu(lvb->lvb_igid), + be16_to_cpu(lvb->lvb_imode)); + mlog(level, "nlink %u, atime_packed 0x%llx, ctime_packed 0x%llx, " + "mtime_packed 0x%llx iattr 0x%x\n", be16_to_cpu(lvb->lvb_inlink), + (long long)be64_to_cpu(lvb->lvb_iatime_packed), + (long long)be64_to_cpu(lvb->lvb_ictime_packed), + (long long)be64_to_cpu(lvb->lvb_imtime_packed), + be32_to_cpu(lvb->lvb_iattr)); +} + + +/* + * OCFS2 Lock Resource Operations + * + * These fine tune the behavior of the generic dlmglue locking infrastructure. + * + * The most basic of lock types can point ->l_priv to their respective + * struct ocfs2_super and allow the default actions to manage things. + * + * Right now, each lock type also needs to implement an init function, + * and trivial lock/unlock wrappers. ocfs2_simple_drop_lockres() + * should be called when the lock is no longer needed (i.e., object + * destruction time). + */ +struct ocfs2_lock_res_ops { + /* + * Translate an ocfs2_lock_res * into an ocfs2_super *. Define + * this callback if ->l_priv is not an ocfs2_super pointer + */ + struct ocfs2_super * (*get_osb)(struct ocfs2_lock_res *); + + /* + * Optionally called in the downconvert thread after a + * successful downconvert. The lockres will not be referenced + * after this callback is called, so it is safe to free + * memory, etc. + * + * The exact semantics of when this is called are controlled + * by ->downconvert_worker() + */ + void (*post_unlock)(struct ocfs2_super *, struct ocfs2_lock_res *); + + /* + * Allow a lock type to add checks to determine whether it is + * safe to downconvert a lock. Return 0 to re-queue the + * downconvert at a later time, nonzero to continue. + * + * For most locks, the default checks that there are no + * incompatible holders are sufficient. + * + * Called with the lockres spinlock held. + */ + int (*check_downconvert)(struct ocfs2_lock_res *, int); + + /* + * Allows a lock type to populate the lock value block. This + * is called on downconvert, and when we drop a lock. + * + * Locks that want to use this should set LOCK_TYPE_USES_LVB + * in the flags field. + * + * Called with the lockres spinlock held. + */ + void (*set_lvb)(struct ocfs2_lock_res *); + + /* + * Called from the downconvert thread when it is determined + * that a lock will be downconverted. This is called without + * any locks held so the function can do work that might + * schedule (syncing out data, etc). + * + * This should return any one of the ocfs2_unblock_action + * values, depending on what it wants the thread to do. + */ + int (*downconvert_worker)(struct ocfs2_lock_res *, int); + + /* + * LOCK_TYPE_* flags which describe the specific requirements + * of a lock type. Descriptions of each individual flag follow. + */ + int flags; +}; + +/* + * Some locks want to "refresh" potentially stale data when a + * meaningful (PRMODE or EXMODE) lock level is first obtained. If this + * flag is set, the OCFS2_LOCK_NEEDS_REFRESH flag will be set on the + * individual lockres l_flags member from the ast function. It is + * expected that the locking wrapper will clear the + * OCFS2_LOCK_NEEDS_REFRESH flag when done. + */ +#define LOCK_TYPE_REQUIRES_REFRESH 0x1 + +/* + * Indicate that a lock type makes use of the lock value block. The + * ->set_lvb lock type callback must be defined. + */ +#define LOCK_TYPE_USES_LVB 0x2 + +static struct ocfs2_lock_res_ops ocfs2_inode_rw_lops = { + .get_osb = ocfs2_get_inode_osb, + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_inode_inode_lops = { + .get_osb = ocfs2_get_inode_osb, + .check_downconvert = ocfs2_check_meta_downconvert, + .set_lvb = ocfs2_set_meta_lvb, + .downconvert_worker = ocfs2_data_convert_worker, + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, +}; + +static struct ocfs2_lock_res_ops ocfs2_super_lops = { + .flags = LOCK_TYPE_REQUIRES_REFRESH, +}; + +static struct ocfs2_lock_res_ops ocfs2_rename_lops = { + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_nfs_sync_lops = { + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_orphan_scan_lops = { + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, +}; + +static struct ocfs2_lock_res_ops ocfs2_dentry_lops = { + .get_osb = ocfs2_get_dentry_osb, + .post_unlock = ocfs2_dentry_post_unlock, + .downconvert_worker = ocfs2_dentry_convert_worker, + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_inode_open_lops = { + .get_osb = ocfs2_get_inode_osb, + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_flock_lops = { + .get_osb = ocfs2_get_file_osb, + .flags = 0, +}; + +static struct ocfs2_lock_res_ops ocfs2_qinfo_lops = { + .set_lvb = ocfs2_set_qinfo_lvb, + .get_osb = ocfs2_get_qinfo_osb, + .flags = LOCK_TYPE_REQUIRES_REFRESH | LOCK_TYPE_USES_LVB, +}; + +static struct ocfs2_lock_res_ops ocfs2_refcount_block_lops = { + .check_downconvert = ocfs2_check_refcount_downconvert, + .downconvert_worker = ocfs2_refcount_convert_worker, + .flags = 0, +}; + +static inline int ocfs2_is_inode_lock(struct ocfs2_lock_res *lockres) +{ + return lockres->l_type == OCFS2_LOCK_TYPE_META || + lockres->l_type == OCFS2_LOCK_TYPE_RW || + lockres->l_type == OCFS2_LOCK_TYPE_OPEN; +} + +static inline struct ocfs2_lock_res *ocfs2_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) +{ + return container_of(lksb, struct ocfs2_lock_res, l_lksb); +} + +static inline struct inode *ocfs2_lock_res_inode(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!ocfs2_is_inode_lock(lockres)); + + return (struct inode *) lockres->l_priv; +} + +static inline struct ocfs2_dentry_lock *ocfs2_lock_res_dl(struct ocfs2_lock_res *lockres) +{ + BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_DENTRY); + + return (struct ocfs2_dentry_lock *)lockres->l_priv; +} + +static inline struct ocfs2_mem_dqinfo *ocfs2_lock_res_qinfo(struct ocfs2_lock_res *lockres) +{ + BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_QINFO); + + return (struct ocfs2_mem_dqinfo *)lockres->l_priv; +} + +static inline struct ocfs2_refcount_tree * +ocfs2_lock_res_refcount_tree(struct ocfs2_lock_res *res) +{ + return container_of(res, struct ocfs2_refcount_tree, rf_lockres); +} + +static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *lockres) +{ + if (lockres->l_ops->get_osb) + return lockres->l_ops->get_osb(lockres); + + return (struct ocfs2_super *)lockres->l_priv; +} + +static int ocfs2_lock_create(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 dlm_flags); +static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, + int wanted); +static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, unsigned long caller_ip); +static inline void ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level) +{ + __ocfs2_cluster_unlock(osb, lockres, level, _RET_IP_); +} + +static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres); +static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres); +static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres); +static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, int level); +static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, + int convert); +#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ + if ((_lockres)->l_type != OCFS2_LOCK_TYPE_DENTRY) \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ + _err, _func, _lockres->l_name); \ + else \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %.*s%08x\n", \ + _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ + (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ +} while (0) +static int ocfs2_downconvert_thread(void *arg); +static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static int ocfs2_inode_lock_update(struct inode *inode, + struct buffer_head **bh); +static void ocfs2_drop_osb_locks(struct ocfs2_super *osb); +static inline int ocfs2_highest_compat_lock_level(int level); +static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static int ocfs2_downconvert_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int new_level, + int lvb, + unsigned int generation); +static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static int ocfs2_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + + +static void ocfs2_build_lock_name(enum ocfs2_lock_type type, + u64 blkno, + u32 generation, + char *name) +{ + int len; + + BUG_ON(type >= OCFS2_NUM_LOCK_TYPES); + + len = snprintf(name, OCFS2_LOCK_ID_MAX_LEN, "%c%s%016llx%08x", + ocfs2_lock_type_char(type), OCFS2_LOCK_ID_PAD, + (long long)blkno, generation); + + BUG_ON(len != (OCFS2_LOCK_ID_MAX_LEN - 1)); + + mlog(0, "built lock resource with name: %s\n", name); +} + +static DEFINE_SPINLOCK(ocfs2_dlm_tracking_lock); + +static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, + struct ocfs2_dlm_debug *dlm_debug) +{ + mlog(0, "Add tracking for lockres %s\n", res->l_name); + + spin_lock(&ocfs2_dlm_tracking_lock); + list_add(&res->l_debug_list, &dlm_debug->d_lockres_tracking); + spin_unlock(&ocfs2_dlm_tracking_lock); +} + +static void ocfs2_remove_lockres_tracking(struct ocfs2_lock_res *res) +{ + spin_lock(&ocfs2_dlm_tracking_lock); + if (!list_empty(&res->l_debug_list)) + list_del_init(&res->l_debug_list); + spin_unlock(&ocfs2_dlm_tracking_lock); +} + +#ifdef CONFIG_OCFS2_FS_STATS +static void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) +{ + res->l_lock_refresh = 0; + memset(&res->l_lock_prmode, 0, sizeof(struct ocfs2_lock_stats)); + memset(&res->l_lock_exmode, 0, sizeof(struct ocfs2_lock_stats)); +} + +static void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, int level, + struct ocfs2_mask_waiter *mw, int ret) +{ + u32 usec; + ktime_t kt; + struct ocfs2_lock_stats *stats; + + if (level == LKM_PRMODE) + stats = &res->l_lock_prmode; + else if (level == LKM_EXMODE) + stats = &res->l_lock_exmode; + else + return; + + kt = ktime_sub(ktime_get(), mw->mw_lock_start); + usec = ktime_to_us(kt); + + stats->ls_gets++; + stats->ls_total += ktime_to_ns(kt); + /* overflow */ + if (unlikely(stats->ls_gets == 0)) { + stats->ls_gets++; + stats->ls_total = ktime_to_ns(kt); + } + + if (stats->ls_max < usec) + stats->ls_max = usec; + + if (ret) + stats->ls_fail++; +} + +static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) +{ + lockres->l_lock_refresh++; +} + +static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) +{ + mw->mw_lock_start = ktime_get(); +} +#else +static inline void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) +{ +} +static inline void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, + int level, struct ocfs2_mask_waiter *mw, int ret) +{ +} +static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) +{ +} +static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) +{ +} +#endif + +static void ocfs2_lock_res_init_common(struct ocfs2_super *osb, + struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + struct ocfs2_lock_res_ops *ops, + void *priv) +{ + res->l_type = type; + res->l_ops = ops; + res->l_priv = priv; + + res->l_level = DLM_LOCK_IV; + res->l_requested = DLM_LOCK_IV; + res->l_blocking = DLM_LOCK_IV; + res->l_action = OCFS2_AST_INVALID; + res->l_unlock_action = OCFS2_UNLOCK_INVALID; + + res->l_flags = OCFS2_LOCK_INITIALIZED; + + ocfs2_add_lockres_tracking(res, osb->osb_dlm_debug); + + ocfs2_init_lock_stats(res); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (type != OCFS2_LOCK_TYPE_OPEN) + lockdep_init_map(&res->l_lockdep_map, ocfs2_lock_type_strings[type], + &lockdep_keys[type], 0); + else + res->l_lockdep_map.key = NULL; +#endif +} + +void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) +{ + /* This also clears out the lock status block */ + memset(res, 0, sizeof(struct ocfs2_lock_res)); + spin_lock_init(&res->l_lock); + init_waitqueue_head(&res->l_event); + INIT_LIST_HEAD(&res->l_blocked_list); + INIT_LIST_HEAD(&res->l_mask_waiters); + INIT_LIST_HEAD(&res->l_holders); +} + +void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + unsigned int generation, + struct inode *inode) +{ + struct ocfs2_lock_res_ops *ops; + + switch(type) { + case OCFS2_LOCK_TYPE_RW: + ops = &ocfs2_inode_rw_lops; + break; + case OCFS2_LOCK_TYPE_META: + ops = &ocfs2_inode_inode_lops; + break; + case OCFS2_LOCK_TYPE_OPEN: + ops = &ocfs2_inode_open_lops; + break; + default: + mlog_bug_on_msg(1, "type: %d\n", type); + ops = NULL; /* thanks, gcc */ + break; + }; + + ocfs2_build_lock_name(type, OCFS2_I(inode)->ip_blkno, + generation, res->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), res, type, ops, inode); +} + +static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + return OCFS2_SB(inode->i_sb); +} + +static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_mem_dqinfo *info = lockres->l_priv; + + return OCFS2_SB(info->dqi_gi.dqi_sb); +} + +static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_file_private *fp = lockres->l_priv; + + return OCFS2_SB(fp->fp_file->f_mapping->host->i_sb); +} + +static __u64 ocfs2_get_dentry_lock_ino(struct ocfs2_lock_res *lockres) +{ + __be64 inode_blkno_be; + + memcpy(&inode_blkno_be, &lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], + sizeof(__be64)); + + return be64_to_cpu(inode_blkno_be); +} + +static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_dentry_lock *dl = lockres->l_priv; + + return OCFS2_SB(dl->dl_inode->i_sb); +} + +void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, + u64 parent, struct inode *inode) +{ + int len; + u64 inode_blkno = OCFS2_I(inode)->ip_blkno; + __be64 inode_blkno_be = cpu_to_be64(inode_blkno); + struct ocfs2_lock_res *lockres = &dl->dl_lockres; + + ocfs2_lock_res_init_once(lockres); + + /* + * Unfortunately, the standard lock naming scheme won't work + * here because we have two 16 byte values to use. Instead, + * we'll stuff the inode number as a binary value. We still + * want error prints to show something without garbling the + * display, so drop a null byte in there before the inode + * number. A future version of OCFS2 will likely use all + * binary lock names. The stringified names have been a + * tremendous aid in debugging, but now that the debugfs + * interface exists, we can mangle things there if need be. + * + * NOTE: We also drop the standard "pad" value (the total lock + * name size stays the same though - the last part is all + * zeros due to the memset in ocfs2_lock_res_init_once() + */ + len = snprintf(lockres->l_name, OCFS2_DENTRY_LOCK_INO_START, + "%c%016llx", + ocfs2_lock_type_char(OCFS2_LOCK_TYPE_DENTRY), + (long long)parent); + + BUG_ON(len != (OCFS2_DENTRY_LOCK_INO_START - 1)); + + memcpy(&lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], &inode_blkno_be, + sizeof(__be64)); + + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, + OCFS2_LOCK_TYPE_DENTRY, &ocfs2_dentry_lops, + dl); +} + +static void ocfs2_super_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* Superblock lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_SUPER, OCFS2_SUPER_BLOCK_BLKNO, + 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_SUPER, + &ocfs2_super_lops, osb); +} + +static void ocfs2_rename_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* Rename lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_RENAME, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_RENAME, + &ocfs2_rename_lops, osb); +} + +static void ocfs2_nfs_sync_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* nfs_sync lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_NFS_SYNC, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_NFS_SYNC, + &ocfs2_nfs_sync_lops, osb); +} + +static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_ORPHAN_SCAN, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_ORPHAN_SCAN, + &ocfs2_orphan_scan_lops, osb); +} + +void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_file_private *fp) +{ + struct inode *inode = fp->fp_file->f_mapping->host; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_FLOCK, oi->ip_blkno, + inode->i_generation, lockres->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, + OCFS2_LOCK_TYPE_FLOCK, &ocfs2_flock_lops, + fp); + lockres->l_flags |= OCFS2_LOCK_NOCACHE; +} + +void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_mem_dqinfo *info) +{ + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_QINFO, info->dqi_gi.dqi_type, + 0, lockres->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(info->dqi_gi.dqi_sb), lockres, + OCFS2_LOCK_TYPE_QINFO, &ocfs2_qinfo_lops, + info); +} + +void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_super *osb, u64 ref_blkno, + unsigned int generation) +{ + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_REFCOUNT, ref_blkno, + generation, lockres->l_name); + ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_REFCOUNT, + &ocfs2_refcount_block_lops, osb); +} + +void ocfs2_lock_res_free(struct ocfs2_lock_res *res) +{ + if (!(res->l_flags & OCFS2_LOCK_INITIALIZED)) + return; + + ocfs2_remove_lockres_tracking(res); + + mlog_bug_on_msg(!list_empty(&res->l_blocked_list), + "Lockres %s is on the blocked list\n", + res->l_name); + mlog_bug_on_msg(!list_empty(&res->l_mask_waiters), + "Lockres %s has mask waiters pending\n", + res->l_name); + mlog_bug_on_msg(spin_is_locked(&res->l_lock), + "Lockres %s is locked\n", + res->l_name); + mlog_bug_on_msg(res->l_ro_holders, + "Lockres %s has %u ro holders\n", + res->l_name, res->l_ro_holders); + mlog_bug_on_msg(res->l_ex_holders, + "Lockres %s has %u ex holders\n", + res->l_name, res->l_ex_holders); + + /* Need to clear out the lock status block for the dlm */ + memset(&res->l_lksb, 0, sizeof(res->l_lksb)); + + res->l_flags = 0UL; +} + +/* + * Keep a list of processes who have interest in a lockres. + * Note: this is now only uesed for check recursive cluster locking. + */ +static inline void ocfs2_add_holder(struct ocfs2_lock_res *lockres, + struct ocfs2_lock_holder *oh) +{ + INIT_LIST_HEAD(&oh->oh_list); + oh->oh_owner_pid = get_pid(task_pid(current)); + + spin_lock(&lockres->l_lock); + list_add_tail(&oh->oh_list, &lockres->l_holders); + spin_unlock(&lockres->l_lock); +} + +static inline void ocfs2_remove_holder(struct ocfs2_lock_res *lockres, + struct ocfs2_lock_holder *oh) +{ + spin_lock(&lockres->l_lock); + list_del(&oh->oh_list); + spin_unlock(&lockres->l_lock); + + put_pid(oh->oh_owner_pid); +} + +static inline int ocfs2_is_locked_by_me(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_lock_holder *oh; + struct pid *pid; + + /* look in the list of holders for one with the current task as owner */ + spin_lock(&lockres->l_lock); + pid = task_pid(current); + list_for_each_entry(oh, &lockres->l_holders, oh_list) { + if (oh->oh_owner_pid == pid) { + spin_unlock(&lockres->l_lock); + return 1; + } + } + spin_unlock(&lockres->l_lock); + + return 0; +} + +static inline void ocfs2_inc_holders(struct ocfs2_lock_res *lockres, + int level) +{ + BUG_ON(!lockres); + + switch(level) { + case DLM_LOCK_EX: + lockres->l_ex_holders++; + break; + case DLM_LOCK_PR: + lockres->l_ro_holders++; + break; + default: + BUG(); + } +} + +static inline void ocfs2_dec_holders(struct ocfs2_lock_res *lockres, + int level) +{ + BUG_ON(!lockres); + + switch(level) { + case DLM_LOCK_EX: + BUG_ON(!lockres->l_ex_holders); + lockres->l_ex_holders--; + break; + case DLM_LOCK_PR: + BUG_ON(!lockres->l_ro_holders); + lockres->l_ro_holders--; + break; + default: + BUG(); + } +} + +/* WARNING: This function lives in a world where the only three lock + * levels are EX, PR, and NL. It *will* have to be adjusted when more + * lock types are added. */ +static inline int ocfs2_highest_compat_lock_level(int level) +{ + int new_level = DLM_LOCK_EX; + + if (level == DLM_LOCK_EX) + new_level = DLM_LOCK_NL; + else if (level == DLM_LOCK_PR) + new_level = DLM_LOCK_PR; + return new_level; +} + +static void lockres_set_flags(struct ocfs2_lock_res *lockres, + unsigned long newflags) +{ + struct ocfs2_mask_waiter *mw, *tmp; + + assert_spin_locked(&lockres->l_lock); + + lockres->l_flags = newflags; + + list_for_each_entry_safe(mw, tmp, &lockres->l_mask_waiters, mw_item) { + if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) + continue; + + list_del_init(&mw->mw_item); + mw->mw_status = 0; + complete(&mw->mw_complete); + } +} +static void lockres_or_flags(struct ocfs2_lock_res *lockres, unsigned long or) +{ + lockres_set_flags(lockres, lockres->l_flags | or); +} +static void lockres_clear_flags(struct ocfs2_lock_res *lockres, + unsigned long clear) +{ + lockres_set_flags(lockres, lockres->l_flags & ~clear); +} + +static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); + BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); + + lockres->l_level = lockres->l_requested; + if (lockres->l_level <= + ocfs2_highest_compat_lock_level(lockres->l_blocking)) { + lockres->l_blocking = DLM_LOCK_NL; + lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); + } + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); + + /* Convert from RO to EX doesn't really need anything as our + * information is already up to data. Convert from NL to + * *anything* however should mark ourselves as needing an + * update */ + if (lockres->l_level == DLM_LOCK_NL && + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + + lockres->l_level = lockres->l_requested; + + /* + * We set the OCFS2_LOCK_UPCONVERT_FINISHING flag before clearing + * the OCFS2_LOCK_BUSY flag to prevent the dc thread from + * downconverting the lock before the upconvert has fully completed. + * Do not prevent the dc thread from downconverting if NONBLOCK lock + * had already returned. + */ + if (!(lockres->l_flags & OCFS2_LOCK_NONBLOCK_FINISHED)) + lockres_or_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + else + lockres_clear_flags(lockres, OCFS2_LOCK_NONBLOCK_FINISHED); + + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON((!(lockres->l_flags & OCFS2_LOCK_BUSY))); + BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); + + if (lockres->l_requested > DLM_LOCK_NL && + !(lockres->l_flags & OCFS2_LOCK_LOCAL) && + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + + lockres->l_level = lockres->l_requested; + lockres_or_flags(lockres, OCFS2_LOCK_ATTACHED); + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, + int level) +{ + int needs_downconvert = 0; + + assert_spin_locked(&lockres->l_lock); + + if (level > lockres->l_blocking) { + /* only schedule a downconvert if we haven't already scheduled + * one that goes low enough to satisfy the level we're + * blocking. this also catches the case where we get + * duplicate BASTs */ + if (ocfs2_highest_compat_lock_level(level) < + ocfs2_highest_compat_lock_level(lockres->l_blocking)) + needs_downconvert = 1; + + lockres->l_blocking = level; + } + + mlog(ML_BASTS, "lockres %s, block %d, level %d, l_block %d, dwn %d\n", + lockres->l_name, level, lockres->l_level, lockres->l_blocking, + needs_downconvert); + + if (needs_downconvert) + lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); + mlog(0, "needs_downconvert = %d\n", needs_downconvert); + return needs_downconvert; +} + +/* + * OCFS2_LOCK_PENDING and l_pending_gen. + * + * Why does OCFS2_LOCK_PENDING exist? To close a race between setting + * OCFS2_LOCK_BUSY and calling ocfs2_dlm_lock(). See ocfs2_unblock_lock() + * for more details on the race. + * + * OCFS2_LOCK_PENDING closes the race quite nicely. However, it introduces + * a race on itself. In o2dlm, we can get the ast before ocfs2_dlm_lock() + * returns. The ast clears OCFS2_LOCK_BUSY, and must therefore clear + * OCFS2_LOCK_PENDING at the same time. When ocfs2_dlm_lock() returns, + * the caller is going to try to clear PENDING again. If nothing else is + * happening, __lockres_clear_pending() sees PENDING is unset and does + * nothing. + * + * But what if another path (eg downconvert thread) has just started a + * new locking action? The other path has re-set PENDING. Our path + * cannot clear PENDING, because that will re-open the original race + * window. + * + * [Example] + * + * ocfs2_meta_lock() + * ocfs2_cluster_lock() + * set BUSY + * set PENDING + * drop l_lock + * ocfs2_dlm_lock() + * ocfs2_locking_ast() ocfs2_downconvert_thread() + * clear PENDING ocfs2_unblock_lock() + * take_l_lock + * !BUSY + * ocfs2_prepare_downconvert() + * set BUSY + * set PENDING + * drop l_lock + * take l_lock + * clear PENDING + * drop l_lock + * + * ocfs2_dlm_lock() + * + * So as you can see, we now have a window where l_lock is not held, + * PENDING is not set, and ocfs2_dlm_lock() has not been called. + * + * The core problem is that ocfs2_cluster_lock() has cleared the PENDING + * set by ocfs2_prepare_downconvert(). That wasn't nice. + * + * To solve this we introduce l_pending_gen. A call to + * lockres_clear_pending() will only do so when it is passed a generation + * number that matches the lockres. lockres_set_pending() will return the + * current generation number. When ocfs2_cluster_lock() goes to clear + * PENDING, it passes the generation it got from set_pending(). In our + * example above, the generation numbers will *not* match. Thus, + * ocfs2_cluster_lock() will not clear the PENDING set by + * ocfs2_prepare_downconvert(). + */ + +/* Unlocked version for ocfs2_locking_ast() */ +static void __lockres_clear_pending(struct ocfs2_lock_res *lockres, + unsigned int generation, + struct ocfs2_super *osb) +{ + assert_spin_locked(&lockres->l_lock); + + /* + * The ast and locking functions can race us here. The winner + * will clear pending, the loser will not. + */ + if (!(lockres->l_flags & OCFS2_LOCK_PENDING) || + (lockres->l_pending_gen != generation)) + return; + + lockres_clear_flags(lockres, OCFS2_LOCK_PENDING); + lockres->l_pending_gen++; + + /* + * The downconvert thread may have skipped us because we + * were PENDING. Wake it up. + */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + ocfs2_wake_downconvert_thread(osb); +} + +/* Locked version for callers of ocfs2_dlm_lock() */ +static void lockres_clear_pending(struct ocfs2_lock_res *lockres, + unsigned int generation, + struct ocfs2_super *osb) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + __lockres_clear_pending(lockres, generation, osb); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +static unsigned int lockres_set_pending(struct ocfs2_lock_res *lockres) +{ + assert_spin_locked(&lockres->l_lock); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + + lockres_or_flags(lockres, OCFS2_LOCK_PENDING); + + return lockres->l_pending_gen; +} + +static void ocfs2_blocking_ast(struct ocfs2_dlm_lksb *lksb, int level) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + int needs_downconvert; + unsigned long flags; + + BUG_ON(level <= DLM_LOCK_NL); + + mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d, " + "type %s\n", lockres->l_name, level, lockres->l_level, + ocfs2_lock_type_string(lockres->l_type)); + + /* + * We can skip the bast for locks which don't enable caching - + * they'll be dropped at the earliest possible time anyway. + */ + if (lockres->l_flags & OCFS2_LOCK_NOCACHE) + return; + + spin_lock_irqsave(&lockres->l_lock, flags); + needs_downconvert = ocfs2_generic_handle_bast(lockres, level); + if (needs_downconvert) + ocfs2_schedule_blocked_lock(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); + + ocfs2_wake_downconvert_thread(osb); +} + +static void ocfs2_locking_ast(struct ocfs2_dlm_lksb *lksb) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + unsigned long flags; + int status; + + spin_lock_irqsave(&lockres->l_lock, flags); + + status = ocfs2_dlm_lock_status(&lockres->l_lksb); + + if (status == -EAGAIN) { + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + goto out; + } + + if (status) { + mlog(ML_ERROR, "lockres %s: lksb status value of %d!\n", + lockres->l_name, status); + spin_unlock_irqrestore(&lockres->l_lock, flags); + return; + } + + mlog(ML_BASTS, "AST fired for lockres %s, action %d, unlock %d, " + "level %d => %d\n", lockres->l_name, lockres->l_action, + lockres->l_unlock_action, lockres->l_level, lockres->l_requested); + + switch(lockres->l_action) { + case OCFS2_AST_ATTACH: + ocfs2_generic_handle_attach_action(lockres); + lockres_clear_flags(lockres, OCFS2_LOCK_LOCAL); + break; + case OCFS2_AST_CONVERT: + ocfs2_generic_handle_convert_action(lockres); + break; + case OCFS2_AST_DOWNCONVERT: + ocfs2_generic_handle_downconvert_action(lockres); + break; + default: + mlog(ML_ERROR, "lockres %s: AST fired with invalid action: %u, " + "flags 0x%lx, unlock: %u\n", + lockres->l_name, lockres->l_action, lockres->l_flags, + lockres->l_unlock_action); + BUG(); + } +out: + /* set it to something invalid so if we get called again we + * can catch it. */ + lockres->l_action = OCFS2_AST_INVALID; + + /* Did we try to cancel this lock? Clear that state */ + if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + + /* + * We may have beaten the locking functions here. We certainly + * know that dlm_lock() has been called :-) + * Because we can't have two lock calls in flight at once, we + * can use lockres->l_pending_gen. + */ + __lockres_clear_pending(lockres, lockres->l_pending_gen, osb); + + wake_up(&lockres->l_event); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + unsigned long flags; + + mlog(ML_BASTS, "UNLOCK AST fired for lockres %s, action = %d\n", + lockres->l_name, lockres->l_unlock_action); + + spin_lock_irqsave(&lockres->l_lock, flags); + if (error) { + mlog(ML_ERROR, "Dlm passes error %d for lock %s, " + "unlock_action %d\n", error, lockres->l_name, + lockres->l_unlock_action); + spin_unlock_irqrestore(&lockres->l_lock, flags); + return; + } + + switch(lockres->l_unlock_action) { + case OCFS2_UNLOCK_CANCEL_CONVERT: + mlog(0, "Cancel convert success for %s\n", lockres->l_name); + lockres->l_action = OCFS2_AST_INVALID; + /* Downconvert thread may have requeued this lock, we + * need to wake it. */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + ocfs2_wake_downconvert_thread(ocfs2_get_lockres_osb(lockres)); + break; + case OCFS2_UNLOCK_DROP_LOCK: + lockres->l_level = DLM_LOCK_IV; + break; + default: + BUG(); + } + + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + wake_up(&lockres->l_event); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +/* + * This is the filesystem locking protocol. It provides the lock handling + * hooks for the underlying DLM. It has a maximum version number. + * The version number allows interoperability with systems running at + * the same major number and an equal or smaller minor number. + * + * Whenever the filesystem does new things with locks (adds or removes a + * lock, orders them differently, does different things underneath a lock), + * the version must be changed. The protocol is negotiated when joining + * the dlm domain. A node may join the domain if its major version is + * identical to all other nodes and its minor version is greater than + * or equal to all other nodes. When its minor version is greater than + * the other nodes, it will run at the minor version specified by the + * other nodes. + * + * If a locking change is made that will not be compatible with older + * versions, the major number must be increased and the minor version set + * to zero. If a change merely adds a behavior that can be disabled when + * speaking to older versions, the minor version must be increased. If a + * change adds a fully backwards compatible change (eg, LVB changes that + * are just ignored by older versions), the version does not need to be + * updated. + */ +static struct ocfs2_locking_protocol lproto = { + .lp_max_version = { + .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR, + .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR, + }, + .lp_lock_ast = ocfs2_locking_ast, + .lp_blocking_ast = ocfs2_blocking_ast, + .lp_unlock_ast = ocfs2_unlock_ast, +}; + +void ocfs2_set_locking_protocol(void) +{ + ocfs2_stack_glue_set_max_proto_version(&lproto.lp_max_version); +} + +static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, + int convert) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + if (convert) + lockres->l_action = OCFS2_AST_INVALID; + else + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); +} + +/* Note: If we detect another process working on the lock (i.e., + * OCFS2_LOCK_BUSY), we'll bail out returning 0. It's up to the caller + * to do the right thing in that case. + */ +static int ocfs2_lock_create(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 dlm_flags) +{ + int ret = 0; + unsigned long flags; + unsigned int gen; + + mlog(0, "lock %s, level = %d, flags = %u\n", lockres->l_name, level, + dlm_flags); + + spin_lock_irqsave(&lockres->l_lock, flags); + if ((lockres->l_flags & OCFS2_LOCK_ATTACHED) || + (lockres->l_flags & OCFS2_LOCK_BUSY)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto bail; + } + + lockres->l_action = OCFS2_AST_ATTACH; + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + gen = lockres_set_pending(lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_dlm_lock(osb->cconn, + level, + &lockres->l_lksb, + dlm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, gen, osb); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 1); + } + + mlog(0, "lock %s, return from ocfs2_dlm_lock\n", lockres->l_name); + +bail: + return ret; +} + +static inline int ocfs2_check_wait_flag(struct ocfs2_lock_res *lockres, + int flag) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&lockres->l_lock, flags); + ret = lockres->l_flags & flag; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ret; +} + +static inline void ocfs2_wait_on_busy_lock(struct ocfs2_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_BUSY)); +} + +static inline void ocfs2_wait_on_refreshing_lock(struct ocfs2_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_REFRESHING)); +} + +/* predict what lock level we'll be dropping down to on behalf + * of another node, and return true if the currently wanted + * level will be compatible with it. */ +static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, + int wanted) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); + + return wanted <= ocfs2_highest_compat_lock_level(lockres->l_blocking); +} + +static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) +{ + INIT_LIST_HEAD(&mw->mw_item); + init_completion(&mw->mw_complete); + ocfs2_init_start_time(mw); +} + +static int ocfs2_wait_for_mask(struct ocfs2_mask_waiter *mw) +{ + wait_for_completion(&mw->mw_complete); + /* Re-arm the completion in case we want to wait on it again */ + reinit_completion(&mw->mw_complete); + return mw->mw_status; +} + +static void lockres_add_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw, + unsigned long mask, + unsigned long goal) +{ + BUG_ON(!list_empty(&mw->mw_item)); + + assert_spin_locked(&lockres->l_lock); + + list_add_tail(&mw->mw_item, &lockres->l_mask_waiters); + mw->mw_mask = mask; + mw->mw_goal = goal; +} + +/* returns 0 if the mw that was removed was already satisfied, -EBUSY + * if the mask still hadn't reached its goal */ +static int __lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw) +{ + int ret = 0; + + assert_spin_locked(&lockres->l_lock); + if (!list_empty(&mw->mw_item)) { + if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) + ret = -EBUSY; + + list_del_init(&mw->mw_item); + init_completion(&mw->mw_complete); + } + + return ret; +} + +static int lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + ret = __lockres_remove_mask_waiter(lockres, mw); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ret; + +} + +static int ocfs2_wait_for_mask_interruptible(struct ocfs2_mask_waiter *mw, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ret = wait_for_completion_interruptible(&mw->mw_complete); + if (ret) + lockres_remove_mask_waiter(lockres, mw); + else + ret = mw->mw_status; + /* Re-arm the completion in case we want to wait on it again */ + reinit_completion(&mw->mw_complete); + return ret; +} + +static int __ocfs2_cluster_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 lkm_flags, + int arg_flags, + int l_subclass, + unsigned long caller_ip) +{ + struct ocfs2_mask_waiter mw; + int wait, catch_signals = !(osb->s_mount_opt & OCFS2_MOUNT_NOINTR); + int ret = 0; /* gcc doesn't realize wait = 1 guarantees ret is set */ + unsigned long flags; + unsigned int gen; + int noqueue_attempted = 0; + int dlm_locked = 0; + int kick_dc = 0; + + if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) { + mlog_errno(-EINVAL); + return -EINVAL; + } + + ocfs2_init_mask_waiter(&mw); + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lkm_flags |= DLM_LKF_VALBLK; + +again: + wait = 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + + if (catch_signals && signal_pending(current)) { + ret = -ERESTARTSYS; + goto unlock; + } + + mlog_bug_on_msg(lockres->l_flags & OCFS2_LOCK_FREEING, + "Cluster lock called on freeing lockres %s! flags " + "0x%lx\n", lockres->l_name, lockres->l_flags); + + /* We only compare against the currently granted level + * here. If the lock is blocked waiting on a downconvert, + * we'll get caught below. */ + if (lockres->l_flags & OCFS2_LOCK_BUSY && + level > lockres->l_level) { + /* is someone sitting in dlm_lock? If so, wait on + * them. */ + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + wait = 1; + goto unlock; + } + + if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) { + /* + * We've upconverted. If the lock now has a level we can + * work with, we take it. If, however, the lock is not at the + * required level, we go thru the full cycle. One way this could + * happen is if a process requesting an upconvert to PR is + * closely followed by another requesting upconvert to an EX. + * If the process requesting EX lands here, we want it to + * continue attempting to upconvert and let the process + * requesting PR take the lock. + * If multiple processes request upconvert to PR, the first one + * here will take the lock. The others will have to go thru the + * OCFS2_LOCK_BLOCKED check to ensure that there is no pending + * downconvert request. + */ + if (level <= lockres->l_level) + goto update_holders; + } + + if (lockres->l_flags & OCFS2_LOCK_BLOCKED && + !ocfs2_may_continue_on_blocked_lock(lockres, level)) { + /* is the lock is currently blocked on behalf of + * another node */ + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); + wait = 1; + goto unlock; + } + + if (level > lockres->l_level) { + if (noqueue_attempted > 0) { + ret = -EAGAIN; + goto unlock; + } + if (lkm_flags & DLM_LKF_NOQUEUE) + noqueue_attempted = 1; + + if (lockres->l_action != OCFS2_AST_INVALID) + mlog(ML_ERROR, "lockres %s has action %u pending\n", + lockres->l_name, lockres->l_action); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + lockres->l_action = OCFS2_AST_ATTACH; + lkm_flags &= ~DLM_LKF_CONVERT; + } else { + lockres->l_action = OCFS2_AST_CONVERT; + lkm_flags |= DLM_LKF_CONVERT; + } + + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + gen = lockres_set_pending(lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + BUG_ON(level == DLM_LOCK_IV); + BUG_ON(level == DLM_LOCK_NL); + + mlog(ML_BASTS, "lockres %s, convert from %d to %d\n", + lockres->l_name, lockres->l_level, level); + + /* call dlm_lock to upgrade lock now */ + ret = ocfs2_dlm_lock(osb->cconn, + level, + &lockres->l_lksb, + lkm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, gen, osb); + if (ret) { + if (!(lkm_flags & DLM_LKF_NOQUEUE) || + (ret != -EAGAIN)) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", + ret, lockres); + } + ocfs2_recover_from_dlm_error(lockres, 1); + goto out; + } + dlm_locked = 1; + + mlog(0, "lock %s, successful return from ocfs2_dlm_lock\n", + lockres->l_name); + + /* At this point we've gone inside the dlm and need to + * complete our work regardless. */ + catch_signals = 0; + + /* wait for busy to clear and carry on */ + goto again; + } + +update_holders: + /* Ok, if we get here then we're good to go. */ + ocfs2_inc_holders(lockres, level); + + ret = 0; +unlock: + lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + + /* ocfs2_unblock_lock reques on seeing OCFS2_LOCK_UPCONVERT_FINISHING */ + kick_dc = (lockres->l_flags & OCFS2_LOCK_BLOCKED); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + if (kick_dc) + ocfs2_wake_downconvert_thread(osb); +out: + /* + * This is helping work around a lock inversion between the page lock + * and dlm locks. One path holds the page lock while calling aops + * which block acquiring dlm locks. The voting thread holds dlm + * locks while acquiring page locks while down converting data locks. + * This block is helping an aop path notice the inversion and back + * off to unlock its page lock before trying the dlm lock again. + */ + if (wait && arg_flags & OCFS2_LOCK_NONBLOCK && + mw.mw_mask & (OCFS2_LOCK_BUSY|OCFS2_LOCK_BLOCKED)) { + wait = 0; + spin_lock_irqsave(&lockres->l_lock, flags); + if (__lockres_remove_mask_waiter(lockres, &mw)) { + if (dlm_locked) + lockres_or_flags(lockres, + OCFS2_LOCK_NONBLOCK_FINISHED); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = -EAGAIN; + } else { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto again; + } + } + if (wait) { + ret = ocfs2_wait_for_mask(&mw); + if (ret == 0) + goto again; + mlog_errno(ret); + } + ocfs2_update_lock_stats(lockres, level, &mw, ret); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (!ret && lockres->l_lockdep_map.key != NULL) { + if (level == DLM_LOCK_PR) + rwsem_acquire_read(&lockres->l_lockdep_map, l_subclass, + !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), + caller_ip); + else + rwsem_acquire(&lockres->l_lockdep_map, l_subclass, + !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), + caller_ip); + } +#endif + return ret; +} + +static inline int ocfs2_cluster_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 lkm_flags, + int arg_flags) +{ + return __ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags, + 0, _RET_IP_); +} + + +static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + unsigned long caller_ip) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + ocfs2_dec_holders(lockres, level); + ocfs2_downconvert_on_unlock(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (lockres->l_lockdep_map.key != NULL) + rwsem_release(&lockres->l_lockdep_map, 1, caller_ip); +#endif +} + +static int ocfs2_create_new_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int ex, + int local) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + unsigned long flags; + u32 lkm_flags = local ? DLM_LKF_LOCAL : 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); + lockres_or_flags(lockres, OCFS2_LOCK_LOCAL); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ocfs2_lock_create(osb, lockres, level, lkm_flags); +} + +/* Grants us an EX lock on the data and metadata resources, skipping + * the normal cluster directory lookup. Use this ONLY on newly created + * inodes which other nodes can't possibly see, and which haven't been + * hashed in the inode hash yet. This can give us a good performance + * increase as it'll skip the network broadcast normally associated + * with creating a new lock resource. */ +int ocfs2_create_new_inode_locks(struct inode *inode) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + BUG_ON(!ocfs2_inode_is_new(inode)); + + mlog(0, "Inode %llu\n", (unsigned long long)OCFS2_I(inode)->ip_blkno); + + /* NOTE: That we don't increment any of the holder counts, nor + * do we add anything to a journal handle. Since this is + * supposed to be a new inode which the cluster doesn't know + * about yet, there is no need to. As far as the LVB handling + * is concerned, this is basically like acquiring an EX lock + * on a resource which has an invalid one -- we'll set it + * valid when we release the EX. */ + + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_rw_lockres, 1, 1); + if (ret) { + mlog_errno(ret); + goto bail; + } + + /* + * We don't want to use DLM_LKF_LOCAL on a meta data lock as they + * don't use a generation in their lock names. + */ + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_inode_lockres, 1, 0); + if (ret) { + mlog_errno(ret); + goto bail; + } + + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_open_lockres, 0, 0); + if (ret) + mlog_errno(ret); + +bail: + return ret; +} + +int ocfs2_rw_lock(struct inode *inode, int write) +{ + int status, level; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu take %s RW lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (ocfs2_mount_local(osb)) + return 0; + + lockres = &OCFS2_I(inode)->ip_rw_lockres; + + level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + + status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, level, 0, + 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +void ocfs2_rw_unlock(struct inode *inode, int write) +{ + int level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_rw_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop %s RW lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, level); +} + +/* + * ocfs2_open_lock always get PR mode lock. + */ +int ocfs2_open_lock(struct inode *inode) +{ + int status = 0; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu take PRMODE open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + if (ocfs2_is_hard_readonly(osb) || ocfs2_mount_local(osb)) + goto out; + + lockres = &OCFS2_I(inode)->ip_open_lockres; + + status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, + DLM_LOCK_PR, 0, 0); + if (status < 0) + mlog_errno(status); + +out: + return status; +} + +int ocfs2_try_open_lock(struct inode *inode, int write) +{ + int status = 0, level; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu try to take %s open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (ocfs2_is_hard_readonly(osb)) { + if (write) + status = -EROFS; + goto out; + } + + if (ocfs2_mount_local(osb)) + goto out; + + lockres = &OCFS2_I(inode)->ip_open_lockres; + + level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + + /* + * The file system may already holding a PRMODE/EXMODE open lock. + * Since we pass DLM_LKF_NOQUEUE, the request won't block waiting on + * other nodes and the -EAGAIN will indicate to the caller that + * this inode is still in use. + */ + status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, + level, DLM_LKF_NOQUEUE, 0); + +out: + return status; +} + +/* + * ocfs2_open_unlock unlock PR and EX mode open locks. + */ +void ocfs2_open_unlock(struct inode *inode) +{ + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_open_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + if (ocfs2_mount_local(osb)) + goto out; + + if(lockres->l_ro_holders) + ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, + DLM_LOCK_PR); + if(lockres->l_ex_holders) + ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, + DLM_LOCK_EX); + +out: + return; +} + +static int ocfs2_flock_handle_signal(struct ocfs2_lock_res *lockres, + int level) +{ + int ret; + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + unsigned long flags; + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + +retry_cancel: + spin_lock_irqsave(&lockres->l_lock, flags); + if (lockres->l_flags & OCFS2_LOCK_BUSY) { + ret = ocfs2_prepare_cancel_convert(osb, lockres); + if (ret) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = ocfs2_cancel_convert(osb, lockres); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + goto retry_cancel; + } + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ocfs2_wait_for_mask(&mw); + goto retry_cancel; + } + + ret = -ERESTARTSYS; + /* + * We may still have gotten the lock, in which case there's no + * point to restarting the syscall. + */ + if (lockres->l_level == level) + ret = 0; + + mlog(0, "Cancel returning %d. flags: 0x%lx, level: %d, act: %d\n", ret, + lockres->l_flags, lockres->l_level, lockres->l_action); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + +out: + return ret; +} + +/* + * ocfs2_file_lock() and ocfs2_file_unlock() map to a single pair of + * flock() calls. The locking approach this requires is sufficiently + * different from all other cluster lock types that we implement a + * separate path to the "low-level" dlm calls. In particular: + * + * - No optimization of lock levels is done - we take at exactly + * what's been requested. + * + * - No lock caching is employed. We immediately downconvert to + * no-lock at unlock time. This also means flock locks never go on + * the blocking list). + * + * - Since userspace can trivially deadlock itself with flock, we make + * sure to allow cancellation of a misbehaving applications flock() + * request. + * + * - Access to any flock lockres doesn't require concurrency, so we + * can simplify the code by requiring the caller to guarantee + * serialization of dlmglue flock calls. + */ +int ocfs2_file_lock(struct file *file, int ex, int trylock) +{ + int ret, level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + unsigned int lkm_flags = trylock ? DLM_LKF_NOQUEUE : 0; + unsigned long flags; + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_lock_res *lockres = &fp->fp_flock; + struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + + if ((lockres->l_flags & OCFS2_LOCK_BUSY) || + (lockres->l_level > DLM_LOCK_NL)) { + mlog(ML_ERROR, + "File lock \"%s\" has busy or locked state: flags: 0x%lx, " + "level: %u\n", lockres->l_name, lockres->l_flags, + lockres->l_level); + return -EINVAL; + } + + spin_lock_irqsave(&lockres->l_lock, flags); + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + /* + * Get the lock at NLMODE to start - that way we + * can cancel the upconvert request if need be. + */ + ret = ocfs2_lock_create(osb, lockres, DLM_LOCK_NL, 0); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_wait_for_mask(&mw); + if (ret) { + mlog_errno(ret); + goto out; + } + spin_lock_irqsave(&lockres->l_lock, flags); + } + + lockres->l_action = OCFS2_AST_CONVERT; + lkm_flags |= DLM_LKF_CONVERT; + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_dlm_lock(osb->cconn, level, &lockres->l_lksb, lkm_flags, + lockres->l_name, OCFS2_LOCK_ID_MAX_LEN - 1); + if (ret) { + if (!trylock || (ret != -EAGAIN)) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ret = -EINVAL; + } + + ocfs2_recover_from_dlm_error(lockres, 1); + lockres_remove_mask_waiter(lockres, &mw); + goto out; + } + + ret = ocfs2_wait_for_mask_interruptible(&mw, lockres); + if (ret == -ERESTARTSYS) { + /* + * Userspace can cause deadlock itself with + * flock(). Current behavior locally is to allow the + * deadlock, but abort the system call if a signal is + * received. We follow this example, otherwise a + * poorly written program could sit in kernel until + * reboot. + * + * Handling this is a bit more complicated for Ocfs2 + * though. We can't exit this function with an + * outstanding lock request, so a cancel convert is + * required. We intentionally overwrite 'ret' - if the + * cancel fails and the lock was granted, it's easier + * to just bubble success back up to the user. + */ + ret = ocfs2_flock_handle_signal(lockres, level); + } else if (!ret && (level > lockres->l_level)) { + /* Trylock failed asynchronously */ + BUG_ON(!trylock); + ret = -EAGAIN; + } + +out: + + mlog(0, "Lock: \"%s\" ex: %d, trylock: %d, returns: %d\n", + lockres->l_name, ex, trylock, ret); + return ret; +} + +void ocfs2_file_unlock(struct file *file) +{ + int ret; + unsigned int gen; + unsigned long flags; + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_lock_res *lockres = &fp->fp_flock; + struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) + return; + + if (lockres->l_level == DLM_LOCK_NL) + return; + + mlog(0, "Unlock: \"%s\" flags: 0x%lx, level: %d, act: %d\n", + lockres->l_name, lockres->l_flags, lockres->l_level, + lockres->l_action); + + spin_lock_irqsave(&lockres->l_lock, flags); + /* + * Fake a blocking ast for the downconvert code. + */ + lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); + lockres->l_blocking = DLM_LOCK_EX; + + gen = ocfs2_prepare_downconvert(lockres, DLM_LOCK_NL); + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_downconvert_lock(osb, lockres, DLM_LOCK_NL, 0, gen); + if (ret) { + mlog_errno(ret); + return; + } + + ret = ocfs2_wait_for_mask(&mw); + if (ret) + mlog_errno(ret); +} + +static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int kick = 0; + + /* If we know that another node is waiting on our lock, kick + * the downconvert thread * pre-emptively when we reach a release + * condition. */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) { + switch(lockres->l_blocking) { + case DLM_LOCK_EX: + if (!lockres->l_ex_holders && !lockres->l_ro_holders) + kick = 1; + break; + case DLM_LOCK_PR: + if (!lockres->l_ex_holders) + kick = 1; + break; + default: + BUG(); + } + } + + if (kick) + ocfs2_wake_downconvert_thread(osb); +} + +#define OCFS2_SEC_BITS 34 +#define OCFS2_SEC_SHIFT (64 - 34) +#define OCFS2_NSEC_MASK ((1ULL << OCFS2_SEC_SHIFT) - 1) + +/* LVB only has room for 64 bits of time here so we pack it for + * now. */ +static u64 ocfs2_pack_timespec(struct timespec *spec) +{ + u64 res; + u64 sec = spec->tv_sec; + u32 nsec = spec->tv_nsec; + + res = (sec << OCFS2_SEC_SHIFT) | (nsec & OCFS2_NSEC_MASK); + + return res; +} + +/* Call this with the lockres locked. I am reasonably sure we don't + * need ip_lock in this function as anyone who would be changing those + * values is supposed to be blocked in ocfs2_inode_lock right now. */ +static void __ocfs2_stuff_meta_lvb(struct inode *inode) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_meta_lvb *lvb; + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + /* + * Invalidate the LVB of a deleted inode - this way other + * nodes are forced to go to disk and discover the new inode + * status. + */ + if (oi->ip_flags & OCFS2_INODE_DELETED) { + lvb->lvb_version = 0; + goto out; + } + + lvb->lvb_version = OCFS2_LVB_VERSION; + lvb->lvb_isize = cpu_to_be64(i_size_read(inode)); + lvb->lvb_iclusters = cpu_to_be32(oi->ip_clusters); + lvb->lvb_iuid = cpu_to_be32(i_uid_read(inode)); + lvb->lvb_igid = cpu_to_be32(i_gid_read(inode)); + lvb->lvb_imode = cpu_to_be16(inode->i_mode); + lvb->lvb_inlink = cpu_to_be16(inode->i_nlink); + lvb->lvb_iatime_packed = + cpu_to_be64(ocfs2_pack_timespec(&inode->i_atime)); + lvb->lvb_ictime_packed = + cpu_to_be64(ocfs2_pack_timespec(&inode->i_ctime)); + lvb->lvb_imtime_packed = + cpu_to_be64(ocfs2_pack_timespec(&inode->i_mtime)); + lvb->lvb_iattr = cpu_to_be32(oi->ip_attr); + lvb->lvb_idynfeatures = cpu_to_be16(oi->ip_dyn_features); + lvb->lvb_igeneration = cpu_to_be32(inode->i_generation); + +out: + mlog_meta_lvb(0, lockres); +} + +static void ocfs2_unpack_timespec(struct timespec *spec, + u64 packed_time) +{ + spec->tv_sec = packed_time >> OCFS2_SEC_SHIFT; + spec->tv_nsec = packed_time & OCFS2_NSEC_MASK; +} + +static void ocfs2_refresh_inode_from_lvb(struct inode *inode) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_meta_lvb *lvb; + + mlog_meta_lvb(0, lockres); + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + /* We're safe here without the lockres lock... */ + spin_lock(&oi->ip_lock); + oi->ip_clusters = be32_to_cpu(lvb->lvb_iclusters); + i_size_write(inode, be64_to_cpu(lvb->lvb_isize)); + + oi->ip_attr = be32_to_cpu(lvb->lvb_iattr); + oi->ip_dyn_features = be16_to_cpu(lvb->lvb_idynfeatures); + ocfs2_set_inode_flags(inode); + + /* fast-symlinks are a special case */ + if (S_ISLNK(inode->i_mode) && !oi->ip_clusters) + inode->i_blocks = 0; + else + inode->i_blocks = ocfs2_inode_sector_count(inode); + + i_uid_write(inode, be32_to_cpu(lvb->lvb_iuid)); + i_gid_write(inode, be32_to_cpu(lvb->lvb_igid)); + inode->i_mode = be16_to_cpu(lvb->lvb_imode); + set_nlink(inode, be16_to_cpu(lvb->lvb_inlink)); + ocfs2_unpack_timespec(&inode->i_atime, + be64_to_cpu(lvb->lvb_iatime_packed)); + ocfs2_unpack_timespec(&inode->i_mtime, + be64_to_cpu(lvb->lvb_imtime_packed)); + ocfs2_unpack_timespec(&inode->i_ctime, + be64_to_cpu(lvb->lvb_ictime_packed)); + spin_unlock(&oi->ip_lock); +} + +static inline int ocfs2_meta_lvb_is_trustable(struct inode *inode, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) + && lvb->lvb_version == OCFS2_LVB_VERSION + && be32_to_cpu(lvb->lvb_igeneration) == inode->i_generation) + return 1; + return 0; +} + +/* Determine whether a lock resource needs to be refreshed, and + * arbitrate who gets to refresh it. + * + * 0 means no refresh needed. + * + * > 0 means you need to refresh this and you MUST call + * ocfs2_complete_lock_res_refresh afterwards. */ +static int ocfs2_should_refresh_lock_res(struct ocfs2_lock_res *lockres) +{ + unsigned long flags; + int status = 0; + +refresh_check: + spin_lock_irqsave(&lockres->l_lock, flags); + if (!(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto bail; + } + + if (lockres->l_flags & OCFS2_LOCK_REFRESHING) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ocfs2_wait_on_refreshing_lock(lockres); + goto refresh_check; + } + + /* Ok, I'll be the one to refresh this lock. */ + lockres_or_flags(lockres, OCFS2_LOCK_REFRESHING); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + status = 1; +bail: + mlog(0, "status %d\n", status); + return status; +} + +/* If status is non zero, I'll mark it as not being in refresh + * anymroe, but i won't clear the needs refresh flag. */ +static inline void ocfs2_complete_lock_res_refresh(struct ocfs2_lock_res *lockres, + int status) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres_clear_flags(lockres, OCFS2_LOCK_REFRESHING); + if (!status) + lockres_clear_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); +} + +/* may or may not return a bh if it went to disk. */ +static int ocfs2_inode_lock_update(struct inode *inode, + struct buffer_head **bh) +{ + int status = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_dinode *fe; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (ocfs2_mount_local(osb)) + goto bail; + + spin_lock(&oi->ip_lock); + if (oi->ip_flags & OCFS2_INODE_DELETED) { + mlog(0, "Orphaned inode %llu was deleted while we " + "were waiting on a lock. ip_flags = 0x%x\n", + (unsigned long long)oi->ip_blkno, oi->ip_flags); + spin_unlock(&oi->ip_lock); + status = -ENOENT; + goto bail; + } + spin_unlock(&oi->ip_lock); + + if (!ocfs2_should_refresh_lock_res(lockres)) + goto bail; + + /* This will discard any caching information we might have had + * for the inode metadata. */ + ocfs2_metadata_cache_purge(INODE_CACHE(inode)); + + ocfs2_extent_map_trunc(inode, 0); + + if (ocfs2_meta_lvb_is_trustable(inode, lockres)) { + mlog(0, "Trusting LVB on inode %llu\n", + (unsigned long long)oi->ip_blkno); + ocfs2_refresh_inode_from_lvb(inode); + } else { + /* Boo, we have to go to disk. */ + /* read bh, cast, ocfs2_refresh_inode */ + status = ocfs2_read_inode_block(inode, bh); + if (status < 0) { + mlog_errno(status); + goto bail_refresh; + } + fe = (struct ocfs2_dinode *) (*bh)->b_data; + + /* This is a good chance to make sure we're not + * locking an invalid object. ocfs2_read_inode_block() + * already checked that the inode block is sane. + * + * We bug on a stale inode here because we checked + * above whether it was wiped from disk. The wiping + * node provides a guarantee that we receive that + * message and can mark the inode before dropping any + * locks associated with it. */ + mlog_bug_on_msg(inode->i_generation != + le32_to_cpu(fe->i_generation), + "Invalid dinode %llu disk generation: %u " + "inode->i_generation: %u\n", + (unsigned long long)oi->ip_blkno, + le32_to_cpu(fe->i_generation), + inode->i_generation); + mlog_bug_on_msg(le64_to_cpu(fe->i_dtime) || + !(fe->i_flags & cpu_to_le32(OCFS2_VALID_FL)), + "Stale dinode %llu dtime: %llu flags: 0x%x\n", + (unsigned long long)oi->ip_blkno, + (unsigned long long)le64_to_cpu(fe->i_dtime), + le32_to_cpu(fe->i_flags)); + + ocfs2_refresh_inode(inode, fe); + ocfs2_track_lock_refresh(lockres); + } + + status = 0; +bail_refresh: + ocfs2_complete_lock_res_refresh(lockres, status); +bail: + return status; +} + +static int ocfs2_assign_bh(struct inode *inode, + struct buffer_head **ret_bh, + struct buffer_head *passed_bh) +{ + int status; + + if (passed_bh) { + /* Ok, the update went to disk for us, use the + * returned bh. */ + *ret_bh = passed_bh; + get_bh(*ret_bh); + + return 0; + } + + status = ocfs2_read_inode_block(inode, ret_bh); + if (status < 0) + mlog_errno(status); + + return status; +} + +/* + * returns < 0 error if the callback will never be called, otherwise + * the result of the lock will be communicated via the callback. + */ +int ocfs2_inode_lock_full_nested(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + int arg_flags, + int subclass) +{ + int status, level, acquired; + u32 dlm_flags; + struct ocfs2_lock_res *lockres = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *local_bh = NULL; + + mlog(0, "inode %llu, take %s META lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + ex ? "EXMODE" : "PRMODE"); + + status = 0; + acquired = 0; + /* We'll allow faking a readonly metadata lock for + * rodevices. */ + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + status = -EROFS; + goto getbh; + } + + if ((arg_flags & OCFS2_META_LOCK_GETBH) || + ocfs2_mount_local(osb)) + goto update; + + if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) + ocfs2_wait_for_recovery(osb); + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + dlm_flags = 0; + if (arg_flags & OCFS2_META_LOCK_NOQUEUE) + dlm_flags |= DLM_LKF_NOQUEUE; + + status = __ocfs2_cluster_lock(osb, lockres, level, dlm_flags, + arg_flags, subclass, _RET_IP_); + if (status < 0) { + if (status != -EAGAIN) + mlog_errno(status); + goto bail; + } + + /* Notify the error cleanup path to drop the cluster lock. */ + acquired = 1; + + /* We wait twice because a node may have died while we were in + * the lower dlm layers. The second time though, we've + * committed to owning this lock so we don't allow signals to + * abort the operation. */ + if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) + ocfs2_wait_for_recovery(osb); + +update: + /* + * We only see this flag if we're being called from + * ocfs2_read_locked_inode(). It means we're locking an inode + * which hasn't been populated yet, so clear the refresh flag + * and let the caller handle it. + */ + if (inode->i_state & I_NEW) { + status = 0; + if (lockres) + ocfs2_complete_lock_res_refresh(lockres, 0); + goto bail; + } + + /* This is fun. The caller may want a bh back, or it may + * not. ocfs2_inode_lock_update definitely wants one in, but + * may or may not read one, depending on what's in the + * LVB. The result of all of this is that we've *only* gone to + * disk if we have to, so the complexity is worthwhile. */ + status = ocfs2_inode_lock_update(inode, &local_bh); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } +getbh: + if (ret_bh) { + status = ocfs2_assign_bh(inode, ret_bh, local_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + +bail: + if (status < 0) { + if (ret_bh && (*ret_bh)) { + brelse(*ret_bh); + *ret_bh = NULL; + } + if (acquired) + ocfs2_inode_unlock(inode, ex); + } + + if (local_bh) + brelse(local_bh); + + return status; +} + +/* + * This is working around a lock inversion between tasks acquiring DLM + * locks while holding a page lock and the downconvert thread which + * blocks dlm lock acquiry while acquiring page locks. + * + * ** These _with_page variantes are only intended to be called from aop + * methods that hold page locks and return a very specific *positive* error + * code that aop methods pass up to the VFS -- test for errors with != 0. ** + * + * The DLM is called such that it returns -EAGAIN if it would have + * blocked waiting for the downconvert thread. In that case we unlock + * our page so the downconvert thread can make progress. Once we've + * done this we have to return AOP_TRUNCATED_PAGE so the aop method + * that called us can bubble that back up into the VFS who will then + * immediately retry the aop call. + */ +int ocfs2_inode_lock_with_page(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct page *page) +{ + int ret; + + ret = ocfs2_inode_lock_full(inode, ret_bh, ex, OCFS2_LOCK_NONBLOCK); + if (ret == -EAGAIN) { + unlock_page(page); + ret = AOP_TRUNCATED_PAGE; + } + + return ret; +} + +int ocfs2_inode_lock_atime(struct inode *inode, + struct vfsmount *vfsmnt, + int *level) +{ + int ret; + + ret = ocfs2_inode_lock(inode, NULL, 0); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + /* + * If we should update atime, we will get EX lock, + * otherwise we just get PR lock. + */ + if (ocfs2_should_update_atime(inode, vfsmnt)) { + struct buffer_head *bh = NULL; + + ocfs2_inode_unlock(inode, 0); + ret = ocfs2_inode_lock(inode, &bh, 1); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + *level = 1; + if (ocfs2_should_update_atime(inode, vfsmnt)) + ocfs2_update_inode_atime(inode, bh); + if (bh) + brelse(bh); + } else + *level = 0; + + return ret; +} + +void ocfs2_inode_unlock(struct inode *inode, + int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_inode_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop %s META lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + ex ? "EXMODE" : "PRMODE"); + + if (!ocfs2_is_hard_readonly(OCFS2_SB(inode->i_sb)) && + !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, level); +} + +/* + * This _tracker variantes are introduced to deal with the recursive cluster + * locking issue. The idea is to keep track of a lock holder on the stack of + * the current process. If there's a lock holder on the stack, we know the + * task context is already protected by cluster locking. Currently, they're + * used in some VFS entry routines. + * + * return < 0 on error, return == 0 if there's no lock holder on the stack + * before this call, return == 1 if this call would be a recursive locking. + */ +int ocfs2_inode_lock_tracker(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct ocfs2_lock_holder *oh) +{ + int status; + int arg_flags = 0, has_locked; + struct ocfs2_lock_res *lockres; + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + has_locked = ocfs2_is_locked_by_me(lockres); + /* Just get buffer head if the cluster lock has been taken */ + if (has_locked) + arg_flags = OCFS2_META_LOCK_GETBH; + + if (likely(!has_locked || ret_bh)) { + status = ocfs2_inode_lock_full(inode, ret_bh, ex, arg_flags); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + } + if (!has_locked) + ocfs2_add_holder(lockres, oh); + + return has_locked; +} + +void ocfs2_inode_unlock_tracker(struct inode *inode, + int ex, + struct ocfs2_lock_holder *oh, + int had_lock) +{ + struct ocfs2_lock_res *lockres; + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + /* had_lock means that the currect process already takes the cluster + * lock previously. If had_lock is 1, we have nothing to do here, and + * it will get unlocked where we got the lock. + */ + if (!had_lock) { + ocfs2_remove_holder(lockres, oh); + ocfs2_inode_unlock(inode, ex); + } +} + +int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno) +{ + struct ocfs2_lock_res *lockres; + struct ocfs2_orphan_scan_lvb *lvb; + int status = 0; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + lockres = &osb->osb_orphan_scan.os_lockres; + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); + if (status < 0) + return status; + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && + lvb->lvb_version == OCFS2_ORPHAN_LVB_VERSION) + *seqno = be32_to_cpu(lvb->lvb_os_seqno); + else + *seqno = osb->osb_orphan_scan.os_seqno + 1; + + return status; +} + +void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno) +{ + struct ocfs2_lock_res *lockres; + struct ocfs2_orphan_scan_lvb *lvb; + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) { + lockres = &osb->osb_orphan_scan.os_lockres; + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + lvb->lvb_version = OCFS2_ORPHAN_LVB_VERSION; + lvb->lvb_os_seqno = cpu_to_be32(seqno); + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); + } +} + +int ocfs2_super_lock(struct ocfs2_super *osb, + int ex) +{ + int status = 0; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + goto bail; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* The super block lock path is really in the best position to + * know when resources covered by the lock need to be + * refreshed, so we do it here. Of course, making sense of + * everything is up to the caller :) */ + status = ocfs2_should_refresh_lock_res(lockres); + if (status) { + status = ocfs2_refresh_slot_info(osb); + + ocfs2_complete_lock_res_refresh(lockres, status); + + if (status < 0) { + ocfs2_cluster_unlock(osb, lockres, level); + mlog_errno(status); + } + ocfs2_track_lock_refresh(lockres); + } +bail: + return status; +} + +void ocfs2_super_unlock(struct ocfs2_super *osb, + int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +int ocfs2_rename_lock(struct ocfs2_super *osb) +{ + int status; + struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +void ocfs2_rename_unlock(struct ocfs2_super *osb) +{ + struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); +} + +int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex) +{ + int status; + struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, ex ? LKM_EXMODE : LKM_PRMODE, + 0, 0); + if (status < 0) + mlog(ML_ERROR, "lock on nfs sync lock failed %d\n", status); + + return status; +} + +void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex) +{ + struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, + ex ? LKM_EXMODE : LKM_PRMODE); +} + +int ocfs2_dentry_lock(struct dentry *dentry, int ex) +{ + int ret; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); + + BUG_ON(!dl); + + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + return -EROFS; + return 0; + } + + if (ocfs2_mount_local(osb)) + return 0; + + ret = ocfs2_cluster_lock(osb, &dl->dl_lockres, level, 0, 0); + if (ret < 0) + mlog_errno(ret); + + return ret; +} + +void ocfs2_dentry_unlock(struct dentry *dentry, int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, &dl->dl_lockres, level); +} + +/* Reference counting of the dlm debug structure. We want this because + * open references on the debug inodes can live on after a mount, so + * we can't rely on the ocfs2_super to always exist. */ +static void ocfs2_dlm_debug_free(struct kref *kref) +{ + struct ocfs2_dlm_debug *dlm_debug; + + dlm_debug = container_of(kref, struct ocfs2_dlm_debug, d_refcnt); + + kfree(dlm_debug); +} + +void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug) +{ + if (dlm_debug) + kref_put(&dlm_debug->d_refcnt, ocfs2_dlm_debug_free); +} + +static void ocfs2_get_dlm_debug(struct ocfs2_dlm_debug *debug) +{ + kref_get(&debug->d_refcnt); +} + +struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void) +{ + struct ocfs2_dlm_debug *dlm_debug; + + dlm_debug = kmalloc(sizeof(struct ocfs2_dlm_debug), GFP_KERNEL); + if (!dlm_debug) { + mlog_errno(-ENOMEM); + goto out; + } + + kref_init(&dlm_debug->d_refcnt); + INIT_LIST_HEAD(&dlm_debug->d_lockres_tracking); + dlm_debug->d_locking_state = NULL; +out: + return dlm_debug; +} + +/* Access to this is arbitrated for us via seq_file->sem. */ +struct ocfs2_dlm_seq_priv { + struct ocfs2_dlm_debug *p_dlm_debug; + struct ocfs2_lock_res p_iter_res; + struct ocfs2_lock_res p_tmp_res; +}; + +static struct ocfs2_lock_res *ocfs2_dlm_next_res(struct ocfs2_lock_res *start, + struct ocfs2_dlm_seq_priv *priv) +{ + struct ocfs2_lock_res *iter, *ret = NULL; + struct ocfs2_dlm_debug *dlm_debug = priv->p_dlm_debug; + + assert_spin_locked(&ocfs2_dlm_tracking_lock); + + list_for_each_entry(iter, &start->l_debug_list, l_debug_list) { + /* discover the head of the list */ + if (&iter->l_debug_list == &dlm_debug->d_lockres_tracking) { + mlog(0, "End of list found, %p\n", ret); + break; + } + + /* We track our "dummy" iteration lockres' by a NULL + * l_ops field. */ + if (iter->l_ops != NULL) { + ret = iter; + break; + } + } + + return ret; +} + +static void *ocfs2_dlm_seq_start(struct seq_file *m, loff_t *pos) +{ + struct ocfs2_dlm_seq_priv *priv = m->private; + struct ocfs2_lock_res *iter; + + spin_lock(&ocfs2_dlm_tracking_lock); + iter = ocfs2_dlm_next_res(&priv->p_iter_res, priv); + if (iter) { + /* Since lockres' have the lifetime of their container + * (which can be inodes, ocfs2_supers, etc) we want to + * copy this out to a temporary lockres while still + * under the spinlock. Obviously after this we can't + * trust any pointers on the copy returned, but that's + * ok as the information we want isn't typically held + * in them. */ + priv->p_tmp_res = *iter; + iter = &priv->p_tmp_res; + } + spin_unlock(&ocfs2_dlm_tracking_lock); + + return iter; +} + +static void ocfs2_dlm_seq_stop(struct seq_file *m, void *v) +{ +} + +static void *ocfs2_dlm_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct ocfs2_dlm_seq_priv *priv = m->private; + struct ocfs2_lock_res *iter = v; + struct ocfs2_lock_res *dummy = &priv->p_iter_res; + + spin_lock(&ocfs2_dlm_tracking_lock); + iter = ocfs2_dlm_next_res(iter, priv); + list_del_init(&dummy->l_debug_list); + if (iter) { + list_add(&dummy->l_debug_list, &iter->l_debug_list); + priv->p_tmp_res = *iter; + iter = &priv->p_tmp_res; + } + spin_unlock(&ocfs2_dlm_tracking_lock); + + return iter; +} + +/* + * Version is used by debugfs.ocfs2 to determine the format being used + * + * New in version 2 + * - Lock stats printed + * New in version 3 + * - Max time in lock stats is in usecs (instead of nsecs) + */ +#define OCFS2_DLM_DEBUG_STR_VERSION 3 +static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) +{ + int i; + char *lvb; + struct ocfs2_lock_res *lockres = v; + + if (!lockres) + return -EINVAL; + + seq_printf(m, "0x%x\t", OCFS2_DLM_DEBUG_STR_VERSION); + + if (lockres->l_type == OCFS2_LOCK_TYPE_DENTRY) + seq_printf(m, "%.*s%08x\t", OCFS2_DENTRY_LOCK_INO_START - 1, + lockres->l_name, + (unsigned int)ocfs2_get_dentry_lock_ino(lockres)); + else + seq_printf(m, "%.*s\t", OCFS2_LOCK_ID_MAX_LEN, lockres->l_name); + + seq_printf(m, "%d\t" + "0x%lx\t" + "0x%x\t" + "0x%x\t" + "%u\t" + "%u\t" + "%d\t" + "%d\t", + lockres->l_level, + lockres->l_flags, + lockres->l_action, + lockres->l_unlock_action, + lockres->l_ro_holders, + lockres->l_ex_holders, + lockres->l_requested, + lockres->l_blocking); + + /* Dump the raw LVB */ + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + for(i = 0; i < DLM_LVB_LEN; i++) + seq_printf(m, "0x%x\t", lvb[i]); + +#ifdef CONFIG_OCFS2_FS_STATS +# define lock_num_prmode(_l) ((_l)->l_lock_prmode.ls_gets) +# define lock_num_exmode(_l) ((_l)->l_lock_exmode.ls_gets) +# define lock_num_prmode_failed(_l) ((_l)->l_lock_prmode.ls_fail) +# define lock_num_exmode_failed(_l) ((_l)->l_lock_exmode.ls_fail) +# define lock_total_prmode(_l) ((_l)->l_lock_prmode.ls_total) +# define lock_total_exmode(_l) ((_l)->l_lock_exmode.ls_total) +# define lock_max_prmode(_l) ((_l)->l_lock_prmode.ls_max) +# define lock_max_exmode(_l) ((_l)->l_lock_exmode.ls_max) +# define lock_refresh(_l) ((_l)->l_lock_refresh) +#else +# define lock_num_prmode(_l) (0) +# define lock_num_exmode(_l) (0) +# define lock_num_prmode_failed(_l) (0) +# define lock_num_exmode_failed(_l) (0) +# define lock_total_prmode(_l) (0ULL) +# define lock_total_exmode(_l) (0ULL) +# define lock_max_prmode(_l) (0) +# define lock_max_exmode(_l) (0) +# define lock_refresh(_l) (0) +#endif + /* The following seq_print was added in version 2 of this output */ + seq_printf(m, "%u\t" + "%u\t" + "%u\t" + "%u\t" + "%llu\t" + "%llu\t" + "%u\t" + "%u\t" + "%u\t", + lock_num_prmode(lockres), + lock_num_exmode(lockres), + lock_num_prmode_failed(lockres), + lock_num_exmode_failed(lockres), + lock_total_prmode(lockres), + lock_total_exmode(lockres), + lock_max_prmode(lockres), + lock_max_exmode(lockres), + lock_refresh(lockres)); + + /* End the line */ + seq_printf(m, "\n"); + return 0; +} + +static const struct seq_operations ocfs2_dlm_seq_ops = { + .start = ocfs2_dlm_seq_start, + .stop = ocfs2_dlm_seq_stop, + .next = ocfs2_dlm_seq_next, + .show = ocfs2_dlm_seq_show, +}; + +static int ocfs2_dlm_debug_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct ocfs2_dlm_seq_priv *priv = seq->private; + struct ocfs2_lock_res *res = &priv->p_iter_res; + + ocfs2_remove_lockres_tracking(res); + ocfs2_put_dlm_debug(priv->p_dlm_debug); + return seq_release_private(inode, file); +} + +static int ocfs2_dlm_debug_open(struct inode *inode, struct file *file) +{ + struct ocfs2_dlm_seq_priv *priv; + struct ocfs2_super *osb; + + priv = __seq_open_private(file, &ocfs2_dlm_seq_ops, sizeof(*priv)); + if (!priv) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + osb = inode->i_private; + ocfs2_get_dlm_debug(osb->osb_dlm_debug); + priv->p_dlm_debug = osb->osb_dlm_debug; + INIT_LIST_HEAD(&priv->p_iter_res.l_debug_list); + + ocfs2_add_lockres_tracking(&priv->p_iter_res, + priv->p_dlm_debug); + + return 0; +} + +static const struct file_operations ocfs2_dlm_debug_fops = { + .open = ocfs2_dlm_debug_open, + .release = ocfs2_dlm_debug_release, + .read = seq_read, + .llseek = seq_lseek, +}; + +static int ocfs2_dlm_init_debug(struct ocfs2_super *osb) +{ + int ret = 0; + struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; + + dlm_debug->d_locking_state = debugfs_create_file("locking_state", + S_IFREG|S_IRUSR, + osb->osb_debug_root, + osb, + &ocfs2_dlm_debug_fops); + if (!dlm_debug->d_locking_state) { + ret = -EINVAL; + mlog(ML_ERROR, + "Unable to create locking state debugfs file.\n"); + goto out; + } + + ocfs2_get_dlm_debug(dlm_debug); +out: + return ret; +} + +static void ocfs2_dlm_shutdown_debug(struct ocfs2_super *osb) +{ + struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; + + if (dlm_debug) { + debugfs_remove(dlm_debug->d_locking_state); + ocfs2_put_dlm_debug(dlm_debug); + } +} + +int ocfs2_dlm_init(struct ocfs2_super *osb) +{ + int status = 0; + struct ocfs2_cluster_connection *conn = NULL; + + if (ocfs2_mount_local(osb)) { + osb->node_num = 0; + goto local; + } + + status = ocfs2_dlm_init_debug(osb); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* launch downconvert thread */ + osb->dc_task = kthread_run(ocfs2_downconvert_thread, osb, "ocfs2dc-%s", + osb->uuid_str); + if (IS_ERR(osb->dc_task)) { + status = PTR_ERR(osb->dc_task); + osb->dc_task = NULL; + mlog_errno(status); + goto bail; + } + + /* for now, uuid == domain */ + status = ocfs2_cluster_connect(osb->osb_cluster_stack, + osb->osb_cluster_name, + strlen(osb->osb_cluster_name), + osb->uuid_str, + strlen(osb->uuid_str), + &lproto, ocfs2_do_node_down, osb, + &conn); + if (status) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_cluster_this_node(conn, &osb->node_num); + if (status < 0) { + mlog_errno(status); + mlog(ML_ERROR, + "could not find this host's node number\n"); + ocfs2_cluster_disconnect(conn, 0); + goto bail; + } + +local: + ocfs2_super_lock_res_init(&osb->osb_super_lockres, osb); + ocfs2_rename_lock_res_init(&osb->osb_rename_lockres, osb); + ocfs2_nfs_sync_lock_res_init(&osb->osb_nfs_sync_lockres, osb); + ocfs2_orphan_scan_lock_res_init(&osb->osb_orphan_scan.os_lockres, osb); + + osb->cconn = conn; +bail: + if (status < 0) { + ocfs2_dlm_shutdown_debug(osb); + if (osb->dc_task) + kthread_stop(osb->dc_task); + } + + return status; +} + +void ocfs2_dlm_shutdown(struct ocfs2_super *osb, + int hangup_pending) +{ + ocfs2_drop_osb_locks(osb); + + /* + * Now that we have dropped all locks and ocfs2_dismount_volume() + * has disabled recovery, the DLM won't be talking to us. It's + * safe to tear things down before disconnecting the cluster. + */ + + if (osb->dc_task) { + kthread_stop(osb->dc_task); + osb->dc_task = NULL; + } + + ocfs2_lock_res_free(&osb->osb_super_lockres); + ocfs2_lock_res_free(&osb->osb_rename_lockres); + ocfs2_lock_res_free(&osb->osb_nfs_sync_lockres); + ocfs2_lock_res_free(&osb->osb_orphan_scan.os_lockres); + + ocfs2_cluster_disconnect(osb->cconn, hangup_pending); + osb->cconn = NULL; + + ocfs2_dlm_shutdown_debug(osb); +} + +static int ocfs2_drop_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + unsigned long flags; + u32 lkm_flags = 0; + + /* We didn't get anywhere near actually using this lockres. */ + if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) + goto out; + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lkm_flags |= DLM_LKF_VALBLK; + + spin_lock_irqsave(&lockres->l_lock, flags); + + mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_FREEING), + "lockres %s, flags 0x%lx\n", + lockres->l_name, lockres->l_flags); + + while (lockres->l_flags & OCFS2_LOCK_BUSY) { + mlog(0, "waiting on busy lock \"%s\": flags = %lx, action = " + "%u, unlock_action = %u\n", + lockres->l_name, lockres->l_flags, lockres->l_action, + lockres->l_unlock_action); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + + /* XXX: Today we just wait on any busy + * locks... Perhaps we need to cancel converts in the + * future? */ + ocfs2_wait_on_busy_lock(lockres); + + spin_lock_irqsave(&lockres->l_lock, flags); + } + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { + if (lockres->l_flags & OCFS2_LOCK_ATTACHED && + lockres->l_level == DLM_LOCK_EX && + !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) + lockres->l_ops->set_lvb(lockres); + } + + if (lockres->l_flags & OCFS2_LOCK_BUSY) + mlog(ML_ERROR, "destroying busy lock: \"%s\"\n", + lockres->l_name); + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + mlog(0, "destroying blocked lock: \"%s\"\n", lockres->l_name); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto out; + } + + lockres_clear_flags(lockres, OCFS2_LOCK_ATTACHED); + + /* make sure we never get here while waiting for an ast to + * fire. */ + BUG_ON(lockres->l_action != OCFS2_AST_INVALID); + + /* is this necessary? */ + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + lockres->l_unlock_action = OCFS2_UNLOCK_DROP_LOCK; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "lock %s\n", lockres->l_name); + + ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, lkm_flags); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); + mlog(ML_ERROR, "lockres flags: %lu\n", lockres->l_flags); + ocfs2_dlm_dump_lksb(&lockres->l_lksb); + BUG(); + } + mlog(0, "lock %s, successful return from ocfs2_dlm_unlock\n", + lockres->l_name); + + ocfs2_wait_on_busy_lock(lockres); +out: + return 0; +} + +static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +/* Mark the lockres as being dropped. It will no longer be + * queued if blocking, but we still may have to wait on it + * being dequeued from the downconvert thread before we can consider + * it safe to drop. + * + * You can *not* attempt to call cluster_lock on this lockres anymore. */ +void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int status; + struct ocfs2_mask_waiter mw; + unsigned long flags, flags2; + + ocfs2_init_mask_waiter(&mw); + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres->l_flags |= OCFS2_LOCK_FREEING; + if (lockres->l_flags & OCFS2_LOCK_QUEUED && current == osb->dc_task) { + /* + * We know the downconvert is queued but not in progress + * because we are the downconvert thread and processing + * different lock. So we can just remove the lock from the + * queue. This is not only an optimization but also a way + * to avoid the following deadlock: + * ocfs2_dentry_post_unlock() + * ocfs2_dentry_lock_put() + * ocfs2_drop_dentry_lock() + * iput() + * ocfs2_evict_inode() + * ocfs2_clear_inode() + * ocfs2_mark_lockres_freeing() + * ... blocks waiting for OCFS2_LOCK_QUEUED + * since we are the downconvert thread which + * should clear the flag. + */ + spin_unlock_irqrestore(&lockres->l_lock, flags); + spin_lock_irqsave(&osb->dc_task_lock, flags2); + list_del_init(&lockres->l_blocked_list); + osb->blocked_lock_count--; + spin_unlock_irqrestore(&osb->dc_task_lock, flags2); + /* + * Warn if we recurse into another post_unlock call. Strictly + * speaking it isn't a problem but we need to be careful if + * that happens (stack overflow, deadlocks, ...) so warn if + * ocfs2 grows a path for which this can happen. + */ + WARN_ON_ONCE(lockres->l_ops->post_unlock); + /* Since the lock is freeing we don't do much in the fn below */ + ocfs2_process_blocked_lock(osb, lockres); + return; + } + while (lockres->l_flags & OCFS2_LOCK_QUEUED) { + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_QUEUED, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "Waiting on lockres %s\n", lockres->l_name); + + status = ocfs2_wait_for_mask(&mw); + if (status) + mlog_errno(status); + + spin_lock_irqsave(&lockres->l_lock, flags); + } + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ocfs2_mark_lockres_freeing(osb, lockres); + ret = ocfs2_drop_lock(osb, lockres); + if (ret) + mlog_errno(ret); +} + +static void ocfs2_drop_osb_locks(struct ocfs2_super *osb) +{ + ocfs2_simple_drop_lockres(osb, &osb->osb_super_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_rename_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_nfs_sync_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_orphan_scan.os_lockres); +} + +int ocfs2_drop_inode_locks(struct inode *inode) +{ + int status, err; + + /* No need to call ocfs2_mark_lockres_freeing here - + * ocfs2_clear_inode has done it for us. */ + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_open_lockres); + if (err < 0) + mlog_errno(err); + + status = err; + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_inode_lockres); + if (err < 0) + mlog_errno(err); + if (err < 0 && !status) + status = err; + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_rw_lockres); + if (err < 0) + mlog_errno(err); + if (err < 0 && !status) + status = err; + + return status; +} + +static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + assert_spin_locked(&lockres->l_lock); + + BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); + + if (lockres->l_level <= new_level) { + mlog(ML_ERROR, "lockres %s, lvl %d <= %d, blcklst %d, mask %d, " + "type %d, flags 0x%lx, hold %d %d, act %d %d, req %d, " + "block %d, pgen %d\n", lockres->l_name, lockres->l_level, + new_level, list_empty(&lockres->l_blocked_list), + list_empty(&lockres->l_mask_waiters), lockres->l_type, + lockres->l_flags, lockres->l_ro_holders, + lockres->l_ex_holders, lockres->l_action, + lockres->l_unlock_action, lockres->l_requested, + lockres->l_blocking, lockres->l_pending_gen); + BUG(); + } + + mlog(ML_BASTS, "lockres %s, level %d => %d, blocking %d\n", + lockres->l_name, lockres->l_level, new_level, lockres->l_blocking); + + lockres->l_action = OCFS2_AST_DOWNCONVERT; + lockres->l_requested = new_level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + return lockres_set_pending(lockres); +} + +static int ocfs2_downconvert_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int new_level, + int lvb, + unsigned int generation) +{ + int ret; + u32 dlm_flags = DLM_LKF_CONVERT; + + mlog(ML_BASTS, "lockres %s, level %d => %d\n", lockres->l_name, + lockres->l_level, new_level); + + /* + * On DLM_LKF_VALBLK, fsdlm behaves differently with o2cb. It always + * expects DLM_LKF_VALBLK being set if the LKB has LVB, so that + * we can recover correctly from node failure. Otherwise, we may get + * invalid LVB in LKB, but without DLM_SBF_VALNOTVALID being set. + */ + if (!ocfs2_is_o2cb_active() && + lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lvb = 1; + + if (lvb) + dlm_flags |= DLM_LKF_VALBLK; + + ret = ocfs2_dlm_lock(osb->cconn, + new_level, + &lockres->l_lksb, + dlm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, generation, osb); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 1); + goto bail; + } + + ret = 0; +bail: + return ret; +} + +/* returns 1 when the caller should unlock and call ocfs2_dlm_unlock */ +static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + assert_spin_locked(&lockres->l_lock); + + if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) { + /* If we're already trying to cancel a lock conversion + * then just drop the spinlock and allow the caller to + * requeue this lock. */ + mlog(ML_BASTS, "lockres %s, skip convert\n", lockres->l_name); + return 0; + } + + /* were we in a convert when we got the bast fire? */ + BUG_ON(lockres->l_action != OCFS2_AST_CONVERT && + lockres->l_action != OCFS2_AST_DOWNCONVERT); + /* set things up for the unlockast to know to just + * clear out the ast_action and unset busy, etc. */ + lockres->l_unlock_action = OCFS2_UNLOCK_CANCEL_CONVERT; + + mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_BUSY), + "lock %s, invalid flags: 0x%lx\n", + lockres->l_name, lockres->l_flags); + + mlog(ML_BASTS, "lockres %s\n", lockres->l_name); + + return 1; +} + +static int ocfs2_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, + DLM_LKF_CANCEL); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 0); + } + + mlog(ML_BASTS, "lockres %s\n", lockres->l_name); + + return ret; +} + +static int ocfs2_unblock_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + struct ocfs2_unblock_ctl *ctl) +{ + unsigned long flags; + int blocking; + int new_level; + int level; + int ret = 0; + int set_lvb = 0; + unsigned int gen; + + spin_lock_irqsave(&lockres->l_lock, flags); + +recheck: + /* + * Is it still blocking? If not, we have no more work to do. + */ + if (!(lockres->l_flags & OCFS2_LOCK_BLOCKED)) { + BUG_ON(lockres->l_blocking != DLM_LOCK_NL); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = 0; + goto leave; + } + + if (lockres->l_flags & OCFS2_LOCK_BUSY) { + /* XXX + * This is a *big* race. The OCFS2_LOCK_PENDING flag + * exists entirely for one reason - another thread has set + * OCFS2_LOCK_BUSY, but has *NOT* yet called dlm_lock(). + * + * If we do ocfs2_cancel_convert() before the other thread + * calls dlm_lock(), our cancel will do nothing. We will + * get no ast, and we will have no way of knowing the + * cancel failed. Meanwhile, the other thread will call + * into dlm_lock() and wait...forever. + * + * Why forever? Because another node has asked for the + * lock first; that's why we're here in unblock_lock(). + * + * The solution is OCFS2_LOCK_PENDING. When PENDING is + * set, we just requeue the unblock. Only when the other + * thread has called dlm_lock() and cleared PENDING will + * we then cancel their request. + * + * All callers of dlm_lock() must set OCFS2_DLM_PENDING + * at the same time they set OCFS2_DLM_BUSY. They must + * clear OCFS2_DLM_PENDING after dlm_lock() returns. + */ + if (lockres->l_flags & OCFS2_LOCK_PENDING) { + mlog(ML_BASTS, "lockres %s, ReQ: Pending\n", + lockres->l_name); + goto leave_requeue; + } + + ctl->requeue = 1; + ret = ocfs2_prepare_cancel_convert(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + if (ret) { + ret = ocfs2_cancel_convert(osb, lockres); + if (ret < 0) + mlog_errno(ret); + } + goto leave; + } + + /* + * This prevents livelocks. OCFS2_LOCK_UPCONVERT_FINISHING flag is + * set when the ast is received for an upconvert just before the + * OCFS2_LOCK_BUSY flag is cleared. Now if the fs received a bast + * on the heels of the ast, we want to delay the downconvert just + * enough to allow the up requestor to do its task. Because this + * lock is in the blocked queue, the lock will be downconverted + * as soon as the requestor is done with the lock. + */ + if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) + goto leave_requeue; + + /* + * How can we block and yet be at NL? We were trying to upconvert + * from NL and got canceled. The code comes back here, and now + * we notice and clear BLOCKING. + */ + if (lockres->l_level == DLM_LOCK_NL) { + BUG_ON(lockres->l_ex_holders || lockres->l_ro_holders); + mlog(ML_BASTS, "lockres %s, Aborting dc\n", lockres->l_name); + lockres->l_blocking = DLM_LOCK_NL; + lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto leave; + } + + /* if we're blocking an exclusive and we have *any* holders, + * then requeue. */ + if ((lockres->l_blocking == DLM_LOCK_EX) + && (lockres->l_ex_holders || lockres->l_ro_holders)) { + mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", + lockres->l_name, lockres->l_ex_holders, + lockres->l_ro_holders); + goto leave_requeue; + } + + /* If it's a PR we're blocking, then only + * requeue if we've got any EX holders */ + if (lockres->l_blocking == DLM_LOCK_PR && + lockres->l_ex_holders) { + mlog(ML_BASTS, "lockres %s, ReQ: EX Holders %u\n", + lockres->l_name, lockres->l_ex_holders); + goto leave_requeue; + } + + /* + * Can we get a lock in this state if the holder counts are + * zero? The meta data unblock code used to check this. + */ + if ((lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + && (lockres->l_flags & OCFS2_LOCK_REFRESHING)) { + mlog(ML_BASTS, "lockres %s, ReQ: Lock Refreshing\n", + lockres->l_name); + goto leave_requeue; + } + + new_level = ocfs2_highest_compat_lock_level(lockres->l_blocking); + + if (lockres->l_ops->check_downconvert + && !lockres->l_ops->check_downconvert(lockres, new_level)) { + mlog(ML_BASTS, "lockres %s, ReQ: Checkpointing\n", + lockres->l_name); + goto leave_requeue; + } + + /* If we get here, then we know that there are no more + * incompatible holders (and anyone asking for an incompatible + * lock is blocked). We can now downconvert the lock */ + if (!lockres->l_ops->downconvert_worker) + goto downconvert; + + /* Some lockres types want to do a bit of work before + * downconverting a lock. Allow that here. The worker function + * may sleep, so we save off a copy of what we're blocking as + * it may change while we're not holding the spin lock. */ + blocking = lockres->l_blocking; + level = lockres->l_level; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ctl->unblock_action = lockres->l_ops->downconvert_worker(lockres, blocking); + + if (ctl->unblock_action == UNBLOCK_STOP_POST) { + mlog(ML_BASTS, "lockres %s, UNBLOCK_STOP_POST\n", + lockres->l_name); + goto leave; + } + + spin_lock_irqsave(&lockres->l_lock, flags); + if ((blocking != lockres->l_blocking) || (level != lockres->l_level)) { + /* If this changed underneath us, then we can't drop + * it just yet. */ + mlog(ML_BASTS, "lockres %s, block=%d:%d, level=%d:%d, " + "Recheck\n", lockres->l_name, blocking, + lockres->l_blocking, level, lockres->l_level); + goto recheck; + } + +downconvert: + ctl->requeue = 0; + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { + if (lockres->l_level == DLM_LOCK_EX) + set_lvb = 1; + + /* + * We only set the lvb if the lock has been fully + * refreshed - otherwise we risk setting stale + * data. Otherwise, there's no need to actually clear + * out the lvb here as it's value is still valid. + */ + if (set_lvb && !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) + lockres->l_ops->set_lvb(lockres); + } + + gen = ocfs2_prepare_downconvert(lockres, new_level); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = ocfs2_downconvert_lock(osb, lockres, new_level, set_lvb, + gen); + +leave: + if (ret) + mlog_errno(ret); + return ret; + +leave_requeue: + spin_unlock_irqrestore(&lockres->l_lock, flags); + ctl->requeue = 1; + + return 0; +} + +static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct inode *inode; + struct address_space *mapping; + struct ocfs2_inode_info *oi; + + inode = ocfs2_lock_res_inode(lockres); + mapping = inode->i_mapping; + + if (S_ISDIR(inode->i_mode)) { + oi = OCFS2_I(inode); + oi->ip_dir_lock_gen++; + mlog(0, "generation: %u\n", oi->ip_dir_lock_gen); + goto out; + } + + if (!S_ISREG(inode->i_mode)) + goto out; + + /* + * We need this before the filemap_fdatawrite() so that it can + * transfer the dirty bit from the PTE to the + * page. Unfortunately this means that even for EX->PR + * downconverts, we'll lose our mappings and have to build + * them up again. + */ + unmap_mapping_range(mapping, 0, 0, 0); + + if (filemap_fdatawrite(mapping)) { + mlog(ML_ERROR, "Could not sync inode %llu for downconvert!", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + } + sync_mapping_buffers(mapping); + if (blocking == DLM_LOCK_EX) { + truncate_inode_pages(mapping, 0); + } else { + /* We only need to wait on the I/O if we're not also + * truncating pages because truncate_inode_pages waits + * for us above. We don't truncate pages if we're + * blocking anything < EXMODE because we want to keep + * them around in that case. */ + filemap_fdatawait(mapping); + } + + forget_all_cached_acls(inode); + +out: + return UNBLOCK_CONTINUE; +} + +static int ocfs2_ci_checkpointed(struct ocfs2_caching_info *ci, + struct ocfs2_lock_res *lockres, + int new_level) +{ + int checkpointed = ocfs2_ci_fully_checkpointed(ci); + + BUG_ON(new_level != DLM_LOCK_NL && new_level != DLM_LOCK_PR); + BUG_ON(lockres->l_level != DLM_LOCK_EX && !checkpointed); + + if (checkpointed) + return 1; + + ocfs2_start_checkpoint(OCFS2_SB(ocfs2_metadata_cache_get_super(ci))); + return 0; +} + +static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + return ocfs2_ci_checkpointed(INODE_CACHE(inode), lockres, new_level); +} + +static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + __ocfs2_stuff_meta_lvb(inode); +} + +/* + * Does the final reference drop on our dentry lock. Right now this + * happens in the downconvert thread, but we could choose to simplify the + * dlmglue API and push these off to the ocfs2_wq in the future. + */ +static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); + ocfs2_dentry_lock_put(osb, dl); +} + +/* + * d_delete() matching dentries before the lock downconvert. + * + * At this point, any process waiting to destroy the + * dentry_lock due to last ref count is stopped by the + * OCFS2_LOCK_QUEUED flag. + * + * We have two potential problems + * + * 1) If we do the last reference drop on our dentry_lock (via dput) + * we'll wind up in ocfs2_release_dentry_lock(), waiting on + * the downconvert to finish. Instead we take an elevated + * reference and push the drop until after we've completed our + * unblock processing. + * + * 2) There might be another process with a final reference, + * waiting on us to finish processing. If this is the case, we + * detect it and exit out - there's no more dentries anyway. + */ +static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); + struct ocfs2_inode_info *oi = OCFS2_I(dl->dl_inode); + struct dentry *dentry; + unsigned long flags; + int extra_ref = 0; + + /* + * This node is blocking another node from getting a read + * lock. This happens when we've renamed within a + * directory. We've forced the other nodes to d_delete(), but + * we never actually dropped our lock because it's still + * valid. The downconvert code will retain a PR for this node, + * so there's no further work to do. + */ + if (blocking == DLM_LOCK_PR) + return UNBLOCK_CONTINUE; + + /* + * Mark this inode as potentially orphaned. The code in + * ocfs2_delete_inode() will figure out whether it actually + * needs to be freed or not. + */ + spin_lock(&oi->ip_lock); + oi->ip_flags |= OCFS2_INODE_MAYBE_ORPHANED; + spin_unlock(&oi->ip_lock); + + /* + * Yuck. We need to make sure however that the check of + * OCFS2_LOCK_FREEING and the extra reference are atomic with + * respect to a reference decrement or the setting of that + * flag. + */ + spin_lock_irqsave(&lockres->l_lock, flags); + spin_lock(&dentry_attach_lock); + if (!(lockres->l_flags & OCFS2_LOCK_FREEING) + && dl->dl_count) { + dl->dl_count++; + extra_ref = 1; + } + spin_unlock(&dentry_attach_lock); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "extra_ref = %d\n", extra_ref); + + /* + * We have a process waiting on us in ocfs2_dentry_iput(), + * which means we can't have any more outstanding + * aliases. There's no need to do any more work. + */ + if (!extra_ref) + return UNBLOCK_CONTINUE; + + spin_lock(&dentry_attach_lock); + while (1) { + dentry = ocfs2_find_local_alias(dl->dl_inode, + dl->dl_parent_blkno, 1); + if (!dentry) + break; + spin_unlock(&dentry_attach_lock); + + if (S_ISDIR(dl->dl_inode->i_mode)) + shrink_dcache_parent(dentry); + + mlog(0, "d_delete(%pd);\n", dentry); + + /* + * The following dcache calls may do an + * iput(). Normally we don't want that from the + * downconverting thread, but in this case it's ok + * because the requesting node already has an + * exclusive lock on the inode, so it can't be queued + * for a downconvert. + */ + d_delete(dentry); + dput(dentry); + + spin_lock(&dentry_attach_lock); + } + spin_unlock(&dentry_attach_lock); + + /* + * If we are the last holder of this dentry lock, there is no + * reason to downconvert so skip straight to the unlock. + */ + if (dl->dl_count == 1) + return UNBLOCK_STOP_POST; + + return UNBLOCK_CONTINUE_POST; +} + +static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + struct ocfs2_refcount_tree *tree = + ocfs2_lock_res_refcount_tree(lockres); + + return ocfs2_ci_checkpointed(&tree->rf_ci, lockres, new_level); +} + +static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct ocfs2_refcount_tree *tree = + ocfs2_lock_res_refcount_tree(lockres); + + ocfs2_metadata_cache_purge(&tree->rf_ci); + + return UNBLOCK_CONTINUE; +} + +static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_qinfo_lvb *lvb; + struct ocfs2_mem_dqinfo *oinfo = ocfs2_lock_res_qinfo(lockres); + struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, + oinfo->dqi_gi.dqi_type); + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + lvb->lvb_version = OCFS2_QINFO_LVB_VERSION; + lvb->lvb_bgrace = cpu_to_be32(info->dqi_bgrace); + lvb->lvb_igrace = cpu_to_be32(info->dqi_igrace); + lvb->lvb_syncms = cpu_to_be32(oinfo->dqi_syncms); + lvb->lvb_blocks = cpu_to_be32(oinfo->dqi_gi.dqi_blocks); + lvb->lvb_free_blk = cpu_to_be32(oinfo->dqi_gi.dqi_free_blk); + lvb->lvb_free_entry = cpu_to_be32(oinfo->dqi_gi.dqi_free_entry); +} + +void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +static int ocfs2_refresh_qinfo(struct ocfs2_mem_dqinfo *oinfo) +{ + struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, + oinfo->dqi_gi.dqi_type); + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_qinfo_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + struct buffer_head *bh = NULL; + struct ocfs2_global_disk_dqinfo *gdinfo; + int status = 0; + + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && + lvb->lvb_version == OCFS2_QINFO_LVB_VERSION) { + info->dqi_bgrace = be32_to_cpu(lvb->lvb_bgrace); + info->dqi_igrace = be32_to_cpu(lvb->lvb_igrace); + oinfo->dqi_syncms = be32_to_cpu(lvb->lvb_syncms); + oinfo->dqi_gi.dqi_blocks = be32_to_cpu(lvb->lvb_blocks); + oinfo->dqi_gi.dqi_free_blk = be32_to_cpu(lvb->lvb_free_blk); + oinfo->dqi_gi.dqi_free_entry = + be32_to_cpu(lvb->lvb_free_entry); + } else { + status = ocfs2_read_quota_phys_block(oinfo->dqi_gqinode, + oinfo->dqi_giblk, &bh); + if (status) { + mlog_errno(status); + goto bail; + } + gdinfo = (struct ocfs2_global_disk_dqinfo *) + (bh->b_data + OCFS2_GLOBAL_INFO_OFF); + info->dqi_bgrace = le32_to_cpu(gdinfo->dqi_bgrace); + info->dqi_igrace = le32_to_cpu(gdinfo->dqi_igrace); + oinfo->dqi_syncms = le32_to_cpu(gdinfo->dqi_syncms); + oinfo->dqi_gi.dqi_blocks = le32_to_cpu(gdinfo->dqi_blocks); + oinfo->dqi_gi.dqi_free_blk = le32_to_cpu(gdinfo->dqi_free_blk); + oinfo->dqi_gi.dqi_free_entry = + le32_to_cpu(gdinfo->dqi_free_entry); + brelse(bh); + ocfs2_track_lock_refresh(lockres); + } + +bail: + return status; +} + +/* Lock quota info, this function expects at least shared lock on the quota file + * so that we can safely refresh quota info from disk. */ +int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + int status = 0; + + /* On RO devices, locking really isn't needed... */ + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + status = -EROFS; + goto bail; + } + if (ocfs2_mount_local(osb)) + goto bail; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + if (!ocfs2_should_refresh_lock_res(lockres)) + goto bail; + /* OK, we have the lock but we need to refresh the quota info */ + status = ocfs2_refresh_qinfo(oinfo); + if (status) + ocfs2_qinfo_unlock(oinfo, ex); + ocfs2_complete_lock_res_refresh(lockres, status); +bail: + return status; +} + +int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex) +{ + int status; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; + struct ocfs2_super *osb = lockres->l_priv; + + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; + struct ocfs2_super *osb = lockres->l_priv; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int status; + struct ocfs2_unblock_ctl ctl = {0, 0,}; + unsigned long flags; + + /* Our reference to the lockres in this function can be + * considered valid until we remove the OCFS2_LOCK_QUEUED + * flag. */ + + BUG_ON(!lockres); + BUG_ON(!lockres->l_ops); + + mlog(ML_BASTS, "lockres %s blocked\n", lockres->l_name); + + /* Detect whether a lock has been marked as going away while + * the downconvert thread was processing other things. A lock can + * still be marked with OCFS2_LOCK_FREEING after this check, + * but short circuiting here will still save us some + * performance. */ + spin_lock_irqsave(&lockres->l_lock, flags); + if (lockres->l_flags & OCFS2_LOCK_FREEING) + goto unqueue; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + status = ocfs2_unblock_lock(osb, lockres, &ctl); + if (status < 0) + mlog_errno(status); + + spin_lock_irqsave(&lockres->l_lock, flags); +unqueue: + if (lockres->l_flags & OCFS2_LOCK_FREEING || !ctl.requeue) { + lockres_clear_flags(lockres, OCFS2_LOCK_QUEUED); + } else + ocfs2_schedule_blocked_lock(osb, lockres); + + mlog(ML_BASTS, "lockres %s, requeue = %s.\n", lockres->l_name, + ctl.requeue ? "yes" : "no"); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + if (ctl.unblock_action != UNBLOCK_CONTINUE + && lockres->l_ops->post_unlock) + lockres->l_ops->post_unlock(osb, lockres); +} + +static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + unsigned long flags; + + assert_spin_locked(&lockres->l_lock); + + if (lockres->l_flags & OCFS2_LOCK_FREEING) { + /* Do not schedule a lock for downconvert when it's on + * the way to destruction - any nodes wanting access + * to the resource will get it soon. */ + mlog(ML_BASTS, "lockres %s won't be scheduled: flags 0x%lx\n", + lockres->l_name, lockres->l_flags); + return; + } + + lockres_or_flags(lockres, OCFS2_LOCK_QUEUED); + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (list_empty(&lockres->l_blocked_list)) { + list_add_tail(&lockres->l_blocked_list, + &osb->blocked_lock_list); + osb->blocked_lock_count++; + } + spin_unlock_irqrestore(&osb->dc_task_lock, flags); +} + +static void ocfs2_downconvert_thread_do_work(struct ocfs2_super *osb) +{ + unsigned long processed; + unsigned long flags; + struct ocfs2_lock_res *lockres; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + /* grab this early so we know to try again if a state change and + * wake happens part-way through our work */ + osb->dc_work_sequence = osb->dc_wake_sequence; + + processed = osb->blocked_lock_count; + /* + * blocked lock processing in this loop might call iput which can + * remove items off osb->blocked_lock_list. Downconvert up to + * 'processed' number of locks, but stop short if we had some + * removed in ocfs2_mark_lockres_freeing when downconverting. + */ + while (processed && !list_empty(&osb->blocked_lock_list)) { + lockres = list_entry(osb->blocked_lock_list.next, + struct ocfs2_lock_res, l_blocked_list); + list_del_init(&lockres->l_blocked_list); + osb->blocked_lock_count--; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + + BUG_ON(!processed); + processed--; + + ocfs2_process_blocked_lock(osb, lockres); + + spin_lock_irqsave(&osb->dc_task_lock, flags); + } + spin_unlock_irqrestore(&osb->dc_task_lock, flags); +} + +static int ocfs2_downconvert_thread_lists_empty(struct ocfs2_super *osb) +{ + int empty = 0; + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (list_empty(&osb->blocked_lock_list)) + empty = 1; + + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + return empty; +} + +static int ocfs2_downconvert_thread_should_wake(struct ocfs2_super *osb) +{ + int should_wake = 0; + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (osb->dc_work_sequence != osb->dc_wake_sequence) + should_wake = 1; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + + return should_wake; +} + +static int ocfs2_downconvert_thread(void *arg) +{ + int status = 0; + struct ocfs2_super *osb = arg; + + /* only quit once we've been asked to stop and there is no more + * work available */ + while (!(kthread_should_stop() && + ocfs2_downconvert_thread_lists_empty(osb))) { + + wait_event_interruptible(osb->dc_event, + ocfs2_downconvert_thread_should_wake(osb) || + kthread_should_stop()); + + mlog(0, "downconvert_thread: awoken\n"); + + ocfs2_downconvert_thread_do_work(osb); + } + + osb->dc_task = NULL; + return status; +} + +void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb) +{ + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + /* make sure the voting thread gets a swipe at whatever changes + * the caller may have made to the voting state */ + osb->dc_wake_sequence++; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + wake_up(&osb->dc_event); +} diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h new file mode 100644 index 00000000..a7fc18ba --- /dev/null +++ b/kmod/src/dlmglue.h @@ -0,0 +1,191 @@ +/* -*- mode: c; c-basic-offset: 8; -*- + * vim: noexpandtab sw=8 ts=8 sts=0: + * + * dlmglue.h + * + * description here + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + + +#ifndef DLMGLUE_H +#define DLMGLUE_H + +#include "dcache.h" + +#define OCFS2_LVB_VERSION 5 + +struct ocfs2_meta_lvb { + __u8 lvb_version; + __u8 lvb_reserved0; + __be16 lvb_idynfeatures; + __be32 lvb_iclusters; + __be32 lvb_iuid; + __be32 lvb_igid; + __be64 lvb_iatime_packed; + __be64 lvb_ictime_packed; + __be64 lvb_imtime_packed; + __be64 lvb_isize; + __be16 lvb_imode; + __be16 lvb_inlink; + __be32 lvb_iattr; + __be32 lvb_igeneration; + __be32 lvb_reserved2; +}; + +#define OCFS2_QINFO_LVB_VERSION 1 + +struct ocfs2_qinfo_lvb { + __u8 lvb_version; + __u8 lvb_reserved[3]; + __be32 lvb_bgrace; + __be32 lvb_igrace; + __be32 lvb_syncms; + __be32 lvb_blocks; + __be32 lvb_free_blk; + __be32 lvb_free_entry; +}; + +#define OCFS2_ORPHAN_LVB_VERSION 1 + +struct ocfs2_orphan_scan_lvb { + __u8 lvb_version; + __u8 lvb_reserved[3]; + __be32 lvb_os_seqno; +}; + +struct ocfs2_lock_holder { + struct list_head oh_list; + struct pid *oh_owner_pid; +}; + +/* ocfs2_inode_lock_full() 'arg_flags' flags */ +/* don't wait on recovery. */ +#define OCFS2_META_LOCK_RECOVERY (0x01) +/* Instruct the dlm not to queue ourselves on the other node. */ +#define OCFS2_META_LOCK_NOQUEUE (0x02) +/* don't block waiting for the downconvert thread, instead return -EAGAIN */ +#define OCFS2_LOCK_NONBLOCK (0x04) +/* just get back disk inode bh if we've got cluster lock. */ +#define OCFS2_META_LOCK_GETBH (0x08) + +/* Locking subclasses of inode cluster lock */ +enum { + OI_LS_NORMAL = 0, + OI_LS_PARENT, + OI_LS_RENAME1, + OI_LS_RENAME2, + OI_LS_REFLINK_TARGET, +}; + +int ocfs2_dlm_init(struct ocfs2_super *osb); +void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); +void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); +void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + unsigned int generation, + struct inode *inode); +void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, + u64 parent, struct inode *inode); +struct ocfs2_file_private; +void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_file_private *fp); +struct ocfs2_mem_dqinfo; +void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_mem_dqinfo *info); +void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_super *osb, u64 ref_blkno, + unsigned int generation); +void ocfs2_lock_res_free(struct ocfs2_lock_res *res); +int ocfs2_create_new_inode_locks(struct inode *inode); +int ocfs2_drop_inode_locks(struct inode *inode); +int ocfs2_rw_lock(struct inode *inode, int write); +void ocfs2_rw_unlock(struct inode *inode, int write); +int ocfs2_open_lock(struct inode *inode); +int ocfs2_try_open_lock(struct inode *inode, int write); +void ocfs2_open_unlock(struct inode *inode); +int ocfs2_inode_lock_atime(struct inode *inode, + struct vfsmount *vfsmnt, + int *level); +int ocfs2_inode_lock_full_nested(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + int arg_flags, + int subclass); +int ocfs2_inode_lock_with_page(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct page *page); +/* Variants without special locking class or flags */ +#define ocfs2_inode_lock_full(i, r, e, f)\ + ocfs2_inode_lock_full_nested(i, r, e, f, OI_LS_NORMAL) +#define ocfs2_inode_lock_nested(i, b, e, s)\ + ocfs2_inode_lock_full_nested(i, b, e, 0, s) +/* 99% of the time we don't want to supply any additional flags -- + * those are for very specific cases only. */ +#define ocfs2_inode_lock(i, b, e) ocfs2_inode_lock_full_nested(i, b, e, 0, OI_LS_NORMAL) +void ocfs2_inode_unlock(struct inode *inode, + int ex); +int ocfs2_super_lock(struct ocfs2_super *osb, + int ex); +void ocfs2_super_unlock(struct ocfs2_super *osb, + int ex); +int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno); +void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno); + +int ocfs2_rename_lock(struct ocfs2_super *osb); +void ocfs2_rename_unlock(struct ocfs2_super *osb); +int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex); +void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex); +int ocfs2_dentry_lock(struct dentry *dentry, int ex); +void ocfs2_dentry_unlock(struct dentry *dentry, int ex); +int ocfs2_file_lock(struct file *file, int ex, int trylock); +void ocfs2_file_unlock(struct file *file); +int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex); +void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex); +struct ocfs2_refcount_tree; +int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex); +void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex); + + +void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +/* for the downconvert thread */ +void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb); + +struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void); +void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug); + +/* To set the locking protocol on module initialization */ +void ocfs2_set_locking_protocol(void); + +/* The _tracker pair is used to avoid cluster recursive locking */ +int ocfs2_inode_lock_tracker(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct ocfs2_lock_holder *oh); +void ocfs2_inode_unlock_tracker(struct inode *inode, + int ex, + struct ocfs2_lock_holder *oh, + int had_lock); + +#endif /* DLMGLUE_H */ From fc21a0253ca65cfe455f069c42c1185c66b23582 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 15:54:08 -0500 Subject: [PATCH 367/920] scoutfs: Hook dlmglue into our build system Signed-off-by: Mark Fasheh --- kmod/src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 2200b81d..756c1b07 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -5,8 +5,8 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - kvec.o inode.o ioctl.o item.o key.o lock.o manifest.o msg.o \ - options.o seg.o server.o scoutfs_trace.o sock.o sort_priv.o \ + dlmglue.o kvec.o inode.o ioctl.o item.o key.o lock.o manifest.o \ + msg.o options.o seg.o server.o scoutfs_trace.o sock.o sort_priv.o \ super.o trans.o xattr.o # From bc2fef7fc87134bea262c20f12cb2fc5ca0bdf84 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 15:55:16 -0500 Subject: [PATCH 368/920] scoutfs: ifdef out ocfs2 specific callbacks and functions We only want the generic stuff. Long term the Ocfs2 specific code would be what's left in fs/ocfs2/dlmglue.[ch]. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 29 ++++++++++++++++++++++++++++- kmod/src/dlmglue.h | 13 +++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 4689940a..b2d1eb46 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -35,6 +35,7 @@ #include #include +#if 0 #define MLOG_MASK_PREFIX ML_DLM_GLUE #include @@ -58,6 +59,7 @@ #include "acl.h" #include "buffer_head_io.h" +#endif struct ocfs2_mask_waiter { struct list_head mw_item; @@ -70,10 +72,12 @@ struct ocfs2_mask_waiter { #endif }; +#if 0 static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres); static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres); static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres); static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres); +#endif /* * Return value from ->downconvert_worker functions. @@ -98,6 +102,7 @@ struct ocfs2_unblock_ctl { /* Lockdep class keys */ struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; +#if 0 static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, int new_level); static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres); @@ -144,6 +149,7 @@ static void ocfs2_dump_meta_lvb_info(u64 level, (long long)be64_to_cpu(lvb->lvb_imtime_packed), be32_to_cpu(lvb->lvb_iattr)); } +#endif /* @@ -234,6 +240,7 @@ struct ocfs2_lock_res_ops { */ #define LOCK_TYPE_USES_LVB 0x2 +#if 0 static struct ocfs2_lock_res_ops ocfs2_inode_rw_lops = { .get_osb = ocfs2_get_inode_osb, .flags = 0, @@ -298,12 +305,13 @@ static inline int ocfs2_is_inode_lock(struct ocfs2_lock_res *lockres) lockres->l_type == OCFS2_LOCK_TYPE_RW || lockres->l_type == OCFS2_LOCK_TYPE_OPEN; } +#endif static inline struct ocfs2_lock_res *ocfs2_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) { return container_of(lksb, struct ocfs2_lock_res, l_lksb); } - +#if 0 static inline struct inode *ocfs2_lock_res_inode(struct ocfs2_lock_res *lockres) { BUG_ON(!ocfs2_is_inode_lock(lockres)); @@ -330,6 +338,7 @@ ocfs2_lock_res_refcount_tree(struct ocfs2_lock_res *res) { return container_of(res, struct ocfs2_refcount_tree, rf_lockres); } +#endif static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *lockres) { @@ -375,9 +384,11 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, static int ocfs2_downconvert_thread(void *arg); static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); +#if 0 static int ocfs2_inode_lock_update(struct inode *inode, struct buffer_head **bh); static void ocfs2_drop_osb_locks(struct ocfs2_super *osb); +#endif static inline int ocfs2_highest_compat_lock_level(int level); static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, int new_level); @@ -536,6 +547,7 @@ void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) INIT_LIST_HEAD(&res->l_holders); } +#if 0 void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, enum ocfs2_lock_type type, unsigned int generation, @@ -721,6 +733,7 @@ void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_REFCOUNT, &ocfs2_refcount_block_lops, osb); } +#endif void ocfs2_lock_res_free(struct ocfs2_lock_res *res) { @@ -1205,6 +1218,7 @@ static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) spin_unlock_irqrestore(&lockres->l_lock, flags); } +#if 0 /* * This is the filesystem locking protocol. It provides the lock handling * hooks for the underlying DLM. It has a maximum version number. @@ -1242,6 +1256,7 @@ void ocfs2_set_locking_protocol(void) { ocfs2_stack_glue_set_max_proto_version(&lproto.lp_max_version); } +#endif static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, int convert) @@ -1670,6 +1685,7 @@ static int ocfs2_create_new_lock(struct ocfs2_super *osb, return ocfs2_lock_create(osb, lockres, level, lkm_flags); } +#if 0 /* Grants us an EX lock on the data and metadata resources, skipping * the normal cluster directory lookup. Use this ONLY on newly created * inodes which other nodes can't possibly see, and which haven't been @@ -2050,6 +2066,7 @@ void ocfs2_file_unlock(struct file *file) if (ret) mlog_errno(ret); } +#endif static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres) @@ -2078,6 +2095,7 @@ static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, ocfs2_wake_downconvert_thread(osb); } +#if 0 #define OCFS2_SEC_BITS 34 #define OCFS2_SEC_SHIFT (64 - 34) #define OCFS2_NSEC_MASK ((1ULL << OCFS2_SEC_SHIFT) - 1) @@ -2193,6 +2211,7 @@ static inline int ocfs2_meta_lvb_is_trustable(struct inode *inode, return 1; return 0; } +#endif /* Determine whether a lock resource needs to be refreshed, and * arbitrate who gets to refresh it. @@ -2246,6 +2265,7 @@ static inline void ocfs2_complete_lock_res_refresh(struct ocfs2_lock_res *lockre wake_up(&lockres->l_event); } +#if 0 /* may or may not return a bh if it went to disk. */ static int ocfs2_inode_lock_update(struct inode *inode, struct buffer_head **bh) @@ -2779,6 +2799,7 @@ void ocfs2_dentry_unlock(struct dentry *dentry, int ex) if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) ocfs2_cluster_unlock(osb, &dl->dl_lockres, level); } +#endif /* Reference counting of the dlm debug structure. We want this because * open references on the debug inodes can live on after a mount, so @@ -3073,6 +3094,7 @@ static void ocfs2_dlm_shutdown_debug(struct ocfs2_super *osb) } } +#if 0 int ocfs2_dlm_init(struct ocfs2_super *osb) { int status = 0; @@ -3164,6 +3186,7 @@ void ocfs2_dlm_shutdown(struct ocfs2_super *osb, ocfs2_dlm_shutdown_debug(osb); } +#endif static int ocfs2_drop_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres) @@ -3327,6 +3350,7 @@ void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, mlog_errno(ret); } +#if 0 static void ocfs2_drop_osb_locks(struct ocfs2_super *osb) { ocfs2_simple_drop_lockres(osb, &osb->osb_super_lockres); @@ -3365,6 +3389,7 @@ int ocfs2_drop_inode_locks(struct inode *inode) return status; } +#endif static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, int new_level) @@ -3682,6 +3707,7 @@ leave_requeue: return 0; } +#if 0 static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, int blocking) { @@ -4042,6 +4068,7 @@ void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex) if (!ocfs2_mount_local(osb)) ocfs2_cluster_unlock(osb, lockres, level); } +#endif static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index a7fc18ba..07354a74 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -27,6 +27,7 @@ #ifndef DLMGLUE_H #define DLMGLUE_H +#if 0 #include "dcache.h" #define OCFS2_LVB_VERSION 5 @@ -69,6 +70,7 @@ struct ocfs2_orphan_scan_lvb { __u8 lvb_reserved[3]; __be32 lvb_os_seqno; }; +#endif struct ocfs2_lock_holder { struct list_head oh_list; @@ -94,13 +96,17 @@ enum { OI_LS_REFLINK_TARGET, }; +#if 0 int ocfs2_dlm_init(struct ocfs2_super *osb); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); +#endif void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, enum ocfs2_lock_type type, unsigned int generation, struct inode *inode); + +#if 0 void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, u64 parent, struct inode *inode); struct ocfs2_file_private; @@ -112,7 +118,9 @@ void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, struct ocfs2_super *osb, u64 ref_blkno, unsigned int generation); +#endif void ocfs2_lock_res_free(struct ocfs2_lock_res *res); +#if 0 int ocfs2_create_new_inode_locks(struct inode *inode); int ocfs2_drop_inode_locks(struct inode *inode); int ocfs2_rw_lock(struct inode *inode, int write); @@ -162,7 +170,7 @@ void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex); struct ocfs2_refcount_tree; int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex); void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex); - +#endif void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); @@ -175,6 +183,7 @@ void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb); struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void); void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug); +#if 0 /* To set the locking protocol on module initialization */ void ocfs2_set_locking_protocol(void); @@ -187,5 +196,5 @@ void ocfs2_inode_unlock_tracker(struct inode *inode, int ex, struct ocfs2_lock_holder *oh, int had_lock); - +#endif #endif /* DLMGLUE_H */ From eae932e0fef9d51ffb014d96305c79190d4e78ed Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 16:00:54 -0500 Subject: [PATCH 369/920] scoutfs: dlmglue fix sched.h header Upstream moved linux/sched.h to linux/sched/signal.h. Centos still uses the old header location. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index b2d1eb46..dae8c556 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #if 0 #define MLOG_MASK_PREFIX ML_DLM_GLUE From 500baca53381113659be76d5c452920601d14a0d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 17:15:23 -0500 Subject: [PATCH 370/920] scoutfs: wrap some mlog calls in dlmglue Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index dae8c556..c9e7a352 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -35,6 +35,21 @@ #include #include +#define mlog(mask, fmt, args...) printk(KERN_INFO fmt , ##args) +#define mlog_errno(st) do { \ + int _st = (st); \ + if (_st != -ERESTARTSYS && _st != -EINTR && \ + _st != AOP_TRUNCATED_PAGE && _st != -ENOSPC) \ + mlog(ML_ERROR, "status = %lld\n", (long long)_st); \ +} while (0) + +#define mlog_bug_on_msg(cond, fmt, args...) do { \ + if (cond) { \ + mlog(ML_ERROR, "bug expression: " #cond "\n"); \ + mlog(ML_ERROR, fmt, ##args); \ + BUG(); \ + } \ +} while (0) #if 0 #define MLOG_MASK_PREFIX ML_DLM_GLUE #include From d4a89a5fbc630a44dc29001af9539dea1f89364e Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 17:46:55 -0500 Subject: [PATCH 371/920] scoutfs: dlmglue ifdef out ocfs2_build_lock_name() This was missed in the initial #ifdef patch. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index c9e7a352..7c25424d 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -417,7 +417,7 @@ static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, static int ocfs2_cancel_convert(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); - +#if 0 static void ocfs2_build_lock_name(enum ocfs2_lock_type type, u64 blkno, u32 generation, @@ -435,7 +435,7 @@ static void ocfs2_build_lock_name(enum ocfs2_lock_type type, mlog(0, "built lock resource with name: %s\n", name); } - +#endif static DEFINE_SPINLOCK(ocfs2_dlm_tracking_lock); static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, From bf6020c22bab3b1ae38703d708c1fa7db7901197 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 17:47:50 -0500 Subject: [PATCH 372/920] scoutfs: hide lockdep_keys in dlmglue for now This belongs behind #ifdef CONFIG_DEBUG_LOCK_ALLOC in the upstream code too. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 7c25424d..1f78808e 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -114,8 +114,10 @@ struct ocfs2_unblock_ctl { enum ocfs2_unblock_action unblock_action; }; +#ifdef CONFIG_DEBUG_LOCK_ALLOC /* Lockdep class keys */ struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; +#endif #if 0 static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, From 498a2f3721f028a029661c22af66c4529a0fa840 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 17:57:34 -0500 Subject: [PATCH 373/920] scoutfs: ifdef out usage of OCFS2_LOCK_TYPE_DENTRY Some of this leaks through even after the big #ifdef'ing - ocfs2 had to special case printing the name of dentry locks. We don't have such a need so it's easy to drop those calls. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 1f78808e..4a2e040b 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -389,7 +389,8 @@ static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, int convert); -#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ +#if 0 +#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ if ((_lockres)->l_type != OCFS2_LOCK_TYPE_DENTRY) \ mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ _err, _func, _lockres->l_name); \ @@ -398,6 +399,13 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ } while (0) +#endif +#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %.*s%08x\n", \ + _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ + (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ +} while (0) + static int ocfs2_downconvert_thread(void *arg); static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); @@ -2957,11 +2965,13 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) seq_printf(m, "0x%x\t", OCFS2_DLM_DEBUG_STR_VERSION); +#if 0 if (lockres->l_type == OCFS2_LOCK_TYPE_DENTRY) seq_printf(m, "%.*s%08x\t", OCFS2_DENTRY_LOCK_INO_START - 1, lockres->l_name, (unsigned int)ocfs2_get_dentry_lock_ino(lockres)); else +#endif seq_printf(m, "%.*s\t", OCFS2_LOCK_ID_MAX_LEN, lockres->l_name); seq_printf(m, "%d\t" From 2142648906ab74cda4e3a894f1a26819c2dccbfe Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 17:59:10 -0500 Subject: [PATCH 374/920] scoutfs: include linux/dlm.h dlmglue needs this as we're no longer hooking it into the stackglue component. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 4a2e040b..ae0d8f27 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -34,6 +34,7 @@ #include #include #include +#include #define mlog(mask, fmt, args...) printk(KERN_INFO fmt , ##args) #define mlog_errno(st) do { \ From 99d00a5a2f6cf9b2f87c41cc4fe763a0a23f6050 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:07:31 -0500 Subject: [PATCH 375/920] scoutfs: dlmglue needs to #include "dlmglue.h" Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index ae0d8f27..c217592d 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -36,6 +36,8 @@ #include #include +#include "dlmglue.h" + #define mlog(mask, fmt, args...) printk(KERN_INFO fmt , ##args) #define mlog_errno(st) do { \ int _st = (st); \ From 9bfb9c059d3c31f3e32e1e0bac507d6e07551906 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:07:52 -0500 Subject: [PATCH 376/920] scoutfs: copy struct ocfs2_lock_res Grab this from fs/ocfs2/ocfs2.h and put it in dlmglue.h. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 107 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 07354a74..241e4f33 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -27,6 +27,113 @@ #ifndef DLMGLUE_H #define DLMGLUE_H +enum ocfs2_ast_action { + OCFS2_AST_INVALID = 0, + OCFS2_AST_ATTACH, + OCFS2_AST_CONVERT, + OCFS2_AST_DOWNCONVERT, +}; + +/* actions for an unlockast function to take. */ +enum ocfs2_unlock_action { + OCFS2_UNLOCK_INVALID = 0, + OCFS2_UNLOCK_CANCEL_CONVERT, + OCFS2_UNLOCK_DROP_LOCK, +}; + +/* ocfs2_lock_res->l_flags flags. */ +#define OCFS2_LOCK_ATTACHED (0x00000001) /* we have initialized + * the lvb */ +#define OCFS2_LOCK_BUSY (0x00000002) /* we are currently in + * dlm_lock */ +#define OCFS2_LOCK_BLOCKED (0x00000004) /* blocked waiting to + * downconvert*/ +#define OCFS2_LOCK_LOCAL (0x00000008) /* newly created inode */ +#define OCFS2_LOCK_NEEDS_REFRESH (0x00000010) +#define OCFS2_LOCK_REFRESHING (0x00000020) +#define OCFS2_LOCK_INITIALIZED (0x00000040) /* track initialization + * for shutdown paths */ +#define OCFS2_LOCK_FREEING (0x00000080) /* help dlmglue track + * when to skip queueing + * a lock because it's + * about to be + * dropped. */ +#define OCFS2_LOCK_QUEUED (0x00000100) /* queued for downconvert */ +#define OCFS2_LOCK_NOCACHE (0x00000200) /* don't use a holder count */ +#define OCFS2_LOCK_PENDING (0x00000400) /* This lockres is pending a + call to dlm_lock. Only + exists with BUSY set. */ +#define OCFS2_LOCK_UPCONVERT_FINISHING (0x00000800) /* blocks the dc thread + * from downconverting + * before the upconvert + * has completed */ + +#define OCFS2_LOCK_NONBLOCK_FINISHED (0x00001000) /* NONBLOCK cluster + * lock has already + * returned, do not block + * dc thread from + * downconverting */ + +struct ocfs2_lock_res_ops; + +typedef void (*ocfs2_lock_callback)(int status, unsigned long data); + +#ifdef CONFIG_OCFS2_FS_STATS +struct ocfs2_lock_stats { + u64 ls_total; /* Total wait in NSEC */ + u32 ls_gets; /* Num acquires */ + u32 ls_fail; /* Num failed acquires */ + + /* Storing max wait in usecs saves 24 bytes per inode */ + u32 ls_max; /* Max wait in USEC */ +}; +#endif + +struct ocfs2_lock_res { + void *l_priv; + struct ocfs2_lock_res_ops *l_ops; + + + struct list_head l_blocked_list; + struct list_head l_mask_waiters; + struct list_head l_holders; + + unsigned long l_flags; + char l_name[OCFS2_LOCK_ID_MAX_LEN]; + unsigned int l_ro_holders; + unsigned int l_ex_holders; + signed char l_level; + signed char l_requested; + signed char l_blocking; + + /* Data packed - type enum ocfs2_lock_type */ + unsigned char l_type; + + /* used from AST/BAST funcs. */ + /* Data packed - enum type ocfs2_ast_action */ + unsigned char l_action; + /* Data packed - enum type ocfs2_unlock_action */ + unsigned char l_unlock_action; + unsigned int l_pending_gen; + + spinlock_t l_lock; + + struct ocfs2_dlm_lksb l_lksb; + + wait_queue_head_t l_event; + + struct list_head l_debug_list; + +#ifdef CONFIG_OCFS2_FS_STATS + struct ocfs2_lock_stats l_lock_prmode; /* PR mode stats */ + u32 l_lock_refresh; /* Disk refreshes */ + struct ocfs2_lock_stats l_lock_exmode; /* EX mode stats */ +#endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map l_lockdep_map; +#endif +}; + #if 0 #include "dcache.h" From 13963d22e3eaa94563fc5c1cc8bd465d03a97870 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:12:54 -0500 Subject: [PATCH 377/920] scoutfs: pull in OCFS2_LOCK_ID_MAX_LEN We need this for the lockres name. It also turns out to be the only thing we need from fs/ocfs2/ocfs2_lockid.h. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 241e4f33..c2efef8a 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -27,6 +27,9 @@ #ifndef DLMGLUE_H #define DLMGLUE_H +/* Max length of lockid name */ +#define OCFS2_LOCK_ID_MAX_LEN 32 + enum ocfs2_ast_action { OCFS2_AST_INVALID = 0, OCFS2_AST_ATTACH, From 1831014c240fd776e3c7bbb7fe9a5271f641b79d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:15:14 -0500 Subject: [PATCH 378/920] scoutfs: remove usage of ocfs2_lock_type_string() This only leaked into the bast function. I retained the debug print - it'll be turned off in our build anyway, and that's what we'd want to do upstream. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index c217592d..1e44b0c9 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -1119,9 +1119,13 @@ static void ocfs2_blocking_ast(struct ocfs2_dlm_lksb *lksb, int level) BUG_ON(level <= DLM_LOCK_NL); +#if 0 mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d, " "type %s\n", lockres->l_name, level, lockres->l_level, ocfs2_lock_type_string(lockres->l_type)); +#endif + mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d\n", + lockres->l_name, level, lockres->l_level); /* * We can skip the bast for locks which don't enable caching - From bb100356d9508898d7b8adc364ab30a9412cebf1 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:37:06 -0500 Subject: [PATCH 379/920] scoutfs: pull in some fields from ocfs2_super for dlmglue This is all the dlmglue global context needed. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index c2efef8a..b21be96a 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -137,6 +137,33 @@ struct ocfs2_lock_res { #endif }; +struct ocfs2_super +{ + struct ocfs2_cluster_connection *cconn; + struct ocfs2_dlm_debug *osb_dlm_debug; + struct dentry *osb_debug_root; + + /* Downconvert thread */ + spinlock_t dc_task_lock; + struct task_struct *dc_task; + wait_queue_head_t dc_event; + unsigned long dc_wake_sequence; + unsigned long dc_work_sequence; + + /* + * Any thread can add locks to the list, but the downconvert + * thread is the only one allowed to remove locks. Any change + * to this rule requires updating + * ocfs2_downconvert_thread_do_work(). + */ + struct list_head blocked_lock_list; + unsigned long blocked_lock_count; + + unsigned long s_mount_opt; +}; +/* For s_mount_opt */ +#define OCFS2_MOUNT_NOINTR (1 << 2) + #if 0 #include "dcache.h" From 1b59ed99fbecf1ff9e5df8660f66e0196e8ca246 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:43:39 -0500 Subject: [PATCH 380/920] scoutfs: remove ocfs2_lock_res->l_type We don't need it - this the only ocfs2-ism in struct ocfs2_lock_res. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 6 ++---- kmod/src/dlmglue.h | 6 +----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 1e44b0c9..3b9519df 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -536,11 +536,9 @@ static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) static void ocfs2_lock_res_init_common(struct ocfs2_super *osb, struct ocfs2_lock_res *res, - enum ocfs2_lock_type type, struct ocfs2_lock_res_ops *ops, void *priv) { - res->l_type = type; res->l_ops = ops; res->l_priv = priv; @@ -3434,10 +3432,10 @@ static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, if (lockres->l_level <= new_level) { mlog(ML_ERROR, "lockres %s, lvl %d <= %d, blcklst %d, mask %d, " - "type %d, flags 0x%lx, hold %d %d, act %d %d, req %d, " + "flags 0x%lx, hold %d %d, act %d %d, req %d, " "block %d, pgen %d\n", lockres->l_name, lockres->l_level, new_level, list_empty(&lockres->l_blocked_list), - list_empty(&lockres->l_mask_waiters), lockres->l_type, + list_empty(&lockres->l_mask_waiters), lockres->l_flags, lockres->l_ro_holders, lockres->l_ex_holders, lockres->l_action, lockres->l_unlock_action, lockres->l_requested, diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index b21be96a..e3034f9e 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -109,9 +109,6 @@ struct ocfs2_lock_res { signed char l_requested; signed char l_blocking; - /* Data packed - type enum ocfs2_lock_type */ - unsigned char l_type; - /* used from AST/BAST funcs. */ /* Data packed - enum type ocfs2_ast_action */ unsigned char l_action; @@ -238,12 +235,11 @@ int ocfs2_dlm_init(struct ocfs2_super *osb); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); #endif void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); +#if 0 void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, enum ocfs2_lock_type type, unsigned int generation, struct inode *inode); - -#if 0 void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, u64 parent, struct inode *inode); struct ocfs2_file_private; From 61499c5d30c00c165bf00b770d6c8b0df1b6950c Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:52:49 -0500 Subject: [PATCH 381/920] scoutfs: pull in struct ocfs2_dlm_debug from fs/ocfs2/ocfs2.h We need this for the dlmglue global context. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index e3034f9e..0a73a7eb 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -134,6 +134,12 @@ struct ocfs2_lock_res { #endif }; +struct ocfs2_dlm_debug { + struct kref d_refcnt; + struct dentry *d_locking_state; + struct list_head d_lockres_tracking; +}; + struct ocfs2_super { struct ocfs2_cluster_connection *cconn; From 114760365c6517c1b83a3150fd6499162af0aa9e Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 18:57:41 -0500 Subject: [PATCH 382/920] scoutfs: fix up ocfs2_log_dlm_error() We're still referencing some ocfs2 specific lock names here, take them out. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 3b9519df..6c0e0d7b 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -404,9 +404,8 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, } while (0) #endif #define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ - mlog(ML_ERROR, "DLM error %d while calling %s on resource %.*s%08x\n", \ - _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ - (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ + _err, _func, (_lockres)->l_name); \ } while (0) static int ocfs2_downconvert_thread(void *arg); From 960f8e08bbb3adb147337f268fbe8a10861da588 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 19:06:18 -0500 Subject: [PATCH 383/920] scoutfs: copy in DLM_LVB_LEN from fs/ocfs2/dlm/dlmapi.h Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 0a73a7eb..f96a879d 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -29,6 +29,7 @@ /* Max length of lockid name */ #define OCFS2_LOCK_ID_MAX_LEN 32 +#define DLM_LVB_LEN 64 enum ocfs2_ast_action { OCFS2_AST_INVALID = 0, From 72a8e9e17127fca92b9b0a6bf754e8a4e6c71e21 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 21:30:26 -0500 Subject: [PATCH 384/920] scoutfs: pull in some of ocfs2 stackglue Dlmglue is built on top of this. Bring in the portions we need which includes the stackglue API as well as most of the fs/dlm implementation. I left off the Ocfs2 specific version and connection handling. Also left out is the old Ocfs2 dlm support which we'll never want. Like dlmglue, we keep as much of the generic stackglue code in tact here. This will make translating to/from upstream patches much easier. Signed-off-by: Mark Fasheh --- kmod/src/Makefile | 2 +- kmod/src/dlmglue.h | 3 +- kmod/src/stackglue.c | 388 +++++++++++++++++++++++++++++++++++++++++++ kmod/src/stackglue.h | 149 +++++++++++++++++ 4 files changed, 540 insertions(+), 2 deletions(-) create mode 100644 kmod/src/stackglue.c create mode 100644 kmod/src/stackglue.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 756c1b07..f1b94903 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -7,7 +7,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ dlmglue.o kvec.o inode.o ioctl.o item.o key.o lock.o manifest.o \ msg.o options.o seg.o server.o scoutfs_trace.o sock.o sort_priv.o \ - super.o trans.o xattr.o + stackglue.o super.o trans.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index f96a879d..beee5cc2 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -27,9 +27,10 @@ #ifndef DLMGLUE_H #define DLMGLUE_H +#include "stackglue.h" + /* Max length of lockid name */ #define OCFS2_LOCK_ID_MAX_LEN 32 -#define DLM_LVB_LEN 64 enum ocfs2_ast_action { OCFS2_AST_INVALID = 0, diff --git a/kmod/src/stackglue.c b/kmod/src/stackglue.c new file mode 100644 index 00000000..49f101d2 --- /dev/null +++ b/kmod/src/stackglue.c @@ -0,0 +1,388 @@ +/* -*- mode: c; c-basic-offset: 8; -*- + * vim: noexpandtab sw=8 ts=8 sts=0: + * + * stackglue.c + * + * Code which implements an OCFS2 specific interface to underlying + * cluster stacks. + * + * Copyright (C) 2007, 2009 Oracle. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stackglue.h" + +static void fsdlm_lock_ast_wrapper(void *astarg) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + int status = lksb->lksb_fsdlm.sb_status; + + /* + * For now we're punting on the issue of other non-standard errors + * where we can't tell if the unlock_ast or lock_ast should be called. + * The main "other error" that's possible is EINVAL which means the + * function was called with invalid args, which shouldn't be possible + * since the caller here is under our control. Other non-standard + * errors probably fall into the same category, or otherwise are fatal + * which means we can't carry on anyway. + */ + + if (status == -DLM_EUNLOCK || status == -DLM_ECANCEL) + lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, 0); + else + lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); +} + +static void fsdlm_blocking_ast_wrapper(void *astarg, int level) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + + lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); +} + +static int user_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen) +{ + int ret; + + if (!lksb->lksb_fsdlm.sb_lvbptr) + lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + + sizeof(struct dlm_lksb); + + ret = dlm_lock(conn->cc_lockspace, mode, &lksb->lksb_fsdlm, + flags|DLM_LKF_NODLCKWT, name, namelen, 0, + fsdlm_lock_ast_wrapper, lksb, + fsdlm_blocking_ast_wrapper); + return ret; +} + +/* + * The ocfs2_dlm_lock() and ocfs2_dlm_unlock() functions take no argument + * for the ast and bast functions. They will pass the lksb to the ast + * and bast. The caller can wrap the lksb with their own structure to + * get more information. + */ +int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen) +{ + if (!lksb->lksb_conn) + lksb->lksb_conn = conn; + else + BUG_ON(lksb->lksb_conn != conn); + return user_dlm_lock(conn, mode, lksb, flags, name, namelen); +} + +static int user_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags) +{ + int ret; + + ret = dlm_unlock(conn->cc_lockspace, lksb->lksb_fsdlm.sb_lkid, + flags, &lksb->lksb_fsdlm, lksb); + return ret; +} + +int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags) +{ + BUG_ON(lksb->lksb_conn == NULL); + + return user_dlm_unlock(conn, lksb, flags); +} + +static int user_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) +{ + return lksb->lksb_fsdlm.sb_status; +} + +int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) +{ + return user_dlm_lock_status(lksb); +} + +static int user_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) +{ + int invalid = lksb->lksb_fsdlm.sb_flags & DLM_SBF_VALNOTVALID; + + return !invalid; +} + +int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) +{ + return user_dlm_lvb_valid(lksb); +} + +static void *user_dlm_lvb(struct ocfs2_dlm_lksb *lksb) +{ + if (!lksb->lksb_fsdlm.sb_lvbptr) + lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + + sizeof(struct dlm_lksb); + return (void *)(lksb->lksb_fsdlm.sb_lvbptr); +} + +void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb) +{ + return user_dlm_lvb(lksb); +} + +void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb) +{ +} + +static int user_plock(struct ocfs2_cluster_connection *conn, + u64 ino, + struct file *file, + int cmd, + struct file_lock *fl) +{ + /* + * This more or less just demuxes the plock request into any + * one of three dlm calls. + * + * Internally, fs/dlm will pass these to a misc device, which + * a userspace daemon will read and write to. + * + * For now, cancel requests (which happen internally only), + * are turned into unlocks. Most of this function taken from + * gfs2_lock. + */ + + if (cmd == F_CANCELLK) { + cmd = F_SETLK; + fl->fl_type = F_UNLCK; + } + + if (IS_GETLK(cmd)) + return dlm_posix_get(conn->cc_lockspace, ino, file, fl); + else if (fl->fl_type == F_UNLCK) + return dlm_posix_unlock(conn->cc_lockspace, ino, file, fl); + else + return dlm_posix_lock(conn->cc_lockspace, ino, file, cmd, fl); +} + +int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, + struct file *file, int cmd, struct file_lock *fl) +{ + return user_plock(conn, ino, file, cmd, fl); +} + +static struct dlm_lockspace_ops *ocfs2_ls_ops = NULL; + +static int user_cluster_connect(struct ocfs2_cluster_connection *conn) +{ + dlm_lockspace_t *fsdlm; +// struct ocfs2_live_connection *lc; + int rc, ops_rv; + + BUG_ON(conn == NULL); + +#if 0 + lc = kzalloc(sizeof(struct ocfs2_live_connection), GFP_KERNEL); + if (!lc) + return -ENOMEM; + + init_waitqueue_head(&lc->oc_wait); + init_completion(&lc->oc_sync_wait); + atomic_set(&lc->oc_this_node, 0); + conn->cc_private = lc; + lc->oc_type = NO_CONTROLD; +#endif + + rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name, + DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN, + ocfs2_ls_ops, conn, &ops_rv, &fsdlm); + if (rc) { + if (rc == -EEXIST || rc == -EPROTO) + printk(KERN_ERR "scoutfs: Unable to create the " + "lockspace %s (%d), because a scoutfs-utils " + "program is running on this file system " + "with the same name lockspace\n", + conn->cc_name, rc); + goto out; + } + + if (ops_rv == -EOPNOTSUPP) { + /* + * If we get this return code, we're on a very old + * version of fs/dlm that doesn't have recovery + * callbacks enabled. + */ +// lc->oc_type = WITH_CONTROLD; + printk(KERN_NOTICE "scoutfs: You seem to be using an older " + "version of dlm_controld and/or scoutfs-utils." + " Please consider upgrading.\n"); + } else if (ops_rv) { + rc = ops_rv; + goto out; + } + conn->cc_lockspace = fsdlm; + +#if 0 + rc = ocfs2_live_connection_attach(conn, lc); + if (rc) + goto out; + + if (lc->oc_type == NO_CONTROLD) { + rc = get_protocol_version(conn); + if (rc) { + printk(KERN_ERR "ocfs2: Could not determine" + " locking version\n"); + user_cluster_disconnect(conn); + goto out; + } + wait_event(lc->oc_wait, (atomic_read(&lc->oc_this_node) > 0)); + } + + /* + * running_proto must have been set before we allowed any mounts + * to proceed. + */ + if (fs_protocol_compare(&running_proto, &conn->cc_version)) { + printk(KERN_ERR + "Unable to mount with fs locking protocol version " + "%u.%u because negotiated protocol is %u.%u\n", + conn->cc_version.pv_major, conn->cc_version.pv_minor, + running_proto.pv_major, running_proto.pv_minor); + rc = -EPROTO; + ocfs2_live_connection_drop(lc); + lc = NULL; + } +#endif +out: +#if 0 + if (rc) + kfree(lc); +#endif + return rc; +} + +int ocfs2_cluster_connect(const char *stack_name, + const char *cluster_name, + int cluster_name_len, + const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn) +{ + int rc = 0; + struct ocfs2_cluster_connection *new_conn; + + BUG_ON(group == NULL); + BUG_ON(conn == NULL); + BUG_ON(recovery_handler == NULL); + + if (grouplen > GROUP_NAME_MAX) { + rc = -EINVAL; + goto out; + } + +#if 0 + if (memcmp(&lproto->lp_max_version, &locking_max_version, + sizeof(struct ocfs2_protocol_version))) { + rc = -EINVAL; + goto out; + } +#endif + new_conn = kzalloc(sizeof(struct ocfs2_cluster_connection), + GFP_KERNEL); + if (!new_conn) { + rc = -ENOMEM; + goto out; + } + + strlcpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1); + new_conn->cc_namelen = grouplen; + if (cluster_name_len) + strlcpy(new_conn->cc_cluster_name, cluster_name, + CLUSTER_NAME_MAX + 1); + new_conn->cc_cluster_name_len = cluster_name_len; + new_conn->cc_recovery_handler = recovery_handler; + new_conn->cc_recovery_data = recovery_data; + + new_conn->cc_proto = lproto; + /* Start the new connection at our maximum compatibility level */ + new_conn->cc_version = lproto->lp_max_version; + +#if 0 + /* This will pin the stack driver if successful */ + rc = ocfs2_stack_driver_get(stack_name); + if (rc) + goto out_free; +#endif + + rc = user_cluster_connect(new_conn); + if (rc) { +// ocfs2_stack_driver_put(); + goto out_free; + } + + *conn = new_conn; + +out_free: + if (rc) + kfree(new_conn); + +out: + return rc; +} + +static int user_cluster_disconnect(struct ocfs2_cluster_connection *conn) +{ + dlm_release_lockspace(conn->cc_lockspace, 2); + conn->cc_lockspace = NULL; + conn->cc_private = NULL; + return 0; +} + +/* If hangup_pending is 0, the stack driver will be dropped */ +int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, + int hangup_pending) +{ + int ret; + + BUG_ON(conn == NULL); + + ret = user_cluster_disconnect(conn); + + /* XXX Should we free it anyway? */ + if (!ret) { + kfree(conn); +#if 0 + if (!hangup_pending) + ocfs2_stack_driver_put(); +#endif + } + + return ret; +} diff --git a/kmod/src/stackglue.h b/kmod/src/stackglue.h new file mode 100644 index 00000000..e3db7678 --- /dev/null +++ b/kmod/src/stackglue.h @@ -0,0 +1,149 @@ +/* -*- mode: c; c-basic-offset: 8; -*- + * vim: noexpandtab sw=8 ts=8 sts=0: + * + * stackglue.h + * + * Glue to the underlying cluster stack. + * + * Copyright (C) 2007 Oracle. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + + +#ifndef STACKGLUE_H +#define STACKGLUE_H + +#include +#include +#include + +#include +#include + +#define DLM_LVB_LEN 64 + +/* Needed for plock-related prototypes */ +struct file; +struct file_lock; + +/* Scoutfs never uses this flag, we define it to zero to avoid errors */ +#define DLM_LKF_LOCAL 0 + +/* + * This shadows DLM_LOCKSPACE_LEN in fs/dlm/dlm_internal.h. That probably + * wants to be in a public header. + */ +#define GROUP_NAME_MAX 64 + +/* This shadows OCFS2_CLUSTER_NAME_LEN */ +#define CLUSTER_NAME_MAX 16 + +/* + * ocfs2_protocol_version changes when ocfs2 does something different in + * its inter-node behavior. See dlmglue.c for more information. + */ +struct ocfs2_protocol_version { + u8 pv_major; + u8 pv_minor; +}; + +/* + * The dlm_lockstatus struct includes lvb space, but the dlm_lksb struct only + * has a pointer to separately allocated lvb space. This struct exists only to + * include in the lksb union to make space for a combined dlm_lksb and lvb. + */ +struct fsdlm_lksb_plus_lvb { + struct dlm_lksb lksb; + char lvb[DLM_LVB_LEN]; +}; + +/* + * A union of all lock status structures. We define it here so that the + * size of the union is known. Lock status structures are embedded in + * ocfs2 inodes. + */ +struct ocfs2_cluster_connection; +struct ocfs2_dlm_lksb { + union { + struct dlm_lksb lksb_fsdlm; + struct fsdlm_lksb_plus_lvb padding; + }; + struct ocfs2_cluster_connection *lksb_conn; +}; + +/* + * The ocfs2_locking_protocol defines the handlers called on ocfs2's behalf. + */ +struct ocfs2_locking_protocol { + struct ocfs2_protocol_version lp_max_version; + void (*lp_lock_ast)(struct ocfs2_dlm_lksb *lksb); + void (*lp_blocking_ast)(struct ocfs2_dlm_lksb *lksb, int level); + void (*lp_unlock_ast)(struct ocfs2_dlm_lksb *lksb, int error); +}; + +/* + * A cluster connection. Mostly opaque to ocfs2, the connection holds + * state for the underlying stack. ocfs2 does use cc_version to determine + * locking compatibility. + */ +struct ocfs2_cluster_connection { + char cc_name[GROUP_NAME_MAX + 1]; + int cc_namelen; + char cc_cluster_name[CLUSTER_NAME_MAX + 1]; + int cc_cluster_name_len; + struct ocfs2_protocol_version cc_version; + struct ocfs2_locking_protocol *cc_proto; + void (*cc_recovery_handler)(int node_num, void *recovery_data); + void *cc_recovery_data; + void *cc_lockspace; + void *cc_private; +}; + +/* In ocfs2_downconvert_lock(), we need to know which stack we are using */ +static inline int ocfs2_is_o2cb_active(void) +{ + return 0; +} + +/* Used by the filesystem */ +int ocfs2_cluster_connect(const char *stack_name, + const char *cluster_name, + int cluster_name_len, + const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn); +int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, + int hangup_pending); + +struct ocfs2_lock_res; +int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen); +int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags); + +int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb); +int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb); +void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb); +void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb); + +int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, + struct file *file, int cmd, struct file_lock *fl); + +#endif /* STACKGLUE_H */ From b1084bee8fb7ba0e08f1a0cdc8530d549ada1901 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 21:56:52 -0500 Subject: [PATCH 385/920] scoutfs: enable ocfs2_dlm_init/ocfs2_dlm_shutdown These work with little modification. We comment out a couple ocfs2-specific lines. And decouple a few more variables from the osb structure. As it stands, ocfs2 could also use these init/shutdown functions with little modification. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 43 +++++++++++++++++++++++++++---------------- kmod/src/dlmglue.h | 10 ++++++---- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 6c0e0d7b..1e8fce4d 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -1247,7 +1247,6 @@ static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) spin_unlock_irqrestore(&lockres->l_lock, flags); } -#if 0 /* * This is the filesystem locking protocol. It provides the lock handling * hooks for the underlying DLM. It has a maximum version number. @@ -1272,15 +1271,18 @@ static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) * updated. */ static struct ocfs2_locking_protocol lproto = { +#if 0 .lp_max_version = { .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR, .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR, }, +#endif .lp_lock_ast = ocfs2_locking_ast, .lp_blocking_ast = ocfs2_blocking_ast, .lp_unlock_ast = ocfs2_unlock_ast, }; +#if 0 void ocfs2_set_locking_protocol(void) { ocfs2_stack_glue_set_max_proto_version(&lproto.lp_max_version); @@ -3093,14 +3095,15 @@ static const struct file_operations ocfs2_dlm_debug_fops = { .llseek = seq_lseek, }; -static int ocfs2_dlm_init_debug(struct ocfs2_super *osb) +static int ocfs2_dlm_init_debug(struct ocfs2_super *osb, + struct dentry *debug_root) { int ret = 0; struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; dlm_debug->d_locking_state = debugfs_create_file("locking_state", S_IFREG|S_IRUSR, - osb->osb_debug_root, + debug_root, osb, &ocfs2_dlm_debug_fops); if (!dlm_debug->d_locking_state) { @@ -3125,26 +3128,32 @@ static void ocfs2_dlm_shutdown_debug(struct ocfs2_super *osb) } } -#if 0 -int ocfs2_dlm_init(struct ocfs2_super *osb) +static void ocfs2_do_node_down(int node_num, void *data) +{ +} + +int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, + char *cluster_name, char *ls_name, struct dentry *debug_root) { int status = 0; struct ocfs2_cluster_connection *conn = NULL; +#if 0 if (ocfs2_mount_local(osb)) { osb->node_num = 0; goto local; } +#endif - status = ocfs2_dlm_init_debug(osb); + status = ocfs2_dlm_init_debug(osb, debug_root); if (status < 0) { mlog_errno(status); goto bail; } /* launch downconvert thread */ - osb->dc_task = kthread_run(ocfs2_downconvert_thread, osb, "ocfs2dc-%s", - osb->uuid_str); + osb->dc_task = kthread_run(ocfs2_downconvert_thread, osb, "scoutdc-%s", + ls_name); if (IS_ERR(osb->dc_task)) { status = PTR_ERR(osb->dc_task); osb->dc_task = NULL; @@ -3153,11 +3162,11 @@ int ocfs2_dlm_init(struct ocfs2_super *osb) } /* for now, uuid == domain */ - status = ocfs2_cluster_connect(osb->osb_cluster_stack, - osb->osb_cluster_name, - strlen(osb->osb_cluster_name), - osb->uuid_str, - strlen(osb->uuid_str), + status = ocfs2_cluster_connect(cluster_stack, + cluster_name, + strlen(cluster_name), + ls_name, + strlen(ls_name), &lproto, ocfs2_do_node_down, osb, &conn); if (status) { @@ -3165,6 +3174,7 @@ int ocfs2_dlm_init(struct ocfs2_super *osb) goto bail; } +#if 0 status = ocfs2_cluster_this_node(conn, &osb->node_num); if (status < 0) { mlog_errno(status); @@ -3179,7 +3189,7 @@ local: ocfs2_rename_lock_res_init(&osb->osb_rename_lockres, osb); ocfs2_nfs_sync_lock_res_init(&osb->osb_nfs_sync_lockres, osb); ocfs2_orphan_scan_lock_res_init(&osb->osb_orphan_scan.os_lockres, osb); - +#endif osb->cconn = conn; bail: if (status < 0) { @@ -3194,7 +3204,7 @@ bail: void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending) { - ocfs2_drop_osb_locks(osb); +// ocfs2_drop_osb_locks(osb); /* * Now that we have dropped all locks and ocfs2_dismount_volume() @@ -3207,17 +3217,18 @@ void ocfs2_dlm_shutdown(struct ocfs2_super *osb, osb->dc_task = NULL; } +#if 0 ocfs2_lock_res_free(&osb->osb_super_lockres); ocfs2_lock_res_free(&osb->osb_rename_lockres); ocfs2_lock_res_free(&osb->osb_nfs_sync_lockres); ocfs2_lock_res_free(&osb->osb_orphan_scan.os_lockres); +#endif ocfs2_cluster_disconnect(osb->cconn, hangup_pending); osb->cconn = NULL; ocfs2_dlm_shutdown_debug(osb); } -#endif static int ocfs2_drop_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index beee5cc2..fc9508e0 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -142,11 +142,14 @@ struct ocfs2_dlm_debug { struct list_head d_lockres_tracking; }; +/* The cluster stack fields */ +#define OCFS2_STACK_LABEL_LEN 4 +#define OCFS2_CLUSTER_NAME_LEN 16 + struct ocfs2_super { struct ocfs2_cluster_connection *cconn; struct ocfs2_dlm_debug *osb_dlm_debug; - struct dentry *osb_debug_root; /* Downconvert thread */ spinlock_t dc_task_lock; @@ -238,10 +241,9 @@ enum { OI_LS_REFLINK_TARGET, }; -#if 0 -int ocfs2_dlm_init(struct ocfs2_super *osb); +int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, + char *cluster_name, char *ls_name, struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); -#endif void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); #if 0 void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, From 4fb011ca71a652d945207e939be60ad41da30626 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 22:18:42 -0500 Subject: [PATCH 386/920] scoutfs: export ocfs2_cluster_(un)lock from dlmglue.c This is what we'll want to build our scoutfs locks on. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 15 +++++++-------- kmod/src/dlmglue.h | 5 +++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 1e8fce4d..cbeab58e 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -377,9 +377,8 @@ static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lock static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level, unsigned long caller_ip); -static inline void ocfs2_cluster_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level) +void ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, int level) { __ocfs2_cluster_unlock(osb, lockres, level, _RET_IP_); } @@ -1671,11 +1670,11 @@ out: return ret; } -static inline int ocfs2_cluster_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - u32 lkm_flags, - int arg_flags) +int ocfs2_cluster_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 lkm_flags, + int arg_flags) { return __ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags, 0, _RET_IP_); diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index fc9508e0..b81c65e4 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -241,6 +241,11 @@ enum { OI_LS_REFLINK_TARGET, }; +int ocfs2_cluster_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, + int level, u32 lkm_flags, int arg_flags); +void ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, int level); + int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, char *cluster_name, char *ls_name, struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); From 6308f347c027c583ad81fa5b4490572b467d8076 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 23 Aug 2017 22:28:37 -0500 Subject: [PATCH 387/920] scoutfs: provide a function to init and uninit our dlmglue context Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 21 +++++++++++++++++++++ kmod/src/dlmglue.h | 3 +++ 2 files changed, 24 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index cbeab58e..e41d346a 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -4279,3 +4279,24 @@ void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb) spin_unlock_irqrestore(&osb->dc_task_lock, flags); wake_up(&osb->dc_event); } + +int ocfs2_init_super(struct ocfs2_super *osb, int flags) +{ + memset(osb, 0, sizeof(*osb)); + + osb->osb_dlm_debug = ocfs2_new_dlm_debug(); + if (!osb->osb_dlm_debug) + return -ENOMEM; + + spin_lock_init(&osb->dc_task_lock); + init_waitqueue_head(&osb->dc_event); + INIT_LIST_HEAD(&osb->blocked_lock_list); + osb->s_mount_opt = flags; + + return 0; +} + +void ocfs2_uninit_super(struct ocfs2_super *osb) +{ + ocfs2_dlm_shutdown_debug(osb); +} diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index b81c65e4..3e25c8c5 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -246,6 +246,9 @@ int ocfs2_cluster_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, void ocfs2_cluster_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level); +int ocfs2_init_super(struct ocfs2_super *osb, int flags); +void ocfs2_uninit_super(struct ocfs2_super *osb); + int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, char *cluster_name, char *ls_name, struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); From 00f5ebf38c7ec7fcba90b0478a91b6c78c11b3ac Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 24 Aug 2017 01:29:33 -0500 Subject: [PATCH 388/920] scoutfs: use dlmglue for lockspace bringup/shutdown Ultimataly the direct dlm lock calls will go away. For now though we grab the lockspace off our cluster connection object. In order to get this going, I stubbed out our recovery callbacks which now gets us a print when a node goes down. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 53 ++++++++++++++++++++++++++------------------ kmod/src/stackglue.c | 26 ++++++++++++++++++++-- kmod/src/super.c | 29 ++++++++++++++++++++++++ kmod/src/super.h | 2 ++ 4 files changed, 86 insertions(+), 24 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 16855530..4ccc4fc3 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -22,18 +22,22 @@ #include "scoutfs_trace.h" #include "msg.h" #include "cmp.h" +#include "dlmglue.h" #define LN_FMT "%u.%u.%llu.%llu" #define LN_ARG(name) \ (name)->zone, (name)->type, le64_to_cpu((name)->first), \ le64_to_cpu((name)->second) +typedef struct ocfs2_super dlmglue_ctxt; + /* * allocated per-super, freed on unmount. */ struct lock_info { struct super_block *sb; - dlm_lockspace_t *ls; + dlmglue_ctxt dlmglue; + bool dlmglue_online; char ls_name[DLM_LOCKSPACE_LEN]; bool shutdown; struct list_head id_head; @@ -394,8 +398,9 @@ check_lock_state: lock->holders++; spin_unlock(&linfo->lock); - ret = dlm_lock(linfo->ls, mode, &lock->lksb, DLM_LKF_NOORDER, - &lock->lock_name, sizeof(struct scoutfs_lock_name), + ret = dlm_lock(linfo->dlmglue.cconn->cc_lockspace, mode, &lock->lksb, + DLM_LKF_NOORDER, &lock->lock_name, + sizeof(struct scoutfs_lock_name), 0, scoutfs_ast, lock, scoutfs_bast); if (ret) { scoutfs_err(sb, "Error %d locking "LN_FMT, ret, @@ -533,7 +538,8 @@ static void unlock_range(struct super_block *sb, struct scoutfs_lock *lock) spin_lock(&linfo->lock); lock->rqmode = DLM_LOCK_IV; spin_unlock(&linfo->lock); - ret = dlm_unlock(linfo->ls, lock->lksb.sb_lkid, 0, &lock->lksb, lock); + ret = dlm_unlock(linfo->dlmglue.cconn->cc_lockspace, lock->lksb.sb_lkid, + 0, &lock->lksb, lock); if (ret) { scoutfs_err(sb, "Error %d unlocking "LN_FMT, ret, LN_ARG(&lock->lock_name)); @@ -601,11 +607,16 @@ static int init_lock_info(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct lock_info *linfo; + int ret; linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); if (!linfo) return -ENOMEM; + ret = ocfs2_init_super(&linfo->dlmglue, 0); + if (ret) + goto out; + spin_lock_init(&linfo->lock); init_waitqueue_head(&linfo->waitq); INIT_LIST_HEAD(&linfo->lru_list); @@ -615,7 +626,6 @@ static int init_lock_info(struct super_block *sb) linfo->sb = sb; linfo->shutdown = false; INIT_LIST_HEAD(&linfo->id_head); - linfo->ls = NULL; snprintf(linfo->ls_name, DLM_LOCKSPACE_LEN, "%llx", le64_to_cpu(sbi->super.hdr.fsid)); @@ -624,6 +634,9 @@ static int init_lock_info(struct super_block *sb) trace_printk("sb %p id %016llx allocated linfo %p held %p\n", sb, le64_to_cpu(sbi->super.id), linfo, linfo); +out: + if (ret) + kfree(linfo); return 0; } @@ -651,17 +664,14 @@ void scoutfs_lock_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_LOCK_INFO(sb, linfo); - int ret; if (linfo) { if (linfo->downconvert_wq) destroy_workqueue(linfo->downconvert_wq); unregister_shrinker(&linfo->shrinker); - if (linfo->ls) { - ret = dlm_release_lockspace(linfo->ls, 2); - if (ret) - scoutfs_info(sb, "Error %d releasing lockspace %s\n", - ret, linfo->ls_name); + if (linfo->dlmglue_online) { + ocfs2_dlm_shutdown(&linfo->dlmglue, 0); + ocfs2_uninit_super(&linfo->dlmglue); } free_lock_tree(sb); @@ -684,23 +694,22 @@ int scoutfs_lock_setup(struct super_block *sb) ret = init_lock_info(sb); if (ret) return ret; - linfo = sbi->lock_info; + linfo->downconvert_wq = alloc_workqueue("scoutfs_dc", WQ_UNBOUND|WQ_HIGHPRI, 0); if (!linfo->downconvert_wq) { - kfree(linfo); - return -ENOMEM; + ret = -ENOMEM; + goto out; } - /* - * Open coded '64' here is for lvb_len. We never use the LVB - * flag so this doesn't matter, but the dlm needs a non-zero - * multiple of 8 - */ - ret = dlm_new_lockspace(linfo->ls_name, sbi->opts.cluster_name, - DLM_LSFL_FS|DLM_LSFL_NEWEXCL, 64, NULL, - NULL, NULL, &linfo->ls); + ret = ocfs2_dlm_init(&linfo->dlmglue, "null", sbi->opts.cluster_name, + linfo->ls_name, sbi->debug_root); + if (ret) + goto out; + linfo->dlmglue_online = true; + +out: if (ret) scoutfs_lock_destroy(sb); diff --git a/kmod/src/stackglue.c b/kmod/src/stackglue.c index 49f101d2..f2ff3b1b 100644 --- a/kmod/src/stackglue.c +++ b/kmod/src/stackglue.c @@ -194,7 +194,29 @@ int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, return user_plock(conn, ino, file, cmd, fl); } -static struct dlm_lockspace_ops *ocfs2_ls_ops = NULL; +static void user_recover_prep(void *arg) +{ + /* XXX: Set FS in recovery here */ +} + +static void user_recover_slot(void *arg, struct dlm_slot *slot) +{ + printk(KERN_INFO "scoutfs: Node %d/%d down. Initiating recovery.\n", + slot->nodeid, slot->slot); +} + +static void user_recover_done(void *arg, struct dlm_slot *slots, + int num_slots, int our_slot, + uint32_t generation) +{ + /* XXX: Do actual fs recovery here */ +} + +static const struct dlm_lockspace_ops ocfs2_ls_ops = { + .recover_prep = user_recover_prep, + .recover_slot = user_recover_slot, + .recover_done = user_recover_done, +}; static int user_cluster_connect(struct ocfs2_cluster_connection *conn) { @@ -218,7 +240,7 @@ static int user_cluster_connect(struct ocfs2_cluster_connection *conn) rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name, DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN, - ocfs2_ls_ops, conn, &ops_rv, &fsdlm); + &ocfs2_ls_ops, conn, &ops_rv, &fsdlm); if (rc) { if (rc == -EEXIST || rc == -EPROTO) printk(KERN_ERR "scoutfs: Unable to create the " diff --git a/kmod/src/super.c b/kmod/src/super.c index 34c0835a..29d29369 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -42,6 +43,7 @@ #include "scoutfs_trace.h" static struct kset *scoutfs_kset; +static struct dentry *scoutfs_debugfs_root; /* * Ask the server for the current statfs fields. The message is very @@ -202,6 +204,24 @@ int scoutfs_read_supers(struct super_block *sb, return 0; } +static int scoutfs_debugfs_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + char name[32]; + + /* + * XXX: Move the name variable to sbi and use it in + * init_lock_info as well. + */ + snprintf(name, 32, "%llx", le64_to_cpu(sbi->super.hdr.fsid)); + + sbi->debug_root = debugfs_create_dir(name, scoutfs_debugfs_root); + if (!sbi->debug_root) + return -ENOMEM; + + return 0; +} + static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; @@ -247,6 +267,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: + scoutfs_debugfs_setup(sb) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_inode_setup(sb) ?: @@ -330,6 +351,7 @@ static void scoutfs_kill_sb(struct super_block *sb) scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); scoutfs_seg_destroy(sb); + debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); if (sbi->kset) kset_unregister(sbi->kset); @@ -349,6 +371,7 @@ MODULE_ALIAS_FS("scoutfs"); /* safe to call at any failure point in _init */ static void teardown_module(void) { + debugfs_remove(scoutfs_debugfs_root); scoutfs_dir_exit(); scoutfs_inode_exit(); if (scoutfs_kset) @@ -374,10 +397,16 @@ static int __init scoutfs_module_init(void) if (!scoutfs_kset) return -ENOMEM; + scoutfs_debugfs_root = debugfs_create_dir("scoutfs", NULL); + if (!scoutfs_debugfs_root) { + ret = -ENOMEM; + goto out; + } ret = scoutfs_inode_init() ?: scoutfs_dir_init() ?: scoutfs_xattr_init() ?: register_filesystem(&scoutfs_fs_type); +out: if (ret) teardown_module(); return ret; diff --git a/kmod/src/super.h b/kmod/src/super.h index c8e8a9a6..933ed2b4 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -61,6 +61,8 @@ struct scoutfs_sb_info { struct scoutfs_counters *counters; struct mount_options opts; + + struct dentry *debug_root; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) From 0011c185a93dd50bd0f29114a025827a95a8e469 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 24 Aug 2017 03:45:58 -0500 Subject: [PATCH 389/920] scoutfs: plug the rest of our locking into dlmglue We move struct ocfs2_lock_res_ops and flags to dlmglue.c so that locks.c can get access to it. Similarly, we export ocfs2_lock_res_init_common() for locks.c can initialize each lockres before use. Also, free_lock_tree() now has to happen before we shut down the dlm - this gives dlmglue the opportunity to unlock their underlying dlm locks before we go off freeing the structures. Signed-off-by: Mark Fasheh --- kmod/src/data.c | 2 +- kmod/src/dir.c | 22 ++++----- kmod/src/dlmglue.c | 13 +++--- kmod/src/dlmglue.h | 108 +++++++++++++++++++++++++++++++++++++++++++++ kmod/src/inode.c | 4 +- kmod/src/ioctl.c | 4 +- kmod/src/lock.c | 85 +++++++++++++++++++++++++++++++---- kmod/src/lock.h | 5 ++- kmod/src/xattr.c | 8 ++-- 9 files changed, 215 insertions(+), 36 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 661cff5b..3ecb3341 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1238,7 +1238,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, blk_off = ext.blk_off + ext.blocks; } - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); out: mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 85730f25..8581a8a5 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -283,7 +283,7 @@ out: else inode = scoutfs_iget(sb, ino); - scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); scoutfs_key_free(sb, key); @@ -390,7 +390,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } out: - scoutfs_unlock(sb, dir_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); kfree(dent); return ret; @@ -545,8 +545,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, out: scoutfs_release_trans(sb); out_unlock: - scoutfs_unlock(sb, dir_lock); - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); @@ -614,8 +614,8 @@ static int scoutfs_link(struct dentry *old_dentry, out: scoutfs_release_trans(sb); out_unlock: - scoutfs_unlock(sb, dir_lock); - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); return ret; } @@ -714,8 +714,8 @@ out_trans: out: scoutfs_key_free(sb, keys[0]); scoutfs_key_free(sb, keys[2]); - scoutfs_unlock(sb, dir_lock); - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); return ret; } @@ -837,7 +837,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) nd_set_link(nd, path); } out: - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); return path; } @@ -934,8 +934,8 @@ out: scoutfs_release_trans(sb); out_unlock: - scoutfs_unlock(sb, dir_lock); - scoutfs_unlock(sb, inode_lock); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); return ret; } diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index e41d346a..59615a31 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -97,6 +97,7 @@ static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres); static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres); #endif +#if 0 /* * Return value from ->downconvert_worker functions. * @@ -111,7 +112,7 @@ enum ocfs2_unblock_action { UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire * ->post_unlock() callback. */ }; - +#endif struct ocfs2_unblock_ctl { int requeue; enum ocfs2_unblock_action unblock_action; @@ -169,7 +170,6 @@ static void ocfs2_dump_meta_lvb_info(u64 level, (long long)be64_to_cpu(lvb->lvb_imtime_packed), be32_to_cpu(lvb->lvb_iattr)); } -#endif /* @@ -260,7 +260,6 @@ struct ocfs2_lock_res_ops { */ #define LOCK_TYPE_USES_LVB 0x2 -#if 0 static struct ocfs2_lock_res_ops ocfs2_inode_rw_lops = { .get_osb = ocfs2_get_inode_osb, .flags = 0, @@ -532,10 +531,10 @@ static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) } #endif -static void ocfs2_lock_res_init_common(struct ocfs2_super *osb, - struct ocfs2_lock_res *res, - struct ocfs2_lock_res_ops *ops, - void *priv) +void ocfs2_lock_res_init_common(struct ocfs2_super *osb, + struct ocfs2_lock_res *res, + struct ocfs2_lock_res_ops *ops, + void *priv) { res->l_ops = ops; res->l_priv = priv; diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 3e25c8c5..7e005da2 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -172,6 +172,110 @@ struct ocfs2_super /* For s_mount_opt */ #define OCFS2_MOUNT_NOINTR (1 << 2) +/* + * Return value from ->downconvert_worker functions. + * + * These control the precise actions of ocfs2_unblock_lock() + * and ocfs2_process_blocked_lock() + * + */ +enum ocfs2_unblock_action { + UNBLOCK_CONTINUE = 0, /* Continue downconvert */ + UNBLOCK_CONTINUE_POST = 1, /* Continue downconvert, fire + * ->post_unlock callback */ + UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire + * ->post_unlock() callback. */ +}; + +/* + * OCFS2 Lock Resource Operations + * + * These fine tune the behavior of the generic dlmglue locking infrastructure. + * + * The most basic of lock types can point ->l_priv to their respective + * struct ocfs2_super and allow the default actions to manage things. + * + * Right now, each lock type also needs to implement an init function, + * and trivial lock/unlock wrappers. ocfs2_simple_drop_lockres() + * should be called when the lock is no longer needed (i.e., object + * destruction time). + */ +struct ocfs2_lock_res_ops { + /* + * Translate an ocfs2_lock_res * into an ocfs2_super *. Define + * this callback if ->l_priv is not an ocfs2_super pointer + */ + struct ocfs2_super * (*get_osb)(struct ocfs2_lock_res *); + + /* + * Optionally called in the downconvert thread after a + * successful downconvert. The lockres will not be referenced + * after this callback is called, so it is safe to free + * memory, etc. + * + * The exact semantics of when this is called are controlled + * by ->downconvert_worker() + */ + void (*post_unlock)(struct ocfs2_super *, struct ocfs2_lock_res *); + + /* + * Allow a lock type to add checks to determine whether it is + * safe to downconvert a lock. Return 0 to re-queue the + * downconvert at a later time, nonzero to continue. + * + * For most locks, the default checks that there are no + * incompatible holders are sufficient. + * + * Called with the lockres spinlock held. + */ + int (*check_downconvert)(struct ocfs2_lock_res *, int); + + /* + * Allows a lock type to populate the lock value block. This + * is called on downconvert, and when we drop a lock. + * + * Locks that want to use this should set LOCK_TYPE_USES_LVB + * in the flags field. + * + * Called with the lockres spinlock held. + */ + void (*set_lvb)(struct ocfs2_lock_res *); + + /* + * Called from the downconvert thread when it is determined + * that a lock will be downconverted. This is called without + * any locks held so the function can do work that might + * schedule (syncing out data, etc). + * + * This should return any one of the ocfs2_unblock_action + * values, depending on what it wants the thread to do. + */ + int (*downconvert_worker)(struct ocfs2_lock_res *, int); + + /* + * LOCK_TYPE_* flags which describe the specific requirements + * of a lock type. Descriptions of each individual flag follow. + */ + int flags; +}; + +/* + * Some locks want to "refresh" potentially stale data when a + * meaningful (PRMODE or EXMODE) lock level is first obtained. If this + * flag is set, the OCFS2_LOCK_NEEDS_REFRESH flag will be set on the + * individual lockres l_flags member from the ast function. It is + * expected that the locking wrapper will clear the + * OCFS2_LOCK_NEEDS_REFRESH flag when done. + */ +#define LOCK_TYPE_REQUIRES_REFRESH 0x1 + +/* + * Indicate that a lock type makes use of the lock value block. The + * ->set_lvb lock type callback must be defined. + */ +#define LOCK_TYPE_USES_LVB 0x2 + + #if 0 #include "dcache.h" @@ -253,6 +357,10 @@ int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, char *cluster_name, char *ls_name, struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); +void ocfs2_lock_res_init_common(struct ocfs2_super *osb, + struct ocfs2_lock_res *res, + struct ocfs2_lock_res_ops *ops, + void *priv); #if 0 void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, enum ocfs2_lock_type type, diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 2b7ddb0a..1b9761b6 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -284,7 +284,7 @@ static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, if (ret == 0) generic_fillattr(inode, stat); - scoutfs_unlock(sb, lock); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); return ret; } @@ -416,7 +416,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) } out: - scoutfs_unlock(sb, lock); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); return inode; } diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 3fe08339..abbf3fa4 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -132,7 +132,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (ret == -ENOENT) { - scoutfs_unlock(sb, lock); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); /* * XXX This will miss dirty items. We'd need to * force writeouts of dirty items in our @@ -181,7 +181,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) scoutfs_key_inc_cur_len(&key); } - scoutfs_unlock(sb, lock); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); out: scoutfs_key_free(sb, next_key); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 4ccc4fc3..360fe216 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -83,7 +83,12 @@ static int invalidate_caches(struct super_block *sb, int mode, static void free_scoutfs_lock(struct scoutfs_lock *lock) { + struct lock_info *linfo; + if (lock) { + linfo = SCOUTFS_SB(lock->sb)->lock_info; + + ocfs2_simple_drop_lockres(&linfo->dlmglue, &lock->lockres); scoutfs_key_free(lock->sb, lock->start); scoutfs_key_free(lock->sb, lock->end); kfree(lock); @@ -113,12 +118,50 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) } } +static struct ocfs2_super *get_ino_lock_osb(struct ocfs2_lock_res *lockres) +{ + struct scoutfs_lock *lock = lockres->l_priv; + struct super_block *sb = lock->sb; + DECLARE_LOCK_INFO(sb, linfo); + + return &linfo->dlmglue; +} + +static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) +{ + struct scoutfs_lock *lock = lockres->l_priv; + struct super_block *sb = lock->sb; + + invalidate_caches(sb, blocking, lock->start, lock->end); + + return UNBLOCK_CONTINUE; +} + +static struct ocfs2_lock_res_ops scoufs_ino_lops = { + .get_osb = get_ino_lock_osb, + .downconvert_worker = ino_lock_downconvert, + /* XXX: .post_unlock for lru */ + /* XXX: .check_downconvert that queries the item cache for dirty items */ + .flags = LOCK_TYPE_REQUIRES_REFRESH, +}; + +static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { + .get_osb = get_ino_lock_osb, + .downconvert_worker = ino_lock_downconvert, + /* XXX: .post_unlock for lru */ + /* XXX: .check_downconvert that queries the item cache for dirty items */ + .flags = 0, +}; + static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_lock_name *lock_name, + struct ocfs2_lock_res_ops *type, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end) { + DECLARE_LOCK_INFO(sb, linfo); +// struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock; lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); @@ -135,6 +178,14 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, lock->mode = DLM_LOCK_IV; INIT_WORK(&lock->dc_work, scoutfs_downconvert_func); INIT_LIST_HEAD(&lock->lru_entry); + ocfs2_lock_res_init_once(&lock->lockres); + BUG_ON(sizeof(struct scoutfs_lock_name) >= + OCFS2_LOCK_ID_MAX_LEN); + /* kzalloc above ensures that l_name is NULL terminated */ + memcpy(&lock->lockres.l_name[0], &lock->lock_name, + sizeof(struct scoutfs_lock_name)); + ocfs2_lock_res_init_common(&linfo->dlmglue, + &lock->lockres, type, lock); } } @@ -152,6 +203,7 @@ static int cmp_lock_names(struct scoutfs_lock_name *a, static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_lock_name *lock_name, + struct ocfs2_lock_res_ops *type, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end) { @@ -187,7 +239,8 @@ search: if (!found) { if (!new) { spin_unlock(&linfo->lock); - new = alloc_scoutfs_lock(sb, lock_name, start, end); + new = alloc_scoutfs_lock(sb, lock_name, type, start, + end); if (!new) return NULL; @@ -349,6 +402,7 @@ static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock) */ static int lock_name_keys(struct super_block *sb, int mode, struct scoutfs_lock_name *lock_name, + struct ocfs2_lock_res_ops *type, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end, struct scoutfs_lock **ret_lock) @@ -357,12 +411,20 @@ static int lock_name_keys(struct super_block *sb, int mode, struct scoutfs_lock *lock; int ret; - lock = find_alloc_scoutfs_lock(sb, lock_name, start, end); + lock = find_alloc_scoutfs_lock(sb, lock_name, type, start, end); if (!lock) return -ENOMEM; trace_scoutfs_lock_resource(sb, lock); + ret = ocfs2_cluster_lock(&linfo->dlmglue, &lock->lockres, mode, + DLM_LKF_NOORDER, 0); + if (ret) + return ret; + + *ret_lock = lock; + return 0; +#if 0 check_lock_state: spin_lock(&linfo->lock); if (linfo->shutdown) { @@ -413,6 +475,7 @@ check_lock_state: out: *ret_lock = lock; return 0; +#endif } int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, @@ -441,7 +504,8 @@ int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, end_ikey.type = ~0; scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, &lock_name, &start, &end, ret_lock); + return lock_name_keys(sb, mode, &lock_name, &scoufs_ino_lops, &start, + &end, ret_lock); } /* @@ -505,10 +569,12 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, end_ikey.ino = cpu_to_be64(ino | ino_mask); scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, &lock_name, &start, &end, ret_lock); + return lock_name_keys(sb, mode, &lock_name, &scoufs_ino_index_lops, + &start, &end, ret_lock); } -void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, + int level) { DECLARE_LOCK_INFO(sb, linfo); @@ -517,12 +583,15 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock) trace_scoutfs_unlock(sb, lock); + ocfs2_cluster_unlock(&linfo->dlmglue, &lock->lockres, level); + +#if 0 spin_lock(&linfo->lock); lock->holders--; if (lock->holders == 0 && (lock->flags & SCOUTFS_LOCK_BLOCKING)) queue_blocking_work(linfo, lock); spin_unlock(&linfo->lock); - +#endif put_scoutfs_lock(sb, lock); } @@ -666,6 +735,8 @@ void scoutfs_lock_destroy(struct super_block *sb) DECLARE_LOCK_INFO(sb, linfo); if (linfo) { + free_lock_tree(sb); /* Do this before uninitializing the dlm. */ + if (linfo->downconvert_wq) destroy_workqueue(linfo->downconvert_wq); unregister_shrinker(&linfo->shrinker); @@ -674,8 +745,6 @@ void scoutfs_lock_destroy(struct super_block *sb) ocfs2_uninit_super(&linfo->dlmglue); } - free_lock_tree(sb); - sbi->lock_info = NULL; trace_printk("sb %p id %016llx freeing linfo %p linfo %p\n", diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 0b118e04..d47f9a7c 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -3,6 +3,7 @@ #include #include "key.h" +#include "dlmglue.h" #define SCOUTFS_LOCK_BLOCKING 0x01 /* Blocking another lock request */ #define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ @@ -22,6 +23,7 @@ struct scoutfs_lock { unsigned int holders; /* Tracks active users of this lock */ unsigned int flags; struct work_struct dc_work; + struct ocfs2_lock_res lockres; }; int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, @@ -29,7 +31,8 @@ int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); -void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock); +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, + int level); int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_shutdown(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index cd860bab..ad24cda9 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -229,7 +229,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = -ERANGE; up_read(&si->xattr_rwsem); - scoutfs_unlock(sb, lck); + scoutfs_unlock(sb, lck, DLM_LOCK_PR); out: scoutfs_key_free(sb, key); @@ -336,7 +336,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, scoutfs_release_trans(sb); unlock: - scoutfs_unlock(sb, lck); + scoutfs_unlock(sb, lck, DLM_LOCK_EX); out: scoutfs_item_free_batch(sb, &list); @@ -436,7 +436,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) } up_read(&si->xattr_rwsem); - scoutfs_unlock(sb, lck); + scoutfs_unlock(sb, lck, DLM_LOCK_PR); out: scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); @@ -490,7 +490,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* don't need to increment past deleted key */ } - scoutfs_unlock(sb, lck); + scoutfs_unlock(sb, lck, DLM_LOCK_EX); out: scoutfs_key_free(sb, key); From dc15c610ca7c4ff53e45774adf52122f579a51e2 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 24 Aug 2017 16:32:25 -0500 Subject: [PATCH 390/920] scoutfs: fix null pointer deref in get_manifest_refs() When we're not the server node, 'mani' is NULL, so derefing it in our loop causes a crash. That said, we don't need it anyway - the loop will eventually end when our btree walk (via btree_prev_overlap_or_next()) ends. Signed-off-by: Mark Fasheh --- kmod/src/manifest.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 1748dc5b..0ce44fe3 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -458,7 +458,6 @@ static int get_manifest_refs(struct super_block *sb, struct scoutfs_key_buf *end, struct list_head *ref_list) { - DECLARE_MANIFEST(sb, mani); struct scoutfs_manifest_btree_key *mkey; struct scoutfs_manifest_entry ment; SCOUTFS_BTREE_ITEM_REF(iref); @@ -499,7 +498,7 @@ static int get_manifest_refs(struct super_block *sb, * cached items to their locks. */ mkey_len = init_btree_key(mkey, 1, 0, key); - for (i = 1; i < mani->nr_levels; i++) { + for (i = 1; ; i++) { mkey->level = i; /* XXX should use level counts to skip searches */ From 3a54b413d506308c2627973c74a4ca27940918f1 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 28 Aug 2017 16:35:04 -0500 Subject: [PATCH 391/920] scoutfs: remove some #ifdef'd out definitions in dlmglue.h These make it hard to read the header and are very ocfs2-specific functions that would get moved when we merge this upstream anyway. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.h | 114 --------------------------------------------- 1 file changed, 114 deletions(-) diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 7e005da2..90e31298 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -275,52 +275,6 @@ struct ocfs2_lock_res_ops { */ #define LOCK_TYPE_USES_LVB 0x2 - -#if 0 -#include "dcache.h" - -#define OCFS2_LVB_VERSION 5 - -struct ocfs2_meta_lvb { - __u8 lvb_version; - __u8 lvb_reserved0; - __be16 lvb_idynfeatures; - __be32 lvb_iclusters; - __be32 lvb_iuid; - __be32 lvb_igid; - __be64 lvb_iatime_packed; - __be64 lvb_ictime_packed; - __be64 lvb_imtime_packed; - __be64 lvb_isize; - __be16 lvb_imode; - __be16 lvb_inlink; - __be32 lvb_iattr; - __be32 lvb_igeneration; - __be32 lvb_reserved2; -}; - -#define OCFS2_QINFO_LVB_VERSION 1 - -struct ocfs2_qinfo_lvb { - __u8 lvb_version; - __u8 lvb_reserved[3]; - __be32 lvb_bgrace; - __be32 lvb_igrace; - __be32 lvb_syncms; - __be32 lvb_blocks; - __be32 lvb_free_blk; - __be32 lvb_free_entry; -}; - -#define OCFS2_ORPHAN_LVB_VERSION 1 - -struct ocfs2_orphan_scan_lvb { - __u8 lvb_version; - __u8 lvb_reserved[3]; - __be32 lvb_os_seqno; -}; -#endif - struct ocfs2_lock_holder { struct list_head oh_list; struct pid *oh_owner_pid; @@ -361,75 +315,7 @@ void ocfs2_lock_res_init_common(struct ocfs2_super *osb, struct ocfs2_lock_res *res, struct ocfs2_lock_res_ops *ops, void *priv); -#if 0 -void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, - enum ocfs2_lock_type type, - unsigned int generation, - struct inode *inode); -void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, - u64 parent, struct inode *inode); -struct ocfs2_file_private; -void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_file_private *fp); -struct ocfs2_mem_dqinfo; -void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_mem_dqinfo *info); -void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_super *osb, u64 ref_blkno, - unsigned int generation); -#endif void ocfs2_lock_res_free(struct ocfs2_lock_res *res); -#if 0 -int ocfs2_create_new_inode_locks(struct inode *inode); -int ocfs2_drop_inode_locks(struct inode *inode); -int ocfs2_rw_lock(struct inode *inode, int write); -void ocfs2_rw_unlock(struct inode *inode, int write); -int ocfs2_open_lock(struct inode *inode); -int ocfs2_try_open_lock(struct inode *inode, int write); -void ocfs2_open_unlock(struct inode *inode); -int ocfs2_inode_lock_atime(struct inode *inode, - struct vfsmount *vfsmnt, - int *level); -int ocfs2_inode_lock_full_nested(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - int arg_flags, - int subclass); -int ocfs2_inode_lock_with_page(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - struct page *page); -/* Variants without special locking class or flags */ -#define ocfs2_inode_lock_full(i, r, e, f)\ - ocfs2_inode_lock_full_nested(i, r, e, f, OI_LS_NORMAL) -#define ocfs2_inode_lock_nested(i, b, e, s)\ - ocfs2_inode_lock_full_nested(i, b, e, 0, s) -/* 99% of the time we don't want to supply any additional flags -- - * those are for very specific cases only. */ -#define ocfs2_inode_lock(i, b, e) ocfs2_inode_lock_full_nested(i, b, e, 0, OI_LS_NORMAL) -void ocfs2_inode_unlock(struct inode *inode, - int ex); -int ocfs2_super_lock(struct ocfs2_super *osb, - int ex); -void ocfs2_super_unlock(struct ocfs2_super *osb, - int ex); -int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno); -void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno); - -int ocfs2_rename_lock(struct ocfs2_super *osb); -void ocfs2_rename_unlock(struct ocfs2_super *osb); -int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex); -void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex); -int ocfs2_dentry_lock(struct dentry *dentry, int ex); -void ocfs2_dentry_unlock(struct dentry *dentry, int ex); -int ocfs2_file_lock(struct file *file, int ex, int trylock); -void ocfs2_file_unlock(struct file *file); -int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex); -void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex); -struct ocfs2_refcount_tree; -int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex); -void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex); -#endif void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); From 0c1a81621bc8111e28af52becfb84184d32ed145 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 28 Aug 2017 16:44:22 -0500 Subject: [PATCH 392/920] scoutfs: #if 0 out lockdep code in dlmglue This portion of the port needs a bit of work before we can use it in scoutfs. In the meantime, disable it so that we can build on debug kernels. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 59615a31..47aca6b0 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -118,7 +118,7 @@ struct ocfs2_unblock_ctl { enum ocfs2_unblock_action unblock_action; }; -#ifdef CONFIG_DEBUG_LOCK_ALLOC +#if 0 && CONFIG_DEBUG_LOCK_ALLOC /* Lockdep class keys */ struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; #endif @@ -550,7 +550,7 @@ void ocfs2_lock_res_init_common(struct ocfs2_super *osb, ocfs2_add_lockres_tracking(res, osb->osb_dlm_debug); ocfs2_init_lock_stats(res); -#ifdef CONFIG_DEBUG_LOCK_ALLOC +#if 0 && CONFIG_DEBUG_LOCK_ALLOC if (type != OCFS2_LOCK_TYPE_OPEN) lockdep_init_map(&res->l_lockdep_map, ocfs2_lock_type_strings[type], &lockdep_keys[type], 0); @@ -1654,7 +1654,7 @@ out: } ocfs2_update_lock_stats(lockres, level, &mw, ret); -#ifdef CONFIG_DEBUG_LOCK_ALLOC +#if 0 && CONFIG_DEBUG_LOCK_ALLOC if (!ret && lockres->l_lockdep_map.key != NULL) { if (level == DLM_LOCK_PR) rwsem_acquire_read(&lockres->l_lockdep_map, l_subclass, @@ -1691,7 +1691,7 @@ static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, ocfs2_dec_holders(lockres, level); ocfs2_downconvert_on_unlock(osb, lockres); spin_unlock_irqrestore(&lockres->l_lock, flags); -#ifdef CONFIG_DEBUG_LOCK_ALLOC +#if 0 && CONFIG_DEBUG_LOCK_ALLOC if (lockres->l_lockdep_map.key != NULL) rwsem_release(&lockres->l_lockdep_map, 1, caller_ip); #endif From e2befc8736e115d9148a3330477a4b83a2416cbd Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 29 Aug 2017 18:45:02 -0500 Subject: [PATCH 393/920] scoutfs: silence dlmglue mlog() These debug prints are spamming the console, send them to the trace buffer instead. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 47aca6b0..63aae298 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -38,7 +38,7 @@ #include "dlmglue.h" -#define mlog(mask, fmt, args...) printk(KERN_INFO fmt , ##args) +#define mlog(mask, fmt, args...) trace_printk(fmt , ##args) #define mlog_errno(st) do { \ int _st = (st); \ if (_st != -ERESTARTSYS && _st != -EINTR && \ From 51e03dcb7ae400985a1082bae7d73ca88c477fe5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 23 Aug 2017 12:31:59 -0700 Subject: [PATCH 394/920] scoutfs: refactor inode locking function This is based on Mark Fasheh 's series that introduced inode refreshing after locking and a trylock for readpage. Rework the inode locking function so that it's more clearly named and takes flags and the inode struct. We have callers that want to lock the logical inode but aren't doing anything with the vfs inode so we provide that specific entry point. Signed-off-by: Zach Brown --- kmod/src/data.c | 3 +-- kmod/src/dir.c | 33 +++++++++++---------------------- kmod/src/inode.c | 5 ++--- kmod/src/lock.c | 11 +++++++++-- kmod/src/lock.h | 6 ++++-- kmod/src/server.c | 2 +- kmod/src/xattr.c | 8 ++++---- 7 files changed, 32 insertions(+), 36 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 3ecb3341..9e136c08 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1187,8 +1187,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 8581a8a5..5ac19786 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -259,8 +259,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, goto out; } - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(dir), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, dir, &dir_lock); if (ret) goto out; @@ -350,8 +349,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (!dir_emit_dots(file, dirent, filldir)) return 0; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &dir_lock); if (ret) return ret; @@ -502,8 +500,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) return ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); if (ret) return ret; @@ -519,8 +516,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, } /* Now that we have ino from scoutfs_new_inode, allocate a lock */ - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); if (ret) goto out; @@ -578,13 +574,11 @@ static int scoutfs_link(struct dentry *old_dentry, if (inode->i_nlink >= SCOUTFS_LINK_MAX) return -EMLINK; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); if (ret) return ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); if (ret) goto out_unlock; @@ -640,13 +634,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) if (S_ISDIR(inode->i_mode) && i_size_read(inode)) return -ENOTEMPTY; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); if (ret) return ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); if (ret) goto out; @@ -812,8 +804,7 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (size > PATH_MAX) return ERR_PTR(-ENAMETOOLONG); - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) return ERR_PTR(ret); @@ -881,8 +872,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(dir), - &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); if (ret) return ret; @@ -897,8 +887,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, goto out; } - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), - &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); if (ret) goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 1b9761b6..df96b762 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -275,8 +275,7 @@ static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct scoutfs_lock *lock = NULL; int ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), - &lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lock); if (ret) return ret; @@ -393,7 +392,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) struct scoutfs_lock *lock = NULL; int ret; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, ino, &lock); + ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); if (ret) return ERR_PTR(ret); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 360fe216..cd1e1aef 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -23,6 +23,7 @@ #include "msg.h" #include "cmp.h" #include "dlmglue.h" +#include "inode.h" #define LN_FMT "%u.%u.%llu.%llu" #define LN_ARG(name) \ @@ -478,8 +479,8 @@ out: #endif } -int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, - struct scoutfs_lock **ret_lock) +int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, + struct scoutfs_lock **ret_lock) { struct scoutfs_lock_name lock_name; struct scoutfs_inode_key start_ikey; @@ -508,6 +509,12 @@ int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, &end, ret_lock); } +int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, + struct inode *inode, struct scoutfs_lock **ret_lock) +{ + return scoutfs_lock_ino(sb, mode, flags, scoutfs_ino(inode), ret_lock); +} + /* * map inode index items to locks. The idea is to not have to * constantly get locks over a reasonable distribution of items, but diff --git a/kmod/src/lock.h b/kmod/src/lock.h index d47f9a7c..20dcaa46 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -26,8 +26,10 @@ struct scoutfs_lock { struct ocfs2_lock_res lockres; }; -int scoutfs_lock_ino_group(struct super_block *sb, int mode, u64 ino, - struct scoutfs_lock **ret_lock); +int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, + struct inode *inode, struct scoutfs_lock **ret_lock); +int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, + struct scoutfs_lock **ret_lock); int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); diff --git a/kmod/src/server.c b/kmod/src/server.c index 427a6463..aa2d7588 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -888,7 +888,7 @@ static void scoutfs_server_func(struct work_struct *work) init_waitqueue_head(&waitq); /* lock attempt will return -ESHUTDOWN once we should not queue */ - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ~0ULL, &lock); + ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ~0ULL, &lock); if (ret) goto out; diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index ad24cda9..6174a011 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -178,7 +178,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, goto out; } - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), &lck); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); if (ret) goto out; @@ -289,7 +289,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, scoutfs_ino(inode), &lck); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &lck); if (ret) goto out; @@ -386,7 +386,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) xkey = key->data; xkey->name[0] = '\0'; - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_PR, scoutfs_ino(inode), &lck); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); if (ret) goto out; @@ -469,7 +469,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) } /* while we read to delete we need to writeback others */ - ret = scoutfs_lock_ino_group(sb, DLM_LOCK_EX, ino, &lck); + ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lck); if (ret) goto out; From d2a1b915fca2faf6f7c91137c59b1a1a29b237b6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Aug 2017 15:10:51 -0700 Subject: [PATCH 395/920] scoutfs: publish refresh_gen from dlmglue In addition to setting NEEDS_REFRESH when locks are acquired out of NL, we now also give them a refresh_gen counter that is increased by incrementing a long lived counter in the super. This gives callers a strictly increasing read-only indication that the lock has changed. They don't have to serialize users to clear NEEDS_REFRESH and transfer it to some other serialized state. scoutfs will use with the multiple inodes that are refreshed with respect to the lock's refresh_gen. Signed-off-by: Zach Brown --- kmod/src/dlmglue.c | 31 +++++++++++++++++++++++++++++-- kmod/src/dlmglue.h | 6 ++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 63aae298..ea819588 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -909,6 +909,23 @@ static void lockres_clear_flags(struct ocfs2_lock_res *lockres, lockres_set_flags(lockres, lockres->l_flags & ~clear); } +/* + * Make sure that a lock gets a strictly increasing number only once + * each time it needs to be refreshed. The gen needs to be larger than + * any previous gen the locked resources has seen so we maintain the gen + * in the super. The caller has serialized on the lock but lots of + * locks can all be racing on the super. + * + * This is used by callers to have a single read-only indicator that + * they need to refresh their resource while they have it locked. + */ +static void lockres_inc_refresh_gen(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + + lockres->l_refresh_gen = atomic64_inc_return(&osb->refresh_gen); +} + static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres) { BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); @@ -935,8 +952,10 @@ static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lo * *anything* however should mark ourselves as needing an * update */ if (lockres->l_level == DLM_LOCK_NL && - lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + lockres_inc_refresh_gen(lockres); + } lockres->l_level = lockres->l_requested; @@ -962,8 +981,10 @@ static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *loc if (lockres->l_requested > DLM_LOCK_NL && !(lockres->l_flags & OCFS2_LOCK_LOCAL) && - lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + lockres_inc_refresh_gen(lockres); + } lockres->l_level = lockres->l_requested; lockres_or_flags(lockres, OCFS2_LOCK_ATTACHED); @@ -2294,6 +2315,11 @@ static inline void ocfs2_complete_lock_res_refresh(struct ocfs2_lock_res *lockre wake_up(&lockres->l_event); } +u64 ocfs2_lock_refresh_gen(struct ocfs2_lock_res *lockres) +{ + return lockres->l_refresh_gen; +} + #if 0 /* may or may not return a bh if it went to disk. */ static int ocfs2_inode_lock_update(struct inode *inode, @@ -4291,6 +4317,7 @@ int ocfs2_init_super(struct ocfs2_super *osb, int flags) init_waitqueue_head(&osb->dc_event); INIT_LIST_HEAD(&osb->blocked_lock_list); osb->s_mount_opt = flags; + atomic64_set(&osb->refresh_gen, 0); return 0; } diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 90e31298..61b4af76 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -103,6 +103,7 @@ struct ocfs2_lock_res { struct list_head l_mask_waiters; struct list_head l_holders; + u64 l_refresh_gen; unsigned long l_flags; char l_name[OCFS2_LOCK_ID_MAX_LEN]; unsigned int l_ro_holders; @@ -167,6 +168,9 @@ struct ocfs2_super struct list_head blocked_lock_list; unsigned long blocked_lock_count; + /* refresh_gen needs to strictly increase as locks come and go */ + atomic64_t refresh_gen; + unsigned long s_mount_opt; }; /* For s_mount_opt */ @@ -317,6 +321,8 @@ void ocfs2_lock_res_init_common(struct ocfs2_super *osb, void *priv); void ocfs2_lock_res_free(struct ocfs2_lock_res *res); +u64 ocfs2_lock_refresh_gen(struct ocfs2_lock_res *lockres); + void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, From fdbe0de8e996e1de355ca67e5cb45aa5cde5b988 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 23 Aug 2017 15:49:25 -0700 Subject: [PATCH 396/920] scoutfs: add flag to refresh inode after locking Lock callers can specify that they want inode fields reread from items after the lock is acquired. dlmglue sets a refresh_gen in the locks that we store in inodes to track when they were last refreshed and if they need a refresh. Signed-off-by: Zach Brown --- kmod/src/dir.c | 4 ++-- kmod/src/inode.c | 50 +++++++++++++++++++++++++++++++++++++++++------- kmod/src/inode.h | 11 ++++++++++- kmod/src/lock.c | 32 +++++++++++++++++++++++++++++-- kmod/src/lock.h | 3 +++ 5 files changed, 88 insertions(+), 12 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 5ac19786..1262e63a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -509,7 +509,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out_unlock; - inode = scoutfs_new_inode(sb, dir, mode, rdev); + inode = scoutfs_new_inode(sb, dir, mode, rdev, dir_lock); if (IS_ERR(inode)) { ret = PTR_ERR(inode); goto out; @@ -881,7 +881,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) goto out_unlock; - inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0); + inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0, dir_lock); if (IS_ERR(inode)) { ret = PTR_ERR(inode); goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index df96b762..d3106a90 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -235,7 +235,19 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) set_item_info(ci, cinode); } -static int refresh_inode(struct inode *inode, struct scoutfs_lock *lock) +/* + * Refresh the vfs inode fields if the lock indicates that the current + * contents could be stale. + * + * This can be racing with many lock holders of an inode. A bunch of + * readers can be checking to refresh while one of them is refreshing. + * + * The vfs inode field updates can't be racing with valid readers of the + * fields because they should have already had a locked refreshed inode + * to be dereferencing its contents. + */ +int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, + int flags) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; @@ -243,15 +255,34 @@ static int refresh_inode(struct inode *inode, struct scoutfs_lock *lock) struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; SCOUTFS_DECLARE_KVEC(val); + const u64 refresh_gen = scoutfs_lock_refresh_gen(lock); int ret; + /* + * Lock refresh gens are supposed to strictly increase. Inodes + * having a greater gen means memory corruption or + * lifetime/logic bugs that could stop the inode from refreshing + * and expose stale data. + */ + BUG_ON(atomic64_read(&si->last_refreshed) > refresh_gen); + + if (atomic64_read(&si->last_refreshed) == refresh_gen) + return 0; + scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); mutex_lock(&si->item_mutex); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), lock->end); - if (ret == 0) - load_inode(inode, &sinode); + if (atomic64_read(&si->last_refreshed) < refresh_gen) { + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), + lock->end); + if (ret == 0) { + load_inode(inode, &sinode); + atomic64_set(&si->last_refreshed, refresh_gen); + } + } else { + ret = 0; + } mutex_unlock(&si->item_mutex); return ret; @@ -279,7 +310,7 @@ static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, if (ret) return ret; - ret = refresh_inode(inode, lock); + ret = scoutfs_inode_refresh(inode, lock, 0); if (ret == 0) generic_fillattr(inode, stat); @@ -404,7 +435,10 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) } if (inode->i_state & I_NEW) { - ret = refresh_inode(inode, lock); + /* XXX ensure refresh, instead clear in drop_inode? */ + atomic64_set(&SCOUTFS_I(inode)->last_refreshed, 0); + + ret = scoutfs_inode_refresh(inode, lock, 0); if (ret) { iget_failed(inode); inode = ERR_PTR(ret); @@ -771,7 +805,8 @@ out: * creating links to it and updating it. @dir can be null. */ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev) + umode_t mode, dev_t rdev, + struct scoutfs_lock *lock) { struct scoutfs_inode_info *ci; struct scoutfs_inode_key ikey; @@ -797,6 +832,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; + atomic64_set(&ci->last_refreshed, scoutfs_lock_refresh_gen(lock)); inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 182779ad..d1fb1b0a 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -3,6 +3,8 @@ #include "key.h" +struct scoutfs_lock; + struct scoutfs_inode_info { /* read or initialized for each inode instance */ u64 ino; @@ -25,6 +27,9 @@ struct scoutfs_inode_info { u64 item_meta_seq; u64 item_data_seq; + /* updated at on each new lock acquisition */ + atomic64_t last_refreshed; + /* initialized once for slab object */ seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ @@ -58,7 +63,8 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end); void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev); + umode_t mode, dev_t rdev, + struct scoutfs_lock *lock); void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); @@ -66,6 +72,9 @@ u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); u64 scoutfs_inode_data_version(struct inode *inode); +int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, + int flags); + int scoutfs_scan_orphans(struct super_block *sb); void scoutfs_inode_queue_writeback(struct inode *inode); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index cd1e1aef..6b0dd76c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -479,6 +479,11 @@ out: #endif } +u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock) +{ + return ocfs2_lock_refresh_gen(&lock->lockres); +} + int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock) { @@ -509,10 +514,33 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, &end, ret_lock); } +/* + * Acquire a lock on an inode. + * + * _REFRESH_INODE indicates that the caller needs to have the vfs inode + * fields current with respect to lock coverage. dlmglue increases the + * lock's refresh_gen once every time its mode is changed from a mode + * that couldn't have the inode cached to one that could. + */ int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, - struct inode *inode, struct scoutfs_lock **ret_lock) + struct inode *inode, struct scoutfs_lock **lock) { - return scoutfs_lock_ino(sb, mode, flags, scoutfs_ino(inode), ret_lock); + int ret; + + ret = scoutfs_lock_ino(sb, mode, flags, scoutfs_ino(inode), lock); + if (ret < 0) + goto out; + + if (flags & SCOUTFS_LKF_REFRESH_INODE) { + ret = scoutfs_inode_refresh(inode, *lock, flags); + if (ret < 0) { + scoutfs_unlock(sb, *lock, mode); + *lock = NULL; + } + } + +out: + return ret; } /* diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 20dcaa46..aa3a9ff0 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -8,6 +8,8 @@ #define SCOUTFS_LOCK_BLOCKING 0x01 /* Blocking another lock request */ #define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ +#define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ + struct scoutfs_lock { struct super_block *sb; struct scoutfs_lock_name lock_name; @@ -26,6 +28,7 @@ struct scoutfs_lock { struct ocfs2_lock_res lockres; }; +u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock); int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, From a08530a24e188ad05024916a92aa46d1f3a6a10f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 23 Aug 2017 15:55:12 -0700 Subject: [PATCH 397/920] scoutfs: add LKF_TRYLOCK Add a flag that tells locking to return -EAGAIN if it hits contention. Signed-off-by: Zach Brown --- kmod/src/lock.c | 17 +++++++++++------ kmod/src/lock.h | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 6b0dd76c..4dc1b254 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -401,7 +401,7 @@ static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock) * The caller provides the opaque lock structure used for storage and * their start and end pointers will be accessed while the lock is held. */ -static int lock_name_keys(struct super_block *sb, int mode, +static int lock_name_keys(struct super_block *sb, int mode, int flags, struct scoutfs_lock_name *lock_name, struct ocfs2_lock_res_ops *type, struct scoutfs_key_buf *start, @@ -410,6 +410,7 @@ static int lock_name_keys(struct super_block *sb, int mode, { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; + int lkm_flags; int ret; lock = find_alloc_scoutfs_lock(sb, lock_name, type, start, end); @@ -418,8 +419,12 @@ static int lock_name_keys(struct super_block *sb, int mode, trace_scoutfs_lock_resource(sb, lock); + lkm_flags = DLM_LKF_NOORDER; + if (flags & SCOUTFS_LKF_TRYLOCK) + lkm_flags |= DLM_LKF_NOQUEUE; /* maybe also NONBLOCK? */ + ret = ocfs2_cluster_lock(&linfo->dlmglue, &lock->lockres, mode, - DLM_LKF_NOORDER, 0); + lkm_flags, 0); if (ret) return ret; @@ -510,8 +515,8 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, end_ikey.type = ~0; scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, &lock_name, &scoufs_ino_lops, &start, - &end, ret_lock); + return lock_name_keys(sb, mode, flags, &lock_name, &scoufs_ino_lops, + &start, &end, ret_lock); } /* @@ -604,8 +609,8 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, end_ikey.ino = cpu_to_be64(ino | ino_mask); scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, &lock_name, &scoufs_ino_index_lops, - &start, &end, ret_lock); + return lock_name_keys(sb, mode, 0, &lock_name, + &scoufs_ino_index_lops, &start, &end, ret_lock); } void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, diff --git a/kmod/src/lock.h b/kmod/src/lock.h index aa3a9ff0..28be1794 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -9,6 +9,7 @@ #define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ +#define SCOUTFS_LKF_TRYLOCK 0x02 /* EAGAIN if contention */ struct scoutfs_lock { struct super_block *sb; From ceccc56c8fd4cb3bc24bf202676e379b20a3655f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 23 Aug 2017 16:07:16 -0700 Subject: [PATCH 398/920] scoutfs: add inode locking flags to callers Now that we have the inode refreshing flags let's add them to the callers that want to have a current inode after they have their lock. Callers locking newly created items use the new inode flag to reset the refresh gen. A few inode tests are moved down to after locking so that it can test the current refreshed inode. Signed-off-by: Zach Brown --- kmod/src/dir.c | 65 +++++++++++++++++++++++++++++++----------------- kmod/src/inode.c | 13 ++++------ 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 1262e63a..a9481ad7 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -500,7 +500,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock); if (ret) return ret; @@ -571,17 +572,22 @@ static int scoutfs_link(struct dentry *old_dentry, DECLARE_ITEM_COUNT(cnt); int ret; - if (inode->i_nlink >= SCOUTFS_LINK_MAX) - return -EMLINK; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock); if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); if (ret) goto out_unlock; + if (inode->i_nlink >= SCOUTFS_LINK_MAX) { + ret = -EMLINK; + goto out_unlock; + } + ret = alloc_dentry_info(dentry); if (ret) goto out_unlock; @@ -631,17 +637,22 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct scoutfs_lock *inode_lock = NULL; int ret = 0; - if (S_ISDIR(inode->i_mode) && i_size_read(inode)) - return -ENOTEMPTY; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock); if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); if (ret) goto out; + if (S_ISDIR(inode->i_mode) && i_size_read(inode)) { + ret = -ENOTEMPTY; + goto out; + } + keys[0] = alloc_dirent_key(sb, dir, dentry); if (!keys[0]) { ret = -ENOMEM; @@ -792,25 +803,32 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; - loff_t size = i_size_read(inode); - char *path; + char *path = NULL; + loff_t size; int ret; - /* XXX corruption */ - if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) - return ERR_PTR(-EIO); - - /* unlikely, but possible I suppose */ - if (size > PATH_MAX) - return ERR_PTR(-ENAMETOOLONG); - - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); if (ret) return ERR_PTR(ret); + size = i_size_read(inode); + + /* XXX corruption */ + if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) { + ret = -EIO; + goto out; + } + + /* unlikely, but possible I suppose */ + if (size > PATH_MAX) { + ret = -ENAMETOOLONG; + goto out; + } + path = kmalloc(size, GFP_NOFS); if (!path) { - path = ERR_PTR(-ENOMEM); + ret = -ENOMEM; goto out; } @@ -821,13 +839,13 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) if (ret == -ENOENT || (ret == 0 && path[size - 1])) ret = -EIO; +out: if (ret < 0) { kfree(path); path = ERR_PTR(ret); } else { nd_set_link(nd, path); } -out: scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); return path; } @@ -872,7 +890,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, dir, &dir_lock); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock); if (ret) return ret; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index d3106a90..4209af81 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -306,15 +306,12 @@ static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct scoutfs_lock *lock = NULL; int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lock); - if (ret) - return ret; - - ret = scoutfs_inode_refresh(inode, lock, 0); - if (ret == 0) + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, + inode, &lock); + if (ret == 0) { generic_fillattr(inode, stat); - - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); + } return ret; } From c0d3f99a6ee79a5c902ace0b3fc3747a58f4abef Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 21 Jul 2017 16:55:01 -0500 Subject: [PATCH 399/920] scoutfs: Cluster coherent read/write With trylock implemented we can add locking in readpage. After that it's pretty easy to implement our own read/write functions which at this point are more or less wrapping the kernel helpers in the correct cluster locking. Data invalidation is a bit interesting. If the lock we are invalidating is an inode group lock, we use the lock boundaries to incrementally search our inode cache. When an inode struct is found, we sync and (optionally) truncate pages. Signed-off-by: Mark Fasheh [zab: adapted to newer lock call, fixed some error handling] Signed-off-by: Zach Brown --- kmod/src/Makefile | 6 ++-- kmod/src/data.c | 43 +++++++++++++++++++++--- kmod/src/file.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/file.h | 9 +++++ kmod/src/inode.c | 5 +++ kmod/src/inode.h | 2 ++ kmod/src/lock.c | 29 +++++++++++++--- 7 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 kmod/src/file.c create mode 100644 kmod/src/file.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f1b94903..40543c43 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -5,9 +5,9 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - dlmglue.o kvec.o inode.o ioctl.o item.o key.o lock.o manifest.o \ - msg.o options.o seg.o server.o scoutfs_trace.o sock.o sort_priv.o \ - stackglue.o super.o trans.o xattr.o + dlmglue.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ + manifest.o msg.o options.o seg.o server.o scoutfs_trace.o sock.o \ + sort_priv.o stackglue.o super.o trans.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/data.c b/kmod/src/data.c index 9e136c08..3f557f9e 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -31,6 +31,7 @@ #include "ioctl.h" #include "client.h" #include "lock.h" +#include "file.h" #define EXTF "[off %llu bno %llu bks %llu fl %x]" #define EXTA(ne) (ne)->blk_off, (ne)->blkno, (ne)->blocks, (ne)->flags @@ -1070,13 +1071,47 @@ out: static int scoutfs_readpage(struct file *file, struct page *page) { - return mpage_readpage(page, scoutfs_get_block); + struct inode *inode = file->f_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int unlock = 1; + int ret; + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE | + SCOUTFS_LKF_TRYLOCK, inode, &inode_lock); + if (ret) { + if (ret == -EAGAIN) + ret = AOP_TRUNCATED_PAGE; + goto out; + } + + ret = mpage_readpage(page, scoutfs_get_block); + unlock = 0; + + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); +out: + if (unlock) + unlock_page(page); + return ret; } static int scoutfs_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { - return mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); + struct inode *inode = file->f_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int ret; + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); + if (ret) + return ret; + + ret = mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); + + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + return ret; } static int scoutfs_writepage(struct page *page, struct writeback_control *wbc) @@ -1256,8 +1291,8 @@ const struct address_space_operations scoutfs_file_aops = { const struct file_operations scoutfs_file_fops = { .read = do_sync_read, .write = do_sync_write, - .aio_read = generic_file_aio_read, - .aio_write = generic_file_aio_write, + .aio_read = scoutfs_file_aio_read, + .aio_write = scoutfs_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, .llseek = generic_file_llseek, diff --git a/kmod/src/file.c b/kmod/src/file.c new file mode 100644 index 00000000..f00fb688 --- /dev/null +++ b/kmod/src/file.c @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "data.h" +#include "scoutfs_trace.h" +#include "item.h" +#include "lock.h" +#include "file.h" + +/* TODO: Direct I/O, AIO */ +ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int ret; + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); + if (ret == 0) { + ret = generic_file_aio_read(iocb, iov, nr_segs, pos); + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + } + + return ret; +} + +ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int ret; + + if (iocb->ki_left == 0) /* Does this even happen? */ + return 0; + + mutex_lock(&inode->i_mutex); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); + if (ret) + goto out; + + /* XXX: remove SUID bit */ + + ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); + + scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); +out: + mutex_unlock(&inode->i_mutex); + + if (ret > 0 || ret == -EIOCBQUEUED) { + ssize_t err; + + err = generic_write_sync(file, pos, ret); + if (err < 0 && ret > 0) + ret = err; + } + + return ret; +} diff --git a/kmod/src/file.h b/kmod/src/file.h new file mode 100644 index 00000000..a55a3bbb --- /dev/null +++ b/kmod/src/file.h @@ -0,0 +1,9 @@ +#ifndef _SCOUTFS_FILE_H_ +#define _SCOUTFS_FILE_H_ + +ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos); +ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos); + +#endif /* _SCOUTFS_FILE_H_ */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 4209af81..d3fe4b30 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -414,6 +414,11 @@ static int scoutfs_iget_set(struct inode *inode, void *arg) return 0; } +struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino) +{ + return ilookup5(sb, ino, scoutfs_iget_test, &ino); +} + struct inode *scoutfs_iget(struct super_block *sb, u64 ino) { struct inode *inode; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index d1fb1b0a..452ed00e 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -2,6 +2,7 @@ #define _SCOUTFS_INODE_H_ #include "key.h" +#include "lock.h" struct scoutfs_lock; @@ -59,6 +60,7 @@ void scoutfs_evict_inode(struct inode *inode); int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); +struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end); void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 4dc1b254..18e68dd8 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "super.h" #include "lock.h" @@ -65,9 +66,12 @@ static void scoutfs_downconvert_func(struct work_struct *work); * to also invalidate all cached overlapping structures. */ static int invalidate_caches(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { + struct scoutfs_key_buf *start = lock->start; + struct scoutfs_key_buf *end = lock->end; + struct inode *inode; + u64 ino, last; int ret; trace_scoutfs_lock_invalidate_sb(sb, mode, start, end); @@ -76,8 +80,23 @@ static int invalidate_caches(struct super_block *sb, int mode, if (ret) return ret; - if (mode == DLM_LOCK_EX) + if (mode == DLM_LOCK_EX) { + if (lock->lock_name.zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->lock_name.first); + last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; + while (ino <= last) { + inode = scoutfs_ilookup(lock->sb, ino); + if (inode && S_ISREG(inode->i_mode)) + truncate_inode_pages(inode->i_mapping, + 0); + + iput(inode); + ino++; + } + } + ret = scoutfs_item_invalidate(sb, start, end); + } return ret; } @@ -133,7 +152,7 @@ static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) struct scoutfs_lock *lock = lockres->l_priv; struct super_block *sb = lock->sb; - invalidate_caches(sb, blocking, lock->start, lock->end); + invalidate_caches(sb, blocking, lock); return UNBLOCK_CONTINUE; } @@ -685,7 +704,7 @@ static void scoutfs_downconvert_func(struct work_struct *work) * invalidate based on what level we're downconverting to (PR, * NL). */ - invalidate_caches(sb, DLM_LOCK_EX, lock->start, lock->end); + invalidate_caches(sb, DLM_LOCK_EX, lock); unlock_range(sb, lock); spin_lock(&linfo->lock); From 1bcad2e9cc326bfca21f667696c8ddd5c7b85e35 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 25 Jul 2017 18:58:20 -0500 Subject: [PATCH 400/920] scoutfs: provide ->permission We need to lock and refresh the VFS inode before it checks permissions in system calls, otherwise we risk checking against stale inode metadata. Signed-off-by: Mark Fasheh [zab: adapted to newer lock call] Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 ++ kmod/src/file.c | 21 +++++++++++++++++++++ kmod/src/file.h | 1 + 3 files changed, 24 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index a9481ad7..f8ae31f6 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -19,6 +19,7 @@ #include #include "format.h" +#include "file.h" #include "dir.h" #include "inode.h" #include "ioctl.h" @@ -1131,6 +1132,7 @@ const struct inode_operations scoutfs_dir_iops = { .listxattr = scoutfs_listxattr, .removexattr = scoutfs_removexattr, .symlink = scoutfs_symlink, + .permission = scoutfs_permission, }; void scoutfs_dir_exit(void) diff --git a/kmod/src/file.c b/kmod/src/file.c index f00fb688..5d8e675e 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -83,3 +83,24 @@ out: return ret; } + +int scoutfs_permission(struct inode *inode, int mask) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *inode_lock = NULL; + int ret; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, + inode, &inode_lock); + if (ret) + return ret; + + ret = generic_permission(inode, mask); + + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + + return ret; +} diff --git a/kmod/src/file.h b/kmod/src/file.h index a55a3bbb..8df0f330 100644 --- a/kmod/src/file.h +++ b/kmod/src/file.h @@ -5,5 +5,6 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos); ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos); +int scoutfs_permission(struct inode *inode, int mask); #endif /* _SCOUTFS_FILE_H_ */ From c4e7b5a6e900ea5508d669a2570612c20baf724a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 4 Aug 2017 16:42:46 -0500 Subject: [PATCH 401/920] scoutfs: provide cluster safe ->llseek Without this we return -ESPIPE when a process tries to seek on a regular file. Signed-off-by: Mark Fasheh [zab: adapted to new lock call] Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- kmod/src/file.c | 38 ++++++++++++++++++++++++++++++++++++++ kmod/src/file.h | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 3f557f9e..b6f2d641 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1295,7 +1295,7 @@ const struct file_operations scoutfs_file_fops = { .aio_write = scoutfs_file_aio_write, .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, - .llseek = generic_file_llseek, + .llseek = scoutfs_file_llseek, }; diff --git a/kmod/src/file.c b/kmod/src/file.c index 5d8e675e..2878d61e 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -104,3 +104,41 @@ int scoutfs_permission(struct inode *inode, int mask) return ret; } + +loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + int ret = 0; + + switch (whence) { + case SEEK_END: + case SEEK_DATA: + case SEEK_HOLE: + /* + * These require a lock and inode refresh as they + * reference i_size. + * + * XXX: SEEK_DATA/SEEK_HOLE can search our extent + * items instead of relying on generic_file_llseek() + * trickery. + */ + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, + SCOUTFS_LKF_REFRESH_INODE, inode, + &lock); + case SEEK_SET: + case SEEK_CUR: + /* No lock required, fall through to the generic helper */ + break; + default: + ret = -EINVAL; + } + + if (ret == 0) + offset = generic_file_llseek(file, offset, whence); + + scoutfs_unlock(sb, lock, DLM_LOCK_PR); + + return ret ? ret : offset; +} diff --git a/kmod/src/file.h b/kmod/src/file.h index 8df0f330..82d86618 100644 --- a/kmod/src/file.h +++ b/kmod/src/file.h @@ -6,5 +6,6 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos); int scoutfs_permission(struct inode *inode, int mask); +loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence); #endif /* _SCOUTFS_FILE_H_ */ From e47d66ddd36a7fd72d923c9adb92407e147b9f1c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 31 Jul 2017 14:11:11 -0700 Subject: [PATCH 402/920] scoutfs: add scoutfs_lock_inodes() Add a function that can lock multiple inodes in order of their inode numbers. It handles nulls and duplicate inodes. Signed-off-by: Zach Brown --- kmod/src/lock.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 5 +++ 2 files changed, 93 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 18e68dd8..c5712bdd 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "super.h" #include "lock.h" @@ -567,6 +568,93 @@ out: return ret; } +struct lock_inodes_arg { + struct inode *inode; + struct scoutfs_lock **lockp; +}; + +/* + * All args with inodes go to the front of the array and are then sorted + * by their inode number. + */ +static int cmp_arg(const void *A, const void *B) +{ + const struct lock_inodes_arg *a = A; + const struct lock_inodes_arg *b = B; + + if (a->inode && b->inode) + return scoutfs_cmp_u64s(scoutfs_ino(a->inode), + scoutfs_ino(b->inode)); + + return a->inode ? -1 : b->inode ? 1 : 0; +} + +static void swap_arg(void *A, void *B, int size) +{ + struct lock_inodes_arg *a = A; + struct lock_inodes_arg *b = B; + + swap(*a, *b); +} + +/* + * Lock all the inodes in inode number order. The inode arguments can + * be in any order and can be duplicated or null. This relies on core + * lock matching to efficiently handle duplicate lock attempts of the + * same group. Callers can try to use the lock range keys for all the + * locks they attempt to acquire without knowing that they map to the + * same groups. + * + * On error no locks are held and all pointers are set to null. Lock + * pointers for null inodes are always set to null. + * + * (pretty great collision with d_lock() here) + */ +int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, + struct inode *a, struct scoutfs_lock **a_lock, + struct inode *b, struct scoutfs_lock **b_lock, + struct inode *c, struct scoutfs_lock **c_lock, + struct inode *d, struct scoutfs_lock **D_lock) +{ + struct lock_inodes_arg args[] = { + {a, a_lock}, {b, b_lock}, {c, c_lock}, {d, D_lock}, + }; + int ret; + int i; + + /* set all lock pointers to null and validating input */ + ret = 0; + for (i = 0; i < ARRAY_SIZE(args); i++) { + if (WARN_ON_ONCE(args[i].inode && !args[i].lockp)) + ret = -EINVAL; + if (args[i].lockp) + *args[i].lockp = NULL; + } + if (ret) + return ret; + + /* sort by having an inode then inode number */ + sort(args, ARRAY_SIZE(args), sizeof(args[0]), cmp_arg, swap_arg); + + /* lock unique inodes */ + for (i = 0; i < ARRAY_SIZE(args) && args[i].inode; i++) { + ret = scoutfs_lock_inode(sb, mode, flags, args[i].inode, + args[i].lockp); + if (ret) + break; + } + + /* unlock on error */ + for (i = ARRAY_SIZE(args) - 1; ret < 0 && i >= 0; i--) { + if (args[i].lockp && *args[i].lockp) { + scoutfs_unlock(sb, *args[i].lockp, mode); + *args[i].lockp = NULL; + } + } + + return ret; +} + /* * map inode index items to locks. The idea is to not have to * constantly get locks over a reasonable distribution of items, but diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 28be1794..5a2f926b 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -37,6 +37,11 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); +int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, + struct inode *a, struct scoutfs_lock **a_lock, + struct inode *b, struct scoutfs_lock **b_lock, + struct inode *c, struct scoutfs_lock **c_lock, + struct inode *d, struct scoutfs_lock **D_lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); From 3233ab47e8e188759e91da2496f5043f03bb0a69 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 31 Jul 2017 14:08:08 -0700 Subject: [PATCH 403/920] scoutfs: add global lock names Add a lock name that has a global scope in a given lockspace. It's not associated with any file system items. We add a scope to the lock name to indicate if a lock is global or not and set that in other lock naming intitialization. We permit lock allocation to accept null start and end keys. Signed-off-by: Zach Brown --- kmod/src/format.h | 6 ++++ kmod/src/lock.c | 70 ++++++++++++++++++++++++++++------------ kmod/src/lock.h | 2 ++ kmod/src/scoutfs_trace.h | 8 +++-- 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 5be9ed58..07e90ad2 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -507,7 +507,13 @@ enum { /* * structures used by dlm */ +#define SCOUTFS_LOCK_SCOPE_GLOBAL 1 +#define SCOUTFS_LOCK_SCOPE_FS_ITEMS 2 + +#define SCOUTFS_LOCK_TYPE_GLOBAL_RENAME 1 + struct scoutfs_lock_name { + __u8 scope; __u8 zone; __u8 type; __le64 first; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c5712bdd..b379d95b 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -27,9 +27,9 @@ #include "dlmglue.h" #include "inode.h" -#define LN_FMT "%u.%u.%llu.%llu" +#define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ - (name)->zone, (name)->type, le64_to_cpu((name)->first), \ + (name)->scope, (name)->zone, (name)->type, le64_to_cpu((name)->first),\ le64_to_cpu((name)->second) typedef struct ocfs2_super dlmglue_ctxt; @@ -174,6 +174,13 @@ static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .flags = 0, }; +static struct ocfs2_lock_res_ops scoutfs_global_lops = { + .get_osb = get_ino_lock_osb, + /* XXX: .post_unlock for lru */ + /* XXX: .check_downconvert that queries the item cache for dirty items */ + .flags = 0, +}; + static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_lock_name *lock_name, struct ocfs2_lock_res_ops *type, @@ -185,38 +192,43 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, // struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock; + if (WARN_ON_ONCE(!!start != !!end)) + return NULL; + lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); - if (lock) { + if (lock == NULL) + return NULL; + + if (start) { lock->start = scoutfs_key_dup(sb, start); lock->end = scoutfs_key_dup(sb, end); if (!lock->start || !lock->end) { free_scoutfs_lock(lock); - lock = NULL; - } else { - RB_CLEAR_NODE(&lock->node); - lock->sb = sb; - lock->lock_name = *lock_name; - lock->mode = DLM_LOCK_IV; - INIT_WORK(&lock->dc_work, scoutfs_downconvert_func); - INIT_LIST_HEAD(&lock->lru_entry); - ocfs2_lock_res_init_once(&lock->lockres); - BUG_ON(sizeof(struct scoutfs_lock_name) >= - OCFS2_LOCK_ID_MAX_LEN); - /* kzalloc above ensures that l_name is NULL terminated */ - memcpy(&lock->lockres.l_name[0], &lock->lock_name, - sizeof(struct scoutfs_lock_name)); - ocfs2_lock_res_init_common(&linfo->dlmglue, - &lock->lockres, type, lock); + return NULL; } } + RB_CLEAR_NODE(&lock->node); + lock->sb = sb; + lock->lock_name = *lock_name; + lock->mode = DLM_LOCK_IV; + INIT_WORK(&lock->dc_work, scoutfs_downconvert_func); + INIT_LIST_HEAD(&lock->lru_entry); + ocfs2_lock_res_init_once(&lock->lockres); + BUG_ON(sizeof(struct scoutfs_lock_name) >= OCFS2_LOCK_ID_MAX_LEN); + /* kzalloc above ensures that l_name is NULL terminated */ + memcpy(&lock->lockres.l_name[0], &lock->lock_name, + sizeof(struct scoutfs_lock_name)); + ocfs2_lock_res_init_common(&linfo->dlmglue, &lock->lockres, type, lock); + return lock; } static int cmp_lock_names(struct scoutfs_lock_name *a, struct scoutfs_lock_name *b) { - return (int)a->zone - (int)b->zone ?: + return (int)a->scope - (int)b->scope ?: + (int)a->zone - (int)b->zone ?: (int)a->type - (int)b->type ?: scoutfs_cmp_u64s(le64_to_cpu(a->first), le64_to_cpu(b->first)) ?: scoutfs_cmp_u64s(le64_to_cpu(b->second), le64_to_cpu(b->second)); @@ -520,6 +532,7 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, ino &= ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK; + lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; lock_name.zone = SCOUTFS_FS_ZONE; lock_name.type = SCOUTFS_INODE_TYPE; lock_name.first = cpu_to_le64(ino); @@ -655,6 +668,22 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, return ret; } +/* + * Acquire a cluster lock with a global scope in the lock space. + */ +int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, + struct scoutfs_lock **lock) +{ + struct scoutfs_lock_name lock_name; + + memset(&lock_name, 0, sizeof(lock_name)); + lock_name.scope = SCOUTFS_LOCK_SCOPE_GLOBAL; + lock_name.type = type; + + return lock_name_keys(sb, mode, flags, &lock_name, &scoutfs_global_lops, + NULL, NULL, lock); +} + /* * map inode index items to locks. The idea is to not have to * constantly get locks over a reasonable distribution of items, but @@ -697,6 +726,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, BUG(); } + lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; lock_name.zone = SCOUTFS_INODE_INDEX_ZONE; lock_name.type = type; lock_name.first = cpu_to_le64(major & ~major_mask); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 5a2f926b..68ba7329 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -42,6 +42,8 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *b, struct scoutfs_lock **b_lock, struct inode *c, struct scoutfs_lock **c_lock, struct inode *d, struct scoutfs_lock **D_lock); +int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, + struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index bd5b3855..a0c75dfc 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -299,6 +299,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck), TP_STRUCT__entry( + __field(u8, name_scope) __field(u8, name_zone) __field(u8, name_type) __field(u64, name_first) @@ -311,6 +312,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(unsigned int, holders) ), TP_fast_assign( + __entry->name_scope = lck->lock_name.scope; __entry->name_zone = lck->lock_name.zone; __entry->name_type = lck->lock_name.type; __entry->name_first = le64_to_cpu(lck->lock_name.first); @@ -322,9 +324,9 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->refcnt = lck->refcnt; __entry->holders = lck->holders; ), - TP_printk("name %u.%u.%llu.%llu seq %u refs %d holders %d mode %s rqmode %s flags 0x%x", - __entry->name_zone, __entry->name_type, __entry->name_first, - __entry->name_second, __entry->seq, + TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d holders %d mode %s rqmode %s flags 0x%x", + __entry->name_scope, __entry->name_zone, __entry->name_type, + __entry->name_first, __entry->name_second, __entry->seq, __entry->refcnt, __entry->holders, lock_mode(__entry->mode), lock_mode(__entry->rqmode), __entry->flags) ); From f634a5b598078437c991e370e6ce1a5f8bbe8ad6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 31 Jul 2017 14:37:36 -0700 Subject: [PATCH 404/920] scoutfs: implement scoutfs_rename() Previously we had lots of inode creation callers that used a function to create the dirent items and we had unlink remove entries by hand. Rename is different because it wants to remove and add multiple links as it does its work, including recreating links that it has deleted. We rework add_entry_item() so that it gets the specific fields it needs instead of getting them from the vfs structs. This makes it clear that callers are responsible for the source of the fields. Specifically we need to be able to add entries during failed rename cleanup without allocating a new readdir pos from the parent dir. With callers now responsible for the inputs to add_entry_items() we move some of its code out into all callers: checking name length, dirtying the parent dir inode, and allocating a readdir pos from the parent. We then refactor most of _unlink() into a a del_entry_items() to match addition. This removes the last user of scoutfs_item_delete_many() and it will be removed in a future commit. With the entry item helpers taking specific fields all the helpers they use also need to use specific fields instead of the vfs structs. To make rename cluster safe we need to get cluster locks for all the inodes that we work with. We also have to check that the locally cached vfs input is still valid after acquiring the locks. We only check the basic structural correctness of the args: that parent dirs don't violate ancestor rules to create loops and that the entries assumed by the rename arguments still exist, or not. Signed-off-by: Zach Brown --- kmod/src/count.h | 23 ++ kmod/src/dir.c | 618 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 527 insertions(+), 114 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 4198dbf5..19fc6606 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -103,6 +103,29 @@ static inline void scoutfs_count_symlink(struct scoutfs_item_count *cnt, scoutfs_count_sym_target(cnt, size); } +/* + * This assumes the worst case of a rename between directories that + * unlinks an existing target. That'll be worse than the common case + * by a few hundred bytes. + */ +static inline void scoutfs_count_rename(struct scoutfs_item_count *cnt, + unsigned old_len, unsigned new_len) +{ + /* dirty dirs and inodes */ + scoutfs_count_dirty_inode(cnt); + scoutfs_count_dirty_inode(cnt); + scoutfs_count_dirty_inode(cnt); + scoutfs_count_dirty_inode(cnt); + + /* unlink old and new, link new */ + scoutfs_count_dirents(cnt, old_len); + scoutfs_count_dirents(cnt, new_len); + scoutfs_count_dirents(cnt, new_len); + + /* orphan the existing target */ + scoutfs_count_orphan(cnt); +} + /* * Setting an xattr can create a full set of items for an xattr with a * max name and length. Any existing items will be dirtied rather than diff --git a/kmod/src/dir.c b/kmod/src/dir.c index f8ae31f6..8ba02ff0 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -156,15 +156,14 @@ static int alloc_dentry_info(struct dentry *dentry) return 0; } -static void update_dentry_info(struct dentry *dentry, - struct scoutfs_dirent *dent) +static void update_dentry_info(struct dentry *dentry, u64 pos) { struct dentry_info *di = dentry->d_fsdata; if (WARN_ON_ONCE(di == NULL)) return; - di->readdir_pos = le64_to_cpu(dent->readdir_pos); + di->readdir_pos = pos; } static u64 dentry_info_pos(struct dentry *dentry) @@ -178,21 +177,20 @@ static u64 dentry_info_pos(struct dentry *dentry) } static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, - struct inode *dir, - struct dentry *dentry) + u64 dir_ino, const char *name, + unsigned name_len) { struct scoutfs_dirent_key *dkey; struct scoutfs_key_buf *key; key = scoutfs_key_alloc(sb, offsetof(struct scoutfs_dirent_key, - name[dentry->d_name.len])); + name[name_len])); if (key) { dkey = key->data; dkey->zone = SCOUTFS_FS_ZONE; - dkey->ino = cpu_to_be64(scoutfs_ino(dir)); + dkey->ino = cpu_to_be64(dir_ino); dkey->type = SCOUTFS_DIRENT_TYPE; - memcpy(dkey->name, (void *)dentry->d_name.name, - dentry->d_name.len); + memcpy(dkey->name, (void *)name, name_len); } return key; @@ -201,7 +199,7 @@ static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, static void init_link_backref_key(struct scoutfs_key_buf *key, struct scoutfs_link_backref_key *lbrkey, u64 ino, u64 dir_ino, - char *name, unsigned name_len) + const char *name, unsigned name_len) { lbrkey->zone = SCOUTFS_FS_ZONE; lbrkey->ino = cpu_to_be64(ino); @@ -216,7 +214,7 @@ static void init_link_backref_key(struct scoutfs_key_buf *key, static struct scoutfs_key_buf *alloc_link_backref_key(struct super_block *sb, u64 ino, u64 dir_ino, - char *name, + const char *name, unsigned name_len) { struct scoutfs_link_backref_key *lbkey; @@ -254,7 +252,8 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, if (ret) goto out; - key = alloc_dirent_key(sb, dir, dentry); + key = alloc_dirent_key(sb, scoutfs_ino(dir), + dentry->d_name.name, dentry->d_name.len); if (!key) { ret = -ENOMEM; goto out; @@ -273,7 +272,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, ret = 0; } else if (ret == 0) { ino = le64_to_cpu(dent.ino); - update_dentry_info(dentry, &dent); + update_dentry_info(dentry, le64_to_cpu(dent.readdir_pos)); } out: if (ret < 0) @@ -313,11 +312,11 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) } static void init_readdir_key(struct scoutfs_key_buf *key, - struct scoutfs_readdir_key *rkey, - struct inode *inode, loff_t pos) + struct scoutfs_readdir_key *rkey, u64 dir_ino, + loff_t pos) { rkey->zone = SCOUTFS_FS_ZONE; - rkey->ino = cpu_to_be64(scoutfs_ino(inode)); + rkey->ino = cpu_to_be64(dir_ino); rkey->type = SCOUTFS_READDIR_TYPE; rkey->pos = cpu_to_be64(pos); @@ -354,7 +353,8 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (ret) return ret; - init_readdir_key(&last_key, &last_rkey, inode, SCOUTFS_DIRENT_LAST_POS); + init_readdir_key(&last_key, &last_rkey, scoutfs_ino(inode), + SCOUTFS_DIRENT_LAST_POS); item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); dent = kmalloc(item_len, GFP_KERNEL); @@ -364,7 +364,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } for (;;) { - init_readdir_key(&key, &rkey, inode, file->f_pos); + init_readdir_key(&key, &rkey, scoutfs_ino(inode), file->f_pos); scoutfs_kvec_init(val, dent, item_len); ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, @@ -395,45 +395,35 @@ out: return ret; } -static int add_entry_items(struct inode *dir, struct scoutfs_lock *dir_lock, - struct dentry *dentry, struct inode *inode, +/* + * Add all the items for the named link to the inode in the dir. Only + * items are modified. The caller is responsible for locking, entering + * a transaction, dirtying items, and managing the vfs structs. + * + * If this returns an error then nothing will have changed. + */ +static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, + const char *name, unsigned name_len, u64 ino, + umode_t mode, struct scoutfs_lock *dir_lock, struct scoutfs_lock *inode_lock) { - struct scoutfs_inode_info *si = SCOUTFS_I(dir); - struct dentry_info *di = dentry->d_fsdata; - struct super_block *sb = dir->i_sb; struct scoutfs_key_buf *ent_key = NULL; struct scoutfs_key_buf *lb_key = NULL; - struct scoutfs_key_buf *del_keys[3]; - struct scoutfs_key_buf *end_keys[3]; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; struct scoutfs_dirent dent; SCOUTFS_DECLARE_KVEC(val); - int del = 0; - u64 pos; + bool del_ent = false; + bool del_rdir = false; int ret; - int err; - - /* caller should have allocated the dentry info */ - if (WARN_ON_ONCE(di == NULL)) - return -EINVAL; - - if (dentry->d_name.len > SCOUTFS_NAME_LEN) - return -ENAMETOOLONG; - - ret = scoutfs_dirty_inode_item(dir, dir_lock->end); - if (ret) - return ret; /* initialize the dent */ - pos = si->next_readdir_pos++; - dent.ino = cpu_to_le64(scoutfs_ino(inode)); + dent.ino = cpu_to_le64(ino); dent.readdir_pos = cpu_to_le64(pos); - dent.type = mode_to_type(inode->i_mode); + dent.type = mode_to_type(mode); /* dirent item for lookup */ - ent_key = alloc_dirent_key(sb, dir, dentry); + ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); if (!ent_key) return -ENOMEM; @@ -442,43 +432,31 @@ static int add_entry_items(struct inode *dir, struct scoutfs_lock *dir_lock, ret = scoutfs_item_create(sb, ent_key, val); if (ret) goto out; - del_keys[del++] = ent_key; - end_keys[del] = dir_lock->end; + del_ent = true; /* readdir item for .. readdir */ - init_readdir_key(&rdir_key, &rkey, dir, pos); - scoutfs_kvec_init(val, &dent, sizeof(dent), - (void *)dentry->d_name.name, dentry->d_name.len); + init_readdir_key(&rdir_key, &rkey, dir_ino, pos); + scoutfs_kvec_init(val, &dent, sizeof(dent), (char *)name, name_len); ret = scoutfs_item_create(sb, &rdir_key, val); if (ret) goto out; - del_keys[del++] = &rdir_key; - end_keys[del] = dir_lock->end; + del_rdir = true; /* link backref item for inode to path resolution */ - lb_key = alloc_link_backref_key(sb, scoutfs_ino(inode), - scoutfs_ino(dir), - (void *)dentry->d_name.name, - dentry->d_name.len); + lb_key = alloc_link_backref_key(sb, ino, dir_ino, name, name_len); if (!lb_key) { ret = -ENOMEM; goto out; } ret = scoutfs_item_create(sb, lb_key, NULL); - if (ret) - goto out; - del_keys[del++] = lb_key; - end_keys[del] = inode_lock->end; - - update_dentry_info(dentry, &dent); - ret = 0; out: - while (ret < 0 && --del >= 0) { - err = scoutfs_item_delete(sb, del_keys[del], end_keys[del]); - /* can always delete dirty while holding */ - BUG_ON(err); + if (ret < 0) { + if (del_ent) + scoutfs_item_delete_dirty(sb, ent_key); + if (del_rdir) + scoutfs_item_delete_dirty(sb, &rdir_key); } scoutfs_key_free(sb, ent_key); @@ -487,6 +465,57 @@ out: return ret; } +/* + * Delete all the items for the named link to the inode in the dir. + * Only items are modified. The caller is responsible for locking, + * entering a transaction, dirtying items, and managing the vfs structs. + * + * The items match the items used in add_entry_items() but we don't have + * to worry about values here and we can dirty all the items before + * starting to delete them which makes cleanup a little easier. + * + * If this returns an error then nothing will have changed. + */ +static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, + const char *name, unsigned name_len, u64 ino, + struct scoutfs_lock *dir_lock, + struct scoutfs_lock *inode_lock) +{ + struct scoutfs_key_buf *ent_key; + struct scoutfs_key_buf *lb_key; + struct scoutfs_key_buf rdir_key; + struct scoutfs_readdir_key rkey; + int ret; + + ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); + if (!ent_key) + return -ENOMEM; + + init_readdir_key(&rdir_key, &rkey, dir_ino, pos); + + lb_key = alloc_link_backref_key(sb, ino, dir_ino, name, name_len); + if (!lb_key) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_item_dirty(sb, ent_key, dir_lock->end) ?: + scoutfs_item_dirty(sb, &rdir_key, dir_lock->end) ?: + scoutfs_item_dirty(sb, lb_key, inode_lock->end); + if (ret) + goto out; + + scoutfs_item_delete_dirty(sb, ent_key); + scoutfs_item_delete_dirty(sb, &rdir_key); + scoutfs_item_delete_dirty(sb, lb_key); + ret = 0; + +out: + kfree(ent_key); + kfree(lb_key); + return ret; +} + static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { @@ -495,8 +524,12 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct inode *inode = NULL; struct scoutfs_lock *dir_lock; struct scoutfs_lock *inode_lock = NULL; + u64 pos; int ret; + if (dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + ret = alloc_dentry_info(dentry); if (ret) return ret; @@ -511,6 +544,10 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out_unlock; + ret = scoutfs_dirty_inode_item(dir, dir_lock->end); + if (ret) + goto out; + inode = scoutfs_new_inode(sb, dir, mode, rdev, dir_lock); if (IS_ERR(inode)) { ret = PTR_ERR(inode); @@ -522,10 +559,16 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out; - ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, + dentry->d_name.len, scoutfs_ino(inode), + inode->i_mode, dir_lock, inode_lock); if (ret) goto out; + update_dentry_info(dentry, pos); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; @@ -571,8 +614,11 @@ static int scoutfs_link(struct dentry *old_dentry, struct scoutfs_lock *dir_lock; struct scoutfs_lock *inode_lock = NULL; DECLARE_ITEM_COUNT(cnt); + u64 pos; int ret; + if (dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, dir, &dir_lock); @@ -598,10 +644,19 @@ static int scoutfs_link(struct dentry *old_dentry, if (ret) goto out_unlock; - ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); + ret = scoutfs_dirty_inode_item(dir, dir_lock->end); if (ret) goto out; + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, + dentry->d_name.len, scoutfs_ino(inode), + inode->i_mode, dir_lock, inode_lock); + if (ret) + goto out; + update_dentry_info(dentry, pos); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_ctime = dir->i_mtime; @@ -620,6 +675,17 @@ out_unlock: return ret; } +static bool should_orphan(struct inode *inode) +{ + if (inode == NULL) + return false; + + if (S_ISDIR(inode->i_mode)) + return inode->i_nlink == 2; + + return inode->i_nlink == 1; +} + /* * Unlink removes the entry from its item and removes the item if ours * was the only remaining entry. @@ -629,16 +695,11 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct super_block *sb = dir->i_sb; struct inode *inode = dentry->d_inode; struct timespec ts = current_kernel_time(); - struct scoutfs_key_buf *keys[3] = {NULL,}; - struct scoutfs_key_buf *ends[3] = {NULL,}; - struct scoutfs_key_buf rdir_key; - struct scoutfs_readdir_key rkey; - DECLARE_ITEM_COUNT(cnt); - struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; + struct scoutfs_lock *dir_lock = NULL; + DECLARE_ITEM_COUNT(cnt); int ret = 0; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, dir, &dir_lock); if (ret) @@ -647,57 +708,34 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) - goto out; + goto unlock; if (S_ISDIR(inode->i_mode) && i_size_read(inode)) { ret = -ENOTEMPTY; - goto out; - } - - keys[0] = alloc_dirent_key(sb, dir, dentry); - if (!keys[0]) { - ret = -ENOMEM; - goto out; - } - ends[0] = dir_lock->end; - - init_readdir_key(&rdir_key, &rkey, dir, dentry_info_pos(dentry)); - keys[1] = &rdir_key; - ends[1] = dir_lock->end; - - keys[2] = alloc_link_backref_key(sb, scoutfs_ino(inode), - scoutfs_ino(dir), - (void *)dentry->d_name.name, - dentry->d_name.len); - if (!keys[2]) { - ret = -ENOMEM; - goto out; + goto unlock; } scoutfs_count_unlink(&cnt, dentry->d_name.len); ret = scoutfs_hold_trans(sb, &cnt); + if (ret) + goto unlock; + + ret = del_entry_items(sb, scoutfs_ino(dir), dentry_info_pos(dentry), + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), dir_lock, inode_lock); if (ret) goto out; - ret = scoutfs_dirty_inode_item(dir, dir_lock->end) ?: - scoutfs_dirty_inode_item(inode, inode_lock->end); - if (ret) - goto out_trans; - - ret = scoutfs_item_delete_many(sb, keys, ARRAY_SIZE(keys), ends); - if (ret) - goto out_trans; - - if ((inode->i_nlink == 1) || - (S_ISDIR(inode->i_mode) && inode->i_nlink == 2)) { + if (should_orphan(inode)) { /* * Insert the orphan item before we modify any inode * metadata so we can gracefully exit should it * fail. */ ret = scoutfs_orphan_inode(inode); + WARN_ON_ONCE(ret); /* XXX returning error but items deleted */ if (ret) - goto out_trans; + goto out; } dir->i_ctime = ts; @@ -713,13 +751,12 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) scoutfs_update_inode_item(inode); scoutfs_update_inode_item(dir); -out_trans: - scoutfs_release_trans(sb); out: - scoutfs_key_free(sb, keys[0]); - scoutfs_key_free(sb, keys[2]); + scoutfs_release_trans(sb); +unlock: scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + return ret; } @@ -881,10 +918,12 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct scoutfs_lock *dir_lock; struct scoutfs_lock *inode_lock = NULL; DECLARE_ITEM_COUNT(cnt); + u64 pos; int ret; /* path_max includes null as does our value for nd_set_link */ - if (name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) + if (dentry->d_name.len > SCOUTFS_NAME_LEN || + name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) return -ENAMETOOLONG; ret = alloc_dentry_info(dentry); @@ -901,6 +940,10 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) goto out_unlock; + ret = scoutfs_dirty_inode_item(dir, dir_lock->end); + if (ret) + goto out; + inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0, dir_lock); if (IS_ERR(inode)) { ret = PTR_ERR(inode); @@ -916,10 +959,16 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) goto out; - ret = add_entry_items(dir, dir_lock, dentry, inode, inode_lock); + pos = SCOUTFS_I(dir)->next_readdir_pos++; + + ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, + dentry->d_name.len, scoutfs_ino(inode), + inode->i_mode, dir_lock, inode_lock); if (ret) goto out; + update_dentry_info(dentry, pos); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -1112,6 +1161,346 @@ out: return ret; } +/* + * Given two parent dir inos, return the ancestor of p2 that is p1's + * child when p1 is also an ancestor of p2: p1/p/[...]/p2. This can + * return p2. + * + * We do this by walking link backref items. Each entry can be thought + * of as a dirent stored at the target. So the parent dir is stored in + * the target. + * + * The caller holds the global rename lock and link backref walk locks + * each inode as it looks up backrefs. + */ +static int item_d_ancestor(struct super_block *sb, u64 p1, u64 p2, u64 *p_ret) +{ + struct scoutfs_link_backref_entry *ent; + LIST_HEAD(list); + u64 dir_ino; + int ret; + u64 p; + + *p_ret = 0; + + ret = scoutfs_dir_get_backref_path(sb, p2, 0, NULL, 0, &list); + if (ret) + goto out; + + p = p2; + list_for_each_entry(ent, &list, head) { + dir_ino = be64_to_cpu(ent->lbkey.dir_ino); + + if (dir_ino == p1) { + *p_ret = p; + ret = 0; + break; + } + p = dir_ino; + } + +out: + scoutfs_dir_free_backref_path(sb, &list); + return ret; +} + +/* + * The vfs checked the relationship between dirs, the source, and target + * before acquiring clusters locks. All that could have changed. If + * we're renaming between parent dirs then we try to verify the basics + * of those checks using our backref items. + * + * Compare this to lock_rename()'s use of d_ancestor() and what it's + * caller does with the returned ancestor. + */ +static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, + u64 old_ino, u64 new_ino) +{ + int ret; + u64 p; + + ret = item_d_ancestor(sb, p1, p2, &p); + if (ret == 0 && p == 0) + ret = item_d_ancestor(sb, p2, p1, &p); + if (ret == 0 && p && (p == old_ino || p == new_ino)) + ret = -EINVAL; + + return ret; +} + +/* + * Make sure that a dirent from the dir to the inode exists at the name. + * The caller has the name locked in the dir. + */ +static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, + unsigned name_len, u64 ino) +{ + struct scoutfs_key_buf *key = NULL; + struct scoutfs_dirent dent; + SCOUTFS_DECLARE_KVEC(val); + int ret; + + key = alloc_dirent_key(sb, dir_ino, name, name_len); + if (!key) + return -ENOMEM; + + scoutfs_kvec_init(val, &dent, sizeof(dent)); + + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), NULL); + if (ret == 0 && le64_to_cpu(dent.ino) != ino) + ret = -ENOENT; + else if (ret == -ENOENT && ino == 0) + ret = 0; + + scoutfs_key_free(sb, key); + return ret; +} + +/* + * The vfs performs checks on cached inodes and dirents before calling + * here. It doesn't hold any locks so all of those checks can be based + * on cached state that has been invalidated by other operations in the + * cluster before we get here. + * + * We do the expedient thing today and verify the basic structural + * checks after we get cluster locks. We perform topology checks + * analagous to the d_ancestor() walks in lock_rename() after acquiring + * a clustered equivalent of the vfs rename lock. We then lock the dir + * and target inodes and verify that the entries assumed by the function + * arguments still exist. + * + * We don't duplicate all the permissions checking in the vfs + * (may_create(), etc, are all static.). This means racing renames can + * succeed after other nodes have gotten success out of changes to + * permissions that should have forbidden renames. + * + * All of this wouldn't be necessary if we could get prepare/complete + * callbacks around rename that'd let us lock the inodes, dirents, and + * topology while the vfs walks dentries and uses inodes. + * + * We acquire the inode locks in inode number order. Because of our + * inode group locking we can't define lock ordering correctness by + * properties that can be different in a given group. This prevents us + * from using parent/child locking orders as two groups can have both + * parent and child relationships to each other. + */ +static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct super_block *sb = old_dir->i_sb; + struct inode *old_inode = old_dentry->d_inode; + struct inode *new_inode = new_dentry->d_inode; + struct scoutfs_lock *rename_lock = NULL; + struct scoutfs_lock *old_dir_lock = NULL; + struct scoutfs_lock *new_dir_lock = NULL; + struct scoutfs_lock *old_inode_lock = NULL; + struct scoutfs_lock *new_inode_lock = NULL; + struct timespec now; + DECLARE_ITEM_COUNT(cnt); + bool ins_new = false; + bool del_new = false; + bool ins_old = false; + u64 new_pos; + int ret; + int err; + + if (new_dentry->d_name.len > SCOUTFS_NAME_LEN) + return -ENAMETOOLONG; + + /* if dirs are different make sure ancestor relationships are valid */ + if (old_dir != new_dir) { + ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, + SCOUTFS_LOCK_TYPE_GLOBAL_RENAME, + &rename_lock); + if (ret) + return ret; + + ret = verify_ancestors(sb, scoutfs_ino(old_dir), + scoutfs_ino(new_dir), + scoutfs_ino(old_inode), + new_inode ? scoutfs_ino(new_inode) : 0); + if (ret) + goto out_unlock; + } + + /* lock all the inodes */ + ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + old_dir, &old_dir_lock, + new_dir, &new_dir_lock, + old_inode, &old_inode_lock, + new_inode, &new_inode_lock); + if (ret) + goto out_unlock; + + /* test dir i_size now that it's refreshed */ + if (new_inode && S_ISDIR(new_inode->i_mode) && i_size_read(new_inode)) { + ret = -ENOTEMPTY; + goto out_unlock; + } + + /* make sure that the entries assumed by the argument still exist */ + ret = verify_entry(sb, scoutfs_ino(old_dir), old_dentry->d_name.name, + old_dentry->d_name.len, scoutfs_ino(old_inode)) ?: + verify_entry(sb, scoutfs_ino(new_dir), new_dentry->d_name.name, + new_dentry->d_name.len, + new_inode ? scoutfs_ino(new_inode) : 0); + if (ret) + goto out_unlock; + + scoutfs_count_rename(&cnt, old_dentry->d_name.len, + new_dentry->d_name.len); + ret = scoutfs_hold_trans(sb, &cnt); + if (ret) + goto out_unlock; + + /* get a pos for the new entry */ + new_pos = SCOUTFS_I(new_dir)->next_readdir_pos++; + + /* dirty the inodes so that updating doesn't fail */ + ret = scoutfs_dirty_inode_item(old_dir, old_dir_lock->end) ?: + scoutfs_dirty_inode_item(old_inode, old_inode_lock->end) ?: + (old_dir != new_dir ? + scoutfs_dirty_inode_item(new_dir, new_dir_lock->end) : 0) ?: + (new_inode ? + scoutfs_dirty_inode_item(new_inode, new_inode_lock->end) : 0); + if (ret) + goto out; + + /* remove the new entry if it exists */ + if (new_inode) { + ret = del_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_pos(new_dentry), + new_dentry->d_name.name, + new_dentry->d_name.len, + scoutfs_ino(new_inode), + new_dir_lock, new_inode_lock); + if (ret) + goto out; + ins_new = true; + } + + /* create the new entry */ + ret = add_entry_items(sb, scoutfs_ino(new_dir), new_pos, + new_dentry->d_name.name, new_dentry->d_name.len, + scoutfs_ino(old_inode), old_inode->i_mode, + new_dir_lock, old_inode_lock); + if (ret) + goto out; + del_new = true; + + /* remove the old entry */ + ret = del_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_pos(old_dentry), + old_dentry->d_name.name, + old_dentry->d_name.len, + scoutfs_ino(old_inode), + old_dir_lock, old_inode_lock); + if (ret) + goto out; + ins_old = true; + + if (should_orphan(new_inode)) { + ret = scoutfs_orphan_inode(new_inode); + if (ret) + goto out; + } + + /* won't fail from here on out, update all the vfs structs */ + + /* the caller will use d_move to move the old_dentry into place */ + update_dentry_info(old_dentry, new_pos); + + i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); + if (!new_inode) + i_size_write(new_dir, i_size_read(new_dir) + + new_dentry->d_name.len); + + if (new_inode) { + drop_nlink(new_inode); + if (S_ISDIR(new_inode->i_mode)) { + drop_nlink(new_dir); + drop_nlink(new_inode); + } + } else if (S_ISDIR(old_inode->i_mode) && (old_dir != new_dir)) { + drop_nlink(old_dir); + inc_nlink(new_dir); + } + + now = CURRENT_TIME; + old_dir->i_ctime = now; + old_dir->i_mtime = now; + if (new_dir != old_dir) { + new_dir->i_ctime = now; + new_dir->i_mtime = now; + } + old_inode->i_ctime = now; + if (new_inode) + old_inode->i_ctime = now; + + scoutfs_update_inode_item(old_dir); + scoutfs_update_inode_item(old_inode); + if (new_dir != old_dir) + scoutfs_update_inode_item(new_dir); + if (new_inode) + scoutfs_update_inode_item(new_inode); + + ret = 0; +out: + if (ret) { + /* + * XXX We have to clean up partial item deletions today + * because we can't have two dirents existing in a + * directory that point to different inodes. If we + * could we'd create the new name then everything after + * that is deletion that will only fail cleanly or + * succeed. Maybe we could have an item replace call + * that gives us the dupe to re-insert on cleanup? Not + * sure. + */ + err = 0; + if (ins_old) + err = add_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_pos(old_dentry), + old_dentry->d_name.name, + old_dentry->d_name.len, + scoutfs_ino(old_inode), + old_inode->i_mode, + old_dir_lock, + old_inode_lock); + + if (del_new && err == 0) + err = del_entry_items(sb, scoutfs_ino(new_dir), + new_pos, + new_dentry->d_name.name, + new_dentry->d_name.len, + scoutfs_ino(old_inode), + new_dir_lock, old_inode_lock); + + if (ins_new && err == 0) + err = add_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_pos(new_dentry), + new_dentry->d_name.name, + new_dentry->d_name.len, + scoutfs_ino(new_inode), + new_inode->i_mode, + new_dir_lock, + new_inode_lock); + /* XXX freak out: panic, go read only, etc */ + BUG_ON(err); + } + + scoutfs_release_trans(sb); + +out_unlock: + scoutfs_unlock(sb, old_inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, new_inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, old_dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, new_dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, rename_lock, DLM_LOCK_EX); + + return ret; +} + const struct file_operations scoutfs_dir_fops = { .readdir = scoutfs_readdir, .unlocked_ioctl = scoutfs_ioctl, @@ -1127,6 +1516,7 @@ const struct inode_operations scoutfs_dir_iops = { .link = scoutfs_link, .unlink = scoutfs_unlink, .rmdir = scoutfs_unlink, + .rename = scoutfs_rename, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, From d2ea247ab9dc5985ca836ed06640719eb02f275a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 31 Jul 2017 14:04:26 -0700 Subject: [PATCH 405/920] scoutfs: remove scoutfs_item_delete_many() It looked like it was easier to have a helper dirty and delete items. But now that we also have to pass in locks the interface gets messy enough that it's easier to have the caller take care of it. Signed-off-by: Zach Brown --- kmod/src/item.c | 30 ------------------------------ kmod/src/item.h | 3 --- 2 files changed, 33 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 94c45c9b..3de2f9a8 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1416,36 +1416,6 @@ void scoutfs_item_delete_dirty(struct super_block *sb, scoutfs_kvec_kfree(del_val); } -/* - * A helper that deletes a set of items. It first dirties the items - * will be pinned so that deletion won't fail as it tries to read and - * populate the items. - * - * It's a little cleaner to have this helper than have the caller - * iterate, but it could also give us the opportunity to reduce item - * searches if we remembered the items we dirtied. - */ -int scoutfs_item_delete_many(struct super_block *sb, - struct scoutfs_key_buf **keys, unsigned nr, - struct scoutfs_key_buf **ends) -{ - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - ret = scoutfs_item_dirty(sb, keys[i], ends[i]); - if (ret) - goto out; - } - - for (i = 0; i < nr; i++) - scoutfs_item_delete_dirty(sb, keys[i]); - -out: - trace_printk("ret %d\n", ret); - return ret; -} - /* * Return the first dirty node in the subtree starting at the given node. */ diff --git a/kmod/src/item.h b/kmod/src/item.h index e7bae7da..bd04ccbc 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -36,9 +36,6 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_key_buf *end); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); -int scoutfs_item_delete_many(struct super_block *sb, - struct scoutfs_key_buf **keys, unsigned nr, - struct scoutfs_key_buf **ends); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); From 8735d319a3540201a07e73e3468070c8a09bf9ab Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 4 Aug 2017 14:53:32 -0700 Subject: [PATCH 406/920] scoutfs: fix inode lock inversions We lock multiple inodes by order of their inode number. This fixes the directory entry paths that hold parent dir and target inode locks. Link and unlink are easy because they just acquire the existing parent dir and target inode locks. Lookup is a little squirrely because we don't want to try and order the parent dir lock with locks down in iget. It turns out that it's safe to drop the dir lock before calling iget as long as iget handles racing the inode cache instantiation with inode deletion. Creation is the remaining pattern and it's a little weird because we want to lock the newly created inode before we create it and the items that store it. We add a function that correctly orders the locks, transaction, and inode cache instantiation. Signed-off-by: Zach Brown --- kmod/src/dir.c | 168 ++++++++++++++++++++++++++++------------------- kmod/src/inode.c | 9 +-- kmod/src/inode.h | 3 +- 3 files changed, 104 insertions(+), 76 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 8ba02ff0..92a9fe2b 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -231,6 +231,17 @@ static struct scoutfs_key_buf *alloc_link_backref_key(struct super_block *sb, return key; } +/* + * Because of rename, locks are ordered by inode number. To hold the + * dir lock while calling iget, we might have to already hold a lesser + * inode's lock while telling iget whether or not to lock. Instead of + * adding all those moving pieces we drop the dir lock before calling + * iget. We don't reuse inode numbers so we don't have to worry about + * the target of the link changing. We will only follow the entry as it + * existed before or after whatever modification is happening under the + * dir lock and that can already legally race before or after our + * lookup. + */ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { @@ -267,6 +278,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), dir_lock->end); + scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); if (ret == -ENOENT) { ino = 0; ret = 0; @@ -282,8 +294,6 @@ out: else inode = scoutfs_iget(sb, ino); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); - scoutfs_key_free(sb, key); return d_splice_alias(inode, dentry); @@ -516,13 +526,82 @@ out: return ret; } +/* + * Inode creation needs to hold dir and inode locks which can be greater + * or less than each other. It seems easiest to keep the dual locking + * here like it is for all the other dual locking of established inodes. + * Except we don't have the inode struct yet when we're getting locks, + * so we roll our own comparion between the two instead of pushing + * complexity down the locking paths that acquire existing inodes in + * order. + */ +static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, + umode_t mode, dev_t rdev, + struct scoutfs_item_count *cnt, + struct scoutfs_lock **dir_lock, + struct scoutfs_lock **inode_lock) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + int ret = 0; + u64 ino; + + ret = alloc_dentry_info(dentry); + if (ret) + return ERR_PTR(ret); + + ret = scoutfs_alloc_ino(sb, &ino); + if (ret) + return ERR_PTR(ret); + + if (ino < scoutfs_ino(dir)) { + ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, inode_lock) ?: + scoutfs_lock_inode(sb, DLM_LOCK_EX, + SCOUTFS_LKF_REFRESH_INODE, dir, + dir_lock); + } else { + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, + SCOUTFS_LKF_REFRESH_INODE, dir, + dir_lock) ?: + scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, inode_lock); + } + if (ret) + goto out_unlock; + + ret = scoutfs_hold_trans(sb, cnt); + if (ret) + goto out_unlock; + + inode = scoutfs_new_inode(sb, dir, mode, rdev, ino, *inode_lock); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + ret = scoutfs_dirty_inode_item(dir, (*dir_lock)->end); +out: + if (ret) + scoutfs_release_trans(sb); +out_unlock: + if (ret) { + scoutfs_unlock(sb, *dir_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, *inode_lock, DLM_LOCK_EX); + *dir_lock = NULL; + *inode_lock = NULL; + + inode = ERR_PTR(ret); + } + + return inode; +} + static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct super_block *sb = dir->i_sb; DECLARE_ITEM_COUNT(cnt); struct inode *inode = NULL; - struct scoutfs_lock *dir_lock; + struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; u64 pos; int ret; @@ -530,34 +609,12 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = alloc_dentry_info(dentry); - if (ret) - return ret; - - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - dir, &dir_lock); - if (ret) - return ret; - scoutfs_count_mknod(&cnt, dentry->d_name.len); - ret = scoutfs_hold_trans(sb, &cnt); - if (ret) - goto out_unlock; - ret = scoutfs_dirty_inode_item(dir, dir_lock->end); - if (ret) - goto out; - - inode = scoutfs_new_inode(sb, dir, mode, rdev, dir_lock); - if (IS_ERR(inode)) { - ret = PTR_ERR(inode); - goto out; - } - - /* Now that we have ino from scoutfs_new_inode, allocate a lock */ - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); - if (ret) - goto out; + inode = lock_hold_create(dir, dentry, mode, rdev, &cnt, + &dir_lock, &inode_lock); + if (IS_ERR(inode)) + return PTR_ERR(inode); pos = SCOUTFS_I(dir)->next_readdir_pos++; @@ -585,9 +642,9 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, d_instantiate(dentry, inode); out: scoutfs_release_trans(sb); -out_unlock: scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) iput(inode); @@ -620,16 +677,12 @@ static int scoutfs_link(struct dentry *old_dentry, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - dir, &dir_lock); + ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock, inode, &inode_lock, + NULL, NULL, NULL, NULL); if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); - if (ret) - goto out_unlock; - if (inode->i_nlink >= SCOUTFS_LINK_MAX) { ret = -EMLINK; goto out_unlock; @@ -700,16 +753,12 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) DECLARE_ITEM_COUNT(cnt); int ret = 0; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - dir, &dir_lock); + ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + dir, &dir_lock, inode, &inode_lock, + NULL, NULL, NULL, NULL); if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); - if (ret) - goto unlock; - if (S_ISDIR(inode->i_mode) && i_size_read(inode)) { ret = -ENOTEMPTY; goto unlock; @@ -915,7 +964,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct super_block *sb = dir->i_sb; const int name_len = strlen(symname) + 1; struct inode *inode = NULL; - struct scoutfs_lock *dir_lock; + struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; DECLARE_ITEM_COUNT(cnt); u64 pos; @@ -930,29 +979,11 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - dir, &dir_lock); - if (ret) - return ret; - scoutfs_count_symlink(&cnt, dentry->d_name.len, name_len); - ret = scoutfs_hold_trans(sb, &cnt); - if (ret) - goto out_unlock; - - ret = scoutfs_dirty_inode_item(dir, dir_lock->end); - if (ret) - goto out; - - inode = scoutfs_new_inode(sb, dir, S_IFLNK|S_IRWXUGO, 0, dir_lock); - if (IS_ERR(inode)) { - ret = PTR_ERR(inode); - goto out; - } - - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &inode_lock); - if (ret) - goto out; + inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, &cnt, + &dir_lock, &inode_lock); + if (IS_ERR(inode)) + return PTR_ERR(inode); ret = symlink_item_ops(sb, SYM_CREATE, scoutfs_ino(inode), inode_lock, symname, name_len); @@ -983,6 +1014,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, d_instantiate(dentry, inode); out: if (ret < 0) { + /* XXX remove inode items */ if (!IS_ERR_OR_NULL(inode)) iput(inode); @@ -991,9 +1023,9 @@ out: } scoutfs_release_trans(sb); -out_unlock: scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + return ret; } diff --git a/kmod/src/inode.c b/kmod/src/inode.c index d3fe4b30..08dc6aa6 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -745,7 +745,7 @@ static bool pool_in_flight(struct free_ino_pool *pool) * net layer calls us when it gets a reply. If there's no more inodes * we'll get ino == ~0 and nr == 0. */ -static int alloc_ino(struct super_block *sb, u64 *ino) +int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) { struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; bool request; @@ -807,7 +807,7 @@ out: * creating links to it and updating it. @dir can be null. */ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev, + umode_t mode, dev_t rdev, u64 ino, struct scoutfs_lock *lock) { struct scoutfs_inode_info *ci; @@ -816,13 +816,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, struct scoutfs_inode sinode; SCOUTFS_DECLARE_KVEC(val); struct inode *inode; - u64 ino; int ret; - ret = alloc_ino(sb, &ino); - if (ret) - return ERR_PTR(ret); - inode = new_inode(sb); if (!inode) return ERR_PTR(-ENOMEM); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 452ed00e..d4f23b46 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -64,8 +64,9 @@ struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end); void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); +int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, - umode_t mode, dev_t rdev, + umode_t mode, dev_t rdev, u64 ino, struct scoutfs_lock *lock); void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); From 826bd7f7bfa6edafbf09637cee59a97e1a6d8721 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 28 Aug 2017 13:53:02 -0700 Subject: [PATCH 407/920] scoutfs: ifdef out some unused dlmglue functions Some dlmglue functions are unused by the current ifdefery. They're throwing warnigns that obscure other warnings in the build. This broadens the ifdef coverage so that we don't get warnings. The unused code will either be promoted to an interface or removed as dlmglue evolves into a reusable component. Signed-off-by: Zach Brown --- kmod/src/dlmglue.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index ea819588..334a3e4f 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -367,10 +367,12 @@ static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *l return (struct ocfs2_super *)lockres->l_priv; } +#if 0 static int ocfs2_lock_create(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level, u32 dlm_flags); +#endif static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, int wanted); static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, @@ -1325,6 +1327,7 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, wake_up(&lockres->l_event); } +#if 0 /* Note: If we detect another process working on the lock (i.e., * OCFS2_LOCK_BUSY), we'll bail out returning 0. It's up to the caller * to do the right thing in that case. @@ -1371,6 +1374,7 @@ static int ocfs2_lock_create(struct ocfs2_super *osb, bail: return ret; } +#endif static inline int ocfs2_check_wait_flag(struct ocfs2_lock_res *lockres, int flag) @@ -1458,6 +1462,7 @@ static int __lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, return ret; } +#if 0 static int lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, struct ocfs2_mask_waiter *mw) { @@ -1486,6 +1491,7 @@ static int ocfs2_wait_for_mask_interruptible(struct ocfs2_mask_waiter *mw, reinit_completion(&mw->mw_complete); return ret; } +#endif static int __ocfs2_cluster_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, @@ -1718,6 +1724,7 @@ static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, #endif } +#if 0 static int ocfs2_create_new_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int ex, @@ -1734,6 +1741,7 @@ static int ocfs2_create_new_lock(struct ocfs2_super *osb, return ocfs2_lock_create(osb, lockres, level, lkm_flags); } +#endif #if 0 /* Grants us an EX lock on the data and metadata resources, skipping @@ -2263,6 +2271,8 @@ static inline int ocfs2_meta_lvb_is_trustable(struct inode *inode, } #endif + +#if 0 /* Determine whether a lock resource needs to be refreshed, and * arbitrate who gets to refresh it. * @@ -2298,6 +2308,7 @@ bail: mlog(0, "status %d\n", status); return status; } +#endif /* If status is non zero, I'll mark it as not being in refresh * anymroe, but i won't clear the needs refresh flag. */ From d228de60c42260f741028548b0ef79bf1f8e56e4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 28 Aug 2017 13:55:04 -0700 Subject: [PATCH 408/920] scoutfs: remove unused lock code There's a fair amount of lock.c that's dead code now that we're using dlmglue. Some of the dead code is seen as unused and is throwing warnings. This silences the errors by removing the code. Signed-off-by: Zach Brown --- kmod/src/lock.c | 62 ------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index b379d95b..dd569cbb 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -350,57 +350,6 @@ static void free_lock_tree(struct super_block *sb) } } -static void scoutfs_ast(void *astarg) -{ - struct scoutfs_lock *lock = astarg; - DECLARE_LOCK_INFO(lock->sb, linfo); - - trace_scoutfs_ast(lock->sb, lock); - - spin_lock(&linfo->lock); - lock->mode = lock->rqmode; - /* Clear blocking flag when we are granted an unlock request */ - if (lock->rqmode == DLM_LOCK_IV) - lock->flags &= ~SCOUTFS_LOCK_BLOCKING; - lock->rqmode = DLM_LOCK_IV; - spin_unlock(&linfo->lock); - - wake_up(&linfo->waitq); -} - -static void queue_blocking_work(struct lock_info *linfo, - struct scoutfs_lock *lock) -{ - assert_spin_locked(&linfo->lock); - if (!(lock->flags & SCOUTFS_LOCK_QUEUED)) { - /* Take a ref for the workqueue */ - lock->flags |= SCOUTFS_LOCK_QUEUED; - lock->refcnt++; - queue_work(linfo->downconvert_wq, &lock->dc_work); - } -} - -static void set_lock_blocking(struct lock_info *linfo, - struct scoutfs_lock *lock) -{ - assert_spin_locked(&linfo->lock); - lock->flags |= SCOUTFS_LOCK_BLOCKING; - if (lock->holders == 0) - queue_blocking_work(linfo, lock); -} - -static void scoutfs_bast(void *astarg, int mode) -{ - struct scoutfs_lock *lock = astarg; - struct lock_info *linfo = SCOUTFS_SB(lock->sb)->lock_info; - - trace_scoutfs_bast(lock->sb, lock); - - spin_lock(&linfo->lock); - set_lock_blocking(linfo, lock); - spin_unlock(&linfo->lock); -} - static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lock, int mode) { @@ -413,17 +362,6 @@ static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lock, return ret; } -static int lock_blocking(struct lock_info *linfo, struct scoutfs_lock *lock) -{ - int ret; - - spin_lock(&linfo->lock); - ret = !!(lock->flags & SCOUTFS_LOCK_BLOCKING); - spin_unlock(&linfo->lock); - - return ret; -} - /* * Acquire a coherent lock on the given range of keys. While the lock * is held other lockers are serialized. Cache coherency is maintained From a15ec9ff841856364c6e6e4dabbe652c259f097d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 28 Aug 2017 14:27:05 -0700 Subject: [PATCH 409/920] scoutfs: comment out dlm posix calls stackglue is trying to call dlm posix symbols that don't exist in some RHEL dlm kernels. We're not using this functionality yet so let's just tear it out for now. Signed-off-by: Zach Brown --- kmod/src/stackglue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/stackglue.c b/kmod/src/stackglue.c index f2ff3b1b..b8a5fbe3 100644 --- a/kmod/src/stackglue.c +++ b/kmod/src/stackglue.c @@ -157,6 +157,7 @@ void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb) { } +#if 0 static int user_plock(struct ocfs2_cluster_connection *conn, u64 ino, struct file *file, @@ -193,6 +194,7 @@ int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, { return user_plock(conn, ino, file, cmd, fl); } +#endif static void user_recover_prep(void *arg) { From ca78757ca54b1e184b37c7418f60fcf2a5e8c792 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Aug 2017 09:38:39 -0700 Subject: [PATCH 410/920] scoutfs: more careful client connect timeouts The client connection loop was a bit of a mess. It only slept between retries in one particular case. Other failures to connect would spin and livelock. It would spin forever. This fixed loop now has a much more orderly reconnect procedure. Each connecting sender always tries once. Then retry attempts backoff exponentially, settling at a nice long timeout. After long enough it'll return errors. This fixes livelocks in the xfstests that mount and unmount around dm-flakey config. generic/{034,039,040} would easily livelock before this fix. Signed-off-by: Zach Brown --- kmod/src/client.c | 72 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 628cca99..8cf43653 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -60,9 +60,15 @@ #define KEEPCNT 3 #define KEEPIDLE 7 #define KEEPINTVL 1 -#define KEEP_TIMEO_SECS (KEEPIDLE + (KEEPCNT * KEEPINTVL)) -#define CONNECT_TIMEO_SECS KEEP_TIMEO_SECS -#define CONNECT_TIMEO_MSECS (KEEP_TIMEO_SECS * MSEC_PER_SEC) + + +/* + * Connection timeouts have to allow for enough time for servers to + * reboot. Figure order minutes at the outside. + */ +#define CONN_RETRY_MIN_MS 10UL +#define CONN_RETRY_MAX_MS (5UL * MSEC_PER_SEC) +#define CONN_RETRY_LIMIT_J (5 * 60 * HZ) struct client_info { struct super_block *sb; @@ -83,6 +89,10 @@ struct client_info { /* blocked senders sit on a waitq that's woken for resends */ wait_queue_head_t waitq; + /* connection timeouts are tracked across attempts */ + unsigned long conn_retry_ms; + unsigned long conn_retry_limit_j; + struct workqueue_struct *recv_wq; struct work_struct recv_work; }; @@ -220,16 +230,22 @@ static void scoutfs_client_recv_func(struct work_struct *work) kfree(rx_alloc); } +static void reset_connect_timeouts(struct client_info *client) +{ + client->conn_retry_ms = CONN_RETRY_MIN_MS; + client->conn_retry_limit_j = jiffies + CONN_RETRY_LIMIT_J; +} + /* - * Spin discovering the address of the server and trying to connect to - * it until either we connect or we're interrupted by a signal. + * Clients who try to send and don't see a connected socket call here to + * connect to the server. They get the server address and try to + * connect. * - * A single mount coming up starts both the server and the client. The - * server takes a few IOs and network messages to get going and communicate - * its address. We want to aggressively retry getting the address so that - * these mounts can be quick. But we back off to avoid storms waiting for - * recovery after an existing server explodes. + * Each sending client will always try to connect once. After that + * it'll sleep and retry connecting at increasing intervals. After long + * enough it will return an error. Future attempts will retry once then + * return errors. */ static int client_connect(struct client_info *client) { @@ -238,29 +254,30 @@ static int client_connect(struct client_info *client) struct sockaddr_in *sin; struct socket *sock = NULL; struct timeval tv; - unsigned int msecs = MSEC_PER_SEC / 10; + int retries; int addrlen; int optval; int ret; BUG_ON(!mutex_is_locked(&client->send_mutex)); - for(;;) { + for(retries = 0; ; retries++) { if (sock) { sock_release(sock); sock = NULL; } - ret = scoutfs_read_supers(sb, &super); - if (ret) - continue; + if (retries) { + /* we tried, and we're past limit, return error */ + if (time_after(jiffies, client->conn_retry_limit_j)) { + ret = -ENOTCONN; + break; + } - if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) { - msleep_interruptible(msecs); - if (msecs < CONNECT_TIMEO_MSECS) - msecs = max(msecs + MSEC_PER_SEC, - CONNECT_TIMEO_MSECS); - continue; + msleep_interruptible(client->conn_retry_ms); + + client->conn_retry_ms = min(client->conn_retry_ms * 2, + CONN_RETRY_MAX_MS); } if (signal_pending(current)) { @@ -268,6 +285,13 @@ static int client_connect(struct client_info *client) break; } + ret = scoutfs_read_supers(sb, &super); + if (ret) + continue; + + if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) + continue; + sin = &client->peername; sin->sin_family = AF_INET; sin->sin_addr.s_addr = le32_to_be32(super.server_addr.addr); @@ -284,8 +308,8 @@ static int client_connect(struct client_info *client) if (ret) continue; - /* start with a connect timeout */ - tv.tv_sec = CONNECT_TIMEO_SECS; + /* use short timeout for connect itself */ + tv.tv_sec = 1; tv.tv_usec = 0; ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); @@ -344,6 +368,7 @@ static int client_connect(struct client_info *client) client->sock_gen++; client->recv_shutdown = false; + reset_connect_timeouts(client); queue_work(client->recv_wq, &client->recv_work); wake_up(&client->waitq); ret = 0; @@ -705,6 +730,7 @@ int scoutfs_client_setup(struct super_block *sb) mutex_init(&client->send_mutex); init_waitqueue_head(&client->waitq); INIT_WORK(&client->recv_work, scoutfs_client_recv_func); + reset_connect_timeouts(client); client->recv_wq = alloc_workqueue("scoutfs_client_recv", WQ_UNBOUND, 1); if (!client->recv_wq) { From a8db7e5b7434800237cd3120833b99e3126a2e82 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 10 Aug 2017 14:49:55 -0700 Subject: [PATCH 411/920] scoutfs: stop iteration at lock end value If item iteration finds a hole in the cache it tries to read items. After the items are read it can look at the cached region and return items or -ENOENT. We recently added an end key to limit how far we can read and cache items. The end key addition correctly limited the cache read to the lock end value. It could never read item cache ranges beyond that. This means that it can't iterate past the end value and should return -ENOENT if it gets past end. But the code forgot to do that, it only checked for iteration past last before returning -ENOENT. It spins continually finding a hole past end but inside last, tries to read items but limits them to end, then finds the same hole again. Triggering this requires a lock end that's nearer than the last iteration key. That's hard to do because most of our item reads are covered by inode group locks which extend well past iteration inside a given inode. Inode index item can easily trigger this if there's no items. I tripped over it when walking empty indexes (data_seq or online_blocks with no regular files). Signed-off-by: Zach Brown --- kmod/src/item.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 3de2f9a8..2979a096 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -841,15 +841,17 @@ static struct cached_item *item_for_next(struct rb_root *root, } /* - * Return the next item starting with the given key, returning the last - * key at the most. + * Return the next item starting with the given key and returning the + * last key at most. * - * While iteration stops the last key we can cache up to the end key so - * that a sequence of small iterations covered by one lock are satisfied - * with a large read of items from segments into the cache. + * If the end key is specified then it limits items that can be read + * into the cache. If it's less than the last key then it also limits + * iteration. These are different values because locking granularity + * can be smaller or larger than the iteration. Callers shouldn't have + * to be aware of that relationship. * - * -ENOENT is returned if there are no items between the given and last - * keys. + * -ENOENT is returned if there are no items between the given and + * last/end keys. * * The next item's key is copied to the caller's key. The caller is * responsible for dealing with key lengths and truncation. @@ -871,6 +873,10 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, bool cached; int ret; + /* use the end key as the last key if it's closer to reduce compares */ + if (end && scoutfs_key_compare(end, last) < 0) + last = end; + /* convenience to avoid searching if caller iterates past their last */ if (scoutfs_key_compare(key, last) > 0) { ret = -ENOENT; From 599269e539dbfa1e7991ecd4c3a2b5301ddf8721 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Aug 2017 11:03:49 -0700 Subject: [PATCH 412/920] scoutfs: don't return uninit index entries Initially the index walking ioctl only ever output a single entry per iteration. So the number of entries to return and the next entry pointer to copy to userspace were maintained in the post-increment of the for loop. When we added locking of the index item results we made it possible to not copy any entries in a loop iteration. When that happened the nr and pointer would be incremented without initializing the entry. The ioctl caller would see a garbage entry in the results. This was visible in scoutfs/002 test results on a volume that had an interesting file population after having run through all the other scoutfs tests. The uninitialized entries would show up as garbage in the size index portion of the test. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index abbf3fa4..2013b31c 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -123,8 +123,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (ret < 0) goto out; - for (nr = 0; nr < walk.nr_entries; - nr++, walk.entries_ptr += sizeof(ent)) { + for (nr = 0; nr < walk.nr_entries; ) { ret = scoutfs_item_next_same(sb, &key, &last_key, NULL, lock->end); if (ret < 0 && ret != -ENOENT) @@ -178,6 +177,9 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) break; } + nr++; + walk.entries_ptr += sizeof(ent); + scoutfs_key_inc_cur_len(&key); } From 76cf28b44215d5555a042807eb42317b6cf12590 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 29 Aug 2017 15:02:04 -0700 Subject: [PATCH 413/920] scoutfs: warn if lock with trans held We can't block on a lock while holding the transaction open because that'd stop lock downconversion from syncing to write out items while it is converting from EX. Add a warning if we try to acquire a blocking lock while holding a transaction. Signed-off-by: Zach Brown --- kmod/src/lock.c | 5 +++++ kmod/src/trans.c | 12 ++++++++++++ kmod/src/trans.h | 1 + 3 files changed, 18 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index dd569cbb..c9fc20b4 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -26,6 +26,7 @@ #include "cmp.h" #include "dlmglue.h" #include "inode.h" +#include "trans.h" #define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ @@ -383,6 +384,10 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, int lkm_flags; int ret; + if (WARN_ON_ONCE(!(flags & SCOUTFS_LKF_TRYLOCK) && + scoutfs_trans_held())) + return -EINVAL; + lock = find_alloc_scoutfs_lock(sb, lock_name, type, start, end); if (!lock) return -ENOMEM; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index e630d0fc..ca05240f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -370,6 +370,18 @@ int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) return ret; } +/* + * Return true if the current task has a transaction held. That is, + * true if the current transaction can't finish and be written out if + * the current task blocks. + */ +bool scoutfs_trans_held(void) +{ + struct scoutfs_reservation *rsv = current->journal_info; + + return rsv && rsv->magic == SCOUTFS_RESERVATION_MAGIC; +} + void scoutfs_trans_track_item(struct super_block *sb, signed items, signed keys, signed vals) { diff --git a/kmod/src/trans.h b/kmod/src/trans.h index fcf0d376..49db305d 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -10,6 +10,7 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, void scoutfs_trans_restart_sync_deadline(struct super_block *sb); int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt); +bool scoutfs_trans_held(void); void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_wake_holders(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, From 2eddeb5db4ce876408c3e3110384a7bb308e6b92 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Sep 2017 18:12:02 -0700 Subject: [PATCH 414/920] scoutfs: delete unused net key types Remove these key types from the format which haven't been used for a while. Signed-off-by: Zach Brown --- kmod/src/format.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 07e90ad2..1f509e0b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -264,9 +264,6 @@ struct scoutfs_segment_block { #define SCOUTFS_FILE_EXTENT_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 -/* XXX don't need these now that we have dlm lock spaces and resources */ -#define SCOUTFS_NET_ADDR_TYPE 254 -#define SCOUTFS_NET_LISTEN_TYPE 255 /* value is struct scoutfs_inode */ struct scoutfs_inode_key { From 82f8daaebfe83d295bb625d8fb54b8b67f540982 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Sep 2017 18:26:59 -0700 Subject: [PATCH 415/920] scoutfs: print trailing key bytes The key printing functions only output the key material that's described by the format. We have some callers that need to increment or decrement keys so they expand them to full size keys. This expansion and extra high precision low significance was hidden from the traces. This adds a helper that prints the key material with the format and then appends an encoding of the trailing bytes. The key printer was a huge mess of cases and ifs that made it hard to integrate a sane helper. We also take the opportunity to break it up into zone|type key printer functions. The isolation makes it much clearer to see what's going on. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 + kmod/src/key.c | 416 ++++++++++++++++++++++++++++++---------------- 2 files changed, 273 insertions(+), 145 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 1f509e0b..f64af13f 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -241,6 +241,7 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_ZONE 1 #define SCOUTFS_NODE_ZONE 2 #define SCOUTFS_FS_ZONE 3 +#define SCOUTFS_MAX_ZONE 4 /* power of 2 is efficient */ /* inode index zone */ #define SCOUTFS_INODE_INDEX_SIZE_TYPE 3 @@ -264,6 +265,7 @@ struct scoutfs_segment_block { #define SCOUTFS_FILE_EXTENT_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 +#define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ /* value is struct scoutfs_inode */ struct scoutfs_inode_key { diff --git a/kmod/src/key.c b/kmod/src/key.c index 40051d8a..ba3954cc 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -109,6 +109,249 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) #define snprintf_null(buf, size, fmt, args...) \ (snprintf((buf), (size), fmt, ##args) + 1) +/* + * Store a formatted string representing the key in the buffer. The key + * must be at least min_len to store the data needed by the format at + * all. fmt_len is the length of data that's used by the format. These + * are different because we have badly designed keys with variable + * length data that isn't described by the key. It's assumed from the + * length of the key. Take dirents -- they need to at least have a + * dirent struct, but the name length is the rest of the key. + * + * (XXX And this goes horribly wrong when we pad out dirent keys to max + * len to increment at high precision. We'll never see these items used + * by real fs code, but temporary keys and range endpoints can be full + * precision and we can try and print them and get very confused. We + * need to rev the format to include explicit lengths.) + * + * If the format doesn't cover the entire key then we append more + * formatting to represent the trailing bytes: runs of zeros compresesd + * to _ and then hex output of non-zero bytes. + */ +static int snprintf_key(char *buf, size_t size, struct scoutfs_key_buf *key, + unsigned min_len, unsigned fmt_len, + const char *fmt, ...) + +{ + va_list args; + char *data; + char *end; + int left; + int part; + int ret; + int nr; + + if (key->key_len < min_len) + return snprintf_null(buf, size, "[trunc len %u < min %u]", + key->key_len, min_len); + + if (fmt_len == 0) + fmt_len = min_len; + + va_start(args, fmt); + ret = vsnprintf(buf, size, fmt, args); + va_end(args); + /* next formatting overwrites null */ + if (buf) { + buf += ret; + size -= min_t(int, size, ret); + } + + data = key->data + fmt_len; + left = key->key_len - fmt_len; + + while (left && (!buf || size > 1)) { + /* compress runs of zero bytes to _ */ + end = memchr_inv(data, 0, left); + nr = end ? end - data : left; + if (nr) { + if (buf) { + *(buf++) = '_'; + size--; + } + ret++; + data += nr; + left -= nr; + continue; + } + + /* + * hex print non-zero bytes. %ph is limited to 64 bytes + * and is buggy in that it still tries to print to buf + * past size. (so buf = null, size = 0 crashes instead + * of printing the length of the formatted string.) + */ + end = memchr(data, 0, left); + nr = end ? end - data : left; + nr = min(nr, 64); + + if (buf) + part = snprintf(buf, size, "%*phN", nr, data); + else + part = nr * 2; + if (buf) { + buf += part; + size -= min_t(int, size, part); + } + ret += part; + + data += nr; + left -= nr; + } + + /* always store and include null */ + if (buf) + *buf = '\0'; + return ret + 1; +} + +typedef int (*key_printer_t)(char *buf, struct scoutfs_key_buf *key, + size_t size); + +static int pr_ino_idx(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + static char *type_strings[] = { + [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", + [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", + [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", + }; + struct scoutfs_inode_index_key *ikey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_inode_index_key), 0, + "iin.%s.%llu.%u.%llu", + type_strings[ikey->type], be64_to_cpu(ikey->major), + be32_to_cpu(ikey->minor), be64_to_cpu(ikey->ino)); +} + +static int pr_free_ext(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_free_extent_blkno_key *fkey = key->data; + + static char *type_strings[] = { + [SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fno", + [SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fks", + }; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_free_extent_blkno_key), 0, + "nod.%llu.%s.%llu.%llu", + be64_to_cpu(fkey->node_id), + type_strings[fkey->type], + be64_to_cpu(fkey->last_blkno), + be64_to_cpu(fkey->blocks)); +} + +static int pr_orphan(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_orphan_key *okey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_orphan_key), 0, + "nod.%llu.orp.%llu", + be64_to_cpu(okey->node_id), + be64_to_cpu(okey->ino)); +} + +static int pr_inode(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_inode_key *ikey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_inode_key), 0, + "fs.%llu.ino", + be64_to_cpu(ikey->ino)); +} + +static int pr_xattr(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_xattr_key *xkey = key->data; + int len = (int)key->key_len - + offsetof(struct scoutfs_xattr_key, name[1]); + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_xattr_key), key->key_len, + "fs.%llu.xat.%.*s", + be64_to_cpu(xkey->ino), len, xkey->name); +} + +static int pr_dirent(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_dirent_key *dkey = key->data; + int len = (int)key->key_len - sizeof(struct scoutfs_dirent_key); + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_dirent_key), key->key_len, + "fs.%llu.dnt.%.*s", + be64_to_cpu(dkey->ino), len, dkey->name); +} + +static int pr_readdir(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_readdir_key *rkey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_readdir_key), 0, + "fs.%llu.rdr.%llu", + be64_to_cpu(rkey->ino), be64_to_cpu(rkey->pos)); +} + +static int pr_link_backref(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_link_backref_key *lkey = key->data; + int len = (int)key->key_len - sizeof(*lkey); + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_link_backref_key), + key->key_len, + "fs.%llu.lbr.%llu.%.*s", + be64_to_cpu(lkey->ino), be64_to_cpu(lkey->dir_ino), + len, lkey->name); +} + +static int pr_symlink(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_symlink_key *skey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_symlink_key), 0, + "fs.%llu.sym", + be64_to_cpu(skey->ino)); +} + +static int pr_file_ext(char *buf, struct scoutfs_key_buf *key, size_t size) +{ + struct scoutfs_file_extent_key *ekey = key->data; + + return snprintf_key(buf, size, key, + sizeof(struct scoutfs_file_extent_key), 0, + "fs.%llu.ext.%llu.%llu.%llu.%x", + be64_to_cpu(ekey->ino), + be64_to_cpu(ekey->last_blk_off), + be64_to_cpu(ekey->last_blkno), + be64_to_cpu(ekey->blocks), + ekey->flags); +} + +const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { + [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_SIZE_TYPE] = + pr_ino_idx, + [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = + pr_ino_idx, + [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = + pr_ino_idx, + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = pr_free_ext, + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = pr_free_ext, + [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = pr_orphan, + [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = pr_inode, + [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = pr_xattr, + [SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = pr_dirent, + [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_readdir, + [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_link_backref, + [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = pr_symlink, + [SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = pr_file_ext, +}; + /* * Write the null-terminated string that describes the key to the * buffer. The bytes copied (including the null) is returned. A null @@ -118,168 +361,51 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) */ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) { - struct scoutfs_inode_key *ikey; - u8 zone = 0; - u8 type = 0; - int len; + u8 zone; + u8 type; if (key == NULL || key->data == NULL) return snprintf_null(buf, size, "[NULL]"); - if (key->key_len == 0) - return snprintf_null(buf, size, "[0 len]"); + /* always at least zone, some id, and type */ + if (key->key_len < (1 + 8 + 1)) + return snprintf_null(buf, size, "[trunc len %u]", key->key_len); zone = *(u8 *)key->data; - /* handle smaller and unknown zones, fall through to fs types */ - switch(zone) { - case SCOUTFS_INODE_INDEX_ZONE: { + /* + * each zone's keys always start with the same fields that let + * us deref any key to get the type. We chose a few representative + * keys from each zone to get the type. + */ + if (zone == SCOUTFS_INODE_INDEX_ZONE) { struct scoutfs_inode_index_key *ikey = key->data; - static char *type_strings[] = { - [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", - [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", - [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", - }; - - if (key->key_len < sizeof(struct scoutfs_inode_index_key)) - break; - - if (type_strings[ikey->type]) - return snprintf_null(buf, size, "iin.%s.%llu.%u.%llu", - type_strings[ikey->type], - be64_to_cpu(ikey->major), - be32_to_cpu(ikey->minor), - be64_to_cpu(ikey->ino)); - else - return snprintf_null(buf, size, "[iin type %u?]", - ikey->type); - } - - /* node zone keys start with zone, node, type */ - case SCOUTFS_NODE_ZONE: { + type = ikey->type; + } else if (zone == SCOUTFS_NODE_ZONE) { struct scoutfs_free_extent_blkno_key *fkey = key->data; - - static char *type_strings[] = { - [SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fno", - [SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fks", - }; - - switch(fkey->type) { - case SCOUTFS_ORPHAN_TYPE: { - struct scoutfs_orphan_key *okey = key->data; - - if (key->key_len < sizeof(struct scoutfs_orphan_key)) - break; - return snprintf_null(buf, size, "nod.%llu.orp.%llu", - be64_to_cpu(okey->node_id), - be64_to_cpu(okey->ino)); - } - - case SCOUTFS_FREE_EXTENT_BLKNO_TYPE: - case SCOUTFS_FREE_EXTENT_BLOCKS_TYPE: - return snprintf_null(buf, size, "nod.%llu.%s.%llu.%llu", - be64_to_cpu(fkey->node_id), - type_strings[fkey->type], - be64_to_cpu(fkey->last_blkno), - be64_to_cpu(fkey->blocks)); - default: - return snprintf_null(buf, size, "[nod type %u?]", - fkey->type); - } - } - - case SCOUTFS_FS_ZONE: - break; - - default: - return snprintf_null(buf, size, "[zone %u?]", zone); - } - - /* everything in the fs tree starts with zone, ino, type */ - ikey = key->data; - switch(ikey->type) { - case SCOUTFS_INODE_TYPE: { + type = fkey->type; + } else if (zone == SCOUTFS_FS_ZONE) { struct scoutfs_inode_key *ikey = key->data; - - if (key->key_len < sizeof(struct scoutfs_inode_key)) - break; - - return snprintf_null(buf, size, "fs.%llu.ino", - be64_to_cpu(ikey->ino)); + type = ikey->type; + } else { + type = 255; } - case SCOUTFS_XATTR_TYPE: { - struct scoutfs_xattr_key *xkey = key->data; - - len = (int)key->key_len - offsetof(struct scoutfs_xattr_key, - name[1]); - if (len <= 0) - break; - - return snprintf_null(buf, size, "fs.%llu.xat.%.*s", - be64_to_cpu(xkey->ino), len, xkey->name); + if (zone > SCOUTFS_MAX_ZONE || type > SCOUTFS_MAX_TYPE || + key_printers[zone][type] == NULL) { + return snprintf_null(buf, size, "[unk zone %u type %u]", + zone, type); } - case SCOUTFS_DIRENT_TYPE: { - struct scoutfs_dirent_key *dkey = key->data; - - len = (int)key->key_len - sizeof(struct scoutfs_dirent_key); - if (len <= 0) - break; - - return snprintf_null(buf, size, "fs.%llu.dnt.%.*s", - be64_to_cpu(dkey->ino), len, dkey->name); - } - - case SCOUTFS_READDIR_TYPE: { - struct scoutfs_readdir_key *rkey = key->data; - - return snprintf_null(buf, size, "fs.%llu.rdr.%llu", - be64_to_cpu(rkey->ino), - be64_to_cpu(rkey->pos)); - } - - case SCOUTFS_LINK_BACKREF_TYPE: { - struct scoutfs_link_backref_key *lkey = key->data; - - len = (int)key->key_len - sizeof(*lkey); - if (len <= 0) - break; - - return snprintf_null(buf, size, "fs.%llu.lbr.%llu.%.*s", - be64_to_cpu(lkey->ino), - be64_to_cpu(lkey->dir_ino), len, - lkey->name); - } - - case SCOUTFS_SYMLINK_TYPE: { - struct scoutfs_symlink_key *skey = key->data; - - return snprintf_null(buf, size, "fs.%llu.sym", - be64_to_cpu(skey->ino)); - } - - case SCOUTFS_FILE_EXTENT_TYPE: { - struct scoutfs_file_extent_key *ekey = key->data; - - return snprintf_null(buf, size, "fs.%llu.ext.%llu.%llu.%llu.%x", - be64_to_cpu(ekey->ino), - be64_to_cpu(ekey->last_blk_off), - be64_to_cpu(ekey->last_blkno), - be64_to_cpu(ekey->blocks), - ekey->flags); - } - - default: - return snprintf_null(buf, size, "[fs type %u?]", type); - } - - return snprintf_null(buf, size, "[fs type %u trunc len %u]", - type, key->key_len); + return key_printers[zone][type](buf, key, size); } /* - * A null buf can be set to find the length of the formatted string. + * Callers never have a pre-existing buffer whose size they need to be + * careful for. For a given static string they're first calling with a + * null buf to find out the formatted length without storing anything. + * Then they're called again with a buffer of that allocation size. As + * long as the formatting is consistent this pattern won't overflow. */ int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) { From 51e8b614e51434c97b5e3ca0a06b49402ad3c626 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 2 Sep 2017 18:30:11 -0700 Subject: [PATCH 416/920] scoutfs: stop livelocking in item_next scoutfs_item_next() could livelock given the right key and segment key boundaries. This was easiest to trigger with an fio command that wrote a lot of data: fio --filesize=100m --nrfiles=25 --name=100m --numjobs=1 \ --iodepth=1 --ioengine=sync --fallocate=0 \ --rw=write --openfiles=256 \ --directory=$TEST_DIR There were two problems. First, if it found a cached region that didn't contain a next item it would try to read the *end* of the existing cached region instead of trying to populate more items by reading from the key past the existing cached region. This is fixed by incrementing the key to read from after setting it to the end of the cached region. Second, it got totally confused by non-merged but adjacent cached regions. It would find a cached region that contains the search key and try to read from the key after that region, but that key could also be cached and just not merged with its previous region. This is fixed by (duh) having an allocated pos key that we set as we walk through cached regions. It used to always try and read from the search key which was bonkers. With these fixes fio now completes. Signed-off-by: Zach Brown --- kmod/src/item.c | 71 ++++++++++++++++++++++------------------ kmod/src/scoutfs_trace.h | 30 +++++++++++++++++ 2 files changed, 70 insertions(+), 31 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 2979a096..15caddaf 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -866,7 +866,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *read_start = NULL; + struct scoutfs_key_buf *pos = NULL; struct scoutfs_key_buf *range_end = NULL; struct cached_item *item; unsigned long flags; @@ -883,57 +883,66 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, goto out; } - read_start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); + pos = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!read_start || !range_end) { + if (!pos || !range_end) { ret = -ENOMEM; goto out; } + scoutfs_key_copy(pos, key); + spin_lock_irqsave(&cac->lock, flags); for(;;) { - /* see if we have a usable item in cache and before last */ - cached = check_range(sb, &cac->ranges, key, range_end); + /* see if we have cache coverage of our iterator pos */ + cached = check_range(sb, &cac->ranges, pos, range_end); - if (cached && (item = item_for_next(&cac->items, key, - range_end, last))) { - scoutfs_key_copy(key, item->key); - if (val) { - item_referenced(cac, item); - ret = scoutfs_kvec_memcpy(val, item->val); - } else { - ret = 0; - } - break; - } + trace_scoutfs_item_next_range_check(sb, !!cached, key, + pos, last, end, range_end); if (!cached) { - /* missing cache starts at key */ - scoutfs_key_copy(read_start, key); + /* populate missing cached range starting at pos */ + spin_unlock_irqrestore(&cac->lock, flags); - } else if (scoutfs_key_compare(range_end, last) < 0) { - /* missing cache starts at range_end */ - scoutfs_key_copy(read_start, range_end); + ret = scoutfs_manifest_read_items(sb, pos, end); - } else { - /* no items and we have cache between key and last */ + spin_lock_irqsave(&cac->lock, flags); + if (ret) + break; + else + continue; + } + + /* see if there's an item in the cached range from pos */ + item = item_for_next(&cac->items, pos, range_end, last); + if (!item) { + if (scoutfs_key_compare(range_end, last) < 0) { + /* keep searching after empty cached range */ + scoutfs_key_copy(pos, range_end); + scoutfs_key_inc(pos); + continue; + } + + /* no item and cache covers last, done */ ret = -ENOENT; break; } - spin_unlock_irqrestore(&cac->lock, flags); - - ret = scoutfs_manifest_read_items(sb, read_start, end); - - spin_lock_irqsave(&cac->lock, flags); - if (ret) - break; + /* we have a next item inside the cached range, done */ + scoutfs_key_copy(key, item->key); + if (val) { + item_referenced(cac, item); + ret = scoutfs_kvec_memcpy(val, item->val); + } else { + ret = 0; + } + break; } spin_unlock_irqrestore(&cac->lock, flags); out: - scoutfs_key_free(sb, read_start); + scoutfs_key_free(sb, pos); scoutfs_key_free(sb, range_end); trace_printk("ret %d\n", ret); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a0c75dfc..40e757e2 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -484,6 +484,36 @@ DEFINE_EVENT(scoutfs_net_class, scoutfs_client_recv_reply, TP_ARGS(sb, name, peer, nh) ); +TRACE_EVENT(scoutfs_item_next_range_check, + TP_PROTO(struct super_block *sb, int cached, + struct scoutfs_key_buf *key, struct scoutfs_key_buf *pos, + struct scoutfs_key_buf *last, struct scoutfs_key_buf *end, + struct scoutfs_key_buf *range_end), + TP_ARGS(sb, cached, key, pos, last, end, range_end), + TP_STRUCT__entry( + __field(void *, sb) + __field(int, cached) + __dynamic_array(char, key, scoutfs_key_str(NULL, key)) + __dynamic_array(char, pos, scoutfs_key_str(NULL, pos)) + __dynamic_array(char, last, scoutfs_key_str(NULL, last)) + __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + __dynamic_array(char, range_end, + scoutfs_key_str(NULL, range_end)) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->cached = cached; + scoutfs_key_str(__get_dynamic_array(key), key); + scoutfs_key_str(__get_dynamic_array(pos), pos); + scoutfs_key_str(__get_dynamic_array(last), last); + scoutfs_key_str(__get_dynamic_array(end), end); + scoutfs_key_str(__get_dynamic_array(range_end), range_end); + ), + TP_printk("sb %p cached %d key %s pos %s last %s end %s range_end %s", + __entry->sb, __entry->cached, __get_str(key), __get_str(pos), + __get_str(last), __get_str(end), __get_str(range_end)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From e7b5cd4c66c4b3f1aec8cacd4360da75f2b141f8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Sep 2017 16:14:31 -0700 Subject: [PATCH 417/920] scoutfs: limit get_block bh use We were seeing __block_write_begin spin when staging writes were called after a read of an offline region saw an error. It turns out that the way __block_write_begin iterates through buffer heads on a page will livelock if b_size is 0. Our get_block was clearing b_blocknr and b_size before doing anything. It'd set them when it allocated blocks or found existing mapped blocks. But it'd leave them 0 on an error and trigger this hang. So we'll back off and only do the same things to the result bh that ext2/3 do, presumably that's what's actually supported. We only set mapped, set or clear new, and set b_size to less than the input b_size. While we're at it we remove a totally bogus extent flag check that's done before seeing if the next extent we found even intersects with the logical block that we're searching for. The extra test is performed again correctly inside the check for the extents overlapping. It is an artifact from the days when the "extents" were a single block and didn't need to check for overlaps. Signed-off-by: Zach Brown --- kmod/src/data.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index b6f2d641..aff8d7fe 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -995,9 +995,6 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, u64 off; int ret; - bh->b_blocknr = 0; - bh->b_size = 0; - ext.blk_off = iblock; ext.blocks = 1; ext.blkno = 0; @@ -1026,11 +1023,6 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, trace_printk("found nei "EXTF"\n", EXTA(&ext)); } - if ((ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) && !si->staging) { - ret = -EINVAL; - goto out; - } - /* use the extent if it intersects */ if (iblock >= ext.blk_off && iblock < (ext.blk_off + ext.blocks)) { @@ -1045,8 +1037,9 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, /* found online extent */ off = iblock - ext.blk_off; map_bh(bh, inode->i_sb, ext.blkno + off); - bh->b_size = min_t(u64, SIZE_MAX, + bh->b_size = min_t(u64, bh->b_size, (ext.blocks - off) << SCOUTFS_BLOCK_SHIFT); + clear_buffer_new(bh); } } From 79110a74ebc8452b1e5b70455c2ce9a95db0b572 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 6 Sep 2017 10:19:15 -0700 Subject: [PATCH 418/920] scoutfs: prevent partial block stage, except final The staging ioctl is just a thin wrapper around writing. If we allowed partial-block staging then the write would zero a newly allocated block and only stage in the partial region of the block, leaving zeros in the file that didn't exist before. We prevent staging when the starting offset isn't block aligned. We prevent staging when the final offset isn't block aligned unless it matches the size because the stage ends in the final partial block of the file. This is verified by an xfstest (scoutfs/003) that is in flight. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 2013b31c..80abaf4a 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -412,14 +412,21 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) struct kiocb kiocb; struct iovec iov; size_t written; + loff_t end_size; + loff_t isize; loff_t pos; int ret; if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; - if (args.count < 0 || (args.offset + args.count < args.offset)) + end_size = args.offset + args.count; + + /* verify arg constraints that aren't dependent on file */ + if (args.count < 0 || (end_size < args.offset) || + args.offset & SCOUTFS_BLOCK_MASK) return -EINVAL; + if (args.count == 0) return 0; @@ -437,11 +444,14 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) mutex_lock(&inode->i_mutex); + isize = i_size_read(inode); + if (!S_ISREG(inode->i_mode) || !(file->f_mode & FMODE_WRITE) || (file->f_flags & (O_APPEND | O_DIRECT | O_DSYNC)) || IS_SYNC(file->f_mapping->host) || - (args.offset + args.count > i_size_read(inode))) { + (end_size > isize) || + ((end_size & SCOUTFS_BLOCK_MASK) && (end_size != isize))) { ret = -EINVAL; goto out; } From 4bb5cadaf0b5dd8d5669a9339a5ccd572b25e9c0 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 6 Sep 2017 16:33:57 -0500 Subject: [PATCH 419/920] scoutfs: remove dead code in lock.[ch] With the dlmglue transition over, we can finally remove some of the no longer used portions of lock.[ch]. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 189 --------------------------------------- kmod/src/lock.h | 9 -- kmod/src/scoutfs_trace.h | 13 +-- kmod/src/super.c | 1 - 4 files changed, 2 insertions(+), 210 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c9fc20b4..4636b475 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -43,14 +43,10 @@ struct lock_info { dlmglue_ctxt dlmglue; bool dlmglue_online; char ls_name[DLM_LOCKSPACE_LEN]; - bool shutdown; - struct list_head id_head; spinlock_t lock; unsigned int seq_cnt; - wait_queue_head_t waitq; struct rb_root lock_tree; - struct workqueue_struct *downconvert_wq; struct shrinker shrinker; struct list_head lru_list; unsigned long long lru_nr; @@ -59,8 +55,6 @@ struct lock_info { #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info -static void scoutfs_downconvert_func(struct work_struct *work); - /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -127,9 +121,6 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) BUG_ON(!lock->refcnt); refs = --lock->refcnt; if (!refs) { - BUG_ON(lock->holders); - /* can't be (even racy) busy without refs */ - BUG_ON(work_busy(&lock->dc_work)); rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); spin_unlock(&linfo->lock); @@ -190,7 +181,6 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, { DECLARE_LOCK_INFO(sb, linfo); -// struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock; if (WARN_ON_ONCE(!!start != !!end)) @@ -212,8 +202,6 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, RB_CLEAR_NODE(&lock->node); lock->sb = sb; lock->lock_name = *lock_name; - lock->mode = DLM_LOCK_IV; - INIT_WORK(&lock->dc_work, scoutfs_downconvert_func); INIT_LIST_HEAD(&lock->lru_entry); ocfs2_lock_res_init_once(&lock->lockres); BUG_ON(sizeof(struct scoutfs_lock_name) >= OCFS2_LOCK_ID_MAX_LEN); @@ -317,9 +305,7 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) if (nr-- == 0) break; - WARN_ON(lock->holders); WARN_ON(lock->refcnt != 1); - WARN_ON(lock->flags & SCOUTFS_LOCK_QUEUED); rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); @@ -351,18 +337,6 @@ static void free_lock_tree(struct super_block *sb) } } -static int lock_granted(struct lock_info *linfo, struct scoutfs_lock *lock, - int mode) -{ - int ret; - - spin_lock(&linfo->lock); - ret = !!(mode == lock->mode); - spin_unlock(&linfo->lock); - - return ret; -} - /* * Acquire a coherent lock on the given range of keys. While the lock * is held other lockers are serialized. Cache coherency is maintained @@ -405,58 +379,6 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, *ret_lock = lock; return 0; -#if 0 -check_lock_state: - spin_lock(&linfo->lock); - if (linfo->shutdown) { - spin_unlock(&linfo->lock); - put_scoutfs_lock(sb, lock); - return -ESHUTDOWN; - } - - if (lock->flags & SCOUTFS_LOCK_BLOCKING) { - spin_unlock(&linfo->lock); - wait_event(linfo->waitq, !lock_blocking(linfo, lock)); - goto check_lock_state; - } - - if (lock->mode > DLM_LOCK_IV) { - if (lock->mode < mode) { - /* - * We already have the lock but at a mode which is not - * compatible with what the caller wants. Set the lock - * blocking to let the downconvert thread do it's work - * so we can reacquire at the correct mode. - */ - set_lock_blocking(linfo, lock); - spin_unlock(&linfo->lock); - goto check_lock_state; - } - lock->holders++; - spin_unlock(&linfo->lock); - goto out; - } - - lock->rqmode = mode; - lock->holders++; - spin_unlock(&linfo->lock); - - ret = dlm_lock(linfo->dlmglue.cconn->cc_lockspace, mode, &lock->lksb, - DLM_LKF_NOORDER, &lock->lock_name, - sizeof(struct scoutfs_lock_name), - 0, scoutfs_ast, lock, scoutfs_bast); - if (ret) { - scoutfs_err(sb, "Error %d locking "LN_FMT, ret, - LN_ARG(&lock->lock_name)); - put_scoutfs_lock(sb, lock); - return ret; - } - - wait_event(linfo->waitq, lock_granted(linfo, lock, mode)); -out: - *ret_lock = lock; - return 0; -#endif } u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock) @@ -705,86 +627,6 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, ocfs2_cluster_unlock(&linfo->dlmglue, &lock->lockres, level); -#if 0 - spin_lock(&linfo->lock); - lock->holders--; - if (lock->holders == 0 && (lock->flags & SCOUTFS_LOCK_BLOCKING)) - queue_blocking_work(linfo, lock); - spin_unlock(&linfo->lock); -#endif - put_scoutfs_lock(sb, lock); -} - -static void unlock_range(struct super_block *sb, struct scoutfs_lock *lock) -{ - DECLARE_LOCK_INFO(sb, linfo); - int ret; - - trace_scoutfs_unlock(sb, lock); - - BUG_ON(!lock->sequence); - - spin_lock(&linfo->lock); - lock->rqmode = DLM_LOCK_IV; - spin_unlock(&linfo->lock); - ret = dlm_unlock(linfo->dlmglue.cconn->cc_lockspace, lock->lksb.sb_lkid, - 0, &lock->lksb, lock); - if (ret) { - scoutfs_err(sb, "Error %d unlocking "LN_FMT, ret, - LN_ARG(&lock->lock_name)); - goto out; - } - - wait_event(linfo->waitq, lock_granted(linfo, lock, DLM_LOCK_IV)); -out: - /* lock was removed from tree, wake up umount process */ - wake_up(&linfo->waitq); -} - -static void scoutfs_downconvert_func(struct work_struct *work) -{ - struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - dc_work); - struct super_block *sb = lock->sb; - DECLARE_LOCK_INFO(sb, linfo); - - trace_scoutfs_downconvert_func(sb, lock); - - spin_lock(&linfo->lock); - lock->flags &= ~SCOUTFS_LOCK_QUEUED; - if (lock->holders) - goto out; /* scoutfs_unlock_range will requeue for us */ - - spin_unlock(&linfo->lock); - - WARN_ON_ONCE(lock->holders); - WARN_ON_ONCE(lock->refcnt == 0); - /* - * Use write mode to invalidate all since we are completely - * dropping the lock. Once we are dowconverting, we can - * invalidate based on what level we're downconverting to (PR, - * NL). - */ - invalidate_caches(sb, DLM_LOCK_EX, lock); - unlock_range(sb, lock); - - spin_lock(&linfo->lock); - /* Check whether we can add the lock to the LRU list: - * - * First, check mode to be sure that the lock wasn't reacquired - * while we slept in unlock_range(). - * - * Next, check refs. refcnt == 1 means the only holder is the - * lock tree so in particular we have nobody in - * scoutfs_lock_range concurrently trying to acquire a lock. - */ - if (lock->mode == DLM_LOCK_IV && lock->refcnt == 1 && - list_empty(&lock->lru_entry)) { - list_add_tail(&lock->lru_entry, &linfo->lru_list); - linfo->lru_nr++; - } -out: - spin_unlock(&linfo->lock); put_scoutfs_lock(sb, lock); } @@ -807,14 +649,11 @@ static int init_lock_info(struct super_block *sb) goto out; spin_lock_init(&linfo->lock); - init_waitqueue_head(&linfo->waitq); INIT_LIST_HEAD(&linfo->lru_list); linfo->shrinker.shrink = shrink_lock_tree; linfo->shrinker.seeks = DEFAULT_SEEKS; register_shrinker(&linfo->shrinker); linfo->sb = sb; - linfo->shutdown = false; - INIT_LIST_HEAD(&linfo->id_head); snprintf(linfo->ls_name, DLM_LOCKSPACE_LEN, "%llx", le64_to_cpu(sbi->super.hdr.fsid)); @@ -830,25 +669,6 @@ out: return 0; } -/* - * Cause all lock attempts from our super to fail, waking anyone who is - * currently blocked attempting to lock. Now that locks can't block we - * can easily tear down subsystems that use locking before freeing lock - * infrastructure. - */ -void scoutfs_lock_shutdown(struct super_block *sb) -{ - DECLARE_LOCK_INFO(sb, linfo); - - if (linfo) { - spin_lock(&linfo->lock); - linfo->shutdown = true; - spin_unlock(&linfo->lock); - - wake_up(&linfo->waitq); - } -} - void scoutfs_lock_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -857,8 +677,6 @@ void scoutfs_lock_destroy(struct super_block *sb) if (linfo) { free_lock_tree(sb); /* Do this before uninitializing the dlm. */ - if (linfo->downconvert_wq) - destroy_workqueue(linfo->downconvert_wq); unregister_shrinker(&linfo->shrinker); if (linfo->dlmglue_online) { ocfs2_dlm_shutdown(&linfo->dlmglue, 0); @@ -885,13 +703,6 @@ int scoutfs_lock_setup(struct super_block *sb) return ret; linfo = sbi->lock_info; - linfo->downconvert_wq = alloc_workqueue("scoutfs_dc", - WQ_UNBOUND|WQ_HIGHPRI, 0); - if (!linfo->downconvert_wq) { - ret = -ENOMEM; - goto out; - } - ret = ocfs2_dlm_init(&linfo->dlmglue, "null", sbi->opts.cluster_name, linfo->ls_name, sbi->debug_root); if (ret) diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 68ba7329..6c5c1aee 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -5,9 +5,6 @@ #include "key.h" #include "dlmglue.h" -#define SCOUTFS_LOCK_BLOCKING 0x01 /* Blocking another lock request */ -#define SCOUTFS_LOCK_QUEUED 0x02 /* Put on drop workqueue */ - #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ #define SCOUTFS_LKF_TRYLOCK 0x02 /* EAGAIN if contention */ @@ -16,16 +13,11 @@ struct scoutfs_lock { struct scoutfs_lock_name lock_name; struct scoutfs_key_buf *start; struct scoutfs_key_buf *end; - int mode; - int rqmode; struct dlm_lksb lksb; unsigned int sequence; /* for debugging and sanity checks */ struct rb_node node; struct list_head lru_entry; unsigned int refcnt; - unsigned int holders; /* Tracks active users of this lock */ - unsigned int flags; - struct work_struct dc_work; struct ocfs2_lock_res lockres; }; @@ -48,7 +40,6 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); int scoutfs_lock_setup(struct super_block *sb); -void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 40e757e2..2147ce2a 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -304,12 +304,8 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(u8, name_type) __field(u64, name_first) __field(u64, name_second) - __field(int, mode) - __field(int, rqmode) __field(unsigned int, seq) - __field(unsigned int, flags) __field(unsigned int, refcnt) - __field(unsigned int, holders) ), TP_fast_assign( __entry->name_scope = lck->lock_name.scope; @@ -317,18 +313,13 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->name_type = lck->lock_name.type; __entry->name_first = le64_to_cpu(lck->lock_name.first); __entry->name_second = le64_to_cpu(lck->lock_name.second); - __entry->mode = lck->mode; - __entry->rqmode = lck->rqmode; __entry->seq = lck->sequence; - __entry->flags = lck->flags; __entry->refcnt = lck->refcnt; - __entry->holders = lck->holders; ), - TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d holders %d mode %s rqmode %s flags 0x%x", + TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d", __entry->name_scope, __entry->name_zone, __entry->name_type, __entry->name_first, __entry->name_second, __entry->seq, - __entry->refcnt, __entry->holders, lock_mode(__entry->mode), - lock_mode(__entry->rqmode), __entry->flags) + __entry->refcnt) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, diff --git a/kmod/src/super.c b/kmod/src/super.c index 29d29369..8dd2ebd5 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -336,7 +336,6 @@ static void scoutfs_kill_sb(struct super_block *sb) if (sb->s_root) { sync_filesystem(sb); - scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); } From 9461104f8ef516628da39146f215256253776a4e Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 6 Sep 2017 18:13:54 -0500 Subject: [PATCH 420/920] scoutfs: Use LRU for locks The move to dlmglue necessitating removing the old lru code. That's fine, it didn't work anyway. We can't drop locks in the shrinker directly, so instead we have the shrinker put them on a workqueue where they are dropped. The rules for the LRU are simple. Locks get a users count. Any process that holds the lock or is in the process of acquiring the lock increments this count. When unlock is called, the count is decremented. We can use the value of this count to manage the LRU - scoutfs_unlock puts locks on the LRU when the count reaches zero, lock_name_keys() always takes them off. If the lock is selected for reclaim, callers wanting to use the lock will need to wait. We acheive this with a pair of flags. SCOUTFS_LOCK_RECLAIM is used to indicate the the lock is now queued for reclaim. Once the is ready to be destroyed, we set SCOUTFS_LOCK_DROPPED flag, telling callers to put the lock and retry their rbtree search. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 97 +++++++++++++++++++++++++++++++++------- kmod/src/lock.h | 12 ++++- kmod/src/scoutfs_trace.h | 8 ++-- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 4636b475..b560d3b3 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -50,11 +50,14 @@ struct lock_info { struct shrinker shrinker; struct list_head lru_list; unsigned long long lru_nr; + struct workqueue_struct *lock_reclaim_wq; }; #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info +static void scoutfs_lock_reclaim(struct work_struct *work); + /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -104,7 +107,6 @@ static void free_scoutfs_lock(struct scoutfs_lock *lock) if (lock) { linfo = SCOUTFS_SB(lock->sb)->lock_info; - ocfs2_simple_drop_lockres(&linfo->dlmglue, &lock->lockres); scoutfs_key_free(lock->sb, lock->start); scoutfs_key_free(lock->sb, lock->end); kfree(lock); @@ -124,6 +126,8 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); spin_unlock(&linfo->lock); + ocfs2_simple_drop_lockres(&linfo->dlmglue, + &lock->lockres); free_scoutfs_lock(lock); return; } @@ -131,6 +135,19 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) } } +static void dec_lock_users(struct scoutfs_lock *lock) +{ + DECLARE_LOCK_INFO(lock->sb, linfo); + + spin_lock(&linfo->lock); + lock->users--; + if (list_empty(&lock->lru_entry) && lock->users == 0) { + list_add_tail(&lock->lru_entry, &linfo->lru_list); + linfo->lru_nr++; + } + spin_unlock(&linfo->lock); +} + static struct ocfs2_super *get_ino_lock_osb(struct ocfs2_lock_res *lockres) { struct scoutfs_lock *lock = lockres->l_priv; @@ -209,6 +226,8 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, memcpy(&lock->lockres.l_name[0], &lock->lock_name, sizeof(struct scoutfs_lock_name)); ocfs2_lock_res_init_common(&linfo->dlmglue, &lock->lockres, type, lock); + INIT_WORK(&lock->reclaim_work, scoutfs_lock_reclaim); + init_waitqueue_head(&lock->waitq); return lock; } @@ -276,16 +295,47 @@ search: rb_insert_color(&found->node, &linfo->lock_tree); } found->refcnt++; + if (test_bit(SCOUTFS_LOCK_RECLAIM, &found->flags)) { + spin_unlock(&linfo->lock); + wait_event(found->waitq, + test_bit(SCOUTFS_LOCK_DROPPED, &found->flags)); + put_scoutfs_lock(sb, found); + goto search; + } + if (!list_empty(&found->lru_entry)) { list_del_init(&found->lru_entry); linfo->lru_nr--; } + found->users++; spin_unlock(&linfo->lock); kfree(new); return found; } +static void scoutfs_lock_reclaim(struct work_struct *work) +{ + struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, + reclaim_work); + struct lock_info *linfo = SCOUTFS_SB(lock->sb)->lock_info; + + trace_scoutfs_lock_reclaim(lock->sb, lock); + + /* + * Drop the last ref on our lock here, allowing us to clean up + * the dlm lock. We might race with another process in + * find_alloc_scoutfs_lock(), hence the dropped flag telling + * those processes to go ahead and drop the lock ref as well. + */ + BUG_ON(lock->users); + + set_bit(SCOUTFS_LOCK_DROPPED, &lock->flags); + wake_up(&lock->waitq); + + put_scoutfs_lock(linfo->sb, lock); +} + static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) { struct lock_info *linfo = container_of(shrink, struct lock_info, @@ -294,7 +344,6 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) struct scoutfs_lock *tmp; unsigned long flags; unsigned long nr; - LIST_HEAD(list); nr = sc->nr_to_scan; if (!nr) @@ -305,20 +354,18 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) if (nr-- == 0) break; - WARN_ON(lock->refcnt != 1); + trace_shrink_lock_tree(linfo->sb, lock); - rb_erase(&lock->node, &linfo->lock_tree); - list_del(&lock->lru_entry); - list_add_tail(&lock->lru_entry, &list); + WARN_ON(lock->users); + + set_bit(SCOUTFS_LOCK_RECLAIM, &lock->flags); + list_del_init(&lock->lru_entry); linfo->lru_nr--; + + queue_work(linfo->lock_reclaim_wq, &lock->reclaim_work); } spin_unlock_irqrestore(&linfo->lock, flags); - list_for_each_entry_safe(lock, tmp, &list, lru_entry) { - trace_shrink_lock_tree(linfo->sb, lock); - list_del(&lock->lru_entry); - free_scoutfs_lock(lock); - } out: return min_t(unsigned long, linfo->lru_nr, INT_MAX); } @@ -374,10 +421,11 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, ret = ocfs2_cluster_lock(&linfo->dlmglue, &lock->lockres, mode, lkm_flags, 0); - if (ret) - return ret; - - *ret_lock = lock; + if (ret) { + dec_lock_users(lock); + put_scoutfs_lock(sb, lock); + } else + *ret_lock = lock; return 0; } @@ -627,6 +675,8 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, ocfs2_cluster_unlock(&linfo->dlmglue, &lock->lockres, level); + dec_lock_users(lock); + put_scoutfs_lock(sb, lock); } @@ -675,9 +725,15 @@ void scoutfs_lock_destroy(struct super_block *sb) DECLARE_LOCK_INFO(sb, linfo); if (linfo) { - free_lock_tree(sb); /* Do this before uninitializing the dlm. */ - unregister_shrinker(&linfo->shrinker); + if (linfo->lock_reclaim_wq) + destroy_workqueue(linfo->lock_reclaim_wq); + /* + * Do this before uninitializing the dlm and after + * draining the reclaim workqueue. + */ + free_lock_tree(sb); + if (linfo->dlmglue_online) { ocfs2_dlm_shutdown(&linfo->dlmglue, 0); ocfs2_uninit_super(&linfo->dlmglue); @@ -703,6 +759,13 @@ int scoutfs_lock_setup(struct super_block *sb) return ret; linfo = sbi->lock_info; + linfo->lock_reclaim_wq = alloc_workqueue("scoutfs_reclaim", + WQ_UNBOUND|WQ_HIGHPRI, 0); + if (!linfo->lock_reclaim_wq) { + ret = -ENOMEM; + goto out; + } + ret = ocfs2_dlm_init(&linfo->dlmglue, "null", sbi->opts.cluster_name, linfo->ls_name, sbi->debug_root); if (ret) diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 6c5c1aee..65c0a4f2 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -8,6 +8,12 @@ #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ #define SCOUTFS_LKF_TRYLOCK 0x02 /* EAGAIN if contention */ +/* flags for scoutfs_lock->flags */ +enum { + SCOUTFS_LOCK_RECLAIM = 0, /* lock is queued for reclaim */ + SCOUTFS_LOCK_DROPPED, /* lock is going away, drop reference */ +}; + struct scoutfs_lock { struct super_block *sb; struct scoutfs_lock_name lock_name; @@ -16,9 +22,13 @@ struct scoutfs_lock { struct dlm_lksb lksb; unsigned int sequence; /* for debugging and sanity checks */ struct rb_node node; - struct list_head lru_entry; unsigned int refcnt; struct ocfs2_lock_res lockres; + struct list_head lru_entry; + struct work_struct reclaim_work; + unsigned int users; /* Tracks active users of this lock */ + unsigned long flags; + wait_queue_head_t waitq; }; u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 2147ce2a..b30504c9 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -306,6 +306,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(u64, name_second) __field(unsigned int, seq) __field(unsigned int, refcnt) + __field(unsigned int, users) ), TP_fast_assign( __entry->name_scope = lck->lock_name.scope; @@ -315,11 +316,12 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->name_second = le64_to_cpu(lck->lock_name.second); __entry->seq = lck->sequence; __entry->refcnt = lck->refcnt; + __entry->users = lck->users; ), - TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d", + TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d users %d", __entry->name_scope, __entry->name_zone, __entry->name_type, __entry->name_first, __entry->name_second, __entry->seq, - __entry->refcnt) + __entry->refcnt, __entry->users) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, @@ -342,7 +344,7 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_bast, TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_downconvert_func, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_reclaim, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); From 2d6d113e03820eca8bbed4f3c44955919dee9768 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 6 Sep 2017 14:34:04 -0700 Subject: [PATCH 421/920] scoutfs: continue index walk after lock We saw inode index queries spinning. They were finding no cached entries in their locked region but the next key in the segments was in the region. This can happen if an item has been deleted in the current transaction. The query won't walk up in to the new dirty seq but it will try to walk the old seq. The item will still be in the segments but won't be visible to item_next because it's marked deleted. The query will spin finding the next stale key to read from and finding it missing in the cache. This is fixed by taking the current coherent cache at its word. When it tells us there's no entries we advance the key to check the manifest for to past the locked region. In this case it'll skip past the cached delete item and look for the next key in the segments. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 80abaf4a..31c07ef9 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -131,7 +131,18 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (ret == -ENOENT) { + /* done if lock covers last iteration key */ + if (scoutfs_key_compare(&last_key, lock->end) <= 0) { + ret = 0; + break; + } + + /* continue iterating after locked empty region */ + scoutfs_key_copy(&key, lock->end); + scoutfs_key_inc_cur_len(&key); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); + /* * XXX This will miss dirty items. We'd need to * force writeouts of dirty items in our From f276771d8cb285988356e1077f1107f7c5fd9a45 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 7 Sep 2017 17:12:04 -0500 Subject: [PATCH 422/920] scoutfs: we need to uninitialize the dlmglue lockres The dlmglue lockres gets put on a debugging list at initialization time, and taken off the list at uninit time. We were missing the uninit portion of this cycle, causing some list debugging warnings. Call ocfs2_lock_res_free() in free_scoutfs_lock(). In addition, we had a raw kfree() of a scoutfs lock in find_alloc_scoutfs_lock() which also needed to be replaced. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index b560d3b3..8013c8d1 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -107,6 +107,7 @@ static void free_scoutfs_lock(struct scoutfs_lock *lock) if (lock) { linfo = SCOUTFS_SB(lock->sb)->lock_info; + ocfs2_lock_res_free(&lock->lockres); scoutfs_key_free(lock->sb, lock->start); scoutfs_key_free(lock->sb, lock->end); kfree(lock); @@ -310,7 +311,7 @@ search: found->users++; spin_unlock(&linfo->lock); - kfree(new); + free_scoutfs_lock(new); return found; } From b1fff0997e7a9a9508e9ffd4824b8834b85fd5f2 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 8 Sep 2017 09:22:13 -0500 Subject: [PATCH 423/920] scoutfs: dlmglue should initialize res->l_debug_list We're missing initialization of this field. It should never cause a problem today because we always do a list add immediately afterwards but let's be extra careful here and initialize it just in case. We also add a sanity check in ocfs2_add_lockres_tracking() that the lockres hasn't already been put on the debug list. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 334a3e4f..49fc1e56 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -456,6 +456,7 @@ static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, mlog(0, "Add tracking for lockres %s\n", res->l_name); spin_lock(&ocfs2_dlm_tracking_lock); + BUG_ON(!list_empty(&res->l_debug_list)); list_add(&res->l_debug_list, &dlm_debug->d_lockres_tracking); spin_unlock(&ocfs2_dlm_tracking_lock); } @@ -570,6 +571,7 @@ void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) INIT_LIST_HEAD(&res->l_blocked_list); INIT_LIST_HEAD(&res->l_mask_waiters); INIT_LIST_HEAD(&res->l_holders); + INIT_LIST_HEAD(&res->l_debug_list); } #if 0 From fbfbe910aae6313926975156f6fdd68738b161f2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 10 Sep 2017 15:05:23 -0700 Subject: [PATCH 424/920] scoutfs: return error from lock_name_keys xfstests generic/028 was crashing dereferencing NULL locks. It'd hit either rename trying to refresh an inode with a NULL lock or lookup trying to pass a NULL lock's end to item lookup. The addition of the lock LRU fixed a bug in lock_name_keys() where it wouldn't drop a lock when _cluster_lock() returned an error. But it always returned 0 instead of returning the error. Returning 0 without setting the lock caused the callers to deref their NULL locks. We also forcefully NULL the lock at the start of the function. It was lucky that callers had already NULLed their locks. If they hadn't they would have been following random on-stack memory and it might have been harder to debug. Signed-off-by: Zach Brown --- kmod/src/lock.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 8013c8d1..459f6c95 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -406,6 +406,8 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, int lkm_flags; int ret; + *ret_lock = NULL; + if (WARN_ON_ONCE(!(flags & SCOUTFS_LKF_TRYLOCK) && scoutfs_trans_held())) return -EINVAL; @@ -425,9 +427,11 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, if (ret) { dec_lock_users(lock); put_scoutfs_lock(sb, lock); - } else + } else { *ret_lock = lock; - return 0; + } + + return ret; } u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock) From 5fa97018e50723e1e3e5f3845e7b789217d4fc9a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 11 Sep 2017 16:38:28 -0500 Subject: [PATCH 425/920] scoutfs: get rid of dlmglues ocfs2_uninit_super All this does is uninitialize the dlmglue debug infrastructure, however we already do that in ocfs2_dlm_shutdown(), resulting in some double frees on unmount. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 5 ----- kmod/src/dlmglue.h | 2 -- kmod/src/lock.c | 4 +--- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 49fc1e56..f1c206aa 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -4334,8 +4334,3 @@ int ocfs2_init_super(struct ocfs2_super *osb, int flags) return 0; } - -void ocfs2_uninit_super(struct ocfs2_super *osb) -{ - ocfs2_dlm_shutdown_debug(osb); -} diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 61b4af76..0d852743 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -309,8 +309,6 @@ void ocfs2_cluster_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level); int ocfs2_init_super(struct ocfs2_super *osb, int flags); -void ocfs2_uninit_super(struct ocfs2_super *osb); - int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, char *cluster_name, char *ls_name, struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 459f6c95..d2023596 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -739,10 +739,8 @@ void scoutfs_lock_destroy(struct super_block *sb) */ free_lock_tree(sb); - if (linfo->dlmglue_online) { + if (linfo->dlmglue_online) ocfs2_dlm_shutdown(&linfo->dlmglue, 0); - ocfs2_uninit_super(&linfo->dlmglue); - } sbi->lock_info = NULL; From ba40899e84abdd1743d3f45b2200db641ae8acb8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 Sep 2017 14:09:16 -0700 Subject: [PATCH 426/920] scoutfs: remove scoutfs_trans_wake_holders() This was used by compaction to wake local holders who were waiting for compaction to free up level 0 segments for them to enter the transaction. Throttling level 0 segment writes works differently now and doesn't involve blocking trans holders. Signed-off-by: Zach Brown --- kmod/src/trans.c | 12 ------------ kmod/src/trans.h | 1 - 2 files changed, 13 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index ca05240f..d00d36c0 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -451,18 +451,6 @@ void scoutfs_release_trans(struct super_block *sb) wake_up(&sbi->trans_hold_wq); } -/* - * This is called to wake people waiting on holders when the conditions - * that they're waiting on change: levels being full, dirty count falling - * under a segment, or holders falling to 0. - */ -void scoutfs_trans_wake_holders(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - wake_up(&sbi->trans_hold_wq); -} - int scoutfs_setup_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 49db305d..d3c6f326 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -12,7 +12,6 @@ void scoutfs_trans_restart_sync_deadline(struct super_block *sb); int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt); bool scoutfs_trans_held(void); void scoutfs_release_trans(struct super_block *sb); -void scoutfs_trans_wake_holders(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, signed keys, signed vals); From e165d89f7f54ac577687dfed38f2a9e7537738e3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 Sep 2017 19:18:49 -0700 Subject: [PATCH 427/920] scoutfs: warn on invalid item counts We had a bug where a caller was slowly increasing their item count for every transaction they attempted in a loop. Eventually the item count grew to be too large to fit in a segment and they slept indefinitely. Let's warn on invalid and impossibly large item counts as we enter transactions. Signed-off-by: Zach Brown --- kmod/src/trans.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index d00d36c0..c5a9189f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -346,6 +346,15 @@ int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) struct scoutfs_reservation *rsv; int ret; + /* + * Caller shouldn't provide garbage counts, nor counts that + * can't fit in segments by themselves. + */ + if (WARN_ON_ONCE(cnt->items <= 0 || cnt->keys < 0 || cnt->vals < 0) || + WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt->items, cnt->keys, + cnt->vals))) + return -EINVAL; + if (current == sbi->trans_task) return 0; From f0a7c4f29448e2cfce9d94c573a37d6112250c45 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 Sep 2017 21:49:23 -0700 Subject: [PATCH 428/920] scoutfs: make trans item count const rhs The item count estimate functions didn't obviously differentiate between adding to a count and resetting it. Most callers initialized the count struct to 0 on the stack, incremented their estimate once, and passed it in. The problem is that those same functions that increment once in callers are also used in other estimates to build counts based on multiple operations. This tripped up the data truncate path. It looped and kept incrementing its count while truncating a file with lots of extents until the count got so large that it didn't fit in a segment by itself and blocked forever. This cleans up the item count code so that it's much harder to get wrong. We differentiate between the SIC_*() high level count estimates that are meant to be passed in to _hold_trans(), and the internal __count_*() functions which are used to add up the item counts that make up an aggregate operation. With this fix the only way to use the count in extent truncation is to correctly reset it for the item count for each transacation. Signed-off-by: Zach Brown --- kmod/src/count.h | 184 +++++++++++++++++++++++++++++++---------------- kmod/src/data.c | 8 +-- kmod/src/dir.c | 26 +++---- kmod/src/inode.c | 4 +- kmod/src/trans.c | 13 ++-- kmod/src/trans.h | 3 +- kmod/src/xattr.c | 4 +- 7 files changed, 146 insertions(+), 96 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 19fc6606..568caed0 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -1,19 +1,34 @@ #ifndef _SCOUTFS_COUNT_H_ #define _SCOUTFS_COUNT_H_ +/* + * Our estimate of the space consumed while dirtying items isn't a + * single value. We're packing items into segments which have different + * overheads for items (header overhead), keys (block aligned), and + * values (can span blocks, not aligned). + * + * The estimate is still a read-only input to entering the transaction. + * We'd like to use it as a clean rhs arg to hold_trans. We define SIC_ + * functions which return the count struct. This lets us have a single + * arg and avoid bugs in initializing and passing in struct pointers + * from callers. The internal __count functions are used compose an + * estimate out of the sets of items it manipulates. We program in much + * clearer C instead of in the preprocessor. + * + * Compilers are able to collapse the inlines into constants for the + * constant estimates. + */ + struct scoutfs_item_count { signed items; signed keys; signed vals; }; -#define DECLARE_ITEM_COUNT(name) \ - struct scoutfs_item_count name = { 0, } - /* * Allocating an inode creates a new set of indexed items. */ -static inline void scoutfs_count_alloc_inode(struct scoutfs_item_count *cnt) +static inline void __count_alloc_inode(struct scoutfs_item_count *cnt) { const int nr_indices = SCOUTFS_INODE_INDEX_NR; @@ -27,7 +42,7 @@ static inline void scoutfs_count_alloc_inode(struct scoutfs_item_count *cnt) * Dirtying an inode dirties the inode item and can delete and create * the full set of indexed items. */ -static inline void scoutfs_count_dirty_inode(struct scoutfs_item_count *cnt) +static inline void __count_dirty_inode(struct scoutfs_item_count *cnt) { const int nr_indices = 2 * SCOUTFS_INODE_INDEX_NR; @@ -37,11 +52,29 @@ static inline void scoutfs_count_dirty_inode(struct scoutfs_item_count *cnt) cnt->vals += sizeof(struct scoutfs_inode); } +static inline const struct scoutfs_item_count SIC_ALLOC_INODE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_alloc_inode(&cnt); + + return cnt; +} + +static inline const struct scoutfs_item_count SIC_DIRTY_INODE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirty_inode(&cnt); + + return cnt; +} + /* * Adding a dirent adds the entry key, readdir key, and backref. */ -static inline void scoutfs_count_dirents(struct scoutfs_item_count *cnt, - unsigned name_len) +static inline void __count_dirents(struct scoutfs_item_count *cnt, + unsigned name_len) { cnt->items += 3; @@ -51,8 +84,8 @@ static inline void scoutfs_count_dirents(struct scoutfs_item_count *cnt, cnt->vals += 2 * offsetof(struct scoutfs_dirent, name[name_len]); } -static inline void scoutfs_count_sym_target(struct scoutfs_item_count *cnt, - unsigned size) +static inline void __count_sym_target(struct scoutfs_item_count *cnt, + unsigned size) { unsigned nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); @@ -61,46 +94,65 @@ static inline void scoutfs_count_sym_target(struct scoutfs_item_count *cnt, cnt->vals += size; } -static inline void scoutfs_count_orphan(struct scoutfs_item_count *cnt) +static inline void __count_orphan(struct scoutfs_item_count *cnt) { cnt->items += 1; cnt->keys += sizeof(struct scoutfs_orphan_key); } -static inline void scoutfs_count_mknod(struct scoutfs_item_count *cnt, - unsigned name_len) +static inline void __count_mknod(struct scoutfs_item_count *cnt, + unsigned name_len) { - scoutfs_count_alloc_inode(cnt); - scoutfs_count_dirents(cnt, name_len); - scoutfs_count_dirty_inode(cnt); + __count_alloc_inode(cnt); + __count_dirents(cnt, name_len); + __count_dirty_inode(cnt); } -static inline void scoutfs_count_link(struct scoutfs_item_count *cnt, - unsigned name_len) +static inline const struct scoutfs_item_count SIC_MKNOD(unsigned name_len) { - scoutfs_count_dirents(cnt, name_len); - scoutfs_count_dirty_inode(cnt); - scoutfs_count_dirty_inode(cnt); + struct scoutfs_item_count cnt = {0,}; + + __count_mknod(&cnt, name_len); + + return cnt; +} + +static inline const struct scoutfs_item_count SIC_LINK(unsigned name_len) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirents(&cnt, name_len); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + + return cnt; } /* * Unlink can add orphan items. */ -static inline void scoutfs_count_unlink(struct scoutfs_item_count *cnt, - unsigned name_len) +static inline const struct scoutfs_item_count SIC_UNLINK(unsigned name_len) { - scoutfs_count_dirents(cnt, name_len); - scoutfs_count_dirty_inode(cnt); - scoutfs_count_dirty_inode(cnt); - scoutfs_count_orphan(cnt); + struct scoutfs_item_count cnt = {0,}; + + __count_dirents(&cnt, name_len); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_orphan(&cnt); + + return cnt; } -static inline void scoutfs_count_symlink(struct scoutfs_item_count *cnt, - unsigned name_len, unsigned size) +static inline const struct scoutfs_item_count SIC_SYMLINK(unsigned name_len, + unsigned size) { - scoutfs_count_mknod(cnt, name_len); - scoutfs_count_sym_target(cnt, size); + struct scoutfs_item_count cnt = {0,}; + + __count_mknod(&cnt, name_len); + __count_sym_target(&cnt, size); + + return cnt; } /* @@ -108,22 +160,26 @@ static inline void scoutfs_count_symlink(struct scoutfs_item_count *cnt, * unlinks an existing target. That'll be worse than the common case * by a few hundred bytes. */ -static inline void scoutfs_count_rename(struct scoutfs_item_count *cnt, - unsigned old_len, unsigned new_len) +static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, + unsigned new_len) { + struct scoutfs_item_count cnt = {0,}; + /* dirty dirs and inodes */ - scoutfs_count_dirty_inode(cnt); - scoutfs_count_dirty_inode(cnt); - scoutfs_count_dirty_inode(cnt); - scoutfs_count_dirty_inode(cnt); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); + __count_dirty_inode(&cnt); /* unlink old and new, link new */ - scoutfs_count_dirents(cnt, old_len); - scoutfs_count_dirents(cnt, new_len); - scoutfs_count_dirents(cnt, new_len); + __count_dirents(&cnt, old_len); + __count_dirents(&cnt, new_len); + __count_dirents(&cnt, new_len); /* orphan the existing target */ - scoutfs_count_orphan(cnt); + __count_orphan(&cnt); + + return cnt; } /* @@ -131,19 +187,22 @@ static inline void scoutfs_count_rename(struct scoutfs_item_count *cnt, * max name and length. Any existing items will be dirtied rather than * deleted so we won't have more items than a max xattr's worth. */ -static inline void scoutfs_count_xattr_set(struct scoutfs_item_count *cnt, - unsigned name_len, unsigned size) +static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned name_len, + unsigned size) { + struct scoutfs_item_count cnt = {0,}; unsigned parts = DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE); - scoutfs_count_dirty_inode(cnt); + __count_dirty_inode(&cnt); - cnt->items += parts; - cnt->keys += parts * (offsetof(struct scoutfs_xattr_key, + cnt.items += parts; + cnt.keys += parts * (offsetof(struct scoutfs_xattr_key, name[name_len]) + sizeof(struct scoutfs_xattr_key_footer)); - cnt->vals += parts * (sizeof(struct scoutfs_xattr_val_header) + + cnt.vals += parts * (sizeof(struct scoutfs_xattr_val_header) + SCOUTFS_XATTR_PART_SIZE); + + return cnt; } /* @@ -152,10 +211,9 @@ static inline void scoutfs_count_xattr_set(struct scoutfs_item_count *cnt, * third new extent and removal can delete an existing extent and create * two new remaining extents. */ -static inline void scoutfs_count_extents(struct scoutfs_item_count *cnt, - unsigned nr_mod, unsigned sz) +static inline void __count_extents(struct scoutfs_item_count *cnt, + unsigned nr_mod, unsigned sz) { - cnt->items += nr_mod * 3; cnt->keys += (nr_mod * 3) * sz; } @@ -165,29 +223,35 @@ static inline void scoutfs_count_extents(struct scoutfs_item_count *cnt, * alloc an block, delete an offline mapping, and insert the new allocated * mapping. */ -static inline void scoutfs_count_write_begin(struct scoutfs_item_count *cnt) +static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) { + struct scoutfs_item_count cnt = {0,}; + BUILD_BUG_ON(sizeof(struct scoutfs_free_extent_blkno_key) != sizeof(struct scoutfs_free_extent_blocks_key)); - scoutfs_count_dirty_inode(cnt); + __count_dirty_inode(&cnt); - scoutfs_count_extents(cnt, 2 * (SCOUTFS_BULK_ALLOC_COUNT + 1), - sizeof(struct scoutfs_free_extent_blkno_key)); - scoutfs_count_extents(cnt, 2, - sizeof(struct scoutfs_file_extent_key)); + __count_extents(&cnt, 2 * (SCOUTFS_BULK_ALLOC_COUNT + 1), + sizeof(struct scoutfs_free_extent_blkno_key)); + __count_extents(&cnt, 2, sizeof(struct scoutfs_file_extent_key)); + + return cnt; } /* * Truncating a block can free an allocated block, delete an online * mapping, and create an offline mapping. */ -static inline void scoutfs_count_trunc_block(struct scoutfs_item_count *cnt) +static inline const struct scoutfs_item_count SIC_TRUNC_BLOCK(void) { - scoutfs_count_extents(cnt, 2 * 1, - sizeof(struct scoutfs_free_extent_blkno_key)); - scoutfs_count_extents(cnt, 2, - sizeof(struct scoutfs_file_extent_key)); + struct scoutfs_item_count cnt = {0,}; + + __count_extents(&cnt, 2 * 1, + sizeof(struct scoutfs_free_extent_blkno_key)); + __count_extents(&cnt, 2, sizeof(struct scoutfs_file_extent_key)); + + return cnt; } #endif diff --git a/kmod/src/data.c b/kmod/src/data.c index aff8d7fe..b0c7c8e2 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -536,7 +536,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, struct native_extent ext; struct native_extent ofl; struct native_extent fr; - DECLARE_ITEM_COUNT(cnt); bool rem_fr = false; bool ins_ext = false; bool holding = false; @@ -602,8 +601,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, if (offline && (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) continue; - scoutfs_count_trunc_block(&cnt); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); if (ret) break; holding = true; @@ -1125,14 +1123,12 @@ static int scoutfs_write_begin(struct file *file, { struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; - DECLARE_ITEM_COUNT(cnt); int ret; trace_printk("ino %llu pos %llu len %u\n", scoutfs_ino(inode), (u64)pos, len); - scoutfs_count_write_begin(&cnt); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_WRITE_BEGIN()); if (ret) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 92a9fe2b..a73bad66 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -537,7 +537,7 @@ out: */ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev, - struct scoutfs_item_count *cnt, + const struct scoutfs_item_count cnt, struct scoutfs_lock **dir_lock, struct scoutfs_lock **inode_lock) { @@ -599,7 +599,6 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct super_block *sb = dir->i_sb; - DECLARE_ITEM_COUNT(cnt); struct inode *inode = NULL; struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; @@ -609,9 +608,9 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - scoutfs_count_mknod(&cnt, dentry->d_name.len); - inode = lock_hold_create(dir, dentry, mode, rdev, &cnt, + inode = lock_hold_create(dir, dentry, mode, rdev, + SIC_MKNOD(dentry->d_name.len), &dir_lock, &inode_lock); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -670,7 +669,6 @@ static int scoutfs_link(struct dentry *old_dentry, struct super_block *sb = dir->i_sb; struct scoutfs_lock *dir_lock; struct scoutfs_lock *inode_lock = NULL; - DECLARE_ITEM_COUNT(cnt); u64 pos; int ret; @@ -692,8 +690,7 @@ static int scoutfs_link(struct dentry *old_dentry, if (ret) goto out_unlock; - scoutfs_count_link(&cnt, dentry->d_name.len); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_LINK(dentry->d_name.len)); if (ret) goto out_unlock; @@ -750,7 +747,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct timespec ts = current_kernel_time(); struct scoutfs_lock *inode_lock = NULL; struct scoutfs_lock *dir_lock = NULL; - DECLARE_ITEM_COUNT(cnt); int ret = 0; ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, @@ -764,8 +760,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto unlock; } - scoutfs_count_unlink(&cnt, dentry->d_name.len); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_UNLINK(dentry->d_name.len)); if (ret) goto unlock; @@ -966,7 +961,6 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct inode *inode = NULL; struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; - DECLARE_ITEM_COUNT(cnt); u64 pos; int ret; @@ -979,8 +973,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - scoutfs_count_symlink(&cnt, dentry->d_name.len, name_len); - inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, &cnt, + inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, + SIC_SYMLINK(dentry->d_name.len, name_len), &dir_lock, &inode_lock); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -1328,7 +1322,6 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, struct scoutfs_lock *old_inode_lock = NULL; struct scoutfs_lock *new_inode_lock = NULL; struct timespec now; - DECLARE_ITEM_COUNT(cnt); bool ins_new = false; bool del_new = false; bool ins_old = false; @@ -1379,9 +1372,8 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (ret) goto out_unlock; - scoutfs_count_rename(&cnt, old_dentry->d_name.len, - new_dentry->d_name.len); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_RENAME(old_dentry->d_name.len, + new_dentry->d_name.len)); if (ret) goto out_unlock; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 08dc6aa6..af14a3b0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -892,7 +892,6 @@ static int delete_inode_items(struct super_block *sb, u64 ino) struct scoutfs_inode sinode; struct scoutfs_key_buf key; SCOUTFS_DECLARE_KVEC(val); - DECLARE_ITEM_COUNT(cnt); bool release = false; umode_t mode; int ret; @@ -917,8 +916,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) trace_delete_inode(sb, ino, mode); /* XXX this is obviously not done yet :) */ - scoutfs_count_dirty_inode(&cnt); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_DIRTY_INODE()); if (ret) goto out; release = true; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index c5a9189f..0cf8b858 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -282,7 +282,7 @@ struct scoutfs_reservation { */ static bool acquired_hold(struct super_block *sb, struct scoutfs_reservation *rsv, - struct scoutfs_item_count *cnt) + const struct scoutfs_item_count *cnt) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_TRANS_INFO(sb, tri); @@ -340,7 +340,8 @@ out: return acquired; } -int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) +int scoutfs_hold_trans(struct super_block *sb, + const struct scoutfs_item_count cnt) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_reservation *rsv; @@ -350,9 +351,9 @@ int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) * Caller shouldn't provide garbage counts, nor counts that * can't fit in segments by themselves. */ - if (WARN_ON_ONCE(cnt->items <= 0 || cnt->keys < 0 || cnt->vals < 0) || - WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt->items, cnt->keys, - cnt->vals))) + if (WARN_ON_ONCE(cnt.items <= 0 || cnt.keys < 0 || cnt.vals < 0) || + WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt.items, cnt.keys, + cnt.vals))) return -EINVAL; if (current == sbi->trans_task) @@ -371,7 +372,7 @@ int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt) BUG_ON(rsv->magic != SCOUTFS_RESERVATION_MAGIC); ret = wait_event_interruptible(sbi->trans_hold_wq, - acquired_hold(sb, rsv, cnt)); + acquired_hold(sb, rsv, &cnt)); if (ret && rsv->holders == 0) { current->journal_info = NULL; kfree(rsv); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index d3c6f326..775c9f62 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -9,7 +9,8 @@ int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); void scoutfs_trans_restart_sync_deadline(struct super_block *sb); -int scoutfs_hold_trans(struct super_block *sb, struct scoutfs_item_count *cnt); +int scoutfs_hold_trans(struct super_block *sb, + const struct scoutfs_item_count cnt); bool scoutfs_trans_held(void); void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 6174a011..3f81ed6e 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -263,7 +263,6 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); - DECLARE_ITEM_COUNT(cnt); struct scoutfs_lock *lck; unsigned int bytes; unsigned int off; @@ -316,8 +315,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, else sif = 0; - scoutfs_count_xattr_set(&cnt, name_len, size); - ret = scoutfs_hold_trans(sb, &cnt); + ret = scoutfs_hold_trans(sb, SIC_XATTR_SET(name_len, size)); if (ret) goto unlock; From 785447147576fae4dc390d9dbfddf45a089fb9fb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 Sep 2017 12:06:03 -0700 Subject: [PATCH 429/920] scoutfs: fix server wq destory warning We were seeing warnings in destroy_workqueue() which meant that work was queued on the server workqueue after it was drained and before it was finally destroyed. The only work that wasn't properly waited for was the commit work. It looks like it'd be idle because the server receive threads all wait for their request processing work to finish. But the way the commit work is batched means that a request can have its commit processed by executing commit work while leaving the work queued for another run. Fix this by specifically waiting for the commit work to finish after the server work has waited for all the recv and compaction work to finish. I wasn't able to reliably trigger the assertion in repeated xfstests runs. This survived many runs also, let's see if it stops the destroy_workqueue() assertion from triggering in the future. Signed-off-by: Zach Brown --- kmod/src/server.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kmod/src/server.c b/kmod/src/server.c index aa2d7588..d42012b0 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -96,6 +96,11 @@ struct commit_waiter { * that the caller can be sure to be woken by the next commit after they * queue and release the lock. * + * It's important to realize that the caller's commit_waiter list node + * might be serviced by a currently running commit work while queueing + * another work run in the future. This caller can return from + * wait_for_commit() while the commit_work is still queued. + * * This could queue delayed work but we're first trying to have batching * work by having concurrent modification line up behind a commit in * flight. Once the commit finishes it'll unlock and hopefully everyone @@ -1077,6 +1082,9 @@ void scoutfs_server_destroy(struct super_block *sb) /* wait for server work to wait for everything to shut down */ cancel_delayed_work_sync(&server->dwork); + /* recv work/compaction could have left commit_work queued */ + cancel_work_sync(&server->commit_work); + destroy_workqueue(server->wq); kfree(server); From 5325aff69894db34956769c59244f74727be7b29 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 17 Sep 2017 08:48:59 -0700 Subject: [PATCH 430/920] scoutfs: add item count update function The item cache maintains a count of the number of dirty items, keys, and values. It updates the counts as it dirties and cleans items. There are callers who want to modify the accounting directly instead of having the accounting updated as a side effect of cleaning and re-dirtying the item. Signed-off-by: Zach Brown --- kmod/src/item.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 15caddaf..38662a98 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -307,6 +307,19 @@ static void update_dirty_parents(struct cached_item *item) scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); } +static void update_dirty_item_counts(struct super_block *sb, signed items, + signed keys, signed vals) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + + cac->nr_dirty_items += items; + cac->dirty_key_bytes += keys; + cac->dirty_val_bytes += vals; + + scoutfs_trans_track_item(sb, items, keys, vals); +} + static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, struct cached_item *item) { @@ -320,11 +333,7 @@ static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, list_del_init(&item->entry); cac->lru_nr--; - cac->nr_dirty_items++; - cac->dirty_key_bytes += item->key->key_len; - cac->dirty_val_bytes += scoutfs_kvec_length(item->val); - - scoutfs_trans_track_item(sb, 1, item->key->key_len, + update_dirty_item_counts(sb, 1, item->key->key_len, scoutfs_kvec_length(item->val)); update_dirty_parents(item); @@ -343,11 +352,7 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, list_add_tail(&item->entry, &cac->lru_list); cac->lru_nr++; - cac->nr_dirty_items--; - cac->dirty_key_bytes -= item->key->key_len; - cac->dirty_val_bytes -= scoutfs_kvec_length(item->val); - - scoutfs_trans_track_item(sb, -1, -item->key->key_len, + update_dirty_item_counts(sb, -1, -item->key->key_len, -scoutfs_kvec_length(item->val)); WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || From c4f3c26343d8fa8c0df7d88b4050b99cb38934d5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 17 Sep 2017 12:58:16 -0700 Subject: [PATCH 431/920] scoutfs: add scoutfs_item_update_dirty() Add a function for updating the value of a dirty item. Callers can use this to make changes without having to worry about failure. Signed-off-by: Zach Brown --- kmod/src/item.c | 28 ++++++++++++++++++++++++++++ kmod/src/item.h | 2 ++ 2 files changed, 30 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 38662a98..f996bd1e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1436,6 +1436,34 @@ void scoutfs_item_delete_dirty(struct super_block *sb, scoutfs_kvec_kfree(del_val); } +/* + * Copy the callers value into the dirty item and truncate its value if + * the existing value is longer. The caller must have ensured that the + * item was dirty and had a large enough value. + */ +void scoutfs_item_update_dirty(struct super_block *sb, + struct scoutfs_key_buf *key, struct kvec *val) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + signed delta; + + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + + BUG_ON(!item || !(item->dirty & ITEM_DIRTY) || + scoutfs_kvec_length(val) > scoutfs_kvec_length(item->val)); + + delta = scoutfs_kvec_length(val) - scoutfs_kvec_length(item->val); + scoutfs_kvec_memcpy_truncate(item->val, val); + update_dirty_item_counts(sb, 0, 0, delta); + + spin_unlock_irqrestore(&cac->lock, flags); +} + /* * Return the first dirty node in the subtree starting at the given node. */ diff --git a/kmod/src/item.h b/kmod/src/item.h index bd04ccbc..5825eb0f 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -36,6 +36,8 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_key_buf *end); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); +void scoutfs_item_update_dirty(struct super_block *sb, + struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *end); From 1012ee5e8f9e142ac3b37a029d61d50a4f35bb79 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 13 Sep 2017 14:41:51 -0700 Subject: [PATCH 432/920] scoutfs: use block mapping items Move to static mapping items instead of unbounded extents. We get more predictable data structures and simpler code but still get reasonably dense metadata. We no longer need all the extent code needed to split and merge extents, test for overlaps, and all that. The functions that use the mappings (get_block, fiemap, truncate) now have a pattern where they decode the mapping item into an allocated native representation, do their work, and encode the result back into the dense item. We do have to grow the largest possible item value to fit the worst case encoding expansion of random block numbers. The local allocators are no longer two extents but are instead simple bitmaps: one for full segments and one for individual blocks. There are helper functions to free and allocate segments and blocks, with careful coordination of, for example, freeing a segment once all of its constituent blocks are free. _fiemap is refactored a bit to make it more clear what's going on. There's one function that either merges the next bit with the currently building extent or fills the current and starts recording from a non-mergable additional block. The old loop worked this way but was implemented with a single squirrely iteration over the extents. This wasn't feasible now that we're also iterating over blocks inside the mapping items. It's a lot clearer to call out to merge or fill the fiemap entry. The dirty item reservation counts for using the mappings is reduced significantly because each modification no longer has to assume that it might merge with two adjacent contiguous neighbours. Signed-off-by: Zach Brown --- kmod/src/count.h | 44 +- kmod/src/data.c | 1769 +++++++++++++++++++++++++-------------------- kmod/src/data.h | 2 + kmod/src/dir.c | 2 +- kmod/src/format.h | 80 +- kmod/src/key.c | 43 +- kmod/src/super.c | 4 + 7 files changed, 1079 insertions(+), 865 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 568caed0..5a11f127 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -206,50 +206,40 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned name_len, } /* - * Both insertion and removal modifications can dirty three extents - * at most: insertion can delete two existing neighbours and create a - * third new extent and removal can delete an existing extent and create - * two new remaining extents. - */ -static inline void __count_extents(struct scoutfs_item_count *cnt, - unsigned nr_mod, unsigned sz) -{ - cnt->items += nr_mod * 3; - cnt->keys += (nr_mod * 3) * sz; -} - -/* - * write_begin can refill local free extents after a bulk alloc rpc, - * alloc an block, delete an offline mapping, and insert the new allocated - * mapping. + * write_begin can add local free segment items, modify another to + * alloc, add a free blkno item, and modify dirty the mapping. */ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) { struct scoutfs_item_count cnt = {0,}; - - BUILD_BUG_ON(sizeof(struct scoutfs_free_extent_blkno_key) != - sizeof(struct scoutfs_free_extent_blocks_key)); + unsigned nr_free = SCOUTFS_BULK_ALLOC_COUNT + 1 + 1; __count_dirty_inode(&cnt); - __count_extents(&cnt, 2 * (SCOUTFS_BULK_ALLOC_COUNT + 1), - sizeof(struct scoutfs_free_extent_blkno_key)); - __count_extents(&cnt, 2, sizeof(struct scoutfs_file_extent_key)); + cnt.items += 1 + nr_free; + cnt.keys += sizeof(struct scoutfs_block_mapping_key) + + (nr_free * sizeof(struct scoutfs_free_bits_key)); + cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + + (nr_free * sizeof(struct scoutfs_free_bits)); return cnt; } /* - * Truncating a block can free an allocated block, delete an online - * mapping, and create an offline mapping. + * Truncating a block mapping item's worth of blocks can modify both + * free blkno and free segno items per block. Then the largest possible + * mapping item. */ static inline const struct scoutfs_item_count SIC_TRUNC_BLOCK(void) { struct scoutfs_item_count cnt = {0,}; + unsigned nr_free = (2 * SCOUTFS_BLOCK_MAPPING_BLOCKS); - __count_extents(&cnt, 2 * 1, - sizeof(struct scoutfs_free_extent_blkno_key)); - __count_extents(&cnt, 2, sizeof(struct scoutfs_file_extent_key)); + cnt.items += 1 + nr_free; + cnt.keys += sizeof(struct scoutfs_block_mapping_key) + + (nr_free * sizeof(struct scoutfs_free_bits_key)); + cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + + (nr_free * sizeof(struct scoutfs_free_bits)); return cnt; } diff --git a/kmod/src/data.c b/kmod/src/data.c index b0c7c8e2..082fc13d 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1,15 +1,15 @@ /* -* Copyright (C) 2017 Versity Software, Inc. All rights reserved. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public -* License v2 as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* General Public License for more details. -*/ + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ #include #include #include @@ -18,6 +18,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -33,50 +34,29 @@ #include "lock.h" #include "file.h" -#define EXTF "[off %llu bno %llu bks %llu fl %x]" -#define EXTA(ne) (ne)->blk_off, (ne)->blkno, (ne)->blocks, (ne)->flags - /* - * scoutfs uses extent items to reference file data. + * scoutfs uses block mapping items at a fixed granularity to describe + * file data block allocations. * - * The extent items map logical file regions to device blocks at 4K - * block granularity. File data isn't overwritten so that overwriting - * doesn't generate extent item locking and modification. + * Each item describes a fixed number of blocks. To keep the overhead + * of the items down the series of mapped blocks is encoded. The + * mapping items also describe offline blocks. They can only be written + * to newly allocated blocks with the staging ioctl. * - * Nodes have their own free extent items stored at their node id to - * avoid lock contention during allocation and freeing. These pools are - * filled and drained with messages to the server who allocates - * segment-sized regions. + * Free segnos and blocks are kept in bitmap items that are private to + * nodes so they can be modified without cluster locks. * * Block allocation maintains a fixed number of allocation cursors that * remember the position of tasks within free regions. This is very - * simple and maintains decent extents for simple streaming writes. It - * eventually won't be good enough and we'll spend complexity on - * delalloc but we want to put that off as long as possible. + * simple and maintains contiguous allocations for simple streaming + * writes. It eventually won't be good enough and we'll spend + * complexity on delalloc but we want to put that off as long as + * possible. * - * There's no unwritten extents. As we dirty file data pages, possibly - * allocating extents for the first time, we track their inodes. Before - * we commit dirty metadata we write out all tracked inodes. This - * ensures that data is persistent before the metadata that references - * it is visible. - * - * Files can have offline extents. They have no allocated file data but - * the offline status represents file data that can be recalled through - * staging. While offline the extents have their physical blkno set to - * the logical blk_off so that all the usual block extent calculations - * still hold. It's mapped back to phys == 0 for fiemap. - * - * Weirdly, the extents are indexed by the *final* logical block and - * blkno of the extent. This lets us search for neighbouring previous - * extents with a _next() call and avoids having to implement item - * reading that iterates backwards through the manifest and segments. - * - * There are two items that track free extents, one indexed by the block - * location of the free extent and one indexed by the size of the free - * extent. This means that one allocation can update a great number of - * items throughout the tree as items are created and deleted as extents - * are split and merged. This can introduce inconsistent failure - * states. We'll some day address that with preallocation and pinning. + * There's no unwritten extents. As we dirty file data pages we track + * their inodes. Before we commit dirty metadata we write out all + * tracked inodes. This ensures that data is persistent before the + * metadata that references it is visible. * * XXX * - truncate @@ -84,6 +64,7 @@ * - better io error propagation * - forced unmount with dirty data * - direct IO + * - need trans around each bulk alloc */ /* more than enough for a few tasks per core on moderate hardware */ @@ -93,7 +74,6 @@ struct data_info { struct rw_semaphore alloc_rwsem; - u64 next_large_blkno; struct list_head cursor_lru; struct hlist_head cursor_hash[CURSOR_HASH_HEADS]; }; @@ -101,20 +81,8 @@ struct data_info { #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info - -/* - * This is the size of extents that are tracked by a cursor and so end - * up being the largest file item extent length given concurrent - * streaming writes. - * - * XXX We probably want this to be a bit larger to further reduce the - * amount of item churn involved in truncating tremendous files. - */ -#define LARGE_EXTENT_BLOCKS SCOUTFS_SEGMENT_BLOCKS - struct task_cursor { u64 blkno; - u64 blocks; struct hlist_node hnode; struct list_head list_head; struct task_struct *task; @@ -122,401 +90,509 @@ struct task_cursor { }; /* - * Both file extent and free extent keys are converted into this native - * form for manipulation. The free extents set blk_off to blkno. + * Block mapping items and their native decoded form can be pretty big. + * Let's allocate them to avoid blowing the stack. */ -struct native_extent { - u64 blk_off; - u64 blkno; - u64 blocks; - u8 flags; -}; +struct block_mapping { + /* native representation */ + unsigned long offline[DIV_ROUND_UP(SCOUTFS_BLOCK_MAPPING_BLOCKS, + BITS_PER_LONG)]; + u64 blknos[SCOUTFS_BLOCK_MAPPING_BLOCKS]; -/* avoiding dynamic on-stack array initializers :/ */ -union extent_key_union { - struct scoutfs_file_extent_key file; - struct scoutfs_free_extent_blkno_key blkno; - struct scoutfs_free_extent_blocks_key blocks; + /* encoded persistent item */ + u8 encoded[SCOUTFS_BLOCK_MAPPING_MAX_BYTES]; } __packed; -#define MAX_KEY_BYTES sizeof(union extent_key_union) -static void init_file_extent_key(struct scoutfs_key_buf *key, void *key_bytes, - struct native_extent *ext, u64 arg) +/* + * We encode u64 blknos as a vlq zigzag encoded delta from the previous + * blkno. zigzag moves the sign bit down into the lsb so that small + * negative values have very few bits set. Then vlq outputs the least + * significant set bits into bytes in groups of 7. + * + * https://en.wikipedia.org/wiki/Variable-length_quantity + * + * The end result is that a series of blknos, which are limited by + * device size and often allocated near each other, are encoded with a + * handful of bytes. + */ +static unsigned zigzag_encode(u8 *bytes, u64 prev, u64 x) { - struct scoutfs_file_extent_key *fkey = key_bytes; + unsigned pos = 0; - fkey->zone = SCOUTFS_FS_ZONE; - fkey->ino = cpu_to_be64(arg); - fkey->type = SCOUTFS_FILE_EXTENT_TYPE; - fkey->last_blk_off = cpu_to_be64(ext->blk_off + ext->blocks - 1); - fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); - fkey->blocks = cpu_to_be64(ext->blocks); - fkey->flags = ext->flags; + x -= prev; + /* careful, relying on shifting extending the sign bit */ + x = (x << 1) ^ ((s64)x >> 63); - scoutfs_key_init(key, fkey, sizeof(struct scoutfs_file_extent_key)); + do { + bytes[pos++] = x & 127; + x >>= 7; + } while (x); + + bytes[pos - 1] |= 128; + + return pos; } -#define INIT_FREE_EXTENT_KEY(which_type, key, key_bytes, ext, arg, type) \ -do { \ - struct which_type *fkey = key_bytes; \ - \ - fkey->zone = SCOUTFS_NODE_ZONE; \ - fkey->node_id = cpu_to_be64(arg); \ - fkey->type = type; \ - fkey->last_blkno = cpu_to_be64(ext->blkno + ext->blocks - 1); \ - fkey->blocks = cpu_to_be64(ext->blocks); \ - \ - scoutfs_key_init(key, fkey, sizeof(struct which_type)); \ -} while (0) - -static void init_extent_key(struct scoutfs_key_buf *key, void *key_bytes, - struct native_extent *ext, u64 arg, u8 type) +static int zigzag_decode(u64 *res, u64 prev, u8 *bytes, unsigned len) { - if (type == SCOUTFS_FILE_EXTENT_TYPE) - init_file_extent_key(key, key_bytes, ext, arg); - else if(type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) - INIT_FREE_EXTENT_KEY(scoutfs_free_extent_blkno_key, - key, key_bytes, ext, arg, type); - else - INIT_FREE_EXTENT_KEY(scoutfs_free_extent_blocks_key, - key, key_bytes, ext, arg, type); -} + unsigned shift = 0; + int ret = -EIO; + u64 x = 0; + int i; + u8 b; -/* XXX could have some sanity checks */ -static void load_file_extent(struct native_extent *ext, - struct scoutfs_key_buf *key) -{ - struct scoutfs_file_extent_key *fkey = key->data; + for (i = 0; i < len; i++) { + b = bytes[i]; + x |= (u64)(b & 127) << shift; + if (b & 128) { + ret = i + 1; + break; + } + shift += 7; - ext->blocks = be64_to_cpu(fkey->blocks); - ext->blk_off = be64_to_cpu(fkey->last_blk_off) - ext->blocks + 1; - ext->blkno = be64_to_cpu(fkey->last_blkno) - ext->blocks + 1; - ext->flags = fkey->flags; -} + /* falls through to return -EIO if we run out of bytes */ + } -#define LOAD_FREE_EXTENT(which_type, ext, key) \ -do { \ - struct which_type *fkey = key->data; \ - \ - ext->blkno = be64_to_cpu(fkey->last_blkno) - \ - be64_to_cpu(fkey->blocks) + 1; \ - ext->blk_off = ext->blkno; \ - ext->blocks = be64_to_cpu(fkey->blocks); \ - ext->flags = 0; \ -} while (0) + x = (x >> 1) ^ (-(x & 1)); + *res = prev + x; -static void load_extent(struct native_extent *ext, struct scoutfs_key_buf *key) -{ - struct scoutfs_free_extent_blocks_key *fkey = key->data; - - BUILD_BUG_ON(offsetof(struct scoutfs_file_extent_key, type) != - offsetof(struct scoutfs_free_extent_blkno_key, type) || - offsetof(struct scoutfs_file_extent_key, type) != - offsetof(struct scoutfs_free_extent_blocks_key, type)); - - if (fkey->type == SCOUTFS_FILE_EXTENT_TYPE) - load_file_extent(ext, key); - else if (fkey->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) - LOAD_FREE_EXTENT(scoutfs_free_extent_blkno_key, ext, key); - else - LOAD_FREE_EXTENT(scoutfs_free_extent_blocks_key, ext, key); + return ret; } /* - * Merge two extents if they're adjacent. First we arrange them to - * only test their adjoining endpoints, then are careful to not reference - * fields after we've modified them. + * Block mappings are encoded into a byte stream. + * + * The first byte's low bits contains the last mapping index that will + * be decoded. + * + * As we walk through the encoded blocks we add control bits to the + * current control byte for the encoding of the block: zero, offline, + * increment from prev, or zigzag encoding. + * + * When the control byte is full we start filling the next byte in the + * output as the control byte for the coming blocks. When we zigzag + * encode blocks we add them to the output stream. The result is an + * interleaving of control bytes and zigzag blocks, when they're needed. + * + * In practice the typical mapping will have a zigzag for the first + * block and then the rest will be described by the control bits. + * Regions of sparse, advancing allocations, and offline are all + * described only by control bits, getting us down to 2 bits per block. */ -static int merge_extents(struct native_extent *mod, - struct native_extent *ext) +static unsigned encode_mapping(struct block_mapping *map) { - struct native_extent *left; - struct native_extent *right; + unsigned shift; + unsigned len; + u64 blkno; + u64 prev; + u8 *enc; + u8 *ctl; + u8 last; + int ret; + int i; - if (mod->blk_off < ext->blk_off) { - left = mod; - right = ext; - } else { - left = ext; - right = mod; + enc = map->encoded; + ctl = enc++; + len = 1; + + /* find the last set block in the mapping */ + last = SCOUTFS_BLOCK_MAPPING_BLOCKS; + for (i = 0; i < SCOUTFS_BLOCK_MAPPING_BLOCKS; i++) { + if (map->blknos[i] || test_bit(i, map->offline)) + last = i; } - if (left->blk_off + left->blocks == right->blk_off && - left->blkno + left->blocks == right->blkno && - left->flags == right->flags) { - mod->blk_off = left->blk_off; - mod->blkno = left->blkno; - mod->blocks = left->blocks + right->blocks; - return 1; + if (last == SCOUTFS_BLOCK_MAPPING_BLOCKS) + return 0; + + /* start with 6 bits of last */ + *ctl = last; + shift = 6; + + prev = 0; + for (i = 0; i <= last; i++) { + blkno = map->blknos[i]; + + + if (shift == 8) { + ctl = enc++; + len++; + *ctl = 0; + shift = 0; + } + + + if (blkno == prev + 1) + *ctl |= (SCOUTFS_BLOCK_ENC_INC << shift); + else if (test_bit(i, map->offline)) + *ctl |= (SCOUTFS_BLOCK_ENC_OFFLINE << shift); + else if (!blkno) + *ctl |= (SCOUTFS_BLOCK_ENC_ZERO << shift); + else { + *ctl |= (SCOUTFS_BLOCK_ENC_DELTA << shift); + + ret = zigzag_encode(enc, prev, blkno); + enc += ret; + len += ret; + } + + shift += 2; + if (blkno) + prev = blkno; } + + return len; +} + +static int decode_mapping(struct block_mapping *map, int size) +{ + unsigned ctl_bits; + u64 blkno; + u64 prev; + u8 *enc; + u8 ctl; + u8 last; + int ret; + int i; + + if (size < 1 || size > SCOUTFS_BLOCK_MAPPING_MAX_BYTES) + return -EIO; + + memset(map->blknos, 0, sizeof(map->blknos)); + memset(map->offline, 0, sizeof(map->offline)); + + enc = map->encoded; + ctl = *(enc++); + size--; + + /* start with lsb 6 bits of last */ + last = ctl & SCOUTFS_BLOCK_MAPPING_MASK; + ctl >>= 6; + ctl_bits = 2; + + prev = 0; + for (i = 0; i <= last; i++) { + + if (ctl_bits == 0) { + if (size-- == 0) + return -EIO; + ctl = *(enc++); + ctl_bits = 8; + } + + + switch(ctl & SCOUTFS_BLOCK_ENC_MASK) { + case SCOUTFS_BLOCK_ENC_INC: + blkno = prev + 1; + break; + case SCOUTFS_BLOCK_ENC_OFFLINE: + set_bit(i, map->offline); + blkno = 0; + break; + case SCOUTFS_BLOCK_ENC_ZERO: + blkno = 0; + break; + case SCOUTFS_BLOCK_ENC_DELTA: + ret = zigzag_decode(&blkno, prev, enc, size); + /* XXX corruption, ran out of encoded bytes */ + if (ret <= 0) + return -EIO; + enc += ret; + size -= ret; + break; + } + + ctl >>= 2; + ctl_bits -= 2; + + map->blknos[i] = blkno; + if (blkno) + prev = blkno; + } + + /* XXX corruption: didn't use up all the bytes */ + if (size != 0) + return -EIO; + return 0; } -/* - * The caller has ensured that the inner extent is entirely within - * the outer extent. Fill out the left and right regions of outter - * that don't overlap with inner. - */ -static void trim_extents(struct native_extent *left, - struct native_extent *right, - struct native_extent *outer, - struct native_extent *inner) +static void init_mapping_key(struct scoutfs_key_buf *key, + struct scoutfs_block_mapping_key *bmk, + u64 ino, u64 iblock) { - left->blk_off = outer->blk_off; - left->blkno = outer->blkno; - left->blocks = inner->blk_off - outer->blk_off; - left->flags = outer->flags; - right->blk_off = inner->blk_off + inner->blocks; - right->blkno = inner->blkno + inner->blocks; - right->blocks = (outer->blk_off + outer->blocks) - right->blk_off; - right->flags = outer->flags; + bmk->zone = SCOUTFS_FS_ZONE; + bmk->ino = cpu_to_be64(ino); + bmk->type = SCOUTFS_BLOCK_MAPPING_TYPE; + bmk->base = cpu_to_be64(iblock >> SCOUTFS_BLOCK_MAPPING_SHIFT); + + scoutfs_key_init(key, bmk, sizeof(struct scoutfs_block_mapping_key)); } -/* return true if inner is fully contained by outer */ -static bool extents_within(struct native_extent *outer, - struct native_extent *inner) -{ - u64 outer_end = outer->blk_off + outer->blocks - 1; - u64 inner_end = inner->blk_off + inner->blocks - 1; - return outer->blk_off <= inner_end && outer_end >= inner_end; +static void init_free_key(struct scoutfs_key_buf *key, + struct scoutfs_free_bits_key *fbk, u64 node_id, + u64 full_bit, u8 type) +{ + fbk->zone = SCOUTFS_NODE_ZONE; + fbk->node_id = cpu_to_be64(node_id); + fbk->type = type; + fbk->base = cpu_to_be64(full_bit >> SCOUTFS_FREE_BITS_SHIFT); + + scoutfs_key_init(key, fbk, sizeof(struct scoutfs_free_bits_key)); } /* - * Find an adjacent extent in the direction of the delta. If we can - * merge with it then we modify the incoming cur extent. nei is set to - * the neighbour we found. If we didn't merge then nei's blocks is set - * to 0. + * Mark the given segno as allocated. We set its bit in a free segno + * item, possibly after creating it. */ -static int try_merge(struct super_block *sb, struct native_extent *cur, - s64 delta, struct native_extent *nei, u64 arg, u8 type) +static int set_segno_free(struct super_block *sb, u64 segno) { - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits_key fbk = {0,}; + struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - struct native_extent ext; + SCOUTFS_DECLARE_KVEC(val); + int bit = 0; int ret; - memset(nei, 0, sizeof(struct native_extent)); + init_free_key(&key, &fbk, sbi->node_id, segno, + SCOUTFS_FREE_BITS_SEGNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, val, + sizeof(struct scoutfs_free_bits), + NULL); + if (ret && ret != -ENOENT) + goto out; - /* short circuit prev search for common first block alloc */ - if (cur->blk_off == 0 && delta < 0) - return 0; + bit = segno & SCOUTFS_FREE_BITS_MASK; - memset(&ext, ~0, sizeof(ext)); - init_extent_key(&last, last_bytes, &ext, arg, type); - - ext.blk_off = cur->blk_off + delta; - ext.blkno = cur->blkno + delta; - ext.blocks = 1; - ext.flags = 0; - init_extent_key(&key, key_bytes, &ext, arg, type); - - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; + if (ret == -ENOENT) { + memset(&frb, 0, sizeof(frb)); + set_bit_le(bit, &frb); + ret = scoutfs_item_create(sb, &key, val); goto out; } - load_extent(&ext, &key); - trace_printk("merge nei "EXTF"\n", EXTA(&ext)); + if (test_and_set_bit_le(bit, frb.bits)) { + ret = -EIO; + goto out; + } - if (merge_extents(cur, &ext)) - *nei = ext; + ret = scoutfs_item_update(sb, &key, val, NULL); +out: + trace_printk("segno %llu base %llu bit %u ret %d\n", + segno, be64_to_cpu(fbk.base), bit, ret); + return ret; +} + +/* + * Create a new free blkno item with all but the given blkno marked + * free. We use the caller's key so they can delete it later if they + * need to. + */ +static int create_blkno_free(struct super_block *sb, u64 blkno, + struct scoutfs_key_buf *key, + struct scoutfs_free_bits_key *fbk) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits frb; + SCOUTFS_DECLARE_KVEC(val); + int bit; + + init_free_key(key, fbk, sbi->node_id, blkno, + SCOUTFS_FREE_BITS_BLKNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + + bit = blkno & SCOUTFS_FREE_BITS_MASK; + memset(&frb, 0xff, sizeof(frb)); + clear_bit_le(bit, frb.bits); + + return scoutfs_item_create(sb, key, val); +} + +/* + * Mark the first block in the segno as allocated. This isn't a general + * purpose bit clear. It knows that it's only called from allocation + * that found the bit so it won't create the segno item. + * + * And because it's allocating a block in the segno, it also has to + * create a free block item that marks the rest of the blknos in segno + * as free. + * + * It deletes the free segno item if it clears the last bit. + */ +static int clear_segno_free(struct super_block *sb, u64 segno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits_key b_fbk; + struct scoutfs_free_bits_key fbk; + struct scoutfs_free_bits frb; + struct scoutfs_key_buf b_key; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + u64 blkno; + int bit; + int ret; + + init_free_key(&key, &fbk, sbi->node_id, segno, + SCOUTFS_FREE_BITS_SEGNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, val, + sizeof(struct scoutfs_free_bits), + NULL); + if (ret) { + /* XXX corruption, caller saw item.. should still exist */ + if (ret == -ENOENT) + ret = -EIO; + goto out; + } + + /* XXX corruption, bit couldn't have been set */ + bit = segno & SCOUTFS_FREE_BITS_MASK; + if (!test_and_clear_bit_le(bit, frb.bits)) { + ret = -EIO; + goto out; + } + + /* create the new blkno item, we can safely delete it */ + blkno = segno << SCOUTFS_SEGMENT_BLOCK_SHIFT; + ret = create_blkno_free(sb, blkno, &b_key, &b_fbk); + if (ret) + goto out; + + if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) + ret = scoutfs_item_delete(sb, &key, NULL); + else + ret = scoutfs_item_update(sb, &key, val, NULL); + if (ret) + scoutfs_item_delete_dirty(sb, &b_key); +out: + return ret; +} + +/* + * Mark the given blkno free. Set its bit in its free blkno item, + * possibly after creating it. If all the bits are set we try to mark + * its segno free and delete the blkno item. + */ +static int set_blkno_free(struct super_block *sb, u64 blkno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits_key fbk; + struct scoutfs_free_bits frb; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + u64 segno; + int bit; + int ret; + + /* get the specified item */ + init_free_key(&key, &fbk, sbi->node_id, blkno, + SCOUTFS_FREE_BITS_BLKNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, val, + sizeof(struct scoutfs_free_bits), + NULL); + if (ret && ret != -ENOENT) + goto out; + + bit = blkno & SCOUTFS_FREE_BITS_MASK; + + if (ret == -ENOENT) { + memset(&frb, 0, sizeof(frb)); + set_bit_le(bit, &frb); + ret = scoutfs_item_create(sb, &key, val); + goto out; + } + + if (test_and_set_bit_le(bit, frb.bits)) { + ret = -EIO; + goto out; + } + + if (!bitmap_full((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) { + ret = scoutfs_item_update(sb, &key, val, NULL); + goto out; + } + + /* dirty so we can safely delete if set segno fails */ + ret = scoutfs_item_dirty(sb, &key, NULL); + if (ret) + goto out; + + segno = blkno >> SCOUTFS_SEGMENT_BLOCK_SHIFT; + ret = set_segno_free(sb, segno); + if (ret) + goto out; + + scoutfs_item_delete_dirty(sb, &key); ret = 0; out: return ret; } /* - * We have two item types for indexing free extents by either the - * location of the extent or the size of the extent. When we create - * logical extents we might be finding neighbouring extents that could - * be merged. We can only search for neighbours in the location items. - * Once we find them we mirror the item modifications for both the - * location and size items. - * - * If this returns an error then nothing will have changed. + * Mark the given blkno as allocated. This is working on behalf of a + * caller who just saw the item, it must exist. We delete the free + * blkno item if all its bits are empty. */ -static int modify_items(struct super_block *sb, struct native_extent *ext, - u64 arg, u8 type, bool create) +static int clear_blkno_free(struct super_block *sb, u64 blkno) { - u8 key_bytes[MAX_KEY_BYTES]; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits_key fbk; + struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - int ret; - int err; - - trace_printk("mod cre %u "EXTF"\n", create, EXTA(ext)); - - BUG_ON(type != SCOUTFS_FILE_EXTENT_TYPE && - type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE); - - init_extent_key(&key, key_bytes, ext, arg, type); - ret = create ? scoutfs_item_create(sb, &key, NULL) : - scoutfs_item_delete(sb, &key, NULL); - - if (ret == 0 && type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { - init_extent_key(&key, key_bytes, ext, arg, - SCOUTFS_FREE_EXTENT_BLOCKS_TYPE); - ret = create ? scoutfs_item_create(sb, &key, NULL) : - scoutfs_item_delete(sb, &key, NULL); - if (ret) { - init_extent_key(&key, key_bytes, ext, arg, type); - err = create ? scoutfs_item_delete(sb, &key, NULL) : - scoutfs_item_create(sb, &key, NULL); - BUG_ON(err); - } - } - - return ret; -} - -/* - * Insert a new extent. We see if it can be merged with adjacent - * existing extents. If this returns an error then the existing extents - * will not have changed. - */ -static int insert_extent(struct super_block *sb, - struct native_extent *caller_ins, - u64 arg, u8 type) -{ - struct native_extent left; - struct native_extent right; - struct native_extent ins = *caller_ins; - bool del_ins = false; - bool ins_left = false; - int err; + SCOUTFS_DECLARE_KVEC(val); + int bit; int ret; - trace_printk("inserting "EXTF"\n", EXTA(caller_ins)); - - /* find previous that might be adjacent */ - ret = try_merge(sb, &ins, -1, &left, arg, type) ?: - try_merge(sb, &ins, 1, &right, arg, type); - if (ret < 0) - goto out; - - trace_printk("merge left "EXTF"\n", EXTA(&left)); - trace_printk("merge right "EXTF"\n", EXTA(&right)); - - ret = modify_items(sb, &ins, arg, type, true); - if (ret) - goto out; - del_ins = true; - - if (left.blocks) { - ret = modify_items(sb, &left, arg, type, false); - if (ret) - goto undo; - ins_left = true; - } - - if (right.blocks) - ret = modify_items(sb, &right, arg, type, false); - -undo: + /* get the specified item */ + init_free_key(&key, &fbk, sbi->node_id, blkno, + SCOUTFS_FREE_BITS_BLKNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, val, + sizeof(struct scoutfs_free_bits), + NULL); if (ret) { - if (ins_left) { - err = modify_items(sb, &left, arg, type, true); - BUG_ON(err); - } - if (del_ins) { - err = modify_items(sb, &ins, arg, type, false); - BUG_ON(err); - } + /* XXX corruption, bits should have existed */ + if (ret == -ENOENT) + ret = -EIO; + goto out; } -out: - return ret; -} - -/* - * Remove a portion of an existing extent. The removal might leave - * behind non-overlapping edges of the existing extent. If this returns - * an error then the existing extent will not have changed. - */ -static int remove_extent(struct super_block *sb, - struct native_extent *rem, u64 arg, u8 type) -{ - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; - struct scoutfs_key_buf key; - struct native_extent left = {0,}; - struct native_extent right = {0,}; - struct native_extent outer; - bool rem_left = false; - bool rem_right = false; - int err = 0; - int ret; - - trace_printk("removing "EXTF"\n", EXTA(rem)); - - memset(&outer, ~0, sizeof(outer)); - init_extent_key(&last, last_bytes, &outer, arg, type); - - /* find outer existing extent that contains removal extent */ - init_extent_key(&key, key_bytes, rem, arg, type); - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); - if (ret) - goto out; - - load_extent(&outer, &key); - - trace_printk("outer "EXTF"\n", EXTA(&outer)); - - if (!extents_within(&outer, rem) || outer.flags != rem->flags) { + /* XXX corruption, bit couldn't have been set */ + bit = blkno & SCOUTFS_FREE_BITS_MASK; + if (!test_and_clear_bit_le(bit, frb.bits)) { ret = -EIO; goto out; } - trim_extents(&left, &right, &outer, rem); - - trace_printk("trim left "EXTF"\n", EXTA(&left)); - trace_printk("trim right "EXTF"\n", EXTA(&right)); - - if (left.blocks) { - ret = modify_items(sb, &left, arg, type, true); - if (ret) - goto out; - rem_left = true; - } - - if (right.blocks) { - ret = modify_items(sb, &right, arg, type, true); - if (ret) - goto out; - rem_right = true; - } - - ret = modify_items(sb, &outer, arg, type, false); - + if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) + ret = scoutfs_item_delete(sb, &key, NULL); + else + ret = scoutfs_item_update(sb, &key, val, NULL); out: - if (ret) { - if (rem_right) { - err = modify_items(sb, &right, arg, type, false); - BUG_ON(err); - } - if (rem_left) { - err = modify_items(sb, &left, arg, type, false); - BUG_ON(err); - } - } - - trace_printk("ret %d\n", ret); return ret; } /* - * Free extents whose blocks fall inside the specified logical block - * range. + * In each iteration iblock is the logical block and i is the index into + * blknos array and the bit in the offline bitmap. The iteration won't + * advance past the last logical block. + */ +#define for_each_block(i, iblock, last) \ + for (i = iblock & SCOUTFS_BLOCK_MAPPING_MASK; \ + i < SCOUTFS_BLOCK_MAPPING_BLOCKS && iblock <= (last); \ + i++, iblock++) + +/* + * Free blocks inside the specified logical block range. * - * If 'offline' is given then blocks are freed but the extent items are - * left behind and their _OFFLINE flag is set. + * If 'offline' is given then blocks are freed an offline mapping is + * left behind. * * This is the low level extent item manipulation code. We hold and * release the transaction so the caller doesn't have to deal with @@ -525,137 +601,119 @@ out: int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, u64 len, bool offline) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; + struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; - struct native_extent found; - struct native_extent first; - struct native_extent rng; - struct native_extent ext; - struct native_extent ofl; - struct native_extent fr; - bool rem_fr = false; - bool ins_ext = false; - bool holding = false; + struct scoutfs_block_mapping_key last_bmk; + struct scoutfs_block_mapping_key bmk; + struct block_mapping *map; + SCOUTFS_DECLARE_KVEC(val); + bool holding; + bool dirtied; + bool modified; + u64 blkno; + u64 last; + int bytes; int ret = 0; - int err; + int i; trace_printk("iblock %llu len %llu offline %u\n", iblock, len, offline); - memset(&ext, ~0, sizeof(ext)); - init_extent_key(&last, last_bytes, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); + if (WARN_ON_ONCE(iblock + len < iblock)) + return -EINVAL; - rng.blk_off = iblock; - rng.blocks = len; - rng.blkno = 0; - rng.flags = 0; + map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); + if (!map) + return -ENOMEM; - while (rng.blocks) { - /* find the next extent that could include our first block */ - first = rng; - first.blocks = 1; - init_extent_key(&key, key_bytes, &first, ino, - SCOUTFS_FILE_EXTENT_TYPE); + last = iblock + len - 1; + init_mapping_key(&last_key, &last_bmk, ino, last); - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); + while (iblock <= last) { + /* find the mapping that could include iblock */ + init_mapping_key(&key, &bmk, ino, iblock); + scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + + ret = scoutfs_item_next(sb, &key, &last_key, val, NULL); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - load_extent(&found, &key); - trace_printk("found "EXTF"\n", EXTA(&found)); - - /* XXX corruption: offline has phys == log */ - if ((found.flags & SCOUTFS_FILE_EXTENT_OFFLINE) && - found.blkno != found.blk_off) { - ret = -EIO; + ret = decode_mapping(map, ret); + if (ret < 0) break; - } - /* we're done if the found extent is past us */ - if (found.blk_off >= rng.blk_off + rng.blocks) { - ret = 0; - break; - } + /* set iblock to the first in the next item inside last */ + iblock = max(iblock, be64_to_cpu(bmk.base) << + SCOUTFS_BLOCK_MAPPING_SHIFT); - /* find the intersection */ - ext.blk_off = max(rng.blk_off, found.blk_off); - ext.blocks = min(rng.blk_off + rng.blocks, - found.blk_off + found.blocks) - ext.blk_off; - ext.blkno = found.blkno + (ext.blk_off - found.blk_off); - ext.flags = found.flags; - - /* next search will be past the extent we truncate */ - rng.blk_off = ext.blk_off + ext.blocks; - if (rng.blk_off < iblock + len) - rng.blocks = (iblock + len) - rng.blk_off; - else - rng.blocks = 0; - - /* done if already offline */ - if (offline && (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE)) - continue; - - ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); - if (ret) - break; - holding = true; - - /* free the old extent if it was allocated */ - if (ext.blkno) { - fr = ext; - fr.blk_off = fr.blkno; - ret = insert_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_TYPE); - if (ret) - break; - rem_fr = true; - } - - /* always remove the overlapping file extent */ - ret = remove_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); - if (ret) - break; - ins_ext = true; - - /* maybe add new file extents with the offline flag set */ - if (offline) { - ofl = ext; - ofl.blkno = ofl.blk_off; - ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; - ret = insert_extent(sb, &ofl, ino, - SCOUTFS_FILE_EXTENT_TYPE); - if (ret) - break; - } - - rem_fr = false; - ins_ext = false; - scoutfs_release_trans(sb); holding = false; + dirtied = false; + modified = false; + for_each_block(i, iblock, last) { + + blkno = map->blknos[i]; + + /* don't need to do anything.. */ + if (!blkno && + !!offline == !!test_bit(i, map->offline)) + continue; + + if (!holding) { + ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); + if (ret) + break; + holding = true; + } + + if (!dirtied) { + /* dirty item with full size encoded */ + ret = scoutfs_item_update(sb, &key, val, NULL); + if (ret) + break; + dirtied = true; + } + + /* free if allocated */ + if (blkno) { + ret = set_blkno_free(sb, blkno); + if (ret) + break; + + map->blknos[i] = 0; + } + + if (offline && !test_bit(i, map->offline)) + set_bit(i, map->offline); + else if (!offline && test_bit(i, map->offline)) + clear_bit(i, map->offline); + + modified = true; + } + + if (modified) { + /* update how ever much of the item we finished */ + bytes = encode_mapping(map); + if (bytes) { + scoutfs_kvec_init(val, map->encoded, bytes); + scoutfs_item_update_dirty(sb, &key, val); + } else { + scoutfs_item_delete_dirty(sb, &key); + } + } + + if (holding) { + scoutfs_release_trans(sb); + holding = false; + } + + if (ret) + break; } - if (ret) { - if (ins_ext) { - err = insert_extent(sb, &ext, ino, - SCOUTFS_FILE_EXTENT_TYPE); - BUG_ON(err); - } - if (rem_fr) { - err = remove_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_TYPE); - BUG_ON(err); - } - } - - if (holding) - scoutfs_release_trans(sb); - + kfree(map); return ret; } @@ -723,7 +781,6 @@ static struct task_cursor *get_cursor(struct data_info *datinf) curs->pid = pid; hlist_add_head(&curs->hnode, head); curs->blkno = 0; - curs->blocks = 0; } list_move(&curs->list_head, &datinf->cursor_lru); @@ -733,8 +790,6 @@ static struct task_cursor *get_cursor(struct data_info *datinf) static int bulk_alloc(struct super_block *sb) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct native_extent ext; u64 *segnos = NULL; int ret; int i; @@ -746,29 +801,7 @@ static int bulk_alloc(struct super_block *sb) } for (i = 0; segnos[i]; i++) { - - /* merge or set this one */ - if (i > 0 && (segnos[i] == segnos[i - 1] + 1)) { - ext.blocks += SCOUTFS_SEGMENT_BLOCKS; - trace_printk("merged segno [%u] %llu blocks %llu\n", - i, segnos[i], ext.blocks); - } else { - ext.blkno = segnos[i] << SCOUTFS_SEGMENT_BLOCK_SHIFT; - ext.blocks = SCOUTFS_SEGMENT_BLOCKS; - trace_printk("set extent segno [%u] %llu blkno %llu\n", - i, segnos[i], ext.blkno); - } - - /* don't write if we merge with the next one */ - if ((segnos[i] + 1) == segnos[i + 1]) - continue; - - trace_printk("inserting [%u] "EXTF"\n", i, EXTA(&ext)); - - ext.blk_off = ext.blkno; - ext.flags = 0; - ret = insert_extent(sb, &ext, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_TYPE); + ret = set_segno_free(sb, segnos[i]); if (ret) break; } @@ -783,196 +816,173 @@ out: } /* - * Allocate a single block for the logical block offset in the file. + * Find the free bit item that contains the blkno and return the next blkno + * set starting with this blkno. * - * We try to merge single block allocations into large extents by using - * per-task cursors. Each cursor tracks a block region that should be - * searched for free extents. If we don't have a cursor, or we find - * free space outside of our cursor, then we look for the next large - * free extent. + * Returns -ENOENT if there's no free blknos at or after the given blkno. */ -static int allocate_block(struct inode *inode, sector_t iblock, u64 *blkno, - bool was_offline) +static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) { - struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_DATA_INFO(sb, datinf); - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; + struct scoutfs_free_bits_key fbk; + struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - struct native_extent last_ext; - struct native_extent found; - struct native_extent ext; - struct native_extent ofl; - struct native_extent fr; - struct task_cursor *curs; - bool alloced = false; - const u64 ino = scoutfs_ino(inode); - bool rem_ext = false; - bool ins_ofl = false; - u8 type; - int err; + SCOUTFS_DECLARE_KVEC(val); + int ret; + int bit; + + init_free_key(&key, &fbk, sbi->node_id, blkno, + SCOUTFS_FREE_BITS_BLKNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + + ret = scoutfs_item_lookup_exact(sb, &key, val, + sizeof(struct scoutfs_free_bits), NULL); + if (ret < 0) + goto out; + + bit = blkno & SCOUTFS_FREE_BITS_MASK; + bit = find_next_bit_le(frb.bits, SCOUTFS_FREE_BITS_BITS, bit); + if (bit >= SCOUTFS_FREE_BITS_BITS) { + ret = -ENOENT; + goto out; + } + + *blkno_ret = (be64_to_cpu(fbk.base) << SCOUTFS_FREE_BITS_SHIFT) + bit; + ret = 0; +out: + return ret; +} + +/* + * Find a free segno to satisfy allocation by finding the first bit set + * in the first free segno item. + */ +static int find_free_segno(struct super_block *sb, u64 *segno) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_free_bits_key last_fbk; + struct scoutfs_free_bits_key fbk; + struct scoutfs_free_bits frb; + struct scoutfs_key_buf last_key; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + int bit; int ret; - memset(&last_ext, ~0, sizeof(last_ext)); + init_free_key(&key, &fbk, sbi->node_id, 0, + SCOUTFS_FREE_BITS_SEGNO_TYPE); + init_free_key(&last_key, &last_fbk, sbi->node_id, ~0, + SCOUTFS_FREE_BITS_SEGNO_TYPE); + scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + + ret = scoutfs_item_next(sb, &key, &last_key, val, NULL); + if (ret < 0) + goto out; + + bit = find_next_bit_le(frb.bits, SCOUTFS_FREE_BITS_BITS, 0); + /* XXX corruption, shouldn't see empty items */ + if (bit >= SCOUTFS_FREE_BITS_BITS) { + ret = -EIO; + goto out; + } + + *segno = (be64_to_cpu(fbk.base) << SCOUTFS_FREE_BITS_SHIFT) + bit; + ret = 0; +out: + return ret; +} + +/* + * Allocate a single block for the logical block offset in the file. + * + * We try to encourage contiguous allocation by having per-task cursors + * that track blocks inside segments. Each new allocating task will get + * a new segment. Lots of concurrent allocations can interleave at + * segment granularity. + */ +static int find_alloc_block(struct super_block *sb, struct block_mapping *map, + struct scoutfs_key_buf *map_key, + unsigned map_ind, bool map_exists) +{ + DECLARE_DATA_INFO(sb, datinf); + struct task_cursor *curs; + SCOUTFS_DECLARE_KVEC(val); + int bytes; + u64 segno; + u64 blkno; + int ret; down_write(&datinf->alloc_rwsem); curs = get_cursor(datinf); - /* start from the cursor or look for the next large extent */ -reset_cursor: - if (curs->blocks) { - ext.blkno = curs->blkno; - ext.blocks = 0; - type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - } else { - ext.blkno = datinf->next_large_blkno; - ext.blocks = LARGE_EXTENT_BLOCKS; - type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; - } - ext.flags = 0; + trace_printk("got curs %p blkno %llu\n", curs, curs->blkno); -retry: - trace_printk("searching %llu,%llu curs %p task %p pid %u %llu,%llu\n", - ext.blkno, ext.blocks, curs, curs->task, curs->pid, - curs->blkno, curs->blocks); - - ext.blk_off = ext.blkno; - init_extent_key(&key, key_bytes, &ext, sbi->node_id, type); - init_extent_key(&last, last_bytes, &last_ext, sbi->node_id, type); - - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); - if (ret < 0) { - if (ret == -ENOENT) { - /* if the cursor's empty fall back to next large */ - if (ext.blkno && ext.blocks == 0) { - curs->blkno = 0; - curs->blocks = 0; - goto reset_cursor; - } - - /* wrap the search for large extents */ - if (ext.blkno > LARGE_EXTENT_BLOCKS && ext.blocks) { - datinf->next_large_blkno = LARGE_EXTENT_BLOCKS; - ext.blkno = datinf->next_large_blkno; - goto retry; - } - - /* ask the server for more extents */ - if (ext.blocks && !alloced) { - ret = bulk_alloc(sb); - if (ret < 0) - goto out; - alloced = true; - goto retry; - } - - /* finally look for any free block at all */ - if (ext.blocks) { - ext.blkno = 0; - ext.blocks = 0; - type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - goto retry; - } - - /* after all that return -ENOSPC */ - ret = -ENOSPC; + /* try to find the next blkno in our cursor if we have one */ + if (curs->blkno) { + ret = find_free_blkno(sb, curs->blkno, &blkno); + if (ret < 0 && ret != -ENOENT) + goto out; + if (ret == 0) { + curs->blkno = blkno; + segno = 0; + } else { + curs->blkno = 0; } - goto out; } - load_extent(&found, &key); - trace_printk("found nei "EXTF"\n", EXTA(&found)); + /* try to find segnos, asking the server for more */ + while (curs->blkno == 0) { + ret = find_free_segno(sb, &segno); + if (ret < 0 && ret != -ENOENT) + goto out; + if (ret == 0) { + blkno = segno << SCOUTFS_SEGMENT_BLOCK_SHIFT; + curs->blkno = blkno; + break; + } - /* look for a new large extent if found is outside cursor */ - if (curs->blocks && - (found.blkno + found.blocks <= curs->blkno || - found.blkno >= curs->blkno + curs->blocks)) { - curs->blkno = 0; - curs->blocks = 0; - goto reset_cursor; - } - - /* - * Set the cursor if: - * - we didn't already have one - * - it's large enough for a large extent with alignment padding - * - the sufficiently large free region is past next large - */ - if (!curs->blocks && - found.blocks >= (2 * LARGE_EXTENT_BLOCKS) && - (found.blkno + found.blocks - (2 * LARGE_EXTENT_BLOCKS) >= - datinf->next_large_blkno)) { - - curs->blkno = ALIGN(max(found.blkno, datinf->next_large_blkno), - LARGE_EXTENT_BLOCKS); - curs->blocks = LARGE_EXTENT_BLOCKS; - found.blkno = curs->blkno; - found.blocks = curs->blocks; - - datinf->next_large_blkno = curs->blkno + LARGE_EXTENT_BLOCKS; - } - - trace_printk("using %llu,%llu curs %llu,%llu\n", - found.blkno, found.blocks, curs->blkno, curs->blocks); - - /* remove old offline block if we're staging */ - if (was_offline) { - ofl.blk_off = iblock; - ofl.blkno = iblock; - ofl.blocks = 1; - ofl.flags = SCOUTFS_FILE_EXTENT_OFFLINE; - ret = remove_extent(sb, &ofl, ino, SCOUTFS_FILE_EXTENT_TYPE); + ret = bulk_alloc(sb); if (ret < 0) goto out; - ins_ofl = true; } - /* insert new file extent */ - *blkno = found.blkno; - ext.blk_off = iblock; - ext.blkno = found.blkno; - ext.blocks = 1; - ext.flags = 0; - ret = insert_extent(sb, &ext, ino, SCOUTFS_FILE_EXTENT_TYPE); - if (ret < 0) - goto out; - rem_ext = true; + trace_printk("found free segno %llu blkno %llu\n", segno, blkno); - /* and remove free extents */ - fr = ext; - fr.blk_off = ext.blkno; - ret = remove_extent(sb, &fr, sbi->node_id, - SCOUTFS_FREE_EXTENT_BLKNO_TYPE); + /* ensure that we can copy in encoded without failing */ + scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + if (map_exists) + ret = scoutfs_item_update(sb, map_key, val, NULL); + else + ret = scoutfs_item_create(sb, map_key, val); if (ret) goto out; - /* advance cursor if we're using it */ - if (curs->blocks) { - if (--curs->blocks == 0) - curs->blkno = 0; - else - curs->blkno++; - } + /* clear the free bit we found */ + if (segno) + ret = clear_segno_free(sb, segno); + else + ret = clear_blkno_free(sb, blkno); + if (ret) + goto out; + + /* update the mapping */ + clear_bit(map_ind, map->offline); + map->blknos[map_ind] = blkno; + + bytes = encode_mapping(map); + scoutfs_kvec_init(val, map->encoded, bytes); + scoutfs_item_update_dirty(sb, map_key, val); + + /* set cursor to next block, clearing if we finish the segment */ + curs->blkno++; + if ((curs->blkno & SCOUTFS_FREE_BITS_MASK) == 0) + curs->blkno = 0; ret = 0; out: - if (ret) { - if (rem_ext) { - err = remove_extent(sb, &ext, ino, - SCOUTFS_FILE_EXTENT_TYPE); - BUG_ON(err); - } - if (ins_ofl) { - err = insert_extent(sb, &ofl, ino, - SCOUTFS_FILE_EXTENT_TYPE); - BUG_ON(err); - } - } - up_write(&datinf->alloc_rwsem); + trace_printk("ret %d\n", ret); return ret; } @@ -982,73 +992,67 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - DECLARE_DATA_INFO(sb, datinf); - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; + struct scoutfs_block_mapping_key bmk; struct scoutfs_key_buf key; - struct native_extent ext; - bool was_offline = false; - u64 blkno; - u64 off; + struct block_mapping *map; + SCOUTFS_DECLARE_KVEC(val); + bool exists; + int ind; int ret; + int i; - ext.blk_off = iblock; - ext.blocks = 1; - ext.blkno = 0; - ext.flags = 0; - init_extent_key(&key, key_bytes, &ext, scoutfs_ino(inode), - SCOUTFS_FILE_EXTENT_TYPE); + map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); + if (!map) + return -ENOMEM; - memset(&ext, ~0, sizeof(ext)); - init_extent_key(&last, last_bytes, &ext, scoutfs_ino(inode), - SCOUTFS_FILE_EXTENT_TYPE); + init_mapping_key(&key, &bmk, scoutfs_ino(inode), iblock); + scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); - /* - * XXX think about how far this next can go, given locking and - * item consistency. - */ - down_read(&datinf->alloc_rwsem); - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); - up_read(&datinf->alloc_rwsem); + /* find the mapping item that covers the logical block */ + ret = scoutfs_item_lookup(sb, &key, val, NULL); if (ret < 0) { - if (ret == -ENOENT) - memset(&ext, 0, sizeof(ext)); - else + if (ret != -ENOENT) goto out; + memset(map->blknos, 0, sizeof(map->blknos)); + memset(map->offline, 0, sizeof(map->offline)); + exists = false; } else { - load_extent(&ext, &key); - trace_printk("found nei "EXTF"\n", EXTA(&ext)); + ret = decode_mapping(map, ret); + if (ret < 0) + goto out; + exists = true; } - /* use the extent if it intersects */ - if (iblock >= ext.blk_off && iblock < (ext.blk_off + ext.blocks)) { + ind = iblock & SCOUTFS_BLOCK_MAPPING_MASK; - if (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) { - /* non-stage can't write to offline */ - if (!si->staging) { - ret = -EINVAL; - goto out; - } - was_offline = true; - } else { - /* found online extent */ - off = iblock - ext.blk_off; - map_bh(bh, inode->i_sb, ext.blkno + off); - bh->b_size = min_t(u64, bh->b_size, - (ext.blocks - off) << SCOUTFS_BLOCK_SHIFT); - clear_buffer_new(bh); - } + /* fail read and write if it's offline and we're not staging */ + if (test_bit(ind, map->offline) && !si->staging) { + ret = -EINVAL; + goto out; } - if (!buffer_mapped(bh) && create) { - ret = allocate_block(inode, iblock, &blkno, was_offline); + /* try to allocate if we're writing */ + if (create && !map->blknos[ind]) { + /* + * XXX can blow the transaction here.. need to back off + * and try again if we've already done a bulk alloc in + * our transaction. + */ + ret = find_alloc_block(sb, map, &key, ind, exists); if (ret) goto out; + } - map_bh(bh, inode->i_sb, blkno); - bh->b_size = SCOUTFS_BLOCK_SHIFT; - set_buffer_new(bh); + /* mark the bh mapped and set the size for as many contig as we see */ + if (map->blknos[ind]) { + for (i = 1; ind + i < SCOUTFS_BLOCK_MAPPING_BLOCKS; i++) { + if (map->blknos[ind + i] != map->blknos[ind] + i) + break; + } + + map_bh(bh, inode->i_sb, map->blknos[ind]); + bh->b_size = min_t(u64, bh->b_size, i << SCOUTFS_BLOCK_SHIFT); + clear_buffer_new(bh); } ret = 0; @@ -1057,6 +1061,8 @@ out: scoutfs_ino(inode), (u64)iblock, create, ret, (u64)bh->b_blocknr, bh->b_size); + kfree(map); + return ret; } @@ -1172,98 +1178,170 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, return ret; } +struct pending_fiemap { + u64 logical; + u64 phys; + u64 size; + u32 flags; +}; + /* - * Return the extents that intersect with the given byte range. It doesn't - * trim the returned extents to the byte range. + * The caller is iterating over mapped blocks. We merge the current + * pending fiemap entry with the next block if we can. If we can't + * merge then we fill the current entry and start on the next. We also + * fill the pending mapping if the caller specifically tells us that + * this will be the last call. + * + * returns 0 to continue, 1 to stop, and -errno to stop with error. + */ +static int merge_or_fill(struct fiemap_extent_info *fieinfo, + struct pending_fiemap *pend, u64 logical, u64 phys, + bool offline, bool last) +{ + u32 flags = offline ? FIEMAP_EXTENT_UNKNOWN : 0; + int ret; + + /* merge if we can, returning if we don't have to fill last */ + if (pend->logical + pend->size == logical && + ((pend->phys == 0 && phys == 0) || + (pend->phys + pend->size == phys)) && + pend->flags == flags) { + pend->size += SCOUTFS_BLOCK_SIZE; + if (!last) + return 0; + } + + if (pend->size) { + if (last) + pend->flags |= FIEMAP_EXTENT_LAST; + + /* returns 1 to end, including if we passed in _LAST */ + ret = fiemap_fill_next_extent(fieinfo, pend->logical, + pend->phys, pend->size, + pend->flags); + if (ret != 0) + return ret; + } + + pend->logical = logical; + pend->phys = phys; + pend->size = SCOUTFS_BLOCK_SIZE; + pend->flags = flags; + + return 0; +} + +/* + * Iterate over non-zero block mapping items merging contiguous blocks and + * filling extent entries as we cross non-contiguous boundaries. We set + * _LAST on the last extent and _UNKNOWN on offline extents. */ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct super_block *sb = inode->i_sb; - const u8 type = SCOUTFS_FILE_EXTENT_TYPE; const u64 ino = scoutfs_ino(inode); - u8 last_bytes[MAX_KEY_BYTES]; - u8 key_bytes[MAX_KEY_BYTES]; - struct scoutfs_key_buf last; + struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; - struct native_extent ext; struct scoutfs_lock *inode_lock = NULL; - u64 logical; + struct block_mapping *map; + struct pending_fiemap pend; + struct scoutfs_block_mapping_key last_bmk; + struct scoutfs_block_mapping_key bmk; + SCOUTFS_DECLARE_KVEC(val); + loff_t i_size; + bool offline; u64 blk_off; u64 final; + u64 logical; u64 phys; - u64 size; - u32 flags; int ret; + int i; ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) return ret; - memset(&ext, ~0, sizeof(ext)); - init_extent_key(&last, last_bytes, &ext, ino, type); + map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); + if (!map) + return -ENOMEM; - blk_off = start >> SCOUTFS_BLOCK_SHIFT; - final = (start + len - 1) >> SCOUTFS_BLOCK_SHIFT; - size = 0; - flags = 0; + /* initialize to impossible to merge */ + memset(&pend, 0, sizeof(pend)); /* XXX overkill? */ mutex_lock(&inode->i_mutex); + /* stop at i_size, we don't allocate outside i_size */ + i_size = i_size_read(inode); + if (i_size == 0) { + ret = 0; + goto out; + } + + blk_off = start >> SCOUTFS_BLOCK_SHIFT; + final = min_t(loff_t, i_size - 1, start + len - 1) >> + SCOUTFS_BLOCK_SHIFT; + init_mapping_key(&last_key, &last_bmk, ino, final); + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) goto out; - for (;;) { - ext.blk_off = blk_off; - ext.blkno = 0; - ext.blocks = 1; - ext.flags = 0; - init_extent_key(&key, key_bytes, &ext, ino, type); + while (blk_off <= final) { + init_mapping_key(&key, &bmk, ino, blk_off); + scoutfs_kvec_init(val, &map->encoded, sizeof(map->encoded)); - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); + ret = scoutfs_item_next(sb, &key, &last_key, val, + inode_lock->end); if (ret < 0) { - if (ret != -ENOENT) - break; - flags |= FIEMAP_EXTENT_LAST; - ret = 0; + if (ret == -ENOENT) + ret = 0; + break; } - load_extent(&ext, &key); - - if (ext.blk_off > final) - flags |= FIEMAP_EXTENT_LAST; - - if (size) { - ret = fiemap_fill_next_extent(fieinfo, logical, phys, - size, flags); - if (ret != 0) { - if (ret == 1) - ret = 0; - break; - } - } - - if (flags & FIEMAP_EXTENT_LAST) + ret = decode_mapping(map, ret); + if (ret < 0) break; - logical = ext.blk_off << SCOUTFS_BLOCK_SHIFT; - phys = ext.blkno << SCOUTFS_BLOCK_SHIFT; - size = ext.blocks << SCOUTFS_BLOCK_SHIFT; - flags = 0; + /* set blk_off to the first in the next item inside last */ + blk_off = max(blk_off, be64_to_cpu(bmk.base) << + SCOUTFS_BLOCK_MAPPING_SHIFT); - if (ext.flags & SCOUTFS_FILE_EXTENT_OFFLINE) { - phys = 0; - flags = FIEMAP_EXTENT_UNKNOWN; + for_each_block(i, blk_off, final) { + offline = !!test_bit(i, map->offline); + + /* nothing to do with sparse regions */ + if (map->blknos[i] == 0 && !offline) + continue; + + trace_printk("blk_off %llu i %u blkno %llu\n", + blk_off, i, map->blknos[i]); + + logical = blk_off << SCOUTFS_BLOCK_SHIFT; + phys = map->blknos[i] << SCOUTFS_BLOCK_SHIFT; + + ret = merge_or_fill(fieinfo, &pend, logical, phys, + offline, false); + if (ret != 0) + break; } - - blk_off = ext.blk_off + ext.blocks; + if (ret != 0) + break; } scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + + if (ret == 0) { + /* catch final last fill */ + ret = merge_or_fill(fieinfo, &pend, 0, 0, false, true); + } + if (ret == 1) + ret = 0; + out: mutex_unlock(&inode->i_mutex); + kfree(map); return ret; } @@ -1302,8 +1380,6 @@ int scoutfs_data_setup(struct super_block *sb) init_rwsem(&datinf->alloc_rwsem); INIT_LIST_HEAD(&datinf->cursor_lru); - /* always search for large aligned extents */ - datinf->next_large_blkno = LARGE_EXTENT_BLOCKS; for (i = 0; i < CURSOR_HASH_HEADS; i++) INIT_HLIST_HEAD(&datinf->cursor_hash[i]); @@ -1340,3 +1416,120 @@ void scoutfs_data_destroy(struct super_block *sb) kfree(datinf); } } + +/* + * Basic correctness tests of u64 and mapping encoding. + */ +int __init scoutfs_data_test(void) +{ + u8 encoded[SCOUTFS_ZIGZAG_MAX_BYTES]; + struct block_mapping *input; + struct block_mapping *output; + u64 blkno; + u8 bits; + u64 prev; + u64 in; + u64 out; + int ret; + int len; + int b; + int i; + + prev = 0; + for (i = 0; i < 10000; i++) { + get_random_bytes_arch(&bits, sizeof(bits)); + get_random_bytes_arch(&in, sizeof(in)); + in &= (1ULL << (bits % 64)) - 1; + + len = zigzag_encode(encoded, prev, in); + + ret = zigzag_decode(&out, prev, encoded, len); + + if (ret <= 0 || ret > SCOUTFS_ZIGZAG_MAX_BYTES || in != out) { + printk("i %d prev %llu in %llu out %llu len %d ret %d\n", + i, prev, in, out, len, ret); + + ret = -EINVAL; + } + if (ret < 0) + return ret; + + prev = out; + } + + input = kmalloc(sizeof(struct block_mapping), GFP_KERNEL); + output = kmalloc(sizeof(struct block_mapping), GFP_KERNEL); + if (!input || !output) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < 1000; i++) { + prev = 0; + for (b = 0; b < SCOUTFS_BLOCK_MAPPING_BLOCKS; b++) { + + if (b % (64 / 2) == 0) + get_random_bytes_arch(&in, sizeof(in)); + + clear_bit(b, input->offline); + + switch(in & SCOUTFS_BLOCK_ENC_MASK) { + case SCOUTFS_BLOCK_ENC_INC: + blkno = prev + 1; + break; + case SCOUTFS_BLOCK_ENC_OFFLINE: + set_bit(b, input->offline); + blkno = 0; + break; + case SCOUTFS_BLOCK_ENC_ZERO: + blkno = 0; + break; + case SCOUTFS_BLOCK_ENC_DELTA: + get_random_bytes_arch(&bits, sizeof(bits)); + get_random_bytes_arch(&blkno, sizeof(blkno)); + blkno &= (1ULL << (bits % 64)) - 1; + break; + } + + input->blknos[b] = blkno; + + in >>= 2; + if (blkno) + prev = blkno; + } + + len = encode_mapping(input); + if (len >= 1 && len < SCOUTFS_BLOCK_MAPPING_MAX_BYTES) + memcpy(output->encoded, input->encoded, len); + ret = decode_mapping(output, len); + if (ret) { + printk("map len %d decoding failed %d\n", len, ret); + ret = -EINVAL; + goto out; + } + + for (b = 0; b < SCOUTFS_BLOCK_MAPPING_BLOCKS; b++) { + if (input->blknos[b] != output->blknos[b] || + !!test_bit(b, input->offline) != + !!test_bit(b, output->offline)) + break; + } + + if (b < SCOUTFS_BLOCK_MAPPING_BLOCKS) { + printk("map ind %u: in %llu %u, out %llu %u\n", + b, input->blknos[b], + !!test_bit(b, input->offline), + output->blknos[b], + !!test_bit(b, output->offline)); + ret = -EINVAL; + goto out; + } + } + + ret = 0; +out: + kfree(input); + kfree(output); + + return ret; +} diff --git a/kmod/src/data.h b/kmod/src/data.h index da624a80..04dd9050 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -12,4 +12,6 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); +int __init scoutfs_data_test(void); + #endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index a73bad66..269b2b84 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -852,7 +852,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, for (i = 0; i < nr; i++) { init_symlink_key(&key, &skey, ino, i); - bytes = min(size, SCOUTFS_MAX_VAL_SIZE); + bytes = min_t(u64, size, SCOUTFS_MAX_VAL_SIZE); scoutfs_kvec_init(val, (void *)target, bytes); if (op == SYM_CREATE) diff --git a/kmod/src/format.h b/kmod/src/format.h index f64af13f..b435bfb7 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -252,8 +252,8 @@ struct scoutfs_segment_block { (SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE - SCOUTFS_INODE_INDEX_SIZE_TYPE + 1) /* node zone */ -#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 11 -#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 12 +#define SCOUTFS_FREE_BITS_SEGNO_TYPE 1 +#define SCOUTFS_FREE_BITS_BLKNO_TYPE 2 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -262,7 +262,7 @@ struct scoutfs_segment_block { #define SCOUTFS_READDIR_TYPE 4 #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 -#define SCOUTFS_FILE_EXTENT_TYPE 7 +#define SCOUTFS_BLOCK_MAPPING_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 #define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ @@ -299,38 +299,70 @@ struct scoutfs_link_backref_key { __u8 name[0]; } __packed; - -/* no value */ -struct scoutfs_file_extent_key { +/* key is bytes of encoded block mapping */ +struct scoutfs_block_mapping_key { __u8 zone; __be64 ino; __u8 type; - __be64 last_blk_off; - __be64 last_blkno; - __be64 blocks; - __u8 flags; + __be64 base; } __packed; -#define SCOUTFS_FILE_EXTENT_OFFLINE (1 << 0) +/* each mapping item describes a fixed number of blocks */ +#define SCOUTFS_BLOCK_MAPPING_SHIFT 6 +#define SCOUTFS_BLOCK_MAPPING_BLOCKS (1 << SCOUTFS_BLOCK_MAPPING_SHIFT) +#define SCOUTFS_BLOCK_MAPPING_MASK (SCOUTFS_BLOCK_MAPPING_BLOCKS - 1) -/* no value */ -struct scoutfs_free_extent_blkno_key { +/* + * The mapping item value is a byte stream that encodes the value of the + * mapped blocks. The first byte contains the last index that contains + * a mapped block in its low bits. The high bits contain the control + * bits for the first (and possibly only) mapped block. + * + * From then on we consume the control bits in the current control byte + * for each mapped block. Each block has two bits that describe the + * block: zero, incremental from previous block, delta encoded, and + * offline. If we run out of control bits then we consume the next byte + * in the stream for additional control bits. If we have a delta + * encoded block then we consume its encoded bytes from the byte stream. + */ + +#define SCOUTFS_BLOCK_ENC_ZERO 0 +#define SCOUTFS_BLOCK_ENC_INC 1 +#define SCOUTFS_BLOCK_ENC_DELTA 2 +#define SCOUTFS_BLOCK_ENC_OFFLINE 3 +#define SCOUTFS_BLOCK_ENC_MASK 3 + +#define SCOUTFS_ZIGZAG_MAX_BYTES (DIV_ROUND_UP(64, 7)) + +/* + * the largest block mapping has: nr byte, ctl bytes for all blocks, and + * worst case zigzag encodings for all blocks. + */ +#define SCOUTFS_BLOCK_MAPPING_MAX_BYTES \ + (1 + (SCOUTFS_BLOCK_MAPPING_BLOCKS / 4) + \ + (SCOUTFS_BLOCK_MAPPING_BLOCKS * SCOUTFS_ZIGZAG_MAX_BYTES)) + +/* free bit bitmaps contain a segment's worth of blocks */ +#define SCOUTFS_FREE_BITS_SHIFT \ + SCOUTFS_SEGMENT_BLOCK_SHIFT +#define SCOUTFS_FREE_BITS_BITS \ + (1 << SCOUTFS_FREE_BITS_SHIFT) +#define SCOUTFS_FREE_BITS_MASK \ + (SCOUTFS_FREE_BITS_BITS - 1) +#define SCOUTFS_FREE_BITS_U64S \ + DIV_ROUND_UP(SCOUTFS_FREE_BITS_BITS, 64) + +struct scoutfs_free_bits_key { __u8 zone; __be64 node_id; __u8 type; - __be64 last_blkno; - __be64 blocks; + __be64 base; } __packed; -struct scoutfs_free_extent_blocks_key { - __u8 zone; - __be64 node_id; - __u8 type; - __be64 blocks; - __be64 last_blkno; +struct scoutfs_free_bits { + __le64 bits[SCOUTFS_FREE_BITS_U64S]; } __packed; -/* no value */ struct scoutfs_orphan_key { __u8 zone; __be64 node_id; @@ -492,9 +524,7 @@ enum { #define SCOUTFS_MAX_KEY_SIZE \ offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) -/* largest single val are dirents, larger broken up into units of this */ -#define SCOUTFS_MAX_VAL_SIZE \ - offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]) +#define SCOUTFS_MAX_VAL_SIZE SCOUTFS_BLOCK_MAPPING_MAX_BYTES #define SCOUTFS_XATTR_MAX_NAME_LEN 255 #define SCOUTFS_XATTR_MAX_SIZE 65536 diff --git a/kmod/src/key.c b/kmod/src/key.c index ba3954cc..d3695ede 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -224,22 +224,20 @@ static int pr_ino_idx(char *buf, struct scoutfs_key_buf *key, size_t size) be32_to_cpu(ikey->minor), be64_to_cpu(ikey->ino)); } -static int pr_free_ext(char *buf, struct scoutfs_key_buf *key, size_t size) +static int pr_free_bits(char *buf, struct scoutfs_key_buf *key, size_t size) { - struct scoutfs_free_extent_blkno_key *fkey = key->data; - static char *type_strings[] = { - [SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fno", - [SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fks", + [SCOUTFS_FREE_BITS_SEGNO_TYPE] = "fsg", + [SCOUTFS_FREE_BITS_BLKNO_TYPE] = "fbk", }; + struct scoutfs_free_bits_key *frk = key->data; return snprintf_key(buf, size, key, - sizeof(struct scoutfs_free_extent_blkno_key), 0, - "nod.%llu.%s.%llu.%llu", - be64_to_cpu(fkey->node_id), - type_strings[fkey->type], - be64_to_cpu(fkey->last_blkno), - be64_to_cpu(fkey->blocks)); + sizeof(struct scoutfs_block_mapping_key), 0, + "nod.%llu.%s.%llu", + be64_to_cpu(frk->node_id), + type_strings[frk->type], + be64_to_cpu(frk->base)); } static int pr_orphan(char *buf, struct scoutfs_key_buf *key, size_t size) @@ -319,18 +317,15 @@ static int pr_symlink(char *buf, struct scoutfs_key_buf *key, size_t size) be64_to_cpu(skey->ino)); } -static int pr_file_ext(char *buf, struct scoutfs_key_buf *key, size_t size) +static int pr_block_mapping(char *buf, struct scoutfs_key_buf *key, size_t size) { - struct scoutfs_file_extent_key *ekey = key->data; + struct scoutfs_block_mapping_key *bmk = key->data; return snprintf_key(buf, size, key, - sizeof(struct scoutfs_file_extent_key), 0, - "fs.%llu.ext.%llu.%llu.%llu.%x", - be64_to_cpu(ekey->ino), - be64_to_cpu(ekey->last_blk_off), - be64_to_cpu(ekey->last_blkno), - be64_to_cpu(ekey->blocks), - ekey->flags); + sizeof(struct scoutfs_block_mapping_key), 0, + "fs.%llu.bmp.%llu", + be64_to_cpu(bmk->ino), + be64_to_cpu(bmk->base)); } const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { @@ -340,8 +335,8 @@ const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { pr_ino_idx, [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = pr_ino_idx, - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = pr_free_ext, - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = pr_free_ext, + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_SEGNO_TYPE] = pr_free_bits, + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_BLKNO_TYPE] = pr_free_bits, [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = pr_orphan, [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = pr_inode, [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = pr_xattr, @@ -349,7 +344,7 @@ const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_readdir, [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_link_backref, [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = pr_symlink, - [SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = pr_file_ext, + [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = pr_block_mapping, }; /* @@ -382,7 +377,7 @@ int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) struct scoutfs_inode_index_key *ikey = key->data; type = ikey->type; } else if (zone == SCOUTFS_NODE_ZONE) { - struct scoutfs_free_extent_blkno_key *fkey = key->data; + struct scoutfs_free_bits_key *fkey = key->data; type = fkey->type; } else if (zone == SCOUTFS_FS_ZONE) { struct scoutfs_inode_key *ikey = key->data; diff --git a/kmod/src/super.c b/kmod/src/super.c index 8dd2ebd5..19db0cd2 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -392,6 +392,10 @@ static int __init scoutfs_module_init(void) scoutfs_init_counters(); + ret = scoutfs_data_test(); + if (ret) + return ret; + scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); if (!scoutfs_kset) return -ENOMEM; From 0b15cfe7f8d91920c748e415fc2b2a6e0ebaa5d9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 19 Sep 2017 09:54:20 -0700 Subject: [PATCH 433/920] scoutfs: split on btree deletion We were getting asserts during deletion that insertion while updating a parent item didn't have enough room in the block. Our btree has variable length keys. During a merge it's possible that the items moved between the blocks can result in the final key of a block changing from a small key to a large key. To update the parent ref the parent block must have as much free space as the difference in the key sizes. We ensure free space in parents during descent by trying to split the block. Deletion wasn't doing that, it was only trying to merge blocks. We need to try to split as well as merge during deletion. And we have to update the merge threshold so that we don't just split the resulting block again if it doesn't have the min free space for a new parent item. Signed-off-by: Zach Brown --- kmod/src/btree.c | 46 +++++++++++++++++++++++++++++++++++++--------- kmod/src/format.h | 5 ----- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index e6ad7f71..38d70586 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -169,6 +169,36 @@ static inline unsigned int all_len_bytes(unsigned key_len, unsigned val_len) len_bytes(key_len, val_len); } +/* number of bytes needed to insert potentially max size child item */ +static inline unsigned int parent_min_free_bytes(void) +{ + return all_len_bytes(SCOUTFS_BTREE_MAX_KEY_LEN, + sizeof(struct scoutfs_btree_ref)); +} + +/* + * The minimum number of bytes we allow in a block. During descent to + * modify if we see a block with fewer used bytes then we'll try to + * merge items from neighbours. If the neighbour also has less than the + * min bytes then the two blocks are merged. + * + * This is carefully calculated so that if two blocks are merged the + * resulting block will have at least parent min free bytes free so + * that it's not immediately split again. + * + * new_used = min_used + min_used - hdr + * new_used <= (bs - parent_min_free) + * + * min_used + min_used - hdr <= (bs - parent_min_free) + * 2 * min_used <= (bs - parent_min_free - hdr) + * min_used <= (bs - parent_min_free - hdr) / 2 + */ +static inline unsigned int min_used_bytes(void) +{ + return (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - + parent_min_free_bytes()) / 2; +} + /* total block bytes used by an existing item */ static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) { @@ -953,8 +983,7 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, int ret; if (right->level) - all_bytes = all_len_bytes(SCOUTFS_BTREE_MAX_KEY_LEN, - sizeof(struct scoutfs_btree_ref)); + all_bytes = parent_min_free_bytes(); else all_bytes = all_len_bytes(key_len, val_len); @@ -1019,7 +1048,7 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, int to_move; int ret; - if (reclaimable_free(bt) <= SCOUTFS_BTREE_FREE_LIMIT) + if (used_total(bt) >= min_used_bytes()) return 0; /* move items right into our block if we have a left sibling */ @@ -1035,10 +1064,10 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, if (ret) return ret; - if (used_total(sib) <= reclaimable_free(bt)) + if (used_total(sib) < min_used_bytes()) to_move = used_total(sib); else - to_move = reclaimable_free(bt) - SCOUTFS_BTREE_FREE_LIMIT; + to_move = min_used_bytes() - used_total(bt); move_items(bt, sib, move_right, to_move); @@ -1316,13 +1345,12 @@ restart: * than try and special case modifying the path to * reflect the tree changes. */ - if (flags & BTW_INSERT) + ret = 0; + if (flags & (BTW_INSERT | BTW_DELETE)) ret = try_split(sb, root, key, key_len, val_len, parent, pos, bt); - else if ((flags & BTW_DELETE) && parent) + if (ret == 0 && (flags & BTW_DELETE) && parent) ret = try_merge(sb, root, parent, pos, bt); - else - ret = 0; if (ret > 0) goto restart; else if (ret < 0) diff --git a/kmod/src/format.h b/kmod/src/format.h index b435bfb7..161b31a2 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -65,11 +65,6 @@ struct scoutfs_block_header { * generous. */ #define SCOUTFS_BTREE_MAX_HEIGHT 20 - -/* btree blocks (beyond the first) need to be at least half full */ -#define SCOUTFS_BTREE_FREE_LIMIT \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 2) - #define SCOUTFS_BTREE_BITS 8 /* From 42b33d616e6d60f90e01c22e31b5fe663d42c38b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 19 Sep 2017 21:36:11 -0700 Subject: [PATCH 434/920] scoutfs: fix btree bit iteration store_pos_bits() was trying to iterate over bits that were different between the existing bits set in the item and the new bits that will be set. It used a too clever for_each helper that tried to only iterate as many times as there were bits. But it messed up and only used ffs to find the next bit for the first iteration. From then on it would iterate over bits that weren't different. This would cause the counts to be changed when the bits didn't change and end up being wildly wrong. Fix this by using a much clearer loop. It still breaks out when there are no more different bits and we're only using a few low bits so the number of iterations is tiny. Signed-off-by: Zach Brown --- kmod/src/btree.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 38d70586..1d6a5c9a 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -409,13 +409,6 @@ static u8 bits_from_counts(struct scoutfs_btree_block *bt) return bits; } -/* - * Iterate through 0-based bit numbers set in 'bits' from least to - * greatest. It modifies 'bits' as it goes! - */ -#define for_each_bit(i, bits) \ - for (i = bits ? ffs(bits) : 0; i-- > 0; bits &= ~(1 < i)) - /* * Store the new bits and update the counts to match the difference from * the previously set bits. Callers use this to keep item bits in sync @@ -424,16 +417,17 @@ static u8 bits_from_counts(struct scoutfs_btree_block *bt) static void store_pos_bits(struct scoutfs_btree_block *bt, int pos, u8 bits) { u8 diff = bits ^ pos_bits(bt, pos); - int b; + int i; + u8 b; - if (!diff) - return; - - for_each_bit(b, diff) { - if (bits & (1 << b)) - le16_add_cpu(&bt->bit_counts[b], 1); - else - le16_add_cpu(&bt->bit_counts[b], -1); + for (i = 0, b = 1; diff != 0; i++, b <<= 1) { + if (diff & b) { + if (bits & b) + le16_add_cpu(&bt->bit_counts[i], 1); + else + le16_add_cpu(&bt->bit_counts[i], -1); + diff ^= b; + } } bt->item_hdrs[pos].bits = bits; From 215ba7d4adc73315690b01e453811d427e67e5a6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 19 Sep 2017 21:42:10 -0700 Subject: [PATCH 435/920] scoutfs: more reliably set btree parent item bits Most paths correctly calculated the bits to set in a parent item by combining the bits set in the child with the half bit of the child block's position in the ring. With the exception of fixing up the parent item bits after descent by walking the path. This mistake caused the parent item half bits to be zero and prevented migrating of blocks from the old half of the ring. Fix it by introducing a helper function to calculate the parent ref item bits and consistently using it. Signed-off-by: Zach Brown --- kmod/src/btree.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 1d6a5c9a..658f7e63 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -409,6 +409,18 @@ static u8 bits_from_counts(struct scoutfs_btree_block *bt) return bits; } +/* + * The bits set in a parent's ref item include the half bit for the + * child blkno so that we can search for blocks in a specific half of + * the ring. + */ +static u8 ref_item_bits(struct scoutfs_btree_ring *bring, + struct scoutfs_btree_block *child) +{ + return bits_from_counts(child) | + half_bit(bring, le64_to_cpu(child->blkno)); +} + /* * Store the new bits and update the counts to match the difference from * the previously set bits. Callers use this to keep item bits in sync @@ -432,7 +444,6 @@ static void store_pos_bits(struct scoutfs_btree_block *bt, int pos, u8 bits) bt->item_hdrs[pos].bits = bits; } - /* * The caller has descended through parents to a final block. Each * block may have had item bits modified and counts updated but they @@ -441,8 +452,9 @@ static void store_pos_bits(struct scoutfs_btree_block *bt, int pos, u8 bits) * bits to the union of all the bits down through the path to the final * block. */ -static void path_repair_reset(struct btree_path *path) +static void path_repair_reset(struct super_block *sb, struct btree_path *path) { + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *parent; struct scoutfs_btree_block *bt; u8 bits; @@ -451,7 +463,7 @@ static void path_repair_reset(struct btree_path *path) bt = path_pop(path, &pos); while ((parent = path_pop(path, &pos))) { - bits = bits_from_counts(bt); + bits = ref_item_bits(bring, bt); store_pos_bits(parent, pos, bits); bt = parent; } @@ -898,8 +910,7 @@ static int get_parent_ref_block(struct super_block *sb, int flags, ret = get_ref_block(sb, flags, ref, bt_ret); if (ret == 0) { - bits = bits_from_counts(*bt_ret) | - half_bit(bring, le64_to_cpu(ref->blkno)); + bits = ref_item_bits(bring, *bt_ret); store_pos_bits(parent, pos, bits); } @@ -919,8 +930,7 @@ static void create_parent_item(struct scoutfs_btree_ring *bring, .blkno = child->blkno, .seq = child->seq, }; - u8 bits = bits_from_counts(child) | - half_bit(bring, le64_to_cpu(ref.blkno)); + u8 bits = ref_item_bits(bring, child); create_item(parent, pos, bits, key, key_len, &ref, sizeof(ref)); } @@ -946,8 +956,7 @@ static void update_parent_bits(struct scoutfs_btree_ring *bring, struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *child) { - u8 bits = bits_from_counts(child) | - half_bit(bring, le64_to_cpu(child->blkno)); + u8 bits = ref_item_bits(bring, child); store_pos_bits(parent, pos, bits); } @@ -1284,7 +1293,7 @@ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, return -EINVAL; restart: - path_repair_reset(path); + path_repair_reset(sb, path); put_btree_block(parent); parent = NULL; put_btree_block(bt); @@ -1516,7 +1525,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(&path); + path_repair_reset(sb, &path); return ret; } @@ -1560,7 +1569,7 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(&path); + path_repair_reset(sb, &path); return ret; } @@ -1598,7 +1607,7 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(&path); + path_repair_reset(sb, &path); return ret; } @@ -1737,7 +1746,7 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, put_btree_block(bt); } - path_repair_reset(&path); + path_repair_reset(sb, &path); return ret; } @@ -1815,7 +1824,7 @@ int scoutfs_btree_write_dirty(struct super_block *sb) ret = btree_walk(sb, root, &path, BTW_DIRTY | BTW_BIT | BTW_DIRTY_OLD, NULL, 0, 0, bit, NULL, NULL, NULL); - path_repair_reset(&path); + path_repair_reset(sb, &path); if (ret == -ENOENT) { root = roots[next_root++]; continue; From 3a5093c6ae2e9a5a4ba307fa6b4de5ac4fe338b6 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 11:13:28 -0500 Subject: [PATCH 436/920] scoutfs: replace trace_printk in alloc.c Signed-off-by: Mark Fasheh --- kmod/src/alloc.c | 6 ++--- kmod/src/scoutfs_trace.h | 52 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 0a938174..b501483c 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -21,6 +21,7 @@ #include "cmp.h" #include "alloc.h" #include "counters.h" +#include "scoutfs_trace.h" /* * scoutfs allocates segments using regions of an allocation bitmap @@ -190,7 +191,7 @@ out: } up_write(&sal->rwsem); - trace_printk("segno %llu ret %d\n", *segno, ret); + trace_scoutfs_alloc_segno(sb, *segno, ret); return ret; } @@ -231,8 +232,7 @@ int scoutfs_alloc_free(struct super_block *sb, u64 segno) out: up_write(&sal->rwsem); - trace_printk("freeing segno %llu ind %llu nr %d ret %d\n", - segno, ind, nr, ret); + trace_scoutfs_alloc_free(sb, segno, ind, nr, ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b30504c9..de2f3b37 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -31,8 +31,58 @@ #include "kvec.h" #include "lock.h" #include "seg.h" +#include "super.h" -struct scoutfs_sb_info; +#define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) +#define FSID_FMT "%llx" + +TRACE_EVENT(scoutfs_alloc_free, + TP_PROTO(struct super_block *sb, __u64 segno, __u64 index, int nr, + int ret), + + TP_ARGS(sb, segno, index, nr, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + __field(__u64, index) + __field(int, nr) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + __entry->index = index; + __entry->nr = nr; + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" freeing segno %llu ind %llu nr %d ret %d", + __entry->fsid, __entry->segno, __entry->index, __entry->nr, + __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_segno, + TP_PROTO(struct super_block *sb, __u64 segno, int ret), + + TP_ARGS(sb, segno, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" segno %llu ret %d", __entry->fsid, __entry->segno, + __entry->ret) +); TRACE_EVENT(scoutfs_write_begin, TP_PROTO(u64 ino, loff_t pos, unsigned len), From 2c1f117bef25f83023dfc59387e781cd03ce8d49 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 16:01:31 -0500 Subject: [PATCH 437/920] scoutfs: replace trace_printk in compact.c Signed-off-by: Mark Fasheh --- kmod/src/compact.c | 2 +- kmod/src/scoutfs_trace.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index e81cea5f..82acc11b 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -618,7 +618,7 @@ static void scoutfs_compact_func(struct work_struct *work) free_cseg_list(sb, &results); WARN_ON_ONCE(ret); - trace_printk("ret %d\n", ret); + trace_scoutfs_compact_func(sb, ret); } void scoutfs_compact_kick(struct super_block *sb) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index de2f3b37..0148f332 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,24 @@ #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_compact_func, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + TRACE_EVENT(scoutfs_alloc_free, TP_PROTO(struct super_block *sb, __u64 segno, __u64 index, int nr, int ret), From a5283e6f2cf9d0008a81b482c771b719dd57abac Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 16:16:51 -0500 Subject: [PATCH 438/920] scoutfs: replace trace_printk in dir.c Signed-off-by: Mark Fasheh --- kmod/src/dir.c | 4 ++-- kmod/src/scoutfs_trace.h | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 269b2b84..14d8d044 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -30,6 +30,7 @@ #include "kvec.h" #include "item.h" #include "lock.h" +#include "scoutfs_trace.h" /* * Directory entries are stored in entries with offsets calculated from @@ -1073,8 +1074,7 @@ static int add_next_linkref(struct super_block *sb, u64 ino, /* next backref key is now in ent */ ret = scoutfs_item_next(sb, &key, &last, NULL, NULL); - trace_printk("ino %llu dir_ino %llu ret %d key_len %u\n", - ino, dir_ino, ret, key.key_len); + trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, ret, key.key_len); if (ret < 0) goto out; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0148f332..628d3859 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,33 @@ #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_dir_add_next_linkref, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, int ret, + unsigned int key_len), + + TP_ARGS(sb, ino, dir_ino, ret, key_len), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, dir_ino) + __field(int, ret) + __field(unsigned int, key_len) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->dir_ino = dir_ino; + __entry->ret = ret; + __entry->key_len = key_len; + ), + + TP_printk(FSID_FMT" ino %llu dir_ino %llu ret %d key_len %u", + __entry->fsid, __entry->ino, __entry->dir_ino, __entry->ret, + __entry->key_len) +); + TRACE_EVENT(scoutfs_compact_func, TP_PROTO(struct super_block *sb, int ret), From 0d28930271c37dedd3711ba4cfaedba34341e29d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 16:21:42 -0500 Subject: [PATCH 439/920] scoutfs: replace trace_printk in super.c Signed-off-by: Mark Fasheh --- kmod/src/scoutfs_trace.h | 18 ++++++++++++++++++ kmod/src/super.c | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 628d3859..763b993d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,24 @@ #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_advance_dirty_super, + TP_PROTO(struct super_block *sb, __u64 seq), + + TP_ARGS(sb, seq), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->seq = seq; + ), + + TP_printk(FSID_FMT" super seq now %llu", __entry->fsid, __entry->seq) +); + TRACE_EVENT(scoutfs_dir_add_next_linkref, TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, int ret, unsigned int key_len), diff --git a/kmod/src/super.c b/kmod/src/super.c index 19db0cd2..0a8b71af 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -116,7 +116,7 @@ void scoutfs_advance_dirty_super(struct super_block *sb) le64_add_cpu(&super->hdr.seq, 1); - trace_printk("super seq now %llu\n", le64_to_cpu(super->hdr.seq)); + trace_scoutfs_advance_dirty_super(sb, le64_to_cpu(super->hdr.seq)); } /* From 87adeb93066669a9aba2fd95397d3479acb6eef6 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 16:36:39 -0500 Subject: [PATCH 440/920] scoutfs: replace trace_printk in manifest.c Signed-off-by: Mark Fasheh --- kmod/src/manifest.c | 2 +- kmod/src/scoutfs_trace.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 0ce44fe3..ecb84a55 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -832,7 +832,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) break; } - trace_printk("level %d\n", level); + trace_scoutfs_manifest_next_compact(sb, level); if (level < 0) { ret = 0; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 763b993d..7bf5aaea 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,24 @@ #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_manifest_next_compact, + TP_PROTO(struct super_block *sb, int level), + + TP_ARGS(sb, level), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, level) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->level = level; + ), + + TP_printk(FSID_FMT" level %d", __entry->fsid, __entry->level) +); + TRACE_EVENT(scoutfs_advance_dirty_super, TP_PROTO(struct super_block *sb, __u64 seq), From 7739a0084ef2d2541e46dfc2b47ce8d677b16e98 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 16:43:44 -0500 Subject: [PATCH 441/920] scoutfs: replace trace_printk in xattr.c Signed-off-by: Mark Fasheh --- kmod/src/scoutfs_trace.h | 27 +++++++++++++++++++++++++++ kmod/src/xattr.c | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7bf5aaea..07ef28af 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,33 @@ #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_xattr_set, + TP_PROTO(struct super_block *sb, size_t name_len, const void *value, + size_t size, int flags), + + TP_ARGS(sb, name_len, value, size, flags), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(size_t, name_len) + __field(const void *, value) + __field(size_t, size) + __field(int, flags) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->name_len = name_len; + __entry->value = value; + __entry->size = size; + __entry->flags = flags; + ), + + TP_printk(FSID_FMT" name_len %zu value %p size %zu flags 0x%x", + __entry->fsid, __entry->name_len, __entry->value, + __entry->size, __entry->flags) +); + TRACE_EVENT(scoutfs_manifest_next_compact, TP_PROTO(struct super_block *sb, int level), diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 3f81ed6e..5c775b80 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -24,6 +24,7 @@ #include "trans.h" #include "xattr.h" #include "lock.h" +#include "scoutfs_trace.h" /* * In the simple case an xattr is stored in a single item whose key and @@ -271,8 +272,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, int sif; int ret; - trace_printk("name_len %zu value %p size %zu flags 0x%x\n", - name_len, value, size, flags); + trace_scoutfs_xattr_set(sb, name_len, value, size, flags); if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN || (value && size > SCOUTFS_XATTR_MAX_SIZE)) From cf3f9fee7582f352e56321a7311fb592cf5e3de4 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 18:03:56 -0500 Subject: [PATCH 442/920] scoutfs: replace trace_printk in lock.c Also clean up these traces a bit and make a lock_info trace class which we can expand in a future patch. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 6 ++---- kmod/src/scoutfs_trace.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index d2023596..12e4669d 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -715,8 +715,7 @@ static int init_lock_info(struct super_block *sb) sbi->lock_info = linfo; - trace_printk("sb %p id %016llx allocated linfo %p held %p\n", - sb, le64_to_cpu(sbi->super.id), linfo, linfo); + trace_init_lock_info(sb, linfo); out: if (ret) kfree(linfo); @@ -744,8 +743,7 @@ void scoutfs_lock_destroy(struct super_block *sb) sbi->lock_info = NULL; - trace_printk("sb %p id %016llx freeing linfo %p linfo %p\n", - sb, le64_to_cpu(sbi->super.id), linfo, linfo); + trace_scoutfs_lock_destroy(sb, linfo); kfree(linfo); } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 07ef28af..fce7f9bb 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -33,9 +33,39 @@ #include "seg.h" #include "super.h" +struct lock_info; + #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +DECLARE_EVENT_CLASS(scoutfs_lock_info_class, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + + TP_ARGS(sb, linfo), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(struct lock_info *, linfo) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->linfo = linfo; + ), + + TP_printk(FSID_FMT" linfo %p", __entry->fsid, __entry->linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, init_lock_info, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_destroy, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + TRACE_EVENT(scoutfs_xattr_set, TP_PROTO(struct super_block *sb, size_t name_len, const void *value, size_t size, int flags), From 44a19b63c038af4e42d21260ca7f9f984e2494f2 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Sep 2017 18:14:12 -0500 Subject: [PATCH 443/920] scoutfs: replace trace_printk in segment.c Signed-off-by: Mark Fasheh --- kmod/src/scoutfs_trace.h | 28 ++++++++++++++++++++++++++++ kmod/src/seg.c | 4 ++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index fce7f9bb..ed2b30fd 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -38,6 +38,34 @@ struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +DECLARE_EVENT_CLASS(scoutfs_segment_class, + TP_PROTO(struct super_block *sb, __u64 segno), + + TP_ARGS(sb, segno), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + ), + + TP_printk(FSID_FMT" segno %llu", __entry->fsid, __entry->segno) +); + +DEFINE_EVENT(scoutfs_segment_class, scoutfs_seg_submit_read, + TP_PROTO(struct super_block *sb, __u64 segno), + TP_ARGS(sb, segno) +); + +DEFINE_EVENT(scoutfs_segment_class, scoutfs_seg_submit_write, + TP_PROTO(struct super_block *sb, __u64 segno), + TP_ARGS(sb, segno) +); + DECLARE_EVENT_CLASS(scoutfs_lock_info_class, TP_PROTO(struct super_block *sb, struct lock_info *linfo), diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 1cda3462..039530d4 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -305,7 +305,7 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, struct scoutfs_segment *seg; unsigned long flags; - trace_printk("segno %llu\n", segno); + trace_scoutfs_seg_submit_read(sb, segno); spin_lock_irqsave(&cac->lock, flags); seg = find_seg(&cac->root, segno); @@ -340,7 +340,7 @@ int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_bio_completion *comp) { - trace_printk("submitting segno %llu\n", seg->segno); + trace_scoutfs_seg_submit_write(sb, seg->segno); scoutfs_bio_submit_comp(sb, WRITE, seg->pages, segno_to_blkno(seg->segno), From 8ad6ff9d410a3812363c52167c51432b02e14617 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 21 Sep 2017 15:27:32 -0500 Subject: [PATCH 444/920] scoutfs: replace trace_printk in inode.c Signed-off-by: Mark Fasheh --- kmod/src/inode.c | 28 +++---- kmod/src/scoutfs_trace.h | 174 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 187 insertions(+), 15 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index af14a3b0..11dc929a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -94,7 +94,7 @@ static void scoutfs_i_callback(struct rcu_head *head) { struct inode *inode = container_of(head, struct inode, i_rcu); - trace_printk("freeing inode %p\n", inode); + trace_scoutfs_i_callback(inode); kmem_cache_free(scoutfs_inode_cachep, SCOUTFS_I(inode)); } @@ -554,9 +554,8 @@ static int update_index(struct super_block *sb, struct scoutfs_inode_info *si, int ret; int err; - trace_printk("ino %llu have %u now %llu.%u then %llu.%u \n", - ino, si->have_item, now_major, now_minor, then_major, - then_minor); + trace_scoutfs_inode_update_index(sb, ino, si->have_item, now_major, + now_minor, then_major, then_minor); if (si->have_item && now_major == then_major && now_minor == then_minor) return 0; @@ -715,7 +714,7 @@ void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr) { struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; - trace_printk("filling ino %llu nr %llu\n", ino, nr); + trace_scoutfs_inode_fill_pool(sb, ino, nr); spin_lock(&pool->lock); @@ -797,8 +796,9 @@ int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) spin_unlock(&pool->lock); out: - trace_printk("ret %d ino %llu pool ino %llu nr %llu req %u (racey)\n", - ret, *ino, pool->ino, pool->nr, pool->in_flight); + + trace_scoutfs_alloc_ino(sb, ret, *ino, pool->ino, pool->nr, + pool->in_flight); return ret; } @@ -913,7 +913,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) } mode = le32_to_cpu(sinode.mode); - trace_delete_inode(sb, ino, mode); + trace_scoutfs_delete_inode(sb, ino, mode); /* XXX this is obviously not done yet :) */ ret = scoutfs_hold_trans(sb, SIC_DIRTY_INODE()); @@ -958,8 +958,8 @@ out: */ void scoutfs_evict_inode(struct inode *inode) { - trace_printk("ino %llu nlink %d bad %d\n", - scoutfs_ino(inode), inode->i_nlink, is_bad_inode(inode)); + trace_scoutfs_evict_inode(inode->i_sb, scoutfs_ino(inode), + inode->i_nlink, is_bad_inode(inode)); if (is_bad_inode(inode)) goto clear; @@ -976,8 +976,8 @@ int scoutfs_drop_inode(struct inode *inode) { int ret = generic_drop_inode(inode); - trace_printk("ret %d nlink %d unhashed %d\n", - ret, inode->i_nlink, inode_unhashed(inode)); + trace_scoutfs_drop_inode(inode->i_sb, scoutfs_ino(inode), + inode->i_nlink, inode_unhashed(inode)); return ret; } @@ -1102,8 +1102,8 @@ int scoutfs_inode_walk_writeback(struct super_block *sb, bool write) ret = filemap_fdatawrite(inode->i_mapping); else ret = filemap_fdatawait(inode->i_mapping); - trace_printk("ino %llu write %d ret %d\n", - scoutfs_ino(inode), write, ret); + trace_scoutfs_inode_walk_writeback(sb, scoutfs_ino(inode), + write, ret); if (ret) { iput(inode); goto out; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index ed2b30fd..68ab2c3a 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -38,6 +38,178 @@ struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_i_callback, + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(struct inode *, inode) + ), + + TP_fast_assign( + __entry->inode = inode; + ), + + /* don't print fsid as we may not have our sb private available */ + TP_printk("freeing inode %p", __entry->inode) +); + +TRACE_EVENT(scoutfs_inode_update_index, + TP_PROTO(struct super_block *sb, __u64 ino, unsigned int have_item, + __u64 now_major, unsigned int now_minor, __u64 then_major, + unsigned int then_minor), + + TP_ARGS(sb, ino, have_item, now_major, now_minor, then_major, + then_minor), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(unsigned int, have_item) + __field(__u64, now_major) + __field(unsigned int, now_minor) + __field(__u64, then_major) + __field(unsigned int, then_minor) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->have_item = have_item; + __entry->now_major = now_major; + __entry->now_minor = now_minor; + __entry->then_major = then_major; + __entry->then_minor = then_minor; + ), + + TP_printk(FSID_FMT" ino %llu have %u now %llu.%u then %llu.%u", + __entry->fsid, __entry->ino, __entry->have_item, + __entry->now_major, __entry->now_minor, __entry->then_major, + __entry->then_minor) +); + +TRACE_EVENT(scoutfs_inode_fill_pool, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 nr), + + TP_ARGS(sb, ino, nr), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, nr) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->nr = nr; + ), + + TP_printk(FSID_FMT" filling ino %llu nr %llu", __entry->fsid, + __entry->ino, __entry->nr) +); + +TRACE_EVENT(scoutfs_alloc_ino, + TP_PROTO(struct super_block *sb, int ret, __u64 ino, __u64 pool_ino, + __u64 nr, unsigned int in_flight), + + TP_ARGS(sb, ret, ino, pool_ino, nr, in_flight), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + __field(__u64, ino) + __field(__u64, pool_ino) + __field(__u64, nr) + __field(unsigned int, in_flight) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + __entry->ino = ino; + __entry->pool_ino = pool_ino; + __entry->nr = nr; + __entry->in_flight = in_flight; + ), + + TP_printk(FSID_FMT" ret %d ino %llu pool ino %llu nr %llu req %u " + "(racey)", __entry->fsid, __entry->ret, __entry->ino, + __entry->pool_ino, __entry->nr, __entry->in_flight) +); + +TRACE_EVENT(scoutfs_evict_inode, + TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, + unsigned int is_bad_ino), + + TP_ARGS(sb, ino, nlink, is_bad_ino), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(unsigned int, nlink) + __field(unsigned int, is_bad_ino) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->is_bad_ino = is_bad_ino; + ), + + TP_printk(FSID_FMT" ino %llu nlink %u bad %d", __entry->fsid, + __entry->ino, __entry->nlink, __entry->is_bad_ino) +); + +TRACE_EVENT(scoutfs_drop_inode, + TP_PROTO(struct super_block *sb, __u64 ino, unsigned int nlink, + unsigned int unhashed), + + TP_ARGS(sb, ino, nlink, unhashed), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(unsigned int, nlink) + __field(unsigned int, unhashed) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->nlink = nlink; + __entry->unhashed = unhashed; + ), + + TP_printk(FSID_FMT" ino %llu nlink %u unhashed %d", __entry->fsid, + __entry->ino, __entry->nlink, __entry->unhashed) +); + +TRACE_EVENT(scoutfs_inode_walk_writeback, + TP_PROTO(struct super_block *sb, __u64 ino, int write, int ret), + + TP_ARGS(sb, ino, write, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(int, write) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->write = write; + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ino %llu write %d ret %d", __entry->fsid, + __entry->ino, __entry->write, __entry->ret) +); + DECLARE_EVENT_CLASS(scoutfs_segment_class, TP_PROTO(struct super_block *sb, __u64 segno), @@ -351,7 +523,7 @@ TRACE_EVENT(scoutfs_orphan_inode, MINOR(__entry->dev), __entry->ino) ); -TRACE_EVENT(delete_inode, +TRACE_EVENT(scoutfs_delete_inode, TP_PROTO(struct super_block *sb, u64 ino, umode_t mode), TP_ARGS(sb, ino, mode), From 285842086dbac1ac00edf178357a85e3037c2a2b Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 21 Sep 2017 19:16:01 -0500 Subject: [PATCH 445/920] scoutfs: replace trace_printk in ioctl.c Signed-off-by: Mark Fasheh --- kmod/src/ioctl.c | 11 +++--- kmod/src/scoutfs_trace.h | 75 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 31c07ef9..a7b794da 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -32,6 +32,7 @@ #include "client.h" #include "lock.h" #include "manifest.h" +#include "scoutfs_trace.h" /* * We make inode index items coherent by locking fixed size regions of @@ -68,10 +69,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) if (copy_from_user(&walk, uwalk, sizeof(walk))) return -EFAULT; - trace_printk("index %u first %llu.%u.%llu last %llu.%u.%llu\n", - walk.index, walk.first.major, walk.first.minor, - walk.first.ino, walk.last.major, walk.last.minor, - walk.last.ino); + trace_scoutfs_ioc_walk_inodes(sb, &walk); if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) type = SCOUTFS_INODE_INDEX_SIZE_TYPE; @@ -340,8 +338,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; - trace_printk("block %llu count %llu vers %llu\n", - args.block, args.count, args.data_version); + trace_scoutfs_ioc_release(sb, &args); if (args.count == 0) return 0; @@ -383,7 +380,7 @@ out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); - trace_printk("ret %d\n", ret); + trace_scoutfs_ioc_release_ret(sb, ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 68ab2c3a..dd409c9a 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -32,12 +32,87 @@ #include "lock.h" #include "seg.h" #include "super.h" +#include "ioctl.h" struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_ioc_release_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_ioc_release, + TP_PROTO(struct super_block *sb, struct scoutfs_ioctl_release *args), + + TP_ARGS(sb, args), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, block) + __field(__u64, count) + __field(__u64, vers) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->block = args->block; + __entry->count = args->count; + __entry->vers = args->data_version; + ), + + TP_printk(FSID_FMT" block %llu count %llu vers %llu", __entry->fsid, + __entry->block, __entry->count, __entry->vers) +); + +TRACE_EVENT(scoutfs_ioc_walk_inodes, + TP_PROTO(struct super_block *sb, struct scoutfs_ioctl_walk_inodes *walk), + + TP_ARGS(sb, walk), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, index) + __field(__u64, first_major) + __field(__u32, first_minor) + __field(__u64, first_ino) + __field(__u64, last_major) + __field(__u32, last_minor) + __field(__u64, last_ino) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->index = walk->index; + __entry->first_major = walk->first.major; + __entry->first_minor = walk->first.minor; + __entry->first_ino = walk->first.ino; + __entry->last_major = walk->last.major; + __entry->last_minor = walk->last.minor; + __entry->last_ino = walk->last.ino; + ), + + TP_printk(FSID_FMT" index %u first %llu.%u.%llu last %llu.%u.%llu", + __entry->fsid, __entry->index, __entry->first_major, + __entry->first_minor, __entry->first_ino, __entry->last_major, + __entry->last_minor, __entry->last_ino) +); + TRACE_EVENT(scoutfs_i_callback, TP_PROTO(struct inode *inode), From deecfa0ad504e29612f1668baa500b75e352a0fe Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 22 Sep 2017 16:26:22 -0500 Subject: [PATCH 446/920] scoutfs: replace trace_printk in trans.c Signed-off-by: Mark Fasheh --- kmod/src/scoutfs_trace.h | 151 +++++++++++++++++++++++++++++++++++++++ kmod/src/trans.c | 27 ++++--- 2 files changed, 163 insertions(+), 15 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index dd409c9a..029ee4c8 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -33,12 +33,163 @@ #include "seg.h" #include "super.h" #include "ioctl.h" +#include "count.h" struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_sync_fs, + TP_PROTO(struct super_block *sb, int wait), + + TP_ARGS(sb, wait), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, wait) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->wait = wait; + ), + + TP_printk(FSID_FMT" wait %d", __entry->fsid, __entry->wait) +); + +TRACE_EVENT(scoutfs_trans_write_func, + TP_PROTO(struct super_block *sb, int dirty), + + TP_ARGS(sb, dirty), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, dirty) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->dirty = dirty; + ), + + TP_printk(FSID_FMT" dirty %d", __entry->fsid, __entry->dirty) +); + +TRACE_EVENT(scoutfs_release_trans, + TP_PROTO(struct super_block *sb, void *rsv, unsigned int rsv_holders, + struct scoutfs_item_count *res, + struct scoutfs_item_count *act, unsigned int tri_holders, + unsigned int tri_writing, unsigned int tri_items, + unsigned int tri_keys, unsigned int tri_vals), + + TP_ARGS(sb, rsv, rsv_holders, res, act, tri_holders, tri_writing, + tri_items, tri_keys, tri_vals), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, rsv) + __field(unsigned int, rsv_holders) + __field(int, res_items) + __field(int, res_keys) + __field(int, res_vals) + __field(int, act_items) + __field(int, act_keys) + __field(int, act_vals) + __field(unsigned int, tri_holders) + __field(unsigned int, tri_writing) + __field(unsigned int, tri_items) + __field(unsigned int, tri_keys) + __field(unsigned int, tri_vals) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->rsv = rsv; + __entry->rsv_holders = rsv_holders; + __entry->res_items = res->items; + __entry->res_keys = res->keys; + __entry->res_vals = res->vals; + __entry->act_items = act->items; + __entry->act_keys = act->keys; + __entry->act_vals = act->vals; + __entry->tri_holders = tri_holders; + __entry->tri_writing = tri_writing; + __entry->tri_items = tri_items; + __entry->tri_keys = tri_keys; + __entry->tri_vals = tri_vals; + ), + + TP_printk(FSID_FMT" rsv %p holders %u reserved %u.%u.%u actual " + "%d.%d.%d, trans holders %u writing %u reserved " + "%u.%u.%u", __entry->fsid, __entry->rsv, + __entry->rsv_holders, __entry->res_items, __entry->res_keys, + __entry->res_vals, __entry->act_items, __entry->act_keys, + __entry->act_vals, __entry->tri_holders, __entry->tri_writing, + __entry->tri_items, __entry->tri_keys, __entry->tri_vals) +); + +TRACE_EVENT(scoutfs_trans_acquired_hold, + TP_PROTO(struct super_block *sb, const struct scoutfs_item_count *cnt, + void *rsv, unsigned int rsv_holders, + struct scoutfs_item_count *res, + struct scoutfs_item_count *act, unsigned int tri_holders, + unsigned int tri_writing, unsigned int tri_items, + unsigned int tri_keys, unsigned int tri_vals), + + TP_ARGS(sb, cnt, rsv, rsv_holders, res, act, tri_holders, tri_writing, + tri_items, tri_keys, tri_vals), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, cnt_items) + __field(int, cnt_keys) + __field(int, cnt_vals) + __field(void *, rsv) + __field(unsigned int, rsv_holders) + __field(int, res_items) + __field(int, res_keys) + __field(int, res_vals) + __field(int, act_items) + __field(int, act_keys) + __field(int, act_vals) + __field(unsigned int, tri_holders) + __field(unsigned int, tri_writing) + __field(unsigned int, tri_items) + __field(unsigned int, tri_keys) + __field(unsigned int, tri_vals) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->cnt_items = cnt->items; + __entry->cnt_keys = cnt->keys; + __entry->cnt_vals = cnt->vals; + __entry->rsv = rsv; + __entry->rsv_holders = rsv_holders; + __entry->res_items = res->items; + __entry->res_keys = res->keys; + __entry->res_vals = res->vals; + __entry->act_items = act->items; + __entry->act_keys = act->keys; + __entry->act_vals = act->vals; + __entry->tri_holders = tri_holders; + __entry->tri_writing = tri_writing; + __entry->tri_items = tri_items; + __entry->tri_keys = tri_keys; + __entry->tri_vals = tri_vals; + ), + + TP_printk(FSID_FMT" cnt %u.%u.%u, rsv %p holders %u reserved %u.%u.%u " + "actual %d.%d.%d, trans holders %u writing %u reserved " + "%u.%u.%u", __entry->fsid, __entry->cnt_items, + __entry->cnt_keys, __entry->cnt_vals, __entry->rsv, + __entry->rsv_holders, __entry->res_items, __entry->res_keys, + __entry->res_vals, __entry->act_items, __entry->act_keys, + __entry->act_vals, __entry->tri_holders, __entry->tri_writing, + __entry->tri_items, __entry->tri_keys, __entry->tri_vals) +); + TRACE_EVENT(scoutfs_ioc_release_ret, TP_PROTO(struct super_block *sb, int ret), diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 0cf8b858..32c4c751 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -121,7 +121,7 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); - trace_printk("items dirty %d\n", scoutfs_item_has_dirty(sb)); + trace_scoutfs_trans_write_func(sb, scoutfs_item_has_dirty(sb)); if (scoutfs_item_has_dirty(sb)) { /* @@ -222,7 +222,7 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) struct write_attempt attempt; int ret; - trace_printk("wait %d\n", wait); + trace_scoutfs_sync_fs(sb, wait); if (!wait) { queue_trans_work(sbi); @@ -294,13 +294,12 @@ static bool acquired_hold(struct super_block *sb, spin_lock(&tri->lock); - trace_printk("cnt %u.%u.%u, rsv %p holders %u reserved %u.%u.%u actual %d.%d.%d, trans holders %u writing %u reserved %u.%u.%u\n", - cnt->items, cnt->keys, cnt->vals, rsv, rsv->holders, - rsv->reserved.items, rsv->reserved.keys, - rsv->reserved.vals, rsv->actual.items, rsv->actual.keys, - rsv->actual.vals, tri->holders, tri->writing, - tri->reserved_items, tri->reserved_keys, - tri->reserved_vals); + trace_scoutfs_trans_acquired_hold(sb, cnt, rsv, rsv->holders, + &rsv->reserved, &rsv->actual, + tri->holders, tri->writing, + tri->reserved_items, + tri->reserved_keys, + tri->reserved_vals); /* use a caller's existing reservation */ if (rsv->holders) @@ -433,12 +432,10 @@ void scoutfs_release_trans(struct super_block *sb) spin_lock(&tri->lock); - trace_printk("rsv %p holders %u reserved %u.%u.%u actual %d.%d.%d, trans holders %u writing %u reserved %u.%u.%u\n", - rsv, rsv->holders, rsv->reserved.items, - rsv->reserved.keys, rsv->reserved.vals, - rsv->actual.items, rsv->actual.keys, rsv->actual.vals, - tri->holders, tri->writing, tri->reserved_items, - tri->reserved_keys, tri->reserved_vals); + trace_scoutfs_release_trans(sb, rsv, rsv->holders, &rsv->reserved, + &rsv->actual, tri->holders, tri->writing, + tri->reserved_items, tri->reserved_keys, + tri->reserved_vals); BUG_ON(rsv->holders <= 0); BUG_ON(tri->holders <= 0); From 2a07e6f642105570b442949aeffd5fa50959cd2d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 22 Sep 2017 17:34:07 -0500 Subject: [PATCH 447/920] scoutfs: replace trace_printk in data.c Signed-off-by: Mark Fasheh --- kmod/src/data.c | 32 +++--- kmod/src/scoutfs_trace.h | 212 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 217 insertions(+), 27 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 082fc13d..9b72d18b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -383,8 +383,8 @@ static int set_segno_free(struct super_block *sb, u64 segno) ret = scoutfs_item_update(sb, &key, val, NULL); out: - trace_printk("segno %llu base %llu bit %u ret %d\n", - segno, be64_to_cpu(fbk.base), bit, ret); + trace_scoutfs_data_set_segno_free(sb, segno, be64_to_cpu(fbk.base), + bit, ret); return ret; } @@ -616,8 +616,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, int ret = 0; int i; - trace_printk("iblock %llu len %llu offline %u\n", - iblock, len, offline); + trace_scoutfs_data_truncate_items(sb, iblock, len, offline); if (WARN_ON_ONCE(iblock + len < iblock)) return -EINVAL; @@ -774,8 +773,7 @@ static struct task_cursor *get_cursor(struct data_info *datinf) if (!curs) { curs = list_last_entry(&datinf->cursor_lru, struct task_cursor, list_head); - trace_printk("resetting curs %p was task %p pid %u\n", - curs, task, pid); + trace_scoutfs_data_get_cursor(curs, task, pid); hlist_del_init(&curs->hnode); curs->task = task; curs->pid = pid; @@ -916,7 +914,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, curs = get_cursor(datinf); - trace_printk("got curs %p blkno %llu\n", curs, curs->blkno); + trace_scoutfs_data_find_alloc_block_curs(sb, curs, curs->blkno); /* try to find the next blkno in our cursor if we have one */ if (curs->blkno) { @@ -947,7 +945,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, goto out; } - trace_printk("found free segno %llu blkno %llu\n", segno, blkno); + trace_scoutfs_data_find_alloc_block_found_seg(sb, segno, blkno); /* ensure that we can copy in encoded without failing */ scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); @@ -983,7 +981,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, out: up_write(&datinf->alloc_rwsem); - trace_printk("ret %d\n", ret); + trace_scoutfs_data_find_alloc_block_ret(sb, ret); return ret; } @@ -1057,9 +1055,8 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, ret = 0; out: - trace_printk("ino %llu iblock %llu create %d ret %d bnr %llu size %zu\n", - scoutfs_ino(inode), (u64)iblock, create, ret, - (u64)bh->b_blocknr, bh->b_size); + trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, + ret, bh->b_blocknr, bh->b_size); kfree(map); @@ -1131,8 +1128,7 @@ static int scoutfs_write_begin(struct file *file, struct super_block *sb = inode->i_sb; int ret; - trace_printk("ino %llu pos %llu len %u\n", - scoutfs_ino(inode), (u64)pos, len); + trace_scoutfs_write_begin(sb, scoutfs_ino(inode), (__u64)pos, len); ret = scoutfs_hold_trans(sb, SIC_WRITE_BEGIN()); if (ret) @@ -1161,8 +1157,8 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct super_block *sb = inode->i_sb; int ret; - trace_printk("ino %llu pgind %lu pos %llu len %u copied %d\n", - scoutfs_ino(inode), page->index, (u64)pos, len, copied); + trace_scoutfs_write_end(sb, scoutfs_ino(inode), page->index, (u64)pos, + len, copied); ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); if (ret > 0) { @@ -1315,8 +1311,8 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (map->blknos[i] == 0 && !offline) continue; - trace_printk("blk_off %llu i %u blkno %llu\n", - blk_off, i, map->blknos[i]); + trace_scoutfs_data_fiemap(sb, blk_off, i, + map->blknos[i]); logical = blk_off << SCOUTFS_BLOCK_SHIFT; phys = map->blknos[i] << SCOUTFS_BLOCK_SHIFT; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 029ee4c8..2f31db6d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -40,6 +40,192 @@ struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_data_fiemap, + TP_PROTO(struct super_block *sb, __u64 off, int i, __u64 blkno), + + + TP_ARGS(sb, off, i, blkno), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, off) + __field(int, i) + __field(__u64, blkno) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->off = off; + __entry->i = i; + __entry->blkno = blkno; + ), + + TP_printk(FSID_FMT" blk_off %llu i %u blkno %llu", __entry->fsid, + __entry->off, __entry->i, __entry->blkno) +); + +TRACE_EVENT(scoutfs_get_block, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, + int create, int ret, __u64 blkno, size_t size), + + TP_ARGS(sb, ino, iblock, create, ret, blkno, size), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, iblock) + __field(int, create) + __field(int, ret) + __field(__u64, blkno) + __field(size_t, size) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->iblock = iblock; + __entry->create = create; + __entry->ret = ret; + __entry->blkno = blkno; + __entry->size = size; + ), + + TP_printk(FSID_FMT" ino %llu iblock %llu create %d ret %d bnr %llu " + "size %zu", __entry->fsid, __entry->ino, __entry->iblock, + __entry->create, __entry->ret, __entry->blkno, __entry->size) +); + +TRACE_EVENT(scoutfs_data_find_alloc_block_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_data_find_alloc_block_found_seg, + TP_PROTO(struct super_block *sb, __u64 segno, __u64 blkno), + + TP_ARGS(sb, segno, blkno), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + __field(__u64, blkno) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + __entry->blkno = blkno; + ), + + TP_printk(FSID_FMT" found free segno %llu blkno %llu", __entry->fsid, + __entry->segno, __entry->blkno) +); + +TRACE_EVENT(scoutfs_data_find_alloc_block_curs, + TP_PROTO(struct super_block *sb, void *curs, __u64 blkno), + + TP_ARGS(sb, curs, blkno), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, curs) + __field(__u64, blkno) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->curs = curs; + __entry->blkno = blkno; + ), + + TP_printk(FSID_FMT" got curs %p blkno %llu", __entry->fsid, + __entry->curs, __entry->blkno) +); + +TRACE_EVENT(scoutfs_data_get_cursor, + TP_PROTO(void *curs, void *task, unsigned int pid), + + TP_ARGS(curs, task, pid), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, curs) + __field(void *, task) + __field(unsigned int, pid) + ), + + TP_fast_assign( + __entry->curs = curs; + __entry->task = task; + __entry->pid = pid; + ), + + TP_printk("resetting curs %p was task %p pid %u", __entry->curs, + __entry->task, __entry->pid) +); + +TRACE_EVENT(scoutfs_data_truncate_items, + TP_PROTO(struct super_block *sb, __u64 iblock, __u64 len, int offline), + + TP_ARGS(sb, iblock, len, offline), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, iblock) + __field(__u64, len) + __field(int, offline) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->iblock = iblock; + __entry->len = len; + __entry->offline = offline; + ), + + TP_printk(FSID_FMT" iblock %llu len %llu offline %u", __entry->fsid, + __entry->iblock, __entry->len, __entry->offline) +); + +TRACE_EVENT(scoutfs_data_set_segno_free, + TP_PROTO(struct super_block *sb, __u64 segno, __u64 base, + unsigned int bit, int ret), + + TP_ARGS(sb, segno, base, bit, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + __field(__u64, base) + __field(unsigned int, bit) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + __entry->base = base; + __entry->bit = bit; + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" segno %llu base %llu bit %u ret %d", __entry->fsid, + __entry->segno, __entry->base, __entry->bit, __entry->ret) +); + TRACE_EVENT(scoutfs_sync_fs, TP_PROTO(struct super_block *sb, int wait), @@ -649,47 +835,55 @@ TRACE_EVENT(scoutfs_alloc_segno, ); TRACE_EVENT(scoutfs_write_begin, - TP_PROTO(u64 ino, loff_t pos, unsigned len), + TP_PROTO(struct super_block *sb, u64 ino, loff_t pos, unsigned len), - TP_ARGS(ino, pos, len), + TP_ARGS(sb, ino, pos, len), TP_STRUCT__entry( + __field(__u64, fsid) __field(__u64, inode) __field(__u64, pos) __field(__u32, len) ), TP_fast_assign( + __entry->fsid = FSID_ARG(sb); __entry->inode = ino; __entry->pos = pos; __entry->len = len; ), - TP_printk("ino %llu pos %llu len %u", + TP_printk(FSID_FMT" ino %llu pos %llu len %u", __entry->fsid, __entry->inode, __entry->pos, __entry->len) ); TRACE_EVENT(scoutfs_write_end, - TP_PROTO(u64 ino, loff_t pos, unsigned len, unsigned copied), + TP_PROTO(struct super_block *sb, u64 ino, unsigned long idx, u64 pos, + unsigned len, unsigned copied), - TP_ARGS(ino, pos, len, copied), + TP_ARGS(sb, ino, idx, pos, len, copied), TP_STRUCT__entry( - __field(__u64, inode) + __field(__u64, fsid) + __field(__u64, ino) + __field(unsigned long, idx) __field(__u64, pos) __field(__u32, len) __field(__u32, copied) ), TP_fast_assign( - __entry->inode = ino; + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->idx = idx; __entry->pos = pos; __entry->len = len; __entry->copied = copied; ), - TP_printk("ino %llu pos %llu len %u", - __entry->inode, __entry->pos, __entry->len) + TP_printk(FSID_FMT" ino %llu pgind %lu pos %llu len %u copied %d", + __entry->fsid, __entry->ino, __entry->idx, __entry->pos, + __entry->len, __entry->copied) ); TRACE_EVENT(scoutfs_dirty_inode, From 3430edb60b26342a5e7c144b7af443afc65f50bf Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 22 Sep 2017 18:02:07 -0500 Subject: [PATCH 448/920] scoutfs: replace trace_printk in item.c Signed-off-by: Mark Fasheh --- kmod/src/item.c | 20 ++--- kmod/src/scoutfs_trace.h | 189 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 12 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index f996bd1e..b0920e2f 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -374,7 +374,7 @@ static void item_referenced(struct item_cache *cac, struct cached_item *item) static void erase_item(struct super_block *sb, struct item_cache *cac, struct cached_item *item) { - trace_printk("erasing item %p\n", item); + trace_scoutfs_erase_item(sb, item); clear_item_dirty(sb, cac, item); rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); @@ -755,7 +755,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - trace_printk("ret %d\n", ret); + trace_scoutfs_item_lookup_ret(sb, ret); return ret; } @@ -950,7 +950,7 @@ out: scoutfs_key_free(sb, pos); scoutfs_key_free(sb, range_end); - trace_printk("ret %d\n", ret); + trace_scoutfs_item_next_ret(sb, ret); return ret; } @@ -969,7 +969,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, int key_len = key->key_len; int ret; - trace_printk("key len %u min val len %d\n", key_len, len); + trace_scoutfs_item_next_same_min(sb, key_len, len); if (WARN_ON_ONCE(!val || scoutfs_kvec_length(val) < len)) return -EINVAL; @@ -978,7 +978,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, if (ret >= 0 && (key->key_len != key_len || ret < len)) ret = -EIO; - trace_printk("ret %d\n", ret); + trace_scoutfs_item_next_same_min_ret(sb, ret); return ret; } @@ -994,13 +994,13 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, int key_len = key->key_len; int ret; - trace_printk("key len %u\n", key_len); + trace_scoutfs_item_next_same(sb, key_len); ret = scoutfs_item_next(sb, key, last, val, end); if (ret >= 0 && (key->key_len != key_len)) ret = -EIO; - trace_printk("ret %d\n", ret); + trace_scoutfs_item_next_same_ret(sb, ret); return ret; } @@ -1304,7 +1304,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); - trace_printk("ret %d\n", ret); + trace_scoutfs_item_dirty_ret(sb, ret); return ret; } @@ -1357,7 +1357,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, out: scoutfs_kvec_kfree(up_val); - trace_printk("ret %d\n", ret); + trace_scoutfs_item_update_ret(sb, ret); return ret; } @@ -1405,7 +1405,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, scoutfs_kvec_kfree(del_val); - trace_printk("ret %d\n", ret); + trace_scoutfs_item_delete_ret(sb, ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 2f31db6d..85deb845 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -40,6 +40,171 @@ struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_item_delete_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_item_dirty_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_item_update_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_item_next_same, + TP_PROTO(struct super_block *sb, unsigned int key_len), + + TP_ARGS(sb, key_len), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(unsigned int, key_len) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->key_len = key_len; + ), + + TP_printk(FSID_FMT" key len %u", __entry->fsid, __entry->key_len) +); + +TRACE_EVENT(scoutfs_item_next_same_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_item_next_same_min, + TP_PROTO(struct super_block *sb, int key_len, int len), + + TP_ARGS(sb, key_len, len), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, key_len) + __field(int, len) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->key_len = key_len; + __entry->len = len; + ), + + TP_printk(FSID_FMT" key len %u min val len %d", __entry->fsid, + __entry->key_len, __entry->len) +); + +TRACE_EVENT(scoutfs_item_next_same_min_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_item_next_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + +TRACE_EVENT(scoutfs_erase_item, + TP_PROTO(struct super_block *sb, void *item), + + TP_ARGS(sb, item), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, item) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->item = item; + ), + + TP_printk(FSID_FMT" erasing item %p", __entry->fsid, __entry->item) +); + TRACE_EVENT(scoutfs_data_fiemap, TP_PROTO(struct super_block *sb, __u64 off, int i, __u64 blkno), @@ -1031,12 +1196,14 @@ DECLARE_EVENT_CLASS(scoutfs_key_class, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), TP_ARGS(sb, key), TP_STRUCT__entry( - __dynamic_array(char, key, scoutfs_key_str(NULL, key)) + __field(__u64, fsid) + __dynamic_array(char, key, scoutfs_key_str(NULL, key)) ), TP_fast_assign( + __entry->fsid = FSID_ARG(sb); scoutfs_key_str(__get_dynamic_array(key), key); ), - TP_printk("key %s", __get_str(key)) + TP_printk(FSID_FMT" key %s", __entry->fsid, __get_str(key)) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, @@ -1044,6 +1211,24 @@ DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, TP_ARGS(sb, key) ); +TRACE_EVENT(scoutfs_item_lookup_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), TP_ARGS(sb, key) From e67e5009406c54870b44bbb785f8383f66e83893 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 22 Sep 2017 18:06:06 -0500 Subject: [PATCH 449/920] scoutfs: turn off tracing in dlmglue.c Put this behind a #define. Leave the asserts (mlog_bug_on_msg) though and redefine their macro to printk instead of going to the trace buffer. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index f1c206aa..7f729239 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -38,6 +38,7 @@ #include "dlmglue.h" +#ifdef TRACE_DLMGLUE #define mlog(mask, fmt, args...) trace_printk(fmt , ##args) #define mlog_errno(st) do { \ int _st = (st); \ @@ -45,11 +46,14 @@ _st != AOP_TRUNCATED_PAGE && _st != -ENOSPC) \ mlog(ML_ERROR, "status = %lld\n", (long long)_st); \ } while (0) - +#else +#define mlog(mask, fmt, args...) +#define mlog_errno(st) +#endif #define mlog_bug_on_msg(cond, fmt, args...) do { \ if (cond) { \ - mlog(ML_ERROR, "bug expression: " #cond "\n"); \ - mlog(ML_ERROR, fmt, ##args); \ + printk(KERN_ERR "bug expression: " #cond "\n"); \ + printk(KERN_ERR fmt, ##args); \ BUG(); \ } \ } while (0) From 43a2d63f79a505a1d3fbdef7339b6950b71e43f8 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 22 Sep 2017 18:53:20 -0500 Subject: [PATCH 450/920] scoutfs: replace trace_printk in bio.c Signed-off-by: Mark Fasheh --- kmod/src/bio.c | 27 ++++--- kmod/src/scoutfs_trace.h | 150 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 14 deletions(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index 916c5be7..f72ff4d1 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -18,6 +18,7 @@ #include "super.h" #include "format.h" #include "bio.h" +#include "scoutfs_trace.h" struct bio_end_io_args { struct super_block *sb; @@ -32,8 +33,8 @@ static void dec_end_io(struct bio_end_io_args *args, int err) if (err && !args->err) args->err = err; - trace_printk("args %p in_flight %d err %d\n", - args, atomic_read(&args->in_flight), err); + trace_scoutfs_dec_end_io(args->sb, args, atomic_read(&args->in_flight), + err); if (atomic_dec_and_test(&args->in_flight)) { args->end_io(args->sb, args->data, args->err); @@ -45,7 +46,7 @@ static void bio_end_io(struct bio *bio, int err) { struct bio_end_io_args *args = bio->bi_private; - trace_printk("bio %p size %u err %d \n", bio, bio->bi_size, err); + trace_scoutfs_bio_end_io(args->sb, bio, bio->bi_size, err); dec_end_io(args, err); bio_put(bio); @@ -113,15 +114,15 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, if (bio_add_page(bio, page, bytes, 0) != bytes) { /* submit the full bio and retry this page */ atomic_inc(&args->in_flight); - trace_printk("bio %p args %p in_flight %d\n", - bio, args, atomic_read(&args->in_flight)); + trace_scoutfs_bio_submit(sb, bio, args, + atomic_read(&args->in_flight)); submit_bio(rw, bio); bio = NULL; i--; continue; } - trace_printk("added page %p to bio %p\n", page, bio); + trace_scoutfs_bio_submit_added(sb, page, bio); blkno += SCOUTFS_BLOCKS_PER_PAGE; nr_blocks -= SCOUTFS_BLOCKS_PER_PAGE; @@ -129,8 +130,8 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, if (bio) { atomic_inc(&args->in_flight); - trace_printk("bio %p args %p in_flight %d\n", - bio, args, atomic_read(&args->in_flight)); + trace_scoutfs_bio_submit_partial(sb, bio, args, + atomic_read(&args->in_flight)); submit_bio(rw, bio); } @@ -144,7 +145,7 @@ void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp) atomic_set(&comp->pending, 1); init_completion(&comp->comp); comp->err = 0; - trace_printk("initing comp %p\n", comp); + trace_scoutfs_bio_init_comp(comp); } static void comp_end_io(struct super_block *sb, void *data, int err) @@ -154,8 +155,7 @@ static void comp_end_io(struct super_block *sb, void *data, int err) if (err && !comp->err) comp->err = err; - trace_printk("ending comp %p pending before %d\n", - comp, atomic_read(&comp->pending)); + trace_comp_end_io(sb, comp); if (atomic_dec_and_test(&comp->pending)) complete(&comp->comp); @@ -167,8 +167,7 @@ void scoutfs_bio_submit_comp(struct super_block *sb, int rw, struct scoutfs_bio_completion *comp) { atomic_inc(&comp->pending); - trace_printk("submitting comp %p pending before %d\n", - comp, atomic_read(&comp->pending)); + trace_scoutfs_bio_submit_comp(sb, comp); scoutfs_bio_submit(sb, rw, pages, blkno, nr_blocks, comp_end_io, comp); } @@ -177,7 +176,7 @@ int scoutfs_bio_wait_comp(struct super_block *sb, struct scoutfs_bio_completion *comp) { comp_end_io(sb, comp, 0); - trace_printk("waiting for comp %p\n", comp); + trace_scoutfs_bio_wait_comp(sb, comp); wait_for_completion(&comp->comp); return comp->err; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 85deb845..50140d25 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -34,12 +34,162 @@ #include "super.h" #include "ioctl.h" #include "count.h" +#include "bio.h" struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +DECLARE_EVENT_CLASS(scoutfs_comp_class, + TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), + + TP_ARGS(sb, comp), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(struct scoutfs_bio_completion *, comp) + __field(int, pending) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->comp = comp; + __entry->pending = atomic_read(&comp->pending); + ), + + TP_printk(FSID_FMT" comp %p pending before %d", __entry->fsid, + __entry->comp, __entry->pending) +); +DEFINE_EVENT(scoutfs_comp_class, comp_end_io, + TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), + TP_ARGS(sb, comp) +); +DEFINE_EVENT(scoutfs_comp_class, scoutfs_bio_submit_comp, + TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), + TP_ARGS(sb, comp) +); +DEFINE_EVENT(scoutfs_comp_class, scoutfs_bio_wait_comp, + TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), + TP_ARGS(sb, comp) +); + +TRACE_EVENT(scoutfs_bio_init_comp, + TP_PROTO(void *comp), + + TP_ARGS(comp), + + TP_STRUCT__entry( + __field(void *, comp) + ), + + TP_fast_assign( + __entry->comp = comp; + ), + + TP_printk("initing comp %p", __entry->comp) +); + +TRACE_EVENT(scoutfs_bio_submit_added, + TP_PROTO(struct super_block *sb, void *page, void *bio), + + TP_ARGS(sb, page, bio), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, page) + __field(void *, bio) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->page = page; + __entry->bio = bio; + ), + + TP_printk(FSID_FMT" added page %p to bio %p", __entry->fsid, + __entry->page, __entry->bio) +); + +DECLARE_EVENT_CLASS(scoutfs_bio_class, + TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), + + TP_ARGS(sb, bio, args, in_flight), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, bio) + __field(void *, args) + __field(int, in_flight) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->bio = bio; + __entry->args = args; + __entry->in_flight = in_flight; + ), + + TP_printk(FSID_FMT" bio %p args %p in_flight %d", __entry->fsid, + __entry->bio, __entry->args, __entry->in_flight) +); + +DEFINE_EVENT(scoutfs_bio_class, scoutfs_bio_submit, + TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), + TP_ARGS(sb, bio, args, in_flight) +); + +DEFINE_EVENT(scoutfs_bio_class, scoutfs_bio_submit_partial, + TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), + TP_ARGS(sb, bio, args, in_flight) +); + +TRACE_EVENT(scoutfs_bio_end_io, + TP_PROTO(struct super_block *sb, void *bio, int size, int err), + + TP_ARGS(sb, bio, size, err), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, bio) + __field(int, size) + __field(int, err) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->bio = bio; + __entry->size = size; + __entry->err = err; + ), + + TP_printk(FSID_FMT" bio %p size %u err %d", __entry->fsid, + __entry->bio, __entry->size, __entry->err) +); + +TRACE_EVENT(scoutfs_dec_end_io, + TP_PROTO(struct super_block *sb, void *args, int in_flight, int err), + + TP_ARGS(sb, args, in_flight, err), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, args) + __field(int, in_flight) + __field(int, err) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->args = args; + __entry->in_flight = in_flight; + __entry->err = err; + ), + + TP_printk(FSID_FMT" args %p in_flight %d err %d", __entry->fsid, + __entry->args, __entry->in_flight, __entry->err) +); + TRACE_EVENT(scoutfs_item_delete_ret, TP_PROTO(struct super_block *sb, int ret), From 15aa09b0c259a63075bad06ecb98e00106fa771c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 21 Sep 2017 13:07:52 -0700 Subject: [PATCH 451/920] scoutfs: add shrink exit trace points Add trace points that show the incoming nr_to_scan and resulting object count for shrinker calls. Signed-off-by: Zach Brown --- kmod/src/item.c | 5 ++++- kmod/src/lock.c | 5 ++++- kmod/src/scoutfs_trace.h | 32 ++++++++++++++++++++++++++++++++ kmod/src/seg.c | 5 ++++- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index b0920e2f..cb3fee9a 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1748,6 +1748,7 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) struct cached_item *end; unsigned long flags; unsigned long nr; + int ret; nr = sc->nr_to_scan; if (nr == 0) @@ -1848,7 +1849,9 @@ abort: spin_unlock_irqrestore(&cac->lock, flags); out: - return min_t(unsigned long, cac->lru_nr, INT_MAX); + ret = min_t(unsigned long, cac->lru_nr, INT_MAX); + trace_scoutfs_item_shrink_exit(sb, sc->nr_to_scan, ret); + return ret; } static void *copy_key_with_len(void *data, struct scoutfs_key_buf *key) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 12e4669d..994121a5 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -345,6 +345,7 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) struct scoutfs_lock *tmp; unsigned long flags; unsigned long nr; + int ret; nr = sc->nr_to_scan; if (!nr) @@ -368,7 +369,9 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) spin_unlock_irqrestore(&linfo->lock, flags); out: - return min_t(unsigned long, linfo->lru_nr, INT_MAX); + ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); + trace_scoutfs_lock_shrink_exit(linfo->sb, sc->nr_to_scan, ret); + return ret; } static void free_lock_tree(struct super_block *sb) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 50140d25..461475e1 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1650,6 +1650,38 @@ TRACE_EVENT(scoutfs_item_next_range_check, __get_str(last), __get_str(end), __get_str(range_end)) ); +DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret), + TP_STRUCT__entry( + __field(void *, sb) + __field(unsigned long, nr_to_scan) + __field(int, ret) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->nr_to_scan = nr_to_scan; + __entry->ret = ret; + ), + TP_printk("sb %p nr_to_scan %lu ret %d", + __entry->sb, __entry->nr_to_scan, __entry->ret) +); + +DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_lock_shrink_exit, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret) +); + +DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_seg_shrink_exit, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret) +); + +DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_item_shrink_exit, + TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), + TP_ARGS(sb, nr_to_scan, ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 039530d4..e7d9d1d0 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -716,6 +716,7 @@ static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) unsigned long flags; unsigned long nr; LIST_HEAD(list); + int ret; nr = sc->nr_to_scan; if (!nr) @@ -746,7 +747,9 @@ static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) } out: - return min_t(unsigned long, cac->lru_nr, INT_MAX); + ret = min_t(unsigned long, cac->lru_nr, INT_MAX); + trace_scoutfs_seg_shrink_exit(sb, sc->nr_to_scan, ret); + return ret; } int scoutfs_seg_setup(struct super_block *sb) From ccefffe74fc66118df094b6d008b71f2f8fe68cc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 21 Sep 2017 13:03:54 -0700 Subject: [PATCH 452/920] scoutfs: add item, range, lock alloc/free counters Add some counters to track allocation and freeing of our structures that are subject to shrinking. This lets us eyeball the counters to see if we have runaway leaks. Signed-off-by: Zach Brown --- kmod/src/counters.h | 10 ++++++++-- kmod/src/item.c | 7 +++++++ kmod/src/lock.c | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 9fe291c0..86b5afa2 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -31,10 +31,14 @@ EXPAND_COUNTER(data_invalidatepage) \ EXPAND_COUNTER(data_writepage) \ EXPAND_COUNTER(data_end_writeback_page) \ + EXPAND_COUNTER(item_alloc) \ + EXPAND_COUNTER(item_free) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_lookup_hit) \ EXPAND_COUNTER(item_lookup_miss) \ EXPAND_COUNTER(item_delete) \ + EXPAND_COUNTER(item_range_alloc) \ + EXPAND_COUNTER(item_range_free) \ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ EXPAND_COUNTER(item_range_insert) \ @@ -43,10 +47,12 @@ EXPAND_COUNTER(item_shrink_dirty_abort) \ EXPAND_COUNTER(item_shrink_skip_inced) \ EXPAND_COUNTER(item_shrink_range) \ - EXPAND_COUNTER(item_shrink) + EXPAND_COUNTER(item_shrink) \ + EXPAND_COUNTER(lock_alloc) \ + EXPAND_COUNTER(lock_free) #define FIRST_COUNTER alloc_alloc -#define LAST_COUNTER item_shrink +#define LAST_COUNTER lock_free #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/item.c b/kmod/src/item.c index cb3fee9a..aed13730 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -103,6 +103,7 @@ static u8 item_flags(struct cached_item *item) static void free_item(struct super_block *sb, struct cached_item *item) { if (!IS_ERR_OR_NULL(item)) { + scoutfs_inc_counter(sb, item_free); WARN_ON_ONCE(!list_empty(&item->entry)); WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); scoutfs_key_free(sb, item->key); @@ -133,6 +134,9 @@ static struct cached_item *alloc_item(struct super_block *sb, } } + if (item) + scoutfs_inc_counter(sb, item_alloc); + return item; } @@ -563,6 +567,7 @@ static bool check_range(struct super_block *sb, struct rb_root *root, static void free_range(struct super_block *sb, struct cached_range *rng) { if (!IS_ERR_OR_NULL(rng)) { + scoutfs_inc_counter(sb, item_range_free); scoutfs_key_free(sb, rng->start); scoutfs_key_free(sb, rng->end); kfree(rng); @@ -1099,6 +1104,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) return -EINVAL; + scoutfs_inc_counter(sb, item_range_alloc); rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); if (rng) { rng->start = scoutfs_key_dup(sb, start); @@ -1657,6 +1663,7 @@ int scoutfs_item_invalidate(struct super_block *sb, /* XXX think about racing with trans write */ + scoutfs_inc_counter(sb, item_range_alloc); rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); if (rng) { rng->start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 994121a5..acd60e6c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -27,6 +27,7 @@ #include "dlmglue.h" #include "inode.h" #include "trans.h" +#include "counters.h" #define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ @@ -107,6 +108,7 @@ static void free_scoutfs_lock(struct scoutfs_lock *lock) if (lock) { linfo = SCOUTFS_SB(lock->sb)->lock_info; + scoutfs_inc_counter(lock->sb, lock_free); ocfs2_lock_res_free(&lock->lockres); scoutfs_key_free(lock->sb, lock->start); scoutfs_key_free(lock->sb, lock->end); @@ -294,6 +296,7 @@ search: found->sequence = ++linfo->seq_cnt; rb_link_node(&found->node, parent, node); rb_insert_color(&found->node, &linfo->lock_tree); + scoutfs_inc_counter(sb, lock_alloc); } found->refcnt++; if (test_bit(SCOUTFS_LOCK_RECLAIM, &found->flags)) { From fd509840d4c9e7aabdff73f0c6cae850207f5ebe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 22 Sep 2017 09:38:04 -0700 Subject: [PATCH 453/920] scoutfs: use pages for seg shrink object count The VM wasn't very excited about trying to reclaim our seg count when we returned small count of the number of large segment objects available for reclaim. Each segment represents a ton of memory so we want to give the VM more visibility into the scale of the cache to encourage it to shrink it. We define the object count for the seg shrinker as the number of pages of segments in the lru. Signed-off-by: Zach Brown --- kmod/src/seg.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index e7d9d1d0..f17a8ebd 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -718,7 +718,7 @@ static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) LIST_HEAD(list); int ret; - nr = sc->nr_to_scan; + nr = DIV_ROUND_UP(sc->nr_to_scan, SCOUTFS_SEGMENT_PAGES); if (!nr) goto out; @@ -747,7 +747,8 @@ static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) } out: - ret = min_t(unsigned long, cac->lru_nr, INT_MAX); + ret = min_t(unsigned long, cac->lru_nr * SCOUTFS_SEGMENT_PAGES, + INT_MAX); trace_scoutfs_seg_shrink_exit(sb, sc->nr_to_scan, ret); return ret; } From c5ddec705812ba473a60810c7c9dc2b08fa0ab42 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Sep 2017 10:23:40 -0700 Subject: [PATCH 454/920] scoutfs: more aggressively shrink items The old item shrinking was very conservative. It would only try and reclaim items from the front of the range of the oldest items in the lru. It would stop making progress if all the items in the front of the lru are in a range whose initial item can't be reclaimed. With the item shrinking not making progress memory fills with items. Eventually the system backs up behind an allocation during segment writing blocking waiting for free pages. We fix this by much more aggressively shrinking items. We now look for a region of items around the oldest items to shrink. If those fall in the middle of a range then we use the memory from the items to construct a new range and split the existing range. Now the only way we'll refuse to shrink items is if they're dirty. We have a reasonably small cap on the number of dirty items so we shoudln't get stuck. Signed-off-by: Zach Brown --- kmod/src/counters.h | 10 +- kmod/src/item.c | 349 +++++++++++++++++++++++++++------------ kmod/src/scoutfs_trace.h | 35 ++++ 3 files changed, 281 insertions(+), 113 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 86b5afa2..2933ae09 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -42,11 +42,13 @@ EXPAND_COUNTER(item_range_hit) \ EXPAND_COUNTER(item_range_miss) \ EXPAND_COUNTER(item_range_insert) \ - EXPAND_COUNTER(item_shrink_no_items) \ + EXPAND_COUNTER(item_shrink_alone) \ + EXPAND_COUNTER(item_shrink_empty_range) \ + EXPAND_COUNTER(item_shrink_next_dirty) \ EXPAND_COUNTER(item_shrink_outside) \ - EXPAND_COUNTER(item_shrink_dirty_abort) \ - EXPAND_COUNTER(item_shrink_skip_inced) \ - EXPAND_COUNTER(item_shrink_range) \ + EXPAND_COUNTER(item_shrink_range_end) \ + EXPAND_COUNTER(item_shrink_split_range) \ + EXPAND_COUNTER(item_shrink_small_split) \ EXPAND_COUNTER(item_shrink) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) diff --git a/kmod/src/item.c b/kmod/src/item.c index aed13730..8c28f06f 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -371,6 +371,19 @@ static void item_referenced(struct item_cache *cac, struct cached_item *item) list_move_tail(&item->entry, &cac->lru_list); } +/* remove the item from its tracking data structures */ +static void unlink_item(struct super_block *sb, struct item_cache *cac, + struct cached_item *item) +{ + clear_item_dirty(sb, cac, item); + rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); + RB_CLEAR_NODE(&item->node); + if (!list_empty(&item->entry)) { + list_del_init(&item->entry); + cac->lru_nr--; + } +} + /* * Safely erase an item from the tree. Make sure to remove its dirty * accounting, use the augmented erase, and free it. @@ -380,13 +393,7 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, { trace_scoutfs_erase_item(sb, item); - clear_item_dirty(sb, cac, item); - rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); - RB_CLEAR_NODE(&item->node); - if (!list_empty(&item->entry)) { - list_del_init(&item->entry); - cac->lru_nr--; - } + unlink_item(sb, cac, item); free_item(sb, item); } @@ -1714,34 +1721,213 @@ static struct cached_item *rb_next_item(struct cached_item *item) return NULL; } +static struct cached_item *rb_prev_item(struct cached_item *item) +{ + struct rb_node *node; + + if (item && (node = rb_prev(&item->node))) + return container_of(node, struct cached_item, node); + + return NULL; +} + /* - * Shrink the item cache. + * Find the bounds of an item cache shrinking operation. Starting from + * an item, walk through either next items to the right or prev items to + * the left. Record items that are valid final shrinking points because + * using their key for a new range end doesn't cross the remaining + * existing item. We stop if we check enough items, hit a dirty item, + * or run out of items in the range. + * + * We only can't use an item as a new range end point if moving its key + * crosses the next item in the cache. This only happens when smaller + * items share a prefix with the next larger item. This only happens + * for item populations with names (dirents, xattrs) that share + * prefixes. We really don't want to be unable to reclaim so we + * aggressively try to walk past all of them. + */ +#define BOUNDARY_MIN 32 +#define BOUNDARY_MAX 300 +static struct cached_item *shrink_boundary(struct super_block *sb, + struct cached_item *item, + struct cached_item **next_ret, + struct scoutfs_key_buf *end, + bool right) +{ + struct cached_item *found = NULL; + struct cached_item *next; + bool cmp; + int i; + + *next_ret = NULL; + + for (i = 0; i < BOUNDARY_MAX; i++) { + if (right) + next = rb_next_item(item); + else + next = rb_prev_item(item); + + if (next) { + if (right) + cmp = scoutfs_key_compare(next->key, end) > 0; + else + cmp = scoutfs_key_compare(next->key, end) < 0; + } else { + cmp = true; + } + if (cmp) { + scoutfs_inc_counter(sb, item_shrink_range_end); + found = item; + *next_ret = NULL; + break; + } + + if (right) { + scoutfs_key_inc_cur_len(item->key); + cmp = scoutfs_key_compare(item->key, next->key) <= 0; + scoutfs_key_dec_cur_len(item->key); + } else { + scoutfs_key_dec_cur_len(item->key); + cmp = scoutfs_key_compare(item->key, next->key) >= 0; + scoutfs_key_inc_cur_len(item->key); + } + if (cmp) { + found = item; + *next_ret = next; + if (i >= BOUNDARY_MIN) + break; + } + + if (next->dirty & ITEM_DIRTY) { + scoutfs_inc_counter(sb, item_shrink_next_dirty); + break; + } + + item = next; + } + + return found; +} + +/* + * The caller found an item in the lru and the range it falls within. + * This frees items around the item. After finding the boundaries we + * have to either update the ranges if items remain or free the item. + * + * We're in the context of a shrinker so we can't allocate. If we + * remove items from the middle of a range we use the memory from some + * removed items to store the new split range. + */ +static int shrink_around(struct super_block *sb, struct cached_range *rng, + struct cached_item *item) +{ + struct item_cache *cac = SCOUTFS_SB(sb)->item_cache; + struct scoutfs_key_buf *rng_end = NULL; + struct scoutfs_key_buf *key; + struct cached_range *new_rng; + struct cached_item *first; + struct cached_item *last; + struct cached_item *prev; + struct cached_item *next; + int nr = 0; + + /* we're re-using item memory as ranges :P */ + BUILD_BUG_ON(sizeof(struct cached_item) < sizeof(struct cached_range)); + + first = shrink_boundary(sb, item, &prev, rng->start, false); + last = shrink_boundary(sb, item, &next, rng->end, true); + + trace_scoutfs_item_shrink_around(sb, rng->start, rng->end, item->key, + prev ? prev->key : NULL, + first ? first->key : NULL, + last ? last->key : NULL, + next ? next->key : NULL); + + /* can't shrink if we can't use neighbours */ + if (!first || !last) { + scoutfs_inc_counter(sb, item_shrink_alone); + return 0; + } + + /* can't split if we don't have an item to use for the range */ + if (next && prev && (first == last)) { + scoutfs_inc_counter(sb, item_shrink_small_split); + return 0; + } + + /* set end of remaining existing range, save old for split or freeing */ + if (prev) { + rng_end = rng->end; + rng->end = first->key; + first->key = NULL; + scoutfs_key_dec_cur_len(rng->end); + } + + /* set start of remaining existing range */ + if (next && !prev) { + scoutfs_key_free(sb, rng->start); + rng->start = last->key; + last->key = NULL; + scoutfs_key_inc_cur_len(rng->start); + } + + /* add new range, stealing existing end */ + if (next && prev) { + item = last; + last = rb_prev_item(last); + + unlink_item(sb, cac, item); + key = item->key; + scoutfs_kvec_kfree(item->val); + nr++; + + new_rng = (void *)item; + item = NULL; + memset(new_rng, 0, sizeof(struct cached_range)); + + new_rng->end = rng_end; + rng_end = NULL; + new_rng->start = key; + scoutfs_key_inc_cur_len(new_rng->start); + insert_range(sb, &cac->ranges, new_rng); + + scoutfs_inc_counter(sb, item_shrink_split_range); + } + + /* totally emptied the range */ + if (!prev && !next) { + rb_erase(&rng->node, &cac->ranges); + free_range(sb, rng); + } + + /* and finally shrink all the surrounding items */ + for (item = first; + item && (next = item == last ? NULL : rb_next_item(item), 1); + item = next) { + if (item->key) + trace_scoutfs_item_shrink(sb, item->key); + scoutfs_inc_counter(sb, item_shrink); + erase_item(sb, cac, item); + nr++; + } + + scoutfs_key_free(sb, rng_end); + + return nr; +} + +/* + * Shrink the item cache. * * Unfortunately this is complicated by the rbtree of ranges that track * the validity of the cache. If we free items we have to make sure * they're not covered by ranges or else they'd be considered a valid - * negative cache hit. We don't want to allocate more memory for new - * range entries that would be required to poke holes int he cached - * range. + * negative cache hit. We aggressively try to free items because if we + * have a structural pattern of keys that we can't free then those build + * up and fill memory. * - * So instead of just freeing the oldest item we shrink the range that - * contains the oldest item. We bias towards freeing the lesser side of - * the range. - * - * Instead of allocating a new range start key we use the key of the - * item we're removing. We have to increment it past the removed key - * value. That increment can move it past the next key in the range if - * the next key is of higher precision. This will be rare and can't go - * on indefinitely so we keep searching until we can inc a key and not - * extend past the next item. Eventually we have a range of items to - * free. - * - * During all of this, we chose to abort if we see dirty items. They - * won't be dirty forever and the mm can call back in. - * - * We can also hit items in the lru which aren't covered by ranges. We - * just free them straight away. And finally if we're completely out of - * items we walk and free the ranges. + * We can also hit items in the lru which aren't covered by ranges, we + * free those immediately. */ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) { @@ -1750,9 +1936,7 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) struct super_block *sb = cac->sb; struct cached_range *rng; struct cached_item *item; - struct cached_item *next; - struct cached_item *begin; - struct cached_item *end; + struct cached_item *first_moved = NULL; unsigned long flags; unsigned long nr; int ret; @@ -1763,24 +1947,9 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) spin_lock_irqsave(&cac->lock, flags); - while (nr > 0) { - item = list_first_entry_or_null(&cac->lru_list, - struct cached_item, entry); - - /* no lru items, if no items at all then free ranges */ - if (!item) { - if (!RB_EMPTY_ROOT(&cac->items)) { - scoutfs_inc_counter(sb, item_shrink_dirty_abort); - goto abort; - } - rng = rb_first_rng(&cac->ranges); - if (!rng) - break; - scoutfs_inc_counter(sb, item_shrink_no_items); - begin = NULL; - end = NULL; - goto free; - } + while (nr && + (item = list_first_entry_or_null(&cac->lru_list, + struct cached_item, entry))) { /* can't have dirty items on the lru */ BUG_ON(item->dirty & ITEM_DIRTY); @@ -1788,71 +1957,33 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) /* if we're not in a range just shrink the item */ rng = walk_ranges(&cac->ranges, item->key, NULL, NULL); if (!rng) { - begin = item; - end = item; scoutfs_inc_counter(sb, item_shrink_outside); - goto free; - } - - /* find the string of items to free, ending with range start */ - item = next_item(&cac->items, rng->start); - begin = item; - end = item; - - while (item) { - /* can't if it's dirty :( */ - if (item->dirty & ITEM_DIRTY) { - scoutfs_inc_counter(sb, item_shrink_dirty_abort); - goto abort; - } - - /* we're going to free this item now */ - end = item; - - /* free items and range if we exhausted the range */ - next = rb_next_item(item); - if (!next || scoutfs_key_compare(next->key, rng->end) > 0) - break; - - /* truncate range using after our key as start, if safe */ - scoutfs_key_inc_cur_len(item->key); - if (scoutfs_key_compare(item->key, next->key) <= 0) { - trace_scoutfs_item_shrink(sb, item->key); - scoutfs_key_free(sb, rng->start); - rng->start = item->key; - item->key = NULL; - rng = NULL; - break; - } - scoutfs_key_dec_cur_len(item->key); - - /* keep searching for valid range start key */ - scoutfs_inc_counter(sb, item_shrink_skip_inced); - item = next; - } - -free: - if (rng) { - trace_scoutfs_item_shrink_range(sb, rng->start, rng->end); - scoutfs_inc_counter(sb, item_shrink_range); - rb_erase(&rng->node, &cac->ranges); - free_range(sb, rng); - } - - /* free items from begin to end */ - for (item = begin; - item && (next = item == end ? NULL : rb_next_item(item), 1); - item = next) { - if (item->key) - trace_scoutfs_item_shrink(sb, item->key); - scoutfs_inc_counter(sb, item_shrink); erase_item(sb, cac, item); + nr--; + continue; } - nr--; + ret = shrink_around(sb, rng, item); + if (ret == 0) { + if (first_moved && first_moved == item) + break; + else if (!first_moved) + first_moved = item; + list_move_tail(&item->entry, &cac->lru_list); + continue; + } + + nr -= min_t(unsigned long, nr, ret); + } + + /* always try to free empty ranges */ + while (RB_EMPTY_ROOT(&cac->items) && + (rng = rb_first_rng(&cac->ranges))) { + scoutfs_inc_counter(sb, item_shrink_empty_range); + rb_erase(&rng->node, &cac->ranges); + free_range(sb, rng); } -abort: spin_unlock_irqrestore(&cac->lock, flags); out: diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 461475e1..678e7c69 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1682,6 +1682,41 @@ DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_item_shrink_exit, TP_ARGS(sb, nr_to_scan, ret) ); +TRACE_EVENT(scoutfs_item_shrink_around, + TP_PROTO(struct super_block *sb, + struct scoutfs_key_buf *rng_start, + struct scoutfs_key_buf *rng_end, struct scoutfs_key_buf *item, + struct scoutfs_key_buf *prev, struct scoutfs_key_buf *first, + struct scoutfs_key_buf *last, struct scoutfs_key_buf *next), + TP_ARGS(sb, rng_start, rng_end, item, prev, first, last, next), + TP_STRUCT__entry( + __field(void *, sb) + __dynamic_array(char, rng_start, + scoutfs_key_str(NULL, rng_start)) + __dynamic_array(char, rng_end, + scoutfs_key_str(NULL, rng_end)) + __dynamic_array(char, item, scoutfs_key_str(NULL, item)) + __dynamic_array(char, prev, scoutfs_key_str(NULL, prev)) + __dynamic_array(char, first, scoutfs_key_str(NULL, first)) + __dynamic_array(char, last, scoutfs_key_str(NULL, last)) + __dynamic_array(char, next, scoutfs_key_str(NULL, next)) + ), + TP_fast_assign( + __entry->sb = sb; + scoutfs_key_str(__get_dynamic_array(rng_start), rng_start); + scoutfs_key_str(__get_dynamic_array(rng_end), rng_end); + scoutfs_key_str(__get_dynamic_array(item), item); + scoutfs_key_str(__get_dynamic_array(prev), prev); + scoutfs_key_str(__get_dynamic_array(first), first); + scoutfs_key_str(__get_dynamic_array(last), last); + scoutfs_key_str(__get_dynamic_array(next), next); + ), + TP_printk("sb %p rng_start %s rng_end %s item %s prev %s first %s last %s next %s", + __entry->sb, __get_str(rng_start), __get_str(rng_end), + __get_str(item), __get_str(prev), __get_str(first), + __get_str(last), __get_str(next)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 4bc565be39f1f46b984525261a45377153dbb38c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Sep 2017 11:39:21 -0700 Subject: [PATCH 455/920] scoutfs: silence bulk_alloc gcc warning Some versions of gcc correctly noticed that bulk_alloc() had a case where it wouldn't initialize ret if the first segno was 0. This won't happen because the client response processing returns an error in this case. So this just shuts up the warning. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 9b72d18b..68ca9345 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -789,7 +789,7 @@ static struct task_cursor *get_cursor(struct data_info *datinf) static int bulk_alloc(struct super_block *sb) { u64 *segnos = NULL; - int ret; + int ret = 0; int i; segnos = scoutfs_client_bulk_alloc(sb); From b6c592f0993cc73d8d287e34aecb68e4a9efb807 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Sep 2017 15:47:37 -0700 Subject: [PATCH 456/920] scoutfs: don't dirty btree buffers The btree sets some buffer head flags before it writes them to satisfy submit_bh() requirements. It was setting dirty which wasn't required. Nothing every cleared dirty so those buffers sat around and were never freed. Each btree block we wrote sat around forever. Eventually the vm gets clogged up and the world backs up trying to allocate pages to write and we see massive stalls. With this fix we no longer see the 'buffers' vm stat continously grow and IO rates are consistent over time. Signed-off-by: Zach Brown --- kmod/src/btree.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 658f7e63..a64e6337 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1844,7 +1844,6 @@ int scoutfs_btree_write_dirty(struct super_block *sb) for_each_dirty_bh(bti, bh, tmp) { lock_buffer(bh); - set_buffer_dirty(bh); set_buffer_mapped(bh); bh->b_end_io = end_buffer_write_sync; get_bh(bh); From ccf5301c37dd785d4ad3b86f9b5f30ef2e239d1b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 10:39:46 -0700 Subject: [PATCH 457/920] scoutfs: add -Werror for build errors We insist on a warning free build but it's up to human diligence to discover and address warnings. We've also caught errors when compilers in automated testing saw problems that the compilers in developer environments didn't. That is, a human only could have noticed by investigating the output from successful test runs. Let's put some weight behind our promise of a warning free build and turn gcc warnings into errors. Signed-off-by: Zach Brown --- kmod/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/Makefile b/kmod/Makefile index bb162626..e8e45a4a 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -17,7 +17,8 @@ SCOUTFS_GIT_DESCRIBE := \ echo not-in-a-git-repository) SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ - CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src + CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ + EXTRA_CFLAGS=-Werror all: module From 17c6025cb7871763c5710a3f115376918d968088 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 26 Sep 2017 12:28:41 -0500 Subject: [PATCH 458/920] scoutfs: clean up some comments in lock.c We finished the lock lru work and can remove these TODO comments. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index acd60e6c..d89ea6ac 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -173,7 +173,6 @@ static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) static struct ocfs2_lock_res_ops scoufs_ino_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, - /* XXX: .post_unlock for lru */ /* XXX: .check_downconvert that queries the item cache for dirty items */ .flags = LOCK_TYPE_REQUIRES_REFRESH, }; @@ -181,14 +180,12 @@ static struct ocfs2_lock_res_ops scoufs_ino_lops = { static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, - /* XXX: .post_unlock for lru */ /* XXX: .check_downconvert that queries the item cache for dirty items */ .flags = 0, }; static struct ocfs2_lock_res_ops scoutfs_global_lops = { .get_osb = get_ino_lock_osb, - /* XXX: .post_unlock for lru */ /* XXX: .check_downconvert that queries the item cache for dirty items */ .flags = 0, }; From c5e6676b041a2fbb2dec74229b4e995a8bb7d538 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 26 Sep 2017 14:23:05 -0500 Subject: [PATCH 459/920] scoutfs: remove some ifdef'd out dlmglue code We can dump the ocfs2-specifics as well as any definitions that have been exported via the header file. This makes reading through and modifying dlmglue much more palatable. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 1734 -------------------------------------------- 1 file changed, 1734 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 7f729239..ffcf264d 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -57,31 +57,6 @@ BUG(); \ } \ } while (0) -#if 0 -#define MLOG_MASK_PREFIX ML_DLM_GLUE -#include - -#include "ocfs2.h" -#include "ocfs2_lockingver.h" - -#include "alloc.h" -#include "dcache.h" -#include "dlmglue.h" -#include "extent_map.h" -#include "file.h" -#include "heartbeat.h" -#include "inode.h" -#include "journal.h" -#include "stackglue.h" -#include "slot_map.h" -#include "super.h" -#include "uptodate.h" -#include "quota.h" -#include "refcounttree.h" -#include "acl.h" - -#include "buffer_head_io.h" -#endif struct ocfs2_mask_waiter { struct list_head mw_item; @@ -94,29 +69,6 @@ struct ocfs2_mask_waiter { #endif }; -#if 0 -static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres); -static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres); -static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres); -static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres); -#endif - -#if 0 -/* - * Return value from ->downconvert_worker functions. - * - * These control the precise actions of ocfs2_unblock_lock() - * and ocfs2_process_blocked_lock() - * - */ -enum ocfs2_unblock_action { - UNBLOCK_CONTINUE = 0, /* Continue downconvert */ - UNBLOCK_CONTINUE_POST = 1, /* Continue downconvert, fire - * ->post_unlock callback */ - UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire - * ->post_unlock() callback. */ -}; -#endif struct ocfs2_unblock_ctl { int requeue; enum ocfs2_unblock_action unblock_action; @@ -127,241 +79,10 @@ struct ocfs2_unblock_ctl { struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; #endif -#if 0 -static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, - int new_level); -static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres); - -static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, - int blocking); - -static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, - int blocking); - -static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); - -static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres); - -static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, - int new_level); -static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, - int blocking); - -#define mlog_meta_lvb(__level, __lockres) ocfs2_dump_meta_lvb_info(__level, __PRETTY_FUNCTION__, __LINE__, __lockres) - -/* This aids in debugging situations where a bad LVB might be involved. */ -static void ocfs2_dump_meta_lvb_info(u64 level, - const char *function, - unsigned int line, - struct ocfs2_lock_res *lockres) -{ - struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - - mlog(level, "LVB information for %s (called from %s:%u):\n", - lockres->l_name, function, line); - mlog(level, "version: %u, clusters: %u, generation: 0x%x\n", - lvb->lvb_version, be32_to_cpu(lvb->lvb_iclusters), - be32_to_cpu(lvb->lvb_igeneration)); - mlog(level, "size: %llu, uid %u, gid %u, mode 0x%x\n", - (unsigned long long)be64_to_cpu(lvb->lvb_isize), - be32_to_cpu(lvb->lvb_iuid), be32_to_cpu(lvb->lvb_igid), - be16_to_cpu(lvb->lvb_imode)); - mlog(level, "nlink %u, atime_packed 0x%llx, ctime_packed 0x%llx, " - "mtime_packed 0x%llx iattr 0x%x\n", be16_to_cpu(lvb->lvb_inlink), - (long long)be64_to_cpu(lvb->lvb_iatime_packed), - (long long)be64_to_cpu(lvb->lvb_ictime_packed), - (long long)be64_to_cpu(lvb->lvb_imtime_packed), - be32_to_cpu(lvb->lvb_iattr)); -} - - -/* - * OCFS2 Lock Resource Operations - * - * These fine tune the behavior of the generic dlmglue locking infrastructure. - * - * The most basic of lock types can point ->l_priv to their respective - * struct ocfs2_super and allow the default actions to manage things. - * - * Right now, each lock type also needs to implement an init function, - * and trivial lock/unlock wrappers. ocfs2_simple_drop_lockres() - * should be called when the lock is no longer needed (i.e., object - * destruction time). - */ -struct ocfs2_lock_res_ops { - /* - * Translate an ocfs2_lock_res * into an ocfs2_super *. Define - * this callback if ->l_priv is not an ocfs2_super pointer - */ - struct ocfs2_super * (*get_osb)(struct ocfs2_lock_res *); - - /* - * Optionally called in the downconvert thread after a - * successful downconvert. The lockres will not be referenced - * after this callback is called, so it is safe to free - * memory, etc. - * - * The exact semantics of when this is called are controlled - * by ->downconvert_worker() - */ - void (*post_unlock)(struct ocfs2_super *, struct ocfs2_lock_res *); - - /* - * Allow a lock type to add checks to determine whether it is - * safe to downconvert a lock. Return 0 to re-queue the - * downconvert at a later time, nonzero to continue. - * - * For most locks, the default checks that there are no - * incompatible holders are sufficient. - * - * Called with the lockres spinlock held. - */ - int (*check_downconvert)(struct ocfs2_lock_res *, int); - - /* - * Allows a lock type to populate the lock value block. This - * is called on downconvert, and when we drop a lock. - * - * Locks that want to use this should set LOCK_TYPE_USES_LVB - * in the flags field. - * - * Called with the lockres spinlock held. - */ - void (*set_lvb)(struct ocfs2_lock_res *); - - /* - * Called from the downconvert thread when it is determined - * that a lock will be downconverted. This is called without - * any locks held so the function can do work that might - * schedule (syncing out data, etc). - * - * This should return any one of the ocfs2_unblock_action - * values, depending on what it wants the thread to do. - */ - int (*downconvert_worker)(struct ocfs2_lock_res *, int); - - /* - * LOCK_TYPE_* flags which describe the specific requirements - * of a lock type. Descriptions of each individual flag follow. - */ - int flags; -}; - -/* - * Some locks want to "refresh" potentially stale data when a - * meaningful (PRMODE or EXMODE) lock level is first obtained. If this - * flag is set, the OCFS2_LOCK_NEEDS_REFRESH flag will be set on the - * individual lockres l_flags member from the ast function. It is - * expected that the locking wrapper will clear the - * OCFS2_LOCK_NEEDS_REFRESH flag when done. - */ -#define LOCK_TYPE_REQUIRES_REFRESH 0x1 - -/* - * Indicate that a lock type makes use of the lock value block. The - * ->set_lvb lock type callback must be defined. - */ -#define LOCK_TYPE_USES_LVB 0x2 - -static struct ocfs2_lock_res_ops ocfs2_inode_rw_lops = { - .get_osb = ocfs2_get_inode_osb, - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_inode_inode_lops = { - .get_osb = ocfs2_get_inode_osb, - .check_downconvert = ocfs2_check_meta_downconvert, - .set_lvb = ocfs2_set_meta_lvb, - .downconvert_worker = ocfs2_data_convert_worker, - .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, -}; - -static struct ocfs2_lock_res_ops ocfs2_super_lops = { - .flags = LOCK_TYPE_REQUIRES_REFRESH, -}; - -static struct ocfs2_lock_res_ops ocfs2_rename_lops = { - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_nfs_sync_lops = { - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_orphan_scan_lops = { - .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, -}; - -static struct ocfs2_lock_res_ops ocfs2_dentry_lops = { - .get_osb = ocfs2_get_dentry_osb, - .post_unlock = ocfs2_dentry_post_unlock, - .downconvert_worker = ocfs2_dentry_convert_worker, - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_inode_open_lops = { - .get_osb = ocfs2_get_inode_osb, - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_flock_lops = { - .get_osb = ocfs2_get_file_osb, - .flags = 0, -}; - -static struct ocfs2_lock_res_ops ocfs2_qinfo_lops = { - .set_lvb = ocfs2_set_qinfo_lvb, - .get_osb = ocfs2_get_qinfo_osb, - .flags = LOCK_TYPE_REQUIRES_REFRESH | LOCK_TYPE_USES_LVB, -}; - -static struct ocfs2_lock_res_ops ocfs2_refcount_block_lops = { - .check_downconvert = ocfs2_check_refcount_downconvert, - .downconvert_worker = ocfs2_refcount_convert_worker, - .flags = 0, -}; - -static inline int ocfs2_is_inode_lock(struct ocfs2_lock_res *lockres) -{ - return lockres->l_type == OCFS2_LOCK_TYPE_META || - lockres->l_type == OCFS2_LOCK_TYPE_RW || - lockres->l_type == OCFS2_LOCK_TYPE_OPEN; -} -#endif - static inline struct ocfs2_lock_res *ocfs2_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) { return container_of(lksb, struct ocfs2_lock_res, l_lksb); } -#if 0 -static inline struct inode *ocfs2_lock_res_inode(struct ocfs2_lock_res *lockres) -{ - BUG_ON(!ocfs2_is_inode_lock(lockres)); - - return (struct inode *) lockres->l_priv; -} - -static inline struct ocfs2_dentry_lock *ocfs2_lock_res_dl(struct ocfs2_lock_res *lockres) -{ - BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_DENTRY); - - return (struct ocfs2_dentry_lock *)lockres->l_priv; -} - -static inline struct ocfs2_mem_dqinfo *ocfs2_lock_res_qinfo(struct ocfs2_lock_res *lockres) -{ - BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_QINFO); - - return (struct ocfs2_mem_dqinfo *)lockres->l_priv; -} - -static inline struct ocfs2_refcount_tree * -ocfs2_lock_res_refcount_tree(struct ocfs2_lock_res *res) -{ - return container_of(res, struct ocfs2_refcount_tree, rf_lockres); -} -#endif static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *lockres) { @@ -371,12 +92,6 @@ static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *l return (struct ocfs2_super *)lockres->l_priv; } -#if 0 -static int ocfs2_lock_create(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - u32 dlm_flags); -#endif static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, int wanted); static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, @@ -415,11 +130,6 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, static int ocfs2_downconvert_thread(void *arg); static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); -#if 0 -static int ocfs2_inode_lock_update(struct inode *inode, - struct buffer_head **bh); -static void ocfs2_drop_osb_locks(struct ocfs2_super *osb); -#endif static inline int ocfs2_highest_compat_lock_level(int level); static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, int new_level); @@ -433,25 +143,6 @@ static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, static int ocfs2_cancel_convert(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); -#if 0 -static void ocfs2_build_lock_name(enum ocfs2_lock_type type, - u64 blkno, - u32 generation, - char *name) -{ - int len; - - BUG_ON(type >= OCFS2_NUM_LOCK_TYPES); - - len = snprintf(name, OCFS2_LOCK_ID_MAX_LEN, "%c%s%016llx%08x", - ocfs2_lock_type_char(type), OCFS2_LOCK_ID_PAD, - (long long)blkno, generation); - - BUG_ON(len != (OCFS2_LOCK_ID_MAX_LEN - 1)); - - mlog(0, "built lock resource with name: %s\n", name); -} -#endif static DEFINE_SPINLOCK(ocfs2_dlm_tracking_lock); static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, @@ -578,194 +269,6 @@ void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) INIT_LIST_HEAD(&res->l_debug_list); } -#if 0 -void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, - enum ocfs2_lock_type type, - unsigned int generation, - struct inode *inode) -{ - struct ocfs2_lock_res_ops *ops; - - switch(type) { - case OCFS2_LOCK_TYPE_RW: - ops = &ocfs2_inode_rw_lops; - break; - case OCFS2_LOCK_TYPE_META: - ops = &ocfs2_inode_inode_lops; - break; - case OCFS2_LOCK_TYPE_OPEN: - ops = &ocfs2_inode_open_lops; - break; - default: - mlog_bug_on_msg(1, "type: %d\n", type); - ops = NULL; /* thanks, gcc */ - break; - }; - - ocfs2_build_lock_name(type, OCFS2_I(inode)->ip_blkno, - generation, res->l_name); - ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), res, type, ops, inode); -} - -static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres) -{ - struct inode *inode = ocfs2_lock_res_inode(lockres); - - return OCFS2_SB(inode->i_sb); -} - -static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_mem_dqinfo *info = lockres->l_priv; - - return OCFS2_SB(info->dqi_gi.dqi_sb); -} - -static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_file_private *fp = lockres->l_priv; - - return OCFS2_SB(fp->fp_file->f_mapping->host->i_sb); -} - -static __u64 ocfs2_get_dentry_lock_ino(struct ocfs2_lock_res *lockres) -{ - __be64 inode_blkno_be; - - memcpy(&inode_blkno_be, &lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], - sizeof(__be64)); - - return be64_to_cpu(inode_blkno_be); -} - -static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_dentry_lock *dl = lockres->l_priv; - - return OCFS2_SB(dl->dl_inode->i_sb); -} - -void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, - u64 parent, struct inode *inode) -{ - int len; - u64 inode_blkno = OCFS2_I(inode)->ip_blkno; - __be64 inode_blkno_be = cpu_to_be64(inode_blkno); - struct ocfs2_lock_res *lockres = &dl->dl_lockres; - - ocfs2_lock_res_init_once(lockres); - - /* - * Unfortunately, the standard lock naming scheme won't work - * here because we have two 16 byte values to use. Instead, - * we'll stuff the inode number as a binary value. We still - * want error prints to show something without garbling the - * display, so drop a null byte in there before the inode - * number. A future version of OCFS2 will likely use all - * binary lock names. The stringified names have been a - * tremendous aid in debugging, but now that the debugfs - * interface exists, we can mangle things there if need be. - * - * NOTE: We also drop the standard "pad" value (the total lock - * name size stays the same though - the last part is all - * zeros due to the memset in ocfs2_lock_res_init_once() - */ - len = snprintf(lockres->l_name, OCFS2_DENTRY_LOCK_INO_START, - "%c%016llx", - ocfs2_lock_type_char(OCFS2_LOCK_TYPE_DENTRY), - (long long)parent); - - BUG_ON(len != (OCFS2_DENTRY_LOCK_INO_START - 1)); - - memcpy(&lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], &inode_blkno_be, - sizeof(__be64)); - - ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, - OCFS2_LOCK_TYPE_DENTRY, &ocfs2_dentry_lops, - dl); -} - -static void ocfs2_super_lock_res_init(struct ocfs2_lock_res *res, - struct ocfs2_super *osb) -{ - /* Superblock lockres doesn't come from a slab so we call init - * once on it manually. */ - ocfs2_lock_res_init_once(res); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_SUPER, OCFS2_SUPER_BLOCK_BLKNO, - 0, res->l_name); - ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_SUPER, - &ocfs2_super_lops, osb); -} - -static void ocfs2_rename_lock_res_init(struct ocfs2_lock_res *res, - struct ocfs2_super *osb) -{ - /* Rename lockres doesn't come from a slab so we call init - * once on it manually. */ - ocfs2_lock_res_init_once(res); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_RENAME, 0, 0, res->l_name); - ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_RENAME, - &ocfs2_rename_lops, osb); -} - -static void ocfs2_nfs_sync_lock_res_init(struct ocfs2_lock_res *res, - struct ocfs2_super *osb) -{ - /* nfs_sync lockres doesn't come from a slab so we call init - * once on it manually. */ - ocfs2_lock_res_init_once(res); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_NFS_SYNC, 0, 0, res->l_name); - ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_NFS_SYNC, - &ocfs2_nfs_sync_lops, osb); -} - -static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res, - struct ocfs2_super *osb) -{ - ocfs2_lock_res_init_once(res); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_ORPHAN_SCAN, 0, 0, res->l_name); - ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_ORPHAN_SCAN, - &ocfs2_orphan_scan_lops, osb); -} - -void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_file_private *fp) -{ - struct inode *inode = fp->fp_file->f_mapping->host; - struct ocfs2_inode_info *oi = OCFS2_I(inode); - - ocfs2_lock_res_init_once(lockres); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_FLOCK, oi->ip_blkno, - inode->i_generation, lockres->l_name); - ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, - OCFS2_LOCK_TYPE_FLOCK, &ocfs2_flock_lops, - fp); - lockres->l_flags |= OCFS2_LOCK_NOCACHE; -} - -void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_mem_dqinfo *info) -{ - ocfs2_lock_res_init_once(lockres); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_QINFO, info->dqi_gi.dqi_type, - 0, lockres->l_name); - ocfs2_lock_res_init_common(OCFS2_SB(info->dqi_gi.dqi_sb), lockres, - OCFS2_LOCK_TYPE_QINFO, &ocfs2_qinfo_lops, - info); -} - -void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, - struct ocfs2_super *osb, u64 ref_blkno, - unsigned int generation) -{ - ocfs2_lock_res_init_once(lockres); - ocfs2_build_lock_name(OCFS2_LOCK_TYPE_REFCOUNT, ref_blkno, - generation, lockres->l_name); - ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_REFCOUNT, - &ocfs2_refcount_block_lops, osb); -} -#endif - void ocfs2_lock_res_free(struct ocfs2_lock_res *res) { if (!(res->l_flags & OCFS2_LOCK_INITIALIZED)) @@ -1143,11 +646,6 @@ static void ocfs2_blocking_ast(struct ocfs2_dlm_lksb *lksb, int level) BUG_ON(level <= DLM_LOCK_NL); -#if 0 - mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d, " - "type %s\n", lockres->l_name, level, lockres->l_level, - ocfs2_lock_type_string(lockres->l_type)); -#endif mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d\n", lockres->l_name, level, lockres->l_level); @@ -1750,179 +1248,6 @@ static int ocfs2_create_new_lock(struct ocfs2_super *osb, #endif #if 0 -/* Grants us an EX lock on the data and metadata resources, skipping - * the normal cluster directory lookup. Use this ONLY on newly created - * inodes which other nodes can't possibly see, and which haven't been - * hashed in the inode hash yet. This can give us a good performance - * increase as it'll skip the network broadcast normally associated - * with creating a new lock resource. */ -int ocfs2_create_new_inode_locks(struct inode *inode) -{ - int ret; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - BUG_ON(!ocfs2_inode_is_new(inode)); - - mlog(0, "Inode %llu\n", (unsigned long long)OCFS2_I(inode)->ip_blkno); - - /* NOTE: That we don't increment any of the holder counts, nor - * do we add anything to a journal handle. Since this is - * supposed to be a new inode which the cluster doesn't know - * about yet, there is no need to. As far as the LVB handling - * is concerned, this is basically like acquiring an EX lock - * on a resource which has an invalid one -- we'll set it - * valid when we release the EX. */ - - ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_rw_lockres, 1, 1); - if (ret) { - mlog_errno(ret); - goto bail; - } - - /* - * We don't want to use DLM_LKF_LOCAL on a meta data lock as they - * don't use a generation in their lock names. - */ - ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_inode_lockres, 1, 0); - if (ret) { - mlog_errno(ret); - goto bail; - } - - ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_open_lockres, 0, 0); - if (ret) - mlog_errno(ret); - -bail: - return ret; -} - -int ocfs2_rw_lock(struct inode *inode, int write) -{ - int status, level; - struct ocfs2_lock_res *lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu take %s RW lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - write ? "EXMODE" : "PRMODE"); - - if (ocfs2_mount_local(osb)) - return 0; - - lockres = &OCFS2_I(inode)->ip_rw_lockres; - - level = write ? DLM_LOCK_EX : DLM_LOCK_PR; - - status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, level, 0, - 0); - if (status < 0) - mlog_errno(status); - - return status; -} - -void ocfs2_rw_unlock(struct inode *inode, int write) -{ - int level = write ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_rw_lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu drop %s RW lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - write ? "EXMODE" : "PRMODE"); - - if (!ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, level); -} - -/* - * ocfs2_open_lock always get PR mode lock. - */ -int ocfs2_open_lock(struct inode *inode) -{ - int status = 0; - struct ocfs2_lock_res *lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu take PRMODE open lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno); - - if (ocfs2_is_hard_readonly(osb) || ocfs2_mount_local(osb)) - goto out; - - lockres = &OCFS2_I(inode)->ip_open_lockres; - - status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, - DLM_LOCK_PR, 0, 0); - if (status < 0) - mlog_errno(status); - -out: - return status; -} - -int ocfs2_try_open_lock(struct inode *inode, int write) -{ - int status = 0, level; - struct ocfs2_lock_res *lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu try to take %s open lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - write ? "EXMODE" : "PRMODE"); - - if (ocfs2_is_hard_readonly(osb)) { - if (write) - status = -EROFS; - goto out; - } - - if (ocfs2_mount_local(osb)) - goto out; - - lockres = &OCFS2_I(inode)->ip_open_lockres; - - level = write ? DLM_LOCK_EX : DLM_LOCK_PR; - - /* - * The file system may already holding a PRMODE/EXMODE open lock. - * Since we pass DLM_LKF_NOQUEUE, the request won't block waiting on - * other nodes and the -EAGAIN will indicate to the caller that - * this inode is still in use. - */ - status = ocfs2_cluster_lock(OCFS2_SB(inode->i_sb), lockres, - level, DLM_LKF_NOQUEUE, 0); - -out: - return status; -} - -/* - * ocfs2_open_unlock unlock PR and EX mode open locks. - */ -void ocfs2_open_unlock(struct inode *inode) -{ - struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_open_lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu drop open lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno); - - if (ocfs2_mount_local(osb)) - goto out; - - if(lockres->l_ro_holders) - ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, - DLM_LOCK_PR); - if(lockres->l_ex_holders) - ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, - DLM_LOCK_EX); - -out: - return; -} - static int ocfs2_flock_handle_signal(struct ocfs2_lock_res *lockres, int level) { @@ -2159,125 +1484,6 @@ static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, ocfs2_wake_downconvert_thread(osb); } -#if 0 -#define OCFS2_SEC_BITS 34 -#define OCFS2_SEC_SHIFT (64 - 34) -#define OCFS2_NSEC_MASK ((1ULL << OCFS2_SEC_SHIFT) - 1) - -/* LVB only has room for 64 bits of time here so we pack it for - * now. */ -static u64 ocfs2_pack_timespec(struct timespec *spec) -{ - u64 res; - u64 sec = spec->tv_sec; - u32 nsec = spec->tv_nsec; - - res = (sec << OCFS2_SEC_SHIFT) | (nsec & OCFS2_NSEC_MASK); - - return res; -} - -/* Call this with the lockres locked. I am reasonably sure we don't - * need ip_lock in this function as anyone who would be changing those - * values is supposed to be blocked in ocfs2_inode_lock right now. */ -static void __ocfs2_stuff_meta_lvb(struct inode *inode) -{ - struct ocfs2_inode_info *oi = OCFS2_I(inode); - struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; - struct ocfs2_meta_lvb *lvb; - - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - - /* - * Invalidate the LVB of a deleted inode - this way other - * nodes are forced to go to disk and discover the new inode - * status. - */ - if (oi->ip_flags & OCFS2_INODE_DELETED) { - lvb->lvb_version = 0; - goto out; - } - - lvb->lvb_version = OCFS2_LVB_VERSION; - lvb->lvb_isize = cpu_to_be64(i_size_read(inode)); - lvb->lvb_iclusters = cpu_to_be32(oi->ip_clusters); - lvb->lvb_iuid = cpu_to_be32(i_uid_read(inode)); - lvb->lvb_igid = cpu_to_be32(i_gid_read(inode)); - lvb->lvb_imode = cpu_to_be16(inode->i_mode); - lvb->lvb_inlink = cpu_to_be16(inode->i_nlink); - lvb->lvb_iatime_packed = - cpu_to_be64(ocfs2_pack_timespec(&inode->i_atime)); - lvb->lvb_ictime_packed = - cpu_to_be64(ocfs2_pack_timespec(&inode->i_ctime)); - lvb->lvb_imtime_packed = - cpu_to_be64(ocfs2_pack_timespec(&inode->i_mtime)); - lvb->lvb_iattr = cpu_to_be32(oi->ip_attr); - lvb->lvb_idynfeatures = cpu_to_be16(oi->ip_dyn_features); - lvb->lvb_igeneration = cpu_to_be32(inode->i_generation); - -out: - mlog_meta_lvb(0, lockres); -} - -static void ocfs2_unpack_timespec(struct timespec *spec, - u64 packed_time) -{ - spec->tv_sec = packed_time >> OCFS2_SEC_SHIFT; - spec->tv_nsec = packed_time & OCFS2_NSEC_MASK; -} - -static void ocfs2_refresh_inode_from_lvb(struct inode *inode) -{ - struct ocfs2_inode_info *oi = OCFS2_I(inode); - struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; - struct ocfs2_meta_lvb *lvb; - - mlog_meta_lvb(0, lockres); - - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - - /* We're safe here without the lockres lock... */ - spin_lock(&oi->ip_lock); - oi->ip_clusters = be32_to_cpu(lvb->lvb_iclusters); - i_size_write(inode, be64_to_cpu(lvb->lvb_isize)); - - oi->ip_attr = be32_to_cpu(lvb->lvb_iattr); - oi->ip_dyn_features = be16_to_cpu(lvb->lvb_idynfeatures); - ocfs2_set_inode_flags(inode); - - /* fast-symlinks are a special case */ - if (S_ISLNK(inode->i_mode) && !oi->ip_clusters) - inode->i_blocks = 0; - else - inode->i_blocks = ocfs2_inode_sector_count(inode); - - i_uid_write(inode, be32_to_cpu(lvb->lvb_iuid)); - i_gid_write(inode, be32_to_cpu(lvb->lvb_igid)); - inode->i_mode = be16_to_cpu(lvb->lvb_imode); - set_nlink(inode, be16_to_cpu(lvb->lvb_inlink)); - ocfs2_unpack_timespec(&inode->i_atime, - be64_to_cpu(lvb->lvb_iatime_packed)); - ocfs2_unpack_timespec(&inode->i_mtime, - be64_to_cpu(lvb->lvb_imtime_packed)); - ocfs2_unpack_timespec(&inode->i_ctime, - be64_to_cpu(lvb->lvb_ictime_packed)); - spin_unlock(&oi->ip_lock); -} - -static inline int ocfs2_meta_lvb_is_trustable(struct inode *inode, - struct ocfs2_lock_res *lockres) -{ - struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - - if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) - && lvb->lvb_version == OCFS2_LVB_VERSION - && be32_to_cpu(lvb->lvb_igeneration) == inode->i_generation) - return 1; - return 0; -} -#endif - - #if 0 /* Determine whether a lock resource needs to be refreshed, and * arbitrate who gets to refresh it. @@ -2337,542 +1543,6 @@ u64 ocfs2_lock_refresh_gen(struct ocfs2_lock_res *lockres) return lockres->l_refresh_gen; } -#if 0 -/* may or may not return a bh if it went to disk. */ -static int ocfs2_inode_lock_update(struct inode *inode, - struct buffer_head **bh) -{ - int status = 0; - struct ocfs2_inode_info *oi = OCFS2_I(inode); - struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; - struct ocfs2_dinode *fe; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - if (ocfs2_mount_local(osb)) - goto bail; - - spin_lock(&oi->ip_lock); - if (oi->ip_flags & OCFS2_INODE_DELETED) { - mlog(0, "Orphaned inode %llu was deleted while we " - "were waiting on a lock. ip_flags = 0x%x\n", - (unsigned long long)oi->ip_blkno, oi->ip_flags); - spin_unlock(&oi->ip_lock); - status = -ENOENT; - goto bail; - } - spin_unlock(&oi->ip_lock); - - if (!ocfs2_should_refresh_lock_res(lockres)) - goto bail; - - /* This will discard any caching information we might have had - * for the inode metadata. */ - ocfs2_metadata_cache_purge(INODE_CACHE(inode)); - - ocfs2_extent_map_trunc(inode, 0); - - if (ocfs2_meta_lvb_is_trustable(inode, lockres)) { - mlog(0, "Trusting LVB on inode %llu\n", - (unsigned long long)oi->ip_blkno); - ocfs2_refresh_inode_from_lvb(inode); - } else { - /* Boo, we have to go to disk. */ - /* read bh, cast, ocfs2_refresh_inode */ - status = ocfs2_read_inode_block(inode, bh); - if (status < 0) { - mlog_errno(status); - goto bail_refresh; - } - fe = (struct ocfs2_dinode *) (*bh)->b_data; - - /* This is a good chance to make sure we're not - * locking an invalid object. ocfs2_read_inode_block() - * already checked that the inode block is sane. - * - * We bug on a stale inode here because we checked - * above whether it was wiped from disk. The wiping - * node provides a guarantee that we receive that - * message and can mark the inode before dropping any - * locks associated with it. */ - mlog_bug_on_msg(inode->i_generation != - le32_to_cpu(fe->i_generation), - "Invalid dinode %llu disk generation: %u " - "inode->i_generation: %u\n", - (unsigned long long)oi->ip_blkno, - le32_to_cpu(fe->i_generation), - inode->i_generation); - mlog_bug_on_msg(le64_to_cpu(fe->i_dtime) || - !(fe->i_flags & cpu_to_le32(OCFS2_VALID_FL)), - "Stale dinode %llu dtime: %llu flags: 0x%x\n", - (unsigned long long)oi->ip_blkno, - (unsigned long long)le64_to_cpu(fe->i_dtime), - le32_to_cpu(fe->i_flags)); - - ocfs2_refresh_inode(inode, fe); - ocfs2_track_lock_refresh(lockres); - } - - status = 0; -bail_refresh: - ocfs2_complete_lock_res_refresh(lockres, status); -bail: - return status; -} - -static int ocfs2_assign_bh(struct inode *inode, - struct buffer_head **ret_bh, - struct buffer_head *passed_bh) -{ - int status; - - if (passed_bh) { - /* Ok, the update went to disk for us, use the - * returned bh. */ - *ret_bh = passed_bh; - get_bh(*ret_bh); - - return 0; - } - - status = ocfs2_read_inode_block(inode, ret_bh); - if (status < 0) - mlog_errno(status); - - return status; -} - -/* - * returns < 0 error if the callback will never be called, otherwise - * the result of the lock will be communicated via the callback. - */ -int ocfs2_inode_lock_full_nested(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - int arg_flags, - int subclass) -{ - int status, level, acquired; - u32 dlm_flags; - struct ocfs2_lock_res *lockres = NULL; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - struct buffer_head *local_bh = NULL; - - mlog(0, "inode %llu, take %s META lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - ex ? "EXMODE" : "PRMODE"); - - status = 0; - acquired = 0; - /* We'll allow faking a readonly metadata lock for - * rodevices. */ - if (ocfs2_is_hard_readonly(osb)) { - if (ex) - status = -EROFS; - goto getbh; - } - - if ((arg_flags & OCFS2_META_LOCK_GETBH) || - ocfs2_mount_local(osb)) - goto update; - - if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) - ocfs2_wait_for_recovery(osb); - - lockres = &OCFS2_I(inode)->ip_inode_lockres; - level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - dlm_flags = 0; - if (arg_flags & OCFS2_META_LOCK_NOQUEUE) - dlm_flags |= DLM_LKF_NOQUEUE; - - status = __ocfs2_cluster_lock(osb, lockres, level, dlm_flags, - arg_flags, subclass, _RET_IP_); - if (status < 0) { - if (status != -EAGAIN) - mlog_errno(status); - goto bail; - } - - /* Notify the error cleanup path to drop the cluster lock. */ - acquired = 1; - - /* We wait twice because a node may have died while we were in - * the lower dlm layers. The second time though, we've - * committed to owning this lock so we don't allow signals to - * abort the operation. */ - if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) - ocfs2_wait_for_recovery(osb); - -update: - /* - * We only see this flag if we're being called from - * ocfs2_read_locked_inode(). It means we're locking an inode - * which hasn't been populated yet, so clear the refresh flag - * and let the caller handle it. - */ - if (inode->i_state & I_NEW) { - status = 0; - if (lockres) - ocfs2_complete_lock_res_refresh(lockres, 0); - goto bail; - } - - /* This is fun. The caller may want a bh back, or it may - * not. ocfs2_inode_lock_update definitely wants one in, but - * may or may not read one, depending on what's in the - * LVB. The result of all of this is that we've *only* gone to - * disk if we have to, so the complexity is worthwhile. */ - status = ocfs2_inode_lock_update(inode, &local_bh); - if (status < 0) { - if (status != -ENOENT) - mlog_errno(status); - goto bail; - } -getbh: - if (ret_bh) { - status = ocfs2_assign_bh(inode, ret_bh, local_bh); - if (status < 0) { - mlog_errno(status); - goto bail; - } - } - -bail: - if (status < 0) { - if (ret_bh && (*ret_bh)) { - brelse(*ret_bh); - *ret_bh = NULL; - } - if (acquired) - ocfs2_inode_unlock(inode, ex); - } - - if (local_bh) - brelse(local_bh); - - return status; -} - -/* - * This is working around a lock inversion between tasks acquiring DLM - * locks while holding a page lock and the downconvert thread which - * blocks dlm lock acquiry while acquiring page locks. - * - * ** These _with_page variantes are only intended to be called from aop - * methods that hold page locks and return a very specific *positive* error - * code that aop methods pass up to the VFS -- test for errors with != 0. ** - * - * The DLM is called such that it returns -EAGAIN if it would have - * blocked waiting for the downconvert thread. In that case we unlock - * our page so the downconvert thread can make progress. Once we've - * done this we have to return AOP_TRUNCATED_PAGE so the aop method - * that called us can bubble that back up into the VFS who will then - * immediately retry the aop call. - */ -int ocfs2_inode_lock_with_page(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - struct page *page) -{ - int ret; - - ret = ocfs2_inode_lock_full(inode, ret_bh, ex, OCFS2_LOCK_NONBLOCK); - if (ret == -EAGAIN) { - unlock_page(page); - ret = AOP_TRUNCATED_PAGE; - } - - return ret; -} - -int ocfs2_inode_lock_atime(struct inode *inode, - struct vfsmount *vfsmnt, - int *level) -{ - int ret; - - ret = ocfs2_inode_lock(inode, NULL, 0); - if (ret < 0) { - mlog_errno(ret); - return ret; - } - - /* - * If we should update atime, we will get EX lock, - * otherwise we just get PR lock. - */ - if (ocfs2_should_update_atime(inode, vfsmnt)) { - struct buffer_head *bh = NULL; - - ocfs2_inode_unlock(inode, 0); - ret = ocfs2_inode_lock(inode, &bh, 1); - if (ret < 0) { - mlog_errno(ret); - return ret; - } - *level = 1; - if (ocfs2_should_update_atime(inode, vfsmnt)) - ocfs2_update_inode_atime(inode, bh); - if (bh) - brelse(bh); - } else - *level = 0; - - return ret; -} - -void ocfs2_inode_unlock(struct inode *inode, - int ex) -{ - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_inode_lockres; - struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - - mlog(0, "inode %llu drop %s META lock\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - ex ? "EXMODE" : "PRMODE"); - - if (!ocfs2_is_hard_readonly(OCFS2_SB(inode->i_sb)) && - !ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(OCFS2_SB(inode->i_sb), lockres, level); -} - -/* - * This _tracker variantes are introduced to deal with the recursive cluster - * locking issue. The idea is to keep track of a lock holder on the stack of - * the current process. If there's a lock holder on the stack, we know the - * task context is already protected by cluster locking. Currently, they're - * used in some VFS entry routines. - * - * return < 0 on error, return == 0 if there's no lock holder on the stack - * before this call, return == 1 if this call would be a recursive locking. - */ -int ocfs2_inode_lock_tracker(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - struct ocfs2_lock_holder *oh) -{ - int status; - int arg_flags = 0, has_locked; - struct ocfs2_lock_res *lockres; - - lockres = &OCFS2_I(inode)->ip_inode_lockres; - has_locked = ocfs2_is_locked_by_me(lockres); - /* Just get buffer head if the cluster lock has been taken */ - if (has_locked) - arg_flags = OCFS2_META_LOCK_GETBH; - - if (likely(!has_locked || ret_bh)) { - status = ocfs2_inode_lock_full(inode, ret_bh, ex, arg_flags); - if (status < 0) { - if (status != -ENOENT) - mlog_errno(status); - return status; - } - } - if (!has_locked) - ocfs2_add_holder(lockres, oh); - - return has_locked; -} - -void ocfs2_inode_unlock_tracker(struct inode *inode, - int ex, - struct ocfs2_lock_holder *oh, - int had_lock) -{ - struct ocfs2_lock_res *lockres; - - lockres = &OCFS2_I(inode)->ip_inode_lockres; - /* had_lock means that the currect process already takes the cluster - * lock previously. If had_lock is 1, we have nothing to do here, and - * it will get unlocked where we got the lock. - */ - if (!had_lock) { - ocfs2_remove_holder(lockres, oh); - ocfs2_inode_unlock(inode, ex); - } -} - -int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno) -{ - struct ocfs2_lock_res *lockres; - struct ocfs2_orphan_scan_lvb *lvb; - int status = 0; - - if (ocfs2_is_hard_readonly(osb)) - return -EROFS; - - if (ocfs2_mount_local(osb)) - return 0; - - lockres = &osb->osb_orphan_scan.os_lockres; - status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); - if (status < 0) - return status; - - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && - lvb->lvb_version == OCFS2_ORPHAN_LVB_VERSION) - *seqno = be32_to_cpu(lvb->lvb_os_seqno); - else - *seqno = osb->osb_orphan_scan.os_seqno + 1; - - return status; -} - -void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno) -{ - struct ocfs2_lock_res *lockres; - struct ocfs2_orphan_scan_lvb *lvb; - - if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) { - lockres = &osb->osb_orphan_scan.os_lockres; - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - lvb->lvb_version = OCFS2_ORPHAN_LVB_VERSION; - lvb->lvb_os_seqno = cpu_to_be32(seqno); - ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); - } -} - -int ocfs2_super_lock(struct ocfs2_super *osb, - int ex) -{ - int status = 0; - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; - - if (ocfs2_is_hard_readonly(osb)) - return -EROFS; - - if (ocfs2_mount_local(osb)) - goto bail; - - status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); - if (status < 0) { - mlog_errno(status); - goto bail; - } - - /* The super block lock path is really in the best position to - * know when resources covered by the lock need to be - * refreshed, so we do it here. Of course, making sense of - * everything is up to the caller :) */ - status = ocfs2_should_refresh_lock_res(lockres); - if (status) { - status = ocfs2_refresh_slot_info(osb); - - ocfs2_complete_lock_res_refresh(lockres, status); - - if (status < 0) { - ocfs2_cluster_unlock(osb, lockres, level); - mlog_errno(status); - } - ocfs2_track_lock_refresh(lockres); - } -bail: - return status; -} - -void ocfs2_super_unlock(struct ocfs2_super *osb, - int ex) -{ - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; - - if (!ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, lockres, level); -} - -int ocfs2_rename_lock(struct ocfs2_super *osb) -{ - int status; - struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; - - if (ocfs2_is_hard_readonly(osb)) - return -EROFS; - - if (ocfs2_mount_local(osb)) - return 0; - - status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); - if (status < 0) - mlog_errno(status); - - return status; -} - -void ocfs2_rename_unlock(struct ocfs2_super *osb) -{ - struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; - - if (!ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); -} - -int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex) -{ - int status; - struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; - - if (ocfs2_is_hard_readonly(osb)) - return -EROFS; - - if (ocfs2_mount_local(osb)) - return 0; - - status = ocfs2_cluster_lock(osb, lockres, ex ? LKM_EXMODE : LKM_PRMODE, - 0, 0); - if (status < 0) - mlog(ML_ERROR, "lock on nfs sync lock failed %d\n", status); - - return status; -} - -void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex) -{ - struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; - - if (!ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, lockres, - ex ? LKM_EXMODE : LKM_PRMODE); -} - -int ocfs2_dentry_lock(struct dentry *dentry, int ex) -{ - int ret; - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_dentry_lock *dl = dentry->d_fsdata; - struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); - - BUG_ON(!dl); - - if (ocfs2_is_hard_readonly(osb)) { - if (ex) - return -EROFS; - return 0; - } - - if (ocfs2_mount_local(osb)) - return 0; - - ret = ocfs2_cluster_lock(osb, &dl->dl_lockres, level, 0, 0); - if (ret < 0) - mlog_errno(ret); - - return ret; -} - -void ocfs2_dentry_unlock(struct dentry *dentry, int ex) -{ - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_dentry_lock *dl = dentry->d_fsdata; - struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); - - if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, &dl->dl_lockres, level); -} -#endif - /* Reference counting of the dlm debug structure. We want this because * open references on the debug inodes can live on after a mount, so * we can't rely on the ocfs2_super to always exist. */ @@ -3433,47 +2103,6 @@ void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, mlog_errno(ret); } -#if 0 -static void ocfs2_drop_osb_locks(struct ocfs2_super *osb) -{ - ocfs2_simple_drop_lockres(osb, &osb->osb_super_lockres); - ocfs2_simple_drop_lockres(osb, &osb->osb_rename_lockres); - ocfs2_simple_drop_lockres(osb, &osb->osb_nfs_sync_lockres); - ocfs2_simple_drop_lockres(osb, &osb->osb_orphan_scan.os_lockres); -} - -int ocfs2_drop_inode_locks(struct inode *inode) -{ - int status, err; - - /* No need to call ocfs2_mark_lockres_freeing here - - * ocfs2_clear_inode has done it for us. */ - - err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), - &OCFS2_I(inode)->ip_open_lockres); - if (err < 0) - mlog_errno(err); - - status = err; - - err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), - &OCFS2_I(inode)->ip_inode_lockres); - if (err < 0) - mlog_errno(err); - if (err < 0 && !status) - status = err; - - err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), - &OCFS2_I(inode)->ip_rw_lockres); - if (err < 0) - mlog_errno(err); - if (err < 0 && !status) - status = err; - - return status; -} -#endif - static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, int new_level) { @@ -3790,369 +2419,6 @@ leave_requeue: return 0; } -#if 0 -static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, - int blocking) -{ - struct inode *inode; - struct address_space *mapping; - struct ocfs2_inode_info *oi; - - inode = ocfs2_lock_res_inode(lockres); - mapping = inode->i_mapping; - - if (S_ISDIR(inode->i_mode)) { - oi = OCFS2_I(inode); - oi->ip_dir_lock_gen++; - mlog(0, "generation: %u\n", oi->ip_dir_lock_gen); - goto out; - } - - if (!S_ISREG(inode->i_mode)) - goto out; - - /* - * We need this before the filemap_fdatawrite() so that it can - * transfer the dirty bit from the PTE to the - * page. Unfortunately this means that even for EX->PR - * downconverts, we'll lose our mappings and have to build - * them up again. - */ - unmap_mapping_range(mapping, 0, 0, 0); - - if (filemap_fdatawrite(mapping)) { - mlog(ML_ERROR, "Could not sync inode %llu for downconvert!", - (unsigned long long)OCFS2_I(inode)->ip_blkno); - } - sync_mapping_buffers(mapping); - if (blocking == DLM_LOCK_EX) { - truncate_inode_pages(mapping, 0); - } else { - /* We only need to wait on the I/O if we're not also - * truncating pages because truncate_inode_pages waits - * for us above. We don't truncate pages if we're - * blocking anything < EXMODE because we want to keep - * them around in that case. */ - filemap_fdatawait(mapping); - } - - forget_all_cached_acls(inode); - -out: - return UNBLOCK_CONTINUE; -} - -static int ocfs2_ci_checkpointed(struct ocfs2_caching_info *ci, - struct ocfs2_lock_res *lockres, - int new_level) -{ - int checkpointed = ocfs2_ci_fully_checkpointed(ci); - - BUG_ON(new_level != DLM_LOCK_NL && new_level != DLM_LOCK_PR); - BUG_ON(lockres->l_level != DLM_LOCK_EX && !checkpointed); - - if (checkpointed) - return 1; - - ocfs2_start_checkpoint(OCFS2_SB(ocfs2_metadata_cache_get_super(ci))); - return 0; -} - -static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, - int new_level) -{ - struct inode *inode = ocfs2_lock_res_inode(lockres); - - return ocfs2_ci_checkpointed(INODE_CACHE(inode), lockres, new_level); -} - -static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres) -{ - struct inode *inode = ocfs2_lock_res_inode(lockres); - - __ocfs2_stuff_meta_lvb(inode); -} - -/* - * Does the final reference drop on our dentry lock. Right now this - * happens in the downconvert thread, but we could choose to simplify the - * dlmglue API and push these off to the ocfs2_wq in the future. - */ -static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); - ocfs2_dentry_lock_put(osb, dl); -} - -/* - * d_delete() matching dentries before the lock downconvert. - * - * At this point, any process waiting to destroy the - * dentry_lock due to last ref count is stopped by the - * OCFS2_LOCK_QUEUED flag. - * - * We have two potential problems - * - * 1) If we do the last reference drop on our dentry_lock (via dput) - * we'll wind up in ocfs2_release_dentry_lock(), waiting on - * the downconvert to finish. Instead we take an elevated - * reference and push the drop until after we've completed our - * unblock processing. - * - * 2) There might be another process with a final reference, - * waiting on us to finish processing. If this is the case, we - * detect it and exit out - there's no more dentries anyway. - */ -static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, - int blocking) -{ - struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); - struct ocfs2_inode_info *oi = OCFS2_I(dl->dl_inode); - struct dentry *dentry; - unsigned long flags; - int extra_ref = 0; - - /* - * This node is blocking another node from getting a read - * lock. This happens when we've renamed within a - * directory. We've forced the other nodes to d_delete(), but - * we never actually dropped our lock because it's still - * valid. The downconvert code will retain a PR for this node, - * so there's no further work to do. - */ - if (blocking == DLM_LOCK_PR) - return UNBLOCK_CONTINUE; - - /* - * Mark this inode as potentially orphaned. The code in - * ocfs2_delete_inode() will figure out whether it actually - * needs to be freed or not. - */ - spin_lock(&oi->ip_lock); - oi->ip_flags |= OCFS2_INODE_MAYBE_ORPHANED; - spin_unlock(&oi->ip_lock); - - /* - * Yuck. We need to make sure however that the check of - * OCFS2_LOCK_FREEING and the extra reference are atomic with - * respect to a reference decrement or the setting of that - * flag. - */ - spin_lock_irqsave(&lockres->l_lock, flags); - spin_lock(&dentry_attach_lock); - if (!(lockres->l_flags & OCFS2_LOCK_FREEING) - && dl->dl_count) { - dl->dl_count++; - extra_ref = 1; - } - spin_unlock(&dentry_attach_lock); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - mlog(0, "extra_ref = %d\n", extra_ref); - - /* - * We have a process waiting on us in ocfs2_dentry_iput(), - * which means we can't have any more outstanding - * aliases. There's no need to do any more work. - */ - if (!extra_ref) - return UNBLOCK_CONTINUE; - - spin_lock(&dentry_attach_lock); - while (1) { - dentry = ocfs2_find_local_alias(dl->dl_inode, - dl->dl_parent_blkno, 1); - if (!dentry) - break; - spin_unlock(&dentry_attach_lock); - - if (S_ISDIR(dl->dl_inode->i_mode)) - shrink_dcache_parent(dentry); - - mlog(0, "d_delete(%pd);\n", dentry); - - /* - * The following dcache calls may do an - * iput(). Normally we don't want that from the - * downconverting thread, but in this case it's ok - * because the requesting node already has an - * exclusive lock on the inode, so it can't be queued - * for a downconvert. - */ - d_delete(dentry); - dput(dentry); - - spin_lock(&dentry_attach_lock); - } - spin_unlock(&dentry_attach_lock); - - /* - * If we are the last holder of this dentry lock, there is no - * reason to downconvert so skip straight to the unlock. - */ - if (dl->dl_count == 1) - return UNBLOCK_STOP_POST; - - return UNBLOCK_CONTINUE_POST; -} - -static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, - int new_level) -{ - struct ocfs2_refcount_tree *tree = - ocfs2_lock_res_refcount_tree(lockres); - - return ocfs2_ci_checkpointed(&tree->rf_ci, lockres, new_level); -} - -static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, - int blocking) -{ - struct ocfs2_refcount_tree *tree = - ocfs2_lock_res_refcount_tree(lockres); - - ocfs2_metadata_cache_purge(&tree->rf_ci); - - return UNBLOCK_CONTINUE; -} - -static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_qinfo_lvb *lvb; - struct ocfs2_mem_dqinfo *oinfo = ocfs2_lock_res_qinfo(lockres); - struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, - oinfo->dqi_gi.dqi_type); - - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - lvb->lvb_version = OCFS2_QINFO_LVB_VERSION; - lvb->lvb_bgrace = cpu_to_be32(info->dqi_bgrace); - lvb->lvb_igrace = cpu_to_be32(info->dqi_igrace); - lvb->lvb_syncms = cpu_to_be32(oinfo->dqi_syncms); - lvb->lvb_blocks = cpu_to_be32(oinfo->dqi_gi.dqi_blocks); - lvb->lvb_free_blk = cpu_to_be32(oinfo->dqi_gi.dqi_free_blk); - lvb->lvb_free_entry = cpu_to_be32(oinfo->dqi_gi.dqi_free_entry); -} - -void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex) -{ - struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; - struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - - if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, lockres, level); -} - -static int ocfs2_refresh_qinfo(struct ocfs2_mem_dqinfo *oinfo) -{ - struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, - oinfo->dqi_gi.dqi_type); - struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; - struct ocfs2_qinfo_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - struct buffer_head *bh = NULL; - struct ocfs2_global_disk_dqinfo *gdinfo; - int status = 0; - - if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && - lvb->lvb_version == OCFS2_QINFO_LVB_VERSION) { - info->dqi_bgrace = be32_to_cpu(lvb->lvb_bgrace); - info->dqi_igrace = be32_to_cpu(lvb->lvb_igrace); - oinfo->dqi_syncms = be32_to_cpu(lvb->lvb_syncms); - oinfo->dqi_gi.dqi_blocks = be32_to_cpu(lvb->lvb_blocks); - oinfo->dqi_gi.dqi_free_blk = be32_to_cpu(lvb->lvb_free_blk); - oinfo->dqi_gi.dqi_free_entry = - be32_to_cpu(lvb->lvb_free_entry); - } else { - status = ocfs2_read_quota_phys_block(oinfo->dqi_gqinode, - oinfo->dqi_giblk, &bh); - if (status) { - mlog_errno(status); - goto bail; - } - gdinfo = (struct ocfs2_global_disk_dqinfo *) - (bh->b_data + OCFS2_GLOBAL_INFO_OFF); - info->dqi_bgrace = le32_to_cpu(gdinfo->dqi_bgrace); - info->dqi_igrace = le32_to_cpu(gdinfo->dqi_igrace); - oinfo->dqi_syncms = le32_to_cpu(gdinfo->dqi_syncms); - oinfo->dqi_gi.dqi_blocks = le32_to_cpu(gdinfo->dqi_blocks); - oinfo->dqi_gi.dqi_free_blk = le32_to_cpu(gdinfo->dqi_free_blk); - oinfo->dqi_gi.dqi_free_entry = - le32_to_cpu(gdinfo->dqi_free_entry); - brelse(bh); - ocfs2_track_lock_refresh(lockres); - } - -bail: - return status; -} - -/* Lock quota info, this function expects at least shared lock on the quota file - * so that we can safely refresh quota info from disk. */ -int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex) -{ - struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; - struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - int status = 0; - - /* On RO devices, locking really isn't needed... */ - if (ocfs2_is_hard_readonly(osb)) { - if (ex) - status = -EROFS; - goto bail; - } - if (ocfs2_mount_local(osb)) - goto bail; - - status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); - if (status < 0) { - mlog_errno(status); - goto bail; - } - if (!ocfs2_should_refresh_lock_res(lockres)) - goto bail; - /* OK, we have the lock but we need to refresh the quota info */ - status = ocfs2_refresh_qinfo(oinfo); - if (status) - ocfs2_qinfo_unlock(oinfo, ex); - ocfs2_complete_lock_res_refresh(lockres, status); -bail: - return status; -} - -int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex) -{ - int status; - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; - struct ocfs2_super *osb = lockres->l_priv; - - - if (ocfs2_is_hard_readonly(osb)) - return -EROFS; - - if (ocfs2_mount_local(osb)) - return 0; - - status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); - if (status < 0) - mlog_errno(status); - - return status; -} - -void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex) -{ - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; - struct ocfs2_super *osb = lockres->l_priv; - - if (!ocfs2_mount_local(osb)) - ocfs2_cluster_unlock(osb, lockres, level); -} -#endif - static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres) { From 28a6b82690f37c0166d0c778c3a62dd208edf87f Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 26 Sep 2017 17:23:23 -0500 Subject: [PATCH 460/920] scoutfs: allow some recursive locking in dlmglue Scoutfs can get into a situation where it wants to acquire a lock twice. This happens when we have a parent and child in the same inode group. A create operation will lock that group twice, once for each inode. Instead of forcing callers to remember which inode groups they've locked, we handle this internally within dlmglue. Add a dlmglue lock type flag that indicates we might recursively lock a resource. During locking, when dlmglue sees that flag and we already have the lock at an appropriate level, it will allow the lock operation to continue even when the lock is marked blocking. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 23 +++++++++++++++++++++++ kmod/src/dlmglue.h | 13 +++++++++++++ kmod/src/lock.c | 4 ++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index ffcf264d..dc1fa4d3 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -918,6 +918,13 @@ static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lock return wanted <= ocfs2_highest_compat_lock_level(lockres->l_blocking); } +static inline int lockres_allow_recursion(struct ocfs2_lock_res *lockres, + int wanted) +{ + return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) + && wanted <= lockres->l_level; +} + static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) { INIT_LIST_HEAD(&mw->mw_item); @@ -1070,6 +1077,7 @@ again: } if (lockres->l_flags & OCFS2_LOCK_BLOCKED && + !lockres_allow_recursion(lockres, level) && !ocfs2_may_continue_on_blocked_lock(lockres, level)) { /* is the lock is currently blocked on behalf of * another node */ @@ -2385,6 +2393,21 @@ recheck: goto recheck; } + if ((lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && + ((lockres->l_blocking == DLM_LOCK_PR && lockres->l_ex_holders) || + (lockres->l_blocking == DLM_LOCK_EX && + (lockres->l_ex_holders || lockres->l_ro_holders)))) { + /* + * Recursive locks may have had their holder count + * incremented while we were sleeping in + * ->downconvert_worker. Recheck here. + */ + mlog(ML_BASTS, "lockres %s, block=%d:%d, level=%d:%d, ro=%d " + "ex=%d, Recheck\n", lockres->l_name, blocking, + lockres->l_blocking, level, lockres->l_level, + lockres->l_ro_holders, lockres->l_ex_holders); + goto recheck; + } downconvert: ctl->requeue = 0; diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 0d852743..4a88b934 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -279,6 +279,19 @@ struct ocfs2_lock_res_ops { */ #define LOCK_TYPE_USES_LVB 0x2 +/* + * Tells dlmglue to override fairness considerations when locking this + * lock type - the blocking flag will be ignored when a lock is + * requested and we already have it at the appropriate level. This + * allows a process to acquire a dlmglue lock on the same resource + * multiple times in a row without deadlocking, even if another node has + * asked for a competing lock on the resource. + * + * Note that lock/unlock calls must always be balanced (1 unlock for + * every lock), even when this flag is set. + */ +#define LOCK_TYPE_RECURSIVE 0x4 + struct ocfs2_lock_holder { struct list_head oh_list; struct pid *oh_owner_pid; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index d89ea6ac..7fc15526 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -174,14 +174,14 @@ static struct ocfs2_lock_res_ops scoufs_ino_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ - .flags = LOCK_TYPE_REQUIRES_REFRESH, + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_RECURSIVE, }; static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ - .flags = 0, + .flags = LOCK_TYPE_RECURSIVE, }; static struct ocfs2_lock_res_ops scoutfs_global_lops = { From 1da18d17cfee59a940aa9fb9d1ec00637ee48ad9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 Sep 2017 11:10:22 -0700 Subject: [PATCH 461/920] scoutfs: use trylock for global server lock Shared unmount hasn't worked for a long time because we didn't have the server work woken out of blocking trying to acquire the lock. In the old lock code the wait conditions didn't test ->shutdown. dlmglue doesn't give us a reasonable way to break a caller out of a blocked lock. We could add some code to do it with a global context that'd have to wake all locks or add a call with a lock resource name, not a held lock, that'd wake that specific lock. Neither sound great. So instead we'll use trylock to get the server lock. It's guaranteed to make reasonble forward progress. The server work is already requeued with a delay to retry. While we're at it we add a global server lock instead of using the weird magical inode lock in the fs space. The server lock doesn't need keys or to participate in item cache consistency, etc. With this unmount works. All mounts will now generate regular background trylock requests. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/server.c | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 161b31a2..17c53a18 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -535,6 +535,7 @@ enum { #define SCOUTFS_LOCK_SCOPE_FS_ITEMS 2 #define SCOUTFS_LOCK_TYPE_GLOBAL_RENAME 1 +#define SCOUTFS_LOCK_TYPE_GLOBAL_SERVER 2 struct scoutfs_lock_name { __u8 scope; diff --git a/kmod/src/server.c b/kmod/src/server.c index d42012b0..a12da887 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -892,8 +892,9 @@ static void scoutfs_server_func(struct work_struct *work) init_waitqueue_head(&waitq); - /* lock attempt will return -ESHUTDOWN once we should not queue */ - ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ~0ULL, &lock); + ret = scoutfs_lock_global(sb, DLM_LOCK_EX, SCOUTFS_LKF_TRYLOCK, + SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, + &lock); if (ret) goto out; From 85dbc21dc63c95a4f603354ac3575e18be04b69b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Sep 2017 14:46:52 -0700 Subject: [PATCH 462/920] scoutfs: use lock end keys in rename verification scoutfs_rename() looks for dirents again after acquiring cluster locks. It needs to pass in the lock end keys to limit the items that are read into the cache. Signed-off-by: Zach Brown --- kmod/src/dir.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 14d8d044..d0bf52ee 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1259,7 +1259,8 @@ static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, * The caller has the name locked in the dir. */ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, - unsigned name_len, u64 ino) + unsigned name_len, u64 ino, + struct scoutfs_lock *lock) { struct scoutfs_key_buf *key = NULL; struct scoutfs_dirent dent; @@ -1272,7 +1273,7 @@ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), NULL); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), lock->end); if (ret == 0 && le64_to_cpu(dent.ino) != ino) ret = -ENOENT; else if (ret == -ENOENT && ino == 0) @@ -1365,10 +1366,12 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, /* make sure that the entries assumed by the argument still exist */ ret = verify_entry(sb, scoutfs_ino(old_dir), old_dentry->d_name.name, - old_dentry->d_name.len, scoutfs_ino(old_inode)) ?: + old_dentry->d_name.len, scoutfs_ino(old_inode), + old_dir_lock) ?: verify_entry(sb, scoutfs_ino(new_dir), new_dentry->d_name.name, new_dentry->d_name.len, - new_inode ? scoutfs_ino(new_inode) : 0); + new_inode ? scoutfs_ino(new_inode) : 0, + new_dir_lock); if (ret) goto out_unlock; From 1193fbc9c5a56eac46f9b79579446a5cc4de04bd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 15:35:40 -0700 Subject: [PATCH 463/920] scoutfs: add a node_id lock A mount's node_id item operations need to be locked. For now let's use a lock that's held for the duration of the mount. It makes it trivial for us to use it with node_id items but we'll have work to do if we want to opportunistically get access to other mount's node_id items while they're still up. Signed-off-by: Zach Brown --- kmod/src/lock.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 2 ++ kmod/src/super.c | 12 ++++++++++-- kmod/src/super.h | 2 ++ 4 files changed, 63 insertions(+), 2 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 7fc15526..75c0fa7c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -190,6 +190,13 @@ static struct ocfs2_lock_res_ops scoutfs_global_lops = { .flags = 0, }; +static struct ocfs2_lock_res_ops scoutfs_node_id_lops = { + .get_osb = get_ino_lock_osb, + /* XXX: .check_downconvert that queries the item cache for dirty items */ + .downconvert_worker = ino_lock_downconvert, + .flags = 0, +}; + static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_lock_name *lock_name, struct ocfs2_lock_res_ops *type, @@ -671,6 +678,48 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, &scoufs_ino_index_lops, &start, &end, ret_lock); } +/* + * The node_id lock protects a mount's private persistent items in the + * node_id zone. It's held for the duration of the mount. It lets the + * mount modify the node_id items at will and signals to other mounts + * that we're still alive and our node_id items shouldn't be reclaimed. + * + * Being held for the entire mount prevents other nodes from reclaiming + * our items, like free blocks, when it would make sense for them to be + * able to. Maybe we have a bunch free and they're trying to allocate + * and are getting ENOSPC. + */ +int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, + u64 node_id, struct scoutfs_lock **lock) +{ + struct scoutfs_lock_name lock_name; + struct scoutfs_orphan_key start_okey; + struct scoutfs_orphan_key end_okey; + struct scoutfs_key_buf start; + struct scoutfs_key_buf end; + + lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; + lock_name.zone = SCOUTFS_NODE_ZONE; + lock_name.type = 0; + lock_name.first = cpu_to_le64(node_id); + lock_name.second = 0; + + start_okey.zone = SCOUTFS_NODE_ZONE; + start_okey.node_id = cpu_to_be64(node_id); + start_okey.type = 0; + start_okey.ino = 0; + scoutfs_key_init(&start, &start_okey, sizeof(start_okey)); + + end_okey.zone = SCOUTFS_NODE_ZONE; + end_okey.node_id = cpu_to_be64(node_id); + end_okey.type = ~0; + end_okey.ino = cpu_to_be64(~0ULL); + scoutfs_key_init(&end, &end_okey, sizeof(end_okey)); + + return lock_name_keys(sb, mode, flags, &lock_name, + &scoutfs_node_id_lops, &start, &end, lock); +} + void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level) { diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 65c0a4f2..cd80a4fd 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -46,6 +46,8 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *d, struct scoutfs_lock **D_lock); int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, struct scoutfs_lock **lock); +int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, + u64 node_id, struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); diff --git a/kmod/src/super.c b/kmod/src/super.c index 0a8b71af..e3d81395 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -290,6 +290,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) */ ret = scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb); + scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, + &sbi->node_id_lock); if (ret) goto out; @@ -313,8 +315,11 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) // scoutfs_scan_orphans(sb); ret = 0; out: - if (ret) - scoutfs_server_destroy(sb); + if (ret) { + scoutfs_server_destroy(sb); + scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); + sbi->node_id_lock = NULL; + } return ret; } @@ -342,6 +347,9 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { + scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); + sbi->node_id_lock = NULL; + scoutfs_lock_destroy(sb); scoutfs_client_destroy(sb); scoutfs_server_destroy(sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 933ed2b4..89a843f9 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -23,7 +23,9 @@ struct btree_info; struct scoutfs_sb_info { struct super_block *sb; + /* assigned once at the start of each mount, read-only */ u64 node_id; + struct scoutfs_lock *node_id_lock; struct scoutfs_super_block super; From 55709c434528bd6333922196edd47949b2a91fc3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 15:45:45 -0700 Subject: [PATCH 464/920] scoutfs: add lock coverage testing to item_lookup* Let's give the item functions the full lock so that they can make sure that the lock has coverage for the keys involved in the operation. This _lookup*() conversion is first so it adds the lock_coverager() helper. Signed-off-by: Zach Brown --- kmod/src/dir.c | 7 +++---- kmod/src/inode.c | 2 +- kmod/src/item.c | 40 ++++++++++++++++++++++++++++++++++++---- kmod/src/item.h | 4 ++-- kmod/src/xattr.c | 2 +- 5 files changed, 43 insertions(+), 12 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index d0bf52ee..b9a0dc9a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -277,8 +277,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), - dir_lock->end); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), dir_lock); scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); if (ret == -ENOENT) { ino = 0; @@ -860,7 +859,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, ret = scoutfs_item_create(sb, &key, val); else if (op == SYM_LOOKUP) ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, - lock->end); + lock); else if (op == SYM_DELETE) ret = scoutfs_item_delete(sb, &key, lock->end); if (ret) @@ -1273,7 +1272,7 @@ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), lock->end); + ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), lock); if (ret == 0 && le64_to_cpu(dent.ino) != ino) ret = -ENOENT; else if (ret == -ENOENT && ino == 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 11dc929a..5eba0cb0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -275,7 +275,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, mutex_lock(&si->item_mutex); if (atomic64_read(&si->last_refreshed) < refresh_gen) { ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), - lock->end); + lock); if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); diff --git a/kmod/src/item.c b/kmod/src/item.c index 8c28f06f..dc2263fe 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -730,6 +730,35 @@ restart: } } +/* + * Return true if the lock protects the use of the key. Some locks not + * intended for item use don't have a key range and we wan't to safely + * detect that. We use the block 'rw' constants just because they're + * convenient. The level test is racey but it's a char.. how racy can + * it be? :). + */ +static bool lock_coverage(struct scoutfs_lock *lock, + struct scoutfs_key_buf *key, int rw) +{ + bool writing = rw & WRITE; + signed char level; + + if (rw & ~WRITE) + return false; + + if (!lock || !lock->start || !lock->end) + return false; + + level = ACCESS_ONCE(lock->lockres.l_level); + + if ((writing && level != DLM_LOCK_EX) || + (!writing && level != DLM_LOCK_EX && level != DLM_LOCK_PR)) + return false; + + return scoutfs_key_compare_ranges(key, key, + lock->start, lock->end) == 0; +} + /* * Find an item with the given key and copy its value into the caller's * value vector. The amount of bytes copied is returned which can be 0 @@ -739,7 +768,7 @@ restart: * and inserted into the cache. */ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val, struct scoutfs_key_buf *end) + struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -747,6 +776,9 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; + if (WARN_ON_ONCE(!lock_coverage(lock, key, READ))) + return -EINVAL; + trace_scoutfs_item_lookup(sb, key); do { @@ -765,7 +797,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); trace_scoutfs_item_lookup_ret(sb, ret); return ret; @@ -786,11 +818,11 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, */ int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size, struct scoutfs_key_buf *end) + int size, struct scoutfs_lock *lock) { int ret; - ret = scoutfs_item_lookup(sb, key, val, end); + ret = scoutfs_item_lookup(sb, key, val, lock); if (ret == size) ret = 0; else if (ret >= 0) diff --git a/kmod/src/item.h b/kmod/src/item.h index 5825eb0f..0fd1eff9 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -13,10 +13,10 @@ struct scoutfs_segment; struct scoutfs_key_buf; int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val, struct scoutfs_key_buf *end); + struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size, struct scoutfs_key_buf *end); + int size, struct scoutfs_lock *lock); int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, struct scoutfs_key_buf *end); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 5c775b80..f72b6746 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -190,7 +190,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, for_each_xattr_item(key, val, &vh, buffer, size, part, off, bytes) { - ret = scoutfs_item_lookup(sb, key, val, lck->end); + ret = scoutfs_item_lookup(sb, key, val, lck); if (ret < 0) { if (ret == -ENOENT) ret = -ENODATA; From 9e3954a918399015a51f3c772c98ebdbb5b0f76e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:18:32 -0700 Subject: [PATCH 465/920] scoutfs: add lock around data item truncation Add cluster lock coverage to scoutfs_data_truncate_items() and plumb the lock down into the item functions. Signed-off-by: Zach Brown --- kmod/src/data.c | 8 +++++--- kmod/src/data.h | 3 ++- kmod/src/ioctl.c | 9 ++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 68ca9345..eeb9081a 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -599,7 +599,8 @@ out: * partial progress. */ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline) + u64 len, bool offline, + struct scoutfs_lock *lock) { struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; @@ -633,7 +634,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, init_mapping_key(&key, &bmk, ino, iblock); scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); - ret = scoutfs_item_next(sb, &key, &last_key, val, NULL); + ret = scoutfs_item_next(sb, &key, &last_key, val, lock->end); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -669,7 +670,8 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, if (!dirtied) { /* dirty item with full size encoded */ - ret = scoutfs_item_update(sb, &key, val, NULL); + ret = scoutfs_item_update(sb, &key, val, + lock->end); if (ret) break; dirtied = true; diff --git a/kmod/src/data.h b/kmod/src/data.h index 04dd9050..0dcd23a6 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -5,7 +5,8 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline); + u64 len, bool offline, + struct scoutfs_lock *lock); int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index a7b794da..17747af8 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -331,6 +331,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; struct scoutfs_ioctl_release args; + struct scoutfs_lock *lock = NULL; loff_t start; loff_t end_inc; int ret; @@ -352,6 +353,11 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) mutex_lock(&inode->i_mutex); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &lock); + if (ret) + goto out; + if (!S_ISREG(inode->i_mode)) { ret = -EINVAL; goto out; @@ -375,8 +381,9 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) truncate_inode_pages_range(&inode->i_data, start, end_inc); ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), args.block, - args.count, true); + args.count, true, lock); out: + scoutfs_unlock(sb, lock, DLM_LOCK_EX); mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); From b2668fee9a57b42d4387fb19d5a0f512b4cbac30 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:25:02 -0700 Subject: [PATCH 466/920] scoutfs: protect node free block items Now that we have a long-lived node_id lock we can use it to protect the free block items in the node zone. Signed-off-by: Zach Brown --- kmod/src/data.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index eeb9081a..3cc5b04e 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -351,6 +351,7 @@ static void init_free_key(struct scoutfs_key_buf *key, static int set_segno_free(struct super_block *sb, u64 segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key fbk = {0,}; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; @@ -363,7 +364,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(struct scoutfs_free_bits), - NULL); + lock); if (ret && ret != -ENOENT) goto out; @@ -381,7 +382,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) goto out; } - ret = scoutfs_item_update(sb, &key, val, NULL); + ret = scoutfs_item_update(sb, &key, val, lock->end); out: trace_scoutfs_data_set_segno_free(sb, segno, be64_to_cpu(fbk.base), bit, ret); @@ -427,6 +428,7 @@ static int create_blkno_free(struct super_block *sb, u64 blkno, static int clear_segno_free(struct super_block *sb, u64 segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key b_fbk; struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; @@ -442,7 +444,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(struct scoutfs_free_bits), - NULL); + lock); if (ret) { /* XXX corruption, caller saw item.. should still exist */ if (ret == -ENOENT) @@ -464,9 +466,9 @@ static int clear_segno_free(struct super_block *sb, u64 segno) goto out; if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, NULL); + ret = scoutfs_item_delete(sb, &key, lock->end); else - ret = scoutfs_item_update(sb, &key, val, NULL); + ret = scoutfs_item_update(sb, &key, val, lock->end); if (ret) scoutfs_item_delete_dirty(sb, &b_key); out: @@ -481,6 +483,7 @@ out: static int set_blkno_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; @@ -495,7 +498,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(struct scoutfs_free_bits), - NULL); + lock); if (ret && ret != -ENOENT) goto out; @@ -514,12 +517,12 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) } if (!bitmap_full((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) { - ret = scoutfs_item_update(sb, &key, val, NULL); + ret = scoutfs_item_update(sb, &key, val, lock->end); goto out; } /* dirty so we can safely delete if set segno fails */ - ret = scoutfs_item_dirty(sb, &key, NULL); + ret = scoutfs_item_dirty(sb, &key, lock->end); if (ret) goto out; @@ -542,6 +545,7 @@ out: static int clear_blkno_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; @@ -555,7 +559,7 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(struct scoutfs_free_bits), - NULL); + lock); if (ret) { /* XXX corruption, bits should have existed */ if (ret == -ENOENT) @@ -571,9 +575,9 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) } if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, NULL); + ret = scoutfs_item_delete(sb, &key, lock->end); else - ret = scoutfs_item_update(sb, &key, val, NULL); + ret = scoutfs_item_update(sb, &key, val, lock->end); out: return ret; } @@ -824,6 +828,7 @@ out: static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; @@ -836,7 +841,7 @@ static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), NULL); + sizeof(struct scoutfs_free_bits), lock); if (ret < 0) goto out; @@ -860,6 +865,7 @@ out: static int find_free_segno(struct super_block *sb, u64 *segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits_key last_fbk; struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; @@ -875,7 +881,7 @@ static int find_free_segno(struct super_block *sb, u64 *segno) SCOUTFS_FREE_BITS_SEGNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_next(sb, &key, &last_key, val, NULL); + ret = scoutfs_item_next(sb, &key, &last_key, val, lock->end); if (ret < 0) goto out; From 3a277bac6fca1cbe62d8a09178c25a9ae14fec96 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:31:28 -0700 Subject: [PATCH 467/920] scoutfs: protect orphan items with node_id_lock Orphan processing only works with orphans on its node today. Protect that orphan item use with the node_id lock. Signed-off-by: Zach Brown --- kmod/src/inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 5eba0cb0..91f98bf9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -865,13 +865,14 @@ static void init_orphan_key(struct scoutfs_key_buf *key, static int remove_orphan_item(struct super_block *sb, u64 ino) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_orphan_key okey; struct scoutfs_key_buf key; int ret; init_orphan_key(&key, &okey, sbi->node_id, ino); - ret = scoutfs_item_delete(sb, &key, NULL); + ret = scoutfs_item_delete(sb, &key, lock->end); if (ret == -ENOENT) ret = 0; @@ -994,6 +995,7 @@ int scoutfs_drop_inode(struct inode *inode) int scoutfs_scan_orphans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_orphan_key okey; struct scoutfs_orphan_key last_okey; struct scoutfs_key_buf key; @@ -1007,7 +1009,7 @@ int scoutfs_scan_orphans(struct super_block *sb) init_orphan_key(&last, &last_okey, sbi->node_id, ~0ULL); while (1) { - ret = scoutfs_item_next_same(sb, &key, &last, NULL, NULL); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, lock->end); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) From 0e4627ea65a8aaf4f7b22ed5a687a49bfaeb2e96 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:45:16 -0700 Subject: [PATCH 468/920] scoutfs: add locking of link backref traversal Add cluster locking around the link backref item lookups during ino to path traversal. Signed-off-by: Zach Brown --- kmod/src/dir.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index b9a0dc9a..73758a5a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1042,6 +1042,9 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino, * * Returns 0 if we added an entry, -ENOENT if we didn't, and -errno for * search errors. + * + * Callers are comfortable with the race inherent to incrementally + * building up a path with individual locked backref item lookups. */ static int add_next_linkref(struct super_block *sb, u64 ino, u64 dir_ino, char *name, unsigned int name_len, @@ -1049,6 +1052,7 @@ static int add_next_linkref(struct super_block *sb, u64 ino, { struct scoutfs_link_backref_key last_lbkey; struct scoutfs_link_backref_entry *ent; + struct scoutfs_lock *lock = NULL; struct scoutfs_key_buf last; struct scoutfs_key_buf key; int len; @@ -1072,12 +1076,18 @@ static int add_next_linkref(struct super_block *sb, u64 ino, init_link_backref_key(&last, &last_lbkey, ino, U64_MAX, NULL, 0); /* next backref key is now in ent */ - ret = scoutfs_item_next(sb, &key, &last, NULL, NULL); + ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); + if (ret) + goto out; + + ret = scoutfs_item_next(sb, &key, &last, NULL, lock->end); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); + lock = NULL; + trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, ret, key.key_len); if (ret < 0) goto out; - len = (int)key.key_len - sizeof(struct scoutfs_link_backref_key); /* XXX corruption */ if (len < 1 || len > SCOUTFS_NAME_LEN) { @@ -1237,6 +1247,10 @@ out: * * Compare this to lock_rename()'s use of d_ancestor() and what it's * caller does with the returned ancestor. + * + * The caller only holds the global rename cluster lock. + * item_d_ancestor is going to walk backref paths and acquire and + * release locks for each target inode in the path. */ static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, u64 old_ino, u64 new_ino) From 1c6e3e39bf5f02773d3f2f3ad78155ca81bfbfff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:50:19 -0700 Subject: [PATCH 469/920] scoutfs: add full lock coverage to _item_next*() Add the full lock argument to _item_next*() so that it can verify lock coverage in addition to limiting item cache population to the range covered by the lock. Signed-off-by: Zach Brown --- kmod/src/data.c | 7 +++---- kmod/src/dir.c | 4 ++-- kmod/src/inode.c | 2 +- kmod/src/ioctl.c | 2 +- kmod/src/item.c | 24 +++++++++++++++--------- kmod/src/item.h | 6 +++--- kmod/src/xattr.c | 4 ++-- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 3cc5b04e..73289828 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -638,7 +638,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, init_mapping_key(&key, &bmk, ino, iblock); scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); - ret = scoutfs_item_next(sb, &key, &last_key, val, lock->end); + ret = scoutfs_item_next(sb, &key, &last_key, val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -881,7 +881,7 @@ static int find_free_segno(struct super_block *sb, u64 *segno) SCOUTFS_FREE_BITS_SEGNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_next(sb, &key, &last_key, val, lock->end); + ret = scoutfs_item_next(sb, &key, &last_key, val, lock); if (ret < 0) goto out; @@ -1296,8 +1296,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, init_mapping_key(&key, &bmk, ino, blk_off); scoutfs_kvec_init(val, &map->encoded, sizeof(map->encoded)); - ret = scoutfs_item_next(sb, &key, &last_key, val, - inode_lock->end); + ret = scoutfs_item_next(sb, &key, &last_key, val, inode_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 73758a5a..e7c38265 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -379,7 +379,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) scoutfs_kvec_init(val, dent, item_len); ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, offsetof(struct scoutfs_dirent, name[1]), - dir_lock->end); + dir_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1080,7 +1080,7 @@ static int add_next_linkref(struct super_block *sb, u64 ino, if (ret) goto out; - ret = scoutfs_item_next(sb, &key, &last, NULL, lock->end); + ret = scoutfs_item_next(sb, &key, &last, NULL, lock); scoutfs_unlock(sb, lock, DLM_LOCK_PR); lock = NULL; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 91f98bf9..b576333a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -1009,7 +1009,7 @@ int scoutfs_scan_orphans(struct super_block *sb) init_orphan_key(&last, &last_okey, sbi->node_id, ~0ULL); while (1) { - ret = scoutfs_item_next_same(sb, &key, &last, NULL, lock->end); + ret = scoutfs_item_next_same(sb, &key, &last, NULL, lock); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 17747af8..cb4ba017 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -123,7 +123,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) for (nr = 0; nr < walk.nr_entries; ) { - ret = scoutfs_item_next_same(sb, &key, &last_key, NULL, lock->end); + ret = scoutfs_item_next_same(sb, &key, &last_key, NULL, lock); if (ret < 0 && ret != -ENOENT) break; diff --git a/kmod/src/item.c b/kmod/src/item.c index dc2263fe..0b62b730 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -911,7 +911,7 @@ static struct cached_item *item_for_next(struct rb_root *root, */ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -923,8 +923,8 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, int ret; /* use the end key as the last key if it's closer to reduce compares */ - if (end && scoutfs_key_compare(end, last) < 0) - last = end; + if (scoutfs_key_compare(lock->end, last) < 0) + last = lock->end; /* convenience to avoid searching if caller iterates past their last */ if (scoutfs_key_compare(key, last) > 0) { @@ -932,6 +932,11 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, goto out; } + if (WARN_ON_ONCE(!lock_coverage(lock, key, READ))) { + ret = -EINVAL; + goto out; + } + pos = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); if (!pos || !range_end) { @@ -948,13 +953,14 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, cached = check_range(sb, &cac->ranges, pos, range_end); trace_scoutfs_item_next_range_check(sb, !!cached, key, - pos, last, end, range_end); + pos, last, lock->end, + range_end); if (!cached) { /* populate missing cached range starting at pos */ spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, pos, end); + ret = scoutfs_manifest_read_items(sb, pos, lock->end); spin_lock_irqsave(&cac->lock, flags); if (ret) @@ -1008,7 +1014,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, int len, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { int key_len = key->key_len; int ret; @@ -1018,7 +1024,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, if (WARN_ON_ONCE(!val || scoutfs_kvec_length(val) < len)) return -EINVAL; - ret = scoutfs_item_next(sb, key, last, val, end); + ret = scoutfs_item_next(sb, key, last, val, lock); if (ret >= 0 && (key->key_len != key_len || ret < len)) ret = -EIO; @@ -1033,14 +1039,14 @@ int scoutfs_item_next_same_min(struct super_block *sb, */ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { int key_len = key->key_len; int ret; trace_scoutfs_item_next_same(sb, key_len); - ret = scoutfs_item_next(sb, key, last, val, end); + ret = scoutfs_item_next(sb, key, last, val, lock); if (ret >= 0 && (key->key_len != key_len)) ret = -EIO; diff --git a/kmod/src/item.h b/kmod/src/item.h index 0fd1eff9..8af12785 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -19,15 +19,15 @@ int scoutfs_item_lookup_exact(struct super_block *sb, int size, struct scoutfs_lock *lock); int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, int len, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index f72b6746..72525470 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -392,7 +392,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) total = 0; for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL, lck->end); + ret = scoutfs_item_next(sb, key, last, NULL, lck); if (ret < 0) { if (ret == -ENOENT) ret = total; @@ -474,7 +474,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) /* the inode is dead so we don't need the xattr sem */ for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL, lck->end); + ret = scoutfs_item_next(sb, key, last, NULL, lck); if (ret < 0) { if (ret == -ENOENT) ret = 0; From 32a68e84cfed0d5011433601b23ab7bfccdddf55 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 16:55:59 -0700 Subject: [PATCH 470/920] scoutfs: add full lock coverage to _item_dirty() Add the full lock argument to _item_dirty() so that it can verify lock coverage in addition to limiting item cache population to the range covered by the lock. This also ropes in scoutfs_dirty_inode_item() which is a thin wrapper around _item_dirty(); Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- kmod/src/dir.c | 18 +++++++++--------- kmod/src/inode.c | 4 ++-- kmod/src/inode.h | 2 +- kmod/src/item.c | 7 +++++-- kmod/src/item.h | 2 +- kmod/src/xattr.c | 2 +- 7 files changed, 20 insertions(+), 17 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 73289828..cce006d3 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -522,7 +522,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) } /* dirty so we can safely delete if set segno fails */ - ret = scoutfs_item_dirty(sb, &key, lock->end); + ret = scoutfs_item_dirty(sb, &key, lock); if (ret) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e7c38265..3b711372 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -509,9 +509,9 @@ static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, goto out; } - ret = scoutfs_item_dirty(sb, ent_key, dir_lock->end) ?: - scoutfs_item_dirty(sb, &rdir_key, dir_lock->end) ?: - scoutfs_item_dirty(sb, lb_key, inode_lock->end); + ret = scoutfs_item_dirty(sb, ent_key, dir_lock) ?: + scoutfs_item_dirty(sb, &rdir_key, dir_lock) ?: + scoutfs_item_dirty(sb, lb_key, inode_lock); if (ret) goto out; @@ -578,7 +578,7 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, goto out; } - ret = scoutfs_dirty_inode_item(dir, (*dir_lock)->end); + ret = scoutfs_dirty_inode_item(dir, *dir_lock); out: if (ret) scoutfs_release_trans(sb); @@ -694,7 +694,7 @@ static int scoutfs_link(struct dentry *old_dentry, if (ret) goto out_unlock; - ret = scoutfs_dirty_inode_item(dir, dir_lock->end); + ret = scoutfs_dirty_inode_item(dir, dir_lock); if (ret) goto out; @@ -1397,12 +1397,12 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, new_pos = SCOUTFS_I(new_dir)->next_readdir_pos++; /* dirty the inodes so that updating doesn't fail */ - ret = scoutfs_dirty_inode_item(old_dir, old_dir_lock->end) ?: - scoutfs_dirty_inode_item(old_inode, old_inode_lock->end) ?: + ret = scoutfs_dirty_inode_item(old_dir, old_dir_lock) ?: + scoutfs_dirty_inode_item(old_inode, old_inode_lock) ?: (old_dir != new_dir ? - scoutfs_dirty_inode_item(new_dir, new_dir_lock->end) : 0) ?: + scoutfs_dirty_inode_item(new_dir, new_dir_lock) : 0) ?: (new_inode ? - scoutfs_dirty_inode_item(new_inode, new_inode_lock->end) : 0); + scoutfs_dirty_inode_item(new_inode, new_inode_lock) : 0); if (ret) goto out; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index b576333a..e4149787 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -498,7 +498,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) * * XXX this will have to do something about variable length inodes */ -int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end) +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; struct scoutfs_inode_key ikey; @@ -510,7 +510,7 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end) scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - ret = scoutfs_item_dirty(sb, &key, end); + ret = scoutfs_item_dirty(sb, &key, lock); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index d4f23b46..15175dd4 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -61,7 +61,7 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); -int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_key_buf *end); +int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); void scoutfs_update_inode_item(struct inode *inode); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); diff --git a/kmod/src/item.c b/kmod/src/item.c index 0b62b730..9e793798 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1329,7 +1329,7 @@ void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) * if it wasn't cached. -ENOENT is returned if the item doesn't exist. */ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1337,6 +1337,9 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + do { spin_lock_irqsave(&cac->lock, flags); @@ -1353,7 +1356,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); trace_scoutfs_item_dirty_ret(sb, ret); return ret; diff --git a/kmod/src/item.h b/kmod/src/item.h index 8af12785..ece0c8c6 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -31,7 +31,7 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_key_buf *end); void scoutfs_item_delete_dirty(struct super_block *sb, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 72525470..e9ff58e1 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -321,7 +321,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, down_write(&si->xattr_rwsem); - ret = scoutfs_dirty_inode_item(inode, lck->end) ?: + ret = scoutfs_dirty_inode_item(inode, lck) ?: scoutfs_item_set_batch(sb, &list, key, last, sif, lck->end); if (ret == 0) { /* XXX do these want i_mutex or anything? */ From 0535e249d1b1eeab91c9199df51b0a093a7f92f1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Sep 2017 17:01:44 -0700 Subject: [PATCH 471/920] scoutfs: add lock arg to scoutfs_update_inode_item Add a full lock argument to scoutfs_update_inode_item() and use it to pass the lock's end key into item_update(). This'll get changed into passing the full lock into _update soon. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- kmod/src/dir.c | 24 ++++++++++++------------ kmod/src/inode.c | 4 ++-- kmod/src/inode.h | 2 +- kmod/src/xattr.c | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index cce006d3..8c249acd 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1175,7 +1175,7 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, scoutfs_inode_inc_data_version(inode); } /* XXX kind of a big hammer, inode life cycle needs work */ - scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(inode, NULL); scoutfs_inode_queue_writeback(inode); } scoutfs_release_trans(sb); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 3b711372..0a46cce2 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -634,8 +634,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, inc_nlink(dir); } - scoutfs_update_inode_item(inode); - scoutfs_update_inode_item(dir); + scoutfs_update_inode_item(inode, inode_lock); + scoutfs_update_inode_item(dir, dir_lock); insert_inode_hash(inode); d_instantiate(dentry, inode); @@ -712,8 +712,8 @@ static int scoutfs_link(struct dentry *old_dentry, inode->i_ctime = dir->i_mtime; inc_nlink(inode); - scoutfs_update_inode_item(inode); - scoutfs_update_inode_item(dir); + scoutfs_update_inode_item(inode, inode_lock); + scoutfs_update_inode_item(dir, dir_lock); atomic_inc(&inode->i_count); d_instantiate(dentry, inode); @@ -792,8 +792,8 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) drop_nlink(dir); drop_nlink(inode); } - scoutfs_update_inode_item(inode); - scoutfs_update_inode_item(dir); + scoutfs_update_inode_item(inode, inode_lock); + scoutfs_update_inode_item(dir, dir_lock); out: scoutfs_release_trans(sb); @@ -1000,8 +1000,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, inode->i_ctime = dir->i_mtime; i_size_write(inode, name_len); - scoutfs_update_inode_item(inode); - scoutfs_update_inode_item(dir); + scoutfs_update_inode_item(inode, inode_lock); + scoutfs_update_inode_item(dir, dir_lock); insert_inode_hash(inode); /* XXX need to set i_op/fop before here for sec callbacks */ @@ -1477,12 +1477,12 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (new_inode) old_inode->i_ctime = now; - scoutfs_update_inode_item(old_dir); - scoutfs_update_inode_item(old_inode); + scoutfs_update_inode_item(old_dir, old_dir_lock); + scoutfs_update_inode_item(old_inode, old_inode_lock); if (new_dir != old_dir) - scoutfs_update_inode_item(new_dir); + scoutfs_update_inode_item(new_dir, new_dir_lock); if (new_inode) - scoutfs_update_inode_item(new_inode); + scoutfs_update_inode_item(new_inode, new_inode_lock); ret = 0; out: diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e4149787..4b144faa 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -596,7 +596,7 @@ static int update_index(struct super_block *sb, struct scoutfs_inode_info *si, * have to deal with errors and unwinding after they've modified the * vfs inode and get here. */ -void scoutfs_update_inode_item(struct inode *inode) +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; @@ -631,7 +631,7 @@ void scoutfs_update_inode_item(struct inode *inode) scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, &key, val, NULL); + err = scoutfs_item_update(sb, &key, val, lock->end); if (err) { scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 15175dd4..f590fdfb 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -62,7 +62,7 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); -void scoutfs_update_inode_item(struct inode *inode); +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock); void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index e9ff58e1..fc97691f 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -327,7 +327,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); inode->i_ctime = CURRENT_TIME; - scoutfs_update_inode_item(inode); + scoutfs_update_inode_item(inode, lck); } up_write(&si->xattr_rwsem); From c3e690a1ac11786dae24ff6183ebf7c932f16523 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Sep 2017 09:48:38 -0700 Subject: [PATCH 472/920] scoutfs: add per_task storage helper Add some functions for storing and using per-task storage in a list. Callers can use this to pass pointers to children in a given scope when interfaces don't allow for passing individual arguments amongst concurrent callers in the scope. Signed-off-by: Zach Brown --- kmod/src/Makefile | 5 +-- kmod/src/per_task.c | 81 +++++++++++++++++++++++++++++++++++++++++++++ kmod/src/per_task.h | 25 ++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 kmod/src/per_task.c create mode 100644 kmod/src/per_task.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 40543c43..30997479 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,8 +6,9 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ dlmglue.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ - manifest.o msg.o options.o seg.o server.o scoutfs_trace.o sock.o \ - sort_priv.o stackglue.o super.o trans.o xattr.o + manifest.o msg.o options.o per_task.o seg.o server.o \ + scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o trans.o \ + xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/per_task.c b/kmod/src/per_task.c new file mode 100644 index 00000000..24b7f412 --- /dev/null +++ b/kmod/src/per_task.c @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "per_task.h" + +/* + * There are times when we'd like to pass data from a caller to its + * callee but we're bouncing through functions and callbacks that don't + * provide per-task storage. We add a trivial little locked list that + * lets a caller store a pointer for callees. The lists are put in the + * scope of the sharing so the contention is rare and limited to real + * concurrency -- imagine, for example, concurrent file reading on an + * inode. + */ + +/* + * Return the pointer that our caller added for us on the given list. + * The expected promise is that the pointer is valid until we return to + * the caller who will remove it from the list. + */ +void *scoutfs_per_task_get(struct scoutfs_per_task *pt) +{ + const struct task_struct *task = current; + struct scoutfs_per_task_entry *ent; + void *ret = NULL; + + spin_lock(&pt->lock); + + list_for_each_entry(ent, &pt->list, head) { + if (ent->task == task){ + ret = ent->ptr; + break; + } + } + + spin_unlock(&pt->lock); + + return ret; +} + +void scoutfs_per_task_add(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr) +{ + ent->task = current; + ent->ptr = ptr; + + spin_lock(&pt->lock); + list_add(&ent->head, &pt->list); + spin_unlock(&pt->lock); +} + +void scoutfs_per_task_del(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent) +{ + BUG_ON(!list_empty(&ent->head) && ent->task != current); + + if (!list_empty(&ent->head)) { + spin_lock(&pt->lock); + list_del_init(&ent->head); + spin_unlock(&pt->lock); + } +} + +void scoutfs_per_task_init(struct scoutfs_per_task *pt) +{ + spin_lock_init(&pt->lock); + INIT_LIST_HEAD(&pt->list); +} diff --git a/kmod/src/per_task.h b/kmod/src/per_task.h new file mode 100644 index 00000000..6cad8098 --- /dev/null +++ b/kmod/src/per_task.h @@ -0,0 +1,25 @@ +#ifndef _SCOUTFS_PER_TASK_H_ +#define _SCOUTFS_PER_TASK_H_ + +struct scoutfs_per_task { + spinlock_t lock; + struct list_head list; +}; + +struct scoutfs_per_task_entry { + struct list_head head; + struct task_struct *task; + void *ptr; +}; + +#define SCOUTFS_DECLARE_PER_TASK_ENTRY(name) \ + struct scoutfs_per_task_entry name + +void *scoutfs_per_task_get(struct scoutfs_per_task *pt); +void scoutfs_per_task_add(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr); +void scoutfs_per_task_del(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent); +void scoutfs_per_task_init(struct scoutfs_per_task *pt); + +#endif From aa7090315415087d75e197251b4bf135715c2a2a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Sep 2017 09:50:08 -0700 Subject: [PATCH 473/920] scoutfs: add lock coverage for data paths Use per_task storage on the inode to pass locks from high level read and write lock holders down into the callbacks that operate under the locks so that the locks can then be passed to the item functions. Signed-off-by: Zach Brown --- kmod/src/data.c | 28 ++++++++++++++++++++++------ kmod/src/file.c | 11 +++++++++++ kmod/src/inode.c | 1 + kmod/src/inode.h | 2 ++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 8c249acd..05083e8b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -908,7 +908,8 @@ out: */ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, struct scoutfs_key_buf *map_key, - unsigned map_ind, bool map_exists) + unsigned map_ind, bool map_exists, + struct scoutfs_lock *data_lock) { DECLARE_DATA_INFO(sb, datinf); struct task_cursor *curs; @@ -958,7 +959,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, /* ensure that we can copy in encoded without failing */ scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); if (map_exists) - ret = scoutfs_item_update(sb, map_key, val, NULL); + ret = scoutfs_item_update(sb, map_key, val, data_lock->end); else ret = scoutfs_item_create(sb, map_key, val); if (ret) @@ -1000,6 +1001,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct super_block *sb = inode->i_sb; struct scoutfs_block_mapping_key bmk; struct scoutfs_key_buf key; + struct scoutfs_lock *lock; struct block_mapping *map; SCOUTFS_DECLARE_KVEC(val); bool exists; @@ -1007,6 +1009,10 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, int ret; int i; + lock = scoutfs_per_task_get(&si->pt_data_lock); + if (WARN_ON_ONCE(!lock)) + return -EINVAL; + map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); if (!map) return -ENOMEM; @@ -1015,7 +1021,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); /* find the mapping item that covers the logical block */ - ret = scoutfs_item_lookup(sb, &key, val, NULL); + ret = scoutfs_item_lookup(sb, &key, val, lock); if (ret < 0) { if (ret != -ENOENT) goto out; @@ -1044,7 +1050,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, * and try again if we've already done a bulk alloc in * our transaction. */ - ret = find_alloc_block(sb, map, &key, ind, exists); + ret = find_alloc_block(sb, map, &key, ind, exists, lock); if (ret) goto out; } @@ -1133,11 +1139,17 @@ static int scoutfs_write_begin(struct file *file, struct page **pagep, void **fsdata) { struct inode *inode = mapping->host; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock; int ret; trace_scoutfs_write_begin(sb, scoutfs_ino(inode), (__u64)pos, len); + lock = scoutfs_per_task_get(&si->pt_data_lock); + if (WARN_ON_ONCE(!lock)) + return -EINVAL; + ret = scoutfs_hold_trans(sb, SIC_WRITE_BEGIN()); if (ret) goto out; @@ -1146,7 +1158,7 @@ static int scoutfs_write_begin(struct file *file, flags |= AOP_FLAG_NOFS; /* generic write_end updates i_size and calls dirty_inode */ - ret = scoutfs_dirty_inode_item(inode, NULL); + ret = scoutfs_dirty_inode_item(inode, lock); if (ret == 0) ret = block_write_begin(mapping, pos, len, flags, pagep, scoutfs_get_block); @@ -1163,11 +1175,15 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct inode *inode = mapping->host; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock; int ret; trace_scoutfs_write_end(sb, scoutfs_ino(inode), page->index, (u64)pos, len, copied); + /* always call write_end, update_inode will bark if there's no lock */ + lock = scoutfs_per_task_get(&si->pt_data_lock); + ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); if (ret > 0) { if (!si->staging) { @@ -1175,7 +1191,7 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, scoutfs_inode_inc_data_version(inode); } /* XXX kind of a big hammer, inode life cycle needs work */ - scoutfs_update_inode_item(inode, NULL); + scoutfs_update_inode_item(inode, lock); scoutfs_inode_queue_writeback(inode); } scoutfs_release_trans(sb); diff --git a/kmod/src/file.c b/kmod/src/file.c index 2878d61e..b59df5e0 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -26,6 +26,8 @@ #include "item.h" #include "lock.h" #include "file.h" +#include "inode.h" +#include "per_task.h" /* TODO: Direct I/O, AIO */ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, @@ -33,14 +35,18 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, { struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); int ret; ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret == 0) { + scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); ret = generic_file_aio_read(iocb, iov, nr_segs, pos); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); } @@ -52,8 +58,10 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, { struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); int ret; if (iocb->ki_left == 0) /* Does this even happen? */ @@ -65,10 +73,13 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, if (ret) goto out; + scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); + /* XXX: remove SUID bit */ ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); out: mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 4b144faa..08e7b31e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -73,6 +73,7 @@ static void scoutfs_inode_ctor(void *obj) mutex_init(&ci->item_mutex); seqcount_init(&ci->seqcount); ci->staging = false; + scoutfs_per_task_init(&ci->pt_data_lock); init_rwsem(&ci->xattr_rwsem); RB_CLEAR_NODE(&ci->writeback_node); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index f590fdfb..dc9a2e60 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -3,6 +3,7 @@ #include "key.h" #include "lock.h" +#include "per_task.h" struct scoutfs_lock; @@ -34,6 +35,7 @@ struct scoutfs_inode_info { /* initialized once for slab object */ seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ + struct scoutfs_per_task pt_data_lock; struct rw_semaphore xattr_rwsem; struct rb_node writeback_node; From 960bc4d53ce0531de3a0710b77a8cecfe905c920 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Oct 2017 13:20:40 -0700 Subject: [PATCH 474/920] scoutfs: add lock coverage for stage ioctl Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index cb4ba017..0b8df1ad 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -421,9 +421,12 @@ out: static long scoutfs_ioc_stage(struct file *file, unsigned long arg) { struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; struct address_space *mapping = inode->i_mapping; struct scoutfs_inode_info *si = SCOUTFS_I(inode); + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); struct scoutfs_ioctl_stage args; + struct scoutfs_lock *lock = NULL; struct kiocb kiocb; struct iovec iov; size_t written; @@ -459,6 +462,13 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) mutex_lock(&inode->i_mutex); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &lock); + if (ret) + goto out; + + scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, lock); + isize = i_size_read(inode); if (!S_ISREG(inode->i_mode) || @@ -492,6 +502,8 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) si->staging = false; current->backing_dev_info = NULL; out: + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, lock, DLM_LOCK_EX); mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); From 950436461a386155135adbc32664f0068bd51696 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Sep 2017 10:28:50 -0700 Subject: [PATCH 475/920] scoutfs: add lock coverage for inode index items Add lock coverage for inode index items. Sadly, this isn't trivial. We have to predict the value of the indexed fields before the operation to lock those items. One value in particular we can't reliably predict: the sequence of the transaction we enter after locking. Also operations can create an absolute ton of index item updates -- rename can modify nr_inodes * items_per_inode * 2 items, so maybe 24 today. And these items can be arbitrarily positioned in the key space. So to handle all this we add functions to gather predicted item values we'll need to lock sort and lock them all, then pass appropriate locks down to the item functions during inode updates. The trickiest bit of the index locking code is having to retry if the sequence number changes. Preparing locks has to guess the sequence number of its upcoming trans and then makes item update decisions based on that. If we enter and have a different sequence number then we need to back off and retry with the correct sequence number (we may find that we'll need to update the indexed meta seq and need to have it locked). The use of the functions is straight forward. Sites figure out the predicted sizes, lock, pass the locks to inode updates, and unlock. While we're at it we replace the individual item field tracking variables in the inode info with an array of indexed values. The code ends up a bit nicer. It also gets rid of the indexed time fields that were left behind and were unused. It's worth noting that we're getting exclusive locks on the index updates. Locking the meta/data seq updates results in complete global serialization of all changes. We'll need concurrent writer locks to get concurrency back. Signed-off-by: Zach Brown --- kmod/src/data.c | 61 ++++- kmod/src/dir.c | 140 ++++++++--- kmod/src/format.h | 12 +- kmod/src/inode.c | 512 +++++++++++++++++++++++++++++++++------ kmod/src/inode.h | 26 +- kmod/src/lock.c | 38 ++- kmod/src/lock.h | 1 + kmod/src/scoutfs_trace.h | 45 ++-- kmod/src/xattr.c | 26 +- 9 files changed, 702 insertions(+), 159 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 05083e8b..bbf21bef 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1133,6 +1133,12 @@ static int scoutfs_writepages(struct address_space *mapping, return mpage_writepages(mapping, wbc, scoutfs_get_block); } +/* fsdata allocated in write_begin and freed in write_end */ +struct write_begin_data { + struct list_head ind_locks; + struct scoutfs_lock *lock; +}; + static int scoutfs_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, @@ -1141,30 +1147,60 @@ static int scoutfs_write_begin(struct file *file, struct inode *inode = mapping->host; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_lock *lock; + struct write_begin_data *wbd; + u64 new_size; + u64 ind_seq; int ret; trace_scoutfs_write_begin(sb, scoutfs_ino(inode), (__u64)pos, len); - lock = scoutfs_per_task_get(&si->pt_data_lock); - if (WARN_ON_ONCE(!lock)) - return -EINVAL; + wbd = kmalloc(sizeof(struct write_begin_data), GFP_NOFS); + if (!wbd) + return -ENOMEM; - ret = scoutfs_hold_trans(sb, SIC_WRITE_BEGIN()); - if (ret) + INIT_LIST_HEAD(&wbd->ind_locks); + *fsdata = wbd; + + wbd->lock = scoutfs_per_task_get(&si->pt_data_lock); + if (WARN_ON_ONCE(!wbd->lock)) { + ret = -EINVAL; + goto out; + } + + /* + * Lock a size update item assuming we perform the full write. + * If If the write is inside i_size then we don't lock and + * nothing will be updated. Lock granularity is larger than + * pages so any size update in this call will be covered by the + * lock. If there's an error and we don't change i_size then + * the item update won't happen and the lock will be unused. + */ + new_size = max(pos + len, i_size_read(inode)); + do { + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &wbd->ind_locks, inode, + new_size, true) ?: + scoutfs_inode_index_lock_hold(sb, &wbd->ind_locks, + ind_seq, SIC_WRITE_BEGIN()); + } while (ret > 0); + if (ret < 0) goto out; /* can't re-enter fs, have trans */ flags |= AOP_FLAG_NOFS; /* generic write_end updates i_size and calls dirty_inode */ - ret = scoutfs_dirty_inode_item(inode, lock); + ret = scoutfs_dirty_inode_item(inode, wbd->lock); if (ret == 0) ret = block_write_begin(mapping, pos, len, flags, pagep, scoutfs_get_block); if (ret) scoutfs_release_trans(sb); out: + if (ret) { + scoutfs_inode_index_unlock(sb, &wbd->ind_locks); + kfree(wbd); + } return ret; } @@ -1175,26 +1211,25 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, struct inode *inode = mapping->host; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_lock *lock; + struct write_begin_data *wbd = fsdata; int ret; trace_scoutfs_write_end(sb, scoutfs_ino(inode), page->index, (u64)pos, len, copied); - /* always call write_end, update_inode will bark if there's no lock */ - lock = scoutfs_per_task_get(&si->pt_data_lock); - ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); if (ret > 0) { if (!si->staging) { scoutfs_inode_set_data_seq(inode); scoutfs_inode_inc_data_version(inode); } - /* XXX kind of a big hammer, inode life cycle needs work */ - scoutfs_update_inode_item(inode, lock); + + scoutfs_update_inode_item(inode, wbd->lock, &wbd->ind_locks); scoutfs_inode_queue_writeback(inode); } scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &wbd->ind_locks); + kfree(wbd); return ret; } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0a46cce2..b8713e35 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -538,11 +538,14 @@ out: static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev, const struct scoutfs_item_count cnt, + u64 dir_size, u64 inode_size, struct scoutfs_lock **dir_lock, - struct scoutfs_lock **inode_lock) + struct scoutfs_lock **inode_lock, + struct list_head *ind_locks) { struct super_block *sb = dir->i_sb; struct inode *inode; + u64 ind_seq; int ret = 0; u64 ino; @@ -568,7 +571,14 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, if (ret) goto out_unlock; - ret = scoutfs_hold_trans(sb, cnt); +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, ind_locks, dir, dir_size, true) ?: + scoutfs_inode_index_prepare_ino(sb, ind_locks, ino, mode, + inode_size) ?: + scoutfs_inode_index_lock_hold(sb, ind_locks, ind_seq, cnt); + if (ret > 0) + goto retry; if (ret) goto out_unlock; @@ -584,6 +594,7 @@ out: scoutfs_release_trans(sb); out_unlock: if (ret) { + scoutfs_inode_index_unlock(sb, ind_locks); scoutfs_unlock(sb, *dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, *inode_lock, DLM_LOCK_EX); *dir_lock = NULL; @@ -602,16 +613,18 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct inode *inode = NULL; struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 dir_size; u64 pos; int ret; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - + dir_size = i_size_read(dir) + dentry->d_name.len; inode = lock_hold_create(dir, dentry, mode, rdev, - SIC_MKNOD(dentry->d_name.len), - &dir_lock, &inode_lock); + SIC_MKNOD(dentry->d_name.len), dir_size, 0, + &dir_lock, &inode_lock, &ind_locks); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -625,7 +638,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, update_dentry_info(dentry, pos); - i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + i_size_write(dir, dir_size); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; @@ -634,13 +647,15 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, inc_nlink(dir); } - scoutfs_update_inode_item(inode, inode_lock); - scoutfs_update_inode_item(dir, dir_lock); + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); + scoutfs_inode_index_unlock(sb, &ind_locks); insert_inode_hash(inode); d_instantiate(dentry, inode); out: scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); @@ -669,6 +684,9 @@ static int scoutfs_link(struct dentry *old_dentry, struct super_block *sb = dir->i_sb; struct scoutfs_lock *dir_lock; struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 dir_size; + u64 ind_seq; u64 pos; int ret; @@ -690,7 +708,17 @@ static int scoutfs_link(struct dentry *old_dentry, if (ret) goto out_unlock; - ret = scoutfs_hold_trans(sb, SIC_LINK(dentry->d_name.len)); + dir_size = i_size_read(dir) + dentry->d_name.len; +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, + dir_size, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, + i_size_read(inode), false) ?: + scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + SIC_LINK(dentry->d_name.len)); + if (ret > 0) + goto retry; if (ret) goto out_unlock; @@ -707,19 +735,20 @@ static int scoutfs_link(struct dentry *old_dentry, goto out; update_dentry_info(dentry, pos); - i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + i_size_write(dir, dir_size); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_ctime = dir->i_mtime; inc_nlink(inode); - scoutfs_update_inode_item(inode, inode_lock); - scoutfs_update_inode_item(dir, dir_lock); + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); atomic_inc(&inode->i_count); d_instantiate(dentry, inode); out: scoutfs_release_trans(sb); out_unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); return ret; @@ -747,6 +776,9 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct timespec ts = current_kernel_time(); struct scoutfs_lock *inode_lock = NULL; struct scoutfs_lock *dir_lock = NULL; + LIST_HEAD(ind_locks); + u64 dir_size; + u64 ind_seq; int ret = 0; ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, @@ -760,7 +792,17 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto unlock; } - ret = scoutfs_hold_trans(sb, SIC_UNLINK(dentry->d_name.len)); + dir_size = i_size_read(dir) - dentry->d_name.len; +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, + dir_size, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, + i_size_read(inode), false) ?: + scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + SIC_UNLINK(dentry->d_name.len)); + if (ret > 0) + goto retry; if (ret) goto unlock; @@ -784,7 +826,7 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) dir->i_ctime = ts; dir->i_mtime = ts; - i_size_write(dir, i_size_read(dir) - dentry->d_name.len); + i_size_write(dir, dir_size); inode->i_ctime = ts; drop_nlink(inode); @@ -792,12 +834,13 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) drop_nlink(dir); drop_nlink(inode); } - scoutfs_update_inode_item(inode, inode_lock); - scoutfs_update_inode_item(dir, dir_lock); + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); out: scoutfs_release_trans(sb); unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); @@ -961,6 +1004,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct inode *inode = NULL; struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; + LIST_HEAD(ind_locks); + u64 dir_size; u64 pos; int ret; @@ -973,9 +1018,11 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; + dir_size = i_size_read(dir) + dentry->d_name.len; inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, SIC_SYMLINK(dentry->d_name.len, name_len), - &dir_lock, &inode_lock); + dir_size, name_len, + &dir_lock, &inode_lock, &ind_locks); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -994,14 +1041,14 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, update_dentry_info(dentry, pos); - i_size_write(dir, i_size_read(dir) + dentry->d_name.len); + i_size_write(dir, dir_size); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_ctime = dir->i_mtime; i_size_write(inode, name_len); - scoutfs_update_inode_item(inode, inode_lock); - scoutfs_update_inode_item(dir, dir_lock); + scoutfs_update_inode_item(inode, inode_lock, &ind_locks); + scoutfs_update_inode_item(dir, dir_lock, &ind_locks); insert_inode_hash(inode); /* XXX need to set i_op/fop before here for sec callbacks */ @@ -1017,6 +1064,7 @@ out: } scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); @@ -1339,6 +1387,10 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, bool ins_new = false; bool del_new = false; bool ins_old = false; + LIST_HEAD(ind_locks); + u64 old_size; + u64 uninitialized_var(new_size); + u64 ind_seq; u64 new_pos; int ret; int err; @@ -1388,8 +1440,35 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (ret) goto out_unlock; - ret = scoutfs_hold_trans(sb, SIC_RENAME(old_dentry->d_name.len, - new_dentry->d_name.len)); + old_size = i_size_read(old_dir) - old_dentry->d_name.len; + if (!new_inode) { + if (old_dir != new_dir) + new_size = i_size_read(new_dir) + + new_dentry->d_name.len; + else + old_size += new_dentry->d_name.len; + } else { + if (old_dir != new_dir) + new_size = i_size_read(new_dir); + } + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_dir, + old_size, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_inode, + i_size_read(old_inode), false) ?: + (new_dir == old_dir ? 0 : + scoutfs_inode_index_prepare(sb, &ind_locks, new_dir, + new_size, false)) ?: + (new_inode == NULL ? 0 : + scoutfs_inode_index_prepare(sb, &ind_locks, new_inode, + i_size_read(new_inode), false)) ?: + scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + SIC_RENAME(old_dentry->d_name.len, + new_dentry->d_name.len)); + if (ret > 0) + goto retry; if (ret) goto out_unlock; @@ -1450,10 +1529,9 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, /* the caller will use d_move to move the old_dentry into place */ update_dentry_info(old_dentry, new_pos); - i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); - if (!new_inode) - i_size_write(new_dir, i_size_read(new_dir) + - new_dentry->d_name.len); + i_size_write(old_dir, old_size); + if (old_dir != new_dir) + i_size_write(new_dir, new_size); if (new_inode) { drop_nlink(new_inode); @@ -1477,12 +1555,13 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (new_inode) old_inode->i_ctime = now; - scoutfs_update_inode_item(old_dir, old_dir_lock); - scoutfs_update_inode_item(old_inode, old_inode_lock); + scoutfs_update_inode_item(old_dir, old_dir_lock, &ind_locks); + scoutfs_update_inode_item(old_inode, old_inode_lock, &ind_locks); if (new_dir != old_dir) - scoutfs_update_inode_item(new_dir, new_dir_lock); + scoutfs_update_inode_item(new_dir, new_dir_lock, &ind_locks); if (new_inode) - scoutfs_update_inode_item(new_inode, new_inode_lock); + scoutfs_update_inode_item(new_inode, new_inode_lock, + &ind_locks); ret = 0; out: @@ -1532,6 +1611,7 @@ out: scoutfs_release_trans(sb); out_unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, old_inode_lock, DLM_LOCK_EX); scoutfs_unlock(sb, new_inode_lock, DLM_LOCK_EX); scoutfs_unlock(sb, old_dir_lock, DLM_LOCK_EX); diff --git a/kmod/src/format.h b/kmod/src/format.h index 17c53a18..89b95ff1 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -239,12 +239,10 @@ struct scoutfs_segment_block { #define SCOUTFS_MAX_ZONE 4 /* power of 2 is efficient */ /* inode index zone */ -#define SCOUTFS_INODE_INDEX_SIZE_TYPE 3 -#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 4 -#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 5 - -#define SCOUTFS_INODE_INDEX_NR \ - (SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE - SCOUTFS_INODE_INDEX_SIZE_TYPE + 1) +#define SCOUTFS_INODE_INDEX_SIZE_TYPE 1 +#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 2 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 3 +#define SCOUTFS_INODE_INDEX_NR 4 /* don't forget to update */ /* node zone */ #define SCOUTFS_FREE_BITS_SEGNO_TYPE 1 @@ -549,6 +547,8 @@ struct scoutfs_lock_name { #define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) #define SCOUTFS_LOCK_INODE_GROUP_OFFSET (~0ULL) +#define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) + /* * messages over the wire. */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 08e7b31e..48baea03 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -32,6 +33,7 @@ #include "kvec.h" #include "item.h" #include "client.h" +#include "cmp.h" /* * XXX @@ -39,6 +41,12 @@ * - use inode item value lengths for forward/back compat */ +/* + * XXX before committing: + * - describe all this better + * - describe data locking size problems + */ + struct free_ino_pool { wait_queue_head_t waitq; spinlock_t lock; @@ -201,14 +209,16 @@ static void set_item_info(struct scoutfs_inode_info *si, { BUG_ON(!mutex_is_locked(&si->item_mutex)); + memset(si->item_majors, 0, sizeof(si->item_majors)); + memset(si->item_minors, 0, sizeof(si->item_minors)); + si->have_item = true; - si->item_size = le64_to_cpu(sinode->size); - si->item_ctime.tv_sec = le64_to_cpu(sinode->ctime.sec); - si->item_ctime.tv_nsec = le32_to_cpu(sinode->ctime.nsec); - si->item_mtime.tv_sec = le64_to_cpu(sinode->mtime.sec); - si->item_mtime.tv_nsec = le32_to_cpu(sinode->mtime.nsec); - si->item_meta_seq = le64_to_cpu(sinode->meta_seq); - si->item_data_seq = le64_to_cpu(sinode->data_seq); + si->item_majors[SCOUTFS_INODE_INDEX_SIZE_TYPE] = + le64_to_cpu(sinode->size); + si->item_majors[SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = + le64_to_cpu(sinode->meta_seq); + si->item_majors[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = + le64_to_cpu(sinode->data_seq); } static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) @@ -517,87 +527,190 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) return ret; } +struct index_lock { + struct list_head head; + struct scoutfs_lock *lock; + u8 type; + u64 major; + u32 minor; + u64 ino; +}; + +static bool will_del_index(struct scoutfs_inode_info *si, + u8 type, u64 major, u32 minor) +{ + return si && si->have_item && + (si->item_majors[type] != major || + si->item_minors[type] != minor); +} + +static bool will_ins_index(struct scoutfs_inode_info *si, + u8 type, u64 major, u32 minor) +{ + return !si || !si->have_item || + (si->item_majors[type] != major || + si->item_minors[type] != minor); +} + +static bool inode_has_index(umode_t mode, u8 type) +{ + switch(type) { + case SCOUTFS_INODE_INDEX_SIZE_TYPE: + case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: + return true; + case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: + return S_ISREG(mode); + default: + return WARN_ON_ONCE(false); + } +} + +static int cmp_index_lock(void *priv, struct list_head *A, struct list_head *B) +{ + struct index_lock *a = list_entry(A, struct index_lock, head); + struct index_lock *b = list_entry(B, struct index_lock, head); + + return ((int)a->type - (int)b->type) ?: + scoutfs_cmp_u64s(a->major, b->major) ?: + scoutfs_cmp_u64s(a->minor, b->minor) ?: + scoutfs_cmp_u64s(a->ino, b->ino); +} + /* - * Make sure inode index items are kept in sync with the fields that are - * set in the inode items. This must be called any time the contents of - * the inode items are updated. - * - * This is effectively a RMW on the inode fields so the caller needs to - * lock the inode so that it's the only one working with the index items - * for a given set of fields in the inode. - * - * But it doesn't need to lock the index item keys. By locking the - * inode we've ensured that we can safely log deletion and insertion - * items in our log. The indexes are eventually consistent so we don't - * need to wrap them locks. - * - * XXX this needs more supporting work from the rest of the - * infrastructure: - * - * - Deleting and creating the items needs to forcefully set those dirty - * items in the cache without first trying to read them from segments. - * - the reading ioctl needs to forcefully invalidate the index items - * as it walks. - * - maybe the reading ioctl needs to verify fields with inodes? - * - final inode deletion needs to invalidate the index items for - * each inode as it deletes items based on the locked inode fields. - * - make sure deletion items safely vanish w/o finding existing item - * - ... error handling :( + * Find the lock that covers the given index item. Returns NULL if + * there isn't a lock that covers the item. We know that the list is + * sorted at this point so we can stop once our search value is less + * than a list entry. */ -static int update_index(struct super_block *sb, struct scoutfs_inode_info *si, - u64 ino, u8 type, u64 now_major, u32 now_minor, - u64 then_major, u32 then_minor) +static struct scoutfs_lock *find_index_lock(struct list_head *lock_list, + u8 type, u64 major, u32 minor, + u64 ino) +{ + struct index_lock *ind_lock; + struct index_lock needle; + int cmp; + + scoutfs_lock_clamp_inode_index(type, &major, &minor, &ino); + needle.type = type; + needle.major = major; + needle.minor = minor; + needle.ino = ino; + + list_for_each_entry(ind_lock, lock_list, head) { + cmp = cmp_index_lock(NULL, &needle.head, &ind_lock->head); + if (cmp == 0) + return ind_lock->lock; + if (cmp < 0) + break; + } + + return NULL; +} + +/* + * The inode info reflects the current inode index items. Create or delete + * index items to bring the index in line with the caller's item. The list + * should contain locks that cover any item modifications that are made. + */ +static int update_index_items(struct super_block *sb, + struct scoutfs_inode_info *si, u64 ino, u8 type, + u64 major, u32 minor, + struct list_head *lock_list) { struct scoutfs_inode_index_key ins_ikey; struct scoutfs_inode_index_key del_ikey; + struct scoutfs_lock *ins_lock; + struct scoutfs_lock *del_lock; struct scoutfs_key_buf ins; struct scoutfs_key_buf del; int ret; int err; - trace_scoutfs_inode_update_index(sb, ino, si->have_item, now_major, - now_minor, then_major, then_minor); - - if (si->have_item && now_major == then_major && now_minor == then_minor) + if (!will_ins_index(si, type, major, minor)) return 0; + trace_scoutfs_create_index_item(sb, type, major, minor, ino); + ins_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; ins_ikey.type = type; - ins_ikey.major = cpu_to_be64(now_major); - ins_ikey.minor = cpu_to_be32(now_minor); + ins_ikey.major = cpu_to_be64(major); + ins_ikey.minor = cpu_to_be32(minor); ins_ikey.ino = cpu_to_be64(ino); scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); + ins_lock = find_index_lock(lock_list, type, major, minor, ino); ret = scoutfs_item_create(sb, &ins, NULL); - if (ret || !si->have_item) + if (ret || !will_del_index(si, type, major, minor)) return ret; + trace_scoutfs_delete_index_item(sb, type, major, minor, ino); + del_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; del_ikey.type = type; - del_ikey.major = cpu_to_be64(then_major); - del_ikey.minor = cpu_to_be32(then_minor); + del_ikey.major = cpu_to_be64(si->item_majors[type]); + del_ikey.minor = cpu_to_be32(si->item_minors[type]); del_ikey.ino = cpu_to_be64(ino); scoutfs_key_init(&del, &del_ikey, sizeof(del_ikey)); - ret = scoutfs_item_delete(sb, &del, NULL); + del_lock = find_index_lock(lock_list, type, si->item_majors[type], + si->item_minors[type], ino); + ret = scoutfs_item_delete(sb, &del, del_lock->end); if (ret) { - err = scoutfs_item_delete(sb, &ins, NULL); + err = scoutfs_item_delete(sb, &ins, ins_lock->end); BUG_ON(err); } return ret; } +static int update_indices(struct super_block *sb, + struct scoutfs_inode_info *si, u64 ino, umode_t mode, + struct scoutfs_inode *sinode, + struct list_head *lock_list) +{ + struct index_update { + u8 type; + u64 major; + u32 minor; + } *upd, upds[] = { + { SCOUTFS_INODE_INDEX_SIZE_TYPE, + le64_to_cpu(sinode->size), 0 }, + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0 }, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0 }, + }; + int ret; + int i; + + for (i = 0, upd = upds; i < ARRAY_SIZE(upds); i++, upd++) { + if (!inode_has_index(mode, upd->type)) + continue; + + ret = update_index_items(sb, si, ino, upd->type, upd->major, + upd->minor, lock_list); + if (ret) + break; + } + + return ret; +} + /* * Every time we modify the inode in memory we copy it to its inode * item. This lets us write out items without having to track down * dirty vfs inodes. * * The caller makes sure that the item is dirty and pinned so they don't - * have to deal with errors and unwinding after they've modified the - * vfs inode and get here. + * have to deal with errors and unwinding after they've modified the vfs + * inode and get here. + * + * Index items that track inode fields are updated here as we update the + * inode item. The caller must have acquired locks on all the index + * items that might change. */ -void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock) +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, + struct list_head *lock_list) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; @@ -617,16 +730,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock) /* only race with other inode field stores once */ store_inode(&sinode, inode); - ret = update_index(sb, si, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, - le64_to_cpu(sinode.size), 0, si->item_size, 0) ?: - update_index(sb, si, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, - le64_to_cpu(sinode.meta_seq), 0, - si->item_meta_seq, 0); - if (ret == 0 && S_ISREG(inode->i_mode)) - ret = update_index(sb, si, ino, - SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, - le64_to_cpu(sinode.data_seq), 0, - si->item_data_seq, 0); + ret = update_indices(sb, si, ino, inode->i_mode, &sinode, lock_list); BUG_ON(ret); scoutfs_inode_init_key(&key, &ikey, ino); @@ -644,12 +748,251 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock) mutex_unlock(&si->item_mutex); } +/* + * We map the item to coarse locks here. This reduces the number of + * locks we track and means that when we later try to find the lock that + * covers an item we can deal with the item update changing a little + * (seq, size) while still being covered. It does mean we have to share + * some logic with lock naming. + */ +static int add_index_lock(struct list_head *list, u64 ino, u8 type, u64 major, + u32 minor) +{ + struct index_lock *ind_lock; + + scoutfs_lock_clamp_inode_index(type, &major, &minor, &ino); + + list_for_each_entry(ind_lock, list, head) { + if (ind_lock->type == type && ind_lock->major == major && + ind_lock->minor == minor && ind_lock->ino == ino) { + return 0; + } + } + + ind_lock = kzalloc(sizeof(struct index_lock), GFP_NOFS); + if (!ind_lock) + return -ENOMEM; + + ind_lock->type = type; + ind_lock->major = major; + ind_lock->minor = minor; + ind_lock->ino = ino; + list_add(&ind_lock->head, list); + + return 0; +} + +static int prepare_index_items(struct scoutfs_inode_info *si, + struct list_head *list, u64 ino, umode_t mode, + u8 type, u64 major, u32 minor) +{ + int ret; + + if (will_ins_index(si, type, major, minor)) { + ret = add_index_lock(list, ino, type, major, minor); + if (ret) + return ret; + } + + if (will_del_index(si, type, major, minor)) { + ret = add_index_lock(list, ino, type, si->item_majors[type], + si->item_minors[type]); + if (ret) + return ret; + } + + return 0; +} + +/* + * Return the data seq that we expect to see in the updated inode. The + * caller tells us if they know they're going to update it. If the + * inode doesn't exist it'll also get the current data_seq. + */ +static u64 upd_data_seq(struct scoutfs_sb_info *sbi, + struct scoutfs_inode_info *si, bool set_data_seq) +{ + if (!si || !si->have_item || set_data_seq) + return sbi->trans_seq; + + return si->item_majors[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE]; +} + +/* + * Prepare locks that will cover the inode index items that will be + * modified when this inode's item is updated during the upcoming + * transaction. + * + * To lock the index items that will be created we need to predict the + * new indexed values. We assume that the meta seq will always be set + * to the current seq. This will usually be a nop in a running + * transaction. The caller tells us what the size will be and whether + * data_seq will also be set to the current transaction. + */ +static int prepare_indices(struct super_block *sb, struct list_head *list, + struct scoutfs_inode_info *si, u64 ino, + umode_t mode, u64 new_size, bool set_data_seq) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct index_update { + u8 type; + u64 major; + u32 minor; + } *upd, upds[] = { + { SCOUTFS_INODE_INDEX_SIZE_TYPE, new_size, 0}, + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, sbi->trans_seq, 0}, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + upd_data_seq(sbi, si, set_data_seq), 0}, + }; + int ret; + int i; + + for (i = 0, upd = upds; i < ARRAY_SIZE(upds); i++, upd++) { + if (!inode_has_index(mode, upd->type)) + continue; + + ret = prepare_index_items(si, list, ino, mode, + upd->type, upd->major, upd->minor); + if (ret) + break; + } + + return ret; +} + +int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, + struct inode *inode, u64 new_size, + bool set_data_seq) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return prepare_indices(sb, list, si, scoutfs_ino(inode), + inode->i_mode, new_size, set_data_seq); +} + +/* + * This is used to initially create the index items for a newly created + * inode. We don't have a populated vfs inode yet. The existing + * indexed values don't matter because it's 'have_item' is false. It + * will try to create all the appropriate index items. + */ +int scoutfs_inode_index_prepare_ino(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode, u64 new_size) +{ + return prepare_indices(sb, list, NULL, ino, mode, new_size, true); +} + +/* + * Prepare the locks needed to delete all the index items associated + * with the inode. We know the items have to exist and can skip straight + * to adding locks for each of them. + */ +static int prepare_index_deletion(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode, struct scoutfs_inode *sinode) +{ + struct index_item { + u8 type; + u64 major; + u32 minor; + } *ind, inds[] = { + { SCOUTFS_INODE_INDEX_SIZE_TYPE, + le64_to_cpu(sinode->size), 0 }, + { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + le64_to_cpu(sinode->meta_seq), 0 }, + { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, + le64_to_cpu(sinode->data_seq), 0 }, + }; + int ret; + int i; + + for (i = 0, ind = inds; i < ARRAY_SIZE(inds); i++, ind++) { + if (!inode_has_index(mode, ind->type)) + continue; + + ret = add_index_lock(list, ino, ind->type, ind->major, + ind->minor); + if (ret) + break; + } + + return ret; +} + +/* + * Sample the transaction sequence before we start checking it to see if + * indexed meta seq and data seq items will change. + */ +int scoutfs_inode_index_start(struct super_block *sb, u64 *seq) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + /* XXX this feels racey in a bad way :) */ + *seq = sbi->trans_seq; + return 0; +} + +/* + * Acquire the prepared index locks and hold the transaction. If the + * sequence number changes as we enter the transaction then we need to + * retry so that we can use the new seq to prepare locks. + * + * Returns > 0 if the seq changed and the locks should be retried. + */ +int scoutfs_inode_index_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct index_lock *ind_lock; + int ret = 0; + + list_sort(NULL, list, cmp_index_lock); + + list_for_each_entry(ind_lock, list, head) { + ret = scoutfs_lock_inode_index(sb, DLM_LOCK_EX, ind_lock->type, + ind_lock->major, ind_lock->ino, + &ind_lock->lock); + if (ret) + goto out; + } + + ret = scoutfs_hold_trans(sb, cnt); + if (ret == 0 && seq != sbi->trans_seq) { + scoutfs_release_trans(sb); + ret = 1; + } + +out: + if (ret) + scoutfs_inode_index_unlock(sb, list); + + return ret; +} + +/* + * Unlocks and frees all the locks on the list. + */ +void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list) +{ + struct index_lock *ind_lock; + struct index_lock *tmp; + + list_for_each_entry_safe(ind_lock, tmp, list, head) { + scoutfs_unlock(sb, ind_lock->lock, DLM_LOCK_EX); + list_del_init(&ind_lock->head); + kfree(ind_lock); + } +} + /* this is called on final inode cleanup so enoent is fine */ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, - u32 minor) + u32 minor, struct list_head *ind_locks) { struct scoutfs_inode_index_key ikey; struct scoutfs_key_buf key; + struct scoutfs_lock *lock; int ret; ikey.zone = SCOUTFS_INODE_INDEX_ZONE; @@ -659,8 +1002,8 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, ikey.ino = cpu_to_be64(ino); scoutfs_key_init(&key, &ikey, sizeof(ikey)); - /* XXX would be deletion under CW that doesn't need to read */ - ret = scoutfs_item_delete(sb, &key, NULL); + lock = find_index_lock(ind_locks, type, major, minor, ino); + ret = scoutfs_item_delete(sb, &key, lock->end); if (ret == -ENOENT) ret = 0; return ret; @@ -676,18 +1019,19 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, * the time they get to it, including being deleted. */ static int remove_index_items(struct super_block *sb, u64 ino, - struct scoutfs_inode *sinode) + struct scoutfs_inode *sinode, + struct list_head *ind_locks) { umode_t mode = le32_to_cpu(sinode->mode); int ret; ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, - le64_to_cpu(sinode->size), 0) ?: + le64_to_cpu(sinode->size), 0, ind_locks) ?: remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, - le64_to_cpu(sinode->meta_seq), 0); + le64_to_cpu(sinode->meta_seq), 0, ind_locks); if (ret == 0 && S_ISREG(mode)) ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, - le64_to_cpu(sinode->data_seq), 0); + le64_to_cpu(sinode->data_seq), 0, ind_locks); return ret; } @@ -825,13 +1169,14 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; - ci->meta_seq = 0; - ci->data_seq = 0; ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; atomic64_set(&ci->last_refreshed, scoutfs_lock_refresh_gen(lock)); + scoutfs_inode_set_meta_seq(inode); + scoutfs_inode_set_data_seq(inode); + inode->i_ino = ino; /* XXX overflow */ inode_init_owner(inode, dir, mode); inode_set_bytes(inode, 0); @@ -890,41 +1235,56 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) */ static int delete_inode_items(struct super_block *sb, u64 ino) { + struct scoutfs_lock *lock = NULL; struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; struct scoutfs_key_buf key; SCOUTFS_DECLARE_KVEC(val); + LIST_HEAD(ind_locks); bool release = false; umode_t mode; + u64 ind_seq; int ret; + ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lock); + if (ret) + return ret; + scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), NULL); + ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; - return ret; + goto out; } /* XXX corruption, inode probably won't be freed without repair */ if (le32_to_cpu(sinode.nlink)) { scoutfs_warn(sb, "Dangling orphan item for inode %llu.", ino); - return -EIO; + ret = -EIO; + goto out; } mode = le32_to_cpu(sinode.mode); trace_scoutfs_delete_inode(sb, ino, mode); - /* XXX this is obviously not done yet :) */ - ret = scoutfs_hold_trans(sb, SIC_DIRTY_INODE()); + /* XXX the trans reservation count is obviously bonkers :) */ +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + prepare_index_deletion(sb, &ind_locks, ino, mode, &sinode) ?: + scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + SIC_DIRTY_INODE()); + if (ret > 0) + goto retry; if (ret) goto out; + release = true; /* first remove index items to try to avoid indexing partial deletion */ - ret = remove_index_items(sb, ino, &sinode); + ret = remove_index_items(sb, ino, &sinode, &ind_locks); if (ret) goto out; @@ -941,7 +1301,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) goto out; #endif - ret = scoutfs_item_delete(sb, &key, NULL); + ret = scoutfs_item_delete(sb, &key, lock->end); if (ret) goto out; @@ -949,6 +1309,8 @@ static int delete_inode_items(struct super_block *sb, u64 ino) out: if (release) scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, lock, DLM_LOCK_EX); return ret; } diff --git a/kmod/src/inode.h b/kmod/src/inode.h index dc9a2e60..1e0b4f5e 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -4,6 +4,7 @@ #include "key.h" #include "lock.h" #include "per_task.h" +#include "count.h" struct scoutfs_lock; @@ -23,11 +24,8 @@ struct scoutfs_inode_info { */ struct mutex item_mutex; bool have_item; - u64 item_size; - struct timespec item_ctime; - struct timespec item_mtime; - u64 item_meta_seq; - u64 item_data_seq; + u64 item_majors[SCOUTFS_INODE_INDEX_NR]; + u32 item_minors[SCOUTFS_INODE_INDEX_NR]; /* updated at on each new lock acquisition */ atomic64_t last_refreshed; @@ -63,13 +61,29 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); + +int scoutfs_inode_index_start(struct super_block *sb, u64 *seq); +int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, + struct inode *inode, u64 new_size, + bool set_data_seq); +int scoutfs_inode_index_prepare_ino(struct super_block *sb, + struct list_head *list, u64 ino, + umode_t mode, u64 new_size); +int scoutfs_inode_index_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt); +void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list); + int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); -void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock); +void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, + struct list_head *ind_locks); + void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev, u64 ino, struct scoutfs_lock *lock); + void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 75c0fa7c..875ac62b 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -612,6 +612,42 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, NULL, NULL, lock); } +/* + * Set the caller's major, minor, and ino to the start of lock that + * covers the incoming index item. This can be used to discover when + * multiple items map to the same lock. + */ +void scoutfs_lock_clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino) +{ + u64 major_mask; + u64 ino_mask; + int bit; + + switch(type) { + case SCOUTFS_INODE_INDEX_SIZE_TYPE: + major_mask = 0; + if (*major) { + bit = fls64(*major); + if (bit > 4) + major_mask = (1 << (bit - 4)) - 1; + } + ino_mask = (1 << 12) - 1; + break; + + case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: + case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: + major_mask = SCOUTFS_LOCK_SEQ_GROUP_MASK; + ino_mask = ~0ULL; + break; + default: + BUG(); + } + + *major &= ~major_mask; + *minor = 0; + *ino &= ~ino_mask; +} + /* * map inode index items to locks. The idea is to not have to * constantly get locks over a reasonable distribution of items, but @@ -647,7 +683,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: - major_mask = (1 << 10) - 1; + major_mask = SCOUTFS_LOCK_SEQ_GROUP_MASK; ino_mask = ~0ULL; break; default: diff --git a/kmod/src/lock.h b/kmod/src/lock.h index cd80a4fd..30e95458 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -36,6 +36,7 @@ int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock); +void scoutfs_lock_clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino); int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 678e7c69..8e0fd843 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -782,38 +782,43 @@ TRACE_EVENT(scoutfs_i_callback, TP_printk("freeing inode %p", __entry->inode) ); -TRACE_EVENT(scoutfs_inode_update_index, - TP_PROTO(struct super_block *sb, __u64 ino, unsigned int have_item, - __u64 now_major, unsigned int now_minor, __u64 then_major, - unsigned int then_minor), +DECLARE_EVENT_CLASS(scoutfs_index_item_class, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), - TP_ARGS(sb, ino, have_item, now_major, now_minor, then_major, - then_minor), + TP_ARGS(sb, type, major, minor, ino), TP_STRUCT__entry( __field(__u64, fsid) + __field(__u8, type) + __field(__u64, major) + __field(__u32, minor) __field(__u64, ino) - __field(unsigned int, have_item) - __field(__u64, now_major) - __field(unsigned int, now_minor) - __field(__u64, then_major) - __field(unsigned int, then_minor) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); + __entry->type = type; + __entry->major = major; + __entry->minor = minor; __entry->ino = ino; - __entry->have_item = have_item; - __entry->now_major = now_major; - __entry->now_minor = now_minor; - __entry->then_major = then_major; - __entry->then_minor = then_minor; ), - TP_printk(FSID_FMT" ino %llu have %u now %llu.%u then %llu.%u", - __entry->fsid, __entry->ino, __entry->have_item, - __entry->now_major, __entry->now_minor, __entry->then_major, - __entry->then_minor) + TP_printk("fsid "FSID_FMT" type %u major %llu minor %u ino %llu", + __entry->fsid, __entry->type, __entry->major, __entry->minor, + __entry->ino) +); + +DEFINE_EVENT(scoutfs_index_item_class, scoutfs_create_index_item, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), + TP_ARGS(sb, type, major, minor, ino) +); + +DEFINE_EVENT(scoutfs_index_item_class, scoutfs_delete_index_item, + TP_PROTO(struct super_block *sb, __u8 type, __u64 major, __u32 minor, + __u64 ino), + TP_ARGS(sb, type, major, minor, ino) ); TRACE_EVENT(scoutfs_inode_fill_pool, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index fc97691f..70b2584f 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -264,10 +264,12 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); - struct scoutfs_lock *lck; + struct scoutfs_lock *lck = NULL; unsigned int bytes; unsigned int off; + LIST_HEAD(ind_locks); LIST_HEAD(list); + u64 ind_seq; u8 part; int sif; int ret; @@ -299,7 +301,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, ret = scoutfs_item_add_batch(sb, &list, key, val); if (ret) - goto unlock; + goto out; } } @@ -315,28 +317,36 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, else sif = 0; - ret = scoutfs_hold_trans(sb, SIC_XATTR_SET(name_len, size)); + down_write(&si->xattr_rwsem); + +retry: + ret = scoutfs_inode_index_start(sb, &ind_seq) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, + i_size_read(inode), false) ?: + scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + SIC_XATTR_SET(name_len, size)); + if (ret > 0) + goto retry; if (ret) goto unlock; - down_write(&si->xattr_rwsem); - ret = scoutfs_dirty_inode_item(inode, lck) ?: scoutfs_item_set_batch(sb, &list, key, last, sif, lck->end); if (ret == 0) { /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); inode->i_ctime = CURRENT_TIME; - scoutfs_update_inode_item(inode, lck); + scoutfs_update_inode_item(inode, lck, &ind_locks); } - up_write(&si->xattr_rwsem); scoutfs_release_trans(sb); unlock: - scoutfs_unlock(sb, lck, DLM_LOCK_EX); + up_write(&si->xattr_rwsem); out: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, lck, DLM_LOCK_EX); scoutfs_item_free_batch(sb, &list); scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); From 47f5946c904989016aa79262de4a9d50e43080ff Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 11:26:04 -0700 Subject: [PATCH 476/920] scoutfs: fix lock name comparison The lock name comparison had a typo where it didn't compare the second fields between the two names. Only inode index items used the second field. This bug could cause lock matching when the names don't match and trigger lock coverage warnings. While we're in there don't rely so heavily on readers knowing the relative precedence of subtraction and (magical gcc empty) ternary operators. Signed-off-by: Zach Brown --- kmod/src/lock.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 875ac62b..60efede4 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -242,11 +242,11 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, static int cmp_lock_names(struct scoutfs_lock_name *a, struct scoutfs_lock_name *b) { - return (int)a->scope - (int)b->scope ?: - (int)a->zone - (int)b->zone ?: - (int)a->type - (int)b->type ?: + return ((int)a->scope - (int)b->scope) ?: + ((int)a->zone - (int)b->zone) ?: + ((int)a->type - (int)b->type) ?: scoutfs_cmp_u64s(le64_to_cpu(a->first), le64_to_cpu(b->first)) ?: - scoutfs_cmp_u64s(le64_to_cpu(b->second), le64_to_cpu(b->second)); + scoutfs_cmp_u64s(le64_to_cpu(a->second), le64_to_cpu(b->second)); } static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, From 0aa16f5ef68f8b4233bf3b98279e9d295a97178a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 11:57:14 -0700 Subject: [PATCH 477/920] scoutfs: add lock arg to _item_create() scoutfs_item_create() hasn't been working with lock coverage. It wouldn't return -ENOENT if it didn't have the lock cached. It would create items outside lock coverate so they wouldn't be invalidated and re-read if another node modified the item. Add a lock arg and teach it to populate the cache so that it's correctly consistent. Signed-off-by: Zach Brown --- kmod/src/data.c | 9 +++++---- kmod/src/dir.c | 8 ++++---- kmod/src/inode.c | 7 ++++--- kmod/src/item.c | 29 +++++++++++++++++++++-------- kmod/src/item.h | 2 +- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index bbf21bef..f1bd42ce 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -373,7 +373,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) if (ret == -ENOENT) { memset(&frb, 0, sizeof(frb)); set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, val); + ret = scoutfs_item_create(sb, &key, val, lock); goto out; } @@ -399,6 +399,7 @@ static int create_blkno_free(struct super_block *sb, u64 blkno, struct scoutfs_free_bits_key *fbk) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits frb; SCOUTFS_DECLARE_KVEC(val); int bit; @@ -411,7 +412,7 @@ static int create_blkno_free(struct super_block *sb, u64 blkno, memset(&frb, 0xff, sizeof(frb)); clear_bit_le(bit, frb.bits); - return scoutfs_item_create(sb, key, val); + return scoutfs_item_create(sb, key, val, lock); } /* @@ -507,7 +508,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) if (ret == -ENOENT) { memset(&frb, 0, sizeof(frb)); set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, val); + ret = scoutfs_item_create(sb, &key, val, lock); goto out; } @@ -961,7 +962,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, if (map_exists) ret = scoutfs_item_update(sb, map_key, val, data_lock->end); else - ret = scoutfs_item_create(sb, map_key, val); + ret = scoutfs_item_create(sb, map_key, val, data_lock); if (ret) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index b8713e35..502d75c6 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -439,7 +439,7 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_create(sb, ent_key, val); + ret = scoutfs_item_create(sb, ent_key, val, dir_lock); if (ret) goto out; del_ent = true; @@ -448,7 +448,7 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, init_readdir_key(&rdir_key, &rkey, dir_ino, pos); scoutfs_kvec_init(val, &dent, sizeof(dent), (char *)name, name_len); - ret = scoutfs_item_create(sb, &rdir_key, val); + ret = scoutfs_item_create(sb, &rdir_key, val, dir_lock); if (ret) goto out; del_rdir = true; @@ -460,7 +460,7 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, goto out; } - ret = scoutfs_item_create(sb, lb_key, NULL); + ret = scoutfs_item_create(sb, lb_key, NULL, inode_lock); out: if (ret < 0) { if (del_ent) @@ -899,7 +899,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, scoutfs_kvec_init(val, (void *)target, bytes); if (op == SYM_CREATE) - ret = scoutfs_item_create(sb, &key, val); + ret = scoutfs_item_create(sb, &key, val, lock); else if (op == SYM_LOOKUP) ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, lock); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 48baea03..998ce53c 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -639,7 +639,7 @@ static int update_index_items(struct super_block *sb, scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); ins_lock = find_index_lock(lock_list, type, major, minor, ino); - ret = scoutfs_item_create(sb, &ins, NULL); + ret = scoutfs_item_create(sb, &ins, NULL, ins_lock); if (ret || !will_del_index(si, type, major, minor)) return ret; @@ -1188,7 +1188,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_create(sb, &key, val); + ret = scoutfs_item_create(sb, &key, val, lock); if (ret) { iput(inode); return ERR_PTR(ret); @@ -1394,6 +1394,7 @@ int scoutfs_orphan_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_orphan_key okey; struct scoutfs_key_buf key; int ret; @@ -1402,7 +1403,7 @@ int scoutfs_orphan_inode(struct inode *inode) init_orphan_key(&key, &okey, sbi->node_id, scoutfs_ino(inode)); - ret = scoutfs_item_create(sb, &key, NULL); + ret = scoutfs_item_create(sb, &key, NULL, lock); return ret; } diff --git a/kmod/src/item.c b/kmod/src/item.c index 9e793798..f4ce689d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1062,7 +1062,7 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, * XXX but it doesn't read.. is that weird? Seems weird. */ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val) + struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1077,13 +1077,26 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, if (!item) return -ENOMEM; - spin_lock_irqsave(&cac->lock, flags); - ret = insert_item(sb, cac, item, false, false); - if (!ret) { - scoutfs_inc_counter(sb, item_create); - mark_item_dirty(sb, cac, item); - } - spin_unlock_irqrestore(&cac->lock, flags); + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + + do { + spin_lock_irqsave(&cac->lock, flags); + + if (!check_range(sb, &cac->ranges, key, NULL)) { + ret = -ENODATA; + } else { + ret = insert_item(sb, cac, item, false, false); + if (!ret) { + scoutfs_inc_counter(sb, item_create); + mark_item_dirty(sb, cac, item); + } + } + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); if (ret) free_item(sb, item); diff --git a/kmod/src/item.h b/kmod/src/item.h index ece0c8c6..7ea43feb 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -29,7 +29,7 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val); + struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, From 6cd64f32286d0ba816dd9bfddecee0e79539139a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 12:06:32 -0700 Subject: [PATCH 478/920] scoutfs: add full lock arg to _item_update() Add the full lock arg to _item_update() so that it can verify lock coverage. Signed-off-by: Zach Brown --- kmod/src/data.c | 13 ++++++------- kmod/src/inode.c | 2 +- kmod/src/item.c | 7 +++++-- kmod/src/item.h | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index f1bd42ce..da146a2a 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -382,7 +382,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) goto out; } - ret = scoutfs_item_update(sb, &key, val, lock->end); + ret = scoutfs_item_update(sb, &key, val, lock); out: trace_scoutfs_data_set_segno_free(sb, segno, be64_to_cpu(fbk.base), bit, ret); @@ -469,7 +469,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) ret = scoutfs_item_delete(sb, &key, lock->end); else - ret = scoutfs_item_update(sb, &key, val, lock->end); + ret = scoutfs_item_update(sb, &key, val, lock); if (ret) scoutfs_item_delete_dirty(sb, &b_key); out: @@ -518,7 +518,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) } if (!bitmap_full((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) { - ret = scoutfs_item_update(sb, &key, val, lock->end); + ret = scoutfs_item_update(sb, &key, val, lock); goto out; } @@ -578,7 +578,7 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) ret = scoutfs_item_delete(sb, &key, lock->end); else - ret = scoutfs_item_update(sb, &key, val, lock->end); + ret = scoutfs_item_update(sb, &key, val, lock); out: return ret; } @@ -675,8 +675,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, if (!dirtied) { /* dirty item with full size encoded */ - ret = scoutfs_item_update(sb, &key, val, - lock->end); + ret = scoutfs_item_update(sb, &key, val, lock); if (ret) break; dirtied = true; @@ -960,7 +959,7 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, /* ensure that we can copy in encoded without failing */ scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); if (map_exists) - ret = scoutfs_item_update(sb, map_key, val, data_lock->end); + ret = scoutfs_item_update(sb, map_key, val, data_lock); else ret = scoutfs_item_create(sb, map_key, val, data_lock); if (ret) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 998ce53c..57ad8287 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -736,7 +736,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, &key, val, lock->end); + err = scoutfs_item_update(sb, &key, val, lock); if (err) { scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); diff --git a/kmod/src/item.c b/kmod/src/item.c index f4ce689d..f242aa88 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1382,7 +1382,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, * Returns -ENOENT if the item doesn't exist. */ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val, struct scoutfs_key_buf *end) + struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1394,6 +1394,9 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, if (invalid_key_val(key, val)) return -EINVAL; + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + if (val) { ret = scoutfs_kvec_dup_flatten(up_val, val); if (ret) @@ -1420,7 +1423,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); out: scoutfs_kvec_kfree(up_val); diff --git a/kmod/src/item.h b/kmod/src/item.h index 7ea43feb..e442c313 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -33,7 +33,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val, struct scoutfs_key_buf *end); + struct kvec *val, struct scoutfs_lock *lock); void scoutfs_item_delete_dirty(struct super_block *sb, struct scoutfs_key_buf *key); void scoutfs_item_update_dirty(struct super_block *sb, From 9b31c9795b3612586d8d946ca9cea3f7e7de3cc3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 12:09:07 -0700 Subject: [PATCH 479/920] scoutfs: add full lock arg to _item_delete() Add the full lock arg to _item_delete() so that it can verify lock coverage. Signed-off-by: Zach Brown --- kmod/src/data.c | 4 ++-- kmod/src/dir.c | 2 +- kmod/src/inode.c | 10 +++++----- kmod/src/item.c | 7 +++++-- kmod/src/item.h | 2 +- kmod/src/xattr.c | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index da146a2a..e42fd0eb 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -467,7 +467,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) goto out; if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); else ret = scoutfs_item_update(sb, &key, val, lock); if (ret) @@ -576,7 +576,7 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) } if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); else ret = scoutfs_item_update(sb, &key, val, lock); out: diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 502d75c6..ac5aebd4 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -904,7 +904,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, lock); else if (op == SYM_DELETE) - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); if (ret) break; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 57ad8287..5d10ef7f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -654,9 +654,9 @@ static int update_index_items(struct super_block *sb, del_lock = find_index_lock(lock_list, type, si->item_majors[type], si->item_minors[type], ino); - ret = scoutfs_item_delete(sb, &del, del_lock->end); + ret = scoutfs_item_delete(sb, &del, del_lock); if (ret) { - err = scoutfs_item_delete(sb, &ins, ins_lock->end); + err = scoutfs_item_delete(sb, &ins, ins_lock); BUG_ON(err); } @@ -1003,7 +1003,7 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, scoutfs_key_init(&key, &ikey, sizeof(ikey)); lock = find_index_lock(ind_locks, type, major, minor, ino); - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); if (ret == -ENOENT) ret = 0; return ret; @@ -1218,7 +1218,7 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) init_orphan_key(&key, &okey, sbi->node_id, ino); - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); if (ret == -ENOENT) ret = 0; @@ -1301,7 +1301,7 @@ retry: goto out; #endif - ret = scoutfs_item_delete(sb, &key, lock->end); + ret = scoutfs_item_delete(sb, &key, lock); if (ret) goto out; diff --git a/kmod/src/item.c b/kmod/src/item.c index f242aa88..2ace7963 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1444,7 +1444,7 @@ out: * deletion items for items that didn't exist in the first place. */ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1453,6 +1453,9 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + scoutfs_kvec_init_null(del_val); do { @@ -1471,7 +1474,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); scoutfs_kvec_kfree(del_val); diff --git a/kmod/src/item.h b/kmod/src/item.h index e442c313..6affdb81 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -39,7 +39,7 @@ void scoutfs_item_delete_dirty(struct super_block *sb, void scoutfs_item_update_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 70b2584f..e26da49d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -491,7 +491,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) break; } - ret = scoutfs_item_delete(sb, key, lck->end); + ret = scoutfs_item_delete(sb, key, lck); if (ret) break; From 365048b785fd3405ede1e835680276ce5477e313 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 12:12:35 -0700 Subject: [PATCH 480/920] scoutfs: add full lock arg to _item_set_batch() Add the full lock arg to _item_set_batch() so that it can verify lock coverage. Signed-off-by: Zach Brown --- kmod/src/item.c | 7 ++++--- kmod/src/item.h | 2 +- kmod/src/xattr.c | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 2ace7963..2b0665cc 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1211,7 +1211,7 @@ out: int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, int sif, - struct scoutfs_key_buf *end) + struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1235,7 +1235,8 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, trace_scoutfs_item_set_batch(sb, first, last); if (WARN_ON_ONCE(scoutfs_key_compare(first, last) > 0) || - WARN_ON_ONCE(scoutfs_key_compare(end, last) < 0)) + WARN_ON_ONCE(!lock_coverage(lock, first, WRITE)) || + WARN_ON_ONCE(!lock_coverage(lock, last, WRITE))) return -EINVAL; range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); @@ -1256,7 +1257,7 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, } spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, range_end, end); + ret = scoutfs_manifest_read_items(sb, range_end, lock->end); spin_lock_irqsave(&cac->lock, flags); if (ret) diff --git a/kmod/src/item.h b/kmod/src/item.h index 6affdb81..ac64e280 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -49,7 +49,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *first, struct scoutfs_key_buf *last, int sif, - struct scoutfs_key_buf *end); + struct scoutfs_lock *lock); void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index e26da49d..20978835 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -331,7 +331,7 @@ retry: goto unlock; ret = scoutfs_dirty_inode_item(inode, lck) ?: - scoutfs_item_set_batch(sb, &list, key, last, sif, lck->end); + scoutfs_item_set_batch(sb, &list, key, last, sif, lck); if (ret == 0) { /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); From d593e2caa0c3b67b06e7844abbbabc3ff73b65a6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Oct 2017 12:17:35 -0700 Subject: [PATCH 481/920] scoutfs: warn if we read items without cache limit All the item ops now know the limit of the items they're allowed to read into the cache. Warn if someone asks to read items without knowing how much they're allowed to read based on their lock coverage. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index ecb84a55..fa895d58 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -535,11 +535,13 @@ out: * The caller found a hole in the item cache that they'd like populated. * * We search the manifest for all the segments we'll need to iterate - * from the key to the end key. If the end key is null then we'll read - * as many items as the intersecting segments contain. + * from the key to the end key, the last key we're allowed to insert + * into the cache. * * If next_key is provided then the segments are only walked to find the * next key after the search key. If none is found -ENOENT is returned. + * There's no limit on the next_key we can return, the caller has + * to deal with that. * * As we insert the batch of items we give the item cache the range of * keys that contain these items. This lets the cache return negative @@ -584,6 +586,9 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, int err; int cmp; + if (WARN_ON_ONCE(!end && !next_key)) + return -EINVAL; + if (end) { scoutfs_key_clone(&seg_end, end); } else { @@ -593,7 +598,6 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, trace_scoutfs_read_items(sb, key, &seg_end); - /* * Ask the manifest server which manifest root to read from. Lock * holding callers will be responsible for this in the future. They'll From 8bbb859f0c713465e260547362476d1c2ef67ac8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Oct 2017 10:21:31 -0700 Subject: [PATCH 482/920] scoutfs: move scoutfs_ioctl definition We're going to be strictly enforcing matching format.h and ioctl.h between userspace and kernel space. Let's get the exported kernel function definition out of ioctl.h. Signed-off-by: Zach Brown --- kmod/src/ioctl.h | 2 -- kmod/src/super.h | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 2c980455..e9a78db0 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -1,8 +1,6 @@ #ifndef _SCOUTFS_IOCTL_H_ #define _SCOUTFS_IOCTL_H_ -long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); - /* XXX I have no idea how these are chosen. */ #define SCOUTFS_IOCTL_MAGIC 's' diff --git a/kmod/src/super.h b/kmod/src/super.h index 89a843f9..e09ac9d5 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -77,4 +77,7 @@ int scoutfs_read_supers(struct super_block *sb, void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); +/* to keep this out of the ioctl.h public interface definition */ +long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + #endif From 80a4b7df2ce270034ab1d0e2346c48ea2b99d111 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Oct 2017 10:37:32 -0700 Subject: [PATCH 483/920] scoutfs: move btree parent min to format.h mkfs needs to know the size of the largest btree when figuring out how big to make the ring. It needs to know how few items we can have in parent blocks and to know that it needs to know how empty the blocks can get. Signed-off-by: Zach Brown --- kmod/src/btree.c | 11 ++--------- kmod/src/format.h | 12 ++++++++++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index a64e6337..32421239 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -169,13 +169,6 @@ static inline unsigned int all_len_bytes(unsigned key_len, unsigned val_len) len_bytes(key_len, val_len); } -/* number of bytes needed to insert potentially max size child item */ -static inline unsigned int parent_min_free_bytes(void) -{ - return all_len_bytes(SCOUTFS_BTREE_MAX_KEY_LEN, - sizeof(struct scoutfs_btree_ref)); -} - /* * The minimum number of bytes we allow in a block. During descent to * modify if we see a block with fewer used bytes then we'll try to @@ -196,7 +189,7 @@ static inline unsigned int parent_min_free_bytes(void) static inline unsigned int min_used_bytes(void) { return (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - - parent_min_free_bytes()) / 2; + SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES) / 2; } /* total block bytes used by an existing item */ @@ -986,7 +979,7 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, int ret; if (right->level) - all_bytes = parent_min_free_bytes(); + all_bytes = SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES; else all_bytes = all_len_bytes(key_len, val_len); diff --git a/kmod/src/format.h b/kmod/src/format.h index 89b95ff1..9bc496b3 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -60,6 +60,18 @@ struct scoutfs_block_header { /* level 0 segments can have two full keys in the value :/ */ #define SCOUTFS_BTREE_MAX_VAL_LEN 768 +/* + * The min number of free bytes we must leave in a parent as we descend + * to modify. This leaves enough free bytes to insert a possibly maximal + * sized key as a seperator for a child block. Fewer bytes then this + * and split/merge might try to insert a max child item in the parent + * that wouldn't fit. + */ +#define SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES \ + (sizeof(struct scoutfs_btree_item_header) + \ + sizeof(struct scoutfs_btree_item) + SCOUTFS_BTREE_MAX_KEY_LEN +\ + sizeof(struct scoutfs_btree_ref)) + /* * A 4EB test image measured a worst case height of 17. This is plenty * generous. From ce4daa817a8c33c333c68d0648a18414bdd5b725 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Oct 2017 10:59:01 -0700 Subject: [PATCH 484/920] scoutfs: add support for format_hash Calculate the hash of format.h and ioctl.h and make sure the hash stored in the super during mkfs matches our calculated hash on mount. Signed-off-by: Zach Brown --- kmod/Makefile | 4 ++++ kmod/src/Makefile | 3 ++- kmod/src/format.h | 1 + kmod/src/super.c | 7 +++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/kmod/Makefile b/kmod/Makefile index e8e45a4a..97c48c17 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -16,7 +16,11 @@ SCOUTFS_GIT_DESCRIBE := \ $(shell git describe --all --abbrev=6 --long 2>/dev/null || \ echo not-in-a-git-repository) +SCOUTFS_FORMAT_HASH := \ + $(shell cat src/format.h src/ioctl.h | md5sum | cut -b1-16) + SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ + SCOUTFS_FORMAT_HASH=$(SCOUTFS_FORMAT_HASH) \ CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ EXTRA_CFLAGS=-Werror diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 30997479..ed9bdcbe 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -1,6 +1,7 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o -CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" +CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ + -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include diff --git a/kmod/src/format.h b/kmod/src/format.h index 9bc496b3..d9be949b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -430,6 +430,7 @@ struct scoutfs_inet_addr { struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; + __le64 format_hash; __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 next_seq; diff --git a/kmod/src/super.c b/kmod/src/super.c index e3d81395..83dabb5e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -184,6 +184,13 @@ int scoutfs_read_supers(struct super_block *sb, continue; } + if (super->format_hash != cpu_to_le64(SCOUTFS_FORMAT_HASH)) { + scoutfs_warn(sb, "super block %u has invalid format hash 0x%llx, expected 0x%llx", + i, le64_to_cpu(super->format_hash), + SCOUTFS_FORMAT_HASH); + continue; + } + if (found < 0 || (le64_to_cpu(super->hdr.seq) > seq)) { *local = *super; seq = le64_to_cpu((*local).hdr.seq); From cb879d9f3744190f47cf44cf9bcb1ef705f056a5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Oct 2017 11:27:23 -0700 Subject: [PATCH 485/920] scoutfs: add network greeting message Add a network greeting message that's exchanged between the client and server on every connection to make sure that we have the correct file system and format hash. Signed-off-by: Zach Brown --- kmod/src/client.c | 30 ++++++++++++++++++++++++++++++ kmod/src/format.h | 5 +++++ kmod/src/server.c | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 8cf43653..16517ccb 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -251,9 +251,11 @@ static int client_connect(struct client_info *client) { struct super_block *sb = client->sb; struct scoutfs_super_block super; + struct scoutfs_net_greeting greet; struct sockaddr_in *sin; struct socket *sock = NULL; struct timeval tv; + struct kvec kv; int retries; int addrlen; int optval; @@ -323,6 +325,34 @@ static int client_connect(struct client_info *client) if (ret) continue; + greet.fsid = super.id; + greet.format_hash = super.format_hash; + kv.iov_base = &greet; + kv.iov_len = sizeof(greet); + ret = scoutfs_sock_sendmsg(sock, &kv, 1); + if (ret) + continue; + + ret = scoutfs_sock_recvmsg(sock, &greet, sizeof(greet)); + if (ret) + continue; + + if (greet.fsid != super.id) { + scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", + SIN_ARG(&client->peername), + le64_to_cpu(greet.fsid), + le64_to_cpu(super.id)); + continue; + } + + if (greet.format_hash != super.format_hash) { + scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", + SIN_ARG(&client->peername), + le64_to_cpu(greet.format_hash), + le64_to_cpu(super.format_hash)); + continue; + } + /* but use a keepalive timeout instead of send timeout */ tv.tv_sec = 0; tv.tv_usec = 0; diff --git a/kmod/src/format.h b/kmod/src/format.h index d9be949b..ffe5ce44 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -566,6 +566,11 @@ struct scoutfs_lock_name { * messages over the wire. */ +struct scoutfs_net_greeting { + __le64 fsid; + __le64 format_hash; +} __packed; + /* * This header precedes and describes all network messages sent over * sockets. The id is set by the request and sent in the reply. The diff --git a/kmod/src/server.c b/kmod/src/server.c index a12da887..007ca7d6 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -765,11 +765,16 @@ static void scoutfs_server_recv_func(struct work_struct *work) recv_work); struct server_info *server = conn->server; struct super_block *sb = server->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; struct socket *sock = conn->sock; struct workqueue_struct *req_wq; + struct scoutfs_net_greeting greet; struct scoutfs_net_header nh; struct server_request *req; + bool passed_greeting; unsigned data_len; + struct kvec kv; int ret; req_wq = alloc_workqueue("scoutfs_server_requests", @@ -779,13 +784,45 @@ static void scoutfs_server_recv_func(struct work_struct *work) goto out; } - for (;;) { + /* first bounce the greeting */ + ret = scoutfs_sock_recvmsg(sock, &greet, sizeof(greet)); + if (ret) + goto out; + /* we'll close conn after failed greeting to let client see ours */ + passed_greeting = false; + + if (greet.fsid != super->id) { + scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(greet.fsid), + le64_to_cpu(super->id)); + } else if (greet.format_hash != super->format_hash) { + scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(greet.format_hash), + le64_to_cpu(super->format_hash)); + } else { + passed_greeting = true; + } + + greet.fsid = super->id; + greet.format_hash = super->format_hash; + kv.iov_base = &greet; + kv.iov_len = sizeof(greet); + ret = scoutfs_sock_sendmsg(sock, &kv, 1); + if (ret) + goto out; + + for (;;) { /* receive the header */ ret = scoutfs_sock_recvmsg(sock, &nh, sizeof(nh)); if (ret) break; + if (!passed_greeting) + break; + trace_scoutfs_server_recv_request(conn->server->sb, &conn->sockname, &conn->peername, &nh); From 8dee30047c6b09c9920b91844db86960e4e6c502 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 10 Oct 2017 15:58:29 -0700 Subject: [PATCH 486/920] scoutfs: fix xattr trans reservation The xattr trans reservation assumed that it was only dirtying items for the new xattr size. It didn't account for dirty deletion items for parts from a larger previous xattr. With this fixed generic/070 no longer triggers warnings. Signed-off-by: Zach Brown --- kmod/src/count.h | 23 +++++++++++++---------- kmod/src/xattr.c | 1 - 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 5a11f127..fedeba69 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -183,24 +183,27 @@ static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, } /* - * Setting an xattr can create a full set of items for an xattr with a - * max name and length. Any existing items will be dirtied rather than - * deleted so we won't have more items than a max xattr's worth. + * Setting an xattr results in a dirty set of items with values for the + * size of the xattr. Any previously existing items from a larger xattr + * are deleted which dirties their key but removes their value. We + * don't know the size of a possibly existing xattr so we assume max + * parts. */ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned name_len, unsigned size) { struct scoutfs_item_count cnt = {0,}; - unsigned parts = DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE); + unsigned val_parts = DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE); __count_dirty_inode(&cnt); - cnt.items += parts; - cnt.keys += parts * (offsetof(struct scoutfs_xattr_key, - name[name_len]) + - sizeof(struct scoutfs_xattr_key_footer)); - cnt.vals += parts * (sizeof(struct scoutfs_xattr_val_header) + - SCOUTFS_XATTR_PART_SIZE); + cnt.items += SCOUTFS_XATTR_MAX_PARTS; + cnt.keys += SCOUTFS_XATTR_MAX_PARTS * + (offsetof(struct scoutfs_xattr_key, name[name_len]) + + sizeof(struct scoutfs_xattr_key_footer)); + cnt.vals += val_parts * + (sizeof(struct scoutfs_xattr_val_header) + + SCOUTFS_XATTR_PART_SIZE); return cnt; } diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 20978835..5f7a1415 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -252,7 +252,6 @@ out: * another. */ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, - const void *value, size_t size, int flags) { From a30f0bf82f0e4ec4eddfff9bc7be3dcaa8356ef5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Oct 2017 13:56:03 -0700 Subject: [PATCH 487/920] scoutfs: stop spurious lockdep warning from dlm The fs/dlm code has a harmless but unannotated inversion between connection and socket locking that triggers during shutdown and disables lockdep. We don't want it to mask our warnings during testing that may happen after the first shared unmount so we disable lockdep around the dlm shutdown. It's not ideal but then neither are distro kernels that ship with lockdep warnings. Signed-off-by: Zach Brown --- kmod/src/lock.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 60efede4..ce1459c0 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -826,8 +826,17 @@ void scoutfs_lock_destroy(struct super_block *sb) */ free_lock_tree(sb); - if (linfo->dlmglue_online) + if (linfo->dlmglue_online) { + /* + * fs/dlm has a harmless but unannotated + * inversion between their connection and socket + * locking that triggers during shutdown and + * disables lockdep. + */ + lockdep_off(); ocfs2_dlm_shutdown(&linfo->dlmglue, 0); + lockdep_on(); + } sbi->lock_info = NULL; From 856f257085bd3e1e6c4acc99cacff8e43c824b7f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Oct 2017 09:55:51 -0700 Subject: [PATCH 488/920] scoutfs: used locked getattr for all inodes We only set the .getattr method to our locked getattr filler for regular files. Set it for all files so that stat, etc, will see the current inode for all file types. Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 ++ kmod/src/inode.c | 7 +++---- kmod/src/inode.h | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index ac5aebd4..8522f945 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -986,6 +986,7 @@ const struct inode_operations scoutfs_symlink_iops = { .readlink = generic_readlink, .follow_link = scoutfs_follow_link, .put_link = scoutfs_put_link, + .getattr = scoutfs_getattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -1637,6 +1638,7 @@ const struct inode_operations scoutfs_dir_iops = { .unlink = scoutfs_unlink, .rmdir = scoutfs_unlink, .rename = scoutfs_rename, + .getattr = scoutfs_getattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 5d10ef7f..be8bbee0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -66,8 +66,6 @@ struct inode_sb_info { struct inode_sb_info *name = SCOUTFS_SB(sb)->inode_sb_info static struct kmem_cache *scoutfs_inode_cachep; -static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, - struct kstat *stat); /* * This is called once before all the allocations and frees of a inode @@ -162,6 +160,7 @@ static const struct inode_operations scoutfs_file_iops = { }; static const struct inode_operations scoutfs_special_iops = { + .getattr = scoutfs_getattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -309,8 +308,8 @@ void scoutfs_inode_init_key(struct scoutfs_key_buf *key, scoutfs_key_init(key, ikey, sizeof(struct scoutfs_inode_key)); } -static int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, - struct kstat *stat) +int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 1e0b4f5e..be47ddb0 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -93,6 +93,8 @@ u64 scoutfs_inode_data_version(struct inode *inode); int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, int flags); +int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, + struct kstat *stat); int scoutfs_scan_orphans(struct super_block *sb); From 9027775ef26f4b8c1e944913f95cbbd6895ae419 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Oct 2017 10:42:03 -0700 Subject: [PATCH 489/920] scoutfs: fix parent dir nlink update in rename Renaming a dir between parents and clobbering an existing empty dir wasn't correctly updating the parent link counts. Updating parent link counts when dirs are moved between parents is an independent operation from decreasing the link count of a victim existing target of the rename. Signed-off-by: Zach Brown --- kmod/src/dir.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 8522f945..0813eec0 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1540,7 +1540,10 @@ retry: drop_nlink(new_dir); drop_nlink(new_inode); } - } else if (S_ISDIR(old_inode->i_mode) && (old_dir != new_dir)) { + + } + + if (S_ISDIR(old_inode->i_mode) && (old_dir != new_dir)) { drop_nlink(old_dir); inc_nlink(new_dir); } From afa30e60fe5e0b7ef6e3f1faa313372d96d18943 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 10 Oct 2017 16:22:39 -0500 Subject: [PATCH 490/920] scoutfs: use inclusive range for scoutfs_data_truncate_items() This makes calling it for truncate less cumbersome - we can safely use ~0ULL for the end point now. Signed-off-by: Mark Fasheh --- kmod/src/data.c | 11 +++++------ kmod/src/data.h | 2 +- kmod/src/ioctl.c | 3 ++- kmod/src/scoutfs_trace.h | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index e42fd0eb..4c8b88bb 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -594,7 +594,8 @@ out: i++, iblock++) /* - * Free blocks inside the specified logical block range. + * Free blocks inside the logical block range from 'iblock' to 'last', + * inclusive. * * If 'offline' is given then blocks are freed an offline mapping is * left behind. @@ -604,7 +605,7 @@ out: * partial progress. */ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline, + u64 last, bool offline, struct scoutfs_lock *lock) { struct scoutfs_key_buf last_key; @@ -617,21 +618,19 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, bool dirtied; bool modified; u64 blkno; - u64 last; int bytes; int ret = 0; int i; - trace_scoutfs_data_truncate_items(sb, iblock, len, offline); + trace_scoutfs_data_truncate_items(sb, iblock, last, offline); - if (WARN_ON_ONCE(iblock + len < iblock)) + if (WARN_ON_ONCE(last < iblock)) return -EINVAL; map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); if (!map) return -ENOMEM; - last = iblock + len - 1; init_mapping_key(&last_key, &last_bmk, ino, last); while (iblock <= last) { diff --git a/kmod/src/data.h b/kmod/src/data.h index 0dcd23a6..a305ad2e 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -5,7 +5,7 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 len, bool offline, + u64 last, bool offline, struct scoutfs_lock *lock); int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 0b8df1ad..9906e617 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -381,7 +381,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) truncate_inode_pages_range(&inode->i_data, start, end_inc); ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), args.block, - args.count, true, lock); + args.block + args.count - 1, true, + lock); out: scoutfs_unlock(sb, lock, DLM_LOCK_EX); mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 8e0fd843..8e3d4eba 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -493,26 +493,26 @@ TRACE_EVENT(scoutfs_data_get_cursor, ); TRACE_EVENT(scoutfs_data_truncate_items, - TP_PROTO(struct super_block *sb, __u64 iblock, __u64 len, int offline), + TP_PROTO(struct super_block *sb, __u64 iblock, __u64 last, int offline), - TP_ARGS(sb, iblock, len, offline), + TP_ARGS(sb, iblock, last, offline), TP_STRUCT__entry( __field(__u64, fsid) __field(__u64, iblock) - __field(__u64, len) + __field(__u64, last) __field(int, offline) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->iblock = iblock; - __entry->len = len; + __entry->last = last; __entry->offline = offline; ), - TP_printk(FSID_FMT" iblock %llu len %llu offline %u", __entry->fsid, - __entry->iblock, __entry->len, __entry->offline) + TP_printk(FSID_FMT" iblock %llu last %llu offline %u", __entry->fsid, + __entry->iblock, __entry->last, __entry->offline) ); TRACE_EVENT(scoutfs_data_set_segno_free, From dd99a0127e192329537786a497b8a160d09f0497 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 16 Oct 2017 18:14:48 -0500 Subject: [PATCH 491/920] scoutfs: rename scoutfs_inode_index_lock_hold Call it scoutfs_inode_index_try_lock_hold since it may fail and unwind as part of normal (not an error) operation. This lets us re-use the name in an upcoming patch. Signed-off-by: Mark Fasheh --- kmod/src/data.c | 5 +++-- kmod/src/dir.c | 12 ++++++------ kmod/src/inode.c | 10 +++++----- kmod/src/inode.h | 6 +++--- kmod/src/xattr.c | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 4c8b88bb..23a98eff 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1179,8 +1179,9 @@ static int scoutfs_write_begin(struct file *file, ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &wbd->ind_locks, inode, new_size, true) ?: - scoutfs_inode_index_lock_hold(sb, &wbd->ind_locks, - ind_seq, SIC_WRITE_BEGIN()); + scoutfs_inode_index_try_lock_hold(sb, &wbd->ind_locks, + ind_seq, + SIC_WRITE_BEGIN()); } while (ret > 0); if (ret < 0) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0813eec0..5c8a7d13 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -576,7 +576,7 @@ retry: scoutfs_inode_index_prepare(sb, ind_locks, dir, dir_size, true) ?: scoutfs_inode_index_prepare_ino(sb, ind_locks, ino, mode, inode_size) ?: - scoutfs_inode_index_lock_hold(sb, ind_locks, ind_seq, cnt); + scoutfs_inode_index_try_lock_hold(sb, ind_locks, ind_seq, cnt); if (ret > 0) goto retry; if (ret) @@ -715,8 +715,8 @@ retry: dir_size, false) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, i_size_read(inode), false) ?: - scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, - SIC_LINK(dentry->d_name.len)); + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_LINK(dentry->d_name.len)); if (ret > 0) goto retry; if (ret) @@ -799,8 +799,8 @@ retry: dir_size, false) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, i_size_read(inode), false) ?: - scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, - SIC_UNLINK(dentry->d_name.len)); + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_UNLINK(dentry->d_name.len)); if (ret > 0) goto retry; if (ret) @@ -1465,7 +1465,7 @@ retry: (new_inode == NULL ? 0 : scoutfs_inode_index_prepare(sb, &ind_locks, new_inode, i_size_read(new_inode), false)) ?: - scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_RENAME(old_dentry->d_name.len, new_dentry->d_name.len)); if (ret > 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index be8bbee0..095c478e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -939,9 +939,9 @@ int scoutfs_inode_index_start(struct super_block *sb, u64 *seq) * * Returns > 0 if the seq changed and the locks should be retried. */ -int scoutfs_inode_index_lock_hold(struct super_block *sb, - struct list_head *list, u64 seq, - const struct scoutfs_item_count cnt) +int scoutfs_inode_index_try_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct index_lock *ind_lock; @@ -1273,8 +1273,8 @@ static int delete_inode_items(struct super_block *sb, u64 ino) retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: prepare_index_deletion(sb, &ind_locks, ino, mode, &sinode) ?: - scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, - SIC_DIRTY_INODE()); + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_DIRTY_INODE()); if (ret > 0) goto retry; if (ret) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index be47ddb0..fe884f13 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -69,9 +69,9 @@ int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, int scoutfs_inode_index_prepare_ino(struct super_block *sb, struct list_head *list, u64 ino, umode_t mode, u64 new_size); -int scoutfs_inode_index_lock_hold(struct super_block *sb, - struct list_head *list, u64 seq, - const struct scoutfs_item_count cnt); +int scoutfs_inode_index_try_lock_hold(struct super_block *sb, + struct list_head *list, u64 seq, + const struct scoutfs_item_count cnt); void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list); int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 5f7a1415..d72942c9 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -322,8 +322,8 @@ retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, i_size_read(inode), false) ?: - scoutfs_inode_index_lock_hold(sb, &ind_locks, ind_seq, - SIC_XATTR_SET(name_len, size)); + scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, + SIC_XATTR_SET(name_len, size)); if (ret > 0) goto retry; if (ret) From 20a22ddc6be4ad63226a15b93bf7bc42646fa04d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 5 Oct 2017 15:28:53 -0500 Subject: [PATCH 492/920] scoutfs: provide ->setattr Simple attr changes are mostly handled by the VFS, we just have to mirror them into our inode. Truncates are done in a seperate set of transactions. We use a flag to indicate an in-progress truncate. This allows us to detect and continue the truncate should the node crash. Index locking is a bit complicated, so we add a helper function to grab index locks and start a transaction. With this patch we now pass the following xfstests: generic/014 generic/101 generic/313 Signed-off-by: Mark Fasheh --- kmod/src/dir.c | 2 + kmod/src/file.c | 7 +- kmod/src/format.h | 3 + kmod/src/inode.c | 156 +++++++++++++++++++++++++++++++++++++++ kmod/src/inode.h | 7 +- kmod/src/scoutfs_trace.h | 57 ++++++++++++++ 6 files changed, 229 insertions(+), 3 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 5c8a7d13..b869fb7b 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -987,6 +987,7 @@ const struct inode_operations scoutfs_symlink_iops = { .follow_link = scoutfs_follow_link, .put_link = scoutfs_put_link, .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -1642,6 +1643,7 @@ const struct inode_operations scoutfs_dir_iops = { .rmdir = scoutfs_unlink, .rename = scoutfs_rename, .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, diff --git a/kmod/src/file.c b/kmod/src/file.c index b59df5e0..9382d96a 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -73,15 +73,18 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, if (ret) goto out; + ret = scoutfs_complete_truncate(inode, inode_lock); + if (ret) + goto out; + scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); /* XXX: remove SUID bit */ ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); - +out: scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); -out: mutex_unlock(&inode->i_mutex); if (ret > 0 || ret == -EIOCBQUEUED) { diff --git a/kmod/src/format.h b/kmod/src/format.h index ffe5ce44..e30a1320 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -483,11 +483,14 @@ struct scoutfs_inode { __le32 gid; __le32 mode; __le32 rdev; + __le32 flags; struct scoutfs_timespec atime; struct scoutfs_timespec ctime; struct scoutfs_timespec mtime; } __packed; +#define SCOUTFS_INO_FLAG_TRUNCATE 0x1 + #define SCOUTFS_ROOT_INO 1 /* like the block size, a reasonable min PATH_MAX across platforms */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 095c478e..cdb7275b 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -152,6 +152,7 @@ void scoutfs_destroy_inode(struct inode *inode) static const struct inode_operations scoutfs_file_iops = { .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -161,6 +162,7 @@ static const struct inode_operations scoutfs_file_iops = { static const struct inode_operations scoutfs_special_iops = { .getattr = scoutfs_getattr, + .setattr = scoutfs_setattr, .setxattr = scoutfs_setxattr, .getxattr = scoutfs_getxattr, .listxattr = scoutfs_listxattr, @@ -242,6 +244,8 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->data_version = le64_to_cpu(cinode->data_version); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); + ci->flags = le32_to_cpu(cinode->flags); + set_item_info(ci, cinode); } @@ -325,6 +329,138 @@ int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, return ret; } +static int set_inode_size(struct inode *inode, struct scoutfs_lock *lock, + u64 new_size, bool truncate) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + LIST_HEAD(ind_locks); + int ret; + + if (!S_ISREG(inode->i_mode)) + return 0; + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, new_size, true, + SIC_DIRTY_INODE()); + if (ret) + return ret; + + truncate_setsize(inode, new_size); + inode->i_ctime = inode->i_mtime = CURRENT_TIME; + if (truncate) + ci->flags |= SCOUTFS_INO_FLAG_TRUNCATE; + scoutfs_inode_set_data_seq(inode); + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + return ret; +} + +static int clear_truncate_flag(struct inode *inode, struct scoutfs_lock *lock) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + struct super_block *sb = inode->i_sb; + LIST_HEAD(ind_locks); + int ret; + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, + i_size_read(inode), false, + SIC_DIRTY_INODE()); + if (ret) + return ret; + + ci->flags &= ~SCOUTFS_INO_FLAG_TRUNCATE; + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + return ret; +} + +int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) +{ + struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 start; + int ret, err; + + trace_scoutfs_complete_truncate(inode, ci->flags); + + if (!(ci->flags & SCOUTFS_INO_FLAG_TRUNCATE)) + return 0; + + start = (i_size_read(inode) + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; + ret = scoutfs_data_truncate_items(inode->i_sb, scoutfs_ino(inode), + start, ~0ULL, false, lock); + err = clear_truncate_flag(inode, lock); + + return ret ? ret : err; +} + +int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; + LIST_HEAD(ind_locks); + bool truncate = false; + u64 attr_size; + int ret; + + trace_scoutfs_setattr(dentry, attr); + + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &lock); + if (ret) + return ret; + + ret = inode_change_ok(inode, attr); + if (ret) + goto out; + + attr_size = (attr->ia_valid & ATTR_SIZE) ? attr->ia_size : + i_size_read(inode); + + if (S_ISREG(inode->i_mode) && attr->ia_valid & ATTR_SIZE) { + /* + * Complete any truncates that may have failed while + * in progress + */ + ret = scoutfs_complete_truncate(inode, lock); + if (ret) + goto out; + + truncate = i_size_read(inode) > attr_size; + + ret = set_inode_size(inode, lock, attr_size, truncate); + if (ret) + goto out; + + if (truncate) { + ret = scoutfs_complete_truncate(inode, lock); + if (ret) + goto out; + } + } + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, + i_size_read(inode), false, + SIC_DIRTY_INODE()); + if (ret) + goto out; + + setattr_copy(inode, attr); + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); +out: + scoutfs_unlock(sb, lock, DLM_LOCK_EX); + return ret; +} + /* * Set a given seq to the current trans seq if it differs. The caller * holds locks and a transaction which prevents the transaction from @@ -486,6 +622,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); cinode->data_version = cpu_to_le64(scoutfs_inode_data_version(inode)); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); + cinode->flags = cpu_to_le32(ci->flags); } /* @@ -970,6 +1107,24 @@ out: return ret; } +int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, + u64 size, bool set_data_seq, + const struct scoutfs_item_count cnt) +{ + struct super_block *sb = inode->i_sb; + int ret; + u64 seq; + + do { + ret = scoutfs_inode_index_start(sb, &seq) ?: + scoutfs_inode_index_prepare(sb, list, inode, size, + set_data_seq) ?: + scoutfs_inode_index_try_lock_hold(sb, list, seq, cnt); + } while (ret > 0); + + return ret; +} + /* * Unlocks and frees all the locks on the list. */ @@ -1172,6 +1327,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; atomic64_set(&ci->last_refreshed, scoutfs_lock_refresh_gen(lock)); + ci->flags = 0; scoutfs_inode_set_meta_seq(inode); scoutfs_inode_set_data_seq(inode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index fe884f13..ca35e4ba 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -15,6 +15,7 @@ struct scoutfs_inode_info { u64 meta_seq; u64 data_seq; u64 data_version; + u32 flags; /* * The in-memory item info caches the current index item values @@ -36,7 +37,6 @@ struct scoutfs_inode_info { struct scoutfs_per_task pt_data_lock; struct rw_semaphore xattr_rwsem; struct rb_node writeback_node; - struct inode inode; }; @@ -72,6 +72,9 @@ int scoutfs_inode_index_prepare_ino(struct super_block *sb, int scoutfs_inode_index_try_lock_hold(struct super_block *sb, struct list_head *list, u64 seq, const struct scoutfs_item_count cnt); +int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, + u64 size, bool set_data_seq, + const struct scoutfs_item_count cnt); void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list); int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); @@ -90,11 +93,13 @@ void scoutfs_inode_inc_data_version(struct inode *inode); u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); u64 scoutfs_inode_data_version(struct inode *inode); +int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock); int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, int flags); int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat); +int scoutfs_setattr(struct dentry *dentry, struct iattr *attr); int scoutfs_scan_orphans(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 8e3d4eba..b920b038 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -41,6 +41,63 @@ struct lock_info; #define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) #define FSID_FMT "%llx" +TRACE_EVENT(scoutfs_setattr, + TP_PROTO(struct dentry *dentry, struct iattr *attr), + + TP_ARGS(dentry, attr), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(unsigned int, d_len) + __string(d_name, dentry->d_name.name) + __field(__u64, i_size) + __field(__u64, ia_size) + __field(unsigned int, ia_valid) + __field(int, size_change) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(dentry->d_inode->i_sb); + __entry->ino = scoutfs_ino(dentry->d_inode); + __entry->d_len = dentry->d_name.len; + __assign_str(d_name, dentry->d_name.name); + __entry->ia_valid = attr->ia_valid; + __entry->size_change = !!(attr->ia_valid & ATTR_SIZE); + __entry->ia_size = attr->ia_size; + __entry->i_size = i_size_read(dentry->d_inode); + ), + + TP_printk(FSID_FMT" %s ino %llu ia_valid 0x%x size change %d ia_size " + "%llu i_size %llu", __entry->fsid, __get_str(d_name), + __entry->ino, __entry->ia_valid, __entry->size_change, + __entry->ia_size, __entry->i_size) +); + +TRACE_EVENT(scoutfs_complete_truncate, + TP_PROTO(struct inode *inode, __u32 flags), + + TP_ARGS(inode, flags), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, i_size) + __field(__u32, flags) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(inode->i_sb); + __entry->ino = scoutfs_ino(inode); + __entry->i_size = i_size_read(inode); + __entry->flags = flags; + ), + + TP_printk(FSID_FMT" ino %llu i_size %llu flags 0x%x", + __entry->fsid, __entry->ino, __entry->i_size, + __entry->flags) +); + DECLARE_EVENT_CLASS(scoutfs_comp_class, TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), From 0712ca6b9b7cbfe30e90d6e33bea3c65c629156b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 18 Oct 2017 15:15:03 -0700 Subject: [PATCH 493/920] scoutfs: correctly set new flag in get_blocks We weren't setting the new flag in the mapped buffer head. This tells the caller that the buffer is newly allocated and needs to be zeroed. Without this we expose unwritten newly allocated block contents. fsx found this almost immediately. With this fixed fsx passes. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 23a98eff..b101fc48 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1052,6 +1052,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, ret = find_alloc_block(sb, map, &key, ind, exists, lock); if (ret) goto out; + set_buffer_new(bh); } /* mark the bh mapped and set the size for as many contig as we see */ @@ -1063,7 +1064,6 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, map_bh(bh, inode->i_sb, map->blknos[ind]); bh->b_size = min_t(u64, bh->b_size, i << SCOUTFS_BLOCK_SHIFT); - clear_buffer_new(bh); } ret = 0; From 95d8f4bf209a12c54ea422c3b90de6a649fbff3d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Oct 2017 13:01:47 -0700 Subject: [PATCH 494/920] scoutfs: only allow recursive blocked hold fsx-mpi spins creating contention between ex holders of locks between nodes. It was tripping assertions in item invalidation as it tried to invalidate dirty items. Tracing showed that we were allowing holders of locks while we were invalidating. Our invalidation function would commit the current transaction, another task would hold the lock and dirty an item, and then invalidation would continue on and try to invalidate the dirty item. The invalidation code has always assumed that it's not running concurrently with item dirtying. The recursive locking change allowed acquireing blocked locks if the recursive flag was set. It'd then check holders after calling downconvert_worker (invalidation for us) and retry the downconvert if a holder appeared. That it allowed recursive holders regardless of who was alredy holding the lock is what let holders arrive once downconvert started on the blocked lock. Not only did this create our problem with invalidation, it also could leave items behind if the holder dirtied an item and dropped the lock between invalidation and before downconvert checked the holders again. The fix is to only allow recursive holders on blocked locks that already have holders. This ensures that holders will never increase past zero on blocked locks. Once the downconvert sees the holders drain it will call invalidation which won't have racing dirtiers. We can remove the holder check after invalidation entirely. With this fixed fsx-mpi no longer tries to invalidate dirty items as it bounces locks back and forth. Signed-off-by: Zach Brown --- kmod/src/dlmglue.c | 22 +++++----------------- kmod/src/dlmglue.h | 9 +++++---- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index dc1fa4d3..c4df1071 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -918,11 +918,14 @@ static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lock return wanted <= ocfs2_highest_compat_lock_level(lockres->l_blocking); } +/* the caller doesn't have to wait on a blocked lock if their wanted level + * is compatible with it and there are already holders of the lock */ static inline int lockres_allow_recursion(struct ocfs2_lock_res *lockres, int wanted) { - return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) - && wanted <= lockres->l_level; + return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && + wanted <= lockres->l_level && + (lockres->l_ex_holders || lockres->l_ro_holders); } static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) @@ -2393,21 +2396,6 @@ recheck: goto recheck; } - if ((lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && - ((lockres->l_blocking == DLM_LOCK_PR && lockres->l_ex_holders) || - (lockres->l_blocking == DLM_LOCK_EX && - (lockres->l_ex_holders || lockres->l_ro_holders)))) { - /* - * Recursive locks may have had their holder count - * incremented while we were sleeping in - * ->downconvert_worker. Recheck here. - */ - mlog(ML_BASTS, "lockres %s, block=%d:%d, level=%d:%d, ro=%d " - "ex=%d, Recheck\n", lockres->l_name, blocking, - lockres->l_blocking, level, lockres->l_level, - lockres->l_ro_holders, lockres->l_ex_holders); - goto recheck; - } downconvert: ctl->requeue = 0; diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 4a88b934..80a155f6 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -282,10 +282,11 @@ struct ocfs2_lock_res_ops { /* * Tells dlmglue to override fairness considerations when locking this * lock type - the blocking flag will be ignored when a lock is - * requested and we already have it at the appropriate level. This - * allows a process to acquire a dlmglue lock on the same resource - * multiple times in a row without deadlocking, even if another node has - * asked for a competing lock on the resource. + * requested and we already have it at the appropriate level and the + * resource is currently held. This allows a process to acquire a + * dlmglue lock on the same resource multiple times in a row without + * deadlocking, even if another node has asked for a competing lock on + * the resource. * * Note that lock/unlock calls must always be balanced (1 unlock for * every lock), even when this flag is set. From 4263a22c1573ce46d3c75ff704aedd5e55628fbb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Oct 2017 13:10:30 -0700 Subject: [PATCH 495/920] scoutfs: actually initialize per_task entry head We had callers using the initialization macro, it just didn't do anything. The uninitialized entries triggered a bug on trying to delete an uninitialized entry. fsx-mpi tripped over this on shutdown after seeing a consistency error. Signed-off-by: Zach Brown --- kmod/src/per_task.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kmod/src/per_task.h b/kmod/src/per_task.h index 6cad8098..6a055391 100644 --- a/kmod/src/per_task.h +++ b/kmod/src/per_task.h @@ -12,8 +12,11 @@ struct scoutfs_per_task_entry { void *ptr; }; -#define SCOUTFS_DECLARE_PER_TASK_ENTRY(name) \ - struct scoutfs_per_task_entry name +#define SCOUTFS_DECLARE_PER_TASK_ENTRY(name) \ + struct scoutfs_per_task_entry name = { \ + .head = LIST_HEAD_INIT((name).head), \ + } + void *scoutfs_per_task_get(struct scoutfs_per_task *pt); void scoutfs_per_task_add(struct scoutfs_per_task *pt, From 4c6253a18e695623c01d4bcab5c030c1c93729bf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Oct 2017 14:17:16 -0700 Subject: [PATCH 496/920] scoutfs: add lock trace event, convert invalidate Expand the generic lock tracing event to trace the level and holders, add an event for acquiring a lock, and switch the invalidation event over to using the lock class. Signed-off-by: Zach Brown --- kmod/src/lock.c | 3 ++- kmod/src/scoutfs_trace.h | 49 ++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ce1459c0..ac4986a2 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -74,7 +74,7 @@ static int invalidate_caches(struct super_block *sb, int mode, u64 ino, last; int ret; - trace_scoutfs_lock_invalidate_sb(sb, mode, start, end); + trace_scoutfs_lock_invalidate(sb, lock); ret = scoutfs_item_writeback(sb, start, end); if (ret) @@ -438,6 +438,7 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, dec_lock_users(lock); put_scoutfs_lock(sb, lock); } else { + trace_scoutfs_lock(sb, lock); *ret_lock = lock; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b920b038..4a4beeaa 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1504,6 +1504,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck), TP_STRUCT__entry( + __field(__u64, fsid) __field(u8, name_scope) __field(u8, name_zone) __field(u8, name_type) @@ -1512,8 +1513,12 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(unsigned int, seq) __field(unsigned int, refcnt) __field(unsigned int, users) + __field(unsigned char, level) + __field(unsigned int, ro) + __field(unsigned int, ex) ), TP_fast_assign( + __entry->fsid = FSID_ARG(sb); __entry->name_scope = lck->lock_name.scope; __entry->name_zone = lck->lock_name.zone; __entry->name_type = lck->lock_name.type; @@ -1522,11 +1527,16 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->seq = lck->sequence; __entry->refcnt = lck->refcnt; __entry->users = lck->users; + /* racey, but safe refs of embedded struct */ + __entry->level = lck->lockres.l_level; + __entry->ro = lck->lockres.l_ro_holders; + __entry->ex = lck->lockres.l_ex_holders; ), - TP_printk("name %u.%u.%u.%llu.%llu seq %u refs %d users %d", - __entry->name_scope, __entry->name_zone, __entry->name_type, - __entry->name_first, __entry->name_second, __entry->seq, - __entry->refcnt, __entry->users) + TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu seq %u refs %d users %d level %u ro %u ex %u", + __entry->fsid, __entry->name_scope, __entry->name_zone, + __entry->name_type, __entry->name_first, + __entry->name_second, __entry->seq, __entry->refcnt, + __entry->users, __entry->level, __entry->ro, __entry->ex) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, @@ -1534,6 +1544,11 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) @@ -1549,6 +1564,11 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_bast, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_reclaim, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) @@ -1559,27 +1579,6 @@ DEFINE_EVENT(scoutfs_lock_class, shrink_lock_tree, TP_ARGS(sb, lck) ); -TRACE_EVENT(scoutfs_lock_invalidate_sb, - TP_PROTO(struct super_block *sb, int mode, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), - TP_ARGS(sb, mode, start, end), - TP_STRUCT__entry( - __field(void *, sb) - __field(int, mode) - __dynamic_array(char, start, scoutfs_key_str(NULL, start)) - __dynamic_array(char, end, scoutfs_key_str(NULL, end)) - ), - TP_fast_assign( - __entry->sb = sb; - __entry->mode = mode; - scoutfs_key_str(__get_dynamic_array(start), start); - scoutfs_key_str(__get_dynamic_array(end), end); - ), - TP_printk("sb %p mode %s start %s end %s", - __entry->sb, lock_mode(__entry->mode), - __get_str(start), __get_str(end)) -); - DECLARE_EVENT_CLASS(scoutfs_seg_class, TP_PROTO(struct scoutfs_segment *seg), TP_ARGS(seg), From 5f74a7280c0d050ab5199d10d3aac0a8ebe7a43b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Oct 2017 16:26:08 -0700 Subject: [PATCH 497/920] scoutfs: refresh inode in xattr set scoutfs_xattr_set() refreshes the cached item inode with its current vfs inode. It has to refresh its vfs item as it acquires the lock before it asserts that vfs inode as current. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index d72942c9..d01b0640 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -289,7 +289,8 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, 0, inode, &lck); + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &lck); if (ret) goto out; From ecbf59d1307e09b0ab92bc77f99247fcf363f941 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Oct 2017 15:38:53 -0700 Subject: [PATCH 498/920] scoutfs: use migration key instead of bits The bit tracking code was a bit much (HA). It introduced a lot of complexity just to provide a way to migrate blocks from the old half of the ring into the current half of the ring. We can get rid of a ton of code and potential for bugs if we simply store a persistent migration key in the super and use it to sweep the tree looking for old blocks to dirty. A simple tree walk that dirties and returns the next key is all we need. Signed-off-by: Zach Brown --- kmod/src/btree.c | 141 ++++++++++++++++++++++++++++++++++------------ kmod/src/format.h | 5 ++ 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 32421239..9e7d25ad 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -53,10 +53,9 @@ * blocks by the time it fills up and wraps around to start overwriting * the old half of the ring. * - * To find the blocks in the old half of the ring we augment the btree - * items to store bits that are or-ed in parent items up to the root. - * Parent items have bits set for the half of the ring that their child - * block is stored in. + * To find the blocks in the old half of the ring we store a migration + * key in the super. Whenever we need to dirty old blocks we sweep leaf + * blocks from that key dirtying old blocks we find. * * Blocks are of a fixed size and are set to 4k to avoid multi-page * blocks. This means they can be smaller than the page size and we can @@ -82,9 +81,9 @@ * order. * * A dense array of item headers after the btree block header stores the - * offsets and bits of the items and is kept sorted by the item's keys. - * The array is small enough that keeping it sorted with memmove() - * involves a few cache lines at most. + * offsets of the items and is kept sorted by the item's keys. The + * array is small enough that keeping it sorted with memmove() involves + * a few cache lines at most. * * Parent blocks in the btree have the same format as leaf blocks. * There's one key for every child reference instead of having separator @@ -100,7 +99,6 @@ * - counters and tracing * - could issue read-ahead around reads up to dirty blkno * - have barrier as we cross to prevent refreshing clobbering stale reads - * - audit split and merge for bit updating * - audit/comment that dirty blknos can wrap around ring * - figure out some max transaction size so ring won't wrap in one * - update the world of comments @@ -138,6 +136,7 @@ enum { BTW_ALLOC = (1 << 7), /* allocate a new block for 0 ref */ BTW_INSERT = (1 << 8), /* walking to insert, try splitting */ BTW_DELETE = (1 << 9), /* walking to delete, try merging */ + BTW_MIGRATE = (1 << 10), /* don't dirty old leaf blocks */ }; /* @@ -383,12 +382,6 @@ static u8 half_bit(struct scoutfs_btree_ring *bring, u64 blkno) SCOUTFS_BTREE_BIT_HALF2; } -static u8 other_half_bit(struct scoutfs_btree_ring *bring, u64 blkno) -{ - return half_bit(bring, blkno) ^ (SCOUTFS_BTREE_BIT_HALF1 | - SCOUTFS_BTREE_BIT_HALF2); -} - static u8 bits_from_counts(struct scoutfs_btree_block *bt) { u8 bits = 0; @@ -462,6 +455,37 @@ static void path_repair_reset(struct super_block *sb, struct btree_path *path) } } +/* + * A block is current if it's in the same half of the ring as the next + * dirty block in the transaction. + */ +static bool blkno_is_current(struct scoutfs_btree_ring *bring, u64 blkno) +{ + u64 half_blkno = le64_to_cpu(bring->first_blkno) + + (le64_to_cpu(bring->nr_blocks) / 2); + u64 next_blkno = le64_to_cpu(bring->first_blkno) + + le64_to_cpu(bring->next_block); + + return (blkno < half_blkno) == (next_blkno < half_blkno); +} + +static bool first_block_in_half(struct scoutfs_btree_ring *bring) +{ + u64 block = le64_to_cpu(bring->next_block); + + return block == 0 || block == (le64_to_cpu(bring->nr_blocks) / 2); +} + +static size_t super_root_offsets[] = { + offsetof(struct scoutfs_super_block, alloc_root), + offsetof(struct scoutfs_super_block, manifest.root), +}; + +#define for_each_super_root(super, i, root) \ + for (i = 0; i < ARRAY_SIZE(super_root_offsets) && \ + (root = ((void *)super + super_root_offsets[i]), 1);\ + i++) + static int cmp_hdr_item_key(void *priv, const void *a_ptr, const void *b_ptr) { struct scoutfs_btree_block *bt = priv; @@ -762,6 +786,7 @@ static int get_ref_block(struct super_block *sb, int flags, DECLARE_BTREE_INFO(sb, bti); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_btree_ring *bring = &super->bring; + struct scoutfs_btree_root *root; struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_block *new; struct buffer_head *bh; @@ -769,6 +794,7 @@ static int get_ref_block(struct super_block *sb, int flags, u64 blkno; u64 seq; int ret; + int i; retry: /* always get the current block, either to return or cow from */ @@ -842,6 +868,15 @@ retry: else le64_add_cpu(&bring->next_block, 1); + /* reset the migration keys if we've just entered a new half */ + if (first_block_in_half(bring)) { + for_each_super_root(super, i, root) { + memset(root->migration_key, 0, + sizeof(root->migration_key)); + root->migration_key_len = cpu_to_le16(1); + } + } + le64_add_cpu(&bring->next_seq, 1); if (half_bit(bring, blkno) == half_bit(bring, bti->first_dirty_blkno)) @@ -1265,6 +1300,12 @@ static void inc_key(u8 *bytes, unsigned *len) * potentially updated bits in the leaf. They must always repair the * path because we can modify parent bits during descent before * returning an error. + * + * Migrating is a special kind of dirtying that returns the parent block + * in the walk if the leaf block is already current and doesn't need to + * be migrated. It's presumed that the caller is iterating over keys + * dirtying old leaf blocks and isn't actually doing anything with the + * blocks themselves. */ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, struct btree_path *path, int flags, @@ -1272,9 +1313,11 @@ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block **bt_ret, void *iter_key, unsigned *iter_len) { + struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *parent = NULL; struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_item *item; + struct scoutfs_btree_ref *ref; unsigned level; unsigned pos; unsigned nr; @@ -1310,7 +1353,16 @@ restart: goto out; } + ref = &root->ref; + while(level-- > 0) { + /* no point in dirtying current leaf blocks for migration */ + if ((flags & BTW_MIGRATE) && level == 0 && + blkno_is_current(bring, le64_to_cpu(ref->blkno))) { + ret = 0; + break; + } + if (parent) ret = get_parent_ref_block(sb, flags, parent, pos, &bt); else @@ -1404,6 +1456,8 @@ restart: put_btree_block(parent); parent = bt; bt = NULL; + + ref = item_val(pos_item(parent, pos)); } out: @@ -1790,40 +1844,53 @@ int scoutfs_btree_write_dirty(struct super_block *sb) DECLARE_BTREE_INFO(sb, bti); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_btree_ring *bring = &super->bring; - struct scoutfs_btree_root *roots[] = { - &super->manifest.root, - &super->alloc_root, - NULL, - }; struct scoutfs_btree_root *root; struct scoutfs_btree_block *bt; DECLARE_BTREE_PATH(path); struct buffer_head *tmp; struct buffer_head *bh; struct blk_plug plug; - unsigned next_root; - u8 bit; + unsigned int walk_len; + unsigned int iter_len; + bool progress; + void *walk_key; + void *iter_key; int ret; + int i; if (bti->first_dirty_bh == NULL) return 0; - /* cow old dirty blocks to balance ring */ - bit = other_half_bit(bring, bti->first_dirty_blkno); - next_root = 0; - root = roots[next_root]; - while (root && bti->old_dirtied < bti->cur_dirtied) { - ret = btree_walk(sb, root, &path, - BTW_DIRTY | BTW_BIT | BTW_DIRTY_OLD, - NULL, 0, 0, bit, NULL, NULL, NULL); - path_repair_reset(sb, &path); - if (ret == -ENOENT) { - root = roots[next_root++]; - continue; + iter_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); + if (!iter_key) + return -ENOMEM; + + progress = true; + while (progress && bti->old_dirtied < bti->cur_dirtied) { + progress = false; + + for_each_super_root(super, i, root) { + walk_key = root->migration_key; + walk_len = le16_to_cpu(root->migration_key_len); + if (walk_len == 0) + continue; + + ret = btree_walk(sb, root, &path, + BTW_DIRTY | BTW_NEXT | BTW_MIGRATE, + walk_key, walk_len, 0, 0, &bt, + iter_key, &iter_len); + path_repair_reset(sb, &path); + if (ret < 0) + goto out; + + root->migration_key_len = cpu_to_le16(iter_len); + if (iter_len) { + memcpy(walk_key, iter_key, iter_len); + progress = true; + } else { + memset(walk_key, 0, SCOUTFS_BTREE_MAX_KEY_LEN); + } } - if (ret < 0) - goto out; } /* checksum everything to reduce time between io submission merging */ @@ -1852,7 +1919,9 @@ int scoutfs_btree_write_dirty(struct super_block *sb) if (!buffer_uptodate(bh)) ret = -EIO; } + out: + kfree(iter_key); return ret; } diff --git a/kmod/src/format.h b/kmod/src/format.h index e30a1320..3c78a8dd 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -103,10 +103,15 @@ struct scoutfs_btree_ref { /* * A height of X means that the first block read will have level X-1 and * the leaves will have level 0. + * + * The migration key is used to walk the tree finding old blocks to migrate + * into the current half of the ring. */ struct scoutfs_btree_root { struct scoutfs_btree_ref ref; __u8 height; + __le16 migration_key_len; + __u8 migration_key[SCOUTFS_BTREE_MAX_KEY_LEN]; } __packed; struct scoutfs_btree_item_header { From 22911afc6ee16ceb63b22e3e9db4e1115f552f5e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Oct 2017 16:50:03 -0700 Subject: [PATCH 499/920] scoutfs: remove btree item bit tracking The augmenting of the btree to track items with bits set was too fiddly for its own good. We were able to migrate old btree blocks with a simple stored key while also fixing livelocks as the parent and item bits got out of sync. This is now unused buggy code that can be removed. Signed-off-by: Zach Brown --- kmod/src/btree.c | 418 ++++++---------------------------------------- kmod/src/format.h | 19 --- 2 files changed, 47 insertions(+), 390 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 9e7d25ad..2376ab54 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -131,12 +131,10 @@ enum { BTW_PREV = (1 << 2), /* return <= key */ BTW_BEFORE = (1 << 3), /* return < key */ BTW_DIRTY = (1 << 4), /* cow stable blocks */ - BTW_BIT = (1 << 5), /* search for the first set bit, not key */ - BTW_DIRTY_OLD = (1 << 6), /* dirty old leaf blocks to balance ring */ - BTW_ALLOC = (1 << 7), /* allocate a new block for 0 ref */ - BTW_INSERT = (1 << 8), /* walking to insert, try splitting */ - BTW_DELETE = (1 << 9), /* walking to delete, try merging */ - BTW_MIGRATE = (1 << 10), /* don't dirty old leaf blocks */ + BTW_ALLOC = (1 << 5), /* allocate a new block for 0 ref */ + BTW_INSERT = (1 << 6), /* walking to insert, try splitting */ + BTW_DELETE = (1 << 7), /* walking to delete, try merging */ + BTW_MIGRATE = (1 << 8), /* don't dirty old leaf blocks */ }; /* @@ -303,158 +301,6 @@ static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, return pos; } -static inline u8 pos_bits(struct scoutfs_btree_block *bt, unsigned int pos) -{ - return bt->item_hdrs[pos].bits; -} - -static inline bool pos_bit_set(struct scoutfs_btree_block *bt, unsigned int pos, - u8 bit) -{ - return bt->item_hdrs[pos].bits & bit; -} - -static inline u16 bit_count(struct scoutfs_btree_block *bt, u8 bit) -{ - int ind; - - BUG_ON(hweight8(bit) != 1); - - ind = ffs(bit) - 1; - return le16_to_cpu(bt->bit_counts[ind]); -} - -/* find the first item pos with the given bit set */ -static int find_pos_bit(struct scoutfs_btree_block *bt, int pos, u8 bit) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - - while (pos < nr && !pos_bit_set(bt, pos, bit)) - pos++; - - return pos; -} - -/* - * Record the path we took through parent blocks. Used to set the bits - * in parent reference items that lead to bits in leaves. - */ -struct btree_path { - unsigned nr; - struct scoutfs_btree_block *bt[SCOUTFS_BTREE_MAX_HEIGHT]; - u16 pos[SCOUTFS_BTREE_MAX_HEIGHT]; -}; - -#define DECLARE_BTREE_PATH(name) \ - struct btree_path name = {0, } - -/* - * Add a block to the path for later traversal for updating bits. Only dirty - * blocks are put in the path and they have an extra ref to keep them pinned - * until we write them out. - */ -static void path_push(struct btree_path *path, - struct scoutfs_btree_block *bt, unsigned pos) -{ - if (path) { - BUG_ON(path->nr >= SCOUTFS_BTREE_MAX_HEIGHT); - - path->bt[path->nr] = bt; - path->pos[path->nr++] = pos; - } -} - -static struct scoutfs_btree_block *path_pop(struct btree_path *path, unsigned *pos) -{ - if (!path || path->nr == 0) - return NULL; - - *pos = path->pos[--path->nr]; - return path->bt[path->nr]; -} - -static u8 half_bit(struct scoutfs_btree_ring *bring, u64 blkno) -{ - u64 half_blkno = le64_to_cpu(bring->first_blkno) + - (le64_to_cpu(bring->nr_blocks) / 2); - - return blkno < half_blkno ? SCOUTFS_BTREE_BIT_HALF1 : - SCOUTFS_BTREE_BIT_HALF2; -} - -static u8 bits_from_counts(struct scoutfs_btree_block *bt) -{ - u8 bits = 0; - int i; - - for (i = 0; i < SCOUTFS_BTREE_BITS; i++) { - if (bt->bit_counts[i]) - bits |= 1 << i; - } - - return bits; -} - -/* - * The bits set in a parent's ref item include the half bit for the - * child blkno so that we can search for blocks in a specific half of - * the ring. - */ -static u8 ref_item_bits(struct scoutfs_btree_ring *bring, - struct scoutfs_btree_block *child) -{ - return bits_from_counts(child) | - half_bit(bring, le64_to_cpu(child->blkno)); -} - -/* - * Store the new bits and update the counts to match the difference from - * the previously set bits. Callers use this to keep item bits in sync - * with the counts of bits in the block headers. - */ -static void store_pos_bits(struct scoutfs_btree_block *bt, int pos, u8 bits) -{ - u8 diff = bits ^ pos_bits(bt, pos); - int i; - u8 b; - - for (i = 0, b = 1; diff != 0; i++, b <<= 1) { - if (diff & b) { - if (bits & b) - le16_add_cpu(&bt->bit_counts[i], 1); - else - le16_add_cpu(&bt->bit_counts[i], -1); - diff ^= b; - } - } - - bt->item_hdrs[pos].bits = bits; -} -/* - * The caller has descended through parents to a final block. Each - * block may have had item bits modified and counts updated but they - * didn't keep parent item bits in sync with modifications to all the - * children. Our job is to ascend back through parents and set their - * bits to the union of all the bits down through the path to the final - * block. - */ -static void path_repair_reset(struct super_block *sb, struct btree_path *path) -{ - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; - struct scoutfs_btree_block *parent; - struct scoutfs_btree_block *bt; - u8 bits; - int pos; - - bt = path_pop(path, &pos); - - while ((parent = path_pop(path, &pos))) { - bits = ref_item_bits(bring, bt); - store_pos_bits(parent, pos, bits); - bt = parent; - } -} - /* * A block is current if it's in the same half of the ring as the next * dirty block in the transaction. @@ -583,8 +429,9 @@ static void compact_items(struct scoutfs_btree_block *bt) * key, or value pointers across item creation. An easy way to verify * this is to audit pos_item() callers. */ -static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, u8 bits, - void *key, unsigned key_len, void *val, unsigned val_len) +static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, + void *key, unsigned key_len, void *val, + unsigned val_len) { unsigned nr = le16_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; @@ -607,9 +454,6 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, u8 bit BUG_ON(le16_to_cpu(bt->free_end) < offsetof(struct scoutfs_btree_block, item_hdrs[nr])); - bt->item_hdrs[pos].bits = 0; - store_pos_bits(bt, pos, bits); - item = pos_item(bt, pos); item->key_len = cpu_to_le16(key_len); item->val_len = cpu_to_le16(val_len); @@ -629,8 +473,6 @@ static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) struct scoutfs_btree_item *item = pos_item(bt, pos); unsigned int nr = le16_to_cpu(bt->nr_items); - store_pos_bits(bt, pos, 0); - if (pos < (nr - 1)) memmove_arr(bt->item_hdrs, pos, pos + 1, nr - 1 - pos); @@ -667,9 +509,8 @@ static void move_items(struct scoutfs_btree_block *dst, while (f < le16_to_cpu(src->nr_items) && to_move > 0) { from = pos_item(src, f); - create_item(dst, t, pos_bits(src, f), item_key(from), - item_key_len(from), item_val(from), - item_val_len(from)); + create_item(dst, t, item_key(from), item_key_len(from), + item_val(from), item_val_len(from)); to_move -= all_item_bytes(from); @@ -769,11 +610,6 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, * deal with this error: either find a new root or return a hard error * if the block is really corrupt. * - * This only sets the caller's reference. It doesn't know if the - * caller's ref is in a parent item and would need to update bits and - * counts based on the blkno. It's up to the callers to take care of - * that. - * * btree callers serialize concurrent writers in a btree but not between * btrees. We have to lock around the shared btree_info. Callers do * lock between all btree writers and writing dirty blocks. We don't @@ -862,6 +698,11 @@ retry: if (!bti->first_dirty_bh) bti->first_dirty_bh = bh; + if (blkno_is_current(bring, blkno)) + bti->cur_dirtied++; + else + bti->old_dirtied++; + /* wrap next block and increase next seq */ if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) bring->next_block = 0; @@ -879,11 +720,6 @@ retry: le64_add_cpu(&bring->next_seq, 1); - if (half_bit(bring, blkno) == half_bit(bring, bti->first_dirty_blkno)) - bti->cur_dirtied++; - else - bti->old_dirtied++; - mutex_unlock(&bti->mutex); if (bt) { @@ -918,33 +754,6 @@ out: return ret; } -/* - * Get the block referenced by the given parent item. The parent item - * and its bits are updated. - */ -static int get_parent_ref_block(struct super_block *sb, int flags, - struct scoutfs_btree_block *parent, unsigned pos, - struct scoutfs_btree_block **bt_ret) -{ - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; - struct scoutfs_btree_item *item; - struct scoutfs_btree_ref *ref; - u8 bits; - int ret; - - /* ref can only be updated, no insertion or compaction */ - item = pos_item(parent, pos); - ref = item_val(item); - - ret = get_ref_block(sb, flags, ref, bt_ret); - if (ret == 0) { - bits = ref_item_bits(bring, *bt_ret); - store_pos_bits(parent, pos, bits); - } - - return ret; -} - /* * Create a new item in the parent which references the child. The caller * specifies the key in the item that describes the items in the child. @@ -958,9 +767,8 @@ static void create_parent_item(struct scoutfs_btree_ring *bring, .blkno = child->blkno, .seq = child->seq, }; - u8 bits = ref_item_bits(bring, child); - create_item(parent, pos, bits, key, key_len, &ref, sizeof(ref)); + create_item(parent, pos, key, key_len, &ref, sizeof(ref)); } /* @@ -979,16 +787,6 @@ static void update_parent_item(struct scoutfs_btree_ring *bring, item_key(item), item_key_len(item)); } -/* the parent item key and value are fine, but child items have changed */ -static void update_parent_bits(struct scoutfs_btree_ring *bring, - struct scoutfs_btree_block *parent, - unsigned pos, struct scoutfs_btree_block *child) -{ - u8 bits = ref_item_bits(bring, child); - - store_pos_bits(parent, pos, bits); -} - /* * See if we need to split this block while descending for insertion so * that we have enough space to insert. Parent blocks need enough space @@ -997,7 +795,6 @@ static void update_parent_bits(struct scoutfs_btree_ring *bring, * * We split to the left so that the greatest key in the existing block * doesn't change so we don't have to update the key in its parent item. - * We still have to update its bits. * * Returns -errno, 0 if nothing done, or 1 if we split. */ @@ -1046,7 +843,6 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, } move_items(left, right, false, used_total(right) / 2); - update_parent_bits(bring, parent, pos, right); item = last_item(left); create_parent_item(bring, parent, pos, left, @@ -1074,6 +870,7 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, { struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *sib; + struct scoutfs_btree_ref *ref; unsigned int sib_pos; bool move_right; int to_move; @@ -1091,7 +888,8 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, move_right = false; } - ret = get_parent_ref_block(sb, BTW_DIRTY, parent, sib_pos, &sib); + ref = item_val(pos_item(parent, sib_pos)); + ret = get_ref_block(sb, BTW_DIRTY, ref, &sib); if (ret) return ret; @@ -1105,16 +903,12 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, /* update our parent's item */ if (!move_right) update_parent_item(bring, parent, pos, bt); - else - update_parent_bits(bring, parent, pos, bt); /* update or delete sibling's parent item */ if (le16_to_cpu(sib->nr_items) == 0) delete_item(parent, sib_pos); else if (move_right) update_parent_item(bring, parent, sib_pos, sib); - else - update_parent_bits(bring, parent, sib_pos, sib); /* and finally shrink the tree if our parent is the root with 1 */ if (le16_to_cpu(parent->nr_items) == 1) { @@ -1128,74 +922,6 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, return 1; } -/* - * This is called before writing dirty blocks to ensure that each batch - * of dirty blocks migrates half as many blocks from the old half of the - * ring as it dirties from the current half. This ensures that by the - * time we fill the current half of the ring it will no longer reference - * the old half. - * - * We've walked to the parent of the leaf level which might have dirtied - * more blocks. Our job is to dirty as many leaves as we need to bring - * the old count back up to equal the current count. The caller will - * keep trying to walk down different paths of each of the btrees. - */ -static int try_dirty_old(struct super_block *sb, struct scoutfs_btree_block *bt, - u8 old_bit) -{ - DECLARE_BTREE_INFO(sb, bti); - struct scoutfs_btree_block *dirtied; - struct scoutfs_btree_item *item; - struct scoutfs_btree_ref *ref; - struct blk_plug plug; - int ret = 0; - int pos = 0; - int nr; - int i; - - if (bti->old_dirtied >= bti->cur_dirtied) - return 0; - - /* called when first parent level is highest level, can have nothing */ - nr = min_t(int, bti->cur_dirtied - bti->old_dirtied, - bit_count(bt, old_bit)); - if (nr == 0) - return -ENOENT; - - blk_start_plug(&plug); - - /* read 'em all */ - for (i = 0, pos = 0; i < nr; i++, pos++) { - pos = find_pos_bit(bt, pos, old_bit); - if (pos >= le16_to_cpu(bt->nr_items)) { - /* XXX bits in headers didn't match count */ - ret = -EIO; - blk_finish_plug(&plug); - goto out; - } - - item = pos_item(bt, pos); - ref = item_val(item); - - sb_breadahead(sb, le64_to_cpu(ref->blkno)); - } - - blk_finish_plug(&plug); - - /* then actually try and dirty the blocks */ - for (i = 0, pos = 0; i < nr; i++, pos++) { - pos = find_pos_bit(bt, pos, old_bit); - - ret = get_parent_ref_block(sb, BTW_DIRTY, bt, pos, &dirtied); - if (ret) - break; - put_btree_block(dirtied); - } - -out: - return ret; -} - /* * A quick and dirty verification of the btree block. We could add a * lot more checks and make it only verified on read or after @@ -1295,12 +1021,6 @@ static void inc_key(u8 *bytes, unsigned *len) * give the caller the nearest key in the direction of iteration that * will land in a different leaf. * - * The caller provides the path to record the parent blocks and items - * used to reach the leaf. We let them repair the path once they've - * potentially updated bits in the leaf. They must always repair the - * path because we can modify parent bits during descent before - * returning an error. - * * Migrating is a special kind of dirtying that returns the parent block * in the walk if the leaf block is already current and doesn't need to * be migrated. It's presumed that the caller is iterating over keys @@ -1308,10 +1028,10 @@ static void inc_key(u8 *bytes, unsigned *len) * blocks themselves. */ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, - struct btree_path *path, int flags, - void *key, unsigned key_len, unsigned int val_len, u8 bit, - struct scoutfs_btree_block **bt_ret, - void *iter_key, unsigned *iter_len) + int flags, void *key, unsigned key_len, + unsigned int val_len, + struct scoutfs_btree_block **bt_ret, void *iter_key, + unsigned *iter_len) { struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *parent = NULL; @@ -1324,12 +1044,10 @@ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, int cmp; int ret; - if (WARN_ON_ONCE((flags & BTW_DIRTY) && path == NULL) || - WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL)) + if (WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL)) return -EINVAL; restart: - path_repair_reset(sb, path); put_btree_block(parent); parent = NULL; put_btree_block(bt); @@ -1363,17 +1081,10 @@ restart: break; } - if (parent) - ret = get_parent_ref_block(sb, flags, parent, pos, &bt); - else - ret = get_ref_block(sb, flags, &root->ref, &bt); + ret = get_ref_block(sb, flags, ref, &bt); if (ret) break; - /* push the parent once we could have updated its bits */ - if (parent) - path_push(path, parent, pos); - /* XXX it'd be nice to make this tunable */ ret = 0 && verify_btree_block(bt, level); if (ret) @@ -1388,15 +1099,15 @@ restart: /* * Splitting and merging can add or remove parents or * change the pos we take through parents to reach the - * block with the search key|bit. In the rare case that - * we split or merge we simply restart the walk rather - * than try and special case modifying the path to - * reflect the tree changes. + * block with the search key. In the rare case that we + * split or merge we simply restart the walk rather than + * try and special case modifying the path to reflect + * the tree changes. */ ret = 0; if (flags & (BTW_INSERT | BTW_DELETE)) ret = try_split(sb, root, key, key_len, val_len, - parent, pos, bt); + parent, pos, bt); if (ret == 0 && (flags & BTW_DELETE) && parent) ret = try_merge(sb, root, parent, pos, bt); if (ret > 0) @@ -1404,41 +1115,18 @@ restart: else if (ret < 0) break; - /* dirtying old stops at the last parent level */ - if ((flags & BTW_DIRTY_OLD) && (level < 2)) { - if (level == 1) { - path_push(path, bt, 0); - ret = try_dirty_old(sb, bt, bit); - } else { - ret = -ENOENT; - } - break; - } - /* done at the leaf */ - if (level == 0) { - path_push(path, bt, 0); + if (level == 0) break; - } nr = le16_to_cpu(bt->nr_items); - /* - * Find the next child block for the search key or bit. - * Key searches should always find a child, bit searches - * can find that the bit isn't set in the first block. - */ - if (flags & BTW_BIT) { - pos = find_pos_bit(bt, 0, bit); - if (pos >= nr) - ret = -ENOENT; - } else { - pos = find_pos(bt, key, key_len, &cmp); - if (pos >= nr) - ret = -EIO; - } - if (ret) + /* Find the next child block for the search key. */ + pos = find_pos(bt, key, key_len, &cmp); + if (pos >= nr) { + ret = -EIO; break; + } /* give the caller the next key to iterate towards */ if (iter_key && (flags & BTW_NEXT) && (pos < (nr - 1))) { @@ -1510,7 +1198,7 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root if (WARN_ON_ONCE(iref->key)) return -EINVAL; - ret = btree_walk(sb, root, NULL, 0, key, key_len, 0, 0, &bt, NULL, NULL); + ret = btree_walk(sb, root, 0, key, key_len, 0, &bt, NULL, NULL); if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { @@ -1550,7 +1238,6 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root void *val, unsigned val_len) { struct scoutfs_btree_block *bt; - DECLARE_BTREE_PATH(path); int pos; int cmp; int ret; @@ -1558,12 +1245,12 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root if (invalid_item(key, key_len, val_len)) return -EINVAL; - ret = btree_walk(sb, root, &path, BTW_DIRTY | BTW_INSERT, key, key_len, - val_len, 0, &bt, NULL, NULL); + ret = btree_walk(sb, root, BTW_DIRTY | BTW_INSERT, key, key_len, + val_len, &bt, NULL, NULL); if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp) { - create_item(bt, pos, 0, key, key_len, val, val_len); + create_item(bt, pos, key, key_len, val, val_len); ret = 0; } else { ret = -EEXIST; @@ -1572,7 +1259,6 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(sb, &path); return ret; } @@ -1586,7 +1272,6 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - DECLARE_BTREE_PATH(path); int pos; int cmp; int ret; @@ -1594,8 +1279,7 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root if (invalid_item(key, key_len, val_len)) return -EINVAL; - ret = btree_walk(sb, root, &path, BTW_DIRTY, key, key_len, 0, 0, &bt, - NULL, NULL); + ret = btree_walk(sb, root, BTW_DIRTY, key, key_len, 0, &bt, NULL, NULL); if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { @@ -1616,7 +1300,6 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(sb, &path); return ret; } @@ -1628,13 +1311,12 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root void *key, unsigned key_len) { struct scoutfs_btree_block *bt; - DECLARE_BTREE_PATH(path); int pos; int cmp; int ret; - ret = btree_walk(sb, root, &path, BTW_DELETE | BTW_DIRTY, key, key_len, - 0, 0, &bt, NULL, NULL); + ret = btree_walk(sb, root, BTW_DELETE | BTW_DIRTY, key, key_len, 0, + &bt, NULL, NULL); if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { @@ -1654,7 +1336,6 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root put_btree_block(bt); } - path_repair_reset(sb, &path); return ret; } @@ -1697,8 +1378,8 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, walk_len = key_len; for (;;) { - ret = btree_walk(sb, root, NULL, flags, walk_key, walk_len, - 0, 0, &bt, iter_key, &iter_len); + ret = btree_walk(sb, root, flags, walk_key, walk_len, 0, &bt, + iter_key, &iter_len); if (ret < 0) break; @@ -1778,12 +1459,10 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len) { struct scoutfs_btree_block *bt; - DECLARE_BTREE_PATH(path); int cmp; int ret; - ret = btree_walk(sb, root, &path, BTW_DIRTY, key, key_len, 0, 0, &bt, - NULL, NULL); + ret = btree_walk(sb, root, BTW_DIRTY, key, key_len, 0, &bt, NULL, NULL); if (ret == 0) { find_pos(bt, key, key_len, &cmp); if (cmp == 0) @@ -1793,7 +1472,6 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, put_btree_block(bt); } - path_repair_reset(sb, &path); return ret; } @@ -1846,7 +1524,6 @@ int scoutfs_btree_write_dirty(struct super_block *sb) struct scoutfs_super_block *super = &sbi->super; struct scoutfs_btree_root *root; struct scoutfs_btree_block *bt; - DECLARE_BTREE_PATH(path); struct buffer_head *tmp; struct buffer_head *bh; struct blk_plug plug; @@ -1875,11 +1552,10 @@ int scoutfs_btree_write_dirty(struct super_block *sb) if (walk_len == 0) continue; - ret = btree_walk(sb, root, &path, + ret = btree_walk(sb, root, BTW_DIRTY | BTW_NEXT | BTW_MIGRATE, - walk_key, walk_len, 0, 0, &bt, + walk_key, walk_len, 0, &bt, iter_key, &iter_len); - path_repair_reset(sb, &path); if (ret < 0) goto out; diff --git a/kmod/src/format.h b/kmod/src/format.h index 3c78a8dd..c0e854d5 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -77,23 +77,6 @@ struct scoutfs_block_header { * generous. */ #define SCOUTFS_BTREE_MAX_HEIGHT 20 -#define SCOUTFS_BTREE_BITS 8 - -/* - * Btree items can have bits associated with them. Their parent items - * reflect all the bits that their child block contain. Thus searches - * can find items with bits set. - * - * @SCOUTFS_BTREE_BIT_HALF1: Tracks blocks found in the first half of - * the ring. It's used to migrate blocks from the old half of the ring - * into the current half as blocks are dirtied. It's not found in leaf - * items but is calculated based on the block number of referenced - * blocks. _HALF2 is identical but for the second half of the ring. - */ -enum { - SCOUTFS_BTREE_BIT_HALF1 = (1 << 0), - SCOUTFS_BTREE_BIT_HALF2 = (1 << 1), -}; struct scoutfs_btree_ref { __le64 blkno; @@ -116,7 +99,6 @@ struct scoutfs_btree_root { struct scoutfs_btree_item_header { __le16 off; - __u8 bits; } __packed; struct scoutfs_btree_item { @@ -134,7 +116,6 @@ struct scoutfs_btree_block { __le16 free_end; __le16 free_reclaim; __le16 nr_items; - __le16 bit_counts[SCOUTFS_BTREE_BITS]; __u8 level; struct scoutfs_btree_item_header item_hdrs[0]; } __packed; From 1c77473551cbb1be423caef72321d4e510408285 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Oct 2017 10:33:08 -0700 Subject: [PATCH 500/920] scoutfs: free both btree iter keys on error I noticed while working on other code that we weren't trying to free potentially allocated btree iter keys if one of them saw an allocation failure. Signed-off-by: Zach Brown --- kmod/src/btree.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 2376ab54..83b82894 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1371,8 +1371,10 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, walk_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); iter_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); - if (!walk_key || !iter_key) - return -ENOMEM; + if (!walk_key || !iter_key) { + ret = -ENOMEM; + goto out; + } memcpy(walk_key, key, key_len); walk_len = key_len; @@ -1414,6 +1416,7 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, break; } +out: kfree(walk_key); kfree(iter_key); From 5c3962d223ecf753d2ad3120e5a1b8b840e88e92 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Nov 2017 13:27:04 -0800 Subject: [PATCH 501/920] scoutfs: trace correct index item deletion The trace point for deleting index items was using the wrong major and minor. Signed-off-by: Zach Brown --- kmod/src/inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index cdb7275b..c9eecb9e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -779,7 +779,8 @@ static int update_index_items(struct super_block *sb, if (ret || !will_del_index(si, type, major, minor)) return ret; - trace_scoutfs_delete_index_item(sb, type, major, minor, ino); + trace_scoutfs_delete_index_item(sb, type, si->item_majors[type], + si->item_minors[type], ino); del_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; del_ikey.type = type; From a3d500c14394bd76a6874d60a271bb8db8f4c868 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 6 Nov 2017 13:41:06 -0800 Subject: [PATCH 502/920] scoutfs: add rename trace point Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 ++ kmod/src/scoutfs_trace.h | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index b869fb7b..3aa361d9 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1397,6 +1397,8 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, int ret; int err; + trace_scoutfs_rename(sb, old_dir, old_dentry, new_dir, new_dentry); + if (new_dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 4a4beeaa..167794e4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1778,6 +1778,44 @@ TRACE_EVENT(scoutfs_item_shrink_around, __get_str(last), __get_str(next)) ); +TRACE_EVENT(scoutfs_rename, + TP_PROTO(struct super_block *sb, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry), + + TP_ARGS(sb, old_dir, old_dentry, new_dir, new_dentry), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, old_dir_ino) + __field(char *, old_name) + __field(unsigned int, old_name_len) + __field(__u64, new_dir_ino) + __field(char *, new_name) + __field(unsigned int, new_name_len) + __field(__u64, new_inode_ino) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->old_dir_ino = scoutfs_ino(old_dir); + __entry->old_name = (char *)old_dentry->d_name.name; + __entry->old_name_len = old_dentry->d_name.len; + __entry->new_dir_ino = scoutfs_ino(new_dir); + __entry->new_name = (char *)new_dentry->d_name.name; + __entry->new_name_len = new_dentry->d_name.len; + __entry->new_inode_ino = new_dentry->d_inode ? + scoutfs_ino(new_dentry->d_inode) : 0; + ), + + TP_printk("fsid "FSID_FMT" old_dir_ino %llu old_name %.*s (len %u) new_dir_ino %llu new_name %.*s (len %u) new_inode_ino %llu", + __entry->fsid, __entry->old_dir_ino, __entry->old_name_len, + __entry->old_name, __entry->old_name_len, + __entry->new_dir_ino, __entry->new_name_len, + __entry->new_name, __entry->new_name_len, + __entry->new_inode_ino) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From fe8e5e095c29d90abf0f0b683f1bba50c4f92710 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 27 Oct 2017 15:43:13 -0500 Subject: [PATCH 503/920] scoutfs: turn on cluster locking stats I accidentally left this off with the initial dlmglue commit. I enabled it here so that I could see our CW locks happening in real time. We don't print lock name yet but that will be remedied in a future patch. Turning this on gives us a debugfs file, /sys/kernel/debug/scoutfs//locking_state which exports the full lock state to userspace. The information exported on each lock is extensive. The export includes each locks name level, blocking level, request state, flags, etc. We also get a count of lock attempts and failures for each level (cw, pr, ex). In addition we also get the total time and max time waited on a given lock request. Signed-off-by: Mark Fasheh --- kmod/Makefile | 2 +- kmod/src/dlmglue.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/Makefile b/kmod/Makefile index 97c48c17..ade36d3e 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -22,7 +22,7 @@ SCOUTFS_FORMAT_HASH := \ SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ SCOUTFS_FORMAT_HASH=$(SCOUTFS_FORMAT_HASH) \ CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ - EXTRA_CFLAGS=-Werror + EXTRA_CFLAGS="-Werror -DCONFIG_OCFS2_FS_STATS" all: module diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index c4df1071..c7bf711c 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -179,9 +179,9 @@ static void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, int level, ktime_t kt; struct ocfs2_lock_stats *stats; - if (level == LKM_PRMODE) + if (level == DLM_LOCK_PR) stats = &res->l_lock_prmode; - else if (level == LKM_EXMODE) + else if (level == DLM_LOCK_EX) stats = &res->l_lock_exmode; else return; From 3a0d6839c81311374631aebe74dd381d78ab8067 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 6 Nov 2017 14:07:03 -0600 Subject: [PATCH 504/920] scoutfs: provide a debug print method to dlmglue This allows us to decode our binary locknames into a string buffer which dlmglue can then print. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 21 ++++++++++++--------- kmod/src/dlmglue.h | 5 +++++ kmod/src/lock.c | 12 ++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index c7bf711c..d9d4a842 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -92,6 +92,15 @@ static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *l return (struct ocfs2_super *)lockres->l_priv; } +static inline void lockres_name(struct ocfs2_lock_res *lockres, char *buf, + unsigned int len) +{ + if (lockres->l_ops->print) + lockres->l_ops->print(lockres, buf, len); + else + snprintf(buf, len, "%s", lockres->l_name); +} + static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, int wanted); static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, @@ -1687,20 +1696,14 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) int i; char *lvb; struct ocfs2_lock_res *lockres = v; + char lockname[256]; if (!lockres) return -EINVAL; - seq_printf(m, "0x%x\t", OCFS2_DLM_DEBUG_STR_VERSION); + lockres_name(lockres, lockname, 256); -#if 0 - if (lockres->l_type == OCFS2_LOCK_TYPE_DENTRY) - seq_printf(m, "%.*s%08x\t", OCFS2_DENTRY_LOCK_INO_START - 1, - lockres->l_name, - (unsigned int)ocfs2_get_dentry_lock_ino(lockres)); - else -#endif - seq_printf(m, "%.*s\t", OCFS2_LOCK_ID_MAX_LEN, lockres->l_name); + seq_printf(m, "0x%x\t%s\t", OCFS2_DLM_DEBUG_STR_VERSION, lockname); seq_printf(m, "%d\t" "0x%lx\t" diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 80a155f6..486613a6 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -256,6 +256,11 @@ struct ocfs2_lock_res_ops { */ int (*downconvert_worker)(struct ocfs2_lock_res *, int); + /* + * Optional: pretty print the lockname into a buffer + */ + void (*print)(struct ocfs2_lock_res *, char *, unsigned int); + /* * LOCK_TYPE_* flags which describe the specific requirements * of a lock type. Descriptions of each individual flag follow. diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ac4986a2..b42d09b8 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -170,10 +170,19 @@ static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) return UNBLOCK_CONTINUE; } +static void lock_name_string(struct ocfs2_lock_res *lockres, char *buf, + unsigned int len) +{ + struct scoutfs_lock *lock = lockres->l_priv; + + snprintf(buf, len, LN_FMT, LN_ARG(&lock->lock_name)); +} + static struct ocfs2_lock_res_ops scoufs_ino_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ + .print = lock_name_string, .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_RECURSIVE, }; @@ -181,12 +190,14 @@ static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ + .print = lock_name_string, .flags = LOCK_TYPE_RECURSIVE, }; static struct ocfs2_lock_res_ops scoutfs_global_lops = { .get_osb = get_ino_lock_osb, /* XXX: .check_downconvert that queries the item cache for dirty items */ + .print = lock_name_string, .flags = 0, }; @@ -194,6 +205,7 @@ static struct ocfs2_lock_res_ops scoutfs_node_id_lops = { .get_osb = get_ino_lock_osb, /* XXX: .check_downconvert that queries the item cache for dirty items */ .downconvert_worker = ino_lock_downconvert, + .print = lock_name_string, .flags = 0, }; From 9fc67bcf1399a4122320a1eea459e3850153344f Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 26 Oct 2017 18:50:58 -0500 Subject: [PATCH 505/920] scoutfs: add helper to check lock holders dlmglue does some holder checks that can become unwieldy, esepcially with the upcoming CW patch. Put them in a helper function. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index d9d4a842..e3ce2f1b 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -401,6 +401,18 @@ static inline int ocfs2_highest_compat_lock_level(int level) return new_level; } +#define H_EX 0x1 +#define H_PR 0x2 +#define H_ANY (H_EX|H_PR) +static int lockres_has_holders(struct ocfs2_lock_res *lockres, int which) +{ + if (which & H_EX && lockres->l_ex_holders) + return 1; + if (which & H_PR && lockres->l_ro_holders) + return 1; + return 0; +} + static void lockres_set_flags(struct ocfs2_lock_res *lockres, unsigned long newflags) { @@ -934,7 +946,7 @@ static inline int lockres_allow_recursion(struct ocfs2_lock_res *lockres, { return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && wanted <= lockres->l_level && - (lockres->l_ex_holders || lockres->l_ro_holders); + lockres_has_holders(lockres, H_ANY); } static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) @@ -1488,11 +1500,11 @@ static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, if (lockres->l_flags & OCFS2_LOCK_BLOCKED) { switch(lockres->l_blocking) { case DLM_LOCK_EX: - if (!lockres->l_ex_holders && !lockres->l_ro_holders) + if (!lockres_has_holders(lockres, H_ANY)) kick = 1; break; case DLM_LOCK_PR: - if (!lockres->l_ex_holders) + if (!lockres_has_holders(lockres, H_EX)) kick = 1; break; default: @@ -2320,7 +2332,7 @@ recheck: * we notice and clear BLOCKING. */ if (lockres->l_level == DLM_LOCK_NL) { - BUG_ON(lockres->l_ex_holders || lockres->l_ro_holders); + BUG_ON(lockres_has_holders(lockres, H_ANY)); mlog(ML_BASTS, "lockres %s, Aborting dc\n", lockres->l_name); lockres->l_blocking = DLM_LOCK_NL; lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); @@ -2330,8 +2342,8 @@ recheck: /* if we're blocking an exclusive and we have *any* holders, * then requeue. */ - if ((lockres->l_blocking == DLM_LOCK_EX) - && (lockres->l_ex_holders || lockres->l_ro_holders)) { + if (lockres->l_blocking == DLM_LOCK_EX && + lockres_has_holders(lockres, H_ANY)) { mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", lockres->l_name, lockres->l_ex_holders, lockres->l_ro_holders); @@ -2341,7 +2353,7 @@ recheck: /* If it's a PR we're blocking, then only * requeue if we've got any EX holders */ if (lockres->l_blocking == DLM_LOCK_PR && - lockres->l_ex_holders) { + lockres_has_holders(lockres, H_EX)) { mlog(ML_BASTS, "lockres %s, ReQ: EX Holders %u\n", lockres->l_name, lockres->l_ex_holders); goto leave_requeue; From e70dbedb7bd2f8728e41bb848fb49345693fca9f Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 26 Oct 2017 19:01:07 -0500 Subject: [PATCH 506/920] scoutfs: dlmglue support for concurrent writer locks This is a bit trickier than just dropping in a cw holders count. dlmglue was comparing levels by a simple greater than or less than check. Since CW locks are not compatible with PR or EX, this check breaks down. Instead we provide a function which can tell us whether a conversion to a given lock levels is is compatible (cache-wise) with the level we have. We also have some slightly more complicated logic in downconvert. As a result we update the helper that dlmglue uses to choose a downconvert level. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 190 +++++++++++++++++++++++++++++++++++++-------- kmod/src/dlmglue.h | 2 + 2 files changed, 158 insertions(+), 34 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index e3ce2f1b..6eb5ba79 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -139,7 +139,6 @@ static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, static int ocfs2_downconvert_thread(void *arg); static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); -static inline int ocfs2_highest_compat_lock_level(int level); static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, int new_level); static int ocfs2_downconvert_lock(struct ocfs2_super *osb, @@ -192,6 +191,8 @@ static void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, int level, stats = &res->l_lock_prmode; else if (level == DLM_LOCK_EX) stats = &res->l_lock_exmode; + else if (level == DLM_LOCK_CW) + stats = &res->l_lock_cwmode; else return; @@ -363,6 +364,9 @@ static inline void ocfs2_inc_holders(struct ocfs2_lock_res *lockres, case DLM_LOCK_PR: lockres->l_ro_holders++; break; + case DLM_LOCK_CW: + lockres->l_cw_holders++; + break; default: BUG(); } @@ -382,34 +386,81 @@ static inline void ocfs2_dec_holders(struct ocfs2_lock_res *lockres, BUG_ON(!lockres->l_ro_holders); lockres->l_ro_holders--; break; + case DLM_LOCK_CW: + BUG_ON(!lockres->l_cw_holders); + lockres->l_cw_holders--; + break; default: BUG(); } } -/* WARNING: This function lives in a world where the only three lock - * levels are EX, PR, and NL. It *will* have to be adjusted when more - * lock types are added. */ -static inline int ocfs2_highest_compat_lock_level(int level) +/* + * Compatibility matrix indexed by lock level - idea borrowed from + * fs/dlm/lock.c. Going across is the level our lock holds, going down + * is the level we're asked to convert to. The UN column and PD + * columns are unused and act as padding. + */ +static const int level_compat_matrix[8][8] = { + /* Lockres granted level */ + /* UN NL CR CW PR PW EX PD */ + {0, 0, 0, 0, 0, 0, 0, 0}, /* UN */ + {0, 1, 1, 1, 1, 1, 1, 0}, /* NL */ + {0, 0, 1, 1, 1, 1, 1, 0}, /* CR */ + {0, 0, 0, 1, 0, 1, 1, 0}, /* CW */ /* <-- Wanted levels */ + {0, 0, 0, 0, 1, 1, 1, 0}, /* PR */ + {0, 0, 0, 0, 0, 1, 1, 0}, /* PW */ + {0, 0, 0, 0, 0, 0, 1, 0}, /* EX */ + {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ +}; + +static inline int __levels_compat(int lockres_level, int wanted) +{ + return level_compat_matrix[wanted + 1][lockres_level + 1]; +} + +static inline int levels_compat(struct ocfs2_lock_res *lockres, int wanted) +{ + return __levels_compat(lockres->l_level, wanted); +} + +/* + * WARNING: We have to adjust this function when adding lock levels to + * dlmglue + * + * Given a lock blocking 'lockres' at 'level', what new level should + * we downconvert to. This function will never return a level which + * would result in an upconvert. + */ +static inline int ocfs2_downconvert_level(struct ocfs2_lock_res *lockres, + int level) { int new_level = DLM_LOCK_EX; if (level == DLM_LOCK_EX) new_level = DLM_LOCK_NL; - else if (level == DLM_LOCK_PR) - new_level = DLM_LOCK_PR; + else if (level == DLM_LOCK_PR) { + if (lockres->l_level == DLM_LOCK_EX) + new_level = DLM_LOCK_PR; + else + new_level = DLM_LOCK_NL; + } else if (level == DLM_LOCK_CW) + new_level = DLM_LOCK_CW; return new_level; } #define H_EX 0x1 #define H_PR 0x2 -#define H_ANY (H_EX|H_PR) +#define H_CW 0x4 +#define H_ANY (H_EX|H_PR|H_CW) static int lockres_has_holders(struct ocfs2_lock_res *lockres, int which) { if (which & H_EX && lockres->l_ex_holders) return 1; if (which & H_PR && lockres->l_ro_holders) return 1; + if (which & H_CW && lockres->l_cw_holders) + return 1; return 0; } @@ -460,14 +511,15 @@ static void lockres_inc_refresh_gen(struct ocfs2_lock_res *lockres) static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres) { + int dc_level = ocfs2_downconvert_level(lockres, lockres->l_blocking); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); lockres->l_level = lockres->l_requested; - if (lockres->l_level <= - ocfs2_highest_compat_lock_level(lockres->l_blocking)) { + if (levels_compat(lockres, dc_level)) { lockres->l_blocking = DLM_LOCK_NL; lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); } @@ -476,20 +528,24 @@ static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres) { + int old_level = lockres->l_level; + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); - /* Convert from RO to EX doesn't really need anything as our - * information is already up to data. Convert from NL to - * *anything* however should mark ourselves as needing an - * update */ - if (lockres->l_level == DLM_LOCK_NL && - lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { - lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); - lockres_inc_refresh_gen(lockres); - } - + /* + * Converting from NL to any mode, or upconverting between + * incompatible modes will require a refresh. + */ lockres->l_level = lockres->l_requested; + if (lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { + if (old_level == DLM_LOCK_NL || + (old_level == DLM_LOCK_CW && + lockres->l_level != DLM_LOCK_NL)) { + lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + lockres_inc_refresh_gen(lockres); + } + } /* * We set the OCFS2_LOCK_UPCONVERT_FINISHING flag before clearing @@ -535,8 +591,8 @@ static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, * one that goes low enough to satisfy the level we're * blocking. this also catches the case where we get * duplicate BASTs */ - if (ocfs2_highest_compat_lock_level(level) < - ocfs2_highest_compat_lock_level(lockres->l_blocking)) + if (ocfs2_downconvert_level(lockres, level) < + ocfs2_downconvert_level(lockres, lockres->l_blocking)) needs_downconvert = 1; lockres->l_blocking = level; @@ -552,6 +608,16 @@ static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, return needs_downconvert; } +static void set_lock_blocking(struct ocfs2_lock_res *lockres, int level) +{ + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + int needs_downconvert; + + needs_downconvert = ocfs2_generic_handle_bast(lockres, level); + if (needs_downconvert) + ocfs2_schedule_blocked_lock(osb, lockres); +} + /* * OCFS2_LOCK_PENDING and l_pending_gen. * @@ -936,7 +1002,7 @@ static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lock { BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); - return wanted <= ocfs2_highest_compat_lock_level(lockres->l_blocking); + return wanted <= ocfs2_downconvert_level(lockres, lockres->l_blocking); } /* the caller doesn't have to wait on a blocked lock if their wanted level @@ -945,7 +1011,7 @@ static inline int lockres_allow_recursion(struct ocfs2_lock_res *lockres, int wanted) { return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && - wanted <= lockres->l_level && + levels_compat(lockres, wanted) && lockres_has_holders(lockres, H_ANY); } @@ -1028,6 +1094,20 @@ static int ocfs2_wait_for_mask_interruptible(struct ocfs2_mask_waiter *mw, } #endif +static inline int cw_incompat_convert(struct ocfs2_lock_res *lockres, + int level) +{ + /* Have CW, want PR/EX */ + if (lockres->l_level == DLM_LOCK_CW && + (level == DLM_LOCK_PR || level == DLM_LOCK_EX)) + return 1; + /* Have EX/PR, want CW */ + if (level == DLM_LOCK_CW && + (lockres->l_level == DLM_LOCK_PR || lockres->l_level == DLM_LOCK_EX)) + return 1; + return 0; +} + static int __ocfs2_cluster_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level, @@ -1073,7 +1153,7 @@ again: * here. If the lock is blocked waiting on a downconvert, * we'll get caught below. */ if (lockres->l_flags & OCFS2_LOCK_BUSY && - level > lockres->l_level) { + !levels_compat(lockres, level)) { /* is someone sitting in dlm_lock? If so, wait on * them. */ lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); @@ -1096,7 +1176,7 @@ again: * OCFS2_LOCK_BLOCKED check to ensure that there is no pending * downconvert request. */ - if (level <= lockres->l_level) + if (levels_compat(lockres, level)) goto update_holders; } @@ -1110,6 +1190,23 @@ again: goto unlock; } + /* + * Convert from PR/EX to CW and vice-versa. Those levels are + * not compatible with each other. As a result, we have to + * wait for holders on the lock to drain. The easiest way to + * do this is by forcing a downconvert. We can then allow the + * process to come back and reacquire the lock at the correct + * level. + */ + if (cw_incompat_convert(lockres, level)) { + /* ocfs2_unblock_lock will drop to NL, then we can upconvert. */ + set_lock_blocking(lockres, DLM_LOCK_EX); + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); + wait = 1; + goto unlock; + } + + /* NL->Anything, PR->EX conditions are handled here */ if (level > lockres->l_level) { if (noqueue_attempted > 0) { ret = -EAGAIN; @@ -1504,7 +1601,11 @@ static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, kick = 1; break; case DLM_LOCK_PR: - if (!lockres_has_holders(lockres, H_EX)) + if (!lockres_has_holders(lockres, H_EX|H_CW)) + kick = 1; + break; + case DLM_LOCK_CW: + if (!lockres_has_holders(lockres, H_EX|H_PR)) kick = 1; break; default: @@ -1740,22 +1841,30 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) seq_printf(m, "0x%x\t", lvb[i]); #ifdef CONFIG_OCFS2_FS_STATS +# define lock_num_cwmode(_l) ((_l)->l_lock_cwmode.ls_gets) # define lock_num_prmode(_l) ((_l)->l_lock_prmode.ls_gets) # define lock_num_exmode(_l) ((_l)->l_lock_exmode.ls_gets) +# define lock_num_cwmode_failed(_l) ((_l)->l_lock_cwmode.ls_fail) # define lock_num_prmode_failed(_l) ((_l)->l_lock_prmode.ls_fail) # define lock_num_exmode_failed(_l) ((_l)->l_lock_exmode.ls_fail) +# define lock_total_cwmode(_l) ((_l)->l_lock_cwmode.ls_total) # define lock_total_prmode(_l) ((_l)->l_lock_prmode.ls_total) # define lock_total_exmode(_l) ((_l)->l_lock_exmode.ls_total) +# define lock_max_cwmode(_l) ((_l)->l_lock_cwmode.ls_max) # define lock_max_prmode(_l) ((_l)->l_lock_prmode.ls_max) # define lock_max_exmode(_l) ((_l)->l_lock_exmode.ls_max) # define lock_refresh(_l) ((_l)->l_lock_refresh) #else +# define lock_num_cwmode(_l) (0) # define lock_num_prmode(_l) (0) # define lock_num_exmode(_l) (0) +# define lock_num_cwmode_failed(_l) (0) # define lock_num_prmode_failed(_l) (0) # define lock_num_exmode_failed(_l) (0) +# define lock_total_cwmode(_l) (0ULL) # define lock_total_prmode(_l) (0ULL) # define lock_total_exmode(_l) (0ULL) +# define lock_max_cwmode(_l) (0) # define lock_max_prmode(_l) (0) # define lock_max_exmode(_l) (0) # define lock_refresh(_l) (0) @@ -2344,18 +2453,31 @@ recheck: * then requeue. */ if (lockres->l_blocking == DLM_LOCK_EX && lockres_has_holders(lockres, H_ANY)) { - mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", + mlog(ML_BASTS, "lockres %s, ReQ: EX/PR/CW Holders %u,%u\n", lockres->l_name, lockres->l_ex_holders, - lockres->l_ro_holders); + lockres->l_ro_holders, lockres->l_cw_holders); goto leave_requeue; } /* If it's a PR we're blocking, then only - * requeue if we've got any EX holders */ + * requeue if we've got any EX or CW holders */ if (lockres->l_blocking == DLM_LOCK_PR && - lockres_has_holders(lockres, H_EX)) { - mlog(ML_BASTS, "lockres %s, ReQ: EX Holders %u\n", - lockres->l_name, lockres->l_ex_holders); + lockres_has_holders(lockres, H_CW|H_EX)) { + mlog(ML_BASTS, "lockres %s, ReQ: EX/CW Holders %u,%u\n", + lockres->l_name, lockres->l_ex_holders, + lockres->l_cw_holders); + goto leave_requeue; + } + + /* + * Same logic as above, we're checking for any holders that + * are incompatible with CW. + */ + if (lockres->l_blocking == DLM_LOCK_CW + && lockres_has_holders(lockres, H_EX|H_PR)) { + mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", + lockres->l_name, lockres->l_ex_holders, + lockres->l_ro_holders); goto leave_requeue; } @@ -2370,7 +2492,7 @@ recheck: goto leave_requeue; } - new_level = ocfs2_highest_compat_lock_level(lockres->l_blocking); + new_level = ocfs2_downconvert_level(lockres, lockres->l_blocking); if (lockres->l_ops->check_downconvert && !lockres->l_ops->check_downconvert(lockres, new_level)) { diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index 486613a6..c805b7e8 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -107,6 +107,7 @@ struct ocfs2_lock_res { unsigned long l_flags; char l_name[OCFS2_LOCK_ID_MAX_LEN]; unsigned int l_ro_holders; + unsigned int l_cw_holders; unsigned int l_ex_holders; signed char l_level; signed char l_requested; @@ -131,6 +132,7 @@ struct ocfs2_lock_res { struct ocfs2_lock_stats l_lock_prmode; /* PR mode stats */ u32 l_lock_refresh; /* Disk refreshes */ struct ocfs2_lock_stats l_lock_exmode; /* EX mode stats */ + struct ocfs2_lock_stats l_lock_cwmode; /* CW mode stats */ #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map l_lockdep_map; From 5fdcd54a5428abd4cac26912c777f08b457bb9c8 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 9 Nov 2017 16:37:41 -0600 Subject: [PATCH 507/920] scoutfs: _force variants of item_create and item_delete These variants will unconditionally overwrite any existing cached items, making them appropriate for us with CW locked inode index items. Signed-off-by: Mark Fasheh --- kmod/src/item.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/item.h | 6 ++++ 2 files changed, 87 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 2b0665cc..9247ade7 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1104,6 +1104,47 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, return ret; } +int scoutfs_item_create_force(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val, struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + unsigned long flags; + int ret; + + if (invalid_key_val(key, val)) + return -EINVAL; + + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + + item = alloc_item(sb, key, val); + if (!item) + return -ENOMEM; + + spin_lock_irqsave(&cac->lock, flags); + + ret = insert_item(sb, cac, item, true, false); + if (ret) { + SK_PRINTK(KERN_EMERG "Scoutfs: corrupted item cache found while" + " creating item "SK_FMT" on fs %llu\n", + SK_ARG(key), + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); + BUG_ON(ret); + } + scoutfs_inc_counter(sb, item_create); + mark_item_dirty(sb, cac, item); + + spin_unlock_irqrestore(&cac->lock, flags); + + if (ret) + free_item(sb, item); + + return ret; +} + /* * Allocate an item with the key and value and add it to the list of * items to be inserted as a batch later. The caller adds in sort order @@ -1483,6 +1524,46 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, return ret; } +int scoutfs_item_delete_force(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + SCOUTFS_DECLARE_KVEC(del_val); + unsigned long flags; + int ret; + + if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + return -EINVAL; + + scoutfs_kvec_init_null(del_val); + + item = alloc_item(sb, key, NULL); + if (!item) + return -ENOMEM; + + spin_lock_irqsave(&cac->lock, flags); + ret = insert_item(sb, cac, item, true, false); + if (ret) { + SK_PRINTK(KERN_EMERG "Scoutfs: corrupted item cache found while" + " deleting item "SK_FMT" on fs %llu\n", + SK_ARG(key), + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); + BUG_ON(ret); + } + scoutfs_inc_counter(sb, item_create); + mark_item_dirty(sb, cac, item); + + become_deletion_item(sb, cac, item, del_val); + spin_unlock_irqrestore(&cac->lock, flags); + + scoutfs_kvec_kfree(del_val); + + return ret; +} + /* * Delete an item that the caller knows must be dirty because they hold * locks and the transaction and have created or dirtied it. This can't diff --git a/kmod/src/item.h b/kmod/src/item.h index ac64e280..453ec5ed 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -30,6 +30,9 @@ int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_lock *lock); +int scoutfs_item_create_force(struct super_block *sb, + struct scoutfs_key_buf *key, + struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, @@ -40,6 +43,9 @@ void scoutfs_item_update_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val); int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); +int scoutfs_item_delete_force(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_lock *lock); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); From e8f87ff90ac217ad82454372ca8024758cc2a5ea Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 24 Oct 2017 16:19:55 -0500 Subject: [PATCH 508/920] scoutfs: use CW locks for inode index updates This will give us concurrency yet still allow our ioctls to drive cache syncing/invalidation on other nodes. Our lock_coverage() checks evolve to handle direct dlm modes, allowing us to verify correct usage of CW locks. As a test, we can run createmany on two nodes at the same time, each working in their own directory. The following commands were run on each node: $ mkdir /scoutfs/`uname -n` $ cd /scoutfs/`uname -n` $ /root/createmany -o ./file_$i 100000 Before this patch that test wouldn't finish in any reasonable amount of time and I would kill it after some number of hours. After this patch, we make swift progress through the test: [root@fstest3 fstest3.site]# /root/createmany -o ./file_$i 100000 - created 10000 (time 1509394646.11 total 0.31 last 0.31) - created 20000 (time 1509394646.38 total 0.59 last 0.28) - created 30000 (time 1509394646.81 total 1.01 last 0.43) - created 40000 (time 1509394647.31 total 1.51 last 0.50) - created 50000 (time 1509394647.82 total 2.02 last 0.51) - created 60000 (time 1509394648.40 total 2.60 last 0.58) - created 70000 (time 1509394649.06 total 3.26 last 0.66) - created 80000 (time 1509394649.72 total 3.93 last 0.66) - created 90000 (time 1509394650.36 total 4.56 last 0.64) total: 100000 creates in 35.02 seconds: 2855.80 creates/second [root@fstest4 fstest4.fstestnet]# /root/createmany -o ./file_$i 100000 - created 10000 (time 1509394647.35 total 0.75 last 0.75) - created 20000 (time 1509394647.89 total 1.28 last 0.54) - created 30000 (time 1509394648.46 total 1.86 last 0.58) - created 40000 (time 1509394648.96 total 2.35 last 0.49) - created 50000 (time 1509394649.51 total 2.90 last 0.55) - created 60000 (time 1509394650.07 total 3.46 last 0.56) - created 70000 (time 1509394650.79 total 4.19 last 0.72) - created 80000 (time 1509394681.26 total 34.66 last 30.47) - created 90000 (time 1509394681.63 total 35.03 last 0.37) total: 100000 creates in 35.50 seconds: 2816.76 creates/second Signed-off-by: Mark Fasheh --- kmod/src/inode.c | 10 +++++----- kmod/src/item.c | 43 ++++++++++++++++++++++++++----------------- kmod/src/lock.c | 4 +++- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index c9eecb9e..e0c5527e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -775,7 +775,7 @@ static int update_index_items(struct super_block *sb, scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); ins_lock = find_index_lock(lock_list, type, major, minor, ino); - ret = scoutfs_item_create(sb, &ins, NULL, ins_lock); + ret = scoutfs_item_create_force(sb, &ins, NULL, ins_lock); if (ret || !will_del_index(si, type, major, minor)) return ret; @@ -791,7 +791,7 @@ static int update_index_items(struct super_block *sb, del_lock = find_index_lock(lock_list, type, si->item_majors[type], si->item_minors[type], ino); - ret = scoutfs_item_delete(sb, &del, del_lock); + ret = scoutfs_item_delete_force(sb, &del, del_lock); if (ret) { err = scoutfs_item_delete(sb, &ins, ins_lock); BUG_ON(err); @@ -1088,7 +1088,7 @@ int scoutfs_inode_index_try_lock_hold(struct super_block *sb, list_sort(NULL, list, cmp_index_lock); list_for_each_entry(ind_lock, list, head) { - ret = scoutfs_lock_inode_index(sb, DLM_LOCK_EX, ind_lock->type, + ret = scoutfs_lock_inode_index(sb, DLM_LOCK_CW, ind_lock->type, ind_lock->major, ind_lock->ino, &ind_lock->lock); if (ret) @@ -1135,7 +1135,7 @@ void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list) struct index_lock *tmp; list_for_each_entry_safe(ind_lock, tmp, list, head) { - scoutfs_unlock(sb, ind_lock->lock, DLM_LOCK_EX); + scoutfs_unlock(sb, ind_lock->lock, DLM_LOCK_CW); list_del_init(&ind_lock->head); kfree(ind_lock); } @@ -1158,7 +1158,7 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, scoutfs_key_init(&key, &ikey, sizeof(ikey)); lock = find_index_lock(ind_locks, type, major, minor, ino); - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_item_delete_force(sb, &key, lock); if (ret == -ENOENT) ret = 0; return ret; diff --git a/kmod/src/item.c b/kmod/src/item.c index 9247ade7..c680e99a 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -738,22 +738,31 @@ restart: * it be? :). */ static bool lock_coverage(struct scoutfs_lock *lock, - struct scoutfs_key_buf *key, int rw) + struct scoutfs_key_buf *key, int op_level) { - bool writing = rw & WRITE; signed char level; - if (rw & ~WRITE) - return false; - if (!lock || !lock->start || !lock->end) return false; level = ACCESS_ONCE(lock->lockres.l_level); - if ((writing && level != DLM_LOCK_EX) || - (!writing && level != DLM_LOCK_EX && level != DLM_LOCK_PR)) + switch (op_level) { + case DLM_LOCK_CW: + if (level != DLM_LOCK_CW) + return false; + break; + case DLM_LOCK_PR: + if (level < DLM_LOCK_PR) + return false; + break; + case DLM_LOCK_EX: + if (level != DLM_LOCK_EX) + return false; + break; + default: return false; + } return scoutfs_key_compare_ranges(key, key, lock->start, lock->end) == 0; @@ -776,7 +785,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, READ))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) return -EINVAL; trace_scoutfs_item_lookup(sb, key); @@ -932,7 +941,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, goto out; } - if (WARN_ON_ONCE(!lock_coverage(lock, key, READ))) { + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) { ret = -EINVAL; goto out; } @@ -1077,7 +1086,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, if (!item) return -ENOMEM; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) return -EINVAL; do { @@ -1117,7 +1126,7 @@ int scoutfs_item_create_force(struct super_block *sb, if (invalid_key_val(key, val)) return -EINVAL; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) return -EINVAL; item = alloc_item(sb, key, val); @@ -1276,8 +1285,8 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, trace_scoutfs_item_set_batch(sb, first, last); if (WARN_ON_ONCE(scoutfs_key_compare(first, last) > 0) || - WARN_ON_ONCE(!lock_coverage(lock, first, WRITE)) || - WARN_ON_ONCE(!lock_coverage(lock, last, WRITE))) + WARN_ON_ONCE(!lock_coverage(lock, first, DLM_LOCK_EX)) || + WARN_ON_ONCE(!lock_coverage(lock, last, DLM_LOCK_EX))) return -EINVAL; range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); @@ -1392,7 +1401,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) return -EINVAL; do { @@ -1436,7 +1445,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, if (invalid_key_val(key, val)) return -EINVAL; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) return -EINVAL; if (val) { @@ -1495,7 +1504,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) return -EINVAL; scoutfs_kvec_init_null(del_val); @@ -1535,7 +1544,7 @@ int scoutfs_item_delete_force(struct super_block *sb, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, WRITE))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) return -EINVAL; scoutfs_kvec_init_null(del_val); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index b42d09b8..c8aff0cc 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -80,7 +80,9 @@ static int invalidate_caches(struct super_block *sb, int mode, if (ret) return ret; - if (mode == DLM_LOCK_EX) { + + if (mode == DLM_LOCK_EX || + (mode == DLM_LOCK_PR && lock->lockres.l_level == DLM_LOCK_CW)) { if (lock->lock_name.zone == SCOUTFS_FS_ZONE) { ino = le64_to_cpu(lock->lock_name.first); last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; From 457d1b54cfd1ea8bb72d50e79c46ebe81ab2180b Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 15 Nov 2017 17:30:45 -0600 Subject: [PATCH 509/920] scoutfs: fix scoutfs_item_create() item leak We'll leak the new item if we don't have lock coverage. Move the check around to fix this. Signed-off-by: Mark Fasheh --- kmod/src/item.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index c680e99a..840030ee 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1082,13 +1082,13 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, if (invalid_key_val(key, val)) return -EINVAL; + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) + return -EINVAL; + item = alloc_item(sb, key, val); if (!item) return -ENOMEM; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) - return -EINVAL; - do { spin_lock_irqsave(&cac->lock, flags); From dbb5541a0c45312e7bfbd7c27943245bd3c65bcd Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 17 Nov 2017 17:49:48 -0600 Subject: [PATCH 510/920] scoutfs: locking_state needs to include cwmode stats This was inadvertantly left out of the main CW locking commit. We simply need to seq_print the new fields. We add them to the end of the line, thus preserving backwards compatibility with old versions of the debug format. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 6eb5ba79..edde124d 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -1889,6 +1889,17 @@ static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) lock_max_exmode(lockres), lock_refresh(lockres)); + seq_printf(m, "%u\t" + "%u\t" + "%u\t" + "%llu\t" + "%u\t", + lockres->l_cw_holders, + lock_num_cwmode(lockres), + lock_num_cwmode_failed(lockres), + lock_total_cwmode(lockres), + lock_max_cwmode(lockres)); + /* End the line */ seq_printf(m, "\n"); return 0; From 5d52bb93ec23f7bcfeeaa4d60894e00808e2c689 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Nov 2017 14:31:36 -0800 Subject: [PATCH 511/920] scoutfs: add item invalidation range trace point Signed-off-by: Zach Brown --- kmod/src/item.c | 2 ++ kmod/src/scoutfs_trace.h | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 840030ee..8f9f17c7 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1819,6 +1819,8 @@ int scoutfs_item_invalidate(struct super_block *sb, unsigned long flags; int ret; + trace_scoutfs_item_invalidate_range(sb, start, end); + /* XXX think about racing with trans write */ scoutfs_inc_counter(sb, item_range_alloc); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 167794e4..3dc087b7 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1478,6 +1478,12 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, TP_ARGS(sb, start, end) ); +DEFINE_EVENT(scoutfs_range_class, scoutfs_item_invalidate_range, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end), + TP_ARGS(sb, start, end) +); + DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), From 3809f35b94ad75e10c28cc5f433315ee17b46071 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Nov 2017 14:31:56 -0800 Subject: [PATCH 512/920] scoutfs: have item range tracepoint include fsid Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3dc087b7..791bad66 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1456,14 +1456,17 @@ DECLARE_EVENT_CLASS(scoutfs_range_class, struct scoutfs_key_buf *end), TP_ARGS(sb, start, end), TP_STRUCT__entry( + __field(__u64, fsid) __dynamic_array(char, start, scoutfs_key_str(NULL, start)) __dynamic_array(char, end, scoutfs_key_str(NULL, end)) ), TP_fast_assign( + __entry->fsid = FSID_ARG(sb); scoutfs_key_str(__get_dynamic_array(start), start); scoutfs_key_str(__get_dynamic_array(end), end); ), - TP_printk("start %s end %s", __get_str(start), __get_str(end)) + TP_printk("fsid "FSID_FMT" start %s end %s", + __entry->fsid, __get_str(start), __get_str(end)) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_set_batch, From 7767a8a48e2ef80cb795cb75a4fdf696d09e4e88 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Nov 2017 18:26:09 -0800 Subject: [PATCH 513/920] scoutfs: add item cache range tracing Add some tracepoints to track operations on our allocated item cache range structs. Signed-off-by: Zach Brown --- kmod/src/item.c | 11 ++++++ kmod/src/scoutfs_trace.h | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 8f9f17c7..e9e7bc65 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -95,6 +95,9 @@ struct cached_range { struct scoutfs_key_buf *end; }; +#define trace_range(which, sb, rng) \ + trace_scoutfs_item_range_##which(sb, (rng), (rng)->start, (rng)->end) + static u8 item_flags(struct cached_item *item) { return item->deletion ? SCOUTFS_ITEM_FLAG_DELETION : 0; @@ -575,6 +578,7 @@ static void free_range(struct super_block *sb, struct cached_range *rng) { if (!IS_ERR_OR_NULL(rng)) { scoutfs_inc_counter(sb, item_range_free); + trace_range(free, sb, rng); scoutfs_key_free(sb, rng->start); scoutfs_key_free(sb, rng->end); kfree(rng); @@ -640,6 +644,7 @@ restart: goto restart; } + trace_range(ins_rb_insert, sb, ins); rb_link_node(&ins->node, parent, node); rb_insert_color(&ins->node, root); } @@ -696,6 +701,7 @@ restart: if (start_cmp > 0 && end_cmp < 0) { swap(rng->end, rem->start); scoutfs_key_dec(rng->end); + trace_range(remove_mid_left, sb, rng); swap(rem->start, rem->end); scoutfs_key_inc(rem->start); @@ -707,12 +713,14 @@ restart: if (start_cmp < 0 && end_cmp < 0) { swap(rem->end, rng->start); scoutfs_key_inc(rng->start); + trace_range(remove_start, sb, rng); continue; } if (start_cmp > 0 && end_cmp > 0) { swap(rem->start, rng->end); scoutfs_key_dec(rng->end); + trace_range(remove_end, sb, rng); continue; } @@ -723,6 +731,7 @@ restart: } if (insert) { + trace_range(rem_rb_insert, sb, rem); rb_link_node(&rem->node, parent, node); rb_insert_color(&rem->node, root); } else { @@ -2014,6 +2023,7 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, rng->end = first->key; first->key = NULL; scoutfs_key_dec_cur_len(rng->end); + trace_range(shrink_end, sb, rng); } /* set start of remaining existing range */ @@ -2022,6 +2032,7 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, rng->start = last->key; last->key = NULL; scoutfs_key_inc_cur_len(rng->start); + trace_range(shrink_start, sb, rng); } /* add new range, stealing existing end */ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 791bad66..aed95f07 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1499,6 +1499,80 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_read_items, TP_ARGS(sb, start, end) ); +DECLARE_EVENT_CLASS(scoutfs_cached_range_class, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end), + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, rng) + __dynamic_array(char, start, scoutfs_key_str(NULL, start)) + __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + ), + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->rng = rng; + scoutfs_key_str(__get_dynamic_array(start), start); + scoutfs_key_str(__get_dynamic_array(end), end); + ), + TP_printk("fsid "FSID_FMT" rng %p start %s end %s", + __entry->fsid, __entry->rng, __get_str(start), __get_str(end)) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_free, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_ins_rb_insert, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_mid_left, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_start, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_end, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_rem_rb_insert, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_delete_enoent, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_start, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + +DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_end, + TP_PROTO(struct super_block *sb, void *rng, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + TP_ARGS(sb, rng, start, end) +); + #define lock_mode(mode) \ __print_symbolic(mode, \ { DLM_LOCK_IV, "IV" }, \ From c292a3ceb84f6e71ebae594a979cbd7861df1f66 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Nov 2017 14:55:42 -0800 Subject: [PATCH 514/920] scoutfs: add coarse lock lifetime tracing Add tracepoints for allocated lock structs entering the tree and finally being freed. This gives visibility into the lifetime of locks without using much higher frequency per-operation tracing that blow out other events. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 ++ kmod/src/scoutfs_trace.h | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c8aff0cc..c91ee5dd 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -128,6 +128,7 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) BUG_ON(!lock->refcnt); refs = --lock->refcnt; if (!refs) { + trace_scoutfs_lock_free(sb, lock); rb_erase(&lock->node, &linfo->lock_tree); list_del(&lock->lru_entry); spin_unlock(&linfo->lock); @@ -312,6 +313,7 @@ search: new = NULL; found->refcnt = 1; /* Freed by shrinker or on umount */ found->sequence = ++linfo->seq_cnt; + trace_scoutfs_lock_rb_insert(sb, found); rb_link_node(&found->node, parent, node); rb_insert_color(&found->node, &linfo->lock_tree); scoutfs_inc_counter(sb, lock_alloc); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index aed95f07..50ec00c0 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1662,6 +1662,16 @@ DEFINE_EVENT(scoutfs_lock_class, shrink_lock_tree, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_rb_insert, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_free, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + DECLARE_EVENT_CLASS(scoutfs_seg_class, TP_PROTO(struct scoutfs_segment *seg), TP_ARGS(seg), From 77f25a71d55ed03931f97d5aa11365e494b56fbf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Nov 2017 17:28:21 -0800 Subject: [PATCH 515/920] scoutfs: stop overlapping inode index lock ranges Lock names don't have minor. They're a unique position in type.major.ino with ino masked to groups. Any index item is mapped to a single lock. But then each lock has a range of items that it covers. The index item key still has a minor from the bad old days of indexing time. When setting the range of keys covered by the lock name we set it to 0/~0 for the range. This is dead wrong because the minor is a higher priority than the inode in the key space. By setting the minor to 0/~0 we are saying that each lock name covers *all the minors and inodes for that major*. This is wrong because there are multiple lock names for different inode groups for each major. We're in effect having the different lock names associated with ranges that all overlap. And this is very bad because it means that a lock can cache keys that are covered by other locks. An index item lock for a small inode can accidentally create a negative item cache region for later inodes covered by an entirely different lock. We saw failures in scoutfs/500 because of this. A node trying to read an existing item would get enoent because it had a false negative cached region from an unrelated lock that overlapped with the lock that it just acquired from a writer and was trying to read the contents from. The fix is to just set the minor to 0. We're not using it. This stops the lock names with fixed majors and inode ranges from accidentally overlapping with each other. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c91ee5dd..cc23e7de 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -723,7 +723,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, end_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; end_ikey.type = type; end_ikey.major = cpu_to_be64(major | major_mask); - end_ikey.minor = cpu_to_be32(U32_MAX); + end_ikey.minor = cpu_to_be32(0); end_ikey.ino = cpu_to_be64(ino | ino_mask); scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); From 413953adbdedc35fc82a07799dd7df55dd604953 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Nov 2017 08:47:26 -0800 Subject: [PATCH 516/920] scoutfs: add 64bit endian swapping helper Signed-off-by: Zach Brown --- kmod/src/endian_swap.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/endian_swap.h b/kmod/src/endian_swap.h index 5693584b..e64d119e 100644 --- a/kmod/src/endian_swap.h +++ b/kmod/src/endian_swap.h @@ -1,9 +1,11 @@ #ifndef _SCOUTFS_ENDIAN_SWAP_H_ #define _SCOUTFS_ENDIAN_SWAP_H_ +#define le64_to_be64(x) cpu_to_be64(le64_to_cpu(x)) #define le32_to_be32(x) cpu_to_be32(le32_to_cpu(x)) #define le16_to_be16(x) cpu_to_be16(le16_to_cpu(x)) +#define be64_to_le64(x) cpu_to_le64(be64_to_cpu(x)) #define be32_to_le32(x) cpu_to_le32(be32_to_cpu(x)) #define be16_to_le16(x) cpu_to_le16(be16_to_cpu(x)) From c36d90e2169d4758134611fbd7780e725f496f6a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 14 Nov 2017 16:55:36 -0800 Subject: [PATCH 517/920] scoutfs: map inode index item locks in one place We have to map many index item keys down to a lock that then has a start and end key range. We also use this mapping over in index item locking to avoid trying to acquire locks multiple times. We were duplicating the mapping calculation in these two places. This refactors these functions to use one range calculation function. It's going to be used in future patches to fix the mapping of the size index items. This should result in no functional changes. Signed-off-by: Zach Brown --- kmod/src/inode.c | 15 +++++- kmod/src/lock.c | 127 +++++++++++++++++++++++------------------------ kmod/src/lock.h | 4 +- 3 files changed, 77 insertions(+), 69 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e0c5527e..e9ebd9a0 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -712,6 +712,17 @@ static int cmp_index_lock(void *priv, struct list_head *A, struct list_head *B) scoutfs_cmp_u64s(a->ino, b->ino); } +static void clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino) +{ + struct scoutfs_inode_index_key start; + + scoutfs_lock_get_index_item_range(type, *major, *ino, &start, NULL); + + *major = be64_to_cpu(start.major); + *minor = be32_to_cpu(start.minor); + *ino = be64_to_cpu(start.ino); +} + /* * Find the lock that covers the given index item. Returns NULL if * there isn't a lock that covers the item. We know that the list is @@ -726,7 +737,7 @@ static struct scoutfs_lock *find_index_lock(struct list_head *lock_list, struct index_lock needle; int cmp; - scoutfs_lock_clamp_inode_index(type, &major, &minor, &ino); + clamp_inode_index(type, &major, &minor, &ino); needle.type = type; needle.major = major; needle.minor = minor; @@ -897,7 +908,7 @@ static int add_index_lock(struct list_head *list, u64 ino, u8 type, u64 major, { struct index_lock *ind_lock; - scoutfs_lock_clamp_inode_index(type, &major, &minor, &ino); + clamp_inode_index(type, &major, &minor, &ino); list_for_each_entry(ind_lock, list, head) { if (ind_lock->type == type && ind_lock->major == major && diff --git a/kmod/src/lock.c b/kmod/src/lock.c index cc23e7de..e9520d58 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -28,6 +28,7 @@ #include "inode.h" #include "trans.h" #include "counters.h" +#include "endian_swap.h" #define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ @@ -630,63 +631,31 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, } /* - * Set the caller's major, minor, and ino to the start of lock that - * covers the incoming index item. This can be used to discover when - * multiple items map to the same lock. + * Set the caller's index items to the range of index item keys that are + * covered by the lock which covers the given type and major. + * + * We're trying to strike a balance between minimizing lock + * communication by locking a large number of items and minimizing + * contention and hold times by locking a small number of items. + * + * The seq indexes have natural batching and limits on the number of + * keys per major value. + * + * The file size index are very different. For them we use a mix of a + * sort of linear-log distribution (top 4 bits of size), and then also a + * lot of inodes per size. + * + * This can also be used to find items that are covered by the same lock + * because their starting keys are the same. */ -void scoutfs_lock_clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino) +void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, + struct scoutfs_inode_index_key *start, + struct scoutfs_inode_index_key *end) { u64 major_mask; u64 ino_mask; int bit; - switch(type) { - case SCOUTFS_INODE_INDEX_SIZE_TYPE: - major_mask = 0; - if (*major) { - bit = fls64(*major); - if (bit > 4) - major_mask = (1 << (bit - 4)) - 1; - } - ino_mask = (1 << 12) - 1; - break; - - case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: - case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: - major_mask = SCOUTFS_LOCK_SEQ_GROUP_MASK; - ino_mask = ~0ULL; - break; - default: - BUG(); - } - - *major &= ~major_mask; - *minor = 0; - *ino &= ~ino_mask; -} - -/* - * map inode index items to locks. The idea is to not have to - * constantly get locks over a reasonable distribution of items, but - * also not have an insane amount of items covered by locks. time and - * seq indexes have natural batching and limits on the number of keys - * per major value. Size keys are very different. For them we use a - * mix of a sort of linear-log distribution (top 4 bits of size), and - * then also a lot of inodes per size. - */ -int scoutfs_lock_inode_index(struct super_block *sb, int mode, - u8 type, u64 major, u64 ino, - struct scoutfs_lock **ret_lock) -{ - struct scoutfs_lock_name lock_name; - struct scoutfs_inode_index_key start_ikey; - struct scoutfs_inode_index_key end_ikey; - struct scoutfs_key_buf start; - struct scoutfs_key_buf end; - u64 major_mask; - u64 ino_mask; - int bit; - switch(type) { case SCOUTFS_INODE_INDEX_SIZE_TYPE: major_mask = 0; @@ -707,24 +676,50 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, BUG(); } + if (start) { + start->zone = SCOUTFS_INODE_INDEX_ZONE; + start->type = type; + start->major = cpu_to_be64(major & ~major_mask); + start->minor = 0; + start->ino = cpu_to_be64(ino & ~ino_mask); + } + + if (end) { + end->zone = SCOUTFS_INODE_INDEX_ZONE; + end->type = type; + end->major = cpu_to_be64(major | major_mask); + end->minor = 0; + end->ino = cpu_to_be64(ino | ino_mask); + } + +} + +/* + * Lock the given index item. We use the index masks to name a reasonable + * batch of logical items to lock and calculate the start and end + * key values that are covered by the lock. + * + */ +int scoutfs_lock_inode_index(struct super_block *sb, int mode, + u8 type, u64 major, u64 ino, + struct scoutfs_lock **ret_lock) +{ + struct scoutfs_lock_name lock_name; + struct scoutfs_inode_index_key start_ikey; + struct scoutfs_inode_index_key end_ikey; + struct scoutfs_key_buf start; + struct scoutfs_key_buf end; + + scoutfs_lock_get_index_item_range(type, major, ino, + &start_ikey, &end_ikey); + lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - lock_name.zone = SCOUTFS_INODE_INDEX_ZONE; - lock_name.type = type; - lock_name.first = cpu_to_le64(major & ~major_mask); - lock_name.second = cpu_to_le64(ino & ~ino_mask); + lock_name.zone = start_ikey.zone; + lock_name.type = start_ikey.type; + lock_name.first = be64_to_le64(start_ikey.major); + lock_name.second = be64_to_le64(start_ikey.ino); - start_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - start_ikey.type = type; - start_ikey.major = cpu_to_be64(major & ~major_mask); - start_ikey.minor = cpu_to_be32(0); - start_ikey.ino = cpu_to_be64(ino & ~ino_mask); scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); - - end_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - end_ikey.type = type; - end_ikey.major = cpu_to_be64(major | major_mask); - end_ikey.minor = cpu_to_be32(0); - end_ikey.ino = cpu_to_be64(ino | ino_mask); scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); return lock_name_keys(sb, mode, 0, &lock_name, diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 30e95458..be75c9ce 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -36,7 +36,9 @@ int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock); -void scoutfs_lock_clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino); +void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, + struct scoutfs_inode_index_key *start, + struct scoutfs_inode_index_key *end); int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); From a5fa9909f5027bda48655365b4e5dc58259718ae Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Nov 2017 12:52:00 -0800 Subject: [PATCH 518/920] scoutfs: fix size index item mapping The mapping of size index item keys to lock names and key ranges was completely bonkers. Its method of setting variable length masks could easily create locks with different names whose key ranges overlapped. We map ranges of sizes to locks and the big change is that all the inodes in these sizes are covered. We can't try to have groups of inodes per size because that would result in too many full precision size locks. With this fix the size index item locks no longer trigger warnings that we're creating locks with overlapping keys. Signed-off-by: Zach Brown --- kmod/src/lock.c | 48 ++++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e9520d58..ca69153f 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -631,8 +631,8 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, } /* - * Set the caller's index items to the range of index item keys that are - * covered by the lock which covers the given type and major. + * Set the caller's keys to the range of index item keys that are + * covered by the lock which covers the given index item. * * We're trying to strike a balance between minimizing lock * communication by locking a large number of items and minimizing @@ -641,9 +641,15 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, * The seq indexes have natural batching and limits on the number of * keys per major value. * - * The file size index are very different. For them we use a mix of a - * sort of linear-log distribution (top 4 bits of size), and then also a - * lot of inodes per size. + * The file size index is very different. We don't control the + * distribution of sizes amongst inodes. We map ranges of sizes to a + * small set of locks by rounding the size down to groups of sizes + * identified by their highest set bit and two next significant bits. + * This results in ranges that increase by quarters of powers of two. + * (small sizes don't have enough bits for this scheme, they're all + * mapped to a range from 0 to 15.) two (0 and 1 are mapped to 0). Each + * lock then covers all the sizes in their range and all the inodes with + * those sizes. * * This can also be used to find items that are covered by the same lock * because their starting keys are the same. @@ -652,25 +658,28 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, struct scoutfs_inode_index_key *start, struct scoutfs_inode_index_key *end) { - u64 major_mask; - u64 ino_mask; + u64 start_major; + u64 end_major; int bit; switch(type) { case SCOUTFS_INODE_INDEX_SIZE_TYPE: - major_mask = 0; - if (major) { - bit = fls64(major); - if (bit > 4) - major_mask = (1 << (bit - 4)) - 1; + bit = major ? fls64(major) : 0; + if (bit < 5) { + /* sizes [ 0 .. 15 ] are in their own lock */ + start_major = 0; + end_major = 15; + } else { + /* last bit, 2 lesser bits, mask */ + start_major = major & (7ULL << (bit - 3)); + end_major = start_major + (1ULL << (bit - 3)) - 1; } - ino_mask = (1 << 12) - 1; break; case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: - major_mask = SCOUTFS_LOCK_SEQ_GROUP_MASK; - ino_mask = ~0ULL; + start_major = major & ~SCOUTFS_LOCK_SEQ_GROUP_MASK; + end_major = major | SCOUTFS_LOCK_SEQ_GROUP_MASK; break; default: BUG(); @@ -679,19 +688,18 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, if (start) { start->zone = SCOUTFS_INODE_INDEX_ZONE; start->type = type; - start->major = cpu_to_be64(major & ~major_mask); + start->major = cpu_to_be64(start_major); start->minor = 0; - start->ino = cpu_to_be64(ino & ~ino_mask); + start->ino = 0; } if (end) { end->zone = SCOUTFS_INODE_INDEX_ZONE; end->type = type; - end->major = cpu_to_be64(major | major_mask); + end->major = cpu_to_be64(end_major); end->minor = 0; - end->ino = cpu_to_be64(ino | ino_mask); + end->ino = cpu_to_be64(~0ULL); } - } /* From e67a5c9ba4ddfa10bb4912297902373433c79e1e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 20 Nov 2017 14:22:49 -0800 Subject: [PATCH 519/920] scoutfs: add _sk console message wrappers Add some _sk suffix variants of the message printing calls so that we can use per-cpu key buffer arguments without the full SK_PCPU() wrapper. Signed-off-by: Zach Brown --- kmod/src/msg.c | 4 ++++ kmod/src/msg.h | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/kmod/src/msg.c b/kmod/src/msg.c index 98235acf..f2ebc81f 100644 --- a/kmod/src/msg.c +++ b/kmod/src/msg.c @@ -3,6 +3,10 @@ #include "msg.h" +/* + * This can be called with pre-emption disabled if the caller is printing + * the contents of formated per-cpu key string buffers. + */ void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...) { diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 8e53290f..9cde9716 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -1,16 +1,32 @@ #ifndef _SCOUTFS_MSG_H_ #define _SCOUTFS_MSG_H_ +#include "key.h" + void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...); +/* + * The _sk variants wrap the message in the SK_PCPU calls which safely + * manage the use of per-cpu key buffers in the arguments. + */ + #define scoutfs_err(sb, fmt, args...) \ scoutfs_msg(sb, KERN_ERR, " error", fmt, ##args) +#define scoutfs_err_sk(sb, fmt, args...) \ + SK_PCPU(scoutfs_err(sb, fmt, ##args)) + #define scoutfs_warn(sb, fmt, args...) \ scoutfs_msg(sb, KERN_WARNING, " warning", fmt, ##args) +#define scoutfs_warn_sk(sb, fmt, args...) \ + SK_PCPU(scoutfs_warn(sb, fmt, ##args)) + #define scoutfs_info(sb, fmt, args...) \ scoutfs_msg(sb, KERN_INFO, "", fmt, ##args) +#define scoutfs_info_sk(sb, fmt, args...) \ + SK_PCPU(scoutfs_info(sb, fmt, ##args)) + #endif From e0f886e892382695524329306a97c1de906d0d34 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 Nov 2017 11:46:06 -0800 Subject: [PATCH 520/920] scoutfs: add run time lock key overlap testing We can't have locks with keys that overlap. This adds an rbtree of locks that are sorted by their key range so that we can find out if we create overlapping locks before they cause item cache consistency problems. Signed-off-by: Zach Brown --- kmod/src/lock.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 1 + 2 files changed, 64 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ca69153f..f27c2916 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -49,6 +49,7 @@ struct lock_info { spinlock_t lock; unsigned int seq_cnt; struct rb_root lock_tree; + struct rb_root lock_range_tree; struct shrinker shrinker; struct list_head lru_list; unsigned long long lru_nr; @@ -115,6 +116,8 @@ static void free_scoutfs_lock(struct scoutfs_lock *lock) ocfs2_lock_res_free(&lock->lockres); scoutfs_key_free(lock->sb, lock->start); scoutfs_key_free(lock->sb, lock->end); + BUG_ON(!RB_EMPTY_NODE(&lock->node)); + BUG_ON(!RB_EMPTY_NODE(&lock->range_node)); kfree(lock); } } @@ -131,6 +134,12 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) if (!refs) { trace_scoutfs_lock_free(sb, lock); rb_erase(&lock->node, &linfo->lock_tree); + RB_CLEAR_NODE(&lock->node); + if(!RB_EMPTY_NODE(&lock->range_node)) { + rb_erase(&lock->range_node, + &linfo->lock_range_tree); + RB_CLEAR_NODE(&lock->range_node); + } list_del(&lock->lru_entry); spin_unlock(&linfo->lock); ocfs2_simple_drop_lockres(&linfo->dlmglue, @@ -230,6 +239,9 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, if (lock == NULL) return NULL; + RB_CLEAR_NODE(&lock->node); + RB_CLEAR_NODE(&lock->range_node); + if (start) { lock->start = scoutfs_key_dup(sb, start); lock->end = scoutfs_key_dup(sb, end); @@ -265,6 +277,46 @@ static int cmp_lock_names(struct scoutfs_lock_name *a, scoutfs_cmp_u64s(le64_to_cpu(a->second), le64_to_cpu(b->second)); } +static int insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct rb_root *root = &linfo->lock_range_tree; + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_lock *lock; + int cmp; + + if (!ins->start) + return 0; + + while (*node) { + parent = *node; + lock = container_of(*node, struct scoutfs_lock, range_node); + + cmp = scoutfs_key_compare_ranges(ins->start, ins->end, + lock->start, lock->end); + if (WARN_ON_ONCE(cmp == 0)) { + scoutfs_warn_sk(sb, "inserting lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT" overlaps with existing lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT"\n", + ins, LN_ARG(&ins->lock_name), + SK_ARG(ins->start), SK_ARG(ins->end), + lock, LN_ARG(&lock->lock_name), + SK_ARG(lock->start), SK_ARG(lock->end)); + return -EINVAL; + } + + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + + rb_link_node(&ins->range_node, parent, node); + rb_insert_color(&ins->range_node, root); + + return 0; +} + static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, struct scoutfs_lock_name *lock_name, struct ocfs2_lock_res_ops *type, @@ -278,6 +330,7 @@ static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, struct rb_node *parent; struct rb_node **node; int cmp; + int ret; search: spin_lock(&linfo->lock); @@ -314,6 +367,14 @@ search: new = NULL; found->refcnt = 1; /* Freed by shrinker or on umount */ found->sequence = ++linfo->seq_cnt; + + ret = insert_range_node(sb, found); + if (ret < 0) { + spin_unlock(&linfo->lock); + free_scoutfs_lock(found); + return NULL; + } + trace_scoutfs_lock_rb_insert(sb, found); rb_link_node(&found->node, parent, node); rb_insert_color(&found->node, &linfo->lock_tree); @@ -817,6 +878,8 @@ static int init_lock_info(struct super_block *sb) linfo->shrinker.seeks = DEFAULT_SEEKS; register_shrinker(&linfo->shrinker); linfo->sb = sb; + linfo->lock_tree = RB_ROOT; + linfo->lock_range_tree = RB_ROOT; snprintf(linfo->ls_name, DLM_LOCKSPACE_LEN, "%llx", le64_to_cpu(sbi->super.hdr.fsid)); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index be75c9ce..9d5af790 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -22,6 +22,7 @@ struct scoutfs_lock { struct dlm_lksb lksb; unsigned int sequence; /* for debugging and sanity checks */ struct rb_node node; + struct rb_node range_node; unsigned int refcnt; struct ocfs2_lock_res lockres; struct list_head lru_entry; From e800cb678584f519844cb9d42878b45beff51aa0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Nov 2017 15:05:17 -0800 Subject: [PATCH 521/920] scoutfs: track segment writes and bytes Add counters for the count of used bytes in segments written. Signed-off-by: Zach Brown --- kmod/src/compact.c | 4 +++- kmod/src/counters.h | 6 ++++-- kmod/src/seg.c | 10 ++++++++++ kmod/src/seg.h | 1 + kmod/src/trans.c | 4 +++- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 82acc11b..fa83741e 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -430,7 +430,9 @@ static int compact_segments(struct super_block *sb, if (ret) break; - scoutfs_inc_counter(sb, compact_segment_written); + scoutfs_inc_counter(sb, compact_segment_writes); + scoutfs_add_counter(sb, compact_segment_write_bytes, + scoutfs_seg_total_bytes(seg)); } return ret; diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 2933ae09..3cfea65f 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -17,12 +17,14 @@ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_shrink) \ EXPAND_COUNTER(seg_free) \ - EXPAND_COUNTER(trans_level0_seg_write) \ + EXPAND_COUNTER(trans_level0_seg_writes) \ + EXPAND_COUNTER(trans_level0_seg_write_bytes) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(compact_operations) \ EXPAND_COUNTER(compact_segment_moved) \ EXPAND_COUNTER(compact_segment_read) \ - EXPAND_COUNTER(compact_segment_written) \ + EXPAND_COUNTER(compact_segment_writes) \ + EXPAND_COUNTER(compact_segment_write_bytes) \ EXPAND_COUNTER(compact_sticky_upper) \ EXPAND_COUNTER(compact_sticky_written) \ EXPAND_COUNTER(data_readpage) \ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index f17a8ebd..1e4ff372 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -557,6 +557,16 @@ int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off) return off; } +/* + * Return the count of bytes of the segment actually used. + */ +u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg) +{ + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + + return le32_to_cpu(sblk->total_bytes); +} + /* * Returns true if the given item population will fit in a single * segment. diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 5a2909d4..6d2f09ad 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -25,6 +25,7 @@ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key_buf *key); int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off); +u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg); int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, struct scoutfs_key_buf *key, struct kvec *val, u8 *flags); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 32c4c751..31b4a768 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -142,7 +142,9 @@ void scoutfs_trans_write_func(struct work_struct *work) if (ret) goto out; - scoutfs_inc_counter(sb, trans_level0_seg_write); + scoutfs_inc_counter(sb, trans_level0_seg_writes); + scoutfs_add_counter(sb, trans_level0_seg_write_bytes, + scoutfs_seg_total_bytes(seg)); } else if (sbi->trans_deadline_expired) { /* From 15fe0eaa95bf56d0d5a6d7b637982c43b02ab62c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Nov 2017 15:20:34 -0800 Subject: [PATCH 522/920] scoutfs: add counters for the source of commits Signed-off-by: Zach Brown --- kmod/src/counters.h | 5 +++++ kmod/src/item.c | 6 ++++-- kmod/src/super.c | 8 ++++++++ kmod/src/trans.c | 12 +++++++++--- kmod/src/trans.h | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 3cfea65f..94101e6a 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -17,6 +17,11 @@ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_shrink) \ EXPAND_COUNTER(seg_free) \ + EXPAND_COUNTER(trans_commit_fsync) \ + EXPAND_COUNTER(trans_commit_full) \ + EXPAND_COUNTER(trans_commit_item_flush) \ + EXPAND_COUNTER(trans_commit_sync_fs) \ + EXPAND_COUNTER(trans_commit_timer) \ EXPAND_COUNTER(trans_level0_seg_writes) \ EXPAND_COUNTER(trans_level0_seg_write_bytes) \ EXPAND_COUNTER(manifest_compact_migrate) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index e9e7bc65..b5afda23 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1805,8 +1805,10 @@ int scoutfs_item_writeback(struct super_block *sb, spin_unlock_irqrestore(&cac->lock, flags); - if (sync) - ret = scoutfs_sync_fs(sb, 1); + if (sync) { + scoutfs_inc_counter(sb, trans_commit_item_flush); + ret = scoutfs_trans_sync(sb, 1); + } return ret; } diff --git a/kmod/src/super.c b/kmod/src/super.c index 83dabb5e..719640e1 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -90,6 +90,14 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) return 0; } +static int scoutfs_sync_fs(struct super_block *sb, int wait) +{ + trace_scoutfs_sync_fs(sb, wait); + scoutfs_inc_counter(sb, trans_commit_sync_fs); + + return scoutfs_trans_sync(sb, wait); +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .drop_inode = scoutfs_drop_inode, diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 31b4a768..2f3fde7e 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -123,7 +123,10 @@ void scoutfs_trans_write_func(struct work_struct *work) trace_scoutfs_trans_write_func(sb, scoutfs_item_has_dirty(sb)); + if (scoutfs_item_has_dirty(sb)) { + if (sbi->trans_deadline_expired) + scoutfs_inc_counter(sb, trans_commit_timer); /* * XXX only straight pass through, we're not worrying * about leaking segnos nor duplicate manifest entries @@ -218,13 +221,12 @@ static void queue_trans_work(struct scoutfs_sb_info *sbi) * before the caller got here that wouldn't be covered by a commit * that's in flight. */ -int scoutfs_sync_fs(struct super_block *sb, int wait) +int scoutfs_trans_sync(struct super_block *sb, int wait) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct write_attempt attempt; int ret; - trace_scoutfs_sync_fs(sb, wait); if (!wait) { queue_trans_work(sbi); @@ -248,7 +250,10 @@ int scoutfs_sync_fs(struct super_block *sb, int wait) int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) { - return scoutfs_sync_fs(file->f_inode->i_sb, 1); + struct super_block *sb = file_inode(file)->i_sb; + + scoutfs_inc_counter(sb, trans_commit_fsync); + return scoutfs_trans_sync(sb, 1); } void scoutfs_trans_restart_sync_deadline(struct super_block *sb) @@ -317,6 +322,7 @@ static bool acquired_hold(struct super_block *sb, vals = tri->reserved_vals + cnt->vals; fits = scoutfs_item_dirty_fits_single(sb, items, keys, vals); if (!fits) { + scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; } diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 775c9f62..0df05ff3 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -4,7 +4,7 @@ #include "count.h" void scoutfs_trans_write_func(struct work_struct *work); -int scoutfs_sync_fs(struct super_block *sb, int wait); +int scoutfs_trans_sync(struct super_block *sb, int wait); int scoutfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); void scoutfs_trans_restart_sync_deadline(struct super_block *sb); From cfe81354ee9ca51b4596f5c2591e62f9c7e39358 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 30 Nov 2017 10:57:14 -0800 Subject: [PATCH 523/920] scoutfs: remove SCOUTFS_LOCK_INODE_GROUP_OFFSET This is an unused artifact from a previous key format. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index c0e854d5..fc15a08a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -547,7 +547,6 @@ struct scoutfs_lock_name { #define SCOUTFS_LOCK_INODE_GROUP_NR 1024 #define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) -#define SCOUTFS_LOCK_INODE_GROUP_OFFSET (~0ULL) #define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) From 8064a161f0ccb716c01eefa77289687b1fb2e3c7 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 4 Dec 2017 19:16:13 -0600 Subject: [PATCH 524/920] scoutfs: better tracking of recursive lock holders This replaces the fragile recursive locking logic in dlmglue. In particular that code fails when we have a pending downconvert and a process comes in for a level that's compatible with the existing level. The downconvert will still happen which causes us to now believe we are holding a lock that we are not! We could go back to checking for holders that raced our downconvert worker but that had problems of its own (see commit e8f7ef0). Instead of trying to infer from lock state what we are allowed to do, let's be explicit. Each lock now has a tree of task refs. If you come in to acquire a lock, we look for our task in that tree. If it's not there, we know this is the first time this task wanted that lock, so we can continue. Otherwise we incremement a count on the task ref and return the already locked lock. Unlock does the opposite - it finds the task ref and decreases the count. On zero it will proceed with the actual unlock. The owning task is the only process allowed to manipulate a task ref, so we only have to lock manipulation of the tree. We make an exception for global locks which might be unlocked from another process context (in this case that means the node id lock). Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 19 ++---- kmod/src/dlmglue.h | 15 +---- kmod/src/lock.c | 150 +++++++++++++++++++++++++++++++++++++++++++-- kmod/src/lock.h | 5 ++ kmod/src/server.c | 3 +- kmod/src/super.c | 6 +- 6 files changed, 162 insertions(+), 36 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index edde124d..82611482 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -419,7 +419,7 @@ static inline int __levels_compat(int lockres_level, int wanted) return level_compat_matrix[wanted + 1][lockres_level + 1]; } -static inline int levels_compat(struct ocfs2_lock_res *lockres, int wanted) +int ocfs2_levels_compat(struct ocfs2_lock_res *lockres, int wanted) { return __levels_compat(lockres->l_level, wanted); } @@ -519,7 +519,7 @@ static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); lockres->l_level = lockres->l_requested; - if (levels_compat(lockres, dc_level)) { + if (ocfs2_levels_compat(lockres, dc_level)) { lockres->l_blocking = DLM_LOCK_NL; lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); } @@ -1005,16 +1005,6 @@ static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lock return wanted <= ocfs2_downconvert_level(lockres, lockres->l_blocking); } -/* the caller doesn't have to wait on a blocked lock if their wanted level - * is compatible with it and there are already holders of the lock */ -static inline int lockres_allow_recursion(struct ocfs2_lock_res *lockres, - int wanted) -{ - return (lockres->l_ops->flags & LOCK_TYPE_RECURSIVE) && - levels_compat(lockres, wanted) && - lockres_has_holders(lockres, H_ANY); -} - static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) { INIT_LIST_HEAD(&mw->mw_item); @@ -1153,7 +1143,7 @@ again: * here. If the lock is blocked waiting on a downconvert, * we'll get caught below. */ if (lockres->l_flags & OCFS2_LOCK_BUSY && - !levels_compat(lockres, level)) { + !ocfs2_levels_compat(lockres, level)) { /* is someone sitting in dlm_lock? If so, wait on * them. */ lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); @@ -1176,12 +1166,11 @@ again: * OCFS2_LOCK_BLOCKED check to ensure that there is no pending * downconvert request. */ - if (levels_compat(lockres, level)) + if (ocfs2_levels_compat(lockres, level)) goto update_holders; } if (lockres->l_flags & OCFS2_LOCK_BLOCKED && - !lockres_allow_recursion(lockres, level) && !ocfs2_may_continue_on_blocked_lock(lockres, level)) { /* is the lock is currently blocked on behalf of * another node */ diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index c805b7e8..af138f80 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -286,20 +286,6 @@ struct ocfs2_lock_res_ops { */ #define LOCK_TYPE_USES_LVB 0x2 -/* - * Tells dlmglue to override fairness considerations when locking this - * lock type - the blocking flag will be ignored when a lock is - * requested and we already have it at the appropriate level and the - * resource is currently held. This allows a process to acquire a - * dlmglue lock on the same resource multiple times in a row without - * deadlocking, even if another node has asked for a competing lock on - * the resource. - * - * Note that lock/unlock calls must always be balanced (1 unlock for - * every lock), even when this flag is set. - */ -#define LOCK_TYPE_RECURSIVE 0x4 - struct ocfs2_lock_holder { struct list_head oh_list; struct pid *oh_owner_pid; @@ -352,6 +338,7 @@ void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb); struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void); void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug); +int ocfs2_levels_compat(struct ocfs2_lock_res *lockres, int wanted); #if 0 /* To set the locking protocol on module initialization */ diff --git a/kmod/src/lock.c b/kmod/src/lock.c index f27c2916..85c08a93 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -61,6 +61,108 @@ struct lock_info { static void scoutfs_lock_reclaim(struct work_struct *work); +struct task_ref { + struct task_struct *task; + struct rb_node node; + int count; + int mode;/* for debugging */ +}; + +static struct task_ref *find_task_ref(struct scoutfs_lock *lock, + struct task_struct *task) +{ + struct rb_node *n; + struct task_ref *tmp; + + spin_lock(&lock->task_refs_lock); + n = lock->task_refs.rb_node; + while (n) { + tmp = rb_entry(n, struct task_ref, node); + + if (tmp->task < task) + n = n->rb_left; + else if (tmp->task > task) + n = n->rb_right; + else { + spin_unlock(&lock->task_refs_lock); + return tmp; + } + } + spin_unlock(&lock->task_refs_lock); + + return NULL; +} + +static struct task_ref *alloc_task_ref(struct task_struct *task, int mode) +{ + struct task_ref *ref = kzalloc(sizeof(*ref), GFP_NOFS); + if (ref) { + ref->task = task; + ref->count = 1; + ref->mode = mode; + RB_CLEAR_NODE(&ref->node); + } + return ref; +} + +static void insert_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) +{ + struct task_ref *tmp; + struct rb_node *parent = NULL; + struct rb_node **p; + + spin_lock(&lock->task_refs_lock); + p = &lock->task_refs.rb_node; + while (*p) { + parent = *p; + + tmp = rb_entry(parent, struct task_ref, node); + + if (tmp->task < ref->task) + p = &(*p)->rb_left; + else if (tmp->task > ref->task) + p = &(*p)->rb_right; + else + BUG(); /* We should never find a duplicate */ + } + + rb_link_node(&ref->node, parent, p); + rb_insert_color(&ref->node, &lock->task_refs); + spin_unlock(&lock->task_refs_lock); +} + +static void get_task_ref(struct task_ref *ref) +{ + ref->count++; +} + +static struct task_ref *new_task_ref(struct scoutfs_lock *lock, + struct task_struct *task, int mode) +{ + struct task_ref *ref = alloc_task_ref(task, mode); + if (ref) + insert_task_ref(lock, ref); + + return ref; +} + +static int put_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) +{ + if (!ref) + return 0; + + ref->count--; + if (ref->count == 0) { + spin_lock(&lock->task_refs_lock); + rb_erase(&ref->node, &lock->task_refs); + spin_unlock(&lock->task_refs_lock); + + kfree(ref); + return 0; + } + return 1; +} + /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -196,7 +298,7 @@ static struct ocfs2_lock_res_ops scoufs_ino_lops = { .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, - .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_RECURSIVE, + .flags = LOCK_TYPE_REQUIRES_REFRESH, }; static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { @@ -204,7 +306,6 @@ static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, - .flags = LOCK_TYPE_RECURSIVE, }; static struct ocfs2_lock_res_ops scoutfs_global_lops = { @@ -251,6 +352,8 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, } } + spin_lock_init(&lock->task_refs_lock); + lock->task_refs = RB_ROOT; RB_CLEAR_NODE(&lock->node); lock->sb = sb; lock->lock_name = *lock_name; @@ -491,6 +594,7 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; + struct task_ref *ref = NULL; int lkm_flags; int ret; @@ -506,13 +610,37 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, trace_scoutfs_lock_resource(sb, lock); + if (!(flags & SCOUTFS_LKF_NO_TASK_REF)) { + ref = find_task_ref(lock, current); + if (ref) { + /* + * We found a ref, which means we have already locked + * this resource. Check that the calling task isn't + * trying to switch modes in the middle of a recursive + * lock request. + */ + BUG_ON(!ocfs2_levels_compat(&lock->lockres, mode)); + get_task_ref(ref); + ret = 0; + goto out; + } + + ref = new_task_ref(lock, current, mode); + if (!ref) { + ret = -ENOMEM; + goto out; + } + } + lkm_flags = DLM_LKF_NOORDER; if (flags & SCOUTFS_LKF_TRYLOCK) lkm_flags |= DLM_LKF_NOQUEUE; /* maybe also NONBLOCK? */ ret = ocfs2_cluster_lock(&linfo->dlmglue, &lock->lockres, mode, lkm_flags, 0); +out: if (ret) { + put_task_ref(lock, ref); dec_lock_users(lock); put_scoutfs_lock(sb, lock); } else { @@ -837,9 +965,10 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, &scoutfs_node_id_lops, &start, &end, lock); } -void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, - int level) +void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, + int level, int flags) { + struct task_ref *ref; DECLARE_LOCK_INFO(sb, linfo); if (!lock) @@ -847,6 +976,13 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, trace_scoutfs_unlock(sb, lock); + if (!(flags & SCOUTFS_LKF_NO_TASK_REF)) { + ref = find_task_ref(lock, current); + BUG_ON(!ref); + if (put_task_ref(lock, ref)) + return; + } + ocfs2_cluster_unlock(&linfo->dlmglue, &lock->lockres, level); dec_lock_users(lock); @@ -854,6 +990,12 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, put_scoutfs_lock(sb, lock); } +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, + int level) +{ + scoutfs_unlock_flags(sb, lock, level, 0); +} + /* * The moment this is done we can have other mounts start asking * us to write back and invalidate, so do this very very late. diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 9d5af790..63a914fe 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -7,6 +7,7 @@ #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ #define SCOUTFS_LKF_TRYLOCK 0x02 /* EAGAIN if contention */ +#define SCOUTFS_LKF_NO_TASK_REF 0x04 /* don't create a task ref */ /* flags for scoutfs_lock->flags */ enum { @@ -30,6 +31,8 @@ struct scoutfs_lock { unsigned int users; /* Tracks active users of this lock */ unsigned long flags; wait_queue_head_t waitq; + struct rb_root task_refs; + spinlock_t task_refs_lock; }; u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock); @@ -54,6 +57,8 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); +void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, + int level, int flags); int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); diff --git a/kmod/src/server.c b/kmod/src/server.c index 007ca7d6..6d178574 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -929,7 +929,8 @@ static void scoutfs_server_func(struct work_struct *work) init_waitqueue_head(&waitq); - ret = scoutfs_lock_global(sb, DLM_LOCK_EX, SCOUTFS_LKF_TRYLOCK, + ret = scoutfs_lock_global(sb, DLM_LOCK_EX, + SCOUTFS_LKF_TRYLOCK|SCOUTFS_LKF_NO_TASK_REF, SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, &lock); if (ret) diff --git a/kmod/src/super.c b/kmod/src/super.c index 719640e1..7a21393c 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -332,7 +332,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) out: if (ret) { scoutfs_server_destroy(sb); - scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); + scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, + SCOUTFS_LKF_NO_TASK_REF); sbi->node_id_lock = NULL; } return ret; @@ -362,7 +363,8 @@ static void scoutfs_kill_sb(struct super_block *sb) kill_block_super(sb); if (sbi) { - scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); + scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, + SCOUTFS_LKF_NO_TASK_REF); sbi->node_id_lock = NULL; scoutfs_lock_destroy(sb); From e15eb13ec9316d566a1e556fcb33488818fa0120 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 1 Dec 2017 12:38:09 -0800 Subject: [PATCH 525/920] scoutfs: centralize teardown in put_super We were trying to tear down our mounted file system resources in the ->kill_sb() callback. This happens relatively early in the unmount process. We call kill_block_super() in our teardown which syncs the mount and tears down the vfs structures. By tearing down in ->kill_sb() we were forced to juggle tearing down before and after the call to kill_block_super(). When we got that wrong we'd tear down too many resources and crash in kill_block_super() or we wouldn't tear down enough and leave work still pending that'd explode as we tried to shut down after kill_block_super(). It turns out the vfs has a callback specifcally to solve this ordering problem. The put_super callback is called after having synced the mount but before its totally torn down. By putting all our shutdown in there we no longer have to worry about racing with active use. Auditing the shutdown dependencies also found some bad cases where we were tearding down subsystems that were still in use. The biggest problem was shutting down locking and networking before shutting down the transaction processing which relies on both. Now we first shut down all the client processing, then all the server processing, then the lowest level common infrastructure. The trickiest part in understanding this is knowing that kill_block_super() only calls put_super during mount failure if mount got far enough to assign the root dentry to s_root. We call put_super manually ourselves in mount failure if it didn't get far enough so that all teardown goes through put_super. (You'll see this s_root test in other upstream file system error paths.) Finally while auding the setup and shutdown paths I noticed a few, trans and counters, that needed simple fixes to properly cleanup errors and only shutdown if they've been setup. This all was stressed with an xfstests that races mount and unmount across the cluster. Before this change it'd crash/hang almost instantly and with this change it runs to completion. Signed-off-by: Zach Brown --- kmod/src/counters.c | 3 +- kmod/src/scoutfs_trace.h | 34 ++++++++++++ kmod/src/super.c | 113 +++++++++++++++++++-------------------- kmod/src/trans.c | 14 +++-- 4 files changed, 100 insertions(+), 64 deletions(-) diff --git a/kmod/src/counters.c b/kmod/src/counters.c index fe8b247c..1a3b1fee 100644 --- a/kmod/src/counters.c +++ b/kmod/src/counters.c @@ -82,13 +82,14 @@ int scoutfs_setup_counters(struct super_block *sb) scoutfs_foreach_counter(sb, pcpu) { ret = percpu_counter_init(pcpu, 0, GFP_KERNEL); if (ret) - return ret; + goto out; } counters->kobj.kset = sbi->kset; init_completion(&counters->comp); ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype, NULL, "counters"); +out: if (ret) { /* tear down partial to avoid destroying null kobjs */ scoutfs_foreach_counter(sb, pcpu) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 50ec00c0..0145ee25 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1909,6 +1909,40 @@ TRACE_EVENT(scoutfs_rename, __entry->new_inode_ino) ); +DECLARE_EVENT_CLASS(scoutfs_super_lifecycle_class, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb), + TP_STRUCT__entry( + __field(__u64, fsid) + __field(void *, sb) + __field(void *, sbi) + __field(void *, s_root) + ), + TP_fast_assign( + __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + __entry->sb = sb; + __entry->sbi = SCOUTFS_SB(sb); + __entry->s_root = sb->s_root; + ), + TP_printk("fsid "FSID_FMT" sb %p sbi %p s_root %p", + __entry->fsid, __entry->sb, __entry->sbi, __entry->s_root) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_fill_super, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_put_super, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + +DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_kill_sb, + TP_PROTO(struct super_block *sb), + TP_ARGS(sb) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 7a21393c..b4f14aa6 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -98,6 +98,42 @@ static int scoutfs_sync_fs(struct super_block *sb, int wait) return scoutfs_trans_sync(sb, wait); } +/* + * This destroys all the state that's built up in the sb info during + * mount. It's called by us on errors during mount if we haven't set + * s_root, by mount after returning errors if we have set s_root, and by + * unmount after having synced the super. + */ +static void scoutfs_put_super(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + trace_scoutfs_put_super(sb); + + scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, + SCOUTFS_LKF_NO_TASK_REF); + sbi->node_id_lock = NULL; + + scoutfs_shutdown_trans(sb); + scoutfs_client_destroy(sb); + scoutfs_data_destroy(sb); + scoutfs_inode_destroy(sb); + scoutfs_item_destroy(sb); + + /* the server locks the listen address and compacts */ + scoutfs_server_destroy(sb); + scoutfs_seg_destroy(sb); + scoutfs_lock_destroy(sb); + + debugfs_remove(sbi->debug_root); + scoutfs_destroy_counters(sb); + if (sbi->kset) + kset_unregister(sbi->kset); + kfree(sbi); + + sb->s_fs_info = NULL; +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .drop_inode = scoutfs_drop_inode, @@ -105,6 +141,7 @@ static const struct super_operations scoutfs_super_ops = { .destroy_inode = scoutfs_destroy_inode, .sync_fs = scoutfs_sync_fs, .statfs = scoutfs_statfs, + .put_super = scoutfs_put_super, }; /* @@ -244,6 +281,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) struct inode *inode; int ret; + trace_scoutfs_fill_super(sb); + sb->s_magic = SCOUTFS_SUPER_MAGIC; sb->s_maxbytes = MAX_LFS_FILESIZE; sb->s_op = &scoutfs_super_ops; @@ -271,12 +310,14 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) /* XXX can have multiple mounts of a device, need mount id */ sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); - if (!sbi->kset) - return -ENOMEM; + if (!sbi->kset) { + ret = -ENOMEM; + goto out; + } ret = scoutfs_parse_options(sb, data, &opts); if (ret) - return ret; + goto out; sbi->opts = opts; @@ -288,23 +329,9 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: - scoutfs_lock_setup(sb); - if (ret) - return ret; - - /* - * The server is a bit magical because it can try to read the - * device in async work context. Once we return an error from - * here the kernel starts tearing down the mount and it isn't - * safe to do IO. So we shut the server down before returning - * an error. - * - * But we still want to start the server before the client to - * help single mounts come up without passing through connection - * timeouts. - */ - ret = scoutfs_server_setup(sb) ?: - scoutfs_client_setup(sb); + scoutfs_lock_setup(sb) ?: + scoutfs_server_setup(sb) ?: + scoutfs_client_setup(sb) ?: scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, &sbi->node_id_lock); if (ret) @@ -330,12 +357,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) // scoutfs_scan_orphans(sb); ret = 0; out: - if (ret) { - scoutfs_server_destroy(sb); - scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, - SCOUTFS_LKF_NO_TASK_REF); - sbi->node_id_lock = NULL; - } + /* on error, generic_shutdown_super calls put_super if s_root */ + if (ret && !sb->s_root) + scoutfs_put_super(sb); + return ret; } @@ -345,42 +370,14 @@ static struct dentry *scoutfs_mount(struct file_system_type *fs_type, int flags, return mount_bdev(fs_type, flags, dev_name, data, scoutfs_fill_super); } +/* + * kill_block_super eventually calls ->put_super if s_root is set + */ static void scoutfs_kill_sb(struct super_block *sb) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - /* - * If we had successfully mounted then make sure dirty data - * writeback and compaction is done before we kill the block - * super and start tearing everything down. - */ - if (sb->s_root) { - sync_filesystem(sb); - - scoutfs_server_destroy(sb); - } + trace_scoutfs_kill_sb(sb); kill_block_super(sb); - - if (sbi) { - scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, - SCOUTFS_LKF_NO_TASK_REF); - sbi->node_id_lock = NULL; - - scoutfs_lock_destroy(sb); - scoutfs_client_destroy(sb); - scoutfs_server_destroy(sb); - scoutfs_shutdown_trans(sb); - scoutfs_data_destroy(sb); - scoutfs_inode_destroy(sb); - scoutfs_item_destroy(sb); - scoutfs_seg_destroy(sb); - debugfs_remove(sbi->debug_root); - scoutfs_destroy_counters(sb); - if (sbi->kset) - kset_unregister(sbi->kset); - kfree(sbi); - } } static struct file_system_type scoutfs_fs_type = { diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 2f3fde7e..712ac1f6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -497,10 +497,14 @@ void scoutfs_shutdown_trans(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_TRANS_INFO(sb, tri); - if (sbi->trans_write_workq) { - cancel_delayed_work_sync(&sbi->trans_write_work); - destroy_workqueue(sbi->trans_write_workq); + if (tri) { + if (sbi->trans_write_workq) { + cancel_delayed_work_sync(&sbi->trans_write_work); + destroy_workqueue(sbi->trans_write_workq); + /* trans work schedules after shutdown see null */ + sbi->trans_write_workq = NULL; + } + kfree(tri); + sbi->trans_info = NULL; } - - kfree(tri); } From ea6aaa083c10c65ec63d692f81acf3171f9d2d44 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Nov 2017 13:30:24 -0800 Subject: [PATCH 526/920] scoutfs: promote inode invalidation to function Hoist the per-inode invalidation up into a function because we're about to add invalidating dentries in parent directories. This should result in no functional change. Signed-off-by: Zach Brown --- kmod/src/lock.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 85c08a93..2ea01f3b 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -163,6 +163,20 @@ static int put_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) return 1; } +static void invalidate_inode(struct super_block *sb, u64 ino) +{ + struct inode *inode; + + inode = scoutfs_ilookup(sb, ino); + if (!inode) + return; + + if (S_ISREG(inode->i_mode)) + truncate_inode_pages(inode->i_mapping, 0); + + iput(inode); +} + /* * Invalidate caches on this because another node wants a lock * with the a lock with the given mode and range. We always have to @@ -174,7 +188,6 @@ static int invalidate_caches(struct super_block *sb, int mode, { struct scoutfs_key_buf *start = lock->start; struct scoutfs_key_buf *end = lock->end; - struct inode *inode; u64 ino, last; int ret; @@ -191,12 +204,7 @@ static int invalidate_caches(struct super_block *sb, int mode, ino = le64_to_cpu(lock->lock_name.first); last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; while (ino <= last) { - inode = scoutfs_ilookup(lock->sb, ino); - if (inode && S_ISREG(inode->i_mode)) - truncate_inode_pages(inode->i_mapping, - 0); - - iput(inode); + invalidate_inode(sb, ino); ino++; } } From c8f8feb7f8bf103088c9a4139158ab91236cce86 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Nov 2017 14:56:45 -0800 Subject: [PATCH 527/920] scoutfs: invalidate dentries as locks are dropped Today we use unconditional dentry revalidation to provide directory entry consistency. Any time the vfs tries to use a cached dentry we tell it to drop it and perform a lookup. This hits our item cache which is kept consistent by the locks. This would just be a waste of cpu if it weren't for how heavy weight the vfs revalidation->lookup path is here. It doesn't just invalidate the entry it uses shrink_dcache_parent() to drop all the cached entries in the subtree rooted at the cached entry. We saw 22 second long cpu livelocks in this shrink_dcache_parent() when creating and archiving empty files. Instead lets let the vfs use dcache entries. We only invalidate them as we're dropping the lock that covers them. (Today coarse inode locks cover all the entries in batches of inodes.) We can use d_drop() to remove entries from the cache to stop them from satisfying lookup without trying to free all the dentries under them. Signed-off-by: Zach Brown --- kmod/src/dir.c | 8 -------- kmod/src/lock.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 3aa361d9..1c9a8b8c 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -120,16 +120,8 @@ static void scoutfs_d_release(struct dentry *dentry) } } -static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) -{ - if (flags & LOOKUP_RCU) - return -ECHILD; - return 0;/* Always revalidate for now */ -} - static const struct dentry_operations scoutfs_dentry_ops = { .d_release = scoutfs_d_release, - .d_revalidate = scoutfs_d_revalidate, }; static int alloc_dentry_info(struct dentry *dentry) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 2ea01f3b..ce53a440 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -163,9 +163,26 @@ static int put_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) return 1; } +/* + * invalidate cached data associated with an inode whose lock is going + * away. + * + * Our inode granular locks mean that we have to invalidate all the + * child dentries of a dir so that they can't satisfy lookup after we + * re-acquire the lock. We're invalidating the lock so there can't be + * active users that could modify the entries in the dcache (lookup, + * create, rename, unlink). We have to make it through all the child + * entries and remove them from the hash so that lookup can't find them. + * They can still be disconnected in the cache or used as working + * directories. A directory can have an enormous number of children so + * we try to be break the lock if needed. + */ static void invalidate_inode(struct super_block *sb, u64 ino) { struct inode *inode; + struct dentry *parent; + struct dentry *child; + struct dentry *saved; inode = scoutfs_ilookup(sb, ino); if (!inode) @@ -174,6 +191,42 @@ static void invalidate_inode(struct super_block *sb, u64 ino) if (S_ISREG(inode->i_mode)) truncate_inode_pages(inode->i_mapping, 0); + if (S_ISDIR(inode->i_mode) && (parent = d_find_alias(inode))) { + saved = NULL; +restart: + spin_lock(&parent->d_lock); + if (saved) { + if (saved->d_parent != parent) + child = NULL; + else + child = saved; + dput(saved); + } else { + child = NULL; + } + + if (child == NULL) + child = list_entry(parent->d_subdirs.next, + struct dentry, d_u.d_child); + + list_for_each_entry_from(child, &parent->d_subdirs,d_u.d_child){ + if (spin_needbreak(&parent->d_lock) || need_resched()) { + saved = child; + dget(saved); + spin_unlock(&parent->d_lock); + cond_resched(); + goto restart; + } + + spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); + __d_drop(child); + spin_unlock(&child->d_lock); + } + spin_unlock(&parent->d_lock); + + dput(parent); + } + iput(inode); } From ec91a4375f9b217bf9f9056bf675bb652f0d5125 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Dec 2017 10:49:24 -0800 Subject: [PATCH 528/920] scoutfs: unlock the server listen lock Turns out the server wasn't explicitly unlocking the listen lock! This ended up working because we only shut down an active server on unmount and unmount will tear down the lock space which will drop the still held listen lock. That's just dumb. But it also forced using an awkward lock flag to avoid setting up a task ref for the lock hold which wouldn't have been torn down otherwise. By adding the lock we restore balance to the force and can get rid of that flag. Cool, cool, cool. Signed-off-by: Zach Brown --- kmod/src/server.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 6d178574..9cba3882 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -915,7 +915,7 @@ static void scoutfs_server_func(struct work_struct *work) static struct sockaddr_in zeros = {0,}; struct socket *new_sock; struct socket *sock = NULL; - struct scoutfs_lock *lock; + struct scoutfs_lock *lock = NULL; struct server_connection *conn; struct server_connection *conn_tmp; struct pending_seq *ps; @@ -930,7 +930,7 @@ static void scoutfs_server_func(struct work_struct *work) init_waitqueue_head(&waitq); ret = scoutfs_lock_global(sb, DLM_LOCK_EX, - SCOUTFS_LKF_TRYLOCK|SCOUTFS_LKF_NO_TASK_REF, + SCOUTFS_LKF_TRYLOCK, SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, &lock); if (ret) @@ -1071,6 +1071,8 @@ out: if (sock) sock_release(sock); + scoutfs_unlock(sb, lock, DLM_LOCK_EX); + /* always requeues, cancel_delayed_work_sync cancels on shutdown */ queue_delayed_work(server->wq, &server->dwork, HZ / 2); } From c1d435937ed596c2e39ae5d3f01add58c1229550 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 Dec 2017 10:23:33 -0800 Subject: [PATCH 529/920] scoutfs: remove dentry invalidation relaxing We walk the list of dentries in subdirs on lock invalidation. This can be a large number so we were trying to back off and give other tasks a chance to schedule and other processes a chance to grab the parent lock while we were iterating. The method for backing off saved our position in the list by getting a reference on a child dentry. It dropped that reference after resuming iteration. But it dropped the reference while holding the parent's lock. This is a deadlock if the put tries to finally remove the dentry because it's been unhashed. We saw this deadlock in practice, the crash dump showed us in the final dentry_kill with the parent locked. Let's just get rid of this premature optimization entirely. Both memory pressure and site logistics will tend to keep child lists in parents reasonably small. A CPU can burn through the locks and list entries for quite a few entries before anything will notice. We can revisit the hot spot later it if bubbles to the surface. Signed-off-by: Zach Brown --- kmod/src/lock.c | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index ce53a440..2e42cfad 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -173,16 +173,12 @@ static int put_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) * active users that could modify the entries in the dcache (lookup, * create, rename, unlink). We have to make it through all the child * entries and remove them from the hash so that lookup can't find them. - * They can still be disconnected in the cache or used as working - * directories. A directory can have an enormous number of children so - * we try to be break the lock if needed. */ static void invalidate_inode(struct super_block *sb, u64 ino) { struct inode *inode; struct dentry *parent; struct dentry *child; - struct dentry *saved; inode = scoutfs_ilookup(sb, ino); if (!inode) @@ -192,32 +188,9 @@ static void invalidate_inode(struct super_block *sb, u64 ino) truncate_inode_pages(inode->i_mapping, 0); if (S_ISDIR(inode->i_mode) && (parent = d_find_alias(inode))) { - saved = NULL; -restart: + spin_lock(&parent->d_lock); - if (saved) { - if (saved->d_parent != parent) - child = NULL; - else - child = saved; - dput(saved); - } else { - child = NULL; - } - - if (child == NULL) - child = list_entry(parent->d_subdirs.next, - struct dentry, d_u.d_child); - - list_for_each_entry_from(child, &parent->d_subdirs,d_u.d_child){ - if (spin_needbreak(&parent->d_lock) || need_resched()) { - saved = child; - dget(saved); - spin_unlock(&parent->d_lock); - cond_resched(); - goto restart; - } - + list_for_each_entry(child, &parent->d_subdirs, d_u.d_child){ spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); __d_drop(child); spin_unlock(&child->d_lock); From 3e18cbdb104153f3788017f2ea278081f6f1e77c Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Tue, 28 Nov 2017 14:45:53 -0700 Subject: [PATCH 530/920] Change docker container to versity/rpm-build The versity/rpm-build container has all the bits that scout needs along with our tooling for building RPMs. Switching allows us to start adding rpm builds soon. This also picks up sparse and other nice bits that we are now iterating on in a separate repository from the original omnibus versity docker repository. --- kmod/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kmod/.gitignore b/kmod/.gitignore index 6117d0be..23820239 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -7,3 +7,6 @@ src/.tmp_versions/ src/Module.symvers src/modules.order cscope.* +*.spec +*.sw[po] +rpmbuild/ From ee96b650f0edb18ab5a0345632bba01c14e8067a Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Mon, 11 Dec 2017 16:51:33 -0600 Subject: [PATCH 531/920] scoutfs: log dlm errors We had disabled ocfs2_log_dlm_error() during the initial import. Re-enable it so the kernel can log dlm errors. One problem is that our binary lock names don't lend themselves legible prints. Add a buffer to the lockres to hold a pretty-printed version of the lock name. We fill it from the ->print callback. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 17 ++++------------- kmod/src/dlmglue.h | 2 ++ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 82611482..c48ba5be 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -120,20 +120,9 @@ static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres); static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, int convert); -#if 0 #define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ - if ((_lockres)->l_type != OCFS2_LOCK_TYPE_DENTRY) \ - mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ - _err, _func, _lockres->l_name); \ - else \ - mlog(ML_ERROR, "DLM error %d while calling %s on resource %.*s%08x\n", \ - _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ - (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ -} while (0) -#endif -#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ - mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ - _err, _func, (_lockres)->l_name); \ + printk(KERN_ERR "DLM error %d while calling %s on resource %s\n", \ + _err, _func, (_lockres)->l_pretty_name); \ } while (0) static int ocfs2_downconvert_thread(void *arg); @@ -255,6 +244,8 @@ void ocfs2_lock_res_init_common(struct ocfs2_super *osb, res->l_flags = OCFS2_LOCK_INITIALIZED; + lockres_name(res, res->l_pretty_name, OCFS2_LOCK_ID_PRETTY_LEN); + ocfs2_add_lockres_tracking(res, osb->osb_dlm_debug); ocfs2_init_lock_stats(res); diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index af138f80..afbe8cfc 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -31,6 +31,7 @@ /* Max length of lockid name */ #define OCFS2_LOCK_ID_MAX_LEN 32 +#define OCFS2_LOCK_ID_PRETTY_LEN 64 enum ocfs2_ast_action { OCFS2_AST_INVALID = 0, @@ -106,6 +107,7 @@ struct ocfs2_lock_res { u64 l_refresh_gen; unsigned long l_flags; char l_name[OCFS2_LOCK_ID_MAX_LEN]; + char l_pretty_name[OCFS2_LOCK_ID_PRETTY_LEN]; unsigned int l_ro_holders; unsigned int l_cw_holders; unsigned int l_ex_holders; From 4a043dfd3faa268c928f626bc43bae6581b5b3fc Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 12 Dec 2017 13:43:11 -0600 Subject: [PATCH 532/920] scoutfs: add some tracing to dlmglue The cluster_lock and cluster_unlock traces are close to each other but not quite there so they have to be two different traces (thanks tracepoints!). The rest (ocfs2_unblock_lock, ocfs2_simple_drop_lock) can use a shared trace class. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 11 +++- kmod/src/scoutfs_trace.h | 127 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index c48ba5be..8ff90e3c 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -38,6 +38,8 @@ #include "dlmglue.h" +#include "scoutfs_trace.h" + #ifdef TRACE_DLMGLUE #define mlog(mask, fmt, args...) trace_printk(fmt , ##args) #define mlog_errno(st) do { \ @@ -1106,6 +1108,8 @@ static int __ocfs2_cluster_lock(struct ocfs2_super *osb, int dlm_locked = 0; int kick_dc = 0; + trace_ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags); + if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) { mlog_errno(-EINVAL); return -EINVAL; @@ -1319,7 +1323,6 @@ int ocfs2_cluster_lock(struct ocfs2_super *osb, 0, _RET_IP_); } - static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level, @@ -1327,6 +1330,8 @@ static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, { unsigned long flags; + trace_ocfs2_cluster_unlock(osb, lockres, level); + spin_lock_irqsave(&lockres->l_lock, flags); ocfs2_dec_holders(lockres, level); ocfs2_downconvert_on_unlock(osb, lockres); @@ -2223,6 +2228,8 @@ void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, { int ret; + trace_ocfs2_simple_drop_lockres(osb, lockres); + ocfs2_mark_lockres_freeing(osb, lockres); ret = ocfs2_drop_lock(osb, lockres); if (ret) @@ -2360,6 +2367,8 @@ static int ocfs2_unblock_lock(struct ocfs2_super *osb, int set_lvb = 0; unsigned int gen; + trace_ocfs2_unblock_lock(osb, lockres); + spin_lock_irqsave(&lockres->l_lock, flags); recheck: diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0145ee25..57fde089 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -35,6 +35,8 @@ #include "ioctl.h" #include "count.h" #include "bio.h" +#include "dlmglue.h" +#include "stackglue.h" struct lock_info; @@ -1943,6 +1945,131 @@ DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_kill_sb, TP_ARGS(sb) ); +TRACE_EVENT(ocfs2_cluster_lock, + TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, + int requested, unsigned int lkm_flags, unsigned int arg_flags), + + TP_ARGS(osb, lockres, requested, lkm_flags, arg_flags), + + TP_STRUCT__entry( + __field(char *, lockspace) + __field(int, lockspace_len) + __field(char *, lockname) + __field(int, requested) + __field(unsigned int, lkm_flags) + __field(unsigned int, arg_flags) + __field(unsigned int, lockres_flags) + __field(int, lockres_level) + __field(int, blocking) + __field(unsigned int, cw_holders) + __field(unsigned int, pr_holders) + __field(unsigned int, ex_holders) + ), + + TP_fast_assign( + __entry->lockspace = osb->cconn->cc_name; + __entry->lockspace_len = osb->cconn->cc_namelen; + __entry->lockname = lockres->l_pretty_name; + __entry->requested = requested; + __entry->lkm_flags = lkm_flags; + __entry->arg_flags = arg_flags; + __entry->lockres_flags = lockres->l_flags; + __entry->lockres_level = lockres->l_level; + __entry->blocking = lockres->l_blocking; + __entry->cw_holders = lockres->l_cw_holders; + __entry->pr_holders = lockres->l_ro_holders; + __entry->ex_holders = lockres->l_ex_holders; + ), + + TP_printk("lockspace %.*s lock %s requested %d lkm_flags 0x%x arg_flags 0x%x lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex %u/%u/%u", + __entry->lockspace_len, __entry->lockspace, __entry->lockname, + __entry->requested, __entry->lkm_flags, __entry->arg_flags, + __entry->lockres_level, __entry->lockres_flags, __entry->blocking, + __entry->cw_holders, __entry->pr_holders, __entry->ex_holders) +); + +TRACE_EVENT(ocfs2_cluster_unlock, + TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, + int level), + + TP_ARGS(osb, lockres, level), + + TP_STRUCT__entry( + __field(char *, lockspace) + __field(int, lockspace_len) + __field(char *, lockname) + __field(int, level) + __field(unsigned int, lockres_flags) + __field(int, lockres_level) + __field(int, blocking) + __field(unsigned int, cw_holders) + __field(unsigned int, pr_holders) + __field(unsigned int, ex_holders) + ), + + TP_fast_assign( + __entry->lockspace = osb->cconn->cc_name; + __entry->lockspace_len = osb->cconn->cc_namelen; + __entry->lockname = lockres->l_pretty_name; + __entry->level = level; + __entry->lockres_flags = lockres->l_flags; + __entry->lockres_level = lockres->l_level; + __entry->blocking = lockres->l_blocking; + __entry->cw_holders = lockres->l_cw_holders; + __entry->pr_holders = lockres->l_ro_holders; + __entry->ex_holders = lockres->l_ex_holders; + ), + + TP_printk("lockspace %.*s lock %s level %d lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", + __entry->lockspace_len, __entry->lockspace, __entry->lockname, + __entry->level, __entry->lockres_level, __entry->lockres_flags, + __entry->blocking, __entry->cw_holders, __entry->pr_holders, + __entry->ex_holders) +); + +DECLARE_EVENT_CLASS(ocfs2_lock_res_class, + TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), + + TP_ARGS(osb, lockres), + + TP_STRUCT__entry( + __field(char *, lockspace) + __field(int, lockspace_len) + __field(char *, lockname) + __field(unsigned int, lockres_flags) + __field(int, lockres_level) + __field(int, blocking) + __field(unsigned int, cw_holders) + __field(unsigned int, pr_holders) + __field(unsigned int, ex_holders) + ), + + TP_fast_assign( + __entry->lockspace = osb->cconn->cc_name; + __entry->lockspace_len = osb->cconn->cc_namelen; + __entry->lockname = lockres->l_pretty_name; + __entry->lockres_flags = lockres->l_flags; + __entry->lockres_level = lockres->l_level; + __entry->blocking = lockres->l_blocking; + __entry->cw_holders = lockres->l_cw_holders; + __entry->pr_holders = lockres->l_ro_holders; + __entry->ex_holders = lockres->l_ex_holders; + ), + + TP_printk("lockspace %.*s lock %s lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", + __entry->lockspace_len, __entry->lockspace, __entry->lockname, + __entry->lockres_level, __entry->lockres_flags, __entry->blocking, + __entry->cw_holders, __entry->pr_holders, __entry->ex_holders) +); + +DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_simple_drop_lockres, + TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), + TP_ARGS(osb, lockres) +); +DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_unblock_lock, + TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), + TP_ARGS(osb, lockres) +); #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From bbbb098e383b7a8b16b6b855c8908a8e38e0988e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 Dec 2017 15:26:33 -0800 Subject: [PATCH 533/920] scoutfs: warn on stale cached segments We've yet to really wire up the eventual consistency of btree ring blocks and segments. The btree block reading code has had a warning that fires if it sees stale blocks for a long time (which we've yet to hit) but we have no such warning in the segment. If we hit stale segments we could have very unpredictable results. So let's add a quick warning to highlight the case to save us heartache if we hit it before implementing full retrying. Signed-off-by: Zach Brown --- kmod/src/compact.c | 2 +- kmod/src/manifest.c | 2 +- kmod/src/seg.c | 51 ++++++++++++++++++++++++++++++++++++--------- kmod/src/seg.h | 3 ++- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index fa83741e..e7fdd551 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -153,7 +153,7 @@ static int read_segment(struct super_block *sb, struct compact_seg *cseg) } else { cseg->seg = seg; scoutfs_inc_counter(sb, compact_segment_read); - ret = scoutfs_seg_wait(sb, cseg->seg); + ret = scoutfs_seg_wait(sb, cseg->seg, cseg->segno, cseg->seq); } /* XXX verify read segment metadata */ diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index fa895d58..2b3d59b0 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -633,7 +633,7 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, if (!ref->seg) break; - err = scoutfs_seg_wait(sb, ref->seg); + err = scoutfs_seg_wait(sb, ref->seg, ref->segno, ref->seq); if (err && !ret) ret = err; } diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 1e4ff372..95624332 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -55,6 +55,14 @@ enum { SF_END_IO = 0, }; +static void *off_ptr(struct scoutfs_segment *seg, u32 off) +{ + unsigned int pg = off >> PAGE_SHIFT; + unsigned int pg_off = off & ~PAGE_MASK; + + return page_address(seg->pages[pg]) + pg_off; +} + static struct scoutfs_segment *alloc_seg(struct super_block *sb, u64 segno) { struct scoutfs_segment *seg; @@ -349,28 +357,51 @@ int scoutfs_seg_submit_write(struct super_block *sb, return 0; } -int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg) +/* + * Wait for IO on the segment to complete. In the cached read fast path + * the bit is already set by the reads that populated the cache. + * + * The caller provides the segno and seq from their segment reference to + * validate that we found the version of the segment that they were + * looking for. If we find an old cached version we return -ESTALE and + * the caller has to retry its reference to find the current segment for + * its operation. (Typically by getting a new manifest btree root and + * searching for keys in the manifest.) + * + * XXX drop stale segments from the cache + * XXX none of the callers perform that retry today. + */ +int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, + u64 segno, u64 seq) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct segment_cache *cac = sbi->segment_cache; + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); int ret; ret = wait_event_interruptible(cac->waitq, test_bit(SF_END_IO, &seg->flags)); - if (!ret) + if (ret) + goto out; + + if (seg->err) { ret = seg->err; + goto out; + } + sblk = off_ptr(seg, 0); + + if (WARN_ON_ONCE(segno != le64_to_cpu(sblk->segno)) || + WARN_ON_ONCE(seq != le64_to_cpu(sblk->seq))) { + ret = -ESTALE; + goto out; + } + + ret = 0; +out: return ret; } -static void *off_ptr(struct scoutfs_segment *seg, u32 off) -{ - unsigned int pg = off >> PAGE_SHIFT; - unsigned int pg_off = off & ~PAGE_MASK; - - return page_address(seg->pages[pg]) + pg_off; -} - static void kvec_from_pages(struct scoutfs_segment *seg, struct kvec *kvec, u32 off, u16 len) { diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 6d2f09ad..4f151490 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -20,7 +20,8 @@ struct scoutfs_segment { struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, u64 segno); -int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg); +int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, + u64 segno, u64 seq); int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key_buf *key); From cdc2249fcf0fe979d0adf7e3fc60ef4d0c0120b9 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 14 Dec 2017 17:13:28 -0600 Subject: [PATCH 534/920] scoutfs: dlmglue/lock counters and tracing We can use the excellent code in counters.h to easily place a whole set of useful counters in dlmglue: - one for every kind of wait in cluster_lock (blocked, busy, etc) - one for each type of dlm operation (lock/unlock requests, converts, etc) - one for each type of downconvert (cw/pr/ex) These will give us a decent idea of the amount and type of lock traffic a given node is seeing. In addition, we add a second trace at the bottom of invalidate_caches. By turning both traces in invalidate_caches on, we can look at our trace log to see how long a given locks downconvert took. Signed-off-by: Mark Fasheh --- kmod/src/counters.h | 15 ++++++++++++-- kmod/src/dlmglue.c | 44 +++++++++++++++++++++++++++++++++------- kmod/src/dlmglue.h | 9 ++++++-- kmod/src/lock.c | 12 ++++++++--- kmod/src/scoutfs_trace.h | 5 +++++ 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 94101e6a..78bcb709 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -38,6 +38,13 @@ EXPAND_COUNTER(data_invalidatepage) \ EXPAND_COUNTER(data_writepage) \ EXPAND_COUNTER(data_end_writeback_page) \ + EXPAND_COUNTER(dlm_cancel_convert) \ + EXPAND_COUNTER(dlm_convert_request) \ + EXPAND_COUNTER(dlm_cw_downconvert) \ + EXPAND_COUNTER(dlm_ex_downconvert) \ + EXPAND_COUNTER(dlm_lock_request) \ + EXPAND_COUNTER(dlm_pr_downconvert) \ + EXPAND_COUNTER(dlm_unlock_request) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_free) \ EXPAND_COUNTER(item_create) \ @@ -58,10 +65,14 @@ EXPAND_COUNTER(item_shrink_small_split) \ EXPAND_COUNTER(item_shrink) \ EXPAND_COUNTER(lock_alloc) \ - EXPAND_COUNTER(lock_free) + EXPAND_COUNTER(lock_blocked_wait) \ + EXPAND_COUNTER(lock_busy_wait) \ + EXPAND_COUNTER(lock_free) \ + EXPAND_COUNTER(lock_incompat_wait) + #define FIRST_COUNTER alloc_alloc -#define LAST_COUNTER lock_free +#define LAST_COUNTER lock_incompat_wait #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 8ff90e3c..4393f595 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -36,6 +36,7 @@ #include #include +#include "counters.h" #include "dlmglue.h" #include "scoutfs_trace.h" @@ -1143,6 +1144,7 @@ again: * them. */ lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); wait = 1; + scoutfs_inc_counter(osb->sb, lock_busy_wait); goto unlock; } @@ -1171,6 +1173,7 @@ again: * another node */ lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); wait = 1; + scoutfs_inc_counter(osb->sb, lock_blocked_wait); goto unlock; } @@ -1187,6 +1190,7 @@ again: set_lock_blocking(lockres, DLM_LOCK_EX); lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); wait = 1; + scoutfs_inc_counter(osb->sb, lock_incompat_wait); goto unlock; } @@ -1216,6 +1220,11 @@ again: gen = lockres_set_pending(lockres); spin_unlock_irqrestore(&lockres->l_lock, flags); + if (lkm_flags & DLM_LKF_CONVERT) + scoutfs_inc_counter(osb->sb, dlm_convert_request); + else + scoutfs_inc_counter(osb->sb, dlm_lock_request); + BUG_ON(level == DLM_LOCK_IV); BUG_ON(level == DLM_LOCK_NL); @@ -1974,8 +1983,9 @@ static void ocfs2_do_node_down(int node_num, void *data) { } -int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, - char *cluster_name, char *ls_name, struct dentry *debug_root) +int ocfs2_dlm_init(struct ocfs2_super *osb, struct super_block *sb, + char *cluster_stack, char *cluster_name, char *ls_name, + struct dentry *debug_root) { int status = 0; struct ocfs2_cluster_connection *conn = NULL; @@ -1986,6 +1996,7 @@ int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, goto local; } #endif + osb->sb = sb; status = ocfs2_dlm_init_debug(osb, debug_root); if (status < 0) { @@ -2149,6 +2160,8 @@ static int ocfs2_drop_lock(struct ocfs2_super *osb, mlog(0, "lock %s, successful return from ocfs2_dlm_unlock\n", lockres->l_name); + scoutfs_inc_counter(osb->sb, dlm_unlock_request); + ocfs2_wait_on_busy_lock(lockres); out: return 0; @@ -2352,6 +2365,8 @@ static int ocfs2_cancel_convert(struct ocfs2_super *osb, mlog(ML_BASTS, "lockres %s\n", lockres->l_name); + scoutfs_inc_counter(osb->sb, dlm_cancel_convert); + return ret; } @@ -2501,11 +2516,6 @@ recheck: goto leave_requeue; } - /* If we get here, then we know that there are no more - * incompatible holders (and anyone asking for an incompatible - * lock is blocked). We can now downconvert the lock */ - if (!lockres->l_ops->downconvert_worker) - goto downconvert; /* Some lockres types want to do a bit of work before * downconverting a lock. Allow that here. The worker function @@ -2513,6 +2523,13 @@ recheck: * it may change while we're not holding the spin lock. */ blocking = lockres->l_blocking; level = lockres->l_level; + + /* If we get here, then we know that there are no more + * incompatible holders (and anyone asking for an incompatible + * lock is blocked). We can now downconvert the lock */ + if (!lockres->l_ops->downconvert_worker) + goto downconvert; + spin_unlock_irqrestore(&lockres->l_lock, flags); ctl->unblock_action = lockres->l_ops->downconvert_worker(lockres, blocking); @@ -2552,6 +2569,19 @@ downconvert: gen = ocfs2_prepare_downconvert(lockres, new_level); spin_unlock_irqrestore(&lockres->l_lock, flags); + + switch (level) { + case DLM_LOCK_EX: + scoutfs_inc_counter(osb->sb, dlm_ex_downconvert); + break; + case DLM_LOCK_PR: + scoutfs_inc_counter(osb->sb, dlm_pr_downconvert); + break; + case DLM_LOCK_CW: + scoutfs_inc_counter(osb->sb, dlm_cw_downconvert); + break; + } + ret = ocfs2_downconvert_lock(osb, lockres, new_level, set_lvb, gen); diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index afbe8cfc..d6b8c639 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -176,6 +176,10 @@ struct ocfs2_super atomic64_t refresh_gen; unsigned long s_mount_opt; + + /* sb is for use with scoutfs counter macros only. Eventually + * we'll roll our own counters code in dlmglue. */ + struct super_block *sb; }; /* For s_mount_opt */ #define OCFS2_MOUNT_NOINTR (1 << 2) @@ -318,8 +322,9 @@ void ocfs2_cluster_unlock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, int level); int ocfs2_init_super(struct ocfs2_super *osb, int flags); -int ocfs2_dlm_init(struct ocfs2_super *osb, char *cluster_stack, - char *cluster_name, char *ls_name, struct dentry *debug_root); +int ocfs2_dlm_init(struct ocfs2_super *osb, struct super_block *sb, + char *cluster_stack, char *cluster_name, char *ls_name, + struct dentry *debug_root); void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); void ocfs2_lock_res_init_common(struct ocfs2_super *osb, diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 2e42cfad..9c8e0ccf 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -223,7 +223,6 @@ static int invalidate_caches(struct super_block *sb, int mode, if (ret) return ret; - if (mode == DLM_LOCK_EX || (mode == DLM_LOCK_PR && lock->lockres.l_level == DLM_LOCK_CW)) { if (lock->lock_name.zone == SCOUTFS_FS_ZONE) { @@ -238,6 +237,12 @@ static int invalidate_caches(struct super_block *sb, int mode, ret = scoutfs_item_invalidate(sb, start, end); } + /* + * Not really tracing the return value here, we're mostly + * interested in elapsed time between the top trace and this one. + */ + trace_scoutfs_lock_invalidate_ret(sb, lock); + return ret; } @@ -1123,8 +1128,9 @@ int scoutfs_lock_setup(struct super_block *sb) goto out; } - ret = ocfs2_dlm_init(&linfo->dlmglue, "null", sbi->opts.cluster_name, - linfo->ls_name, sbi->debug_root); + ret = ocfs2_dlm_init(&linfo->dlmglue, sb, "null", + sbi->opts.cluster_name, linfo->ls_name, + sbi->debug_root); if (ret) goto out; linfo->dlmglue_online = true; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 57fde089..48430a6f 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1654,6 +1654,11 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate, TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate_ret, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); + DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_reclaim, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) From 29d97086aa8e27e9117bc303abb83947c72220c6 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 15 Dec 2017 15:08:30 -0600 Subject: [PATCH 535/920] scoutfs: trace print lockspace and lock names correctly We weren't using the right string macros in the recent lock traces, fix that. Also osb->cconn->cc_name is NULL terminated so we don't need to keep the string length around. Signed-off-by: Mark Fasheh --- kmod/src/scoutfs_trace.h | 60 +++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 48430a6f..7d3901dc 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1957,9 +1957,8 @@ TRACE_EVENT(ocfs2_cluster_lock, TP_ARGS(osb, lockres, requested, lkm_flags, arg_flags), TP_STRUCT__entry( - __field(char *, lockspace) - __field(int, lockspace_len) - __field(char *, lockname) + __string(lockspace, osb->cconn->cc_name) + __string(lockname, lockres->l_pretty_name) __field(int, requested) __field(unsigned int, lkm_flags) __field(unsigned int, arg_flags) @@ -1972,9 +1971,8 @@ TRACE_EVENT(ocfs2_cluster_lock, ), TP_fast_assign( - __entry->lockspace = osb->cconn->cc_name; - __entry->lockspace_len = osb->cconn->cc_namelen; - __entry->lockname = lockres->l_pretty_name; + __assign_str(lockspace, osb->cconn->cc_name); + __assign_str(lockname, lockres->l_pretty_name); __entry->requested = requested; __entry->lkm_flags = lkm_flags; __entry->arg_flags = arg_flags; @@ -1986,11 +1984,12 @@ TRACE_EVENT(ocfs2_cluster_lock, __entry->ex_holders = lockres->l_ex_holders; ), - TP_printk("lockspace %.*s lock %s requested %d lkm_flags 0x%x arg_flags 0x%x lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex %u/%u/%u", - __entry->lockspace_len, __entry->lockspace, __entry->lockname, - __entry->requested, __entry->lkm_flags, __entry->arg_flags, - __entry->lockres_level, __entry->lockres_flags, __entry->blocking, - __entry->cw_holders, __entry->pr_holders, __entry->ex_holders) + TP_printk("lockspace %s lock %s requested %d lkm_flags 0x%x arg_flags 0x%x lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex %u/%u/%u", + __get_str(lockspace), __get_str(lockname), __entry->requested, + __entry->lkm_flags, __entry->arg_flags, + __entry->lockres_level, __entry->lockres_flags, + __entry->blocking, __entry->cw_holders, __entry->pr_holders, + __entry->ex_holders) ); TRACE_EVENT(ocfs2_cluster_unlock, @@ -2000,9 +1999,8 @@ TRACE_EVENT(ocfs2_cluster_unlock, TP_ARGS(osb, lockres, level), TP_STRUCT__entry( - __field(char *, lockspace) - __field(int, lockspace_len) - __field(char *, lockname) + __string(lockspace, osb->cconn->cc_name) + __string(lockname, lockres->l_pretty_name) __field(int, level) __field(unsigned int, lockres_flags) __field(int, lockres_level) @@ -2013,9 +2011,8 @@ TRACE_EVENT(ocfs2_cluster_unlock, ), TP_fast_assign( - __entry->lockspace = osb->cconn->cc_name; - __entry->lockspace_len = osb->cconn->cc_namelen; - __entry->lockname = lockres->l_pretty_name; + __assign_str(lockspace, osb->cconn->cc_name); + __assign_str(lockname, lockres->l_pretty_name); __entry->level = level; __entry->lockres_flags = lockres->l_flags; __entry->lockres_level = lockres->l_level; @@ -2025,11 +2022,11 @@ TRACE_EVENT(ocfs2_cluster_unlock, __entry->ex_holders = lockres->l_ex_holders; ), - TP_printk("lockspace %.*s lock %s level %d lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", - __entry->lockspace_len, __entry->lockspace, __entry->lockname, - __entry->level, __entry->lockres_level, __entry->lockres_flags, - __entry->blocking, __entry->cw_holders, __entry->pr_holders, - __entry->ex_holders) + TP_printk("lockspace %s lock %s level %d lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", + __get_str(lockspace), __get_str(lockname), __entry->level, + __entry->lockres_level, __entry->lockres_flags, + __entry->blocking, __entry->cw_holders, __entry->pr_holders, + __entry->ex_holders) ); DECLARE_EVENT_CLASS(ocfs2_lock_res_class, @@ -2038,9 +2035,8 @@ DECLARE_EVENT_CLASS(ocfs2_lock_res_class, TP_ARGS(osb, lockres), TP_STRUCT__entry( - __field(char *, lockspace) - __field(int, lockspace_len) - __field(char *, lockname) + __string(lockspace, osb->cconn->cc_name) + __string(lockname, lockres->l_pretty_name) __field(unsigned int, lockres_flags) __field(int, lockres_level) __field(int, blocking) @@ -2050,9 +2046,8 @@ DECLARE_EVENT_CLASS(ocfs2_lock_res_class, ), TP_fast_assign( - __entry->lockspace = osb->cconn->cc_name; - __entry->lockspace_len = osb->cconn->cc_namelen; - __entry->lockname = lockres->l_pretty_name; + __assign_str(lockspace, osb->cconn->cc_name); + __assign_str(lockname, lockres->l_pretty_name); __entry->lockres_flags = lockres->l_flags; __entry->lockres_level = lockres->l_level; __entry->blocking = lockres->l_blocking; @@ -2061,10 +2056,11 @@ DECLARE_EVENT_CLASS(ocfs2_lock_res_class, __entry->ex_holders = lockres->l_ex_holders; ), - TP_printk("lockspace %.*s lock %s lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", - __entry->lockspace_len, __entry->lockspace, __entry->lockname, - __entry->lockres_level, __entry->lockres_flags, __entry->blocking, - __entry->cw_holders, __entry->pr_holders, __entry->ex_holders) + TP_printk("lockspace %s lock %s lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", + __get_str(lockspace), __get_str(lockname), + __entry->lockres_level, __entry->lockres_flags, + __entry->blocking, __entry->cw_holders, + __entry->pr_holders, __entry->ex_holders) ); DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_simple_drop_lockres, From bbcf76e154acb505aa3190570146b2b92946d9b5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Dec 2017 09:44:08 -0800 Subject: [PATCH 536/920] scoutfs: reformat counters.h Clean up the counter definition macro. Sort the entries and clean up whitespace so that adding counters in the future will be more orderly and satisfying. Signed-off-by: Zach Brown --- kmod/src/counters.h | 108 ++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 78bcb709..67ceba60 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -11,68 +11,68 @@ * We only have to define each counter here and it'll be enumerated in * other places by this macro. Don't forget to update LAST_COUNTER. */ -#define EXPAND_EACH_COUNTER \ - EXPAND_COUNTER(alloc_alloc) \ - EXPAND_COUNTER(alloc_free) \ - EXPAND_COUNTER(seg_alloc) \ - EXPAND_COUNTER(seg_shrink) \ - EXPAND_COUNTER(seg_free) \ +#define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(alloc_alloc) \ + EXPAND_COUNTER(alloc_free) \ + EXPAND_COUNTER(compact_operations) \ + EXPAND_COUNTER(compact_segment_moved) \ + EXPAND_COUNTER(compact_segment_read) \ + EXPAND_COUNTER(compact_segment_write_bytes) \ + EXPAND_COUNTER(compact_segment_writes) \ + EXPAND_COUNTER(compact_sticky_upper) \ + EXPAND_COUNTER(compact_sticky_written) \ + EXPAND_COUNTER(data_end_writeback_page) \ + EXPAND_COUNTER(data_invalidatepage) \ + EXPAND_COUNTER(data_readpage) \ + EXPAND_COUNTER(data_write_begin) \ + EXPAND_COUNTER(data_write_end) \ + EXPAND_COUNTER(data_writepage) \ + EXPAND_COUNTER(dlm_cancel_convert) \ + EXPAND_COUNTER(dlm_convert_request) \ + EXPAND_COUNTER(dlm_cw_downconvert) \ + EXPAND_COUNTER(dlm_ex_downconvert) \ + EXPAND_COUNTER(dlm_lock_request) \ + EXPAND_COUNTER(dlm_pr_downconvert) \ + EXPAND_COUNTER(dlm_unlock_request) \ + EXPAND_COUNTER(item_alloc) \ + EXPAND_COUNTER(item_create) \ + EXPAND_COUNTER(item_delete) \ + EXPAND_COUNTER(item_free) \ + EXPAND_COUNTER(item_lookup_hit) \ + EXPAND_COUNTER(item_lookup_miss) \ + EXPAND_COUNTER(item_range_alloc) \ + EXPAND_COUNTER(item_range_free) \ + EXPAND_COUNTER(item_range_hit) \ + EXPAND_COUNTER(item_range_insert) \ + EXPAND_COUNTER(item_range_miss) \ + EXPAND_COUNTER(item_shrink) \ + EXPAND_COUNTER(item_shrink_alone) \ + EXPAND_COUNTER(item_shrink_empty_range) \ + EXPAND_COUNTER(item_shrink_next_dirty) \ + EXPAND_COUNTER(item_shrink_outside) \ + EXPAND_COUNTER(item_shrink_range_end) \ + EXPAND_COUNTER(item_shrink_small_split) \ + EXPAND_COUNTER(item_shrink_split_range) \ + EXPAND_COUNTER(lock_alloc) \ + EXPAND_COUNTER(lock_blocked_wait) \ + EXPAND_COUNTER(lock_busy_wait) \ + EXPAND_COUNTER(lock_free) \ + EXPAND_COUNTER(lock_incompat_wait) \ + EXPAND_COUNTER(manifest_compact_migrate) \ + EXPAND_COUNTER(seg_alloc) \ + EXPAND_COUNTER(seg_free) \ + EXPAND_COUNTER(seg_shrink) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_item_flush) \ EXPAND_COUNTER(trans_commit_sync_fs) \ EXPAND_COUNTER(trans_commit_timer) \ - EXPAND_COUNTER(trans_level0_seg_writes) \ EXPAND_COUNTER(trans_level0_seg_write_bytes) \ - EXPAND_COUNTER(manifest_compact_migrate) \ - EXPAND_COUNTER(compact_operations) \ - EXPAND_COUNTER(compact_segment_moved) \ - EXPAND_COUNTER(compact_segment_read) \ - EXPAND_COUNTER(compact_segment_writes) \ - EXPAND_COUNTER(compact_segment_write_bytes) \ - EXPAND_COUNTER(compact_sticky_upper) \ - EXPAND_COUNTER(compact_sticky_written) \ - EXPAND_COUNTER(data_readpage) \ - EXPAND_COUNTER(data_write_begin) \ - EXPAND_COUNTER(data_write_end) \ - EXPAND_COUNTER(data_invalidatepage) \ - EXPAND_COUNTER(data_writepage) \ - EXPAND_COUNTER(data_end_writeback_page) \ - EXPAND_COUNTER(dlm_cancel_convert) \ - EXPAND_COUNTER(dlm_convert_request) \ - EXPAND_COUNTER(dlm_cw_downconvert) \ - EXPAND_COUNTER(dlm_ex_downconvert) \ - EXPAND_COUNTER(dlm_lock_request) \ - EXPAND_COUNTER(dlm_pr_downconvert) \ - EXPAND_COUNTER(dlm_unlock_request) \ - EXPAND_COUNTER(item_alloc) \ - EXPAND_COUNTER(item_free) \ - EXPAND_COUNTER(item_create) \ - EXPAND_COUNTER(item_lookup_hit) \ - EXPAND_COUNTER(item_lookup_miss) \ - EXPAND_COUNTER(item_delete) \ - EXPAND_COUNTER(item_range_alloc) \ - EXPAND_COUNTER(item_range_free) \ - EXPAND_COUNTER(item_range_hit) \ - EXPAND_COUNTER(item_range_miss) \ - EXPAND_COUNTER(item_range_insert) \ - EXPAND_COUNTER(item_shrink_alone) \ - EXPAND_COUNTER(item_shrink_empty_range) \ - EXPAND_COUNTER(item_shrink_next_dirty) \ - EXPAND_COUNTER(item_shrink_outside) \ - EXPAND_COUNTER(item_shrink_range_end) \ - EXPAND_COUNTER(item_shrink_split_range) \ - EXPAND_COUNTER(item_shrink_small_split) \ - EXPAND_COUNTER(item_shrink) \ - EXPAND_COUNTER(lock_alloc) \ - EXPAND_COUNTER(lock_blocked_wait) \ - EXPAND_COUNTER(lock_busy_wait) \ - EXPAND_COUNTER(lock_free) \ - EXPAND_COUNTER(lock_incompat_wait) + EXPAND_COUNTER(trans_level0_seg_writes) -#define FIRST_COUNTER alloc_alloc -#define LAST_COUNTER lock_incompat_wait +#define FIRST_COUNTER alloc_alloc +#define LAST_COUNTER trans_level0_seg_writes #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; From 9ed34f8892c0c8f485311967adc2ec7fd7ce0819 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Dec 2017 11:29:02 -0800 Subject: [PATCH 537/920] scoutfs: add triggers Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/super.c | 3 ++ kmod/src/super.h | 2 + kmod/src/triggers.c | 118 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/triggers.h | 17 +++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 kmod/src/triggers.c create mode 100644 kmod/src/triggers.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index ed9bdcbe..ad177eb2 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -9,7 +9,7 @@ scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ dlmglue.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ manifest.o msg.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o trans.o \ - xattr.o + triggers.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/super.c b/kmod/src/super.c index b4f14aa6..6c12f59b 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -28,6 +28,7 @@ #include "xattr.h" #include "msg.h" #include "counters.h" +#include "triggers.h" #include "trans.h" #include "item.h" #include "manifest.h" @@ -125,6 +126,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); + scoutfs_destroy_triggers(sb); debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); if (sbi->kset) @@ -324,6 +326,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_counters(sb) ?: scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: + scoutfs_setup_triggers(sb) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_inode_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index e09ac9d5..0955d722 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -8,6 +8,7 @@ #include "options.h" struct scoutfs_counters; +struct scoutfs_triggers; struct item_cache; struct manifest; struct segment_cache; @@ -61,6 +62,7 @@ struct scoutfs_sb_info { struct kset *kset; struct scoutfs_counters *counters; + struct scoutfs_triggers *triggers; struct mount_options opts; diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c new file mode 100644 index 00000000..64f83941 --- /dev/null +++ b/kmod/src/triggers.c @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "triggers.h" + +/* + * We have debugfs files we can write to which arm triggers which + * atomically fire once for testing or debugging. + */ + +/* + * The atomic cachelines are kept hot and shared by being read by fast + * paths. They're very rarely modified by debugfs writes which arm them + * and then the next read will atomically clear and return true. + */ +struct scoutfs_triggers { + struct dentry *dir; + atomic_t atomics[SCOUTFS_TRIGGER_NR]; +}; + +#define DECLARE_TRIGGERS(sb, name) \ + struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers + +static char *names[] = { + [SCOUTFS_TRIGGER_SOMETHING] = "something", +}; + +bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t) +{ + DECLARE_TRIGGERS(sb, triggers); + atomic_t *atom; + int old; + int mem; + + BUG_ON(t >= SCOUTFS_TRIGGER_NR); + atom = &triggers->atomics[t]; + + mem = atomic_read(atom); + if (likely(!mem)) + return 0; + + do { + old = mem; + mem = atomic_cmpxchg(atom, old, 0); + } while (mem && mem != old); + + return !!mem; +} + +int scoutfs_setup_triggers(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_triggers *triggers; + int ret; + int i; + + BUILD_BUG_ON(ARRAY_SIZE(names) != SCOUTFS_TRIGGER_NR); + + for (i = 0; i < ARRAY_SIZE(names); i++) { + if (WARN_ON(!names[i])) + return -EINVAL; + } + + triggers = kzalloc(sizeof(struct scoutfs_triggers), GFP_KERNEL); + if (!triggers) + return -ENOMEM; + + sbi->triggers = triggers; + + triggers->dir = debugfs_create_dir("trigger", sbi->debug_root); + if (!triggers->dir) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(triggers->atomics); i++) { + if (!debugfs_create_atomic_t(names[i], 0644, triggers->dir, + &triggers->atomics[i])) { + ret = -ENOMEM; + goto out; + } + } + + ret = 0; +out: + if (ret) + scoutfs_destroy_triggers(sb); + return ret; +} + +void scoutfs_destroy_triggers(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_triggers *triggers = sbi->triggers; + + if (triggers) { + if (triggers->dir) + debugfs_remove_recursive(triggers->dir); + kfree(triggers); + sbi->triggers = NULL; + } +} diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h new file mode 100644 index 00000000..48920844 --- /dev/null +++ b/kmod/src/triggers.h @@ -0,0 +1,17 @@ +#ifndef _SCOUTFS_TRIGGERS_H_ +#define _SCOUTFS_TRIGGERS_H_ + +enum { + SCOUTFS_TRIGGER_SOMETHING, + SCOUTFS_TRIGGER_NR, +}; + +bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t); + +#define scoutfs_trigger(sb, which) \ + scoutfs_trigger_test_and_clear(sb, SCOUTFS_TRIGGER_##which) + +int scoutfs_setup_triggers(struct super_block *sb); +void scoutfs_destroy_triggers(struct super_block *sb); + +#endif From e354fd18b1a4c1745946691f71e9f199f722ee03 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Dec 2017 16:51:04 -0800 Subject: [PATCH 538/920] scoutfs: add sysfs.c, fsid file I wanted to add a sysfs file that exports the fsid for the mount of a given device. But our use of sysfs was confusing and spread through super.c and counters.c. This moves the core of our sysfs use to sysfs.c. Instead of defining the per-mount dir as a kset we define it as an object with attributes which gives us a place to add an fsid attribute. counters still have their own whack of sysfs implementation. We'll let it keep it for now but we could move it into sysfs.c. It's just counter interation around the insane sysfs obj/attr/type nonsense. For now it just needs to know to add its counters dir as a child of the per-mount dir instead of adding it to the kset. Signed-off-by: Zach Brown --- kmod/src/Makefile | 4 +- kmod/src/counters.c | 4 +- kmod/src/super.c | 23 +++---- kmod/src/super.h | 5 +- kmod/src/sysfs.c | 160 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/sysfs.h | 12 ++++ 6 files changed, 186 insertions(+), 22 deletions(-) create mode 100644 kmod/src/sysfs.c create mode 100644 kmod/src/sysfs.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index ad177eb2..d8eecdf5 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -8,8 +8,8 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ dlmglue.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ manifest.o msg.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o trans.o \ - triggers.o xattr.o + scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o sysfs.o \ + trans.o triggers.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/counters.c b/kmod/src/counters.c index 1a3b1fee..5578ae26 100644 --- a/kmod/src/counters.c +++ b/kmod/src/counters.c @@ -16,6 +16,7 @@ #include #include "super.h" +#include "sysfs.h" #include "counters.h" /* @@ -85,10 +86,9 @@ int scoutfs_setup_counters(struct super_block *sb) goto out; } - counters->kobj.kset = sbi->kset; init_completion(&counters->comp); ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype, - NULL, "counters"); + scoutfs_sysfs_sb_dir(sb), "counters"); out: if (ret) { /* tear down partial to avoid destroying null kobjs */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 6c12f59b..9772a8a4 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -41,9 +41,9 @@ #include "client.h" #include "server.h" #include "options.h" +#include "sysfs.h" #include "scoutfs_trace.h" -static struct kset *scoutfs_kset; static struct dentry *scoutfs_debugfs_root; /* @@ -129,8 +129,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_destroy_triggers(sb); debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); - if (sbi->kset) - kset_unregister(sbi->kset); + scoutfs_destroy_sysfs(sb); kfree(sbi); sb->s_fs_info = NULL; @@ -310,12 +309,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - /* XXX can have multiple mounts of a device, need mount id */ - sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); - if (!sbi->kset) { - ret = -ENOMEM; - goto out; - } ret = scoutfs_parse_options(sb, data, &opts); if (ret) @@ -323,7 +316,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->opts = opts; - ret = scoutfs_setup_counters(sb) ?: + ret = scoutfs_setup_sysfs(sb) ?: + scoutfs_setup_counters(sb) ?: scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: scoutfs_setup_triggers(sb) ?: @@ -398,8 +392,7 @@ static void teardown_module(void) debugfs_remove(scoutfs_debugfs_root); scoutfs_dir_exit(); scoutfs_inode_exit(); - if (scoutfs_kset) - kset_unregister(scoutfs_kset); + scoutfs_sysfs_exit(); } static int __init scoutfs_module_init(void) @@ -421,9 +414,9 @@ static int __init scoutfs_module_init(void) if (ret) return ret; - scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); - if (!scoutfs_kset) - return -ENOMEM; + ret = scoutfs_sysfs_init(); + if (ret) + return ret; scoutfs_debugfs_root = debugfs_create_dir("scoutfs", NULL); if (!scoutfs_debugfs_root) { diff --git a/kmod/src/super.h b/kmod/src/super.h index 0955d722..03d5ac7f 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -20,6 +20,7 @@ struct client_info; struct server_info; struct inode_sb_info; struct btree_info; +struct sysfs_info; struct scoutfs_sb_info { struct super_block *sb; @@ -57,9 +58,7 @@ struct scoutfs_sb_info { struct lock_info *lock_info; struct client_info *client_info; struct server_info *server_info; - - /* $sysfs/fs/scoutfs/$id/ */ - struct kset *kset; + struct sysfs_info *sfsinfo; struct scoutfs_counters *counters; struct scoutfs_triggers *triggers; diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c new file mode 100644 index 00000000..f043481e --- /dev/null +++ b/kmod/src/sysfs.c @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "super.h" +#include "sysfs.h" + +static struct kset *scoutfs_kset; + +struct sysfs_info { + struct super_block *sb; + struct kobject sb_id_kobj; + struct completion sb_id_comp; +}; + +#define KOBJ_TO_SB(kobj, which) \ + container_of(kobj, struct sysfs_info, which)->sb + +struct attr_funcs { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); +}; + +#define ATTR_FUNCS_RO(_name) \ + static struct attr_funcs _name##_attr_funcs = __ATTR_RO(_name) + +static ssize_t fsid_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return snprintf(buf, PAGE_SIZE, "%llx\n", le64_to_cpu(super->hdr.fsid)); +} +ATTR_FUNCS_RO(fsid); + +/* + * ops are defined per type, not per attribute. To have attributes with + * different types that want different funcs we wrap them with a struct + * that has per-type funcs. + */ +static ssize_t attr_funcs_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct attr_funcs *af = container_of(attr, struct attr_funcs, attr); + + return af->show(kobj, attr, buf); +} + +#define KTYPE(_name) \ + static void _name##_release(struct kobject *kobj) \ + { \ + struct sysfs_info *sfsinfo; \ + \ + sfsinfo = container_of(kobj, struct sysfs_info, _name##_kobj);\ + \ + complete(&sfsinfo->_name##_comp); \ + } \ + static const struct sysfs_ops _name##_sysfs_ops = { \ + .show = attr_funcs_show, \ + }; \ + \ + static struct kobj_type _name##_ktype = { \ + .default_attrs = _name##_attrs, \ + .sysfs_ops = &_name##_sysfs_ops, \ + .release = _name##_release, \ + }; + + +static struct attribute *sb_id_attrs[] = { + &fsid_attr_funcs.attr, + NULL, +}; +KTYPE(sb_id); + +struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb) +{ + struct sysfs_info *sfsinfo = SCOUTFS_SB(sb)->sfsinfo; + + return &sfsinfo->sb_id_kobj; +} + +static void kobj_del_put_wait(struct kobject *kobj, struct completion *comp) +{ + kobject_del(kobj); + kobject_put(kobj); + wait_for_completion(comp); +} + +#define shutdown_kobj(sfinfo, _name) \ + kobj_del_put_wait(&sfsinfo->_name##_kobj, &sfsinfo->_name##_comp) + +/* + * Only the return from kobj_init_and_add() tells us if the kobj needs + * to be cleaned up or not. This must manually clean up the kobjs and + * only leave full cleanup to _destroy_. + */ +int scoutfs_setup_sysfs(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct sysfs_info *sfsinfo; + int ret; + + sfsinfo = kzalloc(sizeof(struct sysfs_info), GFP_KERNEL); + if (!sfsinfo) + return -ENOMEM; + + sfsinfo->sb = sb; + sbi->sfsinfo = sfsinfo; + + /* XXX can have multiple mounts of a device, need mount id */ + init_completion(&sfsinfo->sb_id_comp); + ret = kobject_init_and_add(&sfsinfo->sb_id_kobj, &sb_id_ktype, + &scoutfs_kset->kobj, "%s", sb->s_id); + if (ret) + kfree(sfsinfo); + + return ret; +} + +void scoutfs_destroy_sysfs(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct sysfs_info *sfsinfo = sbi->sfsinfo; + + if (sfsinfo) { + shutdown_kobj(sfsinfo, sb_id); + + kfree(sfsinfo); + sbi->sfsinfo = NULL; + } +} + +int __init scoutfs_sysfs_init(void) +{ + scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); + if (!scoutfs_kset) + return -ENOMEM; + + return 0; +} + +void __exit scoutfs_sysfs_exit(void) +{ + if (scoutfs_kset) + kset_unregister(scoutfs_kset); +} diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h new file mode 100644 index 00000000..0d94b3a4 --- /dev/null +++ b/kmod/src/sysfs.h @@ -0,0 +1,12 @@ +#ifndef _SCOUTFS_SYSFS_H_ +#define _SCOUTFS_SYSFS_H_ + +struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb); + +int scoutfs_setup_sysfs(struct super_block *sb); +void scoutfs_destroy_sysfs(struct super_block *sb); + +int __init scoutfs_sysfs_init(void); +void __exit scoutfs_sysfs_exit(void); + +#endif From 829126790bbbf5ce67ebb7275f17449c84baa349 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Dec 2017 09:46:08 -0800 Subject: [PATCH 539/920] scoutfs: retry stale btree and segment reads We don't have strict consistency protocols protecting the "physical" caches that hold btree blocks and segments. We have metadata that tells a reader that it's hit a stale cached entry and needs to invalidate and read a the current version from the media. This implements the retrying. If we get stale sequence numbers in segments or btree blocks we invalidate them from the cache and return -ESTALE. This can only happen when reading structures that could have been modified remotely. This means btree reads in the clients and segment reads for everyone. btree reads on the server are always consistent because it is the only writer. Adding retrying to item reading and compaction catches all of these cases. Stale reads are triggered by inconsistency. But that could also be persistent corruption in persistent media. Callers need to be careful to turn their retries into hard errors if they're persistent. Item reading can do this because it knows the btree root seq that anchored the walk. Compaction doesn't do this today. That gets addressed in a big sweep of error handling at some point in the not too distant future. Signed-off-by: Zach Brown --- kmod/src/btree.c | 36 +++++++++++++++++------------------- kmod/src/compact.c | 5 ++++- kmod/src/counters.h | 4 ++++ kmod/src/manifest.c | 25 ++++++++++++++++++++++++- kmod/src/seg.c | 18 +++++++++++++----- kmod/src/triggers.c | 4 +++- kmod/src/triggers.h | 4 +++- 7 files changed, 68 insertions(+), 28 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 83b82894..34d4434a 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -24,6 +24,8 @@ #include "key.h" #include "btree.h" #include "sort_priv.h" +#include "counters.h" +#include "triggers.h" #include "scoutfs_trace.h" @@ -603,12 +605,10 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, * * Btree blocks don't have rigid cache consistency. We can be following * a new root to read refs into previously stale cached blocks. If we - * see that the block metadata doesn't match we first assume that we - * just have a stale block and try and re-read it. If it still doesn't - * match we assume that we're an reader racing with a writer overwriting - * old blocks in the ring. We return an error that tells the caller to - * deal with this error: either find a new root or return a hard error - * if the block is really corrupt. + * hit a cached block that doesn't match the ref (or indeed a corrupt + * block) we return -ESTALE which tells the caller to deal with this + * error: either find a new root or return a hard error if the block is + * really corrupt. * * btree callers serialize concurrent writers in a btree but not between * btrees. We have to lock around the shared btree_info. Callers do @@ -626,13 +626,11 @@ static int get_ref_block(struct super_block *sb, int flags, struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_block *new; struct buffer_head *bh; - int retries = 1; u64 blkno; u64 seq; int ret; int i; -retry: /* always get the current block, either to return or cow from */ if (ref && ref->blkno) { bh = sb_bread(sb, le64_to_cpu(ref->blkno)); @@ -642,17 +640,17 @@ retry: } bt = (void *)bh->b_data; - if (!valid_referenced_block(super, ref, bt, bh)) { - if (retries-- > 0) { - lock_buffer(bh); - clear_buffer_uptodate(bh); - unlock_buffer(bh); - put_bh(bh); - bt = NULL; - goto retry; - } - /* XXX let us know when we eventually hit this */ - ret = WARN_ON_ONCE(-ESTALE); + if (!valid_referenced_block(super, ref, bt, bh) || + scoutfs_trigger(sb, BTREE_STALE_READ)) { + + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + put_bh(bh); + bt = NULL; + + scoutfs_inc_counter(sb, btree_stale_read); + ret = -ESTALE; goto out; } diff --git a/kmod/src/compact.c b/kmod/src/compact.c index e7fdd551..428346a0 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -619,7 +619,10 @@ static void scoutfs_compact_func(struct work_struct *work) free_cseg_list(sb, &curs.csegs); free_cseg_list(sb, &results); - WARN_ON_ONCE(ret); + if (ret == -ESTALE) + scoutfs_inc_counter(sb, compact_stale_error); + + WARN_ON_ONCE(ret && ret != -ESTALE); trace_scoutfs_compact_func(sb, ret); } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 67ceba60..c2d3a453 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -14,11 +14,13 @@ #define EXPAND_EACH_COUNTER \ EXPAND_COUNTER(alloc_alloc) \ EXPAND_COUNTER(alloc_free) \ + EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(compact_operations) \ EXPAND_COUNTER(compact_segment_moved) \ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_write_bytes) \ EXPAND_COUNTER(compact_segment_writes) \ + EXPAND_COUNTER(compact_stale_error) \ EXPAND_COUNTER(compact_sticky_upper) \ EXPAND_COUNTER(compact_sticky_written) \ EXPAND_COUNTER(data_end_writeback_page) \ @@ -59,9 +61,11 @@ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_incompat_wait) \ EXPAND_COUNTER(manifest_compact_migrate) \ + EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(seg_shrink) \ + EXPAND_COUNTER(seg_stale_read) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_item_flush) \ diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2b3d59b0..f7cd99e6 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -26,6 +26,7 @@ #include "manifest.h" #include "trans.h" #include "counters.h" +#include "triggers.h" #include "client.h" #include "scoutfs_trace.h" @@ -527,7 +528,6 @@ out: scoutfs_btree_put_iref(&iref); scoutfs_btree_put_iref(&prev); kfree(mkey); - BUG_ON(ret == -ESTALE); /* XXX caller needs to retry or return error */ return ret; } @@ -575,8 +575,10 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_segment *seg; struct manifest_ref *ref; struct manifest_ref *tmp; + __le64 last_root_seq; LIST_HEAD(ref_list); LIST_HEAD(batch); + bool force_hard; u8 found_flags = 0; u8 item_flags; int found_ctr; @@ -604,6 +606,8 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, * either get a manifest ref in the lvb of their lock or they'll * ask the server the first time the system sees the lock. */ + last_root_seq = 0; +retry_stale: ret = scoutfs_client_get_manifest_root(sb, &root); if (ret) goto out; @@ -771,6 +775,25 @@ out: free_ref(sb, ref); } + /* + * Resample the root and retry reads as long as we see + * inconsistent blocks/segments and new roots to read through. + * Persistent inconsistency in the same root is seen as corrupt + * structures instead. + */ + force_hard = scoutfs_trigger(sb, HARD_STALE_ERROR); + if (ret == -ESTALE || force_hard) { + /* keep trying as long as the root changes */ + if ((last_root_seq != root.ref.seq) && !force_hard) { + last_root_seq = root.ref.seq; + goto retry_stale; + } + + /* persistent error */ + scoutfs_inc_counter(sb, manifest_hard_stale_error); + ret = -EIO; + } + return ret; } diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 95624332..cdb79b2b 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -26,6 +26,7 @@ #include "alloc.h" #include "key.h" #include "counters.h" +#include "triggers.h" #include "scoutfs_trace.h" /* @@ -377,6 +378,8 @@ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct segment_cache *cac = sbi->segment_cache; struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + unsigned long flags; + bool erased; int ret; ret = wait_event_interruptible(cac->waitq, @@ -392,12 +395,17 @@ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, sblk = off_ptr(seg, 0); if (WARN_ON_ONCE(segno != le64_to_cpu(sblk->segno)) || - WARN_ON_ONCE(seq != le64_to_cpu(sblk->seq))) { - ret = -ESTALE; - goto out; - } + WARN_ON_ONCE(seq != le64_to_cpu(sblk->seq)) || + scoutfs_trigger(sb, SEG_STALE_READ)) { + spin_lock_irqsave(&cac->lock, flags); + erased = erase_seg(cac, seg); + spin_unlock_irqrestore(&cac->lock, flags); + if (erased) + scoutfs_seg_put(seg); - ret = 0; + scoutfs_inc_counter(sb, seg_stale_read); + ret = -ESTALE; + } out: return ret; } diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c index 64f83941..200d29e7 100644 --- a/kmod/src/triggers.c +++ b/kmod/src/triggers.c @@ -38,7 +38,9 @@ struct scoutfs_triggers { struct scoutfs_triggers *name = SCOUTFS_SB(sb)->triggers static char *names[] = { - [SCOUTFS_TRIGGER_SOMETHING] = "something", + [SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read", + [SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error", + [SCOUTFS_TRIGGER_SEG_STALE_READ] = "seg_stale_read", }; bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t) diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h index 48920844..1b802c5e 100644 --- a/kmod/src/triggers.h +++ b/kmod/src/triggers.h @@ -2,7 +2,9 @@ #define _SCOUTFS_TRIGGERS_H_ enum { - SCOUTFS_TRIGGER_SOMETHING, + SCOUTFS_TRIGGER_BTREE_STALE_READ, + SCOUTFS_TRIGGER_HARD_STALE_ERROR, + SCOUTFS_TRIGGER_SEG_STALE_READ, SCOUTFS_TRIGGER_NR, }; From 5cc05d663ec409270643e2020adfc4208e84a392 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Tue, 19 Dec 2017 13:50:29 -0800 Subject: [PATCH 540/920] scoutfs: count some lock events by type We use a new event callback in dlmglue so that scout has a chance to do some per-lock type counters. I included the most important dlmglue events - basically those which can cost us network or disk traffic. Right now scout just counts downconvert events since those are the most interesting to us. We also just count on the ino and index locks for now. Signed-off-by: Mark Fasheh --- kmod/src/counters.h | 2 ++ kmod/src/dlmglue.c | 18 ++++++++++++++++-- kmod/src/dlmglue.h | 14 ++++++++++++++ kmod/src/lock.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index c2d3a453..f0c02f50 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -60,6 +60,8 @@ EXPAND_COUNTER(lock_busy_wait) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_incompat_wait) \ + EXPAND_COUNTER(lock_type_ino_downconvert) \ + EXPAND_COUNTER(lock_type_idx_downconvert) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(seg_alloc) \ diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 4393f595..ec04af87 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -104,6 +104,13 @@ static inline void lockres_name(struct ocfs2_lock_res *lockres, char *buf, snprintf(buf, len, "%s", lockres->l_name); } +static inline void lockres_notify_event(struct ocfs2_lock_res *lockres, + enum ocfs2_lock_events event) +{ + if (lockres->l_ops->notify_event) + lockres->l_ops->notify_event(lockres, event); +} + static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, int wanted); static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, @@ -1220,10 +1227,13 @@ again: gen = lockres_set_pending(lockres); spin_unlock_irqrestore(&lockres->l_lock, flags); - if (lkm_flags & DLM_LKF_CONVERT) + if (lkm_flags & DLM_LKF_CONVERT) { scoutfs_inc_counter(osb->sb, dlm_convert_request); - else + lockres_notify_event(lockres, EVENT_DLM_CONVERT); + } else { scoutfs_inc_counter(osb->sb, dlm_lock_request); + lockres_notify_event(lockres, EVENT_DLM_LOCK); + } BUG_ON(level == DLM_LOCK_IV); BUG_ON(level == DLM_LOCK_NL); @@ -2161,6 +2171,7 @@ static int ocfs2_drop_lock(struct ocfs2_super *osb, lockres->l_name); scoutfs_inc_counter(osb->sb, dlm_unlock_request); + lockres_notify_event(lockres, EVENT_DLM_UNLOCK); ocfs2_wait_on_busy_lock(lockres); out: @@ -2366,6 +2377,7 @@ static int ocfs2_cancel_convert(struct ocfs2_super *osb, mlog(ML_BASTS, "lockres %s\n", lockres->l_name); scoutfs_inc_counter(osb->sb, dlm_cancel_convert); + lockres_notify_event(lockres, EVENT_DLM_CONVERT); return ret; } @@ -2532,6 +2544,8 @@ recheck: spin_unlock_irqrestore(&lockres->l_lock, flags); + lockres_notify_event(lockres, EVENT_DLM_DOWNCONVERT_WORK); + ctl->unblock_action = lockres->l_ops->downconvert_worker(lockres, blocking); if (ctl->unblock_action == UNBLOCK_STOP_POST) { diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index d6b8c639..d4cb298e 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -199,6 +199,14 @@ enum ocfs2_unblock_action { * ->post_unlock() callback. */ }; +enum ocfs2_lock_events { + EVENT_DLM_LOCK = 0, + EVENT_DLM_UNLOCK, + EVENT_DLM_CONVERT, + EVENT_DLM_CANCEL_CONVERT, + EVENT_DLM_DOWNCONVERT_WORK, +}; + /* * OCFS2 Lock Resource Operations * @@ -269,6 +277,12 @@ struct ocfs2_lock_res_ops { */ void (*print)(struct ocfs2_lock_res *, char *, unsigned int); + /* + * Optional: Lightweight event callback, intended for quick + * operations like collecting stats, etc. + */ + void (*notify_event)(struct ocfs2_lock_res *, enum ocfs2_lock_events); + /* * LOCK_TYPE_* flags which describe the specific requirements * of a lock type. Descriptions of each individual flag follow. diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 9c8e0ccf..b16b2264 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -332,17 +332,45 @@ static void lock_name_string(struct ocfs2_lock_res *lockres, char *buf, snprintf(buf, len, LN_FMT, LN_ARG(&lock->lock_name)); } +static void count_ino_lock_event(struct ocfs2_lock_res *lockres, + enum ocfs2_lock_events event) +{ + struct scoutfs_lock *lock = container_of(lockres, struct scoutfs_lock, + lockres); + struct super_block *sb = lock->sb; + + if (event == EVENT_DLM_DOWNCONVERT_WORK) + scoutfs_inc_counter(sb, lock_type_ino_downconvert); +} + +static void count_idx_lock_event(struct ocfs2_lock_res *lockres, + enum ocfs2_lock_events event) +{ + struct scoutfs_lock *lock = container_of(lockres, struct scoutfs_lock, + lockres); + struct super_block *sb = lock->sb; + + /* + * Treat all indicies together. Later we can decode the + * lockres name to get at specific indicies. + */ + if (event == EVENT_DLM_DOWNCONVERT_WORK) + scoutfs_inc_counter(sb, lock_type_idx_downconvert); +} + static struct ocfs2_lock_res_ops scoufs_ino_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, + .notify_event = count_ino_lock_event, .flags = LOCK_TYPE_REQUIRES_REFRESH, }; static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, + .notify_event = count_idx_lock_event, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, }; From 3661f06bec593627d54ff05650b60833f9dd71b2 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Wed, 20 Dec 2017 17:04:25 -0800 Subject: [PATCH 541/920] scoutfs: add trigger to drop lock cache We have a corruption that can happen when a lock is reclaimed but it's cache is still dirty. Detect this corruption by placing a trigger in statfs which fires off lock reclaim. Statfs is nice because for scoutfs it's lockless, which means there should not be any references on locks when the trigger is fired. Signed-off-by: Mark Fasheh --- kmod/src/lock.c | 25 +++++++++++++++---------- kmod/src/lock.h | 2 ++ kmod/src/super.c | 8 ++++++++ kmod/src/triggers.c | 1 + kmod/src/triggers.h | 1 + 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index b16b2264..32d7deb6 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -29,6 +29,7 @@ #include "trans.h" #include "counters.h" #include "endian_swap.h" +#include "triggers.h" #define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ @@ -592,19 +593,12 @@ static void scoutfs_lock_reclaim(struct work_struct *work) put_scoutfs_lock(linfo->sb, lock); } -static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) +void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) { - struct lock_info *linfo = container_of(shrink, struct lock_info, - shrinker); + struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; struct scoutfs_lock *lock; struct scoutfs_lock *tmp; unsigned long flags; - unsigned long nr; - int ret; - - nr = sc->nr_to_scan; - if (!nr) - goto out; spin_lock_irqsave(&linfo->lock, flags); list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_entry) { @@ -622,8 +616,19 @@ static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) queue_work(linfo->lock_reclaim_wq, &lock->reclaim_work); } spin_unlock_irqrestore(&linfo->lock, flags); +} + +static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) +{ + struct lock_info *linfo = container_of(shrink, struct lock_info, + shrinker); + unsigned long nr; + int ret; + + nr = sc->nr_to_scan; + if (nr) + scoutfs_free_unused_locks(linfo->sb, nr); -out: ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); trace_scoutfs_lock_shrink_exit(linfo->sb, sc->nr_to_scan, ret); return ret; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 63a914fe..6d0339e8 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -60,6 +60,8 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, int level, int flags); +void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); + int scoutfs_lock_setup(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index 9772a8a4..d39c3dbd 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -88,6 +88,14 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_frsize = SCOUTFS_BLOCK_SIZE; /* the vfs fills f_flags */ + /* + * We don't take cluster locks in statfs which makes it a very + * convenient place to trigger lock reclaim for debugging. We + * try to free as many locks as possible. + */ + if (scoutfs_trigger(sb, STATFS_LOCK_PURGE)) + scoutfs_free_unused_locks(sb, -1UL); + return 0; } diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c index 200d29e7..67cef0e6 100644 --- a/kmod/src/triggers.c +++ b/kmod/src/triggers.c @@ -41,6 +41,7 @@ static char *names[] = { [SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read", [SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error", [SCOUTFS_TRIGGER_SEG_STALE_READ] = "seg_stale_read", + [SCOUTFS_TRIGGER_STATFS_LOCK_PURGE] = "statfs_lock_purge", }; bool scoutfs_trigger_test_and_clear(struct super_block *sb, unsigned int t) diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h index 1b802c5e..900e433f 100644 --- a/kmod/src/triggers.h +++ b/kmod/src/triggers.h @@ -5,6 +5,7 @@ enum { SCOUTFS_TRIGGER_BTREE_STALE_READ, SCOUTFS_TRIGGER_HARD_STALE_ERROR, SCOUTFS_TRIGGER_SEG_STALE_READ, + SCOUTFS_TRIGGER_STATFS_LOCK_PURGE, SCOUTFS_TRIGGER_NR, }; From afc798599f62d616b6ac6528cdfbf00aeb45cd4d Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 21 Dec 2017 10:40:07 -0800 Subject: [PATCH 542/920] scoutfs: invalidate cache when we free locks We weren't invalidating our cache before freeing locks due to memory pressure. This would cause stale data on the node which originally held the lock. Fix this by firing a callback from dlmglue before we free a lock from the system. On the scoutfs side, the callback is wired to call our invalidate function. This will ensure that the right data and metadata hit disk before another node is allowed to acquire that lock. Signed-off-by: Mark Fasheh --- kmod/src/dlmglue.c | 4 ++++ kmod/src/dlmglue.h | 12 ++++++++++++ kmod/src/lock.c | 18 ++++++++++++++++++ kmod/src/super.c | 2 ++ kmod/src/super.h | 2 ++ 5 files changed, 38 insertions(+) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index ec04af87..29e32112 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -2255,6 +2255,10 @@ void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, trace_ocfs2_simple_drop_lockres(osb, lockres); ocfs2_mark_lockres_freeing(osb, lockres); + + if (lockres->l_ops->drop_worker) + lockres->l_ops->drop_worker(lockres); + ret = ocfs2_drop_lock(osb, lockres); if (ret) mlog_errno(ret); diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h index d4cb298e..7a51c528 100644 --- a/kmod/src/dlmglue.h +++ b/kmod/src/dlmglue.h @@ -272,6 +272,18 @@ struct ocfs2_lock_res_ops { */ int (*downconvert_worker)(struct ocfs2_lock_res *, int); + /* + * Called before we free a lock from the system. This allows + * the filesystem to sync and invalidate caches before that + * happens. The concept is identical to ->downconvert_worker + * except for two exceptions: + * - The FS must do the full downconvert work - as if it were + * blocking an EX. + * - We do not return an ocfs2_unblock_action - this worker is not + * allowed to delay dropping of the lock. + */ + void (*drop_worker)(struct ocfs2_lock_res *); + /* * Optional: pretty print the lockname into a buffer */ diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 32d7deb6..e106d08b 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -325,6 +325,21 @@ static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) return UNBLOCK_CONTINUE; } +static void ino_lock_drop(struct ocfs2_lock_res *lockres) +{ + struct scoutfs_lock *lock = lockres->l_priv; + struct super_block *sb = lock->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + /* + * Locks get shut down near the end of our unmount process. By + * now everything that needs to be synced or invalidated, has + * been. + */ + if (!sbi->shutdown) + invalidate_caches(sb, DLM_LOCK_EX, lock); +} + static void lock_name_string(struct ocfs2_lock_res *lockres, char *buf, unsigned int len) { @@ -362,6 +377,7 @@ static void count_idx_lock_event(struct ocfs2_lock_res *lockres, static struct ocfs2_lock_res_ops scoufs_ino_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, + .drop_worker = ino_lock_drop, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, .notify_event = count_ino_lock_event, @@ -371,6 +387,7 @@ static struct ocfs2_lock_res_ops scoufs_ino_lops = { static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { .get_osb = get_ino_lock_osb, .downconvert_worker = ino_lock_downconvert, + .drop_worker = ino_lock_drop, .notify_event = count_idx_lock_event, /* XXX: .check_downconvert that queries the item cache for dirty items */ .print = lock_name_string, @@ -387,6 +404,7 @@ static struct ocfs2_lock_res_ops scoutfs_node_id_lops = { .get_osb = get_ino_lock_osb, /* XXX: .check_downconvert that queries the item cache for dirty items */ .downconvert_worker = ino_lock_downconvert, + .drop_worker = ino_lock_drop, .print = lock_name_string, .flags = 0, }; diff --git a/kmod/src/super.c b/kmod/src/super.c index d39c3dbd..b0c5f6af 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -119,6 +119,8 @@ static void scoutfs_put_super(struct super_block *sb) trace_scoutfs_put_super(sb); + sbi->shutdown = true; + scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, SCOUTFS_LKF_NO_TASK_REF); sbi->node_id_lock = NULL; diff --git a/kmod/src/super.h b/kmod/src/super.h index 03d5ac7f..22ef0b62 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -66,6 +66,8 @@ struct scoutfs_sb_info { struct mount_options opts; struct dentry *debug_root; + + bool shutdown; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) From fb6c128503f0d80f33576e90ba8eff8f07d3af51 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Jan 2018 11:03:35 -0800 Subject: [PATCH 543/920] scoutfs: move unblock_lock trace under lock It samples fields that are only consistent under the lock. We also want to see the fields every time it rechecks the conditions that stop it from downconverting. Signed-off-by: Zach Brown --- kmod/src/dlmglue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c index 29e32112..d2724322 100644 --- a/kmod/src/dlmglue.c +++ b/kmod/src/dlmglue.c @@ -2398,11 +2398,12 @@ static int ocfs2_unblock_lock(struct ocfs2_super *osb, int set_lvb = 0; unsigned int gen; - trace_ocfs2_unblock_lock(osb, lockres); spin_lock_irqsave(&lockres->l_lock, flags); recheck: + trace_ocfs2_unblock_lock(osb, lockres); + /* * Is it still blocking? If not, we have no more work to do. */ From a9c7511c8bb1b82ce17a542c844d9097f0a4f917 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Jan 2018 11:03:58 -0800 Subject: [PATCH 544/920] scoutfs: add more scoutfs lock tracing fields We were missing the blocking level and count of cw holders. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7d3901dc..a70a9ebe 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1599,7 +1599,9 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(unsigned int, refcnt) __field(unsigned int, users) __field(unsigned char, level) - __field(unsigned int, ro) + __field(unsigned char, blocking) + __field(unsigned int, cw) + __field(unsigned int, pr) __field(unsigned int, ex) ), TP_fast_assign( @@ -1614,14 +1616,17 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->users = lck->users; /* racey, but safe refs of embedded struct */ __entry->level = lck->lockres.l_level; - __entry->ro = lck->lockres.l_ro_holders; + __entry->blocking = lck->lockres.l_blocking; + __entry->cw = lck->lockres.l_cw_holders; + __entry->pr = lck->lockres.l_ro_holders; __entry->ex = lck->lockres.l_ex_holders; ), - TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu seq %u refs %d users %d level %u ro %u ex %u", + TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu seq %u refs %d users %d level %u blocking %u cw %u pr %u ex %u", __entry->fsid, __entry->name_scope, __entry->name_zone, __entry->name_type, __entry->name_first, __entry->name_second, __entry->seq, __entry->refcnt, - __entry->users, __entry->level, __entry->ro, __entry->ex) + __entry->users, __entry->level, __entry->blocking, + __entry->cw, __entry->pr, __entry->ex) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, From f54e59eef1df177e841a75230f13bd96726aa514 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Jan 2018 12:45:31 -0800 Subject: [PATCH 545/920] scoutfs: add debugfs "locks" for scoutfs_lock Add a file for showing the scoutfs_lock struct contents. This is the layer above the detailed dlmglue/dlm info provided in the existing "locking_state" file. Signed-off-by: Zach Brown --- kmod/src/lock.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/lock.h | 1 + 2 files changed, 123 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e106d08b..128afa96 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -17,6 +17,9 @@ #include #include #include +#include +#include +#include #include "super.h" #include "lock.h" @@ -55,6 +58,8 @@ struct lock_info { struct list_head lru_list; unsigned long long lru_nr; struct workqueue_struct *lock_reclaim_wq; + struct dentry *debug_locks_dentry; + struct idr debug_locks_idr; }; #define DECLARE_LOCK_INFO(sb, name) \ @@ -283,6 +288,9 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) RB_CLEAR_NODE(&lock->range_node); } list_del(&lock->lru_entry); + if (lock->debug_locks_id) + idr_remove(&linfo->debug_locks_idr, + lock->debug_locks_id); spin_unlock(&linfo->lock); ocfs2_simple_drop_lockres(&linfo->dlmglue, &lock->lockres); @@ -418,6 +426,7 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; + int id; if (WARN_ON_ONCE(!!start != !!end)) return NULL; @@ -426,6 +435,18 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, if (lock == NULL) return NULL; + idr_preload(GFP_NOFS); + spin_lock(&linfo->lock); + id = idr_alloc(&linfo->debug_locks_idr, lock, 1, INT_MAX, GFP_NOWAIT); + if (id > 0) + lock->debug_locks_id = id; + spin_unlock(&linfo->lock); + idr_preload_end(); + if (id <= 0) { + free_scoutfs_lock(lock); + return NULL; + } + RB_CLEAR_NODE(&lock->node); RB_CLEAR_NODE(&lock->range_node); @@ -1106,6 +1127,7 @@ static int init_lock_info(struct super_block *sb) spin_lock_init(&linfo->lock); INIT_LIST_HEAD(&linfo->lru_list); + idr_init(&linfo->debug_locks_idr); linfo->shrinker.shrink = shrink_lock_tree; linfo->shrinker.seeks = DEFAULT_SEEKS; register_shrinker(&linfo->shrinker); @@ -1132,6 +1154,9 @@ void scoutfs_lock_destroy(struct super_block *sb) DECLARE_LOCK_INFO(sb, linfo); if (linfo) { + /* XXX does anything synchronize with open debugfs fds? */ + debugfs_remove(linfo->debug_locks_dentry); + unregister_shrinker(&linfo->shrinker); if (linfo->lock_reclaim_wq) destroy_workqueue(linfo->lock_reclaim_wq); @@ -1140,6 +1165,7 @@ void scoutfs_lock_destroy(struct super_block *sb) * draining the reclaim workqueue. */ free_lock_tree(sb); + idr_destroy(&linfo->debug_locks_idr); if (linfo->dlmglue_online) { /* @@ -1161,6 +1187,94 @@ void scoutfs_lock_destroy(struct super_block *sb) } } +/* _stop is always called no matter what start returns */ +static void *scoutfs_debug_locks_seq_start(struct seq_file *m, loff_t *pos) + __acquires(linfo->lock) +{ + struct super_block *sb = m->private; + DECLARE_LOCK_INFO(sb, linfo); + int id; + + spin_lock(&linfo->lock); + + if (*pos >= INT_MAX) + return NULL; + + id = *pos; + return idr_get_next(&linfo->debug_locks_idr, &id); +} + +static void *scoutfs_debug_locks_seq_next(struct seq_file *m, void *v, + loff_t *pos) +{ + struct super_block *sb = m->private; + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock = v; + int id; + + id = lock->debug_locks_id + 1; + lock = idr_get_next(&linfo->debug_locks_idr, &id); + if (lock) + *pos = lock->debug_locks_id; + return lock; +} + +static void scoutfs_debug_locks_seq_stop(struct seq_file *m, void *v) + __releases(linfo->lock) +{ + struct super_block *sb = m->private; + DECLARE_LOCK_INFO(sb, linfo); + + spin_unlock(&linfo->lock); +} + +/* print an upper or lower case char depending on if the flag is set */ +#define locks_flag_char(lock, nr, c) \ + (test_bit(nr, &(lock)->flags) ? c : tolower(c)) + +#define locks_flags(lock) \ + locks_flag_char(lock, SCOUTFS_LOCK_RECLAIM, 'R'), \ + locks_flag_char(lock, SCOUTFS_LOCK_DROPPED, 'D') + +static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) +{ + struct scoutfs_lock *lock = v; + + SK_PCPU(seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" sequence %u refcnt %u users %u flags %c%c\n", + LN_ARG(&lock->lock_name), SK_ARG(lock->start), + SK_ARG(lock->end), lock->sequence, lock->refcnt, + lock->users, locks_flags(lock))); + + return 0; +} + +static const struct seq_operations scoutfs_debug_locks_seq_ops = { + .start = scoutfs_debug_locks_seq_start, + .next = scoutfs_debug_locks_seq_next, + .stop = scoutfs_debug_locks_seq_stop, + .show = scoutfs_debug_locks_seq_show, +}; + +static int scoutfs_debug_locks_open(struct inode *inode, struct file *file) +{ + struct seq_file *m; + int ret; + + ret = seq_open(file, &scoutfs_debug_locks_seq_ops); + if (ret == 0) { + m = file->private_data; + m->private = inode->i_private; + } + return ret; +} + +static const struct file_operations scoutfs_debug_locks_fops = { + .open = scoutfs_debug_locks_open, + .release = seq_release, + .read = seq_read, + .llseek = seq_lseek, +}; + int scoutfs_lock_setup(struct super_block *sb) { struct lock_info *linfo; @@ -1172,6 +1286,14 @@ int scoutfs_lock_setup(struct super_block *sb) return ret; linfo = sbi->lock_info; + linfo->debug_locks_dentry = debugfs_create_file("locks", + S_IFREG|S_IRUSR, sbi->debug_root, sb, + &scoutfs_debug_locks_fops); + if (!linfo->debug_locks_dentry) { + ret = -ENOMEM; + goto out; + } + linfo->lock_reclaim_wq = alloc_workqueue("scoutfs_reclaim", WQ_UNBOUND|WQ_HIGHPRI, 0); if (!linfo->lock_reclaim_wq) { diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 6d0339e8..75510b49 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -25,6 +25,7 @@ struct scoutfs_lock { struct rb_node node; struct rb_node range_node; unsigned int refcnt; + unsigned int debug_locks_id; struct ocfs2_lock_res lockres; struct list_head lru_entry; struct work_struct reclaim_work; From e803b10bca1539aed13a1ab8a9cf540a5d120347 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 12 Jan 2018 12:53:23 -0800 Subject: [PATCH 546/920] scoutfs: drop lock refcnt/users under task ref If scoutfs_unlock() sees that it isn't the last task using a lock it just returns. It doesn't unlock the lock and it doesn't drop the lock refcnt and users. This leaks the lock refcnt and users because find_alloc_scoutfs_lock() always increments them when it finds a lock. Inflated counts will stop the shrinker from freeing the locks and eventually the counts will wrap and could cause locks to be freed while they're still in use. We can either always drop the refcnt/users in unlock or we can drop them in lock as we notice that our task already has the lock. I chose to have the task ref hold one refcnt/users which are only dropped as the final task unlocks. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 128afa96..4d347c7b 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -732,6 +732,8 @@ static int lock_name_keys(struct super_block *sb, int mode, int flags, */ BUG_ON(!ocfs2_levels_compat(&lock->lockres, mode)); get_task_ref(ref); + dec_lock_users(lock); + put_scoutfs_lock(sb, lock); ret = 0; goto out; } From b015927e7bb700caac9bd95376513db9345f46ef Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 25 Jan 2018 16:01:52 -0800 Subject: [PATCH 547/920] scoutfs: add debug check for scout-107 We have a bug filed where the fs got stuck spinning in scoutfs_dir_get_backref_path(). There's been enough changes lately that we're not sure if this issue still exists. Catch if we have an excessive number of iterations through our loop there and exit with some debug info. Signed-off-by: Mark Fasheh --- kmod/src/dir.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 1c9a8b8c..12a4d90f 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -24,6 +24,7 @@ #include "inode.h" #include "ioctl.h" #include "key.h" +#include "msg.h" #include "super.h" #include "trans.h" #include "xattr.h" @@ -1209,8 +1210,20 @@ int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, { u64 par_ino; int ret; + int iters = 0; retry: + /* + * Debugging for SCOUT-107, can be removed later when we're + * confident we won't hit an endless loop here again. + */ + if (WARN_ONCE(++iters >= 4000, "scoutfs: Excessive retries in " + "dir_get_backref_path. ino %llu dir_ino %llu name %.*s\n", + ino, dir_ino, name_len, name)) { + ret = -EINVAL; + goto out; + } + /* get the next link name to the given inode */ ret = add_next_linkref(sb, ino, dir_ino, name, name_len, list); if (ret < 0) From ac09f03327eb8db3e96a7148e9a2dc1e4dad3ac7 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Thu, 4 Jan 2018 17:01:04 -0800 Subject: [PATCH 548/920] scoutfs: open by handle This is implemented by filling in our export ops functions. When we get those right, the VFS handles most of the details for us. Internally, scoutfs handles are two u64's (ino and parent ino) and a type which indicates whether the handle contains the parent ino or not. Surpisingly enough, no existing type matches this pattern so we use our own types to identify the handle. Most of the export ops are self explanatory scoutfs_encode_fh() takes an inode and an optional parent and encodes those into the smallest handle that would fit. scoutfs_fh_to_[dentry|parent] turn an existing file handle into a dentry. scoutfs_get_parent() is a bit different and would be called on directory inodes to connect a disconnected dentry path. For scoutfs_get_parent(), we can export add_next_linkref() and use the backref mechanism to quickly find a parent directory. scoutfs_get_name() is almost identical to scoutfs_get_parent(). Here we're linking an inode to a name which exists in the parent directory. We can also use add_next_linkref, and simply copy the name from the backref. As a result of this patch we can also now export scoutfs file systems via NFS, however testing NFS thoroughly is outside the scope of this work so export support should be considered experimental at best. Signed-off-by: Mark Fasheh [zab edited <= NAME_MAX] --- kmod/src/Makefile | 4 +- kmod/src/dir.c | 12 +-- kmod/src/dir.h | 4 + kmod/src/export.c | 168 +++++++++++++++++++++++++++++++++++++++ kmod/src/export.h | 8 ++ kmod/src/format.h | 12 +++ kmod/src/scoutfs_trace.h | 84 ++++++++++++++++++++ kmod/src/super.c | 2 + 8 files changed, 287 insertions(+), 7 deletions(-) create mode 100644 kmod/src/export.c create mode 100644 kmod/src/export.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index d8eecdf5..e44808c5 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,8 +6,8 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - dlmglue.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ - manifest.o msg.o options.o per_task.o seg.o server.o \ + dlmglue.o export.o file.o kvec.o inode.o ioctl.o item.o key.o \ + lock.o manifest.o msg.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o sysfs.o \ trans.o triggers.o xattr.o diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 12a4d90f..07f3c062 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1089,9 +1089,9 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino, * Callers are comfortable with the race inherent to incrementally * building up a path with individual locked backref item lookups. */ -static int add_next_linkref(struct super_block *sb, u64 ino, - u64 dir_ino, char *name, unsigned int name_len, - struct list_head *list) +int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, + u64 dir_ino, char *name, unsigned int name_len, + struct list_head *list) { struct scoutfs_link_backref_key last_lbkey; struct scoutfs_link_backref_entry *ent; @@ -1225,7 +1225,8 @@ retry: } /* get the next link name to the given inode */ - ret = add_next_linkref(sb, ino, dir_ino, name, name_len, list); + ret = scoutfs_dir_add_next_linkref(sb, ino, dir_ino, name, name_len, + list); if (ret < 0) goto out; @@ -1233,7 +1234,8 @@ retry: par_ino = first_backref_dir_ino(list); while (par_ino != SCOUTFS_ROOT_INO) { - ret = add_next_linkref(sb, par_ino, 0, NULL, 0, list); + ret = scoutfs_dir_add_next_linkref(sb, par_ino, 0, NULL, 0, + list); if (ret < 0) { if (ret == -ENOENT) { /* restart if there was no parent component */ diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 1a17fd70..79aaaa2a 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -20,6 +20,10 @@ int scoutfs_dir_get_backref_path(struct super_block *sb, u64 target_ino, void scoutfs_dir_free_backref_path(struct super_block *sb, struct list_head *list); +int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, + u64 dir_ino, char *name, unsigned int name_len, + struct list_head *list); + int scoutfs_symlink_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock, u64 i_size); diff --git a/kmod/src/export.c b/kmod/src/export.c new file mode 100644 index 00000000..90c14cb9 --- /dev/null +++ b/kmod/src/export.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include + +#include "export.h" +#include "inode.h" +#include "dir.h" +#include "format.h" +#include "scoutfs_trace.h" + +/* describe the length of the fileid type in terms of number of u32's used. */ +static int scoutfs_fileid_len(int fh_type) +{ + switch (fh_type) { + case FILEID_SCOUTFS: + return 2; + case FILEID_SCOUTFS_WITH_PARENT: + return 4; + } + return FILEID_INVALID; +} + +static bool scoutfs_valid_fileid(int fh_type) +{ + return scoutfs_fileid_len(fh_type) != FILEID_INVALID; +} + +static int scoutfs_encode_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *parent) +{ + struct scoutfs_fid *fid = (struct scoutfs_fid *)fh; + int fh_type = FILEID_SCOUTFS; + int len; + + if (parent) + fh_type = FILEID_SCOUTFS_WITH_PARENT; + + len = scoutfs_fileid_len(fh_type); + + if (*max_len < len) { + *max_len = len; + return FILEID_INVALID; + } + *max_len = len; + + fid->ino = cpu_to_le64(scoutfs_ino(inode)); + if (parent) + fid->parent_ino = cpu_to_le64(scoutfs_ino(parent)); + + trace_scoutfs_encode_fh(inode->i_sb, fh_type, fid); + + return fh_type; +} + +static struct dentry *scoutfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + struct scoutfs_fid *sfid = (struct scoutfs_fid *)fid; + struct inode *inode = NULL; + + if (fh_len < scoutfs_fileid_len(fh_type)) + return NULL; + + trace_scoutfs_fh_to_dentry(sb, fh_type, sfid); + + if (scoutfs_valid_fileid(fh_type)) + inode = scoutfs_iget(sb, le64_to_cpu(sfid->ino)); + + return d_obtain_alias(inode); +} + +static struct dentry *scoutfs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + struct scoutfs_fid *sfid = (struct scoutfs_fid *)fid; + struct inode *inode = NULL; + + if (fh_len < scoutfs_fileid_len(fh_type)) + return NULL; + + trace_scoutfs_fh_to_parent(sb, fh_type, sfid); + + if (scoutfs_valid_fileid(fh_type) && + fh_type == FILEID_SCOUTFS_WITH_PARENT) + inode = scoutfs_iget(sb, le64_to_cpu(sfid->parent_ino)); + + return d_obtain_alias(inode); +} + +static struct dentry *scoutfs_get_parent(struct dentry *child) +{ + struct inode *inode = child->d_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_link_backref_entry *ent; + LIST_HEAD(list); + int ret; + u64 ino; + + ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), 0, NULL, 0, + &list); + if (ret) + return ERR_PTR(ret); + + ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); + ino = be64_to_cpu(ent->lbkey.dir_ino); + scoutfs_dir_free_backref_path(sb, &list); + trace_scoutfs_get_parent(sb, inode, ino); + + inode = scoutfs_iget(sb, ino); + + return d_obtain_alias(inode); +} + +static int scoutfs_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + u64 dir_ino = scoutfs_ino(parent->d_inode); + struct scoutfs_link_backref_entry *ent; + struct inode *inode = child->d_inode; + struct super_block *sb = inode->i_sb; + LIST_HEAD(list); + int ret; + + ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), dir_ino, + NULL, 0, &list); + if (ret) + return ret; + + ret = -ENOENT; + ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); + if (be64_to_cpu(ent->lbkey.ino) == scoutfs_ino(inode) && + be64_to_cpu(ent->lbkey.dir_ino) == dir_ino && + ent->name_len <= NAME_MAX) { + memcpy(name, ent->lbkey.name, ent->name_len); + name[ent->name_len] = '\0'; + ret = 0; + trace_scoutfs_get_name(sb, parent->d_inode, inode, name); + } + scoutfs_dir_free_backref_path(sb, &list); + + return ret; +} + +const struct export_operations scoutfs_export_ops = { + .encode_fh = scoutfs_encode_fh, + .fh_to_dentry = scoutfs_fh_to_dentry, + .fh_to_parent = scoutfs_fh_to_parent, + .get_parent = scoutfs_get_parent, + .get_name = scoutfs_get_name, +}; diff --git a/kmod/src/export.h b/kmod/src/export.h new file mode 100644 index 00000000..7ed9771a --- /dev/null +++ b/kmod/src/export.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_EXPORT_H_ +#define _SCOUTFS_EXPORT_H_ + +#include + +extern const struct export_operations scoutfs_export_ops; + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index fc15a08a..03e49d7c 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -642,4 +642,16 @@ enum { SCOUTFS_NET_STATUS_UNKNOWN, }; +/* + * Scoutfs file handle structure - this can be copied out to userspace + * via open by handle or put on the wire from NFS. + */ +struct scoutfs_fid { + __le64 ino; + __le64 parent_ino; +} __packed; + +#define FILEID_SCOUTFS 0x81 +#define FILEID_SCOUTFS_WITH_PARENT 0x82 + #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a70a9ebe..def4daf7 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -37,6 +37,7 @@ #include "bio.h" #include "dlmglue.h" #include "stackglue.h" +#include "export.h" struct lock_info; @@ -2076,6 +2077,89 @@ DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_unblock_lock, TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), TP_ARGS(osb, lockres) ); + +DECLARE_EVENT_CLASS(scoutfs_fileid_class, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid), + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, fh_type) + __field(u64, ino) + __field(u64, parent_ino) + ), + TP_fast_assign( + __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + __entry->fh_type = fh_type; + __entry->ino = le64_to_cpu(fid->ino); + __entry->parent_ino = fh_type == FILEID_SCOUTFS_WITH_PARENT ? + le64_to_cpu(fid->parent_ino) : 0ULL; + ), + TP_printk("fsid "FSID_FMT" type %d ino %llu parent %llu", + __entry->fsid, __entry->fh_type, __entry->ino, + __entry->parent_ino) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_encode_fh, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_fh_to_dentry, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +DEFINE_EVENT(scoutfs_fileid_class, scoutfs_fh_to_parent, + TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), + TP_ARGS(sb, fh_type, fid) +); + +TRACE_EVENT(scoutfs_get_parent, + TP_PROTO(struct super_block *sb, struct inode *inode, u64 parent), + + TP_ARGS(sb, inode, parent), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, parent) + ), + + TP_fast_assign( + __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + __entry->ino = scoutfs_ino(inode); + __entry->parent = parent; + ), + + TP_printk("fsid "FSID_FMT" child %llu parent %llu", + __entry->fsid, __entry->ino, __entry->parent) +); + +TRACE_EVENT(scoutfs_get_name, + TP_PROTO(struct super_block *sb, struct inode *parent, + struct inode *child, char *name), + + TP_ARGS(sb, parent, child, name), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, parent_ino) + __field(__u64, child_ino) + __string(name, name) + ), + + TP_fast_assign( + __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + __entry->parent_ino = scoutfs_ino(parent); + __entry->child_ino = scoutfs_ino(child); + __assign_str(name, name); + ), + + TP_printk("fsid "FSID_FMT" parent %llu child %llu name: %s", + __entry->fsid, __entry->parent_ino, __entry->child_ino, + __get_str(name)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/super.c b/kmod/src/super.c index b0c5f6af..00667954 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -22,6 +22,7 @@ #include #include "super.h" +#include "export.h" #include "format.h" #include "inode.h" #include "dir.h" @@ -297,6 +298,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sb->s_magic = SCOUTFS_SUPER_MAGIC; sb->s_maxbytes = MAX_LFS_FILESIZE; sb->s_op = &scoutfs_super_ops; + sb->s_export_op = &scoutfs_export_ops; /* btree blocks use long lived bh->b_data refs */ mapping_set_gfp_mask(sb->s_bdev->bd_inode->i_mapping, GFP_NOFS); From 9cc750c4ec0ebf765c398bab10ecdb962427e394 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Jan 2018 12:48:02 -0800 Subject: [PATCH 549/920] scoutfs: remove lock idr in free, not put The idr entry that identifies a lock's position in the debugfs locks file is allocated early in the process of building up a lock. Today the idr entry is only destroyed in put_(), which is called later once reference counts are established. Errors before then just call free_() and can leave idrs around that reference freed memory. This always destroys the idr entry in free_(). We no longer leave idr entries around that reference freed memory. This fixes use after free while walking the debugfs file which can hit in scoutfs/006 which uses the locks file. Signed-off-by: Zach Brown --- kmod/src/lock.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 4d347c7b..1ca89642 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -259,6 +259,13 @@ static void free_scoutfs_lock(struct scoutfs_lock *lock) if (lock) { linfo = SCOUTFS_SB(lock->sb)->lock_info; + if (lock->debug_locks_id) { + spin_lock(&linfo->lock); + idr_remove(&linfo->debug_locks_idr, + lock->debug_locks_id); + spin_unlock(&linfo->lock); + } + scoutfs_inc_counter(lock->sb, lock_free); ocfs2_lock_res_free(&lock->lockres); scoutfs_key_free(lock->sb, lock->start); @@ -288,9 +295,6 @@ static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) RB_CLEAR_NODE(&lock->range_node); } list_del(&lock->lru_entry); - if (lock->debug_locks_id) - idr_remove(&linfo->debug_locks_idr, - lock->debug_locks_id); spin_unlock(&linfo->lock); ocfs2_simple_drop_lockres(&linfo->dlmglue, &lock->lockres); From a49061a7d97e6a584cdaf3dc0b2ed551a6b5ccf9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Jan 2018 13:06:23 -0800 Subject: [PATCH 550/920] scoutfs: remove the size index We aren't using the size index. It has runtime and code maintenance costs that aren't worth paying. Let's remove it. Removing it from the format and no longer maintaining it are straight forward. The bulk of this patch is actually the act of removing it from the index locking functions. We no longer have to predict the size that will be stored during the transaction to lock the index items that will be created during the transaction. A bunch of code to predict the size and then pass it into locking and transactions goes away. Like other inode fields we now update the size as it changes. Signed-off-by: Zach Brown --- kmod/src/data.c | 12 +-------- kmod/src/dir.c | 66 +++++++++++++---------------------------------- kmod/src/format.h | 7 +++-- kmod/src/inode.c | 39 ++++++++++------------------ kmod/src/inode.h | 7 +++-- kmod/src/ioctl.c | 4 +-- kmod/src/ioctl.h | 3 +-- kmod/src/key.c | 3 --- kmod/src/lock.c | 39 +++------------------------- kmod/src/xattr.c | 3 +-- 10 files changed, 45 insertions(+), 138 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index b101fc48..07491291 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1147,7 +1147,6 @@ static int scoutfs_write_begin(struct file *file, struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct write_begin_data *wbd; - u64 new_size; u64 ind_seq; int ret; @@ -1166,19 +1165,10 @@ static int scoutfs_write_begin(struct file *file, goto out; } - /* - * Lock a size update item assuming we perform the full write. - * If If the write is inside i_size then we don't lock and - * nothing will be updated. Lock granularity is larger than - * pages so any size update in this call will be covered by the - * lock. If there's an error and we don't change i_size then - * the item update won't happen and the lock will be unused. - */ - new_size = max(pos + len, i_size_read(inode)); do { ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &wbd->ind_locks, inode, - new_size, true) ?: + true) ?: scoutfs_inode_index_try_lock_hold(sb, &wbd->ind_locks, ind_seq, SIC_WRITE_BEGIN()); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 07f3c062..5ae5cc8b 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -531,7 +531,6 @@ out: static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev, const struct scoutfs_item_count cnt, - u64 dir_size, u64 inode_size, struct scoutfs_lock **dir_lock, struct scoutfs_lock **inode_lock, struct list_head *ind_locks) @@ -566,9 +565,8 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: - scoutfs_inode_index_prepare(sb, ind_locks, dir, dir_size, true) ?: - scoutfs_inode_index_prepare_ino(sb, ind_locks, ino, mode, - inode_size) ?: + scoutfs_inode_index_prepare(sb, ind_locks, dir, true) ?: + scoutfs_inode_index_prepare_ino(sb, ind_locks, ino, mode) ?: scoutfs_inode_index_try_lock_hold(sb, ind_locks, ind_seq, cnt); if (ret > 0) goto retry; @@ -607,16 +605,14 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; LIST_HEAD(ind_locks); - u64 dir_size; u64 pos; int ret; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - dir_size = i_size_read(dir) + dentry->d_name.len; inode = lock_hold_create(dir, dentry, mode, rdev, - SIC_MKNOD(dentry->d_name.len), dir_size, 0, + SIC_MKNOD(dentry->d_name.len), &dir_lock, &inode_lock, &ind_locks); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -631,7 +627,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, update_dentry_info(dentry, pos); - i_size_write(dir, dir_size); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_mtime = inode->i_atime = inode->i_ctime = dir->i_mtime; @@ -704,10 +700,8 @@ static int scoutfs_link(struct dentry *old_dentry, dir_size = i_size_read(dir) + dentry->d_name.len; retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, dir, - dir_size, false) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, inode, - i_size_read(inode), false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_LINK(dentry->d_name.len)); if (ret > 0) @@ -770,7 +764,6 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) struct scoutfs_lock *inode_lock = NULL; struct scoutfs_lock *dir_lock = NULL; LIST_HEAD(ind_locks); - u64 dir_size; u64 ind_seq; int ret = 0; @@ -785,13 +778,10 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) goto unlock; } - dir_size = i_size_read(dir) - dentry->d_name.len; retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, dir, - dir_size, false) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, inode, - i_size_read(inode), false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_UNLINK(dentry->d_name.len)); if (ret > 0) @@ -819,7 +809,7 @@ retry: dir->i_ctime = ts; dir->i_mtime = ts; - i_size_write(dir, dir_size); + i_size_write(dir, i_size_read(dir) - dentry->d_name.len); inode->i_ctime = ts; drop_nlink(inode); @@ -1000,7 +990,6 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; LIST_HEAD(ind_locks); - u64 dir_size; u64 pos; int ret; @@ -1013,10 +1002,8 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) return ret; - dir_size = i_size_read(dir) + dentry->d_name.len; inode = lock_hold_create(dir, dentry, S_IFLNK|S_IRWXUGO, 0, SIC_SYMLINK(dentry->d_name.len, name_len), - dir_size, name_len, &dir_lock, &inode_lock, &ind_locks); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -1036,7 +1023,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, update_dentry_info(dentry, pos); - i_size_write(dir, dir_size); + i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; inode->i_ctime = dir->i_mtime; @@ -1397,8 +1384,6 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, bool del_new = false; bool ins_old = false; LIST_HEAD(ind_locks); - u64 old_size; - u64 uninitialized_var(new_size); u64 ind_seq; u64 new_pos; int ret; @@ -1451,30 +1436,14 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, if (ret) goto out_unlock; - old_size = i_size_read(old_dir) - old_dentry->d_name.len; - if (!new_inode) { - if (old_dir != new_dir) - new_size = i_size_read(new_dir) + - new_dentry->d_name.len; - else - old_size += new_dentry->d_name.len; - } else { - if (old_dir != new_dir) - new_size = i_size_read(new_dir); - } - retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, old_dir, - old_size, false) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, old_inode, - i_size_read(old_inode), false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_dir, false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, old_inode, false) ?: (new_dir == old_dir ? 0 : - scoutfs_inode_index_prepare(sb, &ind_locks, new_dir, - new_size, false)) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, new_dir, false)) ?: (new_inode == NULL ? 0 : - scoutfs_inode_index_prepare(sb, &ind_locks, new_inode, - i_size_read(new_inode), false)) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, new_inode, false)) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_RENAME(old_dentry->d_name.len, new_dentry->d_name.len)); @@ -1540,9 +1509,10 @@ retry: /* the caller will use d_move to move the old_dentry into place */ update_dentry_info(old_dentry, new_pos); - i_size_write(old_dir, old_size); - if (old_dir != new_dir) - i_size_write(new_dir, new_size); + i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); + if (!new_inode) + i_size_write(new_dir, i_size_read(new_dir) + + new_dentry->d_name.len); if (new_inode) { drop_nlink(new_inode); diff --git a/kmod/src/format.h b/kmod/src/format.h index 03e49d7c..90d1d4c7 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -237,10 +237,9 @@ struct scoutfs_segment_block { #define SCOUTFS_MAX_ZONE 4 /* power of 2 is efficient */ /* inode index zone */ -#define SCOUTFS_INODE_INDEX_SIZE_TYPE 1 -#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 2 -#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 3 -#define SCOUTFS_INODE_INDEX_NR 4 /* don't forget to update */ +#define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 +#define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 +#define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ /* node zone */ #define SCOUTFS_FREE_BITS_SEGNO_TYPE 1 diff --git a/kmod/src/inode.c b/kmod/src/inode.c index e9ebd9a0..209bd6f9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -214,8 +214,6 @@ static void set_item_info(struct scoutfs_inode_info *si, memset(si->item_minors, 0, sizeof(si->item_minors)); si->have_item = true; - si->item_majors[SCOUTFS_INODE_INDEX_SIZE_TYPE] = - le64_to_cpu(sinode->size); si->item_majors[SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = le64_to_cpu(sinode->meta_seq); si->item_majors[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = @@ -340,7 +338,7 @@ static int set_inode_size(struct inode *inode, struct scoutfs_lock *lock, if (!S_ISREG(inode->i_mode)) return 0; - ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, new_size, true, + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, true, SIC_DIRTY_INODE()); if (ret) return ret; @@ -365,8 +363,7 @@ static int clear_truncate_flag(struct inode *inode, struct scoutfs_lock *lock) LIST_HEAD(ind_locks); int ret; - ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, - i_size_read(inode), false, + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, SIC_DIRTY_INODE()); if (ret) return ret; @@ -445,8 +442,7 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) } } - ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, - i_size_read(inode), false, + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, SIC_DIRTY_INODE()); if (ret) goto out; @@ -691,7 +687,6 @@ static bool will_ins_index(struct scoutfs_inode_info *si, static bool inode_has_index(umode_t mode, u8 type) { switch(type) { - case SCOUTFS_INODE_INDEX_SIZE_TYPE: case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: return true; case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: @@ -821,8 +816,6 @@ static int update_indices(struct super_block *sb, u64 major; u32 minor; } *upd, upds[] = { - { SCOUTFS_INODE_INDEX_SIZE_TYPE, - le64_to_cpu(sinode->size), 0 }, { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode->meta_seq), 0 }, { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, @@ -900,8 +893,8 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, * We map the item to coarse locks here. This reduces the number of * locks we track and means that when we later try to find the lock that * covers an item we can deal with the item update changing a little - * (seq, size) while still being covered. It does mean we have to share - * some logic with lock naming. + * while still being covered. It does mean we have to share some logic + * with lock naming. */ static int add_index_lock(struct list_head *list, u64 ino, u8 type, u64 major, u32 minor) @@ -979,7 +972,7 @@ static u64 upd_data_seq(struct scoutfs_sb_info *sbi, */ static int prepare_indices(struct super_block *sb, struct list_head *list, struct scoutfs_inode_info *si, u64 ino, - umode_t mode, u64 new_size, bool set_data_seq) + umode_t mode, bool set_data_seq) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct index_update { @@ -987,7 +980,6 @@ static int prepare_indices(struct super_block *sb, struct list_head *list, u64 major; u32 minor; } *upd, upds[] = { - { SCOUTFS_INODE_INDEX_SIZE_TYPE, new_size, 0}, { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, sbi->trans_seq, 0}, { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, upd_data_seq(sbi, si, set_data_seq), 0}, @@ -1009,13 +1001,12 @@ static int prepare_indices(struct super_block *sb, struct list_head *list, } int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, - struct inode *inode, u64 new_size, - bool set_data_seq) + struct inode *inode, bool set_data_seq) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); return prepare_indices(sb, list, si, scoutfs_ino(inode), - inode->i_mode, new_size, set_data_seq); + inode->i_mode, set_data_seq); } /* @@ -1026,9 +1017,9 @@ int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, */ int scoutfs_inode_index_prepare_ino(struct super_block *sb, struct list_head *list, u64 ino, - umode_t mode, u64 new_size) + umode_t mode) { - return prepare_indices(sb, list, NULL, ino, mode, new_size, true); + return prepare_indices(sb, list, NULL, ino, mode, true); } /* @@ -1045,8 +1036,6 @@ static int prepare_index_deletion(struct super_block *sb, u64 major; u32 minor; } *ind, inds[] = { - { SCOUTFS_INODE_INDEX_SIZE_TYPE, - le64_to_cpu(sinode->size), 0 }, { SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode->meta_seq), 0 }, { SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, @@ -1120,7 +1109,7 @@ out: } int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, - u64 size, bool set_data_seq, + bool set_data_seq, const struct scoutfs_item_count cnt) { struct super_block *sb = inode->i_sb; @@ -1129,7 +1118,7 @@ int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, do { ret = scoutfs_inode_index_start(sb, &seq) ?: - scoutfs_inode_index_prepare(sb, list, inode, size, + scoutfs_inode_index_prepare(sb, list, inode, set_data_seq) ?: scoutfs_inode_index_try_lock_hold(sb, list, seq, cnt); } while (ret > 0); @@ -1191,9 +1180,7 @@ static int remove_index_items(struct super_block *sb, u64 ino, umode_t mode = le32_to_cpu(sinode->mode); int ret; - ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_SIZE_TYPE, - le64_to_cpu(sinode->size), 0, ind_locks) ?: - remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, + ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_META_SEQ_TYPE, le64_to_cpu(sinode->meta_seq), 0, ind_locks); if (ret == 0 && S_ISREG(mode)) ret = remove_index(sb, ino, SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE, diff --git a/kmod/src/inode.h b/kmod/src/inode.h index ca35e4ba..6e5309be 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -64,16 +64,15 @@ struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); int scoutfs_inode_index_start(struct super_block *sb, u64 *seq); int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, - struct inode *inode, u64 new_size, - bool set_data_seq); + struct inode *inode, bool set_data_seq); int scoutfs_inode_index_prepare_ino(struct super_block *sb, struct list_head *list, u64 ino, - umode_t mode, u64 new_size); + umode_t mode); int scoutfs_inode_index_try_lock_hold(struct super_block *sb, struct list_head *list, u64 seq, const struct scoutfs_item_count cnt); int scoutfs_inode_index_lock_hold(struct inode *inode, struct list_head *list, - u64 size, bool set_data_seq, + bool set_data_seq, const struct scoutfs_item_count cnt); void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 9906e617..cb86dfa4 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -71,9 +71,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) trace_scoutfs_ioc_walk_inodes(sb, &walk); - if (walk.index == SCOUTFS_IOC_WALK_INODES_SIZE) - type = SCOUTFS_INODE_INDEX_SIZE_TYPE; - else if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) + if (walk.index == SCOUTFS_IOC_WALK_INODES_META_SEQ) type = SCOUTFS_INODE_INDEX_META_SEQ_TYPE; else if (walk.index == SCOUTFS_IOC_WALK_INODES_DATA_SEQ) type = SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index e9a78db0..34917a34 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -51,8 +51,7 @@ struct scoutfs_ioctl_walk_inodes { } __packed; enum { - SCOUTFS_IOC_WALK_INODES_SIZE = 0, - SCOUTFS_IOC_WALK_INODES_META_SEQ, + SCOUTFS_IOC_WALK_INODES_META_SEQ = 0, SCOUTFS_IOC_WALK_INODES_DATA_SEQ, SCOUTFS_IOC_WALK_INODES_UNKNOWN, }; diff --git a/kmod/src/key.c b/kmod/src/key.c index d3695ede..3683bba9 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -211,7 +211,6 @@ typedef int (*key_printer_t)(char *buf, struct scoutfs_key_buf *key, static int pr_ino_idx(char *buf, struct scoutfs_key_buf *key, size_t size) { static char *type_strings[] = { - [SCOUTFS_INODE_INDEX_SIZE_TYPE] = "siz", [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", }; @@ -329,8 +328,6 @@ static int pr_block_mapping(char *buf, struct scoutfs_key_buf *key, size_t size) } const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { - [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_SIZE_TYPE] = - pr_ino_idx, [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = pr_ino_idx, [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 1ca89642..446a97bf 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -947,16 +947,6 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, * The seq indexes have natural batching and limits on the number of * keys per major value. * - * The file size index is very different. We don't control the - * distribution of sizes amongst inodes. We map ranges of sizes to a - * small set of locks by rounding the size down to groups of sizes - * identified by their highest set bit and two next significant bits. - * This results in ranges that increase by quarters of powers of two. - * (small sizes don't have enough bits for this scheme, they're all - * mapped to a range from 0 to 15.) two (0 and 1 are mapped to 0). Each - * lock then covers all the sizes in their range and all the inodes with - * those sizes. - * * This can also be used to find items that are covered by the same lock * because their starting keys are the same. */ @@ -964,32 +954,11 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, struct scoutfs_inode_index_key *start, struct scoutfs_inode_index_key *end) { - u64 start_major; - u64 end_major; - int bit; + u64 start_major = major & ~SCOUTFS_LOCK_SEQ_GROUP_MASK; + u64 end_major = major | SCOUTFS_LOCK_SEQ_GROUP_MASK; - switch(type) { - case SCOUTFS_INODE_INDEX_SIZE_TYPE: - bit = major ? fls64(major) : 0; - if (bit < 5) { - /* sizes [ 0 .. 15 ] are in their own lock */ - start_major = 0; - end_major = 15; - } else { - /* last bit, 2 lesser bits, mask */ - start_major = major & (7ULL << (bit - 3)); - end_major = start_major + (1ULL << (bit - 3)) - 1; - } - break; - - case SCOUTFS_INODE_INDEX_META_SEQ_TYPE: - case SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE: - start_major = major & ~SCOUTFS_LOCK_SEQ_GROUP_MASK; - end_major = major | SCOUTFS_LOCK_SEQ_GROUP_MASK; - break; - default: - BUG(); - } + BUG_ON(type != SCOUTFS_INODE_INDEX_META_SEQ_TYPE && + type != SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE); if (start) { start->zone = SCOUTFS_INODE_INDEX_ZONE; diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index d01b0640..a21fba4a 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -321,8 +321,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: - scoutfs_inode_index_prepare(sb, &ind_locks, inode, - i_size_read(inode), false) ?: + scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_XATTR_SET(name_len, size)); if (ret > 0) From 4ff1e3020f7528df41adca4eb79c587b9e9266c5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Feb 2018 13:25:33 -0800 Subject: [PATCH 551/920] scoutfs: allocate inode numbers per directory Having an inode number allocation pool in the super block meant that all allocations across the mount are interleaved. This means that concurrent file creation in different directories will create overlapping inode numbers. This leads to lock contention as reasonable work loads will tend to distribute work by directories. The easy fix is to have per-directory inode number allocation pools. We take the opportunity to clean up the network request so that the caller gets the allocation instead of having it be fed back in via a weird callback. Signed-off-by: Zach Brown --- kmod/src/client.c | 28 ++++---- kmod/src/client.h | 3 +- kmod/src/dir.c | 2 +- kmod/src/inode.c | 142 ++++++++++++--------------------------- kmod/src/inode.h | 12 +++- kmod/src/scoutfs_trace.h | 22 +++--- kmod/src/server.c | 7 +- 7 files changed, 85 insertions(+), 131 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 16517ccb..174f7f17 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -540,30 +540,30 @@ static int client_request(struct client_info *client, int type, void *data, return ret; } -int scoutfs_client_alloc_inodes(struct super_block *sb) +/* + * Ask for a new run of allocated inode numbers. The server can return + * fewer than @count. It will success with nr == 0 if we've run out. + */ +int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, + u64 *ino, u64 *nr) { struct client_info *client = SCOUTFS_SB(sb)->client_info; struct scoutfs_net_inode_alloc ial; - u64 ino = 0; - u64 nr = 0; + __le64 lecount = cpu_to_le64(count); int ret; - ret = client_request(client, SCOUTFS_NET_ALLOC_INODES, NULL, 0, - &ial, sizeof(ial)); + ret = client_request(client, SCOUTFS_NET_ALLOC_INODES, + &lecount, sizeof(lecount), &ial, sizeof(ial)); if (ret == 0) { - ino = le64_to_cpu(ial.ino); - nr = le64_to_cpu(ial.nr); + *ino = le64_to_cpu(ial.ino); + *nr = le64_to_cpu(ial.nr); - /* catch wrapping */ - if (ino + nr < ino) + if (*nr == 0) + ret = -ENOSPC; + else if (*ino + *nr < *ino) ret = -EINVAL; } - if (ret < 0) - scoutfs_inode_fill_pool(sb, 0, 0); - else - scoutfs_inode_fill_pool(sb, ino, nr); - return ret; } diff --git a/kmod/src/client.h b/kmod/src/client.h index 71c8c4eb..c2bc3bb9 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -1,7 +1,8 @@ #ifndef _SCOUTFS_CLIENT_H_ #define _SCOUTFS_CLIENT_H_ -int scoutfs_client_alloc_inodes(struct super_block *sb); +int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, + u64 *ino, u64 *nr); int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 5ae5cc8b..85b9143e 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -545,7 +545,7 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, if (ret) return ERR_PTR(ret); - ret = scoutfs_alloc_ino(sb, &ino); + ret = scoutfs_alloc_ino(dir, &ino); if (ret) return ERR_PTR(ret); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 209bd6f9..31545f0c 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -47,17 +47,7 @@ * - describe data locking size problems */ -struct free_ino_pool { - wait_queue_head_t waitq; - spinlock_t lock; - u64 ino; - u64 nr; - bool in_flight; -}; - struct inode_sb_info { - struct free_ino_pool pool; - spinlock_t writeback_lock; struct rb_root writeback_inodes; }; @@ -82,6 +72,7 @@ static void scoutfs_inode_ctor(void *obj) scoutfs_per_task_init(&ci->pt_data_lock); init_rwsem(&ci->xattr_rwsem); RB_CLEAR_NODE(&ci->writeback_node); + spin_lock_init(&ci->ino_alloc.lock); inode_init_once(&ci->inode); } @@ -563,8 +554,9 @@ struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino) struct inode *scoutfs_iget(struct super_block *sb, u64 ino) { - struct inode *inode; struct scoutfs_lock *lock = NULL; + struct scoutfs_inode_info *si; + struct inode *inode; int ret; ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); @@ -580,7 +572,10 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) if (inode->i_state & I_NEW) { /* XXX ensure refresh, instead clear in drop_inode? */ - atomic64_set(&SCOUTFS_I(inode)->last_refreshed, 0); + si = SCOUTFS_I(inode); + atomic64_set(&si->last_refreshed, 0); + si->ino_alloc.ino = 0; + si->ino_alloc.nr = 0; ret = scoutfs_inode_refresh(inode, lock, 0); if (ret) { @@ -1205,98 +1200,50 @@ u64 scoutfs_last_ino(struct super_block *sb) } /* - * Network replies refill the pool, providing ino = ~0ULL nr = 0 when - * there's no more inodes (which should never happen in practice.) + * Return an allocated and unused inode number. Returns -ENOSPC if + * we're out of inode. + * + * Each parent directory has its own pool of free inode numbers. Items + * are sorted by their inode numbers as they're stored in segments. + * This will tend to group together files that are created in a + * directory at the same time in segments. Concurrent creation across + * different directories will be stored in their own regions. + * + * Inode numbers are never reclaimed. If the inode is evicted or we're + * unmounted the pending inode numbers will be lost. Asking for a + * relatively small number from the server each time will tend to + * minimize that loss while still being large enough for typical + * directory file counts. */ -void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr) +int scoutfs_alloc_ino(struct inode *parent, u64 *ino_ret) { - struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; - - trace_scoutfs_inode_fill_pool(sb, ino, nr); - - spin_lock(&pool->lock); - - pool->ino = ino; - pool->nr = nr; - pool->in_flight = false; - - spin_unlock(&pool->lock); - - wake_up(&pool->waitq); -} - -static bool pool_in_flight(struct free_ino_pool *pool) -{ - bool in_flight; - - spin_lock(&pool->lock); - in_flight = pool->in_flight; - spin_unlock(&pool->lock); - - return in_flight; -} - -/* - * We have a pool of free inodes given to us by the server. If it - * empties we only ever have one request for new inodes in flight. The - * net layer calls us when it gets a reply. If there's no more inodes - * we'll get ino == ~0 and nr == 0. - */ -int scoutfs_alloc_ino(struct super_block *sb, u64 *ino) -{ - struct free_ino_pool *pool = &SCOUTFS_SB(sb)->inode_sb_info->pool; - bool request; + struct scoutfs_inode_allocator *ia = &SCOUTFS_I(parent)->ino_alloc; + struct super_block *sb = parent->i_sb; + u64 ino; + u64 nr; int ret; - *ino = 0; + spin_lock(&ia->lock); - spin_lock(&pool->lock); - - while (pool->nr == 0 && pool->ino != ~0ULL) { - if (pool->in_flight) { - request = false; - } else { - pool->in_flight = true; - request = true; - } - - spin_unlock(&pool->lock); - - if (request) { - ret = scoutfs_client_alloc_inodes(sb); - if (ret) { - spin_lock(&pool->lock); - pool->in_flight = false; - spin_unlock(&pool->lock); - wake_up(&pool->waitq); - goto out; - } - } - - ret = wait_event_interruptible(pool->waitq, - !pool_in_flight(pool)); - if (ret) + if (ia->nr == 0) { + spin_unlock(&ia->lock); + ret = scoutfs_client_alloc_inodes(sb, 10000, &ino, &nr); + if (ret < 0) goto out; - - spin_lock(&pool->lock); + spin_lock(&ia->lock); + if (ia->nr == 0) { + ia->ino = ino; + ia->nr = nr; + } } - if (pool->nr == 0) { - *ino = 0; - ret = -ENOSPC; - } else { - *ino = pool->ino++; - pool->nr--; - ret = 0; - - } - - spin_unlock(&pool->lock); + *ino_ret = ia->ino++; + ia->nr--; + spin_unlock(&ia->lock); + ret = 0; out: - - trace_scoutfs_alloc_ino(sb, ret, *ino, pool->ino, pool->nr, - pool->in_flight); + trace_scoutfs_alloc_ino(sb, ret, *ino_ret, ia->ino, ia->nr); return ret; } @@ -1327,6 +1274,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->have_item = false; atomic64_set(&ci->last_refreshed, scoutfs_lock_refresh_gen(lock)); ci->flags = 0; + ci->ino_alloc.ino = 0; + ci->ino_alloc.nr = 0; scoutfs_inode_set_meta_seq(inode); scoutfs_inode_set_data_seq(inode); @@ -1653,17 +1602,12 @@ out: int scoutfs_inode_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct free_ino_pool *pool; struct inode_sb_info *inf; inf = kzalloc(sizeof(struct inode_sb_info), GFP_KERNEL); if (!inf) return -ENOMEM; - pool = &inf->pool; - init_waitqueue_head(&pool->waitq); - spin_lock_init(&pool->lock); - spin_lock_init(&inf->writeback_lock); inf->writeback_inodes = RB_ROOT; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 6e5309be..532bb21a 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -8,6 +8,12 @@ struct scoutfs_lock; +struct scoutfs_inode_allocator { + spinlock_t lock; + u64 ino; + u64 nr; +}; + struct scoutfs_inode_info { /* read or initialized for each inode instance */ u64 ino; @@ -31,6 +37,9 @@ struct scoutfs_inode_info { /* updated at on each new lock acquisition */ atomic64_t last_refreshed; + /* reset for every new inode instance */ + struct scoutfs_inode_allocator ino_alloc; + /* initialized once for slab object */ seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ @@ -80,8 +89,7 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, struct list_head *ind_locks); -void scoutfs_inode_fill_pool(struct super_block *sb, u64 ino, u64 nr); -int scoutfs_alloc_ino(struct super_block *sb, u64 *ino); +int scoutfs_alloc_ino(struct inode *parent, u64 *ino); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev, u64 ino, struct scoutfs_lock *lock); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index def4daf7..f8e512eb 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -903,32 +903,30 @@ TRACE_EVENT(scoutfs_inode_fill_pool, ); TRACE_EVENT(scoutfs_alloc_ino, - TP_PROTO(struct super_block *sb, int ret, __u64 ino, __u64 pool_ino, - __u64 nr, unsigned int in_flight), + TP_PROTO(struct super_block *sb, int ret, __u64 ino, __u64 next_ino, + __u64 next_nr), - TP_ARGS(sb, ret, ino, pool_ino, nr, in_flight), + TP_ARGS(sb, ret, ino, next_ino, next_nr), TP_STRUCT__entry( __field(__u64, fsid) __field(int, ret) __field(__u64, ino) - __field(__u64, pool_ino) - __field(__u64, nr) - __field(unsigned int, in_flight) + __field(__u64, next_ino) + __field(__u64, next_nr) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->ret = ret; __entry->ino = ino; - __entry->pool_ino = pool_ino; - __entry->nr = nr; - __entry->in_flight = in_flight; + __entry->next_ino = next_ino; + __entry->next_nr = next_nr; ), - TP_printk(FSID_FMT" ret %d ino %llu pool ino %llu nr %llu req %u " - "(racey)", __entry->fsid, __entry->ret, __entry->ino, - __entry->pool_ino, __entry->nr, __entry->in_flight) + TP_printk(FSID_FMT" ret %d ino %llu next_ino %llu next_nr %llu", + __entry->fsid, __entry->ret, __entry->ino, __entry->next_ino, + __entry->next_nr) ); TRACE_EVENT(scoutfs_evict_inode, diff --git a/kmod/src/server.c b/kmod/src/server.c index 9cba3882..1b206f42 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -304,20 +304,23 @@ static int process_alloc_inodes(struct server_connection *conn, struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_inode_alloc ial; struct commit_waiter cw; + __le64 lecount; u64 ino; u64 nr; int ret; - if (data_len != 0) { + if (data_len != sizeof(lecount)) { ret = -EINVAL; goto out; } + memcpy(&lecount, data, data_len); + down_read(&server->commit_rwsem); spin_lock(&sbi->next_ino_lock); ino = le64_to_cpu(super->next_ino); - nr = min(100000ULL, ~0ULL - ino); + nr = min(le64_to_cpu(lecount), U64_MAX - ino); le64_add_cpu(&super->next_ino, nr); spin_unlock(&sbi->next_ino_lock); From d42a3115c961468a793b72ea2a48ec4ca2e3203d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Feb 2018 13:45:07 -0800 Subject: [PATCH 552/920] scoutfs: fix livelock in item_set_batch scoutfs_item_set_batch() first tries to populate the item cache with the range of keys it's going to be modifying. It does this by walking the input key range and trying to read any missing regions. It made a bad assumption that reading from the final present key of a cached range would read more items into the cache. That was often the case when the last present key landed in a segment that contained more keys. But if the last present key was at the end of a segment the read wouldn't make any difference. It'd keep trying to read that final present key indefinitely. The fix is to try and populate the item cache starting with the first key that's missing from the cache by incrementing the last key that we found in the cache. This stopped scoutfs/507 from reliably getting stuck trying to modify an xattr whose single item happened to land at the end of a segment. Signed-off-by: Zach Brown --- kmod/src/item.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index b5afda23..7216709e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1310,8 +1310,10 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, if (check_range(sb, &cac->ranges, range_end, range_end)) { if (scoutfs_key_compare(range_end, last) >= 0) break; - /* start reading from hole starting at range_end */ + /* start reading after the last key we have cached */ + scoutfs_key_inc(range_end); } else { + /* start reading from the missing first */ scoutfs_key_copy(range_end, first); } From f52dc283220d9bf5d82ca6cc681ff319be092f28 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 2 Feb 2018 09:18:08 -0800 Subject: [PATCH 553/920] scoutfs: simplify lock use of kernel dlm We had an excessive number of layers between scoutfs and the dlm code in the kernel. We had dlmglue, the scoutfs locks, and task refs. Each layer had structs that track the lifetime of the layer below it. We were about to add another layer to hold on to locks just a bit longer so that we can avoid down conversion and transaction commit storms under contention. This collapses all those layers into simple state machine in lock.c that manages the mode of dlm locks on behalf of the file system. The users of the lock interface are mainly unchanged. We did change from a heavier trylock to a lighter nonblock lock attempt and have to change the single rare readpage use. Lock fields change so a few external users of those fields change. This not only removes a lot of code it also contains functional improvements. For example, it can now convert directly to CW locks with a single lock request instead of having to use two by first converting to NL. It introduces the concept of an unlock grace period. Locks won't be dropped on behalf of other nodes soon after being unlocked so that tasks have a chance to batch up work before the other node gets a chance. This can result in two orders of magnitude improvements in the time it takes to, say, change a set of xattrs on the same file population from two nodes concurrently. There are significant changes to trace points, counters, and debug files that follow the implementation changes. Signed-off-by: Zach Brown --- kmod/Makefile | 2 +- kmod/src/Makefile | 8 +- kmod/src/counters.h | 28 +- kmod/src/data.c | 33 +- kmod/src/dlmglue.c | 2803 -------------------------------------- kmod/src/dlmglue.h | 390 ------ kmod/src/inode.c | 4 +- kmod/src/item.c | 49 +- kmod/src/lock.c | 1477 +++++++++++--------- kmod/src/lock.h | 43 +- kmod/src/scoutfs_trace.h | 278 +--- kmod/src/server.c | 6 +- kmod/src/stackglue.c | 412 ------ kmod/src/stackglue.h | 149 -- kmod/src/super.c | 4 +- 15 files changed, 989 insertions(+), 4697 deletions(-) delete mode 100644 kmod/src/dlmglue.c delete mode 100644 kmod/src/dlmglue.h delete mode 100644 kmod/src/stackglue.c delete mode 100644 kmod/src/stackglue.h diff --git a/kmod/Makefile b/kmod/Makefile index ade36d3e..2963f205 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -22,7 +22,7 @@ SCOUTFS_FORMAT_HASH := \ SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ SCOUTFS_FORMAT_HASH=$(SCOUTFS_FORMAT_HASH) \ CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ - EXTRA_CFLAGS="-Werror -DCONFIG_OCFS2_FS_STATS" + EXTRA_CFLAGS="-Werror" all: module diff --git a/kmod/src/Makefile b/kmod/src/Makefile index e44808c5..903d8d29 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,10 +6,10 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - dlmglue.o export.o file.o kvec.o inode.o ioctl.o item.o key.o \ - lock.o manifest.o msg.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sock.o sort_priv.o stackglue.o super.o sysfs.o \ - trans.o triggers.o xattr.o + export.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ + manifest.o msg.o options.o per_task.o seg.o server.o \ + scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ + triggers.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/counters.h b/kmod/src/counters.h index f0c02f50..57cc8c6c 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -29,13 +29,6 @@ EXPAND_COUNTER(data_write_begin) \ EXPAND_COUNTER(data_write_end) \ EXPAND_COUNTER(data_writepage) \ - EXPAND_COUNTER(dlm_cancel_convert) \ - EXPAND_COUNTER(dlm_convert_request) \ - EXPAND_COUNTER(dlm_cw_downconvert) \ - EXPAND_COUNTER(dlm_ex_downconvert) \ - EXPAND_COUNTER(dlm_lock_request) \ - EXPAND_COUNTER(dlm_pr_downconvert) \ - EXPAND_COUNTER(dlm_unlock_request) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_delete) \ @@ -56,12 +49,23 @@ EXPAND_COUNTER(item_shrink_small_split) \ EXPAND_COUNTER(item_shrink_split_range) \ EXPAND_COUNTER(lock_alloc) \ - EXPAND_COUNTER(lock_blocked_wait) \ - EXPAND_COUNTER(lock_busy_wait) \ + EXPAND_COUNTER(lock_ast) \ + EXPAND_COUNTER(lock_ast_edeadlk) \ + EXPAND_COUNTER(lock_ast_error) \ + EXPAND_COUNTER(lock_bast) \ + EXPAND_COUNTER(lock_dlm_call) \ + EXPAND_COUNTER(lock_dlm_call_error) \ EXPAND_COUNTER(lock_free) \ - EXPAND_COUNTER(lock_incompat_wait) \ - EXPAND_COUNTER(lock_type_ino_downconvert) \ - EXPAND_COUNTER(lock_type_idx_downconvert) \ + EXPAND_COUNTER(lock_grace_enforced) \ + EXPAND_COUNTER(lock_grace_expired) \ + EXPAND_COUNTER(lock_grace_extended) \ + EXPAND_COUNTER(lock_invalidate_clean_item) \ + EXPAND_COUNTER(lock_lock) \ + EXPAND_COUNTER(lock_lock_error) \ + EXPAND_COUNTER(lock_nonblock_eagain) \ + EXPAND_COUNTER(lock_shrink) \ + EXPAND_COUNTER(lock_write_dirty_item) \ + EXPAND_COUNTER(lock_unlock) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(seg_alloc) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 07491291..bf56c45b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1076,29 +1076,38 @@ out: return ret; } +/* + * This is almost never used. We can't block on a cluster lock while + * holding the page lock because lock invalidation gets the page lock + * while blocking locks. If we can't use an existing lock then we drop + * the page lock and try again. + */ static int scoutfs_readpage(struct file *file, struct page *page) { struct inode *inode = file->f_inode; struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; - int unlock = 1; + int flags; int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE | - SCOUTFS_LKF_TRYLOCK, inode, &inode_lock); - if (ret) { - if (ret == -EAGAIN) - ret = AOP_TRUNCATED_PAGE; - goto out; + flags = SCOUTFS_LKF_REFRESH_INODE | SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, flags, inode, &inode_lock); + if (ret < 0) { + unlock_page(page); + if (ret == -EAGAIN) { + flags &= ~SCOUTFS_LKF_NONBLOCK; + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, flags, inode, + &inode_lock); + if (ret == 0) { + scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + ret = AOP_TRUNCATED_PAGE; + } + } + return ret; } ret = mpage_readpage(page, scoutfs_get_block); - unlock = 0; - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); -out: - if (unlock) - unlock_page(page); return ret; } diff --git a/kmod/src/dlmglue.c b/kmod/src/dlmglue.c deleted file mode 100644 index d2724322..00000000 --- a/kmod/src/dlmglue.c +++ /dev/null @@ -1,2803 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; -*- - * vim: noexpandtab sw=8 ts=8 sts=0: - * - * dlmglue.c - * - * Code which implements an OCFS2 specific interface to our DLM. - * - * Copyright (C) 2003, 2004 Oracle. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 021110-1307, USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "counters.h" -#include "dlmglue.h" - -#include "scoutfs_trace.h" - -#ifdef TRACE_DLMGLUE -#define mlog(mask, fmt, args...) trace_printk(fmt , ##args) -#define mlog_errno(st) do { \ - int _st = (st); \ - if (_st != -ERESTARTSYS && _st != -EINTR && \ - _st != AOP_TRUNCATED_PAGE && _st != -ENOSPC) \ - mlog(ML_ERROR, "status = %lld\n", (long long)_st); \ -} while (0) -#else -#define mlog(mask, fmt, args...) -#define mlog_errno(st) -#endif -#define mlog_bug_on_msg(cond, fmt, args...) do { \ - if (cond) { \ - printk(KERN_ERR "bug expression: " #cond "\n"); \ - printk(KERN_ERR fmt, ##args); \ - BUG(); \ - } \ -} while (0) - -struct ocfs2_mask_waiter { - struct list_head mw_item; - int mw_status; - struct completion mw_complete; - unsigned long mw_mask; - unsigned long mw_goal; -#ifdef CONFIG_OCFS2_FS_STATS - ktime_t mw_lock_start; -#endif -}; - -struct ocfs2_unblock_ctl { - int requeue; - enum ocfs2_unblock_action unblock_action; -}; - -#if 0 && CONFIG_DEBUG_LOCK_ALLOC -/* Lockdep class keys */ -struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; -#endif - -static inline struct ocfs2_lock_res *ocfs2_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) -{ - return container_of(lksb, struct ocfs2_lock_res, l_lksb); -} - -static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *lockres) -{ - if (lockres->l_ops->get_osb) - return lockres->l_ops->get_osb(lockres); - - return (struct ocfs2_super *)lockres->l_priv; -} - -static inline void lockres_name(struct ocfs2_lock_res *lockres, char *buf, - unsigned int len) -{ - if (lockres->l_ops->print) - lockres->l_ops->print(lockres, buf, len); - else - snprintf(buf, len, "%s", lockres->l_name); -} - -static inline void lockres_notify_event(struct ocfs2_lock_res *lockres, - enum ocfs2_lock_events event) -{ - if (lockres->l_ops->notify_event) - lockres->l_ops->notify_event(lockres, event); -} - -static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, - int wanted); -static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, unsigned long caller_ip); -void ocfs2_cluster_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, int level) -{ - __ocfs2_cluster_unlock(osb, lockres, level, _RET_IP_); -} - -static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres); -static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres); -static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres); -static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, int level); -static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); -static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, - int convert); -#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ - printk(KERN_ERR "DLM error %d while calling %s on resource %s\n", \ - _err, _func, (_lockres)->l_pretty_name); \ -} while (0) - -static int ocfs2_downconvert_thread(void *arg); -static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); -static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, - int new_level); -static int ocfs2_downconvert_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int new_level, - int lvb, - unsigned int generation); -static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); -static int ocfs2_cancel_convert(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); - -static DEFINE_SPINLOCK(ocfs2_dlm_tracking_lock); - -static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, - struct ocfs2_dlm_debug *dlm_debug) -{ - mlog(0, "Add tracking for lockres %s\n", res->l_name); - - spin_lock(&ocfs2_dlm_tracking_lock); - BUG_ON(!list_empty(&res->l_debug_list)); - list_add(&res->l_debug_list, &dlm_debug->d_lockres_tracking); - spin_unlock(&ocfs2_dlm_tracking_lock); -} - -static void ocfs2_remove_lockres_tracking(struct ocfs2_lock_res *res) -{ - spin_lock(&ocfs2_dlm_tracking_lock); - if (!list_empty(&res->l_debug_list)) - list_del_init(&res->l_debug_list); - spin_unlock(&ocfs2_dlm_tracking_lock); -} - -#ifdef CONFIG_OCFS2_FS_STATS -static void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) -{ - res->l_lock_refresh = 0; - memset(&res->l_lock_prmode, 0, sizeof(struct ocfs2_lock_stats)); - memset(&res->l_lock_exmode, 0, sizeof(struct ocfs2_lock_stats)); -} - -static void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, int level, - struct ocfs2_mask_waiter *mw, int ret) -{ - u32 usec; - ktime_t kt; - struct ocfs2_lock_stats *stats; - - if (level == DLM_LOCK_PR) - stats = &res->l_lock_prmode; - else if (level == DLM_LOCK_EX) - stats = &res->l_lock_exmode; - else if (level == DLM_LOCK_CW) - stats = &res->l_lock_cwmode; - else - return; - - kt = ktime_sub(ktime_get(), mw->mw_lock_start); - usec = ktime_to_us(kt); - - stats->ls_gets++; - stats->ls_total += ktime_to_ns(kt); - /* overflow */ - if (unlikely(stats->ls_gets == 0)) { - stats->ls_gets++; - stats->ls_total = ktime_to_ns(kt); - } - - if (stats->ls_max < usec) - stats->ls_max = usec; - - if (ret) - stats->ls_fail++; -} - -static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) -{ - lockres->l_lock_refresh++; -} - -static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) -{ - mw->mw_lock_start = ktime_get(); -} -#else -static inline void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) -{ -} -static inline void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, - int level, struct ocfs2_mask_waiter *mw, int ret) -{ -} -static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) -{ -} -static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) -{ -} -#endif - -void ocfs2_lock_res_init_common(struct ocfs2_super *osb, - struct ocfs2_lock_res *res, - struct ocfs2_lock_res_ops *ops, - void *priv) -{ - res->l_ops = ops; - res->l_priv = priv; - - res->l_level = DLM_LOCK_IV; - res->l_requested = DLM_LOCK_IV; - res->l_blocking = DLM_LOCK_IV; - res->l_action = OCFS2_AST_INVALID; - res->l_unlock_action = OCFS2_UNLOCK_INVALID; - - res->l_flags = OCFS2_LOCK_INITIALIZED; - - lockres_name(res, res->l_pretty_name, OCFS2_LOCK_ID_PRETTY_LEN); - - ocfs2_add_lockres_tracking(res, osb->osb_dlm_debug); - - ocfs2_init_lock_stats(res); -#if 0 && CONFIG_DEBUG_LOCK_ALLOC - if (type != OCFS2_LOCK_TYPE_OPEN) - lockdep_init_map(&res->l_lockdep_map, ocfs2_lock_type_strings[type], - &lockdep_keys[type], 0); - else - res->l_lockdep_map.key = NULL; -#endif -} - -void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) -{ - /* This also clears out the lock status block */ - memset(res, 0, sizeof(struct ocfs2_lock_res)); - spin_lock_init(&res->l_lock); - init_waitqueue_head(&res->l_event); - INIT_LIST_HEAD(&res->l_blocked_list); - INIT_LIST_HEAD(&res->l_mask_waiters); - INIT_LIST_HEAD(&res->l_holders); - INIT_LIST_HEAD(&res->l_debug_list); -} - -void ocfs2_lock_res_free(struct ocfs2_lock_res *res) -{ - if (!(res->l_flags & OCFS2_LOCK_INITIALIZED)) - return; - - ocfs2_remove_lockres_tracking(res); - - mlog_bug_on_msg(!list_empty(&res->l_blocked_list), - "Lockres %s is on the blocked list\n", - res->l_name); - mlog_bug_on_msg(!list_empty(&res->l_mask_waiters), - "Lockres %s has mask waiters pending\n", - res->l_name); - mlog_bug_on_msg(spin_is_locked(&res->l_lock), - "Lockres %s is locked\n", - res->l_name); - mlog_bug_on_msg(res->l_ro_holders, - "Lockres %s has %u ro holders\n", - res->l_name, res->l_ro_holders); - mlog_bug_on_msg(res->l_ex_holders, - "Lockres %s has %u ex holders\n", - res->l_name, res->l_ex_holders); - - /* Need to clear out the lock status block for the dlm */ - memset(&res->l_lksb, 0, sizeof(res->l_lksb)); - - res->l_flags = 0UL; -} - -/* - * Keep a list of processes who have interest in a lockres. - * Note: this is now only uesed for check recursive cluster locking. - */ -static inline void ocfs2_add_holder(struct ocfs2_lock_res *lockres, - struct ocfs2_lock_holder *oh) -{ - INIT_LIST_HEAD(&oh->oh_list); - oh->oh_owner_pid = get_pid(task_pid(current)); - - spin_lock(&lockres->l_lock); - list_add_tail(&oh->oh_list, &lockres->l_holders); - spin_unlock(&lockres->l_lock); -} - -static inline void ocfs2_remove_holder(struct ocfs2_lock_res *lockres, - struct ocfs2_lock_holder *oh) -{ - spin_lock(&lockres->l_lock); - list_del(&oh->oh_list); - spin_unlock(&lockres->l_lock); - - put_pid(oh->oh_owner_pid); -} - -static inline int ocfs2_is_locked_by_me(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_lock_holder *oh; - struct pid *pid; - - /* look in the list of holders for one with the current task as owner */ - spin_lock(&lockres->l_lock); - pid = task_pid(current); - list_for_each_entry(oh, &lockres->l_holders, oh_list) { - if (oh->oh_owner_pid == pid) { - spin_unlock(&lockres->l_lock); - return 1; - } - } - spin_unlock(&lockres->l_lock); - - return 0; -} - -static inline void ocfs2_inc_holders(struct ocfs2_lock_res *lockres, - int level) -{ - BUG_ON(!lockres); - - switch(level) { - case DLM_LOCK_EX: - lockres->l_ex_holders++; - break; - case DLM_LOCK_PR: - lockres->l_ro_holders++; - break; - case DLM_LOCK_CW: - lockres->l_cw_holders++; - break; - default: - BUG(); - } -} - -static inline void ocfs2_dec_holders(struct ocfs2_lock_res *lockres, - int level) -{ - BUG_ON(!lockres); - - switch(level) { - case DLM_LOCK_EX: - BUG_ON(!lockres->l_ex_holders); - lockres->l_ex_holders--; - break; - case DLM_LOCK_PR: - BUG_ON(!lockres->l_ro_holders); - lockres->l_ro_holders--; - break; - case DLM_LOCK_CW: - BUG_ON(!lockres->l_cw_holders); - lockres->l_cw_holders--; - break; - default: - BUG(); - } -} - -/* - * Compatibility matrix indexed by lock level - idea borrowed from - * fs/dlm/lock.c. Going across is the level our lock holds, going down - * is the level we're asked to convert to. The UN column and PD - * columns are unused and act as padding. - */ -static const int level_compat_matrix[8][8] = { - /* Lockres granted level */ - /* UN NL CR CW PR PW EX PD */ - {0, 0, 0, 0, 0, 0, 0, 0}, /* UN */ - {0, 1, 1, 1, 1, 1, 1, 0}, /* NL */ - {0, 0, 1, 1, 1, 1, 1, 0}, /* CR */ - {0, 0, 0, 1, 0, 1, 1, 0}, /* CW */ /* <-- Wanted levels */ - {0, 0, 0, 0, 1, 1, 1, 0}, /* PR */ - {0, 0, 0, 0, 0, 1, 1, 0}, /* PW */ - {0, 0, 0, 0, 0, 0, 1, 0}, /* EX */ - {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ -}; - -static inline int __levels_compat(int lockres_level, int wanted) -{ - return level_compat_matrix[wanted + 1][lockres_level + 1]; -} - -int ocfs2_levels_compat(struct ocfs2_lock_res *lockres, int wanted) -{ - return __levels_compat(lockres->l_level, wanted); -} - -/* - * WARNING: We have to adjust this function when adding lock levels to - * dlmglue - * - * Given a lock blocking 'lockres' at 'level', what new level should - * we downconvert to. This function will never return a level which - * would result in an upconvert. - */ -static inline int ocfs2_downconvert_level(struct ocfs2_lock_res *lockres, - int level) -{ - int new_level = DLM_LOCK_EX; - - if (level == DLM_LOCK_EX) - new_level = DLM_LOCK_NL; - else if (level == DLM_LOCK_PR) { - if (lockres->l_level == DLM_LOCK_EX) - new_level = DLM_LOCK_PR; - else - new_level = DLM_LOCK_NL; - } else if (level == DLM_LOCK_CW) - new_level = DLM_LOCK_CW; - return new_level; -} - -#define H_EX 0x1 -#define H_PR 0x2 -#define H_CW 0x4 -#define H_ANY (H_EX|H_PR|H_CW) -static int lockres_has_holders(struct ocfs2_lock_res *lockres, int which) -{ - if (which & H_EX && lockres->l_ex_holders) - return 1; - if (which & H_PR && lockres->l_ro_holders) - return 1; - if (which & H_CW && lockres->l_cw_holders) - return 1; - return 0; -} - -static void lockres_set_flags(struct ocfs2_lock_res *lockres, - unsigned long newflags) -{ - struct ocfs2_mask_waiter *mw, *tmp; - - assert_spin_locked(&lockres->l_lock); - - lockres->l_flags = newflags; - - list_for_each_entry_safe(mw, tmp, &lockres->l_mask_waiters, mw_item) { - if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) - continue; - - list_del_init(&mw->mw_item); - mw->mw_status = 0; - complete(&mw->mw_complete); - } -} -static void lockres_or_flags(struct ocfs2_lock_res *lockres, unsigned long or) -{ - lockres_set_flags(lockres, lockres->l_flags | or); -} -static void lockres_clear_flags(struct ocfs2_lock_res *lockres, - unsigned long clear) -{ - lockres_set_flags(lockres, lockres->l_flags & ~clear); -} - -/* - * Make sure that a lock gets a strictly increasing number only once - * each time it needs to be refreshed. The gen needs to be larger than - * any previous gen the locked resources has seen so we maintain the gen - * in the super. The caller has serialized on the lock but lots of - * locks can all be racing on the super. - * - * This is used by callers to have a single read-only indicator that - * they need to refresh their resource while they have it locked. - */ -static void lockres_inc_refresh_gen(struct ocfs2_lock_res *lockres) -{ - struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); - - lockres->l_refresh_gen = atomic64_inc_return(&osb->refresh_gen); -} - -static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres) -{ - int dc_level = ocfs2_downconvert_level(lockres, lockres->l_blocking); - - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); - BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); - - lockres->l_level = lockres->l_requested; - if (ocfs2_levels_compat(lockres, dc_level)) { - lockres->l_blocking = DLM_LOCK_NL; - lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); - } - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); -} - -static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres) -{ - int old_level = lockres->l_level; - - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); - - /* - * Converting from NL to any mode, or upconverting between - * incompatible modes will require a refresh. - */ - lockres->l_level = lockres->l_requested; - if (lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { - if (old_level == DLM_LOCK_NL || - (old_level == DLM_LOCK_CW && - lockres->l_level != DLM_LOCK_NL)) { - lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); - lockres_inc_refresh_gen(lockres); - } - } - - /* - * We set the OCFS2_LOCK_UPCONVERT_FINISHING flag before clearing - * the OCFS2_LOCK_BUSY flag to prevent the dc thread from - * downconverting the lock before the upconvert has fully completed. - * Do not prevent the dc thread from downconverting if NONBLOCK lock - * had already returned. - */ - if (!(lockres->l_flags & OCFS2_LOCK_NONBLOCK_FINISHED)) - lockres_or_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); - else - lockres_clear_flags(lockres, OCFS2_LOCK_NONBLOCK_FINISHED); - - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); -} - -static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres) -{ - BUG_ON((!(lockres->l_flags & OCFS2_LOCK_BUSY))); - BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); - - if (lockres->l_requested > DLM_LOCK_NL && - !(lockres->l_flags & OCFS2_LOCK_LOCAL) && - lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) { - lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); - lockres_inc_refresh_gen(lockres); - } - - lockres->l_level = lockres->l_requested; - lockres_or_flags(lockres, OCFS2_LOCK_ATTACHED); - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); -} - -static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, - int level) -{ - int needs_downconvert = 0; - - assert_spin_locked(&lockres->l_lock); - - if (level > lockres->l_blocking) { - /* only schedule a downconvert if we haven't already scheduled - * one that goes low enough to satisfy the level we're - * blocking. this also catches the case where we get - * duplicate BASTs */ - if (ocfs2_downconvert_level(lockres, level) < - ocfs2_downconvert_level(lockres, lockres->l_blocking)) - needs_downconvert = 1; - - lockres->l_blocking = level; - } - - mlog(ML_BASTS, "lockres %s, block %d, level %d, l_block %d, dwn %d\n", - lockres->l_name, level, lockres->l_level, lockres->l_blocking, - needs_downconvert); - - if (needs_downconvert) - lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); - mlog(0, "needs_downconvert = %d\n", needs_downconvert); - return needs_downconvert; -} - -static void set_lock_blocking(struct ocfs2_lock_res *lockres, int level) -{ - struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); - int needs_downconvert; - - needs_downconvert = ocfs2_generic_handle_bast(lockres, level); - if (needs_downconvert) - ocfs2_schedule_blocked_lock(osb, lockres); -} - -/* - * OCFS2_LOCK_PENDING and l_pending_gen. - * - * Why does OCFS2_LOCK_PENDING exist? To close a race between setting - * OCFS2_LOCK_BUSY and calling ocfs2_dlm_lock(). See ocfs2_unblock_lock() - * for more details on the race. - * - * OCFS2_LOCK_PENDING closes the race quite nicely. However, it introduces - * a race on itself. In o2dlm, we can get the ast before ocfs2_dlm_lock() - * returns. The ast clears OCFS2_LOCK_BUSY, and must therefore clear - * OCFS2_LOCK_PENDING at the same time. When ocfs2_dlm_lock() returns, - * the caller is going to try to clear PENDING again. If nothing else is - * happening, __lockres_clear_pending() sees PENDING is unset and does - * nothing. - * - * But what if another path (eg downconvert thread) has just started a - * new locking action? The other path has re-set PENDING. Our path - * cannot clear PENDING, because that will re-open the original race - * window. - * - * [Example] - * - * ocfs2_meta_lock() - * ocfs2_cluster_lock() - * set BUSY - * set PENDING - * drop l_lock - * ocfs2_dlm_lock() - * ocfs2_locking_ast() ocfs2_downconvert_thread() - * clear PENDING ocfs2_unblock_lock() - * take_l_lock - * !BUSY - * ocfs2_prepare_downconvert() - * set BUSY - * set PENDING - * drop l_lock - * take l_lock - * clear PENDING - * drop l_lock - * - * ocfs2_dlm_lock() - * - * So as you can see, we now have a window where l_lock is not held, - * PENDING is not set, and ocfs2_dlm_lock() has not been called. - * - * The core problem is that ocfs2_cluster_lock() has cleared the PENDING - * set by ocfs2_prepare_downconvert(). That wasn't nice. - * - * To solve this we introduce l_pending_gen. A call to - * lockres_clear_pending() will only do so when it is passed a generation - * number that matches the lockres. lockres_set_pending() will return the - * current generation number. When ocfs2_cluster_lock() goes to clear - * PENDING, it passes the generation it got from set_pending(). In our - * example above, the generation numbers will *not* match. Thus, - * ocfs2_cluster_lock() will not clear the PENDING set by - * ocfs2_prepare_downconvert(). - */ - -/* Unlocked version for ocfs2_locking_ast() */ -static void __lockres_clear_pending(struct ocfs2_lock_res *lockres, - unsigned int generation, - struct ocfs2_super *osb) -{ - assert_spin_locked(&lockres->l_lock); - - /* - * The ast and locking functions can race us here. The winner - * will clear pending, the loser will not. - */ - if (!(lockres->l_flags & OCFS2_LOCK_PENDING) || - (lockres->l_pending_gen != generation)) - return; - - lockres_clear_flags(lockres, OCFS2_LOCK_PENDING); - lockres->l_pending_gen++; - - /* - * The downconvert thread may have skipped us because we - * were PENDING. Wake it up. - */ - if (lockres->l_flags & OCFS2_LOCK_BLOCKED) - ocfs2_wake_downconvert_thread(osb); -} - -/* Locked version for callers of ocfs2_dlm_lock() */ -static void lockres_clear_pending(struct ocfs2_lock_res *lockres, - unsigned int generation, - struct ocfs2_super *osb) -{ - unsigned long flags; - - spin_lock_irqsave(&lockres->l_lock, flags); - __lockres_clear_pending(lockres, generation, osb); - spin_unlock_irqrestore(&lockres->l_lock, flags); -} - -static unsigned int lockres_set_pending(struct ocfs2_lock_res *lockres) -{ - assert_spin_locked(&lockres->l_lock); - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); - - lockres_or_flags(lockres, OCFS2_LOCK_PENDING); - - return lockres->l_pending_gen; -} - -static void ocfs2_blocking_ast(struct ocfs2_dlm_lksb *lksb, int level) -{ - struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); - struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); - int needs_downconvert; - unsigned long flags; - - BUG_ON(level <= DLM_LOCK_NL); - - mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d\n", - lockres->l_name, level, lockres->l_level); - - /* - * We can skip the bast for locks which don't enable caching - - * they'll be dropped at the earliest possible time anyway. - */ - if (lockres->l_flags & OCFS2_LOCK_NOCACHE) - return; - - spin_lock_irqsave(&lockres->l_lock, flags); - needs_downconvert = ocfs2_generic_handle_bast(lockres, level); - if (needs_downconvert) - ocfs2_schedule_blocked_lock(osb, lockres); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - wake_up(&lockres->l_event); - - ocfs2_wake_downconvert_thread(osb); -} - -static void ocfs2_locking_ast(struct ocfs2_dlm_lksb *lksb) -{ - struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); - struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); - unsigned long flags; - int status; - - spin_lock_irqsave(&lockres->l_lock, flags); - - status = ocfs2_dlm_lock_status(&lockres->l_lksb); - - if (status == -EAGAIN) { - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); - goto out; - } - - if (status) { - mlog(ML_ERROR, "lockres %s: lksb status value of %d!\n", - lockres->l_name, status); - spin_unlock_irqrestore(&lockres->l_lock, flags); - return; - } - - mlog(ML_BASTS, "AST fired for lockres %s, action %d, unlock %d, " - "level %d => %d\n", lockres->l_name, lockres->l_action, - lockres->l_unlock_action, lockres->l_level, lockres->l_requested); - - switch(lockres->l_action) { - case OCFS2_AST_ATTACH: - ocfs2_generic_handle_attach_action(lockres); - lockres_clear_flags(lockres, OCFS2_LOCK_LOCAL); - break; - case OCFS2_AST_CONVERT: - ocfs2_generic_handle_convert_action(lockres); - break; - case OCFS2_AST_DOWNCONVERT: - ocfs2_generic_handle_downconvert_action(lockres); - break; - default: - mlog(ML_ERROR, "lockres %s: AST fired with invalid action: %u, " - "flags 0x%lx, unlock: %u\n", - lockres->l_name, lockres->l_action, lockres->l_flags, - lockres->l_unlock_action); - BUG(); - } -out: - /* set it to something invalid so if we get called again we - * can catch it. */ - lockres->l_action = OCFS2_AST_INVALID; - - /* Did we try to cancel this lock? Clear that state */ - if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) - lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; - - /* - * We may have beaten the locking functions here. We certainly - * know that dlm_lock() has been called :-) - * Because we can't have two lock calls in flight at once, we - * can use lockres->l_pending_gen. - */ - __lockres_clear_pending(lockres, lockres->l_pending_gen, osb); - - wake_up(&lockres->l_event); - spin_unlock_irqrestore(&lockres->l_lock, flags); -} - -static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) -{ - struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); - unsigned long flags; - - mlog(ML_BASTS, "UNLOCK AST fired for lockres %s, action = %d\n", - lockres->l_name, lockres->l_unlock_action); - - spin_lock_irqsave(&lockres->l_lock, flags); - if (error) { - mlog(ML_ERROR, "Dlm passes error %d for lock %s, " - "unlock_action %d\n", error, lockres->l_name, - lockres->l_unlock_action); - spin_unlock_irqrestore(&lockres->l_lock, flags); - return; - } - - switch(lockres->l_unlock_action) { - case OCFS2_UNLOCK_CANCEL_CONVERT: - mlog(0, "Cancel convert success for %s\n", lockres->l_name); - lockres->l_action = OCFS2_AST_INVALID; - /* Downconvert thread may have requeued this lock, we - * need to wake it. */ - if (lockres->l_flags & OCFS2_LOCK_BLOCKED) - ocfs2_wake_downconvert_thread(ocfs2_get_lockres_osb(lockres)); - break; - case OCFS2_UNLOCK_DROP_LOCK: - lockres->l_level = DLM_LOCK_IV; - break; - default: - BUG(); - } - - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); - lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; - wake_up(&lockres->l_event); - spin_unlock_irqrestore(&lockres->l_lock, flags); -} - -/* - * This is the filesystem locking protocol. It provides the lock handling - * hooks for the underlying DLM. It has a maximum version number. - * The version number allows interoperability with systems running at - * the same major number and an equal or smaller minor number. - * - * Whenever the filesystem does new things with locks (adds or removes a - * lock, orders them differently, does different things underneath a lock), - * the version must be changed. The protocol is negotiated when joining - * the dlm domain. A node may join the domain if its major version is - * identical to all other nodes and its minor version is greater than - * or equal to all other nodes. When its minor version is greater than - * the other nodes, it will run at the minor version specified by the - * other nodes. - * - * If a locking change is made that will not be compatible with older - * versions, the major number must be increased and the minor version set - * to zero. If a change merely adds a behavior that can be disabled when - * speaking to older versions, the minor version must be increased. If a - * change adds a fully backwards compatible change (eg, LVB changes that - * are just ignored by older versions), the version does not need to be - * updated. - */ -static struct ocfs2_locking_protocol lproto = { -#if 0 - .lp_max_version = { - .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR, - .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR, - }, -#endif - .lp_lock_ast = ocfs2_locking_ast, - .lp_blocking_ast = ocfs2_blocking_ast, - .lp_unlock_ast = ocfs2_unlock_ast, -}; - -#if 0 -void ocfs2_set_locking_protocol(void) -{ - ocfs2_stack_glue_set_max_proto_version(&lproto.lp_max_version); -} -#endif - -static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, - int convert) -{ - unsigned long flags; - - spin_lock_irqsave(&lockres->l_lock, flags); - lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); - lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); - if (convert) - lockres->l_action = OCFS2_AST_INVALID; - else - lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; - spin_unlock_irqrestore(&lockres->l_lock, flags); - - wake_up(&lockres->l_event); -} - -#if 0 -/* Note: If we detect another process working on the lock (i.e., - * OCFS2_LOCK_BUSY), we'll bail out returning 0. It's up to the caller - * to do the right thing in that case. - */ -static int ocfs2_lock_create(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - u32 dlm_flags) -{ - int ret = 0; - unsigned long flags; - unsigned int gen; - - mlog(0, "lock %s, level = %d, flags = %u\n", lockres->l_name, level, - dlm_flags); - - spin_lock_irqsave(&lockres->l_lock, flags); - if ((lockres->l_flags & OCFS2_LOCK_ATTACHED) || - (lockres->l_flags & OCFS2_LOCK_BUSY)) { - spin_unlock_irqrestore(&lockres->l_lock, flags); - goto bail; - } - - lockres->l_action = OCFS2_AST_ATTACH; - lockres->l_requested = level; - lockres_or_flags(lockres, OCFS2_LOCK_BUSY); - gen = lockres_set_pending(lockres); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - ret = ocfs2_dlm_lock(osb->cconn, - level, - &lockres->l_lksb, - dlm_flags, - lockres->l_name, - OCFS2_LOCK_ID_MAX_LEN - 1); - lockres_clear_pending(lockres, gen, osb); - if (ret) { - ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); - ocfs2_recover_from_dlm_error(lockres, 1); - } - - mlog(0, "lock %s, return from ocfs2_dlm_lock\n", lockres->l_name); - -bail: - return ret; -} -#endif - -static inline int ocfs2_check_wait_flag(struct ocfs2_lock_res *lockres, - int flag) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&lockres->l_lock, flags); - ret = lockres->l_flags & flag; - spin_unlock_irqrestore(&lockres->l_lock, flags); - - return ret; -} - -static inline void ocfs2_wait_on_busy_lock(struct ocfs2_lock_res *lockres) - -{ - wait_event(lockres->l_event, - !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_BUSY)); -} - -static inline void ocfs2_wait_on_refreshing_lock(struct ocfs2_lock_res *lockres) - -{ - wait_event(lockres->l_event, - !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_REFRESHING)); -} - -/* predict what lock level we'll be dropping down to on behalf - * of another node, and return true if the currently wanted - * level will be compatible with it. */ -static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, - int wanted) -{ - BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); - - return wanted <= ocfs2_downconvert_level(lockres, lockres->l_blocking); -} - -static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) -{ - INIT_LIST_HEAD(&mw->mw_item); - init_completion(&mw->mw_complete); - ocfs2_init_start_time(mw); -} - -static int ocfs2_wait_for_mask(struct ocfs2_mask_waiter *mw) -{ - wait_for_completion(&mw->mw_complete); - /* Re-arm the completion in case we want to wait on it again */ - reinit_completion(&mw->mw_complete); - return mw->mw_status; -} - -static void lockres_add_mask_waiter(struct ocfs2_lock_res *lockres, - struct ocfs2_mask_waiter *mw, - unsigned long mask, - unsigned long goal) -{ - BUG_ON(!list_empty(&mw->mw_item)); - - assert_spin_locked(&lockres->l_lock); - - list_add_tail(&mw->mw_item, &lockres->l_mask_waiters); - mw->mw_mask = mask; - mw->mw_goal = goal; -} - -/* returns 0 if the mw that was removed was already satisfied, -EBUSY - * if the mask still hadn't reached its goal */ -static int __lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, - struct ocfs2_mask_waiter *mw) -{ - int ret = 0; - - assert_spin_locked(&lockres->l_lock); - if (!list_empty(&mw->mw_item)) { - if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) - ret = -EBUSY; - - list_del_init(&mw->mw_item); - init_completion(&mw->mw_complete); - } - - return ret; -} - -#if 0 -static int lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, - struct ocfs2_mask_waiter *mw) -{ - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&lockres->l_lock, flags); - ret = __lockres_remove_mask_waiter(lockres, mw); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - return ret; - -} - -static int ocfs2_wait_for_mask_interruptible(struct ocfs2_mask_waiter *mw, - struct ocfs2_lock_res *lockres) -{ - int ret; - - ret = wait_for_completion_interruptible(&mw->mw_complete); - if (ret) - lockres_remove_mask_waiter(lockres, mw); - else - ret = mw->mw_status; - /* Re-arm the completion in case we want to wait on it again */ - reinit_completion(&mw->mw_complete); - return ret; -} -#endif - -static inline int cw_incompat_convert(struct ocfs2_lock_res *lockres, - int level) -{ - /* Have CW, want PR/EX */ - if (lockres->l_level == DLM_LOCK_CW && - (level == DLM_LOCK_PR || level == DLM_LOCK_EX)) - return 1; - /* Have EX/PR, want CW */ - if (level == DLM_LOCK_CW && - (lockres->l_level == DLM_LOCK_PR || lockres->l_level == DLM_LOCK_EX)) - return 1; - return 0; -} - -static int __ocfs2_cluster_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - u32 lkm_flags, - int arg_flags, - int l_subclass, - unsigned long caller_ip) -{ - struct ocfs2_mask_waiter mw; - int wait, catch_signals = !(osb->s_mount_opt & OCFS2_MOUNT_NOINTR); - int ret = 0; /* gcc doesn't realize wait = 1 guarantees ret is set */ - unsigned long flags; - unsigned int gen; - int noqueue_attempted = 0; - int dlm_locked = 0; - int kick_dc = 0; - - trace_ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags); - - if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) { - mlog_errno(-EINVAL); - return -EINVAL; - } - - ocfs2_init_mask_waiter(&mw); - - if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) - lkm_flags |= DLM_LKF_VALBLK; - -again: - wait = 0; - - spin_lock_irqsave(&lockres->l_lock, flags); - - if (catch_signals && signal_pending(current)) { - ret = -ERESTARTSYS; - goto unlock; - } - - mlog_bug_on_msg(lockres->l_flags & OCFS2_LOCK_FREEING, - "Cluster lock called on freeing lockres %s! flags " - "0x%lx\n", lockres->l_name, lockres->l_flags); - - /* We only compare against the currently granted level - * here. If the lock is blocked waiting on a downconvert, - * we'll get caught below. */ - if (lockres->l_flags & OCFS2_LOCK_BUSY && - !ocfs2_levels_compat(lockres, level)) { - /* is someone sitting in dlm_lock? If so, wait on - * them. */ - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); - wait = 1; - scoutfs_inc_counter(osb->sb, lock_busy_wait); - goto unlock; - } - - if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) { - /* - * We've upconverted. If the lock now has a level we can - * work with, we take it. If, however, the lock is not at the - * required level, we go thru the full cycle. One way this could - * happen is if a process requesting an upconvert to PR is - * closely followed by another requesting upconvert to an EX. - * If the process requesting EX lands here, we want it to - * continue attempting to upconvert and let the process - * requesting PR take the lock. - * If multiple processes request upconvert to PR, the first one - * here will take the lock. The others will have to go thru the - * OCFS2_LOCK_BLOCKED check to ensure that there is no pending - * downconvert request. - */ - if (ocfs2_levels_compat(lockres, level)) - goto update_holders; - } - - if (lockres->l_flags & OCFS2_LOCK_BLOCKED && - !ocfs2_may_continue_on_blocked_lock(lockres, level)) { - /* is the lock is currently blocked on behalf of - * another node */ - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); - wait = 1; - scoutfs_inc_counter(osb->sb, lock_blocked_wait); - goto unlock; - } - - /* - * Convert from PR/EX to CW and vice-versa. Those levels are - * not compatible with each other. As a result, we have to - * wait for holders on the lock to drain. The easiest way to - * do this is by forcing a downconvert. We can then allow the - * process to come back and reacquire the lock at the correct - * level. - */ - if (cw_incompat_convert(lockres, level)) { - /* ocfs2_unblock_lock will drop to NL, then we can upconvert. */ - set_lock_blocking(lockres, DLM_LOCK_EX); - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); - wait = 1; - scoutfs_inc_counter(osb->sb, lock_incompat_wait); - goto unlock; - } - - /* NL->Anything, PR->EX conditions are handled here */ - if (level > lockres->l_level) { - if (noqueue_attempted > 0) { - ret = -EAGAIN; - goto unlock; - } - if (lkm_flags & DLM_LKF_NOQUEUE) - noqueue_attempted = 1; - - if (lockres->l_action != OCFS2_AST_INVALID) - mlog(ML_ERROR, "lockres %s has action %u pending\n", - lockres->l_name, lockres->l_action); - - if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { - lockres->l_action = OCFS2_AST_ATTACH; - lkm_flags &= ~DLM_LKF_CONVERT; - } else { - lockres->l_action = OCFS2_AST_CONVERT; - lkm_flags |= DLM_LKF_CONVERT; - } - - lockres->l_requested = level; - lockres_or_flags(lockres, OCFS2_LOCK_BUSY); - gen = lockres_set_pending(lockres); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - if (lkm_flags & DLM_LKF_CONVERT) { - scoutfs_inc_counter(osb->sb, dlm_convert_request); - lockres_notify_event(lockres, EVENT_DLM_CONVERT); - } else { - scoutfs_inc_counter(osb->sb, dlm_lock_request); - lockres_notify_event(lockres, EVENT_DLM_LOCK); - } - - BUG_ON(level == DLM_LOCK_IV); - BUG_ON(level == DLM_LOCK_NL); - - mlog(ML_BASTS, "lockres %s, convert from %d to %d\n", - lockres->l_name, lockres->l_level, level); - - /* call dlm_lock to upgrade lock now */ - ret = ocfs2_dlm_lock(osb->cconn, - level, - &lockres->l_lksb, - lkm_flags, - lockres->l_name, - OCFS2_LOCK_ID_MAX_LEN - 1); - lockres_clear_pending(lockres, gen, osb); - if (ret) { - if (!(lkm_flags & DLM_LKF_NOQUEUE) || - (ret != -EAGAIN)) { - ocfs2_log_dlm_error("ocfs2_dlm_lock", - ret, lockres); - } - ocfs2_recover_from_dlm_error(lockres, 1); - goto out; - } - dlm_locked = 1; - - mlog(0, "lock %s, successful return from ocfs2_dlm_lock\n", - lockres->l_name); - - /* At this point we've gone inside the dlm and need to - * complete our work regardless. */ - catch_signals = 0; - - /* wait for busy to clear and carry on */ - goto again; - } - -update_holders: - /* Ok, if we get here then we're good to go. */ - ocfs2_inc_holders(lockres, level); - - ret = 0; -unlock: - lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); - - /* ocfs2_unblock_lock reques on seeing OCFS2_LOCK_UPCONVERT_FINISHING */ - kick_dc = (lockres->l_flags & OCFS2_LOCK_BLOCKED); - - spin_unlock_irqrestore(&lockres->l_lock, flags); - if (kick_dc) - ocfs2_wake_downconvert_thread(osb); -out: - /* - * This is helping work around a lock inversion between the page lock - * and dlm locks. One path holds the page lock while calling aops - * which block acquiring dlm locks. The voting thread holds dlm - * locks while acquiring page locks while down converting data locks. - * This block is helping an aop path notice the inversion and back - * off to unlock its page lock before trying the dlm lock again. - */ - if (wait && arg_flags & OCFS2_LOCK_NONBLOCK && - mw.mw_mask & (OCFS2_LOCK_BUSY|OCFS2_LOCK_BLOCKED)) { - wait = 0; - spin_lock_irqsave(&lockres->l_lock, flags); - if (__lockres_remove_mask_waiter(lockres, &mw)) { - if (dlm_locked) - lockres_or_flags(lockres, - OCFS2_LOCK_NONBLOCK_FINISHED); - spin_unlock_irqrestore(&lockres->l_lock, flags); - ret = -EAGAIN; - } else { - spin_unlock_irqrestore(&lockres->l_lock, flags); - goto again; - } - } - if (wait) { - ret = ocfs2_wait_for_mask(&mw); - if (ret == 0) - goto again; - mlog_errno(ret); - } - ocfs2_update_lock_stats(lockres, level, &mw, ret); - -#if 0 && CONFIG_DEBUG_LOCK_ALLOC - if (!ret && lockres->l_lockdep_map.key != NULL) { - if (level == DLM_LOCK_PR) - rwsem_acquire_read(&lockres->l_lockdep_map, l_subclass, - !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), - caller_ip); - else - rwsem_acquire(&lockres->l_lockdep_map, l_subclass, - !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), - caller_ip); - } -#endif - return ret; -} - -int ocfs2_cluster_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - u32 lkm_flags, - int arg_flags) -{ - return __ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags, - 0, _RET_IP_); -} - -static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int level, - unsigned long caller_ip) -{ - unsigned long flags; - - trace_ocfs2_cluster_unlock(osb, lockres, level); - - spin_lock_irqsave(&lockres->l_lock, flags); - ocfs2_dec_holders(lockres, level); - ocfs2_downconvert_on_unlock(osb, lockres); - spin_unlock_irqrestore(&lockres->l_lock, flags); -#if 0 && CONFIG_DEBUG_LOCK_ALLOC - if (lockres->l_lockdep_map.key != NULL) - rwsem_release(&lockres->l_lockdep_map, 1, caller_ip); -#endif -} - -#if 0 -static int ocfs2_create_new_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int ex, - int local) -{ - int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - unsigned long flags; - u32 lkm_flags = local ? DLM_LKF_LOCAL : 0; - - spin_lock_irqsave(&lockres->l_lock, flags); - BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); - lockres_or_flags(lockres, OCFS2_LOCK_LOCAL); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - return ocfs2_lock_create(osb, lockres, level, lkm_flags); -} -#endif - -#if 0 -static int ocfs2_flock_handle_signal(struct ocfs2_lock_res *lockres, - int level) -{ - int ret; - struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); - unsigned long flags; - struct ocfs2_mask_waiter mw; - - ocfs2_init_mask_waiter(&mw); - -retry_cancel: - spin_lock_irqsave(&lockres->l_lock, flags); - if (lockres->l_flags & OCFS2_LOCK_BUSY) { - ret = ocfs2_prepare_cancel_convert(osb, lockres); - if (ret) { - spin_unlock_irqrestore(&lockres->l_lock, flags); - ret = ocfs2_cancel_convert(osb, lockres); - if (ret < 0) { - mlog_errno(ret); - goto out; - } - goto retry_cancel; - } - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - ocfs2_wait_for_mask(&mw); - goto retry_cancel; - } - - ret = -ERESTARTSYS; - /* - * We may still have gotten the lock, in which case there's no - * point to restarting the syscall. - */ - if (lockres->l_level == level) - ret = 0; - - mlog(0, "Cancel returning %d. flags: 0x%lx, level: %d, act: %d\n", ret, - lockres->l_flags, lockres->l_level, lockres->l_action); - - spin_unlock_irqrestore(&lockres->l_lock, flags); - -out: - return ret; -} - -/* - * ocfs2_file_lock() and ocfs2_file_unlock() map to a single pair of - * flock() calls. The locking approach this requires is sufficiently - * different from all other cluster lock types that we implement a - * separate path to the "low-level" dlm calls. In particular: - * - * - No optimization of lock levels is done - we take at exactly - * what's been requested. - * - * - No lock caching is employed. We immediately downconvert to - * no-lock at unlock time. This also means flock locks never go on - * the blocking list). - * - * - Since userspace can trivially deadlock itself with flock, we make - * sure to allow cancellation of a misbehaving applications flock() - * request. - * - * - Access to any flock lockres doesn't require concurrency, so we - * can simplify the code by requiring the caller to guarantee - * serialization of dlmglue flock calls. - */ -int ocfs2_file_lock(struct file *file, int ex, int trylock) -{ - int ret, level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; - unsigned int lkm_flags = trylock ? DLM_LKF_NOQUEUE : 0; - unsigned long flags; - struct ocfs2_file_private *fp = file->private_data; - struct ocfs2_lock_res *lockres = &fp->fp_flock; - struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); - struct ocfs2_mask_waiter mw; - - ocfs2_init_mask_waiter(&mw); - - if ((lockres->l_flags & OCFS2_LOCK_BUSY) || - (lockres->l_level > DLM_LOCK_NL)) { - mlog(ML_ERROR, - "File lock \"%s\" has busy or locked state: flags: 0x%lx, " - "level: %u\n", lockres->l_name, lockres->l_flags, - lockres->l_level); - return -EINVAL; - } - - spin_lock_irqsave(&lockres->l_lock, flags); - if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - /* - * Get the lock at NLMODE to start - that way we - * can cancel the upconvert request if need be. - */ - ret = ocfs2_lock_create(osb, lockres, DLM_LOCK_NL, 0); - if (ret < 0) { - mlog_errno(ret); - goto out; - } - - ret = ocfs2_wait_for_mask(&mw); - if (ret) { - mlog_errno(ret); - goto out; - } - spin_lock_irqsave(&lockres->l_lock, flags); - } - - lockres->l_action = OCFS2_AST_CONVERT; - lkm_flags |= DLM_LKF_CONVERT; - lockres->l_requested = level; - lockres_or_flags(lockres, OCFS2_LOCK_BUSY); - - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - ret = ocfs2_dlm_lock(osb->cconn, level, &lockres->l_lksb, lkm_flags, - lockres->l_name, OCFS2_LOCK_ID_MAX_LEN - 1); - if (ret) { - if (!trylock || (ret != -EAGAIN)) { - ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); - ret = -EINVAL; - } - - ocfs2_recover_from_dlm_error(lockres, 1); - lockres_remove_mask_waiter(lockres, &mw); - goto out; - } - - ret = ocfs2_wait_for_mask_interruptible(&mw, lockres); - if (ret == -ERESTARTSYS) { - /* - * Userspace can cause deadlock itself with - * flock(). Current behavior locally is to allow the - * deadlock, but abort the system call if a signal is - * received. We follow this example, otherwise a - * poorly written program could sit in kernel until - * reboot. - * - * Handling this is a bit more complicated for Ocfs2 - * though. We can't exit this function with an - * outstanding lock request, so a cancel convert is - * required. We intentionally overwrite 'ret' - if the - * cancel fails and the lock was granted, it's easier - * to just bubble success back up to the user. - */ - ret = ocfs2_flock_handle_signal(lockres, level); - } else if (!ret && (level > lockres->l_level)) { - /* Trylock failed asynchronously */ - BUG_ON(!trylock); - ret = -EAGAIN; - } - -out: - - mlog(0, "Lock: \"%s\" ex: %d, trylock: %d, returns: %d\n", - lockres->l_name, ex, trylock, ret); - return ret; -} - -void ocfs2_file_unlock(struct file *file) -{ - int ret; - unsigned int gen; - unsigned long flags; - struct ocfs2_file_private *fp = file->private_data; - struct ocfs2_lock_res *lockres = &fp->fp_flock; - struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); - struct ocfs2_mask_waiter mw; - - ocfs2_init_mask_waiter(&mw); - - if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) - return; - - if (lockres->l_level == DLM_LOCK_NL) - return; - - mlog(0, "Unlock: \"%s\" flags: 0x%lx, level: %d, act: %d\n", - lockres->l_name, lockres->l_flags, lockres->l_level, - lockres->l_action); - - spin_lock_irqsave(&lockres->l_lock, flags); - /* - * Fake a blocking ast for the downconvert code. - */ - lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); - lockres->l_blocking = DLM_LOCK_EX; - - gen = ocfs2_prepare_downconvert(lockres, DLM_LOCK_NL); - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - ret = ocfs2_downconvert_lock(osb, lockres, DLM_LOCK_NL, 0, gen); - if (ret) { - mlog_errno(ret); - return; - } - - ret = ocfs2_wait_for_mask(&mw); - if (ret) - mlog_errno(ret); -} -#endif - -static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int kick = 0; - - /* If we know that another node is waiting on our lock, kick - * the downconvert thread * pre-emptively when we reach a release - * condition. */ - if (lockres->l_flags & OCFS2_LOCK_BLOCKED) { - switch(lockres->l_blocking) { - case DLM_LOCK_EX: - if (!lockres_has_holders(lockres, H_ANY)) - kick = 1; - break; - case DLM_LOCK_PR: - if (!lockres_has_holders(lockres, H_EX|H_CW)) - kick = 1; - break; - case DLM_LOCK_CW: - if (!lockres_has_holders(lockres, H_EX|H_PR)) - kick = 1; - break; - default: - BUG(); - } - } - - if (kick) - ocfs2_wake_downconvert_thread(osb); -} - -#if 0 -/* Determine whether a lock resource needs to be refreshed, and - * arbitrate who gets to refresh it. - * - * 0 means no refresh needed. - * - * > 0 means you need to refresh this and you MUST call - * ocfs2_complete_lock_res_refresh afterwards. */ -static int ocfs2_should_refresh_lock_res(struct ocfs2_lock_res *lockres) -{ - unsigned long flags; - int status = 0; - -refresh_check: - spin_lock_irqsave(&lockres->l_lock, flags); - if (!(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) { - spin_unlock_irqrestore(&lockres->l_lock, flags); - goto bail; - } - - if (lockres->l_flags & OCFS2_LOCK_REFRESHING) { - spin_unlock_irqrestore(&lockres->l_lock, flags); - - ocfs2_wait_on_refreshing_lock(lockres); - goto refresh_check; - } - - /* Ok, I'll be the one to refresh this lock. */ - lockres_or_flags(lockres, OCFS2_LOCK_REFRESHING); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - status = 1; -bail: - mlog(0, "status %d\n", status); - return status; -} -#endif - -/* If status is non zero, I'll mark it as not being in refresh - * anymroe, but i won't clear the needs refresh flag. */ -static inline void ocfs2_complete_lock_res_refresh(struct ocfs2_lock_res *lockres, - int status) -{ - unsigned long flags; - - spin_lock_irqsave(&lockres->l_lock, flags); - lockres_clear_flags(lockres, OCFS2_LOCK_REFRESHING); - if (!status) - lockres_clear_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - wake_up(&lockres->l_event); -} - -u64 ocfs2_lock_refresh_gen(struct ocfs2_lock_res *lockres) -{ - return lockres->l_refresh_gen; -} - -/* Reference counting of the dlm debug structure. We want this because - * open references on the debug inodes can live on after a mount, so - * we can't rely on the ocfs2_super to always exist. */ -static void ocfs2_dlm_debug_free(struct kref *kref) -{ - struct ocfs2_dlm_debug *dlm_debug; - - dlm_debug = container_of(kref, struct ocfs2_dlm_debug, d_refcnt); - - kfree(dlm_debug); -} - -void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug) -{ - if (dlm_debug) - kref_put(&dlm_debug->d_refcnt, ocfs2_dlm_debug_free); -} - -static void ocfs2_get_dlm_debug(struct ocfs2_dlm_debug *debug) -{ - kref_get(&debug->d_refcnt); -} - -struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void) -{ - struct ocfs2_dlm_debug *dlm_debug; - - dlm_debug = kmalloc(sizeof(struct ocfs2_dlm_debug), GFP_KERNEL); - if (!dlm_debug) { - mlog_errno(-ENOMEM); - goto out; - } - - kref_init(&dlm_debug->d_refcnt); - INIT_LIST_HEAD(&dlm_debug->d_lockres_tracking); - dlm_debug->d_locking_state = NULL; -out: - return dlm_debug; -} - -/* Access to this is arbitrated for us via seq_file->sem. */ -struct ocfs2_dlm_seq_priv { - struct ocfs2_dlm_debug *p_dlm_debug; - struct ocfs2_lock_res p_iter_res; - struct ocfs2_lock_res p_tmp_res; -}; - -static struct ocfs2_lock_res *ocfs2_dlm_next_res(struct ocfs2_lock_res *start, - struct ocfs2_dlm_seq_priv *priv) -{ - struct ocfs2_lock_res *iter, *ret = NULL; - struct ocfs2_dlm_debug *dlm_debug = priv->p_dlm_debug; - - assert_spin_locked(&ocfs2_dlm_tracking_lock); - - list_for_each_entry(iter, &start->l_debug_list, l_debug_list) { - /* discover the head of the list */ - if (&iter->l_debug_list == &dlm_debug->d_lockres_tracking) { - mlog(0, "End of list found, %p\n", ret); - break; - } - - /* We track our "dummy" iteration lockres' by a NULL - * l_ops field. */ - if (iter->l_ops != NULL) { - ret = iter; - break; - } - } - - return ret; -} - -static void *ocfs2_dlm_seq_start(struct seq_file *m, loff_t *pos) -{ - struct ocfs2_dlm_seq_priv *priv = m->private; - struct ocfs2_lock_res *iter; - - spin_lock(&ocfs2_dlm_tracking_lock); - iter = ocfs2_dlm_next_res(&priv->p_iter_res, priv); - if (iter) { - /* Since lockres' have the lifetime of their container - * (which can be inodes, ocfs2_supers, etc) we want to - * copy this out to a temporary lockres while still - * under the spinlock. Obviously after this we can't - * trust any pointers on the copy returned, but that's - * ok as the information we want isn't typically held - * in them. */ - priv->p_tmp_res = *iter; - iter = &priv->p_tmp_res; - } - spin_unlock(&ocfs2_dlm_tracking_lock); - - return iter; -} - -static void ocfs2_dlm_seq_stop(struct seq_file *m, void *v) -{ -} - -static void *ocfs2_dlm_seq_next(struct seq_file *m, void *v, loff_t *pos) -{ - struct ocfs2_dlm_seq_priv *priv = m->private; - struct ocfs2_lock_res *iter = v; - struct ocfs2_lock_res *dummy = &priv->p_iter_res; - - spin_lock(&ocfs2_dlm_tracking_lock); - iter = ocfs2_dlm_next_res(iter, priv); - list_del_init(&dummy->l_debug_list); - if (iter) { - list_add(&dummy->l_debug_list, &iter->l_debug_list); - priv->p_tmp_res = *iter; - iter = &priv->p_tmp_res; - } - spin_unlock(&ocfs2_dlm_tracking_lock); - - return iter; -} - -/* - * Version is used by debugfs.ocfs2 to determine the format being used - * - * New in version 2 - * - Lock stats printed - * New in version 3 - * - Max time in lock stats is in usecs (instead of nsecs) - */ -#define OCFS2_DLM_DEBUG_STR_VERSION 3 -static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) -{ - int i; - char *lvb; - struct ocfs2_lock_res *lockres = v; - char lockname[256]; - - if (!lockres) - return -EINVAL; - - lockres_name(lockres, lockname, 256); - - seq_printf(m, "0x%x\t%s\t", OCFS2_DLM_DEBUG_STR_VERSION, lockname); - - seq_printf(m, "%d\t" - "0x%lx\t" - "0x%x\t" - "0x%x\t" - "%u\t" - "%u\t" - "%d\t" - "%d\t", - lockres->l_level, - lockres->l_flags, - lockres->l_action, - lockres->l_unlock_action, - lockres->l_ro_holders, - lockres->l_ex_holders, - lockres->l_requested, - lockres->l_blocking); - - /* Dump the raw LVB */ - lvb = ocfs2_dlm_lvb(&lockres->l_lksb); - for(i = 0; i < DLM_LVB_LEN; i++) - seq_printf(m, "0x%x\t", lvb[i]); - -#ifdef CONFIG_OCFS2_FS_STATS -# define lock_num_cwmode(_l) ((_l)->l_lock_cwmode.ls_gets) -# define lock_num_prmode(_l) ((_l)->l_lock_prmode.ls_gets) -# define lock_num_exmode(_l) ((_l)->l_lock_exmode.ls_gets) -# define lock_num_cwmode_failed(_l) ((_l)->l_lock_cwmode.ls_fail) -# define lock_num_prmode_failed(_l) ((_l)->l_lock_prmode.ls_fail) -# define lock_num_exmode_failed(_l) ((_l)->l_lock_exmode.ls_fail) -# define lock_total_cwmode(_l) ((_l)->l_lock_cwmode.ls_total) -# define lock_total_prmode(_l) ((_l)->l_lock_prmode.ls_total) -# define lock_total_exmode(_l) ((_l)->l_lock_exmode.ls_total) -# define lock_max_cwmode(_l) ((_l)->l_lock_cwmode.ls_max) -# define lock_max_prmode(_l) ((_l)->l_lock_prmode.ls_max) -# define lock_max_exmode(_l) ((_l)->l_lock_exmode.ls_max) -# define lock_refresh(_l) ((_l)->l_lock_refresh) -#else -# define lock_num_cwmode(_l) (0) -# define lock_num_prmode(_l) (0) -# define lock_num_exmode(_l) (0) -# define lock_num_cwmode_failed(_l) (0) -# define lock_num_prmode_failed(_l) (0) -# define lock_num_exmode_failed(_l) (0) -# define lock_total_cwmode(_l) (0ULL) -# define lock_total_prmode(_l) (0ULL) -# define lock_total_exmode(_l) (0ULL) -# define lock_max_cwmode(_l) (0) -# define lock_max_prmode(_l) (0) -# define lock_max_exmode(_l) (0) -# define lock_refresh(_l) (0) -#endif - /* The following seq_print was added in version 2 of this output */ - seq_printf(m, "%u\t" - "%u\t" - "%u\t" - "%u\t" - "%llu\t" - "%llu\t" - "%u\t" - "%u\t" - "%u\t", - lock_num_prmode(lockres), - lock_num_exmode(lockres), - lock_num_prmode_failed(lockres), - lock_num_exmode_failed(lockres), - lock_total_prmode(lockres), - lock_total_exmode(lockres), - lock_max_prmode(lockres), - lock_max_exmode(lockres), - lock_refresh(lockres)); - - seq_printf(m, "%u\t" - "%u\t" - "%u\t" - "%llu\t" - "%u\t", - lockres->l_cw_holders, - lock_num_cwmode(lockres), - lock_num_cwmode_failed(lockres), - lock_total_cwmode(lockres), - lock_max_cwmode(lockres)); - - /* End the line */ - seq_printf(m, "\n"); - return 0; -} - -static const struct seq_operations ocfs2_dlm_seq_ops = { - .start = ocfs2_dlm_seq_start, - .stop = ocfs2_dlm_seq_stop, - .next = ocfs2_dlm_seq_next, - .show = ocfs2_dlm_seq_show, -}; - -static int ocfs2_dlm_debug_release(struct inode *inode, struct file *file) -{ - struct seq_file *seq = file->private_data; - struct ocfs2_dlm_seq_priv *priv = seq->private; - struct ocfs2_lock_res *res = &priv->p_iter_res; - - ocfs2_remove_lockres_tracking(res); - ocfs2_put_dlm_debug(priv->p_dlm_debug); - return seq_release_private(inode, file); -} - -static int ocfs2_dlm_debug_open(struct inode *inode, struct file *file) -{ - struct ocfs2_dlm_seq_priv *priv; - struct ocfs2_super *osb; - - priv = __seq_open_private(file, &ocfs2_dlm_seq_ops, sizeof(*priv)); - if (!priv) { - mlog_errno(-ENOMEM); - return -ENOMEM; - } - - osb = inode->i_private; - ocfs2_get_dlm_debug(osb->osb_dlm_debug); - priv->p_dlm_debug = osb->osb_dlm_debug; - INIT_LIST_HEAD(&priv->p_iter_res.l_debug_list); - - ocfs2_add_lockres_tracking(&priv->p_iter_res, - priv->p_dlm_debug); - - return 0; -} - -static const struct file_operations ocfs2_dlm_debug_fops = { - .open = ocfs2_dlm_debug_open, - .release = ocfs2_dlm_debug_release, - .read = seq_read, - .llseek = seq_lseek, -}; - -static int ocfs2_dlm_init_debug(struct ocfs2_super *osb, - struct dentry *debug_root) -{ - int ret = 0; - struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; - - dlm_debug->d_locking_state = debugfs_create_file("locking_state", - S_IFREG|S_IRUSR, - debug_root, - osb, - &ocfs2_dlm_debug_fops); - if (!dlm_debug->d_locking_state) { - ret = -EINVAL; - mlog(ML_ERROR, - "Unable to create locking state debugfs file.\n"); - goto out; - } - - ocfs2_get_dlm_debug(dlm_debug); -out: - return ret; -} - -static void ocfs2_dlm_shutdown_debug(struct ocfs2_super *osb) -{ - struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; - - if (dlm_debug) { - debugfs_remove(dlm_debug->d_locking_state); - ocfs2_put_dlm_debug(dlm_debug); - } -} - -static void ocfs2_do_node_down(int node_num, void *data) -{ -} - -int ocfs2_dlm_init(struct ocfs2_super *osb, struct super_block *sb, - char *cluster_stack, char *cluster_name, char *ls_name, - struct dentry *debug_root) -{ - int status = 0; - struct ocfs2_cluster_connection *conn = NULL; - -#if 0 - if (ocfs2_mount_local(osb)) { - osb->node_num = 0; - goto local; - } -#endif - osb->sb = sb; - - status = ocfs2_dlm_init_debug(osb, debug_root); - if (status < 0) { - mlog_errno(status); - goto bail; - } - - /* launch downconvert thread */ - osb->dc_task = kthread_run(ocfs2_downconvert_thread, osb, "scoutdc-%s", - ls_name); - if (IS_ERR(osb->dc_task)) { - status = PTR_ERR(osb->dc_task); - osb->dc_task = NULL; - mlog_errno(status); - goto bail; - } - - /* for now, uuid == domain */ - status = ocfs2_cluster_connect(cluster_stack, - cluster_name, - strlen(cluster_name), - ls_name, - strlen(ls_name), - &lproto, ocfs2_do_node_down, osb, - &conn); - if (status) { - mlog_errno(status); - goto bail; - } - -#if 0 - status = ocfs2_cluster_this_node(conn, &osb->node_num); - if (status < 0) { - mlog_errno(status); - mlog(ML_ERROR, - "could not find this host's node number\n"); - ocfs2_cluster_disconnect(conn, 0); - goto bail; - } - -local: - ocfs2_super_lock_res_init(&osb->osb_super_lockres, osb); - ocfs2_rename_lock_res_init(&osb->osb_rename_lockres, osb); - ocfs2_nfs_sync_lock_res_init(&osb->osb_nfs_sync_lockres, osb); - ocfs2_orphan_scan_lock_res_init(&osb->osb_orphan_scan.os_lockres, osb); -#endif - osb->cconn = conn; -bail: - if (status < 0) { - ocfs2_dlm_shutdown_debug(osb); - if (osb->dc_task) - kthread_stop(osb->dc_task); - } - - return status; -} - -void ocfs2_dlm_shutdown(struct ocfs2_super *osb, - int hangup_pending) -{ -// ocfs2_drop_osb_locks(osb); - - /* - * Now that we have dropped all locks and ocfs2_dismount_volume() - * has disabled recovery, the DLM won't be talking to us. It's - * safe to tear things down before disconnecting the cluster. - */ - - if (osb->dc_task) { - kthread_stop(osb->dc_task); - osb->dc_task = NULL; - } - -#if 0 - ocfs2_lock_res_free(&osb->osb_super_lockres); - ocfs2_lock_res_free(&osb->osb_rename_lockres); - ocfs2_lock_res_free(&osb->osb_nfs_sync_lockres); - ocfs2_lock_res_free(&osb->osb_orphan_scan.os_lockres); -#endif - - ocfs2_cluster_disconnect(osb->cconn, hangup_pending); - osb->cconn = NULL; - - ocfs2_dlm_shutdown_debug(osb); -} - -static int ocfs2_drop_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int ret; - unsigned long flags; - u32 lkm_flags = 0; - - /* We didn't get anywhere near actually using this lockres. */ - if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) - goto out; - - if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) - lkm_flags |= DLM_LKF_VALBLK; - - spin_lock_irqsave(&lockres->l_lock, flags); - - mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_FREEING), - "lockres %s, flags 0x%lx\n", - lockres->l_name, lockres->l_flags); - - while (lockres->l_flags & OCFS2_LOCK_BUSY) { - mlog(0, "waiting on busy lock \"%s\": flags = %lx, action = " - "%u, unlock_action = %u\n", - lockres->l_name, lockres->l_flags, lockres->l_action, - lockres->l_unlock_action); - - spin_unlock_irqrestore(&lockres->l_lock, flags); - - /* XXX: Today we just wait on any busy - * locks... Perhaps we need to cancel converts in the - * future? */ - ocfs2_wait_on_busy_lock(lockres); - - spin_lock_irqsave(&lockres->l_lock, flags); - } - - if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { - if (lockres->l_flags & OCFS2_LOCK_ATTACHED && - lockres->l_level == DLM_LOCK_EX && - !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) - lockres->l_ops->set_lvb(lockres); - } - - if (lockres->l_flags & OCFS2_LOCK_BUSY) - mlog(ML_ERROR, "destroying busy lock: \"%s\"\n", - lockres->l_name); - if (lockres->l_flags & OCFS2_LOCK_BLOCKED) - mlog(0, "destroying blocked lock: \"%s\"\n", lockres->l_name); - - if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { - spin_unlock_irqrestore(&lockres->l_lock, flags); - goto out; - } - - lockres_clear_flags(lockres, OCFS2_LOCK_ATTACHED); - - /* make sure we never get here while waiting for an ast to - * fire. */ - BUG_ON(lockres->l_action != OCFS2_AST_INVALID); - - /* is this necessary? */ - lockres_or_flags(lockres, OCFS2_LOCK_BUSY); - lockres->l_unlock_action = OCFS2_UNLOCK_DROP_LOCK; - spin_unlock_irqrestore(&lockres->l_lock, flags); - - mlog(0, "lock %s\n", lockres->l_name); - - ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, lkm_flags); - if (ret) { - ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); - mlog(ML_ERROR, "lockres flags: %lu\n", lockres->l_flags); - ocfs2_dlm_dump_lksb(&lockres->l_lksb); - BUG(); - } - mlog(0, "lock %s, successful return from ocfs2_dlm_unlock\n", - lockres->l_name); - - scoutfs_inc_counter(osb->sb, dlm_unlock_request); - lockres_notify_event(lockres, EVENT_DLM_UNLOCK); - - ocfs2_wait_on_busy_lock(lockres); -out: - return 0; -} - -static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); - -/* Mark the lockres as being dropped. It will no longer be - * queued if blocking, but we still may have to wait on it - * being dequeued from the downconvert thread before we can consider - * it safe to drop. - * - * You can *not* attempt to call cluster_lock on this lockres anymore. */ -void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int status; - struct ocfs2_mask_waiter mw; - unsigned long flags, flags2; - - ocfs2_init_mask_waiter(&mw); - - spin_lock_irqsave(&lockres->l_lock, flags); - lockres->l_flags |= OCFS2_LOCK_FREEING; - if (lockres->l_flags & OCFS2_LOCK_QUEUED && current == osb->dc_task) { - /* - * We know the downconvert is queued but not in progress - * because we are the downconvert thread and processing - * different lock. So we can just remove the lock from the - * queue. This is not only an optimization but also a way - * to avoid the following deadlock: - * ocfs2_dentry_post_unlock() - * ocfs2_dentry_lock_put() - * ocfs2_drop_dentry_lock() - * iput() - * ocfs2_evict_inode() - * ocfs2_clear_inode() - * ocfs2_mark_lockres_freeing() - * ... blocks waiting for OCFS2_LOCK_QUEUED - * since we are the downconvert thread which - * should clear the flag. - */ - spin_unlock_irqrestore(&lockres->l_lock, flags); - spin_lock_irqsave(&osb->dc_task_lock, flags2); - list_del_init(&lockres->l_blocked_list); - osb->blocked_lock_count--; - spin_unlock_irqrestore(&osb->dc_task_lock, flags2); - /* - * Warn if we recurse into another post_unlock call. Strictly - * speaking it isn't a problem but we need to be careful if - * that happens (stack overflow, deadlocks, ...) so warn if - * ocfs2 grows a path for which this can happen. - */ - WARN_ON_ONCE(lockres->l_ops->post_unlock); - /* Since the lock is freeing we don't do much in the fn below */ - ocfs2_process_blocked_lock(osb, lockres); - return; - } - while (lockres->l_flags & OCFS2_LOCK_QUEUED) { - lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_QUEUED, 0); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - mlog(0, "Waiting on lockres %s\n", lockres->l_name); - - status = ocfs2_wait_for_mask(&mw); - if (status) - mlog_errno(status); - - spin_lock_irqsave(&lockres->l_lock, flags); - } - spin_unlock_irqrestore(&lockres->l_lock, flags); -} - -void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int ret; - - trace_ocfs2_simple_drop_lockres(osb, lockres); - - ocfs2_mark_lockres_freeing(osb, lockres); - - if (lockres->l_ops->drop_worker) - lockres->l_ops->drop_worker(lockres); - - ret = ocfs2_drop_lock(osb, lockres); - if (ret) - mlog_errno(ret); -} - -static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, - int new_level) -{ - assert_spin_locked(&lockres->l_lock); - - BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); - - if (lockres->l_level <= new_level) { - mlog(ML_ERROR, "lockres %s, lvl %d <= %d, blcklst %d, mask %d, " - "flags 0x%lx, hold %d %d, act %d %d, req %d, " - "block %d, pgen %d\n", lockres->l_name, lockres->l_level, - new_level, list_empty(&lockres->l_blocked_list), - list_empty(&lockres->l_mask_waiters), - lockres->l_flags, lockres->l_ro_holders, - lockres->l_ex_holders, lockres->l_action, - lockres->l_unlock_action, lockres->l_requested, - lockres->l_blocking, lockres->l_pending_gen); - BUG(); - } - - mlog(ML_BASTS, "lockres %s, level %d => %d, blocking %d\n", - lockres->l_name, lockres->l_level, new_level, lockres->l_blocking); - - lockres->l_action = OCFS2_AST_DOWNCONVERT; - lockres->l_requested = new_level; - lockres_or_flags(lockres, OCFS2_LOCK_BUSY); - return lockres_set_pending(lockres); -} - -static int ocfs2_downconvert_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - int new_level, - int lvb, - unsigned int generation) -{ - int ret; - u32 dlm_flags = DLM_LKF_CONVERT; - - mlog(ML_BASTS, "lockres %s, level %d => %d\n", lockres->l_name, - lockres->l_level, new_level); - - /* - * On DLM_LKF_VALBLK, fsdlm behaves differently with o2cb. It always - * expects DLM_LKF_VALBLK being set if the LKB has LVB, so that - * we can recover correctly from node failure. Otherwise, we may get - * invalid LVB in LKB, but without DLM_SBF_VALNOTVALID being set. - */ - if (!ocfs2_is_o2cb_active() && - lockres->l_ops->flags & LOCK_TYPE_USES_LVB) - lvb = 1; - - if (lvb) - dlm_flags |= DLM_LKF_VALBLK; - - ret = ocfs2_dlm_lock(osb->cconn, - new_level, - &lockres->l_lksb, - dlm_flags, - lockres->l_name, - OCFS2_LOCK_ID_MAX_LEN - 1); - lockres_clear_pending(lockres, generation, osb); - if (ret) { - ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); - ocfs2_recover_from_dlm_error(lockres, 1); - goto bail; - } - - ret = 0; -bail: - return ret; -} - -/* returns 1 when the caller should unlock and call ocfs2_dlm_unlock */ -static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - assert_spin_locked(&lockres->l_lock); - - if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) { - /* If we're already trying to cancel a lock conversion - * then just drop the spinlock and allow the caller to - * requeue this lock. */ - mlog(ML_BASTS, "lockres %s, skip convert\n", lockres->l_name); - return 0; - } - - /* were we in a convert when we got the bast fire? */ - BUG_ON(lockres->l_action != OCFS2_AST_CONVERT && - lockres->l_action != OCFS2_AST_DOWNCONVERT); - /* set things up for the unlockast to know to just - * clear out the ast_action and unset busy, etc. */ - lockres->l_unlock_action = OCFS2_UNLOCK_CANCEL_CONVERT; - - mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_BUSY), - "lock %s, invalid flags: 0x%lx\n", - lockres->l_name, lockres->l_flags); - - mlog(ML_BASTS, "lockres %s\n", lockres->l_name); - - return 1; -} - -static int ocfs2_cancel_convert(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int ret; - - ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, - DLM_LKF_CANCEL); - if (ret) { - ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); - ocfs2_recover_from_dlm_error(lockres, 0); - } - - mlog(ML_BASTS, "lockres %s\n", lockres->l_name); - - scoutfs_inc_counter(osb->sb, dlm_cancel_convert); - lockres_notify_event(lockres, EVENT_DLM_CONVERT); - - return ret; -} - -static int ocfs2_unblock_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, - struct ocfs2_unblock_ctl *ctl) -{ - unsigned long flags; - int blocking; - int new_level; - int level; - int ret = 0; - int set_lvb = 0; - unsigned int gen; - - - spin_lock_irqsave(&lockres->l_lock, flags); - -recheck: - trace_ocfs2_unblock_lock(osb, lockres); - - /* - * Is it still blocking? If not, we have no more work to do. - */ - if (!(lockres->l_flags & OCFS2_LOCK_BLOCKED)) { - BUG_ON(lockres->l_blocking != DLM_LOCK_NL); - spin_unlock_irqrestore(&lockres->l_lock, flags); - ret = 0; - goto leave; - } - - if (lockres->l_flags & OCFS2_LOCK_BUSY) { - /* XXX - * This is a *big* race. The OCFS2_LOCK_PENDING flag - * exists entirely for one reason - another thread has set - * OCFS2_LOCK_BUSY, but has *NOT* yet called dlm_lock(). - * - * If we do ocfs2_cancel_convert() before the other thread - * calls dlm_lock(), our cancel will do nothing. We will - * get no ast, and we will have no way of knowing the - * cancel failed. Meanwhile, the other thread will call - * into dlm_lock() and wait...forever. - * - * Why forever? Because another node has asked for the - * lock first; that's why we're here in unblock_lock(). - * - * The solution is OCFS2_LOCK_PENDING. When PENDING is - * set, we just requeue the unblock. Only when the other - * thread has called dlm_lock() and cleared PENDING will - * we then cancel their request. - * - * All callers of dlm_lock() must set OCFS2_DLM_PENDING - * at the same time they set OCFS2_DLM_BUSY. They must - * clear OCFS2_DLM_PENDING after dlm_lock() returns. - */ - if (lockres->l_flags & OCFS2_LOCK_PENDING) { - mlog(ML_BASTS, "lockres %s, ReQ: Pending\n", - lockres->l_name); - goto leave_requeue; - } - - ctl->requeue = 1; - ret = ocfs2_prepare_cancel_convert(osb, lockres); - spin_unlock_irqrestore(&lockres->l_lock, flags); - if (ret) { - ret = ocfs2_cancel_convert(osb, lockres); - if (ret < 0) - mlog_errno(ret); - } - goto leave; - } - - /* - * This prevents livelocks. OCFS2_LOCK_UPCONVERT_FINISHING flag is - * set when the ast is received for an upconvert just before the - * OCFS2_LOCK_BUSY flag is cleared. Now if the fs received a bast - * on the heels of the ast, we want to delay the downconvert just - * enough to allow the up requestor to do its task. Because this - * lock is in the blocked queue, the lock will be downconverted - * as soon as the requestor is done with the lock. - */ - if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) - goto leave_requeue; - - /* - * How can we block and yet be at NL? We were trying to upconvert - * from NL and got canceled. The code comes back here, and now - * we notice and clear BLOCKING. - */ - if (lockres->l_level == DLM_LOCK_NL) { - BUG_ON(lockres_has_holders(lockres, H_ANY)); - mlog(ML_BASTS, "lockres %s, Aborting dc\n", lockres->l_name); - lockres->l_blocking = DLM_LOCK_NL; - lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); - spin_unlock_irqrestore(&lockres->l_lock, flags); - goto leave; - } - - /* if we're blocking an exclusive and we have *any* holders, - * then requeue. */ - if (lockres->l_blocking == DLM_LOCK_EX && - lockres_has_holders(lockres, H_ANY)) { - mlog(ML_BASTS, "lockres %s, ReQ: EX/PR/CW Holders %u,%u\n", - lockres->l_name, lockres->l_ex_holders, - lockres->l_ro_holders, lockres->l_cw_holders); - goto leave_requeue; - } - - /* If it's a PR we're blocking, then only - * requeue if we've got any EX or CW holders */ - if (lockres->l_blocking == DLM_LOCK_PR && - lockres_has_holders(lockres, H_CW|H_EX)) { - mlog(ML_BASTS, "lockres %s, ReQ: EX/CW Holders %u,%u\n", - lockres->l_name, lockres->l_ex_holders, - lockres->l_cw_holders); - goto leave_requeue; - } - - /* - * Same logic as above, we're checking for any holders that - * are incompatible with CW. - */ - if (lockres->l_blocking == DLM_LOCK_CW - && lockres_has_holders(lockres, H_EX|H_PR)) { - mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", - lockres->l_name, lockres->l_ex_holders, - lockres->l_ro_holders); - goto leave_requeue; - } - - /* - * Can we get a lock in this state if the holder counts are - * zero? The meta data unblock code used to check this. - */ - if ((lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) - && (lockres->l_flags & OCFS2_LOCK_REFRESHING)) { - mlog(ML_BASTS, "lockres %s, ReQ: Lock Refreshing\n", - lockres->l_name); - goto leave_requeue; - } - - new_level = ocfs2_downconvert_level(lockres, lockres->l_blocking); - - if (lockres->l_ops->check_downconvert - && !lockres->l_ops->check_downconvert(lockres, new_level)) { - mlog(ML_BASTS, "lockres %s, ReQ: Checkpointing\n", - lockres->l_name); - goto leave_requeue; - } - - - /* Some lockres types want to do a bit of work before - * downconverting a lock. Allow that here. The worker function - * may sleep, so we save off a copy of what we're blocking as - * it may change while we're not holding the spin lock. */ - blocking = lockres->l_blocking; - level = lockres->l_level; - - /* If we get here, then we know that there are no more - * incompatible holders (and anyone asking for an incompatible - * lock is blocked). We can now downconvert the lock */ - if (!lockres->l_ops->downconvert_worker) - goto downconvert; - - spin_unlock_irqrestore(&lockres->l_lock, flags); - - lockres_notify_event(lockres, EVENT_DLM_DOWNCONVERT_WORK); - - ctl->unblock_action = lockres->l_ops->downconvert_worker(lockres, blocking); - - if (ctl->unblock_action == UNBLOCK_STOP_POST) { - mlog(ML_BASTS, "lockres %s, UNBLOCK_STOP_POST\n", - lockres->l_name); - goto leave; - } - - spin_lock_irqsave(&lockres->l_lock, flags); - if ((blocking != lockres->l_blocking) || (level != lockres->l_level)) { - /* If this changed underneath us, then we can't drop - * it just yet. */ - mlog(ML_BASTS, "lockres %s, block=%d:%d, level=%d:%d, " - "Recheck\n", lockres->l_name, blocking, - lockres->l_blocking, level, lockres->l_level); - goto recheck; - } - -downconvert: - ctl->requeue = 0; - - if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { - if (lockres->l_level == DLM_LOCK_EX) - set_lvb = 1; - - /* - * We only set the lvb if the lock has been fully - * refreshed - otherwise we risk setting stale - * data. Otherwise, there's no need to actually clear - * out the lvb here as it's value is still valid. - */ - if (set_lvb && !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) - lockres->l_ops->set_lvb(lockres); - } - - gen = ocfs2_prepare_downconvert(lockres, new_level); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - switch (level) { - case DLM_LOCK_EX: - scoutfs_inc_counter(osb->sb, dlm_ex_downconvert); - break; - case DLM_LOCK_PR: - scoutfs_inc_counter(osb->sb, dlm_pr_downconvert); - break; - case DLM_LOCK_CW: - scoutfs_inc_counter(osb->sb, dlm_cw_downconvert); - break; - } - - ret = ocfs2_downconvert_lock(osb, lockres, new_level, set_lvb, - gen); - -leave: - if (ret) - mlog_errno(ret); - return ret; - -leave_requeue: - spin_unlock_irqrestore(&lockres->l_lock, flags); - ctl->requeue = 1; - - return 0; -} - -static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - int status; - struct ocfs2_unblock_ctl ctl = {0, 0,}; - unsigned long flags; - - /* Our reference to the lockres in this function can be - * considered valid until we remove the OCFS2_LOCK_QUEUED - * flag. */ - - BUG_ON(!lockres); - BUG_ON(!lockres->l_ops); - - mlog(ML_BASTS, "lockres %s blocked\n", lockres->l_name); - - /* Detect whether a lock has been marked as going away while - * the downconvert thread was processing other things. A lock can - * still be marked with OCFS2_LOCK_FREEING after this check, - * but short circuiting here will still save us some - * performance. */ - spin_lock_irqsave(&lockres->l_lock, flags); - if (lockres->l_flags & OCFS2_LOCK_FREEING) - goto unqueue; - spin_unlock_irqrestore(&lockres->l_lock, flags); - - status = ocfs2_unblock_lock(osb, lockres, &ctl); - if (status < 0) - mlog_errno(status); - - spin_lock_irqsave(&lockres->l_lock, flags); -unqueue: - if (lockres->l_flags & OCFS2_LOCK_FREEING || !ctl.requeue) { - lockres_clear_flags(lockres, OCFS2_LOCK_QUEUED); - } else - ocfs2_schedule_blocked_lock(osb, lockres); - - mlog(ML_BASTS, "lockres %s, requeue = %s.\n", lockres->l_name, - ctl.requeue ? "yes" : "no"); - spin_unlock_irqrestore(&lockres->l_lock, flags); - - if (ctl.unblock_action != UNBLOCK_CONTINUE - && lockres->l_ops->post_unlock) - lockres->l_ops->post_unlock(osb, lockres); -} - -static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres) -{ - unsigned long flags; - - assert_spin_locked(&lockres->l_lock); - - if (lockres->l_flags & OCFS2_LOCK_FREEING) { - /* Do not schedule a lock for downconvert when it's on - * the way to destruction - any nodes wanting access - * to the resource will get it soon. */ - mlog(ML_BASTS, "lockres %s won't be scheduled: flags 0x%lx\n", - lockres->l_name, lockres->l_flags); - return; - } - - lockres_or_flags(lockres, OCFS2_LOCK_QUEUED); - - spin_lock_irqsave(&osb->dc_task_lock, flags); - if (list_empty(&lockres->l_blocked_list)) { - list_add_tail(&lockres->l_blocked_list, - &osb->blocked_lock_list); - osb->blocked_lock_count++; - } - spin_unlock_irqrestore(&osb->dc_task_lock, flags); -} - -static void ocfs2_downconvert_thread_do_work(struct ocfs2_super *osb) -{ - unsigned long processed; - unsigned long flags; - struct ocfs2_lock_res *lockres; - - spin_lock_irqsave(&osb->dc_task_lock, flags); - /* grab this early so we know to try again if a state change and - * wake happens part-way through our work */ - osb->dc_work_sequence = osb->dc_wake_sequence; - - processed = osb->blocked_lock_count; - /* - * blocked lock processing in this loop might call iput which can - * remove items off osb->blocked_lock_list. Downconvert up to - * 'processed' number of locks, but stop short if we had some - * removed in ocfs2_mark_lockres_freeing when downconverting. - */ - while (processed && !list_empty(&osb->blocked_lock_list)) { - lockres = list_entry(osb->blocked_lock_list.next, - struct ocfs2_lock_res, l_blocked_list); - list_del_init(&lockres->l_blocked_list); - osb->blocked_lock_count--; - spin_unlock_irqrestore(&osb->dc_task_lock, flags); - - BUG_ON(!processed); - processed--; - - ocfs2_process_blocked_lock(osb, lockres); - - spin_lock_irqsave(&osb->dc_task_lock, flags); - } - spin_unlock_irqrestore(&osb->dc_task_lock, flags); -} - -static int ocfs2_downconvert_thread_lists_empty(struct ocfs2_super *osb) -{ - int empty = 0; - unsigned long flags; - - spin_lock_irqsave(&osb->dc_task_lock, flags); - if (list_empty(&osb->blocked_lock_list)) - empty = 1; - - spin_unlock_irqrestore(&osb->dc_task_lock, flags); - return empty; -} - -static int ocfs2_downconvert_thread_should_wake(struct ocfs2_super *osb) -{ - int should_wake = 0; - unsigned long flags; - - spin_lock_irqsave(&osb->dc_task_lock, flags); - if (osb->dc_work_sequence != osb->dc_wake_sequence) - should_wake = 1; - spin_unlock_irqrestore(&osb->dc_task_lock, flags); - - return should_wake; -} - -static int ocfs2_downconvert_thread(void *arg) -{ - int status = 0; - struct ocfs2_super *osb = arg; - - /* only quit once we've been asked to stop and there is no more - * work available */ - while (!(kthread_should_stop() && - ocfs2_downconvert_thread_lists_empty(osb))) { - - wait_event_interruptible(osb->dc_event, - ocfs2_downconvert_thread_should_wake(osb) || - kthread_should_stop()); - - mlog(0, "downconvert_thread: awoken\n"); - - ocfs2_downconvert_thread_do_work(osb); - } - - osb->dc_task = NULL; - return status; -} - -void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb) -{ - unsigned long flags; - - spin_lock_irqsave(&osb->dc_task_lock, flags); - /* make sure the voting thread gets a swipe at whatever changes - * the caller may have made to the voting state */ - osb->dc_wake_sequence++; - spin_unlock_irqrestore(&osb->dc_task_lock, flags); - wake_up(&osb->dc_event); -} - -int ocfs2_init_super(struct ocfs2_super *osb, int flags) -{ - memset(osb, 0, sizeof(*osb)); - - osb->osb_dlm_debug = ocfs2_new_dlm_debug(); - if (!osb->osb_dlm_debug) - return -ENOMEM; - - spin_lock_init(&osb->dc_task_lock); - init_waitqueue_head(&osb->dc_event); - INIT_LIST_HEAD(&osb->blocked_lock_list); - osb->s_mount_opt = flags; - atomic64_set(&osb->refresh_gen, 0); - - return 0; -} diff --git a/kmod/src/dlmglue.h b/kmod/src/dlmglue.h deleted file mode 100644 index 7a51c528..00000000 --- a/kmod/src/dlmglue.h +++ /dev/null @@ -1,390 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; -*- - * vim: noexpandtab sw=8 ts=8 sts=0: - * - * dlmglue.h - * - * description here - * - * Copyright (C) 2002, 2004 Oracle. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 021110-1307, USA. - */ - - -#ifndef DLMGLUE_H -#define DLMGLUE_H - -#include "stackglue.h" - -/* Max length of lockid name */ -#define OCFS2_LOCK_ID_MAX_LEN 32 -#define OCFS2_LOCK_ID_PRETTY_LEN 64 - -enum ocfs2_ast_action { - OCFS2_AST_INVALID = 0, - OCFS2_AST_ATTACH, - OCFS2_AST_CONVERT, - OCFS2_AST_DOWNCONVERT, -}; - -/* actions for an unlockast function to take. */ -enum ocfs2_unlock_action { - OCFS2_UNLOCK_INVALID = 0, - OCFS2_UNLOCK_CANCEL_CONVERT, - OCFS2_UNLOCK_DROP_LOCK, -}; - -/* ocfs2_lock_res->l_flags flags. */ -#define OCFS2_LOCK_ATTACHED (0x00000001) /* we have initialized - * the lvb */ -#define OCFS2_LOCK_BUSY (0x00000002) /* we are currently in - * dlm_lock */ -#define OCFS2_LOCK_BLOCKED (0x00000004) /* blocked waiting to - * downconvert*/ -#define OCFS2_LOCK_LOCAL (0x00000008) /* newly created inode */ -#define OCFS2_LOCK_NEEDS_REFRESH (0x00000010) -#define OCFS2_LOCK_REFRESHING (0x00000020) -#define OCFS2_LOCK_INITIALIZED (0x00000040) /* track initialization - * for shutdown paths */ -#define OCFS2_LOCK_FREEING (0x00000080) /* help dlmglue track - * when to skip queueing - * a lock because it's - * about to be - * dropped. */ -#define OCFS2_LOCK_QUEUED (0x00000100) /* queued for downconvert */ -#define OCFS2_LOCK_NOCACHE (0x00000200) /* don't use a holder count */ -#define OCFS2_LOCK_PENDING (0x00000400) /* This lockres is pending a - call to dlm_lock. Only - exists with BUSY set. */ -#define OCFS2_LOCK_UPCONVERT_FINISHING (0x00000800) /* blocks the dc thread - * from downconverting - * before the upconvert - * has completed */ - -#define OCFS2_LOCK_NONBLOCK_FINISHED (0x00001000) /* NONBLOCK cluster - * lock has already - * returned, do not block - * dc thread from - * downconverting */ - -struct ocfs2_lock_res_ops; - -typedef void (*ocfs2_lock_callback)(int status, unsigned long data); - -#ifdef CONFIG_OCFS2_FS_STATS -struct ocfs2_lock_stats { - u64 ls_total; /* Total wait in NSEC */ - u32 ls_gets; /* Num acquires */ - u32 ls_fail; /* Num failed acquires */ - - /* Storing max wait in usecs saves 24 bytes per inode */ - u32 ls_max; /* Max wait in USEC */ -}; -#endif - -struct ocfs2_lock_res { - void *l_priv; - struct ocfs2_lock_res_ops *l_ops; - - - struct list_head l_blocked_list; - struct list_head l_mask_waiters; - struct list_head l_holders; - - u64 l_refresh_gen; - unsigned long l_flags; - char l_name[OCFS2_LOCK_ID_MAX_LEN]; - char l_pretty_name[OCFS2_LOCK_ID_PRETTY_LEN]; - unsigned int l_ro_holders; - unsigned int l_cw_holders; - unsigned int l_ex_holders; - signed char l_level; - signed char l_requested; - signed char l_blocking; - - /* used from AST/BAST funcs. */ - /* Data packed - enum type ocfs2_ast_action */ - unsigned char l_action; - /* Data packed - enum type ocfs2_unlock_action */ - unsigned char l_unlock_action; - unsigned int l_pending_gen; - - spinlock_t l_lock; - - struct ocfs2_dlm_lksb l_lksb; - - wait_queue_head_t l_event; - - struct list_head l_debug_list; - -#ifdef CONFIG_OCFS2_FS_STATS - struct ocfs2_lock_stats l_lock_prmode; /* PR mode stats */ - u32 l_lock_refresh; /* Disk refreshes */ - struct ocfs2_lock_stats l_lock_exmode; /* EX mode stats */ - struct ocfs2_lock_stats l_lock_cwmode; /* CW mode stats */ -#endif -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lockdep_map l_lockdep_map; -#endif -}; - -struct ocfs2_dlm_debug { - struct kref d_refcnt; - struct dentry *d_locking_state; - struct list_head d_lockres_tracking; -}; - -/* The cluster stack fields */ -#define OCFS2_STACK_LABEL_LEN 4 -#define OCFS2_CLUSTER_NAME_LEN 16 - -struct ocfs2_super -{ - struct ocfs2_cluster_connection *cconn; - struct ocfs2_dlm_debug *osb_dlm_debug; - - /* Downconvert thread */ - spinlock_t dc_task_lock; - struct task_struct *dc_task; - wait_queue_head_t dc_event; - unsigned long dc_wake_sequence; - unsigned long dc_work_sequence; - - /* - * Any thread can add locks to the list, but the downconvert - * thread is the only one allowed to remove locks. Any change - * to this rule requires updating - * ocfs2_downconvert_thread_do_work(). - */ - struct list_head blocked_lock_list; - unsigned long blocked_lock_count; - - /* refresh_gen needs to strictly increase as locks come and go */ - atomic64_t refresh_gen; - - unsigned long s_mount_opt; - - /* sb is for use with scoutfs counter macros only. Eventually - * we'll roll our own counters code in dlmglue. */ - struct super_block *sb; -}; -/* For s_mount_opt */ -#define OCFS2_MOUNT_NOINTR (1 << 2) - -/* - * Return value from ->downconvert_worker functions. - * - * These control the precise actions of ocfs2_unblock_lock() - * and ocfs2_process_blocked_lock() - * - */ -enum ocfs2_unblock_action { - UNBLOCK_CONTINUE = 0, /* Continue downconvert */ - UNBLOCK_CONTINUE_POST = 1, /* Continue downconvert, fire - * ->post_unlock callback */ - UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire - * ->post_unlock() callback. */ -}; - -enum ocfs2_lock_events { - EVENT_DLM_LOCK = 0, - EVENT_DLM_UNLOCK, - EVENT_DLM_CONVERT, - EVENT_DLM_CANCEL_CONVERT, - EVENT_DLM_DOWNCONVERT_WORK, -}; - -/* - * OCFS2 Lock Resource Operations - * - * These fine tune the behavior of the generic dlmglue locking infrastructure. - * - * The most basic of lock types can point ->l_priv to their respective - * struct ocfs2_super and allow the default actions to manage things. - * - * Right now, each lock type also needs to implement an init function, - * and trivial lock/unlock wrappers. ocfs2_simple_drop_lockres() - * should be called when the lock is no longer needed (i.e., object - * destruction time). - */ -struct ocfs2_lock_res_ops { - /* - * Translate an ocfs2_lock_res * into an ocfs2_super *. Define - * this callback if ->l_priv is not an ocfs2_super pointer - */ - struct ocfs2_super * (*get_osb)(struct ocfs2_lock_res *); - - /* - * Optionally called in the downconvert thread after a - * successful downconvert. The lockres will not be referenced - * after this callback is called, so it is safe to free - * memory, etc. - * - * The exact semantics of when this is called are controlled - * by ->downconvert_worker() - */ - void (*post_unlock)(struct ocfs2_super *, struct ocfs2_lock_res *); - - /* - * Allow a lock type to add checks to determine whether it is - * safe to downconvert a lock. Return 0 to re-queue the - * downconvert at a later time, nonzero to continue. - * - * For most locks, the default checks that there are no - * incompatible holders are sufficient. - * - * Called with the lockres spinlock held. - */ - int (*check_downconvert)(struct ocfs2_lock_res *, int); - - /* - * Allows a lock type to populate the lock value block. This - * is called on downconvert, and when we drop a lock. - * - * Locks that want to use this should set LOCK_TYPE_USES_LVB - * in the flags field. - * - * Called with the lockres spinlock held. - */ - void (*set_lvb)(struct ocfs2_lock_res *); - - /* - * Called from the downconvert thread when it is determined - * that a lock will be downconverted. This is called without - * any locks held so the function can do work that might - * schedule (syncing out data, etc). - * - * This should return any one of the ocfs2_unblock_action - * values, depending on what it wants the thread to do. - */ - int (*downconvert_worker)(struct ocfs2_lock_res *, int); - - /* - * Called before we free a lock from the system. This allows - * the filesystem to sync and invalidate caches before that - * happens. The concept is identical to ->downconvert_worker - * except for two exceptions: - * - The FS must do the full downconvert work - as if it were - * blocking an EX. - * - We do not return an ocfs2_unblock_action - this worker is not - * allowed to delay dropping of the lock. - */ - void (*drop_worker)(struct ocfs2_lock_res *); - - /* - * Optional: pretty print the lockname into a buffer - */ - void (*print)(struct ocfs2_lock_res *, char *, unsigned int); - - /* - * Optional: Lightweight event callback, intended for quick - * operations like collecting stats, etc. - */ - void (*notify_event)(struct ocfs2_lock_res *, enum ocfs2_lock_events); - - /* - * LOCK_TYPE_* flags which describe the specific requirements - * of a lock type. Descriptions of each individual flag follow. - */ - int flags; -}; - -/* - * Some locks want to "refresh" potentially stale data when a - * meaningful (PRMODE or EXMODE) lock level is first obtained. If this - * flag is set, the OCFS2_LOCK_NEEDS_REFRESH flag will be set on the - * individual lockres l_flags member from the ast function. It is - * expected that the locking wrapper will clear the - * OCFS2_LOCK_NEEDS_REFRESH flag when done. - */ -#define LOCK_TYPE_REQUIRES_REFRESH 0x1 - -/* - * Indicate that a lock type makes use of the lock value block. The - * ->set_lvb lock type callback must be defined. - */ -#define LOCK_TYPE_USES_LVB 0x2 - -struct ocfs2_lock_holder { - struct list_head oh_list; - struct pid *oh_owner_pid; -}; - -/* ocfs2_inode_lock_full() 'arg_flags' flags */ -/* don't wait on recovery. */ -#define OCFS2_META_LOCK_RECOVERY (0x01) -/* Instruct the dlm not to queue ourselves on the other node. */ -#define OCFS2_META_LOCK_NOQUEUE (0x02) -/* don't block waiting for the downconvert thread, instead return -EAGAIN */ -#define OCFS2_LOCK_NONBLOCK (0x04) -/* just get back disk inode bh if we've got cluster lock. */ -#define OCFS2_META_LOCK_GETBH (0x08) - -/* Locking subclasses of inode cluster lock */ -enum { - OI_LS_NORMAL = 0, - OI_LS_PARENT, - OI_LS_RENAME1, - OI_LS_RENAME2, - OI_LS_REFLINK_TARGET, -}; - -int ocfs2_cluster_lock(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, - int level, u32 lkm_flags, int arg_flags); -void ocfs2_cluster_unlock(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres, int level); - -int ocfs2_init_super(struct ocfs2_super *osb, int flags); -int ocfs2_dlm_init(struct ocfs2_super *osb, struct super_block *sb, - char *cluster_stack, char *cluster_name, char *ls_name, - struct dentry *debug_root); -void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); -void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); -void ocfs2_lock_res_init_common(struct ocfs2_super *osb, - struct ocfs2_lock_res *res, - struct ocfs2_lock_res_ops *ops, - void *priv); -void ocfs2_lock_res_free(struct ocfs2_lock_res *res); - -u64 ocfs2_lock_refresh_gen(struct ocfs2_lock_res *lockres); - -void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); -void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, - struct ocfs2_lock_res *lockres); - -/* for the downconvert thread */ -void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb); - -struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void); -void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug); -int ocfs2_levels_compat(struct ocfs2_lock_res *lockres, int wanted); - -#if 0 -/* To set the locking protocol on module initialization */ -void ocfs2_set_locking_protocol(void); - -/* The _tracker pair is used to avoid cluster recursive locking */ -int ocfs2_inode_lock_tracker(struct inode *inode, - struct buffer_head **ret_bh, - int ex, - struct ocfs2_lock_holder *oh); -void ocfs2_inode_unlock_tracker(struct inode *inode, - int ex, - struct ocfs2_lock_holder *oh, - int had_lock); -#endif -#endif /* DLMGLUE_H */ diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 31545f0c..7207466e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -258,7 +258,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; SCOUTFS_DECLARE_KVEC(val); - const u64 refresh_gen = scoutfs_lock_refresh_gen(lock); + const u64 refresh_gen = lock->refresh_gen; int ret; /* @@ -1272,7 +1272,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->data_version = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; - atomic64_set(&ci->last_refreshed, scoutfs_lock_refresh_gen(lock)); + atomic64_set(&ci->last_refreshed, lock->refresh_gen); ci->flags = 0; ci->ino_alloc.ino = 0; ci->ino_alloc.nr = 0; diff --git a/kmod/src/item.c b/kmod/src/item.c index 7216709e..41140ee2 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -741,39 +741,23 @@ restart: /* * Return true if the lock protects the use of the key. Some locks not - * intended for item use don't have a key range and we wan't to safely - * detect that. We use the block 'rw' constants just because they're - * convenient. The level test is racey but it's a char.. how racy can - * it be? :). + * intended for item use don't have a key range and we want to safely + * detect that. The lock mode dereference is racy but the field always + * contains a single non-zero byte. */ static bool lock_coverage(struct scoutfs_lock *lock, - struct scoutfs_key_buf *key, int op_level) + struct scoutfs_key_buf *key, int op_mode) { - signed char level; + signed char mode; if (!lock || !lock->start || !lock->end) return false; - level = ACCESS_ONCE(lock->lockres.l_level); + mode = ACCESS_ONCE(lock->granted_mode); - switch (op_level) { - case DLM_LOCK_CW: - if (level != DLM_LOCK_CW) - return false; - break; - case DLM_LOCK_PR: - if (level < DLM_LOCK_PR) - return false; - break; - case DLM_LOCK_EX: - if (level != DLM_LOCK_EX) - return false; - break; - default: - return false; - } - - return scoutfs_key_compare_ranges(key, key, + return ((op_mode == mode) || + (op_mode == DLM_LOCK_PR && mode == DLM_LOCK_EX)) && + scoutfs_key_compare_ranges(key, key, lock->start, lock->end) == 0; } @@ -1781,6 +1765,8 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) * The caller wants us to write out any dirty items within the given * range. We look for any dirty items within the range and if we find * any we issue a sync which writes out all the dirty items. + * + * Returns a sync error or the number of dirty items written. */ int scoutfs_item_writeback(struct super_block *sb, struct scoutfs_key_buf *start, @@ -1791,6 +1777,7 @@ int scoutfs_item_writeback(struct super_block *sb, struct cached_item *item; unsigned long flags; bool sync = false; + int count = 0; int ret = 0; /* XXX think about racing with trans write */ @@ -1801,8 +1788,10 @@ int scoutfs_item_writeback(struct super_block *sb, item = next_item(&cac->items, start); if (item && !(item->dirty & ITEM_DIRTY)) item = next_dirty(item); - if (item && scoutfs_key_compare(item->key, end) <= 0) + if (item && scoutfs_key_compare(item->key, end) <= 0) { sync = true; + count = cac->nr_dirty_items; + } } spin_unlock_irqrestore(&cac->lock, flags); @@ -1812,12 +1801,14 @@ int scoutfs_item_writeback(struct super_block *sb, ret = scoutfs_trans_sync(sb, 1); } - return ret; + return ret ?: count; } /* * The caller wants us to drop any items within the range on the floor. * They should have ensured that items in this range won't be dirty. + * + * Returns errors or the count of the items invalidated. */ int scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key_buf *start, @@ -1830,6 +1821,7 @@ int scoutfs_item_invalidate(struct super_block *sb, struct cached_item *item; struct rb_node *node; unsigned long flags; + int count = 0; int ret; trace_scoutfs_item_invalidate_range(sb, start, end); @@ -1866,6 +1858,7 @@ int scoutfs_item_invalidate(struct super_block *sb, WARN_ON_ONCE(item->dirty & ITEM_DIRTY); erase_item(sb, cac, item); + count++; } remove_range(sb, &cac->ranges, rng); @@ -1874,7 +1867,7 @@ int scoutfs_item_invalidate(struct super_block *sb, ret = 0; out: - return ret; + return ret ?: count; } static struct cached_item *rb_next_item(struct cached_item *item) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 446a97bf..565a1c8c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -27,147 +27,67 @@ #include "scoutfs_trace.h" #include "msg.h" #include "cmp.h" -#include "dlmglue.h" #include "inode.h" #include "trans.h" #include "counters.h" #include "endian_swap.h" #include "triggers.h" +/* + * scoutfs manages internode item cache consistency using the kernel's + * dlm service. We map ranges of item keys to dlm locks and use each + * lock's modes to govern what we can do with the items under the lock. + * + * The management of locks is based around state updates that queue work + * which then acts on the state. The work calls the dlm to change modes + * and gets completion notification in callbacks. + * + * We free locks that aren't actively protecting items instead of + * converting them to NL and leaving them around. It gives us fewer + * locks consuming resources and fewer locks to wade through to try and + * diagnose a problem. + * + * So far we've only needed a minimal trylock. We don't issue a NOQUEUE + * request to the dlm which can eventually return -EAGAIN if it finds + * contention. We return -EAGAIN ourselves if a user can't immediately + * match an existing granted lock. This is fine for the only rare user + * which can back out of its lock inversion and retry with a full + * blocking lock. This saves us from having to plumb per-waiter flags + * down to dlm requests. + */ + +#define GRACE_WORK_DELAY_JIFFIES msecs_to_jiffies(2) +#define GRACE_UNLOCK_DEADLINE_KT ms_to_ktime(2) + #define LN_FMT "%u.%u.%u.%llu.%llu" #define LN_ARG(name) \ (name)->scope, (name)->zone, (name)->type, le64_to_cpu((name)->first),\ le64_to_cpu((name)->second) -typedef struct ocfs2_super dlmglue_ctxt; - /* * allocated per-super, freed on unmount. */ struct lock_info { struct super_block *sb; - dlmglue_ctxt dlmglue; - bool dlmglue_online; - char ls_name[DLM_LOCKSPACE_LEN]; - spinlock_t lock; - unsigned int seq_cnt; + bool shutdown; struct rb_root lock_tree; struct rb_root lock_range_tree; struct shrinker shrinker; struct list_head lru_list; unsigned long long lru_nr; - struct workqueue_struct *lock_reclaim_wq; + struct workqueue_struct *workq; + dlm_lockspace_t *lockspace; struct dentry *debug_locks_dentry; struct idr debug_locks_idr; + atomic64_t next_refresh_gen; }; #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info -static void scoutfs_lock_reclaim(struct work_struct *work); - -struct task_ref { - struct task_struct *task; - struct rb_node node; - int count; - int mode;/* for debugging */ -}; - -static struct task_ref *find_task_ref(struct scoutfs_lock *lock, - struct task_struct *task) -{ - struct rb_node *n; - struct task_ref *tmp; - - spin_lock(&lock->task_refs_lock); - n = lock->task_refs.rb_node; - while (n) { - tmp = rb_entry(n, struct task_ref, node); - - if (tmp->task < task) - n = n->rb_left; - else if (tmp->task > task) - n = n->rb_right; - else { - spin_unlock(&lock->task_refs_lock); - return tmp; - } - } - spin_unlock(&lock->task_refs_lock); - - return NULL; -} - -static struct task_ref *alloc_task_ref(struct task_struct *task, int mode) -{ - struct task_ref *ref = kzalloc(sizeof(*ref), GFP_NOFS); - if (ref) { - ref->task = task; - ref->count = 1; - ref->mode = mode; - RB_CLEAR_NODE(&ref->node); - } - return ref; -} - -static void insert_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) -{ - struct task_ref *tmp; - struct rb_node *parent = NULL; - struct rb_node **p; - - spin_lock(&lock->task_refs_lock); - p = &lock->task_refs.rb_node; - while (*p) { - parent = *p; - - tmp = rb_entry(parent, struct task_ref, node); - - if (tmp->task < ref->task) - p = &(*p)->rb_left; - else if (tmp->task > ref->task) - p = &(*p)->rb_right; - else - BUG(); /* We should never find a duplicate */ - } - - rb_link_node(&ref->node, parent, p); - rb_insert_color(&ref->node, &lock->task_refs); - spin_unlock(&lock->task_refs_lock); -} - -static void get_task_ref(struct task_ref *ref) -{ - ref->count++; -} - -static struct task_ref *new_task_ref(struct scoutfs_lock *lock, - struct task_struct *task, int mode) -{ - struct task_ref *ref = alloc_task_ref(task, mode); - if (ref) - insert_task_ref(lock, ref); - - return ref; -} - -static int put_task_ref(struct scoutfs_lock *lock, struct task_ref *ref) -{ - if (!ref) - return 0; - - ref->count--; - if (ref->count == 0) { - spin_lock(&lock->task_refs_lock); - rb_erase(&ref->node, &lock->task_refs); - spin_unlock(&lock->task_refs_lock); - - kfree(ref); - return 0; - } - return 1; -} +static void scoutfs_lock_work(struct work_struct *work); +static void scoutfs_lock_grace_work(struct work_struct *work); /* * invalidate cached data associated with an inode whose lock is going @@ -210,29 +130,34 @@ static void invalidate_inode(struct super_block *sb, u64 ino) } /* - * Invalidate caches on this because another node wants a lock - * with the a lock with the given mode and range. We always have to - * write out dirty overlapping items. If they're writing then we need - * to also invalidate all cached overlapping structures. + * Invalidate caches associated with this lock. We're going from the + * previous mode to the next mode. */ -static int invalidate_caches(struct super_block *sb, int mode, - struct scoutfs_lock *lock) +static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, + int prev, int mode) { struct scoutfs_key_buf *start = lock->start; struct scoutfs_key_buf *end = lock->end; u64 ino, last; int ret; - trace_scoutfs_lock_invalidate(sb, lock); + /* any transition from a mode allowed to dirty items has to write */ + if (prev == DLM_LOCK_CW || prev == DLM_LOCK_EX) { + ret = scoutfs_item_writeback(sb, start, end); + if (ret < 0) + return ret; + if (ret > 0) { + scoutfs_add_counter(sb, lock_write_dirty_item, ret); + ret = 0; + } + } - ret = scoutfs_item_writeback(sb, start, end); - if (ret) - return ret; - - if (mode == DLM_LOCK_EX || - (mode == DLM_LOCK_PR && lock->lockres.l_level == DLM_LOCK_CW)) { - if (lock->lock_name.zone == SCOUTFS_FS_ZONE) { - ino = le64_to_cpu(lock->lock_name.first); + /* invalidate items that we could have but won't be able to use */ + if (prev == DLM_LOCK_CW || + (prev == DLM_LOCK_PR && mode != DLM_LOCK_EX) || + (prev == DLM_LOCK_EX && mode != DLM_LOCK_PR)) { + if (lock->name.zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->name.first); last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; while (ino <= last) { invalidate_inode(sb, ino); @@ -241,191 +166,46 @@ static int invalidate_caches(struct super_block *sb, int mode, } ret = scoutfs_item_invalidate(sb, start, end); + if (ret > 0) { + scoutfs_add_counter(sb, lock_invalidate_clean_item, + ret); + ret = 0; + } } - /* - * Not really tracing the return value here, we're mostly - * interested in elapsed time between the top trace and this one. - */ - trace_scoutfs_lock_invalidate_ret(sb, lock); - return ret; } -static void free_scoutfs_lock(struct scoutfs_lock *lock) +static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) { - struct lock_info *linfo; + struct super_block *sb = lock->sb; - if (lock) { - linfo = SCOUTFS_SB(lock->sb)->lock_info; + assert_spin_locked(&linfo->lock); - if (lock->debug_locks_id) { - spin_lock(&linfo->lock); - idr_remove(&linfo->debug_locks_idr, - lock->debug_locks_id); - spin_unlock(&linfo->lock); - } + trace_scoutfs_lock_free(sb, lock); + scoutfs_inc_counter(sb, lock_free); - scoutfs_inc_counter(lock->sb, lock_free); - ocfs2_lock_res_free(&lock->lockres); - scoutfs_key_free(lock->sb, lock->start); - scoutfs_key_free(lock->sb, lock->end); - BUG_ON(!RB_EMPTY_NODE(&lock->node)); - BUG_ON(!RB_EMPTY_NODE(&lock->range_node)); - kfree(lock); + BUG_ON(delayed_work_pending(&lock->grace_work)); + + if (lock->debug_locks_id) + idr_remove(&linfo->debug_locks_idr, lock->debug_locks_id); + if (!RB_EMPTY_NODE(&lock->node)) + rb_erase(&lock->node, &linfo->lock_tree); + if (!RB_EMPTY_NODE(&lock->range_node)) + rb_erase(&lock->range_node, &linfo->lock_range_tree); + if (!list_empty(&lock->lru_head)) { + list_del(&lock->lru_head); + linfo->lru_nr--; } + scoutfs_key_free(sb, lock->start); + scoutfs_key_free(sb, lock->end); + kfree(lock); } -static void put_scoutfs_lock(struct super_block *sb, struct scoutfs_lock *lock) -{ - DECLARE_LOCK_INFO(sb, linfo); - unsigned int refs; - - if (lock) { - spin_lock(&linfo->lock); - BUG_ON(!lock->refcnt); - refs = --lock->refcnt; - if (!refs) { - trace_scoutfs_lock_free(sb, lock); - rb_erase(&lock->node, &linfo->lock_tree); - RB_CLEAR_NODE(&lock->node); - if(!RB_EMPTY_NODE(&lock->range_node)) { - rb_erase(&lock->range_node, - &linfo->lock_range_tree); - RB_CLEAR_NODE(&lock->range_node); - } - list_del(&lock->lru_entry); - spin_unlock(&linfo->lock); - ocfs2_simple_drop_lockres(&linfo->dlmglue, - &lock->lockres); - free_scoutfs_lock(lock); - return; - } - spin_unlock(&linfo->lock); - } -} - -static void dec_lock_users(struct scoutfs_lock *lock) -{ - DECLARE_LOCK_INFO(lock->sb, linfo); - - spin_lock(&linfo->lock); - lock->users--; - if (list_empty(&lock->lru_entry) && lock->users == 0) { - list_add_tail(&lock->lru_entry, &linfo->lru_list); - linfo->lru_nr++; - } - spin_unlock(&linfo->lock); -} - -static struct ocfs2_super *get_ino_lock_osb(struct ocfs2_lock_res *lockres) -{ - struct scoutfs_lock *lock = lockres->l_priv; - struct super_block *sb = lock->sb; - DECLARE_LOCK_INFO(sb, linfo); - - return &linfo->dlmglue; -} - -static int ino_lock_downconvert(struct ocfs2_lock_res *lockres, int blocking) -{ - struct scoutfs_lock *lock = lockres->l_priv; - struct super_block *sb = lock->sb; - - invalidate_caches(sb, blocking, lock); - - return UNBLOCK_CONTINUE; -} - -static void ino_lock_drop(struct ocfs2_lock_res *lockres) -{ - struct scoutfs_lock *lock = lockres->l_priv; - struct super_block *sb = lock->sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - /* - * Locks get shut down near the end of our unmount process. By - * now everything that needs to be synced or invalidated, has - * been. - */ - if (!sbi->shutdown) - invalidate_caches(sb, DLM_LOCK_EX, lock); -} - -static void lock_name_string(struct ocfs2_lock_res *lockres, char *buf, - unsigned int len) -{ - struct scoutfs_lock *lock = lockres->l_priv; - - snprintf(buf, len, LN_FMT, LN_ARG(&lock->lock_name)); -} - -static void count_ino_lock_event(struct ocfs2_lock_res *lockres, - enum ocfs2_lock_events event) -{ - struct scoutfs_lock *lock = container_of(lockres, struct scoutfs_lock, - lockres); - struct super_block *sb = lock->sb; - - if (event == EVENT_DLM_DOWNCONVERT_WORK) - scoutfs_inc_counter(sb, lock_type_ino_downconvert); -} - -static void count_idx_lock_event(struct ocfs2_lock_res *lockres, - enum ocfs2_lock_events event) -{ - struct scoutfs_lock *lock = container_of(lockres, struct scoutfs_lock, - lockres); - struct super_block *sb = lock->sb; - - /* - * Treat all indicies together. Later we can decode the - * lockres name to get at specific indicies. - */ - if (event == EVENT_DLM_DOWNCONVERT_WORK) - scoutfs_inc_counter(sb, lock_type_idx_downconvert); -} - -static struct ocfs2_lock_res_ops scoufs_ino_lops = { - .get_osb = get_ino_lock_osb, - .downconvert_worker = ino_lock_downconvert, - .drop_worker = ino_lock_drop, - /* XXX: .check_downconvert that queries the item cache for dirty items */ - .print = lock_name_string, - .notify_event = count_ino_lock_event, - .flags = LOCK_TYPE_REQUIRES_REFRESH, -}; - -static struct ocfs2_lock_res_ops scoufs_ino_index_lops = { - .get_osb = get_ino_lock_osb, - .downconvert_worker = ino_lock_downconvert, - .drop_worker = ino_lock_drop, - .notify_event = count_idx_lock_event, - /* XXX: .check_downconvert that queries the item cache for dirty items */ - .print = lock_name_string, -}; - -static struct ocfs2_lock_res_ops scoutfs_global_lops = { - .get_osb = get_ino_lock_osb, - /* XXX: .check_downconvert that queries the item cache for dirty items */ - .print = lock_name_string, - .flags = 0, -}; - -static struct ocfs2_lock_res_ops scoutfs_node_id_lops = { - .get_osb = get_ino_lock_osb, - /* XXX: .check_downconvert that queries the item cache for dirty items */ - .downconvert_worker = ino_lock_downconvert, - .drop_worker = ino_lock_drop, - .print = lock_name_string, - .flags = 0, -}; - -static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, - struct scoutfs_lock_name *lock_name, - struct ocfs2_lock_res_ops *type, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) +static struct scoutfs_lock *lock_alloc(struct super_block *sb, + struct scoutfs_lock_name *name, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) { DECLARE_LOCK_INFO(sb, linfo); @@ -439,6 +219,8 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, if (lock == NULL) return NULL; + scoutfs_inc_counter(sb, lock_alloc); + idr_preload(GFP_NOFS); spin_lock(&linfo->lock); id = idr_alloc(&linfo->debug_locks_idr, lock, 1, INT_MAX, GFP_NOWAIT); @@ -447,40 +229,219 @@ static struct scoutfs_lock *alloc_scoutfs_lock(struct super_block *sb, spin_unlock(&linfo->lock); idr_preload_end(); if (id <= 0) { - free_scoutfs_lock(lock); + lock_free(linfo, lock); return NULL; } RB_CLEAR_NODE(&lock->node); RB_CLEAR_NODE(&lock->range_node); + INIT_LIST_HEAD(&lock->lru_head); if (start) { lock->start = scoutfs_key_dup(sb, start); lock->end = scoutfs_key_dup(sb, end); if (!lock->start || !lock->end) { - free_scoutfs_lock(lock); + lock_free(linfo, lock); return NULL; } } - spin_lock_init(&lock->task_refs_lock); - lock->task_refs = RB_ROOT; - RB_CLEAR_NODE(&lock->node); lock->sb = sb; - lock->lock_name = *lock_name; - INIT_LIST_HEAD(&lock->lru_entry); - ocfs2_lock_res_init_once(&lock->lockres); - BUG_ON(sizeof(struct scoutfs_lock_name) >= OCFS2_LOCK_ID_MAX_LEN); - /* kzalloc above ensures that l_name is NULL terminated */ - memcpy(&lock->lockres.l_name[0], &lock->lock_name, - sizeof(struct scoutfs_lock_name)); - ocfs2_lock_res_init_common(&linfo->dlmglue, &lock->lockres, type, lock); - INIT_WORK(&lock->reclaim_work, scoutfs_lock_reclaim); + lock->name = *name; init_waitqueue_head(&lock->waitq); + INIT_WORK(&lock->work, scoutfs_lock_work); + INIT_DELAYED_WORK(&lock->grace_work, scoutfs_lock_grace_work); + lock->granted_mode = DLM_LOCK_IV; + lock->bast_mode = DLM_LOCK_IV; + lock->work_prev_mode = DLM_LOCK_IV; + lock->work_mode = DLM_LOCK_IV; + + trace_scoutfs_lock_alloc(sb, lock); return lock; } +static void lock_inc_count(unsigned int *counts, int mode) +{ + BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); + counts[mode]++; +} + +static void lock_dec_count(unsigned int *counts, int mode) +{ + BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); + counts[mode]--; +} + +/* only PR and EX modes read items to populate the cache. */ +static bool lock_mode_can_read(int mode) +{ + return mode == DLM_LOCK_PR || mode == DLM_LOCK_EX; +} + +/* + * Returns true if a given user mode can be satisfied by a lock with the + * given granted mode. This is directional. A PR user is satisfied by + * an EX grant but not vice versa. + */ +static bool lock_modes_match(int granted, int user) +{ + return (granted == user) || + (granted == DLM_LOCK_EX && user == DLM_LOCK_PR); +} + +/* + * Returns true if all the actively used modes are satisfied by a lock + * of the given granted mode. + */ +static bool lock_counts_match(int granted, unsigned int *counts) +{ + int mode; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (counts[mode] && !lock_modes_match(granted, mode)) + return false; + } + + return true; +} + +/* + * An idle lock has nothing going on and could be safely unlocked and freed. + */ +static bool lock_idle(struct scoutfs_lock *lock) +{ + int mode; + + if (lock->work_mode >= 0 || lock->grace_pending) + return false; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] || lock->users[mode]) + return false; + } + + return true; +} + +/* + * Ensure forward progress on the lock after the caller has changed the lock. + * + * This is the core of the state transition engine that makes locking + * safe. Each transition has to consider the users of the lock, pending + * bast transitions, it's current mode, what mode it should be, and what + * mode to leave it in during the transition. + * + * This can free the lock if it's idle! Callers must not reference the + * lock after calling this. + */ +static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + bool idle; + int mode; + + assert_spin_locked(&linfo->lock); + + /* nothing to do if we're shutting down, stops rearming */ + if (linfo->shutdown) + return; + + /* only idle locks are on the lru */ + idle = lock_idle(lock); + if (list_empty(&lock->lru_head) && idle) { + list_add_tail(&lock->lru_head, &linfo->lru_list); + linfo->lru_nr++; + + } else if (!list_empty(&lock->lru_head) && !idle) { + list_del_init(&lock->lru_head); + linfo->lru_nr--; + } + + /* errored locks are torn down */ + if (lock->error) { + wake_up(&lock->waitq); + goto out; + } + + /* + * Wake any waiters who might be able to use the lock now. + * Notice that this ignores the presence of basts! This lets us + * recursively acquire locks in one task without having to track + * per-task lock references. It comes at the cost of fairness. + * Spinning overlapping users can delay a bast down conversion + * indefinitely. + */ + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] && + lock_modes_match(lock->granted_mode, mode)) { + wake_up(&lock->waitq); + break; + } + } + + /* + * Try to down convert a lock in response to a bast once users + * are done with it. We may have to wait for a grace period + * to expire after an unlock. + */ + if (lock->work_mode < 0 && + lock->bast_mode >= 0 && + lock_counts_match(lock->bast_mode, lock->users) && + !lock->grace_pending) { + + if (ktime_before(ktime_get(), lock->grace_deadline)) { + scoutfs_inc_counter(linfo->sb, lock_grace_enforced); + queue_delayed_work(linfo->workq, &lock->grace_work, + GRACE_WORK_DELAY_JIFFIES); + lock->grace_pending = true; + } else { + lock->work_prev_mode = lock->granted_mode; + lock->work_mode = lock->bast_mode; + lock->granted_mode = lock->bast_mode; + lock->bast_mode = DLM_LOCK_IV; + queue_work(linfo->workq, &lock->work); + } + } + + /* + * Convert on behalf of waiters who aren't satisfied by the + * current mode when it won't conflict with users or a pending + * bast conversion. The new mode may or may not match the + * current granted mode so we may or may not need to block users + * during the transition. + * + * Remember that the presence of waiters doesn't necessarily + * mean that they're blocked. Multiple lock attempts naturally + * line up to add themselves to the waiters count before each + * calls lock_wait() and is transitioned to a user. + */ + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->work_mode < 0 && + lock->waiters[mode] && + !lock_modes_match(lock->granted_mode, mode) && + lock_counts_match(mode, lock->users) && + (lock->bast_mode < 0 || + lock_modes_match(lock->bast_mode, mode))) { + + lock->work_prev_mode = lock->granted_mode; + lock->work_mode = mode; + if (!lock_modes_match(mode, lock->granted_mode)) + lock->granted_mode = DLM_LOCK_NL; + queue_work(linfo->workq, &lock->work); + break; + } + } + +out: + /* + * We can free the lock once it's idle and it's either never + * been initially locked or has been unlocked, both of which we + * indicate with IV. + */ + if (lock_idle(lock) && lock->granted_mode == DLM_LOCK_IV) + lock_free(linfo, lock); +} + static int cmp_lock_names(struct scoutfs_lock_name *a, struct scoutfs_lock_name *b) { @@ -491,7 +452,7 @@ static int cmp_lock_names(struct scoutfs_lock_name *a, scoutfs_cmp_u64s(le64_to_cpu(a->second), le64_to_cpu(b->second)); } -static int insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) +static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) { DECLARE_LOCK_INFO(sb, linfo); struct rb_root *root = &linfo->lock_range_tree; @@ -500,9 +461,6 @@ static int insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) struct scoutfs_lock *lock; int cmp; - if (!ins->start) - return 0; - while (*node) { parent = *node; lock = container_of(*node, struct scoutfs_lock, range_node); @@ -511,11 +469,11 @@ static int insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) lock->start, lock->end); if (WARN_ON_ONCE(cmp == 0)) { scoutfs_warn_sk(sb, "inserting lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT" overlaps with existing lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT"\n", - ins, LN_ARG(&ins->lock_name), + ins, LN_ARG(&ins->name), SK_ARG(ins->start), SK_ARG(ins->end), - lock, LN_ARG(&lock->lock_name), + lock, LN_ARG(&lock->name), SK_ARG(lock->start), SK_ARG(lock->end)); - return -EINVAL; + return false; } if (cmp < 0) @@ -528,26 +486,22 @@ static int insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) rb_link_node(&ins->range_node, parent, node); rb_insert_color(&ins->range_node, root); - return 0; + return true; } -static struct scoutfs_lock *find_alloc_scoutfs_lock(struct super_block *sb, - struct scoutfs_lock_name *lock_name, - struct ocfs2_lock_res_ops *type, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) +static struct scoutfs_lock *lock_rb_walk(struct super_block *sb, + struct scoutfs_lock_name *name, + struct scoutfs_lock *ins) { DECLARE_LOCK_INFO(sb, linfo); - struct scoutfs_lock *new = NULL; struct scoutfs_lock *found; struct scoutfs_lock *lock; struct rb_node *parent; struct rb_node **node; int cmp; - int ret; -search: - spin_lock(&linfo->lock); + assert_spin_locked(&linfo->lock); + node = &linfo->lock_tree.rb_node; parent = NULL; found = NULL; @@ -555,7 +509,7 @@ search: parent = *node; lock = container_of(*node, struct scoutfs_lock, node); - cmp = cmp_lock_names(lock_name, &lock->lock_name); + cmp = cmp_lock_names(name, &lock->name); if (cmp < 0) { node = &(*node)->rb_left; } else if (cmp > 0) { @@ -567,216 +521,320 @@ search: lock = NULL; } - if (!found) { - if (!new) { - spin_unlock(&linfo->lock); - new = alloc_scoutfs_lock(sb, lock_name, type, start, - end); - if (!new) - return NULL; - - goto search; - } - found = new; - new = NULL; - found->refcnt = 1; /* Freed by shrinker or on umount */ - found->sequence = ++linfo->seq_cnt; - - ret = insert_range_node(sb, found); - if (ret < 0) { - spin_unlock(&linfo->lock); - free_scoutfs_lock(found); - return NULL; - } - - trace_scoutfs_lock_rb_insert(sb, found); - rb_link_node(&found->node, parent, node); - rb_insert_color(&found->node, &linfo->lock_tree); - scoutfs_inc_counter(sb, lock_alloc); - } - found->refcnt++; - if (test_bit(SCOUTFS_LOCK_RECLAIM, &found->flags)) { - spin_unlock(&linfo->lock); - wait_event(found->waitq, - test_bit(SCOUTFS_LOCK_DROPPED, &found->flags)); - put_scoutfs_lock(sb, found); - goto search; + if (!found && ins) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &linfo->lock_tree); + found = ins; } - if (!list_empty(&found->lru_entry)) { - list_del_init(&found->lru_entry); - linfo->lru_nr--; - } - found->users++; - spin_unlock(&linfo->lock); - - free_scoutfs_lock(new); return found; } -static void scoutfs_lock_reclaim(struct work_struct *work) -{ - struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - reclaim_work); - struct lock_info *linfo = SCOUTFS_SB(lock->sb)->lock_info; - - trace_scoutfs_lock_reclaim(lock->sb, lock); - - /* - * Drop the last ref on our lock here, allowing us to clean up - * the dlm lock. We might race with another process in - * find_alloc_scoutfs_lock(), hence the dropped flag telling - * those processes to go ahead and drop the lock ref as well. - */ - BUG_ON(lock->users); - - set_bit(SCOUTFS_LOCK_DROPPED, &lock->flags); - wake_up(&lock->waitq); - - put_scoutfs_lock(linfo->sb, lock); -} - -void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) -{ - struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; - struct scoutfs_lock *lock; - struct scoutfs_lock *tmp; - unsigned long flags; - - spin_lock_irqsave(&linfo->lock, flags); - list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_entry) { - if (nr-- == 0) - break; - - trace_shrink_lock_tree(linfo->sb, lock); - - WARN_ON(lock->users); - - set_bit(SCOUTFS_LOCK_RECLAIM, &lock->flags); - list_del_init(&lock->lru_entry); - linfo->lru_nr--; - - queue_work(linfo->lock_reclaim_wq, &lock->reclaim_work); - } - spin_unlock_irqrestore(&linfo->lock, flags); -} - -static int shrink_lock_tree(struct shrinker *shrink, struct shrink_control *sc) -{ - struct lock_info *linfo = container_of(shrink, struct lock_info, - shrinker); - unsigned long nr; - int ret; - - nr = sc->nr_to_scan; - if (nr) - scoutfs_free_unused_locks(linfo->sb, nr); - - ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); - trace_scoutfs_lock_shrink_exit(linfo->sb, sc->nr_to_scan, ret); - return ret; -} - -static void free_lock_tree(struct super_block *sb) +/* + * A dlm lock, conversion, or unlock call has finished. We don't + * strictly serialize the arrival of basts and our dlm calls. It's + * possible and safe for us to get a deadlock notification because we + * tried to convert in conflict with a received bast. We ignore the + * result of the deadlock conversion and processing will retry and this + * time prefer the bast. + */ +static void scoutfs_lock_ast(void *arg) { + struct scoutfs_lock *lock = arg; + struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - struct rb_node *node = rb_first(&linfo->lock_tree); + int status = lock->lksb.sb_status; - while (node) { - struct scoutfs_lock *lock; + scoutfs_inc_counter(sb, lock_ast); - lock = rb_entry(node, struct scoutfs_lock, node); - node = rb_next(node); - put_scoutfs_lock(sb, lock); + spin_lock(&linfo->lock); + + if (status == 0) { + if (lock_mode_can_read(lock->work_mode) && + !lock_mode_can_read(lock->work_prev_mode)) { + lock->refresh_gen = + atomic64_inc_return(&linfo->next_refresh_gen); + } + lock->granted_mode = lock->work_mode; + + } else if (status == -DLM_EUNLOCK) { + lock->granted_mode = DLM_LOCK_IV; + + } else if (status == -EDEADLK) { + /* dlm request conflicted with racing bast, try again */ + scoutfs_inc_counter(sb, lock_ast_edeadlk); + + } else if (!lock->error) { + scoutfs_inc_counter(sb, lock_ast_error); + lock->error = status; } + + lock->work_prev_mode = DLM_LOCK_IV; + lock->work_mode = DLM_LOCK_IV; + + trace_scoutfs_lock_ast(sb, lock); + lock_process(linfo, lock); + + spin_unlock(&linfo->lock); } /* - * Acquire a coherent lock on the given range of keys. While the lock - * is held other lockers are serialized. Cache coherency is maintained - * by the locking infrastructure. Lock acquisition causes writeout from - * or invalidation of other caches. + * A lock on this node has blocked a lock request on another node. * - * The caller provides the opaque lock structure used for storage and - * their start and end pointers will be accessed while the lock is held. + * We can down convert to a PR if we had an EX and they're trying to get + * a PR but all other conflicts cause us to drop our lock and invalidate + * our cache. + */ +static void scoutfs_lock_bast(void *arg, int blocked_mode) +{ + struct scoutfs_lock *lock = arg; + struct super_block *sb = lock->sb; + DECLARE_LOCK_INFO(sb, linfo); + + scoutfs_inc_counter(sb, lock_bast); + + spin_lock(&linfo->lock); + + if (lock->granted_mode == DLM_LOCK_EX && blocked_mode == DLM_LOCK_PR) + lock->bast_mode = DLM_LOCK_PR; + else + lock->bast_mode = DLM_LOCK_NL; + + trace_scoutfs_lock_bast(sb, lock); + lock_process(linfo, lock); + + spin_unlock(&linfo->lock); +} + +/* + * The actual work of sending lock requests to the dlm. There's only + * one of these per lock and the work_mode ensures that there's only one + * transition in flight at a time. + */ +static void scoutfs_lock_work(struct work_struct *work) +{ + struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, + work); + struct super_block *sb = lock->sb; + DECLARE_LOCK_INFO(sb, linfo); + int dlm_flags; + int prev; + int mode; + int ret; + + spin_lock(&linfo->lock); + + /* don't try to call a released lockspace during shutdown */ + if (linfo->shutdown) { + spin_unlock(&linfo->lock); + return; + } + + trace_scoutfs_lock_work(sb, lock); + prev = lock->work_prev_mode; + mode = lock->work_mode; + + spin_unlock(&linfo->lock); + + if (lock->start) { + ret = lock_invalidate(sb, lock, prev, mode); + BUG_ON(ret); + } + + scoutfs_inc_counter(sb, lock_dlm_call); + + if (mode == DLM_LOCK_NL) { + ret = dlm_unlock(linfo->lockspace, lock->lksb.sb_lkid, 0, + &lock->lksb, lock); + } else { + dlm_flags = DLM_LKF_NOORDER; + if (prev >= 0) + dlm_flags |= DLM_LKF_CONVERT; + ret = dlm_lock(linfo->lockspace, mode, &lock->lksb, dlm_flags, + &lock->name, sizeof(lock->name), 0, + scoutfs_lock_ast, lock, scoutfs_lock_bast); + } + /* + * I don't think the lock error handling is correct yet. It + * probably doesn't try to unlock a lock that saw an error. + */ + if (ret) + scoutfs_inc_counter(sb, lock_dlm_call_error); + BUG_ON(ret); + + spin_lock(&linfo->lock); + + if (ret < 0) { + if (!lock->error) + lock->error = ret; + lock->work_prev_mode = DLM_LOCK_IV; + lock->work_mode = DLM_LOCK_IV; + lock_process(linfo, lock); + } + + spin_unlock(&linfo->lock); +} + +/* + * The grace period has elapsed since a down conversion attempt too soon + * after an unlock. It can now be down converted. + */ +static void scoutfs_lock_grace_work(struct work_struct *work) +{ + struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, + grace_work.work); + struct super_block *sb = lock->sb; + DECLARE_LOCK_INFO(sb, linfo); + + BUG_ON(lock->grace_pending == false); + + spin_lock(&linfo->lock); + trace_scoutfs_lock_grace_work(sb, lock); + scoutfs_inc_counter(linfo->sb, lock_grace_expired); + lock->grace_pending = false; + lock_process(linfo, lock); + spin_unlock(&linfo->lock); +} + +/* + * Wait for a lock attempt to be resolved. We return as an active user + * once our mode is satisfied by the lock or we can return errors. + */ +static bool lock_wait(struct lock_info *linfo, struct scoutfs_lock *lock, + int mode, int flags, int *ret) +{ + struct super_block *sb = linfo->sb; + bool done; + + spin_lock(&linfo->lock); + + trace_scoutfs_lock_wait(sb, lock); + + if (lock_modes_match(lock->granted_mode, mode)) { + /* the fast path where we can use the granted mode */ + lock_dec_count(lock->waiters, mode); + lock_inc_count(lock->users, mode); + *ret = 0; + done = true; + + } else if (linfo->shutdown) { + /* locking is going away */ + *ret = -ESHUTDOWN; + done = true; + + } else if (lock->error) { + /* something horrible has happened */ + *ret = lock->error; + done = true; + + } else if (flags & SCOUTFS_LKF_NONBLOCK) { + /* never wait for "nonblocking" callers */ + scoutfs_inc_counter(sb, lock_nonblock_eagain); + *ret = -EAGAIN; + done = true; + + } else { + /* still waiting :/ */ + *ret = 0; + done = false; + } + + lock_process(linfo, lock); + + spin_unlock(&linfo->lock); + + return done; +} + +/* + * Acquire a coherent lock on the given range of keys. On success the + * caller can use the given mode to interact with the item cache. While + * holding the lock the cache won't be invalidated and other conflicting + * lock users will be serialized. The item cache can be invalidated + * once the lock is unlocked. */ static int lock_name_keys(struct super_block *sb, int mode, int flags, - struct scoutfs_lock_name *lock_name, - struct ocfs2_lock_res_ops *type, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct scoutfs_lock **ret_lock) + struct scoutfs_lock_name *name, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_lock **ret_lock) { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; - struct task_ref *ref = NULL; - int lkm_flags; + struct scoutfs_lock *ins; + int wait_ret; int ret; + scoutfs_inc_counter(sb, lock_lock); + *ret_lock = NULL; - if (WARN_ON_ONCE(!(flags & SCOUTFS_LKF_TRYLOCK) && - scoutfs_trans_held())) - return -EINVAL; + /* maybe catch _setup() order mistakes */ + if (WARN_ON_ONCE(!linfo || linfo->lockspace == NULL)) + return -ENOLCK; - lock = find_alloc_scoutfs_lock(sb, lock_name, type, start, end); - if (!lock) - return -ENOMEM; + /* have to lock before entering transactions */ + if (WARN_ON_ONCE(scoutfs_trans_held())) + return -EDEADLK; - trace_scoutfs_lock_resource(sb, lock); + ins = NULL; +retry: + spin_lock(&linfo->lock); - if (!(flags & SCOUTFS_LKF_NO_TASK_REF)) { - ref = find_task_ref(lock, current); - if (ref) { - /* - * We found a ref, which means we have already locked - * this resource. Check that the calling task isn't - * trying to switch modes in the middle of a recursive - * lock request. - */ - BUG_ON(!ocfs2_levels_compat(&lock->lockres, mode)); - get_task_ref(ref); - dec_lock_users(lock); - put_scoutfs_lock(sb, lock); - ret = 0; - goto out; - } + /* don't create locks once we're shutdown */ + if (linfo->shutdown) { + spin_unlock(&linfo->lock); + ret = -ESHUTDOWN; + goto out; + } - ref = new_task_ref(lock, current, mode); - if (!ref) { + lock = lock_rb_walk(sb, name, ins); + if (!lock) { + spin_unlock(&linfo->lock); + ins = lock_alloc(sb, name, start, end); + if (!ins) { ret = -ENOMEM; goto out; } + goto retry; + + } else if (lock == ins) { + if (start && !insert_range_node(sb, ins)) { + lock_free(linfo, ins); + spin_unlock(&linfo->lock); + ret = -EINVAL; + goto out; + } + + } else if (ins) { + lock_free(linfo, ins); } - lkm_flags = DLM_LKF_NOORDER; - if (flags & SCOUTFS_LKF_TRYLOCK) - lkm_flags |= DLM_LKF_NOQUEUE; /* maybe also NONBLOCK? */ + lock_inc_count(lock->waiters, mode); + spin_unlock(&linfo->lock); - ret = ocfs2_cluster_lock(&linfo->dlmglue, &lock->lockres, mode, - lkm_flags, 0); -out: + ret = wait_event_interruptible(lock->waitq, + lock_wait(linfo, lock, mode, flags, + &wait_ret)); + if (ret == 0) + ret = wait_ret; if (ret) { - put_task_ref(lock, ref); - dec_lock_users(lock); - put_scoutfs_lock(sb, lock); + scoutfs_inc_counter(sb, lock_lock_error); + spin_lock(&linfo->lock); + lock_dec_count(lock->waiters, mode); + lock_process(linfo, lock); + spin_unlock(&linfo->lock); } else { - trace_scoutfs_lock(sb, lock); *ret_lock = lock; } - +out: return ret; } -u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock) -{ - return ocfs2_lock_refresh_gen(&lock->lockres); -} - int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock) { - struct scoutfs_lock_name lock_name; + struct scoutfs_lock_name name; struct scoutfs_inode_key start_ikey; struct scoutfs_inode_key end_ikey; struct scoutfs_key_buf start; @@ -784,11 +842,11 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, ino &= ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK; - lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - lock_name.zone = SCOUTFS_FS_ZONE; - lock_name.type = SCOUTFS_INODE_TYPE; - lock_name.first = cpu_to_le64(ino); - lock_name.second = 0; + name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; + name.zone = SCOUTFS_FS_ZONE; + name.type = SCOUTFS_INODE_TYPE; + name.first = cpu_to_le64(ino); + name.second = 0; start_ikey.zone = SCOUTFS_FS_ZONE; start_ikey.ino = cpu_to_be64(ino); @@ -800,8 +858,7 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, end_ikey.type = ~0; scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, flags, &lock_name, &scoufs_ino_lops, - &start, &end, ret_lock); + return lock_name_keys(sb, mode, flags, &name, &start, &end, ret_lock); } /* @@ -926,14 +983,13 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, struct scoutfs_lock **lock) { - struct scoutfs_lock_name lock_name; + struct scoutfs_lock_name name; - memset(&lock_name, 0, sizeof(lock_name)); - lock_name.scope = SCOUTFS_LOCK_SCOPE_GLOBAL; - lock_name.type = type; + memset(&name, 0, sizeof(name)); + name.scope = SCOUTFS_LOCK_SCOPE_GLOBAL; + name.type = type; - return lock_name_keys(sb, mode, flags, &lock_name, &scoutfs_global_lops, - NULL, NULL, lock); + return lock_name_keys(sb, mode, flags, &name, NULL, NULL, lock); } /* @@ -987,7 +1043,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock) { - struct scoutfs_lock_name lock_name; + struct scoutfs_lock_name name; struct scoutfs_inode_index_key start_ikey; struct scoutfs_inode_index_key end_ikey; struct scoutfs_key_buf start; @@ -996,17 +1052,16 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, scoutfs_lock_get_index_item_range(type, major, ino, &start_ikey, &end_ikey); - lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - lock_name.zone = start_ikey.zone; - lock_name.type = start_ikey.type; - lock_name.first = be64_to_le64(start_ikey.major); - lock_name.second = be64_to_le64(start_ikey.ino); + name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; + name.zone = start_ikey.zone; + name.type = start_ikey.type; + name.first = be64_to_le64(start_ikey.major); + name.second = be64_to_le64(start_ikey.ino); scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); - return lock_name_keys(sb, mode, 0, &lock_name, - &scoufs_ino_index_lops, &start, &end, ret_lock); + return lock_name_keys(sb, mode, 0, &name, &start, &end, ret_lock); } /* @@ -1023,17 +1078,17 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock) { - struct scoutfs_lock_name lock_name; + struct scoutfs_lock_name name; struct scoutfs_orphan_key start_okey; struct scoutfs_orphan_key end_okey; struct scoutfs_key_buf start; struct scoutfs_key_buf end; - lock_name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - lock_name.zone = SCOUTFS_NODE_ZONE; - lock_name.type = 0; - lock_name.first = cpu_to_le64(node_id); - lock_name.second = 0; + name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; + name.zone = SCOUTFS_NODE_ZONE; + name.type = 0; + name.first = cpu_to_le64(node_id); + name.second = 0; start_okey.zone = SCOUTFS_NODE_ZONE; start_okey.node_id = cpu_to_be64(node_id); @@ -1047,119 +1102,90 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, end_okey.ino = cpu_to_be64(~0ULL); scoutfs_key_init(&end, &end_okey, sizeof(end_okey)); - return lock_name_keys(sb, mode, flags, &lock_name, - &scoutfs_node_id_lops, &start, &end, lock); -} - -void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, - int level, int flags) -{ - struct task_ref *ref; - DECLARE_LOCK_INFO(sb, linfo); - - if (!lock) - return; - - trace_scoutfs_unlock(sb, lock); - - if (!(flags & SCOUTFS_LKF_NO_TASK_REF)) { - ref = find_task_ref(lock, current); - BUG_ON(!ref); - if (put_task_ref(lock, ref)) - return; - } - - ocfs2_cluster_unlock(&linfo->dlmglue, &lock->lockres, level); - - dec_lock_users(lock); - - put_scoutfs_lock(sb, lock); -} - -void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, - int level) -{ - scoutfs_unlock_flags(sb, lock, level, 0); + return lock_name_keys(sb, mode, flags, &name, &start, &end, lock); } /* - * The moment this is done we can have other mounts start asking - * us to write back and invalidate, so do this very very late. + * As we unlock we start a grace period. If a bast arrives before the + * grace period we'll wait for another full grace period we downconvert + * and invalidate the lock. Each unlock resets the downconvert delay. */ -static int init_lock_info(struct super_block *sb) +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct lock_info *linfo; - int ret; - - linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); - if (!linfo) - return -ENOMEM; - - ret = ocfs2_init_super(&linfo->dlmglue, 0); - if (ret) - goto out; - - spin_lock_init(&linfo->lock); - INIT_LIST_HEAD(&linfo->lru_list); - idr_init(&linfo->debug_locks_idr); - linfo->shrinker.shrink = shrink_lock_tree; - linfo->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&linfo->shrinker); - linfo->sb = sb; - linfo->lock_tree = RB_ROOT; - linfo->lock_range_tree = RB_ROOT; - - snprintf(linfo->ls_name, DLM_LOCKSPACE_LEN, "%llx", - le64_to_cpu(sbi->super.hdr.fsid)); - - sbi->lock_info = linfo; - - trace_init_lock_info(sb, linfo); -out: - if (ret) - kfree(linfo); - - return 0; -} - -void scoutfs_lock_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_LOCK_INFO(sb, linfo); - if (linfo) { - /* XXX does anything synchronize with open debugfs fds? */ - debugfs_remove(linfo->debug_locks_dentry); + if (IS_ERR_OR_NULL(lock)) + return; - unregister_shrinker(&linfo->shrinker); - if (linfo->lock_reclaim_wq) - destroy_workqueue(linfo->lock_reclaim_wq); - /* - * Do this before uninitializing the dlm and after - * draining the reclaim workqueue. - */ - free_lock_tree(sb); - idr_destroy(&linfo->debug_locks_idr); + scoutfs_inc_counter(sb, lock_unlock); - if (linfo->dlmglue_online) { - /* - * fs/dlm has a harmless but unannotated - * inversion between their connection and socket - * locking that triggers during shutdown and - * disables lockdep. - */ - lockdep_off(); - ocfs2_dlm_shutdown(&linfo->dlmglue, 0); - lockdep_on(); - } + spin_lock(&linfo->lock); + trace_scoutfs_lock_unlock(sb, lock); - sbi->lock_info = NULL; - - trace_scoutfs_lock_destroy(sb, linfo); - - kfree(linfo); + lock_dec_count(lock->users, mode); + lock->grace_deadline = ktime_add(ktime_get(), GRACE_UNLOCK_DEADLINE_KT); + if (cancel_delayed_work(&lock->grace_work)) { + scoutfs_inc_counter(linfo->sb, lock_grace_extended); + queue_delayed_work(linfo->workq, &lock->grace_work, + GRACE_WORK_DELAY_JIFFIES); } + + lock_process(linfo, lock); + spin_unlock(&linfo->lock); +} + +static int scoutfs_lock_shrink(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct lock_info *linfo = container_of(shrink, struct lock_info, + shrinker); + struct super_block *sb = linfo->sb; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + unsigned long nr; + int ret; + + nr = sc->nr_to_scan; + if (nr == 0) + goto out; + + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_head) { + + if (nr-- == 0) + break; + + trace_scoutfs_lock_shrink(sb, lock); + scoutfs_inc_counter(sb, lock_shrink); + + WARN_ON_ONCE(!lock_idle(lock)); + + lock->work_prev_mode = lock->granted_mode; + lock->work_mode = DLM_LOCK_NL; + lock->granted_mode = DLM_LOCK_NL; + queue_work(linfo->workq, &lock->work); + + list_del_init(&lock->lru_head); + linfo->lru_nr--; + } + spin_unlock(&linfo->lock); + +out: + ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); + trace_scoutfs_lock_shrink_exit(sb, sc->nr_to_scan, ret); + return ret; +} + +void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) +{ + struct lock_info *linfo = SCOUTFS_SB(sb)->lock_info; + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = INT_MAX, + }; + + linfo->shrinker.shrink(&linfo->shrinker, &sc); } /* _stop is always called no matter what start returns */ @@ -1203,22 +1229,24 @@ static void scoutfs_debug_locks_seq_stop(struct seq_file *m, void *v) spin_unlock(&linfo->lock); } -/* print an upper or lower case char depending on if the flag is set */ -#define locks_flag_char(lock, nr, c) \ - (test_bit(nr, &(lock)->flags) ? c : tolower(c)) - -#define locks_flags(lock) \ - locks_flag_char(lock, SCOUTFS_LOCK_RECLAIM, 'R'), \ - locks_flag_char(lock, SCOUTFS_LOCK_DROPPED, 'D') - static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) { struct scoutfs_lock *lock = v; - SK_PCPU(seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" sequence %u refcnt %u users %u flags %c%c\n", - LN_ARG(&lock->lock_name), SK_ARG(lock->start), - SK_ARG(lock->end), lock->sequence, lock->refcnt, - lock->users, locks_flags(lock))); + SK_PCPU(seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", + LN_ARG(&lock->name), SK_ARG(lock->start), + SK_ARG(lock->end), lock->refresh_gen, lock->error, + lock->granted_mode, lock->bast_mode, + lock->work_prev_mode, lock->work_mode, + lock->waiters[DLM_LOCK_PR], + lock->waiters[DLM_LOCK_EX], + lock->waiters[DLM_LOCK_CW], + lock->users[DLM_LOCK_PR], + lock->users[DLM_LOCK_EX], + lock->users[DLM_LOCK_CW], + lock->lksb.sb_status, + lock->lksb.sb_lkid, + lock->lksb.sb_flags)); return 0; } @@ -1250,16 +1278,153 @@ static const struct file_operations scoutfs_debug_locks_fops = { .llseek = seq_lseek, }; -int scoutfs_lock_setup(struct super_block *sb) +/* + * We're going to be destroying the locks soon. We shouldn't have any + * normal task holders that would have prevented unmount. We can have + * internal threads blocked in locks. We force all currently blocked + * and future lock calls to return -ESHUTDOWN. + */ +void scoutfs_lock_shutdown(struct super_block *sb) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct rb_node *node; + + if (!linfo) + return; + + trace_scoutfs_lock_shutdown(sb, linfo); + + spin_lock(&linfo->lock); + + linfo->shutdown = true; + for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { + lock = rb_entry(node, struct scoutfs_lock, node); + wake_up(&lock->waitq); + } + + spin_unlock(&linfo->lock); +} + +/* + * By the time we get here the caller should have called _shutdown() and + * then called into all the subsystems that held locks to drop them. + * There should be no active users of locks and all future lock calls + * should fail. + * + * Our job is to make sure nothing references the locks and free them. + */ +void scoutfs_lock_destroy(struct super_block *sb) { - struct lock_info *linfo; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + struct rb_node *node; + int mode; int ret; - ret = init_lock_info(sb); - if (ret) - return ret; - linfo = sbi->lock_info; + if (!linfo) + return; + + BUG_ON(!linfo->shutdown); + + trace_scoutfs_lock_destroy(sb, linfo); + + /* stop the shrinker from queueing work */ + unregister_shrinker(&linfo->shrinker); + + /* make sure that no one's actively using locks */ + spin_lock(&linfo->lock); + for (node = rb_first(&linfo->lock_tree); node; node = rb_next(node)) { + lock = rb_entry(node, struct scoutfs_lock, node); + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] || lock->users[mode]) { + scoutfs_warn_sk(sb, "lock name "LN_FMT" start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", + LN_ARG(&lock->name), + SK_ARG(lock->start), + SK_ARG(lock->end), mode); + break; + } + } + + if (cancel_delayed_work(&lock->grace_work)) + lock->grace_pending = false; + + } + spin_unlock(&linfo->lock); + + /* stop the dlm from calling our asts or basts to queue work */ + if (linfo->lockspace) { + /* + * fs/dlm has a harmless but unannotated inversion between their + * connection and socket locking that triggers during shutdown + * and disables lockdep. + */ + lockdep_off(); + ret = dlm_release_lockspace(linfo->lockspace, 2); + lockdep_on(); + if (ret) + scoutfs_warn(sb, "dlm lockspace leave failure: %d", + ret); + } + + if (linfo->workq) { + /* pending grace work queues normal work */ + flush_workqueue(linfo->workq); + /* now all work won't queue itself */ + destroy_workqueue(linfo->workq); + } + + /* XXX does anything synchronize with open debugfs fds? */ + debugfs_remove(linfo->debug_locks_dentry); + + /* free our stale locks that now describe released dlm locks */ + spin_lock(&linfo->lock); + node = rb_first(&linfo->lock_tree); + while (node) { + lock = rb_entry(node, struct scoutfs_lock, node); + node = rb_next(node); + lock_free(linfo, lock); + } + spin_unlock(&linfo->lock); + + idr_destroy(&linfo->debug_locks_idr); + kfree(linfo); + sbi->lock_info = NULL; +} + +int scoutfs_lock_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + char name[DLM_LOCKSPACE_LEN]; + struct lock_info *linfo; + int ret; + + /* we use >= 0 to test iv and use modes as an array index */ + BUILD_BUG_ON(DLM_LOCK_IV >= 0); + BUILD_BUG_ON(DLM_LOCK_NL >= SCOUTFS_LOCK_NR_MODES); + BUILD_BUG_ON(DLM_LOCK_PR >= SCOUTFS_LOCK_NR_MODES); + BUILD_BUG_ON(DLM_LOCK_EX >= SCOUTFS_LOCK_NR_MODES); + BUILD_BUG_ON(DLM_LOCK_CW >= SCOUTFS_LOCK_NR_MODES); + + linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); + if (!linfo) + return -ENOMEM; + + linfo->sb = sb; + spin_lock_init(&linfo->lock); + linfo->lock_tree = RB_ROOT; + linfo->lock_range_tree = RB_ROOT; + linfo->shrinker.shrink = scoutfs_lock_shrink; + linfo->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&linfo->shrinker); + INIT_LIST_HEAD(&linfo->lru_list); + idr_init(&linfo->debug_locks_idr); + atomic64_set(&linfo->next_refresh_gen, 0); + + sbi->lock_info = linfo; + trace_scoutfs_lock_setup(sb, linfo); linfo->debug_locks_dentry = debugfs_create_file("locks", S_IFREG|S_IRUSR, sbi->debug_root, sb, @@ -1269,20 +1434,22 @@ int scoutfs_lock_setup(struct super_block *sb) goto out; } - linfo->lock_reclaim_wq = alloc_workqueue("scoutfs_reclaim", - WQ_UNBOUND|WQ_HIGHPRI, 0); - if (!linfo->lock_reclaim_wq) { + linfo->workq = alloc_workqueue("scoutfs_lock_work", + WQ_UNBOUND|WQ_HIGHPRI, 0); + if (!linfo->workq) { ret = -ENOMEM; goto out; } - ret = ocfs2_dlm_init(&linfo->dlmglue, sb, "null", - sbi->opts.cluster_name, linfo->ls_name, - sbi->debug_root); - if (ret) - goto out; - linfo->dlmglue_online = true; + snprintf(name, DLM_LOCKSPACE_LEN, "scoutfs_fsid_%llx", + le64_to_cpu(sbi->super.hdr.fsid)); + ret = dlm_new_lockspace(name, sbi->opts.cluster_name, + DLM_LSFL_FS | DLM_LSFL_NEWEXCL, 8, + NULL, NULL, NULL, &linfo->lockspace); + if (ret) + scoutfs_warn(sb, "dlm lockspace [%s, %s] join failure: %d", + sbi->opts.cluster_name, name, ret); out: if (ret) scoutfs_lock_destroy(sb); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 75510b49..1cabe132 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -3,40 +3,42 @@ #include #include "key.h" -#include "dlmglue.h" #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ -#define SCOUTFS_LKF_TRYLOCK 0x02 /* EAGAIN if contention */ -#define SCOUTFS_LKF_NO_TASK_REF 0x04 /* don't create a task ref */ +#define SCOUTFS_LKF_NONBLOCK 0x02 /* only use already held locks */ -/* flags for scoutfs_lock->flags */ -enum { - SCOUTFS_LOCK_RECLAIM = 0, /* lock is queued for reclaim */ - SCOUTFS_LOCK_DROPPED, /* lock is going away, drop reference */ -}; +#define SCOUTFS_LOCK_NR_MODES (DLM_LOCK_EX + 1) +/* + * A few fields (start, end, refresh_gen, granted_mode) are referenced + * by code outside lock.c. + */ struct scoutfs_lock { struct super_block *sb; - struct scoutfs_lock_name lock_name; + struct scoutfs_lock_name name; struct scoutfs_key_buf *start; struct scoutfs_key_buf *end; - struct dlm_lksb lksb; - unsigned int sequence; /* for debugging and sanity checks */ struct rb_node node; struct rb_node range_node; - unsigned int refcnt; unsigned int debug_locks_id; - struct ocfs2_lock_res lockres; - struct list_head lru_entry; - struct work_struct reclaim_work; - unsigned int users; /* Tracks active users of this lock */ - unsigned long flags; + u64 refresh_gen; + struct list_head lru_head; wait_queue_head_t waitq; - struct rb_root task_refs; - spinlock_t task_refs_lock; + struct work_struct work; + struct dlm_lksb lksb; + ktime_t grace_deadline; + struct delayed_work grace_work; + bool grace_pending; + + int error; + int granted_mode; + int bast_mode; + int work_prev_mode; + int work_mode; + unsigned int waiters[SCOUTFS_LOCK_NR_MODES]; + unsigned int users[SCOUTFS_LOCK_NR_MODES]; }; -u64 scoutfs_lock_refresh_gen(struct scoutfs_lock *lock); int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, @@ -64,6 +66,7 @@ void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); int scoutfs_lock_setup(struct super_block *sb); +void scoutfs_lock_shutdown(struct super_block *sb); void scoutfs_lock_destroy(struct super_block *sb); #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index f8e512eb..b2ce8ce9 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -35,8 +35,6 @@ #include "ioctl.h" #include "count.h" #include "bio.h" -#include "dlmglue.h" -#include "stackglue.h" #include "export.h" struct lock_info; @@ -1046,7 +1044,12 @@ DECLARE_EVENT_CLASS(scoutfs_lock_info_class, TP_printk(FSID_FMT" linfo %p", __entry->fsid, __entry->linfo) ); -DEFINE_EVENT(scoutfs_lock_info_class, init_lock_info, +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_setup, + TP_PROTO(struct super_block *sb, struct lock_info *linfo), + TP_ARGS(sb, linfo) +); + +DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_shutdown, TP_PROTO(struct super_block *sb, struct lock_info *linfo), TP_ARGS(sb, linfo) ); @@ -1594,94 +1597,85 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(u8, name_type) __field(u64, name_first) __field(u64, name_second) - __field(unsigned int, seq) - __field(unsigned int, refcnt) - __field(unsigned int, users) - __field(unsigned char, level) - __field(unsigned char, blocking) - __field(unsigned int, cw) - __field(unsigned int, pr) - __field(unsigned int, ex) + __field(u64, refresh_gen) + __field(int, error) + __field(int, granted_mode) + __field(int, bast_mode) + __field(int, work_prev_mode) + __field(int, work_mode) + __field(unsigned int, waiters_cw) + __field(unsigned int, waiters_pr) + __field(unsigned int, waiters_ex) + __field(unsigned int, users_cw) + __field(unsigned int, users_pr) + __field(unsigned int, users_ex) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - __entry->name_scope = lck->lock_name.scope; - __entry->name_zone = lck->lock_name.zone; - __entry->name_type = lck->lock_name.type; - __entry->name_first = le64_to_cpu(lck->lock_name.first); - __entry->name_second = le64_to_cpu(lck->lock_name.second); - __entry->seq = lck->sequence; - __entry->refcnt = lck->refcnt; - __entry->users = lck->users; - /* racey, but safe refs of embedded struct */ - __entry->level = lck->lockres.l_level; - __entry->blocking = lck->lockres.l_blocking; - __entry->cw = lck->lockres.l_cw_holders; - __entry->pr = lck->lockres.l_ro_holders; - __entry->ex = lck->lockres.l_ex_holders; + __entry->name_scope = lck->name.scope; + __entry->name_zone = lck->name.zone; + __entry->name_type = lck->name.type; + __entry->name_first = le64_to_cpu(lck->name.first); + __entry->name_second = le64_to_cpu(lck->name.second); + + __entry->refresh_gen = lck->refresh_gen; + __entry->error = lck->error; + __entry->granted_mode = lck->granted_mode; + __entry->bast_mode = lck->bast_mode; + __entry->work_prev_mode = lck->work_prev_mode; + __entry->work_mode = lck->work_mode; + __entry->waiters_pr = lck->waiters[DLM_LOCK_PR]; + __entry->waiters_ex = lck->waiters[DLM_LOCK_EX]; + __entry->waiters_cw = lck->waiters[DLM_LOCK_CW]; + __entry->users_pr = lck->users[DLM_LOCK_PR]; + __entry->users_ex = lck->users[DLM_LOCK_EX]; + __entry->users_cw = lck->users[DLM_LOCK_CW]; ), - TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu seq %u refs %d users %d level %u blocking %u cw %u pr %u ex %u", + TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", __entry->fsid, __entry->name_scope, __entry->name_zone, - __entry->name_type, __entry->name_first, - __entry->name_second, __entry->seq, __entry->refcnt, - __entry->users, __entry->level, __entry->blocking, - __entry->cw, __entry->pr, __entry->ex) + __entry->name_type, __entry->name_first, __entry->name_second, + __entry->refresh_gen, __entry->error, __entry->granted_mode, + __entry->bast_mode, __entry->work_prev_mode, + __entry->work_mode, __entry->waiters_pr, + __entry->waiters_ex, __entry->waiters_cw, __entry->users_pr, + __entry->users_ex, __entry->users_cw) ); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_resource, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_unlock, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_ast, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_bast, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate_ret, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_reclaim, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, shrink_lock_tree, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_rb_insert, - TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), - TP_ARGS(sb, lck) -); - DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_free, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_alloc, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_ast, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_bast, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_work, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_grace_work, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_wait, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_unlock, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_shrink, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); DECLARE_EVENT_CLASS(scoutfs_seg_class, TP_PROTO(struct scoutfs_segment *seg), @@ -1954,128 +1948,6 @@ DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_kill_sb, TP_ARGS(sb) ); -TRACE_EVENT(ocfs2_cluster_lock, - TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, - int requested, unsigned int lkm_flags, unsigned int arg_flags), - - TP_ARGS(osb, lockres, requested, lkm_flags, arg_flags), - - TP_STRUCT__entry( - __string(lockspace, osb->cconn->cc_name) - __string(lockname, lockres->l_pretty_name) - __field(int, requested) - __field(unsigned int, lkm_flags) - __field(unsigned int, arg_flags) - __field(unsigned int, lockres_flags) - __field(int, lockres_level) - __field(int, blocking) - __field(unsigned int, cw_holders) - __field(unsigned int, pr_holders) - __field(unsigned int, ex_holders) - ), - - TP_fast_assign( - __assign_str(lockspace, osb->cconn->cc_name); - __assign_str(lockname, lockres->l_pretty_name); - __entry->requested = requested; - __entry->lkm_flags = lkm_flags; - __entry->arg_flags = arg_flags; - __entry->lockres_flags = lockres->l_flags; - __entry->lockres_level = lockres->l_level; - __entry->blocking = lockres->l_blocking; - __entry->cw_holders = lockres->l_cw_holders; - __entry->pr_holders = lockres->l_ro_holders; - __entry->ex_holders = lockres->l_ex_holders; - ), - - TP_printk("lockspace %s lock %s requested %d lkm_flags 0x%x arg_flags 0x%x lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex %u/%u/%u", - __get_str(lockspace), __get_str(lockname), __entry->requested, - __entry->lkm_flags, __entry->arg_flags, - __entry->lockres_level, __entry->lockres_flags, - __entry->blocking, __entry->cw_holders, __entry->pr_holders, - __entry->ex_holders) -); - -TRACE_EVENT(ocfs2_cluster_unlock, - TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres, - int level), - - TP_ARGS(osb, lockres, level), - - TP_STRUCT__entry( - __string(lockspace, osb->cconn->cc_name) - __string(lockname, lockres->l_pretty_name) - __field(int, level) - __field(unsigned int, lockres_flags) - __field(int, lockres_level) - __field(int, blocking) - __field(unsigned int, cw_holders) - __field(unsigned int, pr_holders) - __field(unsigned int, ex_holders) - ), - - TP_fast_assign( - __assign_str(lockspace, osb->cconn->cc_name); - __assign_str(lockname, lockres->l_pretty_name); - __entry->level = level; - __entry->lockres_flags = lockres->l_flags; - __entry->lockres_level = lockres->l_level; - __entry->blocking = lockres->l_blocking; - __entry->cw_holders = lockres->l_cw_holders; - __entry->pr_holders = lockres->l_ro_holders; - __entry->ex_holders = lockres->l_ex_holders; - ), - - TP_printk("lockspace %s lock %s level %d lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", - __get_str(lockspace), __get_str(lockname), __entry->level, - __entry->lockres_level, __entry->lockres_flags, - __entry->blocking, __entry->cw_holders, __entry->pr_holders, - __entry->ex_holders) -); - -DECLARE_EVENT_CLASS(ocfs2_lock_res_class, - TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), - - TP_ARGS(osb, lockres), - - TP_STRUCT__entry( - __string(lockspace, osb->cconn->cc_name) - __string(lockname, lockres->l_pretty_name) - __field(unsigned int, lockres_flags) - __field(int, lockres_level) - __field(int, blocking) - __field(unsigned int, cw_holders) - __field(unsigned int, pr_holders) - __field(unsigned int, ex_holders) - ), - - TP_fast_assign( - __assign_str(lockspace, osb->cconn->cc_name); - __assign_str(lockname, lockres->l_pretty_name); - __entry->lockres_flags = lockres->l_flags; - __entry->lockres_level = lockres->l_level; - __entry->blocking = lockres->l_blocking; - __entry->cw_holders = lockres->l_cw_holders; - __entry->pr_holders = lockres->l_ro_holders; - __entry->ex_holders = lockres->l_ex_holders; - ), - - TP_printk("lockspace %s lock %s lockres->level %d lockres->flags 0x%x lockres->blocking %d holders cw/pr/ex: %u/%u/%u", - __get_str(lockspace), __get_str(lockname), - __entry->lockres_level, __entry->lockres_flags, - __entry->blocking, __entry->cw_holders, - __entry->pr_holders, __entry->ex_holders) -); - -DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_simple_drop_lockres, - TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), - TP_ARGS(osb, lockres) -); -DEFINE_EVENT(ocfs2_lock_res_class, ocfs2_unblock_lock, - TP_PROTO(struct ocfs2_super *osb, struct ocfs2_lock_res *lockres), - TP_ARGS(osb, lockres) -); - DECLARE_EVENT_CLASS(scoutfs_fileid_class, TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), TP_ARGS(sb, fh_type, fid), diff --git a/kmod/src/server.c b/kmod/src/server.c index 1b206f42..91146e81 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -932,10 +932,8 @@ static void scoutfs_server_func(struct work_struct *work) init_waitqueue_head(&waitq); - ret = scoutfs_lock_global(sb, DLM_LOCK_EX, - SCOUTFS_LKF_TRYLOCK, - SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, - &lock); + ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, + SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, &lock); if (ret) goto out; diff --git a/kmod/src/stackglue.c b/kmod/src/stackglue.c deleted file mode 100644 index b8a5fbe3..00000000 --- a/kmod/src/stackglue.c +++ /dev/null @@ -1,412 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; -*- - * vim: noexpandtab sw=8 ts=8 sts=0: - * - * stackglue.c - * - * Code which implements an OCFS2 specific interface to underlying - * cluster stacks. - * - * Copyright (C) 2007, 2009 Oracle. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "stackglue.h" - -static void fsdlm_lock_ast_wrapper(void *astarg) -{ - struct ocfs2_dlm_lksb *lksb = astarg; - int status = lksb->lksb_fsdlm.sb_status; - - /* - * For now we're punting on the issue of other non-standard errors - * where we can't tell if the unlock_ast or lock_ast should be called. - * The main "other error" that's possible is EINVAL which means the - * function was called with invalid args, which shouldn't be possible - * since the caller here is under our control. Other non-standard - * errors probably fall into the same category, or otherwise are fatal - * which means we can't carry on anyway. - */ - - if (status == -DLM_EUNLOCK || status == -DLM_ECANCEL) - lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, 0); - else - lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); -} - -static void fsdlm_blocking_ast_wrapper(void *astarg, int level) -{ - struct ocfs2_dlm_lksb *lksb = astarg; - - lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); -} - -static int user_dlm_lock(struct ocfs2_cluster_connection *conn, - int mode, - struct ocfs2_dlm_lksb *lksb, - u32 flags, - void *name, - unsigned int namelen) -{ - int ret; - - if (!lksb->lksb_fsdlm.sb_lvbptr) - lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + - sizeof(struct dlm_lksb); - - ret = dlm_lock(conn->cc_lockspace, mode, &lksb->lksb_fsdlm, - flags|DLM_LKF_NODLCKWT, name, namelen, 0, - fsdlm_lock_ast_wrapper, lksb, - fsdlm_blocking_ast_wrapper); - return ret; -} - -/* - * The ocfs2_dlm_lock() and ocfs2_dlm_unlock() functions take no argument - * for the ast and bast functions. They will pass the lksb to the ast - * and bast. The caller can wrap the lksb with their own structure to - * get more information. - */ -int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, - int mode, - struct ocfs2_dlm_lksb *lksb, - u32 flags, - void *name, - unsigned int namelen) -{ - if (!lksb->lksb_conn) - lksb->lksb_conn = conn; - else - BUG_ON(lksb->lksb_conn != conn); - return user_dlm_lock(conn, mode, lksb, flags, name, namelen); -} - -static int user_dlm_unlock(struct ocfs2_cluster_connection *conn, - struct ocfs2_dlm_lksb *lksb, - u32 flags) -{ - int ret; - - ret = dlm_unlock(conn->cc_lockspace, lksb->lksb_fsdlm.sb_lkid, - flags, &lksb->lksb_fsdlm, lksb); - return ret; -} - -int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, - struct ocfs2_dlm_lksb *lksb, - u32 flags) -{ - BUG_ON(lksb->lksb_conn == NULL); - - return user_dlm_unlock(conn, lksb, flags); -} - -static int user_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) -{ - return lksb->lksb_fsdlm.sb_status; -} - -int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) -{ - return user_dlm_lock_status(lksb); -} - -static int user_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) -{ - int invalid = lksb->lksb_fsdlm.sb_flags & DLM_SBF_VALNOTVALID; - - return !invalid; -} - -int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) -{ - return user_dlm_lvb_valid(lksb); -} - -static void *user_dlm_lvb(struct ocfs2_dlm_lksb *lksb) -{ - if (!lksb->lksb_fsdlm.sb_lvbptr) - lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + - sizeof(struct dlm_lksb); - return (void *)(lksb->lksb_fsdlm.sb_lvbptr); -} - -void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb) -{ - return user_dlm_lvb(lksb); -} - -void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb) -{ -} - -#if 0 -static int user_plock(struct ocfs2_cluster_connection *conn, - u64 ino, - struct file *file, - int cmd, - struct file_lock *fl) -{ - /* - * This more or less just demuxes the plock request into any - * one of three dlm calls. - * - * Internally, fs/dlm will pass these to a misc device, which - * a userspace daemon will read and write to. - * - * For now, cancel requests (which happen internally only), - * are turned into unlocks. Most of this function taken from - * gfs2_lock. - */ - - if (cmd == F_CANCELLK) { - cmd = F_SETLK; - fl->fl_type = F_UNLCK; - } - - if (IS_GETLK(cmd)) - return dlm_posix_get(conn->cc_lockspace, ino, file, fl); - else if (fl->fl_type == F_UNLCK) - return dlm_posix_unlock(conn->cc_lockspace, ino, file, fl); - else - return dlm_posix_lock(conn->cc_lockspace, ino, file, cmd, fl); -} - -int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, - struct file *file, int cmd, struct file_lock *fl) -{ - return user_plock(conn, ino, file, cmd, fl); -} -#endif - -static void user_recover_prep(void *arg) -{ - /* XXX: Set FS in recovery here */ -} - -static void user_recover_slot(void *arg, struct dlm_slot *slot) -{ - printk(KERN_INFO "scoutfs: Node %d/%d down. Initiating recovery.\n", - slot->nodeid, slot->slot); -} - -static void user_recover_done(void *arg, struct dlm_slot *slots, - int num_slots, int our_slot, - uint32_t generation) -{ - /* XXX: Do actual fs recovery here */ -} - -static const struct dlm_lockspace_ops ocfs2_ls_ops = { - .recover_prep = user_recover_prep, - .recover_slot = user_recover_slot, - .recover_done = user_recover_done, -}; - -static int user_cluster_connect(struct ocfs2_cluster_connection *conn) -{ - dlm_lockspace_t *fsdlm; -// struct ocfs2_live_connection *lc; - int rc, ops_rv; - - BUG_ON(conn == NULL); - -#if 0 - lc = kzalloc(sizeof(struct ocfs2_live_connection), GFP_KERNEL); - if (!lc) - return -ENOMEM; - - init_waitqueue_head(&lc->oc_wait); - init_completion(&lc->oc_sync_wait); - atomic_set(&lc->oc_this_node, 0); - conn->cc_private = lc; - lc->oc_type = NO_CONTROLD; -#endif - - rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name, - DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN, - &ocfs2_ls_ops, conn, &ops_rv, &fsdlm); - if (rc) { - if (rc == -EEXIST || rc == -EPROTO) - printk(KERN_ERR "scoutfs: Unable to create the " - "lockspace %s (%d), because a scoutfs-utils " - "program is running on this file system " - "with the same name lockspace\n", - conn->cc_name, rc); - goto out; - } - - if (ops_rv == -EOPNOTSUPP) { - /* - * If we get this return code, we're on a very old - * version of fs/dlm that doesn't have recovery - * callbacks enabled. - */ -// lc->oc_type = WITH_CONTROLD; - printk(KERN_NOTICE "scoutfs: You seem to be using an older " - "version of dlm_controld and/or scoutfs-utils." - " Please consider upgrading.\n"); - } else if (ops_rv) { - rc = ops_rv; - goto out; - } - conn->cc_lockspace = fsdlm; - -#if 0 - rc = ocfs2_live_connection_attach(conn, lc); - if (rc) - goto out; - - if (lc->oc_type == NO_CONTROLD) { - rc = get_protocol_version(conn); - if (rc) { - printk(KERN_ERR "ocfs2: Could not determine" - " locking version\n"); - user_cluster_disconnect(conn); - goto out; - } - wait_event(lc->oc_wait, (atomic_read(&lc->oc_this_node) > 0)); - } - - /* - * running_proto must have been set before we allowed any mounts - * to proceed. - */ - if (fs_protocol_compare(&running_proto, &conn->cc_version)) { - printk(KERN_ERR - "Unable to mount with fs locking protocol version " - "%u.%u because negotiated protocol is %u.%u\n", - conn->cc_version.pv_major, conn->cc_version.pv_minor, - running_proto.pv_major, running_proto.pv_minor); - rc = -EPROTO; - ocfs2_live_connection_drop(lc); - lc = NULL; - } -#endif -out: -#if 0 - if (rc) - kfree(lc); -#endif - return rc; -} - -int ocfs2_cluster_connect(const char *stack_name, - const char *cluster_name, - int cluster_name_len, - const char *group, - int grouplen, - struct ocfs2_locking_protocol *lproto, - void (*recovery_handler)(int node_num, - void *recovery_data), - void *recovery_data, - struct ocfs2_cluster_connection **conn) -{ - int rc = 0; - struct ocfs2_cluster_connection *new_conn; - - BUG_ON(group == NULL); - BUG_ON(conn == NULL); - BUG_ON(recovery_handler == NULL); - - if (grouplen > GROUP_NAME_MAX) { - rc = -EINVAL; - goto out; - } - -#if 0 - if (memcmp(&lproto->lp_max_version, &locking_max_version, - sizeof(struct ocfs2_protocol_version))) { - rc = -EINVAL; - goto out; - } -#endif - new_conn = kzalloc(sizeof(struct ocfs2_cluster_connection), - GFP_KERNEL); - if (!new_conn) { - rc = -ENOMEM; - goto out; - } - - strlcpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1); - new_conn->cc_namelen = grouplen; - if (cluster_name_len) - strlcpy(new_conn->cc_cluster_name, cluster_name, - CLUSTER_NAME_MAX + 1); - new_conn->cc_cluster_name_len = cluster_name_len; - new_conn->cc_recovery_handler = recovery_handler; - new_conn->cc_recovery_data = recovery_data; - - new_conn->cc_proto = lproto; - /* Start the new connection at our maximum compatibility level */ - new_conn->cc_version = lproto->lp_max_version; - -#if 0 - /* This will pin the stack driver if successful */ - rc = ocfs2_stack_driver_get(stack_name); - if (rc) - goto out_free; -#endif - - rc = user_cluster_connect(new_conn); - if (rc) { -// ocfs2_stack_driver_put(); - goto out_free; - } - - *conn = new_conn; - -out_free: - if (rc) - kfree(new_conn); - -out: - return rc; -} - -static int user_cluster_disconnect(struct ocfs2_cluster_connection *conn) -{ - dlm_release_lockspace(conn->cc_lockspace, 2); - conn->cc_lockspace = NULL; - conn->cc_private = NULL; - return 0; -} - -/* If hangup_pending is 0, the stack driver will be dropped */ -int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, - int hangup_pending) -{ - int ret; - - BUG_ON(conn == NULL); - - ret = user_cluster_disconnect(conn); - - /* XXX Should we free it anyway? */ - if (!ret) { - kfree(conn); -#if 0 - if (!hangup_pending) - ocfs2_stack_driver_put(); -#endif - } - - return ret; -} diff --git a/kmod/src/stackglue.h b/kmod/src/stackglue.h deleted file mode 100644 index e3db7678..00000000 --- a/kmod/src/stackglue.h +++ /dev/null @@ -1,149 +0,0 @@ -/* -*- mode: c; c-basic-offset: 8; -*- - * vim: noexpandtab sw=8 ts=8 sts=0: - * - * stackglue.h - * - * Glue to the underlying cluster stack. - * - * Copyright (C) 2007 Oracle. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - - -#ifndef STACKGLUE_H -#define STACKGLUE_H - -#include -#include -#include - -#include -#include - -#define DLM_LVB_LEN 64 - -/* Needed for plock-related prototypes */ -struct file; -struct file_lock; - -/* Scoutfs never uses this flag, we define it to zero to avoid errors */ -#define DLM_LKF_LOCAL 0 - -/* - * This shadows DLM_LOCKSPACE_LEN in fs/dlm/dlm_internal.h. That probably - * wants to be in a public header. - */ -#define GROUP_NAME_MAX 64 - -/* This shadows OCFS2_CLUSTER_NAME_LEN */ -#define CLUSTER_NAME_MAX 16 - -/* - * ocfs2_protocol_version changes when ocfs2 does something different in - * its inter-node behavior. See dlmglue.c for more information. - */ -struct ocfs2_protocol_version { - u8 pv_major; - u8 pv_minor; -}; - -/* - * The dlm_lockstatus struct includes lvb space, but the dlm_lksb struct only - * has a pointer to separately allocated lvb space. This struct exists only to - * include in the lksb union to make space for a combined dlm_lksb and lvb. - */ -struct fsdlm_lksb_plus_lvb { - struct dlm_lksb lksb; - char lvb[DLM_LVB_LEN]; -}; - -/* - * A union of all lock status structures. We define it here so that the - * size of the union is known. Lock status structures are embedded in - * ocfs2 inodes. - */ -struct ocfs2_cluster_connection; -struct ocfs2_dlm_lksb { - union { - struct dlm_lksb lksb_fsdlm; - struct fsdlm_lksb_plus_lvb padding; - }; - struct ocfs2_cluster_connection *lksb_conn; -}; - -/* - * The ocfs2_locking_protocol defines the handlers called on ocfs2's behalf. - */ -struct ocfs2_locking_protocol { - struct ocfs2_protocol_version lp_max_version; - void (*lp_lock_ast)(struct ocfs2_dlm_lksb *lksb); - void (*lp_blocking_ast)(struct ocfs2_dlm_lksb *lksb, int level); - void (*lp_unlock_ast)(struct ocfs2_dlm_lksb *lksb, int error); -}; - -/* - * A cluster connection. Mostly opaque to ocfs2, the connection holds - * state for the underlying stack. ocfs2 does use cc_version to determine - * locking compatibility. - */ -struct ocfs2_cluster_connection { - char cc_name[GROUP_NAME_MAX + 1]; - int cc_namelen; - char cc_cluster_name[CLUSTER_NAME_MAX + 1]; - int cc_cluster_name_len; - struct ocfs2_protocol_version cc_version; - struct ocfs2_locking_protocol *cc_proto; - void (*cc_recovery_handler)(int node_num, void *recovery_data); - void *cc_recovery_data; - void *cc_lockspace; - void *cc_private; -}; - -/* In ocfs2_downconvert_lock(), we need to know which stack we are using */ -static inline int ocfs2_is_o2cb_active(void) -{ - return 0; -} - -/* Used by the filesystem */ -int ocfs2_cluster_connect(const char *stack_name, - const char *cluster_name, - int cluster_name_len, - const char *group, - int grouplen, - struct ocfs2_locking_protocol *lproto, - void (*recovery_handler)(int node_num, - void *recovery_data), - void *recovery_data, - struct ocfs2_cluster_connection **conn); -int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, - int hangup_pending); - -struct ocfs2_lock_res; -int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, - int mode, - struct ocfs2_dlm_lksb *lksb, - u32 flags, - void *name, - unsigned int namelen); -int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, - struct ocfs2_dlm_lksb *lksb, - u32 flags); - -int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb); -int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb); -void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb); -void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb); - -int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, - struct file *file, int cmd, struct file_lock *fl); - -#endif /* STACKGLUE_H */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 00667954..ab2bedba 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -122,8 +122,7 @@ static void scoutfs_put_super(struct super_block *sb) sbi->shutdown = true; - scoutfs_unlock_flags(sb, sbi->node_id_lock, DLM_LOCK_EX, - SCOUTFS_LKF_NO_TASK_REF); + scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); sbi->node_id_lock = NULL; scoutfs_shutdown_trans(sb); @@ -133,6 +132,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_item_destroy(sb); /* the server locks the listen address and compacts */ + scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); From c1311783d5e4dbb2ff969126a997ce76ad77eaa8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 2 Feb 2018 09:51:20 -0800 Subject: [PATCH 554/920] scoutfs: add tracking of online and offline blocks Signed-off-by: Zach Brown --- kmod/src/data.c | 26 +++++++++++++++++------ kmod/src/data.h | 4 ++-- kmod/src/format.h | 8 +++++++ kmod/src/inode.c | 54 +++++++++++++++++++++++++++++++++++++++++++++-- kmod/src/inode.h | 6 ++++++ kmod/src/ioctl.c | 5 ++++- kmod/src/ioctl.h | 2 ++ 7 files changed, 93 insertions(+), 12 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index bf56c45b..bae06b3c 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -603,9 +603,13 @@ out: * This is the low level extent item manipulation code. We hold and * release the transaction so the caller doesn't have to deal with * partial progress. + * + * If the inode is provided then we update its tracking of the online + * and offline blocks. If it's not provided then the inode is being + * destroyed and we don't have to keep it updated. */ -int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 last, bool offline, +int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, struct scoutfs_lock *lock) { struct scoutfs_key_buf last_key; @@ -687,12 +691,17 @@ int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, break; map->blknos[i] = 0; + scoutfs_inode_add_online_blocks(inode, -1); } - if (offline && !test_bit(i, map->offline)) + if (offline && !test_bit(i, map->offline)) { set_bit(i, map->offline); - else if (!offline && test_bit(i, map->offline)) + scoutfs_inode_add_offline_blocks(inode, 1); + + } else if (!offline && test_bit(i, map->offline)) { clear_bit(i, map->offline); + scoutfs_inode_add_offline_blocks(inode, -1); + } modified = true; } @@ -905,7 +914,8 @@ out: * a new segment. Lots of concurrent allocations can interleave at * segment granularity. */ -static int find_alloc_block(struct super_block *sb, struct block_mapping *map, +static int find_alloc_block(struct super_block *sb, struct inode *inode, + struct block_mapping *map, struct scoutfs_key_buf *map_key, unsigned map_ind, bool map_exists, struct scoutfs_lock *data_lock) @@ -973,8 +983,10 @@ static int find_alloc_block(struct super_block *sb, struct block_mapping *map, goto out; /* update the mapping */ - clear_bit(map_ind, map->offline); + if (test_and_clear_bit(map_ind, map->offline)) + scoutfs_inode_add_offline_blocks(inode, -1); map->blknos[map_ind] = blkno; + scoutfs_inode_add_online_blocks(inode, 1); bytes = encode_mapping(map); scoutfs_kvec_init(val, map->encoded, bytes); @@ -1049,7 +1061,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, * and try again if we've already done a bulk alloc in * our transaction. */ - ret = find_alloc_block(sb, map, &key, ind, exists, lock); + ret = find_alloc_block(sb, inode, map, &key, ind, exists, lock); if (ret) goto out; set_buffer_new(bh); diff --git a/kmod/src/data.h b/kmod/src/data.h index a305ad2e..e8157065 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -4,8 +4,8 @@ extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; -int scoutfs_data_truncate_items(struct super_block *sb, u64 ino, u64 iblock, - u64 last, bool offline, +int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, struct scoutfs_lock *lock); int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); diff --git a/kmod/src/format.h b/kmod/src/format.h index 90d1d4c7..bb8d674a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -449,6 +449,12 @@ struct scoutfs_timespec { * have changed. It is exposed via an ioctl and is then provided as an * argument to data functions to protect racing modification. * + * @online_blocks: The number of fixed 4k blocks currently allocated and + * storing data in the volume. + * + * @offline_blocks: The number of fixed 4k blocks that could be made + * online by staging. + * * XXX * - otime? * - compat flags? @@ -462,6 +468,8 @@ struct scoutfs_inode { __le64 meta_seq; __le64 data_seq; __le64 data_version; + __le64 online_blocks; + __le64 offline_blocks; __le64 next_readdir_pos; __le32 nlink; __le32 uid; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 7207466e..2146ed9c 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -231,6 +231,8 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->meta_seq = le64_to_cpu(cinode->meta_seq); ci->data_seq = le64_to_cpu(cinode->data_seq); ci->data_version = le64_to_cpu(cinode->data_version); + ci->online_blocks = le64_to_cpu(cinode->online_blocks); + ci->offline_blocks = le64_to_cpu(cinode->offline_blocks); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); ci->flags = le32_to_cpu(cinode->flags); @@ -380,8 +382,9 @@ int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) return 0; start = (i_size_read(inode) + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; - ret = scoutfs_data_truncate_items(inode->i_sb, scoutfs_ino(inode), - start, ~0ULL, false, lock); + ret = scoutfs_data_truncate_items(inode->i_sb, inode, + scoutfs_ino(inode), start, ~0ULL, + false, lock); err = clear_truncate_flag(inode, lock); return ret ? ret : err; @@ -493,6 +496,35 @@ void scoutfs_inode_inc_data_version(struct inode *inode) preempt_enable(); } +static void add_seq_value(struct scoutfs_inode_info *si, u64 *si_u64, u64 val) +{ + preempt_disable(); + write_seqcount_begin(&si->seqcount); + *si_u64 += val; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + +void scoutfs_inode_add_online_blocks(struct inode *inode, u64 val) +{ + struct scoutfs_inode_info *si; + + if (inode) { + si = SCOUTFS_I(inode); + add_seq_value(si, &SCOUTFS_I(inode)->online_blocks, val); + } +} + +void scoutfs_inode_add_offline_blocks(struct inode *inode, u64 val) +{ + struct scoutfs_inode_info *si; + + if (inode) { + si = SCOUTFS_I(inode); + add_seq_value(si, &SCOUTFS_I(inode)->offline_blocks, val); + } +} + static u64 read_seqcount_u64(struct inode *inode, u64 *val) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); @@ -528,6 +560,19 @@ u64 scoutfs_inode_data_version(struct inode *inode) return read_seqcount_u64(inode, &si->data_version); } +u64 scoutfs_inode_online_blocks(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->online_blocks); +} + +u64 scoutfs_inode_offline_blocks(struct inode *inode) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + return read_seqcount_u64(inode, &si->offline_blocks); +} static int scoutfs_iget_test(struct inode *inode, void *arg) { struct scoutfs_inode_info *ci = SCOUTFS_I(inode); @@ -612,6 +657,9 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->meta_seq = cpu_to_le64(scoutfs_inode_meta_seq(inode)); cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); cinode->data_version = cpu_to_le64(scoutfs_inode_data_version(inode)); + cinode->online_blocks = cpu_to_le64(scoutfs_inode_online_blocks(inode)); + cinode->offline_blocks = + cpu_to_le64(scoutfs_inode_offline_blocks(inode)); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); cinode->flags = cpu_to_le32(ci->flags); } @@ -1270,6 +1318,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci = SCOUTFS_I(inode); ci->ino = ino; ci->data_version = 0; + ci->online_blocks = 0; + ci->offline_blocks = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; ci->have_item = false; atomic64_set(&ci->last_refreshed, lock->refresh_gen); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 532bb21a..0bc95a4c 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -21,6 +21,8 @@ struct scoutfs_inode_info { u64 meta_seq; u64 data_seq; u64 data_version; + u64 online_blocks; + u64 offline_blocks; u32 flags; /* @@ -97,9 +99,13 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); +void scoutfs_inode_add_online_blocks(struct inode *inode, u64 val); +void scoutfs_inode_add_offline_blocks(struct inode *inode, u64 val); u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); u64 scoutfs_inode_data_version(struct inode *inode); +u64 scoutfs_inode_online_blocks(struct inode *inode); +u64 scoutfs_inode_offline_blocks(struct inode *inode); int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock); int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index cb86dfa4..57faec81 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -378,7 +378,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SHIFT) - 1; truncate_inode_pages_range(&inode->i_data, start, end_inc); - ret = scoutfs_data_truncate_items(sb, scoutfs_ino(inode), args.block, + ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), + args.block, args.block + args.count - 1, true, lock); out: @@ -522,6 +523,8 @@ static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) stm.meta_seq = scoutfs_inode_meta_seq(inode); stm.data_seq = scoutfs_inode_data_seq(inode); stm.data_version = scoutfs_inode_data_version(inode); + stm.online_blocks = scoutfs_inode_online_blocks(inode); + stm.offline_blocks = scoutfs_inode_offline_blocks(inode); if (copy_to_user((void __user *)arg, &stm, stm.valid_bytes)) return -EFAULT; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 34917a34..33c94b95 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -178,6 +178,8 @@ struct scoutfs_ioctl_stat_more { __u64 meta_seq; __u64 data_seq; __u64 data_version; + __u64 online_blocks; + __u64 offline_blocks; } __packed; #define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ From 302b0f53162bf8d16ba6bf4b1172bcf6f1caa3bc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 14 Feb 2018 13:35:56 -0800 Subject: [PATCH 555/920] scoutfs: track inode 512b block count We weren't doing anything with the inode blocks field. We weren't even initializing it which explains why we'd sometimes see garbage i_blocks values in scoutfs inodes in segments. The logical blocks field reflects the contents of the file regardless of whether its online or not. It's the sum of our online and offline block tracking. So we can initialize it to our persistent online and offline counts and then keep it in sync as blocks are allocated and freed. Signed-off-by: Zach Brown --- kmod/src/data.c | 9 ++++++++- kmod/src/format.h | 3 ++- kmod/src/inode.c | 8 +++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index bae06b3c..16f3b20e 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -692,15 +692,19 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, map->blknos[i] = 0; scoutfs_inode_add_online_blocks(inode, -1); + /* XXX gets tricky with concurrent writes */ + inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; } if (offline && !test_bit(i, map->offline)) { set_bit(i, map->offline); scoutfs_inode_add_offline_blocks(inode, 1); + inode->i_blocks += SCOUTFS_BLOCK_SECTORS; } else if (!offline && test_bit(i, map->offline)) { clear_bit(i, map->offline); scoutfs_inode_add_offline_blocks(inode, -1); + inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; } modified = true; @@ -983,10 +987,13 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, goto out; /* update the mapping */ - if (test_and_clear_bit(map_ind, map->offline)) + if (test_and_clear_bit(map_ind, map->offline)) { scoutfs_inode_add_offline_blocks(inode, -1); + inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; + } map->blknos[map_ind] = blkno; scoutfs_inode_add_online_blocks(inode, 1); + inode->i_blocks += SCOUTFS_BLOCK_SECTORS; bytes = encode_mapping(map); scoutfs_kvec_init(val, map->encoded, bytes); diff --git a/kmod/src/format.h b/kmod/src/format.h index bb8d674a..99241394 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -13,6 +13,8 @@ #define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) #define SCOUTFS_BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_BLOCK_SECTOR_SHIFT (SCOUTFS_BLOCK_SHIFT - 9) +#define SCOUTFS_BLOCK_SECTORS (1 << SCOUTFS_BLOCK_SECTOR_SHIFT) /* * FS data is stored in segments, for now they're fixed size. They'll @@ -464,7 +466,6 @@ struct scoutfs_timespec { */ struct scoutfs_inode { __le64 size; - __le64 blocks; __le64 meta_seq; __le64 data_seq; __le64 data_version; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 2146ed9c..965afbde 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -234,9 +234,15 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->online_blocks = le64_to_cpu(cinode->online_blocks); ci->offline_blocks = le64_to_cpu(cinode->offline_blocks); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); - ci->flags = le32_to_cpu(cinode->flags); + /* + * i_blocks is initialized from online and offline and is then + * maintained as blocks come and go. + */ + inode->i_blocks = (ci->online_blocks = + ci->offline_blocks) + << SCOUTFS_BLOCK_SECTOR_SHIFT; + set_item_info(ci, cinode); } From e31e828aff00c2f01722ea5e69a4433907365203 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Feb 2018 13:31:01 -0800 Subject: [PATCH 556/920] scoutfs: don't livelock conflicting waiters If there are two tasks waiting for conflicting modes, say a writer waiting for a CW index lock and a index walker waiting for a PR index lock, they can livelock. In the ast one of their modes will be granted. We'll wake them under the lock now that they can see that their mode is ready. But then while still under the lock we see a conflicting waiter, and no users, so we immediately start converting the lock away to the other waiting conflicting mode. The woken waiter is scheduled but now sees that the lock isn't granted anymore because it's converting. This bounces back and forth forever. The fix is to refuse to start conversion while there are still waiters for the currently granted mode. Once they finish it'll be able to convert. Signed-off-by: Zach Brown --- kmod/src/lock.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 565a1c8c..3d6e868e 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -405,10 +405,10 @@ static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) /* * Convert on behalf of waiters who aren't satisfied by the - * current mode when it won't conflict with users or a pending - * bast conversion. The new mode may or may not match the - * current granted mode so we may or may not need to block users - * during the transition. + * current mode when it won't conflict with specific waiters, + * matching users, or pending bast conversions. The new mode + * may or may not match the current granted mode so we may or + * may not need to block users during the transition. * * Remember that the presence of waiters doesn't necessarily * mean that they're blocked. Multiple lock attempts naturally @@ -418,6 +418,8 @@ static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { if (lock->work_mode < 0 && lock->waiters[mode] && + (lock->granted_mode < 0 || + !lock->waiters[lock->granted_mode]) && !lock_modes_match(lock->granted_mode, mode) && lock_counts_match(mode, lock->users) && (lock->bast_mode < 0 || From c76c6582f010c3121137dca7c24a280c34f22b92 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 22 Feb 2018 10:29:25 -0800 Subject: [PATCH 557/920] scoutfs: release server conn under mutex I was rarely seeing null derefs during unmount. The per-mount listening scoutfs_server_func() was seeing null sock->ops as it called kernel_sock_shutdown() to shutdown the connected client sockets. sock_release() sets the ops to null. We're not supposed to use a socket after we call it. The per-connection scoutfs_server_recv_func() calls sock_release() as it tears down its connection. But it does this before it removes the connection from the listener's list. There's a brief window where the connection's socket has been released but is still visible on the list. If the listener tries to shutdown during this time it will crash. Hitting this window depends on scheduling races during unmount. The unmount path has the client close its connection to the server then the server closes all its connected clients. If the local mount is the server then it will have recv work see an error as the client disconnects and it will be racing to shut down the connection with the listening thread during unmount. I think I only saw this in my guests because they're running slower debug kernels on my slower laptop. The window of vulnerability while the released socket is on the list is longer. The fix is to release the socket while we hold the mutex and are removing the connection from the list. A released socket is never visible on the list. While we're at it don't use list_for_each_entry_safe() to iterate over the connection list. We're not modifying it. This is an lingering artifact from previous versions of the server code. Signed-off-by: Zach Brown --- kmod/src/server.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 91146e81..ff1ccc69 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -867,10 +867,9 @@ out: destroy_workqueue(req_wq); } - sock_release(conn->sock); - /* process_one_work explicitly allows freeing work in its func */ mutex_lock(&server->mutex); + sock_release(conn->sock); list_del_init(&conn->head); kfree(conn); smp_mb(); @@ -920,7 +919,6 @@ static void scoutfs_server_func(struct work_struct *work) struct socket *sock = NULL; struct scoutfs_lock *lock = NULL; struct server_connection *conn; - struct server_connection *conn_tmp; struct pending_seq *ps; struct pending_seq *ps_tmp; DECLARE_WAIT_QUEUE_HEAD(waitq); @@ -1043,7 +1041,7 @@ static void scoutfs_server_func(struct work_struct *work) /* shutdown send and recv on all accepted sockets */ mutex_lock(&server->mutex); - list_for_each_entry_safe(conn, conn_tmp, &conn_list, head) + list_for_each_entry(conn, &conn_list, head) kernel_sock_shutdown(conn->sock, SHUT_RDWR); mutex_unlock(&server->mutex); From f9e282048f60a191fdcaf0dcee9474d038c12b7b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 27 Feb 2018 15:34:25 -0800 Subject: [PATCH 558/920] scoutfs: revalidate dentries by checking items Initially we had d_revalidate always return that the dentry was invalid. This avoids dentry cache consistency problems across the cluster by always performing lookups. That's slow by itself, but it turns out that the dentry invalidation that happens on revalidation failure is very expensive if you have lots of dentries. So we switched to forcefully dropping dirents as we revoked their lock. That avoided the cost of revalidation failure but it adds the problem that dentries are unhashed when their locks are dropped. This causes paths like getcwd() to return errors when they see unhashed dentries instead of trying to revalidate them. This implements a d_revalidate which actually does work to determine if the dentry is still valid. When we populate dentries under a lock we add them to a list on the lock. As we drop the lock we remove them from the list. But the dentry is not modified. This lets paths like getcwd() still work. Then we implement revalidation that does the actual item lookups if the dentry's lock has been dropped. This lets revalidation return success and avoid the terrible invalidation costs from returning failure and then calling lookup to populate a new dentry. This brings us more in line with the revalidation behaviour of other systems that maintain multi-node dcache consistency. Signed-off-by: Zach Brown --- kmod/src/counters.h | 7 ++ kmod/src/dir.c | 162 +++++++++++++++++++++++++++++++++++++------- kmod/src/lock.c | 111 ++++++++++++++++++++++-------- kmod/src/lock.h | 18 +++++ 4 files changed, 245 insertions(+), 53 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 57cc8c6c..61397277 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -29,6 +29,13 @@ EXPAND_COUNTER(data_write_begin) \ EXPAND_COUNTER(data_write_end) \ EXPAND_COUNTER(data_writepage) \ + EXPAND_COUNTER(dentry_revalidate_error) \ + EXPAND_COUNTER(dentry_revalidate_invalid) \ + EXPAND_COUNTER(dentry_revalidate_locked) \ + EXPAND_COUNTER(dentry_revalidate_orphan) \ + EXPAND_COUNTER(dentry_revalidate_rcu) \ + EXPAND_COUNTER(dentry_revalidate_root) \ + EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_delete) \ diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 85b9143e..7cbc9bb6 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -31,6 +31,7 @@ #include "kvec.h" #include "item.h" #include "lock.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -101,28 +102,35 @@ static unsigned int dentry_type(unsigned int type) } /* - * Each dentry stores the values that are needed to build the keys of - * the items that are removed on unlink so that we don't to search - * through items on unlink. + * @readdir_pos lets us remove items on final unlink without having to + * look them up. + * + * @lock_cov tells revalidation that the dentry is still locked and valid. */ struct dentry_info { u64 readdir_pos; + struct scoutfs_lock_coverage lock_cov; }; static struct kmem_cache *dentry_info_cache; static void scoutfs_d_release(struct dentry *dentry) { + struct super_block *sb = dentry->d_sb; struct dentry_info *di = dentry->d_fsdata; if (di) { + scoutfs_lock_del_coverage(sb, &di->lock_cov); kmem_cache_free(dentry_info_cache, di); dentry->d_fsdata = NULL; } } +static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags); + static const struct dentry_operations scoutfs_dentry_ops = { .d_release = scoutfs_d_release, + .d_revalidate = scoutfs_d_revalidate, }; static int alloc_dentry_info(struct dentry *dentry) @@ -137,6 +145,8 @@ static int alloc_dentry_info(struct dentry *dentry) if (!di) return -ENOMEM; + scoutfs_lock_init_coverage(&di->lock_cov); + spin_lock(&dentry->d_lock); if (!dentry->d_fsdata) { dentry->d_fsdata = di; @@ -150,7 +160,8 @@ static int alloc_dentry_info(struct dentry *dentry) return 0; } -static void update_dentry_info(struct dentry *dentry, u64 pos) +static void update_dentry_info(struct super_block *sb, struct dentry *dentry, + u64 pos, struct scoutfs_lock *lock) { struct dentry_info *di = dentry->d_fsdata; @@ -158,6 +169,7 @@ static void update_dentry_info(struct dentry *dentry, u64 pos) return; di->readdir_pos = pos; + scoutfs_lock_add_coverage(sb, lock, &di->lock_cov); } static u64 dentry_info_pos(struct dentry *dentry) @@ -225,6 +237,116 @@ static struct scoutfs_key_buf *alloc_link_backref_key(struct super_block *sb, return key; } +/* + * Looks for the dirent item and fills the caller's dirent if it finds + * it. Returns item lookup errors including -ENOENT if it's not found. + */ +static int lookup_dirent(struct super_block *sb, struct inode *dir, + const char *name, unsigned name_len, + struct scoutfs_dirent *dent, + struct scoutfs_lock *lock) +{ + struct scoutfs_key_buf *key = NULL; + SCOUTFS_DECLARE_KVEC(val); + int ret; + + key = alloc_dirent_key(sb, scoutfs_ino(dir), name, name_len); + if (!key) { + ret = -ENOMEM; + goto out; + } + + scoutfs_kvec_init(val, dent, sizeof(struct scoutfs_dirent)); + + ret = scoutfs_item_lookup_exact(sb, key, val, + sizeof(struct scoutfs_dirent), lock); +out: + scoutfs_key_free(sb, key); + return ret; +} + +static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct super_block *sb = dentry->d_sb; + struct dentry_info *di = dentry->d_fsdata; + struct scoutfs_lock *lock = NULL; + struct scoutfs_dirent dent; + struct dentry *parent = NULL; + struct inode *dir; + u64 dentry_ino; + int ret; + + /* don't think this happens but we can find out */ + if (IS_ROOT(dentry)) { + scoutfs_inc_counter(sb, dentry_revalidate_root); + if (!dentry->d_inode || + (scoutfs_ino(dentry->d_inode) != SCOUTFS_ROOT_INO)) { + ret = -EIO; + } else { + ret = 1; + } + goto out; + } + + /* XXX what are the rules for _RCU? */ + if (flags & LOOKUP_RCU) { + scoutfs_inc_counter(sb, dentry_revalidate_rcu); + ret = -ECHILD; + goto out; + } + + if (WARN_ON_ONCE(di == NULL)) { + ret = 0; + goto out; + } + + if (scoutfs_lock_is_covered(sb, &di->lock_cov)) { + scoutfs_inc_counter(sb, dentry_revalidate_locked); + ret = 1; + goto out; + } + + parent = dget_parent(dentry); + if (!parent || !parent->d_inode) { + scoutfs_inc_counter(sb, dentry_revalidate_orphan); + ret = 0; + goto out; + } + dir = parent->d_inode; + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, dir, &lock); + if (ret) + goto out; + + ret = lookup_dirent(sb, dir, dentry->d_name.name, dentry->d_name.len, + &dent, lock); + if (ret == -ENOENT) + dent.ino = 0; + else if (ret < 0) + goto out; + + dentry_ino = dentry->d_inode ? scoutfs_ino(dentry->d_inode) : 0; + + if ((dentry_ino == le64_to_cpu(dent.ino))) { + update_dentry_info(sb, dentry, le64_to_cpu(dent.readdir_pos), + lock); + scoutfs_inc_counter(sb, dentry_revalidate_valid); + ret = 1; + } else { + scoutfs_inc_counter(sb, dentry_revalidate_invalid); + ret = 0; + } + +out: + dput(parent); + scoutfs_unlock(sb, lock, DLM_LOCK_PR); + + if (ret < 0 && ret != -ECHILD) + scoutfs_inc_counter(sb, dentry_revalidate_error); + + return ret; +} + /* * Because of rename, locks are ordered by inode number. To hold the * dir lock while calling iget, we might have to already hold a lesser @@ -240,10 +362,8 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct super_block *sb = dir->i_sb; - struct scoutfs_key_buf *key = NULL; - struct scoutfs_dirent dent; struct scoutfs_lock *dir_lock = NULL; - SCOUTFS_DECLARE_KVEC(val); + struct scoutfs_dirent dent; struct inode *inode; u64 ino = 0; int ret; @@ -257,28 +377,22 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, if (ret) goto out; - key = alloc_dirent_key(sb, scoutfs_ino(dir), - dentry->d_name.name, dentry->d_name.len); - if (!key) { - ret = -ENOMEM; - goto out; - } - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, dir, &dir_lock); if (ret) goto out; - scoutfs_kvec_init(val, &dent, sizeof(dent)); - - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), dir_lock); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); + ret = lookup_dirent(sb, dir, dentry->d_name.name, dentry->d_name.len, + &dent, dir_lock); if (ret == -ENOENT) { ino = 0; ret = 0; } else if (ret == 0) { ino = le64_to_cpu(dent.ino); - update_dentry_info(dentry, le64_to_cpu(dent.readdir_pos)); + update_dentry_info(sb, dentry, le64_to_cpu(dent.readdir_pos), + dir_lock); } + scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); + out: if (ret < 0) inode = ERR_PTR(ret); @@ -287,8 +401,6 @@ out: else inode = scoutfs_iget(sb, ino); - scoutfs_key_free(sb, key); - return d_splice_alias(inode, dentry); } @@ -625,7 +737,7 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, if (ret) goto out; - update_dentry_info(dentry, pos); + update_dentry_info(sb, dentry, pos, dir_lock); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -720,7 +832,7 @@ retry: inode->i_mode, dir_lock, inode_lock); if (ret) goto out; - update_dentry_info(dentry, pos); + update_dentry_info(sb, dentry, pos, dir_lock); i_size_write(dir, dir_size); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -1021,7 +1133,7 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, if (ret) goto out; - update_dentry_info(dentry, pos); + update_dentry_info(sb, dentry, pos, dir_lock); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -1507,7 +1619,7 @@ retry: /* won't fail from here on out, update all the vfs structs */ /* the caller will use d_move to move the old_dentry into place */ - update_dentry_info(old_dentry, new_pos); + update_dentry_info(sb, old_dentry, new_pos, new_dir_lock); i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); if (!new_inode) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 3d6e868e..e3178bc8 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -92,41 +92,17 @@ static void scoutfs_lock_grace_work(struct work_struct *work); /* * invalidate cached data associated with an inode whose lock is going * away. - * - * Our inode granular locks mean that we have to invalidate all the - * child dentries of a dir so that they can't satisfy lookup after we - * re-acquire the lock. We're invalidating the lock so there can't be - * active users that could modify the entries in the dcache (lookup, - * create, rename, unlink). We have to make it through all the child - * entries and remove them from the hash so that lookup can't find them. */ static void invalidate_inode(struct super_block *sb, u64 ino) { struct inode *inode; - struct dentry *parent; - struct dentry *child; inode = scoutfs_ilookup(sb, ino); - if (!inode) - return; - - if (S_ISREG(inode->i_mode)) - truncate_inode_pages(inode->i_mapping, 0); - - if (S_ISDIR(inode->i_mode) && (parent = d_find_alias(inode))) { - - spin_lock(&parent->d_lock); - list_for_each_entry(child, &parent->d_subdirs, d_u.d_child){ - spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); - __d_drop(child); - spin_unlock(&child->d_lock); - } - spin_unlock(&parent->d_lock); - - dput(parent); + if (inode) { + if (S_ISREG(inode->i_mode)) + truncate_inode_pages(inode->i_mapping, 0); + iput(inode); } - - iput(inode); } /* @@ -138,6 +114,8 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, { struct scoutfs_key_buf *start = lock->start; struct scoutfs_key_buf *end = lock->end; + struct scoutfs_lock_coverage *cov; + struct scoutfs_lock_coverage *tmp; u64 ino, last; int ret; @@ -156,6 +134,21 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, if (prev == DLM_LOCK_CW || (prev == DLM_LOCK_PR && mode != DLM_LOCK_EX) || (prev == DLM_LOCK_EX && mode != DLM_LOCK_PR)) { + +retry: + spin_lock(&lock->cov_list_lock); + list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) { + if (!spin_trylock(&cov->cov_lock)) { + spin_unlock(&lock->cov_list_lock); + cpu_relax(); + goto retry; + } + list_del_init(&cov->head); + cov->lock = NULL; + spin_unlock(&cov->cov_lock); + } + spin_unlock(&lock->cov_list_lock); + if (lock->name.zone == SCOUTFS_FS_ZONE) { ino = le64_to_cpu(lock->name.first); last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; @@ -237,6 +230,9 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, RB_CLEAR_NODE(&lock->range_node); INIT_LIST_HEAD(&lock->lru_head); + spin_lock_init(&lock->cov_list_lock); + INIT_LIST_HEAD(&lock->cov_list); + if (start) { lock->start = scoutfs_key_dup(sb, start); lock->end = scoutfs_key_dup(sb, end); @@ -1136,6 +1132,65 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) spin_unlock(&linfo->lock); } +void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov) +{ + spin_lock_init(&cov->cov_lock); + cov->lock = NULL; + INIT_LIST_HEAD(&cov->head); +} + +/* + * Record that the given coverage struct is protected by the given lock. + * Once the lock is dropped the coverage list head will be removed and + * callers can use that to see that the cov isn't covered any more. The + * cov might be on another lock so we're careful to remove it. + */ +void scoutfs_lock_add_coverage(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_lock_coverage *cov) +{ + spin_lock(&cov->cov_lock); + + if (cov->lock) { + spin_lock(&cov->lock->cov_list_lock); + list_del_init(&cov->head); + spin_unlock(&cov->lock->cov_list_lock); + cov->lock = NULL; + } + + cov->lock = lock; + spin_lock(&cov->lock->cov_list_lock); + list_add(&cov->head, &lock->cov_list); + spin_unlock(&cov->lock->cov_list_lock); + + spin_unlock(&cov->cov_lock); +} + +bool scoutfs_lock_is_covered(struct super_block *sb, + struct scoutfs_lock_coverage *cov) +{ + bool covered; + + spin_lock(&cov->cov_lock); + covered = !list_empty_careful(&cov->head); + spin_unlock(&cov->cov_lock); + + return covered; +} + +void scoutfs_lock_del_coverage(struct super_block *sb, + struct scoutfs_lock_coverage *cov) +{ + spin_lock(&cov->cov_lock); + if (cov->lock) { + spin_lock(&cov->lock->cov_list_lock); + list_del_init(&cov->head); + spin_unlock(&cov->lock->cov_list_lock); + cov->lock = NULL; + } + spin_unlock(&cov->cov_lock); +} + static int scoutfs_lock_shrink(struct shrinker *shrink, struct shrink_control *sc) { diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 1cabe132..12e8c610 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -30,6 +30,9 @@ struct scoutfs_lock { struct delayed_work grace_work; bool grace_pending; + spinlock_t cov_list_lock; + struct list_head cov_list; + int error; int granted_mode; int bast_mode; @@ -39,6 +42,12 @@ struct scoutfs_lock { unsigned int users[SCOUTFS_LOCK_NR_MODES]; }; +struct scoutfs_lock_coverage { + spinlock_t cov_lock; + struct scoutfs_lock *lock; + struct list_head head; +}; + int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, @@ -63,6 +72,15 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, int level, int flags); +void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov); +void scoutfs_lock_add_coverage(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_lock_coverage *cov); +bool scoutfs_lock_is_covered(struct super_block *sb, + struct scoutfs_lock_coverage *cov); +void scoutfs_lock_del_coverage(struct super_block *sb, + struct scoutfs_lock_coverage *cov); + void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); int scoutfs_lock_setup(struct super_block *sb); From 241b52d55a0c187d68b39efc3dc9ae4660ae13ec Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Feb 2018 12:46:06 -0800 Subject: [PATCH 559/920] scoutfs: reserve at least one xattr item value Even when we're setting an xattr with no value we still have a file system item value that contains the xattr value header which tells us that this is the last value. This fixes a warning that would be issued if we tried to set an xattr with a zero length value. We'd try to dirty an item value with the header after having reserved zero bytes for item values. To hit the warning the inode couldn't already be dirty so that the xattr value didn't get to hide in the unsed reservation for dirtying the inode item's value. Signed-off-by: Zach Brown --- kmod/src/count.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index fedeba69..1139c4cd 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -184,19 +184,22 @@ static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, /* * Setting an xattr results in a dirty set of items with values for the - * size of the xattr. Any previously existing items from a larger xattr - * are deleted which dirties their key but removes their value. We - * don't know the size of a possibly existing xattr so we assume max - * parts. + * size of the xattr. There's always at least one item with a value + * header. Any previously existing items from a larger xattr are + * deleted which dirties their key but removes their value. We don't + * know the size of a possibly existing xattr so we assume max parts. */ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned name_len, unsigned size) { struct scoutfs_item_count cnt = {0,}; - unsigned val_parts = DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE); + unsigned val_parts; __count_dirty_inode(&cnt); + val_parts = max_t(unsigned, 1, + DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE)); + cnt.items += SCOUTFS_XATTR_MAX_PARTS; cnt.keys += SCOUTFS_XATTR_MAX_PARTS * (offsetof(struct scoutfs_xattr_key, name[name_len]) + From 6adb24f0f52291b81d3a770d83646efb08f4f638 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 1 Mar 2018 17:32:21 -0800 Subject: [PATCH 560/920] scoutfs: clean up compaction destruction We're seeing warnings from trying to destroy the server work queue while it's still active. Auditing shows that almost all of the sources of queued work are shutdown before we destroy the work queue. Except for the compaction func. It queues itself via the sneaky call to scoutfs_compact_kick() inside scoutfs_client_finish_compaction(). What a mess. We only wait for work to finish running in scoutfs_compact_destroy(), we don't forbid further queueing. So with just the right races it looks possible to have the compact func executing after we return from _destroy(). It can then later try to queue the commit_work in the server workqueue. It's pretty hard to imagine this race, but it's made a bit easier by the startling fact that we don't free the compact info struct. That makes it a little easier to imagine use-after-destroy not exploding. So let's forcibly forbid chain queueing during compaction shutdown by using cancel_work_sync(). It marks the work canceling while flushing so the queue_work in the work func won't do anything. This should ensure that the compaction func isn't running when destroy returns. Also while we're at it actually free the allocated compaction info struct! Cool cool cool. Signed-off-by: Zach Brown --- kmod/src/compact.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 428346a0..96a38952 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -666,8 +666,11 @@ void scoutfs_compact_destroy(struct super_block *sb) DECLARE_COMPACT_INFO(sb, ci); if (ci) { - flush_work(&ci->work); + /* stop compaction from requeueing itself */ + cancel_work_sync(&ci->work); destroy_workqueue(ci->workq); sbi->compact_info = NULL; + + kfree(ci); } } From 8ec5b7efe37cff7bfe60541c318c5b76714dbe6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Mar 2018 15:35:15 -0800 Subject: [PATCH 561/920] scoutfs: remove bio page add trace This is a very chatty trace evenet that doesn't add much value. Let's remove it and make a lot more room for other more interesting trace events. Signed-off-by: Zach Brown --- kmod/src/bio.c | 2 -- kmod/src/scoutfs_trace.h | 21 --------------------- 2 files changed, 23 deletions(-) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index f72ff4d1..b8e94b46 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -122,8 +122,6 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, continue; } - trace_scoutfs_bio_submit_added(sb, page, bio); - blkno += SCOUTFS_BLOCKS_PER_PAGE; nr_blocks -= SCOUTFS_BLOCKS_PER_PAGE; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b2ce8ce9..5b2e930d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -148,27 +148,6 @@ TRACE_EVENT(scoutfs_bio_init_comp, TP_printk("initing comp %p", __entry->comp) ); -TRACE_EVENT(scoutfs_bio_submit_added, - TP_PROTO(struct super_block *sb, void *page, void *bio), - - TP_ARGS(sb, page, bio), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(void *, page) - __field(void *, bio) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->page = page; - __entry->bio = bio; - ), - - TP_printk(FSID_FMT" added page %p to bio %p", __entry->fsid, - __entry->page, __entry->bio) -); - DECLARE_EVENT_CLASS(scoutfs_bio_class, TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), From 2136a973ed0a441f02a0ab96803078377664e3ee Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Mar 2018 15:49:46 -0800 Subject: [PATCH 562/920] scoutfs: copy names in rename trace event The rename trace event was recording and later dereferencing pointers to dentry names that could be long gone by the time the output is generated. We need to copy the name strings into the trace buffers. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5b2e930d..58293a52 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1865,31 +1865,25 @@ TRACE_EVENT(scoutfs_rename, TP_STRUCT__entry( __field(__u64, fsid) __field(__u64, old_dir_ino) - __field(char *, old_name) - __field(unsigned int, old_name_len) + __string(old_name, old_dentry->d_name.name) __field(__u64, new_dir_ino) - __field(char *, new_name) - __field(unsigned int, new_name_len) + __string(new_name, new_dentry->d_name.name) __field(__u64, new_inode_ino) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->old_dir_ino = scoutfs_ino(old_dir); - __entry->old_name = (char *)old_dentry->d_name.name; - __entry->old_name_len = old_dentry->d_name.len; + __assign_str(old_name, old_dentry->d_name.name) __entry->new_dir_ino = scoutfs_ino(new_dir); - __entry->new_name = (char *)new_dentry->d_name.name; - __entry->new_name_len = new_dentry->d_name.len; + __assign_str(new_name, new_dentry->d_name.name) __entry->new_inode_ino = new_dentry->d_inode ? scoutfs_ino(new_dentry->d_inode) : 0; ), - TP_printk("fsid "FSID_FMT" old_dir_ino %llu old_name %.*s (len %u) new_dir_ino %llu new_name %.*s (len %u) new_inode_ino %llu", - __entry->fsid, __entry->old_dir_ino, __entry->old_name_len, - __entry->old_name, __entry->old_name_len, - __entry->new_dir_ino, __entry->new_name_len, - __entry->new_name, __entry->new_name_len, + TP_printk("fsid "FSID_FMT" old_dir_ino %llu old_name %s new_dir_ino %llu new_name %s new_inode_ino %llu", + __entry->fsid, __entry->old_dir_ino, __get_str(old_name), + __entry->new_dir_ino, __get_str(new_name), __entry->new_inode_ino) ); From 951b6d8dcd9e9f63b017bc8ee7f4acc984e51aaf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Mar 2018 10:03:47 -0800 Subject: [PATCH 563/920] scoutfs: add d_revalidate trace Add a trace event to get some visibility into dentry revalidation. Signed-off-by: Zach Brown --- kmod/src/dir.c | 9 ++++++--- kmod/src/scoutfs_trace.h | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 7cbc9bb6..e79d3245 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -269,9 +269,10 @@ static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) { struct super_block *sb = dentry->d_sb; struct dentry_info *di = dentry->d_fsdata; + struct dentry *parent = dget_parent(dentry); struct scoutfs_lock *lock = NULL; struct scoutfs_dirent dent; - struct dentry *parent = NULL; + bool is_covered = false; struct inode *dir; u64 dentry_ino; int ret; @@ -300,13 +301,13 @@ static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) goto out; } - if (scoutfs_lock_is_covered(sb, &di->lock_cov)) { + is_covered = scoutfs_lock_is_covered(sb, &di->lock_cov); + if (is_covered) { scoutfs_inc_counter(sb, dentry_revalidate_locked); ret = 1; goto out; } - parent = dget_parent(dentry); if (!parent || !parent->d_inode) { scoutfs_inc_counter(sb, dentry_revalidate_orphan); ret = 0; @@ -338,6 +339,8 @@ static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) } out: + trace_scoutfs_d_revalidate(sb, dentry, flags, parent, is_covered, ret); + dput(parent); scoutfs_unlock(sb, lock, DLM_LOCK_PR); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 58293a52..7a07b7cb 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,7 @@ #include "count.h" #include "bio.h" #include "export.h" +#include "dir.h" struct lock_info; @@ -1887,6 +1888,45 @@ TRACE_EVENT(scoutfs_rename, __entry->new_inode_ino) ); +TRACE_EVENT(scoutfs_d_revalidate, + TP_PROTO(struct super_block *sb, + struct dentry *dentry, int flags, struct dentry *parent, + bool is_covered, int ret), + + TP_ARGS(sb, dentry, flags, parent, is_covered, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __string(name, dentry->d_name.name) + __field(__u64, ino) + __field(__u64, parent_ino) + __field(int, flags) + __field(int, is_root) + __field(int, is_covered) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __assign_str(name, dentry->d_name.name) + __entry->ino = dentry->d_inode ? + scoutfs_ino(dentry->d_inode) : 0; + __entry->parent_ino = parent->d_inode ? + scoutfs_ino(parent->d_inode) : 0; + __entry->flags = flags; + __entry->is_root = IS_ROOT(dentry); + __entry->is_covered = is_covered; + __entry->ret = ret; + ), + + TP_printk("fsid "FSID_FMT" name %s ino %llu parent_ino %llu flags 0x%x s_root %u is_covered %u ret %d", + __entry->fsid, __get_str(name), __entry->ino, + __entry->parent_ino, __entry->flags, + __entry->is_root, + __entry->is_covered, + __entry->ret) +); + DECLARE_EVENT_CLASS(scoutfs_super_lifecycle_class, TP_PROTO(struct super_block *sb), TP_ARGS(sb), From 2aa613dae537edd9b4c00014e23e14712e8f0474 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Mar 2018 15:19:07 -0800 Subject: [PATCH 564/920] scoutfs: add scoutfs_item_range_cached() Add a quick helper for querying if a given range of keys is covered by the item cache. Signed-off-by: Zach Brown --- kmod/src/item.c | 60 +++++++++++++++++++++++++++++++++++++++++-------- kmod/src/item.h | 3 +++ 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 41140ee2..a5714e2d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1683,6 +1683,19 @@ static struct cached_item *next_dirty(struct cached_item *item) return NULL; } +static bool dirty_item_within(struct rb_root *root, + struct scoutfs_key_buf *from, + struct scoutfs_key_buf *end) +{ + struct cached_item *item; + + item = next_item(root, from); + if (item && !(item->dirty & ITEM_DIRTY)) + item = next_dirty(item); + + return item && scoutfs_key_compare(item->key, end) <= 0; +} + bool scoutfs_item_has_dirty(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1697,6 +1710,41 @@ bool scoutfs_item_has_dirty(struct super_block *sb) return has; } +/* + * Return true if the item cache covers the given range. If dirty is + * provided then we only return true if there are dirty items in the + * range. + * + * If the start of the query range doesn't overlap a cached range then + * we see if the next cached range starts before the end of the query range. + */ +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, bool dirty) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_range *next; + struct cached_range *rng; + unsigned long flags; + bool cached = false; + + spin_lock_irqsave(&cac->lock, flags); + + if (dirty) { + if (dirty_item_within(&cac->items, start, end)) + cached = true; + } else { + rng = walk_ranges(&cac->ranges, start, NULL, &next); + if (rng || (next && scoutfs_key_compare(next->start, end) <= 0)) + cached = true; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + return cached; +} + /* * Returns true if adding more items with the given count, keys, and values * still fits in a single item along with the current dirty items. @@ -1774,7 +1822,6 @@ int scoutfs_item_writeback(struct super_block *sb, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct cached_item *item; unsigned long flags; bool sync = false; int count = 0; @@ -1784,14 +1831,9 @@ int scoutfs_item_writeback(struct super_block *sb, spin_lock_irqsave(&cac->lock, flags); - if (cac->nr_dirty_items) { - item = next_item(&cac->items, start); - if (item && !(item->dirty & ITEM_DIRTY)) - item = next_dirty(item); - if (item && scoutfs_key_compare(item->key, end) <= 0) { - sync = true; - count = cac->nr_dirty_items; - } + if (cac->nr_dirty_items && dirty_item_within(&cac->items, start, end)) { + sync = true; + count = cac->nr_dirty_items; } spin_unlock_irqrestore(&cac->lock, flags); diff --git a/kmod/src/item.h b/kmod/src/item.h index 453ec5ed..7265c7d4 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -59,6 +59,9 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, bool dirty); bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, u32 key_bytes, u32 val_bytes); int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); From 9ad0f81084cca4be9b7de3f2c8f0c6ecb1d51f6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 11:30:45 -0800 Subject: [PATCH 565/920] scoutfs: add some lock/item consistency checks Add some tests to the locking paths to see if we violate item caching rules. As we finish locking calls we make sure that the item cache is consistent with the lock mode. And we make sure that we don't free locks before they've been unlocked and had a chance to check the item cache. Signed-off-by: Zach Brown --- kmod/src/lock.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e3178bc8..d5eeced1 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -178,6 +178,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) trace_scoutfs_lock_free(sb, lock); scoutfs_inc_counter(sb, lock_free); + BUG_ON(!linfo->shutdown && lock->granted_mode != DLM_LOCK_IV); BUG_ON(delayed_work_pending(&lock->grace_work)); if (lock->debug_locks_id) @@ -542,6 +543,8 @@ static void scoutfs_lock_ast(void *arg) struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); int status = lock->lksb.sb_status; + bool cached; + bool dirty; scoutfs_inc_counter(sb, lock_ast); @@ -571,8 +574,41 @@ static void scoutfs_lock_ast(void *arg) lock->work_mode = DLM_LOCK_IV; trace_scoutfs_lock_ast(sb, lock); - lock_process(linfo, lock); + /* + * Catch lock modes with cached items that violate the item + * cache consistency rules. + * + * We can never have dirty items if we're calling the dlm and + * changing lock modes. We can't have cached items if we're not + * in the two modes that allow caching. + */ + cached = lock->start && scoutfs_item_range_cached(sb, lock->start, + lock->end, false); + dirty = lock->start && scoutfs_item_range_cached(sb, lock->start, + lock->end, true); + if (WARN_ON_ONCE(dirty || + (cached && lock->granted_mode != DLM_LOCK_PR && + lock->granted_mode != DLM_LOCK_EX))) { + scoutfs_err_sk(sb, "lock item cache consistency violation, cached %u dirty %u: name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", + cached, dirty, + LN_ARG(&lock->name), SK_ARG(lock->start), + SK_ARG(lock->end), lock->refresh_gen, lock->error, + lock->granted_mode, lock->bast_mode, + lock->work_prev_mode, lock->work_mode, + lock->waiters[DLM_LOCK_PR], + lock->waiters[DLM_LOCK_EX], + lock->waiters[DLM_LOCK_CW], + lock->users[DLM_LOCK_PR], + lock->users[DLM_LOCK_EX], + lock->users[DLM_LOCK_CW], + lock->lksb.sb_status, + lock->lksb.sb_lkid, + lock->lksb.sb_flags); + BUG(); + } + + lock_process(linfo, lock); spin_unlock(&linfo->lock); } From d58c8d5993ea8c0468ac79461af0e68ea5da7e41 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 13:27:54 -0800 Subject: [PATCH 566/920] scoutfs: move lock work after dependencies Some of the lock processing path was happening too early. Both maintainance of the locks on the LRU and waking waiters depends on whether there is work pending and on the the granted mode. Those are changed in the middle by processing so we need to move these two bits of work down so that they can consume the updated state. Signed-off-by: Zach Brown --- kmod/src/lock.c | 56 ++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index d5eeced1..72eeb88f 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -343,39 +343,12 @@ static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) if (linfo->shutdown) return; - /* only idle locks are on the lru */ - idle = lock_idle(lock); - if (list_empty(&lock->lru_head) && idle) { - list_add_tail(&lock->lru_head, &linfo->lru_list); - linfo->lru_nr++; - - } else if (!list_empty(&lock->lru_head) && !idle) { - list_del_init(&lock->lru_head); - linfo->lru_nr--; - } - /* errored locks are torn down */ if (lock->error) { wake_up(&lock->waitq); goto out; } - /* - * Wake any waiters who might be able to use the lock now. - * Notice that this ignores the presence of basts! This lets us - * recursively acquire locks in one task without having to track - * per-task lock references. It comes at the cost of fairness. - * Spinning overlapping users can delay a bast down conversion - * indefinitely. - */ - for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { - if (lock->waiters[mode] && - lock_modes_match(lock->granted_mode, mode)) { - wake_up(&lock->waitq); - break; - } - } - /* * Try to down convert a lock in response to a bast once users * are done with it. We may have to wait for a grace period @@ -431,13 +404,40 @@ static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) } } + /* + * Wake any waiters who might be able to use the lock now. + * Notice that this ignores the presence of basts! This lets us + * recursively acquire locks in one task without having to track + * per-task lock references. It comes at the cost of fairness. + * Spinning overlapping users can delay a bast down conversion + * indefinitely. + */ + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (lock->waiters[mode] && + lock_modes_match(lock->granted_mode, mode)) { + wake_up(&lock->waitq); + break; + } + } + out: + /* only idle locks are on the lru */ + idle = lock_idle(lock); + if (list_empty(&lock->lru_head) && idle) { + list_add_tail(&lock->lru_head, &linfo->lru_list); + linfo->lru_nr++; + + } else if (!list_empty(&lock->lru_head) && !idle) { + list_del_init(&lock->lru_head); + linfo->lru_nr--; + } + /* * We can free the lock once it's idle and it's either never * been initially locked or has been unlocked, both of which we * indicate with IV. */ - if (lock_idle(lock) && lock->granted_mode == DLM_LOCK_IV) + if (idle && lock->granted_mode == DLM_LOCK_IV) lock_free(linfo, lock); } From 0b54d71b986c6381350bcb89fdd2a6bbfacbf0ee Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 13 Mar 2018 12:31:40 -0700 Subject: [PATCH 567/920] scoutfs: avoid double unlock We weren't sufficiently careful in reacting to basts. If a bast arrived whlie an unlock is in flight we'd turn around and try to unlock again, returning an error, and exploding. More carefully only act on basts if we have an active mode that needs to be unlocked. Now if the racey bast arrives we'll ignore it and end up freeing the lock in processing after the unlock succeeds. Signed-off-by: Zach Brown --- kmod/src/lock.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 72eeb88f..0f0c4f13 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -355,6 +355,7 @@ static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) * to expire after an unlock. */ if (lock->work_mode < 0 && + lock->granted_mode >= 0 && lock->bast_mode >= 0 && lock_counts_match(lock->bast_mode, lock->users) && !lock->grace_pending) { @@ -613,11 +614,14 @@ static void scoutfs_lock_ast(void *arg) } /* - * A lock on this node has blocked a lock request on another node. + * A lock on this node has blocked a lock request on another node. We + * translate the dlm's communication of the blocking mode to the mode + * that we should convert our lock to. We can only either downconvert + * to a matching PR or unlock. * - * We can down convert to a PR if we had an EX and they're trying to get - * a PR but all other conflicts cause us to drop our lock and invalidate - * our cache. + * These are truly asynchronous and can arrive multiple times, at any time. + * We're careful to only set the bast mode here and let lock processing + * sort out the state machine. */ static void scoutfs_lock_bast(void *arg, int blocked_mode) { From 9f51b63f8d3f1d40e30ae329a7cb9c41de5528bb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Mar 2018 15:31:39 -0700 Subject: [PATCH 568/920] scoutfs: check snprintf_key() format args Add the function attribute to snprintf_key() to have the compiler verify its print format and args. I noticed some buggy changes that didn't throw errors. Happily none of the existing calls had problems. Signed-off-by: Zach Brown --- kmod/src/key.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kmod/src/key.c b/kmod/src/key.c index 3683bba9..7406a5fe 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -128,9 +128,10 @@ void scoutfs_key_dec(struct scoutfs_key_buf *key) * formatting to represent the trailing bytes: runs of zeros compresesd * to _ and then hex output of non-zero bytes. */ -static int snprintf_key(char *buf, size_t size, struct scoutfs_key_buf *key, - unsigned min_len, unsigned fmt_len, - const char *fmt, ...) +static int __printf(6, 7) snprintf_key(char *buf, size_t size, + struct scoutfs_key_buf *key, + unsigned min_len, unsigned fmt_len, + const char *fmt, ...) { va_list args; From 77f29fa0214db3d9e4f2c8311a2ccd419a2725d4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:18:40 -0800 Subject: [PATCH 569/920] scoutfs: allow null val in scoutfs_item_lookup Some callers may want to just test if an item is present and not necessarily want to setup storage for copying the value in. Signed-off-by: Zach Brown --- kmod/src/item.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index a5714e2d..9ea9727d 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -764,7 +764,7 @@ static bool lock_coverage(struct scoutfs_lock *lock, /* * Find an item with the given key and copy its value into the caller's * value vector. The amount of bytes copied is returned which can be 0 - * or truncated if the caller's buffer isn't big enough. + * or truncated if the caller's buffer isn't big enough or if val is null. * * The end key limits how many keys after the search key can be read * and inserted into the cache. @@ -789,7 +789,10 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, item = find_item(sb, &cac->items, key); if (item) { item_referenced(cac, item); - ret = scoutfs_kvec_memcpy(val, item->val); + if (val) + ret = scoutfs_kvec_memcpy(val, item->val); + else + ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; } else { From 7fb6841b1eb18f6c809d3016c3e8139d414ce243 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:23:35 -0800 Subject: [PATCH 570/920] scoutfs: free val while deleting items It was silly to hand off deleted values to callers to free. We can just free as we delete and save a bunch of caller value manipulation. Signed-off-by: Zach Brown --- kmod/src/item.c | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 9ea9727d..ad223020 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -402,16 +402,14 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, /* * Turn an item that the caller has found while holding the lock into a - * deletion item. The caller will free whatever we put in the deletion - * value after releasing the lock. + * deletion item. */ static void become_deletion_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *item, - struct kvec *del_val) + struct cached_item *item) { clear_item_dirty(sb, cac, item); - scoutfs_kvec_clone(del_val, item->val); + scoutfs_kvec_kfree(item->val); scoutfs_kvec_init_null(item->val); item->deletion = 1; mark_item_dirty(sb, cac, item); @@ -1262,7 +1260,6 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct scoutfs_key_buf *range_end; - SCOUTFS_DECLARE_KVEC(del_val); struct cached_item *exist; struct cached_item *item; struct cached_item *tmp; @@ -1347,16 +1344,13 @@ int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, } } } - } /* delete everything in the range */ for (exist = item_for_next(&cac->items, first, NULL, last); exist; exist = next_item_node(&cac->items, exist, last)) { - scoutfs_kvec_init_null(del_val); - become_deletion_item(sb, cac, exist, del_val); - scoutfs_kvec_kfree(del_val); + become_deletion_item(sb, cac, exist); } /* insert the caller's items, overwriting any existing */ @@ -1498,21 +1492,18 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct cached_item *item; - SCOUTFS_DECLARE_KVEC(del_val); unsigned long flags; int ret; if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) return -EINVAL; - scoutfs_kvec_init_null(del_val); - do { spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); if (item) { - become_deletion_item(sb, cac, item, del_val); + become_deletion_item(sb, cac, item); ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; @@ -1525,8 +1516,6 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, } while (ret == -ENODATA && (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); - scoutfs_kvec_kfree(del_val); - trace_scoutfs_item_delete_ret(sb, ret); return ret; } @@ -1538,14 +1527,12 @@ int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct cached_item *item; - SCOUTFS_DECLARE_KVEC(del_val); unsigned long flags; int ret; if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) return -EINVAL; - scoutfs_kvec_init_null(del_val); item = alloc_item(sb, key, NULL); if (!item) @@ -1563,10 +1550,9 @@ int scoutfs_item_delete_force(struct super_block *sb, scoutfs_inc_counter(sb, item_create); mark_item_dirty(sb, cac, item); - become_deletion_item(sb, cac, item, del_val); + become_deletion_item(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); - scoutfs_kvec_kfree(del_val); return ret; } @@ -1581,21 +1567,17 @@ void scoutfs_item_delete_dirty(struct super_block *sb, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - SCOUTFS_DECLARE_KVEC(del_val); struct cached_item *item; unsigned long flags; - scoutfs_kvec_init_null(del_val); spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); if (item) - become_deletion_item(sb, cac, item, del_val); + become_deletion_item(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); - - scoutfs_kvec_kfree(del_val); } /* From 4dad03a3dd07d02f14b6138cd8b51376021d8cd1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:29:44 -0800 Subject: [PATCH 571/920] scoutfs: add item_is_dirty() helper We had an absolute ton of open coding of testing an item's dirty flag. Let's hide it off in a helper so we're less likely to mess it up. Signed-off-by: Zach Brown --- kmod/src/item.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index ad223020..25608651 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -224,6 +224,11 @@ static struct cached_item *next_item(struct rb_root *root, #define LEFT_DIRTY 0x2 #define RIGHT_DIRTY 0x4 +static bool item_is_dirty(struct cached_item *item) +{ + return (item->dirty & ITEM_DIRTY) != 0; +} + /* * Return the given dirty bit if the item with the given node is dirty * or has dirty children. @@ -333,7 +338,7 @@ static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) return; - if (item->dirty & ITEM_DIRTY) + if (item_is_dirty(item)) return; item->dirty |= ITEM_DIRTY; @@ -352,7 +357,7 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) return; - if (!(item->dirty & ITEM_DIRTY)) + if (!item_is_dirty(item)) return; item->dirty &= ~ITEM_DIRTY; @@ -474,7 +479,7 @@ restart: rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); - BUG_ON(ins->dirty & ITEM_DIRTY); + BUG_ON(item_is_dirty(ins)); list_add_tail(&ins->entry, &cac->lru_list); cac->lru_nr++; @@ -1598,7 +1603,7 @@ void scoutfs_item_update_dirty(struct super_block *sb, item = find_item(sb, &cac->items, key); - BUG_ON(!item || !(item->dirty & ITEM_DIRTY) || + BUG_ON(!item || !item_is_dirty(item) || scoutfs_kvec_length(val) > scoutfs_kvec_length(item->val)); delta = scoutfs_kvec_length(val) - scoutfs_kvec_length(item->val); @@ -1621,7 +1626,7 @@ static struct cached_item *first_dirty(struct rb_node *node) if (item->dirty & LEFT_DIRTY) { node = item->node.rb_left; - } else if (item->dirty & ITEM_DIRTY) { + } else if (item_is_dirty(item)) { ret = item; break; } else if (item->dirty & RIGHT_DIRTY) { @@ -1659,7 +1664,7 @@ static struct cached_item *next_dirty(struct cached_item *item) /* done if our next greatest parent itself is dirty */ item = container_of(parent, struct cached_item, node); - if (item->dirty & ITEM_DIRTY) + if (item_is_dirty(item)) return item; /* continue to check right subtree */ @@ -1675,7 +1680,7 @@ static bool dirty_item_within(struct rb_root *root, struct cached_item *item; item = next_item(root, from); - if (item && !(item->dirty & ITEM_DIRTY)) + if (item && !item_is_dirty(item)) item = next_dirty(item); return item && scoutfs_key_compare(item->key, end) <= 0; @@ -1883,7 +1888,7 @@ int scoutfs_item_invalidate(struct super_block *sb, else next = NULL; - WARN_ON_ONCE(item->dirty & ITEM_DIRTY); + WARN_ON_ONCE(item_is_dirty(item)); erase_item(sb, cac, item); count++; } @@ -1984,7 +1989,7 @@ static struct cached_item *shrink_boundary(struct super_block *sb, break; } - if (next->dirty & ITEM_DIRTY) { + if (item_is_dirty(next)) { scoutfs_inc_counter(sb, item_shrink_next_dirty); break; } @@ -2140,7 +2145,7 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) struct cached_item, entry))) { /* can't have dirty items on the lru */ - BUG_ON(item->dirty & ITEM_DIRTY); + BUG_ON(item_is_dirty(item)); /* if we're not in a range just shrink the item */ rng = walk_ranges(&cac->ranges, item->key, NULL, NULL); From acfc4b357b02b9755b1b98ce146c5143a79a6fd8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:35:56 -0800 Subject: [PATCH 572/920] scoutfs: add item saving and restoring Add item cache functions for saving and restoring items. This lets callers more easily undo changes while they have transactions pinned. Signed-off-by: Zach Brown --- kmod/src/item.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/item.h | 6 +++ 2 files changed, 124 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 25608651..09474959 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1558,6 +1558,124 @@ int scoutfs_item_delete_force(struct super_block *sb, become_deletion_item(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); + return ret; +} + +/* + * Delete an item and give it to the caller so that they can restore it + * later. + * + * The deleted items can be dirty or not.. we maintain an accurate dirty + * count as we remove the deleted items and leave their dirty flag set + * so that restore can mark them dirty again. + * + * Returns -ENOENT if the item didn't exist and couldn't be deleted. + */ +int scoutfs_item_delete_save(struct super_block *sb, + struct scoutfs_key_buf *key, + struct list_head *list, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *item; + struct cached_item *del; + unsigned long flags; + bool was_dirty; + int ret; + + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) + return -EINVAL; + + del = alloc_item(sb, key, NULL); + if (!del) + return -ENOMEM; + + do { + spin_lock_irqsave(&cac->lock, flags); + + item = find_item(sb, &cac->items, key); + if (item) { + was_dirty = item_is_dirty(item); + unlink_item(sb, cac, item); + list_add_tail(&item->entry, list); + if (was_dirty) + item->dirty |= ITEM_DIRTY; + + ret = insert_item(sb, cac, del, false, false); + BUG_ON(ret); + become_deletion_item(sb, cac, del); + del = NULL; + ret = 0; + } else if (check_range(sb, &cac->ranges, key, NULL)) { + ret = -ENOENT; + } else { + ret = -ENODATA; + } + + spin_unlock_irqrestore(&cac->lock, flags); + + } while (ret == -ENODATA && + (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + + free_item(sb, del); + + return ret; +} + +/* + * Restore a set of previousl saved items. They're returned to the + * cached and marked dirty if they were dirty when they were saved. + * Restored items completely overwrite any existing cached items. + * + * The caller must have held locks covering the save and restore so that + * the cached ranges still exist. + */ +int scoutfs_item_restore(struct super_block *sb, struct list_head *list, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct cached_item *existing; + struct cached_item *item; + struct cached_item *tmp; + unsigned long flags; + bool was_dirty; + int mode; + int ret; + + if (list_empty(list)) + return 0; + + spin_lock_irqsave(&cac->lock, flags); + + /* make sure all the items are locked and cached */ + list_for_each_entry(item, list, entry) { + mode = item_is_dirty(item) ? DLM_LOCK_EX : DLM_LOCK_PR; + if (WARN_ON_ONCE(!lock_coverage(lock, item->key, mode)) || + WARN_ON_ONCE(!check_range(sb, &cac->ranges, item->key, + NULL))) { + ret = -EINVAL; + goto out; + } + } + + list_for_each_entry_safe(item, tmp, list, entry) { + was_dirty = item_is_dirty(item); + item->dirty &= ~ITEM_DIRTY; + list_del_init(&item->entry); + + existing = find_item(sb, &cac->items, item->key); + if (existing) + erase_item(sb, cac, existing); + insert_item(sb, cac, item, false, false); + if (was_dirty) + mark_item_dirty(sb, cac, item); + } + + ret = 0; +out: + spin_unlock_irqrestore(&cac->lock, flags); return ret; } diff --git a/kmod/src/item.h b/kmod/src/item.h index 7265c7d4..521483a9 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -46,6 +46,12 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_lock *lock); +int scoutfs_item_delete_save(struct super_block *sb, + struct scoutfs_key_buf *key, + struct list_head *list, + struct scoutfs_lock *lock); +int scoutfs_item_restore(struct super_block *sb, struct list_head *list, + struct scoutfs_lock *lock); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *key, struct kvec *val); From 4101c655a548e78616e4bc61ef1dc64fb01b7a05 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:43:29 -0800 Subject: [PATCH 573/920] scoutfs: rework set_xattr to honor XATTR_ flags We weren't properly honoring the XATTR_{CREATE,REPLACE} flags. For a start we weren't even passing them in to our _xattr_set() from _setxattr(). So that's something. We left it to scoutfs_item_set_batch() to return errors if we were. This is wrong because the xattr flags are xattr granular, not item granular. We don't want _REPLACE to fail when replacing a larger xattr value because later items in the xattr don't have matching existing items. (And it had some bugs where it could livelock if you set flags and items already existed. :high_fives:). Now that we have the _save and _restore calls we can avoid _set_batch's bad semantics and bugs entirely. It's easy for us to compare the flags to item lookups, delete the old, create the new, and restore the old on errors. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 119 ++++++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index a21fba4a..9661102a 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -242,14 +242,12 @@ out: * The confusing swiss army knife of creating, modifying, and deleting * xattrs. * - * This always removes the old existing xattr. If value is set then - * we're replacing it with a new xattr. The flags cause creation to - * fail if the xattr already exists (_CREATE) or doesn't already exist - * (_REPLACE). xattrs can have a zero length value. + * This always removes the old existing xattr items. * - * To modify xattrs built of individual items we use the batch - * interface. It provides atomic transitions from one group of items to - * another. + * If the value pointer is set then we're replacing it with a new xattr. + * The flags cause creation to fail if the xattr already exists + * (_CREATE) or doesn't already exist (_REPLACE). xattrs can have a + * zero length value. */ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) @@ -258,8 +256,8 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_key_buf *last; - struct scoutfs_key_buf *key; + struct scoutfs_key_buf *last = NULL; + struct scoutfs_key_buf *key = NULL; struct scoutfs_xattr_val_header vh; size_t name_len = strlen(name); SCOUTFS_DECLARE_KVEC(val); @@ -267,16 +265,17 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, unsigned int bytes; unsigned int off; LIST_HEAD(ind_locks); - LIST_HEAD(list); + LIST_HEAD(saved); u64 ind_seq; - u8 part; - int sif; + int part; int ret; trace_scoutfs_xattr_set(sb, name_len, value, size, flags); if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN || - (value && size > SCOUTFS_XATTR_MAX_SIZE)) + (value && size > SCOUTFS_XATTR_MAX_SIZE) || + ((flags & XATTR_CREATE) && (flags & XATTR_REPLACE)) || + (flags & ~(XATTR_CREATE | XATTR_REPLACE))) return -EINVAL; if (unknown_prefix(name)) @@ -294,31 +293,20 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (ret) goto out; - /* build up batch of new items for the new xattr */ - if (value) { - for_each_xattr_item(key, val, &vh, (void *)value, size, - part, off, bytes) { - - ret = scoutfs_item_add_batch(sb, &list, key, val); - if (ret) - goto out; - } + /* see if we violate the flag constraints */ + if ((flags & (XATTR_CREATE | XATTR_REPLACE))) { + set_xattr_key_part(key, 0); + ret = scoutfs_item_lookup(sb, key, NULL, lck); + if (ret == -ENOENT && (flags & XATTR_REPLACE)) + ret = -ENODATA; + else if (ret == 0 && (flags & XATTR_CREATE)) + ret = -EEXIST; + else if (ret == -ENOENT && (flags & XATTR_CREATE)) + ret = 0; + if (ret < 0) + goto out; } - /* XXX could add range deletion items around xattr items here */ - - /* reset key to first */ - set_xattr_key_part(key, 0); - - if (flags & XATTR_CREATE) - sif = SIF_EXCLUSIVE; - else if (flags & XATTR_REPLACE) - sif = SIF_REPLACE; - else - sif = 0; - - down_write(&si->xattr_rwsem); - retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: @@ -327,26 +315,61 @@ retry: if (ret > 0) goto retry; if (ret) - goto unlock; + goto out; - ret = scoutfs_dirty_inode_item(inode, lck) ?: - scoutfs_item_set_batch(sb, &list, key, last, sif, lck); - if (ret == 0) { - /* XXX do these want i_mutex or anything? */ - inode_inc_iversion(inode); - inode->i_ctime = CURRENT_TIME; - scoutfs_update_inode_item(inode, lck, &ind_locks); + down_write(&si->xattr_rwsem); + + ret = scoutfs_dirty_inode_item(inode, lck); + if (ret < 0) + goto release; + + /* delete and save any existing xattr items */ + for (part = 0; part < SCOUTFS_XATTR_MAX_PARTS; part++) { + set_xattr_key_part(key, part); + ret = scoutfs_item_delete_save(sb, key, &saved, lck); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret < 0) + goto release; } - scoutfs_release_trans(sb); + /* create items for the new xattr */ + if (value) { + for_each_xattr_item(key, val, &vh, (void *)value, size, + part, off, bytes) { + + ret = scoutfs_item_create(sb, key, val, lck); + if (ret < 0) { + /* remove any previously created items */ + while (--part >= 0) { + set_xattr_key_part(key, part); + scoutfs_item_delete_dirty(sb, key); + } + goto release; + } + } + } + + /* XXX do these want i_mutex or anything? */ + inode_inc_iversion(inode); + inode->i_ctime = CURRENT_TIME; + scoutfs_update_inode_item(inode, lck, &ind_locks); + ret = 0; + +release: + /* restore the old xattr if we modified it then errored */ + if (ret < 0) + scoutfs_item_restore(sb, &saved, lck); -unlock: up_write(&si->xattr_rwsem); + scoutfs_release_trans(sb); out: scoutfs_inode_index_unlock(sb, &ind_locks); scoutfs_unlock(sb, lck, DLM_LOCK_EX); - scoutfs_item_free_batch(sb, &list); + scoutfs_item_free_batch(sb, &saved); scoutfs_key_free(sb, key); scoutfs_key_free(sb, last); @@ -359,7 +382,7 @@ int scoutfs_setxattr(struct dentry *dentry, const char *name, if (size == 0) value = ""; /* set empty value */ - return scoutfs_xattr_set(dentry, name, value, size, 0); + return scoutfs_xattr_set(dentry, name, value, size, flags); } int scoutfs_removexattr(struct dentry *dentry, const char *name) From c438f5d887418ec51327fa86f7782baf18b078bf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 9 Mar 2018 16:49:32 -0800 Subject: [PATCH 574/920] scoutfs: remove scoutfs_item_set_batch() scoutfs_item_set_batch() has a rocky history of being a giant pain in the butt. It's been a lot simpler to have callers use individual item ops instead of trying to describe a compound item operation to sometihng like _set_batch(). Its last user has gone away so we can remove it and never speak of it again. And there was much rejoycing. Signed-off-by: Zach Brown --- kmod/src/item.c | 137 --------------------------------------- kmod/src/item.h | 10 --- kmod/src/scoutfs_trace.h | 6 -- 3 files changed, 153 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 09474959..2f165f3b 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -47,11 +47,6 @@ static bool invalid_key_val(struct scoutfs_key_buf *key, struct kvec *val) (val && (scoutfs_kvec_length(val) > SCOUTFS_MAX_VAL_SIZE))); } -static bool invalid_flags(int sif) -{ - return (sif & SIF_EXCLUSIVE) && (sif & SIF_REPLACE); -} - struct item_cache { struct super_block *sb; @@ -1241,138 +1236,6 @@ out: return ret; } -/* - * Atomically set the caller's items to be the only cached items in the - * caller's range. Any existing items that overlap with the caller's - * items are replaced. Any existing items in the range that aren't in - * the caller's list will be replaced with deletion items. The deletion - * items and the caller's inserted items will all be marked dirty. - * - * In practice this is used for relatively few items at a time, at most - * on the order of 16. So we're not too worried with it walking a small - * number of items a few times when the caller provides flags that have - * to check for existing items. - * - * Returns -ENODATA if SIF_REPLACE is set and a batch item doesn't have - * a matching existing item or -EEXIST if SIF_EXCLUSIVE is set and a - * batch item does have an existing item. - */ -int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, int sif, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *range_end; - struct cached_item *exist; - struct cached_item *item; - struct cached_item *tmp; - unsigned long flags; - int cmp; - int ret; - - if (WARN_ON_ONCE(invalid_flags(sif))) - return -EINVAL; - - list_for_each_entry(item, list, entry) { - if (invalid_key_val(item->key, item->val)) - return -EINVAL; - } - - trace_scoutfs_item_set_batch(sb, first, last); - - if (WARN_ON_ONCE(scoutfs_key_compare(first, last) > 0) || - WARN_ON_ONCE(!lock_coverage(lock, first, DLM_LOCK_EX)) || - WARN_ON_ONCE(!lock_coverage(lock, last, DLM_LOCK_EX))) - return -EINVAL; - - range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!range_end) - return -ENOMEM; - - spin_lock_irqsave(&cac->lock, flags); - - /* make sure all of first through last are cached */ - scoutfs_key_copy(range_end, first); - for (;;) { - if (check_range(sb, &cac->ranges, range_end, range_end)) { - if (scoutfs_key_compare(range_end, last) >= 0) - break; - /* start reading after the last key we have cached */ - scoutfs_key_inc(range_end); - } else { - /* start reading from the missing first */ - scoutfs_key_copy(range_end, first); - } - - spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, range_end, lock->end); - spin_lock_irqsave(&cac->lock, flags); - - if (ret) - goto out; - } - - /* check for _EXCLUSIVE or _REPLACE errors before destroying items */ - if (!list_empty(list) && (sif & (SIF_EXCLUSIVE | SIF_REPLACE))) { - - item = list_first_entry(list, struct cached_item, entry); - exist = item_for_next(&cac->items, first, NULL, last); - - while (item) { - /* compare keys, with bias to finding _REPLACE err */ - if (exist) - cmp = scoutfs_key_compare(item->key, - exist->key); - else - cmp = -1; - - if (cmp < 0) { - if (sif & SIF_REPLACE) { - ret = -ENODATA; - goto out; - } - if (item->entry.next != list) - item = list_next_entry(item, entry); - else - item = NULL; - - } else if (cmp > 0) { - exist = next_item_node(&cac->items, exist, last); - - } else { - /* cmp == 0 */ - if (sif & SIF_EXCLUSIVE) { - ret = -EEXIST; - goto out; - } - } - } - } - - /* delete everything in the range */ - for (exist = item_for_next(&cac->items, first, NULL, last); - exist; exist = next_item_node(&cac->items, exist, last)) { - - become_deletion_item(sb, cac, exist); - } - - /* insert the caller's items, overwriting any existing */ - list_for_each_entry_safe(item, tmp, list, entry) { - list_del_init(&item->entry); - insert_item(sb, cac, item, true, false); - mark_item_dirty(sb, cac, item); - } - - ret = 0; -out: - spin_unlock_irqrestore(&cac->lock, flags); - scoutfs_key_free(sb, range_end); - - return ret; -} - void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) { struct cached_item *item; diff --git a/kmod/src/item.h b/kmod/src/item.h index 521483a9..da487f85 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -3,12 +3,6 @@ #include -/* behavioural flags for the item functions */ -enum { - SIF_EXCLUSIVE = (1 << 1), - SIF_REPLACE = (1 << 2), -}; - struct scoutfs_segment; struct scoutfs_key_buf; @@ -58,10 +52,6 @@ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); -int scoutfs_item_set_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, int sif, - struct scoutfs_lock *lock); void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7a07b7cb..343f6f1a 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1453,12 +1453,6 @@ DECLARE_EVENT_CLASS(scoutfs_range_class, __entry->fsid, __get_str(start), __get_str(end)) ); -DEFINE_EVENT(scoutfs_range_class, scoutfs_item_set_batch, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), - TP_ARGS(sb, start, end) -); - DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), From c4de85fd825e3ab973a5250495fade1aa9b35cfd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Mar 2018 15:03:47 -0700 Subject: [PATCH 575/920] scoutfs: cleanup xattr item storage Honoring the XATTR_REMOVE flag in xattr deletion exposed an interesting bug in getxattr(). We were unconditinally returning the max xattr value size when someone tried to probe an existing xattrs' value size by calling getxattr with size == 0. Some kernel paths did this to probe the existance of xattrs. They expected to get an error if the xattr didn't exist, but we were giving them the max possible size. This kernel path then tried to remove the xattrs with XATTR_REMOVE and that now failed and caused a bunch of errors in xfstests. The fix is to return the real xattr value size when getxattr is called with size == 0. To do that with the old format we'd have to iterate over all the items which happened to be pretty awkward in the current code paths. So we're taking this opportunity to land a change that had been brewing for a while. We now form the xattr keys from the hash of the name and the item values now store a logical contiquous header, the name, and the value. This makes it very easy for us to have the full xattr value length in the header and return it from getxattr when size == 0. Now all tests pass while honororing the XATTR_CREATE and XATTR_REMOVE flags. And the code is a whole lot easier to follow. And we've removed another barrier for moving to small fixed size keys. Signed-off-by: Zach Brown --- kmod/src/count.h | 37 ++- kmod/src/format.h | 34 ++- kmod/src/inode.c | 3 + kmod/src/inode.h | 1 + kmod/src/key.c | 8 +- kmod/src/scoutfs_trace.h | 5 + kmod/src/super.c | 2 - kmod/src/xattr.c | 636 ++++++++++++++++++++++----------------- kmod/src/xattr.h | 2 - 9 files changed, 406 insertions(+), 322 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 1139c4cd..30b55ca5 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -183,30 +183,35 @@ static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, } /* - * Setting an xattr results in a dirty set of items with values for the - * size of the xattr. There's always at least one item with a value - * header. Any previously existing items from a larger xattr are - * deleted which dirties their key but removes their value. We don't - * know the size of a possibly existing xattr so we assume max parts. + * Creating an xattr results in a dirty set of items with values that + * store the xattr header, name, and value. There's always at least one + * item with the header and name. Any previously existing items are + * deleted which dirties their key but removes their value. The two + * sets of items are indexed by different ids so their items don't + * overlap. */ -static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned name_len, +static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, + bool creating, + unsigned name_len, unsigned size) { struct scoutfs_item_count cnt = {0,}; - unsigned val_parts; + unsigned int new_parts; __count_dirty_inode(&cnt); - val_parts = max_t(unsigned, 1, - DIV_ROUND_UP(size, SCOUTFS_XATTR_PART_SIZE)); + if (old_parts) { + cnt.items += old_parts; + cnt.keys += old_parts * sizeof(struct scoutfs_xattr_key); + } - cnt.items += SCOUTFS_XATTR_MAX_PARTS; - cnt.keys += SCOUTFS_XATTR_MAX_PARTS * - (offsetof(struct scoutfs_xattr_key, name[name_len]) + - sizeof(struct scoutfs_xattr_key_footer)); - cnt.vals += val_parts * - (sizeof(struct scoutfs_xattr_val_header) + - SCOUTFS_XATTR_PART_SIZE); + if (creating) { + new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size) + + cnt.items += new_parts; + cnt.keys += new_parts * sizeof(struct scoutfs_xattr_key); + cnt.vals += sizeof(struct scoutfs_xattr) + name_len + size; + } return cnt; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 99241394..ff3e4872 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -362,22 +362,24 @@ struct scoutfs_orphan_key { __be64 ino; } __packed; -/* value is each item's part of the full xattr value for the off/len */ struct scoutfs_xattr_key { __u8 zone; __be64 ino; __u8 type; - __u8 name[0]; -} __packed; - -struct scoutfs_xattr_key_footer { - __u8 null; + __be32 name_hash; + __be64 id; __u8 part; } __packed; -struct scoutfs_xattr_val_header { - __le16 part_len; - __u8 last_part; +/* + * The first xattr part item has a header that describes the xattr. The + * name and value are then packed into the following bytes in the first + * part item and overflow into the values of the rest of the part items. + */ +struct scoutfs_xattr { + __u8 name_len; + __le16 val_len; + __u8 name[0]; } __packed; /* size determines nr needed to store full target path in their values */ @@ -472,6 +474,7 @@ struct scoutfs_inode { __le64 online_blocks; __le64 offline_blocks; __le64 next_readdir_pos; + __le64 next_xattr_id; __le32 nlink; __le32 uid; __le32 gid; @@ -529,12 +532,13 @@ enum { #define SCOUTFS_MAX_VAL_SIZE SCOUTFS_BLOCK_MAPPING_MAX_BYTES -#define SCOUTFS_XATTR_MAX_NAME_LEN 255 -#define SCOUTFS_XATTR_MAX_SIZE 65536 -#define SCOUTFS_XATTR_PART_SIZE \ - (SCOUTFS_MAX_VAL_SIZE - sizeof(struct scoutfs_xattr_val_header)) -#define SCOUTFS_XATTR_MAX_PARTS \ - DIV_ROUND_UP(SCOUTFS_XATTR_MAX_SIZE, SCOUTFS_XATTR_PART_SIZE) +#define SCOUTFS_XATTR_MAX_NAME_LEN 255 +#define SCOUTFS_XATTR_MAX_VAL_LEN 65535 +#define SCOUTFS_XATTR_MAX_PART_SIZE 512U + +#define SCOUTFS_XATTR_NR_PARTS(name_len, val_len) \ + DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ + SCOUTFS_XATTR_MAX_PART_SIZE); /* * structures used by dlm diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 965afbde..fa4d3dae 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -234,6 +234,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) ci->online_blocks = le64_to_cpu(cinode->online_blocks); ci->offline_blocks = le64_to_cpu(cinode->offline_blocks); ci->next_readdir_pos = le64_to_cpu(cinode->next_readdir_pos); + ci->next_xattr_id = le64_to_cpu(cinode->next_xattr_id); ci->flags = le32_to_cpu(cinode->flags); /* @@ -667,6 +668,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->offline_blocks = cpu_to_le64(scoutfs_inode_offline_blocks(inode)); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); + cinode->next_xattr_id = cpu_to_le64(ci->next_xattr_id); cinode->flags = cpu_to_le32(ci->flags); } @@ -1327,6 +1329,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->online_blocks = 0; ci->offline_blocks = 0; ci->next_readdir_pos = SCOUTFS_DIRENT_FIRST_POS; + ci->next_xattr_id = 0; ci->have_item = false; atomic64_set(&ci->last_refreshed, lock->refresh_gen); ci->flags = 0; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 0bc95a4c..39313955 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -18,6 +18,7 @@ struct scoutfs_inode_info { /* read or initialized for each inode instance */ u64 ino; u64 next_readdir_pos; + u64 next_xattr_id; u64 meta_seq; u64 data_seq; u64 data_version; diff --git a/kmod/src/key.c b/kmod/src/key.c index 7406a5fe..5befe0e0 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -264,13 +264,13 @@ static int pr_inode(char *buf, struct scoutfs_key_buf *key, size_t size) static int pr_xattr(char *buf, struct scoutfs_key_buf *key, size_t size) { struct scoutfs_xattr_key *xkey = key->data; - int len = (int)key->key_len - - offsetof(struct scoutfs_xattr_key, name[1]); return snprintf_key(buf, size, key, sizeof(struct scoutfs_xattr_key), key->key_len, - "fs.%llu.xat.%.*s", - be64_to_cpu(xkey->ino), len, xkey->name); + "fs.%llu.xat.%08x.%llu.%u", + be64_to_cpu(xkey->ino), + be32_to_cpu(xkey->name_hash), + be64_to_cpu(xkey->id), xkey->part); } static int pr_dirent(char *buf, struct scoutfs_key_buf *key, size_t size) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 343f6f1a..a7751374 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1435,6 +1435,11 @@ DEFINE_EVENT(scoutfs_key_class, scoutfs_item_shrink, TP_ARGS(sb, key) ); +DEFINE_EVENT(scoutfs_key_class, scoutfs_xattr_get_next_key, + TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_ARGS(sb, key) +); + DECLARE_EVENT_CLASS(scoutfs_range_class, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), diff --git a/kmod/src/super.c b/kmod/src/super.c index ab2bedba..f9d5b80e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -26,7 +26,6 @@ #include "format.h" #include "inode.h" #include "dir.h" -#include "xattr.h" #include "msg.h" #include "counters.h" #include "triggers.h" @@ -437,7 +436,6 @@ static int __init scoutfs_module_init(void) } ret = scoutfs_inode_init() ?: scoutfs_dir_init() ?: - scoutfs_xattr_init() ?: register_filesystem(&scoutfs_fs_type); out: if (ret) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 9661102a..727d8e43 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -14,6 +14,7 @@ #include #include #include +#include #include "format.h" #include "inode.h" @@ -27,110 +28,61 @@ #include "scoutfs_trace.h" /* - * In the simple case an xattr is stored in a single item whose key and - * value contain the key and value from the xattr. + * Extended attributes are packed into multiple smaller file system + * items. The common case only uses one item. * - * But xattr values can be larger than our max item value length. In - * that case the rest of the xattr value is stored in additional items. - * Each item key contains a footer struct after the name which - * identifies the position of the item in the series that make up the - * total xattr. + * The xattr keys contain the hash of the xattr name and a unique + * identifier used to differentiate xattrs whose names hash to the same + * value. xattr lookup has to walk all the xattrs with the matching + * name hash to compare the names. * - * That xattrs are then spread out across multiple items does mean that - * we need locking other than the item cache locking which only protects - * each item call, the i_mutex which isn't held on getxattr, and cluster - * locking which doesn't serialize local matches on the same node. We - * use a rwsem in the inode. + * We use a rwsem in the inode to serialize modification of multiple + * items to make sure that we don't let readers race and see an + * inconsistent mix of the items that make up xattrs. * * XXX * - add acl support and call generic xattr->handlers for SYSTEM */ -/* - * We have a static full xattr name with all 1s so that we can construct - * precise final keys for the range of items that cover all the xattrs - * on an inode. We could instead construct a smaller last key for the - * next inode with a null name but that could be accidentally create - * lock contention with that next inode. We want lock ranges to be as - * precise as possible. - */ -static char last_xattr_name[SCOUTFS_XATTR_MAX_NAME_LEN]; - -/* account for the footer after the name */ -static unsigned xattr_key_bytes(unsigned name_len) +static u32 xattr_name_hash(const char *name, unsigned int name_len) { - return offsetof(struct scoutfs_xattr_key, name[name_len]) + - sizeof(struct scoutfs_xattr_key_footer); + return crc32c(U32_MAX, name, name_len); } -static unsigned xattr_key_name_len(struct scoutfs_key_buf *key) +/* only compare names if the lens match, callers might not have both names */ +static u32 xattr_names_equal(const char *a_name, unsigned int a_len, + const char *b_name, unsigned int b_len) { - return key->key_len - xattr_key_bytes(0); + return a_len == b_len && memcmp(a_name, b_name, a_len) == 0; } -static struct scoutfs_xattr_key_footer * -xattr_key_footer(struct scoutfs_key_buf *key) +static unsigned int xattr_full_bytes(struct scoutfs_xattr *xat) { - return key->data + key->key_len - - sizeof(struct scoutfs_xattr_key_footer); + return offsetof(struct scoutfs_xattr, + name[xat->name_len + le16_to_cpu(xat->val_len)]); } -static struct scoutfs_key_buf *alloc_xattr_key(struct super_block *sb, - u64 ino, const char *name, - unsigned int name_len, u8 part) +static unsigned int xattr_nr_parts(struct scoutfs_xattr *xat) { - struct scoutfs_xattr_key_footer *foot; - struct scoutfs_xattr_key *xkey; - struct scoutfs_key_buf *key; - - key = scoutfs_key_alloc(sb, xattr_key_bytes(name_len)); - if (key) { - xkey = key->data; - foot = xattr_key_footer(key); - - xkey->zone = SCOUTFS_FS_ZONE; - xkey->ino = cpu_to_be64(ino); - xkey->type = SCOUTFS_XATTR_TYPE; - - if (name && name_len) - memcpy(xkey->name, name, name_len); - - foot->null = '\0'; - foot->part = part; - } - - return key; + return SCOUTFS_XATTR_NR_PARTS(xat->name_len, + le16_to_cpu(xat->val_len)); } -static void set_xattr_key_part(struct scoutfs_key_buf *key, u8 part) +/* If no name is provided then the hash arg is used, caller can modify part */ +static void init_xattr_key(struct scoutfs_key_buf *key, + struct scoutfs_xattr_key *xak, u64 ino, + u32 name_hash, u64 id) { - struct scoutfs_xattr_key_footer *foot = xattr_key_footer(key); + xak->zone = SCOUTFS_FS_ZONE; + xak->ino = cpu_to_be64(ino); + xak->type = SCOUTFS_XATTR_TYPE; + xak->name_hash = cpu_to_be32(name_hash); + xak->id = cpu_to_be64(id); + xak->part = 0; - foot->part = part; + scoutfs_key_init(key, xak, sizeof(struct scoutfs_xattr_key)); } -/* - * This walks the keys and values for the items that make up the xattr - * items that describe the value in the caller's buffer. The caller is - * responsible for breaking out when it hits an existing final item that - * hasn't consumed the buffer. - * - * Each iteration sets the val header in case the caller is writing - * items. If they're reading items they'll just overwrite it. - */ -#define for_each_xattr_item(key, val, vh, buffer, size, part, off, bytes) \ - for (part = 0, off = 0; \ - ((off < size) || (part == 0 && size == 0)) && \ - (bytes = min_t(size_t, SCOUTFS_XATTR_PART_SIZE, size - off), \ - set_xattr_key_part(key, part), \ - (vh)->part_len = cpu_to_le16(bytes), \ - (vh)->last_part = off + bytes == size ? 1 : 0, \ - scoutfs_kvec_init(val, vh, \ - sizeof(struct scoutfs_xattr_val_header), \ - buffer + off, bytes), \ - 1); \ - part++, off += bytes) - static int unknown_prefix(const char *name) { return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && @@ -139,6 +91,182 @@ static int unknown_prefix(const char *name) strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); } +/* + * Find the next xattr and copy the key, xattr header, and as much of + * the name and value into the callers buffer as we can. Returns the + * number of bytes copied which include the header, name, and value and + * can be limited by the xattr length or the callers buffer. The caller + * is responsible for comparing their lengths, the header, and the + * returned length before safely using the xattr. + * + * If a name is provided then we'll iterate over items with a matching + * name_hash until we find a matching name. If we don't find a matching + * name then we return -ENOENT. + * + * If a name isn't provided then we'll return the next xattr from the + * given name_hash and id position. + * + * Returns -ENOENT if it didn't find a next item. + */ +static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, + struct scoutfs_xattr *xat, unsigned int bytes, + const char *name, unsigned int name_len, + u64 name_hash, u64 id, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr_key last_xak; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + u8 last_part; + int total; + u8 part; + int ret; + + /* need to be able to see the name we're looking for */ + if (WARN_ON_ONCE(name_len > 0 && bytes < offsetof(struct scoutfs_xattr, + name[name_len]))) + return -EINVAL; + + if (name_len) + name_hash = xattr_name_hash(name, name_len); + + init_xattr_key(&key, xak, scoutfs_ino(inode), name_hash, id); + init_xattr_key(&last, &last_xak, scoutfs_ino(inode), U32_MAX, U64_MAX); + + last_part = 0; + part = 0; + total = 0; + + for (;;) { + xak->part = part; + scoutfs_kvec_init(val, (void *)xat + total, bytes - total); + ret = scoutfs_item_next(sb, &key, &last, val, lock); + if (ret < 0) { + /* XXX corruption, ran out of parts */ + if (ret == -ENOENT && part > 0) + ret = -EIO; + break; + } + + trace_scoutfs_xattr_get_next_key(sb, &key); + + /* XXX corruption */ + if (xak->part != part) { + ret = -EIO; + break; + } + + /* + * XXX corruption: We should have seen a valid header in + * the first part and if the next xattr name fits in our + * buffer then the item must have included it. + */ + if (part == 0 && + (ret < sizeof(struct scoutfs_xattr) || + (xat->name_len <= name_len && + ret < offsetof(struct scoutfs_xattr, + name[xat->name_len])) || + xat->name_len > SCOUTFS_XATTR_MAX_NAME_LEN || + le16_to_cpu(xat->val_len) > SCOUTFS_XATTR_MAX_VAL_LEN)) { + ret = -EIO; + break; + } + + if (part == 0 && name_len) { + /* ran out of names that could match */ + if (be32_to_cpu(xak->name_hash) != name_hash) { + ret = -ENOENT; + break; + } + + /* keep looking for our name */ + if (!xattr_names_equal(name, name_len, + xat->name, xat->name_len)) { + part = 0; + be64_add_cpu(&xak->id, 1); + continue; + } + + /* use the matching name we found */ + last_part = xattr_nr_parts(xat) - 1; + } + + total += ret; + if (total == bytes || part == last_part) { + /* copied as much as we could */ + ret = total; + break; + } + part++; + } + + return ret; +} + +/* + * Create all the items associated with the given xattr. If this + * returns an error it will have already cleaned up any items it created + * before seeing the error. + */ +static int create_xattr_items(struct inode *inode, u64 id, + struct scoutfs_xattr *xat, unsigned int bytes, + struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr_key xak; + struct scoutfs_key_buf key; + SCOUTFS_DECLARE_KVEC(val); + unsigned int part_bytes; + int total; + int ret; + + init_xattr_key(&key, &xak, scoutfs_ino(inode), + xattr_name_hash(xat->name, xat->name_len), id); + + total = 0; + ret = 0; + while (total < bytes) { + part_bytes = min(bytes - total, SCOUTFS_XATTR_MAX_PART_SIZE); + scoutfs_kvec_init(val, (void *)xat + total, part_bytes); + + ret = scoutfs_item_create(sb, &key, val, lock); + if (ret) { + while (xak.part-- > 0) + scoutfs_item_delete_dirty(sb, &key); + break; + } + + total += part_bytes; + xak.part++; + } + + return ret; +} + +/* + * Delete and save the items that make up the given xattr. If this + * returns an error then the deleted and saved items are left on the + * list for the caller to restore. + */ +static int delete_xattr_items(struct inode *inode, u64 name_hash, u64 id, + u8 nr_parts, struct list_head *list, + struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr_key xak; + struct scoutfs_key_buf key; + int ret; + + init_xattr_key(&key, &xak, scoutfs_ino(inode), name_hash, id); + + do { + ret = scoutfs_item_delete_save(sb, &key, list, lock); + } while (ret == 0 && ++xak.part < nr_parts); + + return ret; +} + /* * Copy the value for the given xattr name into the caller's buffer, if it * fits. Return the bytes copied or -ERANGE if it doesn't fit. @@ -147,18 +275,13 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, size_t size) { struct inode *inode = dentry->d_inode; - struct super_block *sb = inode->i_sb; struct scoutfs_inode_info *si = SCOUTFS_I(inode); - struct scoutfs_xattr_val_header vh; - struct scoutfs_key_buf *key = NULL; - struct scoutfs_key_buf *last = NULL; - SCOUTFS_DECLARE_KVEC(val); - struct scoutfs_lock *lck; - unsigned int total; + struct super_block *sb = inode->i_sb; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *lck = NULL; + struct scoutfs_xattr_key xak; unsigned int bytes; - unsigned int off; size_t name_len; - u8 part; int ret; if (unknown_prefix(name)) @@ -168,16 +291,11 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) return -ENODATA; - /* honestly, userspace, just alloc a max size buffer */ - if (size == 0) - return SCOUTFS_XATTR_MAX_SIZE; - - key = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0); - last = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0xff); - if (!key || !last) { - ret = -ENOMEM; - goto out; - } + /* only need enough for caller's name and value sizes */ + bytes = sizeof(struct scoutfs_xattr) + name_len + size; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) + return -ENOMEM; ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); if (ret) @@ -185,56 +303,40 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, down_read(&si->xattr_rwsem); - total = 0; - vh.last_part = 0; - - for_each_xattr_item(key, val, &vh, buffer, size, part, off, bytes) { - - ret = scoutfs_item_lookup(sb, key, val, lck); - if (ret < 0) { - if (ret == -ENOENT) - ret = -ENODATA; - break; - } - - /* XXX corruption: no header, more val than header len */ - ret -= sizeof(struct scoutfs_xattr_val_header); - if (ret < 0 || ret > le16_to_cpu(vh.part_len)) { - ret = -EIO; - break; - } - - /* not enough buffer if we didn't copy the part */ - if (ret < le16_to_cpu(vh.part_len)) { - ret = -ERANGE; - break; - } - - total += ret; - - /* XXX corruption: total xattr val too long */ - if (total > SCOUTFS_XATTR_MAX_SIZE) { - ret = -EIO; - break; - } - - /* done if we fully copied last part */ - if (vh.last_part) { - ret = total; - break; - } - } - - /* not enough buffer if we didn't see last */ - if (ret >= 0 && !vh.last_part) - ret = -ERANGE; + ret = get_next_xattr(inode, &xak, xat, bytes, + name, name_len, 0, 0, lck); up_read(&si->xattr_rwsem); scoutfs_unlock(sb, lck, DLM_LOCK_PR); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENODATA; + goto out; + } + + /* the caller just wants to know the size */ + if (size == 0) { + ret = le16_to_cpu(xat->val_len); + goto out; + } + + /* the caller's buffer wasn't big enough */ + if (size < le16_to_cpu(xat->val_len)) { + ret = -ERANGE; + goto out; + } + + /* XXX corruption, the items didn't match the header */ + if (ret < xattr_full_bytes(xat)) { + ret = -EIO; + goto out; + } + + ret = le16_to_cpu(xat->val_len); + memcpy(buffer, &xat->name[xat->name_len], ret); out: - scoutfs_key_free(sb, key); - scoutfs_key_free(sb, last); + kfree(xat); return ret; } @@ -244,46 +346,47 @@ out: * * This always removes the old existing xattr items. * - * If the value pointer is set then we're replacing it with a new xattr. - * The flags cause creation to fail if the xattr already exists - * (_CREATE) or doesn't already exist (_REPLACE). xattrs can have a - * zero length value. + * If the value pointer is set then we're adding a new xattr. The flags + * cause creation to fail if the xattr already exists (_CREATE) or + * doesn't already exist (_REPLACE). xattrs can have a zero length + * value. */ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) - { struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_key_buf *last = NULL; - struct scoutfs_key_buf *key = NULL; - struct scoutfs_xattr_val_header vh; - size_t name_len = strlen(name); - SCOUTFS_DECLARE_KVEC(val); + struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; - unsigned int bytes; - unsigned int off; + size_t name_len = strlen(name); + struct scoutfs_xattr_key xak; LIST_HEAD(ind_locks); LIST_HEAD(saved); + u8 found_parts; + unsigned int bytes; u64 ind_seq; - int part; + u64 id; int ret; trace_scoutfs_xattr_set(sb, name_len, value, size, flags); - if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN || - (value && size > SCOUTFS_XATTR_MAX_SIZE) || - ((flags & XATTR_CREATE) && (flags & XATTR_REPLACE)) || + /* mirror the syscall's errors for large names and values */ + if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) + return -ERANGE; + if (value && size > SCOUTFS_XATTR_MAX_VAL_LEN) + return -E2BIG; + + if (((flags & XATTR_CREATE) && (flags & XATTR_REPLACE)) || (flags & ~(XATTR_CREATE | XATTR_REPLACE))) return -EINVAL; if (unknown_prefix(name)) return -EOPNOTSUPP; - key = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0); - last = alloc_xattr_key(sb, scoutfs_ino(inode), name, name_len, 0xff); - if (!key || !last) { + bytes = sizeof(struct scoutfs_xattr) + name_len + size; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) { ret = -ENOMEM; goto out; } @@ -293,64 +396,70 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (ret) goto out; - /* see if we violate the flag constraints */ - if ((flags & (XATTR_CREATE | XATTR_REPLACE))) { - set_xattr_key_part(key, 0); - ret = scoutfs_item_lookup(sb, key, NULL, lck); - if (ret == -ENOENT && (flags & XATTR_REPLACE)) - ret = -ENODATA; - else if (ret == 0 && (flags & XATTR_CREATE)) - ret = -EEXIST; - else if (ret == -ENOENT && (flags & XATTR_CREATE)) - ret = 0; - if (ret < 0) - goto out; + down_write(&si->xattr_rwsem); + + /* find an existing xattr to delete */ + ret = get_next_xattr(inode, &xak, xat, + sizeof(struct scoutfs_xattr) + name_len, + name, name_len, 0, 0, lck); + if (ret < 0 && ret != -ENOENT) + goto unlock; + + /* check existence constraint flags */ + if (ret == -ENOENT && (flags & XATTR_REPLACE)) { + ret = -ENODATA; + goto unlock; + } else if (ret >= 0 && (flags & XATTR_CREATE)) { + ret = -EEXIST; + goto unlock; + } + + /* not an error to delete something that doesn't exist */ + if (ret == -ENOENT && !value) { + ret = 0; + goto unlock; + } + + /* found fields in xak will also be used */ + found_parts = ret >= 0 ? xattr_nr_parts(xat) : 0; + + /* prepare our xattr */ + if (value) { + id = si->next_xattr_id++; + xat->name_len = name_len; + xat->val_len = cpu_to_le16(size); + memcpy(xat->name, name, name_len); + memcpy(&xat->name[xat->name_len], value, size); } retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, - SIC_XATTR_SET(name_len, size)); + SIC_XATTR_SET(found_parts, + value != NULL, + name_len, size)); if (ret > 0) goto retry; if (ret) - goto out; - - down_write(&si->xattr_rwsem); + goto unlock; ret = scoutfs_dirty_inode_item(inode, lck); if (ret < 0) goto release; - /* delete and save any existing xattr items */ - for (part = 0; part < SCOUTFS_XATTR_MAX_PARTS; part++) { - set_xattr_key_part(key, part); - ret = scoutfs_item_delete_save(sb, key, &saved, lck); - if (ret == -ENOENT) { - ret = 0; - break; - } - if (ret < 0) - goto release; - } - - /* create items for the new xattr */ - if (value) { - for_each_xattr_item(key, val, &vh, (void *)value, size, - part, off, bytes) { - - ret = scoutfs_item_create(sb, key, val, lck); - if (ret < 0) { - /* remove any previously created items */ - while (--part >= 0) { - set_xattr_key_part(key, part); - scoutfs_item_delete_dirty(sb, key); - } - goto release; - } - } + ret = 0; + if (found_parts) + ret = delete_xattr_items(inode, be32_to_cpu(xak.name_hash), + be64_to_cpu(xak.id), found_parts, + &saved, lck); + if (value && ret == 0) + ret = create_xattr_items(inode, id, xat, bytes, lck); + if (ret < 0) { + scoutfs_item_restore(sb, &saved, lck); + goto release; } + scoutfs_item_free_batch(sb, &saved); /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); @@ -359,19 +468,13 @@ retry: ret = 0; release: - /* restore the old xattr if we modified it then errored */ - if (ret < 0) - scoutfs_item_restore(sb, &saved, lck); - - up_write(&si->xattr_rwsem); scoutfs_release_trans(sb); - -out: scoutfs_inode_index_unlock(sb, &ind_locks); +unlock: + up_write(&si->xattr_rwsem); scoutfs_unlock(sb, lck, DLM_LOCK_EX); - scoutfs_item_free_batch(sb, &saved); - scoutfs_key_free(sb, key); - scoutfs_key_free(sb, last); +out: + kfree(xat); return ret; } @@ -395,61 +498,43 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_xattr_key_footer *foot; - struct scoutfs_xattr_key *xkey; - struct scoutfs_key_buf *key; - struct scoutfs_key_buf *last; - struct scoutfs_lock *lck; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *lck = NULL; + struct scoutfs_xattr_key xak; + unsigned int bytes; ssize_t total; - int name_len; + u32 name_hash; + u64 id; int ret; - key = alloc_xattr_key(sb, scoutfs_ino(inode), - NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); - last = alloc_xattr_key(sb, scoutfs_ino(inode), last_xattr_name, - SCOUTFS_XATTR_MAX_NAME_LEN, 0xff); - if (!key || !last) { + /* need a buffer large enough for all possible names */ + bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) { ret = -ENOMEM; goto out; } - xkey = key->data; - xkey->name[0] = '\0'; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); if (ret) goto out; down_read(&si->xattr_rwsem); + name_hash = 0; + id = 0; total = 0; + for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL, lck); + ret = get_next_xattr(inode, &xak, xat, bytes, + NULL, 0, name_hash, id, lck); if (ret < 0) { if (ret == -ENOENT) ret = total; break; } - /* not used until we verify key len */ - foot = xattr_key_footer(key); - - /* XXX corruption */ - if (key->key_len < xattr_key_bytes(1) || - foot->null != '\0' || foot->part != 0) { - ret = -EIO; - break; - } - - name_len = xattr_key_name_len(key); - - /* XXX corruption? */ - if (name_len > SCOUTFS_XATTR_MAX_NAME_LEN) { - ret = -EIO; - break; - } - - total += name_len + 1; + total += xat->name_len + 1; if (size) { if (total > size) { @@ -457,26 +542,27 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) break; } - memcpy(buffer, xkey->name, name_len); - buffer += name_len; + memcpy(buffer, xat->name, xat->name_len); + buffer += xat->name_len; *(buffer++) = '\0'; } - set_xattr_key_part(key, 0xff); + name_hash = be32_to_cpu(xak.name_hash); + id = be64_to_cpu(xak.id) + 1; } up_read(&si->xattr_rwsem); scoutfs_unlock(sb, lck, DLM_LOCK_PR); out: - scoutfs_key_free(sb, key); - scoutfs_key_free(sb, last); + kfree(xat); return ret; } /* * Delete all the xattr items associated with this inode. The caller - * holds a transaction. + * holds a transaction. The inode is dead so we don't need the xattr + * rwsem. * * XXX This isn't great because it reads in all the items so that it can * create deletion items for each. It would be better to have the @@ -485,53 +571,37 @@ out: */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { - struct scoutfs_key_buf *key; - struct scoutfs_key_buf *last; + struct scoutfs_xattr_key last_xak; + struct scoutfs_xattr_key xak; + struct scoutfs_key_buf last; + struct scoutfs_key_buf key; struct scoutfs_lock *lck; int ret; - key = alloc_xattr_key(sb, ino, NULL, SCOUTFS_XATTR_MAX_NAME_LEN, 0); - last = alloc_xattr_key(sb, ino, last_xattr_name, - SCOUTFS_XATTR_MAX_NAME_LEN, 0xff); - if (!key || !last) { - ret = -ENOMEM; - goto out; - } + init_xattr_key(&key, &xak, ino, 0, 0); + init_xattr_key(&last, &last_xak, ino, U32_MAX, U64_MAX); /* while we read to delete we need to writeback others */ ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lck); if (ret) goto out; - /* the inode is dead so we don't need the xattr sem */ - for (;;) { - ret = scoutfs_item_next(sb, key, last, NULL, lck); + ret = scoutfs_item_next(sb, &key, &last, NULL, lck); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - ret = scoutfs_item_delete(sb, key, lck); + ret = scoutfs_item_delete(sb, &key, lck); if (ret) break; - /* don't need to increment past deleted key */ + xak.part++; } scoutfs_unlock(sb, lck, DLM_LOCK_EX); - out: - scoutfs_key_free(sb, key); - scoutfs_key_free(sb, last); - return ret; } - -int scoutfs_xattr_init(void) -{ - memset(last_xattr_name, 0xff, sizeof(last_xattr_name)); - - return 0; -} diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index 1035d622..e0fadf32 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -10,6 +10,4 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); int scoutfs_xattr_drop(struct super_block *sb, u64 ino); -int scoutfs_xattr_init(void); - #endif From 3818f727767fbd15d542eb6c7b4b99f255bea5c0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Feb 2018 10:27:51 -0800 Subject: [PATCH 576/920] scoutfs: fix inefficient backwards item reading Iterating over items backwards would result in a lot of extra work. When an item isn't present in the cache we go and search the segments for the item. Once we find the item in its stack of segments we also read in and cache all the items from the missing item to the end of all the segments. This reduced complexity a bit but had very bad worst case performance. If you read items backwards you constantly get cache misses that each search the segments for the item and then try to cache everything to the end of the segment. You're essentially working uncached and are doing quite a lot of work to get that single missed item cached each time. This adds the complexity to cache all the items in the segment stack around the missed item, not just after the missed item. Now reverse iteration hits cached items for everything in the segment after the initial miss. To make this work we have to pass the full lock coverage range to the item reading path. Then we search the manifest for segments that contain the missing key and use those segment's ranges to determine the full range of items that we'll cache. Then we again search the manifest for all the level 0 segments that intersect that range. That range extension is only for cached reads, it doesn't apply to the 'next' call which ignores caching. That operation is getting different enough that we pull it out into its own function. Signed-off-by: Zach Brown --- kmod/src/counters.h | 3 + kmod/src/item.c | 27 ++- kmod/src/manifest.c | 434 ++++++++++++++++++++++++++------------- kmod/src/manifest.h | 1 + kmod/src/scoutfs_trace.h | 37 +++- 5 files changed, 350 insertions(+), 152 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 61397277..379e062d 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -37,6 +37,8 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(item_alloc) \ + EXPAND_COUNTER(item_batch_duplicate) \ + EXPAND_COUNTER(item_batch_inserted) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_delete) \ EXPAND_COUNTER(item_free) \ @@ -75,6 +77,7 @@ EXPAND_COUNTER(lock_unlock) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ + EXPAND_COUNTER(manifest_read_excluded_key) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(seg_shrink) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index 2f165f3b..03c10fe3 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -800,7 +800,8 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); trace_scoutfs_item_lookup_ret(sb, ret); return ret; @@ -963,7 +964,8 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, /* populate missing cached range starting at pos */ spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, pos, lock->end); + ret = scoutfs_manifest_read_items(sb, pos, lock->start, + lock->end); spin_lock_irqsave(&cac->lock, flags); if (ret) @@ -1099,7 +1101,8 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); if (ret) free_item(sb, item); @@ -1224,8 +1227,12 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); - if (insert_item(sb, cac, item, false, true)) + if (insert_item(sb, cac, item, false, true)) { + scoutfs_inc_counter(sb, item_batch_duplicate); list_add(&item->entry, list); + } else { + scoutfs_inc_counter(sb, item_batch_inserted); + } } spin_unlock_irqrestore(&cac->lock, flags); @@ -1280,7 +1287,8 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); trace_scoutfs_item_dirty_ret(sb, ret); return ret; @@ -1334,7 +1342,8 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); out: scoutfs_kvec_kfree(up_val); @@ -1382,7 +1391,8 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); trace_scoutfs_item_delete_ret(sb, ret); return ret; @@ -1479,7 +1489,8 @@ int scoutfs_item_delete_save(struct super_block *sb, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, lock->start, + lock->end)) == 0); free_item(sb, del); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index f7cd99e6..80e00c11 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -434,30 +435,76 @@ static int btree_prev_overlap_or_next(struct super_block *sb, } /* - * starting with the caller's key. The entries will be ordered by the - * order that they should be read: level 0 from newest to oldest then - * increasing higher order levels. + * Get references to all the level 0 segments whose item ranges + * intersect with the callers range. We walk the manifest backwards so + * that we end up adding refs to the caller's list reverse sorted by + * sequence, which is what they want to be able to use the segment with + * the newest item. * - * We have to get all the level 0 segments that intersect with the range - * of items that we want to search because the level 0 segments can - * arbitrarily overlap with each other. - * - * We only need to search for the starting key in all the higher levels. - * They do not overlap so we can iterate through the key space in each - * segment starting with the key. In each level we need the first - * existing segment that intersects with the range, even if it doesn't - * contain the key. The key might fall between segments at that level. - * - * This is walking stable btree roots. The blocks won't be changed as - * long as we read valid blocks. They can be overwritten in which case - * we'll return -ESTALE and the caller can retry with a newer root or - * return hard errors. + * This can return -ESTALE if it reads through stale btree blocks. */ -static int get_manifest_refs(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - struct list_head *ref_list) +static int get_zero_refs(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct list_head *ref_list) +{ + struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_entry ment; + SCOUTFS_BTREE_ITEM_REF(iref); + SCOUTFS_BTREE_ITEM_REF(prev); + unsigned mkey_len; + int ret; + + scoutfs_manifest_init_entry(&ment, 0, 0, 0, start, NULL); + mkey = alloc_btree_key_val(&ment, &mkey_len, NULL, NULL); + if (!mkey) + return -ENOMEM; + + /* get level 0 segments that overlap with the missing range */ + mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); + ret = scoutfs_btree_prev(sb, root, mkey, mkey_len, &iref); + while (ret == 0) { + init_ment_iref(&ment, &iref); + + if (scoutfs_key_compare_ranges(start, end, &ment.first, + &ment.last) == 0) { + ret = alloc_manifest_ref(sb, ref_list, &ment); + if (ret) + break; + } + + swap(prev, iref); + ret = scoutfs_btree_before(sb, root, prev.key, prev.key_len, + &iref); + scoutfs_btree_put_iref(&prev); + } + if (ret == -ENOENT) + ret = 0; + + scoutfs_btree_put_iref(&iref); + scoutfs_btree_put_iref(&prev); + kfree(mkey); + return ret; +} + +/* + * Get references to all segments in non-zero levels that contain the + * caller's search key. The item ranges of segments at each non-zero + * level don't overlap so we can iterate through the key space in each + * segment starting with the search key. In each level we need the + * first existing segment that intersects with the range, even if it + * doesn't contain the key. The key might fall between segments at that + * level. If a segment is entirely outside of the caller's range then + * we can't trust its contents. + * + * This can return -ESTALE if it reads through stale btree blocks. + */ +static int get_nonzero_refs(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *end, + struct list_head *ref_list) { struct scoutfs_manifest_btree_key *mkey; struct scoutfs_manifest_entry ment; @@ -472,38 +519,10 @@ static int get_manifest_refs(struct super_block *sb, if (!mkey) return -ENOMEM; - /* get level 0 segments that overlap with the missing range */ - mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); - ret = scoutfs_btree_prev(sb, root, mkey, mkey_len, &iref); - while (ret == 0) { - init_ment_iref(&ment, &iref); - - if (scoutfs_key_compare_ranges(key, end, &ment.first, - &ment.last) == 0) { - ret = alloc_manifest_ref(sb, ref_list, &ment); - if (ret) - goto out; - } - - swap(prev, iref); - ret = scoutfs_btree_before(sb, root, prev.key, prev.key_len, - &iref); - scoutfs_btree_put_iref(&prev); - } - if (ret != -ENOENT) - goto out; - - /* - * XXX Today we need to read the next segment if our starting key - * falls between segments. That won't be the case once we tie - * cached items to their locks. - */ mkey_len = init_btree_key(mkey, 1, 0, key); for (i = 1; ; i++) { mkey->level = i; - /* XXX should use level counts to skip searches */ - scoutfs_btree_put_iref(&iref); ret = btree_prev_overlap_or_next(sb, root, mkey, mkey_len, key, i, &iref); @@ -515,7 +534,8 @@ static int get_manifest_refs(struct super_block *sb, init_ment_iref(&ment, &iref); - if (ment.level != i) + if (ment.level != i || + scoutfs_key_compare(&ment.first, end) > 0) continue; ret = alloc_manifest_ref(sb, ref_list, &ment); @@ -531,24 +551,73 @@ out: return ret; } +/* + * See if the caller is a remote btree reader who has read a stale btree + * block and should keep trying. If they see repeated errors on the + * same root then we assume that it's persistent corruption. + */ +static int handle_stale_btree(struct super_block *sb, + struct scoutfs_btree_root *root, + __le64 last_root_seq, int ret) +{ + bool force_hard = scoutfs_trigger(sb, HARD_STALE_ERROR); + + if (ret == -ESTALE || force_hard) { + if ((last_root_seq != root->ref.seq) && !force_hard) + return -EAGAIN; + + scoutfs_inc_counter(sb, manifest_hard_stale_error); + return -EIO; + } + + return 0; +} + +static int cmp_ment_ref_segno(void *priv, struct list_head *A, + struct list_head *B) +{ + struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); + struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); + + return scoutfs_cmp_u64s(a->segno, b->segno); +} + +/* + * Sort by from most to least recent item contents.. from lowest to higest + * level and from highest to loweset seq in level 0. + */ +static int cmp_ment_ref_level_seq(void *priv, struct list_head *A, + struct list_head *B) +{ + struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); + struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); + + if (a->level == 0 && b->level == 0) + return -scoutfs_cmp_u64s(a->seq, b->seq); + + return a->level < b->level ? -1 : a->level > b->level ? 1 : 0; +} + /* * The caller found a hole in the item cache that they'd like populated. + * We can only trust items in the segments within their range (they hold + * a lock) and they're going to keep calling ("He'll keep calling me, + * he'll keep calling me") until we insert a range into the cache that + * contains the search key. * - * We search the manifest for all the segments we'll need to iterate - * from the key to the end key, the last key we're allowed to insert - * into the cache. + * We search the manifest for all the non-zero segments that contain the + * key. We adjust the search range if the segments don't cover the + * whole locked range. We have to be careful not to shrink the range + * past the key, it could be outside the segments and we still want to + * negatively cache it. Once we have the search range we get the level + * zero segments that overlap. * - * If next_key is provided then the segments are only walked to find the - * next key after the search key. If none is found -ENOENT is returned. - * There's no limit on the next_key we can return, the caller has - * to deal with that. + * Once we have the segments we iterate over them and allocate the items + * to insert into the cache. We find the next item in each segment, + * ignore deletion items, prefer more recent segments, and advance past + * the items that we used. * - * As we insert the batch of items we give the item cache the range of - * keys that contain these items. This lets the cache return negative - * cache lookups for missing items within the range. - * - * Returns 0 if we inserted items with a range covering the starting - * key. The caller should be able to make progress. + * Returns 0 if we successfully inserted items. * * Returns -errno if we failed to make any change in the cache. * @@ -560,16 +629,17 @@ out: * The segments are immutable at this point so we can use their contents * as long as we hold refs. */ -static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, - struct scoutfs_key_buf *next_key) +int scoutfs_manifest_read_items(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end) { struct scoutfs_key_buf item_key; struct scoutfs_key_buf found_key; struct scoutfs_key_buf batch_end; + struct scoutfs_key_buf seg_start; struct scoutfs_key_buf seg_end; struct scoutfs_btree_root root; - struct scoutfs_inode_key junk; SCOUTFS_DECLARE_KVEC(item_val); SCOUTFS_DECLARE_KVEC(found_val); struct scoutfs_segment *seg; @@ -578,7 +648,6 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, __le64 last_root_seq; LIST_HEAD(ref_list); LIST_HEAD(batch); - bool force_hard; u8 found_flags = 0; u8 item_flags; int found_ctr; @@ -588,18 +657,6 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, int err; int cmp; - if (WARN_ON_ONCE(!end && !next_key)) - return -EINVAL; - - if (end) { - scoutfs_key_clone(&seg_end, end); - } else { - scoutfs_key_init(&seg_end, &junk, sizeof(junk)); - scoutfs_key_set_max(&seg_end); - } - - trace_scoutfs_read_items(sb, key, &seg_end); - /* * Ask the manifest server which manifest root to read from. Lock * holding callers will be responsible for this in the future. They'll @@ -608,15 +665,41 @@ static int read_items(struct super_block *sb, struct scoutfs_key_buf *key, */ last_root_seq = 0; retry_stale: + + scoutfs_key_clone(&seg_start, start); + scoutfs_key_clone(&seg_end, end); + ret = scoutfs_client_get_manifest_root(sb, &root); if (ret) goto out; - /* get refs on all the segments */ - ret = get_manifest_refs(sb, &root, key, &seg_end, &ref_list); + /* get non-zero segments that intersect with the missed key */ + ret = get_nonzero_refs(sb, &root, key, &seg_end, &ref_list); if (ret) goto out; + /* clamp start and end to the segment boundaries, including key */ + list_for_each_entry(ref, &ref_list, entry) { + + if (scoutfs_key_compare(ref->first, &seg_start) > 0 && + scoutfs_key_compare(ref->first, key) <= 0) + scoutfs_key_clone(&seg_start, ref->first); + + if (scoutfs_key_compare(ref->last, &seg_end) < 0 && + scoutfs_key_compare(ref->last, key) >= 0) + scoutfs_key_clone(&seg_end, ref->last); + } + + trace_scoutfs_read_item_keys(sb, key, start, end, &seg_start, &seg_end); + + /* then get level 0s that intersect with our search range */ + ret = get_zero_refs(sb, &root, &seg_start, &seg_end, &ref_list); + if (ret) + goto out; + + /* sort by segment to issue advancing reads */ + list_sort(NULL, &ref_list, cmp_ment_ref_segno); + /* submit reads for all the segments */ list_for_each_entry(ref, &ref_list, entry) { @@ -644,24 +727,12 @@ retry_stale: if (ret) goto out; - /* start from the next item from the key in each segment */ - list_for_each_entry(ref, &ref_list, entry) - ref->off = scoutfs_seg_find_off(ref->seg, key); + /* now sort refs by item age */ + list_sort(NULL, &ref_list, cmp_ment_ref_level_seq); - /* - * Find the limit of the range we can safely walk. We have all - * the level 0 segments that intersect with the caller's range. - * But we only have the level > 0 segments that intersected with - * the starting key. We have to stop at the nearest end of - * those segments because other segments might overlap after - * that. - */ - list_for_each_entry(ref, &ref_list, entry) { - if (ref->level > 0 && - scoutfs_key_compare(ref->last, &seg_end) < 0) { - scoutfs_key_clone(&seg_end, ref->last); - } - } + /* walk items from the start of our range */ + list_for_each_entry(ref, &ref_list, entry) + ref->off = scoutfs_seg_find_off(ref->seg, &seg_start); found_ctr = 0; @@ -709,16 +780,6 @@ retry_stale: found = true; } - if (next_key) { - if (found) { - scoutfs_key_copy(next_key, &found_key); - ret = 0; - } else { - ret = -ENOENT; - } - break; - } - /* ran out of keys in segs, range extends to seg end */ if (!found) { scoutfs_key_clone(&batch_end, &seg_end); @@ -765,50 +826,147 @@ retry_stale: ret = 0; } - if (next_key || ret) + if (ret < 0) { scoutfs_item_free_batch(sb, &batch); - else - ret = scoutfs_item_insert_batch(sb, &batch, key, &batch_end); + } else { + if (scoutfs_key_compare(key, &batch_end) > 0) + scoutfs_inc_counter(sb, manifest_read_excluded_key); + ret = scoutfs_item_insert_batch(sb, &batch, &seg_start, + &batch_end); + } out: list_for_each_entry_safe(ref, tmp, &ref_list, entry) { list_del_init(&ref->entry); free_ref(sb, ref); } - /* - * Resample the root and retry reads as long as we see - * inconsistent blocks/segments and new roots to read through. - * Persistent inconsistency in the same root is seen as corrupt - * structures instead. - */ - force_hard = scoutfs_trigger(sb, HARD_STALE_ERROR); - if (ret == -ESTALE || force_hard) { - /* keep trying as long as the root changes */ - if ((last_root_seq != root.ref.seq) && !force_hard) { - last_root_seq = root.ref.seq; - goto retry_stale; - } - - /* persistent error */ - scoutfs_inc_counter(sb, manifest_hard_stale_error); - ret = -EIO; + ret = handle_stale_btree(sb, &root, last_root_seq, ret); + if (ret == -EAGAIN) { + last_root_seq = root.ref.seq; + goto retry_stale; } return ret; } -int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end) -{ - return read_items(sb, key, end, NULL); -} - +/* + * Give the caller a hint to the next key that they'll find after their + * search key. + * + * We read the segments that intersect the key and return either the + * next item we see or the nearest segment limit. + * + * This is a hint because we can return deleted items or the next + * nearest segment limit can be well before the next items in the next + * segments. The caller needs to very carefully iterate using the next + * key we return. + * + * Returns 0 if it set next_key and -ENOENT if the key was after all the + * segments in the manifest. + */ int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *next_key) { - return read_items(sb, key, NULL, next_key); + struct scoutfs_key_buf item_key; + struct scoutfs_key_buf end; + struct scoutfs_btree_root root; + struct scoutfs_inode_key end_key; + struct scoutfs_segment *seg; + struct manifest_ref *ref; + struct manifest_ref *tmp; + __le64 last_root_seq; + LIST_HEAD(ref_list); + bool found; + int ret; + int err; + + last_root_seq = 0; +retry_stale: + ret = scoutfs_client_get_manifest_root(sb, &root); + if (ret) + goto out; + + scoutfs_key_init(&end, &end_key, sizeof(end_key)); + scoutfs_key_set_max(&end); + + ret = get_zero_refs(sb, &root, key, &end, &ref_list) ?: + get_nonzero_refs(sb, &root, key, &end, &ref_list); + if (ret) + goto out; + + if (list_empty(&ref_list)) { + ret = -ENOENT; + goto out; + } + + list_sort(NULL, &ref_list, cmp_ment_ref_segno); + + list_for_each_entry(ref, &ref_list, entry) { + seg = scoutfs_seg_submit_read(sb, ref->segno); + if (IS_ERR(seg)) { + ret = PTR_ERR(seg); + break; + } + + ref->seg = seg; + } + + list_for_each_entry(ref, &ref_list, entry) { + if (!ref->seg) + break; + + err = scoutfs_seg_wait(sb, ref->seg, ref->segno, ref->seq); + if (err && !ret) + ret = err; + } + if (ret) + goto out; + + list_sort(NULL, &ref_list, cmp_ment_ref_level_seq); + + /* default to returning the nearest segment limit and find offsets */ + found = false; + list_for_each_entry(ref, &ref_list, entry) { + if (ref->level > 0 && + (!found || scoutfs_key_compare(ref->last, next_key) < 0)) { + scoutfs_key_copy(next_key, ref->last); + found = true; + } + + ref->off = scoutfs_seg_find_off(ref->seg, key); + } + + /* return the nearest item in the segments */ + list_for_each_entry_safe(ref, tmp, &ref_list, entry) { + if (ref->off < 0) + continue; + + ret = scoutfs_seg_item_ptrs(ref->seg, ref->off, &item_key, + NULL, NULL); + if (ret < 0) + continue; + + if (!found || scoutfs_key_compare(&item_key, next_key) < 0) { + scoutfs_key_copy(next_key, &item_key); + found = true; + } + } + + ret = 0; +out: + list_for_each_entry_safe(ref, tmp, &ref_list, entry) { + list_del_init(&ref->entry); + free_ref(sb, ref); + } + + ret = handle_stale_btree(sb, &root, last_root_seq, ret); + if (ret == -EAGAIN) { + last_root_seq = root.ref.seq; + goto retry_stale; + } + + return ret; } /* diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index f36a58fe..020a76fc 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -33,6 +33,7 @@ int scoutfs_manifest_unlock(struct super_block *sb); int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key_buf *start, struct scoutfs_key_buf *end); int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key_buf *key, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a7751374..92071f62 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1388,6 +1388,37 @@ DEFINE_EVENT(scoutfs_manifest_class, scoutfs_read_item_segment, TP_ARGS(sb, level, segno, seq, first, last) ); +TRACE_EVENT(scoutfs_read_item_keys, + TP_PROTO(struct super_block *sb, + struct scoutfs_key_buf *key, + struct scoutfs_key_buf *start, + struct scoutfs_key_buf *end, + struct scoutfs_key_buf *seg_start, + struct scoutfs_key_buf *seg_end), + TP_ARGS(sb, key, start, end, seg_start, seg_end), + TP_STRUCT__entry( + __field(__u64, fsid) + __dynamic_array(char, key, scoutfs_key_str(NULL, key)) + __dynamic_array(char, start, scoutfs_key_str(NULL, start)) + __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + __dynamic_array(char, seg_start, + scoutfs_key_str(NULL, seg_start)) + __dynamic_array(char, seg_end, + scoutfs_key_str(NULL, seg_end)) + ), + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + scoutfs_key_str(__get_dynamic_array(key), key); + scoutfs_key_str(__get_dynamic_array(start), start); + scoutfs_key_str(__get_dynamic_array(end), end); + scoutfs_key_str(__get_dynamic_array(seg_start), seg_start); + scoutfs_key_str(__get_dynamic_array(seg_end), seg_end); + ), + TP_printk("fsid "FSID_FMT" key %s start %s end %s seg_start %s seg_end %s", + __entry->fsid, __get_str(key), __get_str(start), + __get_str(end), __get_str(seg_start), __get_str(seg_end)) +); + DECLARE_EVENT_CLASS(scoutfs_key_class, TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), TP_ARGS(sb, key), @@ -1476,12 +1507,6 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, TP_ARGS(sb, start, end) ); -DEFINE_EVENT(scoutfs_range_class, scoutfs_read_items, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), - TP_ARGS(sb, start, end) -); - DECLARE_EVENT_CLASS(scoutfs_cached_range_class, TP_PROTO(struct super_block *sb, void *rng, struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), From 995e43aa18dff86a374f98e670dcb9f7ee369b49 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 15 Mar 2018 15:32:13 -0700 Subject: [PATCH 577/920] scoutfs: hold the alloc sem during truncate The super info's alloc_rwsem protects the local node free segment and block bitmap items. The truncate code wasn't holding using the rwsem so it could race with other local node allocator item users and corrupt the bitmaps. In the best case this could corrupt structures that trigger EIO. The corrupt items could also create duplicate block allocations that clobber each other and corrupt data. Signed-off-by: Zach Brown --- kmod/src/data.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 16f3b20e..10c7833c 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -612,13 +612,14 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, u64 ino, u64 iblock, u64 last, bool offline, struct scoutfs_lock *lock) { + DECLARE_DATA_INFO(sb, datinf); struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; struct scoutfs_block_mapping_key last_bmk; struct scoutfs_block_mapping_key bmk; struct block_mapping *map; SCOUTFS_DECLARE_KVEC(val); - bool holding; + bool holding = false; bool dirtied; bool modified; u64 blkno; @@ -642,6 +643,13 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, init_mapping_key(&key, &bmk, ino, iblock); scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); + if (ret) + break; + holding = true; + + down_write(&datinf->alloc_rwsem); + ret = scoutfs_item_next(sb, &key, &last_key, val, lock); if (ret < 0) { if (ret == -ENOENT) @@ -657,7 +665,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, iblock = max(iblock, be64_to_cpu(bmk.base) << SCOUTFS_BLOCK_MAPPING_SHIFT); - holding = false; dirtied = false; modified = false; for_each_block(i, iblock, last) { @@ -669,13 +676,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, !!offline == !!test_bit(i, map->offline)) continue; - if (!holding) { - ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); - if (ret) - break; - holding = true; - } - if (!dirtied) { /* dirty item with full size encoded */ ret = scoutfs_item_update(sb, &key, val, lock); @@ -721,15 +721,19 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, } } - if (holding) { - scoutfs_release_trans(sb); - holding = false; - } + up_write(&datinf->alloc_rwsem); + scoutfs_release_trans(sb); + holding = false; if (ret) break; } + if (holding) { + up_write(&datinf->alloc_rwsem); + scoutfs_release_trans(sb); + } + kfree(map); return ret; } From 22f1ded17b0407af2e83e995dfc800822fbda502 Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Tue, 28 Nov 2017 14:47:20 -0700 Subject: [PATCH 578/920] Add RPM builds for scoutfs-kmod This adds in the makefile targets and spec file template we need to build RPMs. Most of the heavy lifting is taken care of by our docker container and the rpmbuild.sh script distributed in it. The git versioning comes from 'git describe --long', which gives us the tag, the number of commits and the abbreviated commit name. This allows us to use the number of commits as the RPM release version, letting yum understand how to process 'yum update' ordering. yum update shows us the proper processing, along with how our versioning lines up in the RPMs: ---> Package kmod-scoutfs.x86_64 0:0-0.3.gb83d29d.el7 will be updated ---> Package kmod-scoutfs.x86_64 0:0-0.4.g2e5324e.el7 will be an update The rpm file name is: kmod-scoutfs-0-0.4.g2e5324e.el7.x86_64.rpm When we build release RPMS, we'll toggle _release, giving us a rpm name and version like kmod-scoutfs-0-1.4.g2e5324e.el7.x86_64.rpm. The toggle of 0/1 is enough to tell yum that all of the non-release RPMs with the same version are older than the released RPMs. This allows for the release to yum update cleanly over development versions. The git hash helps map RPM names to the git version and the contents of the .note-git_describe, for this RPM it was: heads/nic/rpms-0-g2e5324. The RPM doesn't contain the branch name, but we can add that and other info later if needed. We are not naming the module for a kernel version, that does not seem to be standard practice upstream. Instead, we'll make use of our Artifactory repos and upload the RPMs to the correct places (7.3 vs 7.4 directories, etc). --- kmod/.gitignore | 2 ++ kmod/Makefile | 48 ++++++++++++++++++++++++++ kmod/scoutfs-kmod.spec.in | 72 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 kmod/scoutfs-kmod.spec.in diff --git a/kmod/.gitignore b/kmod/.gitignore index 23820239..2fa4c0bc 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -10,3 +10,5 @@ cscope.* *.spec *.sw[po] rpmbuild/ + +scoutfs-*.git*/ diff --git a/kmod/Makefile b/kmod/Makefile index 2963f205..c4a40dc6 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -24,11 +24,59 @@ SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ EXTRA_CFLAGS="-Werror" +# move damage locally, this will also help make it easier to cleanup after the build +RPM_DIR = $(shell pwd)/rpmbuild + +# - We use the git describe from tags to set up the RPM versioning +RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') +RPM_RELEASE := $(shell git describe --long --tags | awk -F '-' '{print $$2"."$$3}') +FULL_VERSION := $(RPM_VERSION).$(RPM_RELEASE) +TARFILE = $(RPM_DIR)/SOURCES/scoutfs-kmod-$(FULL_VERSION).tar + +.PHONY: .FORCE + all: module module: make $(SCOUTFS_ARGS) $(SP) make C=2 CF="-D__CHECK_ENDIAN__" $(SCOUTFS_ARGS) + +modules_install: + make $(SCOUTFS_ARGS) modules_install + + +# remake this each time.. +$(RPM_DIR): .FORCE + @echo "## Clean up on isle $(RPM_DIR)..." + rm -frv $(RPM_DIR)/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + mkdir -p $(RPM_DIR)/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + + +%.spec: %.spec.in .FORCE + sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ + -e s'/@@TAR_VERSION@@/$(FULL_VERSION)/g' \ + -e s'/@@RELEASE@@/$(RPM_RELEASE)/g' < $< > $@+ + mv $@+ $@ + + +# NOTE: Both tar & rpm are capable of being built natively on Linux, provided +# you have a local install of rpmbuild.sh for the rpm target. +# Normal exection is to use docker, as that pulls our canned image and tooling for the user. +# ./indocker.sh make tar +# ./indocker.sh make rpm +# +tar: $(RPM_DIR) scoutfs-kmod.spec + git archive --format=tar --prefix scoutfs-$(FULL_VERSION)/ HEAD^{tree} > $(TARFILE) + @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-$(FULL_VERSION)/\1@" scoutfs-kmod.spec + gzip -f -9 $(TARFILE) + + +$(TARFILE).gz: tar + +rpm: $(TARFILE).gz scoutfs-kmod.spec + rpmbuild.sh $(TARFILE).gz + + clean: make $(SCOUTFS_ARGS) clean diff --git a/kmod/scoutfs-kmod.spec.in b/kmod/scoutfs-kmod.spec.in new file mode 100644 index 00000000..72ec10ea --- /dev/null +++ b/kmod/scoutfs-kmod.spec.in @@ -0,0 +1,72 @@ +%define kmod_name scoutfs +#%%trace + +%define _tar_version @@TAR_VERSION@@ +# official builds set this to 1, we use 0 for internal/dev-test +%{!?_release: %global _release 0} + +Name: %{kmod_name} +Summary: %{kmod_name} kernel module +Version: @@VERSION@@ +Release: %{_release}.@@RELEASE@@%{?dist} +License: GPLv2 +Group: System/Kernel +URL: http://versity.com + +BuildRequires: %kernel_module_package_buildreqs +ExclusiveArch: x86_64 + +# Sources. +Source0: scoutfs-kmod-%{_tar_version}.tar.gz + +# Build only for standard kernel variant(s); for debug packages, append "debug" +# after "default" (separated by space) +%kernel_module_package default + + +# Disable the building of the debug package(s). +%define debug_package %{nil} + +%description +%{kmod_name} - kernel module + + +%prep + + +%setup -q -n %{kmod_name}-%{_tar_version} +set -- * +mkdir source +mv "$@" source/ +mkdir obj + + +%build +echo "Building for kernel: %{kernel_version} flavors: '%{flavors_to_build}'" +echo "Build var: kmodtool = %{kmodtool}" +echo "Build var: kverrel = %{kverrel}" +for flavor in %flavors_to_build; do + rm -rf obj/$flavor + cp -r source obj/$flavor + make SK_KSRC=%{kernel_source $flavor} -C obj/$flavor module +done + + +%install +export INSTALL_MOD_PATH=$RPM_BUILD_ROOT +export INSTALL_MOD_DIR=extra/%{name} +for flavor in %flavors_to_build ; do + # TODO add Makefile rule + #make SK_KSRC=%{kernel_source $flavor} -C obj/$flavor modules_install + make -C %{kernel_source $flavor} modules_install \ + M=$PWD/obj/$flavor/src +done + + +%clean +rm -rf %{buildroot} + + +%changelog +* Fri Nov 17 2017 Nic Henke - 1.0 +- Initial version. From 9d18d3a7aac8eab8fb2fe0ef55291e4994df5a9b Mon Sep 17 00:00:00 2001 From: Nic Henke Date: Thu, 29 Mar 2018 10:48:03 -0600 Subject: [PATCH 579/920] Add script to build rpms and populate distro release To better support building RPMs for multiple distribution versions, we need a bit of help to organize the RPMs. We take the path of adding a 'rpms/7.4.1708' style directory, with the implicit knowledge this is for CentOS and RHEL. Other distribution handling is left for the future. To ease DOCKER_IMAGE selection for different distribution versions, the environment variable DISTRO_VERS can be used. This simplifies a bunch of call locations and scripting when we don't need to change the Docker image flavor beyond this distribution version toggle. i.e: DISTRO_VERS=el73 ./indocker.sh ./build_rpms.sh The directory tree ends up looking like this: rpms/7.4.1708/kmod-scoutfs-1.0-0.git.5fee207.el7.x86_64.rpm rpms/7.3.1611/kmod-scoutfs-1.0-0.git.5fee207.el7.x86_64.rpm --- kmod/.gitignore | 1 + kmod/build_rpms.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100755 kmod/build_rpms.sh diff --git a/kmod/.gitignore b/kmod/.gitignore index 2fa4c0bc..a1fd5abc 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -10,5 +10,6 @@ cscope.* *.spec *.sw[po] rpmbuild/ +rpms/ scoutfs-*.git*/ diff --git a/kmod/build_rpms.sh b/kmod/build_rpms.sh new file mode 100755 index 00000000..c9d5c479 --- /dev/null +++ b/kmod/build_rpms.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e -o pipefail + +# Build RPMs, populating a directory structure that indicates the OS release. +# NOTE: This expects to run in a CentOS or RHEL environment, preferrably one of the versity +# rpm-build Docker containers. + +OS_RELEASE=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' /etc/redhat-release) +echo "OS RELEASE: $OS_RELEASE" + +make rpm + +rpm_dist="rpms/$OS_RELEASE" +rm -fvr "$rpm_dist" +mkdir -p "$rpm_dist" + +cp -v rpmbuild/RPMS/x86_64/kmod-scoutfs*.rpm "$rpm_dist/" From 9c1b39340405234d0fa864b3071859c63e0efa0b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Mar 2018 13:20:16 -0700 Subject: [PATCH 580/920] scoutfs: don't track offline sparse blocks There were some mistakes in tracking offline blocks. Online and offline block counts are meant to only refer to actual data contents. Sparse blocks in an archived file shouldn't be counted as offline. But the code was marking unallocated blocks as offline. This could corrupt the offline block count if a release extended past i_size and marked the blocks in the mapping item as offline even though they're past i_size. We could have clamped the block walking to not go past i_size. But we still would have had the problem of having offline blocks track sparse blocks. Instead we can fix the problem by only marking blocks offline if they had allocated blocks. This means that sparse regions are never marked offline and will always read zeros. Now a release that extends past i_size will not do anything to the unallocated blocks in the mapping item past i_size and the offline block count will be consistent. (Also the 'modified' and 'dirty' booleans were redundant, we only need one of the two.) Signed-off-by: Zach Brown --- kmod/src/data.c | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 10c7833c..17a04e62 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -598,7 +598,8 @@ out: * inclusive. * * If 'offline' is given then blocks are freed an offline mapping is - * left behind. + * left behind. Only blocks that have been allocated can be marked + * offline. * * This is the low level extent item manipulation code. We hold and * release the transaction so the caller doesn't have to deal with @@ -621,7 +622,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, SCOUTFS_DECLARE_KVEC(val); bool holding = false; bool dirtied; - bool modified; u64 blkno; int bytes; int ret = 0; @@ -666,14 +666,13 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, SCOUTFS_BLOCK_MAPPING_SHIFT); dirtied = false; - modified = false; for_each_block(i, iblock, last) { blkno = map->blknos[i]; /* don't need to do anything.. */ if (!blkno && - !!offline == !!test_bit(i, map->offline)) + !(!offline && test_bit(i, map->offline))) continue; if (!dirtied) { @@ -684,33 +683,34 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, dirtied = true; } - /* free if allocated */ - if (blkno) { - ret = set_blkno_free(sb, blkno); - if (ret) - break; - - map->blknos[i] = 0; - scoutfs_inode_add_online_blocks(inode, -1); - /* XXX gets tricky with concurrent writes */ - inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; - } - - if (offline && !test_bit(i, map->offline)) { - set_bit(i, map->offline); - scoutfs_inode_add_offline_blocks(inode, 1); - inode->i_blocks += SCOUTFS_BLOCK_SECTORS; - - } else if (!offline && test_bit(i, map->offline)) { + /* truncating offline block */ + if (!offline && test_bit(i, map->offline)) { clear_bit(i, map->offline); scoutfs_inode_add_offline_blocks(inode, -1); inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; } - modified = true; + /* nothing more to do if unallocated */ + if (!blkno) + continue; + + /* free the allocated block, maybe marking offline */ + ret = set_blkno_free(sb, blkno); + if (ret) + break; + + map->blknos[i] = 0; + scoutfs_inode_add_online_blocks(inode, -1); + inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; + + if (offline) { + set_bit(i, map->offline); + scoutfs_inode_add_offline_blocks(inode, 1); + inode->i_blocks += SCOUTFS_BLOCK_SECTORS; + } } - if (modified) { + if (dirtied) { /* update how ever much of the item we finished */ bytes = encode_mapping(map); if (bytes) { From 08f544cc154773bd4f6aebe3b736f59b40eb745b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Mar 2018 10:39:58 -0700 Subject: [PATCH 581/920] scoutfs: remove scoutfs_item_lookup_exact() size Every caller of scoutfs_item_lookup_exact() provided a size that matches the value buffer. Let's remove the redundant arg and use the value buffer length as the exact size to match. Signed-off-by: Zach Brown --- kmod/src/data.c | 19 +++++-------------- kmod/src/dir.c | 8 +++----- kmod/src/inode.c | 5 ++--- kmod/src/item.c | 9 +++++---- kmod/src/item.h | 2 +- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 17a04e62..f7f82c2f 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -362,9 +362,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) init_free_key(&key, &fbk, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret && ret != -ENOENT) goto out; @@ -443,9 +441,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) init_free_key(&key, &fbk, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret) { /* XXX corruption, caller saw item.. should still exist */ if (ret == -ENOENT) @@ -497,9 +493,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) init_free_key(&key, &fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret && ret != -ENOENT) goto out; @@ -558,9 +552,7 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) init_free_key(&key, &fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret) { /* XXX corruption, bits should have existed */ if (ret == -ENOENT) @@ -856,8 +848,7 @@ static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) SCOUTFS_FREE_BITS_BLKNO_TYPE); scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, - sizeof(struct scoutfs_free_bits), lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret < 0) goto out; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e79d3245..0c6ff515 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -258,8 +258,7 @@ static int lookup_dirent(struct super_block *sb, struct inode *dir, scoutfs_kvec_init(val, dent, sizeof(struct scoutfs_dirent)); - ret = scoutfs_item_lookup_exact(sb, key, val, - sizeof(struct scoutfs_dirent), lock); + ret = scoutfs_item_lookup_exact(sb, key, val, lock); out: scoutfs_key_free(sb, key); return ret; @@ -999,8 +998,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, if (op == SYM_CREATE) ret = scoutfs_item_create(sb, &key, val, lock); else if (op == SYM_LOOKUP) - ret = scoutfs_item_lookup_exact(sb, &key, val, bytes, - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); else if (op == SYM_DELETE) ret = scoutfs_item_delete(sb, &key, lock); if (ret) @@ -1445,7 +1443,7 @@ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, scoutfs_kvec_init(val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, sizeof(dent), lock); + ret = scoutfs_item_lookup_exact(sb, key, val, lock); if (ret == 0 && le64_to_cpu(dent.ino) != ino) ret = -ENOENT; else if (ret == -ENOENT && ino == 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index fa4d3dae..20ae32fd 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -286,8 +286,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, mutex_lock(&si->item_mutex); if (atomic64_read(&si->last_refreshed) < refresh_gen) { - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), - lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); @@ -1415,7 +1414,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) scoutfs_inode_init_key(&key, &ikey, ino); scoutfs_kvec_init(val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, sizeof(sinode), lock); + ret = scoutfs_item_lookup_exact(sb, &key, val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/item.c b/kmod/src/item.c index 03c10fe3..d0454552 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -809,21 +809,22 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, /* * This requires that the item at the specified key has a value of the - * same length as the specified value. Callers are asserting that + * same length as the caller's value buffer. Callers are asserting that * mismatched size are corruption so it returns -EIO if the sizes don't * match. This isn't the fast path so we don't mind the copying * overhead that comes from only detecting the size mismatch after the * copy by reusing the more permissive _lookup(). * - * The end key limits how many keys after the search key can be read - * and inserted into the cache. + * The end key limits how many keys after the search key can be read and + * inserted into the cache. * * Returns 0 or -errno. */ int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size, struct scoutfs_lock *lock) + struct scoutfs_lock *lock) { + int size = scoutfs_kvec_length(val); int ret; ret = scoutfs_item_lookup(sb, key, val, lock); diff --git a/kmod/src/item.h b/kmod/src/item.h index da487f85..6507d85a 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -10,7 +10,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, - int size, struct scoutfs_lock *lock); + struct scoutfs_lock *lock); int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, struct scoutfs_key_buf *last, struct kvec *val, struct scoutfs_lock *lock); From 982a0a313eb22b4d9056b6d141b33dd96a693648 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Mar 2018 10:47:15 -0700 Subject: [PATCH 582/920] scoutfs: allocate contiguous dirent for creation The values used in dirent item creation are one of the few places we have value kvecs with multiple entries. Let's instead allocate and copy the dirent struct and name into a contiguous buffer so that we can move towards single vector values. Signed-off-by: Zach Brown --- kmod/src/dir.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 0c6ff515..aa124018 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -526,26 +526,30 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, { struct scoutfs_key_buf *ent_key = NULL; struct scoutfs_key_buf *lb_key = NULL; + struct scoutfs_dirent *dent = NULL; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; - struct scoutfs_dirent dent; SCOUTFS_DECLARE_KVEC(val); bool del_ent = false; bool del_rdir = false; int ret; + ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); + dent = kmalloc(offsetof(struct scoutfs_dirent, name[name_len]), + GFP_NOFS); + if (!ent_key || !dent) { + ret = -ENOMEM; + goto out; + } + /* initialize the dent */ - dent.ino = cpu_to_le64(ino); - dent.readdir_pos = cpu_to_le64(pos); - dent.type = mode_to_type(mode); + dent->ino = cpu_to_le64(ino); + dent->readdir_pos = cpu_to_le64(pos); + dent->type = mode_to_type(mode); + memcpy(dent->name, name, name_len); /* dirent item for lookup */ - ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); - if (!ent_key) - return -ENOMEM; - - scoutfs_kvec_init(val, &dent, sizeof(dent)); - + scoutfs_kvec_init(val, dent, sizeof(struct scoutfs_dirent)); ret = scoutfs_item_create(sb, ent_key, val, dir_lock); if (ret) goto out; @@ -553,7 +557,8 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, /* readdir item for .. readdir */ init_readdir_key(&rdir_key, &rkey, dir_ino, pos); - scoutfs_kvec_init(val, &dent, sizeof(dent), (char *)name, name_len); + scoutfs_kvec_init(val, dent, offsetof(struct scoutfs_dirent, + name[name_len])); ret = scoutfs_item_create(sb, &rdir_key, val, dir_lock); if (ret) @@ -578,6 +583,7 @@ out: scoutfs_key_free(sb, ent_key); scoutfs_key_free(sb, lb_key); + kfree(dent); return ret; } From 966d0176f656e1175841901bb7d8428279d9a7bc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Mar 2018 10:56:25 -0700 Subject: [PATCH 583/920] scoutfs: remove seg kvec_from_pages Item values have been limited to a single value vector entry for a while. They can't span 4K blocks in the segment format so they can't cross kernel pages which are never smaller than 4K. We don't need a helper to build a vector of their contents across disjoint pages. This removes the last user of multi-element kvecs. Signed-off-by: Zach Brown --- kmod/src/seg.c | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index cdb79b2b..645a3134 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -410,20 +410,6 @@ out: return ret; } -static void kvec_from_pages(struct scoutfs_segment *seg, - struct kvec *kvec, u32 off, u16 len) -{ - u32 first; - - first = min_t(int, len, PAGE_SIZE - (off & ~PAGE_MASK)); - - if (first == len) - scoutfs_kvec_init(kvec, off_ptr(seg, off), len); - else - scoutfs_kvec_init(kvec, off_ptr(seg, off), first, - off_ptr(seg, off + first), len - first); -} - static u32 item_bytes(u8 nr_links, u16 key_len, u16 val_len) { return offsetof(struct scoutfs_segment_item, skip_links[nr_links]) + @@ -440,9 +426,9 @@ static inline void *item_key_ptr(struct scoutfs_segment_item *item) return (void *)item + item_bytes(item->nr_links, 0, 0); } -static inline int item_val_off(struct scoutfs_segment_item *item, int item_off) +static inline void *item_val_ptr(struct scoutfs_segment_item *item) { - return item_key_off(item, item_off) + le16_to_cpu(item->key_len); + return item_key_ptr(item) + le16_to_cpu(item->key_len); } static void item_ptrs(struct scoutfs_segment *seg, int off, @@ -454,7 +440,7 @@ static void item_ptrs(struct scoutfs_segment *seg, int off, scoutfs_key_init(key, item_key_ptr(item), le16_to_cpu(item->key_len)); if (val) - kvec_from_pages(seg, val, item_val_off(item, off), + scoutfs_kvec_init(val, item_val_ptr(item), le16_to_cpu(item->val_len)); } From b0bd273accb27b7f6f7f8eb61698387ea5e88112 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Dec 2017 09:25:10 -0800 Subject: [PATCH 584/920] scoutfs: remove support for multi-element kvecs Originally the item interfaces were written with full support for vectored keys and values. Callers constructed keys and values made up of header structs and data buffers. Segments supported much larger values which could span pages when stored in memory. But over time we've pulled that support back. Keys are described by a key struct instead of a multi-element kvec. Values are now much smaller and don't span pages. The item interfaces still use the kvec arrays but everyone only uses a single element. So let's make the world a whole lot less awful but having the item interfaces only supporting a single value buffer specified by a kvec. A bunch of code disappears and the result is much easier to understand. Signed-off-by: Zach Brown --- kmod/src/Makefile | 7 +- kmod/src/compact.c | 18 +-- kmod/src/data.c | 91 ++++++------ kmod/src/dir.c | 37 +++-- kmod/src/inode.c | 24 ++-- kmod/src/item.c | 107 +++++++++----- kmod/src/kvec.c | 292 --------------------------------------- kmod/src/kvec.h | 66 +-------- kmod/src/manifest.c | 10 +- kmod/src/scoutfs_trace.h | 1 - kmod/src/seg.c | 16 ++- kmod/src/xattr.c | 12 +- 12 files changed, 181 insertions(+), 500 deletions(-) delete mode 100644 kmod/src/kvec.c diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 903d8d29..9b72ebee 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,10 +6,9 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - export.o file.o kvec.o inode.o ioctl.o item.o key.o lock.o \ - manifest.o msg.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ - triggers.o xattr.o + export.o file.o inode.o ioctl.o item.o key.o lock.o manifest.o \ + msg.o options.o per_task.o seg.o server.o scoutfs_trace.o sock.o \ + sort_priv.o super.o sysfs.o trans.o triggers.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 96a38952..58087381 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -16,7 +16,6 @@ #include "super.h" #include "format.h" -#include "kvec.h" #include "seg.h" #include "bio.h" #include "cmp.h" @@ -186,7 +185,7 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, struct compact_seg *upper = curs->upper; struct compact_seg *lower = curs->lower; struct scoutfs_key_buf lower_key; - SCOUTFS_DECLARE_KVEC(lower_val); + struct kvec lower_val; u8 lower_flags; int cmp; int ret; @@ -205,7 +204,7 @@ retry: goto out; ret = scoutfs_seg_item_ptrs(lower->seg, lower->off, - &lower_key, lower_val, + &lower_key, &lower_val, &lower_flags); if (ret == 0) break; @@ -232,7 +231,7 @@ retry: if (cmp > 0) { scoutfs_key_clone(item_key, &lower_key); - scoutfs_kvec_clone(item_val, lower_val); + *item_val = lower_val; *item_flags = lower_flags; } @@ -278,13 +277,13 @@ static int compact_segments(struct super_block *sb, struct list_head *results) { struct scoutfs_key_buf item_key; - SCOUTFS_DECLARE_KVEC(item_val); struct scoutfs_segment *seg; struct compact_seg *cseg; struct compact_seg *upper; struct compact_seg *lower; unsigned next_segno = 0; bool append_filled = false; + struct kvec item_val; int ret = 0; u8 flags; @@ -363,7 +362,7 @@ static int compact_segments(struct super_block *sb, break; if (!append_filled) - ret = next_item(sb, curs, &item_key, item_val, &flags); + ret = next_item(sb, curs, &item_key, &item_val, &flags); else ret = 1; if (ret <= 0) @@ -410,13 +409,14 @@ static int compact_segments(struct super_block *sb, list_add_tail(&cseg->entry, results); for (;;) { - if (!scoutfs_seg_append_item(sb, seg, &item_key, item_val, - flags, curs->links)) { + if (!scoutfs_seg_append_item(sb, seg, &item_key, + &item_val, flags, + curs->links)) { append_filled = true; ret = 0; break; } - ret = next_item(sb, curs, &item_key, item_val, &flags); + ret = next_item(sb, curs, &item_key, &item_val, &flags); if (ret <= 0) { append_filled = false; break; diff --git a/kmod/src/data.c b/kmod/src/data.c index f7f82c2f..19108c40 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -25,6 +25,7 @@ #include "inode.h" #include "key.h" #include "data.h" +#include "kvec.h" #include "trans.h" #include "counters.h" #include "scoutfs_trace.h" @@ -355,14 +356,14 @@ static int set_segno_free(struct super_block *sb, u64 segno) struct scoutfs_free_bits_key fbk = {0,}; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int bit = 0; int ret; init_free_key(&key, &fbk, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret && ret != -ENOENT) goto out; @@ -371,7 +372,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) if (ret == -ENOENT) { memset(&frb, 0, sizeof(frb)); set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, val, lock); + ret = scoutfs_item_create(sb, &key, &val, lock); goto out; } @@ -380,7 +381,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) goto out; } - ret = scoutfs_item_update(sb, &key, val, lock); + ret = scoutfs_item_update(sb, &key, &val, lock); out: trace_scoutfs_data_set_segno_free(sb, segno, be64_to_cpu(fbk.base), bit, ret); @@ -399,18 +400,18 @@ static int create_blkno_free(struct super_block *sb, u64 blkno, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; struct scoutfs_free_bits frb; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int bit; init_free_key(key, fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); bit = blkno & SCOUTFS_FREE_BITS_MASK; memset(&frb, 0xff, sizeof(frb)); clear_bit_le(bit, frb.bits); - return scoutfs_item_create(sb, key, val, lock); + return scoutfs_item_create(sb, key, &val, lock); } /* @@ -433,15 +434,15 @@ static int clear_segno_free(struct super_block *sb, u64 segno) struct scoutfs_free_bits frb; struct scoutfs_key_buf b_key; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; u64 blkno; int bit; int ret; init_free_key(&key, &fbk, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret) { /* XXX corruption, caller saw item.. should still exist */ if (ret == -ENOENT) @@ -465,7 +466,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) ret = scoutfs_item_delete(sb, &key, lock); else - ret = scoutfs_item_update(sb, &key, val, lock); + ret = scoutfs_item_update(sb, &key, &val, lock); if (ret) scoutfs_item_delete_dirty(sb, &b_key); out: @@ -484,7 +485,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; u64 segno; int bit; int ret; @@ -492,8 +493,8 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) /* get the specified item */ init_free_key(&key, &fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret && ret != -ENOENT) goto out; @@ -502,7 +503,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) if (ret == -ENOENT) { memset(&frb, 0, sizeof(frb)); set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, val, lock); + ret = scoutfs_item_create(sb, &key, &val, lock); goto out; } @@ -512,7 +513,7 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) } if (!bitmap_full((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) { - ret = scoutfs_item_update(sb, &key, val, lock); + ret = scoutfs_item_update(sb, &key, &val, lock); goto out; } @@ -544,15 +545,15 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int bit; int ret; /* get the specified item */ init_free_key(&key, &fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret) { /* XXX corruption, bits should have existed */ if (ret == -ENOENT) @@ -570,7 +571,7 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) ret = scoutfs_item_delete(sb, &key, lock); else - ret = scoutfs_item_update(sb, &key, val, lock); + ret = scoutfs_item_update(sb, &key, &val, lock); out: return ret; } @@ -611,7 +612,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_block_mapping_key last_bmk; struct scoutfs_block_mapping_key bmk; struct block_mapping *map; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; bool holding = false; bool dirtied; u64 blkno; @@ -633,7 +634,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, while (iblock <= last) { /* find the mapping that could include iblock */ init_mapping_key(&key, &bmk, ino, iblock); - scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + kvec_init(&val, map->encoded, sizeof(map->encoded)); ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); if (ret) @@ -642,7 +643,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, down_write(&datinf->alloc_rwsem); - ret = scoutfs_item_next(sb, &key, &last_key, val, lock); + ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -669,7 +670,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, if (!dirtied) { /* dirty item with full size encoded */ - ret = scoutfs_item_update(sb, &key, val, lock); + ret = scoutfs_item_update(sb, &key, &val, lock); if (ret) break; dirtied = true; @@ -706,8 +707,8 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, /* update how ever much of the item we finished */ bytes = encode_mapping(map); if (bytes) { - scoutfs_kvec_init(val, map->encoded, bytes); - scoutfs_item_update_dirty(sb, &key, val); + kvec_init(&val, map->encoded, bytes); + scoutfs_item_update_dirty(sb, &key, &val); } else { scoutfs_item_delete_dirty(sb, &key); } @@ -840,15 +841,15 @@ static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int ret; int bit; init_free_key(&key, &fbk, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret < 0) goto out; @@ -878,7 +879,7 @@ static int find_free_segno(struct super_block *sb, u64 *segno) struct scoutfs_free_bits frb; struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int bit; int ret; @@ -886,9 +887,9 @@ static int find_free_segno(struct super_block *sb, u64 *segno) SCOUTFS_FREE_BITS_SEGNO_TYPE); init_free_key(&last_key, &last_fbk, sbi->node_id, ~0, SCOUTFS_FREE_BITS_SEGNO_TYPE); - scoutfs_kvec_init(val, &frb, sizeof(struct scoutfs_free_bits)); + kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_next(sb, &key, &last_key, val, lock); + ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); if (ret < 0) goto out; @@ -921,7 +922,7 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, { DECLARE_DATA_INFO(sb, datinf); struct task_cursor *curs; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int bytes; u64 segno; u64 blkno; @@ -965,11 +966,11 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, trace_scoutfs_data_find_alloc_block_found_seg(sb, segno, blkno); /* ensure that we can copy in encoded without failing */ - scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + kvec_init(&val, map->encoded, sizeof(map->encoded)); if (map_exists) - ret = scoutfs_item_update(sb, map_key, val, data_lock); + ret = scoutfs_item_update(sb, map_key, &val, data_lock); else - ret = scoutfs_item_create(sb, map_key, val, data_lock); + ret = scoutfs_item_create(sb, map_key, &val, data_lock); if (ret) goto out; @@ -991,8 +992,8 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, inode->i_blocks += SCOUTFS_BLOCK_SECTORS; bytes = encode_mapping(map); - scoutfs_kvec_init(val, map->encoded, bytes); - scoutfs_item_update_dirty(sb, map_key, val); + kvec_init(&val, map->encoded, bytes); + scoutfs_item_update_dirty(sb, map_key, &val); /* set cursor to next block, clearing if we finish the segment */ curs->blkno++; @@ -1016,7 +1017,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct scoutfs_key_buf key; struct scoutfs_lock *lock; struct block_mapping *map; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; bool exists; int ind; int ret; @@ -1031,10 +1032,10 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, return -ENOMEM; init_mapping_key(&key, &bmk, scoutfs_ino(inode), iblock); - scoutfs_kvec_init(val, map->encoded, sizeof(map->encoded)); + kvec_init(&val, map->encoded, sizeof(map->encoded)); /* find the mapping item that covers the logical block */ - ret = scoutfs_item_lookup(sb, &key, val, lock); + ret = scoutfs_item_lookup(sb, &key, &val, lock); if (ret < 0) { if (ret != -ENOENT) goto out; @@ -1316,7 +1317,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct pending_fiemap pend; struct scoutfs_block_mapping_key last_bmk; struct scoutfs_block_mapping_key bmk; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; loff_t i_size; bool offline; u64 blk_off; @@ -1358,9 +1359,9 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, while (blk_off <= final) { init_mapping_key(&key, &bmk, ino, blk_off); - scoutfs_kvec_init(val, &map->encoded, sizeof(map->encoded)); + kvec_init(&val, &map->encoded, sizeof(map->encoded)); - ret = scoutfs_item_next(sb, &key, &last_key, val, inode_lock); + ret = scoutfs_item_next(sb, &key, &last_key, &val, inode_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/dir.c b/kmod/src/dir.c index aa124018..3f980737 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -247,7 +247,7 @@ static int lookup_dirent(struct super_block *sb, struct inode *dir, struct scoutfs_lock *lock) { struct scoutfs_key_buf *key = NULL; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int ret; key = alloc_dirent_key(sb, scoutfs_ino(dir), name, name_len); @@ -256,9 +256,9 @@ static int lookup_dirent(struct super_block *sb, struct inode *dir, goto out; } - scoutfs_kvec_init(val, dent, sizeof(struct scoutfs_dirent)); + kvec_init(&val, dent, sizeof(struct scoutfs_dirent)); - ret = scoutfs_item_lookup_exact(sb, key, val, lock); + ret = scoutfs_item_lookup_exact(sb, key, &val, lock); out: scoutfs_key_free(sb, key); return ret; @@ -457,9 +457,9 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct scoutfs_readdir_key rkey; struct scoutfs_readdir_key last_rkey; struct scoutfs_lock *dir_lock; - SCOUTFS_DECLARE_KVEC(val); unsigned int item_len; unsigned int name_len; + struct kvec val; u64 pos; int ret; @@ -483,8 +483,8 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) for (;;) { init_readdir_key(&key, &rkey, scoutfs_ino(inode), file->f_pos); - scoutfs_kvec_init(val, dent, item_len); - ret = scoutfs_item_next_same_min(sb, &key, &last_key, val, + kvec_init(&val, dent, item_len); + ret = scoutfs_item_next_same_min(sb, &key, &last_key, &val, offsetof(struct scoutfs_dirent, name[1]), dir_lock); if (ret < 0) { @@ -529,9 +529,9 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, struct scoutfs_dirent *dent = NULL; struct scoutfs_key_buf rdir_key; struct scoutfs_readdir_key rkey; - SCOUTFS_DECLARE_KVEC(val); bool del_ent = false; bool del_rdir = false; + struct kvec val; int ret; ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); @@ -549,18 +549,17 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, memcpy(dent->name, name, name_len); /* dirent item for lookup */ - scoutfs_kvec_init(val, dent, sizeof(struct scoutfs_dirent)); - ret = scoutfs_item_create(sb, ent_key, val, dir_lock); + kvec_init(&val, dent, sizeof(struct scoutfs_dirent)); + ret = scoutfs_item_create(sb, ent_key, &val, dir_lock); if (ret) goto out; del_ent = true; /* readdir item for .. readdir */ init_readdir_key(&rdir_key, &rkey, dir_ino, pos); - scoutfs_kvec_init(val, dent, offsetof(struct scoutfs_dirent, - name[name_len])); + kvec_init(&val, dent, offsetof(struct scoutfs_dirent, name[name_len])); - ret = scoutfs_item_create(sb, &rdir_key, val, dir_lock); + ret = scoutfs_item_create(sb, &rdir_key, &val, dir_lock); if (ret) goto out; del_rdir = true; @@ -984,7 +983,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, { struct scoutfs_symlink_key skey; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; unsigned bytes; unsigned nr; int ret; @@ -999,12 +998,12 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, init_symlink_key(&key, &skey, ino, i); bytes = min_t(u64, size, SCOUTFS_MAX_VAL_SIZE); - scoutfs_kvec_init(val, (void *)target, bytes); + kvec_init(&val, (void *)target, bytes); if (op == SYM_CREATE) - ret = scoutfs_item_create(sb, &key, val, lock); + ret = scoutfs_item_create(sb, &key, &val, lock); else if (op == SYM_LOOKUP) - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); else if (op == SYM_DELETE) ret = scoutfs_item_delete(sb, &key, lock); if (ret) @@ -1440,16 +1439,16 @@ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, { struct scoutfs_key_buf *key = NULL; struct scoutfs_dirent dent; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int ret; key = alloc_dirent_key(sb, dir_ino, name, name_len); if (!key) return -ENOMEM; - scoutfs_kvec_init(val, &dent, sizeof(dent)); + kvec_init(&val, &dent, sizeof(dent)); - ret = scoutfs_item_lookup_exact(sb, key, val, lock); + ret = scoutfs_item_lookup_exact(sb, key, &val, lock); if (ret == 0 && le64_to_cpu(dent.ino) != ino) ret = -ENOENT; else if (ret == -ENOENT && ino == 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 20ae32fd..a5ae0d0a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -266,7 +266,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, struct scoutfs_key_buf key; struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; const u64 refresh_gen = lock->refresh_gen; int ret; @@ -282,11 +282,11 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, return 0; scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + kvec_init(&val, &sinode, sizeof(sinode)); mutex_lock(&si->item_mutex); if (atomic64_read(&si->last_refreshed) < refresh_gen) { - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); @@ -909,7 +909,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, struct scoutfs_inode_key ikey; struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; int ret; int err; @@ -925,9 +925,9 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, BUG_ON(ret); scoutfs_inode_init_key(&key, &ikey, ino); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + kvec_init(&val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, &key, val, lock); + err = scoutfs_item_update(sb, &key, &val, lock); if (err) { scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); @@ -1314,8 +1314,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, struct scoutfs_inode_key ikey; struct scoutfs_key_buf key; struct scoutfs_inode sinode; - SCOUTFS_DECLARE_KVEC(val); struct inode *inode; + struct kvec val; int ret; inode = new_inode(sb); @@ -1347,9 +1347,9 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, store_inode(&sinode, inode); scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_item_create(sb, &key, val, lock); + ret = scoutfs_item_create(sb, &key, &val, lock); if (ret) { iput(inode); return ERR_PTR(ret); @@ -1400,9 +1400,9 @@ static int delete_inode_items(struct super_block *sb, u64 ino) struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); LIST_HEAD(ind_locks); bool release = false; + struct kvec val; umode_t mode; u64 ind_seq; int ret; @@ -1412,9 +1412,9 @@ static int delete_inode_items(struct super_block *sb, u64 ino) return ret; scoutfs_inode_init_key(&key, &ikey, ino); - scoutfs_kvec_init(val, &sinode, sizeof(sinode)); + kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/item.c b/kmod/src/item.c index d0454552..fccc49c7 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -44,7 +44,7 @@ static bool invalid_key_val(struct scoutfs_key_buf *key, struct kvec *val) { return WARN_ON_ONCE(key->key_len > SCOUTFS_MAX_KEY_SIZE || - (val && (scoutfs_kvec_length(val) > SCOUTFS_MAX_VAL_SIZE))); + (val && (val->iov_len > SCOUTFS_MAX_VAL_SIZE))); } struct item_cache { @@ -79,8 +79,8 @@ struct cached_item { unsigned deletion:1; struct scoutfs_key_buf *key; - - SCOUTFS_DECLARE_KVEC(val); + void *val; + unsigned int val_len; }; struct cached_range { @@ -105,11 +105,16 @@ static void free_item(struct super_block *sb, struct cached_item *item) WARN_ON_ONCE(!list_empty(&item->entry)); WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); scoutfs_key_free(sb, item->key); - scoutfs_kvec_kfree(item->val); + kfree(item->val); kfree(item); } } +/* + * The value vec may be null if the item has no value. Values are + * allocated separately so that we can free them when deleting or swap + * them in place when updating items. + */ static struct cached_item *alloc_item(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) @@ -121,12 +126,15 @@ static struct cached_item *alloc_item(struct super_block *sb, RB_CLEAR_NODE(&item->node); INIT_LIST_HEAD(&item->entry); - if (!val) - scoutfs_kvec_init_null(item->val); - item->key = scoutfs_key_dup(sb, key); - if (!item->key || - (val && scoutfs_kvec_dup_flatten(item->val, val))) { + if (val) { + item->val = kmalloc(val->iov_len, GFP_NOFS); + item->val_len = val->iov_len; + if (item->val) + memcpy(item->val, val->iov_base, val->iov_len); + } + + if (!item->key || (val && !item->val)) { free_item(sb, item); item = NULL; } @@ -138,6 +146,25 @@ static struct cached_item *alloc_item(struct super_block *sb, return item; } +/* + * Copy the cached item's value into the caller's single value vector. + * The number of bytes that fit in the vec and were copied is returned. + * A null val returns 0. + */ +static int copy_item_val(struct kvec *val, struct cached_item *item) +{ + int ret; + + if (val) { + ret = min_t(size_t, item->val_len, val->iov_len); + memcpy(val->iov_base, item->val, ret); + } else { + ret = 0; + } + + return ret; +} + /* * Walk the item rbtree and return the item found and the next and * prev items. @@ -340,9 +367,7 @@ static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, list_del_init(&item->entry); cac->lru_nr--; - update_dirty_item_counts(sb, 1, item->key->key_len, - scoutfs_kvec_length(item->val)); - + update_dirty_item_counts(sb, 1, item->key->key_len, item->val_len); update_dirty_parents(item); } @@ -359,8 +384,7 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, list_add_tail(&item->entry, &cac->lru_list); cac->lru_nr++; - update_dirty_item_counts(sb, -1, -item->key->key_len, - -scoutfs_kvec_length(item->val)); + update_dirty_item_counts(sb, -1, -item->key->key_len, -item->val_len); WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || cac->dirty_val_bytes < 0); @@ -408,9 +432,13 @@ static void become_deletion_item(struct super_block *sb, struct item_cache *cac, struct cached_item *item) { + /* uses val_len to update item accounting */ clear_item_dirty(sb, cac, item); - scoutfs_kvec_kfree(item->val); - scoutfs_kvec_init_null(item->val); + + kfree(item->val); + item->val = NULL; + item->val_len = 0; + item->deletion = 1; mark_item_dirty(sb, cac, item); scoutfs_inc_counter(sb, item_delete); @@ -788,7 +816,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, if (item) { item_referenced(cac, item); if (val) - ret = scoutfs_kvec_memcpy(val, item->val); + ret = copy_item_val(val, item); else ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { @@ -824,11 +852,10 @@ int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val, struct scoutfs_lock *lock) { - int size = scoutfs_kvec_length(val); int ret; ret = scoutfs_item_lookup(sb, key, val, lock); - if (ret == size) + if (ret == val->iov_len) ret = 0; else if (ret >= 0) ret = -EIO; @@ -994,7 +1021,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, scoutfs_key_copy(key, item->key); if (val) { item_referenced(cac, item); - ret = scoutfs_kvec_memcpy(val, item->val); + ret = copy_item_val(val, item); } else { ret = 0; } @@ -1027,7 +1054,7 @@ int scoutfs_item_next_same_min(struct super_block *sb, trace_scoutfs_item_next_same_min(sb, key_len, len); - if (WARN_ON_ONCE(!val || scoutfs_kvec_length(val) < len)) + if (WARN_ON_ONCE(!val || val->iov_len < len)) return -EINVAL; ret = scoutfs_item_next(sb, key, last, val, lock); @@ -1306,9 +1333,9 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - SCOUTFS_DECLARE_KVEC(up_val); struct cached_item *item; unsigned long flags; + void *up_val = NULL; int ret; if (invalid_key_val(key, val)) @@ -1318,11 +1345,12 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, return -EINVAL; if (val) { - ret = scoutfs_kvec_dup_flatten(up_val, val); - if (ret) + up_val = kmalloc(val->iov_len, GFP_NOFS); + if (!up_val) { + ret = -ENOMEM; goto out; - } else { - scoutfs_kvec_init_null(up_val); + } + memcpy(up_val, val->iov_base, val->iov_len); } do { @@ -1331,7 +1359,8 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, item = find_item(sb, &cac->items, key); if (item) { clear_item_dirty(sb, cac, item); - scoutfs_kvec_swap(up_val, item->val); + swap(up_val, item->val); + item->val_len = val ? val->iov_len : 0; mark_item_dirty(sb, cac, item); ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { @@ -1346,7 +1375,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, (ret = scoutfs_manifest_read_items(sb, key, lock->start, lock->end)) == 0); out: - scoutfs_kvec_kfree(up_val); + kfree(up_val); trace_scoutfs_item_update_ret(sb, ret); return ret; @@ -1412,7 +1441,6 @@ int scoutfs_item_delete_force(struct super_block *sb, if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) return -EINVAL; - item = alloc_item(sb, key, NULL); if (!item) return -ENOMEM; @@ -1568,7 +1596,6 @@ void scoutfs_item_delete_dirty(struct super_block *sb, struct cached_item *item; unsigned long flags; - spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); @@ -1581,7 +1608,9 @@ void scoutfs_item_delete_dirty(struct super_block *sb, /* * Copy the callers value into the dirty item and truncate its value if * the existing value is longer. The caller must have ensured that the - * item was dirty and had a large enough value. + * item was dirty and had a large enough value. If the updated value is + * smaller then it will sit in the larger item allocation until the + * value is eventually freed along with the item. */ void scoutfs_item_update_dirty(struct super_block *sb, struct scoutfs_key_buf *key, struct kvec *val) @@ -1590,17 +1619,19 @@ void scoutfs_item_update_dirty(struct super_block *sb, struct item_cache *cac = sbi->item_cache; struct cached_item *item; unsigned long flags; + unsigned int new_len = val ? val->iov_len : 0; signed delta; spin_lock_irqsave(&cac->lock, flags); item = find_item(sb, &cac->items, key); - BUG_ON(!item || !item_is_dirty(item) || - scoutfs_kvec_length(val) > scoutfs_kvec_length(item->val)); + BUG_ON(!item || !item_is_dirty(item) || new_len > item->val_len); - delta = scoutfs_kvec_length(val) - scoutfs_kvec_length(item->val); - scoutfs_kvec_memcpy_truncate(item->val, val); + delta = new_len - item->val_len; + if (val) + memcpy(item->val, val->iov_base, new_len); + item->val_len = new_len; update_dirty_item_counts(sb, 0, 0, delta); spin_unlock_irqrestore(&cac->lock, flags); @@ -1767,13 +1798,15 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) struct cached_item *item = NULL; struct cached_item *del; unsigned long flags; + struct kvec val; bool appended; spin_lock_irqsave(&cac->lock, flags); item = first_dirty(cac->items.rb_node); while (item) { - appended = scoutfs_seg_append_item(sb, seg, item->key, item->val, + kvec_init(&val, item->val, item->val_len); + appended = scoutfs_seg_append_item(sb, seg, item->key, &val, item_flags(item), links); /* trans reservation should have limited dirty */ BUG_ON(!appended); @@ -2064,7 +2097,7 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, unlink_item(sb, cac, item); key = item->key; - scoutfs_kvec_kfree(item->val); + kfree(item->val); nr++; new_rng = (void *)item; diff --git a/kmod/src/kvec.c b/kmod/src/kvec.c deleted file mode 100644 index 21d7d68e..00000000 --- a/kmod/src/kvec.c +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "inode.h" -#include "dir.h" -#include "xattr.h" -#include "msg.h" -#include "counters.h" -#include "trans.h" -#include "kvec.h" -#include "scoutfs_trace.h" - -struct iter { - struct kvec *kvec; - size_t count; - size_t off; - size_t i; -}; - -static void iter_advance(struct iter *iter, size_t len) -{ - iter->off += len; - iter->count -= len; - - while (iter->i < SCOUTFS_KVEC_NR && iter->off >= iter->kvec->iov_len) { - iter->off -= iter->kvec->iov_len; - iter->kvec++; - iter->i++; - } -} - -static void iter_init(struct iter *iter, struct kvec *kvec) -{ - iter->kvec = kvec; - iter->i = 0; - iter->off = 0; - iter->count = scoutfs_kvec_length(kvec); - - iter_advance(iter, 0); -} - -static void *iter_ptr(struct iter *iter) -{ - if (iter->i < SCOUTFS_KVEC_NR) - return iter->kvec->iov_base + iter->off; - else - return NULL; -} - -/* count of contiguous bytes available at the next vector */ -static size_t iter_contig(struct iter *iter) -{ - if (iter->i < SCOUTFS_KVEC_NR) - return iter->kvec->iov_len - iter->off; - else - return 0; -} - -/* - * Return the result of memcmp between the min of the two total lengths. - * If their shorter lengths are equal than the shorter length is considered - * smaller than the longer. - */ -int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b) -{ - struct iter a_iter; - struct iter b_iter; - size_t len; - int ret; - - iter_init(&a_iter, a); - iter_init(&b_iter, b); - - while ((len = min(iter_contig(&a_iter), iter_contig(&b_iter)))) { - ret = memcmp(iter_ptr(&a_iter), iter_ptr(&b_iter), len); - if (ret) - return ret; - - iter_advance(&a_iter, len); - iter_advance(&b_iter, len); - } - - return iter_contig(&a_iter) ? 1 : iter_contig(&b_iter) ? -1 : 0; -} - -/* - * Return -1 if [a,b] doesn't overlap with and is to the left of [c,d], - * 1 if it doesn't overlap and is to the right of, and 0 if they - * overlap. - */ -int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, - struct kvec *c, struct kvec *d) -{ - return scoutfs_kvec_memcmp(b, c) < 0 ? -1 : - scoutfs_kvec_memcmp(a, d) > 0 ? 1 : 0; -} - -/* - * Set just the pointers and length fields in the dst vector to point to - * the source vector. - */ -void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src) -{ - int i; - - for (i = 0; i < SCOUTFS_KVEC_NR; i++) - dst[i] = src[i]; -} - -/* - * Copy as much of src as fits in dst. Null base pointers termintae the - * copy. The number of bytes copied is returned. Only the buffers - * pointed to by dst are changed, the kvec elements are not changed. - */ -int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src) -{ - struct iter dst_iter; - struct iter src_iter; - size_t copied = 0; - size_t len; - - iter_init(&dst_iter, dst); - iter_init(&src_iter, src); - - while ((len = min(iter_contig(&dst_iter), iter_contig(&src_iter)))) { - memcpy(iter_ptr(&dst_iter), iter_ptr(&src_iter), len); - - copied += len; - iter_advance(&dst_iter, len); - iter_advance(&src_iter, len); - } - - return copied; -} - -/* - * Copy bytes in src into dst, stopping if dst is full. The number of copied - * bytes is returned and the lengths of dst are updated if the size changes. - * The pointers in dst are not changed. - */ -int scoutfs_kvec_memcpy_truncate(struct kvec *dst, struct kvec *src) -{ - int copied = scoutfs_kvec_memcpy(dst, src); - size_t bytes; - int i; - - if (copied < scoutfs_kvec_length(dst)) { - bytes = copied; - for (i = 0; i < SCOUTFS_KVEC_NR; i++) { - dst[i].iov_len = min(dst[i].iov_len, bytes); - bytes -= dst[i].iov_len; - } - } - - return copied; -} - -/* - * Copy the src key vector into one new allocation in the dst. The existing - * dst is clobbered. The source isn't changed. - */ -int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src) -{ - void *ptr; - size_t len = scoutfs_kvec_length(src); - - ptr = kmalloc(len, GFP_NOFS); - if (!ptr) { - scoutfs_kvec_init_null(dst); - return -ENOMEM; - } - - scoutfs_kvec_init(dst, ptr, len); - scoutfs_kvec_memcpy(dst, src); - return 0; -} - -/* - * Free all the set pointers in the kvec. - */ -void scoutfs_kvec_kfree(struct kvec *kvec) -{ - int i; - - for (i = 0; i < SCOUTFS_KVEC_NR; i++) { - kfree(kvec[i].iov_base); - kvec[i].iov_base = NULL; - } -} - -void scoutfs_kvec_init_null(struct kvec *kvec) -{ - memset(kvec, 0, SCOUTFS_KVEC_BYTES); -} - -void scoutfs_kvec_swap(struct kvec *a, struct kvec *b) -{ - SCOUTFS_DECLARE_KVEC(tmp); - - memcpy(tmp, a, SCOUTFS_KVEC_BYTES); - memcpy(a, b, SCOUTFS_KVEC_BYTES); - memcpy(b, tmp, SCOUTFS_KVEC_BYTES); -} - -int scoutfs_kvec_alloc_key(struct kvec *kvec) -{ - const size_t len = SCOUTFS_MAX_KEY_SIZE; - void *ptr; - - ptr = kzalloc(len, GFP_NOFS); - if (!ptr) { - scoutfs_kvec_init_null(kvec); - return -ENOMEM; - } - - scoutfs_kvec_init(kvec, ptr, len); - return 0; -} - -void scoutfs_kvec_init_key(struct kvec *kvec) -{ - scoutfs_kvec_init(kvec, kvec[0].iov_base, SCOUTFS_MAX_KEY_SIZE); -} - -void scoutfs_kvec_set_max_key(struct kvec *kvec) -{ - __u8 *type = kvec[0].iov_base; - - *type = 255; - scoutfs_kvec_init(kvec, type, 1); -} - -/* - * Increase the kvec as though it is a big endian value. Carry - * increments of the least significant byte as long as it wraps. - */ -void scoutfs_kvec_be_inc(struct kvec *kvec) -{ - int i; - int b; - - for (i = SCOUTFS_KVEC_NR - 1; i >= 0; i--) { - for (b = (int)kvec[i].iov_len - 1; b >= 0; b--) { - if (++((u8 *)kvec[i].iov_base)[b]) - return; - } - } -} - -void scoutfs_kvec_be_dec(struct kvec *kvec) -{ - int i; - int b; - - for (i = SCOUTFS_KVEC_NR - 1; i >= 0; i--) { - for (b = (int)kvec[i].iov_len - 1; b >= 0; b--) { - if (--((u8 *)kvec[i].iov_base)[b] != 0xff) - return; - } - } -} - -/* - * Clone the source kvec into the dst if the dst is empty or if - * the src kvec is less than the dst. - */ -void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src) -{ - if (scoutfs_kvec_length(dst) == 0 || - scoutfs_kvec_memcmp(src, dst) < 0) - scoutfs_kvec_clone(dst, src); -} diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h index c078e802..9341f724 100644 --- a/kmod/src/kvec.h +++ b/kmod/src/kvec.h @@ -3,70 +3,10 @@ #include -/* - * The item APIs use kvecs to represent variable size item keys and - * values. - */ - -/* - * This ends up defining the max item size as nr - 1 * page _size. - */ -#define SCOUTFS_KVEC_NR 2 -#define SCOUTFS_KVEC_BYTES (SCOUTFS_KVEC_NR * sizeof(struct kvec)) - -#define SCOUTFS_DECLARE_KVEC(name) \ - struct kvec name[SCOUTFS_KVEC_NR] - -static inline void scoutfs_kvec_init_all(struct kvec *kvec, - void *ptr0, size_t len0, - void *ptr1, size_t len1, - void *ptr2, ...) +static inline void kvec_init(struct kvec *kv, void *base, size_t len) { - BUG_ON(ptr2 != NULL); - - kvec[0].iov_base = ptr0; - kvec[0].iov_len = len0; - kvec[1].iov_base = ptr1; - kvec[1].iov_len = len1; + kv->iov_base = base; + kv->iov_len = len; } -/* - * Provide a nice variadic initialization function without having to - * iterate over the callers arg types. We play some macro games to pad - * out the callers ptr/len pairs to the full possible number. This will - * produce confusing errors if an odd number of arguments is given and - * the padded ptr/length types aren't compatible with the fixed - * arguments in the static inline. - */ -#define scoutfs_kvec_init(val, ...) \ - scoutfs_kvec_init_all(val, __VA_ARGS__, NULL, 0, NULL, 0) - -static inline int scoutfs_kvec_length(struct kvec *kvec) -{ - BUILD_BUG_ON(sizeof(struct kvec) != sizeof(struct iovec)); - BUILD_BUG_ON(offsetof(struct kvec, iov_len) != - offsetof(struct iovec, iov_len)); - BUILD_BUG_ON(member_sizeof(struct kvec, iov_len) != - member_sizeof(struct iovec, iov_len)); - - return iov_length((struct iovec *)kvec, SCOUTFS_KVEC_NR); -} - -void scoutfs_kvec_clone(struct kvec *dst, struct kvec *src); -int scoutfs_kvec_memcmp(struct kvec *a, struct kvec *b); -int scoutfs_kvec_cmp_overlap(struct kvec *a, struct kvec *b, - struct kvec *c, struct kvec *d); -int scoutfs_kvec_memcpy(struct kvec *dst, struct kvec *src); -int scoutfs_kvec_memcpy_truncate(struct kvec *dst, struct kvec *src); -int scoutfs_kvec_dup_flatten(struct kvec *dst, struct kvec *src); -void scoutfs_kvec_kfree(struct kvec *kvec); -void scoutfs_kvec_init_null(struct kvec *kvec); -void scoutfs_kvec_swap(struct kvec *a, struct kvec *b); -int scoutfs_kvec_alloc_key(struct kvec *kvec); -void scoutfs_kvec_init_key(struct kvec *kvec); -void scoutfs_kvec_set_max_key(struct kvec *kvec); -void scoutfs_kvec_clone_less(struct kvec *dst, struct kvec *src); -void scoutfs_kvec_be_inc(struct kvec *kvec); -void scoutfs_kvec_be_dec(struct kvec *kvec); - #endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 80e00c11..2f00a7e3 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -640,12 +640,12 @@ int scoutfs_manifest_read_items(struct super_block *sb, struct scoutfs_key_buf seg_start; struct scoutfs_key_buf seg_end; struct scoutfs_btree_root root; - SCOUTFS_DECLARE_KVEC(item_val); - SCOUTFS_DECLARE_KVEC(found_val); struct scoutfs_segment *seg; struct manifest_ref *ref; struct manifest_ref *tmp; __le64 last_root_seq; + struct kvec found_val; + struct kvec item_val; LIST_HEAD(ref_list); LIST_HEAD(batch); u8 found_flags = 0; @@ -753,7 +753,7 @@ retry_stale: * that our segments can see. */ ret = scoutfs_seg_item_ptrs(ref->seg, ref->off, - &item_key, item_val, + &item_key, &item_val, &item_flags); if (ret < 0 || scoutfs_key_compare(&item_key, &seg_end) > 0) { @@ -774,7 +774,7 @@ retry_stale: /* remember new least key */ scoutfs_key_clone(&found_key, &item_key); - scoutfs_kvec_clone(found_val, item_val); + found_val = item_val; found_flags = item_flags; ref->found_ctr = ++found_ctr; found = true; @@ -798,7 +798,7 @@ retry_stale: */ if (!(found_flags & SCOUTFS_ITEM_FLAG_DELETION)) { ret = scoutfs_item_add_batch(sb, &batch, &found_key, - found_val); + &found_val); if (ret) { if (added) ret = 0; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 92071f62..a761f468 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -28,7 +28,6 @@ #include "key.h" #include "format.h" -#include "kvec.h" #include "lock.h" #include "seg.h" #include "super.h" diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 645a3134..6e116d11 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -439,9 +439,10 @@ static void item_ptrs(struct scoutfs_segment *seg, int off, if (key) scoutfs_key_init(key, item_key_ptr(item), le16_to_cpu(item->key_len)); - if (val) - scoutfs_kvec_init(val, item_val_ptr(item), - le16_to_cpu(item->val_len)); + if (val) { + val->iov_base = item_val_ptr(item); + val->iov_len = le16_to_cpu(item->val_len); + } } static void first_last_keys(struct scoutfs_segment *seg, @@ -645,14 +646,14 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; struct scoutfs_key_buf item_key; - SCOUTFS_DECLARE_KVEC(item_val); + struct kvec item_val; u8 nr_links; u32 val_len; u32 bytes; u32 off; int i; - val_len = scoutfs_kvec_length(val); + val_len = val ? val->iov_len : 0; /* initialize the segment and skip links as the first item is appended */ if (sblk->nr_items == 0) { @@ -701,9 +702,10 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg links[i] = &item->skip_links[i]; } - item_ptrs(seg, off, &item_key, item_val); + item_ptrs(seg, off, &item_key, &item_val); scoutfs_key_copy(&item_key, key); - scoutfs_kvec_memcpy(item_val, val); + if (val_len) + memcpy(item_val.iov_base, val->iov_base, val_len); return true; } diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 727d8e43..cd6df413 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -117,7 +117,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, struct scoutfs_xattr_key last_xak; struct scoutfs_key_buf last; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); + struct kvec val; u8 last_part; int total; u8 part; @@ -140,8 +140,8 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, for (;;) { xak->part = part; - scoutfs_kvec_init(val, (void *)xat + total, bytes - total); - ret = scoutfs_item_next(sb, &key, &last, val, lock); + kvec_init(&val, (void *)xat + total, bytes - total); + ret = scoutfs_item_next(sb, &key, &last, &val, lock); if (ret < 0) { /* XXX corruption, ran out of parts */ if (ret == -ENOENT && part > 0) @@ -216,8 +216,8 @@ static int create_xattr_items(struct inode *inode, u64 id, struct super_block *sb = inode->i_sb; struct scoutfs_xattr_key xak; struct scoutfs_key_buf key; - SCOUTFS_DECLARE_KVEC(val); unsigned int part_bytes; + struct kvec val; int total; int ret; @@ -228,9 +228,9 @@ static int create_xattr_items(struct inode *inode, u64 id, ret = 0; while (total < bytes) { part_bytes = min(bytes - total, SCOUTFS_XATTR_MAX_PART_SIZE); - scoutfs_kvec_init(val, (void *)xat + total, part_bytes); + kvec_init(&val, (void *)xat + total, part_bytes); - ret = scoutfs_item_create(sb, &key, val, lock); + ret = scoutfs_item_create(sb, &key, &val, lock); if (ret) { while (xak.part-- > 0) scoutfs_item_delete_dirty(sb, &key); From 0bfc4b72c5437fd13ee8b2438a31c405e4750d17 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jan 2018 11:54:22 -0800 Subject: [PATCH 585/920] scoutfs: fix old comment in item.c Signed-off-by: Zach Brown --- kmod/src/item.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index fccc49c7..af78d0f2 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1040,8 +1040,7 @@ out: /* * Like _next but requires that the found keys be the same length as the * search key and that values be of at least a minimum size. It treats - * size mismatches as a sign of corruption. A found key larger than the - * found key buffer gives -ENOBUFS and is a sign of corruption. + * size mismatches as a sign of corruption and returns -EIO. */ int scoutfs_item_next_same_min(struct super_block *sb, struct scoutfs_key_buf *key, From df6a8af71f2be54c390411be86179bc736c52c3c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jan 2018 11:54:51 -0800 Subject: [PATCH 586/920] scoutfs: remove name from dirent keys Directory entries were the last items that had large variable length keys because they stored the entry name in the key. We'd like to have small fixed size keys so let's store dirents with small keys. Entries for lookup are stored at the hash of the name instead of the full name. The key also contains the unique readdir pos so that we don't have to deal with collision on creation. The lookup procedure now does need to iterate over all the readdir positions for the hash value and compare the names. Entries for link backref walking are stored with the entry's position in the parent dir instead of the entry's name. The name is then stored in the value. Inode to path conversion can still walk the backref items without having to lookup dirent items. These changes mean that all directory entry items are now stored at a small key with some u64s (hash, pos, parent dir, etc) and have a value with the dirent struct and full entry name. This lets us use the same key and value format for the three entry key types. We no longer have to allocate keys, we can store them on the stack. We store the entry's hash and pos in the dirent struct in the item value so that any item has all the fields to reference all the other item keys. We store the same values in the dentry_info so that deletion (unlink and rename) can find all the entries. The ino_path ioctl can now much more clearly iterate over parent directories and entry positions instead of oh so cleverly iterating over null terminated names in the parent directories. The ioctl interface structs and implementation become simpler. Signed-off-by: Zach Brown --- kmod/src/count.h | 9 +- kmod/src/dir.c | 519 +++++++++++++++++++++++----------------------- kmod/src/dir.h | 12 +- kmod/src/export.c | 13 +- kmod/src/format.h | 37 ++-- kmod/src/ioctl.c | 95 +++------ kmod/src/ioctl.h | 69 ++++-- kmod/src/key.c | 38 +--- 8 files changed, 375 insertions(+), 417 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 30b55ca5..f115d034 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -71,17 +71,14 @@ static inline const struct scoutfs_item_count SIC_DIRTY_INODE(void) } /* - * Adding a dirent adds the entry key, readdir key, and backref. + * Directory entries are stored in three items. */ static inline void __count_dirents(struct scoutfs_item_count *cnt, unsigned name_len) { - cnt->items += 3; - cnt->keys += offsetof(struct scoutfs_dirent_key, name[name_len]) + - sizeof(struct scoutfs_readdir_key) + - offsetof(struct scoutfs_link_backref_key, name[name_len]); - cnt->vals += 2 * offsetof(struct scoutfs_dirent, name[name_len]); + cnt->keys += 3 * sizeof(struct scoutfs_dirent_key); + cnt->vals += 3 * offsetof(struct scoutfs_dirent, name[name_len]); } static inline void __count_sym_target(struct scoutfs_item_count *cnt, diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 3f980737..f32dfe6c 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -35,34 +35,31 @@ #include "scoutfs_trace.h" /* - * Directory entries are stored in entries with offsets calculated from - * the hash of their entry name. + * Directory entries are stored in three different items. Each has the + * same key format and all have identical values which contain the full + * entry name. * - * Having a single index of items used for both lookup and readdir - * iteration reduces the storage overhead of directories. It also - * avoids having to manage the allocation of readdir positions as - * directories age and the aggregate create count inches towards the - * small 31 bit position limit. The downside is that dirent name - * operations produce random item access patterns. + * Entries for name lookup are stored at the hash of the name and the + * readdir position. Including the position lets us create names + * without having to read the items to check for hash collisions. + * Lookup iterates over all the positions with the same hash values and + * compares the names. * - * Hash values are limited to 31 bits primarily to support older - * deployed protocols that only support 31 bits of file entry offsets, - * but also to avoid unlikely bugs in programs that store offsets in - * signed ints. + * Entries for readdir are stored in an increasing unique readdir + * position. This results in returning entries in creation order which + * matches inode allocation order and avoids random inode access + * patterns during readdir. * - * We have to worry about hash collisions. We linearly probe a fixed - * number of hash values past the natural value. In a typical small - * directory this search will terminate immediately because adjacent - * items will have distant offset values. It's only as the directory - * gets very large that hash values will start to be this dense and - * sweeping over items in a btree leaf is reasonably efficient. + * Entries for link backref traversal are stored at the target inode + * sorted by the parent dir and the entry's position in the parent dir. + * This keeps link backref users away from the higher contention area of + * dirent items in parent dirs. * - * For each directory entry item stored in a directory inode there is a - * corresponding link backref item stored at the target inode. This - * lets us find all the paths that refer to a given inode. The link - * backref offset comes from an advancing counter in the inode and the - * item value contains the dir inode and dirent offset of the referring - * link. + * All the entries have a dirent struct with the full name in their + * value. The dirent struct contains the name hash and readdir position + * so that any item use can reference all the items for a given entry. + * This is important for deleting all the items given a dentry that was + * populated by lookup. */ static unsigned int mode_to_type(umode_t mode) @@ -102,14 +99,15 @@ static unsigned int dentry_type(unsigned int type) } /* - * @readdir_pos lets us remove items on final unlink without having to - * look them up. + * @lock_cov: tells revalidation that the dentry is still locked and valid. * - * @lock_cov tells revalidation that the dentry is still locked and valid. + * @pos, @hash: lets us remove items on final unlink without having to + * look them up. */ struct dentry_info { - u64 readdir_pos; struct scoutfs_lock_coverage lock_cov; + u64 hash; + u64 pos; }; static struct kmem_cache *dentry_info_cache; @@ -161,15 +159,26 @@ static int alloc_dentry_info(struct dentry *dentry) } static void update_dentry_info(struct super_block *sb, struct dentry *dentry, - u64 pos, struct scoutfs_lock *lock) + u64 hash, u64 pos, struct scoutfs_lock *lock) { struct dentry_info *di = dentry->d_fsdata; if (WARN_ON_ONCE(di == NULL)) return; - di->readdir_pos = pos; scoutfs_lock_add_coverage(sb, lock, &di->lock_cov); + di->hash = hash; + di->pos = pos; +} + +static u64 dentry_info_hash(struct dentry *dentry) +{ + struct dentry_info *di = dentry->d_fsdata; + + if (WARN_ON_ONCE(di == NULL)) + return 0; + + return di->hash; } static u64 dentry_info_pos(struct dentry *dentry) @@ -179,88 +188,102 @@ static u64 dentry_info_pos(struct dentry *dentry) if (WARN_ON_ONCE(di == NULL)) return 0; - return di->readdir_pos; + return di->pos; } -static struct scoutfs_key_buf *alloc_dirent_key(struct super_block *sb, - u64 dir_ino, const char *name, - unsigned name_len) +static void init_dirent_key(struct scoutfs_key_buf *key, + struct scoutfs_dirent_key *dkey, u8 type, + u64 ino, u64 major, u64 minor) { - struct scoutfs_dirent_key *dkey; - struct scoutfs_key_buf *key; + dkey->zone = SCOUTFS_FS_ZONE; + dkey->ino = cpu_to_be64(ino); + dkey->type = type; + dkey->major = cpu_to_be64(major); + dkey->minor = cpu_to_be64(minor); - key = scoutfs_key_alloc(sb, offsetof(struct scoutfs_dirent_key, - name[name_len])); - if (key) { - dkey = key->data; - dkey->zone = SCOUTFS_FS_ZONE; - dkey->ino = cpu_to_be64(dir_ino); - dkey->type = SCOUTFS_DIRENT_TYPE; - memcpy(dkey->name, (void *)name, name_len); - } - - return key; + scoutfs_key_init(key, dkey, sizeof(struct scoutfs_dirent_key)); } -static void init_link_backref_key(struct scoutfs_key_buf *key, - struct scoutfs_link_backref_key *lbrkey, - u64 ino, u64 dir_ino, - const char *name, unsigned name_len) +static unsigned int dirent_bytes(unsigned int name_len) { - lbrkey->zone = SCOUTFS_FS_ZONE; - lbrkey->ino = cpu_to_be64(ino); - lbrkey->type = SCOUTFS_LINK_BACKREF_TYPE; - lbrkey->dir_ino = cpu_to_be64(dir_ino); - if (name_len) - memcpy(lbrkey->name, name, name_len); - - scoutfs_key_init(key, lbrkey, offsetof(struct scoutfs_link_backref_key, - name[name_len])); + return offsetof(struct scoutfs_dirent, name[name_len]); } -static struct scoutfs_key_buf *alloc_link_backref_key(struct super_block *sb, - u64 ino, u64 dir_ino, - const char *name, - unsigned name_len) +static struct scoutfs_dirent *alloc_dirent(unsigned int name_len) { - struct scoutfs_link_backref_key *lbkey; - struct scoutfs_key_buf *key; + return kmalloc(dirent_bytes(name_len), GFP_NOFS); +} - key = scoutfs_key_alloc(sb, offsetof(struct scoutfs_link_backref_key, - name[name_len])); - if (key) { - lbkey = key->data; - init_link_backref_key(key, lbkey, ino, dir_ino, - name, name_len); - } +static u64 dirent_name_hash(const char *name, unsigned int name_len) +{ + unsigned int half = (name_len + 1) / 2; - return key; + return crc32c(~0, name, half) | + ((u64)crc32c(~0, name + name_len - half, half) << 32); +} + +static u64 dirent_names_equal(const char *a_name, unsigned int a_len, + const char *b_name, unsigned int b_len) +{ + return a_len == b_len && memcmp(a_name, b_name, a_len) == 0; } /* * Looks for the dirent item and fills the caller's dirent if it finds * it. Returns item lookup errors including -ENOENT if it's not found. */ -static int lookup_dirent(struct super_block *sb, struct inode *dir, - const char *name, unsigned name_len, - struct scoutfs_dirent *dent, +static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, + unsigned name_len, u64 hash, + struct scoutfs_dirent *dent_ret, struct scoutfs_lock *lock) { - struct scoutfs_key_buf *key = NULL; + struct scoutfs_dirent_key last_dkey; + struct scoutfs_dirent_key dkey; + struct scoutfs_key_buf last_key; + struct scoutfs_key_buf key; + struct scoutfs_dirent *dent = NULL; struct kvec val; int ret; - key = alloc_dirent_key(sb, scoutfs_ino(dir), name, name_len); - if (!key) { + dent = alloc_dirent(SCOUTFS_NAME_LEN); + if (!dent) { ret = -ENOMEM; goto out; } - kvec_init(&val, dent, sizeof(struct scoutfs_dirent)); + init_dirent_key(&key, &dkey, SCOUTFS_DIRENT_TYPE, + dir_ino, hash, 0); + init_dirent_key(&last_key, &last_dkey, SCOUTFS_DIRENT_TYPE, + dir_ino, hash, U64_MAX); + kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); + + for (;;) { + ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); + if (ret < 0) + break; + + ret -= sizeof(struct scoutfs_dirent); + /* XXX corruption */ + if (ret < 1 || ret > SCOUTFS_NAME_LEN) { + ret = -EIO; + goto out; + } + + if (dirent_names_equal(name, name_len, dent->name, ret)) { + *dent_ret = *dent; + ret = 0; + break; + } + + if (be64_to_cpu(dkey.minor) == U64_MAX) { + ret = -ENOENT; + break; + } + be64_add_cpu(&dkey.minor, 1); + } - ret = scoutfs_item_lookup_exact(sb, key, &val, lock); out: - scoutfs_key_free(sb, key); + kfree(dent); return ret; } @@ -318,18 +341,24 @@ static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) if (ret) goto out; - ret = lookup_dirent(sb, dir, dentry->d_name.name, dentry->d_name.len, + ret = lookup_dirent(sb, scoutfs_ino(dir), + dentry->d_name.name, dentry->d_name.len, + dirent_name_hash(dentry->d_name.name, + dentry->d_name.len), &dent, lock); - if (ret == -ENOENT) + if (ret == -ENOENT) { dent.ino = 0; - else if (ret < 0) + dent.hash = 0; + dent.pos = 0; + } else if (ret < 0) { goto out; + } dentry_ino = dentry->d_inode ? scoutfs_ino(dentry->d_inode) : 0; if ((dentry_ino == le64_to_cpu(dent.ino))) { - update_dentry_info(sb, dentry, le64_to_cpu(dent.readdir_pos), - lock); + update_dentry_info(sb, dentry, le64_to_cpu(dent.hash), + le64_to_cpu(dent.pos), lock); scoutfs_inc_counter(sb, dentry_revalidate_valid); ret = 1; } else { @@ -368,8 +397,11 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, struct scoutfs_dirent dent; struct inode *inode; u64 ino = 0; + u64 hash; int ret; + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + if (dentry->d_name.len > SCOUTFS_NAME_LEN) { ret = -ENAMETOOLONG; goto out; @@ -383,15 +415,15 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, if (ret) goto out; - ret = lookup_dirent(sb, dir, dentry->d_name.name, dentry->d_name.len, - &dent, dir_lock); + ret = lookup_dirent(sb, scoutfs_ino(dir), dentry->d_name.name, + dentry->d_name.len, hash, &dent, dir_lock); if (ret == -ENOENT) { ino = 0; ret = 0; } else if (ret == 0) { ino = le64_to_cpu(dent.ino); - update_dentry_info(sb, dentry, le64_to_cpu(dent.readdir_pos), - dir_lock); + update_dentry_info(sb, dentry, le64_to_cpu(dent.hash), + le64_to_cpu(dent.pos), dir_lock); } scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); @@ -428,18 +460,6 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) return 1; } -static void init_readdir_key(struct scoutfs_key_buf *key, - struct scoutfs_readdir_key *rkey, u64 dir_ino, - loff_t pos) -{ - rkey->zone = SCOUTFS_FS_ZONE; - rkey->ino = cpu_to_be64(dir_ino); - rkey->type = SCOUTFS_READDIR_TYPE; - rkey->pos = cpu_to_be64(pos); - - scoutfs_key_init(key, rkey, sizeof(struct scoutfs_readdir_key)); -} - /* * readdir simply iterates over the dirent items for the dir inode and * uses their offset as the readdir position. @@ -454,10 +474,9 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct scoutfs_dirent *dent; struct scoutfs_key_buf key; struct scoutfs_key_buf last_key; - struct scoutfs_readdir_key rkey; - struct scoutfs_readdir_key last_rkey; + struct scoutfs_dirent_key dkey; + struct scoutfs_dirent_key last_dkey; struct scoutfs_lock *dir_lock; - unsigned int item_len; unsigned int name_len; struct kvec val; u64 pos; @@ -466,27 +485,26 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) if (!dir_emit_dots(file, dirent, filldir)) return 0; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &dir_lock); - if (ret) - return ret; - - init_readdir_key(&last_key, &last_rkey, scoutfs_ino(inode), - SCOUTFS_DIRENT_LAST_POS); - - item_len = offsetof(struct scoutfs_dirent, name[SCOUTFS_NAME_LEN]); - dent = kmalloc(item_len, GFP_KERNEL); + dent = alloc_dirent(SCOUTFS_NAME_LEN); if (!dent) { ret = -ENOMEM; goto out; } + init_dirent_key(&last_key, &last_dkey, SCOUTFS_READDIR_TYPE, + scoutfs_ino(inode), SCOUTFS_DIRENT_LAST_POS, 0); + kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); + + ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &dir_lock); + if (ret) + goto out; + for (;;) { - init_readdir_key(&key, &rkey, scoutfs_ino(inode), file->f_pos); + init_dirent_key(&key, &dkey, SCOUTFS_READDIR_TYPE, + scoutfs_ino(inode), file->f_pos, 0); - kvec_init(&val, dent, item_len); ret = scoutfs_item_next_same_min(sb, &key, &last_key, &val, - offsetof(struct scoutfs_dirent, name[1]), - dir_lock); + dirent_bytes(1), dir_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -494,7 +512,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } name_len = ret - sizeof(struct scoutfs_dirent); - pos = be64_to_cpu(rkey.pos); + pos = be64_to_cpu(dkey.major); if (filldir(dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) { @@ -519,69 +537,63 @@ out: * * If this returns an error then nothing will have changed. */ -static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, - const char *name, unsigned name_len, u64 ino, - umode_t mode, struct scoutfs_lock *dir_lock, +static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, + u64 pos, const char *name, unsigned name_len, + u64 ino, umode_t mode, struct scoutfs_lock *dir_lock, struct scoutfs_lock *inode_lock) { - struct scoutfs_key_buf *ent_key = NULL; - struct scoutfs_key_buf *lb_key = NULL; - struct scoutfs_dirent *dent = NULL; + struct scoutfs_dirent_key rdir_dkey; + struct scoutfs_dirent_key ent_dkey; + struct scoutfs_dirent_key lb_dkey; struct scoutfs_key_buf rdir_key; - struct scoutfs_readdir_key rkey; + struct scoutfs_key_buf ent_key; + struct scoutfs_key_buf lb_key; + struct scoutfs_dirent *dent; bool del_ent = false; bool del_rdir = false; struct kvec val; int ret; - ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); - dent = kmalloc(offsetof(struct scoutfs_dirent, name[name_len]), - GFP_NOFS); - if (!ent_key || !dent) { + dent = alloc_dirent(name_len); + if (!dent) { ret = -ENOMEM; goto out; } /* initialize the dent */ dent->ino = cpu_to_le64(ino); - dent->readdir_pos = cpu_to_le64(pos); + dent->hash = cpu_to_le64(hash); + dent->pos = cpu_to_le64(pos); dent->type = mode_to_type(mode); memcpy(dent->name, name, name_len); - /* dirent item for lookup */ - kvec_init(&val, dent, sizeof(struct scoutfs_dirent)); - ret = scoutfs_item_create(sb, ent_key, &val, dir_lock); + init_dirent_key(&ent_key, &ent_dkey, SCOUTFS_DIRENT_TYPE, + dir_ino, hash, pos); + init_dirent_key(&rdir_key, &rdir_dkey, SCOUTFS_READDIR_TYPE, + dir_ino, pos, 0); + init_dirent_key(&lb_key, &lb_dkey, SCOUTFS_LINK_BACKREF_TYPE, + ino, dir_ino, pos); + kvec_init(&val, dent, dirent_bytes(name_len)); + + ret = scoutfs_item_create(sb, &ent_key, &val, dir_lock); if (ret) goto out; del_ent = true; - /* readdir item for .. readdir */ - init_readdir_key(&rdir_key, &rkey, dir_ino, pos); - kvec_init(&val, dent, offsetof(struct scoutfs_dirent, name[name_len])); - ret = scoutfs_item_create(sb, &rdir_key, &val, dir_lock); if (ret) goto out; del_rdir = true; - /* link backref item for inode to path resolution */ - lb_key = alloc_link_backref_key(sb, ino, dir_ino, name, name_len); - if (!lb_key) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_item_create(sb, lb_key, NULL, inode_lock); + ret = scoutfs_item_create(sb, &lb_key, &val, inode_lock); out: if (ret < 0) { if (del_ent) - scoutfs_item_delete_dirty(sb, ent_key); + scoutfs_item_delete_dirty(sb, &ent_key); if (del_rdir) scoutfs_item_delete_dirty(sb, &rdir_key); } - scoutfs_key_free(sb, ent_key); - scoutfs_key_free(sb, lb_key); kfree(dent); return ret; @@ -592,49 +604,40 @@ out: * Only items are modified. The caller is responsible for locking, * entering a transaction, dirtying items, and managing the vfs structs. * - * The items match the items used in add_entry_items() but we don't have - * to worry about values here and we can dirty all the items before - * starting to delete them which makes cleanup a little easier. - * * If this returns an error then nothing will have changed. */ -static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 pos, - const char *name, unsigned name_len, u64 ino, - struct scoutfs_lock *dir_lock, +static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, + u64 pos, u64 ino, struct scoutfs_lock *dir_lock, struct scoutfs_lock *inode_lock) { - struct scoutfs_key_buf *ent_key; - struct scoutfs_key_buf *lb_key; + struct scoutfs_dirent_key rdir_dkey; + struct scoutfs_dirent_key ent_dkey; + struct scoutfs_dirent_key lb_dkey; struct scoutfs_key_buf rdir_key; - struct scoutfs_readdir_key rkey; + struct scoutfs_key_buf ent_key; + struct scoutfs_key_buf lb_key; + LIST_HEAD(dir_saved); + LIST_HEAD(inode_saved); int ret; - ent_key = alloc_dirent_key(sb, dir_ino, name, name_len); - if (!ent_key) - return -ENOMEM; + init_dirent_key(&ent_key, &ent_dkey, SCOUTFS_DIRENT_TYPE, + dir_ino, hash, pos); + init_dirent_key(&rdir_key, &rdir_dkey, SCOUTFS_READDIR_TYPE, + dir_ino, pos, 0); + init_dirent_key(&lb_key, &lb_dkey, SCOUTFS_LINK_BACKREF_TYPE, + ino, dir_ino, pos); - init_readdir_key(&rdir_key, &rkey, dir_ino, pos); - - lb_key = alloc_link_backref_key(sb, ino, dir_ino, name, name_len); - if (!lb_key) { - ret = -ENOMEM; - goto out; + ret = scoutfs_item_delete_save(sb, &ent_key, &dir_saved, dir_lock) ?: + scoutfs_item_delete_save(sb, &rdir_key, &dir_saved, dir_lock) ?: + scoutfs_item_delete_save(sb, &lb_key, &inode_saved, inode_lock); + if (ret < 0) { + scoutfs_item_restore(sb, &dir_saved, dir_lock); + scoutfs_item_restore(sb, &inode_saved, inode_lock); + } else { + scoutfs_item_free_batch(sb, &dir_saved); + scoutfs_item_free_batch(sb, &inode_saved); } - ret = scoutfs_item_dirty(sb, ent_key, dir_lock) ?: - scoutfs_item_dirty(sb, &rdir_key, dir_lock) ?: - scoutfs_item_dirty(sb, lb_key, inode_lock); - if (ret) - goto out; - - scoutfs_item_delete_dirty(sb, ent_key); - scoutfs_item_delete_dirty(sb, &rdir_key); - scoutfs_item_delete_dirty(sb, lb_key); - ret = 0; - -out: - kfree(ent_key); - kfree(lb_key); return ret; } @@ -724,12 +727,14 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; LIST_HEAD(ind_locks); + u64 hash; u64 pos; int ret; if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); inode = lock_hold_create(dir, dentry, mode, rdev, SIC_MKNOD(dentry->d_name.len), &dir_lock, &inode_lock, &ind_locks); @@ -738,13 +743,14 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, pos = SCOUTFS_I(dir)->next_readdir_pos++; - ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, - dentry->d_name.len, scoutfs_ino(inode), - inode->i_mode, dir_lock, inode_lock); + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); if (ret) goto out; - update_dentry_info(sb, dentry, pos, dir_lock); + update_dentry_info(sb, dentry, hash, pos, dir_lock); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -795,9 +801,12 @@ static int scoutfs_link(struct dentry *old_dentry, LIST_HEAD(ind_locks); u64 dir_size; u64 ind_seq; + u64 hash; u64 pos; int ret; + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; @@ -834,12 +843,13 @@ retry: pos = SCOUTFS_I(dir)->next_readdir_pos++; - ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, - dentry->d_name.len, scoutfs_ino(inode), - inode->i_mode, dir_lock, inode_lock); + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); if (ret) goto out; - update_dentry_info(sb, dentry, pos, dir_lock); + update_dentry_info(sb, dentry, hash, pos, dir_lock); i_size_write(dir, dir_size); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -908,9 +918,9 @@ retry: if (ret) goto unlock; - ret = del_entry_items(sb, scoutfs_ino(dir), dentry_info_pos(dentry), - dentry->d_name.name, dentry->d_name.len, - scoutfs_ino(inode), dir_lock, inode_lock); + ret = del_entry_items(sb, scoutfs_ino(dir), dentry_info_hash(dentry), + dentry_info_pos(dentry), scoutfs_ino(inode), + dir_lock, inode_lock); if (ret) goto out; @@ -1108,9 +1118,12 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, struct scoutfs_lock *dir_lock = NULL; struct scoutfs_lock *inode_lock = NULL; LIST_HEAD(ind_locks); + u64 hash; u64 pos; int ret; + hash = dirent_name_hash(dentry->d_name.name, dentry->d_name.len); + /* path_max includes null as does our value for nd_set_link */ if (dentry->d_name.len > SCOUTFS_NAME_LEN || name_len > PATH_MAX || name_len > SCOUTFS_SYMLINK_MAX_SIZE) @@ -1133,13 +1146,14 @@ static int scoutfs_symlink(struct inode *dir, struct dentry *dentry, pos = SCOUTFS_I(dir)->next_readdir_pos++; - ret = add_entry_items(sb, scoutfs_ino(dir), pos, dentry->d_name.name, - dentry->d_name.len, scoutfs_ino(inode), - inode->i_mode, dir_lock, inode_lock); + ret = add_entry_items(sb, scoutfs_ino(dir), hash, pos, + dentry->d_name.name, dentry->d_name.len, + scoutfs_ino(inode), inode->i_mode, dir_lock, + inode_lock); if (ret) goto out; - update_dentry_info(sb, dentry, pos, dir_lock); + update_dentry_info(sb, dentry, hash, pos, dir_lock); i_size_write(dir, i_size_read(dir) + dentry->d_name.len); dir->i_mtime = dir->i_ctime = CURRENT_TIME; @@ -1185,7 +1199,7 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino, /* * Find the next link backref key for the given ino starting from the - * given dir inode and null terminated name. If we find a backref item + * given dir inode and final entry position. If we find a backref item * we add an allocated copy of it to the head of the caller's list. * * Returns 0 if we added an entry, -ENOENT if we didn't, and -errno for @@ -1195,40 +1209,37 @@ int scoutfs_symlink_drop(struct super_block *sb, u64 ino, * building up a path with individual locked backref item lookups. */ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, - u64 dir_ino, char *name, unsigned int name_len, + u64 dir_ino, u64 dir_pos, struct list_head *list) { - struct scoutfs_link_backref_key last_lbkey; struct scoutfs_link_backref_entry *ent; - struct scoutfs_lock *lock = NULL; - struct scoutfs_key_buf last; + struct scoutfs_dirent_key last_dkey; + struct scoutfs_dirent_key dkey; + struct scoutfs_key_buf last_key; struct scoutfs_key_buf key; + struct scoutfs_lock *lock = NULL; + struct kvec val; int len; int ret; ent = kmalloc(offsetof(struct scoutfs_link_backref_entry, - lbkey.name[SCOUTFS_NAME_LEN + 1]), GFP_KERNEL); + dent.name[SCOUTFS_NAME_LEN]), GFP_KERNEL); if (!ent) return -ENOMEM; INIT_LIST_HEAD(&ent->head); - /* put search key in ent */ - init_link_backref_key(&key, &ent->lbkey, ino, dir_ino, name, name_len); - /* we actually have room for a full backref item */ - scoutfs_key_init_buf_len(&key, key.data, key.key_len, - offsetof(struct scoutfs_link_backref_key, - name[SCOUTFS_NAME_LEN + 1])); + init_dirent_key(&key, &dkey, SCOUTFS_LINK_BACKREF_TYPE, + ino, dir_ino, dir_pos); + init_dirent_key(&last_key, &last_dkey, SCOUTFS_LINK_BACKREF_TYPE, + ino, U64_MAX, U64_MAX); + kvec_init(&val, &ent->dent, dirent_bytes(SCOUTFS_NAME_LEN)); - /* small last key to avoid full name copy, XXX enforce no U64_MAX ino */ - init_link_backref_key(&last, &last_lbkey, ino, U64_MAX, NULL, 0); - - /* next backref key is now in ent */ ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); if (ret) goto out; - ret = scoutfs_item_next(sb, &key, &last, NULL, lock); + ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); scoutfs_unlock(sb, lock, DLM_LOCK_PR); lock = NULL; @@ -1236,15 +1247,17 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, if (ret < 0) goto out; - len = (int)key.key_len - sizeof(struct scoutfs_link_backref_key); + len = ret - sizeof(struct scoutfs_dirent); /* XXX corruption */ if (len < 1 || len > SCOUTFS_NAME_LEN) { ret = -EIO; goto out; } - ent->name_len = len; list_add(&ent->head, list); + ent->dir_ino = be64_to_cpu(dkey.major); + ent->dir_pos = be64_to_cpu(dkey.minor); + ent->name_len = len; ret = 0; out: if (list_empty(&ent->head)) @@ -1257,7 +1270,7 @@ static u64 first_backref_dir_ino(struct list_head *list) struct scoutfs_link_backref_entry *ent; ent = list_first_entry(list, struct scoutfs_link_backref_entry, head); - return be64_to_cpu(ent->lbkey.dir_ino); + return ent->dir_ino; } void scoutfs_dir_free_backref_path(struct super_block *sb, @@ -1310,8 +1323,7 @@ void scoutfs_dir_free_backref_path(struct super_block *sb, * sync if we see our dirty seq. */ int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, - char *name, u16 name_len, - struct list_head *list) + u64 dir_pos, struct list_head *list) { u64 par_ino; int ret; @@ -1323,15 +1335,14 @@ retry: * confident we won't hit an endless loop here again. */ if (WARN_ONCE(++iters >= 4000, "scoutfs: Excessive retries in " - "dir_get_backref_path. ino %llu dir_ino %llu name %.*s\n", - ino, dir_ino, name_len, name)) { + "dir_get_backref_path. ino %llu dir_ino %llu pos %llu\n", + ino, dir_ino, dir_pos)) { ret = -EINVAL; goto out; } /* get the next link name to the given inode */ - ret = scoutfs_dir_add_next_linkref(sb, ino, dir_ino, name, name_len, - list); + ret = scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, list); if (ret < 0) goto out; @@ -1339,8 +1350,7 @@ retry: par_ino = first_backref_dir_ino(list); while (par_ino != SCOUTFS_ROOT_INO) { - ret = scoutfs_dir_add_next_linkref(sb, par_ino, 0, NULL, 0, - list); + ret = scoutfs_dir_add_next_linkref(sb, par_ino, 0, 0, list); if (ret < 0) { if (ret == -ENOENT) { /* restart if there was no parent component */ @@ -1374,26 +1384,23 @@ static int item_d_ancestor(struct super_block *sb, u64 p1, u64 p2, u64 *p_ret) { struct scoutfs_link_backref_entry *ent; LIST_HEAD(list); - u64 dir_ino; int ret; u64 p; *p_ret = 0; - ret = scoutfs_dir_get_backref_path(sb, p2, 0, NULL, 0, &list); + ret = scoutfs_dir_get_backref_path(sb, p2, 0, 0, &list); if (ret) goto out; p = p2; list_for_each_entry(ent, &list, head) { - dir_ino = be64_to_cpu(ent->lbkey.dir_ino); - - if (dir_ino == p1) { + if (ent->dir_ino == p1) { *p_ret = p; ret = 0; break; } - p = dir_ino; + p = ent->dir_ino; } out: @@ -1434,27 +1441,18 @@ static int verify_ancestors(struct super_block *sb, u64 p1, u64 p2, * The caller has the name locked in the dir. */ static int verify_entry(struct super_block *sb, u64 dir_ino, const char *name, - unsigned name_len, u64 ino, + unsigned name_len, u64 hash, u64 ino, struct scoutfs_lock *lock) { - struct scoutfs_key_buf *key = NULL; struct scoutfs_dirent dent; - struct kvec val; int ret; - key = alloc_dirent_key(sb, dir_ino, name, name_len); - if (!key) - return -ENOMEM; - - kvec_init(&val, &dent, sizeof(dent)); - - ret = scoutfs_item_lookup_exact(sb, key, &val, lock); + ret = lookup_dirent(sb, dir_ino, name, name_len, hash, &dent, lock); if (ret == 0 && le64_to_cpu(dent.ino) != ino) ret = -ENOENT; else if (ret == -ENOENT && ino == 0) ret = 0; - scoutfs_key_free(sb, key); return ret; } @@ -1503,12 +1501,19 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, bool ins_old = false; LIST_HEAD(ind_locks); u64 ind_seq; + u64 old_hash; + u64 new_hash; u64 new_pos; int ret; int err; trace_scoutfs_rename(sb, old_dir, old_dentry, new_dir, new_dentry); + old_hash = dirent_name_hash(old_dentry->d_name.name, + old_dentry->d_name.len); + new_hash = dirent_name_hash(new_dentry->d_name.name, + new_dentry->d_name.len); + if (new_dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; @@ -1545,10 +1550,10 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, /* make sure that the entries assumed by the argument still exist */ ret = verify_entry(sb, scoutfs_ino(old_dir), old_dentry->d_name.name, - old_dentry->d_name.len, scoutfs_ino(old_inode), - old_dir_lock) ?: + old_dentry->d_name.len, old_hash, + scoutfs_ino(old_inode), old_dir_lock) ?: verify_entry(sb, scoutfs_ino(new_dir), new_dentry->d_name.name, - new_dentry->d_name.len, + new_dentry->d_name.len, new_hash, new_inode ? scoutfs_ino(new_inode) : 0, new_dir_lock); if (ret) @@ -1586,9 +1591,8 @@ retry: /* remove the new entry if it exists */ if (new_inode) { ret = del_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_hash(new_dentry), dentry_info_pos(new_dentry), - new_dentry->d_name.name, - new_dentry->d_name.len, scoutfs_ino(new_inode), new_dir_lock, new_inode_lock); if (ret) @@ -1597,7 +1601,7 @@ retry: } /* create the new entry */ - ret = add_entry_items(sb, scoutfs_ino(new_dir), new_pos, + ret = add_entry_items(sb, scoutfs_ino(new_dir), new_hash, new_pos, new_dentry->d_name.name, new_dentry->d_name.len, scoutfs_ino(old_inode), old_inode->i_mode, new_dir_lock, old_inode_lock); @@ -1607,9 +1611,8 @@ retry: /* remove the old entry */ ret = del_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_hash(old_dentry), dentry_info_pos(old_dentry), - old_dentry->d_name.name, - old_dentry->d_name.len, scoutfs_ino(old_inode), old_dir_lock, old_inode_lock); if (ret) @@ -1625,7 +1628,7 @@ retry: /* won't fail from here on out, update all the vfs structs */ /* the caller will use d_move to move the old_dentry into place */ - update_dentry_info(sb, old_dentry, new_pos, new_dir_lock); + update_dentry_info(sb, old_dentry, new_hash, new_pos, new_dir_lock); i_size_write(old_dir, i_size_read(old_dir) - old_dentry->d_name.len); if (!new_inode) @@ -1664,7 +1667,6 @@ retry: if (new_inode) scoutfs_update_inode_item(new_inode, new_inode_lock, &ind_locks); - ret = 0; out: if (ret) { @@ -1677,10 +1679,14 @@ out: * succeed. Maybe we could have an item replace call * that gives us the dupe to re-insert on cleanup? Not * sure. + * + * It's safe to use dentry_info here 'cause they haven't + * been updated if we saw an error. */ err = 0; if (ins_old) err = add_entry_items(sb, scoutfs_ino(old_dir), + dentry_info_hash(old_dentry), dentry_info_pos(old_dentry), old_dentry->d_name.name, old_dentry->d_name.len, @@ -1691,14 +1697,13 @@ out: if (del_new && err == 0) err = del_entry_items(sb, scoutfs_ino(new_dir), - new_pos, - new_dentry->d_name.name, - new_dentry->d_name.len, + new_hash, new_pos, scoutfs_ino(old_inode), new_dir_lock, old_inode_lock); if (ins_new && err == 0) err = add_entry_items(sb, scoutfs_ino(new_dir), + dentry_info_hash(new_dentry), dentry_info_pos(new_dentry), new_dentry->d_name.name, new_dentry->d_name.len, diff --git a/kmod/src/dir.h b/kmod/src/dir.h index 79aaaa2a..ee43930e 100644 --- a/kmod/src/dir.h +++ b/kmod/src/dir.h @@ -10,18 +10,20 @@ extern const struct inode_operations scoutfs_symlink_iops; struct scoutfs_link_backref_entry { struct list_head head; + u64 dir_ino; + u64 dir_pos; u16 name_len; - struct scoutfs_link_backref_key lbkey; + struct scoutfs_dirent dent; + /* the full name is allocated and stored in dent.name[0] */ }; -int scoutfs_dir_get_backref_path(struct super_block *sb, u64 target_ino, - u64 dir_ino, char *name, u16 name_len, - struct list_head *list); +int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, + u64 dir_pos, struct list_head *list); void scoutfs_dir_free_backref_path(struct super_block *sb, struct list_head *list); int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, - u64 dir_ino, char *name, unsigned int name_len, + u64 dir_ino, u64 dir_pos, struct list_head *list); int scoutfs_symlink_drop(struct super_block *sb, u64 ino, diff --git a/kmod/src/export.c b/kmod/src/export.c index 90c14cb9..5ee59b41 100644 --- a/kmod/src/export.c +++ b/kmod/src/export.c @@ -114,13 +114,12 @@ static struct dentry *scoutfs_get_parent(struct dentry *child) int ret; u64 ino; - ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), 0, NULL, 0, - &list); + ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), 0, 0, &list); if (ret) return ERR_PTR(ret); ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); - ino = be64_to_cpu(ent->lbkey.dir_ino); + ino = ent->dir_ino; scoutfs_dir_free_backref_path(sb, &list); trace_scoutfs_get_parent(sb, inode, ino); @@ -140,16 +139,16 @@ static int scoutfs_get_name(struct dentry *parent, char *name, int ret; ret = scoutfs_dir_add_next_linkref(sb, scoutfs_ino(inode), dir_ino, - NULL, 0, &list); + 0, &list); if (ret) return ret; ret = -ENOENT; ent = list_first_entry(&list, struct scoutfs_link_backref_entry, head); - if (be64_to_cpu(ent->lbkey.ino) == scoutfs_ino(inode) && - be64_to_cpu(ent->lbkey.dir_ino) == dir_ino && + if (le64_to_cpu(ent->dent.ino) == scoutfs_ino(inode) && + ent->dir_ino == dir_ino && ent->name_len <= NAME_MAX) { - memcpy(name, ent->lbkey.name, ent->name_len); + memcpy(name, ent->dent.name, ent->name_len); name[ent->name_len] = '\0'; ret = 0; trace_scoutfs_get_name(sb, parent->d_inode, inode, name); diff --git a/kmod/src/format.h b/kmod/src/format.h index ff3e4872..0509e647 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -266,29 +266,13 @@ struct scoutfs_inode_key { __u8 type; } __packed; -/* value is struct scoutfs_dirent without the name */ +/* value is struct scoutfs_dirent with the name */ struct scoutfs_dirent_key { __u8 zone; __be64 ino; __u8 type; - __u8 name[0]; -} __packed; - -/* value is struct scoutfs_dirent with the name */ -struct scoutfs_readdir_key { - __u8 zone; - __be64 ino; - __u8 type; - __be64 pos; -} __packed; - -/* value is empty */ -struct scoutfs_link_backref_key { - __u8 zone; - __be64 ino; - __u8 type; - __be64 dir_ino; - __u8 name[0]; + __be64 major; + __be64 minor; } __packed; /* key is bytes of encoded block mapping */ @@ -494,13 +478,17 @@ struct scoutfs_inode { #define SCOUTFS_SYMLINK_MAX_SIZE 4096 /* - * Dirents are stored in items with an offset of the hash of their name. - * Colliding names are packed into the value. + * Dirents are stored in multiple places to isolate contention when + * performing different operations: hashed by name for creation and + * lookup, at incrementing positions for readdir and resolving inodes to + * paths. Each entry has all the metadata needed to reference all the + * items (so an entry cached by lookup can be used to unlink all the + * items). */ struct scoutfs_dirent { __le64 ino; - __le64 counter; - __le64 readdir_pos; + __le64 hash; + __le64 pos; __u8 type; __u8 name[0]; } __packed; @@ -526,9 +514,8 @@ enum { SCOUTFS_DT_WHT, }; -/* ino_path can search for backref items with a null term */ #define SCOUTFS_MAX_KEY_SIZE \ - offsetof(struct scoutfs_link_backref_key, name[SCOUTFS_NAME_LEN + 1]) + sizeof(struct scoutfs_dirent_key) #define SCOUTFS_MAX_VAL_SIZE SCOUTFS_BLOCK_MAPPING_MAX_BYTES diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 57faec81..fdfabab6 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -201,116 +201,79 @@ out: return ret; } -struct ino_path_cursor { - __u64 dir_ino; - __u8 name[SCOUTFS_NAME_LEN + 1]; -} __packed; - /* - * see the definition of scoutfs_ioctl_ino_path for ioctl semantics. - * - * The null termination of the cursor name is a trick to skip past the - * last name we read without having to try and "increment" the name. - * Adding a null sorts the cursor after the non-null name and before all - * the next names because the item names aren't null terminated. + * See the comment above the definition of struct scoutfs_ioctl_ino_path + * for ioctl semantics. */ static long scoutfs_ioc_ino_path(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_ino_path __user *uargs; + struct scoutfs_ioctl_ino_path_result __user *ures; + struct scoutfs_link_backref_entry *last_ent; struct scoutfs_link_backref_entry *ent; - struct ino_path_cursor __user *ucurs; struct scoutfs_ioctl_ino_path args; - char __user *upath; LIST_HEAD(list); - u64 dir_ino; - u16 name_len; + u16 copied; char term; - char *name; int ret; - BUILD_BUG_ON(SCOUTFS_IOC_INO_PATH_CURSOR_BYTES != - sizeof(struct ino_path_cursor)); - if (!capable(CAP_DAC_READ_SEARCH)) return -EPERM; - uargs = (void __user *)arg; - if (copy_from_user(&args, uargs, sizeof(args))) + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; - if (args.cursor_bytes != sizeof(struct ino_path_cursor)) - return -EINVAL; + ures = (void __user *)(unsigned long)args.result_ptr; - ucurs = (void __user *)(unsigned long)args.cursor_ptr; - upath = (void __user *)(unsigned long)args.path_ptr; - - if (get_user(dir_ino, &ucurs->dir_ino)) - return -EFAULT; - - /* alloc/copy the small cursor name, requires and includes null */ - name_len = strnlen_user(ucurs->name, sizeof(ucurs->name)); - if (name_len < 1 || name_len > sizeof(ucurs->name)) - return -EINVAL; - - name = kmalloc(name_len, GFP_KERNEL); - if (!name) - return -ENOMEM; - - if (copy_from_user(name, ucurs->name, name_len)) { - ret = -EFAULT; + ret = scoutfs_dir_get_backref_path(sb, args.ino, args.dir_ino, + args.dir_pos, &list); + if (ret < 0) goto out; - } - ret = scoutfs_dir_get_backref_path(sb, args.ino, dir_ino, name, - name_len, &list); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - goto out; - } - - ret = 0; + last_ent = list_last_entry(&list, struct scoutfs_link_backref_entry, + head); + copied = 0; list_for_each_entry(ent, &list, head) { - if (ret + ent->name_len + 1 > args.path_bytes) { + + if (offsetof(struct scoutfs_ioctl_ino_path_result, + path[copied + ent->name_len + 1]) + > args.result_bytes) { ret = -ENAMETOOLONG; goto out; } - if (copy_to_user(upath, ent->lbkey.name, ent->name_len)) { + if (copy_to_user(&ures->path[copied], + ent->dent.name, ent->name_len)) { ret = -EFAULT; goto out; } - upath += ent->name_len; - ret += ent->name_len; + copied += ent->name_len; - if (ent->head.next == &list) + if (ent == last_ent) term = '\0'; else term = '/'; - if (put_user(term, upath)) { + if (put_user(term, &ures->path[copied])) { ret = -EFAULT; break; } - upath++; - ret++; + copied++; } - /* copy the last entry into the cursor */ - ent = list_last_entry(&list, struct scoutfs_link_backref_entry, head); - - if (put_user(be64_to_cpu(ent->lbkey.dir_ino), &ucurs->dir_ino) || - copy_to_user(ucurs->name, ent->lbkey.name, ent->name_len) || - put_user('\0', &ucurs->name[ent->name_len])) { + /* fill the result header now that we know the copied path length */ + if (put_user(last_ent->dir_ino, &ures->dir_ino) || + put_user(last_ent->dir_pos, &ures->dir_pos) || + put_user(copied, &ures->path_bytes)) { ret = -EFAULT; + } else { + ret = 0; } out: scoutfs_dir_free_backref_path(sb, &list); - kfree(name); return ret; } diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 33c94b95..721f1cde 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -64,24 +64,36 @@ enum { struct scoutfs_ioctl_walk_inodes) /* - * Fill the path buffer with the next path to the target inode. An - * iteration cursor is stored in the cursor buffer which advances - * through the paths to the inode at each call. + * Fill the result buffer with the next absolute path to the target + * inode searching from a given position in a parent directory. * * @ino: The target ino that we're finding paths to. Constant across * all the calls that make up an iteration over all the inode's paths. * - * @cursor_ptr: A pointer to the buffer that will hold the iteration - * cursor. It must be initialized to 0 before iterating. Each call - * modifies it to skip past the result of that call. + * @dir_ino: The inode number of the directory containing the entry to + * our inode to search from. If this parent directory contains no more + * entries to our inode then we'll search through other parent directory + * inodes in inode order. * - * @cusur_bytes: The length of the cursor buffer. Must be - * SCOUTFS_IOC_INO_PATH_CURSOR_BYTES. + * @dir_pos: The position in the dir_ino parent directory of the entry + * to our inode to search from. If there is no entry at this position + * then we'll search through other entry positions in increasing order. + * If we exhaust the parent directory then we'll search through + * additional parent directories in inode order. * - * @path_ptr: The buffer to store each found path. + * @result_ptr: A pointer to the buffer where the result struct and + * absolute path will be stored. * - * @path_bytes: The size of the buffer that will the found path - * including null termination. (PATH_MAX is a solid choice.) + * @result_bytes: The size of the buffer that will contain the result + * struct and the null terminated absolute path name. + * + * To start iterating set the desired target inode, dir_ino to 0, + * dir_pos to 0, and set result_ptr and _bytes to a sufficiently large + * buffeer (sizeof(result) + PATH_MAX is a solid choice). + * + * After each returned result set the next search dir_ino and dir_pos to + * the returned dir_ino and dir_pos. Then increment the search dir_pos, + * and if it wrapped to 0, increment dir_ino. * * This only walks back through full hard links. None of the returned * paths will reflect symlinks to components in the path. @@ -90,28 +102,39 @@ enum { * returned paths to the inode. It requires CAP_DAC_READ_SEARCH which * bypasses permissions checking. * - * ENAMETOOLONG is returned when the next path found from the cursor - * doesn't fit in the path buffer. - * * This call is not serialized with any modification (create, rename, * unlink) of the path components. It will return all the paths that * were stable both before and after the call. It may or may not return * paths which are created or unlinked during the call. * - * The number of bytes in the path, including the null terminator, are - * returned when a path is found. 0 is returned when there are no more - * paths to the link to the inode from the cursor. + * On success 0 is returned and result struct is filled with the next + * absolute path. The path_bytes length of the path includes a null + * terminating byte. dir_ino and dir_pos refer to the position of the + * final component in its parent directory and can be advanced to search + * for the next terminal entry whose path is then built by walking up + * parent directories. + * + * ENOENT is returned when no paths are found. + * + * ENAMETOOLONG is returned when the result struct and path found + * doesn't fit in the result buffer. + * + * Many other errnos indicate hard failure to find the next path. */ struct scoutfs_ioctl_ino_path { __u64 ino; - __u64 cursor_ptr; - __u64 path_ptr; - __u16 cursor_bytes; - __u16 path_bytes; + __u64 dir_ino; + __u64 dir_pos; + __u64 result_ptr; + __u16 result_bytes; } __packed; -#define SCOUTFS_IOC_INO_PATH_CURSOR_BYTES \ - (sizeof(__u64) + SCOUTFS_NAME_LEN + 1) +struct scoutfs_ioctl_ino_path_result { + __u64 dir_ino; + __u64 dir_pos; + __u16 path_bytes; + __u8 path[0]; +} __packed; /* Get a single path from the root to the given inode number */ #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ diff --git a/kmod/src/key.c b/kmod/src/key.c index 5befe0e0..9e8bb5cc 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -276,35 +276,17 @@ static int pr_xattr(char *buf, struct scoutfs_key_buf *key, size_t size) static int pr_dirent(char *buf, struct scoutfs_key_buf *key, size_t size) { struct scoutfs_dirent_key *dkey = key->data; - int len = (int)key->key_len - sizeof(struct scoutfs_dirent_key); + char *which = dkey->type == SCOUTFS_DIRENT_TYPE ? "dnt" : + dkey->type == SCOUTFS_READDIR_TYPE ? "rdr" : + dkey->type == SCOUTFS_LINK_BACKREF_TYPE ? "lbr" : + "unk"; return snprintf_key(buf, size, key, sizeof(struct scoutfs_dirent_key), key->key_len, - "fs.%llu.dnt.%.*s", - be64_to_cpu(dkey->ino), len, dkey->name); -} - -static int pr_readdir(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_readdir_key *rkey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_readdir_key), 0, - "fs.%llu.rdr.%llu", - be64_to_cpu(rkey->ino), be64_to_cpu(rkey->pos)); -} - -static int pr_link_backref(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_link_backref_key *lkey = key->data; - int len = (int)key->key_len - sizeof(*lkey); - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_link_backref_key), - key->key_len, - "fs.%llu.lbr.%llu.%.*s", - be64_to_cpu(lkey->ino), be64_to_cpu(lkey->dir_ino), - len, lkey->name); + "fs.%llu.%s.%llu.%llu", + be64_to_cpu(dkey->ino), which, + be64_to_cpu(dkey->major), + be64_to_cpu(dkey->minor)); } static int pr_symlink(char *buf, struct scoutfs_key_buf *key, size_t size) @@ -339,8 +321,8 @@ const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = pr_inode, [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = pr_xattr, [SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = pr_dirent, - [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_readdir, - [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_link_backref, + [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_dirent, + [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_dirent, [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = pr_symlink, [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = pr_block_mapping, }; From 9148f24aa2a1fec192b1687341f13227e0b2595c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 9 Jan 2018 09:33:56 -0800 Subject: [PATCH 587/920] scoutfs: use single small key struct Variable length keys lead to having a key struct point to the buffer that contains the key. With dirents and xattrs now using small keys we can convert everyone to using a single key struct and significantly simplify the system. We no longer have a seperate generic key buf struct that points to specific per-type key storage. All items use the key struct and fill out the appropriate fields. All the code that paired a generic key buf struct and a specific key type struct is collapsed down to a key struct. There's no longer the difference between a key buf that shares a read-only key, has it's own precise allocation, or has a max size allocation for incrementing and decrementing. Each key user now has an init function fills out its fields. It looks a lot like the old pattern but we no longer have seperate key storage that the buf points to. A bunch of code now takes the address of static key storage instead of managing allocated keys. Conversely, swapping now uses the full keys instead of pointers to the keys. We don't need all the functions that worked on the generic key buf struct because they had different lengths. Copy, clone, length init, memcpy, all of that goes away. The item API had some functions that tested the length of keys and values. The key length tests vanish, and that gets rid of the _same() call. The _same_min() call only had one user who didn't also test for the value length being too large. Let's leave caller key constraints in callers instead of trying to hide them on the other side of a bunch of item calls. We no longer have to track the number of key bytes when calculating if an item population will fit in segments. This removes the key length from reservations, transactions, and segment writing. The item cache key querying ioctls no longer have to deal with variable length keys. The simply specify the start key, the ioctls return the number of keys copied instead of bytes, and the caller is responsible for incrementing the next search key. The segment no longer has to store the key length. It stores the key struct in the item header. The fancy variable length key formatting and printing can be removed. We have a single format for the universal key struct. The SK_ wrappers that bracked calls to use preempt safe per cpu buffers can turn back into their normal calls. Manifest entries are now a fixed size. We can simply split them between btree keys and values and initialize them instead of allocating them. This means that level 0 entries don't have their own format that sorts by the seq. They're sorted by the key like all the other levels. Compaction needs to sweep all of them looking for the oldest and read can stop sweeping once it can no longer overlap. This makes rare compaction more expensive and common reading less expensive, which is the right tradeoff. Signed-off-by: Zach Brown --- kmod/src/client.c | 16 +- kmod/src/cmp.h | 14 +- kmod/src/compact.c | 48 ++-- kmod/src/count.h | 23 +- kmod/src/data.c | 125 ++++------ kmod/src/dir.c | 137 +++++------ kmod/src/format.h | 176 +++++++------- kmod/src/inode.c | 139 +++++------ kmod/src/inode.h | 6 +- kmod/src/ioctl.c | 129 ++++------- kmod/src/ioctl.h | 11 +- kmod/src/item.c | 487 +++++++++++++++------------------------ kmod/src/item.h | 64 +++-- kmod/src/key.c | 448 +++-------------------------------- kmod/src/key.h | 254 ++++++++++---------- kmod/src/lock.c | 165 ++++++------- kmod/src/lock.h | 8 +- kmod/src/manifest.c | 456 ++++++++++++++---------------------- kmod/src/manifest.h | 19 +- kmod/src/msg.h | 14 -- kmod/src/scoutfs_trace.h | 321 +++++++++++--------------- kmod/src/seg.c | 82 +++---- kmod/src/seg.h | 17 +- kmod/src/server.c | 61 +---- kmod/src/server.h | 10 +- kmod/src/super.c | 1 + kmod/src/trans.c | 21 +- kmod/src/trans.h | 2 +- kmod/src/xattr.c | 96 ++++---- 29 files changed, 1221 insertions(+), 2129 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 174f7f17..e40cc214 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -589,22 +589,14 @@ int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level) { struct client_info *client = SCOUTFS_SB(sb)->client_info; - struct scoutfs_net_manifest_entry *net_ment; + struct scoutfs_net_manifest_entry net_ment; struct scoutfs_manifest_entry ment; - int ret; scoutfs_seg_init_ment(&ment, level, seg); - net_ment = scoutfs_alloc_net_ment(&ment); - if (net_ment) { - ret = client_request(client, SCOUTFS_NET_RECORD_SEGMENT, - net_ment, scoutfs_net_ment_bytes(net_ment), - NULL, 0); - kfree(net_ment); - } else { - ret = -ENOMEM; - } + scoutfs_init_ment_to_net(&net_ment, &ment); - return ret; + return client_request(client, SCOUTFS_NET_RECORD_SEGMENT, &net_ment, + sizeof(net_ment), NULL, 0); } static int sort_cmp_u64s(const void *A, const void *B) diff --git a/kmod/src/cmp.h b/kmod/src/cmp.h index 3230c043..23c6d8a6 100644 --- a/kmod/src/cmp.h +++ b/kmod/src/cmp.h @@ -1,7 +1,19 @@ #ifndef _SCOUTFS_CMP_H_ #define _SCOUTFS_CMP_H_ -#include +/* + * A generic ternary comparison macro with strict type checking. + */ +#define scoutfs_cmp(a, b) \ +({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + int _ret; \ + \ + (void) (&_a == &_b); \ + _ret = _a < _b ? -1 : _a > _b ? 1 : 0; \ + _ret; \ +}) static inline int scoutfs_cmp_u64s(u64 a, u64 b) { diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 58087381..4047e5e7 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -66,8 +66,8 @@ struct compact_seg { u64 segno; u64 seq; u8 level; - struct scoutfs_key_buf *first; - struct scoutfs_key_buf *last; + struct scoutfs_key first; + struct scoutfs_key last; struct scoutfs_segment *seg; int off; bool part_of_move; @@ -101,27 +101,20 @@ static void free_cseg(struct super_block *sb, struct compact_seg *cseg) WARN_ON_ONCE(!list_empty(&cseg->entry)); scoutfs_seg_put(cseg->seg); - scoutfs_key_free(sb, cseg->first); - scoutfs_key_free(sb, cseg->last); - kfree(cseg); } static struct compact_seg *alloc_cseg(struct super_block *sb, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) + struct scoutfs_key *first, + struct scoutfs_key *last) { struct compact_seg *cseg; cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); if (cseg) { INIT_LIST_HEAD(&cseg->entry); - cseg->first = scoutfs_key_dup(sb, first); - cseg->last = scoutfs_key_dup(sb, last); - if (!cseg->first || !cseg->last) { - free_cseg(sb, cseg); - cseg = NULL; - } + cseg->first = *first; + cseg->last = *last; } return cseg; @@ -179,12 +172,12 @@ static struct compact_seg *next_spos(struct compact_cursor *curs, * incremental update items. */ static int next_item(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_key_buf *item_key, struct kvec *item_val, + struct scoutfs_key *item_key, struct kvec *item_val, u8 *item_flags) { struct compact_seg *upper = curs->upper; struct compact_seg *lower = curs->lower; - struct scoutfs_key_buf lower_key; + struct scoutfs_key lower_key; struct kvec lower_val; u8 lower_flags; int cmp; @@ -192,8 +185,8 @@ static int next_item(struct super_block *sb, struct compact_cursor *curs, retry: if (upper) { - ret = scoutfs_seg_item_ptrs(upper->seg, upper->off, - item_key, item_val, item_flags); + ret = scoutfs_seg_get_item(upper->seg, upper->off, + item_key, item_val, item_flags); if (ret < 0) upper = NULL; } @@ -203,9 +196,9 @@ retry: if (ret) goto out; - ret = scoutfs_seg_item_ptrs(lower->seg, lower->off, - &lower_key, &lower_val, - &lower_flags); + ret = scoutfs_seg_get_item(lower->seg, lower->off, + &lower_key, &lower_val, + &lower_flags); if (ret == 0) break; lower = next_spos(curs, lower); @@ -230,7 +223,7 @@ retry: cmp = 1; if (cmp > 0) { - scoutfs_key_clone(item_key, &lower_key); + *item_key = lower_key; *item_val = lower_val; *item_flags = lower_flags; } @@ -243,7 +236,7 @@ retry: */ if (curs->sticky && curs->lower && (!lower || lower == curs->last_lower) && - scoutfs_key_compare(item_key, curs->last_lower->last) > 0) { + scoutfs_key_compare(item_key, &curs->last_lower->last) > 0) { ret = 0; goto out; } @@ -276,7 +269,7 @@ static int compact_segments(struct super_block *sb, struct scoutfs_bio_completion *comp, struct list_head *results) { - struct scoutfs_key_buf item_key; + struct scoutfs_key item_key; struct scoutfs_segment *seg; struct compact_seg *cseg; struct compact_seg *upper; @@ -315,7 +308,7 @@ static int compact_segments(struct super_block *sb, * entry iterator that reading and compacting * can use. */ - cseg = alloc_cseg(sb, upper->first, upper->last); + cseg = alloc_cseg(sb, &upper->first, &upper->last); if (!cseg) { ret = -ENOMEM; break; @@ -535,7 +528,7 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) } scoutfs_manifest_init_entry(&ment, cseg->level, 0, cseg->seq, - cseg->first, NULL); + &cseg->first, NULL); ret = scoutfs_manifest_del(sb, &ment); BUG_ON(ret); } @@ -548,7 +541,7 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) else scoutfs_manifest_init_entry(&ment, cseg->level, cseg->segno, cseg->seq, - cseg->first, cseg->last); + &cseg->first, &cseg->last); ret = scoutfs_manifest_add(sb, &ment); BUG_ON(ret); } @@ -590,7 +583,8 @@ static void scoutfs_compact_func(struct work_struct *work) /* trace compaction ranges */ list_for_each_entry(cseg, &curs.csegs, entry) { trace_scoutfs_compact_input(sb, cseg->level, cseg->segno, - cseg->seq, cseg->first, cseg->last); + cseg->seq, &cseg->first, + &cseg->last); } if (ret == 0 && !list_empty(&curs.csegs)) { diff --git a/kmod/src/count.h b/kmod/src/count.h index f115d034..759c736d 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -2,10 +2,8 @@ #define _SCOUTFS_COUNT_H_ /* - * Our estimate of the space consumed while dirtying items isn't a - * single value. We're packing items into segments which have different - * overheads for items (header overhead), keys (block aligned), and - * values (can span blocks, not aligned). + * Our estimate of the space consumed while dirtying items is based on + * the number of items and the size of their values. * * The estimate is still a read-only input to entering the transaction. * We'd like to use it as a clean rhs arg to hold_trans. We define SIC_ @@ -21,7 +19,6 @@ struct scoutfs_item_count { signed items; - signed keys; signed vals; }; @@ -33,8 +30,6 @@ static inline void __count_alloc_inode(struct scoutfs_item_count *cnt) const int nr_indices = SCOUTFS_INODE_INDEX_NR; cnt->items += 1 + nr_indices; - cnt->keys += sizeof(struct scoutfs_inode_key) + - (nr_indices * sizeof(struct scoutfs_inode_index_key)); cnt->vals += sizeof(struct scoutfs_inode); } @@ -47,8 +42,6 @@ static inline void __count_dirty_inode(struct scoutfs_item_count *cnt) const int nr_indices = 2 * SCOUTFS_INODE_INDEX_NR; cnt->items += 1 + nr_indices; - cnt->keys += sizeof(struct scoutfs_inode_key) + - (nr_indices * sizeof(struct scoutfs_inode_index_key)); cnt->vals += sizeof(struct scoutfs_inode); } @@ -77,7 +70,6 @@ static inline void __count_dirents(struct scoutfs_item_count *cnt, unsigned name_len) { cnt->items += 3; - cnt->keys += 3 * sizeof(struct scoutfs_dirent_key); cnt->vals += 3 * offsetof(struct scoutfs_dirent, name[name_len]); } @@ -87,7 +79,6 @@ static inline void __count_sym_target(struct scoutfs_item_count *cnt, unsigned nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); cnt->items += nr; - cnt->keys += nr * sizeof(struct scoutfs_symlink_key); cnt->vals += size; } @@ -95,7 +86,6 @@ static inline void __count_orphan(struct scoutfs_item_count *cnt) { cnt->items += 1; - cnt->keys += sizeof(struct scoutfs_orphan_key); } static inline void __count_mknod(struct scoutfs_item_count *cnt, @@ -197,16 +187,13 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, __count_dirty_inode(&cnt); - if (old_parts) { + if (old_parts) cnt.items += old_parts; - cnt.keys += old_parts * sizeof(struct scoutfs_xattr_key); - } if (creating) { new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size) cnt.items += new_parts; - cnt.keys += new_parts * sizeof(struct scoutfs_xattr_key); cnt.vals += sizeof(struct scoutfs_xattr) + name_len + size; } @@ -225,8 +212,6 @@ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) __count_dirty_inode(&cnt); cnt.items += 1 + nr_free; - cnt.keys += sizeof(struct scoutfs_block_mapping_key) + - (nr_free * sizeof(struct scoutfs_free_bits_key)); cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + (nr_free * sizeof(struct scoutfs_free_bits)); @@ -244,8 +229,6 @@ static inline const struct scoutfs_item_count SIC_TRUNC_BLOCK(void) unsigned nr_free = (2 * SCOUTFS_BLOCK_MAPPING_BLOCKS); cnt.items += 1 + nr_free; - cnt.keys += sizeof(struct scoutfs_block_mapping_key) + - (nr_free * sizeof(struct scoutfs_free_bits_key)); cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + (nr_free * sizeof(struct scoutfs_free_bits)); diff --git a/kmod/src/data.c b/kmod/src/data.c index 19108c40..fb4d19bb 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -319,30 +319,25 @@ static int decode_mapping(struct block_mapping *map, int size) return 0; } -static void init_mapping_key(struct scoutfs_key_buf *key, - struct scoutfs_block_mapping_key *bmk, - u64 ino, u64 iblock) +static void init_mapping_key(struct scoutfs_key *key, u64 ino, u64 iblock) { - - bmk->zone = SCOUTFS_FS_ZONE; - bmk->ino = cpu_to_be64(ino); - bmk->type = SCOUTFS_BLOCK_MAPPING_TYPE; - bmk->base = cpu_to_be64(iblock >> SCOUTFS_BLOCK_MAPPING_SHIFT); - - scoutfs_key_init(key, bmk, sizeof(struct scoutfs_block_mapping_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skm_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_BLOCK_MAPPING_TYPE, + .skm_base = cpu_to_le64(iblock >> SCOUTFS_BLOCK_MAPPING_SHIFT), + }; } - -static void init_free_key(struct scoutfs_key_buf *key, - struct scoutfs_free_bits_key *fbk, u64 node_id, - u64 full_bit, u8 type) +static void init_free_key(struct scoutfs_key *key, u64 node_id, u64 full_bit, + u8 type) { - fbk->zone = SCOUTFS_NODE_ZONE; - fbk->node_id = cpu_to_be64(node_id); - fbk->type = type; - fbk->base = cpu_to_be64(full_bit >> SCOUTFS_FREE_BITS_SHIFT); - - scoutfs_key_init(key, fbk, sizeof(struct scoutfs_free_bits_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_NODE_ZONE, + .skf_node_id = cpu_to_le64(node_id), + .sk_type = type, + .skf_base = cpu_to_le64(full_bit >> SCOUTFS_FREE_BITS_SHIFT), + }; } /* @@ -353,15 +348,13 @@ static int set_segno_free(struct super_block *sb, u64 segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key fbk = {0,}; struct scoutfs_free_bits frb; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct kvec val; int bit = 0; int ret; - init_free_key(&key, &fbk, sbi->node_id, segno, - SCOUTFS_FREE_BITS_SEGNO_TYPE); + init_free_key(&key, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret && ret != -ENOENT) @@ -383,7 +376,7 @@ static int set_segno_free(struct super_block *sb, u64 segno) ret = scoutfs_item_update(sb, &key, &val, lock); out: - trace_scoutfs_data_set_segno_free(sb, segno, be64_to_cpu(fbk.base), + trace_scoutfs_data_set_segno_free(sb, segno, le64_to_cpu(key.skf_base), bit, ret); return ret; } @@ -394,8 +387,7 @@ out: * need to. */ static int create_blkno_free(struct super_block *sb, u64 blkno, - struct scoutfs_key_buf *key, - struct scoutfs_free_bits_key *fbk) + struct scoutfs_key *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; @@ -403,8 +395,7 @@ static int create_blkno_free(struct super_block *sb, u64 blkno, struct kvec val; int bit; - init_free_key(key, fbk, sbi->node_id, blkno, - SCOUTFS_FREE_BITS_BLKNO_TYPE); + init_free_key(key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); bit = blkno & SCOUTFS_FREE_BITS_MASK; @@ -429,18 +420,15 @@ static int clear_segno_free(struct super_block *sb, u64 segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key b_fbk; - struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; - struct scoutfs_key_buf b_key; - struct scoutfs_key_buf key; + struct scoutfs_key b_key; + struct scoutfs_key key; struct kvec val; u64 blkno; int bit; int ret; - init_free_key(&key, &fbk, sbi->node_id, segno, - SCOUTFS_FREE_BITS_SEGNO_TYPE); + init_free_key(&key, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret) { @@ -459,7 +447,7 @@ static int clear_segno_free(struct super_block *sb, u64 segno) /* create the new blkno item, we can safely delete it */ blkno = segno << SCOUTFS_SEGMENT_BLOCK_SHIFT; - ret = create_blkno_free(sb, blkno, &b_key, &b_fbk); + ret = create_blkno_free(sb, blkno, &b_key); if (ret) goto out; @@ -482,17 +470,15 @@ static int set_blkno_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct kvec val; u64 segno; int bit; int ret; /* get the specified item */ - init_free_key(&key, &fbk, sbi->node_id, blkno, - SCOUTFS_FREE_BITS_BLKNO_TYPE); + init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret && ret != -ENOENT) @@ -542,16 +528,14 @@ static int clear_blkno_free(struct super_block *sb, u64 blkno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct kvec val; int bit; int ret; /* get the specified item */ - init_free_key(&key, &fbk, sbi->node_id, blkno, - SCOUTFS_FREE_BITS_BLKNO_TYPE); + init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); if (ret) { @@ -607,10 +591,8 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; - struct scoutfs_block_mapping_key last_bmk; - struct scoutfs_block_mapping_key bmk; + struct scoutfs_key last_key; + struct scoutfs_key key; struct block_mapping *map; struct kvec val; bool holding = false; @@ -629,11 +611,11 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, if (!map) return -ENOMEM; - init_mapping_key(&last_key, &last_bmk, ino, last); + init_mapping_key(&last_key, ino, last); while (iblock <= last) { /* find the mapping that could include iblock */ - init_mapping_key(&key, &bmk, ino, iblock); + init_mapping_key(&key, ino, iblock); kvec_init(&val, map->encoded, sizeof(map->encoded)); ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); @@ -655,7 +637,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, break; /* set iblock to the first in the next item inside last */ - iblock = max(iblock, be64_to_cpu(bmk.base) << + iblock = max(iblock, le64_to_cpu(key.skm_base) << SCOUTFS_BLOCK_MAPPING_SHIFT); dirtied = false; @@ -838,15 +820,13 @@ static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct kvec val; int ret; int bit; - init_free_key(&key, &fbk, sbi->node_id, blkno, - SCOUTFS_FREE_BITS_BLKNO_TYPE); + init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); @@ -860,7 +840,8 @@ static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) goto out; } - *blkno_ret = (be64_to_cpu(fbk.base) << SCOUTFS_FREE_BITS_SHIFT) + bit; + *blkno_ret = (le64_to_cpu(key.skf_base) << SCOUTFS_FREE_BITS_SHIFT) + + bit; ret = 0; out: return ret; @@ -874,18 +855,15 @@ static int find_free_segno(struct super_block *sb, u64 *segno) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits_key last_fbk; - struct scoutfs_free_bits_key fbk; struct scoutfs_free_bits frb; - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; + struct scoutfs_key last_key; + struct scoutfs_key key; struct kvec val; int bit; int ret; - init_free_key(&key, &fbk, sbi->node_id, 0, - SCOUTFS_FREE_BITS_SEGNO_TYPE); - init_free_key(&last_key, &last_fbk, sbi->node_id, ~0, + init_free_key(&key, sbi->node_id, 0, SCOUTFS_FREE_BITS_SEGNO_TYPE); + init_free_key(&last_key, sbi->node_id, U64_MAX, SCOUTFS_FREE_BITS_SEGNO_TYPE); kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); @@ -900,7 +878,7 @@ static int find_free_segno(struct super_block *sb, u64 *segno) goto out; } - *segno = (be64_to_cpu(fbk.base) << SCOUTFS_FREE_BITS_SHIFT) + bit; + *segno = (le64_to_cpu(key.skf_base) << SCOUTFS_FREE_BITS_SHIFT) + bit; ret = 0; out: return ret; @@ -916,7 +894,7 @@ out: */ static int find_alloc_block(struct super_block *sb, struct inode *inode, struct block_mapping *map, - struct scoutfs_key_buf *map_key, + struct scoutfs_key *map_key, unsigned map_ind, bool map_exists, struct scoutfs_lock *data_lock) { @@ -1013,8 +991,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_block_mapping_key bmk; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct scoutfs_lock *lock; struct block_mapping *map; struct kvec val; @@ -1031,7 +1008,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, if (!map) return -ENOMEM; - init_mapping_key(&key, &bmk, scoutfs_ino(inode), iblock); + init_mapping_key(&key, scoutfs_ino(inode), iblock); kvec_init(&val, map->encoded, sizeof(map->encoded)); /* find the mapping item that covers the logical block */ @@ -1310,13 +1287,11 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, { struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; + struct scoutfs_key last_key; + struct scoutfs_key key; struct scoutfs_lock *inode_lock = NULL; struct block_mapping *map; struct pending_fiemap pend; - struct scoutfs_block_mapping_key last_bmk; - struct scoutfs_block_mapping_key bmk; struct kvec val; loff_t i_size; bool offline; @@ -1351,14 +1326,14 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, blk_off = start >> SCOUTFS_BLOCK_SHIFT; final = min_t(loff_t, i_size - 1, start + len - 1) >> SCOUTFS_BLOCK_SHIFT; - init_mapping_key(&last_key, &last_bmk, ino, final); + init_mapping_key(&last_key, ino, final); ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) goto out; while (blk_off <= final) { - init_mapping_key(&key, &bmk, ino, blk_off); + init_mapping_key(&key, ino, blk_off); kvec_init(&val, &map->encoded, sizeof(map->encoded)); ret = scoutfs_item_next(sb, &key, &last_key, &val, inode_lock); @@ -1373,7 +1348,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, break; /* set blk_off to the first in the next item inside last */ - blk_off = max(blk_off, be64_to_cpu(bmk.base) << + blk_off = max(blk_off, le64_to_cpu(key.skm_base) << SCOUTFS_BLOCK_MAPPING_SHIFT); for_each_block(i, blk_off, final) { diff --git a/kmod/src/dir.c b/kmod/src/dir.c index f32dfe6c..7b813922 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -191,17 +191,16 @@ static u64 dentry_info_pos(struct dentry *dentry) return di->pos; } -static void init_dirent_key(struct scoutfs_key_buf *key, - struct scoutfs_dirent_key *dkey, u8 type, - u64 ino, u64 major, u64 minor) +static void init_dirent_key(struct scoutfs_key *key, u8 type, u64 ino, + u64 major, u64 minor) { - dkey->zone = SCOUTFS_FS_ZONE; - dkey->ino = cpu_to_be64(ino); - dkey->type = type; - dkey->major = cpu_to_be64(major); - dkey->minor = cpu_to_be64(minor); - - scoutfs_key_init(key, dkey, sizeof(struct scoutfs_dirent_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skd_ino = cpu_to_le64(ino), + .sk_type = type, + .skd_major = cpu_to_le64(major), + .skd_minor = cpu_to_le64(minor), + }; } static unsigned int dirent_bytes(unsigned int name_len) @@ -237,10 +236,8 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, struct scoutfs_dirent *dent_ret, struct scoutfs_lock *lock) { - struct scoutfs_dirent_key last_dkey; - struct scoutfs_dirent_key dkey; - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; + struct scoutfs_key last_key; + struct scoutfs_key key; struct scoutfs_dirent *dent = NULL; struct kvec val; int ret; @@ -251,10 +248,8 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, goto out; } - init_dirent_key(&key, &dkey, SCOUTFS_DIRENT_TYPE, - dir_ino, hash, 0); - init_dirent_key(&last_key, &last_dkey, SCOUTFS_DIRENT_TYPE, - dir_ino, hash, U64_MAX); + init_dirent_key(&key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, 0); + init_dirent_key(&last_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, U64_MAX); kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); for (;;) { @@ -275,11 +270,11 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, break; } - if (be64_to_cpu(dkey.minor) == U64_MAX) { + if (le64_to_cpu(key.skd_minor) == U64_MAX) { ret = -ENOENT; break; } - be64_add_cpu(&dkey.minor, 1); + le64_add_cpu(&key.skd_minor, 1); } out: @@ -472,13 +467,11 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; struct scoutfs_dirent *dent; - struct scoutfs_key_buf key; - struct scoutfs_key_buf last_key; - struct scoutfs_dirent_key dkey; - struct scoutfs_dirent_key last_dkey; + struct scoutfs_key key; + struct scoutfs_key last_key; struct scoutfs_lock *dir_lock; - unsigned int name_len; struct kvec val; + int name_len; u64 pos; int ret; @@ -491,8 +484,8 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) goto out; } - init_dirent_key(&last_key, &last_dkey, SCOUTFS_READDIR_TYPE, - scoutfs_ino(inode), SCOUTFS_DIRENT_LAST_POS, 0); + init_dirent_key(&last_key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), + SCOUTFS_DIRENT_LAST_POS, 0); kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &dir_lock); @@ -500,11 +493,10 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) goto out; for (;;) { - init_dirent_key(&key, &dkey, SCOUTFS_READDIR_TYPE, - scoutfs_ino(inode), file->f_pos, 0); + init_dirent_key(&key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), + file->f_pos, 0); - ret = scoutfs_item_next_same_min(sb, &key, &last_key, &val, - dirent_bytes(1), dir_lock); + ret = scoutfs_item_next(sb, &key, &last_key, &val, dir_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -512,7 +504,13 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } name_len = ret - sizeof(struct scoutfs_dirent); - pos = be64_to_cpu(dkey.major); + /* XXX corruption */ + if (name_len < 1 || name_len > SCOUTFS_NAME_LEN) { + ret = -EIO; + goto out; + } + + pos = le64_to_cpu(key.skd_major); if (filldir(dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) { @@ -542,12 +540,9 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, u64 ino, umode_t mode, struct scoutfs_lock *dir_lock, struct scoutfs_lock *inode_lock) { - struct scoutfs_dirent_key rdir_dkey; - struct scoutfs_dirent_key ent_dkey; - struct scoutfs_dirent_key lb_dkey; - struct scoutfs_key_buf rdir_key; - struct scoutfs_key_buf ent_key; - struct scoutfs_key_buf lb_key; + struct scoutfs_key rdir_key; + struct scoutfs_key ent_key; + struct scoutfs_key lb_key; struct scoutfs_dirent *dent; bool del_ent = false; bool del_rdir = false; @@ -567,12 +562,9 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, dent->type = mode_to_type(mode); memcpy(dent->name, name, name_len); - init_dirent_key(&ent_key, &ent_dkey, SCOUTFS_DIRENT_TYPE, - dir_ino, hash, pos); - init_dirent_key(&rdir_key, &rdir_dkey, SCOUTFS_READDIR_TYPE, - dir_ino, pos, 0); - init_dirent_key(&lb_key, &lb_dkey, SCOUTFS_LINK_BACKREF_TYPE, - ino, dir_ino, pos); + init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); + init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); + init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); kvec_init(&val, dent, dirent_bytes(name_len)); ret = scoutfs_item_create(sb, &ent_key, &val, dir_lock); @@ -610,22 +602,16 @@ static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, u64 pos, u64 ino, struct scoutfs_lock *dir_lock, struct scoutfs_lock *inode_lock) { - struct scoutfs_dirent_key rdir_dkey; - struct scoutfs_dirent_key ent_dkey; - struct scoutfs_dirent_key lb_dkey; - struct scoutfs_key_buf rdir_key; - struct scoutfs_key_buf ent_key; - struct scoutfs_key_buf lb_key; + struct scoutfs_key rdir_key; + struct scoutfs_key ent_key; + struct scoutfs_key lb_key; LIST_HEAD(dir_saved); LIST_HEAD(inode_saved); int ret; - init_dirent_key(&ent_key, &ent_dkey, SCOUTFS_DIRENT_TYPE, - dir_ino, hash, pos); - init_dirent_key(&rdir_key, &rdir_dkey, SCOUTFS_READDIR_TYPE, - dir_ino, pos, 0); - init_dirent_key(&lb_key, &lb_dkey, SCOUTFS_LINK_BACKREF_TYPE, - ino, dir_ino, pos); + init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); + init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); + init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); ret = scoutfs_item_delete_save(sb, &ent_key, &dir_saved, dir_lock) ?: scoutfs_item_delete_save(sb, &rdir_key, &dir_saved, dir_lock) ?: @@ -959,15 +945,14 @@ unlock: return ret; } -static void init_symlink_key(struct scoutfs_key_buf *key, - struct scoutfs_symlink_key *skey, u64 ino, u8 nr) +static void init_symlink_key(struct scoutfs_key *key, u64 ino, u8 nr) { - skey->zone = SCOUTFS_FS_ZONE; - skey->ino = cpu_to_be64(ino); - skey->type = SCOUTFS_SYMLINK_TYPE; - skey->nr = nr; - - scoutfs_key_init(key, skey, sizeof(struct scoutfs_symlink_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .sks_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_SYMLINK_TYPE, + .sks_nr = cpu_to_le64(nr), + }; } /* @@ -991,8 +976,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, struct scoutfs_lock *lock, const char *target, size_t size) { - struct scoutfs_symlink_key skey; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct kvec val; unsigned bytes; unsigned nr; @@ -1006,7 +990,7 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, nr = DIV_ROUND_UP(size, SCOUTFS_MAX_VAL_SIZE); for (i = 0; i < nr; i++) { - init_symlink_key(&key, &skey, ino, i); + init_symlink_key(&key, ino, i); bytes = min_t(u64, size, SCOUTFS_MAX_VAL_SIZE); kvec_init(&val, (void *)target, bytes); @@ -1213,10 +1197,8 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, struct list_head *list) { struct scoutfs_link_backref_entry *ent; - struct scoutfs_dirent_key last_dkey; - struct scoutfs_dirent_key dkey; - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; + struct scoutfs_key last_key; + struct scoutfs_key key; struct scoutfs_lock *lock = NULL; struct kvec val; int len; @@ -1229,10 +1211,9 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, INIT_LIST_HEAD(&ent->head); - init_dirent_key(&key, &dkey, SCOUTFS_LINK_BACKREF_TYPE, - ino, dir_ino, dir_pos); - init_dirent_key(&last_key, &last_dkey, SCOUTFS_LINK_BACKREF_TYPE, - ino, U64_MAX, U64_MAX); + init_dirent_key(&key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, dir_pos); + init_dirent_key(&last_key, SCOUTFS_LINK_BACKREF_TYPE, ino, U64_MAX, + U64_MAX); kvec_init(&val, &ent->dent, dirent_bytes(SCOUTFS_NAME_LEN)); ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); @@ -1243,7 +1224,7 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, scoutfs_unlock(sb, lock, DLM_LOCK_PR); lock = NULL; - trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, ret, key.key_len); + trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, ret); if (ret < 0) goto out; @@ -1255,8 +1236,8 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, } list_add(&ent->head, list); - ent->dir_ino = be64_to_cpu(dkey.major); - ent->dir_pos = be64_to_cpu(dkey.minor); + ent->dir_ino = le64_to_cpu(key.skd_major); + ent->dir_pos = le64_to_cpu(key.skd_minor); ent->name_len = len; ret = 0; out: diff --git a/kmod/src/format.h b/kmod/src/format.h index 0509e647..27e467a5 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -52,6 +52,75 @@ struct scoutfs_block_header { __le64 blkno; } __packed; +/* + * scoutfs identifies all file system metadata items by a small key + * struct. + * + * Each item type maps their logical structures to the fixed fields in + * sort order. This lets us print keys without needing per-type + * formats. + * + * The keys are compared by considering the fields in struct order from + * most to least significant. They are considered a multi precision + * value when navigating the keys in ordered key space. We can + * increment them, subtract them from each other, etc. + */ +struct scoutfs_key { + __u8 sk_zone; + __le64 _sk_first; + __u8 sk_type; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; +}__packed; + +/* inode index */ +#define skii_major _sk_second +#define skii_ino _sk_third + +/* node free bit map */ +#define skf_node_id _sk_first +#define skf_base _sk_second + +/* node orphan inode */ +#define sko_node_id _sk_first +#define sko_ino _sk_second + +/* inode */ +#define ski_ino _sk_first + +/* xattr parts */ +#define skx_ino _sk_first +#define skx_name_hash _sk_second +#define skx_id _sk_third +#define skx_part _sk_fourth + +/* directory entries */ +#define skd_ino _sk_first +#define skd_major _sk_second +#define skd_minor _sk_third + +/* symlink target */ +#define sks_ino _sk_first +#define sks_nr _sk_second + +/* file data mapping */ +#define skm_ino _sk_first +#define skm_base _sk_second + +/* + * The btree still uses memcmp() to compare keys. We should fix that + * before too long. + */ +struct scoutfs_key_be { + __u8 sk_zone; + __be64 _sk_first; + __u8 sk_type; + __be64 _sk_second; + __be64 _sk_third; + __u8 _sk_fourth; +}__packed; + /* * Assert that we'll be able to represent all possible keys with 8 64bit * primary sort values. @@ -143,34 +212,24 @@ struct scoutfs_manifest { } __packed; /* - * Manifest entries are packed into btree keys and values in a very - * fiddly way so that we can sort them with memcmp first by level then - * by their position in the level. First comes the level. + * Manifest entries are split across btree keys and values. Putting + * some entry fields in the value keeps the key smaller and increases + * the fanout of the btree which keeps the tree smaller and reduces + * block IO. * - * Level 0 segments are sorted by their seq so they don't have the first - * segment key in the manifest btree key. Both of their keys are in the - * value. - * - * Level 1 segments are sorted by their first key so their last key is - * in the value. - * - * We go to all this trouble so that we can communicate a version of the - * manifest with one btree root, have dense btree keys which are used as - * seperators in parent blocks, and don't duplicate the large keys in - * the manifest btree key and value. + * The key is made up of the level, first key, and seq. At level 0 + * segments can completely overlap and have identical key ranges but we + * avoid duplicate btree keys by including the unique seq. */ - struct scoutfs_manifest_btree_key { __u8 level; - __u8 bkey[0]; + struct scoutfs_key_be first_key; + __be64 seq; } __packed; struct scoutfs_manifest_btree_val { __le64 segno; - __le64 seq; - __le16 first_key_len; - __le16 last_key_len; - __u8 keys[0]; + struct scoutfs_key last_key; } __packed; #define SCOUTFS_ALLOC_REGION_SHIFT 8 @@ -201,15 +260,12 @@ struct scoutfs_alloc_region_btree_val { * They're not allowed to cross a block boundary. */ struct scoutfs_segment_item { - __le16 key_len; + struct scoutfs_key key; __le16 val_len; __u8 flags; __u8 nr_links; __le32 skip_links[0]; - /* - * __u8 key_bytes[key_len] - * __u8 val_bytes[val_len] - */ + /* __u8 val_bytes[val_len] */ } __packed; #define SCOUTFS_ITEM_FLAG_DELETION (1 << 0) @@ -259,30 +315,6 @@ struct scoutfs_segment_block { #define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ -/* value is struct scoutfs_inode */ -struct scoutfs_inode_key { - __u8 zone; - __be64 ino; - __u8 type; -} __packed; - -/* value is struct scoutfs_dirent with the name */ -struct scoutfs_dirent_key { - __u8 zone; - __be64 ino; - __u8 type; - __be64 major; - __be64 minor; -} __packed; - -/* key is bytes of encoded block mapping */ -struct scoutfs_block_mapping_key { - __u8 zone; - __be64 ino; - __u8 type; - __be64 base; -} __packed; - /* each mapping item describes a fixed number of blocks */ #define SCOUTFS_BLOCK_MAPPING_SHIFT 6 #define SCOUTFS_BLOCK_MAPPING_BLOCKS (1 << SCOUTFS_BLOCK_MAPPING_SHIFT) @@ -328,33 +360,10 @@ struct scoutfs_block_mapping_key { #define SCOUTFS_FREE_BITS_U64S \ DIV_ROUND_UP(SCOUTFS_FREE_BITS_BITS, 64) -struct scoutfs_free_bits_key { - __u8 zone; - __be64 node_id; - __u8 type; - __be64 base; -} __packed; - struct scoutfs_free_bits { __le64 bits[SCOUTFS_FREE_BITS_U64S]; } __packed; -struct scoutfs_orphan_key { - __u8 zone; - __be64 node_id; - __u8 type; - __be64 ino; -} __packed; - -struct scoutfs_xattr_key { - __u8 zone; - __be64 ino; - __u8 type; - __be32 name_hash; - __be64 id; - __u8 part; -} __packed; - /* * The first xattr part item has a header that describes the xattr. The * name and value are then packed into the following bytes in the first @@ -366,27 +375,11 @@ struct scoutfs_xattr { __u8 name[0]; } __packed; -/* size determines nr needed to store full target path in their values */ -struct scoutfs_symlink_key { - __u8 zone; - __be64 ino; - __u8 type; - __u8 nr; -} __packed; - struct scoutfs_betimespec { __be64 sec; __be32 nsec; } __packed; -struct scoutfs_inode_index_key { - __u8 zone; - __u8 type; - __be64 major; - __be32 minor; - __be64 ino; -} __packed; - /* XXX does this exist upstream somewhere? */ #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) @@ -514,9 +507,6 @@ enum { SCOUTFS_DT_WHT, }; -#define SCOUTFS_MAX_KEY_SIZE \ - sizeof(struct scoutfs_dirent_key) - #define SCOUTFS_MAX_VAL_SIZE SCOUTFS_BLOCK_MAPPING_MAX_BYTES #define SCOUTFS_XATTR_MAX_NAME_LEN 255 @@ -591,8 +581,8 @@ struct scoutfs_net_key_range { struct scoutfs_net_manifest_entry { __le64 segno; __le64 seq; - __le16 first_key_len; - __le16 last_key_len; + struct scoutfs_key first; + struct scoutfs_key last; __u8 level; __u8 keys[0]; } __packed; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index a5ae0d0a..6708c0a8 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -247,6 +247,15 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) set_item_info(ci, cinode); } +static void init_inode_key(struct scoutfs_key *key, u64 ino) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .ski_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_INODE_TYPE, + }; +} + /* * Refresh the vfs inode fields if the lock indicates that the current * contents could be stale. @@ -263,8 +272,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_key_buf key; - struct scoutfs_inode_key ikey; + struct scoutfs_key key; struct scoutfs_inode sinode; struct kvec val; const u64 refresh_gen = lock->refresh_gen; @@ -281,7 +289,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, if (atomic64_read(&si->last_refreshed) == refresh_gen) return 0; - scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); + init_inode_key(&key, scoutfs_ino(inode)); kvec_init(&val, &sinode, sizeof(sinode)); mutex_lock(&si->item_mutex); @@ -299,16 +307,6 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, return ret; } -void scoutfs_inode_init_key(struct scoutfs_key_buf *key, - struct scoutfs_inode_key *ikey, u64 ino) -{ - ikey->zone = SCOUTFS_FS_ZONE; - ikey->ino = cpu_to_be64(ino); - ikey->type = SCOUTFS_INODE_TYPE; - - scoutfs_key_init(key, ikey, sizeof(struct scoutfs_inode_key)); -} - int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat) { @@ -694,14 +692,13 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct scoutfs_inode sinode; int ret; store_inode(&sinode, inode); - scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); + init_inode_key(&key, scoutfs_ino(inode)); ret = scoutfs_item_dirty(sb, &key, lock); if (!ret) @@ -759,13 +756,13 @@ static int cmp_index_lock(void *priv, struct list_head *A, struct list_head *B) static void clamp_inode_index(u8 type, u64 *major, u32 *minor, u64 *ino) { - struct scoutfs_inode_index_key start; + struct scoutfs_key start; scoutfs_lock_get_index_item_range(type, *major, *ino, &start, NULL); - *major = be64_to_cpu(start.major); - *minor = be32_to_cpu(start.minor); - *ino = be64_to_cpu(start.ino); + *major = le64_to_cpu(start.skii_major); + *minor = 0; + *ino = le64_to_cpu(start.skii_ino); } /* @@ -799,6 +796,17 @@ static struct scoutfs_lock *find_index_lock(struct list_head *lock_list, return NULL; } +void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, + u32 minor, u64 ino) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_INODE_INDEX_ZONE, + .sk_type = type, + .skii_major = cpu_to_le64(major), + .skii_ino = cpu_to_le64(ino), + }; +} + /* * The inode info reflects the current inode index items. Create or delete * index items to bring the index in line with the caller's item. The list @@ -809,12 +817,10 @@ static int update_index_items(struct super_block *sb, u64 major, u32 minor, struct list_head *lock_list) { - struct scoutfs_inode_index_key ins_ikey; - struct scoutfs_inode_index_key del_ikey; struct scoutfs_lock *ins_lock; struct scoutfs_lock *del_lock; - struct scoutfs_key_buf ins; - struct scoutfs_key_buf del; + struct scoutfs_key ins; + struct scoutfs_key del; int ret; int err; @@ -823,12 +829,7 @@ static int update_index_items(struct super_block *sb, trace_scoutfs_create_index_item(sb, type, major, minor, ino); - ins_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - ins_ikey.type = type; - ins_ikey.major = cpu_to_be64(major); - ins_ikey.minor = cpu_to_be32(minor); - ins_ikey.ino = cpu_to_be64(ino); - scoutfs_key_init(&ins, &ins_ikey, sizeof(ins_ikey)); + scoutfs_inode_init_index_key(&ins, type, major, minor, ino); ins_lock = find_index_lock(lock_list, type, major, minor, ino); ret = scoutfs_item_create_force(sb, &ins, NULL, ins_lock); @@ -838,12 +839,8 @@ static int update_index_items(struct super_block *sb, trace_scoutfs_delete_index_item(sb, type, si->item_majors[type], si->item_minors[type], ino); - del_ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - del_ikey.type = type; - del_ikey.major = cpu_to_be64(si->item_majors[type]); - del_ikey.minor = cpu_to_be32(si->item_minors[type]); - del_ikey.ino = cpu_to_be64(ino); - scoutfs_key_init(&del, &del_ikey, sizeof(del_ikey)); + scoutfs_inode_init_index_key(&del, type, si->item_majors[type], + si->item_minors[type], ino); del_lock = find_index_lock(lock_list, type, si->item_majors[type], si->item_minors[type], ino); @@ -906,8 +903,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct scoutfs_inode sinode; struct kvec val; int ret; @@ -924,7 +920,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, ret = update_indices(sb, si, ino, inode->i_mode, &sinode, lock_list); BUG_ON(ret); - scoutfs_inode_init_key(&key, &ikey, ino); + init_inode_key(&key, ino); kvec_init(&val, &sinode, sizeof(sinode)); err = scoutfs_item_update(sb, &key, &val, lock); @@ -1195,17 +1191,11 @@ void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list) static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, u32 minor, struct list_head *ind_locks) { - struct scoutfs_inode_index_key ikey; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct scoutfs_lock *lock; int ret; - ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - ikey.type = type; - ikey.major = cpu_to_be64(major); - ikey.minor = cpu_to_be32(minor); - ikey.ino = cpu_to_be64(ino); - scoutfs_key_init(&key, &ikey, sizeof(ikey)); + scoutfs_inode_init_index_key(&key, type, major, minor, ino); lock = find_index_lock(ind_locks, type, major, minor, ino); ret = scoutfs_item_delete_force(sb, &key, lock); @@ -1311,8 +1301,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, struct scoutfs_lock *lock) { struct scoutfs_inode_info *ci; - struct scoutfs_inode_key ikey; - struct scoutfs_key_buf key; + struct scoutfs_key key; struct scoutfs_inode sinode; struct inode *inode; struct kvec val; @@ -1346,7 +1335,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, set_inode_ops(inode); store_inode(&sinode, inode); - scoutfs_inode_init_key(&key, &ikey, scoutfs_ino(inode)); + init_inode_key(&key, scoutfs_ino(inode)); kvec_init(&val, &sinode, sizeof(sinode)); ret = scoutfs_item_create(sb, &key, &val, lock); @@ -1358,26 +1347,24 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, return inode; } -static void init_orphan_key(struct scoutfs_key_buf *key, - struct scoutfs_orphan_key *okey, u64 node_id, u64 ino) +static void init_orphan_key(struct scoutfs_key *key, u64 node_id, u64 ino) { - okey->zone = SCOUTFS_NODE_ZONE; - okey->node_id = cpu_to_be64(node_id); - okey->type = SCOUTFS_ORPHAN_TYPE; - okey->ino = cpu_to_be64(ino); - - scoutfs_key_init(key, okey, sizeof(struct scoutfs_orphan_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_NODE_ZONE, + .sko_node_id = cpu_to_le64(node_id), + .sk_type = SCOUTFS_ORPHAN_TYPE, + .sko_ino = cpu_to_le64(ino), + }; } static int remove_orphan_item(struct super_block *sb, u64 ino) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_orphan_key okey; - struct scoutfs_key_buf key; + struct scoutfs_key key; int ret; - init_orphan_key(&key, &okey, sbi->node_id, ino); + init_orphan_key(&key, sbi->node_id, ino); ret = scoutfs_item_delete(sb, &key, lock); if (ret == -ENOENT) @@ -1397,9 +1384,8 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) static int delete_inode_items(struct super_block *sb, u64 ino) { struct scoutfs_lock *lock = NULL; - struct scoutfs_inode_key ikey; struct scoutfs_inode sinode; - struct scoutfs_key_buf key; + struct scoutfs_key key; LIST_HEAD(ind_locks); bool release = false; struct kvec val; @@ -1411,7 +1397,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) if (ret) return ret; - scoutfs_inode_init_key(&key, &ikey, ino); + init_inode_key(&key, ino); kvec_init(&val, &sinode, sizeof(sinode)); ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); @@ -1520,30 +1506,32 @@ int scoutfs_scan_orphans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_orphan_key okey; - struct scoutfs_orphan_key last_okey; - struct scoutfs_key_buf key; - struct scoutfs_key_buf last; + struct scoutfs_key key; + struct scoutfs_key last; int err = 0; int ret; trace_scoutfs_scan_orphans(sb); - init_orphan_key(&key, &okey, sbi->node_id, 0); - init_orphan_key(&last, &last_okey, sbi->node_id, ~0ULL); + init_orphan_key(&key, sbi->node_id, 0); + init_orphan_key(&last, sbi->node_id, ~0ULL); while (1) { - ret = scoutfs_item_next_same(sb, &key, &last, NULL, lock); + ret = scoutfs_item_next(sb, &key, &last, NULL, lock); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) goto out; - ret = delete_inode_items(sb, be64_to_cpu(okey.ino)); + ret = delete_inode_items(sb, le64_to_cpu(key.sko_ino)); if (ret && ret != -ENOENT && !err) err = ret; - scoutfs_key_inc_cur_len(&key); + if (le64_to_cpu(key.sko_ino) == U64_MAX) { + ret = -ENOENT; + break; + } + le64_add_cpu(&key.sko_ino, 1); } ret = 0; @@ -1556,13 +1544,12 @@ int scoutfs_orphan_inode(struct inode *inode) struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_orphan_key okey; - struct scoutfs_key_buf key; + struct scoutfs_key key; int ret; trace_scoutfs_orphan_inode(sb, inode); - init_orphan_key(&key, &okey, sbi->node_id, scoutfs_ino(inode)); + init_orphan_key(&key, sbi->node_id, scoutfs_ino(inode)); ret = scoutfs_item_create(sb, &key, NULL, lock); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 39313955..d46f24d1 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -5,6 +5,7 @@ #include "lock.h" #include "per_task.h" #include "count.h" +#include "format.h" struct scoutfs_lock; @@ -62,9 +63,6 @@ static inline u64 scoutfs_ino(struct inode *inode) return SCOUTFS_I(inode)->ino; } -void scoutfs_inode_init_key(struct scoutfs_key_buf *key, - struct scoutfs_inode_key *ikey, u64 ino); - struct inode *scoutfs_alloc_inode(struct super_block *sb); void scoutfs_destroy_inode(struct inode *inode); int scoutfs_drop_inode(struct inode *inode); @@ -74,6 +72,8 @@ int scoutfs_orphan_inode(struct inode *inode); struct inode *scoutfs_iget(struct super_block *sb, u64 ino); struct inode *scoutfs_ilookup(struct super_block *sb, u64 ino); +void scoutfs_inode_init_index_key(struct scoutfs_key *key, u8 type, u64 major, + u32 minor, u64 ino); int scoutfs_inode_index_start(struct super_block *sb, u64 *seq); int scoutfs_inode_index_prepare(struct super_block *sb, struct list_head *list, struct inode *inode, bool set_data_seq); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index fdfabab6..bd135650 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -55,11 +55,9 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) struct scoutfs_ioctl_walk_inodes __user *uwalk = (void __user *)arg; struct scoutfs_ioctl_walk_inodes walk; struct scoutfs_ioctl_walk_inodes_entry ent; - struct scoutfs_inode_index_key last_ikey; - struct scoutfs_inode_index_key ikey; - struct scoutfs_key_buf *next_key; - struct scoutfs_key_buf last_key; - struct scoutfs_key_buf key; + struct scoutfs_key next_key; + struct scoutfs_key last_key; + struct scoutfs_key key; struct scoutfs_lock *lock; u64 last_seq; int ret = 0; @@ -93,23 +91,10 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) } } - next_key = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!next_key) - return -ENOMEM; - - ikey.zone = SCOUTFS_INODE_INDEX_ZONE; - ikey.type = type; - ikey.major = cpu_to_be64(walk.first.major); - ikey.minor = cpu_to_be32(walk.first.minor); - ikey.ino = cpu_to_be64(walk.first.ino); - scoutfs_key_init(&key, &ikey, sizeof(ikey)); - - last_ikey.zone = ikey.zone; - last_ikey.type = ikey.type; - last_ikey.major = cpu_to_be64(walk.last.major); - last_ikey.minor = cpu_to_be32(walk.last.minor); - last_ikey.ino = cpu_to_be64(walk.last.ino); - scoutfs_key_init(&last_key, &last_ikey, sizeof(last_ikey)); + scoutfs_inode_init_index_key(&key, type, walk.first.major, + walk.first.minor, walk.first.ino); + scoutfs_inode_init_index_key(&last_key, type, walk.last.major, + walk.last.minor, walk.last.ino); /* cap nr to the max the ioctl can return to a compat task */ walk.nr_entries = min_t(u64, walk.nr_entries, INT_MAX); @@ -121,21 +106,21 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) for (nr = 0; nr < walk.nr_entries; ) { - ret = scoutfs_item_next_same(sb, &key, &last_key, NULL, lock); + ret = scoutfs_item_next(sb, &key, &last_key, NULL, lock); if (ret < 0 && ret != -ENOENT) break; if (ret == -ENOENT) { /* done if lock covers last iteration key */ - if (scoutfs_key_compare(&last_key, lock->end) <= 0) { + if (scoutfs_key_compare(&last_key, &lock->end) <= 0) { ret = 0; break; } /* continue iterating after locked empty region */ - scoutfs_key_copy(&key, lock->end); - scoutfs_key_inc_cur_len(&key); + key = lock->end; + scoutfs_key_inc(&key); scoutfs_unlock(sb, lock, DLM_LOCK_PR); @@ -146,37 +131,32 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) * It'd mean adding a lock to the inode index * items which isn't quite there yet. */ - ret = scoutfs_manifest_next_key(sb, &key, next_key); + ret = scoutfs_manifest_next_key(sb, &key, &next_key); if (ret < 0 && ret != -ENOENT) goto out; if (ret == -ENOENT || - scoutfs_key_compare(next_key, &last_key) > 0) { + scoutfs_key_compare(&next_key, &last_key) > 0) { ret = 0; goto out; } - /* if it's within last it should be same size */ - if (next_key->key_len != key.key_len) { - ret = -EIO; - goto out; - } + key = next_key; - scoutfs_key_copy(&key, next_key); - - ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, ikey.type, - be64_to_cpu(ikey.major), - be64_to_cpu(ikey.ino), - &lock); + ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, + key.sk_type, + le64_to_cpu(key.skii_major), + le64_to_cpu(key.skii_ino), + &lock); if (ret < 0) goto out; continue; } - ent.major = be64_to_cpu(ikey.major); - ent.minor = be32_to_cpu(ikey.minor); - ent.ino = be64_to_cpu(ikey.ino); + ent.major = le64_to_cpu(key.skii_major); + ent.minor = 0; + ent.ino = le64_to_cpu(key.skii_ino); if (copy_to_user((void __user *)walk.entries_ptr, &ent, sizeof(ent))) { @@ -187,14 +167,12 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) nr++; walk.entries_ptr += sizeof(ent); - scoutfs_key_inc_cur_len(&key); + scoutfs_key_inc(&key); } scoutfs_unlock(sb, lock, DLM_LOCK_PR); out: - scoutfs_key_free(sb, next_key); - if (nr > 0) ret = nr; @@ -499,66 +477,47 @@ static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; struct scoutfs_ioctl_item_cache_keys ick; - struct scoutfs_key_buf *key; - struct page *page; - unsigned bytes; - void *buf; + struct scoutfs_key __user *ukeys; + struct scoutfs_key keys[16]; + unsigned int nr; int total; int ret; if (copy_from_user(&ick, (void __user *)arg, sizeof(ick))) return -EFAULT; - if ((!!ick.key_ptr != !!ick.key_len) || - ick.key_len > SCOUTFS_MAX_KEY_SIZE || - ick.which > SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES) + if (ick.which > SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES) return -EINVAL; - /* don't overflow signed 32bit syscall return longs */ - ick.buf_len = min_t(u64, ick.buf_len, S32_MAX); - - key = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - page = alloc_page(GFP_KERNEL); - if (!key || !page) { - ret = -ENOMEM; - goto out; - } - - if (copy_from_user(key->data, (void __user *)ick.key_ptr, ick.key_len)) { - ret = -EFAULT; - goto out; - } - scoutfs_key_init_buf_len(key, key->data, ick.key_len, - SCOUTFS_MAX_KEY_SIZE); - scoutfs_key_inc(key); - - buf = page_address(page); + ukeys = (void __user *)(long)ick.buf_ptr; total = 0; ret = 0; - while (ick.buf_len) { - bytes = min_t(u64, ick.buf_len, PAGE_SIZE); + while (ick.buf_nr) { + nr = min_t(size_t, ick.buf_nr, ARRAY_SIZE(keys)); if (ick.which == SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS) - ret = scoutfs_item_copy_keys(sb, key, buf, bytes); + ret = scoutfs_item_copy_keys(sb, &ick.key, keys, nr); else - ret = scoutfs_item_copy_range_keys(sb, key, buf, bytes); - - if (ret > 0 && copy_to_user((void __user *)ick.buf_ptr, buf, ret)) - ret = -EFAULT; + ret = scoutfs_item_copy_range_keys(sb, &ick.key, keys, + nr); + BUG_ON(ret > nr); /* stack overflow \o/ */ if (ret <= 0) break; - ick.buf_len -= ret; - ick.buf_ptr += ret; + if (copy_to_user(ukeys, keys, ret * sizeof(keys[0]))) { + ret = -EFAULT; + break; + } + + ick.key = keys[ret - 1]; + scoutfs_key_inc(&ick.key); + + ukeys += ret; + ick.buf_nr -= ret; total += ret; ret = 0; } -out: - scoutfs_key_free(sb, key); - if (page) - __free_page(page); - return ret ?: total; } diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 721f1cde..915a130b 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -208,11 +208,16 @@ struct scoutfs_ioctl_stat_more { #define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_stat_more) +/* + * Fills the buffer with either the keys for the cached items or the + * keys for the cached ranges found starting with the given key. The + * number of keys filled in the buffer is returned. When filling range + * keys the returned number will always be a multiple of two. + */ struct scoutfs_ioctl_item_cache_keys { - __u64 key_ptr; - __u64 key_len; + struct scoutfs_key key; __u64 buf_ptr; - __u64 buf_len; + __u16 buf_nr; __u8 which; } __packed; diff --git a/kmod/src/item.c b/kmod/src/item.c index af78d0f2..86697a31 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -41,10 +41,9 @@ * clobber them in creation and skip them in lookups. */ -static bool invalid_key_val(struct scoutfs_key_buf *key, struct kvec *val) +static bool invalid_key_val(struct scoutfs_key *key, struct kvec *val) { - return WARN_ON_ONCE(key->key_len > SCOUTFS_MAX_KEY_SIZE || - (val && (val->iov_len > SCOUTFS_MAX_VAL_SIZE))); + return WARN_ON_ONCE(val && (val->iov_len > SCOUTFS_MAX_VAL_SIZE)); } struct item_cache { @@ -55,7 +54,6 @@ struct item_cache { struct rb_root ranges; long nr_dirty_items; - long dirty_key_bytes; long dirty_val_bytes; struct shrinker shrinker; @@ -78,7 +76,7 @@ struct cached_item { long dirty; unsigned deletion:1; - struct scoutfs_key_buf *key; + struct scoutfs_key key; void *val; unsigned int val_len; }; @@ -86,12 +84,12 @@ struct cached_item { struct cached_range { struct rb_node node; - struct scoutfs_key_buf *start; - struct scoutfs_key_buf *end; + struct scoutfs_key start; + struct scoutfs_key end; }; #define trace_range(which, sb, rng) \ - trace_scoutfs_item_range_##which(sb, (rng), (rng)->start, (rng)->end) + trace_scoutfs_item_range_##which(sb, (rng), &(rng)->start, &(rng)->end) static u8 item_flags(struct cached_item *item) { @@ -104,7 +102,6 @@ static void free_item(struct super_block *sb, struct cached_item *item) scoutfs_inc_counter(sb, item_free); WARN_ON_ONCE(!list_empty(&item->entry)); WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); - scoutfs_key_free(sb, item->key); kfree(item->val); kfree(item); } @@ -116,33 +113,32 @@ static void free_item(struct super_block *sb, struct cached_item *item) * them in place when updating items. */ static struct cached_item *alloc_item(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct kvec *val) { struct cached_item *item; item = kzalloc(sizeof(struct cached_item), GFP_NOFS); - if (item) { - RB_CLEAR_NODE(&item->node); - INIT_LIST_HEAD(&item->entry); + if (!item) + goto out; - item->key = scoutfs_key_dup(sb, key); - if (val) { - item->val = kmalloc(val->iov_len, GFP_NOFS); - item->val_len = val->iov_len; - if (item->val) - memcpy(item->val, val->iov_base, val->iov_len); - } + item->key = *key; + RB_CLEAR_NODE(&item->node); + INIT_LIST_HEAD(&item->entry); - if (!item->key || (val && !item->val)) { + if (val) { + item->val = kmalloc(val->iov_len, GFP_NOFS); + if (!item->val) { free_item(sb, item); item = NULL; + goto out; } + item->val_len = val->iov_len; + memcpy(item->val, val->iov_base, val->iov_len); } - if (item) - scoutfs_inc_counter(sb, item_alloc); - + scoutfs_inc_counter(sb, item_alloc); +out: return item; } @@ -170,7 +166,7 @@ static int copy_item_val(struct kvec *val, struct cached_item *item) * prev items. */ static struct cached_item *walk_items(struct rb_root *root, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct cached_item **prev, struct cached_item **next) { @@ -184,7 +180,7 @@ static struct cached_item *walk_items(struct rb_root *root, while (node) { item = container_of(node, struct cached_item, node); - cmp = scoutfs_key_compare(key, item->key); + cmp = scoutfs_key_compare(key, &item->key); if (cmp < 0) { *next = item; node = node->rb_left; @@ -209,7 +205,7 @@ static struct cached_item *walk_items(struct rb_root *root, */ static struct cached_item *find_item(struct super_block *sb, struct rb_root *root, - struct scoutfs_key_buf *key) + struct scoutfs_key *key) { struct cached_item *prev; struct cached_item *next; @@ -229,7 +225,7 @@ static struct cached_item *find_item(struct super_block *sb, } static struct cached_item *next_item(struct rb_root *root, - struct scoutfs_key_buf *key) + struct scoutfs_key *key) { struct cached_item *prev; struct cached_item *next; @@ -342,16 +338,15 @@ static void update_dirty_parents(struct cached_item *item) } static void update_dirty_item_counts(struct super_block *sb, signed items, - signed keys, signed vals) + signed vals) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; cac->nr_dirty_items += items; - cac->dirty_key_bytes += keys; cac->dirty_val_bytes += vals; - scoutfs_trans_track_item(sb, items, keys, vals); + scoutfs_trans_track_item(sb, items, vals); } static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, @@ -367,7 +362,7 @@ static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, list_del_init(&item->entry); cac->lru_nr--; - update_dirty_item_counts(sb, 1, item->key->key_len, item->val_len); + update_dirty_item_counts(sb, 1, item->val_len); update_dirty_parents(item); } @@ -384,10 +379,9 @@ static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, list_add_tail(&item->entry, &cac->lru_list); cac->lru_nr++; - update_dirty_item_counts(sb, -1, -item->key->key_len, -item->val_len); + update_dirty_item_counts(sb, -1, -item->val_len); - WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_key_bytes < 0 || - cac->dirty_val_bytes < 0); + WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_val_bytes < 0); update_dirty_parents(item); } @@ -477,7 +471,7 @@ restart: parent = *node; item = container_of(*node, struct cached_item, node); - cmp = scoutfs_key_compare(ins->key, item->key); + cmp = scoutfs_key_compare(&ins->key, &item->key); if (cmp < 0) { if (ins->dirty) item->dirty |= LEFT_DIRTY; @@ -497,7 +491,7 @@ restart: } } - trace_scoutfs_item_insertion(sb, ins->key); + trace_scoutfs_item_insertion(sb, &ins->key); rb_link_node(&ins->node, parent, node); rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); @@ -530,7 +524,7 @@ static struct cached_range *rb_next_rng(struct cached_range *rng) } static struct cached_range *walk_ranges(struct rb_root *root, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct cached_range **prev, struct cached_range **next) { @@ -547,7 +541,7 @@ static struct cached_range *walk_ranges(struct rb_root *root, rng = container_of(node, struct cached_range, node); cmp = scoutfs_key_compare_ranges(key, key, - rng->start, rng->end); + &rng->start, &rng->end); if (cmp < 0) { if (next) *next = rng; @@ -573,8 +567,8 @@ static struct cached_range *walk_ranges(struct rb_root *root, * cached range. */ static bool check_range(struct super_block *sb, struct rb_root *root, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end) + struct scoutfs_key *key, + struct scoutfs_key *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -585,15 +579,15 @@ static bool check_range(struct super_block *sb, struct rb_root *root, if (rng) { scoutfs_inc_counter(sb, item_range_hit); if (end) - scoutfs_key_copy(end, rng->end); + *end = rng->end; return true; } if (end) { if (next) - scoutfs_key_copy(end, next->start); + *end = next->start; else - scoutfs_key_set_max(end); + scoutfs_key_set_ones(end); } scoutfs_inc_counter(sb, item_range_miss); @@ -605,8 +599,6 @@ static void free_range(struct super_block *sb, struct cached_range *rng) if (!IS_ERR_OR_NULL(rng)) { scoutfs_inc_counter(sb, item_range_free); trace_range(free, sb, rng); - scoutfs_key_free(sb, rng->start); - scoutfs_key_free(sb, rng->end); kfree(rng); } } @@ -638,8 +630,8 @@ restart: parent = *node; rng = container_of(*node, struct cached_range, node); - cmp = scoutfs_key_compare_ranges(ins->start, ins->end, - rng->start, rng->end); + cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, + &rng->start, &rng->end); /* simple iteration until we overlap */ if (cmp < 0) { node = &(*node)->rb_left; @@ -649,8 +641,8 @@ restart: continue; } - start_cmp = scoutfs_key_compare(ins->start, rng->start); - end_cmp = scoutfs_key_compare(ins->end, rng->end); + start_cmp = scoutfs_key_compare(&ins->start, &rng->start); + end_cmp = scoutfs_key_compare(&ins->end, &rng->end); /* free our insertion if we're entirely within an existing */ if (start_cmp >= 0 && end_cmp <= 0) { @@ -709,8 +701,8 @@ restart: parent = *node; rng = container_of(*node, struct cached_range, node); - cmp = scoutfs_key_compare_ranges(rem->start, rem->end, - rng->start, rng->end); + cmp = scoutfs_key_compare_ranges(&rem->start, &rem->end, + &rng->start, &rng->end); /* simple iteration until we overlap */ if (cmp < 0) { node = &(*node)->rb_left; @@ -720,17 +712,17 @@ restart: continue; } - start_cmp = scoutfs_key_compare(rem->start, rng->start); - end_cmp = scoutfs_key_compare(rem->end, rng->end); + start_cmp = scoutfs_key_compare(&rem->start, &rng->start); + end_cmp = scoutfs_key_compare(&rem->end, &rng->end); /* remove the middle of an existing range, insert other half */ if (start_cmp > 0 && end_cmp < 0) { swap(rng->end, rem->start); - scoutfs_key_dec(rng->end); + scoutfs_key_dec(&rng->end); trace_range(remove_mid_left, sb, rng); swap(rem->start, rem->end); - scoutfs_key_inc(rem->start); + scoutfs_key_inc(&rem->start); insert = true; goto restart; } @@ -738,14 +730,14 @@ restart: /* remove partial overlap from existing */ if (start_cmp < 0 && end_cmp < 0) { swap(rem->end, rng->start); - scoutfs_key_inc(rng->start); + scoutfs_key_inc(&rng->start); trace_range(remove_start, sb, rng); continue; } if (start_cmp > 0 && end_cmp > 0) { swap(rem->start, rng->end); - scoutfs_key_dec(rng->end); + scoutfs_key_dec(&rng->end); trace_range(remove_end, sb, rng); continue; } @@ -765,26 +757,16 @@ restart: } } -/* - * Return true if the lock protects the use of the key. Some locks not - * intended for item use don't have a key range and we want to safely - * detect that. The lock mode dereference is racy but the field always - * contains a single non-zero byte. - */ +/* Return true if the lock protects the use of the key. */ static bool lock_coverage(struct scoutfs_lock *lock, - struct scoutfs_key_buf *key, int op_mode) + struct scoutfs_key *key, int op_mode) { - signed char mode; - - if (!lock || !lock->start || !lock->end) - return false; - - mode = ACCESS_ONCE(lock->granted_mode); + signed char mode = ACCESS_ONCE(lock->granted_mode); return ((op_mode == mode) || (op_mode == DLM_LOCK_PR && mode == DLM_LOCK_EX)) && scoutfs_key_compare_ranges(key, key, - lock->start, lock->end) == 0; + &lock->start, &lock->end) == 0; } /* @@ -795,7 +777,7 @@ static bool lock_coverage(struct scoutfs_lock *lock, * The end key limits how many keys after the search key can be read * and inserted into the cache. */ -int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -828,8 +810,8 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); trace_scoutfs_item_lookup_ret(sb, ret); return ret; @@ -849,7 +831,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, * Returns 0 or -errno. */ int scoutfs_item_lookup_exact(struct super_block *sb, - struct scoutfs_key_buf *key, struct kvec *val, + struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) { int ret; @@ -869,7 +851,7 @@ int scoutfs_item_lookup_exact(struct super_block *sb, */ static struct cached_item *next_item_node(struct rb_root *root, struct cached_item *item, - struct scoutfs_key_buf *last) + struct scoutfs_key *last) { struct rb_node *node; @@ -882,7 +864,7 @@ static struct cached_item *next_item_node(struct rb_root *root, item = container_of(node, struct cached_item, node); - if (scoutfs_key_compare(item->key, last) > 0) { + if (scoutfs_key_compare(&item->key, last) > 0) { item = NULL; break; } @@ -900,9 +882,9 @@ static struct cached_item *next_item_node(struct rb_root *root, * bounds of the end of the cache and the caller's last key. */ static struct cached_item *item_for_next(struct rb_root *root, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *range_end, - struct scoutfs_key_buf *last) + struct scoutfs_key *key, + struct scoutfs_key *range_end, + struct scoutfs_key *last) { struct cached_item *item; @@ -912,7 +894,7 @@ static struct cached_item *item_for_next(struct rb_root *root, item = next_item(root, key); if (item) { - if (scoutfs_key_compare(item->key, last) > 0) + if (scoutfs_key_compare(&item->key, last) > 0) item = NULL; else if (item->deletion) item = next_item_node(root, item, last); @@ -941,22 +923,22 @@ static struct cached_item *item_for_next(struct rb_root *root, * of value bytes copied is returned. The copied value can be truncated * by the caller's value buffer length. */ -int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val, +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *pos = NULL; - struct scoutfs_key_buf *range_end = NULL; + struct scoutfs_key pos; + struct scoutfs_key range_end; struct cached_item *item; unsigned long flags; bool cached; int ret; /* use the end key as the last key if it's closer to reduce compares */ - if (scoutfs_key_compare(lock->end, last) < 0) - last = lock->end; + if (scoutfs_key_compare(&lock->end, last) < 0) + last = &lock->end; /* convenience to avoid searching if caller iterates past their last */ if (scoutfs_key_compare(key, last) > 0) { @@ -969,31 +951,25 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, goto out; } - pos = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - range_end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - if (!pos || !range_end) { - ret = -ENOMEM; - goto out; - } - - scoutfs_key_copy(pos, key); + pos = *key; spin_lock_irqsave(&cac->lock, flags); for(;;) { /* see if we have cache coverage of our iterator pos */ - cached = check_range(sb, &cac->ranges, pos, range_end); + cached = check_range(sb, &cac->ranges, &pos, &range_end); trace_scoutfs_item_next_range_check(sb, !!cached, key, - pos, last, lock->end, - range_end); + &pos, last, &lock->end, + &range_end); if (!cached) { /* populate missing cached range starting at pos */ spin_unlock_irqrestore(&cac->lock, flags); - ret = scoutfs_manifest_read_items(sb, pos, lock->start, - lock->end); + ret = scoutfs_manifest_read_items(sb, &pos, + &lock->start, + &lock->end); spin_lock_irqsave(&cac->lock, flags); if (ret) @@ -1003,12 +979,12 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, } /* see if there's an item in the cached range from pos */ - item = item_for_next(&cac->items, pos, range_end, last); + item = item_for_next(&cac->items, &pos, &range_end, last); if (!item) { - if (scoutfs_key_compare(range_end, last) < 0) { + if (scoutfs_key_compare(&range_end, last) < 0) { /* keep searching after empty cached range */ - scoutfs_key_copy(pos, range_end); - scoutfs_key_inc(pos); + pos = range_end; + scoutfs_key_inc(&pos); continue; } @@ -1018,7 +994,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, } /* we have a next item inside the cached range, done */ - scoutfs_key_copy(key, item->key); + *key = item->key; if (val) { item_referenced(cac, item); ret = copy_item_val(val, item); @@ -1030,71 +1006,17 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); out: - scoutfs_key_free(sb, pos); - scoutfs_key_free(sb, range_end); trace_scoutfs_item_next_ret(sb, ret); return ret; } -/* - * Like _next but requires that the found keys be the same length as the - * search key and that values be of at least a minimum size. It treats - * size mismatches as a sign of corruption and returns -EIO. - */ -int scoutfs_item_next_same_min(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, - struct kvec *val, int len, - struct scoutfs_lock *lock) -{ - int key_len = key->key_len; - int ret; - - trace_scoutfs_item_next_same_min(sb, key_len, len); - - if (WARN_ON_ONCE(!val || val->iov_len < len)) - return -EINVAL; - - ret = scoutfs_item_next(sb, key, last, val, lock); - if (ret >= 0 && (key->key_len != key_len || ret < len)) - ret = -EIO; - - trace_scoutfs_item_next_same_min_ret(sb, ret); - - return ret; -} - -/* - * Like _next but requires that the found keys be the same length as the - * search key. It treats size mismatches as a sign of corruption. - */ -int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_lock *lock) -{ - int key_len = key->key_len; - int ret; - - trace_scoutfs_item_next_same(sb, key_len); - - ret = scoutfs_item_next(sb, key, last, val, lock); - if (ret >= 0 && (key->key_len != key_len)) - ret = -EIO; - - trace_scoutfs_item_next_same_ret(sb, ret); - - return ret; -} - /* * Create a new dirty item in the cache. Returns -EEXIST if an item * already exists with the given key. - * - * XXX but it doesn't read.. is that weird? Seems weird. */ -int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, - struct kvec *val, struct scoutfs_lock *lock) +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1128,8 +1050,8 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); if (ret) free_item(sb, item); @@ -1138,7 +1060,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, } int scoutfs_item_create_force(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1161,10 +1083,9 @@ int scoutfs_item_create_force(struct super_block *sb, ret = insert_item(sb, cac, item, true, false); if (ret) { - SK_PRINTK(KERN_EMERG "Scoutfs: corrupted item cache found while" - " creating item "SK_FMT" on fs %llu\n", - SK_ARG(key), - le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); + printk(KERN_EMERG "Scoutfs: corrupted item cache found while" + " creating item "SK_FMT" on fs %llu\n", SK_ARG(key), + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); BUG_ON(ret); } scoutfs_inc_counter(sb, item_create); @@ -1184,7 +1105,7 @@ int scoutfs_item_create_force(struct super_block *sb, * and we add with _tail to maintain that order. */ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *key, struct kvec *val) + struct scoutfs_key *key, struct kvec *val) { struct cached_item *item; int ret; @@ -1220,8 +1141,8 @@ int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, * that will be inserted. */ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_key *start, + struct scoutfs_key *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1238,16 +1159,15 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, scoutfs_inc_counter(sb, item_range_alloc); rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); - if (rng) { - rng->start = scoutfs_key_dup(sb, start); - rng->end = scoutfs_key_dup(sb, end); - } - if (!rng || !rng->start || !rng->end) { + if (!rng) { free_range(sb, rng); ret = -ENOMEM; goto out; } + rng->start = *start; + rng->end = *end; + spin_lock_irqsave(&cac->lock, flags); insert_range(sb, &cac->ranges, rng); @@ -1286,7 +1206,7 @@ void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) * If the item exists make sure it's dirty and pinned. It can be read * if it wasn't cached. -ENOENT is returned if the item doesn't exist. */ -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1314,8 +1234,8 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); trace_scoutfs_item_dirty_ret(sb, ret); return ret; @@ -1327,7 +1247,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, * * Returns -ENOENT if the item doesn't exist. */ -int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1371,8 +1291,8 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); out: kfree(up_val); @@ -1392,7 +1312,7 @@ out: * there are any ways for userspace to overwhelm the system with * deletion items for items that didn't exist in the first place. */ -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1420,15 +1340,15 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); trace_scoutfs_item_delete_ret(sb, ret); return ret; } int scoutfs_item_delete_force(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1447,10 +1367,9 @@ int scoutfs_item_delete_force(struct super_block *sb, spin_lock_irqsave(&cac->lock, flags); ret = insert_item(sb, cac, item, true, false); if (ret) { - SK_PRINTK(KERN_EMERG "Scoutfs: corrupted item cache found while" - " deleting item "SK_FMT" on fs %llu\n", - SK_ARG(key), - le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); + printk(KERN_EMERG "Scoutfs: corrupted item cache found while" + " deleting item "SK_FMT" on fs %llu\n", SK_ARG(key), + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); BUG_ON(ret); } scoutfs_inc_counter(sb, item_create); @@ -1473,7 +1392,7 @@ int scoutfs_item_delete_force(struct super_block *sb, * Returns -ENOENT if the item didn't exist and couldn't be deleted. */ int scoutfs_item_delete_save(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct list_head *list, struct scoutfs_lock *lock) { @@ -1517,8 +1436,8 @@ int scoutfs_item_delete_save(struct super_block *sb, spin_unlock_irqrestore(&cac->lock, flags); } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, lock->start, - lock->end)) == 0); + (ret = scoutfs_manifest_read_items(sb, key, &lock->start, + &lock->end)) == 0); free_item(sb, del); @@ -1554,8 +1473,8 @@ int scoutfs_item_restore(struct super_block *sb, struct list_head *list, /* make sure all the items are locked and cached */ list_for_each_entry(item, list, entry) { mode = item_is_dirty(item) ? DLM_LOCK_EX : DLM_LOCK_PR; - if (WARN_ON_ONCE(!lock_coverage(lock, item->key, mode)) || - WARN_ON_ONCE(!check_range(sb, &cac->ranges, item->key, + if (WARN_ON_ONCE(!lock_coverage(lock, &item->key, mode)) || + WARN_ON_ONCE(!check_range(sb, &cac->ranges, &item->key, NULL))) { ret = -EINVAL; goto out; @@ -1567,7 +1486,7 @@ int scoutfs_item_restore(struct super_block *sb, struct list_head *list, item->dirty &= ~ITEM_DIRTY; list_del_init(&item->entry); - existing = find_item(sb, &cac->items, item->key); + existing = find_item(sb, &cac->items, &item->key); if (existing) erase_item(sb, cac, existing); insert_item(sb, cac, item, false, false); @@ -1588,7 +1507,7 @@ out: * fail. */ void scoutfs_item_delete_dirty(struct super_block *sb, - struct scoutfs_key_buf *key) + struct scoutfs_key *key) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1612,7 +1531,7 @@ void scoutfs_item_delete_dirty(struct super_block *sb, * value is eventually freed along with the item. */ void scoutfs_item_update_dirty(struct super_block *sb, - struct scoutfs_key_buf *key, struct kvec *val) + struct scoutfs_key *key, struct kvec *val) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1631,7 +1550,7 @@ void scoutfs_item_update_dirty(struct super_block *sb, if (val) memcpy(item->val, val->iov_base, new_len); item->val_len = new_len; - update_dirty_item_counts(sb, 0, 0, delta); + update_dirty_item_counts(sb, 0, delta); spin_unlock_irqrestore(&cac->lock, flags); } @@ -1697,8 +1616,8 @@ static struct cached_item *next_dirty(struct cached_item *item) } static bool dirty_item_within(struct rb_root *root, - struct scoutfs_key_buf *from, - struct scoutfs_key_buf *end) + struct scoutfs_key *from, + struct scoutfs_key *end) { struct cached_item *item; @@ -1706,7 +1625,7 @@ static bool dirty_item_within(struct rb_root *root, if (item && !item_is_dirty(item)) item = next_dirty(item); - return item && scoutfs_key_compare(item->key, end) <= 0; + return item && scoutfs_key_compare(&item->key, end) <= 0; } bool scoutfs_item_has_dirty(struct super_block *sb) @@ -1732,8 +1651,8 @@ bool scoutfs_item_has_dirty(struct super_block *sb) * we see if the next cached range starts before the end of the query range. */ bool scoutfs_item_range_cached(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, bool dirty) + struct scoutfs_key *start, + struct scoutfs_key *end, bool dirty) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1749,7 +1668,8 @@ bool scoutfs_item_range_cached(struct super_block *sb, cached = true; } else { rng = walk_ranges(&cac->ranges, start, NULL, &next); - if (rng || (next && scoutfs_key_compare(next->start, end) <= 0)) + if (rng || + (next && scoutfs_key_compare(&next->start, end) <= 0)) cached = true; } @@ -1763,7 +1683,7 @@ bool scoutfs_item_range_cached(struct super_block *sb, * still fits in a single item along with the current dirty items. */ bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, - u32 key_bytes, u32 val_bytes) + u32 val_bytes) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1772,7 +1692,6 @@ bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, spin_lock_irqsave(&cac->lock, flags); fits = scoutfs_seg_fits_single(nr_items + cac->nr_dirty_items, - key_bytes + cac->dirty_key_bytes, val_bytes + cac->dirty_val_bytes); spin_unlock_irqrestore(&cac->lock, flags); @@ -1805,7 +1724,7 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) item = first_dirty(cac->items.rb_node); while (item) { kvec_init(&val, item->val, item->val_len); - appended = scoutfs_seg_append_item(sb, seg, item->key, &val, + appended = scoutfs_seg_append_item(sb, seg, &item->key, &val, item_flags(item), links); /* trans reservation should have limited dirty */ BUG_ON(!appended); @@ -1832,8 +1751,8 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) * Returns a sync error or the number of dirty items written. */ int scoutfs_item_writeback(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_key *start, + struct scoutfs_key *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1868,8 +1787,8 @@ int scoutfs_item_writeback(struct super_block *sb, * Returns errors or the count of the items invalidated. */ int scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_key *start, + struct scoutfs_key *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; @@ -1887,23 +1806,19 @@ int scoutfs_item_invalidate(struct super_block *sb, scoutfs_inc_counter(sb, item_range_alloc); rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); - if (rng) { - rng->start = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - rng->end = scoutfs_key_alloc(sb, SCOUTFS_MAX_KEY_SIZE); - } - if (!rng || !rng->start || !rng->end) { + if (!rng) { free_range(sb, rng); ret = -ENOMEM; goto out; } - scoutfs_key_copy(rng->start, start); - scoutfs_key_copy(rng->end, end); + rng->start = *start; + rng->end = *end; spin_lock_irqsave(&cac->lock, flags); for (item = next_item(&cac->items, start); - item && scoutfs_key_compare(item->key, end) <= 0; + item && scoutfs_key_compare(&item->key, end) <= 0; item = next) { /* XXX seems like this should be a helper? */ @@ -1967,7 +1882,7 @@ static struct cached_item *rb_prev_item(struct cached_item *item) static struct cached_item *shrink_boundary(struct super_block *sb, struct cached_item *item, struct cached_item **next_ret, - struct scoutfs_key_buf *end, + struct scoutfs_key *end, bool right) { struct cached_item *found = NULL; @@ -1985,9 +1900,9 @@ static struct cached_item *shrink_boundary(struct super_block *sb, if (next) { if (right) - cmp = scoutfs_key_compare(next->key, end) > 0; + cmp = scoutfs_key_compare(&next->key, end) > 0; else - cmp = scoutfs_key_compare(next->key, end) < 0; + cmp = scoutfs_key_compare(&next->key, end) < 0; } else { cmp = true; } @@ -1999,13 +1914,13 @@ static struct cached_item *shrink_boundary(struct super_block *sb, } if (right) { - scoutfs_key_inc_cur_len(item->key); - cmp = scoutfs_key_compare(item->key, next->key) <= 0; - scoutfs_key_dec_cur_len(item->key); + scoutfs_key_inc(&item->key); + cmp = scoutfs_key_compare(&item->key, &next->key) <= 0; + scoutfs_key_dec(&item->key); } else { - scoutfs_key_dec_cur_len(item->key); - cmp = scoutfs_key_compare(item->key, next->key) >= 0; - scoutfs_key_inc_cur_len(item->key); + scoutfs_key_dec(&item->key); + cmp = scoutfs_key_compare(&item->key, &next->key) >= 0; + scoutfs_key_inc(&item->key); } if (cmp) { found = item; @@ -2038,8 +1953,8 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, struct cached_item *item) { struct item_cache *cac = SCOUTFS_SB(sb)->item_cache; - struct scoutfs_key_buf *rng_end = NULL; - struct scoutfs_key_buf *key; + struct scoutfs_key rng_end; + struct scoutfs_key key; struct cached_range *new_rng; struct cached_item *first; struct cached_item *last; @@ -2050,14 +1965,14 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, /* we're re-using item memory as ranges :P */ BUILD_BUG_ON(sizeof(struct cached_item) < sizeof(struct cached_range)); - first = shrink_boundary(sb, item, &prev, rng->start, false); - last = shrink_boundary(sb, item, &next, rng->end, true); + first = shrink_boundary(sb, item, &prev, &rng->start, false); + last = shrink_boundary(sb, item, &next, &rng->end, true); - trace_scoutfs_item_shrink_around(sb, rng->start, rng->end, item->key, - prev ? prev->key : NULL, - first ? first->key : NULL, - last ? last->key : NULL, - next ? next->key : NULL); + trace_scoutfs_item_shrink_around(sb, &rng->start, &rng->end, &item->key, + prev ? &prev->key : NULL, + first ? &first->key : NULL, + last ? &last->key : NULL, + next ? &next->key : NULL); /* can't shrink if we can't use neighbours */ if (!first || !last) { @@ -2075,17 +1990,14 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, if (prev) { rng_end = rng->end; rng->end = first->key; - first->key = NULL; - scoutfs_key_dec_cur_len(rng->end); + scoutfs_key_dec(&rng->end); trace_range(shrink_end, sb, rng); } /* set start of remaining existing range */ if (next && !prev) { - scoutfs_key_free(sb, rng->start); rng->start = last->key; - last->key = NULL; - scoutfs_key_inc_cur_len(rng->start); + scoutfs_key_inc(&rng->start); trace_range(shrink_start, sb, rng); } @@ -2104,9 +2016,8 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, memset(new_rng, 0, sizeof(struct cached_range)); new_rng->end = rng_end; - rng_end = NULL; new_rng->start = key; - scoutfs_key_inc_cur_len(new_rng->start); + scoutfs_key_inc(&new_rng->start); insert_range(sb, &cac->ranges, new_rng); scoutfs_inc_counter(sb, item_shrink_split_range); @@ -2122,15 +2033,12 @@ static int shrink_around(struct super_block *sb, struct cached_range *rng, for (item = first; item && (next = item == last ? NULL : rb_next_item(item), 1); item = next) { - if (item->key) - trace_scoutfs_item_shrink(sb, item->key); + trace_scoutfs_item_shrink(sb, &item->key); scoutfs_inc_counter(sb, item_shrink); erase_item(sb, cac, item); nr++; } - scoutfs_key_free(sb, rng_end); - return nr; } @@ -2173,7 +2081,7 @@ static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) BUG_ON(item_is_dirty(item)); /* if we're not in a range just shrink the item */ - rng = walk_ranges(&cac->ranges, item->key, NULL, NULL); + rng = walk_ranges(&cac->ranges, &item->key, NULL, NULL); if (!rng) { scoutfs_inc_counter(sb, item_shrink_outside); erase_item(sb, cac, item); @@ -2210,37 +2118,21 @@ out: return ret; } -static void *copy_key_with_len(void *data, struct scoutfs_key_buf *key) -{ - u16 len = key->key_len; - - memcpy(data, &len, sizeof(len)); - data += sizeof(len); - memcpy(data, key->data, len); - - return data + len; -} - /* - * Copy the next cached ranges starting with the key into the caller's - * buffer. Each range copied by storing each keys size in a u16 - * followed by the binary key data. The number of bytes of full copied - * ranges is returned. The caller's key is incremented past the last - * key returned so that they can iterate without worrying about - * examining the returned keys. + * Copy the keys of the sorted cached ranges starting with the search + * key into the caller's key array. The number of copied range keys is + * returned which will always be a multiple of two. */ int scoutfs_item_copy_range_keys(struct super_block *sb, - struct scoutfs_key_buf *key, void *data, - unsigned len) + struct scoutfs_key *key, + struct scoutfs_key *keys, unsigned nr) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; struct rb_node *node = cac->ranges.rb_node; struct cached_range *next = NULL; - struct scoutfs_key_buf *last = NULL; struct cached_range *rng; unsigned long flags; - unsigned bytes; int ret = 0; int cmp; @@ -2250,7 +2142,7 @@ int scoutfs_item_copy_range_keys(struct super_block *sb, rng = container_of(node, struct cached_range, node); cmp = scoutfs_key_compare_ranges(key, key, - rng->start, rng->end); + &rng->start, &rng->end); if (cmp < 0) { next = rng; node = node->rb_left; @@ -2263,21 +2155,11 @@ int scoutfs_item_copy_range_keys(struct super_block *sb, } for (rng = next; rng; rng = rb_next_rng(rng)) { - bytes = 2 + rng->start->key_len + 2 + rng->end->key_len; - if (len < bytes) + if (ret + 2 > nr) break; - data = copy_key_with_len(data, rng->start); - data = copy_key_with_len(data, rng->end); - len -= bytes; - ret += bytes; - - last = rng->end; - } - - if (last) { - scoutfs_key_copy(key, last); - scoutfs_key_inc(key); + keys[ret++] = rng->start; + keys[ret++] = rng->end; } spin_unlock_irqrestore(&cac->lock, flags); @@ -2285,38 +2167,31 @@ int scoutfs_item_copy_range_keys(struct super_block *sb, return ret; } -/* like copy_range_keys, but for present items */ -int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key_buf *key, - void *data, unsigned len) +/* + * Copy keys for the sorted cached items starting with the search key + * into the caller's key array. The number of copied keys is returned. + */ +int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *keys, unsigned nr) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct scoutfs_key_buf *last = NULL; struct cached_item *item = NULL; unsigned long flags; - unsigned bytes; int ret = 0; spin_lock_irqsave(&cac->lock, flags); - for (item = next_item(&cac->items, key); item; item = rb_next_item(item)) { + for (item = next_item(&cac->items, key); item; + item = rb_next_item(item)) { + + if (ret == nr) + break; + if (item->deletion) continue; - bytes = 2 + item->key->key_len; - if (len < bytes) - break; - - data = copy_key_with_len(data, item->key); - len -= bytes; - ret += bytes; - - last = item->key; - } - - if (last) { - scoutfs_key_copy(key, last); - scoutfs_key_inc(key); + keys[ret++] = item->key; } spin_unlock_irqrestore(&cac->lock, flags); diff --git a/kmod/src/item.h b/kmod/src/item.h index 6507d85a..328345b9 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -4,75 +4,67 @@ #include struct scoutfs_segment; -struct scoutfs_key_buf; +struct scoutfs_key; -int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_lookup_exact(struct super_block *sb, - struct scoutfs_key_buf *key, struct kvec *val, + struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_next(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val, +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_next_same_min(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, - struct kvec *val, int len, - struct scoutfs_lock *lock); -int scoutfs_item_next_same(struct super_block *sb, struct scoutfs_key_buf *key, - struct scoutfs_key_buf *last, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_item_create(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_create_force(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock); -int scoutfs_item_update(struct super_block *sb, struct scoutfs_key_buf *key, +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); void scoutfs_item_delete_dirty(struct super_block *sb, - struct scoutfs_key_buf *key); + struct scoutfs_key *key); void scoutfs_item_update_dirty(struct super_block *sb, - struct scoutfs_key_buf *key, struct kvec *val); -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct kvec *val); +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock); int scoutfs_item_delete_force(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct scoutfs_lock *lock); int scoutfs_item_delete_save(struct super_block *sb, - struct scoutfs_key_buf *key, + struct scoutfs_key *key, struct list_head *list, struct scoutfs_lock *lock); int scoutfs_item_restore(struct super_block *sb, struct list_head *list, struct scoutfs_lock *lock); int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *key, struct kvec *val); + struct scoutfs_key *key, struct kvec *val); int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end); + struct scoutfs_key *start, + struct scoutfs_key *end); void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); bool scoutfs_item_has_dirty(struct super_block *sb); bool scoutfs_item_range_cached(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, bool dirty); + struct scoutfs_key *start, + struct scoutfs_key *end, bool dirty); bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, - u32 key_bytes, u32 val_bytes); + u32 val_bytes); int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); int scoutfs_item_writeback(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end); + struct scoutfs_key *start, + struct scoutfs_key *end); int scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end); + struct scoutfs_key *start, + struct scoutfs_key *end); int scoutfs_item_copy_range_keys(struct super_block *sb, - struct scoutfs_key_buf *key, void *data, - unsigned len); -int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key_buf *key, - void *data, unsigned len); + struct scoutfs_key *key, + struct scoutfs_key *keys, unsigned nr); +int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *keys, unsigned nr); int scoutfs_item_setup(struct super_block *sb); void scoutfs_item_destroy(struct super_block *sb); diff --git a/kmod/src/key.c b/kmod/src/key.c index 9e8bb5cc..23aa8265 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -11,422 +11,48 @@ * General Public License for more details. */ #include -#include -#include +#include +#include +#include "format.h" #include "key.h" -struct scoutfs_key_buf *scoutfs_key_alloc(struct super_block *sb, u16 len) +char *scoutfs_zone_strings[SCOUTFS_MAX_ZONE] = { + [SCOUTFS_INODE_INDEX_ZONE] = "ind", + [SCOUTFS_NODE_ZONE] = "nod", + [SCOUTFS_FS_ZONE] = "fs", +}; + +char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { + [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", + [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_SEGNO_TYPE] = "fsg", + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_BLKNO_TYPE] = "fbk", + [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = "orp", + [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = "ino", + [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = "xat", + [SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = "dnt", + [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = "rdr", + [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = "lbr", + [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = "sym", + [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = "bmp", +}; + +char scoutfs_unknown_u8_strings[U8_MAX][U8_STR_MAX]; + +int __init scoutfs_key_init(void) { - struct scoutfs_key_buf *key; - - if (WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE)) - return NULL; - - key = kmalloc(sizeof(struct scoutfs_key_buf) + len, GFP_NOFS); - if (key) { - key->data = key + 1; - key->key_len = len; - key->buf_len = len; - } - - return key; -} - -struct scoutfs_key_buf *scoutfs_key_dup(struct super_block *sb, - struct scoutfs_key_buf *key) -{ - struct scoutfs_key_buf *dup; - - dup = scoutfs_key_alloc(sb, key->key_len); - if (dup) - memcpy(dup->data, key->data, dup->key_len); - return dup; -} - -void scoutfs_key_free(struct super_block *sb, struct scoutfs_key_buf *key) -{ - kfree(key); -} - -/* - * Keys are large multi-byte big-endian values. To correctly increase - * or decrease keys we need to start by extending the key to the full - * precision using the max key size, setting the least significant bytes - * to 0. - */ -static void extend_zeros(struct scoutfs_key_buf *key) -{ - if (key->key_len < SCOUTFS_MAX_KEY_SIZE && - !WARN_ON_ONCE(key->buf_len != SCOUTFS_MAX_KEY_SIZE)) { - memset(key->data + key->key_len, 0, - key->buf_len - key->key_len); - key->key_len = key->buf_len; - } -} - -/* - * There are callers that work with a range of keys of a uniform length - * who know that it's safe to increment their keys that aren't full - * precision. These are exceptional so a specific function variant - * marks them. - */ -void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key) -{ - u8 *bytes = key->data; - int i; - - for (i = key->key_len - 1; i >= 0; i--) { - if (++bytes[i] != 0) - break; - } -} - -void scoutfs_key_inc(struct scoutfs_key_buf *key) -{ - extend_zeros(key); - scoutfs_key_inc_cur_len(key); -} - -void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key) -{ - u8 *bytes = key->data; - int i; - - for (i = key->key_len - 1; i >= 0; i--) { - if (--bytes[i] != 255) - break; - } -} - -void scoutfs_key_dec(struct scoutfs_key_buf *key) -{ - extend_zeros(key); - scoutfs_key_dec_cur_len(key); -} - -/* return the bytes of the string including the null term */ -#define snprintf_null(buf, size, fmt, args...) \ - (snprintf((buf), (size), fmt, ##args) + 1) - -/* - * Store a formatted string representing the key in the buffer. The key - * must be at least min_len to store the data needed by the format at - * all. fmt_len is the length of data that's used by the format. These - * are different because we have badly designed keys with variable - * length data that isn't described by the key. It's assumed from the - * length of the key. Take dirents -- they need to at least have a - * dirent struct, but the name length is the rest of the key. - * - * (XXX And this goes horribly wrong when we pad out dirent keys to max - * len to increment at high precision. We'll never see these items used - * by real fs code, but temporary keys and range endpoints can be full - * precision and we can try and print them and get very confused. We - * need to rev the format to include explicit lengths.) - * - * If the format doesn't cover the entire key then we append more - * formatting to represent the trailing bytes: runs of zeros compresesd - * to _ and then hex output of non-zero bytes. - */ -static int __printf(6, 7) snprintf_key(char *buf, size_t size, - struct scoutfs_key_buf *key, - unsigned min_len, unsigned fmt_len, - const char *fmt, ...) - -{ - va_list args; - char *data; - char *end; - int left; - int part; int ret; - int nr; + int i; - if (key->key_len < min_len) - return snprintf_null(buf, size, "[trunc len %u < min %u]", - key->key_len, min_len); - - if (fmt_len == 0) - fmt_len = min_len; - - va_start(args, fmt); - ret = vsnprintf(buf, size, fmt, args); - va_end(args); - /* next formatting overwrites null */ - if (buf) { - buf += ret; - size -= min_t(int, size, ret); + for (i = 0; i <= U8_MAX; i++) { + ret = snprintf(scoutfs_unknown_u8_strings[i], U8_STR_MAX, + "u%u", i); + if (WARN_ONCE(ret <= 0 || ret >= U8_STR_MAX, + "snprintf("__stringify(U8_STR_MAX)") ret %d\n", + ret)) + return -EINVAL; } - data = key->data + fmt_len; - left = key->key_len - fmt_len; - - while (left && (!buf || size > 1)) { - /* compress runs of zero bytes to _ */ - end = memchr_inv(data, 0, left); - nr = end ? end - data : left; - if (nr) { - if (buf) { - *(buf++) = '_'; - size--; - } - ret++; - data += nr; - left -= nr; - continue; - } - - /* - * hex print non-zero bytes. %ph is limited to 64 bytes - * and is buggy in that it still tries to print to buf - * past size. (so buf = null, size = 0 crashes instead - * of printing the length of the formatted string.) - */ - end = memchr(data, 0, left); - nr = end ? end - data : left; - nr = min(nr, 64); - - if (buf) - part = snprintf(buf, size, "%*phN", nr, data); - else - part = nr * 2; - if (buf) { - buf += part; - size -= min_t(int, size, part); - } - ret += part; - - data += nr; - left -= nr; - } - - /* always store and include null */ - if (buf) - *buf = '\0'; - return ret + 1; -} - -typedef int (*key_printer_t)(char *buf, struct scoutfs_key_buf *key, - size_t size); - -static int pr_ino_idx(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - static char *type_strings[] = { - [SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", - [SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", - }; - struct scoutfs_inode_index_key *ikey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_inode_index_key), 0, - "iin.%s.%llu.%u.%llu", - type_strings[ikey->type], be64_to_cpu(ikey->major), - be32_to_cpu(ikey->minor), be64_to_cpu(ikey->ino)); -} - -static int pr_free_bits(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - static char *type_strings[] = { - [SCOUTFS_FREE_BITS_SEGNO_TYPE] = "fsg", - [SCOUTFS_FREE_BITS_BLKNO_TYPE] = "fbk", - }; - struct scoutfs_free_bits_key *frk = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_block_mapping_key), 0, - "nod.%llu.%s.%llu", - be64_to_cpu(frk->node_id), - type_strings[frk->type], - be64_to_cpu(frk->base)); -} - -static int pr_orphan(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_orphan_key *okey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_orphan_key), 0, - "nod.%llu.orp.%llu", - be64_to_cpu(okey->node_id), - be64_to_cpu(okey->ino)); -} - -static int pr_inode(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_inode_key *ikey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_inode_key), 0, - "fs.%llu.ino", - be64_to_cpu(ikey->ino)); -} - -static int pr_xattr(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_xattr_key *xkey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_xattr_key), key->key_len, - "fs.%llu.xat.%08x.%llu.%u", - be64_to_cpu(xkey->ino), - be32_to_cpu(xkey->name_hash), - be64_to_cpu(xkey->id), xkey->part); -} - -static int pr_dirent(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_dirent_key *dkey = key->data; - char *which = dkey->type == SCOUTFS_DIRENT_TYPE ? "dnt" : - dkey->type == SCOUTFS_READDIR_TYPE ? "rdr" : - dkey->type == SCOUTFS_LINK_BACKREF_TYPE ? "lbr" : - "unk"; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_dirent_key), key->key_len, - "fs.%llu.%s.%llu.%llu", - be64_to_cpu(dkey->ino), which, - be64_to_cpu(dkey->major), - be64_to_cpu(dkey->minor)); -} - -static int pr_symlink(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_symlink_key *skey = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_symlink_key), 0, - "fs.%llu.sym", - be64_to_cpu(skey->ino)); -} - -static int pr_block_mapping(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - struct scoutfs_block_mapping_key *bmk = key->data; - - return snprintf_key(buf, size, key, - sizeof(struct scoutfs_block_mapping_key), 0, - "fs.%llu.bmp.%llu", - be64_to_cpu(bmk->ino), - be64_to_cpu(bmk->base)); -} - -const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { - [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = - pr_ino_idx, - [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = - pr_ino_idx, - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_SEGNO_TYPE] = pr_free_bits, - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_BLKNO_TYPE] = pr_free_bits, - [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = pr_orphan, - [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = pr_inode, - [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = pr_xattr, - [SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = pr_dirent, - [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_dirent, - [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_dirent, - [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = pr_symlink, - [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = pr_block_mapping, -}; - -/* - * Write the null-terminated string that describes the key to the - * buffer. The bytes copied (including the null) is returned. A null - * buffer can be used to find the string size without writing anything. - * - * XXX nonprintable characters in the trace? - */ -int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size) -{ - u8 zone; - u8 type; - - if (key == NULL || key->data == NULL) - return snprintf_null(buf, size, "[NULL]"); - - /* always at least zone, some id, and type */ - if (key->key_len < (1 + 8 + 1)) - return snprintf_null(buf, size, "[trunc len %u]", key->key_len); - - zone = *(u8 *)key->data; - - /* - * each zone's keys always start with the same fields that let - * us deref any key to get the type. We chose a few representative - * keys from each zone to get the type. - */ - if (zone == SCOUTFS_INODE_INDEX_ZONE) { - struct scoutfs_inode_index_key *ikey = key->data; - type = ikey->type; - } else if (zone == SCOUTFS_NODE_ZONE) { - struct scoutfs_free_bits_key *fkey = key->data; - type = fkey->type; - } else if (zone == SCOUTFS_FS_ZONE) { - struct scoutfs_inode_key *ikey = key->data; - type = ikey->type; - } else { - type = 255; - } - - if (zone > SCOUTFS_MAX_ZONE || type > SCOUTFS_MAX_TYPE || - key_printers[zone][type] == NULL) { - return snprintf_null(buf, size, "[unk zone %u type %u]", - zone, type); - } - - return key_printers[zone][type](buf, key, size); -} - -/* - * Callers never have a pre-existing buffer whose size they need to be - * careful for. For a given static string they're first calling with a - * null buf to find out the formatted length without storing anything. - * Then they're called again with a buffer of that allocation size. As - * long as the formatting is consistent this pattern won't overflow. - */ -int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key) -{ - return scoutfs_key_str_size(buf, key, buf ? INT_MAX : 0); -} - -#define MAX_STR_COUNT 10 - -struct key_strings { - bool started; - int next_str; - char strings[MAX_STR_COUNT][SK_STR_BYTES]; -}; - -static DEFINE_PER_CPU(struct key_strings, percpu_key_strings); - -void scoutfs_key_start_percpu(void) -{ - struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); - - BUG_ON(ks->started); - ks->started = true; - get_cpu(); -} - -char *scoutfs_key_percpu_string(void) -{ - struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); - char *str; - - BUG_ON(!ks->started); - - str = ks->strings[ks->next_str++]; - BUG_ON(ks->next_str >= MAX_STR_COUNT); - - return str; -} - -void scoutfs_key_finish_percpu(void) -{ - struct key_strings *ks = this_cpu_ptr(&percpu_key_strings); - - BUG_ON(!ks->started); - - ks->next_str = 0; - ks->started = false; - - put_cpu(); + return 0; } diff --git a/kmod/src/key.h b/kmod/src/key.h index bdeafb57..eb157279 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -3,88 +3,87 @@ #include #include "format.h" +#include "cmp.h" +#include "endian_swap.h" -struct scoutfs_key_buf { - void *data; - u16 key_len; - u16 buf_len; -}; +extern char *scoutfs_zone_strings[SCOUTFS_MAX_ZONE]; +extern char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE]; +#define U8_STR_MAX 5 /* u%3u'\0' */ +extern char scoutfs_unknown_u8_strings[U8_MAX][U8_STR_MAX]; -struct scoutfs_key_buf *scoutfs_key_alloc(struct super_block *sb, u16 len); -struct scoutfs_key_buf *scoutfs_key_dup(struct super_block *sb, - struct scoutfs_key_buf *key); -void scoutfs_key_free(struct super_block *sb, struct scoutfs_key_buf *key); -void scoutfs_key_inc(struct scoutfs_key_buf *key); -void scoutfs_key_inc_cur_len(struct scoutfs_key_buf *key); -void scoutfs_key_dec(struct scoutfs_key_buf *key); -void scoutfs_key_dec_cur_len(struct scoutfs_key_buf *key); +int __init scoutfs_key_init(void); -int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key, size_t size); -int scoutfs_key_str(char *buf, struct scoutfs_key_buf *key); -void scoutfs_key_start_percpu(void); -char *scoutfs_key_percpu_string(void); -void scoutfs_key_finish_percpu(void); - -#define SK_PCPU(statements) do { \ - scoutfs_key_start_percpu(); \ - { statements; } \ - scoutfs_key_finish_percpu(); \ -} while (0) - -/* - * The biggest keys are typically a little struct then a large name. The - * string representation will tend to be mostly the name, but some of the - * strict fields can blow up from say 8 bytes to 20 bytes. So we give - * a lot of padding for that. - */ -#define SK_STR_BYTES (100 + SCOUTFS_MAX_KEY_SIZE) - -#define SK_FMT "%s" -#define SK_ARG(k) \ -({ \ - char *__str = scoutfs_key_percpu_string(); \ - scoutfs_key_str_size(__str, k, SK_STR_BYTES); \ - __str; \ -}) - -#define SK_TRACE_PRINTK(args...) SK_PCPU(trace_printk(args)) -#define SK_PRINTK(args...) SK_PCPU(printk(args)) - -/* - * Initialize a small key in a larger allocated buffer. This lets - * callers, for example, search for a small key and get a larger key - * copied in. - */ -static inline void scoutfs_key_init_buf_len(struct scoutfs_key_buf *key, - void *data, u16 key_len, - u16 buf_len) +static inline char *sk_zone_str(u8 zone) { - WARN_ON_ONCE(buf_len > SCOUTFS_MAX_KEY_SIZE); - WARN_ON_ONCE(key_len > buf_len); + if (zone >= SCOUTFS_MAX_ZONE || scoutfs_zone_strings[zone] == NULL) + return scoutfs_unknown_u8_strings[zone]; - key->data = data; - key->key_len = key_len; - key->buf_len = buf_len; + return scoutfs_zone_strings[zone]; +} + +static inline char *sk_type_str(u8 zone, u8 type) +{ + if (zone >= SCOUTFS_MAX_ZONE || type >= SCOUTFS_MAX_TYPE || + scoutfs_type_strings[zone][type] == NULL) + return scoutfs_unknown_u8_strings[type]; + + return scoutfs_type_strings[zone][type]; +} + +#define SK_FMT "%s.%llu.%s.%llu.%llu.%u" +/* This does not support null keys */ +#define SK_ARG(key) sk_zone_str((key)->sk_zone), \ + le64_to_cpu((key)->_sk_first), \ + sk_type_str((key)->sk_zone, (key)->sk_type), \ + le64_to_cpu((key)->_sk_second), \ + le64_to_cpu((key)->_sk_third), \ + (key)->_sk_fourth + +static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) +{ + key->sk_zone = 0; + key->_sk_first = 0; + key->sk_type = 0; + key->_sk_second = 0; + key->_sk_third = 0; + key->_sk_fourth = 0; +} + +static inline void scoutfs_key_copy_or_zeros(struct scoutfs_key *dst, + struct scoutfs_key *src) +{ + if (src) + *dst = *src; + else + scoutfs_key_set_zeros(dst); +} + +static inline void scoutfs_key_set_ones(struct scoutfs_key *key) +{ + key->sk_zone = U8_MAX; + key->_sk_first = cpu_to_le64(U64_MAX); + key->sk_type = U8_MAX; + key->_sk_second = cpu_to_le64(U64_MAX); + key->_sk_third = cpu_to_le64(U64_MAX); + key->_sk_fourth = U8_MAX; } /* - * Point the key buf, usually statically allocated, at an existing - * contiguous key stored elsewhere. + * Return a -1/0/1 comparison of keys. + * + * It turns out that these ternary chains are consistently cheaper than + * other alternatives across keys that first differ in any of the + * values. Say maybe 20% faster than memcmp. */ -static inline void scoutfs_key_init(struct scoutfs_key_buf *key, - void *data, u16 len) +static inline int scoutfs_key_compare(struct scoutfs_key *a, + struct scoutfs_key *b) { - scoutfs_key_init_buf_len(key, data, len, len); -} - -/* - * Compare the fs keys in segment sort order. - */ -static inline int scoutfs_key_compare(struct scoutfs_key_buf *a, - struct scoutfs_key_buf *b) -{ - return memcmp(a->data, b->data, min(a->key_len, b->key_len)) ?: - a->key_len < b->key_len ? -1 : a->key_len > b->key_len ? 1 : 0; + return scoutfs_cmp(a->sk_zone, b->sk_zone) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_first), le64_to_cpu(b->_sk_first)) ?: + scoutfs_cmp(a->sk_type, b->sk_type) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_second), le64_to_cpu(b->_sk_second)) ?: + scoutfs_cmp(le64_to_cpu(a->_sk_third), le64_to_cpu(b->_sk_third)) ?: + scoutfs_cmp(a->_sk_fourth, b->_sk_fourth); } /* @@ -93,68 +92,85 @@ static inline int scoutfs_key_compare(struct scoutfs_key_buf *a, * 1: a_start > b_end * else 0: ranges overlap */ -static inline int scoutfs_key_compare_ranges(struct scoutfs_key_buf *a_start, - struct scoutfs_key_buf *a_end, - struct scoutfs_key_buf *b_start, - struct scoutfs_key_buf *b_end) +static inline int scoutfs_key_compare_ranges(struct scoutfs_key *a_start, + struct scoutfs_key *a_end, + struct scoutfs_key *b_start, + struct scoutfs_key *b_end) { return scoutfs_key_compare(a_end, b_start) < 0 ? -1 : scoutfs_key_compare(a_start, b_end) > 0 ? 1 : 0; } -/* - * Copy as much of the contents of the source buffer that fits into the - * dest buffer. - */ -static inline void scoutfs_key_copy(struct scoutfs_key_buf *dst, - struct scoutfs_key_buf *src) +static inline void scoutfs_key_inc(struct scoutfs_key *key) { - dst->key_len = min(dst->buf_len, src->key_len); - memcpy(dst->data, src->data, dst->key_len); -} - -/* - * Initialize the dst buffer to point to the source buffer in all ways, - * including the buf len. The contents of the buffer are shared by the - * fields describing the buffers are not. - */ -static inline void scoutfs_key_clone(struct scoutfs_key_buf *dst, - struct scoutfs_key_buf *src) -{ - *dst = *src; -} - -/* - * Memset as much of the length as fits in the buffer and set that to - * the new key length. - */ -static inline void scoutfs_key_memset(struct scoutfs_key_buf *key, int c, - u16 len) -{ - if (WARN_ON_ONCE(len > SCOUTFS_MAX_KEY_SIZE)) + if (++key->_sk_fourth != 0) return; - key->key_len = min(key->buf_len, len); - memset(key->data, c, key->key_len); + le64_add_cpu(&key->_sk_third, 1); + if (key->_sk_third != 0) + return; + + le64_add_cpu(&key->_sk_second, 1); + if (key->_sk_second != 0) + return; + + if (++key->sk_type != 0) + return; + + le64_add_cpu(&key->_sk_first, 1); + if (key->_sk_first != 0) + return; + + key->sk_zone++; } -/* - * Set the contents of the buffer to the smallest possible key by sort - * order. It might be truncated if the buffer isn't large enough. - */ -static inline void scoutfs_key_set_min(struct scoutfs_key_buf *key) +static inline void scoutfs_key_dec(struct scoutfs_key *key) { - scoutfs_key_memset(key, 0, sizeof(struct scoutfs_inode_key)); + if (--key->_sk_fourth != U8_MAX) + return; + + le64_add_cpu(&key->_sk_third, -1); + if (key->_sk_third != cpu_to_le64(U64_MAX)) + return; + + le64_add_cpu(&key->_sk_second, -1); + if (key->_sk_second != cpu_to_le64(U64_MAX)) + return; + + if (--key->sk_type != U8_MAX) + return; + + le64_add_cpu(&key->_sk_first, -1); + if (key->_sk_first != cpu_to_le64(U64_MAX)) + return; + + key->sk_zone--; } -/* - * Set the contents of the buffer to the largest possible key by sort - * order. It might be truncated if the buffer isn't large enough. - */ -static inline void scoutfs_key_set_max(struct scoutfs_key_buf *key) +static inline void scoutfs_key_to_be(struct scoutfs_key_be *be, + struct scoutfs_key *key) { - scoutfs_key_memset(key, 0xff, sizeof(struct scoutfs_inode_key)); + BUILD_BUG_ON(sizeof(struct scoutfs_key_be) != + sizeof(struct scoutfs_key)); + + be->sk_zone = key->sk_zone; + be->_sk_first = le64_to_be64(key->_sk_first); + be->sk_type = key->sk_type; + be->_sk_second = le64_to_be64(key->_sk_second); + be->_sk_third = le64_to_be64(key->_sk_third); + be->_sk_fourth = key->_sk_fourth; +} + +static inline void scoutfs_key_from_be(struct scoutfs_key *key, + struct scoutfs_key_be *be) +{ + key->sk_zone = be->sk_zone; + key->_sk_first = be64_to_le64(be->_sk_first); + key->sk_type = be->sk_type; + key->_sk_second = be64_to_le64(be->_sk_second); + key->_sk_third = be64_to_le64(be->_sk_third); + key->_sk_fourth = be->_sk_fourth; } #endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 0f0c4f13..c6545c19 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -112,8 +112,8 @@ static void invalidate_inode(struct super_block *sb, u64 ino) static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, int prev, int mode) { - struct scoutfs_key_buf *start = lock->start; - struct scoutfs_key_buf *end = lock->end; + struct scoutfs_key *start = &lock->start; + struct scoutfs_key *end = &lock->end; struct scoutfs_lock_coverage *cov; struct scoutfs_lock_coverage *tmp; u64 ino, last; @@ -191,15 +191,13 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) list_del(&lock->lru_head); linfo->lru_nr--; } - scoutfs_key_free(sb, lock->start); - scoutfs_key_free(sb, lock->end); kfree(lock); } static struct scoutfs_lock *lock_alloc(struct super_block *sb, struct scoutfs_lock_name *name, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_key *start, + struct scoutfs_key *end) { DECLARE_LOCK_INFO(sb, linfo); @@ -235,12 +233,8 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, INIT_LIST_HEAD(&lock->cov_list); if (start) { - lock->start = scoutfs_key_dup(sb, start); - lock->end = scoutfs_key_dup(sb, end); - if (!lock->start || !lock->end) { - lock_free(linfo, lock); - return NULL; - } + lock->start = *start; + lock->end = *end; } lock->sb = sb; @@ -465,14 +459,14 @@ static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) parent = *node; lock = container_of(*node, struct scoutfs_lock, range_node); - cmp = scoutfs_key_compare_ranges(ins->start, ins->end, - lock->start, lock->end); + cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, + &lock->start, &lock->end); if (WARN_ON_ONCE(cmp == 0)) { - scoutfs_warn_sk(sb, "inserting lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT" overlaps with existing lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT"\n", - ins, LN_ARG(&ins->name), - SK_ARG(ins->start), SK_ARG(ins->end), - lock, LN_ARG(&lock->name), - SK_ARG(lock->start), SK_ARG(lock->end)); + scoutfs_warn(sb, "inserting lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT" overlaps with existing lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT"\n", + ins, LN_ARG(&ins->name), + SK_ARG(&ins->start), SK_ARG(&ins->end), + lock, LN_ARG(&lock->name), + SK_ARG(&lock->start), SK_ARG(&lock->end)); return false; } @@ -544,8 +538,8 @@ static void scoutfs_lock_ast(void *arg) struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); int status = lock->lksb.sb_status; - bool cached; - bool dirty; + bool cached = false; + bool dirty = false; scoutfs_inc_counter(sb, lock_ast); @@ -584,17 +578,20 @@ static void scoutfs_lock_ast(void *arg) * changing lock modes. We can't have cached items if we're not * in the two modes that allow caching. */ - cached = lock->start && scoutfs_item_range_cached(sb, lock->start, - lock->end, false); - dirty = lock->start && scoutfs_item_range_cached(sb, lock->start, - lock->end, true); + if (!RB_EMPTY_NODE(&lock->range_node)) { + cached = scoutfs_item_range_cached(sb, &lock->start, + &lock->end, false); + dirty = scoutfs_item_range_cached(sb, &lock->start, &lock->end, + true); + } + if (WARN_ON_ONCE(dirty || (cached && lock->granted_mode != DLM_LOCK_PR && lock->granted_mode != DLM_LOCK_EX))) { - scoutfs_err_sk(sb, "lock item cache consistency violation, cached %u dirty %u: name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", + scoutfs_err(sb, "lock item cache consistency violation, cached %u dirty %u: name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", cached, dirty, - LN_ARG(&lock->name), SK_ARG(lock->start), - SK_ARG(lock->end), lock->refresh_gen, lock->error, + LN_ARG(&lock->name), SK_ARG(&lock->start), + SK_ARG(&lock->end), lock->refresh_gen, lock->error, lock->granted_mode, lock->bast_mode, lock->work_prev_mode, lock->work_mode, lock->waiters[DLM_LOCK_PR], @@ -674,7 +671,7 @@ static void scoutfs_lock_work(struct work_struct *work) spin_unlock(&linfo->lock); - if (lock->start) { + if (!RB_EMPTY_NODE(&lock->range_node)) { ret = lock_invalidate(sb, lock, prev, mode); BUG_ON(ret); } @@ -793,8 +790,7 @@ static bool lock_wait(struct lock_info *linfo, struct scoutfs_lock *lock, */ static int lock_name_keys(struct super_block *sb, int mode, int flags, struct scoutfs_lock_name *name, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, + struct scoutfs_key *start, struct scoutfs_key *end, struct scoutfs_lock **ret_lock) { DECLARE_LOCK_INFO(sb, linfo); @@ -873,10 +869,8 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock) { struct scoutfs_lock_name name; - struct scoutfs_inode_key start_ikey; - struct scoutfs_inode_key end_ikey; - struct scoutfs_key_buf start; - struct scoutfs_key_buf end; + struct scoutfs_key start; + struct scoutfs_key end; ino &= ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK; @@ -886,15 +880,17 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, name.first = cpu_to_le64(ino); name.second = 0; - start_ikey.zone = SCOUTFS_FS_ZONE; - start_ikey.ino = cpu_to_be64(ino); - start_ikey.type = 0; - scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); + start = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .ski_ino = cpu_to_le64(ino), + .sk_type = 0, + }; - end_ikey.zone = SCOUTFS_FS_ZONE; - end_ikey.ino = cpu_to_be64(ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1); - end_ikey.type = ~0; - scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); + end = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .ski_ino = cpu_to_le64(ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1), + .sk_type = U8_MAX, + }; return lock_name_keys(sb, mode, flags, &name, &start, &end, ret_lock); } @@ -1045,8 +1041,8 @@ int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, * because their starting keys are the same. */ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, - struct scoutfs_inode_index_key *start, - struct scoutfs_inode_index_key *end) + struct scoutfs_key *start, + struct scoutfs_key *end) { u64 start_major = major & ~SCOUTFS_LOCK_SEQ_GROUP_MASK; u64 end_major = major | SCOUTFS_LOCK_SEQ_GROUP_MASK; @@ -1054,21 +1050,12 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, BUG_ON(type != SCOUTFS_INODE_INDEX_META_SEQ_TYPE && type != SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE); - if (start) { - start->zone = SCOUTFS_INODE_INDEX_ZONE; - start->type = type; - start->major = cpu_to_be64(start_major); - start->minor = 0; - start->ino = 0; - } + if (start) + scoutfs_inode_init_index_key(start, type, start_major, 0, 0); - if (end) { - end->zone = SCOUTFS_INODE_INDEX_ZONE; - end->type = type; - end->major = cpu_to_be64(end_major); - end->minor = 0; - end->ino = cpu_to_be64(~0ULL); - } + if (end) + scoutfs_inode_init_index_key(end, type, end_major, U32_MAX, + U64_MAX); } /* @@ -1082,22 +1069,16 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, struct scoutfs_lock **ret_lock) { struct scoutfs_lock_name name; - struct scoutfs_inode_index_key start_ikey; - struct scoutfs_inode_index_key end_ikey; - struct scoutfs_key_buf start; - struct scoutfs_key_buf end; + struct scoutfs_key start; + struct scoutfs_key end; - scoutfs_lock_get_index_item_range(type, major, ino, - &start_ikey, &end_ikey); + scoutfs_lock_get_index_item_range(type, major, ino, &start, &end); name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - name.zone = start_ikey.zone; - name.type = start_ikey.type; - name.first = be64_to_le64(start_ikey.major); - name.second = be64_to_le64(start_ikey.ino); - - scoutfs_key_init(&start, &start_ikey, sizeof(start_ikey)); - scoutfs_key_init(&end, &end_ikey, sizeof(end_ikey)); + name.zone = start.sk_zone; + name.type = start.sk_type; + name.first = start.skii_major; + name.second = start.skii_ino; return lock_name_keys(sb, mode, 0, &name, &start, &end, ret_lock); } @@ -1117,10 +1098,8 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock) { struct scoutfs_lock_name name; - struct scoutfs_orphan_key start_okey; - struct scoutfs_orphan_key end_okey; - struct scoutfs_key_buf start; - struct scoutfs_key_buf end; + struct scoutfs_key start; + struct scoutfs_key end; name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; name.zone = SCOUTFS_NODE_ZONE; @@ -1128,17 +1107,17 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, name.first = cpu_to_le64(node_id); name.second = 0; - start_okey.zone = SCOUTFS_NODE_ZONE; - start_okey.node_id = cpu_to_be64(node_id); - start_okey.type = 0; - start_okey.ino = 0; - scoutfs_key_init(&start, &start_okey, sizeof(start_okey)); + start = (struct scoutfs_key) { + .sk_zone = SCOUTFS_NODE_ZONE, + .sko_node_id = cpu_to_le64(node_id), + .sk_type = 0, + }; - end_okey.zone = SCOUTFS_NODE_ZONE; - end_okey.node_id = cpu_to_be64(node_id); - end_okey.type = ~0; - end_okey.ino = cpu_to_be64(~0ULL); - scoutfs_key_init(&end, &end_okey, sizeof(end_okey)); + end = (struct scoutfs_key) { + .sk_zone = SCOUTFS_NODE_ZONE, + .sko_node_id = cpu_to_le64(node_id), + .sk_type = U8_MAX, + }; return lock_name_keys(sb, mode, flags, &name, &start, &end, lock); } @@ -1330,9 +1309,9 @@ static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) { struct scoutfs_lock *lock = v; - SK_PCPU(seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", - LN_ARG(&lock->name), SK_ARG(lock->start), - SK_ARG(lock->end), lock->refresh_gen, lock->error, + seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", + LN_ARG(&lock->name), SK_ARG(&lock->start), + SK_ARG(&lock->end), lock->refresh_gen, lock->error, lock->granted_mode, lock->bast_mode, lock->work_prev_mode, lock->work_mode, lock->waiters[DLM_LOCK_PR], @@ -1343,7 +1322,7 @@ static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) lock->users[DLM_LOCK_CW], lock->lksb.sb_status, lock->lksb.sb_lkid, - lock->lksb.sb_flags)); + lock->lksb.sb_flags); return 0; } @@ -1437,10 +1416,10 @@ void scoutfs_lock_destroy(struct super_block *sb) for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { if (lock->waiters[mode] || lock->users[mode]) { - scoutfs_warn_sk(sb, "lock name "LN_FMT" start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", + scoutfs_warn(sb, "lock name "LN_FMT" start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", LN_ARG(&lock->name), - SK_ARG(lock->start), - SK_ARG(lock->end), mode); + SK_ARG(&lock->start), + SK_ARG(&lock->end), mode); break; } } diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 12e8c610..1898a6a3 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -16,8 +16,8 @@ struct scoutfs_lock { struct super_block *sb; struct scoutfs_lock_name name; - struct scoutfs_key_buf *start; - struct scoutfs_key_buf *end; + struct scoutfs_key start; + struct scoutfs_key end; struct rb_node node; struct rb_node range_node; unsigned int debug_locks_id; @@ -53,8 +53,8 @@ int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock); void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, - struct scoutfs_inode_index_key *start, - struct scoutfs_inode_index_key *end); + struct scoutfs_key *start, + struct scoutfs_key *end); int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 2f00a7e3..a41ba62a 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -50,7 +50,7 @@ struct manifest { unsigned long flags; - struct scoutfs_key_buf *compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + struct scoutfs_key compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; }; #define MANI_FLAG_LEVEL0_FULL (1 << 0) @@ -77,8 +77,8 @@ struct manifest_ref { int off; u8 level; - struct scoutfs_key_buf *first; - struct scoutfs_key_buf *last; + struct scoutfs_key first; + struct scoutfs_key last; }; /* @@ -123,95 +123,29 @@ bool scoutfs_manifest_level0_full(struct super_block *sb) void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, u64 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) + struct scoutfs_key *first, + struct scoutfs_key *last) { ment->level = level; ment->segno = segno; ment->seq = seq; - - if (first) - scoutfs_key_clone(&ment->first, first); - else - scoutfs_key_init(&ment->first, NULL, 0); - - if (last) - scoutfs_key_clone(&ment->last, last); - else - scoutfs_key_init(&ment->last, NULL, 0); + scoutfs_key_copy_or_zeros(&ment->first, first); + scoutfs_key_copy_or_zeros(&ment->last, last); } -/* - * level 0 segments have the extra seq up in the btree key. - */ -static struct scoutfs_manifest_btree_key * -alloc_btree_key_val_lens(unsigned first_len, unsigned last_len) +static void init_btree_key(struct scoutfs_manifest_btree_key *mkey, + u8 level, u64 seq, struct scoutfs_key *first) { - return kmalloc(sizeof(struct scoutfs_manifest_btree_key) + - sizeof(u64) + - sizeof(struct scoutfs_manifest_btree_val) + - first_len + last_len, GFP_NOFS); + mkey->level = level; + scoutfs_key_to_be(&mkey->first_key, first); + mkey->seq = cpu_to_be64(seq); } -/* - * Initialize the btree key and value for a manifest entry in one contiguous - * allocation. - */ -static struct scoutfs_manifest_btree_key * -alloc_btree_key_val(struct scoutfs_manifest_entry *ment, unsigned *mkey_len, - struct scoutfs_manifest_btree_val **mval_ret, - unsigned *mval_len_ret) +static void init_btree_val(struct scoutfs_manifest_btree_val *mval, + u64 segno, struct scoutfs_key *last) { - struct scoutfs_manifest_btree_key *mkey; - struct scoutfs_manifest_btree_val *mval; - struct scoutfs_key_buf b_first; - struct scoutfs_key_buf b_last; - unsigned bkey_len; - unsigned mval_len; - __be64 seq; - - mkey = alloc_btree_key_val_lens(ment->first.key_len, ment->last.key_len); - if (!mkey) - return NULL; - - if (ment->level == 0) { - seq = cpu_to_be64(ment->seq); - bkey_len = sizeof(seq); - memcpy(mkey->bkey, &seq, bkey_len); - } else { - bkey_len = ment->first.key_len; - } - - *mkey_len = offsetof(struct scoutfs_manifest_btree_key, bkey[bkey_len]); - mval = (void *)mkey + *mkey_len; - - if (ment->level == 0) { - scoutfs_key_init(&b_first, mval->keys, ment->first.key_len); - scoutfs_key_init(&b_last, mval->keys + ment->first.key_len, - ment->last.key_len); - mval_len = sizeof(struct scoutfs_manifest_btree_val) + - ment->first.key_len + ment->last.key_len; - } else { - scoutfs_key_init(&b_first, mkey->bkey, ment->first.key_len); - scoutfs_key_init(&b_last, mval->keys, ment->last.key_len); - mval_len = sizeof(struct scoutfs_manifest_btree_val) + - ment->last.key_len; - } - - mkey->level = ment->level; - mval->segno = cpu_to_le64(ment->segno); - mval->seq = cpu_to_le64(ment->seq); - mval->first_key_len = cpu_to_le16(ment->first.key_len); - mval->last_key_len = cpu_to_le16(ment->last.key_len); - - scoutfs_key_copy(&b_first, &ment->first); - scoutfs_key_copy(&b_last, &ment->last); - - if (mval_ret) { - *mval_ret = mval; - *mval_len_ret = mval_len; - } - return mkey; + mval->segno = cpu_to_le64(segno); + mval->last_key = *last; } /* initialize a native manifest entry to point to the btree key and value */ @@ -222,50 +156,12 @@ static void init_ment_iref(struct scoutfs_manifest_entry *ment, struct scoutfs_manifest_btree_val *mval = iref->val; ment->level = mkey->level; + scoutfs_key_from_be(&ment->first, &mkey->first_key); + ment->seq = be64_to_cpu(mkey->seq); ment->segno = le64_to_cpu(mval->segno); - ment->seq = le64_to_cpu(mval->seq); - - if (ment->level == 0) { - scoutfs_key_init(&ment->first, mval->keys, - le16_to_cpu(mval->first_key_len)); - scoutfs_key_init(&ment->last, mval->keys + - le16_to_cpu(mval->first_key_len), - le16_to_cpu(mval->last_key_len)); - } else { - scoutfs_key_init(&ment->first, mkey->bkey, - le16_to_cpu(mval->first_key_len)); - scoutfs_key_init(&ment->last, mval->keys, - le16_to_cpu(mval->last_key_len)); - } + ment->last = mval->last_key; } -/* - * Fill the callers max-size btree key with the given values and return - * its length. - */ -static unsigned init_btree_key(struct scoutfs_manifest_btree_key *mkey, - u8 level, u64 seq, struct scoutfs_key_buf *first) -{ - struct scoutfs_key_buf b_first; - unsigned bkey_len; - __be64 bseq; - - mkey->level = level; - - if (level == 0) { - bseq = cpu_to_be64(seq); - bkey_len = sizeof(bseq); - memcpy(mkey->bkey, &bseq, bkey_len); - } else if (first) { - scoutfs_key_init(&b_first, mkey->bkey, first->key_len); - scoutfs_key_copy(&b_first, first); - bkey_len = first->key_len; - } else { - bkey_len = 0; - } - - return offsetof(struct scoutfs_manifest_btree_key, bkey[bkey_len]); -} /* * Insert a new manifest entry in the ring. The ring allocates a new @@ -279,29 +175,25 @@ int scoutfs_manifest_add(struct super_block *sb, DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_btree_key *mkey; - struct scoutfs_manifest_btree_val *mval; - unsigned mkey_len; - unsigned mval_len; + struct scoutfs_manifest_btree_key mkey; + struct scoutfs_manifest_btree_val mval; int ret; lockdep_assert_held(&mani->rwsem); - mkey = alloc_btree_key_val(ment, &mkey_len, &mval, &mval_len); - if (!mkey) - return -ENOMEM; + init_btree_key(&mkey, ment->level, ment->seq, &ment->first); + init_btree_val(&mval, ment->segno, &ment->last); trace_scoutfs_manifest_add(sb, ment->level, ment->segno, ment->seq, &ment->first, &ment->last); - ret = scoutfs_btree_insert(sb, &super->manifest.root, mkey, mkey_len, - mval, mval_len); + ret = scoutfs_btree_insert(sb, &super->manifest.root, + &mkey, sizeof(mkey), &mval, sizeof(mval)); if (ret == 0) { mani->nr_levels = max_t(u8, mani->nr_levels, ment->level + 1); add_level_count(sb, ment->level, 1); } - kfree(mkey); return ret; } @@ -317,8 +209,7 @@ int scoutfs_manifest_del(struct super_block *sb, DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_btree_key *mkey; - unsigned mkey_len; + struct scoutfs_manifest_btree_key mkey; int ret; trace_scoutfs_manifest_delete(sb, ment->level, ment->segno, ment->seq, @@ -326,15 +217,13 @@ int scoutfs_manifest_del(struct super_block *sb, lockdep_assert_held(&mani->rwsem); - mkey = alloc_btree_key_val(ment, &mkey_len, NULL, NULL); - if (!mkey) - return -ENOMEM; + init_btree_key(&mkey, ment->level, ment->seq, &ment->first); - ret = scoutfs_btree_delete(sb, &super->manifest.root, mkey, mkey_len); + ret = scoutfs_btree_delete(sb, &super->manifest.root, + &mkey, sizeof(mkey)); if (ret == 0) add_level_count(sb, ment->level, -1ULL); - kfree(mkey); return ret; } @@ -367,8 +256,6 @@ static void free_ref(struct super_block *sb, struct manifest_ref *ref) if (!IS_ERR_OR_NULL(ref)) { WARN_ON_ONCE(!list_empty(&ref->entry)); scoutfs_seg_put(ref->seg); - scoutfs_key_free(sb, ref->first); - scoutfs_key_free(sb, ref->last); kfree(ref); } } @@ -383,15 +270,11 @@ static int alloc_manifest_ref(struct super_block *sb, struct list_head *ref_list struct manifest_ref *ref; ref = kzalloc(sizeof(struct manifest_ref), GFP_NOFS); - if (ref) { - ref->first = scoutfs_key_dup(sb, &ment->first); - ref->last = scoutfs_key_dup(sb, &ment->last); - } - if (!ref || !ref->first || !ref->last) { - free_ref(sb, ref); + if (!ref) return -ENOMEM; - } + ref->first = ment->first; + ref->last = ment->last; ref->level = ment->level; ref->segno = ment->segno; ref->seq = ment->seq; @@ -410,7 +293,7 @@ static int alloc_manifest_ref(struct super_block *sb, struct list_head *ref_list static int btree_prev_overlap_or_next(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len, - struct scoutfs_key_buf *start, u8 level, + struct scoutfs_key *start, u8 level, struct scoutfs_btree_item_ref *iref) { struct scoutfs_manifest_entry ment; @@ -436,55 +319,63 @@ static int btree_prev_overlap_or_next(struct super_block *sb, /* * Get references to all the level 0 segments whose item ranges - * intersect with the callers range. We walk the manifest backwards so - * that we end up adding refs to the caller's list reverse sorted by - * sequence, which is what they want to be able to use the segment with - * the newest item. + * intersect with the callers range. The entries are sorted by their + * first key so we can stop searching once our end key can only keep + * being less than the increasing start key. * * This can return -ESTALE if it reads through stale btree blocks. */ static int get_zero_refs(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, + struct scoutfs_key *start, + struct scoutfs_key *end, struct list_head *ref_list) { - struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_btree_key mkey; struct scoutfs_manifest_entry ment; SCOUTFS_BTREE_ITEM_REF(iref); - SCOUTFS_BTREE_ITEM_REF(prev); - unsigned mkey_len; + struct scoutfs_key zeros; + int cmp; int ret; - scoutfs_manifest_init_entry(&ment, 0, 0, 0, start, NULL); - mkey = alloc_btree_key_val(&ment, &mkey_len, NULL, NULL); - if (!mkey) - return -ENOMEM; + scoutfs_key_set_zeros(&zeros); + init_btree_key(&mkey, 0, 0, &zeros); + + for (;;) { + ret = scoutfs_btree_next(sb, root, &mkey, sizeof(mkey), &iref); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } - /* get level 0 segments that overlap with the missing range */ - mkey_len = init_btree_key(mkey, 0, ~0ULL, NULL); - ret = scoutfs_btree_prev(sb, root, mkey, mkey_len, &iref); - while (ret == 0) { init_ment_iref(&ment, &iref); + scoutfs_btree_put_iref(&iref); - if (scoutfs_key_compare_ranges(start, end, &ment.first, - &ment.last) == 0) { + /* done if we went past level 0 */ + if (ment.level > 0) { + ret = 0; + break; + } + + cmp = scoutfs_key_compare_ranges(start, end, &ment.first, + &ment.last); + /* done if all the ments will be greater */ + if (cmp < 0) { + ret = 0; + break; + } + + if (cmp == 0) { ret = alloc_manifest_ref(sb, ref_list, &ment); if (ret) break; } - swap(prev, iref); - ret = scoutfs_btree_before(sb, root, prev.key, prev.key_len, - &iref); - scoutfs_btree_put_iref(&prev); + scoutfs_key_inc(&ment.first); + init_btree_key(&mkey, ment.level, ment.seq, &ment.first); } - if (ret == -ENOENT) - ret = 0; - scoutfs_btree_put_iref(&iref); - scoutfs_btree_put_iref(&prev); - kfree(mkey); return ret; } @@ -502,37 +393,29 @@ static int get_zero_refs(struct super_block *sb, */ static int get_nonzero_refs(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *end, + struct scoutfs_key *key, + struct scoutfs_key *end, struct list_head *ref_list) { - struct scoutfs_manifest_btree_key *mkey; + struct scoutfs_manifest_btree_key mkey; struct scoutfs_manifest_entry ment; SCOUTFS_BTREE_ITEM_REF(iref); - SCOUTFS_BTREE_ITEM_REF(prev); - unsigned mkey_len; int ret; int i; - scoutfs_manifest_init_entry(&ment, 0, 0, 0, key, NULL); - mkey = alloc_btree_key_val(&ment, &mkey_len, NULL, NULL); - if (!mkey) - return -ENOMEM; - - mkey_len = init_btree_key(mkey, 1, 0, key); for (i = 1; ; i++) { - mkey->level = i; + init_btree_key(&mkey, i, 0, key); - scoutfs_btree_put_iref(&iref); - ret = btree_prev_overlap_or_next(sb, root, mkey, mkey_len, key, - i, &iref); + ret = btree_prev_overlap_or_next(sb, root, &mkey, sizeof(mkey), + key, i, &iref); if (ret < 0) { if (ret == -ENOENT) ret = 0; - goto out; + break; } init_ment_iref(&ment, &iref); + scoutfs_btree_put_iref(&iref); if (ment.level != i || scoutfs_key_compare(&ment.first, end) > 0) @@ -540,14 +423,9 @@ static int get_nonzero_refs(struct super_block *sb, ret = alloc_manifest_ref(sb, ref_list, &ment); if (ret) - goto out; + break; } - ret = 0; -out: - scoutfs_btree_put_iref(&iref); - scoutfs_btree_put_iref(&prev); - kfree(mkey); return ret; } @@ -630,15 +508,15 @@ static int cmp_ment_ref_level_seq(void *priv, struct list_head *A, * as long as we hold refs. */ int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end) + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end) { - struct scoutfs_key_buf item_key; - struct scoutfs_key_buf found_key; - struct scoutfs_key_buf batch_end; - struct scoutfs_key_buf seg_start; - struct scoutfs_key_buf seg_end; + struct scoutfs_key item_key; + struct scoutfs_key found_key; + struct scoutfs_key batch_end; + struct scoutfs_key seg_start; + struct scoutfs_key seg_end; struct scoutfs_btree_root root; struct scoutfs_segment *seg; struct manifest_ref *ref; @@ -666,8 +544,8 @@ int scoutfs_manifest_read_items(struct super_block *sb, last_root_seq = 0; retry_stale: - scoutfs_key_clone(&seg_start, start); - scoutfs_key_clone(&seg_end, end); + seg_start = *start; + seg_end = *end; ret = scoutfs_client_get_manifest_root(sb, &root); if (ret) @@ -681,13 +559,13 @@ retry_stale: /* clamp start and end to the segment boundaries, including key */ list_for_each_entry(ref, &ref_list, entry) { - if (scoutfs_key_compare(ref->first, &seg_start) > 0 && - scoutfs_key_compare(ref->first, key) <= 0) - scoutfs_key_clone(&seg_start, ref->first); + if (scoutfs_key_compare(&ref->first, &seg_start) > 0 && + scoutfs_key_compare(&ref->first, key) <= 0) + seg_start = ref->first; - if (scoutfs_key_compare(ref->last, &seg_end) < 0 && - scoutfs_key_compare(ref->last, key) >= 0) - scoutfs_key_clone(&seg_end, ref->last); + if (scoutfs_key_compare(&ref->last, &seg_end) < 0 && + scoutfs_key_compare(&ref->last, key) >= 0) + seg_end = ref->last; } trace_scoutfs_read_item_keys(sb, key, start, end, &seg_start, &seg_end); @@ -703,8 +581,9 @@ retry_stale: /* submit reads for all the segments */ list_for_each_entry(ref, &ref_list, entry) { - trace_scoutfs_read_item_segment(sb, ref->level, ref->segno, - ref->seq, ref->first, ref->last); + trace_scoutfs_read_item_segment(sb, ref->level, ref->segno, + ref->seq, &ref->first, + &ref->last); seg = scoutfs_seg_submit_read(sb, ref->segno); if (IS_ERR(seg)) { @@ -752,9 +631,9 @@ retry_stale: * items or if the next item is past the keys * that our segments can see. */ - ret = scoutfs_seg_item_ptrs(ref->seg, ref->off, - &item_key, &item_val, - &item_flags); + ret = scoutfs_seg_get_item(ref->seg, ref->off, + &item_key, &item_val, + &item_flags); if (ret < 0 || scoutfs_key_compare(&item_key, &seg_end) > 0) { ref->off = -1; @@ -773,7 +652,7 @@ retry_stale: } /* remember new least key */ - scoutfs_key_clone(&found_key, &item_key); + found_key = item_key; found_val = item_val; found_flags = item_flags; ref->found_ctr = ++found_ctr; @@ -782,7 +661,7 @@ retry_stale: /* ran out of keys in segs, range extends to seg end */ if (!found) { - scoutfs_key_clone(&batch_end, &seg_end); + batch_end = seg_end; ret = 0; break; } @@ -808,7 +687,7 @@ retry_stale: } /* the last successful key determines range end until run out */ - scoutfs_key_clone(&batch_end, &found_key); + batch_end = found_key; /* if we just saw the end key then we're done */ if (scoutfs_key_compare(&found_key, &seg_end) == 0) { @@ -864,14 +743,12 @@ out: * Returns 0 if it set next_key and -ENOENT if the key was after all the * segments in the manifest. */ -int scoutfs_manifest_next_key(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *next_key) +int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next_key) { - struct scoutfs_key_buf item_key; - struct scoutfs_key_buf end; + struct scoutfs_key item_key; + struct scoutfs_key end; struct scoutfs_btree_root root; - struct scoutfs_inode_key end_key; struct scoutfs_segment *seg; struct manifest_ref *ref; struct manifest_ref *tmp; @@ -887,8 +764,7 @@ retry_stale: if (ret) goto out; - scoutfs_key_init(&end, &end_key, sizeof(end_key)); - scoutfs_key_set_max(&end); + scoutfs_key_set_ones(&end); ret = get_zero_refs(sb, &root, key, &end, &ref_list) ?: get_nonzero_refs(sb, &root, key, &end, &ref_list); @@ -929,8 +805,9 @@ retry_stale: found = false; list_for_each_entry(ref, &ref_list, entry) { if (ref->level > 0 && - (!found || scoutfs_key_compare(ref->last, next_key) < 0)) { - scoutfs_key_copy(next_key, ref->last); + (!found || + scoutfs_key_compare(&ref->last, next_key) < 0)) { + *next_key = ref->last; found = true; } @@ -942,13 +819,13 @@ retry_stale: if (ref->off < 0) continue; - ret = scoutfs_seg_item_ptrs(ref->seg, ref->off, &item_key, - NULL, NULL); + ret = scoutfs_seg_get_item(ref->seg, ref->off, &item_key, + NULL, NULL); if (ret < 0) continue; if (!found || scoutfs_key_compare(&item_key, next_key) < 0) { - scoutfs_key_copy(next_key, &item_key); + *next_key = item_key; found = true; } } @@ -996,19 +873,23 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_manifest_btree_key mkey; + struct scoutfs_manifest_entry next; struct scoutfs_manifest_entry ment; struct scoutfs_manifest_entry over; - struct scoutfs_manifest_btree_key *mkey = NULL; SCOUTFS_BTREE_ITEM_REF(iref); SCOUTFS_BTREE_ITEM_REF(over_iref); SCOUTFS_BTREE_ITEM_REF(prev); - unsigned mkey_len; + struct scoutfs_key zeros; + bool wrapped; bool sticky; int level; int ret; int nr = 0; int i; + scoutfs_key_set_zeros(&zeros); + down_write(&mani->rwsem); for (level = mani->nr_levels - 1; level >= 0; level--) { @@ -1024,45 +905,60 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) goto out; } - /* alloc a full size mkey, fill it with whatever search key */ - - mkey = alloc_btree_key_val_lens(SCOUTFS_MAX_KEY_SIZE, 0); - if (!mkey) { - ret = -ENOMEM; - goto out; - } - - /* find the oldest level 0 or the next higher order level by key */ + /* fill ment and ret == 0 if we find an entry at the level */ if (level == 0) { + /* find the oldest level 0 */ - mkey_len = init_btree_key(mkey, 0, 0, NULL); - ret = scoutfs_btree_next(sb, &super->manifest.root, - mkey, mkey_len, &iref); + init_btree_key(&mkey, 0, 0, &zeros); + ment.seq = U64_MAX; + + for (;;) { + ret = scoutfs_btree_next(sb, &super->manifest.root, + &mkey, sizeof(mkey), &iref); + if (ret < 0) { + if (ret == -ENOENT && ment.seq != U64_MAX) + ret = 0; + break; + } + + init_ment_iref(&next, &iref); + scoutfs_btree_put_iref(&iref); + + if (next.level > 0) { + if (ment.seq == U64_MAX) + ret = -ENOENT; + break; + } + + if (next.seq < ment.seq) + ment = next; + + scoutfs_key_inc(&next.first); + init_btree_key(&mkey, next.level, next.seq, + &next.first); + } + } else { /* find the next segment after the compaction at this level */ - mkey_len = init_btree_key(mkey, level, 0, - mani->compact_keys[level]); - + init_btree_key(&mkey, level, 0, &mani->compact_keys[level]); + wrapped = false; +again: ret = scoutfs_btree_next(sb, &super->manifest.root, - mkey, mkey_len, &iref); + &mkey, sizeof(mkey), &iref); if (ret == 0) { init_ment_iref(&ment, &iref); + scoutfs_btree_put_iref(&iref); if (ment.level != level) ret = -ENOENT; } - if (ret == -ENOENT) { - /* .. possibly wrapping to the first key in level */ - mkey_len = init_btree_key(mkey, level, 0, NULL); - scoutfs_btree_put_iref(&iref); - ret = scoutfs_btree_next(sb, &super->manifest.root, - mkey, mkey_len, &iref); + /* try again if we wrapped */ + if (ret == -ENOENT && !wrapped) { + init_btree_key(&mkey, level, 0, &zeros); + wrapped = true; + goto again; } } - if (ret == 0) { - init_ment_iref(&ment, &iref); - if (ment.level != level) - goto out; - } + if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1076,10 +972,10 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) nr++; /* and add a fanout's worth of lower overlapping segments */ - mkey_len = init_btree_key(mkey, level + 1, 0, &ment.first); + init_btree_key(&mkey, level + 1, 0, &ment.first); ret = btree_prev_overlap_or_next(sb, &super->manifest.root, - mkey, mkey_len, - &ment.first, level + 1, &over_iref); + &mkey, sizeof(mkey), &ment.first, + level + 1, &over_iref); sticky = false; for (i = 0; ret == 0 && i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { init_ment_iref(&over, &over_iref); @@ -1112,14 +1008,13 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1, sticky); /* record the next key to start from */ - scoutfs_key_copy(mani->compact_keys[level], &ment.last); - scoutfs_key_inc(mani->compact_keys[level]); + mani->compact_keys[level] = ment.last; + scoutfs_key_inc(&mani->compact_keys[level]); ret = 0; out: up_write(&mani->rwsem); - kfree(mkey); scoutfs_btree_put_iref(&iref); scoutfs_btree_put_iref(&over_iref); scoutfs_btree_put_iref(&prev); @@ -1140,18 +1035,8 @@ int scoutfs_manifest_setup(struct super_block *sb) init_rwsem(&mani->rwsem); - for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) { - mani->compact_keys[i] = scoutfs_key_alloc(sb, - SCOUTFS_MAX_KEY_SIZE); - if (!mani->compact_keys[i]) { - while (--i >= 0) - scoutfs_key_free(sb, mani->compact_keys[i]); - kfree(mani); - return -ENOMEM; - } - - scoutfs_key_set_min(mani->compact_keys[i]); - } + for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) + scoutfs_key_set_zeros(&mani->compact_keys[i]); for (i = ARRAY_SIZE(super->manifest.level_counts) - 1; i >= 0; i--) { if (super->manifest.level_counts[i]) { @@ -1177,11 +1062,8 @@ void scoutfs_manifest_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct manifest *mani = sbi->manifest; - int i; if (mani) { - for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) - scoutfs_key_free(sb, mani->compact_keys[i]); kfree(mani); sbi->manifest = NULL; } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index 020a76fc..cd1a095d 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -15,14 +15,14 @@ struct scoutfs_manifest_entry { u8 level; u64 segno; u64 seq; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; + struct scoutfs_key first; + struct scoutfs_key last; }; void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, u64 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last); + struct scoutfs_key *first, + struct scoutfs_key *last); int scoutfs_manifest_add(struct super_block *sb, struct scoutfs_manifest_entry *ment); int scoutfs_manifest_del(struct super_block *sb, @@ -32,12 +32,11 @@ int scoutfs_manifest_lock(struct super_block *sb); int scoutfs_manifest_unlock(struct super_block *sb); int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end); -int scoutfs_manifest_next_key(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *next_key); + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end); +int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next_key); int scoutfs_manifest_next_compact(struct super_block *sb, void *data); diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 9cde9716..eff34766 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -6,27 +6,13 @@ void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...); -/* - * The _sk variants wrap the message in the SK_PCPU calls which safely - * manage the use of per-cpu key buffers in the arguments. - */ - #define scoutfs_err(sb, fmt, args...) \ scoutfs_msg(sb, KERN_ERR, " error", fmt, ##args) -#define scoutfs_err_sk(sb, fmt, args...) \ - SK_PCPU(scoutfs_err(sb, fmt, ##args)) - #define scoutfs_warn(sb, fmt, args...) \ scoutfs_msg(sb, KERN_WARNING, " warning", fmt, ##args) -#define scoutfs_warn_sk(sb, fmt, args...) \ - SK_PCPU(scoutfs_warn(sb, fmt, ##args)) - #define scoutfs_info(sb, fmt, args...) \ scoutfs_msg(sb, KERN_INFO, "", fmt, ##args) -#define scoutfs_info_sk(sb, fmt, args...) \ - SK_PCPU(scoutfs_info(sb, fmt, ##args)) - #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a761f468..20c3a16b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -317,45 +317,6 @@ TRACE_EVENT(scoutfs_item_next_same_ret, TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) ); -TRACE_EVENT(scoutfs_item_next_same_min, - TP_PROTO(struct super_block *sb, int key_len, int len), - - TP_ARGS(sb, key_len, len), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(int, key_len) - __field(int, len) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->key_len = key_len; - __entry->len = len; - ), - - TP_printk(FSID_FMT" key len %u min val len %d", __entry->fsid, - __entry->key_len, __entry->len) -); - -TRACE_EVENT(scoutfs_item_next_same_min_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(int, ret) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->ret = ret; - ), - - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) -); - TRACE_EVENT(scoutfs_item_next_ret, TP_PROTO(struct super_block *sb, int ret), @@ -619,25 +580,22 @@ TRACE_EVENT(scoutfs_release_trans, struct scoutfs_item_count *res, struct scoutfs_item_count *act, unsigned int tri_holders, unsigned int tri_writing, unsigned int tri_items, - unsigned int tri_keys, unsigned int tri_vals), + unsigned int tri_vals), TP_ARGS(sb, rsv, rsv_holders, res, act, tri_holders, tri_writing, - tri_items, tri_keys, tri_vals), + tri_items, tri_vals), TP_STRUCT__entry( __field(__u64, fsid) __field(void *, rsv) __field(unsigned int, rsv_holders) __field(int, res_items) - __field(int, res_keys) __field(int, res_vals) __field(int, act_items) - __field(int, act_keys) __field(int, act_vals) __field(unsigned int, tri_holders) __field(unsigned int, tri_writing) __field(unsigned int, tri_items) - __field(unsigned int, tri_keys) __field(unsigned int, tri_vals) ), @@ -646,25 +604,21 @@ TRACE_EVENT(scoutfs_release_trans, __entry->rsv = rsv; __entry->rsv_holders = rsv_holders; __entry->res_items = res->items; - __entry->res_keys = res->keys; __entry->res_vals = res->vals; __entry->act_items = act->items; - __entry->act_keys = act->keys; __entry->act_vals = act->vals; __entry->tri_holders = tri_holders; __entry->tri_writing = tri_writing; __entry->tri_items = tri_items; - __entry->tri_keys = tri_keys; __entry->tri_vals = tri_vals; ), - TP_printk(FSID_FMT" rsv %p holders %u reserved %u.%u.%u actual " - "%d.%d.%d, trans holders %u writing %u reserved " - "%u.%u.%u", __entry->fsid, __entry->rsv, - __entry->rsv_holders, __entry->res_items, __entry->res_keys, - __entry->res_vals, __entry->act_items, __entry->act_keys, + TP_printk(FSID_FMT" rsv %p holders %u reserved %u.%u actual " + "%d.%d, trans holders %u writing %u reserved " + "%u.%u", __entry->fsid, __entry->rsv, __entry->rsv_holders, + __entry->res_items, __entry->res_vals, __entry->act_items, __entry->act_vals, __entry->tri_holders, __entry->tri_writing, - __entry->tri_items, __entry->tri_keys, __entry->tri_vals) + __entry->tri_items, __entry->tri_vals) ); TRACE_EVENT(scoutfs_trans_acquired_hold, @@ -673,59 +627,50 @@ TRACE_EVENT(scoutfs_trans_acquired_hold, struct scoutfs_item_count *res, struct scoutfs_item_count *act, unsigned int tri_holders, unsigned int tri_writing, unsigned int tri_items, - unsigned int tri_keys, unsigned int tri_vals), + unsigned int tri_vals), TP_ARGS(sb, cnt, rsv, rsv_holders, res, act, tri_holders, tri_writing, - tri_items, tri_keys, tri_vals), + tri_items, tri_vals), TP_STRUCT__entry( __field(__u64, fsid) __field(int, cnt_items) - __field(int, cnt_keys) __field(int, cnt_vals) __field(void *, rsv) __field(unsigned int, rsv_holders) __field(int, res_items) - __field(int, res_keys) __field(int, res_vals) __field(int, act_items) - __field(int, act_keys) __field(int, act_vals) __field(unsigned int, tri_holders) __field(unsigned int, tri_writing) __field(unsigned int, tri_items) - __field(unsigned int, tri_keys) __field(unsigned int, tri_vals) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->cnt_items = cnt->items; - __entry->cnt_keys = cnt->keys; __entry->cnt_vals = cnt->vals; __entry->rsv = rsv; __entry->rsv_holders = rsv_holders; __entry->res_items = res->items; - __entry->res_keys = res->keys; __entry->res_vals = res->vals; __entry->act_items = act->items; - __entry->act_keys = act->keys; __entry->act_vals = act->vals; __entry->tri_holders = tri_holders; __entry->tri_writing = tri_writing; __entry->tri_items = tri_items; - __entry->tri_keys = tri_keys; __entry->tri_vals = tri_vals; ), - TP_printk(FSID_FMT" cnt %u.%u.%u, rsv %p holders %u reserved %u.%u.%u " - "actual %d.%d.%d, trans holders %u writing %u reserved " - "%u.%u.%u", __entry->fsid, __entry->cnt_items, - __entry->cnt_keys, __entry->cnt_vals, __entry->rsv, - __entry->rsv_holders, __entry->res_items, __entry->res_keys, - __entry->res_vals, __entry->act_items, __entry->act_keys, + TP_printk(FSID_FMT" cnt %u.%u, rsv %p holders %u reserved %u.%u " + "actual %d.%d, trans holders %u writing %u reserved " + "%u.%u", __entry->fsid, __entry->cnt_items, + __entry->cnt_vals, __entry->rsv, __entry->rsv_holders, + __entry->res_items, __entry->res_vals, __entry->act_items, __entry->act_vals, __entry->tri_holders, __entry->tri_writing, - __entry->tri_items, __entry->tri_keys, __entry->tri_vals) + __entry->tri_items, __entry->tri_vals) ); TRACE_EVENT(scoutfs_ioc_release_ret, @@ -1102,30 +1047,30 @@ TRACE_EVENT(scoutfs_advance_dirty_super, ); TRACE_EVENT(scoutfs_dir_add_next_linkref, - TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, int ret, - unsigned int key_len), + TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, + __u64 dir_pos, int ret), - TP_ARGS(sb, ino, dir_ino, ret, key_len), + TP_ARGS(sb, ino, dir_ino, dir_pos, ret), TP_STRUCT__entry( __field(__u64, fsid) __field(__u64, ino) __field(__u64, dir_ino) + __field(__u64, dir_pos) __field(int, ret) - __field(unsigned int, key_len) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->ino = ino; __entry->dir_ino = dir_ino; + __entry->dir_pos = dir_pos; __entry->ret = ret; - __entry->key_len = key_len; ), - TP_printk(FSID_FMT" ino %llu dir_ino %llu ret %d key_len %u", - __entry->fsid, __entry->ino, __entry->dir_ino, __entry->ret, - __entry->key_len) + TP_printk(FSID_FMT" ino %llu dir_ino %llu dis_pos %llu ret %d", + __entry->fsid, __entry->ino, __entry->dir_ino, + __entry->dir_ino, __entry->ret) ); TRACE_EVENT(scoutfs_compact_func, @@ -1342,98 +1287,97 @@ TRACE_EVENT(scoutfs_scan_orphans, DECLARE_EVENT_CLASS(scoutfs_manifest_class, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + struct scoutfs_key *first, struct scoutfs_key *last), TP_ARGS(sb, level, segno, seq, first, last), TP_STRUCT__entry( __field(u8, level) __field(u64, segno) __field(u64, seq) - __dynamic_array(char, first, scoutfs_key_str(NULL, first)) - __dynamic_array(char, last, scoutfs_key_str(NULL, last)) + __field_struct(struct scoutfs_key, first) + __field_struct(struct scoutfs_key, last) ), TP_fast_assign( __entry->level = level; __entry->segno = segno; __entry->seq = seq; - scoutfs_key_str(__get_dynamic_array(first), first); - scoutfs_key_str(__get_dynamic_array(last), last); + scoutfs_key_copy_or_zeros(&__entry->first, first); + scoutfs_key_copy_or_zeros(&__entry->last, last); ), - TP_printk("level %u segno %llu seq %llu first %s last %s", + TP_printk("level %u segno %llu seq %llu first "SK_FMT" last "SK_FMT, __entry->level, __entry->segno, __entry->seq, - __get_str(first), __get_str(last)) + SK_ARG(&__entry->first), SK_ARG(&__entry->last)) ); DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_add, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + struct scoutfs_key *first, struct scoutfs_key *last), TP_ARGS(sb, level, segno, seq, first, last) ); DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_delete, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + struct scoutfs_key *first, struct scoutfs_key *last), TP_ARGS(sb, level, segno, seq, first, last) ); DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_input, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + struct scoutfs_key *first, struct scoutfs_key *last), TP_ARGS(sb, level, segno, seq, first, last) ); DEFINE_EVENT(scoutfs_manifest_class, scoutfs_read_item_segment, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key_buf *first, struct scoutfs_key_buf *last), + struct scoutfs_key *first, struct scoutfs_key *last), TP_ARGS(sb, level, segno, seq, first, last) ); TRACE_EVENT(scoutfs_read_item_keys, TP_PROTO(struct super_block *sb, - struct scoutfs_key_buf *key, - struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end, - struct scoutfs_key_buf *seg_start, - struct scoutfs_key_buf *seg_end), + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct scoutfs_key *seg_start, + struct scoutfs_key *seg_end), TP_ARGS(sb, key, start, end, seg_start, seg_end), TP_STRUCT__entry( __field(__u64, fsid) - __dynamic_array(char, key, scoutfs_key_str(NULL, key)) - __dynamic_array(char, start, scoutfs_key_str(NULL, start)) - __dynamic_array(char, end, scoutfs_key_str(NULL, end)) - __dynamic_array(char, seg_start, - scoutfs_key_str(NULL, seg_start)) - __dynamic_array(char, seg_end, - scoutfs_key_str(NULL, seg_end)) + __field_struct(struct scoutfs_key, key) + __field_struct(struct scoutfs_key, start) + __field_struct(struct scoutfs_key, end) + __field_struct(struct scoutfs_key, seg_start) + __field_struct(struct scoutfs_key, seg_end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_str(__get_dynamic_array(key), key); - scoutfs_key_str(__get_dynamic_array(start), start); - scoutfs_key_str(__get_dynamic_array(end), end); - scoutfs_key_str(__get_dynamic_array(seg_start), seg_start); - scoutfs_key_str(__get_dynamic_array(seg_end), seg_end); + scoutfs_key_copy_or_zeros(&__entry->key, key); + scoutfs_key_copy_or_zeros(&__entry->start, start); + scoutfs_key_copy_or_zeros(&__entry->end, end); + scoutfs_key_copy_or_zeros(&__entry->seg_start, seg_start); + scoutfs_key_copy_or_zeros(&__entry->seg_end, seg_end); ), - TP_printk("fsid "FSID_FMT" key %s start %s end %s seg_start %s seg_end %s", - __entry->fsid, __get_str(key), __get_str(start), - __get_str(end), __get_str(seg_start), __get_str(seg_end)) + TP_printk("fsid "FSID_FMT" key "SK_FMT" start "SK_FMT" end "SK_FMT" seg_start "SK_FMT" seg_end "SK_FMT"", + __entry->fsid, SK_ARG(&__entry->key), SK_ARG(&__entry->start), + SK_ARG(&__entry->end), SK_ARG(&__entry->seg_start), + SK_ARG(&__entry->seg_end)) ); DECLARE_EVENT_CLASS(scoutfs_key_class, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key), TP_STRUCT__entry( __field(__u64, fsid) - __dynamic_array(char, key, scoutfs_key_str(NULL, key)) + __field_struct(struct scoutfs_key, key) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_str(__get_dynamic_array(key), key); + scoutfs_key_copy_or_zeros(&__entry->key, key); ), - TP_printk(FSID_FMT" key %s", __entry->fsid, __get_str(key)) + TP_printk(FSID_FMT" key "SK_FMT, __entry->fsid, SK_ARG(&__entry->key)) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key) ); @@ -1456,127 +1400,129 @@ TRACE_EVENT(scoutfs_item_lookup_ret, ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_shrink, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_xattr_get_next_key, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *key), + TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key) ); DECLARE_EVENT_CLASS(scoutfs_range_class, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), + TP_PROTO(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end), TP_ARGS(sb, start, end), TP_STRUCT__entry( __field(__u64, fsid) - __dynamic_array(char, start, scoutfs_key_str(NULL, start)) - __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + __field_struct(struct scoutfs_key, start) + __field_struct(struct scoutfs_key, end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_str(__get_dynamic_array(start), start); - scoutfs_key_str(__get_dynamic_array(end), end); + scoutfs_key_copy_or_zeros(&__entry->start, start); + scoutfs_key_copy_or_zeros(&__entry->end, end); ), - TP_printk("fsid "FSID_FMT" start %s end %s", - __entry->fsid, __get_str(start), __get_str(end)) + TP_printk("fsid "FSID_FMT" start "SK_FMT" end "SK_FMT, + __entry->fsid, SK_ARG(&__entry->start), + SK_ARG(&__entry->end)) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), + TP_PROTO(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end), TP_ARGS(sb, start, end) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_invalidate_range, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), + TP_PROTO(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end), TP_ARGS(sb, start, end) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, - TP_PROTO(struct super_block *sb, struct scoutfs_key_buf *start, - struct scoutfs_key_buf *end), + TP_PROTO(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end), TP_ARGS(sb, start, end) ); DECLARE_EVENT_CLASS(scoutfs_cached_range_class, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end), TP_STRUCT__entry( __field(__u64, fsid) __field(void *, rng) - __dynamic_array(char, start, scoutfs_key_str(NULL, start)) - __dynamic_array(char, end, scoutfs_key_str(NULL, end)) + __field_struct(struct scoutfs_key, start) + __field_struct(struct scoutfs_key, end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->rng = rng; - scoutfs_key_str(__get_dynamic_array(start), start); - scoutfs_key_str(__get_dynamic_array(end), end); + scoutfs_key_copy_or_zeros(&__entry->start, start); + scoutfs_key_copy_or_zeros(&__entry->end, end); ), - TP_printk("fsid "FSID_FMT" rng %p start %s end %s", - __entry->fsid, __entry->rng, __get_str(start), __get_str(end)) + TP_printk("fsid "FSID_FMT" rng %p start "SK_FMT" end "SK_FMT, + __entry->fsid, __entry->rng, SK_ARG(&__entry->start), + SK_ARG(&__entry->end)) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_free, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_ins_rb_insert, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_mid_left, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_start, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_end, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_rem_rb_insert, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_delete_enoent, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_start, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_end, TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key_buf *start, struct scoutfs_key_buf *end), + struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end) ); @@ -1784,32 +1730,32 @@ DEFINE_EVENT(scoutfs_net_class, scoutfs_client_recv_reply, TRACE_EVENT(scoutfs_item_next_range_check, TP_PROTO(struct super_block *sb, int cached, - struct scoutfs_key_buf *key, struct scoutfs_key_buf *pos, - struct scoutfs_key_buf *last, struct scoutfs_key_buf *end, - struct scoutfs_key_buf *range_end), + struct scoutfs_key *key, struct scoutfs_key *pos, + struct scoutfs_key *last, struct scoutfs_key *end, + struct scoutfs_key *range_end), TP_ARGS(sb, cached, key, pos, last, end, range_end), TP_STRUCT__entry( __field(void *, sb) __field(int, cached) - __dynamic_array(char, key, scoutfs_key_str(NULL, key)) - __dynamic_array(char, pos, scoutfs_key_str(NULL, pos)) - __dynamic_array(char, last, scoutfs_key_str(NULL, last)) - __dynamic_array(char, end, scoutfs_key_str(NULL, end)) - __dynamic_array(char, range_end, - scoutfs_key_str(NULL, range_end)) + __field_struct(struct scoutfs_key, key) + __field_struct(struct scoutfs_key, pos) + __field_struct(struct scoutfs_key, last) + __field_struct(struct scoutfs_key, end) + __field_struct(struct scoutfs_key, range_end) ), TP_fast_assign( __entry->sb = sb; __entry->cached = cached; - scoutfs_key_str(__get_dynamic_array(key), key); - scoutfs_key_str(__get_dynamic_array(pos), pos); - scoutfs_key_str(__get_dynamic_array(last), last); - scoutfs_key_str(__get_dynamic_array(end), end); - scoutfs_key_str(__get_dynamic_array(range_end), range_end); + scoutfs_key_copy_or_zeros(&__entry->key, key); + scoutfs_key_copy_or_zeros(&__entry->pos, pos); + scoutfs_key_copy_or_zeros(&__entry->last, last); + scoutfs_key_copy_or_zeros(&__entry->end, end); + scoutfs_key_copy_or_zeros(&__entry->range_end, range_end); ), - TP_printk("sb %p cached %d key %s pos %s last %s end %s range_end %s", - __entry->sb, __entry->cached, __get_str(key), __get_str(pos), - __get_str(last), __get_str(end), __get_str(range_end)) + TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" last "SK_FMT" end "SK_FMT" range_end "SK_FMT, + __entry->sb, __entry->cached, SK_ARG(&__entry->key), + SK_ARG(&__entry->pos), SK_ARG(&__entry->last), + SK_ARG(&__entry->end), SK_ARG(&__entry->range_end)) ); DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, @@ -1846,37 +1792,36 @@ DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_item_shrink_exit, TRACE_EVENT(scoutfs_item_shrink_around, TP_PROTO(struct super_block *sb, - struct scoutfs_key_buf *rng_start, - struct scoutfs_key_buf *rng_end, struct scoutfs_key_buf *item, - struct scoutfs_key_buf *prev, struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last, struct scoutfs_key_buf *next), + struct scoutfs_key *rng_start, + struct scoutfs_key *rng_end, struct scoutfs_key *item, + struct scoutfs_key *prev, struct scoutfs_key *first, + struct scoutfs_key *last, struct scoutfs_key *next), TP_ARGS(sb, rng_start, rng_end, item, prev, first, last, next), TP_STRUCT__entry( __field(void *, sb) - __dynamic_array(char, rng_start, - scoutfs_key_str(NULL, rng_start)) - __dynamic_array(char, rng_end, - scoutfs_key_str(NULL, rng_end)) - __dynamic_array(char, item, scoutfs_key_str(NULL, item)) - __dynamic_array(char, prev, scoutfs_key_str(NULL, prev)) - __dynamic_array(char, first, scoutfs_key_str(NULL, first)) - __dynamic_array(char, last, scoutfs_key_str(NULL, last)) - __dynamic_array(char, next, scoutfs_key_str(NULL, next)) + __field_struct(struct scoutfs_key, rng_start) + __field_struct(struct scoutfs_key, rng_end) + __field_struct(struct scoutfs_key, item) + __field_struct(struct scoutfs_key, prev) + __field_struct(struct scoutfs_key, first) + __field_struct(struct scoutfs_key, last) + __field_struct(struct scoutfs_key, next) ), TP_fast_assign( __entry->sb = sb; - scoutfs_key_str(__get_dynamic_array(rng_start), rng_start); - scoutfs_key_str(__get_dynamic_array(rng_end), rng_end); - scoutfs_key_str(__get_dynamic_array(item), item); - scoutfs_key_str(__get_dynamic_array(prev), prev); - scoutfs_key_str(__get_dynamic_array(first), first); - scoutfs_key_str(__get_dynamic_array(last), last); - scoutfs_key_str(__get_dynamic_array(next), next); + scoutfs_key_copy_or_zeros(&__entry->rng_start, rng_start); + scoutfs_key_copy_or_zeros(&__entry->rng_end, rng_end); + scoutfs_key_copy_or_zeros(&__entry->item, item); + scoutfs_key_copy_or_zeros(&__entry->prev, prev); + scoutfs_key_copy_or_zeros(&__entry->first, first); + scoutfs_key_copy_or_zeros(&__entry->last, last); + scoutfs_key_copy_or_zeros(&__entry->next, next); ), - TP_printk("sb %p rng_start %s rng_end %s item %s prev %s first %s last %s next %s", - __entry->sb, __get_str(rng_start), __get_str(rng_end), - __get_str(item), __get_str(prev), __get_str(first), - __get_str(last), __get_str(next)) + TP_printk("sb %p rng_start "SK_FMT" rng_end "SK_FMT" item "SK_FMT" prev "SK_FMT" first "SK_FMT" last "SK_FMT" next "SK_FMT, + __entry->sb, SK_ARG(&__entry->rng_start), + SK_ARG(&__entry->rng_end), SK_ARG(&__entry->item), + SK_ARG(&__entry->prev), SK_ARG(&__entry->first), + SK_ARG(&__entry->last), SK_ARG(&__entry->next)) ); TRACE_EVENT(scoutfs_rename, diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 6e116d11..80ccdacd 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -410,49 +410,39 @@ out: return ret; } -static u32 item_bytes(u8 nr_links, u16 key_len, u16 val_len) +static u32 item_bytes(u8 nr_links, u16 val_len) { return offsetof(struct scoutfs_segment_item, skip_links[nr_links]) + - key_len + val_len; -} - -static inline int item_key_off(struct scoutfs_segment_item *item, int item_off) -{ - return item_off + item_bytes(item->nr_links, 0, 0); -} - -static inline void *item_key_ptr(struct scoutfs_segment_item *item) -{ - return (void *)item + item_bytes(item->nr_links, 0, 0); + val_len; } static inline void *item_val_ptr(struct scoutfs_segment_item *item) { - return item_key_ptr(item) + le16_to_cpu(item->key_len); + return (void *)item + item_bytes(item->nr_links, 0); } -static void item_ptrs(struct scoutfs_segment *seg, int off, - struct scoutfs_key_buf *key, struct kvec *val) +/* copy the item key into the caller's key and init their val to ref the val */ +static void get_item_key_val(struct scoutfs_segment *seg, int off, + struct scoutfs_key *key, struct kvec *val) { struct scoutfs_segment_item *item = off_ptr(seg, off); if (key) - scoutfs_key_init(key, item_key_ptr(item), - le16_to_cpu(item->key_len)); - if (val) { - val->iov_base = item_val_ptr(item); - val->iov_len = le16_to_cpu(item->val_len); - } + *key = item->key; + + if (val) + kvec_init(val, item_val_ptr(item), le16_to_cpu(item->val_len)); } static void first_last_keys(struct scoutfs_segment *seg, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) + struct scoutfs_key *first, + struct scoutfs_key *last) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - item_ptrs(seg, sizeof(struct scoutfs_segment_block), first, NULL); - item_ptrs(seg, le32_to_cpu(sblk->last_item_off), last, NULL); + get_item_key_val(seg, sizeof(struct scoutfs_segment_block), + first, NULL); + get_item_key_val(seg, le32_to_cpu(sblk->last_item_off), last, NULL); } static int check_caller_off(struct scoutfs_segment_block *sblk, int off) @@ -475,9 +465,8 @@ static int check_caller_off(struct scoutfs_segment_block *sblk, int off) * All other offsets must be initial values less than the segment header * size, notably including 0, or returned from _next_off(). */ -int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, - struct scoutfs_key_buf *key, struct kvec *val, - u8 *flags) +int scoutfs_seg_get_item(struct scoutfs_segment *seg, int off, + struct scoutfs_key *key, struct kvec *val,u8 *flags) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; @@ -486,7 +475,7 @@ int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, if (off < 0) return off; - item_ptrs(seg, off, key, val); + get_item_key_val(seg, off, key, val); if (flags) { item = off_ptr(seg, off); @@ -524,12 +513,10 @@ static u8 skip_most_nr(u32 nr_items) * than the items and descend down to lower more frequent links when the * search key is less. */ -int scoutfs_seg_find_off(struct scoutfs_segment *seg, - struct scoutfs_key_buf *key) +int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key *key) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; - struct scoutfs_key_buf item_key; __le32 *links; int cmp; int ret; @@ -544,10 +531,8 @@ int scoutfs_seg_find_off(struct scoutfs_segment *seg, off = le32_to_cpu(links[i]); item = off_ptr(seg, off); - scoutfs_key_init(&item_key, item_key_ptr(item), - le16_to_cpu(item->key_len)); - cmp = scoutfs_key_compare(key, &item_key); + cmp = scoutfs_key_compare(key, &item->key); if (cmp == 0) { ret = off; break; @@ -607,16 +592,15 @@ u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg) * than two links per item. We assume the worst case items have the * max number of links. */ -bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes) +bool scoutfs_seg_fits_single(u32 nr_items, u32 val_bytes) { u32 header = sizeof(struct scoutfs_segment_block); - u32 items = nr_items * item_bytes(2, 0, 0); - u32 item_pad = item_bytes(skip_most_nr(nr_items), SCOUTFS_MAX_KEY_SIZE, + u32 items = nr_items * item_bytes(2, 0); + u32 item_pad = item_bytes(skip_most_nr(nr_items), SCOUTFS_MAX_VAL_SIZE) - 1; u32 padding = (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) * item_pad; - return (header + items + key_bytes + val_bytes + padding) - <= SCOUTFS_SEGMENT_SIZE; + return (header + items + val_bytes + padding) <= SCOUTFS_SEGMENT_SIZE; } static u32 align_item_off(struct scoutfs_segment *seg, u32 item_off, u32 bytes) @@ -638,14 +622,13 @@ static u32 align_item_off(struct scoutfs_segment *seg, u32 item_off, u32 bytes) * We return true if we appended and false if the segment was full. */ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val, + struct scoutfs_key *key, struct kvec *val, u8 flags, __le32 **links) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_segment_block *sblk = off_ptr(seg, 0); struct scoutfs_segment_item *item; - struct scoutfs_key_buf item_key; struct kvec item_val; u8 nr_links; u32 val_len; @@ -674,12 +657,12 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg */ off = le32_to_cpu(sblk->last_item_off); if (off) { - item_ptrs(seg, off, &item_key, NULL); - BUG_ON(scoutfs_key_compare(key, &item_key) <= 0); + item = off_ptr(seg, off); + BUG_ON(scoutfs_key_compare(key, &item->key) <= 0); } nr_links = skip_next_nr(le32_to_cpu(sblk->nr_items)); - bytes = item_bytes(nr_links, key->key_len, val_len); + bytes = item_bytes(nr_links, val_len); off = align_item_off(seg, le32_to_cpu(sblk->total_bytes), bytes); if ((off + bytes) > SCOUTFS_SEGMENT_SIZE) @@ -690,7 +673,7 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg le32_add_cpu(&sblk->nr_items, 1); item = off_ptr(seg, off); - item->key_len = cpu_to_le16(key->key_len); + item->key = *key; item->val_len = cpu_to_le16(val_len); item->flags = flags; @@ -702,8 +685,7 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg links[i] = &item->skip_links[i]; } - item_ptrs(seg, off, &item_key, &item_val); - scoutfs_key_copy(&item_key, key); + get_item_key_val(seg, off, NULL, &item_val); if (val_len) memcpy(item_val.iov_base, val->iov_base, val_len); @@ -714,8 +696,8 @@ void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, struct scoutfs_segment *seg) { struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; + struct scoutfs_key first; + struct scoutfs_key last; first_last_keys(seg, &first, &last); diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 4f151490..74f7f961 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -2,7 +2,7 @@ #define _SCOUTFS_SEG_H_ struct scoutfs_bio_completion; -struct scoutfs_key_buf; +struct scoutfs_key; struct scoutfs_manifest_entry; struct kvec; @@ -23,24 +23,21 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, u64 segno, u64 seq); -int scoutfs_seg_find_off(struct scoutfs_segment *seg, - struct scoutfs_key_buf *key); +int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key *key); int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off); u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg); -int scoutfs_seg_item_ptrs(struct scoutfs_segment *seg, int off, - struct scoutfs_key_buf *key, struct kvec *val, - u8 *flags); +int scoutfs_seg_get_item(struct scoutfs_segment *seg, int off, + struct scoutfs_key *key, struct kvec *val, u8 *flags); void scoutfs_seg_get(struct scoutfs_segment *seg); void scoutfs_seg_put(struct scoutfs_segment *seg); int scoutfs_seg_alloc(struct super_block *sb, u64 segno, struct scoutfs_segment **seg_ret); -int scoutfs_seg_free_segno(struct super_block *sb, - struct scoutfs_segment *seg); -bool scoutfs_seg_fits_single(u32 nr_items, u32 key_bytes, u32 val_bytes); +int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); +bool scoutfs_seg_fits_single(u32 nr_items, u32 val_bytes); bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key_buf *key, struct kvec *val, + struct scoutfs_key *key, struct kvec *val, u8 flags, __le32 **links); void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, struct scoutfs_segment *seg); diff --git a/kmod/src/server.c b/kmod/src/server.c index ff1ccc69..7482e4bd 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -232,67 +232,24 @@ static int send_reply(struct server_connection *conn, u64 id, return ret; } -void scoutfs_init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last) +void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, + struct scoutfs_manifest_entry *ment) { - scoutfs_key_init(first, net_ment->keys, - le16_to_cpu(net_ment->first_key_len)); - scoutfs_key_init(last, net_ment->keys + - le16_to_cpu(net_ment->first_key_len), - le16_to_cpu(net_ment->last_key_len)); -} - -/* - * Allocate a contiguous manifest entry for communication over the network. - */ -struct scoutfs_net_manifest_entry * -scoutfs_alloc_net_ment(struct scoutfs_manifest_entry *ment) -{ - struct scoutfs_net_manifest_entry *net_ment; - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - net_ment = kmalloc(offsetof(struct scoutfs_net_manifest_entry, - keys[ment->first.key_len + - ment->last.key_len]), GFP_NOFS); - if (!net_ment) - return NULL; - net_ment->segno = cpu_to_le64(ment->segno); net_ment->seq = cpu_to_le64(ment->seq); - net_ment->first_key_len = cpu_to_le16(ment->first.key_len); - net_ment->last_key_len = cpu_to_le16(ment->last.key_len); + net_ment->first = ment->first; + net_ment->last = ment->last; net_ment->level = ment->level; - - scoutfs_init_net_ment_keys(net_ment, &first, &last); - scoutfs_key_copy(&first, &ment->first); - scoutfs_key_copy(&last, &ment->last); - - return net_ment; } -/* point a native manifest entry at a contiguous net manifest */ -void scoutfs_init_ment_net_ment(struct scoutfs_manifest_entry *ment, +void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment) { - struct scoutfs_key_buf first; - struct scoutfs_key_buf last; - - scoutfs_init_net_ment_keys(net_ment, &first, &last); - scoutfs_key_clone(&ment->first, &first); - scoutfs_key_clone(&ment->last, &last); - ment->segno = le64_to_cpu(net_ment->segno); ment->seq = le64_to_cpu(net_ment->seq); ment->level = net_ment->level; -} - -unsigned scoutfs_net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment) -{ - return offsetof(struct scoutfs_net_manifest_entry, - keys[le16_to_cpu(net_ment->first_key_len) + - le16_to_cpu(net_ment->last_key_len)]); + ment->first = net_ment->first; + ment->last = net_ment->last; } static int process_alloc_inodes(struct server_connection *conn, @@ -381,7 +338,7 @@ static int process_record_segment(struct server_connection *conn, u64 id, net_ment = data; - if (data_len != scoutfs_net_ment_bytes(net_ment)) { + if (data_len != sizeof(*net_ment)) { ret = -EINVAL; goto out; } @@ -399,7 +356,7 @@ retry: goto retry; } - scoutfs_init_ment_net_ment(&ment, net_ment); + scoutfs_init_ment_from_net(&ment, net_ment); ret = scoutfs_manifest_add(sb, &ment); scoutfs_manifest_unlock(sb); diff --git a/kmod/src/server.h b/kmod/src/server.h index 8cb7c05c..f6e076db 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -1,14 +1,10 @@ #ifndef _SCOUTFS_SERVER_H_ #define _SCOUTFS_SERVER_H_ -void scoutfs_init_net_ment_keys(struct scoutfs_net_manifest_entry *net_ment, - struct scoutfs_key_buf *first, - struct scoutfs_key_buf *last); -struct scoutfs_net_manifest_entry * -scoutfs_alloc_net_ment(struct scoutfs_manifest_entry *ment); -void scoutfs_init_ment_net_ment(struct scoutfs_manifest_entry *ment, +void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, + struct scoutfs_manifest_entry *ment); +void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment); -unsigned scoutfs_net_ment_bytes(struct scoutfs_net_manifest_entry *net_ment); int scoutfs_client_get_compaction(struct super_block *sb, void *curs); int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, diff --git a/kmod/src/super.c b/kmod/src/super.c index f9d5b80e..9d97e578 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -419,6 +419,7 @@ static int __init scoutfs_module_init(void) ".string \""SCOUTFS_GIT_DESCRIBE"\\n\"\n" ".previous\n"); + scoutfs_key_init(); scoutfs_init_counters(); ret = scoutfs_data_test(); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 712ac1f6..2c39951f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -60,7 +60,6 @@ struct trans_info { spinlock_t lock; unsigned reserved_items; - unsigned reserved_keys; unsigned reserved_vals; unsigned holders; bool writing; @@ -295,7 +294,6 @@ static bool acquired_hold(struct super_block *sb, DECLARE_TRANS_INFO(sb, tri); bool acquired = false; unsigned items; - unsigned keys; unsigned vals; bool fits; @@ -305,7 +303,6 @@ static bool acquired_hold(struct super_block *sb, &rsv->reserved, &rsv->actual, tri->holders, tri->writing, tri->reserved_items, - tri->reserved_keys, tri->reserved_vals); /* use a caller's existing reservation */ @@ -318,9 +315,8 @@ static bool acquired_hold(struct super_block *sb, /* see if we can reserve space for our item count */ items = tri->reserved_items + cnt->items; - keys = tri->reserved_keys + cnt->keys; vals = tri->reserved_vals + cnt->vals; - fits = scoutfs_item_dirty_fits_single(sb, items, keys, vals); + fits = scoutfs_item_dirty_fits_single(sb, items, vals); if (!fits) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); @@ -328,11 +324,9 @@ static bool acquired_hold(struct super_block *sb, } tri->reserved_items = items; - tri->reserved_keys = keys; tri->reserved_vals = vals; rsv->reserved.items = cnt->items; - rsv->reserved.keys = cnt->keys; rsv->reserved.vals = cnt->vals; hold: @@ -358,9 +352,8 @@ int scoutfs_hold_trans(struct super_block *sb, * Caller shouldn't provide garbage counts, nor counts that * can't fit in segments by themselves. */ - if (WARN_ON_ONCE(cnt.items <= 0 || cnt.keys < 0 || cnt.vals < 0) || - WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt.items, cnt.keys, - cnt.vals))) + if (WARN_ON_ONCE(cnt.items <= 0 || cnt.vals < 0) || + WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt.items, cnt.vals))) return -EINVAL; if (current == sbi->trans_task) @@ -400,7 +393,7 @@ bool scoutfs_trans_held(void) } void scoutfs_trans_track_item(struct super_block *sb, signed items, - signed keys, signed vals) + signed vals) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_reservation *rsv = current->journal_info; @@ -411,11 +404,9 @@ void scoutfs_trans_track_item(struct super_block *sb, signed items, BUG_ON(!rsv || rsv->magic != SCOUTFS_RESERVATION_MAGIC); rsv->actual.items += items; - rsv->actual.keys += keys; rsv->actual.vals += vals; WARN_ON_ONCE(rsv->actual.items > rsv->reserved.items); - WARN_ON_ONCE(rsv->actual.keys > rsv->reserved.keys); WARN_ON_ONCE(rsv->actual.vals > rsv->reserved.vals); } @@ -442,15 +433,13 @@ void scoutfs_release_trans(struct super_block *sb) trace_scoutfs_release_trans(sb, rsv, rsv->holders, &rsv->reserved, &rsv->actual, tri->holders, tri->writing, - tri->reserved_items, tri->reserved_keys, - tri->reserved_vals); + tri->reserved_items, tri->reserved_vals); BUG_ON(rsv->holders <= 0); BUG_ON(tri->holders <= 0); if (--rsv->holders == 0) { tri->reserved_items -= rsv->reserved.items; - tri->reserved_keys -= rsv->reserved.keys; tri->reserved_vals -= rsv->reserved.vals; current->journal_info = NULL; kfree(rsv); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 0df05ff3..04e28f9c 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -14,7 +14,7 @@ int scoutfs_hold_trans(struct super_block *sb, bool scoutfs_trans_held(void); void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, - signed keys, signed vals); + signed vals); int scoutfs_setup_trans(struct super_block *sb); void scoutfs_shutdown_trans(struct super_block *sb); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index cd6df413..d2e57277 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -68,19 +68,17 @@ static unsigned int xattr_nr_parts(struct scoutfs_xattr *xat) le16_to_cpu(xat->val_len)); } -/* If no name is provided then the hash arg is used, caller can modify part */ -static void init_xattr_key(struct scoutfs_key_buf *key, - struct scoutfs_xattr_key *xak, u64 ino, - u32 name_hash, u64 id) +static void init_xattr_key(struct scoutfs_key *key, u64 ino, u32 name_hash, + u64 id) { - xak->zone = SCOUTFS_FS_ZONE; - xak->ino = cpu_to_be64(ino); - xak->type = SCOUTFS_XATTR_TYPE; - xak->name_hash = cpu_to_be32(name_hash); - xak->id = cpu_to_be64(id); - xak->part = 0; - - scoutfs_key_init(key, xak, sizeof(struct scoutfs_xattr_key)); + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skx_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_XATTR_TYPE, + .skx_name_hash = cpu_to_le64(name_hash), + .skx_id = cpu_to_le64(id), + .skx_part = 0, + }; } static int unknown_prefix(const char *name) @@ -108,15 +106,13 @@ static int unknown_prefix(const char *name) * * Returns -ENOENT if it didn't find a next item. */ -static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, +static int get_next_xattr(struct inode *inode, struct scoutfs_key *key, struct scoutfs_xattr *xat, unsigned int bytes, const char *name, unsigned int name_len, u64 name_hash, u64 id, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_xattr_key last_xak; - struct scoutfs_key_buf last; - struct scoutfs_key_buf key; + struct scoutfs_key last; struct kvec val; u8 last_part; int total; @@ -131,17 +127,17 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, if (name_len) name_hash = xattr_name_hash(name, name_len); - init_xattr_key(&key, xak, scoutfs_ino(inode), name_hash, id); - init_xattr_key(&last, &last_xak, scoutfs_ino(inode), U32_MAX, U64_MAX); + init_xattr_key(key, scoutfs_ino(inode), name_hash, id); + init_xattr_key(&last, scoutfs_ino(inode), U32_MAX, U64_MAX); last_part = 0; part = 0; total = 0; for (;;) { - xak->part = part; + key->skx_part = part; kvec_init(&val, (void *)xat + total, bytes - total); - ret = scoutfs_item_next(sb, &key, &last, &val, lock); + ret = scoutfs_item_next(sb, key, &last, &val, lock); if (ret < 0) { /* XXX corruption, ran out of parts */ if (ret == -ENOENT && part > 0) @@ -149,10 +145,10 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, break; } - trace_scoutfs_xattr_get_next_key(sb, &key); + trace_scoutfs_xattr_get_next_key(sb, key); /* XXX corruption */ - if (xak->part != part) { + if (key->skx_part != part) { ret = -EIO; break; } @@ -175,7 +171,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, if (part == 0 && name_len) { /* ran out of names that could match */ - if (be32_to_cpu(xak->name_hash) != name_hash) { + if (le64_to_cpu(key->skx_name_hash) != name_hash) { ret = -ENOENT; break; } @@ -184,7 +180,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_xattr_key *xak, if (!xattr_names_equal(name, name_len, xat->name, xat->name_len)) { part = 0; - be64_add_cpu(&xak->id, 1); + le64_add_cpu(&key->skx_id, 1); continue; } @@ -214,14 +210,13 @@ static int create_xattr_items(struct inode *inode, u64 id, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_xattr_key xak; - struct scoutfs_key_buf key; + struct scoutfs_key key; unsigned int part_bytes; struct kvec val; int total; int ret; - init_xattr_key(&key, &xak, scoutfs_ino(inode), + init_xattr_key(&key, scoutfs_ino(inode), xattr_name_hash(xat->name, xat->name_len), id); total = 0; @@ -232,13 +227,13 @@ static int create_xattr_items(struct inode *inode, u64 id, ret = scoutfs_item_create(sb, &key, &val, lock); if (ret) { - while (xak.part-- > 0) + while (key.skx_part-- > 0) scoutfs_item_delete_dirty(sb, &key); break; } total += part_bytes; - xak.part++; + key.skx_part++; } return ret; @@ -249,20 +244,19 @@ static int create_xattr_items(struct inode *inode, u64 id, * returns an error then the deleted and saved items are left on the * list for the caller to restore. */ -static int delete_xattr_items(struct inode *inode, u64 name_hash, u64 id, +static int delete_xattr_items(struct inode *inode, u32 name_hash, u64 id, u8 nr_parts, struct list_head *list, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_xattr_key xak; - struct scoutfs_key_buf key; + struct scoutfs_key key; int ret; - init_xattr_key(&key, &xak, scoutfs_ino(inode), name_hash, id); + init_xattr_key(&key, scoutfs_ino(inode), name_hash, id); do { ret = scoutfs_item_delete_save(sb, &key, list, lock); - } while (ret == 0 && ++xak.part < nr_parts); + } while (ret == 0 && ++key.skx_part < nr_parts); return ret; } @@ -279,7 +273,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, struct super_block *sb = inode->i_sb; struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; - struct scoutfs_xattr_key xak; + struct scoutfs_key key; unsigned int bytes; size_t name_len; int ret; @@ -303,7 +297,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, down_read(&si->xattr_rwsem); - ret = get_next_xattr(inode, &xak, xat, bytes, + ret = get_next_xattr(inode, &key, xat, bytes, name, name_len, 0, 0, lck); up_read(&si->xattr_rwsem); @@ -360,7 +354,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; size_t name_len = strlen(name); - struct scoutfs_xattr_key xak; + struct scoutfs_key key; LIST_HEAD(ind_locks); LIST_HEAD(saved); u8 found_parts; @@ -399,7 +393,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, down_write(&si->xattr_rwsem); /* find an existing xattr to delete */ - ret = get_next_xattr(inode, &xak, xat, + ret = get_next_xattr(inode, &key, xat, sizeof(struct scoutfs_xattr) + name_len, name, name_len, 0, 0, lck); if (ret < 0 && ret != -ENOENT) @@ -420,7 +414,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto unlock; } - /* found fields in xak will also be used */ + /* found fields in key will also be used */ found_parts = ret >= 0 ? xattr_nr_parts(xat) : 0; /* prepare our xattr */ @@ -450,8 +444,8 @@ retry: ret = 0; if (found_parts) - ret = delete_xattr_items(inode, be32_to_cpu(xak.name_hash), - be64_to_cpu(xak.id), found_parts, + ret = delete_xattr_items(inode, le64_to_cpu(key.skx_name_hash), + le64_to_cpu(key.skx_id), found_parts, &saved, lck); if (value && ret == 0) ret = create_xattr_items(inode, id, xat, bytes, lck); @@ -500,7 +494,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct super_block *sb = inode->i_sb; struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; - struct scoutfs_xattr_key xak; + struct scoutfs_key key; unsigned int bytes; ssize_t total; u32 name_hash; @@ -526,7 +520,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) total = 0; for (;;) { - ret = get_next_xattr(inode, &xak, xat, bytes, + ret = get_next_xattr(inode, &key, xat, bytes, NULL, 0, name_hash, id, lck); if (ret < 0) { if (ret == -ENOENT) @@ -547,8 +541,8 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) *(buffer++) = '\0'; } - name_hash = be32_to_cpu(xak.name_hash); - id = be64_to_cpu(xak.id) + 1; + name_hash = le64_to_cpu(key.skx_name_hash); + id = le64_to_cpu(key.skx_id) + 1; } up_read(&si->xattr_rwsem); @@ -571,15 +565,13 @@ out: */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) { - struct scoutfs_xattr_key last_xak; - struct scoutfs_xattr_key xak; - struct scoutfs_key_buf last; - struct scoutfs_key_buf key; + struct scoutfs_key last; + struct scoutfs_key key; struct scoutfs_lock *lck; int ret; - init_xattr_key(&key, &xak, ino, 0, 0); - init_xattr_key(&last, &last_xak, ino, U32_MAX, U64_MAX); + init_xattr_key(&key, ino, 0, 0); + init_xattr_key(&last, ino, U32_MAX, U64_MAX); /* while we read to delete we need to writeback others */ ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lck); @@ -598,7 +590,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino) if (ret) break; - xak.part++; + key.skx_part++; } scoutfs_unlock(sb, lck, DLM_LOCK_EX); From 4b413ed8041e4232b4b94fc1d992104824d93e96 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 26 Mar 2018 14:35:47 -0700 Subject: [PATCH 588/920] scoutfs: add seg item append trace point Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 33 +++++++++++++++++++++++++++++++++ kmod/src/seg.c | 6 ++++++ 2 files changed, 39 insertions(+) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 20c3a16b..95d4676c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1667,6 +1667,39 @@ DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_free, TP_ARGS(seg) ); +TRACE_EVENT(scoutfs_seg_append_item, + TP_PROTO(struct super_block *sb, u64 segno, u64 seq, u32 nr_items, + u32 total_bytes, struct scoutfs_key *key, u16 val_len), + + TP_ARGS(sb, segno, seq, nr_items, total_bytes, key, val_len), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, segno) + __field(__u64, seq) + __field(__u32, nr_items) + __field(__u32, total_bytes) + __field_struct(struct scoutfs_key, key) + __field(__u16, val_len) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + __entry->seq = seq; + __entry->nr_items = nr_items; + __entry->total_bytes = total_bytes; + __entry->key = *key; + __entry->val_len = val_len; + ), + + TP_printk("fsid "FSID_FMT" segno %llu seq %llu nr_items %u total_bytes %u key "SK_FMT" val_len %u", + __entry->fsid, __entry->segno, __entry->seq, + __entry->nr_items, __entry->total_bytes, + SK_ARG(&__entry->key), + __entry->val_len) +); + DECLARE_EVENT_CLASS(scoutfs_net_class, TP_PROTO(struct super_block *sb, struct sockaddr_in *name, struct sockaddr_in *peer, struct scoutfs_net_header *nh), diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 80ccdacd..600c4560 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -650,6 +650,12 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg links[i] = &sblk->skip_links[i]; } + trace_scoutfs_seg_append_item(sb, le64_to_cpu(sblk->segno), + le64_to_cpu(sblk->seq), + le32_to_cpu(sblk->nr_items), + le32_to_cpu(sblk->total_bytes), + key, val_len); + /* * It's very bad data corruption if we write out of order items * to a segment. It'll mislead the key search during read and From 704714c2ee46c96006ea8e25ff574c1a1cfbfd9c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 27 Mar 2018 12:44:27 -0700 Subject: [PATCH 589/920] scoutfs: add scoutfs_bug_on() Add a BUG_ON() wrapper that identifies the file system via the super block and prints the condition and some additional formatted output. Signed-off-by: Zach Brown --- kmod/src/msg.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kmod/src/msg.h b/kmod/src/msg.h index eff34766..0586f75c 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -15,4 +15,12 @@ void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, #define scoutfs_info(sb, fmt, args...) \ scoutfs_msg(sb, KERN_INFO, "", fmt, ##args) +#define scoutfs_bug_on(sb, cond, fmt, args...) \ +do { \ + if (cond) { \ + scoutfs_err(sb, "(" __stringify(cond) "), " fmt, ##args); \ + BUG(); \ + } \ +} while (0) \ + #endif From 62e26c5d96de7ea62131b1622e71fe39a51be03e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 27 Mar 2018 12:45:07 -0700 Subject: [PATCH 590/920] scoutfs: scoutfs_bug_on to show bad append order Use scoutfs_bug_on() to freak out if we append items to a segment out of order. We don't really return errors from this path, we should, but for now at least share the keys that show the problem. Signed-off-by: Zach Brown --- kmod/src/seg.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 600c4560..fea5ea4f 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -27,6 +27,7 @@ #include "key.h" #include "counters.h" #include "triggers.h" +#include "msg.h" #include "scoutfs_trace.h" /* @@ -664,7 +665,9 @@ bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg off = le32_to_cpu(sblk->last_item_off); if (off) { item = off_ptr(seg, off); - BUG_ON(scoutfs_key_compare(key, &item->key) <= 0); + scoutfs_bug_on(sb, scoutfs_key_compare(key, &item->key) <= 0, + "key "SK_FMT" item->key "SK_FMT, + SK_ARG(key), SK_ARG(&item->key)); } nr_links = skip_next_nr(le32_to_cpu(sblk->nr_items)); From 5001631dd9fa9b2c8ae216caccf8befdf261f1a3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 27 Mar 2018 12:46:22 -0700 Subject: [PATCH 591/920] scoutfs: add item deletion tracing Add some traces for item deletion functions with their return values. Signed-off-by: Zach Brown --- kmod/src/item.c | 17 +++++++++++------ kmod/src/scoutfs_trace.h | 20 ++++++++++++++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 86697a31..77cd5057 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1343,7 +1343,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, (ret = scoutfs_manifest_read_items(sb, key, &lock->start, &lock->end)) == 0); - trace_scoutfs_item_delete_ret(sb, ret); + trace_scoutfs_item_delete(sb, key, ret); return ret; } @@ -1404,12 +1404,16 @@ int scoutfs_item_delete_save(struct super_block *sb, bool was_dirty; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) - return -EINVAL; + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + ret = -EINVAL; + goto out; + } del = alloc_item(sb, key, NULL); - if (!del) - return -ENOMEM; + if (!del) { + ret = -ENOMEM; + goto out; + } do { spin_lock_irqsave(&cac->lock, flags); @@ -1440,7 +1444,8 @@ int scoutfs_item_delete_save(struct super_block *sb, &lock->end)) == 0); free_item(sb, del); - +out: + trace_scoutfs_item_delete_save(sb, key, ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 95d4676c..1ac7765b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -227,22 +227,34 @@ TRACE_EVENT(scoutfs_dec_end_io, __entry->args, __entry->in_flight, __entry->err) ); -TRACE_EVENT(scoutfs_item_delete_ret, - TP_PROTO(struct super_block *sb, int ret), +DECLARE_EVENT_CLASS(scoutfs_key_ret_class, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), - TP_ARGS(sb, ret), + TP_ARGS(sb, key, ret), TP_STRUCT__entry( __field(__u64, fsid) + __field_struct(struct scoutfs_key, key) __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); + __entry->key = *key; __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk("fsid "FSID_FMT" key "SK_FMT" ret %d", + __entry->fsid, SK_ARG(&__entry->key), __entry->ret) +); + +DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_delete, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), + TP_ARGS(sb, key, ret) +); +DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_delete_save, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), + TP_ARGS(sb, key, ret) ); TRACE_EVENT(scoutfs_item_dirty_ret, From 045380ca55b316f286e56978c33e8bcb1c71bb21 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Mar 2018 08:58:07 -0700 Subject: [PATCH 592/920] scoutfs: don't negatively cache unread segments Previously we changed item reading to try and read from the start of its locked range instead of from the key that wasn't found in the cache. This greatly improved the performance of access patterns that didn't proceed in key order. We rightly shrank the range of items that we'd claim to cache by the segments that we read. But we missed the case where our search key falls between two segments and we chose to read the next segment instead of the previous. If the previous segment in this case overlapped with the lock range then we were claiming to cache the segments contents but weren't reading it. This would result in bad negative caching of items that existed. scoutfs/500 was tripping over this as it tried to rename a file created by another node. The local renaming node would try to look up a key that only existed in level 0 and not read but negatively cache the items in the previous level 1 segment. We fix this by shrinking the caching range down as we're considering manifest entries instead of up as we process each segment read because we have to shrink based on the segments in the manifest, not the ones we chose to read. With this fixed the rename can see those items in the level 1 segment again. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 98 ++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index a41ba62a..d667f339 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -285,33 +285,53 @@ static int alloc_manifest_ref(struct super_block *sb, struct list_head *ref_list } /* - * Return the previous entry if it's in the right level and it overlaps - * with the start key by having a last key that's >=. If no such entry - * exists it just returns the next entry after the key and doesn't test - * it at all. If this returns 0 then the caller has to put the iref. + * Give the caller the next entry that overlaps with the given key at th + * egiven level. We first check the previous entry before the key to + * see if it overlaps. If it does then we return it. If it doesn't + * then we return the raw next entry after the key. The caller has to + * test it. + * + * If a start key is provided then the caller is working with cache + * ranges. If we find a previous entry that doesn't contain the key + * then we see if we should shrink the range to make sure that it + * doesn't include this segment whose items we're not using. + * + * Returns 0 with the iref pointing to the btree item with the entry, + * callers has to put the iref when they're done. */ static int btree_prev_overlap_or_next(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + void *bkey, unsigned bkey_len, + struct scoutfs_key *key, struct scoutfs_key *start, u8 level, struct scoutfs_btree_item_ref *iref) { struct scoutfs_manifest_entry ment; int ret; - ret = scoutfs_btree_prev(sb, root, key, key_len, iref); + ret = scoutfs_btree_prev(sb, root, bkey, bkey_len, iref); if (ret < 0 && ret != -ENOENT) return ret; if (ret == 0) { init_ment_iref(&ment, iref); + + /* shrink range so it doesn't cover skipped prev */ + if (start && ment.level == level && + scoutfs_key_compare(&ment.last, key) < 0 && + scoutfs_key_compare(&ment.last, start) >= 0) { + *start = ment.last; + scoutfs_key_inc(start); + } + + /* skip prev that doesn't contain the key */ if (ment.level != level || - scoutfs_key_compare(&ment.last, start) < 0) + scoutfs_key_compare(&ment.last, key) < 0) ret = -ENOENT; } if (ret == -ENOENT) { scoutfs_btree_put_iref(iref); - ret = scoutfs_btree_next(sb, root, key, key_len, iref); + ret = scoutfs_btree_next(sb, root, bkey, bkey_len, iref); } return ret; @@ -381,19 +401,26 @@ static int get_zero_refs(struct super_block *sb, /* * Get references to all segments in non-zero levels that contain the - * caller's search key. The item ranges of segments at each non-zero - * level don't overlap so we can iterate through the key space in each - * segment starting with the search key. In each level we need the - * first existing segment that intersects with the range, even if it - * doesn't contain the key. The key might fall between segments at that - * level. If a segment is entirely outside of the caller's range then - * we can't trust its contents. + * caller's key. The item ranges of segments at each non-zero level + * don't overlap so we can iterate through the key space in each segment + * starting with the search key. In each level we need the first + * existing segment that intersects with the range, even if it doesn't + * contain the key. The key might fall between segments at that level. + * + * The caller can provide the range of items that they're going to + * consider authoritative for the range of segments that we give them. + * We have to shrink this range if we give them segments that don't + * cover the range. This includes implicitly negative cached space + * that's created by using the segment after the hole between segments. + * If a segment is entirely outside of the caller's range then we can't + * trust its contents. * * This can return -ESTALE if it reads through stale btree blocks. */ static int get_nonzero_refs(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, + struct scoutfs_key *start, struct scoutfs_key *end, struct list_head *ref_list) { @@ -403,11 +430,15 @@ static int get_nonzero_refs(struct super_block *sb, int ret; int i; + if (WARN_ON_ONCE(!!start != !!end) || + WARN_ON_ONCE(start && scoutfs_key_compare(start, end) > 0)) + return -EINVAL; + for (i = 1; ; i++) { init_btree_key(&mkey, i, 0, key); ret = btree_prev_overlap_or_next(sb, root, &mkey, sizeof(mkey), - key, i, &iref); + key, start, i, &iref); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -418,12 +449,20 @@ static int get_nonzero_refs(struct super_block *sb, scoutfs_btree_put_iref(&iref); if (ment.level != i || - scoutfs_key_compare(&ment.first, end) > 0) + (end && scoutfs_key_compare(&ment.first, end) > 0)) continue; ret = alloc_manifest_ref(sb, ref_list, &ment); if (ret) break; + + if (start && scoutfs_key_compare(&ment.first, start) > 0 && + scoutfs_key_compare(&ment.first, key) <= 0) + *start = ment.first; + + if (end && scoutfs_key_compare(&ment.last, end) < 0 && + scoutfs_key_compare(&ment.last, key) >= 0) + *end = ment.last; } return ret; @@ -551,23 +590,11 @@ retry_stale: if (ret) goto out; - /* get non-zero segments that intersect with the missed key */ - ret = get_nonzero_refs(sb, &root, key, &seg_end, &ref_list); + /* get non-zero segments that intersect with the key, shrinks range */ + ret = get_nonzero_refs(sb, &root, key, &seg_start, &seg_end, &ref_list); if (ret) goto out; - /* clamp start and end to the segment boundaries, including key */ - list_for_each_entry(ref, &ref_list, entry) { - - if (scoutfs_key_compare(&ref->first, &seg_start) > 0 && - scoutfs_key_compare(&ref->first, key) <= 0) - seg_start = ref->first; - - if (scoutfs_key_compare(&ref->last, &seg_end) < 0 && - scoutfs_key_compare(&ref->last, key) >= 0) - seg_end = ref->last; - } - trace_scoutfs_read_item_keys(sb, key, start, end, &seg_start, &seg_end); /* then get level 0s that intersect with our search range */ @@ -747,7 +774,6 @@ int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next_key) { struct scoutfs_key item_key; - struct scoutfs_key end; struct scoutfs_btree_root root; struct scoutfs_segment *seg; struct manifest_ref *ref; @@ -764,10 +790,8 @@ retry_stale: if (ret) goto out; - scoutfs_key_set_ones(&end); - - ret = get_zero_refs(sb, &root, key, &end, &ref_list) ?: - get_nonzero_refs(sb, &root, key, &end, &ref_list); + ret = get_zero_refs(sb, &root, key, key, &ref_list) ?: + get_nonzero_refs(sb, &root, key, NULL, NULL, &ref_list); if (ret) goto out; @@ -975,7 +999,7 @@ again: init_btree_key(&mkey, level + 1, 0, &ment.first); ret = btree_prev_overlap_or_next(sb, &super->manifest.root, &mkey, sizeof(mkey), &ment.first, - level + 1, &over_iref); + NULL, level + 1, &over_iref); sticky = false; for (i = 0; ret == 0 && i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { init_ment_iref(&over, &over_iref); From 966c8b8cbc5cb58bbbff8196107505591d4fe5e2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Apr 2018 09:18:28 -0700 Subject: [PATCH 593/920] scoutfs: alloc inos at multiple of lock group Inode allocations come from batches that are reserved for directories. As the batch is exhausted a new one is acquired and allocated from. The batch size was arbitrarily set to the human friendly 10000. This doesn't interact well with the lock group size being a power of two. Each allocation batch will straddle an inode group with its previous and next inode batch. This often doesn't matter because dirctories very rarely have more than 9000 entries. But as entries pass 10000 they'd see surprising contention with other inode ranges in directories. Tweak the allocation size to be a multiple of the lock group size to stop this from happening. Signed-off-by: Zach Brown --- kmod/src/inode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 6708c0a8..2812ae25 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -1272,7 +1272,9 @@ int scoutfs_alloc_ino(struct inode *parent, u64 *ino_ret) if (ia->nr == 0) { spin_unlock(&ia->lock); - ret = scoutfs_client_alloc_inodes(sb, 10000, &ino, &nr); + ret = scoutfs_client_alloc_inodes(sb, + SCOUTFS_LOCK_INODE_GROUP_NR * 10, + &ino, &nr); if (ret < 0) goto out; spin_lock(&ia->lock); From e1f32a0f8befc82a6f3c54bbd36ff6f2d3b8f3b3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 10 Apr 2018 15:32:58 -0700 Subject: [PATCH 594/920] scoutfs: fix spurious hard stale block errors The stale block handling code only handled the case where we read through a stale root into blocks that have been overwritten in the persistent store. In this case you'll get a new root and the read will be OK. It didn't handle the case where we have stale blocks cached at the blocks of the legitamate current root. In this case we get ESTALE from each stale block and because the root doesn't change when we retry we assume the persistent structure is corrupt. This case can happen when the btree ring wraps and there are still blocks cached at the head of the ring. This became much more possible when we moved to small fixed size keys. The fix is to retry reading individual blocks or segments before returning -ESTALE and expecting the caller to get a new root and try again. In the stale cache case this will allow the more recent correct blocks to be read. Signed-off-by: Zach Brown --- kmod/src/btree.c | 21 +++++++++++++++------ kmod/src/manifest.c | 22 ++++++++++++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 34d4434a..9e5effd0 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -604,11 +604,12 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, * dirtying, and allocate new blocks. * * Btree blocks don't have rigid cache consistency. We can be following - * a new root to read refs into previously stale cached blocks. If we - * hit a cached block that doesn't match the ref (or indeed a corrupt - * block) we return -ESTALE which tells the caller to deal with this - * error: either find a new root or return a hard error if the block is - * really corrupt. + * block references into cached blocks that are now stale or can be + * following a stale root into blocks that have been overwritten. If we + * hit a block that looks stale we first invalidate the cache and retry, + * returning -ESTALE if it still looks wrong. The caller can retry the + * read from a more current root or decide that this is a persistent + * error. * * btree callers serialize concurrent writers in a btree but not between * btrees. We have to lock around the shared btree_info. Callers do @@ -626,6 +627,7 @@ static int get_ref_block(struct super_block *sb, int flags, struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_block *new; struct buffer_head *bh; + bool retried = false; u64 blkno; u64 seq; int ret; @@ -633,6 +635,7 @@ static int get_ref_block(struct super_block *sb, int flags, /* always get the current block, either to return or cow from */ if (ref && ref->blkno) { +retry: bh = sb_bread(sb, le64_to_cpu(ref->blkno)); if (!bh) { ret = -EIO; @@ -643,13 +646,19 @@ static int get_ref_block(struct super_block *sb, int flags, if (!valid_referenced_block(super, ref, bt, bh) || scoutfs_trigger(sb, BTREE_STALE_READ)) { + scoutfs_inc_counter(sb, btree_stale_read); + lock_buffer(bh); clear_buffer_uptodate(bh); unlock_buffer(bh); put_bh(bh); bt = NULL; - scoutfs_inc_counter(sb, btree_stale_read); + if (!retried) { + retried = true; + goto retry; + } + ret = -ESTALE; goto out; } diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index d667f339..fcf404a2 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -76,6 +76,7 @@ struct manifest_ref { int found_ctr; int off; u8 level; + bool retried; struct scoutfs_key first; struct scoutfs_key last; @@ -469,9 +470,11 @@ static int get_nonzero_refs(struct super_block *sb, } /* - * See if the caller is a remote btree reader who has read a stale btree - * block and should keep trying. If they see repeated errors on the - * same root then we assume that it's persistent corruption. + * If we saw persistent stale blocks or segment reads while walking the + * manifest then we might be trying to read through an old stale root + * that has been overwritten. We can ask for a new root and try again. + * If we don't get a new root and the errors persist then the we've hit + * corruption. */ static int handle_stale_btree(struct super_block *sb, struct scoutfs_btree_root *root, @@ -605,8 +608,12 @@ retry_stale: /* sort by segment to issue advancing reads */ list_sort(NULL, &ref_list, cmp_ment_ref_segno); +resubmit: /* submit reads for all the segments */ list_for_each_entry(ref, &ref_list, entry) { + /* don't resubmit if we've read */ + if (ref->seg) + continue; trace_scoutfs_read_item_segment(sb, ref->level, ref->segno, ref->seq, &ref->first, @@ -624,9 +631,16 @@ retry_stale: /* always wait for submitted segments */ list_for_each_entry(ref, &ref_list, entry) { if (!ref->seg) - break; + continue; err = scoutfs_seg_wait(sb, ref->seg, ref->segno, ref->seq); + if (err == -ESTALE && !ref->retried) { + ref->retried = true; + err = 0; + scoutfs_seg_put(ref->seg); + ref->seg = NULL; + goto resubmit; + } if (err && !ret) ret = err; } From 90de34361c6834e0a6ca1ea49f70cbdd1cdc5fef Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Apr 2018 09:56:31 -0700 Subject: [PATCH 595/920] scoutfs: add trigger for advancing btree ring Add a trigger that lets us force advancing the btree block to the start of the next half. It's only safe to do this once migration has moved all the blocks out of the old half. Signed-off-by: Zach Brown --- kmod/src/btree.c | 44 ++++++++++++++++++++++++++++++++++++++++---- kmod/src/triggers.c | 1 + kmod/src/triggers.h | 1 + 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 9e5effd0..104b4e54 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -324,6 +324,25 @@ static bool first_block_in_half(struct scoutfs_btree_ring *bring) return block == 0 || block == (le64_to_cpu(bring->nr_blocks) / 2); } +/* Set next_block to the start of the other half */ +static void advance_to_next_half(struct scoutfs_btree_ring *bring) +{ + u64 block = le64_to_cpu(bring->next_block); + u64 half = le64_to_cpu(bring->nr_blocks) / 2; + u64 offset; + + if (block >= half) { + offset = le64_to_cpu(bring->nr_blocks) - block; + block = 0; + } else { + offset = half - block; + block = half; + } + + bring->next_block = cpu_to_le64(block); + le64_add_cpu(&bring->next_seq, offset); +} + static size_t super_root_offsets[] = { offsetof(struct scoutfs_super_block, alloc_root), offsetof(struct scoutfs_super_block, manifest.root), @@ -334,6 +353,19 @@ static size_t super_root_offsets[] = { (root = ((void *)super + super_root_offsets[i]), 1);\ i++) +static bool all_roots_migrated(struct scoutfs_super_block *super) +{ + struct scoutfs_btree_root *root; + int i; + + for_each_super_root(super, i, root) { + if (root->migration_key_len) + return false; + } + + return true; +} + static int cmp_hdr_item_key(void *priv, const void *a_ptr, const void *b_ptr) { struct scoutfs_btree_block *bt = priv; @@ -711,10 +743,16 @@ retry: bti->old_dirtied++; /* wrap next block and increase next seq */ + le64_add_cpu(&bring->next_block, 1); + le64_add_cpu(&bring->next_seq, 1); + if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) bring->next_block = 0; - else - le64_add_cpu(&bring->next_block, 1); + + /* advance to the next half when asked and migration made it safe */ + if (all_roots_migrated(super) && + scoutfs_trigger(sb, BTREE_ADVANCE_RING_HALF)) + advance_to_next_half(bring); /* reset the migration keys if we've just entered a new half */ if (first_block_in_half(bring)) { @@ -725,8 +763,6 @@ retry: } } - le64_add_cpu(&bring->next_seq, 1); - mutex_unlock(&bti->mutex); if (bt) { diff --git a/kmod/src/triggers.c b/kmod/src/triggers.c index 67cef0e6..a94f2b65 100644 --- a/kmod/src/triggers.c +++ b/kmod/src/triggers.c @@ -39,6 +39,7 @@ struct scoutfs_triggers { static char *names[] = { [SCOUTFS_TRIGGER_BTREE_STALE_READ] = "btree_stale_read", + [SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF] = "btree_advance_ring_half", [SCOUTFS_TRIGGER_HARD_STALE_ERROR] = "hard_stale_error", [SCOUTFS_TRIGGER_SEG_STALE_READ] = "seg_stale_read", [SCOUTFS_TRIGGER_STATFS_LOCK_PURGE] = "statfs_lock_purge", diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h index 900e433f..d1fa9e71 100644 --- a/kmod/src/triggers.h +++ b/kmod/src/triggers.h @@ -3,6 +3,7 @@ enum { SCOUTFS_TRIGGER_BTREE_STALE_READ, + SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF, SCOUTFS_TRIGGER_HARD_STALE_ERROR, SCOUTFS_TRIGGER_SEG_STALE_READ, SCOUTFS_TRIGGER_STATFS_LOCK_PURGE, From 31286ad71456e3ba8c1de3c9e6cfdfff00b75921 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Apr 2018 10:30:25 -0700 Subject: [PATCH 596/920] scoutfs: add options debugfs dir Add a debugfs dir that will offer debugging options for an actively mounted volume. Signed-off-by: Zach Brown --- kmod/src/options.c | 61 +++++++++++++++++++++++++++++++++++++++++----- kmod/src/options.h | 11 +++++++++ kmod/src/super.c | 2 ++ kmod/src/super.h | 2 ++ 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/kmod/src/options.c b/kmod/src/options.c index 9f909e62..2d468553 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -22,12 +24,7 @@ #include "msg.h" #include "options.h" - -enum { - Opt_listen = 0, - Opt_cluster, - Opt_err, -}; +#include "super.h" static const match_table_t tokens = { {Opt_listen, "listen=%s"}, @@ -35,6 +32,20 @@ static const match_table_t tokens = { {Opt_err, NULL} }; +struct options_sb_info { + struct dentry *debugfs_dir; +}; + +u32 scoutfs_option_u32(struct super_block *sb, int token) +{ + switch(token) { + default: break; + } + + WARN_ON_ONCE(1); + return 0; +} + int scoutfs_parse_options(struct super_block *sb, char *options, struct mount_options *parsed) { @@ -79,3 +90,41 @@ int scoutfs_parse_options(struct super_block *sb, char *options, return 0; } + +int scoutfs_options_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct options_sb_info *osi; + int ret; + + osi = kzalloc(sizeof(struct options_sb_info), GFP_KERNEL); + if (!osi) + return -ENOMEM; + + sbi->options = osi; + + osi->debugfs_dir = debugfs_create_dir("options", sbi->debug_root); + if (!osi->debugfs_dir) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_options_destroy(sb); + return ret; +} + +void scoutfs_options_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct options_sb_info *osi = sbi->options; + + if (osi) { + if (osi->debugfs_dir) + debugfs_remove_recursive(osi->debugfs_dir); + kfree(osi); + sbi->options = NULL; + } +} diff --git a/kmod/src/options.h b/kmod/src/options.h index 30009faf..8bf58b36 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -4,6 +4,12 @@ #include #include "format.h" +enum { + Opt_listen = 0, + Opt_cluster, + Opt_err, +}; + #define MAX_CLUSTER_NAME_LEN 17 struct mount_options { @@ -13,5 +19,10 @@ struct mount_options int scoutfs_parse_options(struct super_block *sb, char *options, struct mount_options *parsed); +int scoutfs_options_setup(struct super_block *sb); +void scoutfs_options_destroy(struct super_block *sb); + +u32 scoutfs_option_u32(struct super_block *sb, int token); +#define scoutfs_option_bool scoutfs_option_u32 #endif /* _SCOUTFS_OPTIONS_H_ */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 9d97e578..fdfd4080 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -137,6 +137,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_lock_destroy(sb); scoutfs_destroy_triggers(sb); + scoutfs_options_destroy(sb); debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); scoutfs_destroy_sysfs(sb); @@ -331,6 +332,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_setup_counters(sb) ?: scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: + scoutfs_options_setup(sb) ?: scoutfs_setup_triggers(sb) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 22ef0b62..195e54d0 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -21,6 +21,7 @@ struct server_info; struct inode_sb_info; struct btree_info; struct sysfs_info; +struct options_sb_info; struct scoutfs_sb_info { struct super_block *sb; @@ -64,6 +65,7 @@ struct scoutfs_sb_info { struct scoutfs_triggers *triggers; struct mount_options opts; + struct options_sb_info *options; struct dentry *debug_root; From e145267c050fa6526901ed95990db0b3249585c1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Apr 2018 11:59:53 -0700 Subject: [PATCH 597/920] scoutfs: allow smaller btree keys and values Now that we're using small file system keys we can dramatically shrink the maximum allowed btree keys and values. This more accurately matches the current users and less us fit more possible items in each block. Which lets us turn the block size way down and still have multiple worst case largest items per block. Signed-off-by: Zach Brown --- kmod/src/btree.c | 11 ++++------- kmod/src/format.h | 12 +++--------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 104b4e54..7fdb2b72 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -142,11 +142,10 @@ enum { /* * This greatest key value is stored down the right spine of the tree * and has to be sorted by memcmp() greater than all possible keys in - * all btrees. We give it room for a decent number of big-endian - * primary sort values. + * all btrees. */ -static char max_key[SCOUTFS_BTREE_GREATEST_KEY_LEN] = { - [0 ... (SCOUTFS_BTREE_GREATEST_KEY_LEN - 1)] = 0xff, +static char max_key[SCOUTFS_BTREE_MAX_KEY_LEN] = { + [0 ... (SCOUTFS_BTREE_MAX_KEY_LEN - 1)] = 0xff, }; /* number of contiguous bytes used by the item header, key, and value */ @@ -1262,9 +1261,7 @@ static bool invalid_item(void *key, unsigned key_len, unsigned val_len) { return WARN_ON_ONCE(key_len == 0) || WARN_ON_ONCE(key_len > SCOUTFS_BTREE_MAX_KEY_LEN) || - WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN) || - WARN_ON_ONCE(key_len > SCOUTFS_BTREE_GREATEST_KEY_LEN && - cmp_keys(key, key_len, max_key, sizeof(max_key)) > 0); + WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN); } /* diff --git a/kmod/src/format.h b/kmod/src/format.h index 27e467a5..29ea140c 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -121,15 +121,9 @@ struct scoutfs_key_be { __u8 _sk_fourth; }__packed; -/* - * Assert that we'll be able to represent all possible keys with 8 64bit - * primary sort values. - */ -#define SCOUTFS_BTREE_GREATEST_KEY_LEN 32 -/* level >0 segments can have a full key and some metadata */ -#define SCOUTFS_BTREE_MAX_KEY_LEN 320 -/* level 0 segments can have two full keys in the value :/ */ -#define SCOUTFS_BTREE_MAX_VAL_LEN 768 +/* chose reasonable max key and value lens that have room for some u64s */ +#define SCOUTFS_BTREE_MAX_KEY_LEN 40 +#define SCOUTFS_BTREE_MAX_VAL_LEN 64 /* * The min number of free bytes we must leave in a parent as we descend From c118f7cc036b070d30faa522760362d815b6d800 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 12 Apr 2018 12:09:16 -0700 Subject: [PATCH 598/920] scoutfs: add option to force tiny btree blocks Add a tunable option to force using tiny btree blocks on an active mount. This lets us quickly exercise large btrees. Signed-off-by: Zach Brown --- kmod/src/btree.c | 23 +++++++++++++++++------ kmod/src/format.h | 8 ++++++++ kmod/src/options.c | 14 +++++++++++++- kmod/src/options.h | 5 +++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 7fdb2b72..15e28c8c 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -26,6 +26,7 @@ #include "sort_priv.h" #include "counters.h" #include "triggers.h" +#include "options.h" #include "scoutfs_trace.h" @@ -184,9 +185,9 @@ static inline unsigned int all_len_bytes(unsigned key_len, unsigned val_len) * 2 * min_used <= (bs - parent_min_free - hdr) * min_used <= (bs - parent_min_free - hdr) / 2 */ -static inline unsigned int min_used_bytes(void) +static inline int min_used_bytes(int block_size) { - return (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - + return (block_size - sizeof(struct scoutfs_btree_block) - SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES) / 2; } @@ -852,7 +853,9 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, bool put_parent = false; int ret; - if (right->level) + if (scoutfs_option_bool(sb, Opt_btree_force_tiny_blocks)) + all_bytes = SCOUTFS_BLOCK_SIZE - SCOUTFS_BTREE_TINY_BLOCK_SIZE; + else if (right->level) all_bytes = SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES; else all_bytes = all_len_bytes(key_len, val_len); @@ -913,12 +916,20 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *sib; struct scoutfs_btree_ref *ref; + unsigned int min_used; unsigned int sib_pos; bool move_right; int to_move; int ret; - if (used_total(bt) >= min_used_bytes()) + BUILD_BUG_ON(min_used_bytes(SCOUTFS_BTREE_TINY_BLOCK_SIZE) < 0); + + if (scoutfs_option_bool(sb, Opt_btree_force_tiny_blocks)) + min_used = min_used_bytes(SCOUTFS_BTREE_TINY_BLOCK_SIZE); + else + min_used = min_used_bytes(SCOUTFS_BLOCK_SIZE); + + if (used_total(bt) >= min_used) return 0; /* move items right into our block if we have a left sibling */ @@ -935,10 +946,10 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, if (ret) return ret; - if (used_total(sib) < min_used_bytes()) + if (used_total(sib) < min_used) to_move = used_total(sib); else - to_move = min_used_bytes() - used_total(bt); + to_move = min_used - used_total(bt); move_items(bt, sib, move_right, to_move); diff --git a/kmod/src/format.h b/kmod/src/format.h index 29ea140c..b935d033 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -137,6 +137,14 @@ struct scoutfs_key_be { sizeof(struct scoutfs_btree_item) + SCOUTFS_BTREE_MAX_KEY_LEN +\ sizeof(struct scoutfs_btree_ref)) +/* + * When debugging we can tune the splitting and merging thresholds to + * create much larger trees by having blocks with many fewer items. We + * implement this by pretending the blocks are tiny. They're still + * large enough for a handful of items. + */ +#define SCOUTFS_BTREE_TINY_BLOCK_SIZE 512 + /* * A 4EB test image measured a worst case height of 17. This is plenty * generous. diff --git a/kmod/src/options.c b/kmod/src/options.c index 2d468553..f4b7e472 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -34,12 +34,17 @@ static const match_table_t tokens = { struct options_sb_info { struct dentry *debugfs_dir; + u32 btree_force_tiny_blocks; }; u32 scoutfs_option_u32(struct super_block *sb, int token) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct options_sb_info *osi = sbi->options; + switch(token) { - default: break; + case Opt_btree_force_tiny_blocks: + return osi->btree_force_tiny_blocks; } WARN_ON_ONCE(1); @@ -109,6 +114,13 @@ int scoutfs_options_setup(struct super_block *sb) goto out; } + if (!debugfs_create_bool("btree_force_tiny_blocks", 0644, + osi->debugfs_dir, + &osi->btree_force_tiny_blocks)) { + ret = -ENOMEM; + goto out; + } + ret = 0; out: if (ret) diff --git a/kmod/src/options.h b/kmod/src/options.h index 8bf58b36..0c038c92 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -7,6 +7,11 @@ enum { Opt_listen = 0, Opt_cluster, + /* + * For debugging we can quickly create huge trees by limiting + * the number of items in each block as though the blocks were tiny. + */ + Opt_btree_force_tiny_blocks, Opt_err, }; From 676d1e32ef672f5438bddfbccaa85e7561f05691 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Apr 2018 09:31:01 -0700 Subject: [PATCH 599/920] scoutfs: more carefully trace backref walk loop We were only issuing one kernel warning when we couldn't resolve a path to an inode due to excessive retries. It was hard to capture and we only saw details from the first instance. This adds a counter for each time we see excessive retries and returns -ELOOP in that case. We also extend the link backref adding trace point to include the found entry, if any. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/dir.c | 28 ++++++++++++++-------------- kmod/src/scoutfs_trace.h | 19 ++++++++++++++----- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 379e062d..f552617a 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -36,6 +36,7 @@ EXPAND_COUNTER(dentry_revalidate_rcu) \ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ + EXPAND_COUNTER(dir_backref_excessive_retries) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_batch_duplicate) \ EXPAND_COUNTER(item_batch_inserted) \ diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 7b813922..bf2c052a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1206,8 +1206,10 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, ent = kmalloc(offsetof(struct scoutfs_link_backref_entry, dent.name[SCOUTFS_NAME_LEN]), GFP_KERNEL); - if (!ent) - return -ENOMEM; + if (!ent) { + ret = -ENOMEM; + goto out; + } INIT_LIST_HEAD(&ent->head); @@ -1223,8 +1225,6 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); scoutfs_unlock(sb, lock, DLM_LOCK_PR); lock = NULL; - - trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, ret); if (ret < 0) goto out; @@ -1241,7 +1241,12 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, ent->name_len = len; ret = 0; out: - if (list_empty(&ent->head)) + trace_scoutfs_dir_add_next_linkref(sb, ino, dir_ino, dir_pos, ret, + ent ? ent->dir_ino : 0, + ent ? ent->dir_pos : 0, + ent ? ent->name_len : 0); + + if (ent && list_empty(&ent->head)) kfree(ent); return ret; } @@ -1306,19 +1311,14 @@ void scoutfs_dir_free_backref_path(struct super_block *sb, int scoutfs_dir_get_backref_path(struct super_block *sb, u64 ino, u64 dir_ino, u64 dir_pos, struct list_head *list) { + int retries = 10; u64 par_ino; int ret; - int iters = 0; retry: - /* - * Debugging for SCOUT-107, can be removed later when we're - * confident we won't hit an endless loop here again. - */ - if (WARN_ONCE(++iters >= 4000, "scoutfs: Excessive retries in " - "dir_get_backref_path. ino %llu dir_ino %llu pos %llu\n", - ino, dir_ino, dir_pos)) { - ret = -EINVAL; + if (retries-- == 0) { + scoutfs_inc_counter(sb, dir_backref_excessive_retries); + ret = -ELOOP; goto out; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 1ac7765b..81baeb1f 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1060,9 +1060,11 @@ TRACE_EVENT(scoutfs_advance_dirty_super, TRACE_EVENT(scoutfs_dir_add_next_linkref, TP_PROTO(struct super_block *sb, __u64 ino, __u64 dir_ino, - __u64 dir_pos, int ret), + __u64 dir_pos, int ret, __u64 found_dir_ino, + __u64 found_dir_pos, unsigned int name_len), - TP_ARGS(sb, ino, dir_ino, dir_pos, ret), + TP_ARGS(sb, ino, dir_ino, dir_pos, ret, found_dir_pos, found_dir_ino, + name_len), TP_STRUCT__entry( __field(__u64, fsid) @@ -1070,6 +1072,9 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, __field(__u64, dir_ino) __field(__u64, dir_pos) __field(int, ret) + __field(__u64, found_dir_ino) + __field(__u64, found_dir_pos) + __field(unsigned int, name_len) ), TP_fast_assign( @@ -1078,11 +1083,15 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, __entry->dir_ino = dir_ino; __entry->dir_pos = dir_pos; __entry->ret = ret; + __entry->found_dir_ino = dir_ino; + __entry->found_dir_pos = dir_pos; + __entry->name_len = name_len; ), - TP_printk(FSID_FMT" ino %llu dir_ino %llu dis_pos %llu ret %d", - __entry->fsid, __entry->ino, __entry->dir_ino, - __entry->dir_ino, __entry->ret) + TP_printk("fsid "FSID_FMT" ino %llu dir_ino %llu dir_pos %llu ret %d found_dir_ino %llu found_dir_pos %llu name_len %u", + __entry->fsid, __entry->ino, __entry->dir_pos, + __entry->dir_ino, __entry->ret, __entry->found_dir_pos, + __entry->found_dir_ino, __entry->name_len) ); TRACE_EVENT(scoutfs_compact_func, From 81b315950819ad4602d736b7696d6ac799f85e9c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Apr 2018 11:22:57 -0700 Subject: [PATCH 600/920] scoutfs: return errors from read_items The introduction of the helper to handle stale segment retrying was masking errors. It's meant to pass through the caller's return status when it doesn't return -EAGAIN to trigger stale read retries. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index fcf404a2..bc5d5d42 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -490,7 +490,7 @@ static int handle_stale_btree(struct super_block *sb, return -EIO; } - return 0; + return ret; } static int cmp_ment_ref_segno(void *priv, struct list_head *A, From 8061a5cd2899beb17bf2b82c48c47d87e5b46908 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Apr 2018 11:38:37 -0700 Subject: [PATCH 601/920] scoutfs: add server bind warning Emit an error message if the server fails to bind. It can mean that there is a bad configured address. But we might want to be able to bind if the address becomes available, so we don't hard error. We only emit the message once for a series of failures. Signed-off-by: Zach Brown --- kmod/src/server.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 7482e4bd..89a39d81 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -47,6 +47,7 @@ struct server_info { struct mutex mutex; bool shutting_down; + bool bind_warned; struct task_struct *listen_task; struct socket *listen_sock; @@ -906,8 +907,20 @@ static void scoutfs_server_func(struct work_struct *work) goto out; addrlen = sizeof(sin); - ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen) ?: - kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); + ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen); + if (ret) { + if (!server->bind_warned) { + scoutfs_err(sb, "server failed to bind to "SIN_FMT", errno %d%s. Retrying indefinitely..", + SIN_ARG(&sin), ret, + ret == -EADDRNOTAVAIL ? " (Bad address?)" + : ""); + server->bind_warned = true; + } + goto out; + } + server->bind_warned = false; + + kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); if (ret) goto out; From ac259c82a07529128f2a0294ff5467ab95694021 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Apr 2018 11:41:37 -0700 Subject: [PATCH 602/920] scoutfs: allow interrupting client sends Waiting for replies to sent requests wasn't interruptible. This was preventing ctl-c from breaking out of mount when a server wasn't yet around to accept connections. The only complication was that the receive thread was accessing the sender's struct outside of the lock. An interrupted sender could remove their struct while receive was processing it. We rework recv processing so that it only uses the sender struct under the lock. This introduces a cpu copy of the payload but they're small and relatively infrequent control messages. Signed-off-by: Zach Brown --- kmod/src/client.c | 61 +++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index e40cc214..dd0d6b9a 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -158,10 +158,8 @@ static void scoutfs_client_recv_func(struct work_struct *work) recv_work); struct waiting_sender *sender; struct scoutfs_net_header nh; - void *rx_alloc = NULL; - int result = 0; + void *rx = NULL; u16 data_len; - void *rx; int ret; for (;;) { @@ -176,32 +174,12 @@ static void scoutfs_client_recv_func(struct work_struct *work) &client->sockname, &client->peername, &nh); - /* see if we have a waiting sender */ - spin_lock(&client->recv_lock); - sender = walk_sender_tree(client, le64_to_cpu(nh.id), NULL); - spin_unlock(&client->recv_lock); - - if (sender) { - if (sender->rx_size < data_len) { - /* protocol mismatch is fatal */ - rx = NULL; - result = -EIO; - } else { - rx = sender->rx; - result = 0; - } - } else { - rx = NULL; - } - + /* receive the payload */ + kfree(rx); + rx = kmalloc(data_len, GFP_NOFS); if (!rx) { - kfree(rx_alloc); - rx_alloc = kmalloc(data_len, GFP_NOFS); - if (!rx_alloc) { - ret = -ENOMEM; - break; - } - rx = rx_alloc; + ret = -ENOMEM; + break; } /* recv failure can be server crashing, not fatal */ @@ -210,14 +188,21 @@ static void scoutfs_client_recv_func(struct work_struct *work) break; } + /* give the payload to a sender if there is one */ + spin_lock(&client->recv_lock); + sender = walk_sender_tree(client, le64_to_cpu(nh.id), NULL); if (sender) { - /* lock to keep sender around until after we wake */ - spin_lock(&client->recv_lock); - sender->result = result; + /* protocol mismatch is fatal */ + if (sender->rx_size < data_len) { + sender->result = -EIO; + } else { + memcpy(sender->rx, rx, data_len); + sender->result = 0; + } smp_mb(); /* store result before waking */ wake_up_process(sender->task); - spin_unlock(&client->recv_lock); } + spin_unlock(&client->recv_lock); } /* make senders reconnect if we see an rx error */ @@ -227,7 +212,7 @@ static void scoutfs_client_recv_func(struct work_struct *work) client->recv_shutdown = true; } - kfree(rx_alloc); + kfree(rx); } static void reset_connect_timeouts(struct client_info *client) @@ -513,10 +498,14 @@ static int client_request(struct client_info *client, int type, void *data, sent_to_gen = client->sock_gen; } - /* XXX would need to protect erase during rx if interruptible */ mutex_unlock(&client->send_mutex); - wait_event(client->waitq, sender_should_wake(client, &sender)); + ret = wait_event_interruptible(client->waitq, + sender_should_wake(client, &sender)); + if (ret < 0 && sender.result == -EINPROGRESS) { + sender.result = ret; + ret = 0; + } mutex_lock(&client->send_mutex); @@ -529,7 +518,7 @@ static int client_request(struct client_info *client, int type, void *data, mutex_unlock(&client->send_mutex); - /* safe to remove, we only finish after canceling recv or we're woke */ + /* only we remove senders, recv only uses senders under the lock */ spin_lock(&client->recv_lock); rb_erase(&sender.node, &client->sender_root); spin_unlock(&client->recv_lock); From c9573d13bb19ed7f228a8c259487e49c21acb2b4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Apr 2018 10:32:21 -0700 Subject: [PATCH 603/920] scoutfs: add scoutfs_corruption() Add a helper for printing a message warning about corruption. Signed-off-by: Zach Brown --- kmod/src/format.h | 9 +++++++++ kmod/src/msg.h | 22 ++++++++++++++++++++++ kmod/src/super.h | 2 ++ 3 files changed, 33 insertions(+) diff --git a/kmod/src/format.h b/kmod/src/format.h index b935d033..8022f36a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -645,4 +645,13 @@ struct scoutfs_fid { #define FILEID_SCOUTFS 0x81 #define FILEID_SCOUTFS_WITH_PARENT 0x82 +/* + * Identifiers for sources of corruption that can generate messages. + */ +enum { + SC_NR_SOURCES, +}; + +#define SC_NR_LONGS DIV_ROUND_UP(SC_NR_SOURCES, BITS_PER_LONG) + #endif diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 0586f75c..75791309 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -1,6 +1,7 @@ #ifndef _SCOUTFS_MSG_H_ #define _SCOUTFS_MSG_H_ +#include #include "key.h" void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, @@ -23,4 +24,25 @@ do { \ } \ } while (0) \ +/* + * Each message is only generated once per volume. Remounting resets + * the messages. + */ +#define scoutfs_corruption(sb, which, counter, fmt, args...) \ +do { \ + __typeof__(sb) _sb = (sb); \ + struct scoutfs_sb_info *_sbi = SCOUTFS_SB(_sb); \ + unsigned int _bit = (which); \ + \ + if (WARN_ON_ONCE(_bit >= SC_NR_SOURCES)) \ + break; \ + \ + scoutfs_inc_counter(_sb, counter); \ + if (!test_and_set_bit(_bit, _sbi->corruption_messages_once)) { \ + scoutfs_err(_sb, "corruption (see scoutfs-corruption(5)): " \ + #which ": " fmt, ##args); \ + dump_stack(); \ + } \ +} while (0) \ + #endif diff --git a/kmod/src/super.h b/kmod/src/super.h index 195e54d0..fdc15efc 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -70,6 +70,8 @@ struct scoutfs_sb_info { struct dentry *debug_root; bool shutdown; + + unsigned long corruption_messages_once[SC_NR_LONGS]; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) From 3efcc87413ab51293f127fb9015ec3947975c2b3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Apr 2018 10:32:44 -0700 Subject: [PATCH 604/920] scoutfs: add corruption messages for namei Add scoutfs_corruption() calls for corruption associated with mapping names to inodes. Signed-off-by: Zach Brown --- kmod/src/counters.h | 6 ++++++ kmod/src/dir.c | 36 ++++++++++++++++++++++++++++++------ kmod/src/format.h | 6 ++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index f552617a..a919b191 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -23,6 +23,12 @@ EXPAND_COUNTER(compact_stale_error) \ EXPAND_COUNTER(compact_sticky_upper) \ EXPAND_COUNTER(compact_sticky_written) \ + EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ + EXPAND_COUNTER(corrupt_dirent_name_len) \ + EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ + EXPAND_COUNTER(corrupt_symlink_inode_size) \ + EXPAND_COUNTER(corrupt_symlink_missing_item) \ + EXPAND_COUNTER(corrupt_symlink_not_null_term) \ EXPAND_COUNTER(data_end_writeback_page) \ EXPAND_COUNTER(data_invalidatepage) \ EXPAND_COUNTER(data_readpage) \ diff --git a/kmod/src/dir.c b/kmod/src/dir.c index bf2c052a..292df2d5 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -258,8 +258,11 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, break; ret -= sizeof(struct scoutfs_dirent); - /* XXX corruption */ if (ret < 1 || ret > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_NAME_LEN, + corrupt_dirent_name_len, + "dir_ino %llu hash %llu key "SK_FMT" len %d", + dir_ino, hash, SK_ARG(&key), ret); ret = -EIO; goto out; } @@ -504,8 +507,12 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } name_len = ret - sizeof(struct scoutfs_dirent); - /* XXX corruption */ if (name_len < 1 || name_len > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_READDIR_NAME_LEN, + corrupt_dirent_readdir_name_len, + "dir_ino %llu pos %llu key "SK_FMT" len %d", + scoutfs_ino(inode), file->f_pos, + SK_ARG(&key), name_len); ret = -EIO; goto out; } @@ -1034,8 +1041,11 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) size = i_size_read(inode); - /* XXX corruption */ if (size == 0 || size > SCOUTFS_SYMLINK_MAX_SIZE) { + scoutfs_corruption(sb, SC_SYMLINK_INODE_SIZE, + corrupt_symlink_inode_size, + "ino %llu size %llu", + scoutfs_ino(inode), (u64)size); ret = -EIO; goto out; } @@ -1055,10 +1065,21 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) ret = symlink_item_ops(sb, SYM_LOOKUP, scoutfs_ino(inode), inode_lock, path, size); - /* XXX corruption: missing items or not null term */ - if (ret == -ENOENT || (ret == 0 && path[size - 1])) + if (ret == -ENOENT) { + scoutfs_corruption(sb, SC_SYMLINK_MISSING_ITEM, + corrupt_symlink_missing_item, + "ino %llu size %llu", scoutfs_ino(inode), + size); ret = -EIO; + } else if (ret == 0 && path[size - 1]) { + scoutfs_corruption(sb, SC_SYMLINK_NOT_NULL_TERM, + corrupt_symlink_not_null_term, + "ino %llu last %u", + scoutfs_ino(inode), path[size - 1]); + ret = -EIO; + } + out: if (ret < 0) { kfree(path); @@ -1229,8 +1250,11 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, goto out; len = ret - sizeof(struct scoutfs_dirent); - /* XXX corruption */ if (len < 1 || len > SCOUTFS_NAME_LEN) { + scoutfs_corruption(sb, SC_DIRENT_BACKREF_NAME_LEN, + corrupt_dirent_backref_name_len, + "ino %llu dir_ino %llu pos %llu key "SK_FMT" len %d", + ino, dir_ino, dir_pos, SK_ARG(&key), len); ret = -EIO; goto out; } diff --git a/kmod/src/format.h b/kmod/src/format.h index 8022f36a..54857d24 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -649,6 +649,12 @@ struct scoutfs_fid { * Identifiers for sources of corruption that can generate messages. */ enum { + SC_DIRENT_NAME_LEN = 0, + SC_DIRENT_BACKREF_NAME_LEN, + SC_DIRENT_READDIR_NAME_LEN, + SC_SYMLINK_INODE_SIZE, + SC_SYMLINK_MISSING_ITEM, + SC_SYMLINK_NOT_NULL_TERM, SC_NR_SOURCES, }; From fe8b1550616778a218f6ef5550221bb0eaef7799 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Apr 2018 14:54:19 -0700 Subject: [PATCH 605/920] scoutfs: add btree corruption messages Signed-off-by: Zach Brown --- kmod/src/btree.c | 19 +++++++++++++++++++ kmod/src/counters.h | 2 ++ kmod/src/format.h | 2 ++ 3 files changed, 23 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 15e28c8c..d8ff6d51 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -27,6 +27,7 @@ #include "counters.h" #include "triggers.h" #include "options.h" +#include "msg.h" #include "scoutfs_trace.h" @@ -1145,6 +1146,15 @@ restart: /* XXX more aggressive block verification, before ref updates? */ if (bt->level != level) { + scoutfs_corruption(sb, SC_BTREE_BLOCK_LEVEL, + corrupt_btree_block_level, + "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u expected %u", + root->height, + le64_to_cpu(root->ref.blkno), + le64_to_cpu(root->ref.seq), + le64_to_cpu(bt->blkno), + le64_to_cpu(bt->seq), bt->level, + level); ret = -EIO; break; } @@ -1177,6 +1187,15 @@ restart: /* Find the next child block for the search key. */ pos = find_pos(bt, key, key_len, &cmp); if (pos >= nr) { + scoutfs_corruption(sb, SC_BTREE_NO_CHILD_REF, + corrupt_btree_block_level, + "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u nr %u pos %u cmp %d", + root->height, + le64_to_cpu(root->ref.blkno), + le64_to_cpu(root->ref.seq), + le64_to_cpu(bt->blkno), + le64_to_cpu(bt->seq), bt->level, + nr, pos, cmp); ret = -EIO; break; } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a919b191..6ecbd8e5 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -23,6 +23,8 @@ EXPAND_COUNTER(compact_stale_error) \ EXPAND_COUNTER(compact_sticky_upper) \ EXPAND_COUNTER(compact_sticky_written) \ + EXPAND_COUNTER(corrupt_btree_block_level) \ + EXPAND_COUNTER(corrupt_btree_no_child_ref) \ EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 54857d24..4b60a75c 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -655,6 +655,8 @@ enum { SC_SYMLINK_INODE_SIZE, SC_SYMLINK_MISSING_ITEM, SC_SYMLINK_NOT_NULL_TERM, + SC_BTREE_BLOCK_LEVEL, + SC_BTREE_NO_CHILD_REF, SC_NR_SOURCES, }; From 7d7f8e45b77ce91f3cdd0f7d9b012fe22ce2130e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Apr 2018 15:06:46 -0700 Subject: [PATCH 606/920] scoutfs: more carefully manage private bh bits The management of _checked and _valid_crc private bits in the buffer_head wasn't quite right. _checked indicates that the block has been checked and that the expensive crc verification doesn't need to be recalculated. _valid_crc then indicates the result of the crc verification. _checked is read without locks. First, we didn't make sure that _valid_crc was stored before _checked. Multiple tasks could race to see _checked before _valid_crc. So we add some memory barriers. Then we didn't clear _checked when re-reading a stale block. This meant that the moment the block was read its private flags could still indicate that it had a valid crc. We clear the private bits before we read so that we'll recalculate the crc. Signed-off-by: Zach Brown --- kmod/src/btree.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index d8ff6d51..aead4756 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -611,6 +611,7 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, __le32 existing; u32 calc; + smp_rmb(); /* load checked before crc */ if (!buffer_scoutfs_checked(bh)) { lock_buffer(bh); if (!buffer_scoutfs_checked(bh)) { @@ -619,11 +620,13 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, calc = crc32c(~0, bt, SCOUTFS_BLOCK_SIZE); bt->crc = existing; - set_buffer_scoutfs_checked(bh); if (calc == le32_to_cpu(existing)) set_buffer_scoutfs_valid_crc(bh); else clear_buffer_scoutfs_valid_crc(bh); + + smp_wmb(); /* store crc before checked */ + set_buffer_scoutfs_checked(bh); } unlock_buffer(bh); } @@ -683,6 +686,9 @@ retry: lock_buffer(bh); clear_buffer_uptodate(bh); + clear_buffer_scoutfs_valid_crc(bh); + smp_wmb(); /* store crc before checked */ + clear_buffer_scoutfs_checked(bh); unlock_buffer(bh); put_bh(bh); bt = NULL; From 24cc5cc2962dec69413686b9257b554865ca022a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Apr 2018 15:20:01 -0700 Subject: [PATCH 607/920] scoutfs: lock manifest root request The manifest root request processing samples the stable_manifest_root in the server info. The stable_manifest_root is updated after a commit has suceeded. The read of stable_manifest_root in request processing was locking the manifest. The update during commit doesn't lock the manifest so these paths were racing. The race is very tight, a few cpu stores, but it could in theory give a client a malformed root that could be misinterpreted as corruption. Add a seqcount around the store of the stable manifest root during commit and its load during request processing. This ensures that clients always get a consistent manifest root. Signed-off-by: Zach Brown --- kmod/src/server.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 89a39d81..5642c18f 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -61,6 +61,7 @@ struct server_info { wait_queue_head_t compaction_waitq; /* server remembers the stable manifest root for clients */ + seqcount_t stable_seqcount; struct scoutfs_btree_root stable_manifest_root; /* server tracks seq use */ @@ -166,8 +167,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) scoutfs_btree_write_complete(sb); + write_seqcount_begin(&server->stable_seqcount); server->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; + write_seqcount_end(&server->stable_seqcount); + scoutfs_advance_dirty_super(sb); } else { ret = 0; @@ -530,15 +534,15 @@ static int process_get_manifest_root(struct server_connection *conn, u64 id, u8 type, void *data, unsigned data_len) { struct server_info *server = conn->server; - struct super_block *sb = server->sb; struct scoutfs_btree_root root; + unsigned int start; int ret; if (data_len == 0) { - scoutfs_manifest_lock(sb); - memcpy(&root, &server->stable_manifest_root, - sizeof(struct scoutfs_btree_root)); - scoutfs_manifest_unlock(sb); + do { + start = read_seqcount_begin(&server->stable_seqcount); + root = server->stable_manifest_root; + } while (read_seqcount_retry(&server->stable_seqcount, start)); ret = 0; } else { ret = -EINVAL; @@ -1062,6 +1066,7 @@ int scoutfs_server_setup(struct super_block *sb) init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); init_waitqueue_head(&server->compaction_waitq); + seqcount_init(&server->stable_seqcount); spin_lock_init(&server->seq_lock); INIT_LIST_HEAD(&server->pending_seqs); From ae6907623cddfccd906b73e6dbd45079eff5b762 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 May 2018 08:51:45 -0700 Subject: [PATCH 608/920] scoutfs: add btree rw error traces and counters Add some trivial traces and counters around btree block IO errors. Signed-off-by: Zach Brown --- kmod/src/btree.c | 6 +++++- kmod/src/counters.h | 2 ++ kmod/src/scoutfs_trace.h | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index aead4756..565474df 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -674,6 +674,8 @@ static int get_ref_block(struct super_block *sb, int flags, retry: bh = sb_bread(sb, le64_to_cpu(ref->blkno)); if (!bh) { + trace_scoutfs_btree_read_error(sb, ref); + scoutfs_inc_counter(sb, btree_read_error); ret = -EIO; goto out; } @@ -1671,8 +1673,10 @@ int scoutfs_btree_write_dirty(struct super_block *sb) ret = 0; for_each_dirty_bh(bti, bh, tmp) { wait_on_buffer(bh); - if (!buffer_uptodate(bh)) + if (!buffer_uptodate(bh)) { + scoutfs_inc_counter(sb, btree_write_error); ret = -EIO; + } } out: diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 6ecbd8e5..adae0a23 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -14,7 +14,9 @@ #define EXPAND_EACH_COUNTER \ EXPAND_COUNTER(alloc_alloc) \ EXPAND_COUNTER(alloc_free) \ + EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ + EXPAND_COUNTER(btree_write_error) \ EXPAND_COUNTER(compact_operations) \ EXPAND_COUNTER(compact_segment_moved) \ EXPAND_COUNTER(compact_segment_read) \ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 81baeb1f..537f2a3c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2065,6 +2065,27 @@ TRACE_EVENT(scoutfs_get_name, __get_str(name)) ); +TRACE_EVENT(scoutfs_btree_read_error, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *ref), + + TP_ARGS(sb, ref), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, blkno) + __field(__u64, seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->blkno = le64_to_cpu(ref->blkno); + __entry->seq = le64_to_cpu(ref->seq); + ), + + TP_printk("fsid "FSID_FMT" blkno %llu seq %llu", + __entry->fsid, __entry->blkno, __entry->seq) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From f3007f10cad38d7415d83f4d8a829aa9857df2c0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 May 2018 08:53:58 -0700 Subject: [PATCH 609/920] scoutfs: shut down server on commit errors We hadn't yet implemented any error handling in the server when commits fail. Commit errors are serious and we take them as a sign that something has gone horribly wrong. This patch prints commit error warnings to the console and shuts down. Clients will try to reconnect and resend their requests. The hope is that another server will be able to make progress. But this same node could become the server again and it could well be that the errors are persistent. The next steps are to implement server startup backoff, client retry backoff, and hard failure policies. Signed-off-by: Zach Brown --- kmod/src/server.c | 96 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 5642c18f..d5317d04 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -93,6 +93,19 @@ struct commit_waiter { int ret; }; +/* + * Trigger a server shutdown by shutting down the listening socket. The + * server thread will break out of accept and exit. + */ +static void shut_down_server(struct server_info *server) +{ + mutex_lock(&server->mutex); + server->shutting_down = true; + if (server->listen_sock) + kernel_sock_shutdown(server->listen_sock, SHUT_RDWR); + mutex_unlock(&server->mutex); +} + /* * This is called while still holding the rwsem that prevents commits so * that the caller can be sure to be woken by the next commit after they @@ -120,9 +133,22 @@ static void queue_commit_work(struct server_info *server, queue_work(server->wq, &server->commit_work); } -static int wait_for_commit(struct commit_waiter *cw) +/* + * Commit errors are fatal and shut down the server. This is called + * from request processing which shutdown will wait for. + */ +static int wait_for_commit(struct server_info *server, + struct commit_waiter *cw, u64 id, u8 type) { + struct super_block *sb = server->sb; + wait_for_completion(&cw->comp); + if (cw->ret < 0) { + scoutfs_err(sb, "commit error %d processing req id %llu type %u", + cw->ret, id, type); + + shut_down_server(server); + } return cw->ret; } @@ -157,26 +183,39 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); - if (scoutfs_btree_has_dirty(sb)) { - ret = scoutfs_alloc_apply_pending(sb) ?: - scoutfs_btree_write_dirty(sb) ?: - scoutfs_write_dirty_super(sb); - - /* we'd need to loop or something */ - BUG_ON(ret); - - scoutfs_btree_write_complete(sb); - - write_seqcount_begin(&server->stable_seqcount); - server->stable_manifest_root = - SCOUTFS_SB(sb)->super.manifest.root; - write_seqcount_end(&server->stable_seqcount); - - scoutfs_advance_dirty_super(sb); - } else { + if (!scoutfs_btree_has_dirty(sb)) { ret = 0; + goto out; } + ret = scoutfs_alloc_apply_pending(sb); + if (ret) { + scoutfs_err(sb, "server error freeing segments: %d", ret); + goto out; + } + + ret = scoutfs_btree_write_dirty(sb); + if (ret) { + scoutfs_err(sb, "server error writing btree blocks: %d", ret); + goto out; + } + + ret = scoutfs_write_dirty_super(sb); + if (ret) { + scoutfs_err(sb, "server error writing super block: %d", ret); + goto out; + } + + scoutfs_btree_write_complete(sb); + + write_seqcount_begin(&server->stable_seqcount); + server->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; + write_seqcount_end(&server->stable_seqcount); + + scoutfs_advance_dirty_super(sb); + ret = 0; + +out: node = llist_del_all(&server->commit_waiters); /* waiters always wait on completion, cw could be free after complete */ @@ -292,7 +331,7 @@ static int process_alloc_inodes(struct server_connection *conn, ial.ino = cpu_to_le64(ino); ial.nr = cpu_to_le64(nr); - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, id, type); out: return send_reply(conn, id, type, ret, &ial, sizeof(ial)); } @@ -321,7 +360,7 @@ static int process_alloc_segno(struct server_connection *conn, up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, id, type); out: return send_reply(conn, id, type, ret, &lesegno, sizeof(lesegno)); } @@ -371,7 +410,7 @@ retry: up_read(&server->commit_rwsem); if (ret == 0) { - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, id, type); if (ret == 0) scoutfs_compact_kick(sb); } @@ -424,7 +463,7 @@ static int process_bulk_alloc(struct server_connection *conn, u64 id, u8 type, up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, id, type); out: ret = send_reply(conn, id, type, ret, ns, size); kfree(ns); @@ -493,7 +532,7 @@ static int process_advance_seq(struct server_connection *conn, u64 id, u8 type, spin_unlock(&server->seq_lock); queue_commit_work(server, &cw); up_read(&server->commit_rwsem); - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, id, type); out: return send_reply(conn, id, type, ret, &next_seq, sizeof(next_seq)); @@ -629,7 +668,7 @@ int scoutfs_client_get_compaction(struct super_block *sb, void *curs) up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, U64_MAX, 1); return ret; } @@ -668,7 +707,7 @@ int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(&cw); + ret = wait_for_commit(server, &cw, U64_MAX, 2); scoutfs_compact_kick(sb); @@ -1088,12 +1127,7 @@ void scoutfs_server_destroy(struct super_block *sb) struct server_info *server = sbi->server_info; if (server) { - /* break server thread out of blocking socket calls */ - mutex_lock(&server->mutex); - server->shutting_down = true; - if (server->listen_sock) - kernel_sock_shutdown(server->listen_sock, SHUT_RDWR); - mutex_unlock(&server->mutex); + shut_down_server(server); /* wait for server work to wait for everything to shut down */ cancel_delayed_work_sync(&server->dwork); From 55e063d2a153d713ced7c581d0ebf14c330a3e6b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 2 May 2018 09:20:42 -0700 Subject: [PATCH 610/920] scoutfs: get rid of silly lock destroy BUG_ON The BUG_ON() at the start of scoutfs_lock_destroy() was intended to ensure that scoutfs_lock_shutdown() had been called first. But that doesn't happen in the case where we get an error during mount. The _destroy() function is careful to notice active use and only tears down resources that were created. The BUG_ON() can just be removed. Signed-off-by: Zach Brown --- kmod/src/lock.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c6545c19..f9935c56 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1402,8 +1402,6 @@ void scoutfs_lock_destroy(struct super_block *sb) if (!linfo) return; - BUG_ON(!linfo->shutdown); - trace_scoutfs_lock_destroy(sb, linfo); /* stop the shrinker from queueing work */ From 4fc554584ac48cde936a887319127e3fb575ff6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Apr 2018 10:06:06 -0700 Subject: [PATCH 611/920] scoutfs: add SCOUTFS_BLOCK_MAX Add the max possible logical block / physical blkno number given u64 bytes recorded at block size granularity. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/format.h b/kmod/src/format.h index 4b60a75c..3cefe482 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -15,6 +15,7 @@ #define SCOUTFS_BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) #define SCOUTFS_BLOCK_SECTOR_SHIFT (SCOUTFS_BLOCK_SHIFT - 9) #define SCOUTFS_BLOCK_SECTORS (1 << SCOUTFS_BLOCK_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_MAX (U64_MAX >> SCOUTFS_BLOCK_SHIFT) /* * FS data is stored in segments, for now they're fixed size. They'll From 4ceb123473584df07879cb783949806b39a25992 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Jun 2018 11:41:23 -0700 Subject: [PATCH 612/920] scoutfs: include counters.h for messages The corruption helpers use counters and callers shouldn't have to include the counters header themselves. Signed-off-by: Zach Brown --- kmod/src/msg.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 75791309..8c7cea53 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -3,6 +3,7 @@ #include #include "key.h" +#include "counters.h" void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...); From 036577890fc891e6e6c595d33e999f16b5804403 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Apr 2018 14:18:36 -0700 Subject: [PATCH 613/920] scoutfs: add atomic online/offline blocks calls Add functions that atomically change and query the online and offline block counts as a pair. They're semantically linked and we shouldn't present counts that don't match if they're in the process of being updated. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/format.h | 1 + kmod/src/inode.c | 62 ++++++++++++++++++++++++++++----------------- kmod/src/inode.h | 2 ++ kmod/src/ioctl.c | 3 +-- 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index adae0a23..bd22c0ab 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -30,6 +30,7 @@ EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ + EXPAND_COUNTER(corrupt_inode_block_counts) \ EXPAND_COUNTER(corrupt_symlink_inode_size) \ EXPAND_COUNTER(corrupt_symlink_missing_item) \ EXPAND_COUNTER(corrupt_symlink_not_null_term) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 3cefe482..012a6b70 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -658,6 +658,7 @@ enum { SC_SYMLINK_NOT_NULL_TERM, SC_BTREE_BLOCK_LEVEL, SC_BTREE_NO_CHILD_REF, + SC_INODE_BLOCK_COUNTS, SC_NR_SOURCES, }; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 2812ae25..726f4483 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -500,32 +500,32 @@ void scoutfs_inode_inc_data_version(struct inode *inode) preempt_enable(); } -static void add_seq_value(struct scoutfs_inode_info *si, u64 *si_u64, u64 val) -{ - preempt_disable(); - write_seqcount_begin(&si->seqcount); - *si_u64 += val; - write_seqcount_end(&si->seqcount); - preempt_enable(); -} - -void scoutfs_inode_add_online_blocks(struct inode *inode, u64 val) +void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) { struct scoutfs_inode_info *si; - if (inode) { + if (inode && (on || off)) { si = SCOUTFS_I(inode); - add_seq_value(si, &SCOUTFS_I(inode)->online_blocks, val); - } -} + preempt_disable(); + write_seqcount_begin(&si->seqcount); -void scoutfs_inode_add_offline_blocks(struct inode *inode, u64 val) -{ - struct scoutfs_inode_info *si; + /* inode and extents out of sync, bad callers */ + if (((s64)si->online_blocks + on < 0) || + ((s64)si->offline_blocks + off < 0)) { + scoutfs_corruption(inode->i_sb, SC_INODE_BLOCK_COUNTS, + corrupt_inode_block_counts, + "ino %llu size %llu online %llu + %lld offline %llu + %lld", + scoutfs_ino(inode), i_size_read(inode), + si->online_blocks, on, si->offline_blocks, off); + } - if (inode) { - si = SCOUTFS_I(inode); - add_seq_value(si, &SCOUTFS_I(inode)->offline_blocks, val); + si->online_blocks += on; + si->offline_blocks += off; + /* XXX not sure if this is right */ + inode->i_blocks += (on + off) * SCOUTFS_BLOCK_SECTORS; + + write_seqcount_end(&si->seqcount); + preempt_enable(); } } @@ -577,6 +577,19 @@ u64 scoutfs_inode_offline_blocks(struct inode *inode) return read_seqcount_u64(inode, &si->offline_blocks); } + +void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + unsigned int seq; + + do { + seq = read_seqcount_begin(&si->seqcount); + *on = SCOUTFS_I(inode)->online_blocks; + *off = SCOUTFS_I(inode)->offline_blocks; + } while (read_seqcount_retry(&si->seqcount, seq)); +} + static int scoutfs_iget_test(struct inode *inode, void *arg) { struct scoutfs_inode_info *ci = SCOUTFS_I(inode); @@ -644,6 +657,10 @@ out: static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) { struct scoutfs_inode_info *ci = SCOUTFS_I(inode); + u64 online_blocks; + u64 offline_blocks; + + scoutfs_inode_get_onoff(inode, &online_blocks, &offline_blocks); cinode->size = cpu_to_le64(i_size_read(inode)); cinode->nlink = cpu_to_le32(inode->i_nlink); @@ -661,9 +678,8 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->meta_seq = cpu_to_le64(scoutfs_inode_meta_seq(inode)); cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); cinode->data_version = cpu_to_le64(scoutfs_inode_data_version(inode)); - cinode->online_blocks = cpu_to_le64(scoutfs_inode_online_blocks(inode)); - cinode->offline_blocks = - cpu_to_le64(scoutfs_inode_offline_blocks(inode)); + cinode->online_blocks = cpu_to_le64(online_blocks); + cinode->offline_blocks = cpu_to_le64(offline_blocks); cinode->next_readdir_pos = cpu_to_le64(ci->next_readdir_pos); cinode->next_xattr_id = cpu_to_le64(ci->next_xattr_id); cinode->flags = cpu_to_le32(ci->flags); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index d46f24d1..4dd5f641 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -102,11 +102,13 @@ void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); void scoutfs_inode_add_online_blocks(struct inode *inode, u64 val); void scoutfs_inode_add_offline_blocks(struct inode *inode, u64 val); +void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off); u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); u64 scoutfs_inode_data_version(struct inode *inode); u64 scoutfs_inode_online_blocks(struct inode *inode); u64 scoutfs_inode_offline_blocks(struct inode *inode); +void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off); int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock); int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index bd135650..206a054d 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -464,8 +464,7 @@ static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) stm.meta_seq = scoutfs_inode_meta_seq(inode); stm.data_seq = scoutfs_inode_data_seq(inode); stm.data_version = scoutfs_inode_data_version(inode); - stm.online_blocks = scoutfs_inode_online_blocks(inode); - stm.offline_blocks = scoutfs_inode_offline_blocks(inode); + scoutfs_inode_get_onoff(inode, &stm.online_blocks, &stm.offline_blocks); if (copy_to_user((void __user *)arg, &stm, stm.valid_bytes)) return -EFAULT; From 869d11fd0fdd485f5449676fcbfbdb282246c987 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Apr 2018 14:38:47 -0700 Subject: [PATCH 614/920] scoutfs: add core extent functions Add a file of extent functions that callers will use to manipulate and store extents in different persistent formats. Signed-off-by: Zach Brown --- kmod/src/Makefile | 7 +- kmod/src/counters.h | 7 + kmod/src/extents.c | 343 +++++++++++++++++++++++++++++++++++++++ kmod/src/extents.h | 41 +++++ kmod/src/format.h | 2 + kmod/src/scoutfs_trace.h | 45 +++++ 6 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 kmod/src/extents.c create mode 100644 kmod/src/extents.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 9b72ebee..9e8a6703 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,9 +6,10 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ - export.o file.o inode.o ioctl.o item.o key.o lock.o manifest.o \ - msg.o options.o per_task.o seg.o server.o scoutfs_trace.o sock.o \ - sort_priv.o super.o sysfs.o trans.o triggers.o xattr.o + export.o extents.o file.o inode.o ioctl.o item.o key.o lock.o \ + manifest.o msg.o options.o per_task.o seg.o server.o \ + scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ + triggers.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/counters.h b/kmod/src/counters.h index bd22c0ab..9d6cfc04 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -31,6 +31,8 @@ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ EXPAND_COUNTER(corrupt_inode_block_counts) \ + EXPAND_COUNTER(corrupt_extent_add_cleanup) \ + EXPAND_COUNTER(corrupt_extent_rem_cleanup) \ EXPAND_COUNTER(corrupt_symlink_inode_size) \ EXPAND_COUNTER(corrupt_symlink_missing_item) \ EXPAND_COUNTER(corrupt_symlink_not_null_term) \ @@ -48,6 +50,11 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ + EXPAND_COUNTER(extent_add) \ + EXPAND_COUNTER(extent_delete) \ + EXPAND_COUNTER(extent_insert) \ + EXPAND_COUNTER(extent_next) \ + EXPAND_COUNTER(extent_remove) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_batch_duplicate) \ EXPAND_COUNTER(item_batch_inserted) \ diff --git a/kmod/src/extents.c b/kmod/src/extents.c new file mode 100644 index 00000000..1a461805 --- /dev/null +++ b/kmod/src/extents.c @@ -0,0 +1,343 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "extents.h" +#include "counters.h" +#include "scoutfs_trace.h" +#include "msg.h" + +/* + * These low level functions take on the fiddly details of extent + * manipulation. Callers handle serialization and storage and call in + * here to add or remove extents. This slices and dices the extents + * while dodging all the fence posts. + */ + +/* return the last logical position that is in the extent, inclusive */ +static u64 extent_end(struct scoutfs_extent *ext) +{ + return ext->start + ext->len - 1; +} + +/* returns true if the two extents overlap */ +static bool extents_overlap(struct scoutfs_extent *a, struct scoutfs_extent *b) +{ + return extent_end(a) >= b->start && a->start <= extent_end(b); +} + +/* Returns true if a is entirely within b */ +static bool extent_within(struct scoutfs_extent *a, struct scoutfs_extent *b) +{ + return a->start >= b->start && extent_end(a) <= extent_end(b); +} + +/* + * Returns true if two extents can be merged because they're adjacent, + * mapping is equally set or not, mappings are adjacent if they're set, + * and all the rest of the fields match. + */ +static bool extents_can_merge(struct scoutfs_extent *a, + struct scoutfs_extent *b) +{ + if (a->start > b->start) + swap(a, b); + + return (a->owner == b->owner) && + ((a->start + a->len) == b->start) && + (!!a->map == !!b->map) && + (!a->map || ((a->map + a->len) == b->map)) && + (a->type == b->type) && + (a->flags == b->flags); +} + +int scoutfs_extent_init(struct scoutfs_extent *ext, u8 type, u64 owner, + u64 start, u64 len, u64 map, u8 flags) +{ + /* don't allow 0 len or len wrapping map or start */ + if ((start + len <= start) || (map + len <= map)) + return -EIO; + + ext->owner = owner; + ext->start = start; + ext->len = len; + ext->map = map; + ext->type = type; + ext->flags = flags; + + return 0; +} + +/* + * Returns true if the two extents intersect and modifies a to be the + * intersection of the two extents. Callers only need to initialize a's + * start and len when probing for an intersection and we'll copy the + * rest from b. + */ +bool scoutfs_extent_intersection(struct scoutfs_extent *a, + struct scoutfs_extent *b) +{ + u64 new_start; + u64 new_end; + + if (extents_overlap(a, b)) { + new_end = min(extent_end(a), extent_end(b)); + new_start = max(a->start, b->start); + + a->owner = b->owner; + a->start = new_start; + a->len = new_end - new_start + 1; + a->map = b->map ? (new_start - b->start) + b->map: 0; + a->type = b->type; + a->flags = b->flags; + return true; + } + + return false; +} + +static int extent_insert(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ins, void *data) +{ + scoutfs_inc_counter(sb, extent_insert); + trace_scoutfs_extent_insert(sb, ins); + return iof(sb, SEI_INSERT, ins, data); +} + +static int extent_delete(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *del, void *data) +{ + scoutfs_inc_counter(sb, extent_delete); + trace_scoutfs_extent_delete(sb, del); + return iof(sb, SEI_DELETE, del, data); +} + +/* + * Find the next extent using the given extent as the starting search + * position. This just passes the extent through to the underlying key + * building and searching routines. + * + * Callers have to be very careful when building the search extent. + * Most extents are indexed by their final logical position and some + * have all the metadata in the key. So a typical pattern is to search + * for an intersection by searching from a single block extent with the + * rest of the fields set to zero. + * + * But some callers are searching indexes of free extents where both the + * length and start are meaningful. + * + * The io function is responsible for ensuring that we return next + * extents with the same type and owner as the given extent. + */ +int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ext, void *data) +{ + int ret; + + scoutfs_inc_counter(sb, extent_next); + trace_scoutfs_extent_next_input(sb, ext); + ret = iof(sb, SEI_NEXT, ext, data); + if (ret == 0) + trace_scoutfs_extent_next_output(sb, ext); + return ret; +} + +/* + * Search for a next extent and see if we can merge it with the caller's + * extent. The caller has initialized next for us to search from. If + * we can merge then we update the callers extent, delete the old + * extent, and return 1. If we return an error or 0 then nothing will + * have changed. + */ +static int try_merge_next(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ext, + struct scoutfs_extent *next, void *data) +{ + int ret; + + ret = scoutfs_extent_next(sb, iof, next, data); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + if (extents_overlap(ext, next)) { + ret = -EIO; + goto out; + } + + if (!extents_can_merge(ext, next)) { + ret = 0; + goto out; + } + + if (next->start < ext->start) { + ext->start = next->start; + ext->map = next->map; + ext->len += next->len; + } else { + ext->len += next->len; + } + + ret = extent_delete(sb, iof, next, data); + if (ret == 0) + ret = 1; +out: + return ret; +} + +/* + * The process of modifying an extent creates and deletes many + * intermediate extents. If we hit an error we need to undo the + * process. If we then hit an error we can be left with inconsistent + * extent items. + * + * We could fix this for extents that are stored in the item cache + * because it has tools for ensuring that operations can't fail. + * Extents that are stored in the btree currently can't avoid errors. + * We'd have to predirty blocks, allow deletion to fall below thresholds + * if merging saw an error, and preallocate blocks to be used for + * splitting/growth. It'd probably be worth it. + */ +#define extent_cleanup(cond, ext_func, sb, iof, clean, data, which, ctr, ext) \ +do { \ + __typeof__(sb) _sb = (sb); \ + int _ret; \ + \ + if ((cond) && (_ret = ext_func(_sb, iof, clean, data)) < 0) \ + scoutfs_corruption(_sb, which, ctr, \ + "ext "SE_FMT" clean "SE_FMT" ret %d", \ + SE_ARG(ext), SE_ARG(clean), _ret); \ +} while (0) + +/* + * Add a new extent. It can not overlap with any existing extents. It + * may be merged with neighbouring extents. + */ +int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *add, void *data) +{ + struct scoutfs_extent right; + struct scoutfs_extent left; + struct scoutfs_extent ext; + bool ins_left = false; + bool ins_right = false; + int ret; + + scoutfs_inc_counter(sb, extent_add); + trace_scoutfs_extent_add(sb, add); + ext = *add; + + /* see if we are merging with and deleting a left neighbour */ + if (ext.start) { + scoutfs_extent_init(&left, ext.type, ext.owner, + ext.start - 1, 1, 0, 0); + ret = try_merge_next(sb, iof, &ext, &left, data); + if (ret < 0) + goto out; + if (ret > 0) + ins_left = true; + } + + /* see if we are merging with and deleting a right neighbour */ + if (ext.start + ext.len <= SCOUTFS_BLOCK_MAX) { + scoutfs_extent_init(&right, ext.type, ext.owner, + ext.start, 1, 0, 0); + ret = try_merge_next(sb, iof, &ext, &right, data); + if (ret < 0) + goto out; + if (ret > 0) + ins_right = true; + } + + /* finally insert our new (possibly merged) extent */ + ret = extent_insert(sb, iof, &ext, data); +out: + extent_cleanup(ret < 0 && ins_right, + extent_insert, sb, iof, &right, data, + SC_EXTENT_ADD_CLEANUP, corrupt_extent_add_cleanup, + add); + extent_cleanup(ret < 0 && ins_left, + extent_insert, sb, iof, &left, data, + SC_EXTENT_ADD_CLEANUP, corrupt_extent_add_cleanup, + add); + + return ret; +} + + +/* + * Remove a region of an existing extent. The region to remove must be + * be fully within an existing extent. This creates the items left + * behind on either end of the removed region as appropriate. + */ +int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *rem, void *data) +{ + struct scoutfs_extent right; + struct scoutfs_extent left; + struct scoutfs_extent ext; + bool ins_ext = false; + bool del_left = false; + int ret; + + scoutfs_inc_counter(sb, extent_remove); + trace_scoutfs_extent_remove(sb, rem); + + scoutfs_extent_init(&ext, rem->type, rem->owner, rem->start, 1, 0, 0); + ret = scoutfs_extent_next(sb, iof, &ext, data); + if (ret < 0) + goto out; + + /* make sure they're correct */ + if (!extent_within(rem, &ext)) { + ret = -EIO; + goto out; + } + + ret = extent_delete(sb, iof, &ext, data); + if (ret) + goto out; + ins_ext = true; + + if (rem->start != ext.start) { + scoutfs_extent_init(&left, ext.type, ext.owner, + ext.start, rem->start - ext.start, + ext.map, ext.flags); + ret = extent_insert(sb, iof, &left, data); + if (ret) + goto out; + del_left = true; + } + + if (extent_end(rem) != extent_end(&ext)) { + scoutfs_extent_init(&right, ext.type, ext.owner, + rem->start + rem->len, + extent_end(&ext) - extent_end(rem), + ext.map ? rem->map + rem->len : 0, + ext.flags); + ret = extent_insert(sb, iof, &right, data); + } + +out: + extent_cleanup(ret < 0 && del_left, + extent_delete, sb, iof, &left, data, + SC_EXTENT_REM_CLEANUP, corrupt_extent_rem_cleanup, rem); + extent_cleanup(ret < 0 && ins_ext, + extent_insert, sb, iof, &ext, data, + SC_EXTENT_REM_CLEANUP, corrupt_extent_rem_cleanup, rem); + + return ret; +} diff --git a/kmod/src/extents.h b/kmod/src/extents.h new file mode 100644 index 00000000..ab25c93f --- /dev/null +++ b/kmod/src/extents.h @@ -0,0 +1,41 @@ +#ifndef _SCOUTFS_EXTENTS_H_ +#define _SCOUTFS_EXTENTS_H_ + +/* + * Native storage for an extent. Read and write translates between + * these and persistent storage. + */ +struct scoutfs_extent { + u64 owner; + u64 start; + u64 len; + u64 map; + u8 type; + u8 flags; +}; + +#define SE_FMT "%llu.%llu.%llu.%llu.%u.%x" +#define SE_ARG(ext) (ext)->owner, (ext)->start, (ext)->len, (ext)->map, \ + (ext)->type, (ext)->flags + +enum { + SEI_NEXT, + SEI_INSERT, + SEI_DELETE, +}; +typedef int (*scoutfs_extent_io_t)(struct super_block *sb, int op, + struct scoutfs_extent *ext, void *data); + +int scoutfs_extent_init(struct scoutfs_extent *ext, u8 type, u64 owner, + u64 start, u64 len, u64 map, u8 flags); +bool scoutfs_extent_intersection(struct scoutfs_extent *a, + struct scoutfs_extent *b); + +int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ext, void *data); +int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *add, void *data); +int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *rem, void *data); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 012a6b70..7735440b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -659,6 +659,8 @@ enum { SC_BTREE_BLOCK_LEVEL, SC_BTREE_NO_CHILD_REF, SC_INODE_BLOCK_COUNTS, + SC_EXTENT_ADD_CLEANUP, + SC_EXTENT_REM_CLEANUP, SC_NR_SOURCES, }; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 537f2a3c..2ab9c3f3 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -36,6 +36,7 @@ #include "bio.h" #include "export.h" #include "dir.h" +#include "extents.h" struct lock_info; @@ -2086,6 +2087,50 @@ TRACE_EVENT(scoutfs_btree_read_error, __entry->fsid, __entry->blkno, __entry->seq) ); +DECLARE_EVENT_CLASS(scoutfs_extent_class, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + + TP_ARGS(sb, ext), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field_struct(struct scoutfs_extent, ext) + ), + + TP_fast_assign( + __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + __entry->ext = *ext; + ), + + TP_printk("fsid "FSID_FMT" ext "SE_FMT, + __entry->fsid, SE_ARG(&__entry->ext)) +); + +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_insert, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_delete, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_input, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_output, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_add, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_remove, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From abbe76093b870813e98a4f511ba4e22e4d6b19e9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Apr 2018 14:40:57 -0700 Subject: [PATCH 615/920] scoutfs: store file data in extents Store file data mappings and free block ranges in extents instead of in block mapping items and bitmaps. This adds the new functionality and refactors the functions that use it. The old functions are no longer called and we stop at ifdeffing them out to keep the change small. We'll remove all the dead code in a future change. Signed-off-by: Zach Brown --- kmod/src/count.h | 38 ++- kmod/src/data.c | 709 ++++++++++++++++++++++++--------------- kmod/src/data.h | 2 - kmod/src/format.h | 27 +- kmod/src/key.c | 3 + kmod/src/scoutfs_trace.h | 41 +++ kmod/src/super.c | 4 - 7 files changed, 535 insertions(+), 289 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 759c736d..799c2e31 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -201,19 +201,25 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, } /* - * write_begin can add local free segment items, modify another to - * alloc, add a free blkno item, and modify dirty the mapping. + * write_begin can have to allocate all the blocks in the page and can + * have to add a big allocation from the server to do so: + * - merge added free extents from the server + * - remove a free extent per block + * - remove an offline extent for every other block + * - add a file extent per block */ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) { struct scoutfs_item_count cnt = {0,}; - unsigned nr_free = SCOUTFS_BULK_ALLOC_COUNT + 1 + 1; + unsigned nr_free = (SCOUTFS_BULK_ALLOC_COUNT + + SCOUTFS_BLOCKS_PER_PAGE) * 3; + unsigned nr_file = (DIV_ROUND_UP(SCOUTFS_BLOCKS_PER_PAGE, 2) + + SCOUTFS_BLOCKS_PER_PAGE) * 3; __count_dirty_inode(&cnt); - cnt.items += 1 + nr_free; - cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + - (nr_free * sizeof(struct scoutfs_free_bits)); + cnt.items += nr_free + nr_file; + cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); return cnt; } @@ -235,4 +241,24 @@ static inline const struct scoutfs_item_count SIC_TRUNC_BLOCK(void) return cnt; } +/* + * Truncating an extent can: + * - delete existing file extent, + * - create two surrounding file extents, + * - add an offline file extent, + * - delete two existing free extents + * - create a merged free extent + */ +static inline const struct scoutfs_item_count SIC_TRUNC_EXTENT(void) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int nr_file = 1 + 2 + 1; + unsigned int nr_free = (2 + 1) * 2; + + cnt.items += nr_file + nr_free; + cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + + return cnt; +} + #endif diff --git a/kmod/src/data.c b/kmod/src/data.c index fb4d19bb..4343a90d 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -34,18 +35,11 @@ #include "client.h" #include "lock.h" #include "file.h" +#include "extents.h" /* - * scoutfs uses block mapping items at a fixed granularity to describe - * file data block allocations. - * - * Each item describes a fixed number of blocks. To keep the overhead - * of the items down the series of mapped blocks is encoded. The - * mapping items also describe offline blocks. They can only be written - * to newly allocated blocks with the staging ioctl. - * - * Free segnos and blocks are kept in bitmap items that are private to - * nodes so they can be modified without cluster locks. + * scoutfs uses extent items to track file data block mappings and free + * blocks. * * Block allocation maintains a fixed number of allocation cursors that * remember the position of tasks within free regions. This is very @@ -90,6 +84,7 @@ struct task_cursor { pid_t pid; }; +#if 0 /* * Block mapping items and their native decoded form can be pretty big. * Let's allocate them to avoid blowing the stack. @@ -569,6 +564,255 @@ out: for (i = iblock & SCOUTFS_BLOCK_MAPPING_MASK; \ i < SCOUTFS_BLOCK_MAPPING_BLOCKS && iblock <= (last); \ i++, iblock++) +#endif + +static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FS_ZONE, + .skfe_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_FILE_EXTENT_TYPE, + .skfe_last = cpu_to_le64(last), + }; +} + +static void init_free_extent_key(struct scoutfs_key *key, u8 type, u64 node_id, + u64 major, u64 minor) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_NODE_ZONE, + .sknf_node_id = cpu_to_le64(node_id), + .sk_type = type, + .sknf_major = cpu_to_le64(major), + .sknf_minor = cpu_to_le64(minor), + }; +} + +static int init_extent_from_item(struct scoutfs_extent *ext, + struct scoutfs_key *key, + struct scoutfs_file_extent *fex) +{ + u64 owner; + u64 start; + u64 map; + u64 len; + u8 flags; + + if (key->sk_type != SCOUTFS_FILE_EXTENT_TYPE && + key->sk_type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + key->sk_type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) + return -EIO; /* XXX corruption, unknown key type */ + + if (key->sk_type == SCOUTFS_FILE_EXTENT_TYPE) { + owner = le64_to_cpu(key->skfe_ino); + len = le64_to_cpu(fex->len); + start = le64_to_cpu(key->skfe_last) - len + 1; + map = le64_to_cpu(fex->blkno); + flags = fex->flags; + + } else { + owner = le64_to_cpu(key->sknf_node_id); + start = le64_to_cpu(key->sknf_major); + len = le64_to_cpu(key->sknf_minor); + if (key->sk_type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) + swap(start, len); + start -= len - 1; + map = 0; + flags = 0; + } + + return scoutfs_extent_init(ext, key->sk_type, owner, start, len, map, + flags); +} + +/* + * Read and write file extent and free extent items. + * + * File extents and free extents are indexed by the last position in the + * extent so that we can find intersections with _next. + * + * We also index free extents by their length. We implement that by + * keeping their _BLOCKS_ item in sync with the primary _BLKNO_ item + * that callers operate on. + */ +static int data_extent_io(struct super_block *sb, int op, + struct scoutfs_extent *ext, void *data) +{ + struct scoutfs_lock *lock = data; + struct scoutfs_file_extent fex; + struct scoutfs_key last; + struct scoutfs_key key; + struct kvec val; + bool mirror = false; + u8 mirror_type; + u8 mirror_op = 0; + int expected; + int ret; + int err; + + if (WARN_ON_ONCE(ext->type != SCOUTFS_FILE_EXTENT_TYPE && + ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) + return -EINVAL; + + if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + (op == SEI_INSERT || op == SEI_DELETE)) { + mirror = true; + mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; + mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; + } + + if (ext->type == SCOUTFS_FILE_EXTENT_TYPE) { + init_file_extent_key(&key, ext->owner, + ext->start + ext->len - 1); + init_file_extent_key(&last, ext->owner, U64_MAX); + fex.blkno = cpu_to_le64(ext->map); + fex.len = cpu_to_le64(ext->len); + fex.flags = ext->flags; + kvec_init(&val, &fex, sizeof(fex)); + } else { + init_free_extent_key(&key, ext->type, ext->owner, + ext->start + ext->len - 1, ext->len); + if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) + swap(key.sknf_major, key.sknf_minor); + init_free_extent_key(&last, ext->type, ext->owner, + U64_MAX, U64_MAX); + kvec_init(&val, NULL, 0); + } + + if (op == SEI_NEXT) { + expected = val.iov_len; + ret = scoutfs_item_next(sb, &key, &last, &val, lock); + if (ret >= 0 && ret != expected) + ret = -EIO; + if (ret == expected) + ret = init_extent_from_item(ext, &key, &fex); + + } else if (op == SEI_INSERT) { + ret = scoutfs_item_create(sb, &key, &val, lock); + + } else if (op == SEI_DELETE) { + ret = scoutfs_item_delete(sb, &key, lock); + + } else { + ret = WARN_ON_ONCE(-EINVAL); + } + + if (ret == 0 && mirror) { + swap(ext->type, mirror_type); + ret = data_extent_io(sb, op, ext, data); + swap(ext->type, mirror_type); + if (ret) { + err = data_extent_io(sb, mirror_op, ext, data); + BUG_ON(err); + } + } + + return ret; +} + +/* + * Find and remove or mark offline the next extent that intersects with + * the caller's range. The caller is responsible for transactions and + * locks. + * + * Returns: + * - -errno on errors + * - 0 if there are no more extents to stop iteration + * - +iblock of next logical block to truncate the next block from + * + * Since our extents are block granular we can never have > S64_MAX + * iblock values. Returns -ENOENT if no extent was found and -errno on + * errors. + */ +static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_extent next; + struct scoutfs_extent rem; + struct scoutfs_extent fr; + struct scoutfs_extent ofl; + bool rem_fr = false; + bool add_rem = false; + s64 ret; + int err; + + scoutfs_extent_init(&next, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &next, lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + trace_scoutfs_data_truncate_next(sb, &next); + + scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, last - iblock + 1, 0, 0); + if (!scoutfs_extent_intersection(&rem, &next)) { + ret = 0; + goto out; + } + + trace_scoutfs_data_truncate_remove(sb, &rem); + + /* nothing to do if the extent's already offline */ + if (offline && (rem.flags & SEF_OFFLINE)) { + ret = 1; + goto out; + } + + /* free an allocated mapping */ + if (rem.map) { + scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, rem.map, rem.len, 0, 0); + ret = scoutfs_extent_add(sb, data_extent_io, &fr, + sbi->node_id_lock); + if (ret) + goto out; + rem_fr = true; + } + + /* remove the mapping */ + ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); + if (ret) + goto out; + add_rem = true; + + /* add an offline extent */ + if (offline) { + scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, rem.owner, + rem.start, rem.len, 0, SEF_OFFLINE); + trace_scoutfs_data_truncate_offline(sb, &ofl); + ret = scoutfs_extent_add(sb, data_extent_io, &ofl, lock); + if (ret) + goto out; + } + + scoutfs_inode_add_onoff(inode, rem.map ? -rem.len : 0, + (rem.flags & SEF_OFFLINE ? -rem.len : 0) + + (offline ? ofl.len : 0)); + ret = 1; +out: + if (ret < 0) { + err = 0; + if (add_rem) + err |= scoutfs_extent_add(sb, data_extent_io, &rem, + lock); + if (rem_fr) + err |= scoutfs_extent_remove(sb, data_extent_io, &fr, + sbi->node_id_lock); + BUG_ON(err); /* inconsistency, could save/restore */ + + } else if (ret > 0) { + ret = rem.start + rem.len; + } + + return ret; +} /* * Free blocks inside the logical block range from 'iblock' to 'last', @@ -591,125 +835,37 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_key last_key; - struct scoutfs_key key; - struct block_mapping *map; - struct kvec val; - bool holding = false; - bool dirtied; - u64 blkno; - int bytes; - int ret = 0; - int i; + s64 ret = 0; + + WARN_ON_ONCE(inode && !mutex_is_locked(&inode->i_mutex)); + + /* clamp last to the last possible block? */ + if (last > SCOUTFS_BLOCK_MAX) + last = SCOUTFS_BLOCK_MAX; trace_scoutfs_data_truncate_items(sb, iblock, last, offline); if (WARN_ON_ONCE(last < iblock)) return -EINVAL; - map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); - if (!map) - return -ENOMEM; - - init_mapping_key(&last_key, ino, last); - while (iblock <= last) { - /* find the mapping that could include iblock */ - init_mapping_key(&key, ino, iblock); - kvec_init(&val, map->encoded, sizeof(map->encoded)); - - ret = scoutfs_hold_trans(sb, SIC_TRUNC_BLOCK()); + ret = scoutfs_hold_trans(sb, SIC_TRUNC_EXTENT()); if (ret) break; - holding = true; down_write(&datinf->alloc_rwsem); - - ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - ret = decode_mapping(map, ret); - if (ret < 0) - break; - - /* set iblock to the first in the next item inside last */ - iblock = max(iblock, le64_to_cpu(key.skm_base) << - SCOUTFS_BLOCK_MAPPING_SHIFT); - - dirtied = false; - for_each_block(i, iblock, last) { - - blkno = map->blknos[i]; - - /* don't need to do anything.. */ - if (!blkno && - !(!offline && test_bit(i, map->offline))) - continue; - - if (!dirtied) { - /* dirty item with full size encoded */ - ret = scoutfs_item_update(sb, &key, &val, lock); - if (ret) - break; - dirtied = true; - } - - /* truncating offline block */ - if (!offline && test_bit(i, map->offline)) { - clear_bit(i, map->offline); - scoutfs_inode_add_offline_blocks(inode, -1); - inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; - } - - /* nothing more to do if unallocated */ - if (!blkno) - continue; - - /* free the allocated block, maybe marking offline */ - ret = set_blkno_free(sb, blkno); - if (ret) - break; - - map->blknos[i] = 0; - scoutfs_inode_add_online_blocks(inode, -1); - inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; - - if (offline) { - set_bit(i, map->offline); - scoutfs_inode_add_offline_blocks(inode, 1); - inode->i_blocks += SCOUTFS_BLOCK_SECTORS; - } - } - - if (dirtied) { - /* update how ever much of the item we finished */ - bytes = encode_mapping(map); - if (bytes) { - kvec_init(&val, map->encoded, bytes); - scoutfs_item_update_dirty(sb, &key, &val); - } else { - scoutfs_item_delete_dirty(sb, &key); - } - } - + ret = truncate_one_extent(sb, inode, ino, iblock, last, + offline, lock); up_write(&datinf->alloc_rwsem); scoutfs_release_trans(sb); - holding = false; - if (ret) + if (ret <= 0) break; + + iblock = ret; + ret = 0; } - if (holding) { - up_write(&datinf->alloc_rwsem); - scoutfs_release_trans(sb); - } - - kfree(map); return ret; } @@ -785,6 +941,8 @@ static struct task_cursor *get_cursor(struct data_info *datinf) static int bulk_alloc(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_extent ext; u64 *segnos = NULL; int ret = 0; int i; @@ -796,7 +954,13 @@ static int bulk_alloc(struct super_block *sb) } for (i = 0; segnos[i]; i++) { - ret = set_segno_free(sb, segnos[i]); + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, + segnos[i] << SCOUTFS_SEGMENT_BLOCK_SHIFT, + SCOUTFS_SEGMENT_BLOCKS, 0, 0); + trace_scoutfs_data_bulk_alloc(sb, &ext); + ret = scoutfs_extent_add(sb, data_extent_io, &ext, + sbi->node_id_lock); if (ret) break; } @@ -810,6 +974,7 @@ out: return ret; } +#if 0 /* * Find the free bit item that contains the blkno and return the next blkno * set starting with this blkno. @@ -883,27 +1048,33 @@ static int find_free_segno(struct super_block *sb, u64 *segno) out: return ret; } +#endif /* * Allocate a single block for the logical block offset in the file. + * The caller tells us if the block was offline or not. We modify the + * extent items and the caller will search for the resulting extent. * * We try to encourage contiguous allocation by having per-task cursors - * that track blocks inside segments. Each new allocating task will get - * a new segment. Lots of concurrent allocations can interleave at - * segment granularity. + * that track large extents. Each new allocating task will get a new + * extent. */ +/* XXX initially tied to segment size, should be a lot larger */ +#define LARGE_EXTENT_BLOCKS SCOUTFS_SEGMENT_BLOCKS static int find_alloc_block(struct super_block *sb, struct inode *inode, - struct block_mapping *map, - struct scoutfs_key *map_key, - unsigned map_ind, bool map_exists, - struct scoutfs_lock *data_lock) + u64 iblock, bool was_offline, + struct scoutfs_lock *lock) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); + const u64 ino = scoutfs_ino(inode); + struct scoutfs_extent ext; + struct scoutfs_extent ofl; + struct scoutfs_extent fr; struct task_cursor *curs; - struct kvec val; - int bytes; - u64 segno; - u64 blkno; + bool add_ofl = false; + bool add_fr = false; + int err; int ret; down_write(&datinf->alloc_rwsem); @@ -912,74 +1083,103 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, trace_scoutfs_data_find_alloc_block_curs(sb, curs, curs->blkno); - /* try to find the next blkno in our cursor if we have one */ + /* see if our cursor is still free */ if (curs->blkno) { - ret = find_free_blkno(sb, curs->blkno, &blkno); - if (ret < 0 && ret != -ENOENT) + /* look for the extent that overlaps our iblock */ + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, curs->blkno, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, + sbi->node_id_lock); + if (ret && ret != -ENOENT) goto out; - if (ret == 0) { - curs->blkno = blkno; - segno = 0; - } else { + + if (ret == 0) + trace_scoutfs_data_alloc_block_cursor(sb, &ext); + + /* find a new large extent if our cursor isn't free */ + if (ret < 0 || ext.start > curs->blkno) curs->blkno = 0; - } } - /* try to find segnos, asking the server for more */ + /* try to find a new large extent, possibly asking for more */ while (curs->blkno == 0) { - ret = find_free_segno(sb, &segno); - if (ret < 0 && ret != -ENOENT) + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + sbi->node_id, 0, 2 * LARGE_EXTENT_BLOCKS, + 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, + sbi->node_id_lock); + if (ret && ret != -ENOENT) goto out; + + /* XXX should try to look for smaller free extents :/ */ + + /* + * set our cursor to the aligned start of a large extent + * We'll then remove it and the next aligned free large + * extent will start much later. This stops us from + * constantly setting cursors to the start of a large + * free extent that keeps have its start allocated. + */ if (ret == 0) { - blkno = segno << SCOUTFS_SEGMENT_BLOCK_SHIFT; - curs->blkno = blkno; + trace_scoutfs_data_alloc_block_free(sb, &ext); + curs->blkno = ALIGN(ext.start, LARGE_EXTENT_BLOCKS); break; } + /* try to get allocation from the server if we're out */ ret = bulk_alloc(sb); if (ret < 0) goto out; } - trace_scoutfs_data_find_alloc_block_found_seg(sb, segno, blkno); - /* ensure that we can copy in encoded without failing */ - kvec_init(&val, map->encoded, sizeof(map->encoded)); - if (map_exists) - ret = scoutfs_item_update(sb, map_key, &val, data_lock); - else - ret = scoutfs_item_create(sb, map_key, &val, data_lock); + /* remove the free block we're using */ + scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, curs->blkno, 1, 0, 0); + ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); if (ret) goto out; + add_fr = true; - /* clear the free bit we found */ - if (segno) - ret = clear_segno_free(sb, segno); - else - ret = clear_blkno_free(sb, blkno); - if (ret) - goto out; - - /* update the mapping */ - if (test_and_clear_bit(map_ind, map->offline)) { - scoutfs_inode_add_offline_blocks(inode, -1); - inode->i_blocks -= SCOUTFS_BLOCK_SECTORS; + /* remove an offline file extent */ + if (was_offline) { + scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, 1, 0, SEF_OFFLINE); + ret = scoutfs_extent_remove(sb, data_extent_io, &ofl, lock); + if (ret) + goto out; + add_ofl = true; } - map->blknos[map_ind] = blkno; - scoutfs_inode_add_online_blocks(inode, 1); - inode->i_blocks += SCOUTFS_BLOCK_SECTORS; - bytes = encode_mapping(map); - kvec_init(&val, map->encoded, bytes); - scoutfs_item_update_dirty(sb, map_key, &val); + /* add (and hopefully merge!) the new allocation */ + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, 1, curs->blkno, 0); + trace_scoutfs_data_alloc_block(sb, &ext); + ret = scoutfs_extent_add(sb, data_extent_io, &ext, lock); + if (ret) + goto out; - /* set cursor to next block, clearing if we finish the segment */ + scoutfs_inode_add_onoff(inode, 1, was_offline ? -1ULL : 0); + + /* set cursor to next block, clearing if we finish a large extent */ + BUILD_BUG_ON(!is_power_of_2(LARGE_EXTENT_BLOCKS)); curs->blkno++; - if ((curs->blkno & SCOUTFS_FREE_BITS_MASK) == 0) + if ((curs->blkno & (LARGE_EXTENT_BLOCKS - 1)) == 0) curs->blkno = 0; ret = 0; out: + if (ret) { + err = 0; + if (add_ofl) + err |= scoutfs_extent_add(sb, data_extent_io, &ofl, + lock); + if (add_fr) + err |= scoutfs_extent_add(sb, data_extent_io, &fr, + sbi->node_id_lock); + BUG_ON(err); /* inconsistency */ + } + up_write(&datinf->alloc_rwsem); trace_scoutfs_data_find_alloc_block_ret(sb, ret); @@ -991,80 +1191,68 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; - struct scoutfs_key key; + struct scoutfs_extent ext; struct scoutfs_lock *lock; - struct block_mapping *map; - struct kvec val; - bool exists; - int ind; + u64 offset; int ret; - int i; + + WARN_ON_ONCE(create && !mutex_is_locked(&inode->i_mutex)); lock = scoutfs_per_task_get(&si->pt_data_lock); if (WARN_ON_ONCE(!lock)) return -EINVAL; - map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); - if (!map) - return -ENOMEM; +restart: + /* look for the extent that overlaps our iblock */ + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, + scoutfs_ino(inode), iblock, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + if (ret && ret != -ENOENT) + goto out; - init_mapping_key(&key, scoutfs_ino(inode), iblock); - kvec_init(&val, map->encoded, sizeof(map->encoded)); + if (ret == 0) + trace_scoutfs_data_get_block_next(sb, &ext); - /* find the mapping item that covers the logical block */ - ret = scoutfs_item_lookup(sb, &key, &val, lock); - if (ret < 0) { - if (ret != -ENOENT) - goto out; - memset(map->blknos, 0, sizeof(map->blknos)); - memset(map->offline, 0, sizeof(map->offline)); - exists = false; - } else { - ret = decode_mapping(map, ret); - if (ret < 0) - goto out; - exists = true; - } + /* didn't find an extent or it's past our iblock */ + if (ret == -ENOENT || ext.start > iblock) + memset(&ext, 0, sizeof(ext)); - ind = iblock & SCOUTFS_BLOCK_MAPPING_MASK; + if (ext.len) + trace_scoutfs_data_get_block_intersection(sb, &ext); /* fail read and write if it's offline and we're not staging */ - if (test_bit(ind, map->offline) && !si->staging) { + if ((ext.flags & SEF_OFFLINE) && !si->staging) { ret = -EINVAL; goto out; } /* try to allocate if we're writing */ - if (create && !map->blknos[ind]) { + if (create && !ext.map) { /* * XXX can blow the transaction here.. need to back off * and try again if we've already done a bulk alloc in * our transaction. */ - ret = find_alloc_block(sb, inode, map, &key, ind, exists, lock); + ret = find_alloc_block(sb, inode, iblock, + ext.flags & SEF_OFFLINE, lock); if (ret) goto out; set_buffer_new(bh); + /* restart the search now that it's been allocated */ + goto restart; } - /* mark the bh mapped and set the size for as many contig as we see */ - if (map->blknos[ind]) { - for (i = 1; ind + i < SCOUTFS_BLOCK_MAPPING_BLOCKS; i++) { - if (map->blknos[ind + i] != map->blknos[ind] + i) - break; - } - - map_bh(bh, inode->i_sb, map->blknos[ind]); - bh->b_size = min_t(u64, bh->b_size, i << SCOUTFS_BLOCK_SHIFT); + /* map the bh and set the size to as much of the extent as we can */ + if (ext.map) { + offset = iblock - ext.start; + map_bh(bh, inode->i_sb, ext.map + offset); + bh->b_size = min_t(u64, bh->b_size, + (ext.len - offset) << SCOUTFS_BLOCK_SHIFT); } - ret = 0; out: trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, ret, bh->b_blocknr, bh->b_size); - - kfree(map); - return ret; } @@ -1231,6 +1419,7 @@ struct pending_fiemap { u32 flags; }; +#if 0 /* * The caller is iterating over mapped blocks. We merge the current * pending fiemap entry with the next block if we can. If we can't @@ -1276,43 +1465,31 @@ static int merge_or_fill(struct fiemap_extent_info *fieinfo, return 0; } +#endif /* - * Iterate over non-zero block mapping items merging contiguous blocks and - * filling extent entries as we cross non-contiguous boundaries. We set - * _LAST on the last extent and _UNKNOWN on offline extents. + * Return all the file's extents whose blocks overlap with the caller's + * byte region. We set _LAST on the last extent and _UNKNOWN on offline + * extents. */ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct super_block *sb = inode->i_sb; - const u64 ino = scoutfs_ino(inode); - struct scoutfs_key last_key; - struct scoutfs_key key; struct scoutfs_lock *inode_lock = NULL; - struct block_mapping *map; - struct pending_fiemap pend; - struct kvec val; + struct scoutfs_extent ext; loff_t i_size; - bool offline; u64 blk_off; - u64 final; - u64 logical; - u64 phys; + u64 logical = 0; + u64 phys = 0; + u64 size = 0; + u32 flags = 0; int ret; - int i; ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) return ret; - map = kmalloc(sizeof(struct block_mapping), GFP_NOFS); - if (!map) - return -ENOMEM; - - /* initialize to impossible to merge */ - memset(&pend, 0, sizeof(pend)); - /* XXX overkill? */ mutex_lock(&inode->i_mutex); @@ -1323,68 +1500,46 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, goto out; } - blk_off = start >> SCOUTFS_BLOCK_SHIFT; - final = min_t(loff_t, i_size - 1, start + len - 1) >> - SCOUTFS_BLOCK_SHIFT; - init_mapping_key(&last_key, ino, final); - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) goto out; - while (blk_off <= final) { - init_mapping_key(&key, ino, blk_off); - kvec_init(&val, &map->encoded, sizeof(map->encoded)); + blk_off = start >> SCOUTFS_BLOCK_SHIFT; - ret = scoutfs_item_next(sb, &key, &last_key, &val, inode_lock); - if (ret < 0) { + for (;;) { + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, + scoutfs_ino(inode), blk_off, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, inode_lock); + /* fiemap will return last and stop when we see enoent */ + if (ret < 0 && ret != -ENOENT) + break; + + if (ret == 0) + trace_scoutfs_data_fiemap_extent(sb, &ext); + + if (size) { if (ret == -ENOENT) - ret = 0; - break; - } - - ret = decode_mapping(map, ret); - if (ret < 0) - break; - - /* set blk_off to the first in the next item inside last */ - blk_off = max(blk_off, le64_to_cpu(key.skm_base) << - SCOUTFS_BLOCK_MAPPING_SHIFT); - - for_each_block(i, blk_off, final) { - offline = !!test_bit(i, map->offline); - - /* nothing to do with sparse regions */ - if (map->blknos[i] == 0 && !offline) - continue; - - trace_scoutfs_data_fiemap(sb, blk_off, i, - map->blknos[i]); - - logical = blk_off << SCOUTFS_BLOCK_SHIFT; - phys = map->blknos[i] << SCOUTFS_BLOCK_SHIFT; - - ret = merge_or_fill(fieinfo, &pend, logical, phys, - offline, false); - if (ret != 0) + flags |= FIEMAP_EXTENT_LAST; + ret = fiemap_fill_next_extent(fieinfo, logical, phys, + size, flags); + if (ret || (logical + size >= (start + len))) { + if (ret == 1) + ret = 0; break; + } } - if (ret != 0) - break; + + logical = ext.start << SCOUTFS_BLOCK_SHIFT; + phys = ext.map << SCOUTFS_BLOCK_SHIFT; + size = ext.len << SCOUTFS_BLOCK_SHIFT; + flags = (ext.flags & SEF_OFFLINE) ? FIEMAP_EXTENT_UNKNOWN : 0; + + blk_off = ext.start + ext.len; } scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); - - if (ret == 0) { - /* catch final last fill */ - ret = merge_or_fill(fieinfo, &pend, 0, 0, false, true); - } - if (ret == 1) - ret = 0; - out: mutex_unlock(&inode->i_mutex); - kfree(map); return ret; } @@ -1460,6 +1615,7 @@ void scoutfs_data_destroy(struct super_block *sb) } } +#if 0 /* * Basic correctness tests of u64 and mapping encoding. */ @@ -1576,3 +1732,4 @@ out: return ret; } +#endif diff --git a/kmod/src/data.h b/kmod/src/data.h index e8157065..c9214c5c 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -13,6 +13,4 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); -int __init scoutfs_data_test(void); - #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 7735440b..766aad25 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -83,6 +83,11 @@ struct scoutfs_key { #define skf_node_id _sk_first #define skf_base _sk_second +/* node free extent */ +#define sknf_node_id _sk_first +#define sknf_major _sk_second +#define sknf_minor _sk_third + /* node orphan inode */ #define sko_node_id _sk_first #define sko_ino _sk_second @@ -109,6 +114,10 @@ struct scoutfs_key { #define skm_ino _sk_first #define skm_base _sk_second +/* file extent */ +#define skfe_ino _sk_first +#define skfe_last _sk_second + /* * The btree still uses memcmp() to compare keys. We should fix that * before too long. @@ -305,6 +314,8 @@ struct scoutfs_segment_block { /* node zone */ #define SCOUTFS_FREE_BITS_SEGNO_TYPE 1 #define SCOUTFS_FREE_BITS_BLKNO_TYPE 2 +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 3 +#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 4 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -315,6 +326,7 @@ struct scoutfs_segment_block { #define SCOUTFS_SYMLINK_TYPE 6 #define SCOUTFS_BLOCK_MAPPING_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 +#define SCOUTFS_FILE_EXTENT_TYPE 9 #define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ @@ -367,6 +379,18 @@ struct scoutfs_free_bits { __le64 bits[SCOUTFS_FREE_BITS_U64S]; } __packed; +/* + * File extents have more data than easily fits in the key so we move + * the non-indexed fields into the value. + */ +struct scoutfs_file_extent { + __le64 blkno; + __le64 len; + __u8 flags; +} __packed; + +#define SEF_OFFLINE 0x1 + /* * The first xattr part item has a header that describes the xattr. The * name and value are then packed into the following bytes in the first @@ -510,7 +534,6 @@ enum { SCOUTFS_DT_WHT, }; -#define SCOUTFS_MAX_VAL_SIZE SCOUTFS_BLOCK_MAPPING_MAX_BYTES #define SCOUTFS_XATTR_MAX_NAME_LEN 255 #define SCOUTFS_XATTR_MAX_VAL_LEN 65535 @@ -520,6 +543,8 @@ enum { DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ SCOUTFS_XATTR_MAX_PART_SIZE); +#define SCOUTFS_MAX_VAL_SIZE SCOUTFS_XATTR_MAX_PART_SIZE + /* * structures used by dlm */ diff --git a/kmod/src/key.c b/kmod/src/key.c index 23aa8265..6b34b432 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -28,6 +28,8 @@ char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_SEGNO_TYPE] = "fsg", [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_BLKNO_TYPE] = "fbk", + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fbn", + [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fbs", [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = "orp", [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = "ino", [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = "xat", @@ -36,6 +38,7 @@ char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = "lbr", [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = "sym", [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = "bmp", + [SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = "fex", }; char scoutfs_unknown_u8_strings[U8_MAX][U8_STR_MAX]; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 2ab9c3f3..c42b6c00 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2131,6 +2131,47 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_remove, TP_ARGS(sb, ext) ); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_next, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_remove, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_offline, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_bulk_alloc, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_cursor, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_free, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_next, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_intersection, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/super.c b/kmod/src/super.c index fdfd4080..915e33fe 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -424,10 +424,6 @@ static int __init scoutfs_module_init(void) scoutfs_key_init(); scoutfs_init_counters(); - ret = scoutfs_data_test(); - if (ret) - return ret; - ret = scoutfs_sysfs_init(); if (ret) return ret; From 70b2a50c9a2ec94b44ab1385b2c5be330b7b6c4f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Apr 2018 14:46:20 -0700 Subject: [PATCH 616/920] scoutfs: remove individual online/offline calls Remove the functions that operate on online and offline blocks independently now that the file data mapping code isn't using it any more. Signed-off-by: Zach Brown --- kmod/src/inode.c | 14 -------------- kmod/src/inode.h | 4 ---- 2 files changed, 18 deletions(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 726f4483..1ab2c127 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -564,20 +564,6 @@ u64 scoutfs_inode_data_version(struct inode *inode) return read_seqcount_u64(inode, &si->data_version); } -u64 scoutfs_inode_online_blocks(struct inode *inode) -{ - struct scoutfs_inode_info *si = SCOUTFS_I(inode); - - return read_seqcount_u64(inode, &si->online_blocks); -} - -u64 scoutfs_inode_offline_blocks(struct inode *inode) -{ - struct scoutfs_inode_info *si = SCOUTFS_I(inode); - - return read_seqcount_u64(inode, &si->offline_blocks); -} - void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 4dd5f641..7ae34de8 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -100,14 +100,10 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); -void scoutfs_inode_add_online_blocks(struct inode *inode, u64 val); -void scoutfs_inode_add_offline_blocks(struct inode *inode, u64 val); void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off); u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); u64 scoutfs_inode_data_version(struct inode *inode); -u64 scoutfs_inode_online_blocks(struct inode *inode); -u64 scoutfs_inode_offline_blocks(struct inode *inode); void scoutfs_inode_get_onoff(struct inode *inode, s64 *on, s64 *off); int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock); From 5eddd10eb7ee4e3bfdd25cb2a752ca58f5308a0a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 6 Apr 2018 14:55:59 -0700 Subject: [PATCH 617/920] scoutfs: remove dead block mapping code Remove all the code for tracking block mapping items and storing free blocks in bitmaps. Signed-off-by: Zach Brown --- kmod/src/count.h | 17 - kmod/src/data.c | 733 --------------------------------------- kmod/src/format.h | 66 +--- kmod/src/key.c | 3 - kmod/src/scoutfs_trace.h | 26 -- 5 files changed, 3 insertions(+), 842 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 799c2e31..863a789c 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -224,23 +224,6 @@ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) return cnt; } -/* - * Truncating a block mapping item's worth of blocks can modify both - * free blkno and free segno items per block. Then the largest possible - * mapping item. - */ -static inline const struct scoutfs_item_count SIC_TRUNC_BLOCK(void) -{ - struct scoutfs_item_count cnt = {0,}; - unsigned nr_free = (2 * SCOUTFS_BLOCK_MAPPING_BLOCKS); - - cnt.items += 1 + nr_free; - cnt.vals += SCOUTFS_BLOCK_MAPPING_MAX_BYTES + - (nr_free * sizeof(struct scoutfs_free_bits)); - - return cnt; -} - /* * Truncating an extent can: * - delete existing file extent, diff --git a/kmod/src/data.c b/kmod/src/data.c index 4343a90d..e0d273df 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "format.h" @@ -84,488 +83,6 @@ struct task_cursor { pid_t pid; }; -#if 0 -/* - * Block mapping items and their native decoded form can be pretty big. - * Let's allocate them to avoid blowing the stack. - */ -struct block_mapping { - /* native representation */ - unsigned long offline[DIV_ROUND_UP(SCOUTFS_BLOCK_MAPPING_BLOCKS, - BITS_PER_LONG)]; - u64 blknos[SCOUTFS_BLOCK_MAPPING_BLOCKS]; - - /* encoded persistent item */ - u8 encoded[SCOUTFS_BLOCK_MAPPING_MAX_BYTES]; -} __packed; - -/* - * We encode u64 blknos as a vlq zigzag encoded delta from the previous - * blkno. zigzag moves the sign bit down into the lsb so that small - * negative values have very few bits set. Then vlq outputs the least - * significant set bits into bytes in groups of 7. - * - * https://en.wikipedia.org/wiki/Variable-length_quantity - * - * The end result is that a series of blknos, which are limited by - * device size and often allocated near each other, are encoded with a - * handful of bytes. - */ -static unsigned zigzag_encode(u8 *bytes, u64 prev, u64 x) -{ - unsigned pos = 0; - - x -= prev; - /* careful, relying on shifting extending the sign bit */ - x = (x << 1) ^ ((s64)x >> 63); - - do { - bytes[pos++] = x & 127; - x >>= 7; - } while (x); - - bytes[pos - 1] |= 128; - - return pos; -} - -static int zigzag_decode(u64 *res, u64 prev, u8 *bytes, unsigned len) -{ - unsigned shift = 0; - int ret = -EIO; - u64 x = 0; - int i; - u8 b; - - for (i = 0; i < len; i++) { - b = bytes[i]; - x |= (u64)(b & 127) << shift; - if (b & 128) { - ret = i + 1; - break; - } - shift += 7; - - /* falls through to return -EIO if we run out of bytes */ - } - - x = (x >> 1) ^ (-(x & 1)); - *res = prev + x; - - return ret; -} - -/* - * Block mappings are encoded into a byte stream. - * - * The first byte's low bits contains the last mapping index that will - * be decoded. - * - * As we walk through the encoded blocks we add control bits to the - * current control byte for the encoding of the block: zero, offline, - * increment from prev, or zigzag encoding. - * - * When the control byte is full we start filling the next byte in the - * output as the control byte for the coming blocks. When we zigzag - * encode blocks we add them to the output stream. The result is an - * interleaving of control bytes and zigzag blocks, when they're needed. - * - * In practice the typical mapping will have a zigzag for the first - * block and then the rest will be described by the control bits. - * Regions of sparse, advancing allocations, and offline are all - * described only by control bits, getting us down to 2 bits per block. - */ -static unsigned encode_mapping(struct block_mapping *map) -{ - unsigned shift; - unsigned len; - u64 blkno; - u64 prev; - u8 *enc; - u8 *ctl; - u8 last; - int ret; - int i; - - enc = map->encoded; - ctl = enc++; - len = 1; - - /* find the last set block in the mapping */ - last = SCOUTFS_BLOCK_MAPPING_BLOCKS; - for (i = 0; i < SCOUTFS_BLOCK_MAPPING_BLOCKS; i++) { - if (map->blknos[i] || test_bit(i, map->offline)) - last = i; - } - - if (last == SCOUTFS_BLOCK_MAPPING_BLOCKS) - return 0; - - /* start with 6 bits of last */ - *ctl = last; - shift = 6; - - prev = 0; - for (i = 0; i <= last; i++) { - blkno = map->blknos[i]; - - - if (shift == 8) { - ctl = enc++; - len++; - *ctl = 0; - shift = 0; - } - - - if (blkno == prev + 1) - *ctl |= (SCOUTFS_BLOCK_ENC_INC << shift); - else if (test_bit(i, map->offline)) - *ctl |= (SCOUTFS_BLOCK_ENC_OFFLINE << shift); - else if (!blkno) - *ctl |= (SCOUTFS_BLOCK_ENC_ZERO << shift); - else { - *ctl |= (SCOUTFS_BLOCK_ENC_DELTA << shift); - - ret = zigzag_encode(enc, prev, blkno); - enc += ret; - len += ret; - } - - shift += 2; - if (blkno) - prev = blkno; - } - - - return len; -} - -static int decode_mapping(struct block_mapping *map, int size) -{ - unsigned ctl_bits; - u64 blkno; - u64 prev; - u8 *enc; - u8 ctl; - u8 last; - int ret; - int i; - - if (size < 1 || size > SCOUTFS_BLOCK_MAPPING_MAX_BYTES) - return -EIO; - - memset(map->blknos, 0, sizeof(map->blknos)); - memset(map->offline, 0, sizeof(map->offline)); - - enc = map->encoded; - ctl = *(enc++); - size--; - - /* start with lsb 6 bits of last */ - last = ctl & SCOUTFS_BLOCK_MAPPING_MASK; - ctl >>= 6; - ctl_bits = 2; - - prev = 0; - for (i = 0; i <= last; i++) { - - if (ctl_bits == 0) { - if (size-- == 0) - return -EIO; - ctl = *(enc++); - ctl_bits = 8; - } - - - switch(ctl & SCOUTFS_BLOCK_ENC_MASK) { - case SCOUTFS_BLOCK_ENC_INC: - blkno = prev + 1; - break; - case SCOUTFS_BLOCK_ENC_OFFLINE: - set_bit(i, map->offline); - blkno = 0; - break; - case SCOUTFS_BLOCK_ENC_ZERO: - blkno = 0; - break; - case SCOUTFS_BLOCK_ENC_DELTA: - ret = zigzag_decode(&blkno, prev, enc, size); - /* XXX corruption, ran out of encoded bytes */ - if (ret <= 0) - return -EIO; - enc += ret; - size -= ret; - break; - } - - ctl >>= 2; - ctl_bits -= 2; - - map->blknos[i] = blkno; - if (blkno) - prev = blkno; - } - - /* XXX corruption: didn't use up all the bytes */ - if (size != 0) - return -EIO; - - return 0; -} - -static void init_mapping_key(struct scoutfs_key *key, u64 ino, u64 iblock) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_FS_ZONE, - .skm_ino = cpu_to_le64(ino), - .sk_type = SCOUTFS_BLOCK_MAPPING_TYPE, - .skm_base = cpu_to_le64(iblock >> SCOUTFS_BLOCK_MAPPING_SHIFT), - }; -} - -static void init_free_key(struct scoutfs_key *key, u64 node_id, u64 full_bit, - u8 type) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_NODE_ZONE, - .skf_node_id = cpu_to_le64(node_id), - .sk_type = type, - .skf_base = cpu_to_le64(full_bit >> SCOUTFS_FREE_BITS_SHIFT), - }; -} - -/* - * Mark the given segno as allocated. We set its bit in a free segno - * item, possibly after creating it. - */ -static int set_segno_free(struct super_block *sb, u64 segno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key key; - struct kvec val; - int bit = 0; - int ret; - - init_free_key(&key, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); - if (ret && ret != -ENOENT) - goto out; - - bit = segno & SCOUTFS_FREE_BITS_MASK; - - if (ret == -ENOENT) { - memset(&frb, 0, sizeof(frb)); - set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, &val, lock); - goto out; - } - - if (test_and_set_bit_le(bit, frb.bits)) { - ret = -EIO; - goto out; - } - - ret = scoutfs_item_update(sb, &key, &val, lock); -out: - trace_scoutfs_data_set_segno_free(sb, segno, le64_to_cpu(key.skf_base), - bit, ret); - return ret; -} - -/* - * Create a new free blkno item with all but the given blkno marked - * free. We use the caller's key so they can delete it later if they - * need to. - */ -static int create_blkno_free(struct super_block *sb, u64 blkno, - struct scoutfs_key *key) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct kvec val; - int bit; - - init_free_key(key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - - bit = blkno & SCOUTFS_FREE_BITS_MASK; - memset(&frb, 0xff, sizeof(frb)); - clear_bit_le(bit, frb.bits); - - return scoutfs_item_create(sb, key, &val, lock); -} - -/* - * Mark the first block in the segno as allocated. This isn't a general - * purpose bit clear. It knows that it's only called from allocation - * that found the bit so it won't create the segno item. - * - * And because it's allocating a block in the segno, it also has to - * create a free block item that marks the rest of the blknos in segno - * as free. - * - * It deletes the free segno item if it clears the last bit. - */ -static int clear_segno_free(struct super_block *sb, u64 segno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key b_key; - struct scoutfs_key key; - struct kvec val; - u64 blkno; - int bit; - int ret; - - init_free_key(&key, sbi->node_id, segno, SCOUTFS_FREE_BITS_SEGNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); - if (ret) { - /* XXX corruption, caller saw item.. should still exist */ - if (ret == -ENOENT) - ret = -EIO; - goto out; - } - - /* XXX corruption, bit couldn't have been set */ - bit = segno & SCOUTFS_FREE_BITS_MASK; - if (!test_and_clear_bit_le(bit, frb.bits)) { - ret = -EIO; - goto out; - } - - /* create the new blkno item, we can safely delete it */ - blkno = segno << SCOUTFS_SEGMENT_BLOCK_SHIFT; - ret = create_blkno_free(sb, blkno, &b_key); - if (ret) - goto out; - - if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, lock); - else - ret = scoutfs_item_update(sb, &key, &val, lock); - if (ret) - scoutfs_item_delete_dirty(sb, &b_key); -out: - return ret; -} - -/* - * Mark the given blkno free. Set its bit in its free blkno item, - * possibly after creating it. If all the bits are set we try to mark - * its segno free and delete the blkno item. - */ -static int set_blkno_free(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key key; - struct kvec val; - u64 segno; - int bit; - int ret; - - /* get the specified item */ - init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); - if (ret && ret != -ENOENT) - goto out; - - bit = blkno & SCOUTFS_FREE_BITS_MASK; - - if (ret == -ENOENT) { - memset(&frb, 0, sizeof(frb)); - set_bit_le(bit, &frb); - ret = scoutfs_item_create(sb, &key, &val, lock); - goto out; - } - - if (test_and_set_bit_le(bit, frb.bits)) { - ret = -EIO; - goto out; - } - - if (!bitmap_full((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) { - ret = scoutfs_item_update(sb, &key, &val, lock); - goto out; - } - - /* dirty so we can safely delete if set segno fails */ - ret = scoutfs_item_dirty(sb, &key, lock); - if (ret) - goto out; - - segno = blkno >> SCOUTFS_SEGMENT_BLOCK_SHIFT; - ret = set_segno_free(sb, segno); - if (ret) - goto out; - - scoutfs_item_delete_dirty(sb, &key); - ret = 0; -out: - return ret; -} - -/* - * Mark the given blkno as allocated. This is working on behalf of a - * caller who just saw the item, it must exist. We delete the free - * blkno item if all its bits are empty. - */ -static int clear_blkno_free(struct super_block *sb, u64 blkno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key key; - struct kvec val; - int bit; - int ret; - - /* get the specified item */ - init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); - if (ret) { - /* XXX corruption, bits should have existed */ - if (ret == -ENOENT) - ret = -EIO; - goto out; - } - - /* XXX corruption, bit couldn't have been set */ - bit = blkno & SCOUTFS_FREE_BITS_MASK; - if (!test_and_clear_bit_le(bit, frb.bits)) { - ret = -EIO; - goto out; - } - - if (bitmap_empty((long *)frb.bits, SCOUTFS_FREE_BITS_BITS)) - ret = scoutfs_item_delete(sb, &key, lock); - else - ret = scoutfs_item_update(sb, &key, &val, lock); -out: - return ret; -} - -/* - * In each iteration iblock is the logical block and i is the index into - * blknos array and the bit in the offline bitmap. The iteration won't - * advance past the last logical block. - */ -#define for_each_block(i, iblock, last) \ - for (i = iblock & SCOUTFS_BLOCK_MAPPING_MASK; \ - i < SCOUTFS_BLOCK_MAPPING_BLOCKS && iblock <= (last); \ - i++, iblock++) -#endif - static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) { *key = (struct scoutfs_key) { @@ -974,82 +491,6 @@ out: return ret; } -#if 0 -/* - * Find the free bit item that contains the blkno and return the next blkno - * set starting with this blkno. - * - * Returns -ENOENT if there's no free blknos at or after the given blkno. - */ -static int find_free_blkno(struct super_block *sb, u64 blkno, u64 *blkno_ret) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key key; - struct kvec val; - int ret; - int bit; - - init_free_key(&key, sbi->node_id, blkno, SCOUTFS_FREE_BITS_BLKNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); - if (ret < 0) - goto out; - - bit = blkno & SCOUTFS_FREE_BITS_MASK; - bit = find_next_bit_le(frb.bits, SCOUTFS_FREE_BITS_BITS, bit); - if (bit >= SCOUTFS_FREE_BITS_BITS) { - ret = -ENOENT; - goto out; - } - - *blkno_ret = (le64_to_cpu(key.skf_base) << SCOUTFS_FREE_BITS_SHIFT) + - bit; - ret = 0; -out: - return ret; -} - -/* - * Find a free segno to satisfy allocation by finding the first bit set - * in the first free segno item. - */ -static int find_free_segno(struct super_block *sb, u64 *segno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; - struct scoutfs_free_bits frb; - struct scoutfs_key last_key; - struct scoutfs_key key; - struct kvec val; - int bit; - int ret; - - init_free_key(&key, sbi->node_id, 0, SCOUTFS_FREE_BITS_SEGNO_TYPE); - init_free_key(&last_key, sbi->node_id, U64_MAX, - SCOUTFS_FREE_BITS_SEGNO_TYPE); - kvec_init(&val, &frb, sizeof(struct scoutfs_free_bits)); - - ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); - if (ret < 0) - goto out; - - bit = find_next_bit_le(frb.bits, SCOUTFS_FREE_BITS_BITS, 0); - /* XXX corruption, shouldn't see empty items */ - if (bit >= SCOUTFS_FREE_BITS_BITS) { - ret = -EIO; - goto out; - } - - *segno = (le64_to_cpu(key.skf_base) << SCOUTFS_FREE_BITS_SHIFT) + bit; - ret = 0; -out: - return ret; -} -#endif - /* * Allocate a single block for the logical block offset in the file. * The caller tells us if the block was offline or not. We modify the @@ -1412,61 +853,6 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, return ret; } -struct pending_fiemap { - u64 logical; - u64 phys; - u64 size; - u32 flags; -}; - -#if 0 -/* - * The caller is iterating over mapped blocks. We merge the current - * pending fiemap entry with the next block if we can. If we can't - * merge then we fill the current entry and start on the next. We also - * fill the pending mapping if the caller specifically tells us that - * this will be the last call. - * - * returns 0 to continue, 1 to stop, and -errno to stop with error. - */ -static int merge_or_fill(struct fiemap_extent_info *fieinfo, - struct pending_fiemap *pend, u64 logical, u64 phys, - bool offline, bool last) -{ - u32 flags = offline ? FIEMAP_EXTENT_UNKNOWN : 0; - int ret; - - /* merge if we can, returning if we don't have to fill last */ - if (pend->logical + pend->size == logical && - ((pend->phys == 0 && phys == 0) || - (pend->phys + pend->size == phys)) && - pend->flags == flags) { - pend->size += SCOUTFS_BLOCK_SIZE; - if (!last) - return 0; - } - - if (pend->size) { - if (last) - pend->flags |= FIEMAP_EXTENT_LAST; - - /* returns 1 to end, including if we passed in _LAST */ - ret = fiemap_fill_next_extent(fieinfo, pend->logical, - pend->phys, pend->size, - pend->flags); - if (ret != 0) - return ret; - } - - pend->logical = logical; - pend->phys = phys; - pend->size = SCOUTFS_BLOCK_SIZE; - pend->flags = flags; - - return 0; -} -#endif - /* * Return all the file's extents whose blocks overlap with the caller's * byte region. We set _LAST on the last extent and _UNKNOWN on offline @@ -1614,122 +1000,3 @@ void scoutfs_data_destroy(struct super_block *sb) kfree(datinf); } } - -#if 0 -/* - * Basic correctness tests of u64 and mapping encoding. - */ -int __init scoutfs_data_test(void) -{ - u8 encoded[SCOUTFS_ZIGZAG_MAX_BYTES]; - struct block_mapping *input; - struct block_mapping *output; - u64 blkno; - u8 bits; - u64 prev; - u64 in; - u64 out; - int ret; - int len; - int b; - int i; - - prev = 0; - for (i = 0; i < 10000; i++) { - get_random_bytes_arch(&bits, sizeof(bits)); - get_random_bytes_arch(&in, sizeof(in)); - in &= (1ULL << (bits % 64)) - 1; - - len = zigzag_encode(encoded, prev, in); - - ret = zigzag_decode(&out, prev, encoded, len); - - if (ret <= 0 || ret > SCOUTFS_ZIGZAG_MAX_BYTES || in != out) { - printk("i %d prev %llu in %llu out %llu len %d ret %d\n", - i, prev, in, out, len, ret); - - ret = -EINVAL; - } - if (ret < 0) - return ret; - - prev = out; - } - - input = kmalloc(sizeof(struct block_mapping), GFP_KERNEL); - output = kmalloc(sizeof(struct block_mapping), GFP_KERNEL); - if (!input || !output) { - ret = -ENOMEM; - goto out; - } - - for (i = 0; i < 1000; i++) { - prev = 0; - for (b = 0; b < SCOUTFS_BLOCK_MAPPING_BLOCKS; b++) { - - if (b % (64 / 2) == 0) - get_random_bytes_arch(&in, sizeof(in)); - - clear_bit(b, input->offline); - - switch(in & SCOUTFS_BLOCK_ENC_MASK) { - case SCOUTFS_BLOCK_ENC_INC: - blkno = prev + 1; - break; - case SCOUTFS_BLOCK_ENC_OFFLINE: - set_bit(b, input->offline); - blkno = 0; - break; - case SCOUTFS_BLOCK_ENC_ZERO: - blkno = 0; - break; - case SCOUTFS_BLOCK_ENC_DELTA: - get_random_bytes_arch(&bits, sizeof(bits)); - get_random_bytes_arch(&blkno, sizeof(blkno)); - blkno &= (1ULL << (bits % 64)) - 1; - break; - } - - input->blknos[b] = blkno; - - in >>= 2; - if (blkno) - prev = blkno; - } - - len = encode_mapping(input); - if (len >= 1 && len < SCOUTFS_BLOCK_MAPPING_MAX_BYTES) - memcpy(output->encoded, input->encoded, len); - ret = decode_mapping(output, len); - if (ret) { - printk("map len %d decoding failed %d\n", len, ret); - ret = -EINVAL; - goto out; - } - - for (b = 0; b < SCOUTFS_BLOCK_MAPPING_BLOCKS; b++) { - if (input->blknos[b] != output->blknos[b] || - !!test_bit(b, input->offline) != - !!test_bit(b, output->offline)) - break; - } - - if (b < SCOUTFS_BLOCK_MAPPING_BLOCKS) { - printk("map ind %u: in %llu %u, out %llu %u\n", - b, input->blknos[b], - !!test_bit(b, input->offline), - output->blknos[b], - !!test_bit(b, output->offline)); - ret = -EINVAL; - goto out; - } - } - - ret = 0; -out: - kfree(input); - kfree(output); - - return ret; -} -#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 766aad25..03bcd856 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -79,10 +79,6 @@ struct scoutfs_key { #define skii_major _sk_second #define skii_ino _sk_third -/* node free bit map */ -#define skf_node_id _sk_first -#define skf_base _sk_second - /* node free extent */ #define sknf_node_id _sk_first #define sknf_major _sk_second @@ -110,10 +106,6 @@ struct scoutfs_key { #define sks_ino _sk_first #define sks_nr _sk_second -/* file data mapping */ -#define skm_ino _sk_first -#define skm_base _sk_second - /* file extent */ #define skfe_ino _sk_first #define skfe_last _sk_second @@ -312,10 +304,8 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ /* node zone */ -#define SCOUTFS_FREE_BITS_SEGNO_TYPE 1 -#define SCOUTFS_FREE_BITS_BLKNO_TYPE 2 -#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 3 -#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 4 +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 +#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -324,61 +314,11 @@ struct scoutfs_segment_block { #define SCOUTFS_READDIR_TYPE 4 #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 -#define SCOUTFS_BLOCK_MAPPING_TYPE 7 +#define SCOUTFS_FILE_EXTENT_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 -#define SCOUTFS_FILE_EXTENT_TYPE 9 #define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ -/* each mapping item describes a fixed number of blocks */ -#define SCOUTFS_BLOCK_MAPPING_SHIFT 6 -#define SCOUTFS_BLOCK_MAPPING_BLOCKS (1 << SCOUTFS_BLOCK_MAPPING_SHIFT) -#define SCOUTFS_BLOCK_MAPPING_MASK (SCOUTFS_BLOCK_MAPPING_BLOCKS - 1) - -/* - * The mapping item value is a byte stream that encodes the value of the - * mapped blocks. The first byte contains the last index that contains - * a mapped block in its low bits. The high bits contain the control - * bits for the first (and possibly only) mapped block. - * - * From then on we consume the control bits in the current control byte - * for each mapped block. Each block has two bits that describe the - * block: zero, incremental from previous block, delta encoded, and - * offline. If we run out of control bits then we consume the next byte - * in the stream for additional control bits. If we have a delta - * encoded block then we consume its encoded bytes from the byte stream. - */ - -#define SCOUTFS_BLOCK_ENC_ZERO 0 -#define SCOUTFS_BLOCK_ENC_INC 1 -#define SCOUTFS_BLOCK_ENC_DELTA 2 -#define SCOUTFS_BLOCK_ENC_OFFLINE 3 -#define SCOUTFS_BLOCK_ENC_MASK 3 - -#define SCOUTFS_ZIGZAG_MAX_BYTES (DIV_ROUND_UP(64, 7)) - -/* - * the largest block mapping has: nr byte, ctl bytes for all blocks, and - * worst case zigzag encodings for all blocks. - */ -#define SCOUTFS_BLOCK_MAPPING_MAX_BYTES \ - (1 + (SCOUTFS_BLOCK_MAPPING_BLOCKS / 4) + \ - (SCOUTFS_BLOCK_MAPPING_BLOCKS * SCOUTFS_ZIGZAG_MAX_BYTES)) - -/* free bit bitmaps contain a segment's worth of blocks */ -#define SCOUTFS_FREE_BITS_SHIFT \ - SCOUTFS_SEGMENT_BLOCK_SHIFT -#define SCOUTFS_FREE_BITS_BITS \ - (1 << SCOUTFS_FREE_BITS_SHIFT) -#define SCOUTFS_FREE_BITS_MASK \ - (SCOUTFS_FREE_BITS_BITS - 1) -#define SCOUTFS_FREE_BITS_U64S \ - DIV_ROUND_UP(SCOUTFS_FREE_BITS_BITS, 64) - -struct scoutfs_free_bits { - __le64 bits[SCOUTFS_FREE_BITS_U64S]; -} __packed; - /* * File extents have more data than easily fits in the key so we move * the non-indexed fields into the value. diff --git a/kmod/src/key.c b/kmod/src/key.c index 6b34b432..9c1603b5 100644 --- a/kmod/src/key.c +++ b/kmod/src/key.c @@ -26,8 +26,6 @@ char *scoutfs_zone_strings[SCOUTFS_MAX_ZONE] = { char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_SEGNO_TYPE] = "fsg", - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_BITS_BLKNO_TYPE] = "fbk", [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fbn", [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fbs", [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = "orp", @@ -37,7 +35,6 @@ char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = "rdr", [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = "lbr", [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = "sym", - [SCOUTFS_FS_ZONE][SCOUTFS_BLOCK_MAPPING_TYPE] = "bmp", [SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = "fex", }; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index c42b6c00..3f172a2d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -526,32 +526,6 @@ TRACE_EVENT(scoutfs_data_truncate_items, __entry->iblock, __entry->last, __entry->offline) ); -TRACE_EVENT(scoutfs_data_set_segno_free, - TP_PROTO(struct super_block *sb, __u64 segno, __u64 base, - unsigned int bit, int ret), - - TP_ARGS(sb, segno, base, bit, ret), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(__u64, segno) - __field(__u64, base) - __field(unsigned int, bit) - __field(int, ret) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->segno = segno; - __entry->base = base; - __entry->bit = bit; - __entry->ret = ret; - ), - - TP_printk(FSID_FMT" segno %llu base %llu bit %u ret %d", __entry->fsid, - __entry->segno, __entry->base, __entry->bit, __entry->ret) -); - TRACE_EVENT(scoutfs_sync_fs, TP_PROTO(struct super_block *sb, int wait), From 19f7e0284ba279a5afe718fc3bb4af153125cf9c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Apr 2018 17:18:59 -0700 Subject: [PATCH 618/920] scoutfs: add online/offline block trace event Signed-off-by: Zach Brown --- kmod/src/inode.c | 4 ++++ kmod/src/scoutfs_trace.h | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 1ab2c127..2ae48bb5 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -524,6 +524,10 @@ void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) /* XXX not sure if this is right */ inode->i_blocks += (on + off) * SCOUTFS_BLOCK_SECTORS; + trace_scoutfs_online_offline_blocks(inode, on, off, + si->online_blocks, + si->offline_blocks); + write_seqcount_end(&si->seqcount); preempt_enable(); } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3f172a2d..7929bb54 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2146,6 +2146,33 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, TP_ARGS(sb, ext) ); +TRACE_EVENT(scoutfs_online_offline_blocks, + TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, + u64 on_now, u64 off_now), + + TP_ARGS(inode, on_delta, off_delta, on_now, off_now), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__s64, on_delta) + __field(__s64, off_delta) + __field(__u64, on_now) + __field(__u64, off_now) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(inode->i_sb); + __entry->on_delta = on_delta; + __entry->off_delta = off_delta; + __entry->on_now = on_now; + __entry->off_now = off_now; + ), + + TP_printk("fsid "FSID_FMT" on_delta %lld off_delta %lld on_now %llu off_now %llu ", + __entry->fsid, __entry->on_delta, __entry->off_delta, + __entry->on_now, __entry->off_now) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From c01a715852cfa95a5abae90a5f05110fe79b68af Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 10 Apr 2018 13:01:31 -0700 Subject: [PATCH 619/920] scoutfs: use extents in the server allocator Have the server use the extent core to maintain free extent items in the allocation btree instead of the bitmap items. We add a client request to allocate an extent of a given length. The existing segment alloc and free now work with a segment's worth of blocks. The server maintains counters in the super block of free blocks instead of free segments. We maintain an allocation cursor so that allocation results tend to cycle through the device. It's stored in the super so that it is maintained across server instances. This doesn't remove unused dead code to keep the commit from getting too noisy. It'll be removed in a future commit. Signed-off-by: Zach Brown --- kmod/src/client.c | 19 ++ kmod/src/client.h | 1 + kmod/src/compact.c | 6 +- kmod/src/count.h | 3 +- kmod/src/counters.h | 7 + kmod/src/data.c | 77 ++++---- kmod/src/format.h | 23 ++- kmod/src/scoutfs_trace.h | 53 ++++- kmod/src/seg.c | 3 +- kmod/src/server.c | 414 ++++++++++++++++++++++++++++++++++++++- kmod/src/server.h | 1 + kmod/src/super.c | 3 +- 12 files changed, 545 insertions(+), 65 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index dd0d6b9a..1ad2de21 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -556,6 +556,25 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, return ret; } +int scoutfs_client_alloc_extent(struct super_block *sb, u64 len, u64 *start) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + __le64 lelen = cpu_to_le64(len); + __le64 lestart; + int ret; + + ret = client_request(client, SCOUTFS_NET_ALLOC_EXTENT, + &lelen, sizeof(lelen), &lestart, sizeof(lestart)); + if (ret == 0) { + if (lestart == 0) + ret = -ENOSPC; + else + *start = le64_to_cpu(lestart); + } + + return ret; +} + int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) { struct client_info *client = SCOUTFS_SB(sb)->client_info; diff --git a/kmod/src/client.h b/kmod/src/client.h index c2bc3bb9..aa098f02 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -3,6 +3,7 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); +int scoutfs_client_alloc_extent(struct super_block *sb, u64 len, u64 *start); int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 4047e5e7..7ae99183 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -494,7 +494,7 @@ void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno) /* * Commit the result of a compaction based on the state of the cursor. - * The net caller stops the manifest from being written while we're + * The server caller stops the manifest from being written while we're * making changes. We lock the manifest to atomically make our changes. * * The erorr handling is sketchy here because calling the manifest from @@ -513,7 +513,7 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) /* free unused segnos that were allocated for the compaction */ for (i = 0; i < curs->nr_segnos; i++) { if (curs->segnos[i]) { - ret = scoutfs_alloc_free(sb, curs->segnos[i]); + ret = scoutfs_server_free_segno(sb, curs->segnos[i]); BUG_ON(ret); } } @@ -523,7 +523,7 @@ int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) /* delete input segments, probably freeing their segnos */ list_for_each_entry(cseg, &curs->csegs, entry) { if (!cseg->part_of_move) { - ret = scoutfs_alloc_free(sb, cseg->segno); + ret = scoutfs_server_free_segno(sb, cseg->segno); BUG_ON(ret); } diff --git a/kmod/src/count.h b/kmod/src/count.h index 863a789c..95aed312 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -211,8 +211,7 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) { struct scoutfs_item_count cnt = {0,}; - unsigned nr_free = (SCOUTFS_BULK_ALLOC_COUNT + - SCOUTFS_BLOCKS_PER_PAGE) * 3; + unsigned nr_free = (1 + SCOUTFS_BLOCKS_PER_PAGE) * 3; unsigned nr_file = (DIV_ROUND_UP(SCOUTFS_BLOCKS_PER_PAGE, 2) + SCOUTFS_BLOCKS_PER_PAGE) * 3; diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 9d6cfc04..b07b5e46 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -101,6 +101,13 @@ EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(seg_shrink) \ EXPAND_COUNTER(seg_stale_read) \ + EXPAND_COUNTER(server_alloc_segno) \ + EXPAND_COUNTER(server_extent_alloc) \ + EXPAND_COUNTER(server_extent_alloc_error) \ + EXPAND_COUNTER(server_free_extent) \ + EXPAND_COUNTER(server_free_pending_extent) \ + EXPAND_COUNTER(server_free_pending_error) \ + EXPAND_COUNTER(server_free_segno) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_item_flush) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index e0d273df..12a39425 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -456,38 +456,24 @@ static struct task_cursor *get_cursor(struct data_info *datinf) return curs; } -static int bulk_alloc(struct super_block *sb) +static int get_server_extent(struct super_block *sb, u64 len) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_extent ext; - u64 *segnos = NULL; - int ret = 0; - int i; + u64 start; + int ret; - segnos = scoutfs_client_bulk_alloc(sb); - if (IS_ERR(segnos)) { - ret = PTR_ERR(segnos); + ret = scoutfs_client_alloc_extent(sb, len, &start); + if (ret) goto out; - } - for (i = 0; segnos[i]; i++) { - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, - segnos[i] << SCOUTFS_SEGMENT_BLOCK_SHIFT, - SCOUTFS_SEGMENT_BLOCKS, 0, 0); - trace_scoutfs_data_bulk_alloc(sb, &ext); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, - sbi->node_id_lock); - if (ret) - break; - } + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, start, len, 0, 0); + trace_scoutfs_data_get_server_extent(sb, &ext); + ret = scoutfs_extent_add(sb, data_extent_io, &ext, sbi->node_id_lock); + /* XXX don't free extent on error, crash recovery with server */ out: - if (!IS_ERR_OR_NULL(segnos)) - kfree(segnos); - - /* XXX don't orphan segnos on error, crash recovery with server */ - return ret; } @@ -500,8 +486,10 @@ out: * that track large extents. Each new allocating task will get a new * extent. */ -/* XXX initially tied to segment size, should be a lot larger */ -#define LARGE_EXTENT_BLOCKS SCOUTFS_SEGMENT_BLOCKS +#define CURSOR_BLOCKS (1 * 1024 * 1024 / BLOCK_SIZE) +#define CURSOR_BLOCKS_MASK (CURSOR_BLOCKS - 1) +#define CURSOR_BLOCKS_SEARCH (CURSOR_BLOCKS + CURSOR_BLOCKS - 1) +#define CURSOR_BLOCKS_ALLOC (CURSOR_BLOCKS * 64) static int find_alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, bool was_offline, struct scoutfs_lock *lock) @@ -543,16 +531,26 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, } /* try to find a new large extent, possibly asking for more */ - while (curs->blkno == 0) { + if (curs->blkno == 0) { scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, 2 * LARGE_EXTENT_BLOCKS, + sbi->node_id, 0, CURSOR_BLOCKS_SEARCH, 0, 0); ret = scoutfs_extent_next(sb, data_extent_io, &ext, sbi->node_id_lock); - if (ret && ret != -ENOENT) + if (ret == -ENOENT) { + /* try to get allocation from the server if we're out */ + ret = get_server_extent(sb, CURSOR_BLOCKS_ALLOC); + if (ret == 0) + ret = scoutfs_extent_next(sb, data_extent_io, + &ext, + sbi->node_id_lock); + } + if (ret) { + /* XXX should try to look for smaller free extents :/ */ + if (ret == -ENOENT) + ret = -ENOSPC; goto out; - - /* XXX should try to look for smaller free extents :/ */ + } /* * set our cursor to the aligned start of a large extent @@ -561,19 +559,10 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, * constantly setting cursors to the start of a large * free extent that keeps have its start allocated. */ - if (ret == 0) { - trace_scoutfs_data_alloc_block_free(sb, &ext); - curs->blkno = ALIGN(ext.start, LARGE_EXTENT_BLOCKS); - break; - } - - /* try to get allocation from the server if we're out */ - ret = bulk_alloc(sb); - if (ret < 0) - goto out; + trace_scoutfs_data_alloc_block_free(sb, &ext); + curs->blkno = ALIGN(ext.start, CURSOR_BLOCKS); } - /* remove the free block we're using */ scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, sbi->node_id, curs->blkno, 1, 0, 0); @@ -603,9 +592,9 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, scoutfs_inode_add_onoff(inode, 1, was_offline ? -1ULL : 0); /* set cursor to next block, clearing if we finish a large extent */ - BUILD_BUG_ON(!is_power_of_2(LARGE_EXTENT_BLOCKS)); + BUILD_BUG_ON(!is_power_of_2(CURSOR_BLOCKS)); curs->blkno++; - if ((curs->blkno & (LARGE_EXTENT_BLOCKS - 1)) == 0) + if ((curs->blkno & CURSOR_BLOCKS_MASK) == 0) curs->blkno = 0; ret = 0; diff --git a/kmod/src/format.h b/kmod/src/format.h index 03bcd856..f0a28347 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -236,6 +236,19 @@ struct scoutfs_manifest_btree_val { struct scoutfs_key last_key; } __packed; +/* + * Free extents are stored in the server in an allocation btree. The + * type differentiates whether start or length is in stored in the major + * value and is the primary sort key. 'start' is set to the final block + * in the extent so that overlaping queries can be done with next + * instead prev. + */ +struct scoutfs_extent_btree_key { + __u8 type; + __be64 major; + __be64 minor; +} __packed; + #define SCOUTFS_ALLOC_REGION_SHIFT 8 #define SCOUTFS_ALLOC_REGION_BITS (1 << SCOUTFS_ALLOC_REGION_SHIFT) #define SCOUTFS_ALLOC_REGION_MASK (SCOUTFS_ALLOC_REGION_BITS - 1) @@ -303,7 +316,7 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 #define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ -/* node zone */ +/* node zone (also used in server alloc btree) */ #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 #define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 @@ -370,6 +383,9 @@ struct scoutfs_super_block { __le64 alloc_uninit; __le64 total_segs; __le64 free_segs; + __le64 total_blocks; + __le64 free_blocks; + __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; struct scoutfs_btree_root alloc_root; @@ -564,9 +580,9 @@ struct scoutfs_net_segnos { } __packed; struct scoutfs_net_statfs { - __le64 total_segs; /* total segments in device */ + __le64 total_blocks; /* total blocks in device */ __le64 next_ino; /* next unused inode number */ - __le64 bfree; /* total free small blocks */ + __le64 bfree; /* free blocks */ __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; @@ -582,6 +598,7 @@ struct scoutfs_net_statfs { enum { SCOUTFS_NET_ALLOC_INODES = 0, + SCOUTFS_NET_ALLOC_EXTENT, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_BULK_ALLOC, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7929bb54..2c435714 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2117,7 +2117,7 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_offline, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_bulk_alloc, +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_server_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); @@ -2145,6 +2145,30 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_next, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_allocated, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_segno_next, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_segno_allocated, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_free_pending_extent, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_extent_io, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); TRACE_EVENT(scoutfs_online_offline_blocks, TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, @@ -2173,6 +2197,33 @@ TRACE_EVENT(scoutfs_online_offline_blocks, __entry->on_now, __entry->off_now) ); +DECLARE_EVENT_CLASS(scoutfs_segno_class, + TP_PROTO(struct super_block *sb, u64 segno), + + TP_ARGS(sb, segno), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__s64, segno) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->segno = segno; + ), + + TP_printk("fsid "FSID_FMT" segno %llu", + __entry->fsid, __entry->segno) +); +DEFINE_EVENT(scoutfs_segno_class, scoutfs_alloc_segno, + TP_PROTO(struct super_block *sb, u64 segno), + TP_ARGS(sb, segno) +); +DEFINE_EVENT(scoutfs_segno_class, scoutfs_free_segno, + TP_PROTO(struct super_block *sb, u64 segno), + TP_ARGS(sb, segno) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index fea5ea4f..3b402f93 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -28,6 +28,7 @@ #include "counters.h" #include "triggers.h" #include "msg.h" +#include "server.h" #include "scoutfs_trace.h" /* @@ -298,7 +299,7 @@ out: */ int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg) { - return scoutfs_alloc_free(sb, seg->segno); + return scoutfs_server_free_segno(sb, seg->segno); } /* diff --git a/kmod/src/server.c b/kmod/src/server.c index d5317d04..64c862c1 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "format.h" #include "counters.h" @@ -67,6 +68,10 @@ struct server_info { /* server tracks seq use */ spinlock_t seq_lock; struct list_head pending_seqs; + + /* server tracks pending frees to be applied during commit */ + struct rw_semaphore alloc_rwsem; + struct list_head pending_frees; }; struct server_request { @@ -93,6 +98,350 @@ struct commit_waiter { int ret; }; +static void init_extent_btree_key(struct scoutfs_extent_btree_key *ebk, + u8 type, u64 major, u64 minor) +{ + ebk->type = type; + ebk->major = cpu_to_be64(major); + ebk->minor = cpu_to_be64(minor); +} + +static int init_extent_from_btree_key(struct scoutfs_extent *ext, u8 type, + struct scoutfs_extent_btree_key *ebk, + unsigned int key_bytes) +{ + u64 start; + u64 len; + + /* btree _next doesn't have last key limit */ + if (ebk->type != type) + return -ENOENT; + + if (key_bytes != sizeof(struct scoutfs_extent_btree_key) || + (ebk->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + ebk->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) + return -EIO; /* XXX corruption, bad key */ + + start = be64_to_cpu(ebk->major); + len = be64_to_cpu(ebk->minor); + if (ebk->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) + swap(start, len); + start -= len - 1; + + return scoutfs_extent_init(ext, ebk->type, 0, start, len, 0, 0); +} + +/* + * This is called by the extent core on behalf of the server who holds + * the appropriate locks to protect the many btree items that can be + * accessed on behalf of one extent operation. + */ +static int server_extent_io(struct super_block *sb, int op, + struct scoutfs_extent *ext, void *data) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_extent_btree_key ebk; + SCOUTFS_BTREE_ITEM_REF(iref); + bool mirror = false; + u8 mirror_type; + u8 mirror_op = 0; + int ret; + int err; + + trace_scoutfs_server_extent_io(sb, ext); + + if (WARN_ON_ONCE(ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) + return -EINVAL; + + if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && + (op == SEI_INSERT || op == SEI_DELETE)) { + mirror = true; + mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; + mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; + } + + init_extent_btree_key(&ebk, ext->type, ext->start + ext->len - 1, + ext->len); + if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) + swap(ebk.major, ebk.minor); + + if (op == SEI_NEXT) { + ret = scoutfs_btree_next(sb, &super->alloc_root, + &ebk, sizeof(ebk), &iref); + if (ret == 0) { + ret = init_extent_from_btree_key(ext, ext->type, + iref.key, + iref.key_len); + scoutfs_btree_put_iref(&iref); + } + + } else if (op == SEI_INSERT) { + ret = scoutfs_btree_insert(sb, &super->alloc_root, + &ebk, sizeof(ebk), NULL, 0); + + } else if (op == SEI_DELETE) { + ret = scoutfs_btree_delete(sb, &super->alloc_root, + &ebk, sizeof(ebk)); + + } else { + ret = WARN_ON_ONCE(-EINVAL); + } + + if (ret == 0 && mirror) { + swap(ext->type, mirror_type); + ret = server_extent_io(sb, op, ext, data); + swap(ext->type, mirror_type); + if (ret) { + err = server_extent_io(sb, mirror_op, ext, data); + BUG_ON(err); + } + } + + return ret; +} + +/* + * Allocate an extent of the given length in the first smallest free + * extent that contains it. We allocate in multiples of segment blocks + * and expose that to callers today. + * + * This doesn't have the cursor that segment allocation does. It's + * possible that a recently freed segment can merge to form a larger + * free extent that can be very quickly allocated to a node. The hope is + * that doesn't happen very often. + */ +static int alloc_extent(struct super_block *sb, u64 len, u64 *start) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_extent ext; + int ret; + + *start = 0; + + down_write(&server->alloc_rwsem); + + if (len & (SCOUTFS_SEGMENT_BLOCKS - 1)) { + ret = -EINVAL; + goto out; + } + + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, 0, + 0, len, 0, 0); + ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); + if (ret) { + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; + } + + trace_scoutfs_server_alloc_extent_next(sb, &ext); + + ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + ext.len = len; + + ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); + if (ret) + goto out; + + trace_scoutfs_server_alloc_extent_allocated(sb, &ext); + le64_add_cpu(&super->free_blocks, -ext.len); + + *start = ext.start; + ret = 0; + +out: + up_write(&server->alloc_rwsem); + + if (ret) + scoutfs_inc_counter(sb, server_extent_alloc_error); + else + scoutfs_inc_counter(sb, server_extent_alloc); + + return ret; +} + +struct pending_free_extent { + struct list_head head; + u64 start; + u64 len; +}; + +/* + * Now that the transaction's done we can apply all the pending frees. + * The list entries are totally unsorted so this is the first time that + * we can discover corruption from duplicated frees, etc. This can also + * fail on normal transient io or memory errors. + * + * We can't unwind if this fails. The caller can freak out or keep + * trying forever. + */ +static int apply_pending_frees(struct super_block *sb) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct pending_free_extent *pfe; + struct pending_free_extent *tmp; + struct scoutfs_extent ext; + int ret; + + down_write(&server->alloc_rwsem); + + list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { + scoutfs_inc_counter(sb, server_free_pending_extent); + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, + pfe->start, pfe->len, 0, 0); + trace_scoutfs_server_free_pending_extent(sb, &ext); + ret = scoutfs_extent_add(sb, server_extent_io, &ext, NULL); + if (ret) { + scoutfs_inc_counter(sb, server_free_pending_error); + break; + } + + le64_add_cpu(&super->free_blocks, pfe->len); + list_del_init(&pfe->head); + kfree(pfe); + } + + up_write(&server->alloc_rwsem); + + return 0; +} + +/* + * If there are still pending frees to destroy it means the server didn't + * shut down cleanly and that's not well supported today so we want to + * have it holler if this happens. In the future we'd cleanly support + * forced shutdown that had been told that it's OK to throw away dirty + * state. + */ +static int destroy_pending_frees(struct super_block *sb) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct pending_free_extent *pfe; + struct pending_free_extent *tmp; + + WARN_ON_ONCE(!list_empty(&server->pending_frees)); + + down_write(&server->alloc_rwsem); + + list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { + list_del_init(&pfe->head); + kfree(pfe); + } + + up_write(&server->alloc_rwsem); + + return 0; +} + +/* + * We can't satisfy allocations with freed extents until the removed + * references to the freed extents have been committed. We add freed + * extents to a list that is only applied to the persistent indexes as + * the transaction is being committed and the current transaction won't + * try to allocate any more extents. If we didn't do this then we could + * write to referenced data as part of the commit that frees it. If the + * commit was interrupted the stable data could have been overwritten. + */ +static int free_extent(struct super_block *sb, u64 start, u64 len) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct pending_free_extent *pfe; + int ret; + + scoutfs_inc_counter(sb, server_free_extent); + + down_write(&server->alloc_rwsem); + + pfe = kmalloc(sizeof(struct pending_free_extent), GFP_NOFS); + if (!pfe) { + ret = -ENOMEM; + } else { + pfe->start = start; + pfe->len = len; + list_add_tail(&pfe->head, &server->pending_frees); + ret = 0; + } + + up_write(&server->alloc_rwsem); + + return ret; +} + +/* + * This is called by the compaction code which is running in the server. + * The server caller has held all the locks, etc. + */ +int scoutfs_server_free_segno(struct super_block *sb, u64 segno) +{ + scoutfs_inc_counter(sb, server_free_segno); + trace_scoutfs_free_segno(sb, segno); + return free_extent(sb, segno << SCOUTFS_SEGMENT_BLOCK_SHIFT, + SCOUTFS_SEGMENT_BLOCKS); +} + +/* + * Allocate a segment on behalf of compaction or a node wanting to write + * a level 0 segment. It has to be aligned to the segment size because + * we address segments with aligned segment numbers instead of block + * offsets. + * + * We can use a simple cursor sweep of the index by start because all + * server extents are multiples of the segment size. Sweeping through + * the volume tries to spread out new segment writes and make it more + * rare to write to a recently freed segment which can cause a client to + * have to re-read the manifest. + */ +static int alloc_segno(struct super_block *sb, u64 *segno) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_extent ext; + u64 curs; + int ret; + + down_write(&server->alloc_rwsem); + + curs = ALIGN(le64_to_cpu(super->alloc_cursor), SCOUTFS_SEGMENT_BLOCKS); + *segno = 0; + + do { + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, + curs, 1, 0, 0); + ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); + } while (ret == -ENOENT && curs && (curs = 0, 1)); + if (ret) { + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; + } + + trace_scoutfs_server_alloc_segno_next(sb, &ext); + + /* use cursor if within extent, otherwise start of next extent */ + if (ext.start < curs) + ext.start = curs; + ext.len = SCOUTFS_SEGMENT_BLOCKS; + + ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); + if (ret) + goto out; + + super->alloc_cursor = cpu_to_le64(ext.start + ext.len); + + *segno = ext.start >> SCOUTFS_SEGMENT_BLOCK_SHIFT; + + trace_scoutfs_server_alloc_segno_allocated(sb, &ext); + trace_scoutfs_alloc_segno(sb, *segno); + scoutfs_inc_counter(sb, server_alloc_segno); + +out: + up_write(&server->alloc_rwsem); + return ret; +} + /* * Trigger a server shutdown by shutting down the listening socket. The * server thread will break out of accept and exit. @@ -188,9 +537,9 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - ret = scoutfs_alloc_apply_pending(sb); + ret = apply_pending_frees(sb); if (ret) { - scoutfs_err(sb, "server error freeing segments: %d", ret); + scoutfs_err(sb, "server error freeing extents: %d", ret); goto out; } @@ -336,6 +685,50 @@ out: return send_reply(conn, id, type, ret, &ial, sizeof(ial)); } +/* + * Give the client an extent allocation of len blocks. We leave the + * details to the extent allocator. + */ +static int process_alloc_extent(struct server_connection *conn, + u64 id, u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct commit_waiter cw; + __le64 lestart; + __le64 lelen; + u64 start; + int ret; + + if (data_len != sizeof(lelen)) { + ret = -EINVAL; + goto out; + } + + memcpy(&lelen, data, data_len); + + down_read(&server->commit_rwsem); + ret = alloc_extent(sb, le64_to_cpu(lelen), &start); + if (ret == -ENOSPC) { + start = 0; + ret = 0; + } + if (ret == 0) { + lestart = cpu_to_le64(start); + queue_commit_work(server, &cw); + } + up_read(&server->commit_rwsem); + + if (ret == 0) + ret = wait_for_commit(server, &cw, id, type); +out: + return send_reply(conn, id, type, ret, &lestart, sizeof(lestart)); +} + +/* + * We still special case segno allocation because it's aligned and we'd + * like to keep that detail in the server. + */ static int process_alloc_segno(struct server_connection *conn, u64 id, u8 type, void *data, unsigned data_len) { @@ -352,7 +745,7 @@ static int process_alloc_segno(struct server_connection *conn, } down_read(&server->commit_rwsem); - ret = scoutfs_alloc_segno(sb, &segno); + ret = alloc_segno(sb, &segno); if (ret == 0) { lesegno = cpu_to_le64(segno); queue_commit_work(server, &cw); @@ -607,14 +1000,15 @@ static int process_statfs(struct server_connection *conn, u64 id, u8 type, if (data_len == 0) { /* uuid and total_segs are constant, so far */ memcpy(nstatfs.uuid, super->uuid, sizeof(nstatfs.uuid)); - nstatfs.total_segs = super->total_segs; spin_lock(&sbi->next_ino_lock); nstatfs.next_ino = super->next_ino; spin_unlock(&sbi->next_ino_lock); - /* alloc locks the bfree calculation */ - nstatfs.bfree = cpu_to_le64(scoutfs_alloc_bfree(sb)); + down_read(&server->alloc_rwsem); + nstatfs.total_blocks = super->total_blocks; + nstatfs.bfree = super->free_blocks; + up_read(&server->alloc_rwsem); ret = 0; } else { ret = -EINVAL; @@ -657,7 +1051,7 @@ int scoutfs_client_get_compaction(struct super_block *sb, void *curs) /* allow for expansion slop from sticky and alignment */ for (i = 0; i < nr + SCOUTFS_COMPACTION_SLOP; i++) { - ret = scoutfs_alloc_segno(sb, &segno); + ret = alloc_segno(sb, &segno); if (ret < 0) break; scoutfs_compact_add_segno(sb, curs, segno); @@ -728,6 +1122,7 @@ static void scoutfs_server_process_func(struct work_struct *work) struct server_connection *conn = req->conn; static process_func_t process_funcs[] = { [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, + [SCOUTFS_NET_ALLOC_EXTENT] = process_alloc_extent, [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, @@ -994,7 +1389,6 @@ static void scoutfs_server_func(struct work_struct *work) /* finally start up the server subsystems before accepting */ ret = scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: - scoutfs_alloc_setup(sb) ?: scoutfs_compact_setup(sb); if (ret) goto shutdown; @@ -1067,7 +1461,7 @@ shutdown: /* shut down all the server subsystems */ scoutfs_compact_destroy(sb); - scoutfs_alloc_destroy(sb); + destroy_pending_frees(sb); scoutfs_manifest_destroy(sb); scoutfs_btree_destroy(sb); @@ -1108,6 +1502,8 @@ int scoutfs_server_setup(struct super_block *sb) seqcount_init(&server->stable_seqcount); spin_lock_init(&server->seq_lock); INIT_LIST_HEAD(&server->pending_seqs); + init_rwsem(&server->alloc_rwsem); + INIT_LIST_HEAD(&server->pending_frees); server->wq = alloc_workqueue("scoutfs_server", WQ_NON_REENTRANT, 0); if (!server->wq) { diff --git a/kmod/src/server.h b/kmod/src/server.h index f6e076db..bbc73901 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -9,6 +9,7 @@ void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, int scoutfs_client_get_compaction(struct super_block *sb, void *curs); int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, void *list); +int scoutfs_server_free_segno(struct super_block *sb, u64 segno); int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index 915e33fe..8b728d22 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -74,8 +74,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_bfree = le64_to_cpu(nstatfs.bfree); kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SIZE; - kst->f_blocks = le64_to_cpu(nstatfs.total_segs) * - SCOUTFS_SEGMENT_BLOCKS; + kst->f_blocks = le64_to_cpu(nstatfs.total_blocks); kst->f_bavail = kst->f_bfree; kst->f_ffree = kst->f_bfree * 16; From 1b3645db8be8e24057018b76a565d638b44bfd19 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Apr 2018 14:36:35 -0700 Subject: [PATCH 620/920] scoutfs: remove dead server allocator code Remove the bitmap segno allocator code that the server used to use to manage allocations. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/alloc.c | 355 --------------------------------------- kmod/src/alloc.h | 15 -- kmod/src/client.c | 92 ---------- kmod/src/compact.c | 1 - kmod/src/counters.h | 4 +- kmod/src/format.h | 25 --- kmod/src/scoutfs_trace.h | 48 ------ kmod/src/seg.c | 1 - kmod/src/server.c | 55 ------ kmod/src/super.c | 1 - 11 files changed, 2 insertions(+), 597 deletions(-) delete mode 100644 kmod/src/alloc.c delete mode 100644 kmod/src/alloc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 9e8a6703..61b12f70 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -5,7 +5,7 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += alloc.o bio.o btree.o client.o compact.o counters.o data.o dir.o \ +scoutfs-y += bio.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o key.o lock.o \ manifest.o msg.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c deleted file mode 100644 index b501483c..00000000 --- a/kmod/src/alloc.c +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "btree.h" -#include "cmp.h" -#include "alloc.h" -#include "counters.h" -#include "scoutfs_trace.h" - -/* - * scoutfs allocates segments using regions of an allocation bitmap - * stored in btree items. - * - * Freed segments are recorded in nodes in an rbtree. The frees can't - * satisfy allocation until they're committed to prevent overwriting - * live data so they're only applied to the region nodes as their - * transaction is written. - * - * We allocate by sweeping a cursor through the volume. This gives - * racing unlocked readers more time to try to sample a stale freed - * segment, when its safe to do so, before it is reallocated and - * rewritten and they're forced to retry their racey read. - */ - -struct seg_alloc { - struct rw_semaphore rwsem; - struct rb_root pending_root; - u64 next_segno; -}; - -#define DECLARE_SEG_ALLOC(sb, name) \ - struct seg_alloc *name = SCOUTFS_SB(sb)->seg_alloc - -struct pending_region { - struct rb_node node; - u64 ind; - struct scoutfs_alloc_region_btree_val reg_val; -}; - -static struct pending_region *find_pending(struct rb_root *root, u64 ind) -{ - struct rb_node *node = root->rb_node; - struct pending_region *pend; - - while (node) { - pend = container_of(node, struct pending_region, node); - - if (ind < pend->ind) - node = node->rb_left; - else if (ind > pend->ind) - node = node->rb_right; - else - return pend; - } - - return NULL; -} - -static void insert_pending(struct rb_root *root, struct pending_region *ins) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct pending_region *pend; - - while (*node) { - parent = *node; - pend = container_of(*node, struct pending_region, node); - - if (ins->ind < pend->ind) - node = &(*node)->rb_left; - else if (ins->ind > pend->ind) - node = &(*node)->rb_right; - else - BUG(); - } - - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); -} - -static int copy_region_item(struct scoutfs_alloc_region_btree_key *reg_key, - struct scoutfs_alloc_region_btree_val *reg_val, - struct scoutfs_btree_item_ref *iref) -{ - if (iref->key_len != sizeof(struct scoutfs_alloc_region_btree_key) || - iref->val_len != sizeof(struct scoutfs_alloc_region_btree_val)) - return -EIO; - - memcpy(reg_key, iref->key, iref->key_len); - memcpy(reg_val, iref->val, iref->val_len); - return 0; -} - -/* - * We're careful to copy the bitmaps out to aligned versions so that - * we can use native bitops that require aligned longs. - */ -int scoutfs_alloc_segno(struct super_block *sb, u64 *segno) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_alloc_region_btree_key reg_key; - struct scoutfs_alloc_region_btree_val __aligned(sizeof(long)) reg_val; - SCOUTFS_BTREE_ITEM_REF(iref); - DECLARE_SEG_ALLOC(sb, sal); - u64 ind; - int ret; - int nr; - - down_write(&sal->rwsem); - - /* initially sweep through all segments */ - if (super->alloc_uninit != super->total_segs) { - *segno = le64_to_cpu(super->alloc_uninit); - /* done when inc hits total_segs */ - le64_add_cpu(&super->alloc_uninit, 1); - ret = 0; - goto out; - } - - /* but usually search for region nodes */ - ind = sal->next_segno >> SCOUTFS_ALLOC_REGION_SHIFT; - nr = sal->next_segno & SCOUTFS_ALLOC_REGION_MASK; - - for (;;) { - reg_key.index = cpu_to_be64(ind); - ret = scoutfs_btree_next(sb, &super->alloc_root, - ®_key, sizeof(reg_key), &iref); - if (ret == -ENOENT && ind != 0) { - ind = 0; - nr = 0; - continue; - } - if (ret < 0) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - ret = copy_region_item(®_key, ®_val, &iref); - scoutfs_btree_put_iref(&iref); - if (ret) - goto out; - - ind = be64_to_cpu(reg_key.index); - nr = find_next_bit_le(reg_val.bits, SCOUTFS_ALLOC_REGION_BITS, nr); - if (nr < SCOUTFS_ALLOC_REGION_BITS) { - break; - } - - /* possible for nr to be after all free bits, keep going */ - ind++; - nr = 0; - } - - clear_bit_le(nr, reg_val.bits); - - if (bitmap_empty((long *)reg_val.bits, SCOUTFS_ALLOC_REGION_BITS)) - ret = scoutfs_btree_delete(sb, &super->alloc_root, - ®_key, sizeof(reg_key)); - else - ret = scoutfs_btree_update(sb, &super->alloc_root, - ®_key, sizeof(reg_key), - ®_val, sizeof(reg_val)); - if (ret) - goto out; - - *segno = (ind << SCOUTFS_ALLOC_REGION_SHIFT) + nr; - sal->next_segno = *segno + 1; - - ret = 0; -out: - if (ret == 0) { - scoutfs_inc_counter(sb, alloc_alloc); - le64_add_cpu(&super->free_segs, -1); - } - up_write(&sal->rwsem); - - trace_scoutfs_alloc_segno(sb, *segno, ret); - return ret; -} - -/* - * Record newly freed sgements in pending regions. These are applied to - * persistent regions in btree items as the transaction commits. - */ -int scoutfs_alloc_free(struct super_block *sb, u64 segno) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct pending_region *pend; - DECLARE_SEG_ALLOC(sb, sal); - u64 ind; - int ret; - int nr; - - ind = segno >> SCOUTFS_ALLOC_REGION_SHIFT; - nr = segno & SCOUTFS_ALLOC_REGION_MASK; - - down_write(&sal->rwsem); - - pend = find_pending(&sal->pending_root, ind); - if (!pend) { - pend = kzalloc(sizeof(struct pending_region), GFP_NOFS); - if (!pend) { - ret = -ENOMEM; - goto out; - } - - pend->ind = ind; - insert_pending(&sal->pending_root, pend); - } - - set_bit_le(nr, pend->reg_val.bits); - scoutfs_inc_counter(sb, alloc_free); - le64_add_cpu(&super->free_segs, 1); - ret = 0; -out: - up_write(&sal->rwsem); - - trace_scoutfs_alloc_free(sb, segno, ind, nr, ret); - return ret; -} - -/* - * Apply the pending frees to create the final set of dirty btree - * blocks. The caller will write the btree blocks. We're destroying - * the pending free record here so from this point on the pending free - * blocks could be visible to allocation. The caller can't finish with - * the transaction until the btree is written successfully. - */ -int scoutfs_alloc_apply_pending(struct super_block *sb) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_SEG_ALLOC(sb, sal); - struct pending_region *pend; - struct rb_node *node; - struct scoutfs_alloc_region_btree_key reg_key; - struct scoutfs_alloc_region_btree_val __aligned(sizeof(long)) reg_val; - SCOUTFS_BTREE_ITEM_REF(iref); - int ret; - - down_write(&sal->rwsem); - - ret = 0; - while ((node = rb_first(&sal->pending_root))) { - pend = container_of(node, struct pending_region, node); - - /* see if we have a region for this index */ - reg_key.index = cpu_to_be64(pend->ind); - ret = scoutfs_btree_lookup(sb, &super->alloc_root, - ®_key, sizeof(reg_key), &iref); - if (ret == -ENOENT) { - /* create a new item if we don't */ - ret = scoutfs_btree_insert(sb, &super->alloc_root, - ®_key, sizeof(reg_key), - &pend->reg_val, - sizeof(pend->reg_val)); - } else if (ret == 0) { - /* and update the existing item if we do */ - ret = copy_region_item(®_key, ®_val, &iref); - scoutfs_btree_put_iref(&iref); - if (ret) - break; - - bitmap_or((long *)reg_val.bits, (long *)reg_val.bits, - (long *)pend->reg_val.bits, - SCOUTFS_ALLOC_REGION_BITS); - - ret = scoutfs_btree_update(sb, &super->alloc_root, - ®_key, sizeof(reg_key), - ®_val, sizeof(reg_val)); - } - if (ret < 0) - break; - - rb_erase(&pend->node, &sal->pending_root); - kfree(pend); - } - - up_write(&sal->rwsem); - - return ret; -} - -/* - * Return the number of blocks free for statfs. - */ -u64 scoutfs_alloc_bfree(struct super_block *sb) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_SEG_ALLOC(sb, sal); - u64 bfree; - - down_read(&sal->rwsem); - bfree = le64_to_cpu(super->free_segs) << SCOUTFS_SEGMENT_BLOCK_SHIFT; - up_read(&sal->rwsem); - - return bfree; -} - -int scoutfs_alloc_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct seg_alloc *sal; - - /* bits need to be aligned so hosts can use native bitops */ - BUILD_BUG_ON(offsetof(struct scoutfs_alloc_region_btree_val, bits) & - (sizeof(long) - 1)); - - sal = kzalloc(sizeof(struct seg_alloc), GFP_KERNEL); - if (!sal) - return -ENOMEM; - - init_rwsem(&sal->rwsem); - sal->pending_root = RB_ROOT; - - /* XXX read next_segno from super? */ - - sbi->seg_alloc = sal; - - return 0; -} - -void scoutfs_alloc_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_SEG_ALLOC(sb, sal); - struct pending_region *pend; - struct rb_node *node; - - if (sal) { - while ((node = rb_first(&sal->pending_root))) { - pend = container_of(node, struct pending_region, node); - rb_erase(&pend->node, &sal->pending_root); - kfree(pend); - } - kfree(sal); - sbi->seg_alloc = NULL; - } -} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h deleted file mode 100644 index d6c8a5b0..00000000 --- a/kmod/src/alloc.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef _SCOUTFS_ALLOC_H_ -#define _SCOUTFS_ALLOC_H_ - -struct scoutfs_alloc_region; - -int scoutfs_alloc_segno(struct super_block *sb, u64 *segno); -int scoutfs_alloc_free(struct super_block *sb, u64 segno); - -int scoutfs_alloc_apply_pending(struct super_block *sb); -u64 scoutfs_alloc_bfree(struct super_block *sb); - -int scoutfs_alloc_setup(struct super_block *sb); -void scoutfs_alloc_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/client.c b/kmod/src/client.c index 1ad2de21..c3c0bb91 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include "format.h" @@ -607,97 +606,6 @@ int scoutfs_client_record_segment(struct super_block *sb, sizeof(net_ment), NULL, 0); } -static int sort_cmp_u64s(const void *A, const void *B) -{ - const u64 *a = A; - const u64 *b = B; - - return *a < *b ? -1 : *a > *b ? 1 : 0; -} - -static void sort_swap_u64s(void *A, void *B, int size) -{ - u64 *a = A; - u64 *b = B; - - swap(*a, *b); -} - -/* - * Returns a 0-terminated allocated array of segnos, the caller is - * responsible for freeing it. - * - * This double alloc is silly. But the caller does have an easier time - * with native u64s. We'll probably clean this up. - */ -u64 *scoutfs_client_bulk_alloc(struct super_block *sb) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - struct scoutfs_net_segnos *ns = NULL; - u64 *segnos = NULL; - size_t size; - unsigned nr; - u64 prev; - int ret; - int i; - - size = offsetof(struct scoutfs_net_segnos, - segnos[SCOUTFS_BULK_ALLOC_COUNT]); - ns = kmalloc(size, GFP_NOFS); - if (!ns) { - ret = -ENOMEM; - goto out; - } - - ret = client_request(client, SCOUTFS_NET_BULK_ALLOC, NULL, 0, ns, size); - if (ret) - goto out; - - nr = le16_to_cpu(ns->nr); - if (nr == 0) { - ret = -ENOSPC; - goto out; - } - - if (nr > SCOUTFS_BULK_ALLOC_COUNT) { - ret = -EINVAL; - goto out; - } - - segnos = kmalloc_array(nr + 1, sizeof(*segnos), GFP_NOFS); - if (segnos == NULL) { - ret = -ENOMEM; - goto out; - } - - for (i = 0; i < nr; i++) - segnos[i] = le64_to_cpu(ns->segnos[i]); - segnos[nr] = 0; - - /* sort segnos for the caller so they can merge easily */ - sort(segnos, nr, sizeof(segnos[0]), sort_cmp_u64s, sort_swap_u64s); - - /* make sure they're all non-zero and unique */ - prev = 0; - for (i = 0; i < nr; i++) { - if (segnos[i] == prev) { - ret = -EINVAL; - goto out; - } - prev = segnos[i]; - } - - ret = 0; -out: - kfree(ns); - if (ret) { - kfree(segnos); - segnos = ERR_PTR(ret); - } - - return segnos; -} - int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) { struct client_info *client = SCOUTFS_SB(sb)->client_info; diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 7ae99183..3efb88ed 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -22,7 +22,6 @@ #include "compact.h" #include "manifest.h" #include "counters.h" -#include "alloc.h" #include "server.h" #include "scoutfs_trace.h" diff --git a/kmod/src/counters.h b/kmod/src/counters.h index b07b5e46..621d72c0 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,8 +12,6 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ - EXPAND_COUNTER(alloc_alloc) \ - EXPAND_COUNTER(alloc_free) \ EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(btree_write_error) \ @@ -117,7 +115,7 @@ EXPAND_COUNTER(trans_level0_seg_writes) -#define FIRST_COUNTER alloc_alloc +#define FIRST_COUNTER btree_read_error #define LAST_COUNTER trans_level0_seg_writes #undef EXPAND_COUNTER diff --git a/kmod/src/format.h b/kmod/src/format.h index f0a28347..87655716 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -249,19 +249,6 @@ struct scoutfs_extent_btree_key { __be64 minor; } __packed; -#define SCOUTFS_ALLOC_REGION_SHIFT 8 -#define SCOUTFS_ALLOC_REGION_BITS (1 << SCOUTFS_ALLOC_REGION_SHIFT) -#define SCOUTFS_ALLOC_REGION_MASK (SCOUTFS_ALLOC_REGION_BITS - 1) - -struct scoutfs_alloc_region_btree_key { - __be64 index; -} __packed; - -/* The bits need to be aligned so that the hosts can use native long bit ops */ -struct scoutfs_alloc_region_btree_val { - __le64 bits[SCOUTFS_ALLOC_REGION_BITS / 64]; -} __packed; - /* * The max number of links defines the max number of entries that we can * index in o(log n) and the static list head storage size in the @@ -380,9 +367,6 @@ struct scoutfs_super_block { __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 next_seq; - __le64 alloc_uninit; - __le64 total_segs; - __le64 free_segs; __le64 total_blocks; __le64 free_blocks; __le64 alloc_cursor; @@ -571,14 +555,6 @@ struct scoutfs_net_manifest_entry { __u8 keys[0]; } __packed; -/* XXX I dunno, totally made up */ -#define SCOUTFS_BULK_ALLOC_COUNT 32 - -struct scoutfs_net_segnos { - __le16 nr; - __le64 segnos[0]; -} __packed; - struct scoutfs_net_statfs { __le64 total_blocks; /* total blocks in device */ __le64 next_ino; /* next unused inode number */ @@ -601,7 +577,6 @@ enum { SCOUTFS_NET_ALLOC_EXTENT, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, - SCOUTFS_NET_BULK_ALLOC, SCOUTFS_NET_ADVANCE_SEQ, SCOUTFS_NET_GET_LAST_SEQ, SCOUTFS_NET_GET_MANIFEST_ROOT, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 2c435714..5e480364 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1087,54 +1087,6 @@ TRACE_EVENT(scoutfs_compact_func, TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) ); -TRACE_EVENT(scoutfs_alloc_free, - TP_PROTO(struct super_block *sb, __u64 segno, __u64 index, int nr, - int ret), - - TP_ARGS(sb, segno, index, nr, ret), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(__u64, segno) - __field(__u64, index) - __field(int, nr) - __field(int, ret) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->segno = segno; - __entry->index = index; - __entry->nr = nr; - __entry->ret = ret; - ), - - TP_printk(FSID_FMT" freeing segno %llu ind %llu nr %d ret %d", - __entry->fsid, __entry->segno, __entry->index, __entry->nr, - __entry->ret) -); - -TRACE_EVENT(scoutfs_alloc_segno, - TP_PROTO(struct super_block *sb, __u64 segno, int ret), - - TP_ARGS(sb, segno, ret), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(__u64, segno) - __field(int, ret) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->segno = segno; - __entry->ret = ret; - ), - - TP_printk(FSID_FMT" segno %llu ret %d", __entry->fsid, __entry->segno, - __entry->ret) -); - TRACE_EVENT(scoutfs_write_begin, TP_PROTO(struct super_block *sb, u64 ino, loff_t pos, unsigned len), diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 3b402f93..a69a226d 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -23,7 +23,6 @@ #include "kvec.h" #include "cmp.h" #include "manifest.h" -#include "alloc.h" #include "key.h" #include "counters.h" #include "triggers.h" diff --git a/kmod/src/server.c b/kmod/src/server.c index 64c862c1..b9538f87 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include "format.h" @@ -27,7 +26,6 @@ #include "inode.h" #include "btree.h" #include "manifest.h" -#include "alloc.h" #include "seg.h" #include "compact.h" #include "scoutfs_trace.h" @@ -811,58 +809,6 @@ out: return send_reply(conn, id, type, ret, NULL, 0); } -static int process_bulk_alloc(struct server_connection *conn, u64 id, u8 type, - void *data, unsigned data_len) -{ - struct server_info *server = conn->server; - struct super_block *sb = server->sb; - struct scoutfs_net_segnos *ns = NULL; - struct commit_waiter cw; - size_t size; - u64 segno; - int ret; - int i; - - if (data_len != 0) { - ret = -EINVAL; - goto out; - } - - size = offsetof(struct scoutfs_net_segnos, - segnos[SCOUTFS_BULK_ALLOC_COUNT]); - ns = kmalloc(size, GFP_NOFS); - if (!ns) { - ret = -ENOMEM; - goto out; - } - - down_read(&server->commit_rwsem); - - ns->nr = cpu_to_le16(SCOUTFS_BULK_ALLOC_COUNT); - for (i = 0; i < SCOUTFS_BULK_ALLOC_COUNT; i++) { - ret = scoutfs_alloc_segno(sb, &segno); - if (ret) { - while (i-- > 0) - scoutfs_alloc_free(sb, - le64_to_cpu(ns->segnos[i])); - break; - } - - ns->segnos[i] = cpu_to_le64(segno); - } - - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - - if (ret == 0) - ret = wait_for_commit(server, &cw, id, type); -out: - ret = send_reply(conn, id, type, ret, ns, size); - kfree(ns); - return ret; -} - struct pending_seq { struct list_head head; u64 seq; @@ -1125,7 +1071,6 @@ static void scoutfs_server_process_func(struct work_struct *work) [SCOUTFS_NET_ALLOC_EXTENT] = process_alloc_extent, [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, - [SCOUTFS_NET_BULK_ALLOC] = process_bulk_alloc, [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, diff --git a/kmod/src/super.c b/kmod/src/super.c index 8b728d22..6b853766 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -34,7 +34,6 @@ #include "manifest.h" #include "seg.h" #include "bio.h" -#include "alloc.h" #include "compact.h" #include "data.h" #include "lock.h" From dd091e18a98538200bc6a9ae8c66f08fd8a6d1df Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 18 Apr 2018 09:26:30 -0700 Subject: [PATCH 621/920] scoutfs: add trans item tracking trace Add a trace event that records the changes to a reservation's dirty item count. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 33 +++++++++++++++++++++++++++++++++ kmod/src/trans.c | 14 ++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5e480364..5c3178ff 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -660,6 +660,39 @@ TRACE_EVENT(scoutfs_trans_acquired_hold, __entry->tri_items, __entry->tri_vals) ); +TRACE_EVENT(scoutfs_trans_track_item, + TP_PROTO(struct super_block *sb, int delta_items, int delta_vals, + int act_items, int act_vals, int res_items, int res_vals), + + TP_ARGS(sb, delta_items, delta_vals, act_items, act_vals, res_items, + res_vals), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, delta_items) + __field(int, delta_vals) + __field(int, act_items) + __field(int, act_vals) + __field(int, res_items) + __field(int, res_vals) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->delta_items = delta_items; + __entry->delta_vals = delta_vals; + __entry->act_items = act_items; + __entry->act_vals = act_vals; + __entry->res_items = res_items; + __entry->res_vals = res_vals; + ), + + TP_printk("fsid "FSID_FMT" delta_items %d delta_vals %d act_items %d act_vals %d res_items %d res_vals %d", + __entry->fsid, __entry->delta_items, __entry->delta_vals, + __entry->act_items, __entry->act_vals, __entry->res_items, + __entry->res_vals) +); + TRACE_EVENT(scoutfs_ioc_release_ret, TP_PROTO(struct super_block *sb, int ret), diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 2c39951f..9e45b7e2 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -392,6 +392,16 @@ bool scoutfs_trans_held(void) return rsv && rsv->magic == SCOUTFS_RESERVATION_MAGIC; } +/* + * Record a transaction holder's individual contribution to the dirty + * items in the current transaction. We're making sure that the + * reservation matches the possible item manipulations while they hold + * the reservation. + * + * It is possible and legitimate for an individual contribution to be + * negative if they delete dirty items. The item cache makes sure that + * the total dirty item count doesn't fall below zero. + */ void scoutfs_trans_track_item(struct super_block *sb, signed items, signed vals) { @@ -406,6 +416,10 @@ void scoutfs_trans_track_item(struct super_block *sb, signed items, rsv->actual.items += items; rsv->actual.vals += vals; + trace_scoutfs_trans_track_item(sb, items, vals, rsv->actual.items, + rsv->actual.vals, rsv->reserved.items, + rsv->reserved.vals); + WARN_ON_ONCE(rsv->actual.items > rsv->reserved.items); WARN_ON_ONCE(rsv->actual.vals > rsv->reserved.vals); } From fe94eb7363bcd54d8487fedc7d31d9ce5146058a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Apr 2018 14:31:16 -0700 Subject: [PATCH 622/920] scoutfs: add unwritten extents Now that we have extents we can address the fragmentation of concurrent writes with large preallocated unwritten extents instead of trying to allocate from disjoint free space with cursors. First we add support for unwritten extents. Truncate needs to make sure it doesn't treat truncated unwritten blocks as online just because they're not offline. If we try to write into them we convert them to written extents. And fiemap needs to flag them as unwritten and be sure to check for extents past i_size. Then we allocate unwritten extents only if we're extending a contiguous file. We try to preallocate the size of the file and cap it to a meg. This ends up with a power of two progression of preallocation sizes, which nicely balances extent sizes and wasted allocation as file sizes increase. We need to be careful to truncate the preallocated regions if the entire file is released. We take that as an indication that the user doesn't want the file consuming any more space. This removes most of the use of the cursor code. It will be completely removed in a further patch. Signed-off-by: Zach Brown --- kmod/src/data.c | 259 +++++++++++++++++++++++---------------- kmod/src/format.h | 1 + kmod/src/ioctl.c | 20 +++ kmod/src/scoutfs_trace.h | 67 +++++----- 4 files changed, 207 insertions(+), 140 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 12a39425..1fc9bb20 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -40,17 +40,12 @@ * scoutfs uses extent items to track file data block mappings and free * blocks. * - * Block allocation maintains a fixed number of allocation cursors that - * remember the position of tasks within free regions. This is very - * simple and maintains contiguous allocations for simple streaming - * writes. It eventually won't be good enough and we'll spend - * complexity on delalloc but we want to put that off as long as - * possible. + * Typically we'll allocate a single block in get_block if a mapping + * isn't found. * - * There's no unwritten extents. As we dirty file data pages we track - * their inodes. Before we commit dirty metadata we write out all - * tracked inodes. This ensures that data is persistent before the - * metadata that references it is visible. + * We special case extending contiguous files. In that case we'll preallocate + * an unwritten extent at the end of the file. The size of the preallocation + * is based on the file size and is capped. * * XXX * - truncate @@ -253,6 +248,8 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, struct scoutfs_extent ofl; bool rem_fr = false; bool add_rem = false; + s64 offline_delta = 0; + s64 online_delta = 0; s64 ret; int err; @@ -309,9 +306,15 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, goto out; } - scoutfs_inode_add_onoff(inode, rem.map ? -rem.len : 0, - (rem.flags & SEF_OFFLINE ? -rem.len : 0) + - (offline ? ofl.len : 0)); + if (rem.map && !(rem.flags & SEF_UNWRITTEN)) + online_delta += -rem.len; + if (rem.flags & SEF_OFFLINE) + offline_delta += -rem.len; + if (offline) + offline_delta += ofl.len; + + scoutfs_inode_add_onoff(inode, online_delta, offline_delta); + ret = 1; out: if (ret < 0) { @@ -396,6 +399,7 @@ static inline struct hlist_head *cursor_head(struct data_info *datinf, return &datinf->cursor_hash[h]; } +#if 0 static struct task_cursor *search_head(struct hlist_head *head, struct task_struct *task, pid_t pid) { @@ -455,6 +459,7 @@ static struct task_cursor *get_cursor(struct data_info *datinf) return curs; } +#endif static int get_server_extent(struct super_block *sb, u64 len) { @@ -482,96 +487,86 @@ out: * The caller tells us if the block was offline or not. We modify the * extent items and the caller will search for the resulting extent. * - * We try to encourage contiguous allocation by having per-task cursors - * that track large extents. Each new allocating task will get a new - * extent. + * If we're writing to the final block of the file then we try to + * preallocate unwritten blocks past i_size for future extending writes + * to use. We only base this decision on the file size. Truncating + * down the size, unlink, or releasing all blocks in the file will + * remove these preallocated blocks. Truncating past them will preserve + * them and treat them as 0. + * + * This assumes that there can't be existing unwritten extents in the + * inode that would overlap with our allocations. Writes are serialized + * and the caller only calls us if an extent doesn't exist. Unwritten + * extents are only created adjacent to i_size extensions. The only way + * to pull i_size back behind unwritten extents is to truncate and it + * frees them. Corrupt disk images could have fragmented unwritten + * extents past i_size in inodes and that'd manifest as errors inserting + * overlapping new allocations. */ -#define CURSOR_BLOCKS (1 * 1024 * 1024 / BLOCK_SIZE) -#define CURSOR_BLOCKS_MASK (CURSOR_BLOCKS - 1) -#define CURSOR_BLOCKS_SEARCH (CURSOR_BLOCKS + CURSOR_BLOCKS - 1) -#define CURSOR_BLOCKS_ALLOC (CURSOR_BLOCKS * 64) -static int find_alloc_block(struct super_block *sb, struct inode *inode, - u64 iblock, bool was_offline, - struct scoutfs_lock *lock) +#define MAX_UNWRITTEN_BLOCKS ((u64)SCOUTFS_SEGMENT_BLOCKS) +#define SERVER_ALLOC_BLOCKS (MAX_UNWRITTEN_BLOCKS * 32) +static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, + bool was_offline, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); const u64 ino = scoutfs_ino(inode); + struct scoutfs_extent unwr; struct scoutfs_extent ext; struct scoutfs_extent ofl; + struct scoutfs_extent blk; struct scoutfs_extent fr; - struct task_cursor *curs; bool add_ofl = false; bool add_fr = false; + bool rem_blk = false; + u64 offline; + u64 online; + u64 len; int err; int ret; down_write(&datinf->alloc_rwsem); - curs = get_cursor(datinf); + scoutfs_inode_get_onoff(inode, &online, &offline); - trace_scoutfs_data_find_alloc_block_curs(sb, curs, curs->blkno); + /* exponentially prealloc unwritten extents to a limit */ + if (iblock > 1 && iblock == (online + offline)) + len = min(iblock, MAX_UNWRITTEN_BLOCKS); + else + len = 1; - /* see if our cursor is still free */ - if (curs->blkno) { - /* look for the extent that overlaps our iblock */ - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, curs->blkno, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, - sbi->node_id_lock); - if (ret && ret != -ENOENT) - goto out; + trace_scoutfs_data_alloc_block(sb, inode, iblock, was_offline, + online, offline, len); + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + sbi->node_id, 0, len, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, + sbi->node_id_lock); + if (ret == -ENOENT) { + /* try to get allocation from the server if we're out */ + ret = get_server_extent(sb, SERVER_ALLOC_BLOCKS); if (ret == 0) - trace_scoutfs_data_alloc_block_cursor(sb, &ext); - - /* find a new large extent if our cursor isn't free */ - if (ret < 0 || ext.start > curs->blkno) - curs->blkno = 0; + ret = scoutfs_extent_next(sb, data_extent_io, &ext, + sbi->node_id_lock); + } + if (ret) { + /* XXX should try to look for smaller free extents :/ */ + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; } - /* try to find a new large extent, possibly asking for more */ - if (curs->blkno == 0) { - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, CURSOR_BLOCKS_SEARCH, - 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, - sbi->node_id_lock); - if (ret == -ENOENT) { - /* try to get allocation from the server if we're out */ - ret = get_server_extent(sb, CURSOR_BLOCKS_ALLOC); - if (ret == 0) - ret = scoutfs_extent_next(sb, data_extent_io, - &ext, - sbi->node_id_lock); - } - if (ret) { - /* XXX should try to look for smaller free extents :/ */ - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } + trace_scoutfs_data_alloc_block_next(sb, &ext); - /* - * set our cursor to the aligned start of a large extent - * We'll then remove it and the next aligned free large - * extent will start much later. This stops us from - * constantly setting cursors to the start of a large - * free extent that keeps have its start allocated. - */ - trace_scoutfs_data_alloc_block_free(sb, &ext); - curs->blkno = ALIGN(ext.start, CURSOR_BLOCKS); - } - - /* remove the free block we're using */ + /* remove the free extent we're using */ scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, curs->blkno, 1, 0, 0); + sbi->node_id, ext.start, len, 0, 0); ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); if (ret) goto out; add_fr = true; - /* remove an offline file extent */ + /* remove an offline block extent */ if (was_offline) { scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, ino, iblock, 1, 0, SEF_OFFLINE); @@ -581,26 +576,32 @@ static int find_alloc_block(struct super_block *sb, struct inode *inode, add_ofl = true; } - /* add (and hopefully merge!) the new allocation */ - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, curs->blkno, 0); - trace_scoutfs_data_alloc_block(sb, &ext); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, lock); + /* add the block that the caller is writing */ + scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, 1, ext.start, 0); + ret = scoutfs_extent_add(sb, data_extent_io, &blk, lock); if (ret) goto out; + rem_blk = true; + + /* and maybe add the remaining unwritten extent */ + if (len > 1) { + scoutfs_extent_init(&unwr, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock + 1, len - 1, ext.start + 1, + SEF_UNWRITTEN); + ret = scoutfs_extent_add(sb, data_extent_io, &unwr, lock); + if (ret) + goto out; + } scoutfs_inode_add_onoff(inode, 1, was_offline ? -1ULL : 0); - - /* set cursor to next block, clearing if we finish a large extent */ - BUILD_BUG_ON(!is_power_of_2(CURSOR_BLOCKS)); - curs->blkno++; - if ((curs->blkno & CURSOR_BLOCKS_MASK) == 0) - curs->blkno = 0; - ret = 0; out: if (ret) { err = 0; + if (rem_blk) + err |= scoutfs_extent_remove(sb, data_extent_io, &blk, + lock); if (add_ofl) err |= scoutfs_extent_add(sb, data_extent_io, &ofl, lock); @@ -612,7 +613,52 @@ out: up_write(&datinf->alloc_rwsem); - trace_scoutfs_data_find_alloc_block_ret(sb, ret); + trace_scoutfs_data_alloc_block_ret(sb, ret); + return ret; +} + +/* + * Remove the unwritten flag from an existing extent. We don't have to + * wait for dirty block IO to complete before clearing the unwritten + * flag in metadata because we have strict synchronization between data + * and metadata. All dirty data in the current transaction is written + * before the metadata in the transaction that references it is + * committed. + * + * The extent is unwritten so it can't be offline nor online. We remove + * the unwritten flag, possibly splitting and merging. We record the + * extent as online now as initial block allocation would. + */ +static int convert_unwritten(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 start, u64 len, + struct scoutfs_lock *lock) +{ + struct scoutfs_extent conv; + int err; + int ret; + + if (WARN_ON_ONCE(!ext->map) || + WARN_ON_ONCE(!(ext->flags & SEF_UNWRITTEN))) + return -EINVAL; + + scoutfs_extent_init(&conv, ext->type, ext->owner, start, len, + ext->map + (start - ext->start), ext->flags); + ret = scoutfs_extent_remove(sb, data_extent_io, &conv, lock); + if (ret) + goto out; + + conv.flags &= ~SEF_UNWRITTEN; + ret = scoutfs_extent_add(sb, data_extent_io, &conv, lock); + if (ret) { + conv.flags |= SEF_UNWRITTEN; + err = scoutfs_extent_add(sb, data_extent_io, &conv, lock); + BUG_ON(err); + goto out; + } + + ret = 0; +out: + scoutfs_inode_add_onoff(inode, len, 0); return ret; } @@ -656,15 +702,18 @@ restart: goto out; } + /* convert unwritten to written */ + if (create && (ext.flags & SEF_UNWRITTEN)) { + ret = convert_unwritten(sb, inode, &ext, iblock, 1, lock); + if (ret) + goto out; + goto restart; + } + /* try to allocate if we're writing */ if (create && !ext.map) { - /* - * XXX can blow the transaction here.. need to back off - * and try again if we've already done a bulk alloc in - * our transaction. - */ - ret = find_alloc_block(sb, inode, iblock, - ext.flags & SEF_OFFLINE, lock); + ret = alloc_block(sb, inode, iblock, ext.flags & SEF_OFFLINE, + lock); if (ret) goto out; set_buffer_new(bh); @@ -853,7 +902,6 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; struct scoutfs_extent ext; - loff_t i_size; u64 blk_off; u64 logical = 0; u64 phys = 0; @@ -868,13 +916,6 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); - /* stop at i_size, we don't allocate outside i_size */ - i_size = i_size_read(inode); - if (i_size == 0) { - ret = 0; - goto out; - } - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); if (ret) goto out; @@ -907,7 +948,11 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, logical = ext.start << SCOUTFS_BLOCK_SHIFT; phys = ext.map << SCOUTFS_BLOCK_SHIFT; size = ext.len << SCOUTFS_BLOCK_SHIFT; - flags = (ext.flags & SEF_OFFLINE) ? FIEMAP_EXTENT_UNKNOWN : 0; + flags = 0; + if (ext.flags & SEF_OFFLINE) + flags |= FIEMAP_EXTENT_UNKNOWN; + if (ext.flags & SEF_UNWRITTEN) + flags |= FIEMAP_EXTENT_UNWRITTEN; blk_off = ext.start + ext.len; } @@ -961,7 +1006,6 @@ int scoutfs_data_setup(struct super_block *sb) for (i = 0; i < NR_CURSORS; i++) { curs = kzalloc(sizeof(struct task_cursor), GFP_KERNEL); if (!curs) { - destroy_cursors(datinf); kfree(datinf); return -ENOMEM; } @@ -984,8 +1028,5 @@ void scoutfs_data_destroy(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct data_info *datinf = sbi->data_info; - if (datinf) { - destroy_cursors(datinf); - kfree(datinf); - } + kfree(datinf); } diff --git a/kmod/src/format.h b/kmod/src/format.h index 87655716..cecd5c8b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -330,6 +330,7 @@ struct scoutfs_file_extent { } __packed; #define SEF_OFFLINE 0x1 +#define SEF_UNWRITTEN 0x2 /* * The first xattr part item has a header that describes the xattr. The diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 206a054d..31a4a612 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -262,6 +262,10 @@ out: * offline. Attempts to use the blocks in the future will trigger * recall from the archive. * + * If the file's online blocks drop to 0 then we also truncate any + * blocks beyond i_size. This honors the intent of fully releasing a file + * without the user needing to know to release past i_size or truncate. + * * XXX permissions? * XXX a lot of this could be generic file write prep */ @@ -273,6 +277,9 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) struct scoutfs_lock *lock = NULL; loff_t start; loff_t end_inc; + u64 online; + u64 offline; + u64 isize; int ret; if (copy_from_user(&args, (void __user *)arg, sizeof(args))) @@ -323,6 +330,19 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) args.block, args.block + args.count - 1, true, lock); + if (ret == 0) { + scoutfs_inode_get_onoff(inode, &online, &offline); + isize = i_size_read(inode); + if (online == 0 && isize) { + start = (isize + SCOUTFS_BLOCK_SIZE - 1) + >> SCOUTFS_BLOCK_SHIFT; + ret = scoutfs_data_truncate_items(sb, inode, + scoutfs_ino(inode), + start, U64_MAX, + false, lock); + } + } + out: scoutfs_unlock(sb, lock, DLM_LOCK_EX); mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5c3178ff..29c3b3bf 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -421,7 +421,41 @@ TRACE_EVENT(scoutfs_get_block, __entry->create, __entry->ret, __entry->blkno, __entry->size) ); -TRACE_EVENT(scoutfs_data_find_alloc_block_ret, +TRACE_EVENT(scoutfs_data_alloc_block, + TP_PROTO(struct super_block *sb, struct inode *inode, u64 iblock, + bool was_offline, u64 online_blocks, u64 offline_blocks, + u64 len), + + TP_ARGS(sb, inode, iblock, was_offline, online_blocks, offline_blocks, + len), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, iblock) + __field(__u8, was_offline) + __field(__u64, online_blocks) + __field(__u64, offline_blocks) + __field(__u64, len) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = scoutfs_ino(inode); + __entry->iblock = iblock; + __entry->was_offline = was_offline; + __entry->online_blocks = online_blocks; + __entry->offline_blocks = offline_blocks; + __entry->len = len; + ), + + TP_printk("fsid "FSID_FMT" ino %llu iblock %llu was_offline %u online_blocks %llu offline_blocks %llu len %llu", + __entry->fsid, __entry->ino, __entry->iblock, + __entry->was_offline, __entry->online_blocks, + __entry->offline_blocks, __entry->len) +); + +TRACE_EVENT(scoutfs_data_alloc_block_ret, TP_PROTO(struct super_block *sb, int ret), TP_ARGS(sb, ret), @@ -439,27 +473,6 @@ TRACE_EVENT(scoutfs_data_find_alloc_block_ret, TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) ); -TRACE_EVENT(scoutfs_data_find_alloc_block_found_seg, - TP_PROTO(struct super_block *sb, __u64 segno, __u64 blkno), - - TP_ARGS(sb, segno, blkno), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(__u64, segno) - __field(__u64, blkno) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->segno = segno; - __entry->blkno = blkno; - ), - - TP_printk(FSID_FMT" found free segno %llu blkno %llu", __entry->fsid, - __entry->segno, __entry->blkno) -); - TRACE_EVENT(scoutfs_data_find_alloc_block_curs, TP_PROTO(struct super_block *sb, void *curs, __u64 blkno), @@ -2106,15 +2119,7 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_server_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_cursor, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_free, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block, +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_next, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); From 874a44aef0f890fc7bab7b7eb9ff9c3ebeea5f57 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 20 Apr 2018 10:47:14 -0700 Subject: [PATCH 623/920] scoutfs: remove dead file allocation cursor code This is no longer used now that we allocate large extents for concurrently extending files by preallocating unwritten extents. Signed-off-by: Zach Brown --- kmod/src/data.c | 110 ------------------------------------------------ 1 file changed, 110 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 1fc9bb20..d3f89b3c 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -56,28 +56,13 @@ * - need trans around each bulk alloc */ -/* more than enough for a few tasks per core on moderate hardware */ -#define NR_CURSORS 4096 -#define CURSOR_HASH_HEADS (PAGE_SIZE / sizeof(void *) / 2) -#define CURSOR_HASH_BITS ilog2(CURSOR_HASH_HEADS) - struct data_info { struct rw_semaphore alloc_rwsem; - struct list_head cursor_lru; - struct hlist_head cursor_hash[CURSOR_HASH_HEADS]; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -struct task_cursor { - u64 blkno; - struct hlist_node hnode; - struct list_head list_head; - struct task_struct *task; - pid_t pid; -}; - static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) { *key = (struct scoutfs_key) { @@ -389,78 +374,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return ret; } -static inline struct hlist_head *cursor_head(struct data_info *datinf, - struct task_struct *task, - pid_t pid) -{ - unsigned h = hash_ptr(task, CURSOR_HASH_BITS) ^ - hash_long(pid, CURSOR_HASH_BITS); - - return &datinf->cursor_hash[h]; -} - -#if 0 -static struct task_cursor *search_head(struct hlist_head *head, - struct task_struct *task, pid_t pid) -{ - struct task_cursor *curs; - - hlist_for_each_entry(curs, head, hnode) { - if (curs->task == task && curs->pid == pid) - return curs; - } - - return NULL; -} - -static void destroy_cursors(struct data_info *datinf) -{ - struct task_cursor *curs; - struct hlist_node *tmp; - int i; - - for (i = 0; i < CURSOR_HASH_HEADS; i++) { - hlist_for_each_entry_safe(curs, tmp, &datinf->cursor_hash[i], - hnode) { - hlist_del_init(&curs->hnode); - kfree(curs); - } - } -} - -/* - * These cheesy cursors are only meant to encourage nice IO patterns for - * concurrent tasks either streaming large file writes or creating lots - * of small files. It will do very poorly in many other situations. To - * do better we'd need to go further down the road to delalloc and take - * more surrounding context into account. - */ -static struct task_cursor *get_cursor(struct data_info *datinf) -{ - struct task_struct *task = current; - pid_t pid = current->pid; - struct hlist_head *head; - struct task_cursor *curs; - - head = cursor_head(datinf, task, pid); - curs = search_head(head, task, pid); - if (!curs) { - curs = list_last_entry(&datinf->cursor_lru, - struct task_cursor, list_head); - trace_scoutfs_data_get_cursor(curs, task, pid); - hlist_del_init(&curs->hnode); - curs->task = task; - curs->pid = pid; - hlist_add_head(&curs->hnode, head); - curs->blkno = 0; - } - - list_move(&curs->list_head, &datinf->cursor_lru); - - return curs; -} -#endif - static int get_server_extent(struct super_block *sb, u64 len) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -987,36 +900,13 @@ const struct file_operations scoutfs_file_fops = { int scoutfs_data_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct hlist_head *head; struct data_info *datinf; - struct task_cursor *curs; - int i; datinf = kzalloc(sizeof(struct data_info), GFP_KERNEL); if (!datinf) return -ENOMEM; init_rwsem(&datinf->alloc_rwsem); - INIT_LIST_HEAD(&datinf->cursor_lru); - - for (i = 0; i < CURSOR_HASH_HEADS; i++) - INIT_HLIST_HEAD(&datinf->cursor_hash[i]); - - /* just allocate all of these up front */ - for (i = 0; i < NR_CURSORS; i++) { - curs = kzalloc(sizeof(struct task_cursor), GFP_KERNEL); - if (!curs) { - kfree(datinf); - return -ENOMEM; - } - - curs->pid = i; - - head = cursor_head(datinf, curs->task, curs->pid); - hlist_add_head(&curs->hnode, head); - - list_add(&curs->list_head, &datinf->cursor_lru); - } sbi->data_info = datinf; From 41c29c48ddb1ebd8014025a6db456b2366bb07e7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 22 May 2018 11:14:14 -0700 Subject: [PATCH 624/920] scoutfs: add extent corruption cases The extent code was originally written to panic if it hit errors during cleanup that resulted in inconsistent metadata. The more reasonble strategy is to warn about the corruption and act accordingly and leave it to corrective measures to resolve the corruption. In this case we continue returning the error that caused us to try and clean up. Signed-off-by: Zach Brown --- kmod/src/counters.h | 3 +++ kmod/src/data.c | 54 ++++++++++++++++++++++----------------------- kmod/src/extents.c | 52 ++++++++++--------------------------------- kmod/src/extents.h | 25 +++++++++++++++++++++ kmod/src/format.h | 3 +++ kmod/src/server.c | 9 ++++++-- 6 files changed, 76 insertions(+), 70 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 621d72c0..9d5db004 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -25,12 +25,15 @@ EXPAND_COUNTER(compact_sticky_written) \ EXPAND_COUNTER(corrupt_btree_block_level) \ EXPAND_COUNTER(corrupt_btree_no_child_ref) \ + EXPAND_COUNTER(corrupt_data_extent_trunc_cleanup) \ + EXPAND_COUNTER(corrupt_data_extent_alloc_cleanup) \ EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ EXPAND_COUNTER(corrupt_inode_block_counts) \ EXPAND_COUNTER(corrupt_extent_add_cleanup) \ EXPAND_COUNTER(corrupt_extent_rem_cleanup) \ + EXPAND_COUNTER(corrupt_server_extent_cleanup) \ EXPAND_COUNTER(corrupt_symlink_inode_size) \ EXPAND_COUNTER(corrupt_symlink_missing_item) \ EXPAND_COUNTER(corrupt_symlink_not_null_term) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index d3f89b3c..478746a6 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -35,6 +35,7 @@ #include "lock.h" #include "file.h" #include "extents.h" +#include "msg.h" /* * scoutfs uses extent items to track file data block mappings and free @@ -236,7 +237,6 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, s64 offline_delta = 0; s64 online_delta = 0; s64 ret; - int err; scoutfs_extent_init(&next, SCOUTFS_FILE_EXTENT_TYPE, ino, iblock, 1, 0, 0); @@ -302,19 +302,17 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, ret = 1; out: - if (ret < 0) { - err = 0; - if (add_rem) - err |= scoutfs_extent_add(sb, data_extent_io, &rem, - lock); - if (rem_fr) - err |= scoutfs_extent_remove(sb, data_extent_io, &fr, - sbi->node_id_lock); - BUG_ON(err); /* inconsistency, could save/restore */ + scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, + data_extent_io, &rem, lock, + SC_DATA_EXTENT_TRUNC_CLEANUP, + corrupt_data_extent_trunc_cleanup, &rem); + scoutfs_extent_cleanup(ret < 0 && rem_fr, scoutfs_extent_remove, sb, + data_extent_io, &fr, sbi->node_id_lock, + SC_DATA_EXTENT_TRUNC_CLEANUP, + corrupt_data_extent_trunc_cleanup, &rem); - } else if (ret > 0) { + if (ret > 0) ret = rem.start + rem.len; - } return ret; } @@ -435,7 +433,6 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, u64 offline; u64 online; u64 len; - int err; int ret; down_write(&datinf->alloc_rwsem); @@ -471,6 +468,10 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, trace_scoutfs_data_alloc_block_next(sb, &ext); + /* initialize the new mapped block extent, referenced by cleanup */ + scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, 1, ext.start, 0); + /* remove the free extent we're using */ scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, sbi->node_id, ext.start, len, 0, 0); @@ -490,8 +491,6 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, } /* add the block that the caller is writing */ - scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, ext.start, 0); ret = scoutfs_extent_add(sb, data_extent_io, &blk, lock); if (ret) goto out; @@ -510,19 +509,18 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, scoutfs_inode_add_onoff(inode, 1, was_offline ? -1ULL : 0); ret = 0; out: - if (ret) { - err = 0; - if (rem_blk) - err |= scoutfs_extent_remove(sb, data_extent_io, &blk, - lock); - if (add_ofl) - err |= scoutfs_extent_add(sb, data_extent_io, &ofl, - lock); - if (add_fr) - err |= scoutfs_extent_add(sb, data_extent_io, &fr, - sbi->node_id_lock); - BUG_ON(err); /* inconsistency */ - } + scoutfs_extent_cleanup(ret < 0 && rem_blk, scoutfs_extent_remove, sb, + data_extent_io, &blk, lock, + SC_DATA_EXTENT_ALLOC_CLEANUP, + corrupt_data_extent_alloc_cleanup, &blk); + scoutfs_extent_cleanup(ret < 0 && add_ofl, scoutfs_extent_add, sb, + data_extent_io, &ofl, lock, + SC_DATA_EXTENT_ALLOC_CLEANUP, + corrupt_data_extent_alloc_cleanup, &blk); + scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, + data_extent_io, &fr, sbi->node_id_lock, + SC_DATA_EXTENT_ALLOC_CLEANUP, + corrupt_data_extent_alloc_cleanup, &blk); up_write(&datinf->alloc_rwsem); diff --git a/kmod/src/extents.c b/kmod/src/extents.c index 1a461805..bbb18c71 100644 --- a/kmod/src/extents.c +++ b/kmod/src/extents.c @@ -198,30 +198,6 @@ out: return ret; } -/* - * The process of modifying an extent creates and deletes many - * intermediate extents. If we hit an error we need to undo the - * process. If we then hit an error we can be left with inconsistent - * extent items. - * - * We could fix this for extents that are stored in the item cache - * because it has tools for ensuring that operations can't fail. - * Extents that are stored in the btree currently can't avoid errors. - * We'd have to predirty blocks, allow deletion to fall below thresholds - * if merging saw an error, and preallocate blocks to be used for - * splitting/growth. It'd probably be worth it. - */ -#define extent_cleanup(cond, ext_func, sb, iof, clean, data, which, ctr, ext) \ -do { \ - __typeof__(sb) _sb = (sb); \ - int _ret; \ - \ - if ((cond) && (_ret = ext_func(_sb, iof, clean, data)) < 0) \ - scoutfs_corruption(_sb, which, ctr, \ - "ext "SE_FMT" clean "SE_FMT" ret %d", \ - SE_ARG(ext), SE_ARG(clean), _ret); \ -} while (0) - /* * Add a new extent. It can not overlap with any existing extents. It * may be merged with neighbouring extents. @@ -265,15 +241,12 @@ int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, /* finally insert our new (possibly merged) extent */ ret = extent_insert(sb, iof, &ext, data); out: - extent_cleanup(ret < 0 && ins_right, - extent_insert, sb, iof, &right, data, - SC_EXTENT_ADD_CLEANUP, corrupt_extent_add_cleanup, - add); - extent_cleanup(ret < 0 && ins_left, - extent_insert, sb, iof, &left, data, - SC_EXTENT_ADD_CLEANUP, corrupt_extent_add_cleanup, - add); - + scoutfs_extent_cleanup(ret < 0 && ins_right, extent_insert, sb, iof, + &right, data, SC_EXTENT_ADD_CLEANUP, + corrupt_extent_add_cleanup, add); + scoutfs_extent_cleanup(ret < 0 && ins_left, extent_insert, sb, iof, + &left, data, SC_EXTENT_ADD_CLEANUP, + corrupt_extent_add_cleanup, add); return ret; } @@ -332,12 +305,11 @@ int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, } out: - extent_cleanup(ret < 0 && del_left, - extent_delete, sb, iof, &left, data, - SC_EXTENT_REM_CLEANUP, corrupt_extent_rem_cleanup, rem); - extent_cleanup(ret < 0 && ins_ext, - extent_insert, sb, iof, &ext, data, - SC_EXTENT_REM_CLEANUP, corrupt_extent_rem_cleanup, rem); - + scoutfs_extent_cleanup(ret < 0 && del_left, extent_delete, sb, iof, + &left, data, SC_EXTENT_REM_CLEANUP, + corrupt_extent_rem_cleanup, rem); + scoutfs_extent_cleanup(ret < 0 && ins_ext, extent_insert, sb, iof, + &ext, data, SC_EXTENT_REM_CLEANUP, + corrupt_extent_rem_cleanup, rem); return ret; } diff --git a/kmod/src/extents.h b/kmod/src/extents.h index ab25c93f..d97f1f32 100644 --- a/kmod/src/extents.h +++ b/kmod/src/extents.h @@ -38,4 +38,29 @@ int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, struct scoutfs_extent *rem, void *data); +/* + * The process of modifying an extent creates and deletes many + * intermediate extents. If we hit an error we need to undo the + * process. If we then hit an error we can be left with inconsistent + * extent items. + * + * We could fix this for extents that are stored in the item cache + * because it has tools for ensuring that operations can't fail. + * Extents that are stored in the btree currently can't avoid errors. + * We'd have to predirty blocks, allow deletion to fall below thresholds + * if merging saw an error, and preallocate blocks to be used for + * splitting/growth. It'd probably be worth it. + */ +#define scoutfs_extent_cleanup(cond, ext_func, sb, iof, clean, data, \ + which, ctr, ext) \ +do { \ + __typeof__(sb) _sb = (sb); \ + int _ret; \ + \ + if ((cond) && (_ret = ext_func(_sb, iof, clean, data)) < 0) \ + scoutfs_corruption(_sb, which, ctr, \ + "ext "SE_FMT" clean "SE_FMT" ret %d", \ + SE_ARG(ext), SE_ARG(clean), _ret); \ +} while (0) + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index cecd5c8b..f9a33b83 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -619,6 +619,9 @@ enum { SC_INODE_BLOCK_COUNTS, SC_EXTENT_ADD_CLEANUP, SC_EXTENT_REM_CLEANUP, + SC_DATA_EXTENT_TRUNC_CLEANUP, + SC_DATA_EXTENT_ALLOC_CLEANUP, + SC_SERVER_EXTENT_CLEANUP, SC_NR_SOURCES, }; diff --git a/kmod/src/server.c b/kmod/src/server.c index b9538f87..e7c0c3dd 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -190,9 +190,14 @@ static int server_extent_io(struct super_block *sb, int op, swap(ext->type, mirror_type); ret = server_extent_io(sb, op, ext, data); swap(ext->type, mirror_type); - if (ret) { + if (ret < 0) { err = server_extent_io(sb, mirror_op, ext, data); - BUG_ON(err); + if (err) + scoutfs_corruption(sb, + SC_SERVER_EXTENT_CLEANUP, + corrupt_server_extent_cleanup, + "op %u ext "SE_FMT" ret %d", + op, SE_ARG(ext), err); } } From 5f0c87970c9faa6104d5104b18f0448692ffc09a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 4 May 2018 16:31:36 -0700 Subject: [PATCH 625/920] scoutfs: fix level 0 key iteration increment Compaction has to find the oldest level 0 segment for compaction. It iterates over the level 0 segments by their manifest entry's btree key. It was incorrectly incrementing the btree search key. It was incrementing the first key stored in the entry, but that's not the least significant field. The seq is the least significant field so this iteration could skip over segments written at different times with the same first key. The fix to have it visit all the entries is to increment the lowest precision seq field. Right now we have a single level 0 segment so this code never actually matters. Signed-off-by: Zach Brown --- kmod/src/manifest.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index bc5d5d42..5ecd9f61 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -971,8 +971,7 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) if (next.seq < ment.seq) ment = next; - scoutfs_key_inc(&next.first); - init_btree_key(&mkey, next.level, next.seq, + init_btree_key(&mkey, next.level, next.seq + 1, &next.first); } From 345721c933f3188162107a8814cac71024e4b9ef Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 17 May 2018 09:45:11 -0700 Subject: [PATCH 626/920] scoutfs: preserve sticky deletion items We limit the number of lower segments that a compaction will read. A sticky compaction happens when the upper segment overlaps more lower segments. The remaining items in the upper segment are written back to the upper level -- they're stuck. A future compaction will attempt to compact the remaining items with the next set of overlapping lower segments. Deletion items are rightly discarded as they're compacted to the lowest level -- at that point they have no more matching items in lower segments to destroy and are done. Deletion items were being dropped instead of being written back into the upper level of a sticky compaction. The test for discarding the deletion items only considered the lowest level of the compaction, not the level that the items were being written to. We need to be careful to preserve the deletion items in the case of compaction to the lowest level writing sticky items back to the upper segment. Signed-off-by: Zach Brown --- kmod/src/compact.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index 3efb88ed..b18b11d6 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -250,8 +250,12 @@ retry: * all the duplicate items that they find. When we're * compacting to the last level we can remove them by retrying * the search after we've advanced past them. + * + * If we're filling the remaining items in a sticky merge into + * the upper level then we have to preserve the deletion items. */ if ((curs->lower_level == curs->last_level) && + (!curs->sticky || lower) && ((*item_flags) & SCOUTFS_ITEM_FLAG_DELETION)) goto retry; From e227c6446ee8cf032432a821307f6598335b9a66 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 23 May 2018 08:10:00 -0700 Subject: [PATCH 627/920] scoutfs: don't advance btree after wrapping The btree writes its blocks to a fixed ring of preallocated blocks. We added a trigger to force the index to advance to the next half of the ring to test conditions where the cached btree blocks are out of date with respect to the blocks on disk. We have to be careful to only advance the index once all the live blocks are migrated out of the half that we're about to advance to. The trigger tested that condition. But it missed the case where the normal btree block allocation *just* advanced into the next ring. In this case the migration needs to occur to make it safe to advance *again* to the previous half. But it missed this case because the migration keys are reset after we test the trigger. This resulted in leaving live btree blocks in the half that we advance to and start overwriting. The server got -ESTALE as it tried to read through blocks that had been overwritten and hilarity ensued. This precise condition of having the trigger fire just as we wrapped was amazingly caught by scoutfs/505 in xfstests. Signed-off-by: Zach Brown --- kmod/src/btree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 565474df..028c81c3 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -758,8 +758,8 @@ retry: if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) bring->next_block = 0; - /* advance to the next half when asked and migration made it safe */ - if (all_roots_migrated(super) && + /* force advancing if migration's done and we didn't just wrap */ + if (all_roots_migrated(super) && !first_block_in_half(bring) && scoutfs_trigger(sb, BTREE_ADVANCE_RING_HALF)) advance_to_next_half(bring); From 1c5d84fa3e11eb469da20058bd1468542874da10 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 Apr 2018 13:28:08 -0700 Subject: [PATCH 628/920] scoutfs: add counters for items written in level 0 Signed-off-by: Zach Brown --- kmod/src/counters.h | 7 ++++--- kmod/src/item.c | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 9d5db004..cc4fc7a8 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -115,11 +115,12 @@ EXPAND_COUNTER(trans_commit_sync_fs) \ EXPAND_COUNTER(trans_commit_timer) \ EXPAND_COUNTER(trans_level0_seg_write_bytes) \ - EXPAND_COUNTER(trans_level0_seg_writes) - + EXPAND_COUNTER(trans_level0_seg_writes) \ + EXPAND_COUNTER(trans_write_item) \ + EXPAND_COUNTER(trans_write_deletion_item) #define FIRST_COUNTER btree_read_error -#define LAST_COUNTER trans_level0_seg_writes +#define LAST_COUNTER trans_write_deletion_item #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/item.c b/kmod/src/item.c index 77cd5057..b6432f19 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1734,6 +1734,11 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) /* trans reservation should have limited dirty */ BUG_ON(!appended); + if (item->deletion) + scoutfs_inc_counter(sb, trans_write_deletion_item); + else + scoutfs_inc_counter(sb, trans_write_item); + clear_item_dirty(sb, cac, item); del = item; From 9c80f109d54958b26a287ec5807dbc55204c96d8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 Apr 2018 13:22:24 -0700 Subject: [PATCH 629/920] scoutfs: don't always write deletion items Items deleted from the item cache would always write deletion items to segments. We need to write deletion items so that compaction can eventually combine them with the existing item and remove both. We don't need them for items that were only created in the current transaction. Writing a deletion item for them only results in a lot of extra work compacting the item down to the final segment level so that it can be removed. The upcoming extent code really demonstrated the cost of this overhead. It happens to create and delete quite a lot of temporary extent items during the transaction as all the different kinds of indexed extents change. This change tracks whether a given item in the cache reflects an item that is present in the persistent storage. This lets us free items that have only existed in the current transaction. This made a meaningful difference when writing a 4MB file with the current block mapping items, but it made an enormous difference when writing that same file with the extent items. It went from writing 1024 deletion items for 11 real items to only writing those real items. items deletions block mappings before: 25 5 block mappings after: 25 0 extents before: 11 1024 extents after: 11 0 Signed-off-by: Zach Brown --- kmod/src/item.c | 81 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 16 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index b6432f19..6dc2731e 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -35,10 +35,13 @@ * negative lookups cache hits for items that don't exist without having * to constantly perform expensive segment searches. * - * Deletions are recorded with items in the rbtree which record the key - * of the deletion. They're removed once they're written to a level0 - * segment. While they're present in the cache we have to be careful to - * clobber them in creation and skip them in lookups. + * Deletions of persistent items are recorded with items in the rbtree + * which record the key of the deletion. They're removed once they're + * written to a level0 segment. While they're present in the cache we + * have to be careful to clobber them in creation and skip them in + * lookups. We only need deletion items for keys that exist in + * segments. We can immediately free non-persistent items when they're + * deleted. */ static bool invalid_key_val(struct scoutfs_key *key, struct kvec *val) @@ -68,13 +71,19 @@ struct item_cache { * The entry list_head typically stores clean items on an lru for shrinking. * It's also briefly used to track items in a batch after they're * allocated but before they're inserted for the first time. + * + * The persistent bit indicates that the item's key is present in + * segments. If we delete persistent items we have to write a deletion + * item to delete to remove the existing item. We can free deleted items + * that aren't persistent without writing them. */ struct cached_item { struct rb_node node; struct list_head entry; long dirty; - unsigned deletion:1; + unsigned deletion:1, + persistent:1; struct scoutfs_key key; void *val; @@ -419,13 +428,22 @@ static void erase_item(struct super_block *sb, struct item_cache *cac, } /* - * Turn an item that the caller has found while holding the lock into a - * deletion item. + * Delete an item from the cache. If it wasn't persistent we can just + * free the item. The caller must not try to use the item after calling + * this. + * + * If it was persistent we have to write a deletion item so that + * compaction will remove the old item. We only need the key for the + * deletion item so we can free the value. */ -static void become_deletion_item(struct super_block *sb, - struct item_cache *cac, - struct cached_item *item) +static void delete_item(struct super_block *sb, struct item_cache *cac, + struct cached_item *item) { + if (!item->persistent) { + erase_item(sb, cac, item); + return; + } + /* uses val_len to update item accounting */ clear_item_dirty(sb, cac, item); @@ -445,9 +463,11 @@ static void become_deletion_item(struct super_block *sb, * We distinguish between callers seeing trying to insert a new logical * item and others trying to populate the cache. * - * New logical item creaters have made sure the items are participating + * New logical item creators have made sure the items are participating * in consistent locking. It's safe for them to clobber dirty deletion - * items with a new version of the item. + * items with a new version of the item. The newly inserted item needs + * to retain the persistence of the item it replaces so that if it is later + * deleted it will still write a deletion item. * * Cache readers can only populate items that weren't present already. * In particular, they absolutely cannot replace dirty old inode index items @@ -487,6 +507,8 @@ restart: /* sadly there's no augmented replace */ erase_item(sb, cac, item); + if (item->persistent) + ins->persistent = 1; goto restart; } } @@ -1059,6 +1081,15 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, return ret; } +/* + * "force" an item creation without first reading to see if the item + * exist. The caller is asserting that they know it's correct to + * overwrite a possibly existing item with this newly created item. + * + * Because this can be overwriting an existing item we need to be sure + * that we write a deletion item if it's deleted so we force its + * persistent flag. + */ int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) @@ -1079,6 +1110,8 @@ int scoutfs_item_create_force(struct super_block *sb, if (!item) return -ENOMEM; + item->persistent = 1; + spin_lock_irqsave(&cac->lock, flags); ret = insert_item(sb, cac, item, true, false); @@ -1174,6 +1207,7 @@ int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, list_for_each_entry_safe(item, tmp, list, entry) { list_del_init(&item->entry); + item->persistent = 1; if (insert_item(sb, cac, item, false, true)) { scoutfs_inc_counter(sb, item_batch_duplicate); list_add(&item->entry, list); @@ -1329,7 +1363,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, item = find_item(sb, &cac->items, key); if (item) { - become_deletion_item(sb, cac, item); + delete_item(sb, cac, item); ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { ret = -ENOENT; @@ -1347,6 +1381,14 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, return ret; } +/* + * "force" a deletion by creating a deletion item without first reading + * the existing item. + * + * The caller knows that there is an existing item but doesn't want to + * pay the cost of reading it before writing a deletion item. We mark + * the allocated deletion item persistent to ensure that it's written. + */ int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock) @@ -1364,6 +1406,8 @@ int scoutfs_item_delete_force(struct super_block *sb, if (!item) return -ENOMEM; + item->persistent = 1; + spin_lock_irqsave(&cac->lock, flags); ret = insert_item(sb, cac, item, true, false); if (ret) { @@ -1375,7 +1419,7 @@ int scoutfs_item_delete_force(struct super_block *sb, scoutfs_inc_counter(sb, item_create); mark_item_dirty(sb, cac, item); - become_deletion_item(sb, cac, item); + delete_item(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); return ret; @@ -1426,9 +1470,10 @@ int scoutfs_item_delete_save(struct super_block *sb, if (was_dirty) item->dirty |= ITEM_DIRTY; + del->persistent = item->persistent; ret = insert_item(sb, cac, del, false, false); BUG_ON(ret); - become_deletion_item(sb, cac, del); + delete_item(sb, cac, del); del = NULL; ret = 0; } else if (check_range(sb, &cac->ranges, key, NULL)) { @@ -1523,7 +1568,7 @@ void scoutfs_item_delete_dirty(struct super_block *sb, item = find_item(sb, &cac->items, key); if (item) - become_deletion_item(sb, cac, item); + delete_item(sb, cac, item); spin_unlock_irqrestore(&cac->lock, flags); } @@ -1739,7 +1784,11 @@ int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) else scoutfs_inc_counter(sb, trans_write_item); + /* non-persistent should have been freed (safe to write) */ + WARN_ON_ONCE(item->deletion && !item->persistent); + clear_item_dirty(sb, cac, item); + item->persistent = 1; del = item; item = next_dirty(item); From 27d1f3bcf7bd7ac0daa308bab64d27af6c52fb40 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 5 Jun 2018 11:33:24 -0700 Subject: [PATCH 630/920] scoutfs: inode read shouldn't modify online blocks There was a typo in the addition of i_blocks tracking that would set online blocks to the value of offline blocks when reading an existing inode into memory. Signed-off-by: Zach Brown --- kmod/src/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 2ae48bb5..b873e33f 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -241,7 +241,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) * i_blocks is initialized from online and offline and is then * maintained as blocks come and go. */ - inode->i_blocks = (ci->online_blocks = + ci->offline_blocks) + inode->i_blocks = (ci->online_blocks + ci->offline_blocks) << SCOUTFS_BLOCK_SECTOR_SHIFT; set_item_info(ci, cinode); From 08a6fab7256634690e7bb87f51088de3512a6e26 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 7 May 2018 11:42:31 -0700 Subject: [PATCH 631/920] scoutfs: always trace item create/delete ret Add a trace event for item creation and always trace the return value of create and delete events. Signed-off-by: Zach Brown --- kmod/src/item.c | 25 ++++++++++++++++--------- kmod/src/scoutfs_trace.h | 4 ++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 6dc2731e..15f4d326 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1046,15 +1046,17 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, unsigned long flags; int ret; - if (invalid_key_val(key, val)) - return -EINVAL; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) - return -EINVAL; + if (invalid_key_val(key, val) || + WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + ret = -EINVAL; + goto out; + } item = alloc_item(sb, key, val); - if (!item) - return -ENOMEM; + if (!item) { + ret = -ENOMEM; + goto out; + } do { spin_lock_irqsave(&cac->lock, flags); @@ -1078,6 +1080,8 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, if (ret) free_item(sb, item); +out: + trace_scoutfs_item_create(sb, key, ret); return ret; } @@ -1355,8 +1359,10 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) - return -EINVAL; + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + ret = -EINVAL; + goto out; + } do { spin_lock_irqsave(&cac->lock, flags); @@ -1377,6 +1383,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, (ret = scoutfs_manifest_read_items(sb, key, &lock->start, &lock->end)) == 0); +out: trace_scoutfs_item_delete(sb, key, ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 29c3b3bf..d31c8d3d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -249,6 +249,10 @@ DECLARE_EVENT_CLASS(scoutfs_key_ret_class, __entry->fsid, SK_ARG(&__entry->key), __entry->ret) ); +DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_create, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), + TP_ARGS(sb, key, ret) +); DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_delete, TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), TP_ARGS(sb, key, ret) From 9c74f2011d43f5328b16a3a94f71814caff6ac06 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 31 May 2018 14:53:31 -0700 Subject: [PATCH 632/920] scoutfs: add server work tracing Add some server workqueue and work tracing to chase down the destruction of an active workqueue. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 45 ++++++++++++++++++++++++++++++++++++++++ kmod/src/server.c | 12 +++++++++++ 2 files changed, 57 insertions(+) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index d31c8d3d..0f88e423 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1759,6 +1759,51 @@ DEFINE_EVENT(scoutfs_net_class, scoutfs_client_recv_reply, TP_ARGS(sb, name, peer, nh) ); +DECLARE_EVENT_CLASS(scoutfs_work_class, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret), + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, data) + __field(int, ret) + ), + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->data = data; + __entry->ret = ret; + ), + TP_printk("fsid "FSID_FMT" data %llu ret %d", + __entry->fsid, __entry->data, __entry->ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_recv_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_recv_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_workqueue_destroy, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); + TRACE_EVENT(scoutfs_item_next_range_check, TP_PROTO(struct super_block *sb, int cached, struct scoutfs_key *key, struct scoutfs_key *pos, diff --git a/kmod/src/server.c b/kmod/src/server.c index e7c0c3dd..1290f75b 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -533,6 +533,8 @@ static void scoutfs_server_commit_func(struct work_struct *work) struct llist_node *node; int ret; + trace_scoutfs_server_commit_work_enter(sb, 0, 0); + down_write(&server->commit_rwsem); if (!scoutfs_btree_has_dirty(sb)) { @@ -577,6 +579,7 @@ out: } up_write(&server->commit_rwsem); + trace_scoutfs_server_commit_work_exit(sb, 0, ret); } /* @@ -1126,6 +1129,8 @@ static void scoutfs_server_recv_func(struct work_struct *work) struct kvec kv; int ret; + trace_scoutfs_server_recv_work_enter(sb, 0, 0); + req_wq = alloc_workqueue("scoutfs_server_requests", WQ_NON_REENTRANT, 0); if (!req_wq) { @@ -1221,6 +1226,8 @@ out: smp_mb(); wake_up_process(server->listen_task); mutex_unlock(&server->mutex); + + trace_scoutfs_server_recv_work_exit(sb, 0, ret); } /* @@ -1274,6 +1281,8 @@ static void scoutfs_server_func(struct work_struct *work) int optval; int ret; + trace_scoutfs_server_work_enter(sb, 0, 0); + init_waitqueue_head(&waitq); ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, @@ -1431,6 +1440,8 @@ out: /* always requeues, cancel_delayed_work_sync cancels on shutdown */ queue_delayed_work(server->wq, &server->dwork, HZ / 2); + + trace_scoutfs_server_work_exit(sb, 0, ret); } int scoutfs_server_setup(struct super_block *sb) @@ -1480,6 +1491,7 @@ void scoutfs_server_destroy(struct super_block *sb) /* recv work/compaction could have left commit_work queued */ cancel_work_sync(&server->commit_work); + trace_scoutfs_server_workqueue_destroy(sb, 0, 0); destroy_workqueue(server->wq); kfree(server); From dab0fd7d9a2aae70fcfb11e993e76d4150f5d5f2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 Jun 2018 14:05:34 -0700 Subject: [PATCH 633/920] scoutfs: update inode item after releasing The release ioctl forgot to update the inode item after truncating online block mappings. This meant that the offline block count update was lost when the inode was evicted and re-read, leading to inconsistent offline block counts. Signed-off-by: Zach Brown --- kmod/src/count.h | 6 +++++- kmod/src/data.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 95aed312..ae0c98ec 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -231,12 +231,16 @@ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) * - delete two existing free extents * - create a merged free extent */ -static inline const struct scoutfs_item_count SIC_TRUNC_EXTENT(void) +static inline const struct scoutfs_item_count +SIC_TRUNC_EXTENT(struct inode *inode) { struct scoutfs_item_count cnt = {0,}; unsigned int nr_file = 1 + 2 + 1; unsigned int nr_free = (2 + 1) * 2; + if (inode) + __count_dirty_inode(&cnt); + cnt.items += nr_file + nr_free; cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); diff --git a/kmod/src/data.c b/kmod/src/data.c index 478746a6..83c3bee8 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -325,19 +325,22 @@ out: * left behind. Only blocks that have been allocated can be marked * offline. * - * This is the low level extent item manipulation code. We hold and - * release the transaction so the caller doesn't have to deal with - * partial progress. - * * If the inode is provided then we update its tracking of the online * and offline blocks. If it's not provided then the inode is being - * destroyed and we don't have to keep it updated. + * destroyed and isn't reachable, we don't need to update it. + * + * The caller is in charge of locking the inode and extents, but we may + * have to modify far more items than fit in a transaction so we're in + * charge of batching updates into transactions. If the inode is + * provided then we're responsible for updating its item as we go. */ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, u64 ino, u64 iblock, u64 last, bool offline, struct scoutfs_lock *lock) { + struct scoutfs_item_count cnt = SIC_TRUNC_EXTENT(inode); DECLARE_DATA_INFO(sb, datinf); + LIST_HEAD(ind_locks); s64 ret = 0; WARN_ON_ONCE(inode && !mutex_is_locked(&inode->i_mutex)); @@ -352,15 +355,30 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return -EINVAL; while (iblock <= last) { - ret = scoutfs_hold_trans(sb, SIC_TRUNC_EXTENT()); + if (inode) + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, + true, cnt); + else + ret = scoutfs_hold_trans(sb, cnt); if (ret) break; + if (inode) + ret = scoutfs_dirty_inode_item(inode, lock); + else + ret = 0; + down_write(&datinf->alloc_rwsem); - ret = truncate_one_extent(sb, inode, ino, iblock, last, - offline, lock); + if (ret == 0) + ret = truncate_one_extent(sb, inode, ino, iblock, last, + offline, lock); up_write(&datinf->alloc_rwsem); + + if (inode) + scoutfs_update_inode_item(inode, lock, &ind_locks); scoutfs_release_trans(sb); + if (inode) + scoutfs_inode_index_unlock(sb, &ind_locks); if (ret <= 0) break; From 1fca13b092f127d05f5836bdf1dabfcc9730d0df Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 15 Jun 2018 15:16:48 -0700 Subject: [PATCH 634/920] scoutfs: add fallocate Add an fallocate operation. This changes the possible combinations of flags in extents and makes it possible to create extents beyond i_size. This will confuse the rest of the code in a few places and that will be fixed up next. Signed-off-by: Zach Brown --- kmod/src/count.h | 21 ++++ kmod/src/counters.h | 1 + kmod/src/data.c | 226 +++++++++++++++++++++++++++++++++++++++ kmod/src/data.h | 1 + kmod/src/format.h | 1 + kmod/src/scoutfs_trace.h | 29 +++++ 6 files changed, 279 insertions(+) diff --git a/kmod/src/count.h b/kmod/src/count.h index ae0c98ec..fc2f993e 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -247,4 +247,25 @@ SIC_TRUNC_EXTENT(struct inode *inode) return cnt; } +/* + * Fallocating an extent can, at most: + * - allocate from the server: delete two free and insert merged + * - free an allocated extent: delete one and create two split + * - remove an unallocated file extent: delete one and create two split + * - add an fallocated flie extent: delete two and inset one merged + */ +static inline const struct scoutfs_item_count SIC_FALLOCATE_ONE(void) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int nr_free = ((1 + 2) * 2) * 2; + unsigned int nr_file = (1 + 2) * 2; + + __count_dirty_inode(&cnt); + + cnt.items += nr_free + nr_file; + cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + + return cnt; +} + #endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index cc4fc7a8..b8432d56 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -27,6 +27,7 @@ EXPAND_COUNTER(corrupt_btree_no_child_ref) \ EXPAND_COUNTER(corrupt_data_extent_trunc_cleanup) \ EXPAND_COUNTER(corrupt_data_extent_alloc_cleanup) \ + EXPAND_COUNTER(corrupt_data_extent_fallocate_cleanup) \ EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 83c3bee8..706a9382 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -820,6 +821,230 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, return ret; } +/* + * Update one extent on behalf of fallocate. + * + * The caller has searched for the next extent that intersects with the + * region including first_block and last_block. The next extent will be + * zeroed if it wasn't found. We don't know the state of the offsets + * past the next extent. + * + * The caller has held transactions and acquired locks. We only ever + * make one extent modification here. + * + * If this returns 0 then the caller's extent is clobbered. It is set + * to the newly fallocated extent so that the caller can continue with + * the fallocate operation. + */ +static int fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, + u64 len, u8 flags, u8 rem_flags, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_extent fal; + struct scoutfs_extent rem; + struct scoutfs_extent fr; + bool add_rem = false; + bool add_fr = false; + int ret; + + if (WARN_ON_ONCE(len == 0) || + WARN_ON_ONCE(start + len < start)) { + ret = -EINVAL; + goto out; + } + + /* find a sufficiently large free extent */ + scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + sbi->node_id, 0, len, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &fr, + sbi->node_id_lock); + if (ret == -ENOENT) { + /* try to get allocation from the server if we're out */ + ret = get_server_extent(sb, SERVER_ALLOC_BLOCKS); + if (ret == 0) + ret = scoutfs_extent_next(sb, data_extent_io, &fr, + sbi->node_id_lock); + /* XXX try to find smaller free extents */ + } + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; + } + + /* trim our allocation from the length indexed extent */ + scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, fr.start, min(fr.len, len), 0, 0); + + ret = scoutfs_extent_init(&fal, SCOUTFS_FILE_EXTENT_TYPE, ino, + start, fr.len, fr.start, flags); + if (WARN_ON_ONCE(ret)) + goto out; + + ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); + if (ret) + goto out; + add_fr = true; + + /* remove a region of the existing extent */ + if (rem_flags) { + scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, + fal.start, fal.len, 0, rem_flags); + ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); + if (ret) + goto out; + add_rem = true; + } + + ret = scoutfs_extent_add(sb, data_extent_io, &fal, lock); +out: + scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, + data_extent_io, &rem, lock, + SC_DATA_EXTENT_FALLOCATE_CLEANUP, + corrupt_data_extent_fallocate_cleanup, &fal); + scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, + data_extent_io, &fr, sbi->node_id_lock, + SC_DATA_EXTENT_FALLOCATE_CLEANUP, + corrupt_data_extent_alloc_cleanup, &fal); + return ret; +} + +/* + * Modify the extents that map the blocks that store the len byte region + * starting at offset. + * + * The caller has only prevented freezing by entering a fs write + * context. We're responsible for all other locking and consistency. + */ +long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + DECLARE_DATA_INFO(sb, datinf); + struct scoutfs_extent ext; + LIST_HEAD(ind_locks); + u64 last_block; + u64 iblock; + u64 blocks; + loff_t end; + u8 rem_flags; + u8 flags; + int ret; + + mutex_lock(&inode->i_mutex); + + /* XXX support more flags */ + if (mode & ~(FALLOC_FL_KEEP_SIZE)) { + ret = -EOPNOTSUPP; + goto out; + } + + /* catch wrapping */ + if (offset + len < offset) { + ret = -EINVAL; + goto out; + } + + if (len == 0) { + ret = 0; + goto out; + } + + ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + inode, &lock); + if (ret) + goto out; + + inode_dio_wait(inode); + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (offset + len > i_size_read(inode))) { + ret = inode_newsize_ok(inode, offset + len); + if (ret) + goto out; + } + + iblock = offset >> SCOUTFS_BLOCK_SHIFT; + last_block = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; + + for (; iblock <= last_block; iblock = ext.start + ext.len) { + + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, + ino, iblock, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + if (ret < 0 && ret != -ENOENT) + goto out; + + blocks = last_block - iblock + 1; + flags = SEF_UNWRITTEN; + rem_flags = 0; + + if (ret == -ENOENT || ext.start > last_block) { + /* no next extent or past us, all remaining blocks */ + + } else if (iblock < ext.start) { + /* sparse region until next extent */ + blocks = min(blocks, ext.start - iblock); + + } else if (ext.map > 0) { + /* skip past an allocated extent */ + blocks = min(blocks, (ext.start + ext.len) - iblock); + iblock += blocks; + blocks = 0; + + } else { + /* allocating a portion of an unallocated extent */ + blocks = min(blocks, (ext.start + ext.len) - iblock); + flags |= ext.flags; + rem_flags = ext.flags; + /* XXX corruption; why'd we store map == flags == 0? */ + if (rem_flags == 0) { + ret = -EIO; + goto out; + } + } + + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_FALLOCATE_ONE()); + if (ret) + goto out; + + if (blocks > 0) { + down_write(&datinf->alloc_rwsem); + ret = fallocate_one_extent(sb, ino, iblock, blocks, + flags, rem_flags, lock); + up_write(&datinf->alloc_rwsem); + } + + if (ret == 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { + end = (iblock + blocks) << SCOUTFS_BLOCK_SHIFT; + if (end == 0 || end > offset + len) + end = offset + len; + if (end > i_size_read(inode)) + i_size_write(inode, end); + scoutfs_update_inode_item(inode, lock, &ind_locks); + } + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + + if (ret) + goto out; + + iblock += blocks; + } + ret = 0; +out: + scoutfs_unlock(sb, lock, DLM_LOCK_EX); + mutex_unlock(&inode->i_mutex); + + trace_scoutfs_data_fallocate(sb, ino, mode, offset, len, ret); + return ret; +} + + /* * Return all the file's extents whose blocks overlap with the caller's * byte region. We set _LAST on the last extent and _UNKNOWN on offline @@ -910,6 +1135,7 @@ const struct file_operations scoutfs_file_fops = { .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, .llseek = scoutfs_file_llseek, + .fallocate = scoutfs_fallocate, }; diff --git a/kmod/src/data.h b/kmod/src/data.h index c9214c5c..bd9f84fa 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -9,6 +9,7 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock); int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); +long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index f9a33b83..e3aa67d7 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -622,6 +622,7 @@ enum { SC_DATA_EXTENT_TRUNC_CLEANUP, SC_DATA_EXTENT_ALLOC_CLEANUP, SC_SERVER_EXTENT_CLEANUP, + SC_DATA_EXTENT_FALLOCATE_CLEANUP, SC_NR_SOURCES, }; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0f88e423..6fa4863b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -370,6 +370,35 @@ TRACE_EVENT(scoutfs_erase_item, TP_printk(FSID_FMT" erasing item %p", __entry->fsid, __entry->item) ); +TRACE_EVENT(scoutfs_data_fallocate, + TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, + loff_t len, int ret), + + TP_ARGS(sb, ino, mode, offset, len, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(int, mode) + __field(__u64, offset) + __field(__u64, len) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->mode = mode; + __entry->offset = offset; + __entry->len = len; + __entry->ret = ret; + ), + + TP_printk("fsid "FSID_FMT" ino %llu mode 0x%x offset %llu len %llu ret %d", + __entry->fsid, __entry->ino, __entry->mode, __entry->offset, + __entry->len, __entry->ret) +); + TRACE_EVENT(scoutfs_data_fiemap, TP_PROTO(struct super_block *sb, __u64 off, int i, __u64 blkno), From 600ecd9fad1d51ae4665661b107e0671242f7359 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 18 Jun 2018 16:28:07 -0700 Subject: [PATCH 635/920] scoutfs: adapt to fallcated extents The addition of fallocate() now means that offline extents can be unwritten and allocated and that extents can now be found outside of i_size. Truncating needs to know about the possible flag combinations, writing preallocation needs to know to update an existing extent or allocate up to the next extent, get_block can't map unwritten extents for read, extent conversion needs to also clear offline, and truncate needs to drop extents outside i_size even if truncating to the existing file size. Signed-off-by: Zach Brown --- kmod/src/data.c | 195 ++++++++++++++++++++++----------------- kmod/src/inode.c | 3 +- kmod/src/scoutfs_trace.h | 34 +++---- 3 files changed, 130 insertions(+), 102 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 706a9382..d2600cbe 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -259,8 +259,8 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, trace_scoutfs_data_truncate_remove(sb, &rem); - /* nothing to do if the extent's already offline */ - if (offline && (rem.flags & SEF_OFFLINE)) { + /* nothing to do if the extent's already offline and unallocated */ + if ((offline && (rem.flags & SEF_OFFLINE)) && !rem.map) { ret = 1; goto out; } @@ -276,7 +276,7 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, rem_fr = true; } - /* remove the mapping */ + /* remove the extent */ ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); if (ret) goto out; @@ -294,9 +294,9 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, if (rem.map && !(rem.flags & SEF_UNWRITTEN)) online_delta += -rem.len; - if (rem.flags & SEF_OFFLINE) + if (!offline && (rem.flags & SEF_OFFLINE)) offline_delta += -rem.len; - if (offline) + if (offline && !(rem.flags & SEF_OFFLINE)) offline_delta += ofl.len; scoutfs_inode_add_onoff(inode, online_delta, offline_delta); @@ -413,69 +413,75 @@ out: } /* - * Allocate a single block for the logical block offset in the file. - * The caller tells us if the block was offline or not. We modify the - * extent items and the caller will search for the resulting extent. + * The caller is writing to a logical block that doesn't have an + * allocated extent. * - * If we're writing to the final block of the file then we try to - * preallocate unwritten blocks past i_size for future extending writes - * to use. We only base this decision on the file size. Truncating - * down the size, unlink, or releasing all blocks in the file will - * remove these preallocated blocks. Truncating past them will preserve - * them and treat them as 0. + * We always allocate an extent starting at the logical block. The + * caller has considered overlapping and following extents and has given + * us a maximum length that we could safely allocate. Preallocation + * heuristics decide to use this length or only a single block. * - * This assumes that there can't be existing unwritten extents in the - * inode that would overlap with our allocations. Writes are serialized - * and the caller only calls us if an extent doesn't exist. Unwritten - * extents are only created adjacent to i_size extensions. The only way - * to pull i_size back behind unwritten extents is to truncate and it - * frees them. Corrupt disk images could have fragmented unwritten - * extents past i_size in inodes and that'd manifest as errors inserting - * overlapping new allocations. + * If the caller passes in an existing extent then we remove the + * allocated region from the existing extent. We then add a single + * block extent for the caller to write into. Then if we allocated + * multiple blocks we add an unwritten extent for the rest of the blocks + * in the extent. + * + * Preallocation is used if we're strictly contiguously extending + * writes. That is, if the logical block offset equals the number of + * online blocks. We try to preallocate the number of blocks existing + * so that small files don't waste inordinate amounts of space and large + * files will eventually see large extents. This only works for + * contiguous single stream writes or stages of files from the first + * block. It doesn't work for concurrent stages, releasing behind + * staging, sparse files, multi-node writes, etc. fallocate() is always + * a better tool to use. + * + * On success we update the caller's extent to the single block + * allocated extent for the logical block for use in block mapping. */ -#define MAX_UNWRITTEN_BLOCKS ((u64)SCOUTFS_SEGMENT_BLOCKS) -#define SERVER_ALLOC_BLOCKS (MAX_UNWRITTEN_BLOCKS * 32) -static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, - bool was_offline, struct scoutfs_lock *lock) +#define MAX_STREAMING_PREALLOC_BLOCKS ((u64)SCOUTFS_SEGMENT_BLOCKS) +#define SERVER_ALLOC_BLOCKS (MAX_STREAMING_PREALLOC_BLOCKS * 32) +static int alloc_block(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 iblock, u64 len, + struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); const u64 ino = scoutfs_ino(inode); struct scoutfs_extent unwr; - struct scoutfs_extent ext; - struct scoutfs_extent ofl; + struct scoutfs_extent old; struct scoutfs_extent blk; struct scoutfs_extent fr; - bool add_ofl = false; + bool add_old = false; bool add_fr = false; bool rem_blk = false; u64 offline; u64 online; - u64 len; int ret; down_write(&datinf->alloc_rwsem); scoutfs_inode_get_onoff(inode, &online, &offline); - /* exponentially prealloc unwritten extents to a limit */ - if (iblock > 1 && iblock == (online + offline)) - len = min(iblock, MAX_UNWRITTEN_BLOCKS); + /* strictly contiguous extending writes will try to preallocate */ + if (iblock > 1 && iblock == online) + len = min3(len, iblock, MAX_STREAMING_PREALLOC_BLOCKS); else len = 1; - trace_scoutfs_data_alloc_block(sb, inode, iblock, was_offline, - online, offline, len); + trace_scoutfs_data_alloc_block(sb, inode, ext, iblock, len, + online, offline); - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, sbi->node_id, 0, len, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, + ret = scoutfs_extent_next(sb, data_extent_io, &fr, sbi->node_id_lock); if (ret == -ENOENT) { /* try to get allocation from the server if we're out */ ret = get_server_extent(sb, SERVER_ALLOC_BLOCKS); if (ret == 0) - ret = scoutfs_extent_next(sb, data_extent_io, &ext, + ret = scoutfs_extent_next(sb, data_extent_io, &fr, sbi->node_id_lock); } if (ret) { @@ -485,28 +491,28 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, goto out; } - trace_scoutfs_data_alloc_block_next(sb, &ext); + trace_scoutfs_data_alloc_block_next(sb, &fr); /* initialize the new mapped block extent, referenced by cleanup */ scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, ext.start, 0); + iblock, 1, fr.start, 0); /* remove the free extent we're using */ scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, ext.start, len, 0, 0); + sbi->node_id, fr.start, len, 0, 0); ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); if (ret) goto out; add_fr = true; - /* remove an offline block extent */ - if (was_offline) { - scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, 0, SEF_OFFLINE); - ret = scoutfs_extent_remove(sb, data_extent_io, &ofl, lock); + /* remove an existing offline or unwritten block extent */ + if (ext->flags) { + scoutfs_extent_init(&old, SCOUTFS_FILE_EXTENT_TYPE, ino, + iblock, len, 0, ext->flags); + ret = scoutfs_extent_remove(sb, data_extent_io, &old, lock); if (ret) goto out; - add_ofl = true; + add_old = true; } /* add the block that the caller is writing */ @@ -518,22 +524,23 @@ static int alloc_block(struct super_block *sb, struct inode *inode, u64 iblock, /* and maybe add the remaining unwritten extent */ if (len > 1) { scoutfs_extent_init(&unwr, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock + 1, len - 1, ext.start + 1, - SEF_UNWRITTEN); + iblock + 1, len - 1, fr.start + 1, + ext->flags | SEF_UNWRITTEN); ret = scoutfs_extent_add(sb, data_extent_io, &unwr, lock); if (ret) goto out; } - scoutfs_inode_add_onoff(inode, 1, was_offline ? -1ULL : 0); + scoutfs_inode_add_onoff(inode, 1, + (ext->flags & SEF_OFFLINE) ? -1ULL : 0); ret = 0; out: scoutfs_extent_cleanup(ret < 0 && rem_blk, scoutfs_extent_remove, sb, data_extent_io, &blk, lock, SC_DATA_EXTENT_ALLOC_CLEANUP, corrupt_data_extent_alloc_cleanup, &blk); - scoutfs_extent_cleanup(ret < 0 && add_ofl, scoutfs_extent_add, sb, - data_extent_io, &ofl, lock, + scoutfs_extent_cleanup(ret < 0 && add_old, scoutfs_extent_add, sb, + data_extent_io, &old, lock, SC_DATA_EXTENT_ALLOC_CLEANUP, corrupt_data_extent_alloc_cleanup, &blk); scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, @@ -543,21 +550,22 @@ out: up_write(&datinf->alloc_rwsem); - trace_scoutfs_data_alloc_block_ret(sb, ret); + trace_scoutfs_data_alloc_block_ret(sb, ext, ret); + if (ret == 0) + *ext = blk; return ret; } /* - * Remove the unwritten flag from an existing extent. We don't have to - * wait for dirty block IO to complete before clearing the unwritten - * flag in metadata because we have strict synchronization between data - * and metadata. All dirty data in the current transaction is written - * before the metadata in the transaction that references it is - * committed. + * A caller is writing into unwritten allocated space. This can also be + * called for staging writes so we clear both the unwritten and offline + * flags. We record the extent as online as allocating writes would. * - * The extent is unwritten so it can't be offline nor online. We remove - * the unwritten flag, possibly splitting and merging. We record the - * extent as online now as initial block allocation would. + * We don't have to wait for dirty block IO to complete before clearing + * the unwritten flag in metadata because we have strict synchronization + * between data and metadata. All dirty data in the current transaction + * is written before the metadata in the transaction that references it + * is committed. */ static int convert_unwritten(struct super_block *sb, struct inode *inode, struct scoutfs_extent *ext, u64 start, u64 len, @@ -577,18 +585,20 @@ static int convert_unwritten(struct super_block *sb, struct inode *inode, if (ret) goto out; - conv.flags &= ~SEF_UNWRITTEN; + conv.flags &= ~(SEF_UNWRITTEN | SEF_OFFLINE); ret = scoutfs_extent_add(sb, data_extent_io, &conv, lock); if (ret) { - conv.flags |= SEF_UNWRITTEN; + conv.flags = ext->flags; err = scoutfs_extent_add(sb, data_extent_io, &conv, lock); BUG_ON(err); goto out; } + scoutfs_inode_add_onoff(inode, len, + (ext->flags & SEF_OFFLINE) ? -len : 0); + *ext = conv; ret = 0; out: - scoutfs_inode_add_onoff(inode, len, 0); return ret; } @@ -597,18 +607,23 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + struct scoutfs_lock *lock = NULL; struct scoutfs_extent ext; - struct scoutfs_lock *lock; + u64 next_iblock = 0; u64 offset; + u64 len; int ret; WARN_ON_ONCE(create && !mutex_is_locked(&inode->i_mutex)); + /* make sure caller holds a cluster lock */ lock = scoutfs_per_task_get(&si->pt_data_lock); - if (WARN_ON_ONCE(!lock)) - return -EINVAL; + if (WARN_ON_ONCE(!lock) || + WARN_ON_ONCE(!create && si->staging)) { + ret = -EINVAL; + goto out; + } -restart: /* look for the extent that overlaps our iblock */ scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, scoutfs_ino(inode), iblock, 1, 0, 0); @@ -616,8 +631,12 @@ restart: if (ret && ret != -ENOENT) goto out; - if (ret == 0) + if (ret == 0) { trace_scoutfs_data_get_block_next(sb, &ext); + /* remember start of next to limit preallocation */ + if (ext.start > iblock) + next_iblock = ext.start; + } /* didn't find an extent or it's past our iblock */ if (ret == -ENOENT || ext.start > iblock) @@ -635,31 +654,37 @@ restart: /* convert unwritten to written */ if (create && (ext.flags & SEF_UNWRITTEN)) { ret = convert_unwritten(sb, inode, &ext, iblock, 1, lock); - if (ret) - goto out; - goto restart; + if (ret == 0) + set_buffer_new(bh); + goto out; } - /* try to allocate if we're writing */ + /* allocate an extent from our logical block */ if (create && !ext.map) { - ret = alloc_block(sb, inode, iblock, ext.flags & SEF_OFFLINE, - lock); - if (ret) - goto out; - set_buffer_new(bh); - /* restart the search now that it's been allocated */ - goto restart; + /* limit possible alloc to this extent, next, or logical max */ + if (ext.len > 0) + len = ext.len - (iblock - ext.start); + else if (next_iblock > iblock) + len = ext.start - iblock; + else + len = SCOUTFS_BLOCK_MAX - iblock; + + ret = alloc_block(sb, inode, &ext, iblock, len, lock); + if (ret == 0) + set_buffer_new(bh); + } else { + ret = 0; } - /* map the bh and set the size to as much of the extent as we can */ - if (ext.map) { +out: + /* map usable extent, else leave bh unmapped for sparse reads */ + if (ret == 0 && ext.map && !(ext.flags & SEF_UNWRITTEN)) { offset = iblock - ext.start; map_bh(bh, inode->i_sb, ext.map + offset); bh->b_size = min_t(u64, bh->b_size, (ext.len - offset) << SCOUTFS_BLOCK_SHIFT); } - ret = 0; -out: + trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, ret, bh->b_blocknr, bh->b_size); return ret; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index b873e33f..20676cee 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -427,7 +427,8 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) if (ret) goto out; - truncate = i_size_read(inode) > attr_size; + /* truncating to current size truncates extents past size */ + truncate = i_size_read(inode) >= attr_size; ret = set_inode_size(inode, lock, attr_size, truncate); if (ret) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 6fa4863b..9eb5a75e 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -455,55 +455,57 @@ TRACE_EVENT(scoutfs_get_block, ); TRACE_EVENT(scoutfs_data_alloc_block, - TP_PROTO(struct super_block *sb, struct inode *inode, u64 iblock, - bool was_offline, u64 online_blocks, u64 offline_blocks, - u64 len), + TP_PROTO(struct super_block *sb, struct inode *inode, + struct scoutfs_extent *ext, u64 iblock, u64 len, + u64 online_blocks, u64 offline_blocks), - TP_ARGS(sb, inode, iblock, was_offline, online_blocks, offline_blocks, - len), + TP_ARGS(sb, inode, ext, iblock, len, online_blocks, offline_blocks), TP_STRUCT__entry( __field(__u64, fsid) __field(__u64, ino) + __field_struct(struct scoutfs_extent, ext) __field(__u64, iblock) - __field(__u8, was_offline) + __field(__u64, len) __field(__u64, online_blocks) __field(__u64, offline_blocks) - __field(__u64, len) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->ino = scoutfs_ino(inode); + __entry->ext = *ext; __entry->iblock = iblock; - __entry->was_offline = was_offline; + __entry->len = len; __entry->online_blocks = online_blocks; __entry->offline_blocks = offline_blocks; - __entry->len = len; ), - TP_printk("fsid "FSID_FMT" ino %llu iblock %llu was_offline %u online_blocks %llu offline_blocks %llu len %llu", - __entry->fsid, __entry->ino, __entry->iblock, - __entry->was_offline, __entry->online_blocks, - __entry->offline_blocks, __entry->len) + TP_printk("fsid "FSID_FMT" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", + __entry->fsid, __entry->ino, SE_ARG(&__entry->ext), + __entry->iblock, __entry->len, __entry->online_blocks, + __entry->offline_blocks) ); TRACE_EVENT(scoutfs_data_alloc_block_ret, - TP_PROTO(struct super_block *sb, int ret), + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext, int ret), - TP_ARGS(sb, ret), + TP_ARGS(sb, ext, ret), TP_STRUCT__entry( __field(__u64, fsid) + __field_struct(struct scoutfs_extent, ext) __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); + __entry->ext = *ext; __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(FSID_FMT" ext "SE_FMT" ret %d", __entry->fsid, + SE_ARG(&__entry->ext), __entry->ret) ); TRACE_EVENT(scoutfs_data_find_alloc_block_curs, From d53ec115bcbbdd18d83706e498ddf572602d88dd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 20 Jun 2018 15:31:24 -0700 Subject: [PATCH 636/920] scoutfs: add scoutfs_item_prev() Add scoutfs_item_prev() for searching for an item before a given key. This wasn't initially implemented because it's rarely needed and for a long time the segment reading and item cache populating code had a strong bias for iterating forward from the given search key. Since we've added limiting item cache reading to the keys covered by locks and reading in entire segments it's now very easy to iterate backwards through keys just like scoutfs_item_next() iterates forwards. The only remaining forward iteration bias was in check_range(). It had to give callers the start of the cached range that it found. Signed-off-by: Zach Brown --- kmod/src/item.c | 233 ++++++++++++++++++++++++++++++++++----- kmod/src/item.h | 3 + kmod/src/scoutfs_trace.h | 48 ++++++++ 3 files changed, 255 insertions(+), 29 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 15f4d326..60c461fb 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -242,6 +242,15 @@ static struct cached_item *next_item(struct rb_root *root, return walk_items(root, key, &prev, &next) ?: next; } +static struct cached_item *prev_item(struct rb_root *root, + struct scoutfs_key *key) +{ + struct cached_item *prev; + struct cached_item *next; + + return walk_items(root, key, &prev, &next) ?: prev; +} + /* * We store the dirty bits in a single value so that the simple * augmented rbtree implementation gets a single scalar value to compare @@ -581,36 +590,35 @@ static struct cached_range *walk_ranges(struct rb_root *root, } /* - * Return true if the given key is covered by a cached range. end is - * set to the end of the cached range. + * Return true if the given key is covered by a cached range. start and + * end are set to the existing cached range. * - * Return false if the given key isn't covered by a cached range and is - * instead in an uncached hole. end is set to the start of the next - * cached range. + * Return false if the key is not covered by a range. start and end are + * set to zero. (Nothing uses these today, this is to avoid tracing + * uninitialized keys in this case.) */ static bool check_range(struct super_block *sb, struct rb_root *root, - struct scoutfs_key *key, + struct scoutfs_key *key, struct scoutfs_key *start, struct scoutfs_key *end) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct item_cache *cac = sbi->item_cache; - struct cached_range *next; struct cached_range *rng; - rng = walk_ranges(&cac->ranges, key, NULL, &next); + rng = walk_ranges(&cac->ranges, key, NULL, NULL); if (rng) { scoutfs_inc_counter(sb, item_range_hit); + if (start) + *start = rng->start; if (end) *end = rng->end; return true; } - if (end) { - if (next) - *end = next->start; - else - scoutfs_key_set_ones(end); - } + if (start) + scoutfs_key_set_zeros(start); + if (end) + scoutfs_key_set_zeros(end); scoutfs_inc_counter(sb, item_range_miss); return false; @@ -823,7 +831,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, ret = copy_item_val(val, item); else ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL)) { + } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -929,14 +937,14 @@ static struct cached_item *item_for_next(struct rb_root *root, * Return the next item starting with the given key and returning the * last key at most. * - * If the end key is specified then it limits items that can be read - * into the cache. If it's less than the last key then it also limits - * iteration. These are different values because locking granularity - * can be smaller or larger than the iteration. Callers shouldn't have - * to be aware of that relationship. + * The range covered by the lock also limits the last item that can be + * returned. -ENOENT can be returned when there are no next items + * covered by the lock but there are still items before the last key + * outside of the lock. The caller needs to know to reacquire the next + * lock to continue iteration. * - * -ENOENT is returned if there are no items between the given and - * last/end keys. + * -ENOENT is returned if there are no items between the given and last + * keys inside the range covered by the lock. * * The next item's key is copied to the caller's key. The caller is * responsible for dealing with key lengths and truncation. @@ -979,7 +987,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, for(;;) { /* see if we have cache coverage of our iterator pos */ - cached = check_range(sb, &cac->ranges, &pos, &range_end); + cached = check_range(sb, &cac->ranges, &pos, NULL, &range_end); trace_scoutfs_item_next_range_check(sb, !!cached, key, &pos, last, &lock->end, @@ -1033,6 +1041,173 @@ out: return ret; } +/* + * Return the prev linked node in the tree that isn't a deletion item + * and which is still within the first allowed key value. + */ +static struct cached_item *prev_item_node(struct rb_root *root, + struct cached_item *item, + struct scoutfs_key *first) +{ + struct rb_node *node; + + while (item) { + node = rb_prev(&item->node); + if (!node) { + item = NULL; + break; + } + + item = container_of(node, struct cached_item, node); + + if (scoutfs_key_compare(&item->key, first) < 0) { + item = NULL; + break; + } + + if (!item->deletion) + break; + } + + return item; +} + +/* + * Find the prev item to return from the "_prev" item interface. It's the + * prev item from the key that isn't a deletion item and is within the + * bounds of the start of the cache and the caller's first key. + */ +static struct cached_item *item_for_prev(struct rb_root *root, + struct scoutfs_key *key, + struct scoutfs_key *range_start, + struct scoutfs_key *first) +{ + struct cached_item *item; + + /* limit by the greater of the two */ + if (range_start && scoutfs_key_compare(range_start, first) > 0) + first = range_start; + + item = prev_item(root, key); + if (item) { + if (scoutfs_key_compare(&item->key, first) < 0) + item = NULL; + else if (item->deletion) + item = prev_item_node(root, item, first); + } + + return item; +} + +/* + * Return the prev item starting with the given key and returning the + * first key at least. + * + * The range covered by the lock also limits the first item that can be + * returned. -ENOENT can be returned when there are no prev items + * covered by the lock but there are still items after the first key + * outside of the lock. The caller needs to know to reacquire the next + * lock to continue iteration. + * + * -ENOENT is returned if there are no items between the given and + * first key inside the range covered by the lock. + * + * The prev item's key is copied to the caller's key. The caller is + * responsible for dealing with key lengths and truncation. + * + * The prev item's value is copied into the callers value. The number + * of value bytes copied is returned. The copied value can be truncated + * by the caller's value buffer length. + */ +int scoutfs_item_prev(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *first, struct kvec *val, + struct scoutfs_lock *lock) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache *cac = sbi->item_cache; + struct scoutfs_key range_start; + struct scoutfs_key pos; + struct cached_item *item; + unsigned long flags; + bool cached; + int ret; + + /* use the start key as the first key if it's closer */ + if (scoutfs_key_compare(&lock->start, first) > 0) + first = &lock->start; + + /* convenience to avoid searching if caller iterates past their last */ + if (scoutfs_key_compare(key, first) < 0) { + ret = -ENOENT; + goto out; + } + + if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) { + ret = -EINVAL; + goto out; + } + + pos = *key; + + spin_lock_irqsave(&cac->lock, flags); + + for(;;) { + /* see if we have cache coverage of our iterator pos */ + cached = check_range(sb, &cac->ranges, &pos, + &range_start, NULL); + + trace_scoutfs_item_prev_range_check(sb, !!cached, key, + &pos, first, &lock->start, + &range_start); + + if (!cached) { + /* populate missing cached range starting at pos */ + spin_unlock_irqrestore(&cac->lock, flags); + + ret = scoutfs_manifest_read_items(sb, &pos, + &lock->start, + &lock->end); + + spin_lock_irqsave(&cac->lock, flags); + if (ret) + break; + else + continue; + } + + /* see if there's an item in the cached range from pos */ + item = item_for_prev(&cac->items, &pos, &range_start, first); + if (!item) { + if (scoutfs_key_compare(&range_start, first) > 0) { + /* keep searching before empty cached range */ + pos = range_start; + scoutfs_key_dec(&pos); + continue; + } + + /* no item and cache covers first, done */ + ret = -ENOENT; + break; + } + + /* we have a prev item inside the cached range, done */ + *key = item->key; + if (val) { + item_referenced(cac, item); + ret = copy_item_val(val, item); + } else { + ret = 0; + } + break; + } + + spin_unlock_irqrestore(&cac->lock, flags); +out: + + trace_scoutfs_item_prev_ret(sb, ret); + return ret; +} + /* * Create a new dirty item in the cache. Returns -EEXIST if an item * already exists with the given key. @@ -1061,7 +1236,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, do { spin_lock_irqsave(&cac->lock, flags); - if (!check_range(sb, &cac->ranges, key, NULL)) { + if (!check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENODATA; } else { ret = insert_item(sb, cac, item, false, false); @@ -1263,7 +1438,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, if (item) { mark_item_dirty(sb, cac, item); ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL)) { + } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1320,7 +1495,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, item->val_len = val ? val->iov_len : 0; mark_item_dirty(sb, cac, item); ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL)) { + } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1371,7 +1546,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, if (item) { delete_item(sb, cac, item); ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL)) { + } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1483,7 +1658,7 @@ int scoutfs_item_delete_save(struct super_block *sb, delete_item(sb, cac, del); del = NULL; ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL)) { + } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { ret = -ENOENT; } else { ret = -ENODATA; @@ -1532,7 +1707,7 @@ int scoutfs_item_restore(struct super_block *sb, struct list_head *list, mode = item_is_dirty(item) ? DLM_LOCK_EX : DLM_LOCK_PR; if (WARN_ON_ONCE(!lock_coverage(lock, &item->key, mode)) || WARN_ON_ONCE(!check_range(sb, &cac->ranges, &item->key, - NULL))) { + NULL, NULL))) { ret = -EINVAL; goto out; } diff --git a/kmod/src/item.h b/kmod/src/item.h index 328345b9..c281b895 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -14,6 +14,9 @@ int scoutfs_item_lookup_exact(struct super_block *sb, int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *last, struct kvec *val, struct scoutfs_lock *lock); +int scoutfs_item_prev(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *first, struct kvec *val, + struct scoutfs_lock *lock); int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_item_create_force(struct super_block *sb, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 9eb5a75e..b665dbab 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -352,6 +352,24 @@ TRACE_EVENT(scoutfs_item_next_ret, TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) ); +TRACE_EVENT(scoutfs_item_prev_ret, + TP_PROTO(struct super_block *sb, int ret), + + TP_ARGS(sb, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ret = ret; + ), + + TP_printk("fsid "FSID_FMT" ret %d", __entry->fsid, __entry->ret) +); + TRACE_EVENT(scoutfs_erase_item, TP_PROTO(struct super_block *sb, void *item), @@ -1865,6 +1883,36 @@ TRACE_EVENT(scoutfs_item_next_range_check, SK_ARG(&__entry->end), SK_ARG(&__entry->range_end)) ); +TRACE_EVENT(scoutfs_item_prev_range_check, + TP_PROTO(struct super_block *sb, int cached, + struct scoutfs_key *key, struct scoutfs_key *pos, + struct scoutfs_key *first, struct scoutfs_key *start, + struct scoutfs_key *range_start), + TP_ARGS(sb, cached, key, pos, first, start, range_start), + TP_STRUCT__entry( + __field(void *, sb) + __field(int, cached) + __field_struct(struct scoutfs_key, key) + __field_struct(struct scoutfs_key, pos) + __field_struct(struct scoutfs_key, first) + __field_struct(struct scoutfs_key, start) + __field_struct(struct scoutfs_key, range_start) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->cached = cached; + scoutfs_key_copy_or_zeros(&__entry->key, key); + scoutfs_key_copy_or_zeros(&__entry->pos, pos); + scoutfs_key_copy_or_zeros(&__entry->first, first); + scoutfs_key_copy_or_zeros(&__entry->start, start); + scoutfs_key_copy_or_zeros(&__entry->range_start, range_start); + ), + TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" first "SK_FMT" start "SK_FMT" range_start "SK_FMT, + __entry->sb, __entry->cached, SK_ARG(&__entry->key), + SK_ARG(&__entry->pos), SK_ARG(&__entry->first), + SK_ARG(&__entry->start), SK_ARG(&__entry->range_start)) +); + DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), TP_ARGS(sb, nr_to_scan, ret), From 04660dbfee44ff4e1737b536faf68db273c75c1b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 20 Jun 2018 15:36:37 -0700 Subject: [PATCH 637/920] scoutfs: add scoutfs_extent_prev() Add an extent function for iterating backwards through extents. We add the wrapper and have the extent IO functions call their storage _prev functions. Data extent IO can now call the new scoutfs_item_prev(). Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/data.c | 12 ++++++++++-- kmod/src/extents.c | 13 +++++++++++++ kmod/src/extents.h | 3 +++ kmod/src/scoutfs_trace.h | 8 ++++++++ kmod/src/server.c | 10 +++++++--- 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index b8432d56..6d26c4f7 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -56,6 +56,7 @@ EXPAND_COUNTER(extent_delete) \ EXPAND_COUNTER(extent_insert) \ EXPAND_COUNTER(extent_next) \ + EXPAND_COUNTER(extent_prev) \ EXPAND_COUNTER(extent_remove) \ EXPAND_COUNTER(item_alloc) \ EXPAND_COUNTER(item_batch_duplicate) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index d2600cbe..50196f37 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -139,6 +139,7 @@ static int data_extent_io(struct super_block *sb, int op, { struct scoutfs_lock *lock = data; struct scoutfs_file_extent fex; + struct scoutfs_key first; struct scoutfs_key last; struct scoutfs_key key; struct kvec val; @@ -164,6 +165,7 @@ static int data_extent_io(struct super_block *sb, int op, if (ext->type == SCOUTFS_FILE_EXTENT_TYPE) { init_file_extent_key(&key, ext->owner, ext->start + ext->len - 1); + init_file_extent_key(&first, ext->owner, 0); init_file_extent_key(&last, ext->owner, U64_MAX); fex.blkno = cpu_to_le64(ext->map); fex.len = cpu_to_le64(ext->len); @@ -174,14 +176,20 @@ static int data_extent_io(struct super_block *sb, int op, ext->start + ext->len - 1, ext->len); if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) swap(key.sknf_major, key.sknf_minor); + init_free_extent_key(&first, ext->type, ext->owner, + 0, 0); init_free_extent_key(&last, ext->type, ext->owner, U64_MAX, U64_MAX); kvec_init(&val, NULL, 0); } - if (op == SEI_NEXT) { + if (op == SEI_NEXT || op == SEI_PREV) { expected = val.iov_len; - ret = scoutfs_item_next(sb, &key, &last, &val, lock); + + if (op == SEI_NEXT) + ret = scoutfs_item_next(sb, &key, &last, &val, lock); + else + ret = scoutfs_item_prev(sb, &key, &first, &val, lock); if (ret >= 0 && ret != expected) ret = -EIO; if (ret == expected) diff --git a/kmod/src/extents.c b/kmod/src/extents.c index bbb18c71..c5bacdfd 100644 --- a/kmod/src/extents.c +++ b/kmod/src/extents.c @@ -153,6 +153,19 @@ int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, return ret; } +int scoutfs_extent_prev(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ext, void *data) +{ + int ret; + + scoutfs_inc_counter(sb, extent_prev); + trace_scoutfs_extent_prev_input(sb, ext); + ret = iof(sb, SEI_PREV, ext, data); + if (ret == 0) + trace_scoutfs_extent_prev_output(sb, ext); + return ret; +} + /* * Search for a next extent and see if we can merge it with the caller's * extent. The caller has initialized next for us to search from. If diff --git a/kmod/src/extents.h b/kmod/src/extents.h index d97f1f32..2cba23ab 100644 --- a/kmod/src/extents.h +++ b/kmod/src/extents.h @@ -20,6 +20,7 @@ struct scoutfs_extent { enum { SEI_NEXT, + SEI_PREV, SEI_INSERT, SEI_DELETE, }; @@ -33,6 +34,8 @@ bool scoutfs_extent_intersection(struct scoutfs_extent *a, int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, struct scoutfs_extent *ext, void *data); +int scoutfs_extent_prev(struct super_block *sb, scoutfs_extent_io_t iof, + struct scoutfs_extent *ext, void *data); int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, struct scoutfs_extent *add, void *data); int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b665dbab..87495177 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2222,6 +2222,14 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_output, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_input, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_output, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_add, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) diff --git a/kmod/src/server.c b/kmod/src/server.c index 1290f75b..d830d53e 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -164,9 +164,13 @@ static int server_extent_io(struct super_block *sb, int op, if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) swap(ebk.major, ebk.minor); - if (op == SEI_NEXT) { - ret = scoutfs_btree_next(sb, &super->alloc_root, - &ebk, sizeof(ebk), &iref); + if (op == SEI_NEXT || op == SEI_PREV) { + if (op == SEI_NEXT) + ret = scoutfs_btree_next(sb, &super->alloc_root, + &ebk, sizeof(ebk), &iref); + else + ret = scoutfs_btree_prev(sb, &super->alloc_root, + &ebk, sizeof(ebk), &iref); if (ret == 0) { ret = init_extent_from_btree_key(ext, ext->type, iref.key, From 2efba47b77b30407862f219fd9bf6ac25fb7ff30 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 20 Jun 2018 15:55:27 -0700 Subject: [PATCH 638/920] scoutfs: satisfy large allocs with smaller extents The previous fallocate and get_block allocators only looked for free extents larger than the requested allocation size. This prematurely returns -ENOSPC if a very large allocation is attempted. Some xfstests stress low free space situations by fallocating almost all the free space in the volume. This adds an allocation helper function that finds the biggest free extent to satisfy an allocation, psosibly after trying to get more free extents from the server. It looks for previous extents in the index of extents by length. This builds on the previously added item and extent _prev operations. Allocators need to then know the size of the allocation they got instead of assuming they got what they asked for. The server can also return a smaller extent so it needs to communicate the extent length, not just its start. Signed-off-by: Zach Brown --- kmod/src/client.c | 22 ++++-- kmod/src/client.h | 3 +- kmod/src/data.c | 165 +++++++++++++++++++++++---------------- kmod/src/format.h | 5 ++ kmod/src/scoutfs_trace.h | 4 + kmod/src/server.c | 30 ++++--- 6 files changed, 143 insertions(+), 86 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index c3c0bb91..fe837420 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -555,20 +555,28 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, return ret; } -int scoutfs_client_alloc_extent(struct super_block *sb, u64 len, u64 *start) +/* + * Ask the server for an extent of at most @blocks blocks. It can return + * smaller extents. + */ +int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, + u64 *len) + { struct client_info *client = SCOUTFS_SB(sb)->client_info; - __le64 lelen = cpu_to_le64(len); - __le64 lestart; + __le64 leblocks = cpu_to_le64(blocks); + struct scoutfs_net_extent nex; int ret; ret = client_request(client, SCOUTFS_NET_ALLOC_EXTENT, - &lelen, sizeof(lelen), &lestart, sizeof(lestart)); + &leblocks, sizeof(leblocks), &nex, sizeof(nex)); if (ret == 0) { - if (lestart == 0) + if (nex.len == 0) { ret = -ENOSPC; - else - *start = le64_to_cpu(lestart); + } else { + *start = le64_to_cpu(nex.start); + *len = le64_to_cpu(nex.len); + } } return ret; diff --git a/kmod/src/client.h b/kmod/src/client.h index aa098f02..259454dd 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -3,7 +3,8 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); -int scoutfs_client_alloc_extent(struct super_block *sb, u64 len, u64 *start); +int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, + u64 *len); int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); diff --git a/kmod/src/data.c b/kmod/src/data.c index 50196f37..96fa9b9f 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -58,6 +58,20 @@ * - need trans around each bulk alloc */ +/* + * The largest extent that we'll store in a single item. This will + * determine the granularity of interleaved concurrent allocations on a + * node. Sequential max length allocations could still see contiguous + * physical extent allocations. It limits the amount of IO needed to + * invalidate a lock. And it determines the granularity of parallel + * writes to a file between nodes. + */ +#define MAX_EXTENT_BLOCKS (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SHIFT) +/* + * We ask for a fixed size from the server today. + */ +#define SERVER_ALLOC_BLOCKS (MAX_EXTENT_BLOCKS * 8) + struct data_info { struct rw_semaphore alloc_rwsem; }; @@ -399,14 +413,16 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return ret; } -static int get_server_extent(struct super_block *sb, u64 len) +static int get_server_extent(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_extent ext; u64 start; + u64 len; int ret; - ret = scoutfs_client_alloc_extent(sb, len, &start); + ret = scoutfs_client_alloc_extent(sb, SERVER_ALLOC_BLOCKS, + &start, &len); if (ret) goto out; @@ -420,6 +436,57 @@ out: return ret; } +/* + * Find a free extent to satisfy an allocation of at most @len blocks. + * + * Returns 0 and fills the caller's extent with a _BLKNO_TYPE extent if + * we found a match. It's len may be less than desired. No stored + * extents have been modified. + * + * Returns -errno on error and -ENOSPC if no free extents were found. + * + * The caller's extent is always clobbered. + */ +static int find_free_extent(struct super_block *sb, u64 len, + struct scoutfs_extent *ext) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret; + + len = min(len, MAX_EXTENT_BLOCKS); + + for (;;) { + /* first try to find the first sufficient extent */ + scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + sbi->node_id, 0, len, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, ext, + sbi->node_id_lock); + + /* if none big enough, look for last largest smaller */ + if (ret == -ENOENT && len > 1) + ret = scoutfs_extent_prev(sb, data_extent_io, ext, + sbi->node_id_lock); + + /* ask the server for more if we think it'll help */ + if (ret == -ENOENT || ext->len < len) { + ret = get_server_extent(sb); + if (ret == 0) + continue; + } + + /* use the extent we found or return errors */ + break; + } + + if (ret == 0) + scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + sbi->node_id, ext->start, + min(ext->len, len), 0, 0); + + trace_scoutfs_data_find_free_extent(sb, ext); + return ret; +} + /* * The caller is writing to a logical block that doesn't have an * allocated extent. @@ -448,8 +515,6 @@ out: * On success we update the caller's extent to the single block * allocated extent for the logical block for use in block mapping. */ -#define MAX_STREAMING_PREALLOC_BLOCKS ((u64)SCOUTFS_SEGMENT_BLOCKS) -#define SERVER_ALLOC_BLOCKS (MAX_STREAMING_PREALLOC_BLOCKS * 32) static int alloc_block(struct super_block *sb, struct inode *inode, struct scoutfs_extent *ext, u64 iblock, u64 len, struct scoutfs_lock *lock) @@ -474,30 +539,16 @@ static int alloc_block(struct super_block *sb, struct inode *inode, /* strictly contiguous extending writes will try to preallocate */ if (iblock > 1 && iblock == online) - len = min3(len, iblock, MAX_STREAMING_PREALLOC_BLOCKS); + len = min3(len, iblock, MAX_EXTENT_BLOCKS); else len = 1; trace_scoutfs_data_alloc_block(sb, inode, ext, iblock, len, online, offline); - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, len, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &fr, - sbi->node_id_lock); - if (ret == -ENOENT) { - /* try to get allocation from the server if we're out */ - ret = get_server_extent(sb, SERVER_ALLOC_BLOCKS); - if (ret == 0) - ret = scoutfs_extent_next(sb, data_extent_io, &fr, - sbi->node_id_lock); - } - if (ret) { - /* XXX should try to look for smaller free extents :/ */ - if (ret == -ENOENT) - ret = -ENOSPC; + ret = find_free_extent(sb, len, &fr); + if (ret < 0) goto out; - } trace_scoutfs_data_alloc_block_next(sb, &fr); @@ -505,9 +556,7 @@ static int alloc_block(struct super_block *sb, struct inode *inode, scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, iblock, 1, fr.start, 0); - /* remove the free extent we're using */ - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, fr.start, len, 0, 0); + /* remove the free extent that we're allocating */ ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); if (ret) goto out; @@ -855,21 +904,14 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, } /* - * Update one extent on behalf of fallocate. + * Allocate one extent on behalf of fallocate. The caller has given us + * the largest extent we can add, its flags, and the flags of an + * existing overlapping extent to remove. * - * The caller has searched for the next extent that intersects with the - * region including first_block and last_block. The next extent will be - * zeroed if it wasn't found. We don't know the state of the offsets - * past the next extent. - * - * The caller has held transactions and acquired locks. We only ever - * make one extent modification here. - * - * If this returns 0 then the caller's extent is clobbered. It is set - * to the newly fallocated extent so that the caller can continue with - * the fallocate operation. + * We allocate the largest extent that we can and return its length or + * -errno. */ -static int fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, +static s64 fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, u64 len, u8 flags, u8 rem_flags, struct scoutfs_lock *lock) { @@ -879,7 +921,7 @@ static int fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, struct scoutfs_extent fr; bool add_rem = false; bool add_fr = false; - int ret; + s64 ret; if (WARN_ON_ONCE(len == 0) || WARN_ON_ONCE(start + len < start)) { @@ -887,28 +929,9 @@ static int fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, goto out; } - /* find a sufficiently large free extent */ - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, len, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &fr, - sbi->node_id_lock); - if (ret == -ENOENT) { - /* try to get allocation from the server if we're out */ - ret = get_server_extent(sb, SERVER_ALLOC_BLOCKS); - if (ret == 0) - ret = scoutfs_extent_next(sb, data_extent_io, &fr, - sbi->node_id_lock); - /* XXX try to find smaller free extents */ - } - if (ret < 0) { - if (ret == -ENOENT) - ret = -ENOSPC; + ret = find_free_extent(sb, len, &fr); + if (ret < 0) goto out; - } - - /* trim our allocation from the length indexed extent */ - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, fr.start, min(fr.len, len), 0, 0); ret = scoutfs_extent_init(&fal, SCOUTFS_FILE_EXTENT_TYPE, ino, start, fr.len, fr.start, flags); @@ -931,6 +954,8 @@ static int fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, } ret = scoutfs_extent_add(sb, data_extent_io, &fal, lock); + if (ret == 0) + ret = fal.len; out: scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, data_extent_io, &rem, lock, @@ -961,7 +986,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) LIST_HEAD(ind_locks); u64 last_block; u64 iblock; - u64 blocks; + s64 blocks; loff_t end; u8 rem_flags; u8 flags; @@ -1003,7 +1028,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) iblock = offset >> SCOUTFS_BLOCK_SHIFT; last_block = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; - for (; iblock <= last_block; iblock = ext.start + ext.len) { + while(iblock <= last_block) { scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, iblock, 1, 0, 0); @@ -1020,17 +1045,19 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) } else if (iblock < ext.start) { /* sparse region until next extent */ - blocks = min(blocks, ext.start - iblock); + blocks = min_t(u64, blocks, ext.start - iblock); } else if (ext.map > 0) { /* skip past an allocated extent */ - blocks = min(blocks, (ext.start + ext.len) - iblock); + blocks = min_t(u64, blocks, + (ext.start + ext.len) - iblock); iblock += blocks; blocks = 0; } else { /* allocating a portion of an unallocated extent */ - blocks = min(blocks, (ext.start + ext.len) - iblock); + blocks = min_t(u64, blocks, + (ext.start + ext.len) - iblock); flags |= ext.flags; rem_flags = ext.flags; /* XXX corruption; why'd we store map == flags == 0? */ @@ -1047,9 +1074,13 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) if (blocks > 0) { down_write(&datinf->alloc_rwsem); - ret = fallocate_one_extent(sb, ino, iblock, blocks, - flags, rem_flags, lock); + blocks = fallocate_one_extent(sb, ino, iblock, blocks, + flags, rem_flags, lock); up_write(&datinf->alloc_rwsem); + if (blocks < 0) + ret = blocks; + else + ret = 0; } if (ret == 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { @@ -1068,7 +1099,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) iblock += blocks; } - ret = 0; + out: scoutfs_unlock(sb, lock, DLM_LOCK_EX); mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/format.h b/kmod/src/format.h index e3aa67d7..65c10b69 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -563,6 +563,11 @@ struct scoutfs_net_statfs { __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; +struct scoutfs_net_extent { + __le64 start; + __le64 len; +} __packed; + /* XXX eventually we'll have net compaction and will need agents to agree */ /* one upper segment and fanout lower segments */ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 87495177..9d32367b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2255,6 +2255,10 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_server_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_find_free_extent, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_next, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) diff --git a/kmod/src/server.c b/kmod/src/server.c index d830d53e..ee52e8cc 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -218,7 +218,8 @@ static int server_extent_io(struct super_block *sb, int op, * free extent that can be very quickly allocated to a node. The hope is * that doesn't happen very often. */ -static int alloc_extent(struct super_block *sb, u64 len, u64 *start) +static int alloc_extent(struct super_block *sb, u64 blocks, + u64 *start, u64 *len) { struct server_info *server = SCOUTFS_SB(sb)->server_info; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -226,17 +227,20 @@ static int alloc_extent(struct super_block *sb, u64 len, u64 *start) int ret; *start = 0; + *len = 0; down_write(&server->alloc_rwsem); - if (len & (SCOUTFS_SEGMENT_BLOCKS - 1)) { + if (blocks & (SCOUTFS_SEGMENT_BLOCKS - 1)) { ret = -EINVAL; goto out; } scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, 0, - 0, len, 0, 0); + 0, blocks, 0, 0); ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); + if (ret == -ENOENT) + ret = scoutfs_extent_prev(sb, server_extent_io, &ext, NULL); if (ret) { if (ret == -ENOENT) ret = -ENOSPC; @@ -246,7 +250,7 @@ static int alloc_extent(struct super_block *sb, u64 len, u64 *start) trace_scoutfs_server_alloc_extent_next(sb, &ext); ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - ext.len = len; + ext.len = min(blocks, ext.len); ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); if (ret) @@ -256,6 +260,7 @@ static int alloc_extent(struct super_block *sb, u64 len, u64 *start) le64_add_cpu(&super->free_blocks, -ext.len); *start = ext.start; + *len = ext.len; ret = 0; out: @@ -705,26 +710,29 @@ static int process_alloc_extent(struct server_connection *conn, struct server_info *server = conn->server; struct super_block *sb = server->sb; struct commit_waiter cw; - __le64 lestart; - __le64 lelen; + struct scoutfs_net_extent nex; + __le64 leblocks; u64 start; + u64 len; int ret; - if (data_len != sizeof(lelen)) { + if (data_len != sizeof(leblocks)) { ret = -EINVAL; goto out; } - memcpy(&lelen, data, data_len); + memcpy(&leblocks, data, data_len); down_read(&server->commit_rwsem); - ret = alloc_extent(sb, le64_to_cpu(lelen), &start); + ret = alloc_extent(sb, le64_to_cpu(leblocks), &start, &len); if (ret == -ENOSPC) { start = 0; + len = 0; ret = 0; } if (ret == 0) { - lestart = cpu_to_le64(start); + nex.start = cpu_to_le64(start); + nex.len = cpu_to_le64(len); queue_commit_work(server, &cw); } up_read(&server->commit_rwsem); @@ -732,7 +740,7 @@ static int process_alloc_extent(struct server_connection *conn, if (ret == 0) ret = wait_for_commit(server, &cw, id, type); out: - return send_reply(conn, id, type, ret, &lestart, sizeof(lestart)); + return send_reply(conn, id, type, ret, &nex, sizeof(nex)); } /* From 876414065bd486bff54d11dea81031848062d7f5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 21 Jun 2018 10:23:52 -0700 Subject: [PATCH 639/920] scoutfs: warn if we try IO outside the device We've had bugs in allocators that return success and crazy block numbers. The bad block numbers eventually make their way down to the context-free kernel warning that IO was attempted outside the device. This at least gives us a stack trace to help find where it's coming from. Signed-off-by: Zach Brown --- kmod/src/bio.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/bio.c b/kmod/src/bio.c index b8e94b46..f47ed7b2 100644 --- a/kmod/src/bio.c +++ b/kmod/src/bio.c @@ -69,6 +69,7 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, { unsigned int nr_pages = DIV_ROUND_UP(nr_blocks, SCOUTFS_BLOCKS_PER_PAGE); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct bio_end_io_args *args; struct blk_plug plug; unsigned int bytes; @@ -77,6 +78,12 @@ void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, int ret = 0; int i; + if (super->total_blocks && + WARN_ON_ONCE(blkno >= le64_to_cpu(super->total_blocks))) { + end_io(sb, data, -EIO); + return; + } + args = kmalloc(sizeof(struct bio_end_io_args), GFP_NOFS); if (!args) { end_io(sb, data, -ENOMEM); From 002daf3c1c5775615835dcbfbc97ed4b594b6bcb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 21 Jun 2018 10:27:16 -0700 Subject: [PATCH 640/920] scoutfs: return -ENOSPC to client alloc segno The server send_reply interface is confusing. It uses errors to shut down the connection. Clients getting enospc needs to happen in the message reply payload. The segno allocation server processing needs to set the segno to 0 so that the client gets it and translates that into -ENOSPC. Signed-off-by: Zach Brown --- kmod/src/server.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index ee52e8cc..6eed54ca 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -753,7 +753,7 @@ static int process_alloc_segno(struct server_connection *conn, struct server_info *server = conn->server; struct super_block *sb = server->sb; struct commit_waiter cw; - __le64 lesegno; + __le64 lesegno = 0; u64 segno; int ret; @@ -767,10 +767,12 @@ static int process_alloc_segno(struct server_connection *conn, if (ret == 0) { lesegno = cpu_to_le64(segno); queue_commit_work(server, &cw); + } else if (ret == -ENOSPC) { + ret = 0; } up_read(&server->commit_rwsem); - if (ret == 0) + if (ret == 0 && lesegno != 0) ret = wait_for_commit(server, &cw, id, type); out: return send_reply(conn, id, type, ret, &lesegno, sizeof(lesegno)); From 0c7ea66f57ad0b0bfbb0aba6f5cb3de9b8650ef8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 Apr 2018 09:15:19 -0700 Subject: [PATCH 641/920] scoutfs: add SIC_EXACT Add an item count call that lets the caller give the exact item count instead of basing it on the operation they're performing. Signed-off-by: Zach Brown --- kmod/src/count.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kmod/src/count.h b/kmod/src/count.h index fc2f993e..0135fb81 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -22,6 +22,18 @@ struct scoutfs_item_count { signed vals; }; +/* The caller knows exactly what they're doing. */ +static inline const struct scoutfs_item_count SIC_EXACT(signed items, + signed vals) +{ + struct scoutfs_item_count cnt = { + .items = items, + .vals = vals, + }; + + return cnt; +} + /* * Allocating an inode creates a new set of indexed items. */ From 59170f41b13760578f1f2a02114704cdf5ed341c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 2 Apr 2018 09:17:53 -0700 Subject: [PATCH 642/920] scoutfs: revive item deletion path The inode deletion path had bit rotted. Delete the ifdefs that were stopping it from deleting all the items associated with an inode. There can be a lot of xattr and data mapping items so we have them manage their own transactions (data already did). The xattr deletion code was trying to get a lock while the caller already held it so delete that. Then we accurately account for the small number of remaining items that finally delete the inode. Signed-off-by: Zach Brown --- kmod/src/count.h | 18 ++++++++++++++++++ kmod/src/inode.c | 38 ++++++++++++++++++++++---------------- kmod/src/scoutfs_trace.h | 11 +++++++---- kmod/src/xattr.c | 40 +++++++++++++++++++++++++--------------- kmod/src/xattr.h | 3 ++- 5 files changed, 74 insertions(+), 36 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 0135fb81..41817ea7 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -117,6 +117,24 @@ static inline const struct scoutfs_item_count SIC_MKNOD(unsigned name_len) return cnt; } +/* + * Dropping the inode deletes all its items. Potentially enormous numbers + * of items (data mapping, xattrs) are deleted in their own transactions. + */ +static inline const struct scoutfs_item_count SIC_DROP_INODE(int mode, + u64 size) +{ + struct scoutfs_item_count cnt = {0,}; + + if (S_ISLNK(mode)) + __count_sym_target(&cnt, size); + __count_dirty_inode(&cnt); + __count_orphan(&cnt); + + cnt.vals = 0; + return cnt; +} + static inline const struct scoutfs_item_count SIC_LINK(unsigned name_len) { struct scoutfs_item_count cnt = {0,}; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 20676cee..55400260 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -1400,6 +1400,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) struct kvec val; umode_t mode; u64 ind_seq; + u64 size; int ret; ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lock); @@ -1424,14 +1425,27 @@ static int delete_inode_items(struct super_block *sb, u64 ino) } mode = le32_to_cpu(sinode.mode); - trace_scoutfs_delete_inode(sb, ino, mode); + size = le64_to_cpu(sinode.size); + trace_scoutfs_delete_inode(sb, ino, mode, size); - /* XXX the trans reservation count is obviously bonkers :) */ + /* remove data items in their own transactions */ + if (S_ISREG(mode)) { + ret = scoutfs_data_truncate_items(sb, NULL, ino, 0, ~0ULL, + false, lock); + if (ret) + goto out; + } + + ret = scoutfs_xattr_drop(sb, ino, lock); + if (ret) + goto out; + + /* then delete the small known number of remaining inode items */ retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: prepare_index_deletion(sb, &ind_locks, ino, mode, &sinode) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, - SIC_DIRTY_INODE()); + SIC_DROP_INODE(mode, size)); if (ret > 0) goto retry; if (ret) @@ -1439,24 +1453,16 @@ retry: release = true; - /* first remove index items to try to avoid indexing partial deletion */ ret = remove_index_items(sb, ino, &sinode, &ind_locks); if (ret) goto out; -#if 0 - ret = scoutfs_xattr_drop(sb, ino); - if (ret) - goto out; + if (S_ISLNK(mode)) { + ret = scoutfs_symlink_drop(sb, ino, lock, size); + if (ret) + goto out; + } - if (S_ISLNK(mode)) - ret = scoutfs_symlink_drop(sb, ino, i_size); - else if (S_ISREG(mode)) - ret = scoutfs_truncate_extent_items(sb, ino, 0, ~0ULL, false); - if (ret) - goto out; - -#endif ret = scoutfs_item_delete(sb, &key, lock); if (ret) goto out; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 9d32367b..fca0b99d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1296,24 +1296,27 @@ TRACE_EVENT(scoutfs_orphan_inode, ); TRACE_EVENT(scoutfs_delete_inode, - TP_PROTO(struct super_block *sb, u64 ino, umode_t mode), + TP_PROTO(struct super_block *sb, u64 ino, umode_t mode, u64 size), - TP_ARGS(sb, ino, mode), + TP_ARGS(sb, ino, mode, size), TP_STRUCT__entry( __field(dev_t, dev) __field(__u64, ino) __field(umode_t, mode) + __field(__u64, size) ), TP_fast_assign( __entry->dev = sb->s_dev; __entry->ino = ino; __entry->mode = mode; + __entry->size = size; ), - TP_printk("dev %d,%d ino %llu, mode 0x%x", MAJOR(__entry->dev), - MINOR(__entry->dev), __entry->ino, __entry->mode) + TP_printk("dev %d,%d ino %llu, mode 0x%x size %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, + __entry->mode, __entry->size) ); TRACE_EVENT(scoutfs_scan_orphans, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index d2e57277..877626a4 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -554,46 +554,56 @@ out: } /* - * Delete all the xattr items associated with this inode. The caller - * holds a transaction. The inode is dead so we don't need the xattr - * rwsem. + * Delete all the xattr items associated with this inode. The inode is + * dead so we don't need the xattr rwsem. * * XXX This isn't great because it reads in all the items so that it can * create deletion items for each. It would be better to have the * caller create range deletion items for all the items covered by the * inode. That wouldn't require reading at all. */ -int scoutfs_xattr_drop(struct super_block *sb, u64 ino) +int scoutfs_xattr_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock) { struct scoutfs_key last; struct scoutfs_key key; - struct scoutfs_lock *lck; + unsigned int items = 16; + bool holding = false; int ret; init_xattr_key(&key, ino, 0, 0); init_xattr_key(&last, ino, U32_MAX, U64_MAX); - /* while we read to delete we need to writeback others */ - ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lck); - if (ret) - goto out; - for (;;) { - ret = scoutfs_item_next(sb, &key, &last, NULL, lck); + ret = scoutfs_item_next(sb, &key, &last, NULL, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - ret = scoutfs_item_delete(sb, &key, lck); + if (!holding) { + ret = scoutfs_hold_trans(sb, SIC_EXACT(items, 0)); + if (ret) + break; + holding = true; + } + + ret = scoutfs_item_delete(sb, &key, lock); if (ret) break; - key.skx_part++; + if (--items == 0) { + scoutfs_release_trans(sb); + holding = false; + items = 16; + } + + /* don't need to inc, next won't see deleted item */ } - scoutfs_unlock(sb, lck, DLM_LOCK_EX); -out: + if (holding) + scoutfs_release_trans(sb); + return ret; } diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index e0fadf32..6c205358 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -8,6 +8,7 @@ int scoutfs_setxattr(struct dentry *dentry, const char *name, int scoutfs_removexattr(struct dentry *dentry, const char *name); ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); -int scoutfs_xattr_drop(struct super_block *sb, u64 ino); +int scoutfs_xattr_drop(struct super_block *sb, u64 ino, + struct scoutfs_lock *lock); #endif From fddc3a7a7596156170554b3b726b2bdf2012f903 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 21 Jun 2018 17:07:29 -0700 Subject: [PATCH 643/920] scoutfs: minimize commit writeback latencies Our simple transaction machinery causes high commit latencies if we let too much dirty file data accumulate. Small files have a natural limit on the amount of dirty data because they have more dirty items per dirty page. They fill up the single segment sooner and kick off a commit which finds a relatively small amount of dirty file data. But large files can reference quite a lot of dirty data with a small amount of extent items which don't fill up the transaction's segment. During large streaming writes we can fill up memory with dirty file data before filling a segment with mapping extent metadata. This can lead to high commit latencies when memory is full of dirty file pages. Regularly kicking off background writeback behind streaming write positions reduces the amount of dirty data that commits will find and have to write out. Signed-off-by: Zach Brown --- kmod/src/data.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index 96fa9b9f..07b90294 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -874,6 +875,20 @@ out: return ret; } +/* kinda like __filemap_fdatawrite_range! :P */ +static int writepages_sync_none(struct address_space *mapping, loff_t start, + loff_t end) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = LONG_MAX, + .range_start = start, + .range_end = end, + }; + + return mapping->a_ops->writepages(mapping, &wbc); +} + static int scoutfs_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) @@ -900,6 +915,27 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &wbd->ind_locks); kfree(wbd); + + /* + * Currently transactions are kept very simple. Only one is + * open at a time and commit excludes concurrent dirtying. It + * writes out all dirty file data during commit. This can lead + * to very long commit latencies with lots of dirty file data. + * + * This hack tries to minimize these writeback latencies while + * keeping concurrent large file strreaming writes from + * suffering too terribly. Every N bytes we kick off background + * writbeack on the previous N bytes. By the time transaction + * commit comes along it will find that dirty file blocks have + * already been written. + */ +#define BACKGROUND_WRITEBACK_BYTES (16 * 1024 * 1024) +#define BACKGROUND_WRITEBACK_MASK (BACKGROUND_WRITEBACK_BYTES - 1) + if (ret > 0 && ((pos + ret) & BACKGROUND_WRITEBACK_MASK) == 0) + writepages_sync_none(mapping, + pos + ret - BACKGROUND_WRITEBACK_BYTES, + pos + ret - 1); + return ret; } From 5935a3f43e46488e1c477a7efb0f7e18e1fc7341 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Jun 2018 14:17:29 -0700 Subject: [PATCH 644/920] scoutfs: remove unused trace events These trace events were all orphaned long ago by commits which removed their callers but forgot to remove their definitions. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 112 --------------------------------------- 1 file changed, 112 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index fca0b99d..56eae869 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -298,42 +298,6 @@ TRACE_EVENT(scoutfs_item_update_ret, TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) ); -TRACE_EVENT(scoutfs_item_next_same, - TP_PROTO(struct super_block *sb, unsigned int key_len), - - TP_ARGS(sb, key_len), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(unsigned int, key_len) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->key_len = key_len; - ), - - TP_printk(FSID_FMT" key len %u", __entry->fsid, __entry->key_len) -); - -TRACE_EVENT(scoutfs_item_next_same_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(int, ret) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->ret = ret; - ), - - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) -); - TRACE_EVENT(scoutfs_item_next_ret, TP_PROTO(struct super_block *sb, int ret), @@ -526,49 +490,6 @@ TRACE_EVENT(scoutfs_data_alloc_block_ret, SE_ARG(&__entry->ext), __entry->ret) ); -TRACE_EVENT(scoutfs_data_find_alloc_block_curs, - TP_PROTO(struct super_block *sb, void *curs, __u64 blkno), - - TP_ARGS(sb, curs, blkno), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(void *, curs) - __field(__u64, blkno) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->curs = curs; - __entry->blkno = blkno; - ), - - TP_printk(FSID_FMT" got curs %p blkno %llu", __entry->fsid, - __entry->curs, __entry->blkno) -); - -TRACE_EVENT(scoutfs_data_get_cursor, - TP_PROTO(void *curs, void *task, unsigned int pid), - - TP_ARGS(curs, task, pid), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(void *, curs) - __field(void *, task) - __field(unsigned int, pid) - ), - - TP_fast_assign( - __entry->curs = curs; - __entry->task = task; - __entry->pid = pid; - ), - - TP_printk("resetting curs %p was task %p pid %u", __entry->curs, - __entry->task, __entry->pid) -); - TRACE_EVENT(scoutfs_data_truncate_items, TP_PROTO(struct super_block *sb, __u64 iblock, __u64 last, int offline), @@ -889,27 +810,6 @@ DEFINE_EVENT(scoutfs_index_item_class, scoutfs_delete_index_item, TP_ARGS(sb, type, major, minor, ino) ); -TRACE_EVENT(scoutfs_inode_fill_pool, - TP_PROTO(struct super_block *sb, __u64 ino, __u64 nr), - - TP_ARGS(sb, ino, nr), - - TP_STRUCT__entry( - __field(__u64, fsid) - __field(__u64, ino) - __field(__u64, nr) - ), - - TP_fast_assign( - __entry->fsid = FSID_ARG(sb); - __entry->ino = ino; - __entry->nr = nr; - ), - - TP_printk(FSID_FMT" filling ino %llu nr %llu", __entry->fsid, - __entry->ino, __entry->nr) -); - TRACE_EVENT(scoutfs_alloc_ino, TP_PROTO(struct super_block *sb, int ret, __u64 ino, __u64 next_ino, __u64 next_nr), @@ -1495,12 +1395,6 @@ DEFINE_EVENT(scoutfs_range_class, scoutfs_item_invalidate_range, TP_ARGS(sb, start, end) ); -DEFINE_EVENT(scoutfs_range_class, scoutfs_item_shrink_range, - TP_PROTO(struct super_block *sb, struct scoutfs_key *start, - struct scoutfs_key *end), - TP_ARGS(sb, start, end) -); - DECLARE_EVENT_CLASS(scoutfs_cached_range_class, TP_PROTO(struct super_block *sb, void *rng, struct scoutfs_key *start, struct scoutfs_key *end), @@ -1558,12 +1452,6 @@ DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_rem_rb_insert, TP_ARGS(sb, rng, start, end) ); -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_delete_enoent, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_start, TP_PROTO(struct super_block *sb, void *rng, struct scoutfs_key *start, struct scoutfs_key *end), From dfac36a9aa48ebffadf2dc8c92fa72049ec1766b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Jun 2018 14:54:08 -0700 Subject: [PATCH 645/920] scoutfs: trace key struct The userspace trace event printing code has trouble with arguments that refer to fields in entries. Add macros to make entries for all the fields and use them as the formatted arguments. We also remove the mapping of zone and type to strings. It's smaller to print the values directly and gets rid of some silly code. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/key.c | 58 -------------- kmod/src/key.h | 71 ++++++++++------- kmod/src/scoutfs_trace.h | 166 +++++++++++++++++++-------------------- kmod/src/super.c | 1 - 5 files changed, 125 insertions(+), 173 deletions(-) delete mode 100644 kmod/src/key.c diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 61b12f70..73589b6c 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -6,7 +6,7 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += bio.o btree.o client.o compact.o counters.o data.o dir.o \ - export.o extents.o file.o inode.o ioctl.o item.o key.o lock.o \ + export.o extents.o file.o inode.o ioctl.o item.o lock.o \ manifest.o msg.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ triggers.o xattr.o diff --git a/kmod/src/key.c b/kmod/src/key.c deleted file mode 100644 index 9c1603b5..00000000 --- a/kmod/src/key.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2018 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include - -#include "format.h" -#include "key.h" - -char *scoutfs_zone_strings[SCOUTFS_MAX_ZONE] = { - [SCOUTFS_INODE_INDEX_ZONE] = "ind", - [SCOUTFS_NODE_ZONE] = "nod", - [SCOUTFS_FS_ZONE] = "fs", -}; - -char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = { - [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] = "msq", - [SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] = "dsq", - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = "fbn", - [SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = "fbs", - [SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = "orp", - [SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = "ino", - [SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = "xat", - [SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = "dnt", - [SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = "rdr", - [SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = "lbr", - [SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = "sym", - [SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = "fex", -}; - -char scoutfs_unknown_u8_strings[U8_MAX][U8_STR_MAX]; - -int __init scoutfs_key_init(void) -{ - int ret; - int i; - - for (i = 0; i <= U8_MAX; i++) { - ret = snprintf(scoutfs_unknown_u8_strings[i], U8_STR_MAX, - "u%u", i); - if (WARN_ONCE(ret <= 0 || ret >= U8_STR_MAX, - "snprintf("__stringify(U8_STR_MAX)") ret %d\n", - ret)) - return -EINVAL; - } - - return 0; -} diff --git a/kmod/src/key.h b/kmod/src/key.h index eb157279..abe89ee0 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -6,39 +6,52 @@ #include "cmp.h" #include "endian_swap.h" -extern char *scoutfs_zone_strings[SCOUTFS_MAX_ZONE]; -extern char *scoutfs_type_strings[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE]; -#define U8_STR_MAX 5 /* u%3u'\0' */ -extern char scoutfs_unknown_u8_strings[U8_MAX][U8_STR_MAX]; +#define SK_FMT "%u.%llu.%u.%llu.%llu.%u" -int __init scoutfs_key_init(void); - -static inline char *sk_zone_str(u8 zone) -{ - if (zone >= SCOUTFS_MAX_ZONE || scoutfs_zone_strings[zone] == NULL) - return scoutfs_unknown_u8_strings[zone]; - - return scoutfs_zone_strings[zone]; -} - -static inline char *sk_type_str(u8 zone, u8 type) -{ - if (zone >= SCOUTFS_MAX_ZONE || type >= SCOUTFS_MAX_TYPE || - scoutfs_type_strings[zone][type] == NULL) - return scoutfs_unknown_u8_strings[type]; - - return scoutfs_type_strings[zone][type]; -} - -#define SK_FMT "%s.%llu.%s.%llu.%llu.%u" /* This does not support null keys */ -#define SK_ARG(key) sk_zone_str((key)->sk_zone), \ - le64_to_cpu((key)->_sk_first), \ - sk_type_str((key)->sk_zone, (key)->sk_type), \ - le64_to_cpu((key)->_sk_second), \ - le64_to_cpu((key)->_sk_third), \ +#define SK_ARG(key) (key)->sk_zone, \ + le64_to_cpu((key)->_sk_first), \ + (key)->sk_type, \ + le64_to_cpu((key)->_sk_second), \ + le64_to_cpu((key)->_sk_third), \ (key)->_sk_fourth +/* userspace trace event printing doesn't like arguments with structure + * field references. So we explode structures into their fields instead + * of + */ +#define sk_trace_define(name) \ + __field(__u8, name##_zone) \ + __field(__u64, name##_first) \ + __field(__u8, name##_type) \ + __field(__u64, name##_second) \ + __field(__u64, name##_third) \ + __field(__u8, name##_fourth) + +#define sk_trace_assign(name, key) \ +do { \ + __typeof__(key) _key = (key); \ + if (_key) { \ + __entry->name##_zone = _key->sk_zone; \ + __entry->name##_first = le64_to_cpu(_key->_sk_first); \ + __entry->name##_type = _key->sk_type; \ + __entry->name##_second = le64_to_cpu(_key->_sk_second);\ + __entry->name##_third = le64_to_cpu(_key->_sk_third); \ + __entry->name##_fourth = _key->_sk_fourth; \ + } else { \ + __entry->name##_zone = 0; \ + __entry->name##_first = 0; \ + __entry->name##_type = 0; \ + __entry->name##_second = 0; \ + __entry->name##_third = 0; \ + __entry->name##_fourth = 0; \ + } \ +} while (0) + +#define sk_trace_args(name) \ + __entry->name##_zone, __entry->name##_first, __entry->name##_type, \ + __entry->name##_second, __entry->name##_third, __entry->name##_fourth + static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) { key->sk_zone = 0; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 56eae869..dc7568b0 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -235,18 +235,18 @@ DECLARE_EVENT_CLASS(scoutfs_key_ret_class, TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_key, key) + sk_trace_define(key) __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - __entry->key = *key; + sk_trace_assign(key, key); __entry->ret = ret; ), TP_printk("fsid "FSID_FMT" key "SK_FMT" ret %d", - __entry->fsid, SK_ARG(&__entry->key), __entry->ret) + __entry->fsid, sk_trace_args(key), __entry->ret) ); DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_create, @@ -1243,19 +1243,19 @@ DECLARE_EVENT_CLASS(scoutfs_manifest_class, __field(u8, level) __field(u64, segno) __field(u64, seq) - __field_struct(struct scoutfs_key, first) - __field_struct(struct scoutfs_key, last) + sk_trace_define(first) + sk_trace_define(last) ), TP_fast_assign( __entry->level = level; __entry->segno = segno; __entry->seq = seq; - scoutfs_key_copy_or_zeros(&__entry->first, first); - scoutfs_key_copy_or_zeros(&__entry->last, last); + sk_trace_assign(first, first); + sk_trace_assign(last, last); ), TP_printk("level %u segno %llu seq %llu first "SK_FMT" last "SK_FMT, __entry->level, __entry->segno, __entry->seq, - SK_ARG(&__entry->first), SK_ARG(&__entry->last)) + sk_trace_args(first), sk_trace_args(last)) ); DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_add, @@ -1292,24 +1292,24 @@ TRACE_EVENT(scoutfs_read_item_keys, TP_ARGS(sb, key, start, end, seg_start, seg_end), TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_key, key) - __field_struct(struct scoutfs_key, start) - __field_struct(struct scoutfs_key, end) - __field_struct(struct scoutfs_key, seg_start) - __field_struct(struct scoutfs_key, seg_end) + sk_trace_define(key) + sk_trace_define(start) + sk_trace_define(end) + sk_trace_define(seg_start) + sk_trace_define(seg_end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_copy_or_zeros(&__entry->key, key); - scoutfs_key_copy_or_zeros(&__entry->start, start); - scoutfs_key_copy_or_zeros(&__entry->end, end); - scoutfs_key_copy_or_zeros(&__entry->seg_start, seg_start); - scoutfs_key_copy_or_zeros(&__entry->seg_end, seg_end); + sk_trace_assign(key, key); + sk_trace_assign(start, start); + sk_trace_assign(end, end); + sk_trace_assign(seg_start, seg_start); + sk_trace_assign(seg_end, seg_end); ), TP_printk("fsid "FSID_FMT" key "SK_FMT" start "SK_FMT" end "SK_FMT" seg_start "SK_FMT" seg_end "SK_FMT"", - __entry->fsid, SK_ARG(&__entry->key), SK_ARG(&__entry->start), - SK_ARG(&__entry->end), SK_ARG(&__entry->seg_start), - SK_ARG(&__entry->seg_end)) + __entry->fsid, sk_trace_args(key), sk_trace_args(start), + sk_trace_args(end), sk_trace_args(seg_start), + sk_trace_args(seg_end)) ); DECLARE_EVENT_CLASS(scoutfs_key_class, @@ -1317,13 +1317,13 @@ DECLARE_EVENT_CLASS(scoutfs_key_class, TP_ARGS(sb, key), TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_key, key) + sk_trace_define(key) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_copy_or_zeros(&__entry->key, key); + sk_trace_assign(key, key); ), - TP_printk(FSID_FMT" key "SK_FMT, __entry->fsid, SK_ARG(&__entry->key)) + TP_printk(FSID_FMT" key "SK_FMT, __entry->fsid, sk_trace_args(key)) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, @@ -1370,17 +1370,16 @@ DECLARE_EVENT_CLASS(scoutfs_range_class, TP_ARGS(sb, start, end), TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_key, start) - __field_struct(struct scoutfs_key, end) + sk_trace_define(start) + sk_trace_define(end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - scoutfs_key_copy_or_zeros(&__entry->start, start); - scoutfs_key_copy_or_zeros(&__entry->end, end); + sk_trace_assign(start, start); + sk_trace_assign(end, end); ), TP_printk("fsid "FSID_FMT" start "SK_FMT" end "SK_FMT, - __entry->fsid, SK_ARG(&__entry->start), - SK_ARG(&__entry->end)) + __entry->fsid, sk_trace_args(start), sk_trace_args(end)) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, @@ -1402,18 +1401,18 @@ DECLARE_EVENT_CLASS(scoutfs_cached_range_class, TP_STRUCT__entry( __field(__u64, fsid) __field(void *, rng) - __field_struct(struct scoutfs_key, start) - __field_struct(struct scoutfs_key, end) + sk_trace_define(start) + sk_trace_define(end) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->rng = rng; - scoutfs_key_copy_or_zeros(&__entry->start, start); - scoutfs_key_copy_or_zeros(&__entry->end, end); + sk_trace_assign(start, start); + sk_trace_assign(end, end); ), TP_printk("fsid "FSID_FMT" rng %p start "SK_FMT" end "SK_FMT, - __entry->fsid, __entry->rng, SK_ARG(&__entry->start), - SK_ARG(&__entry->end)) + __entry->fsid, __entry->rng, sk_trace_args(start), + sk_trace_args(end)) ); DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_free, @@ -1617,7 +1616,7 @@ TRACE_EVENT(scoutfs_seg_append_item, __field(__u64, seq) __field(__u32, nr_items) __field(__u32, total_bytes) - __field_struct(struct scoutfs_key, key) + sk_trace_define(key) __field(__u16, val_len) ), @@ -1627,15 +1626,14 @@ TRACE_EVENT(scoutfs_seg_append_item, __entry->seq = seq; __entry->nr_items = nr_items; __entry->total_bytes = total_bytes; - __entry->key = *key; + sk_trace_assign(key, key); __entry->val_len = val_len; ), TP_printk("fsid "FSID_FMT" segno %llu seq %llu nr_items %u total_bytes %u key "SK_FMT" val_len %u", __entry->fsid, __entry->segno, __entry->seq, __entry->nr_items, __entry->total_bytes, - SK_ARG(&__entry->key), - __entry->val_len) + sk_trace_args(key), __entry->val_len) ); DECLARE_EVENT_CLASS(scoutfs_net_class, @@ -1753,25 +1751,25 @@ TRACE_EVENT(scoutfs_item_next_range_check, TP_STRUCT__entry( __field(void *, sb) __field(int, cached) - __field_struct(struct scoutfs_key, key) - __field_struct(struct scoutfs_key, pos) - __field_struct(struct scoutfs_key, last) - __field_struct(struct scoutfs_key, end) - __field_struct(struct scoutfs_key, range_end) + sk_trace_define(key) + sk_trace_define(pos) + sk_trace_define(last) + sk_trace_define(end) + sk_trace_define(range_end) ), TP_fast_assign( __entry->sb = sb; __entry->cached = cached; - scoutfs_key_copy_or_zeros(&__entry->key, key); - scoutfs_key_copy_or_zeros(&__entry->pos, pos); - scoutfs_key_copy_or_zeros(&__entry->last, last); - scoutfs_key_copy_or_zeros(&__entry->end, end); - scoutfs_key_copy_or_zeros(&__entry->range_end, range_end); + sk_trace_assign(key, key); + sk_trace_assign(pos, pos); + sk_trace_assign(last, last); + sk_trace_assign(end, end); + sk_trace_assign(range_end, range_end); ), TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" last "SK_FMT" end "SK_FMT" range_end "SK_FMT, - __entry->sb, __entry->cached, SK_ARG(&__entry->key), - SK_ARG(&__entry->pos), SK_ARG(&__entry->last), - SK_ARG(&__entry->end), SK_ARG(&__entry->range_end)) + __entry->sb, __entry->cached, sk_trace_args(key), + sk_trace_args(pos), sk_trace_args(last), + sk_trace_args(end), sk_trace_args(range_end)) ); TRACE_EVENT(scoutfs_item_prev_range_check, @@ -1783,25 +1781,25 @@ TRACE_EVENT(scoutfs_item_prev_range_check, TP_STRUCT__entry( __field(void *, sb) __field(int, cached) - __field_struct(struct scoutfs_key, key) - __field_struct(struct scoutfs_key, pos) - __field_struct(struct scoutfs_key, first) - __field_struct(struct scoutfs_key, start) - __field_struct(struct scoutfs_key, range_start) + sk_trace_define(key) + sk_trace_define(pos) + sk_trace_define(first) + sk_trace_define(start) + sk_trace_define(range_start) ), TP_fast_assign( __entry->sb = sb; __entry->cached = cached; - scoutfs_key_copy_or_zeros(&__entry->key, key); - scoutfs_key_copy_or_zeros(&__entry->pos, pos); - scoutfs_key_copy_or_zeros(&__entry->first, first); - scoutfs_key_copy_or_zeros(&__entry->start, start); - scoutfs_key_copy_or_zeros(&__entry->range_start, range_start); + sk_trace_assign(key, key); + sk_trace_assign(pos, pos); + sk_trace_assign(first, first); + sk_trace_assign(start, start); + sk_trace_assign(range_start, range_start); ), TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" first "SK_FMT" start "SK_FMT" range_start "SK_FMT, - __entry->sb, __entry->cached, SK_ARG(&__entry->key), - SK_ARG(&__entry->pos), SK_ARG(&__entry->first), - SK_ARG(&__entry->start), SK_ARG(&__entry->range_start)) + __entry->sb, __entry->cached, sk_trace_args(key), + sk_trace_args(pos), sk_trace_args(first), + sk_trace_args(start), sk_trace_args(range_start)) ); DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, @@ -1845,29 +1843,29 @@ TRACE_EVENT(scoutfs_item_shrink_around, TP_ARGS(sb, rng_start, rng_end, item, prev, first, last, next), TP_STRUCT__entry( __field(void *, sb) - __field_struct(struct scoutfs_key, rng_start) - __field_struct(struct scoutfs_key, rng_end) - __field_struct(struct scoutfs_key, item) - __field_struct(struct scoutfs_key, prev) - __field_struct(struct scoutfs_key, first) - __field_struct(struct scoutfs_key, last) - __field_struct(struct scoutfs_key, next) + sk_trace_define(rng_start) + sk_trace_define(rng_end) + sk_trace_define(item) + sk_trace_define(prev) + sk_trace_define(first) + sk_trace_define(last) + sk_trace_define(next) ), TP_fast_assign( __entry->sb = sb; - scoutfs_key_copy_or_zeros(&__entry->rng_start, rng_start); - scoutfs_key_copy_or_zeros(&__entry->rng_end, rng_end); - scoutfs_key_copy_or_zeros(&__entry->item, item); - scoutfs_key_copy_or_zeros(&__entry->prev, prev); - scoutfs_key_copy_or_zeros(&__entry->first, first); - scoutfs_key_copy_or_zeros(&__entry->last, last); - scoutfs_key_copy_or_zeros(&__entry->next, next); + sk_trace_assign(rng_start, rng_start); + sk_trace_assign(rng_end, rng_end); + sk_trace_assign(item, item); + sk_trace_assign(prev, prev); + sk_trace_assign(first, first); + sk_trace_assign(last, last); + sk_trace_assign(next, next); ), TP_printk("sb %p rng_start "SK_FMT" rng_end "SK_FMT" item "SK_FMT" prev "SK_FMT" first "SK_FMT" last "SK_FMT" next "SK_FMT, - __entry->sb, SK_ARG(&__entry->rng_start), - SK_ARG(&__entry->rng_end), SK_ARG(&__entry->item), - SK_ARG(&__entry->prev), SK_ARG(&__entry->first), - SK_ARG(&__entry->last), SK_ARG(&__entry->next)) + __entry->sb, sk_trace_args(rng_start), + sk_trace_args(rng_end), sk_trace_args(item), + sk_trace_args(prev), sk_trace_args(first), + sk_trace_args(last), sk_trace_args(next)) ); TRACE_EVENT(scoutfs_rename, diff --git a/kmod/src/super.c b/kmod/src/super.c index 6b853766..aaea4cba 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -419,7 +419,6 @@ static int __init scoutfs_module_init(void) ".string \""SCOUTFS_GIT_DESCRIBE"\\n\"\n" ".previous\n"); - scoutfs_key_init(); scoutfs_init_counters(); ret = scoutfs_sysfs_init(); From 53e8ab0f7b68f44886671fd79685fb537ef18b13 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Jun 2018 15:04:21 -0700 Subject: [PATCH 646/920] scoutfs: trace extent struct The userspace trace event printing code has trouble with arguments that refer to fields in entries. Add macros to make entries for all the fields and use them as the formatted arguments. Signed-off-by: Zach Brown --- kmod/src/extents.h | 25 +++++++++++++++++++++++++ kmod/src/scoutfs_trace.h | 18 +++++++++--------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/kmod/src/extents.h b/kmod/src/extents.h index 2cba23ab..478892a8 100644 --- a/kmod/src/extents.h +++ b/kmod/src/extents.h @@ -18,6 +18,31 @@ struct scoutfs_extent { #define SE_ARG(ext) (ext)->owner, (ext)->start, (ext)->len, (ext)->map, \ (ext)->type, (ext)->flags +#define se_trace_define(name) \ + __field(__u64, name##_owner) \ + __field(__u64, name##_start) \ + __field(__u64, name##_len) \ + __field(__u64, name##_map) \ + __field(__u8, name##_type) \ + __field(__u8, name##_flags) + +/* doesn't support null extent pointers */ +#define se_trace_assign(name, ext) \ +do { \ + __typeof__(ext) _ext = (ext); \ + \ + __entry->name##_owner = _ext->owner; \ + __entry->name##_start = _ext->start; \ + __entry->name##_len = _ext->len; \ + __entry->name##_map = _ext->map; \ + __entry->name##_type = _ext->type; \ + __entry->name##_flags = _ext->flags; \ +} while (0) + +#define se_trace_args(name) \ + __entry->name##_owner, __entry->name##_start, __entry->name##_len, \ + __entry->name##_map, __entry->name##_type, __entry->name##_flags + enum { SEI_NEXT, SEI_PREV, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index dc7568b0..e43084d0 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -446,7 +446,7 @@ TRACE_EVENT(scoutfs_data_alloc_block, TP_STRUCT__entry( __field(__u64, fsid) __field(__u64, ino) - __field_struct(struct scoutfs_extent, ext) + se_trace_define(ext) __field(__u64, iblock) __field(__u64, len) __field(__u64, online_blocks) @@ -456,7 +456,7 @@ TRACE_EVENT(scoutfs_data_alloc_block, TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->ino = scoutfs_ino(inode); - __entry->ext = *ext; + se_trace_assign(ext, ext); __entry->iblock = iblock; __entry->len = len; __entry->online_blocks = online_blocks; @@ -464,7 +464,7 @@ TRACE_EVENT(scoutfs_data_alloc_block, ), TP_printk("fsid "FSID_FMT" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", - __entry->fsid, __entry->ino, SE_ARG(&__entry->ext), + __entry->fsid, __entry->ino, se_trace_args(ext), __entry->iblock, __entry->len, __entry->online_blocks, __entry->offline_blocks) ); @@ -476,18 +476,18 @@ TRACE_EVENT(scoutfs_data_alloc_block_ret, TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_extent, ext) + se_trace_define(ext) __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - __entry->ext = *ext; + se_trace_assign(ext, ext); __entry->ret = ret; ), TP_printk(FSID_FMT" ext "SE_FMT" ret %d", __entry->fsid, - SE_ARG(&__entry->ext), __entry->ret) + se_trace_args(ext), __entry->ret) ); TRACE_EVENT(scoutfs_data_truncate_items, @@ -2083,16 +2083,16 @@ DECLARE_EVENT_CLASS(scoutfs_extent_class, TP_STRUCT__entry( __field(__u64, fsid) - __field_struct(struct scoutfs_extent, ext) + se_trace_define(ext) ), TP_fast_assign( __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; - __entry->ext = *ext; + se_trace_assign(ext, ext); ), TP_printk("fsid "FSID_FMT" ext "SE_FMT, - __entry->fsid, SE_ARG(&__entry->ext)) + __entry->fsid, se_trace_args(ext)) ); DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_insert, From 5d9ad0923a7c7d2d300de315430e12cfd84a44d2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 26 Jun 2018 15:38:12 -0700 Subject: [PATCH 647/920] scoutfs: trace net structs The userspace trace event printing code has trouble with arguments that refer to fields in entries. Add macros to make entries for all the fields and use them as the formatted arguments. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 39 +++++++++++------------------------- kmod/src/server.h | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index e43084d0..fd3a0f6c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -37,6 +37,7 @@ #include "export.h" #include "dir.h" #include "extents.h" +#include "server.h" struct lock_info; @@ -1641,36 +1642,20 @@ DECLARE_EVENT_CLASS(scoutfs_net_class, struct sockaddr_in *peer, struct scoutfs_net_header *nh), TP_ARGS(sb, name, peer, nh), TP_STRUCT__entry( - __field(unsigned int, major) - __field(unsigned int, minor) - __field(u32, name_addr) - __field(u16, name_port) - __field(u32, peer_addr) - __field(u16, peer_port) - __field(u64, id) - __field(u8, type) - __field(u8, status) - __field(u16, data_len) + __field(__u64, fsid) + si4_trace_define(name) + si4_trace_define(peer) + snh_trace_define(nh) ), TP_fast_assign( - __entry->major = MAJOR(sb->s_bdev->bd_dev); - __entry->minor = MINOR(sb->s_bdev->bd_dev); - /* sparse can't handle this cpp nightmare */ - __entry->name_addr = (u32 __force)name->sin_addr.s_addr; - __entry->name_port = be16_to_cpu(name->sin_port); - __entry->peer_addr = (u32 __force)peer->sin_addr.s_addr; - __entry->peer_port = be16_to_cpu(peer->sin_port); - __entry->id = le64_to_cpu(nh->id); - __entry->type = nh->type; - __entry->status = nh->status; - __entry->data_len = le16_to_cpu(nh->data_len); + __entry->fsid = FSID_ARG(sb); + si4_trace_assign(name, name); + si4_trace_assign(peer, peer); + snh_trace_assign(nh, nh); ), - TP_printk("dev %u:%u %pI4:%u -> %pI4:%u id %llu type %u status %u data_len %u", - __entry->major, __entry->minor, - &__entry->name_addr, __entry->name_port, - &__entry->peer_addr, __entry->peer_port, - __entry->id, __entry->type, __entry->status, - __entry->data_len) + TP_printk("fsid "FSID_FMT" name "SI4_FMT" peer "SI4_FMT" nh "SNH_FMT, + __entry->fsid, si4_trace_args(name), si4_trace_args(peer), + snh_trace_args(nh)) ); DEFINE_EVENT(scoutfs_net_class, scoutfs_client_send_request, diff --git a/kmod/src/server.h b/kmod/src/server.h index bbc73901..96f3a5df 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -1,6 +1,49 @@ #ifndef _SCOUTFS_SERVER_H_ #define _SCOUTFS_SERVER_H_ +#define SI4_FMT "%u.%u.%u.%u:%u" + +#define si4_trace_define(name) \ + __field(__u32, name##_addr) \ + __field(__u16, name##_port) + +#define si4_trace_assign(name, sin) \ +do { \ + __typeof__(sin) _sin = (sin); \ + \ + __entry->name##_addr = be32_to_cpu(_sin->sin_addr.s_addr); \ + __entry->name##_port = be16_to_cpu(_sin->sin_port); \ +} while(0) + +#define si4_trace_args(name) \ + (__entry->name##_addr >> 24), \ + (__entry->name##_addr >> 16) & 255, \ + (__entry->name##_addr >> 0) & 255, \ + __entry->name##_addr & 255, \ + __entry->name##_port + +#define SNH_FMT "id %llu data_len %u type %u status %u" + +#define snh_trace_define(name) \ + __field(__u64, name##_id) \ + __field(__u16, name##_data_len) \ + __field(__u8, name##_type) \ + __field(__u8, name##_status) + +#define snh_trace_assign(name, nh) \ +do { \ + __typeof__(nh) _nh = (nh); \ + \ + __entry->name##_id = le64_to_cpu(_nh->id); \ + __entry->name##_data_len = le16_to_cpu(_nh->data_len); \ + __entry->name##_type = _nh->type; \ + __entry->name##_status = _nh->status; \ +} while (0) + +#define snh_trace_args(name) \ + __entry->name##_id, __entry->name##_data_len, __entry->name##_type, \ + __entry->name##_status + void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment); void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, From e19716a0f22386286a97c1618be5d0ec58558f76 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 Jun 2018 15:09:28 -0700 Subject: [PATCH 648/920] scoutfs: clean up super block use The code that works with the super block had drifted a bit. We still had two from an old design and we weren't doing anything with its crc. Move to only using one super block at a fixed blkno and store and verify its crc field by sharing code with the btree block checksumming. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/block.c | 43 +++++++++++++++++++ kmod/src/block.h | 10 +++++ kmod/src/btree.c | 57 +++++++++++-------------- kmod/src/client.c | 2 +- kmod/src/format.h | 20 +++------ kmod/src/server.c | 2 +- kmod/src/super.c | 106 ++++++++++++++++++++++------------------------ kmod/src/super.h | 4 +- 9 files changed, 140 insertions(+), 106 deletions(-) create mode 100644 kmod/src/block.c create mode 100644 kmod/src/block.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 73589b6c..c9375a0b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -5,7 +5,7 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += bio.o btree.o client.o compact.o counters.o data.o dir.o \ +scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o lock.o \ manifest.o msg.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ diff --git a/kmod/src/block.c b/kmod/src/block.c new file mode 100644 index 00000000..bfe07e16 --- /dev/null +++ b/kmod/src/block.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "format.h" +#include "super.h" +#include "block.h" + +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr) +{ + int off = offsetof(struct scoutfs_block_header, crc) + + FIELD_SIZEOF(struct scoutfs_block_header, crc); + u32 calc = crc32c(~0, (char *)hdr + off, SCOUTFS_BLOCK_SIZE - off); + + return cpu_to_le32(calc); +} + +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr) +{ + return hdr->crc == scoutfs_block_calc_crc(hdr); +} + +bool scoutfs_block_valid_ref(struct super_block *sb, + struct scoutfs_block_header *hdr, + __le64 seq, __le64 blkno) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return hdr->fsid == super->hdr.fsid && hdr->seq == seq && + hdr->blkno == blkno; +} diff --git a/kmod/src/block.h b/kmod/src/block.h new file mode 100644 index 00000000..5bf42331 --- /dev/null +++ b/kmod/src/block.h @@ -0,0 +1,10 @@ +#ifndef _SCOUTFS_BLOCK_H_ +#define _SCOUTFS_BLOCK_H_ + +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr); +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr); +bool scoutfs_block_valid_ref(struct super_block *sb, + struct scoutfs_block_header *hdr, + __le64 seq, __le64 blkno); + +#endif diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 028c81c3..eee9c547 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -28,6 +28,7 @@ #include "triggers.h" #include "options.h" #include "msg.h" +#include "block.h" #include "scoutfs_trace.h" @@ -603,24 +604,16 @@ BUFFER_FNS(ScoutfsValidCrc, scoutfs_valid_crc) /* crc matched */ * Make sure that we've found a valid block and that it's the block that * we're looking for. */ -static bool valid_referenced_block(struct scoutfs_super_block *super, +static bool valid_referenced_block(struct super_block *sb, struct scoutfs_btree_ref *ref, struct scoutfs_btree_block *bt, struct buffer_head *bh) { - __le32 existing; - u32 calc; - smp_rmb(); /* load checked before crc */ if (!buffer_scoutfs_checked(bh)) { lock_buffer(bh); if (!buffer_scoutfs_checked(bh)) { - existing = bt->crc; - bt->crc = 0; - calc = crc32c(~0, bt, SCOUTFS_BLOCK_SIZE); - bt->crc = existing; - - if (calc == le32_to_cpu(existing)) + if (scoutfs_block_valid_crc(&bt->hdr)) set_buffer_scoutfs_valid_crc(bh); else clear_buffer_scoutfs_valid_crc(bh); @@ -631,8 +624,8 @@ static bool valid_referenced_block(struct scoutfs_super_block *super, unlock_buffer(bh); } - return buffer_scoutfs_valid_crc(bh) && super->hdr.fsid == bt->fsid && - ref->blkno == bt->blkno && ref->seq == bt->seq; + return buffer_scoutfs_valid_crc(bh) && + scoutfs_block_valid_ref(sb, &bt->hdr, ref->seq, ref->blkno); } /* @@ -681,7 +674,7 @@ retry: } bt = (void *)bh->b_data; - if (!valid_referenced_block(super, ref, bt, bh) || + if (!valid_referenced_block(sb, ref, bt, bh) || scoutfs_trigger(sb, BTREE_STALE_READ)) { scoutfs_inc_counter(sb, btree_stale_read); @@ -706,7 +699,7 @@ retry: /* done if not dirtying or already dirty */ if (!(flags & BTW_DIRTY) || - (le64_to_cpu(bt->seq) >= bti->first_dirty_seq)) { + (le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq)) { ret = 0; goto out; } @@ -784,15 +777,15 @@ retry: bt = new; new = NULL; memset(bt, 0, SCOUTFS_BLOCK_SIZE); - bt->fsid = super->hdr.fsid; + bt->hdr.fsid = super->hdr.fsid; bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); } - bt->blkno = cpu_to_le64(blkno); - bt->seq = cpu_to_le64(seq); + bt->hdr.blkno = cpu_to_le64(blkno); + bt->hdr.seq = cpu_to_le64(seq); if (ref) { - ref->blkno = bt->blkno; - ref->seq = bt->seq; + ref->blkno = bt->hdr.blkno; + ref->seq = bt->hdr.seq; } ret = 0; @@ -816,8 +809,8 @@ static void create_parent_item(struct scoutfs_btree_ring *bring, void *key, unsigned key_len) { struct scoutfs_btree_ref ref = { - .blkno = child->blkno, - .seq = child->seq, + .blkno = child->hdr.blkno, + .seq = child->hdr.seq, }; create_item(parent, pos, key, key_len, &ref, sizeof(ref)); @@ -888,8 +881,8 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, parent->level = root->height; root->height++; - root->ref.blkno = parent->blkno; - root->ref.seq = parent->seq; + root->ref.blkno = parent->hdr.blkno; + root->ref.seq = parent->hdr.seq; pos = 0; create_parent_item(bring, parent, pos, right, @@ -975,8 +968,8 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, /* and finally shrink the tree if our parent is the root with 1 */ if (le16_to_cpu(parent->nr_items) == 1) { root->height--; - root->ref.blkno = bt->blkno; - root->ref.seq = bt->seq; + root->ref.blkno = bt->hdr.blkno; + root->ref.seq = bt->hdr.seq; } put_btree_block(sib); @@ -1041,7 +1034,7 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) out: if (bad) { printk("bt %p blkno %llu level %d end %u reclaim %u nr %u (after %u bytes %u)\n", - bt, le64_to_cpu(bt->blkno), level, + bt, le64_to_cpu(bt->hdr.blkno), level, le16_to_cpu(bt->free_end), le16_to_cpu(bt->free_reclaim), le16_to_cpu(bt->nr_items), after_off, bytes); @@ -1160,8 +1153,8 @@ restart: root->height, le64_to_cpu(root->ref.blkno), le64_to_cpu(root->ref.seq), - le64_to_cpu(bt->blkno), - le64_to_cpu(bt->seq), bt->level, + le64_to_cpu(bt->hdr.blkno), + le64_to_cpu(bt->hdr.seq), bt->level, level); ret = -EIO; break; @@ -1201,8 +1194,8 @@ restart: root->height, le64_to_cpu(root->ref.blkno), le64_to_cpu(root->ref.seq), - le64_to_cpu(bt->blkno), - le64_to_cpu(bt->seq), bt->level, + le64_to_cpu(bt->hdr.blkno), + le64_to_cpu(bt->hdr.seq), bt->level, nr, pos, cmp); ret = -EIO; break; @@ -1653,8 +1646,8 @@ int scoutfs_btree_write_dirty(struct super_block *sb) /* checksum everything to reduce time between io submission merging */ for_each_dirty_bh(bti, bh, tmp) { bt = (void *)bh->b_data; - bt->crc = 0; - bt->crc = cpu_to_le32(crc32c(~0, bt, SCOUTFS_BLOCK_SIZE)); + bt->hdr._pad = 0; + bt->hdr.crc = scoutfs_block_calc_crc(&bt->hdr); } blk_start_plug(&plug); diff --git a/kmod/src/client.c b/kmod/src/client.c index fe837420..69b12248 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -271,7 +271,7 @@ static int client_connect(struct client_info *client) break; } - ret = scoutfs_read_supers(sb, &super); + ret = scoutfs_read_super(sb, &super); if (ret) continue; diff --git a/kmod/src/format.h b/kmod/src/format.h index 65c10b69..9addabf5 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -33,17 +33,15 @@ #define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) /* - * The super blocks leave some room at the start of the first block for - * platform structures like boot loaders. + * The super block leaves some room before the first block for platform + * structures like boot loaders. */ -#define SCOUTFS_SUPER_BLKNO ((64 * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_SUPER_NR 2 +#define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) /* - * This header is found at the start of every block so that we can - * verify that it's what we were looking for. The crc and padding - * starts the block so that its calculation operations on a nice 64bit - * aligned region. + * This header is stored at the start of btree blocks and the super + * block for verification. The crc is calculated by zeroing the crc and + * padding so the buffer is large and aligned. */ struct scoutfs_block_header { __le32 crc; @@ -183,11 +181,7 @@ struct scoutfs_btree_item { } __packed; struct scoutfs_btree_block { - __le64 fsid; - __le64 blkno; - __le64 seq; - __le32 crc; - __le32 _pad; + struct scoutfs_block_header hdr; __le16 free_end; __le16 free_reclaim; __le16 nr_items; diff --git a/kmod/src/server.c b/kmod/src/server.c index 6eed54ca..a88f8721 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1340,7 +1340,7 @@ static void scoutfs_server_func(struct work_struct *work) goto out; /* publish the address for clients to connect to */ - ret = scoutfs_read_supers(sb, super); + ret = scoutfs_read_super(sb, super); if (ret) goto out; diff --git a/kmod/src/super.c b/kmod/src/super.c index aaea4cba..8e7e12fd 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -22,6 +22,7 @@ #include #include "super.h" +#include "block.h" #include "export.h" #include "format.h" #include "inode.h" @@ -155,7 +156,7 @@ static const struct super_operations scoutfs_super_ops = { }; /* - * The caller advances the block number and sequence number in the super + * The caller advances the sequence number in the super block header * every time it wants to dirty it and eventually write it to reference * dirty data that's been written. */ @@ -164,13 +165,7 @@ void scoutfs_advance_dirty_super(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - le64_add_cpu(&super->hdr.blkno, 1); - if (le64_to_cpu(super->hdr.blkno) == (SCOUTFS_SUPER_BLKNO + - SCOUTFS_SUPER_NR)) - super->hdr.blkno = cpu_to_le64(SCOUTFS_SUPER_BLKNO); - le64_add_cpu(&super->hdr.seq, 1); - trace_scoutfs_advance_dirty_super(sb, le64_to_cpu(super->hdr.seq)); } @@ -193,6 +188,8 @@ int scoutfs_write_dirty_super(struct super_block *sb) super = page_address(page); memcpy(super, &sbi->super, sizeof(*super)); + super->hdr._pad = 0; + super->hdr.crc = scoutfs_block_calc_crc(&super->hdr); ret = scoutfs_bio_write(sb, &page, le64_to_cpu(super->hdr.blkno), 1); WARN_ON_ONCE(ret); @@ -203,67 +200,64 @@ int scoutfs_write_dirty_super(struct super_block *sb) } /* - * Read the pair of super blocks and store the most recent one in the sb - * info. Clients reference but don't modify the super. The server has - * to re-read the super every time it comes up so that it can work from - * the most recent persistent state. + * Read the super block. If it's valid store it in the caller's super + * struct. */ -int scoutfs_read_supers(struct super_block *sb, - struct scoutfs_super_block *local) +int scoutfs_read_super(struct super_block *sb, + struct scoutfs_super_block *super_res) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; struct page *page; - int found = -1; + __le32 calc; int ret; - int i; - u64 seq = 0; page = alloc_page(GFP_KERNEL); if (!page) return -ENOMEM; - for (i = 0; i < SCOUTFS_SUPER_NR; i++) { - - ret = scoutfs_bio_read(sb, &page, SCOUTFS_SUPER_BLKNO + i, 1); - if (ret) { - scoutfs_warn(sb, "couldn't read super block %u", i); - continue; - } - - super = scoutfs_page_block_address(&page, 0); - - if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { - scoutfs_warn(sb, "super block %u has invalid id %llx", - i, le64_to_cpu(super->id)); - continue; - } - - if (super->format_hash != cpu_to_le64(SCOUTFS_FORMAT_HASH)) { - scoutfs_warn(sb, "super block %u has invalid format hash 0x%llx, expected 0x%llx", - i, le64_to_cpu(super->format_hash), - SCOUTFS_FORMAT_HASH); - continue; - } - - if (found < 0 || (le64_to_cpu(super->hdr.seq) > seq)) { - *local = *super; - seq = le64_to_cpu((*local).hdr.seq); - found = i; - } + ret = scoutfs_bio_read(sb, &page, SCOUTFS_SUPER_BLKNO, 1); + if (ret) { + scoutfs_err(sb, "error reading super block: %d", ret); + goto out; } + super = scoutfs_page_block_address(&page, 0); + + if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { + scoutfs_err(sb, "super block has invalid id %llx", + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + calc = scoutfs_block_calc_crc(&super->hdr); + if (calc != super->hdr.crc) { + scoutfs_err(sb, "super block has invalid crc 0x%08x, calculated 0x%08x", + le32_to_cpu(super->hdr.crc), le32_to_cpu(calc)); + ret = -EINVAL; + } + + if (le64_to_cpu(super->hdr.blkno) != SCOUTFS_SUPER_BLKNO) { + scoutfs_err(sb, "super block has invalid block number %llu, data read from %llu", + le64_to_cpu(super->hdr.blkno), SCOUTFS_SUPER_BLKNO); + ret = -EINVAL; + goto out; + } + + + if (super->format_hash != cpu_to_le64(SCOUTFS_FORMAT_HASH)) { + scoutfs_err(sb, "super block has invalid format hash 0x%llx, expected 0x%llx", + le64_to_cpu(super->format_hash), + SCOUTFS_FORMAT_HASH); + ret = -EINVAL; + goto out; + } + + *super_res = *super; + ret = 0; +out: __free_page(page); - - if (found < 0) { - scoutfs_err(sb, "unable to read valid super block"); - return -EINVAL; - } - - scoutfs_info(sb, "using super %u with seq %llu", - found, le64_to_cpu(sbi->super.hdr.seq)); - - return 0; + return ret; } static int scoutfs_debugfs_setup(struct super_block *sb) @@ -328,7 +322,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) ret = scoutfs_setup_sysfs(sb) ?: scoutfs_setup_counters(sb) ?: - scoutfs_read_supers(sb, &SCOUTFS_SB(sb)->super) ?: + scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: scoutfs_options_setup(sb) ?: scoutfs_setup_triggers(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index fdc15efc..8e9cece1 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -79,8 +79,8 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } -int scoutfs_read_supers(struct super_block *sb, - struct scoutfs_super_block *local); +int scoutfs_read_super(struct super_block *sb, + struct scoutfs_super_block *super_res); void scoutfs_advance_dirty_super(struct super_block *sb); int scoutfs_write_dirty_super(struct super_block *sb); From 784cda9beea39df4a9ef3d06cc2049a55fb422af Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 29 Jun 2018 13:10:52 -0700 Subject: [PATCH 649/920] scoutfs: more carefully set lock bast mode Locks get a bast call from the dlm when a remote node is blocked waiting for the mode of a lock to change. We'd set the mode that we need to convert to and kick off lock work to make forward progress. The bast calls can happen at any old time. If a call came in as we were unlocking a lock we'd set its bast mode even though it was being unlocked and would not need to be down converted. Usually this bad mode would be fine because the lock was idle and would just be freed after being locked. But if someone was actively waiting for the lock it would get stuck in an unlocked state. The bad bast mode would prevent it from being upconverted, but the waiters would stop it from being freed. We fix this by only setting the mode from the bast call if there is really work to do. This avoids setting the bast for unlocked locks which will let the lock state machine re-acquire them and make forward progress on behalf of the waiters. Signed-off-by: Zach Brown --- kmod/src/lock.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index f9935c56..20ebc225 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -281,6 +281,16 @@ static bool lock_modes_match(int granted, int user) (granted == DLM_LOCK_EX && user == DLM_LOCK_PR); } +/* + * This isn't strictly the same as being compatible.. callers + * need to be very careful to understand the combinations of + * modes that are possible for them to attempt. + */ +static bool lock_mode_valid_and_greater(int mode, int other) +{ + return mode != DLM_LOCK_IV && mode > other; +} + /* * Returns true if all the actively used modes are satisfied by a lock * of the given granted mode. @@ -616,24 +626,35 @@ static void scoutfs_lock_ast(void *arg) * that we should convert our lock to. We can only either downconvert * to a matching PR or unlock. * - * These are truly asynchronous and can arrive multiple times, at any time. - * We're careful to only set the bast mode here and let lock processing - * sort out the state machine. + * These are truly asynchronous and can arrive multiple times, at any + * time. We're careful to only set the lock's bast mode here if the + * mode that's required conflicts with the lock's current mode, the mode + * the work might be converting to, or the next mode from a previous + * bast. This stops us from setting the lock's bast mode when it isn't + * needed a confusing the state machine, like for a lock that's being + * unlocked. */ static void scoutfs_lock_bast(void *arg, int blocked_mode) { struct scoutfs_lock *lock = arg; struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); + int bast_mode; scoutfs_inc_counter(sb, lock_bast); spin_lock(&linfo->lock); if (lock->granted_mode == DLM_LOCK_EX && blocked_mode == DLM_LOCK_PR) - lock->bast_mode = DLM_LOCK_PR; + bast_mode = DLM_LOCK_PR; else - lock->bast_mode = DLM_LOCK_NL; + bast_mode = DLM_LOCK_NL; + + /* greater is safe, only try nl < all or pr < ex */ + if (lock_mode_valid_and_greater(lock->granted_mode, bast_mode) || + lock_mode_valid_and_greater(lock->work_mode, bast_mode) || + lock_mode_valid_and_greater(lock->bast_mode, bast_mode)) + lock->bast_mode = bast_mode; trace_scoutfs_lock_bast(sb, lock); lock_process(linfo, lock); From 295bf6b73bc9e3e3ee34d2c5ec66bcb54cfe26da Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 5 Jul 2018 10:28:02 -0700 Subject: [PATCH 650/920] scoutfs: return free extents to server Freed file data extents are tracked in free extent items in each node. They could only be re-used in the future for file data extent allocation on that node. Allocations on other nodes or, critically, segment allocation on the server could never see those free extents. With the right allocation patterns, particularly allocating on node X and freeing on node Y, all the free extents can build up on a node and starve other allocations. This adds a simple high water mark after which nodes start returning free extents to the server. From there they can satisfy segment allocations or be sent to other nodes for file data extent allocation. Signed-off-by: Zach Brown --- kmod/src/client.c | 12 ++++ kmod/src/client.h | 2 + kmod/src/count.h | 15 +++++ kmod/src/data.c | 139 ++++++++++++++++++++++++++++++++++++++- kmod/src/format.h | 15 +++++ kmod/src/scoutfs_trace.h | 12 ++++ kmod/src/server.c | 49 ++++++++++++++ kmod/src/super.c | 3 +- 8 files changed, 244 insertions(+), 3 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 69b12248..fedaee8d 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -582,6 +582,18 @@ int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, return ret; } +int scoutfs_client_free_extents(struct super_block *sb, + struct scoutfs_net_extent_list *nexl) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + unsigned int bytes; + + bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(le64_to_cpu(nexl->nr)); + + return client_request(client, SCOUTFS_NET_FREE_EXTENTS, + nexl, bytes, NULL, 0); +} + int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) { struct client_info *client = SCOUTFS_SB(sb)->client_info; diff --git a/kmod/src/client.h b/kmod/src/client.h index 259454dd..f0a8d609 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -5,6 +5,8 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, u64 *len); +int scoutfs_client_free_extents(struct super_block *sb, + struct scoutfs_net_extent_list *nexl); int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); diff --git a/kmod/src/count.h b/kmod/src/count.h index 41817ea7..db35ebec 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -277,6 +277,21 @@ SIC_TRUNC_EXTENT(struct inode *inode) return cnt; } +/* + * Returning extents to the server can, at most: + * - delete MAX_NR extents with indexed copies + * - create an extent for the leftovers of the last extent + */ +static inline const struct scoutfs_item_count SIC_RETURN_EXTENTS(void) +{ + struct scoutfs_item_count cnt = {0,}; + unsigned int nr = SCOUTFS_NET_EXTENT_LIST_MAX_NR + 1; + + cnt.items += (nr * 2); + + return cnt; +} + /* * Fallocating an extent can, at most: * - allocate from the server: delete two free and insert merged diff --git a/kmod/src/data.c b/kmod/src/data.c index 07b90294..5485af0b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -38,6 +39,7 @@ #include "file.h" #include "extents.h" #include "msg.h" +#include "count.h" /* * scoutfs uses extent items to track file data block mappings and free @@ -72,9 +74,17 @@ * We ask for a fixed size from the server today. */ #define SERVER_ALLOC_BLOCKS (MAX_EXTENT_BLOCKS * 8) +/* + * Send free extents back to the server if we have plenty locally. + */ +#define NODE_FREE_HIGH_WATER_BLOCKS (SERVER_ALLOC_BLOCKS * 16) struct data_info { + struct super_block *sb; struct rw_semaphore alloc_rwsem; + atomic64_t node_free_blocks; + struct workqueue_struct *workq; + struct work_struct return_work; }; #define DECLARE_DATA_INFO(sb, name) \ @@ -148,10 +158,16 @@ static int init_extent_from_item(struct scoutfs_extent *ext, * We also index free extents by their length. We implement that by * keeping their _BLOCKS_ item in sync with the primary _BLKNO_ item * that callers operate on. + * + * The count of free blocks stored in node items is kept consistent by + * updating the count every time we create or delete items. Updated + * extents are deleted and then recreated so the count can bounce around + * a bit, but it's OK for it to be imprecise at the margins. */ static int data_extent_io(struct super_block *sb, int op, struct scoutfs_extent *ext, void *data) { + DECLARE_DATA_INFO(sb, datinf); struct scoutfs_lock *lock = data; struct scoutfs_file_extent fex; struct scoutfs_key first; @@ -230,6 +246,13 @@ static int data_extent_io(struct super_block *sb, int op, } } + if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + if (op == SEI_INSERT) + atomic64_add(ext->len, &datinf->node_free_blocks); + else if (op == SEI_DELETE) + atomic64_sub(ext->len, &datinf->node_free_blocks); + } + return ret; } @@ -252,6 +275,7 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_DATA_INFO(sb, datinf); struct scoutfs_extent next; struct scoutfs_extent rem; struct scoutfs_extent fr; @@ -324,6 +348,11 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, scoutfs_inode_add_onoff(inode, online_delta, offline_delta); + /* start returning free extents to the server after a small delay */ + if (rem.map && (atomic64_read(&datinf->node_free_blocks) > + NODE_FREE_HIGH_WATER_BLOCKS)) + queue_work(datinf->workq, &datinf->return_work); + ret = 1; out: scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, @@ -1238,6 +1267,94 @@ const struct file_operations scoutfs_file_fops = { .fallocate = scoutfs_fallocate, }; +/* + * Return extents to the server if we're over the high water mark. Each + * work call sends one batch of extents so that the work can be easily + * canceled to stop progress during unmount. + */ +static void scoutfs_data_return_server_extents_worker(struct work_struct *work) +{ + struct data_info *datinf = container_of(work, struct data_info, + return_work); + struct super_block *sb = datinf->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_extent_list *nexl; + struct scoutfs_extent ext; + u64 nr = 0; + u64 free; + int bytes; + int ret; + int err; + + trace_scoutfs_data_return_server_extents_enter(sb, 0, 0); + + bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(SCOUTFS_NET_EXTENT_LIST_MAX_NR); + nexl = kmalloc(bytes, GFP_NOFS); + if (!nexl) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_hold_trans(sb, SIC_RETURN_EXTENTS()); + if (ret) + goto out; + + down_write(&datinf->alloc_rwsem); + + free = atomic64_read(&datinf->node_free_blocks); + + while (nr < SCOUTFS_NET_EXTENT_LIST_MAX_NR && + free > NODE_FREE_HIGH_WATER_BLOCKS) { + + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, + sbi->node_id, 0, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, + sbi->node_id_lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + trace_scoutfs_data_return_server_extent(sb, &ext); + + ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + ext.len = min(ext.len, free - NODE_FREE_HIGH_WATER_BLOCKS); + + ret = scoutfs_extent_remove(sb, data_extent_io, &ext, + sbi->node_id_lock); + if (ret) + break; + + nexl->extents[nr].start = cpu_to_le64(ext.start); + nexl->extents[nr].len = cpu_to_le64(ext.len); + + nr++; + free -= ext.len; + } + + nexl->nr = cpu_to_le64(nr); + + up_write(&datinf->alloc_rwsem); + + if (nr > 0) { + err = scoutfs_client_free_extents(sb, nexl); + /* XXX leaked extents if free failed */ + if (ret == 0 && err < 0) + ret = err; + } + + scoutfs_release_trans(sb); +out: + kfree(nexl); + + trace_scoutfs_data_return_server_extents_exit(sb, nr, ret); + + /* keep returning if we're still over the water mark */ + if (ret == 0 && (atomic64_read(&datinf->node_free_blocks) > + NODE_FREE_HIGH_WATER_BLOCKS)) + queue_work(datinf->workq, &datinf->return_work); +} int scoutfs_data_setup(struct super_block *sb) { @@ -1248,10 +1365,19 @@ int scoutfs_data_setup(struct super_block *sb) if (!datinf) return -ENOMEM; + datinf->sb = sb; init_rwsem(&datinf->alloc_rwsem); + atomic64_set(&datinf->node_free_blocks, 0); + INIT_WORK(&datinf->return_work, + scoutfs_data_return_server_extents_worker); + + datinf->workq = alloc_workqueue("scoutfs_data", 0, 1); + if (!datinf->workq) { + kfree(datinf); + return -ENOMEM; + } sbi->data_info = datinf; - return 0; } @@ -1260,5 +1386,14 @@ void scoutfs_data_destroy(struct super_block *sb) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct data_info *datinf = sbi->data_info; - kfree(datinf); + if (datinf) { + if (datinf->workq) { + cancel_work_sync(&datinf->return_work); + destroy_workqueue(datinf->workq); + datinf->workq = NULL; + } + + sbi->data_info = NULL; + kfree(datinf); + } } diff --git a/kmod/src/format.h b/kmod/src/format.h index 9addabf5..e1dfc9a2 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -562,6 +562,20 @@ struct scoutfs_net_extent { __le64 len; } __packed; +struct scoutfs_net_extent_list { + __le64 nr; + struct { + __le64 start; + __le64 len; + } __packed extents[0]; +} __packed; + +#define SCOUTFS_NET_EXTENT_LIST_BYTES(nr) \ + offsetof(struct scoutfs_net_extent_list, extents[nr]) + +/* arbitrarily makes a nice ~1k extent list payload */ +#define SCOUTFS_NET_EXTENT_LIST_MAX_NR 64 + /* XXX eventually we'll have net compaction and will need agents to agree */ /* one upper segment and fanout lower segments */ @@ -575,6 +589,7 @@ struct scoutfs_net_extent { enum { SCOUTFS_NET_ALLOC_INODES = 0, SCOUTFS_NET_ALLOC_EXTENT, + SCOUTFS_NET_FREE_EXTENTS, SCOUTFS_NET_ALLOC_SEGNO, SCOUTFS_NET_RECORD_SEGMENT, SCOUTFS_NET_ADVANCE_SEQ, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index fd3a0f6c..b223d70d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1726,6 +1726,14 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_server_workqueue_destroy, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); +DEFINE_EVENT(scoutfs_work_class, scoutfs_data_return_server_extents_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_data_return_server_extents_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); TRACE_EVENT(scoutfs_item_next_range_check, TP_PROTO(struct super_block *sb, int cached, @@ -2149,6 +2157,10 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); +DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_return_server_extent, + TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), + TP_ARGS(sb, ext) +); DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_next, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) diff --git a/kmod/src/server.c b/kmod/src/server.c index a88f8721..da284854 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -743,6 +743,54 @@ out: return send_reply(conn, id, type, ret, &nex, sizeof(nex)); } +static bool invalid_net_extent_list(struct scoutfs_net_extent_list *nexl, + unsigned data_len) +{ + return (data_len < sizeof(struct scoutfs_net_extent_list)) || + (le64_to_cpu(nexl->nr) > SCOUTFS_NET_EXTENT_LIST_MAX_NR) || + (data_len != offsetof(struct scoutfs_net_extent_list, + extents[le64_to_cpu(nexl->nr)])); +} + +static int process_free_extents(struct server_connection *conn, + u64 id, u8 type, void *data, unsigned data_len) +{ + struct server_info *server = conn->server; + struct super_block *sb = server->sb; + struct scoutfs_net_extent_list *nexl; + struct commit_waiter cw; + int ret = 0; + int err; + u64 i; + + nexl = data; + if (invalid_net_extent_list(nexl, data_len)) { + ret = -EINVAL; + goto out; + } + + down_read(&server->commit_rwsem); + + for (i = 0; i < le64_to_cpu(nexl->nr); i++) { + ret = free_extent(sb, le64_to_cpu(nexl->extents[i].start), + le64_to_cpu(nexl->extents[i].len)); + if (ret) + break; + } + + if (i > 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + + if (i > 0) { + err = wait_for_commit(server, &cw, id, type); + if (ret == 0) + ret = err; + } +out: + return send_reply(conn, id, type, ret, NULL, 0); +} + /* * We still special case segno allocation because it's aligned and we'd * like to keep that detail in the server. @@ -1091,6 +1139,7 @@ static void scoutfs_server_process_func(struct work_struct *work) static process_func_t process_funcs[] = { [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, [SCOUTFS_NET_ALLOC_EXTENT] = process_alloc_extent, + [SCOUTFS_NET_FREE_EXTENTS] = process_free_extents, [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, diff --git a/kmod/src/super.c b/kmod/src/super.c index 8e7e12fd..4279ea96 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -120,12 +120,13 @@ static void scoutfs_put_super(struct super_block *sb) sbi->shutdown = true; + scoutfs_data_destroy(sb); + scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); sbi->node_id_lock = NULL; scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); - scoutfs_data_destroy(sb); scoutfs_inode_destroy(sb); scoutfs_item_destroy(sb); From 17dec65a527991642833d2ba2973a81cd2f32152 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Jul 2018 11:37:33 -0700 Subject: [PATCH 651/920] scoutfs: add bidirectional network messages The client and server networking code was a bit too rudimentary. The existing code only had support for the client synchronously and actively sending requests that the server could only passively respond to. We're going to need the server to be able to send requests to connected clients and it can't block waiting for responses from each one. This refactors sending and receiving in both the client and server code into shared networking code. It's built around a connection struct that then holds the message state. Both peers on the connection can send requests and send responses. The existing code only retransmitted requests down newly established connections. Requests could be processed twice. This adds robust reliability guarantees. Requests are resend until their response is received. Requests are only processed once by a given peer, regardless of the connection's transport socket. Responses are reiably resent until acknowledged. This only adds the new refactored code and disables the old unused code to keep the diff foot print minmal. A following commit will remove all the unused code. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/client.c | 231 ++++-- kmod/src/counters.h | 12 + kmod/src/format.h | 85 ++- kmod/src/net.c | 1495 ++++++++++++++++++++++++++++++++++++++ kmod/src/net.h | 61 ++ kmod/src/scoutfs_trace.h | 60 +- kmod/src/server.c | 401 +++++----- kmod/src/server.h | 16 +- kmod/src/super.c | 3 + kmod/src/super.h | 2 + 11 files changed, 2025 insertions(+), 343 deletions(-) create mode 100644 kmod/src/net.c create mode 100644 kmod/src/net.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index c9375a0b..237e0f11 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -7,7 +7,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o lock.o \ - manifest.o msg.o options.o per_task.o seg.o server.o \ + manifest.o msg.o net.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ triggers.o xattr.o diff --git a/kmod/src/client.c b/kmod/src/client.c index fedaee8d..c7e2cdac 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -32,24 +32,18 @@ #include "msg.h" #include "server.h" #include "client.h" -#include "sock.h" +#include "net.h" #include "endian_swap.h" /* - * Client callers block sending requests to the server. Senders connect - * and send down the socket in their blocked context under a mutex. - * Once a socket is connected recv work is fired up. Destroying a - * socket shuts down the socket and cancels the work. - * - * Clients are responsible for resending their requests after - * reconnecting to a new socket. These new socket connections might be - * connecting to the same server. The message sending and processing - * paths are responsible for dealing with duplicate requests. + * The client always maintains a connection to the server. It reads the + * super to get the address it should try and connect to. */ #define SIN_FMT "%pIS:%u" #define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) +#if 0 /* * Have a pretty aggressive keepalive timeout of around 10 seconds. The * TCP keepalives are being processed out of task context so they should @@ -59,6 +53,7 @@ #define KEEPCNT 3 #define KEEPIDLE 7 #define KEEPINTVL 1 +#endif /* @@ -72,30 +67,17 @@ struct client_info { struct super_block *sb; - /* spinlock protects quick critical sections between send,recv,umount */ - spinlock_t recv_lock; - struct rb_root sender_root; + struct scoutfs_net_connection *conn; + atomic_t shutting_down; - /* the sock mutex serializes connecting and sending */ - struct mutex send_mutex; - bool recv_shutdown; - u64 next_id; - u64 sock_gen; - struct socket *sock; - struct sockaddr_in peername; - struct sockaddr_in sockname; - - /* blocked senders sit on a waitq that's woken for resends */ - wait_queue_head_t waitq; + struct workqueue_struct *workq; + struct delayed_work connect_dwork; /* connection timeouts are tracked across attempts */ unsigned long conn_retry_ms; - unsigned long conn_retry_limit_j; - - struct workqueue_struct *recv_wq; - struct work_struct recv_work; }; +#if 0 struct waiting_sender { struct rb_node node; struct task_struct *task; @@ -213,14 +195,20 @@ static void scoutfs_client_recv_func(struct work_struct *work) kfree(rx); } +#endif -static void reset_connect_timeouts(struct client_info *client) +static void reset_connect_timeout(struct client_info *client) { client->conn_retry_ms = CONN_RETRY_MIN_MS; - client->conn_retry_limit_j = jiffies + CONN_RETRY_LIMIT_J; } +static void grow_connect_timeout(struct client_info *client) +{ + client->conn_retry_ms = min(client->conn_retry_ms * 2, + CONN_RETRY_MAX_MS); +} +#if 0 /* * Clients who try to send and don't see a connected socket call here to * connect to the server. They get the server address and try to @@ -527,6 +515,7 @@ static int client_request(struct client_info *client, int type, void *data, return ret; } +#endif /* * Ask for a new run of allocated inode numbers. The server can return @@ -540,8 +529,10 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, __le64 lecount = cpu_to_le64(count); int ret; - ret = client_request(client, SCOUTFS_NET_ALLOC_INODES, - &lecount, sizeof(lecount), &ial, sizeof(ial)); + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ALLOC_INODES, + &lecount, sizeof(lecount), + &ial, sizeof(ial)); if (ret == 0) { *ino = le64_to_cpu(ial.ino); *nr = le64_to_cpu(ial.nr); @@ -568,8 +559,10 @@ int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, struct scoutfs_net_extent nex; int ret; - ret = client_request(client, SCOUTFS_NET_ALLOC_EXTENT, - &leblocks, sizeof(leblocks), &nex, sizeof(nex)); + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ALLOC_EXTENT, + &leblocks, sizeof(leblocks), + &nex, sizeof(nex)); if (ret == 0) { if (nex.len == 0) { ret = -ENOSPC; @@ -590,8 +583,9 @@ int scoutfs_client_free_extents(struct super_block *sb, bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(le64_to_cpu(nexl->nr)); - return client_request(client, SCOUTFS_NET_FREE_EXTENTS, - nexl, bytes, NULL, 0); + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_FREE_EXTENTS, + nexl, bytes, NULL, 0); } int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) @@ -600,8 +594,9 @@ int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) __le64 lesegno; int ret; - ret = client_request(client, SCOUTFS_NET_ALLOC_SEGNO, NULL, 0, - &lesegno, sizeof(lesegno)); + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ALLOC_SEGNO, + NULL, 0, &lesegno, sizeof(lesegno)); if (ret == 0) { if (lesegno == 0) ret = -ENOSPC; @@ -622,8 +617,9 @@ int scoutfs_client_record_segment(struct super_block *sb, scoutfs_seg_init_ment(&ment, level, seg); scoutfs_init_ment_to_net(&net_ment, &ment); - return client_request(client, SCOUTFS_NET_RECORD_SEGMENT, &net_ment, - sizeof(net_ment), NULL, 0); + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_RECORD_SEGMENT, + &net_ment, sizeof(net_ment), NULL, 0); } int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) @@ -633,8 +629,10 @@ int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) __le64 after; int ret; - ret = client_request(client, SCOUTFS_NET_ADVANCE_SEQ, - &before, sizeof(before), &after, sizeof(after)); + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_ADVANCE_SEQ, + &before, sizeof(before), + &after, sizeof(after)); if (ret == 0) *seq = le64_to_cpu(after); @@ -647,8 +645,9 @@ int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq) __le64 last_seq; int ret; - ret = client_request(client, SCOUTFS_NET_GET_LAST_SEQ, - NULL, 0, &last_seq, sizeof(last_seq)); + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_LAST_SEQ, + NULL, 0, &last_seq, sizeof(last_seq)); if (ret == 0) *seq = le64_to_cpu(last_seq); @@ -660,8 +659,10 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; - return client_request(client, SCOUTFS_NET_GET_MANIFEST_ROOT, - NULL, 0, root, sizeof(struct scoutfs_btree_root)); + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, + NULL, 0, root, + sizeof(struct scoutfs_btree_root)); } int scoutfs_client_statfs(struct super_block *sb, @@ -669,53 +670,147 @@ int scoutfs_client_statfs(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; - return client_request(client, SCOUTFS_NET_STATFS, NULL, 0, nstatfs, - sizeof(struct scoutfs_net_statfs)); + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_STATFS, NULL, 0, + nstatfs, + sizeof(struct scoutfs_net_statfs)); +} + +/* + * Attempt to connect to the listening address that the server wrote in + * the super block. We keep trying indefinitely with an increasing + * delay if we fail to either read the address or connect to it. + * + * We're careful to only ever have one connection attempt in flight. We + * only queue this work on mount, on error, or from the connection + * callback. + */ +static void scoutfs_client_connect_worker(struct work_struct *work) +{ + struct client_info *client = container_of(work, struct client_info, + connect_dwork.work); + struct super_block *sb = client->sb; + struct scoutfs_super_block super; + struct sockaddr_in sin; + int ret; + + ret = scoutfs_read_super(sb, &super); + if (ret) + goto out; + + if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) { + ret = -EADDRNOTAVAIL; + goto out; + } + + memset(&sin, 0, sizeof(sin)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = le32_to_be32(super.server_addr.addr); + sin.sin_port = le16_to_be16(super.server_addr.port); + + scoutfs_net_connect(sb, client->conn, &sin, client->conn_retry_ms); + ret = 0; +out: + if (ret && !atomic_read(&client->shutting_down)) { + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(client->conn_retry_ms)); + grow_connect_timeout(client); + } +} + +static void client_notify_up(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + reset_connect_timeout(client); +} + +/* + * Called when either a connect attempt or established connection times + * out and fails. + */ +static void client_notify_down(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + if (!atomic_read(&client->shutting_down)) { + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(client->conn_retry_ms)); + grow_connect_timeout(client); + } } int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct client_info *client; + int ret; client = kzalloc(sizeof(struct client_info), GFP_KERNEL); - if (!client) - return -ENOMEM; + if (!client) { + ret = -ENOMEM; + goto out; + } + sbi->client_info = client; client->sb = sb; - spin_lock_init(&client->recv_lock); - client->sender_root = RB_ROOT; - mutex_init(&client->send_mutex); - init_waitqueue_head(&client->waitq); - INIT_WORK(&client->recv_work, scoutfs_client_recv_func); - reset_connect_timeouts(client); + atomic_set(&client->shutting_down, 0); + INIT_DELAYED_WORK(&client->connect_dwork, + scoutfs_client_connect_worker); - client->recv_wq = alloc_workqueue("scoutfs_client_recv", WQ_UNBOUND, 1); - if (!client->recv_wq) { - kfree(client); - return -ENOMEM; + /* client doesn't process any incoming requests yet */ + client->conn = scoutfs_net_alloc_conn(sb, client_notify_up, + client_notify_down, NULL, + "client"); + if (!client->conn) { + ret = -ENOMEM; + goto out; } - sbi->client_info = client; - return 0; + client->workq = alloc_workqueue("scoutfs_client_workq", WQ_UNBOUND, 1); + if (!client->workq) { + ret = -ENOMEM; + goto out; + } + + reset_connect_timeout(client); + /* delay initial connect to give a local server some time to setup */ + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(client->conn_retry_ms)); + ret = 0; + +out: + if (ret) + scoutfs_client_destroy(sb); + return ret; } /* - * There must be no more callers to the client send functions by the - * time we get here. We just need to free the socket if it's - * still sitting around. + * There must be no more callers to the client request functions by the + * time we get here. */ void scoutfs_client_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_connection *conn; if (client) { - shutdown_sock_sync(client); + /* stop notify_down from queueing connect work */ + atomic_set(&client->shutting_down, 1); - cancel_work_sync(&client->recv_work); - destroy_workqueue(client->recv_wq); + /* make sure worker isn't using the conn */ + cancel_delayed_work_sync(&client->connect_dwork); + /* make racing conn use explode */ + conn = client->conn; + client->conn = NULL; + scoutfs_net_free_conn(sb, conn); + + if (client->workq) + destroy_workqueue(client->workq); kfree(client); sbi->client_info = NULL; } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 6d26c4f7..94d175c2 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -100,6 +100,18 @@ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(manifest_read_excluded_key) \ + EXPAND_COUNTER(net_dropped_ack) \ + EXPAND_COUNTER(net_dropped_response) \ + EXPAND_COUNTER(net_dropped_request) \ + EXPAND_COUNTER(net_send_bytes) \ + EXPAND_COUNTER(net_send_error) \ + EXPAND_COUNTER(net_send_messages) \ + EXPAND_COUNTER(net_recv_bytes) \ + EXPAND_COUNTER(net_recv_error) \ + EXPAND_COUNTER(net_recv_invalid_message) \ + EXPAND_COUNTER(net_recv_messages) \ + EXPAND_COUNTER(net_unknown_message) \ + EXPAND_COUNTER(net_unknown_request) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(seg_shrink) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index e1dfc9a2..6f2d46f8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -353,8 +353,6 @@ struct scoutfs_inet_addr { __le16 port; } __packed; -#define SCOUTFS_DEFAULT_PORT 12345 - struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -506,6 +504,10 @@ struct scoutfs_lock_name { * messages over the wire. */ +/* + * Greetings verify identity of communicating nodes. The sender + * sends their credentials and the receiver verifies them. + */ struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; @@ -517,15 +519,70 @@ struct scoutfs_net_greeting { * type is strictly redundant in the reply because the id will find the * send but we include it in both packets to make it easier to observe * replies without having the id from their previous request. + * + * Error is only set to a translated errno on response messages and + * data_len will be 0. */ struct scoutfs_net_header { __le64 id; __le16 data_len; - __u8 type; - __u8 status; + __u8 msg; + __u8 cmd; + __u8 error; __u8 data[0]; } __packed; +/* + * Greetings are the first messages sent down every newly established + * socket on the connection. Every other message gets a unique + * increasing id over the life time of the connection. + */ +#define SCOUTFS_NET_ID_GREETING 1 + +enum { + SCOUTFS_NET_MSG_REQUEST = 0, + SCOUTFS_NET_MSG_RESPONSE, + SCOUTFS_NET_MSG_ACK, + SCOUTFS_NET_MSG_UNKNOWN, +}; + +enum { + SCOUTFS_NET_CMD_GREETING = 0, + SCOUTFS_NET_CMD_ALLOC_INODES, + SCOUTFS_NET_CMD_ALLOC_EXTENT, + SCOUTFS_NET_CMD_FREE_EXTENTS, + SCOUTFS_NET_CMD_ALLOC_SEGNO, + SCOUTFS_NET_CMD_RECORD_SEGMENT, + SCOUTFS_NET_CMD_ADVANCE_SEQ, + SCOUTFS_NET_CMD_GET_LAST_SEQ, + SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, + SCOUTFS_NET_CMD_STATFS, + SCOUTFS_NET_CMD_UNKNOWN, +}; + +/* + * Define a macro to evaluate another macro for each of the errnos we + * translate over the wire. This lets us keep our enum in sync with the + * mapping arrays to and from host errnos. + */ +#define EXPAND_EACH_NET_ERRNO \ + EXPAND_NET_ERRNO(ENOENT) \ + EXPAND_NET_ERRNO(ENOMEM) \ + EXPAND_NET_ERRNO(EIO) \ + EXPAND_NET_ERRNO(ENOSPC) \ + EXPAND_NET_ERRNO(EINVAL) + +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) SCOUTFS_NET_ERR_##which, +enum { + SCOUTFS_NET_ERR_NONE = 0, + EXPAND_EACH_NET_ERRNO + SCOUTFS_NET_ERR_UNKNOWN, +}; + +/* arbitrarily chosen to be safely less than mss and allow 1k with header */ +#define SCOUTFS_NET_MAX_DATA_LEN 1100 + /* * When there's no more free inodes this will be sent with ino = ~0 and * nr = 0. @@ -586,26 +643,6 @@ struct scoutfs_net_extent_list { #define SCOUTFS_COMPACTION_MAX_UPDATE \ (2 * (SCOUTFS_COMPACTION_MAX_INPUT + SCOUTFS_COMPACTION_SLOP)) -enum { - SCOUTFS_NET_ALLOC_INODES = 0, - SCOUTFS_NET_ALLOC_EXTENT, - SCOUTFS_NET_FREE_EXTENTS, - SCOUTFS_NET_ALLOC_SEGNO, - SCOUTFS_NET_RECORD_SEGMENT, - SCOUTFS_NET_ADVANCE_SEQ, - SCOUTFS_NET_GET_LAST_SEQ, - SCOUTFS_NET_GET_MANIFEST_ROOT, - SCOUTFS_NET_STATFS, - SCOUTFS_NET_UNKNOWN, -}; - -enum { - SCOUTFS_NET_STATUS_REQUEST = 0, - SCOUTFS_NET_STATUS_SUCCESS, - SCOUTFS_NET_STATUS_ERROR, - SCOUTFS_NET_STATUS_UNKNOWN, -}; - /* * Scoutfs file handle structure - this can be copied out to userspace * via open by handle or put on the wire from NFS. diff --git a/kmod/src/net.c b/kmod/src/net.c new file mode 100644 index 00000000..1ab0b377 --- /dev/null +++ b/kmod/src/net.c @@ -0,0 +1,1495 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "inode.h" +#include "btree.h" +#include "manifest.h" +#include "seg.h" +#include "compact.h" +#include "scoutfs_trace.h" +#include "msg.h" +#include "net.h" +#include "endian_swap.h" + + +/* + * scoutfs networking reliably delivers requests and responses between + * nodes. + * + * Nodes decide to be either a connecting client or a listening server. + * Both set up a connection and specify the set of request commands they + * can process. + * + * The networking core maintains reliable request processing as the + * nodes reconnect. Requests are resent as connections are + * re-established until a response is received. Responses are resent + * until an ack is received. The connections are not bound to the + * addresses of the underlying socket transports and can reliably + * deliver messages across renumbering. + * + * XXX: + * - assign node_ids and validate with the greeting + * - defer accepted conn destruction until reconnect timeout + * - trace command and response data payloads + * - checksum message contents? + * - explicit shutdown message to free accepted, timeout and fence otherwise + * - shutdown server if accept can't alloc resources for new conn? + */ + +/* + * A connection's shutdown work executes in its own workqueue so that the + * work can free the connection's workq. + */ +struct net_info { + struct workqueue_struct *shutdown_workq; +}; + +struct scoutfs_net_connection { + struct super_block *sb; + scoutfs_net_notify_t notify_up; + scoutfs_net_notify_t notify_down; + scoutfs_net_request_t *req_funcs; + + spinlock_t lock; + + unsigned long valid_greeting:1, /* other commands can proceed */ + established:1, /* added sends queue send work */ + shutting_down:1; /* shutdown work has been queued */ + + struct sockaddr_in connect_sin; + unsigned long connect_timeout_ms; + + struct socket *sock; + struct sockaddr_in sockname; + struct sockaddr_in peername; + + struct list_head accepted_head; + struct scoutfs_net_connection *listening_conn; + struct list_head accepted_list; + wait_queue_head_t accepted_waitq; + + u64 next_send_id; + u64 last_proc_id; + struct list_head send_queue; + struct list_head resend_queue; + + struct workqueue_struct *workq; + struct work_struct listen_work; + struct work_struct connect_work; + struct work_struct send_work; + struct work_struct recv_work; + struct work_struct shutdown_work; + /* message_recv proc_work also executes in the conn workq */ +}; + +/* + * Messages to be sent are allocated and put on the send queue. + * + * Request and response messages are put on the resend queue until their + * response or ack messages are received, respectively, and they can be + * freed. + * + * The send worker is the only context that references messages while + * not holding the lock. It does this while blocking sending the + * message down the socket. To free messages we mark them dead and have + * the send worker free them while under the lock so that we don't have + * to risk freeing messages from under the unlocked send worker. + */ +struct message_send { + struct list_head head; + scoutfs_net_response_t resp_func; + void *resp_data; + unsigned long dead:1; + struct scoutfs_net_header nh; +}; + +/* + * Incoming received messages are processed in concurrent blocking work + * contexts. + */ +struct message_recv { + struct scoutfs_net_connection *conn; + struct work_struct proc_work; + struct scoutfs_net_header nh; +}; + +#define DEFINE_CONN_FROM_WORK(name, work, member) \ + struct scoutfs_net_connection *name = \ + container_of(work, struct scoutfs_net_connection, member) + +/* Total message bytes including header and payload */ +static int nh_bytes(unsigned int data_len) +{ + return offsetof(struct scoutfs_net_header, data[data_len]); +} + +static struct message_send *search_list(struct scoutfs_net_connection *conn, + struct list_head *list, + u8 msg, u8 cmd, u64 id) +{ + struct message_send *msend; + + assert_spin_locked(&conn->lock); + + list_for_each_entry(msend, list, head) { + if (msend->nh.msg == msg && msend->nh.cmd == cmd && + le64_to_cpu(msend->nh.id) == id) + return msend; + } + + return NULL; +} + +/* + * Find an active send on the lists. It's almost certainly waiting on + * the resend queue but it could be actively being sent. + */ +static struct message_send *find_send(struct scoutfs_net_connection *conn, + u8 msg, u8 cmd, u64 id) +{ + struct message_send *msend; + + msend = search_list(conn, &conn->resend_queue, msg, cmd, id) ?: + search_list(conn, &conn->send_queue, msg, cmd, id); + if (msend && msend->dead) + msend = NULL; + return msend; +} + +/* + * Complete a send message by moving it to the send queue and marking it + * to be freed. + * + * Request messages have their response function called. Their response + * processing can return an error if the response is invalid. The + * request message is still removed and freed in that case. + */ +static int complete_send(struct scoutfs_net_connection *conn, + struct message_send *msend, + void *resp, unsigned int resp_len, int error) +{ + struct super_block *sb = conn->sb; + int ret = 0; + + if (WARN_ON_ONCE(msend->dead) || + WARN_ON_ONCE(list_empty(&msend->head))) + return -EINVAL; + + assert_spin_locked(&conn->lock); + + if (msend->resp_func) + ret = msend->resp_func(sb, conn, resp, resp_len, error, + msend->resp_data); + msend->dead = 1; + list_move(&msend->head, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); + + return ret; +} + + +/* + * Translate a positive error on the wire to a negative host errno. + */ +static inline int net_err_to_host(u8 net_err) +{ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) [SCOUTFS_NET_ERR_##which] = which, + static u8 host_errnos[] = { + EXPAND_EACH_NET_ERRNO + }; + + if (net_err == SCOUTFS_NET_ERR_NONE) + return 0; + + if (net_err < ARRAY_SIZE(host_errnos) && host_errnos[net_err]) + return -host_errnos[net_err]; + + return -EINVAL; +} + +/* + * Translate a negative host errno to a positive error on the wire. + * + * The caller is our kernel run time which should have been careful with + * errnos. But mistakes happen so let's holler and translate unknown + * errors. A fun bit of trivia: sparse's array bounds detection once + * got confused by conditions in WARN_ON_ONCE(); + */ +static inline u8 net_err_from_host(struct super_block *sb, int error) +{ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) [which] = SCOUTFS_NET_ERR_##which, + static u8 net_errs[] = { + EXPAND_EACH_NET_ERRNO + }; + int ind = -error; + + if (error == 0) + return SCOUTFS_NET_ERR_NONE; + + if (error > 0 || ind >= ARRAY_SIZE(net_errs) || net_errs[ind] == 0) { + static bool warned; + if (!warned) { + warned = 1; + scoutfs_warn(sb, "host errno %d sent as EINVAL\n", + error); + } + + return -EINVAL; + } + + return net_errs[ind]; +} + +/* + * Shutdown the connection. This is called by many contexts including + * work that most complete to finish shutting down. We queue specific + * shutdown work that can wait on all the connection's other work. + * We're sure to only queue the shutdown work once. + */ +static void shutdown_conn_locked(struct scoutfs_net_connection *conn) +{ + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + + assert_spin_locked(&conn->lock); + + if (!conn->shutting_down) { + conn->established = 0; + conn->shutting_down = 1; + queue_work(ninf->shutdown_workq, &conn->shutdown_work); + } +} + +static void shutdown_conn(struct scoutfs_net_connection *conn) +{ + spin_lock(&conn->lock); + shutdown_conn_locked(conn); + spin_unlock(&conn->lock); +} + +/* + * Allocate a message and put it on the send queue. + * + * A 0 id means that we'll assign the next id from the connection once + * we hold the lock and is only valid for sending requests. + * + * This can race with connections that are either starting up and + * shutting down. We only directly queue the send work if the + * connection has passed the greeting and isn't being shut down. At all + * other times we add new sends to the resend queue. + */ +static int submit_send(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 msg, u8 cmd, u64 id, u8 net_err, + void *data, u16 data_len, + scoutfs_net_response_t resp_func, void *resp_data, + u64 *id_ret) +{ + struct message_send *msend; + + if (WARN_ON_ONCE(msg >= SCOUTFS_NET_MSG_UNKNOWN) || + WARN_ON_ONCE(cmd >= SCOUTFS_NET_CMD_UNKNOWN) || + WARN_ON_ONCE(net_err >= SCOUTFS_NET_ERR_UNKNOWN) || + WARN_ON_ONCE(data_len > SCOUTFS_NET_MAX_DATA_LEN) || + WARN_ON_ONCE(data_len && (!data || net_err)) || + WARN_ON_ONCE(net_err && (msg != SCOUTFS_NET_MSG_RESPONSE)) || + WARN_ON_ONCE(id == 0 && msg != SCOUTFS_NET_MSG_REQUEST) || + WARN_ON_ONCE((cmd == SCOUTFS_NET_CMD_GREETING) != + (id == SCOUTFS_NET_ID_GREETING))) + return -EINVAL; + + msend = kmalloc(offsetof(struct message_send, + nh.data[data_len]), GFP_NOFS); + if (!msend) + return -ENOMEM; + + spin_lock(&conn->lock); + + msend->resp_func = resp_func; + msend->resp_data = resp_data; + msend->dead = 0; + + if (id == 0) + id = conn->next_send_id++; + msend->nh.id = cpu_to_le64(id); + msend->nh.msg = msg; + msend->nh.cmd = cmd; + msend->nh.error = net_err; + msend->nh.data_len = cpu_to_le16(data_len); + if (data_len) + memcpy(msend->nh.data, data, data_len); + + if (conn->established && + (conn->valid_greeting || cmd == SCOUTFS_NET_CMD_GREETING)) { + list_add_tail(&msend->head, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); + } else { + list_add_tail(&msend->head, &conn->resend_queue); + } + + if (id_ret) + *id_ret = le64_to_cpu(msend->nh.id); + + spin_unlock(&conn->lock); + + return 0; +} + +/* + * Messages can flow once we receive a valid greeting from our peer. + * Response callers are already called under the lock, request callers + * need to acquire it. + * + * At this point greeting request processing has queued the greeting + * response message on the send queue. All the sends waiting to be + * resent need to be added to the end of the send queue after the + * greeting response. Greeting acks are sent differently and can be + * received after resend messages. + */ +static void saw_valid_greeting(struct scoutfs_net_connection *conn) +{ + struct super_block *sb = conn->sb; + + assert_spin_locked(&conn->lock); + + conn->valid_greeting = 1; + if (conn->notify_up) + conn->notify_up(sb, conn); + list_splice_tail_init(&conn->resend_queue, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); +} + +static int greeting_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, + void *data) +{ + struct scoutfs_net_greeting *gr = resp; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + int ret = 0; + + if (error) { + ret = error; + goto out; + } + + if (resp_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + saw_valid_greeting(conn); + +out: + return ret; +} + +/* + * Process an incoming greeting request. We try to send responses to + * failed greetings so that the sender can log some detail before + * shutting down. A failure to send a greeting response shuts down the + * connection. + */ +static int greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_net_greeting *gr = arg; + struct scoutfs_net_greeting greet; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + int ret = 0; + + if (arg_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", + SIN_ARG(&conn->peername), + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + greet.fsid = super->id; + greet.format_hash = super->format_hash; +out: + ret = scoutfs_net_response(sb, conn, cmd, id, ret, + &greet, sizeof(greet)); + if (ret == 0) { + spin_lock(&conn->lock); + saw_valid_greeting(conn); + spin_unlock(&conn->lock); + } + return ret; +} + + +/* + * Process an incoming response. The greeting should ensure that the + * sender won't send us unknown commands. We return an error if we see + * an unknown command because the greeting should agree on an understood + * protocol. The request function sends a response and returns an error + * if they couldn't. + */ +static int process_request(struct scoutfs_net_connection *conn, + struct message_recv *mrecv) +{ + struct super_block *sb = conn->sb; + scoutfs_net_request_t req_func = NULL; + + if (conn->listening_conn != NULL && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING) { + req_func = greeting_request; + } else if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) { + req_func = conn->req_funcs[mrecv->nh.cmd]; + } if (req_func == NULL) { + scoutfs_inc_counter(sb, net_unknown_request); + return -EINVAL; + } + + return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); +} + +/* + * An incoming response finds the queued request and calls its response + * function. We call the function and remove it from the lists before + * trying to send the ack so that we only call the response function + * once. Future duplicate responses will just resend the ack in + * response. + */ +static int process_response(struct scoutfs_net_connection *conn, + struct message_recv *mrecv) +{ + struct super_block *sb = conn->sb; + struct message_send *msend; + int ret = 0; + + spin_lock(&conn->lock); + + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, mrecv->nh.cmd, + le64_to_cpu(mrecv->nh.id)); + if (msend) + ret = complete_send(conn, msend, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), + net_err_to_host(mrecv->nh.error)); + else + scoutfs_inc_counter(sb, net_dropped_response); + + spin_unlock(&conn->lock); + + if (ret == 0) + ret = submit_send(sb, conn, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, + le64_to_cpu(mrecv->nh.id), 0, NULL, 0, NULL, + NULL, NULL); + return ret; +} + +/* + * An incoming ack frees the pending response. + */ +static void process_ack(struct scoutfs_net_connection *conn, + struct message_recv *mrecv) +{ + struct super_block *sb = conn->sb; + struct message_send *msend; + + spin_lock(&conn->lock); + + msend = find_send(conn, SCOUTFS_NET_MSG_RESPONSE, mrecv->nh.cmd, + le64_to_cpu(mrecv->nh.id)); + if (msend) + complete_send(conn, msend, NULL, 0, 0); + else + scoutfs_inc_counter(sb, net_dropped_ack); + + spin_unlock(&conn->lock); +} + +/* + * Process an incoming received message in its own concurrent blocking + * work context. + */ +static void scoutfs_net_proc_worker(struct work_struct *work) +{ + struct message_recv *mrecv = container_of(work, struct message_recv, + proc_work); + struct scoutfs_net_connection *conn = mrecv->conn; + struct super_block *sb = conn->sb; + int ret; + + trace_scoutfs_net_proc_work_enter(sb, 0, 0); + + switch (mrecv->nh.msg) { + case SCOUTFS_NET_MSG_REQUEST: + ret = process_request(conn, mrecv); + break; + case SCOUTFS_NET_MSG_RESPONSE: + ret = process_response(conn, mrecv); + break; + case SCOUTFS_NET_MSG_ACK: + process_ack(conn, mrecv); + ret = 0; + break; + default: + scoutfs_inc_counter(sb, net_unknown_message); + ret = -ENOMSG; + break; + } + + /* process_one_work explicitly allows freeing work in its func */ + kfree(mrecv); + + /* shut down the connection if processing returns fatal errors */ + if (ret) + shutdown_conn(conn); + + trace_scoutfs_net_proc_work_exit(sb, 0, ret); +} + +static int recvmsg_full(struct socket *sock, void *buf, unsigned len) +{ + struct msghdr msg; + struct kvec kv; + int ret; + + while (len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)&kv; + msg.msg_iovlen = 1; + msg.msg_flags = MSG_NOSIGNAL; + kv.iov_base = buf; + kv.iov_len = len; + + ret = kernel_recvmsg(sock, &msg, &kv, 1, len, msg.msg_flags); + if (ret <= 0) + return -ECONNABORTED; + + len -= ret; + buf += ret; + } + + return 0; +} + +static bool invalid_message(struct scoutfs_net_header *nh) +{ + /* ids must be non-zero */ + if (nh->id == 0) + return true; + + /* greeting messages must have the greeting id */ + if ((nh->cmd == SCOUTFS_NET_CMD_GREETING) != + (le64_to_cpu(nh->id) == SCOUTFS_NET_ID_GREETING)) + return true; + + /* greeting should negotiate understood protocol */ + if (nh->msg >= SCOUTFS_NET_MSG_UNKNOWN || + nh->cmd >= SCOUTFS_NET_CMD_UNKNOWN || + nh->error >= SCOUTFS_NET_ERR_UNKNOWN) + return true; + + /* errors can't have payloads */ + if (nh->data_len != 0 && nh->error != SCOUTFS_NET_ERR_NONE) + return true; + + /* payloads have a limit */ + if (le16_to_cpu(nh->data_len) > SCOUTFS_NET_MAX_DATA_LEN) + return true; + + /* only responses can carry errors */ + if (nh->error != SCOUTFS_NET_ERR_NONE && + nh->msg != SCOUTFS_NET_MSG_RESPONSE) + return true; + + return false; +} + +/* + * Always block receiving from the socket. Errors trigger shutting down + * the connection. + */ +static void scoutfs_net_recv_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, recv_work); + struct super_block *sb = conn->sb; + struct socket *sock = conn->sock; + struct scoutfs_net_header nh; + struct message_recv *mrecv; + unsigned int data_len; + int ret; + + trace_scoutfs_net_recv_work_enter(sb, 0, 0); + + for (;;) { + /* receive the header */ + ret = recvmsg_full(sock, &nh, sizeof(nh)); + if (ret) + break; + + /* receiving an invalid message breaks the connection */ + if (invalid_message(&nh)) { + scoutfs_inc_counter(sb, net_recv_invalid_message); + ret = -EBADMSG; + break; + } + + data_len = le16_to_cpu(nh.data_len); + + scoutfs_inc_counter(sb, net_recv_messages); + scoutfs_add_counter(sb, net_recv_bytes, nh_bytes(data_len)); + trace_scoutfs_net_recv_message(sb, &conn->sockname, + &conn->peername, &nh); + + /* invalid message checked data len */ + mrecv = kmalloc(offsetof(struct message_recv, + nh.data[data_len]), GFP_NOFS); + if (!mrecv) { + ret = -ENOMEM; + break; + } + + mrecv->conn = conn; + INIT_WORK(&mrecv->proc_work, scoutfs_net_proc_worker); + mrecv->nh = nh; + + /* receive the data payload */ + ret = recvmsg_full(sock, mrecv->nh.data, data_len); + if (ret) { + kfree(mrecv); + break; + } + + /* + * Check and maintain the last processed id for + * non-greeting requests before introducing reordering + * by queueing concurrent work. + */ + spin_lock(&conn->lock); + if (mrecv->nh.msg == SCOUTFS_NET_MSG_REQUEST && + mrecv->nh.cmd != SCOUTFS_NET_CMD_GREETING) { + if (le64_to_cpu(mrecv->nh.id) <= conn->last_proc_id) { + scoutfs_inc_counter(sb, net_dropped_request); + kfree(mrecv); + mrecv = NULL; + } else { + conn->last_proc_id = le64_to_cpu(mrecv->nh.id); + } + } + spin_unlock(&conn->lock); + + if (mrecv) + queue_work(conn->workq, &mrecv->proc_work); + } + + if (ret) + scoutfs_inc_counter(sb, net_recv_error); + + /* recv stopping always shuts down the connection */ + shutdown_conn(conn); + + trace_scoutfs_net_recv_work_exit(sb, 0, ret); +} + +static int sendmsg_full(struct socket *sock, void *buf, unsigned len) +{ + struct msghdr msg; + struct kvec kv; + int ret; + + while (len) { + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = (struct iovec *)&kv; + msg.msg_iovlen = 1; + msg.msg_flags = MSG_NOSIGNAL; + kv.iov_base = buf; + kv.iov_len = len; + + ret = kernel_sendmsg(sock, &msg, &kv, 1, len); + if (ret <= 0) + return -ECONNABORTED; + + len -= ret; + buf += ret; + } + + return 0; +} + +/* + * Each connection has a single worker that sends queued messages down + * the connection's socket. The work is queued whenever a message is + * put on the send queue. The worker uses blocking sends so that we + * don't have to worry about resuming partial sends or hooking into + * data_ready. Send errors shut down the connection. + * + * The worker is responsible for freeing messages so that other contexts + * don't have to worry about freeing a message while we're blocked + * sending it without the lock held. + */ +static void scoutfs_net_send_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, send_work); + struct super_block *sb = conn->sb; + struct message_send *msend; + int ret = 0; + int len; + + trace_scoutfs_net_send_work_enter(sb, 0, 0); + + spin_lock(&conn->lock); + + while ((msend = list_first_entry_or_null(&conn->send_queue, + struct message_send, head))) { + + if (msend->dead) { + list_del_init(&msend->head); + kfree(msend); + continue; + } + + spin_unlock(&conn->lock); + + len = nh_bytes(le16_to_cpu(msend->nh.data_len)); + + scoutfs_inc_counter(sb, net_send_messages); + scoutfs_add_counter(sb, net_send_bytes, len); + trace_scoutfs_net_send_message(sb, &conn->sockname, + &conn->peername, &msend->nh); + + ret = sendmsg_full(conn->sock, &msend->nh, len); + + spin_lock(&conn->lock); + + if (ret) + break; + + /* acks are always freed, others will be resent if not dead */ + if (msend->nh.msg == SCOUTFS_NET_MSG_ACK) + msend->dead = 1; + else if (!msend->dead) + list_move_tail(&msend->head, &conn->resend_queue); + } + + spin_unlock(&conn->lock); + + if (ret) { + scoutfs_inc_counter(sb, net_send_error); + shutdown_conn(conn); + } + + trace_scoutfs_net_send_work_exit(sb, 0, ret); +} + +static void destroy_conn(struct scoutfs_net_connection *conn) +{ + struct scoutfs_net_connection *listener; + struct message_send *msend; + struct message_send *tmp; + + WARN_ON_ONCE(conn->sock != NULL); + WARN_ON_ONCE(!list_empty(&conn->accepted_list)); + + /* free all messages, refactor and complete for forced unmount? */ + list_splice_init(&conn->resend_queue, &conn->send_queue); + list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { + list_del_init(&msend->head); + kfree(msend); + } + + /* accepted sockets are removed from their listener's list */ + if (conn->listening_conn) { + listener = conn->listening_conn; + + spin_lock(&listener->lock); + list_del_init(&conn->accepted_head); + if (list_empty(&listener->accepted_list)) + wake_up(&listener->accepted_waitq); + spin_unlock(&listener->lock); + } + + destroy_workqueue(conn->workq); + kfree(conn); +} + +/* + * Have a pretty aggressive keepalive timeout of around 10 seconds. The + * TCP keepalives are being processed out of task context so they should + * be responsive even when mounts are under load. + */ +#define KEEPCNT 3 +#define KEEPIDLE 7 +#define KEEPINTVL 1 +static int sock_opts_and_names(struct scoutfs_net_connection *conn, + struct socket *sock) +{ + struct timeval tv; + int addrlen; + int optval; + int ret; + + /* but use a keepalive timeout instead of send timeout */ + tv.tv_sec = 0; + tv.tv_usec = 0; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) + goto out; + + optval = KEEPCNT; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = KEEPIDLE; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = KEEPINTVL; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + optval = 1; + ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(sock, (struct sockaddr *)&conn->sockname, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getpeername(sock, (struct sockaddr *)&conn->peername, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; +out: + return ret; +} + +/* + * Each bound and listening connection has long running work that blocks + * accepting new connections. The listening socket has been setup by + * the time this is queued. + * + * Any errors on the listening sock tear down all the connections that + * were accepted. + */ +static void scoutfs_net_listen_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, listen_work); + struct super_block *sb = conn->sb; + struct scoutfs_net_connection *acc_conn; + DECLARE_WAIT_QUEUE_HEAD(waitq); + struct socket *acc_sock; + LIST_HEAD(conn_list); + int ret; + + trace_scoutfs_net_listen_work_enter(sb, 0, 0); + + for (;;) { + ret = kernel_accept(conn->sock, &acc_sock, 0); + if (ret < 0) + break; + + /* inherit accepted request funcs from listening conn */ + acc_conn = scoutfs_net_alloc_conn(sb, NULL, NULL, + conn->req_funcs, "accepted"); + if (!acc_conn) { + sock_release(acc_sock); + ret = -ENOMEM; + continue; + } + + ret = sock_opts_and_names(acc_conn, acc_sock); + if (ret) { + sock_release(acc_sock); + destroy_conn(acc_conn); + continue; + } + + scoutfs_info(sb, "server accepted "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&acc_conn->sockname), + SIN_ARG(&acc_conn->peername)); + + /* acc_conn isn't visible, conn unlock orders stores */ + spin_lock(&conn->lock); + + acc_conn->sock = acc_sock; + acc_conn->listening_conn = conn; + acc_conn->established = 1; + list_add_tail(&acc_conn->accepted_head, &conn->accepted_list); + + spin_unlock(&conn->lock); + + queue_work(acc_conn->workq, &acc_conn->recv_work); + } + + /* listening stopping shuts down connection */ + shutdown_conn(conn); + + trace_scoutfs_net_listen_work_exit(sb, 0, ret); +} + +/* + * Try once to connect to the caller's address. This is racing with + * shutdown if the caller frees the connection while we're connecting. + * Shutdown will wait for our executing work to finish. + */ +static void scoutfs_net_connect_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, connect_work); + struct super_block *sb = conn->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting greet; + struct socket *sock; + struct timeval tv; + int ret; + + trace_scoutfs_net_connect_work_enter(sb, 0, 0); + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + /* caller specified connect timeout */ + tv.tv_sec = conn->connect_timeout_ms / MSEC_PER_SEC; + tv.tv_usec = (conn->connect_timeout_ms % MSEC_PER_SEC) * USEC_PER_MSEC; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, + (char *)&tv, sizeof(tv)); + if (ret) { + sock_release(sock); + goto out; + } + + /* shutdown now owns sock, can break blocking connect */ + spin_lock(&conn->lock); + conn->sock = sock; + spin_unlock(&conn->lock); + + ret = kernel_connect(sock, (struct sockaddr *)&conn->connect_sin, + sizeof(struct sockaddr_in), 0); + if (ret) + goto out; + + ret = sock_opts_and_names(conn, sock); + if (ret) + goto out; + + /* greeting is about to queue send work */ + spin_lock(&conn->lock); + conn->established = 1; + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->recv_work); + + /* queue a new updated greeting send */ + greet.fsid = super->id; + greet.format_hash = super->format_hash; + + ret = submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, 0, + &greet, sizeof(greet), greeting_response, NULL, NULL); + if (ret) + goto out; + + scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, + SIN_ARG(&conn->sockname), + SIN_ARG(&conn->peername)); +out: + if (ret) + shutdown_conn(conn); + + trace_scoutfs_net_connect_work_exit(sb, 0, ret); +} + +static bool empty_accepted_list(struct scoutfs_net_connection *conn) +{ + bool empty; + + spin_lock(&conn->lock); + empty = list_empty(&conn->accepted_list); + spin_unlock(&conn->lock); + + return empty; +} + +/* listening and their accepting sockets have a fixed locking order */ +enum { + CONN_LOCK_LISTENER, + CONN_LOCK_ACCEPTED, +}; + +/* + * Safely shut down an active connection. This can be triggered by + * errors in workers or by an external call to free the connection. The + * shutting down flag ensures that this only executes once for each live + * socket. + * + * Our reliability guarantee requires request processing to make forward + * progress once we've received and recorded a request id. We wait for + * processing work that is in flight and its sends will be queued for + * resending because the connection is not established while it's + * shutting down. + */ +static void scoutfs_net_shutdown_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); + struct super_block *sb = conn->sb; + struct scoutfs_net_connection *acc_conn; + struct message_send *msend; + + trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); + + /* connected and accepted conns print a message */ + if (conn->peername.sin_family) + scoutfs_info(sb, "%s "SIN_FMT" -> "SIN_FMT, + conn->listening_conn ? "server closing" : + "client disconnected", + SIN_ARG(&conn->sockname), + SIN_ARG(&conn->peername)); + + /* ensure that sockets return errors, wakes blocked socket work */ + if (conn->sock) + kernel_sock_shutdown(conn->sock, SHUT_RDWR); + + /* wait for socket and proc work to finish, includes chained work */ + drain_workqueue(conn->workq); + + /* tear down the sock now that all work is done */ + if (conn->sock) { + sock_release(conn->sock); + conn->sock = NULL; + } + + /* listening connections shut down all the connections they accepted */ + spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); + list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { + spin_lock_nested(&acc_conn->lock, CONN_LOCK_ACCEPTED); + shutdown_conn_locked(acc_conn); + spin_unlock(&acc_conn->lock); + } + spin_unlock(&conn->lock); + wait_event(conn->accepted_waitq, empty_accepted_list(conn)); + + spin_lock(&conn->lock); + + /* all queued sends will be resent, protocol handles dupes */ + list_splice_tail_init(&conn->send_queue, &conn->resend_queue); + + /* clear greeting state for next negotiation */ + conn->valid_greeting = 0; + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_RESPONSE, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_ACK, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); + if (msend) + complete_send(conn, msend, NULL, 0, 0); + + spin_unlock(&conn->lock); + + memset(&conn->peername, 0, sizeof(conn->peername)); + + /* tell the caller that the connection is down */ + if (conn->notify_down) + conn->notify_down(sb, conn); + + /* accepted conns are destroyed */ + if (conn->listening_conn) { + destroy_conn(conn); + } else { + spin_lock(&conn->lock); + conn->shutting_down = 0; + spin_unlock(&conn->lock); + } + + trace_scoutfs_net_shutdown_work_exit(sb, 0, 0); +} + +struct scoutfs_net_connection * +scoutfs_net_alloc_conn(struct super_block *sb, + scoutfs_net_notify_t notify_up, + scoutfs_net_notify_t notify_down, + scoutfs_net_request_t *req_funcs, char *name_suffix) +{ + struct scoutfs_net_connection *conn; + + /* we handle greetings, the caller shouldn't attempt to */ + if (WARN_ON_ONCE(req_funcs != NULL && + req_funcs[SCOUTFS_NET_CMD_GREETING] != NULL)) + return NULL; + + conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); + if (!conn) + return NULL; + + conn->workq = alloc_workqueue("scoutfs_net_%s", + WQ_UNBOUND | WQ_NON_REENTRANT, 0, + name_suffix); + if (!conn->workq) { + kfree(conn); + return NULL; + } + + conn->sb = sb; + conn->notify_up = notify_up; + conn->notify_down = notify_down; + conn->req_funcs = req_funcs; + spin_lock_init(&conn->lock); + INIT_LIST_HEAD(&conn->accepted_head); + INIT_LIST_HEAD(&conn->accepted_list); + init_waitqueue_head(&conn->accepted_waitq); + conn->next_send_id = SCOUTFS_NET_ID_GREETING + 1; + INIT_LIST_HEAD(&conn->send_queue); + INIT_LIST_HEAD(&conn->resend_queue); + INIT_WORK(&conn->listen_work, scoutfs_net_listen_worker); + INIT_WORK(&conn->connect_work, scoutfs_net_connect_worker); + INIT_WORK(&conn->send_work, scoutfs_net_send_worker); + INIT_WORK(&conn->recv_work, scoutfs_net_recv_worker); + INIT_WORK(&conn->shutdown_work, scoutfs_net_shutdown_worker); + + return conn; +} + +/* + * Shutdown the connection. Once this returns no network traffic + * or work will be executing. The caller can then connect or bind and + * listen again. Additional shutdown calls will already find it shutdown. + */ +void scoutfs_net_shutdown(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + shutdown_conn(conn); + flush_work(&conn->shutdown_work); +} + +/* + * Destroy the connection after the shutdown work has stopped all concurrent + * processing on the connection. + */ +void scoutfs_net_free_conn(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + if (conn) { + scoutfs_net_shutdown(sb, conn); + destroy_conn(conn); + } +} + +/* + * Associate a bound socket with the caller's connection. We call bind + * and listen to assign the listening address and give it to the caller. + * + * If this returns success then the caller has to call either listen or + * free_conn. + */ +int scoutfs_net_bind(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin) +{ + struct socket *sock = NULL; + int addrlen; + int ret; + + /* caller state machine shouldn't let this happen */ + if (WARN_ON_ONCE(conn->sock)) + return -EINVAL; + + ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_bind(sock, (struct sockaddr *)sin, addrlen); + if (ret) + goto out; + + ret = kernel_listen(sock, 255); + if (ret) + goto out; + + addrlen = sizeof(struct sockaddr_in); + ret = kernel_getsockname(sock, (struct sockaddr *)&conn->sockname, + &addrlen); + if (ret == 0 && addrlen != sizeof(struct sockaddr_in)) + ret = -EAFNOSUPPORT; + if (ret) + goto out; + + conn->sock = sock; + *sin = conn->sockname; + ret = 0; +out: + if (ret < 0 && sock) + sock_release(sock); + return ret; +} + +/* + * Kick off blocking background work to accept connections from the + * connection's listening socket that was created with a previous bind + * call. + * + * The callback notify_down will be called once the listening socket is + * shut down either by errors or the caller freeing the conn. + */ +void scoutfs_net_listen(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + queue_work(conn->workq, &conn->listen_work); +} + +/* + * Start connecting to the given address. notify_up may be called if + * the connection completes. notify_down will be called when either the + * connection disconnects or times out. Both could be called before + * this function returns. The caller must be careful not to call + * connect again until notify_down has been called. + */ +void scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms) +{ + spin_lock(&conn->lock); + conn->connect_sin = *sin; + conn->connect_timeout_ms = timeout_ms; + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->connect_work); +} + +/* + * Submit a request down the connection. It's up to the caller to + * ensure that the conn is allocated. Sends submitted when the + * connection isn't established will be resent in order the next time + * it's established. + */ +int scoutfs_net_submit_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret) +{ + return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, cmd, 0, 0, + arg, arg_len, resp_func, resp_data, id_ret); +} + +/* + * Send a response. Responses don't get callbacks and use the request's + * id so caller's don't need to get an id in return. + * + * The data payload is ignored if an error is sent so that callers have + * simple processing exit paths. + * + * An error is returned if the response could not be sent. + */ +int scoutfs_net_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, int error, void *resp, u16 resp_len) +{ + if (error) { + resp = NULL; + resp_len = 0; + } + + return submit_send(sb, conn, SCOUTFS_NET_MSG_RESPONSE, + cmd, id, net_err_from_host(sb, error), + resp, resp_len, NULL, NULL, NULL); +} + +void scoutfs_net_cancel_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id) +{ + struct message_send *msend; + + spin_lock(&conn->lock); + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, cmd, id); + if (msend) + complete_send(conn, msend, NULL, 0, -ECANCELED); + spin_unlock(&conn->lock); +} + +struct sync_request_completion { + struct completion comp; + void *resp; + unsigned int resp_len; + int error; +}; + +static int sync_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + struct sync_request_completion *sreq = data; + + if (error == 0 && resp_len != sreq->resp_len) + error = -EMSGSIZE; + + if (error) + sreq->error = error; + else if (resp_len) + memcpy(sreq->resp, resp, resp_len); + + complete(&sreq->comp); + + return 0; +} + +/* + * Send a request and wait for a response to be copied into the given + * buffer. Errors returned can come from the remote request processing + * or local failure to send. + * + * The wait for the response is interruptible and can return + * -ERESTARTSYS if it is interrupted. + * + * -EOVERFLOW is returned if the response message's data_length doesn't + * match the caller's resp_len buffer. + */ +int scoutfs_net_sync_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, unsigned arg_len, + void *resp, size_t resp_len) +{ + struct sync_request_completion sreq; + int ret; + u64 id; + + init_completion(&sreq.comp); + sreq.resp = resp; + sreq.resp_len = resp_len; + sreq.error = 0; + + ret = scoutfs_net_submit_request(sb, conn, cmd, arg, arg_len, + sync_response, &sreq, &id); + + ret = wait_for_completion_interruptible(&sreq.comp); + if (ret == -ERESTARTSYS) + scoutfs_net_cancel_request(sb, conn, cmd, id); + else + ret = sreq.error; + + return ret; +} + +int scoutfs_net_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct net_info *ninf; + int ret; + + /* fail the build if host errnos don't fit in the u8 mapping arrays */ +#undef EXPAND_NET_ERRNO +#define EXPAND_NET_ERRNO(which) BUILD_BUG_ON(which >= U8_MAX); + EXPAND_EACH_NET_ERRNO + + ninf = kzalloc(sizeof(struct net_info), GFP_KERNEL); + if (!ninf) { + ret = -ENOMEM; + goto out; + } + + sbi->net_info = ninf; + + ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", + WQ_UNBOUND, 0); + if (!ninf->shutdown_workq) { + ret = -ENOMEM; + goto out; + } + + ret = 0; +out: + if (ret) + scoutfs_net_destroy(sb); + return ret; +} + +void scoutfs_net_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + + if (ninf) { + if (ninf->shutdown_workq) + destroy_workqueue(ninf->shutdown_workq); + kfree(ninf); + sbi->net_info = NULL; + } +} diff --git a/kmod/src/net.h b/kmod/src/net.h new file mode 100644 index 00000000..4e270760 --- /dev/null +++ b/kmod/src/net.h @@ -0,0 +1,61 @@ +#ifndef _SCOUTFS_NET_H_ +#define _SCOUTFS_NET_H_ + +#include + +#define SIN_FMT "%pIS:%u" +#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) + +struct scoutfs_net_connection; + +/* These are called in their own blocking context */ +typedef int (*scoutfs_net_request_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len); + +/* These are called with a spinlock held, funcs must be fast and nonblocking */ +typedef int (*scoutfs_net_response_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data); + +typedef void (*scoutfs_net_notify_t)(struct super_block *sb, + struct scoutfs_net_connection *conn); + +struct scoutfs_net_connection * +scoutfs_net_alloc_conn(struct super_block *sb, + scoutfs_net_notify_t notify_up, + scoutfs_net_notify_t notify_down, + scoutfs_net_request_t *req_funcs, char *name_suffix); +void scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms); +int scoutfs_net_bind(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin); +void scoutfs_net_listen(struct super_block *sb, + struct scoutfs_net_connection *conn); +int scoutfs_net_submit_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret); +void scoutfs_net_cancel_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id); +int scoutfs_net_sync_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, void *arg, unsigned arg_len, + void *resp, size_t resp_len); +int scoutfs_net_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, int error, void *resp, u16 resp_len); +void scoutfs_net_shutdown(struct super_block *sb, + struct scoutfs_net_connection *conn); +void scoutfs_net_free_conn(struct super_block *sb, + struct scoutfs_net_connection *conn); + +int scoutfs_net_setup(struct super_block *sb); +void scoutfs_net_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b223d70d..962bfca7 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1658,25 +1658,13 @@ DECLARE_EVENT_CLASS(scoutfs_net_class, snh_trace_args(nh)) ); -DEFINE_EVENT(scoutfs_net_class, scoutfs_client_send_request, +DEFINE_EVENT(scoutfs_net_class, scoutfs_net_send_message, TP_PROTO(struct super_block *sb, struct sockaddr_in *name, struct sockaddr_in *peer, struct scoutfs_net_header *nh), TP_ARGS(sb, name, peer, nh) ); -DEFINE_EVENT(scoutfs_net_class, scoutfs_server_recv_request, - TP_PROTO(struct super_block *sb, struct sockaddr_in *name, - struct sockaddr_in *peer, struct scoutfs_net_header *nh), - TP_ARGS(sb, name, peer, nh) -); - -DEFINE_EVENT(scoutfs_net_class, scoutfs_server_send_reply, - TP_PROTO(struct super_block *sb, struct sockaddr_in *name, - struct sockaddr_in *peer, struct scoutfs_net_header *nh), - TP_ARGS(sb, name, peer, nh) -); - -DEFINE_EVENT(scoutfs_net_class, scoutfs_client_recv_reply, +DEFINE_EVENT(scoutfs_net_class, scoutfs_net_recv_message, TP_PROTO(struct super_block *sb, struct sockaddr_in *name, struct sockaddr_in *peer, struct scoutfs_net_header *nh), TP_ARGS(sb, name, peer, nh) @@ -1706,11 +1694,51 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); -DEFINE_EVENT(scoutfs_work_class, scoutfs_server_recv_work_enter, +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); -DEFINE_EVENT(scoutfs_work_class, scoutfs_server_recv_work_exit, +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_listen_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_listen_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_connect_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_connect_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_send_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_send_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_recv_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_recv_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); diff --git a/kmod/src/server.c b/kmod/src/server.c index da284854..13428ad2 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -32,9 +32,20 @@ #include "msg.h" #include "client.h" #include "server.h" -#include "sock.h" +#include "net.h" #include "endian_swap.h" +/* + * Every active mount can act as the server that listens on a net + * connection and accepts connections from all the other mounts acting + * as clients. + * + * It queues long-lived work that blocks trying to acquire a lock. If + * it acquires the lock it listens on a socket and serves requests. If + * it sees errors it shuts down the server in the hopes that another + * mount will have less trouble. + */ + #define SIN_FMT "%pIS:%u" #define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) @@ -43,19 +54,14 @@ struct server_info { struct workqueue_struct *wq; struct delayed_work dwork; - - struct mutex mutex; - bool shutting_down; + struct completion shutdown_comp; bool bind_warned; - struct task_struct *listen_task; - struct socket *listen_sock; /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; struct work_struct commit_work; - /* adding new segments can have to wait for compaction */ wait_queue_head_t compaction_waitq; @@ -72,6 +78,10 @@ struct server_info { struct list_head pending_frees; }; +#define DECLARE_SERVER_INFO(sb, name) \ + struct server_info *name = SCOUTFS_SB(sb)->server_info + +#if 0 struct server_request { struct server_connection *conn; struct work_struct work; @@ -89,6 +99,7 @@ struct server_connection { struct work_struct recv_work; struct mutex send_mutex; }; +#endif struct commit_waiter { struct completion comp; @@ -454,17 +465,9 @@ out: return ret; } -/* - * Trigger a server shutdown by shutting down the listening socket. The - * server thread will break out of accept and exit. - */ -static void shut_down_server(struct server_info *server) +static void shutdown_server(struct server_info *server) { - mutex_lock(&server->mutex); - server->shutting_down = true; - if (server->listen_sock) - kernel_sock_shutdown(server->listen_sock, SHUT_RDWR); - mutex_unlock(&server->mutex); + complete(&server->shutdown_comp); } /* @@ -495,21 +498,11 @@ static void queue_commit_work(struct server_info *server, } /* - * Commit errors are fatal and shut down the server. This is called - * from request processing which shutdown will wait for. + * Wait for a commit during request processing and return its status. */ -static int wait_for_commit(struct server_info *server, - struct commit_waiter *cw, u64 id, u8 type) +static inline int wait_for_commit(struct commit_waiter *cw) { - struct super_block *sb = server->sb; - wait_for_completion(&cw->comp); - if (cw->ret < 0) { - scoutfs_err(sb, "commit error %d processing req id %llu type %u", - cw->ret, id, type); - - shut_down_server(server); - } return cw->ret; } @@ -546,17 +539,18 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); - if (!scoutfs_btree_has_dirty(sb)) { - ret = 0; - goto out; - } - + /* try to free first which can dirty the btrees */ ret = apply_pending_frees(sb); if (ret) { scoutfs_err(sb, "server error freeing extents: %d", ret); goto out; } + if (!scoutfs_btree_has_dirty(sb)) { + ret = 0; + goto out; + } + ret = scoutfs_btree_write_dirty(sb); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); @@ -591,6 +585,7 @@ out: trace_scoutfs_server_commit_work_exit(sb, 0, ret); } +#if 0 /* * Request processing synchronously sends their reply from within their * processing work. If this fails the socket is shutdown. @@ -639,6 +634,7 @@ static int send_reply(struct server_connection *conn, u64 id, return ret; } +#endif void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment) @@ -660,26 +656,26 @@ void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, ment->last = net_ment->last; } -static int process_alloc_inodes(struct server_connection *conn, - u64 id, u8 type, void *data, unsigned data_len) +static int server_alloc_inodes(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_net_inode_alloc ial; + struct scoutfs_net_inode_alloc ial = { 0, }; struct commit_waiter cw; __le64 lecount; u64 ino; u64 nr; int ret; - if (data_len != sizeof(lecount)) { + if (arg_len != sizeof(lecount)) { ret = -EINVAL; goto out; } - memcpy(&lecount, data, data_len); + memcpy(&lecount, arg, arg_len); down_read(&server->commit_rwsem); @@ -695,52 +691,48 @@ static int process_alloc_inodes(struct server_connection *conn, ial.ino = cpu_to_le64(ino); ial.nr = cpu_to_le64(nr); - ret = wait_for_commit(server, &cw, id, type); + ret = wait_for_commit(&cw); out: - return send_reply(conn, id, type, ret, &ial, sizeof(ial)); + return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); } /* * Give the client an extent allocation of len blocks. We leave the * details to the extent allocator. */ -static int process_alloc_extent(struct server_connection *conn, - u64 id, u8 type, void *data, unsigned data_len) +static int server_alloc_extent(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; - struct scoutfs_net_extent nex; + struct scoutfs_net_extent nex = {0,}; __le64 leblocks; u64 start; u64 len; int ret; - if (data_len != sizeof(leblocks)) { + if (arg_len != sizeof(leblocks)) { ret = -EINVAL; goto out; } - memcpy(&leblocks, data, data_len); + memcpy(&leblocks, arg, arg_len); down_read(&server->commit_rwsem); ret = alloc_extent(sb, le64_to_cpu(leblocks), &start, &len); - if (ret == -ENOSPC) { - start = 0; - len = 0; - ret = 0; - } - if (ret == 0) { - nex.start = cpu_to_le64(start); - nex.len = cpu_to_le64(len); - queue_commit_work(server, &cw); - } - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(server, &cw, id, type); + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + if (ret) + goto out; + + nex.start = cpu_to_le64(start); + nex.len = cpu_to_le64(len); out: - return send_reply(conn, id, type, ret, &nex, sizeof(nex)); + return scoutfs_net_response(sb, conn, cmd, id, ret, &nex, sizeof(nex)); } static bool invalid_net_extent_list(struct scoutfs_net_extent_list *nexl, @@ -752,19 +744,19 @@ static bool invalid_net_extent_list(struct scoutfs_net_extent_list *nexl, extents[le64_to_cpu(nexl->nr)])); } -static int process_free_extents(struct server_connection *conn, - u64 id, u8 type, void *data, unsigned data_len) +static int server_free_extents(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_net_extent_list *nexl; struct commit_waiter cw; - int ret = 0; + int ret; int err; u64 i; - nexl = data; - if (invalid_net_extent_list(nexl, data_len)) { + nexl = arg; + if (invalid_net_extent_list(nexl, arg_len)) { ret = -EINVAL; goto out; } @@ -783,70 +775,66 @@ static int process_free_extents(struct server_connection *conn, up_read(&server->commit_rwsem); if (i > 0) { - err = wait_for_commit(server, &cw, id, type); + err = wait_for_commit(&cw); if (ret == 0) ret = err; } + out: - return send_reply(conn, id, type, ret, NULL, 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } /* * We still special case segno allocation because it's aligned and we'd * like to keep that detail in the server. */ -static int process_alloc_segno(struct server_connection *conn, - u64 id, u8 type, void *data, unsigned data_len) +static int server_alloc_segno(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; __le64 lesegno = 0; u64 segno; int ret; - if (data_len != 0) { + if (arg_len != 0) { ret = -EINVAL; goto out; } down_read(&server->commit_rwsem); ret = alloc_segno(sb, &segno); - if (ret == 0) { - lesegno = cpu_to_le64(segno); + if (ret == 0) queue_commit_work(server, &cw); - } else if (ret == -ENOSPC) { - ret = 0; - } up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + if (ret) + goto out; - if (ret == 0 && lesegno != 0) - ret = wait_for_commit(server, &cw, id, type); + lesegno = cpu_to_le64(segno); out: - return send_reply(conn, id, type, ret, &lesegno, sizeof(lesegno)); + return scoutfs_net_response(sb, conn, cmd, id, ret, + &lesegno, sizeof(lesegno)); } -static int process_record_segment(struct server_connection *conn, u64 id, - u8 type, void *data, unsigned data_len) +static int server_record_segment(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_net_manifest_entry *net_ment; struct scoutfs_manifest_entry ment; struct commit_waiter cw; int ret; - if (data_len < sizeof(struct scoutfs_net_manifest_entry)) { + if (arg_len != sizeof(struct scoutfs_net_manifest_entry)) { ret = -EINVAL; goto out; } - net_ment = data; - - if (data_len != sizeof(*net_ment)) { - ret = -EINVAL; - goto out; - } + net_ment = arg; retry: down_read(&server->commit_rwsem); @@ -871,12 +859,13 @@ retry: up_read(&server->commit_rwsem); if (ret == 0) { - ret = wait_for_commit(server, &cw, id, type); + ret = wait_for_commit(&cw); if (ret == 0) scoutfs_compact_kick(sb); } + out: - return send_reply(conn, id, type, ret, NULL, 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } struct pending_seq { @@ -896,21 +885,21 @@ struct pending_seq { * XXX The pending seq tracking should be persistent so that it survives * server failover. */ -static int process_advance_seq(struct server_connection *conn, u64 id, u8 type, - void *data, unsigned data_len) +static int server_advance_seq(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct pending_seq *next_ps; struct pending_seq *ps; struct commit_waiter cw; - __le64 * __packed their_seq = data; + __le64 * __packed their_seq = arg; __le64 next_seq; int ret; - if (data_len != sizeof(__le64)) { + if (arg_len != sizeof(__le64)) { ret = -EINVAL; goto out; } @@ -941,24 +930,24 @@ static int process_advance_seq(struct server_connection *conn, u64 id, u8 type, spin_unlock(&server->seq_lock); queue_commit_work(server, &cw); up_read(&server->commit_rwsem); - ret = wait_for_commit(server, &cw, id, type); - + ret = wait_for_commit(&cw); out: - return send_reply(conn, id, type, ret, &next_seq, sizeof(next_seq)); + return scoutfs_net_response(sb, conn, cmd, id, ret, + &next_seq, sizeof(next_seq)); } -static int process_get_last_seq(struct server_connection *conn, u64 id, - u8 type, void *data, unsigned data_len) +static int server_get_last_seq(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct pending_seq *ps; __le64 last_seq; int ret; - if (data_len != 0) { + if (arg_len != 0) { ret = -EINVAL; goto out; } @@ -975,18 +964,20 @@ static int process_get_last_seq(struct server_connection *conn, u64 id, spin_unlock(&server->seq_lock); ret = 0; out: - return send_reply(conn, id, type, ret, &last_seq, sizeof(last_seq)); + return scoutfs_net_response(sb, conn, cmd, id, ret, + &last_seq, sizeof(last_seq)); } -static int process_get_manifest_root(struct server_connection *conn, u64 id, - u8 type, void *data, unsigned data_len) +static int server_get_manifest_root(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_btree_root root; unsigned int start; int ret; - if (data_len == 0) { + if (arg_len == 0) { do { start = read_seqcount_begin(&server->stable_seqcount); root = server->stable_manifest_root; @@ -996,24 +987,25 @@ static int process_get_manifest_root(struct server_connection *conn, u64 id, ret = -EINVAL; } - return send_reply(conn, id, type, ret, &root, sizeof(root)); + return scoutfs_net_response(sb, conn, cmd, id, ret, + &root, sizeof(root)); } /* * Sample the super stats that the client wants for statfs by serializing * with each component. */ -static int process_statfs(struct server_connection *conn, u64 id, u8 type, - void *data, unsigned data_len) +static int server_statfs(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct server_info *server = conn->server; - struct super_block *sb = server->sb; + DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_statfs nstatfs; int ret; - if (data_len == 0) { + if (arg_len == 0) { /* uuid and total_segs are constant, so far */ memcpy(nstatfs.uuid, super->uuid, sizeof(nstatfs.uuid)); @@ -1030,7 +1022,8 @@ static int process_statfs(struct server_connection *conn, u64 id, u8 type, ret = -EINVAL; } - return send_reply(conn, id, type, ret, &nstatfs, sizeof(nstatfs)); + return scoutfs_net_response(sb, conn, cmd, id, ret, + &nstatfs, sizeof(nstatfs)); } /* @@ -1078,7 +1071,7 @@ int scoutfs_client_get_compaction(struct super_block *sb, void *curs) up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(server, &cw, U64_MAX, 1); + ret = wait_for_commit(&cw); return ret; } @@ -1117,13 +1110,14 @@ int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, up_read(&server->commit_rwsem); if (ret == 0) - ret = wait_for_commit(server, &cw, U64_MAX, 2); + ret = wait_for_commit(&cw); scoutfs_compact_kick(sb); return ret; } +#if 0 typedef int (*process_func_t)(struct server_connection *conn, u64 id, u8 type, void *data, unsigned data_len); @@ -1168,7 +1162,9 @@ static void scoutfs_server_process_func(struct work_struct *work) /* process_one_work explicitly allows freeing work in its func */ kfree(req); } +#endif +#if 0 /* * Always block receiving from the socket. This owns the socket. If * receive fails this shuts down and frees the socket. @@ -1292,6 +1288,7 @@ out: trace_scoutfs_server_recv_work_exit(sb, 0, ret); } +#endif /* * This relies on the caller having read the current super and advanced @@ -1308,6 +1305,7 @@ static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) return scoutfs_write_dirty_super(sb); } +#if 0 static bool barrier_list_empty_careful(struct list_head *list) { /* store caller's task state before loading wake condition */ @@ -1315,7 +1313,27 @@ static bool barrier_list_empty_careful(struct list_head *list) return list_empty_careful(list); } +#endif +static scoutfs_net_request_t server_req_funcs[] = { + [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, + [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, + [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, + [SCOUTFS_NET_CMD_ALLOC_SEGNO] = server_alloc_segno, + [SCOUTFS_NET_CMD_RECORD_SEGMENT] = server_record_segment, + [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, + [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, + [SCOUTFS_NET_CMD_GET_MANIFEST_ROOT] = server_get_manifest_root, + [SCOUTFS_NET_CMD_STATFS] = server_statfs, +}; + +static void server_notify_down(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + DECLARE_SERVER_INFO(sb, server); + + shutdown_server(server); +} /* * This work is always running or has a delayed timer set while a super * is mounted. It tries to grab the lock to become the server. If it @@ -1323,51 +1341,45 @@ static bool barrier_list_empty_careful(struct list_head *list) * anything goes wrong it releases the lock and sets a timer to try to * become the server all over again. */ -static void scoutfs_server_func(struct work_struct *work) +static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, dwork.work); struct super_block *sb = server->sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_net_connection *conn = NULL; static struct sockaddr_in zeros = {0,}; - struct socket *new_sock; - struct socket *sock = NULL; struct scoutfs_lock *lock = NULL; - struct server_connection *conn; struct pending_seq *ps; struct pending_seq *ps_tmp; DECLARE_WAIT_QUEUE_HEAD(waitq); struct sockaddr_in sin; LIST_HEAD(conn_list); - int addrlen; - int optval; int ret; trace_scoutfs_server_work_enter(sb, 0, 0); - init_waitqueue_head(&waitq); + init_completion(&server->shutdown_comp); ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, &lock); if (ret) goto out; + conn = scoutfs_net_alloc_conn(sb, NULL, server_notify_down, + server_req_funcs, "server"); + if (!conn) { + ret = -ENOMEM; + goto out; + } + sin.sin_family = AF_INET; sin.sin_addr.s_addr = le32_to_be32(sbi->opts.listen_addr.addr); sin.sin_port = le16_to_be16(sbi->opts.listen_addr.port); - optval = 1; - ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock) ?: - kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, - (char *)&optval, sizeof(optval)) ?: - kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&optval, sizeof(optval)); - if (ret) - goto out; - - addrlen = sizeof(sin); - ret = kernel_bind(sock, (struct sockaddr *)&sin, addrlen); + /* get the address of our listening socket */ + ret = scoutfs_net_bind(sb, conn, &sin); if (ret) { if (!server->bind_warned) { scoutfs_err(sb, "server failed to bind to "SIN_FMT", errno %d%s. Retrying indefinitely..", @@ -1378,15 +1390,6 @@ static void scoutfs_server_func(struct work_struct *work) } goto out; } - server->bind_warned = false; - - kernel_getsockname(sock, (struct sockaddr *)&sin, &addrlen); - if (ret) - goto out; - - ret = kernel_listen(sock, 255); - if (ret) - goto out; /* publish the address for clients to connect to */ ret = scoutfs_read_super(sb, super); @@ -1398,17 +1401,7 @@ static void scoutfs_server_func(struct work_struct *work) if (ret) goto out; - /* either see shutting down or they'll shutdown our sock */ - mutex_lock(&server->mutex); - server->listen_task = current; - server->listen_sock = sock; - if (server->shutting_down) - ret = -ESHUTDOWN; - mutex_unlock(&server->mutex); - if (ret) - goto out; - - /* finally start up the server subsystems before accepting */ + /* start up the server subsystems before accepting */ ret = scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_compact_setup(sb); @@ -1420,69 +1413,24 @@ static void scoutfs_server_func(struct work_struct *work) scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); - for (;;) { - ret = kernel_accept(sock, &new_sock, 0); - if (ret < 0) - break; + /* start accepting connections and processing work */ + scoutfs_net_listen(sb, conn); - conn = kmalloc(sizeof(struct server_connection), GFP_NOFS); - if (!conn) { - sock_release(new_sock); - ret = -ENOMEM; - continue; - } - - addrlen = sizeof(struct sockaddr_in); - ret = kernel_getsockname(new_sock, - (struct sockaddr *)&conn->sockname, - &addrlen) ?: - kernel_getpeername(new_sock, - (struct sockaddr *)&conn->peername, - &addrlen); - if (ret) { - sock_release(new_sock); - continue; - } - - /* - * XXX yeah, ok, killing the sock and accepting a new - * one is racey. think about that in all the code. Are - * we destroying a resource to shutdown that the thing - * we're canceling creates? - */ - - conn->server = server; - conn->sock = new_sock; - mutex_init(&conn->send_mutex); - - scoutfs_info(sb, "server accepted "SIN_FMT" -> "SIN_FMT, - SIN_ARG(&conn->peername), - SIN_ARG(&conn->sockname)); - - /* recv work owns the conn once its in the list */ - mutex_lock(&server->mutex); - list_add(&conn->head, &conn_list); - mutex_unlock(&server->mutex); - - INIT_WORK(&conn->recv_work, scoutfs_server_recv_func); - queue_work(server->wq, &conn->recv_work); - } - - /* shutdown send and recv on all accepted sockets */ - mutex_lock(&server->mutex); - list_for_each_entry(conn, &conn_list, head) - kernel_sock_shutdown(conn->sock, SHUT_RDWR); - mutex_unlock(&server->mutex); - - /* wait for all recv work to finish and free connections */ - wait_event(waitq, barrier_list_empty_careful(&conn_list)); + /* wait for listening down or umount, conn can still be live */ + wait_for_completion_interruptible(&server->shutdown_comp); scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); shutdown: + /* wait for request processing */ + scoutfs_net_shutdown(sb, conn); + /* wait for commit queued by request processing */ + flush_work(&server->commit_work); /* shut down all the server subsystems */ scoutfs_compact_destroy(sb); + /* (wait for possible double commit work queued by compaction) */ + flush_work(&server->commit_work); destroy_pending_frees(sb); scoutfs_manifest_destroy(sb); scoutfs_btree_destroy(sb); @@ -1496,9 +1444,7 @@ shutdown: write_server_addr(sb, &zeros); out: - if (sock) - sock_release(sock); - + scoutfs_net_free_conn(sb, conn); scoutfs_unlock(sb, lock, DLM_LOCK_EX); /* always requeues, cancel_delayed_work_sync cancels on shutdown */ @@ -1517,8 +1463,9 @@ int scoutfs_server_setup(struct super_block *sb) return -ENOMEM; server->sb = sb; - INIT_DELAYED_WORK(&server->dwork, scoutfs_server_func); - mutex_init(&server->mutex); + init_completion(&server->shutdown_comp); + server->bind_warned = false; + INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); init_rwsem(&server->commit_rwsem); init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); @@ -1547,7 +1494,7 @@ void scoutfs_server_destroy(struct super_block *sb) struct server_info *server = sbi->server_info; if (server) { - shut_down_server(server); + shutdown_server(server); /* wait for server work to wait for everything to shut down */ cancel_delayed_work_sync(&server->dwork); diff --git a/kmod/src/server.h b/kmod/src/server.h index 96f3a5df..506aa546 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -22,13 +22,14 @@ do { \ __entry->name##_addr & 255, \ __entry->name##_port -#define SNH_FMT "id %llu data_len %u type %u status %u" +#define SNH_FMT "id %llu data_len %u msg %u cmd %u error %u" #define snh_trace_define(name) \ __field(__u64, name##_id) \ __field(__u16, name##_data_len) \ - __field(__u8, name##_type) \ - __field(__u8, name##_status) + __field(__u8, name##_msg) \ + __field(__u8, name##_cmd) \ + __field(__u8, name##_error) #define snh_trace_assign(name, nh) \ do { \ @@ -36,13 +37,14 @@ do { \ \ __entry->name##_id = le64_to_cpu(_nh->id); \ __entry->name##_data_len = le16_to_cpu(_nh->data_len); \ - __entry->name##_type = _nh->type; \ - __entry->name##_status = _nh->status; \ + __entry->name##_msg = _nh->msg; \ + __entry->name##_cmd = _nh->cmd; \ + __entry->name##_error = _nh->error; \ } while (0) #define snh_trace_args(name) \ - __entry->name##_id, __entry->name##_data_len, __entry->name##_type, \ - __entry->name##_status + __entry->name##_id, __entry->name##_data_len, __entry->name##_msg, \ + __entry->name##_cmd, __entry->name##_error void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment); diff --git a/kmod/src/super.c b/kmod/src/super.c index 4279ea96..d9540334 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -38,6 +38,7 @@ #include "compact.h" #include "data.h" #include "lock.h" +#include "net.h" #include "client.h" #include "server.h" #include "options.h" @@ -133,6 +134,7 @@ static void scoutfs_put_super(struct super_block *sb) /* the server locks the listen address and compacts */ scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); + scoutfs_net_destroy(sb); scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); @@ -333,6 +335,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: scoutfs_lock_setup(sb) ?: + scoutfs_net_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, diff --git a/kmod/src/super.h b/kmod/src/super.h index 8e9cece1..99944c8d 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -22,6 +22,7 @@ struct inode_sb_info; struct btree_info; struct sysfs_info; struct options_sb_info; +struct net_info; struct scoutfs_sb_info { struct super_block *sb; @@ -42,6 +43,7 @@ struct scoutfs_sb_info { struct data_info *data_info; struct inode_sb_info *inode_sb_info; struct btree_info *btree_info; + struct net_info *net_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From d708421cfb1b16d6d772644eb497da90c2511485 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 23 Jul 2018 15:32:13 -0700 Subject: [PATCH 652/920] scoutfs: remove unused client and server code The previous commit added shared networking code and disabled the old unused code. This removes all that unused client and server code that was refactored to become the shared networking code. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/client.c | 446 ---------------------------------------------- kmod/src/server.c | 258 --------------------------- kmod/src/sock.c | 96 ---------- kmod/src/sock.h | 7 - 5 files changed, 1 insertion(+), 808 deletions(-) delete mode 100644 kmod/src/sock.c delete mode 100644 kmod/src/sock.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 237e0f11..d4d0db75 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -8,7 +8,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o lock.o \ manifest.o msg.o net.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sock.o sort_priv.o super.o sysfs.o trans.o \ + scoutfs_trace.o sort_priv.o super.o sysfs.o trans.o \ triggers.o xattr.o # diff --git a/kmod/src/client.c b/kmod/src/client.c index c7e2cdac..a21cbefd 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -30,7 +30,6 @@ #include "compact.h" #include "scoutfs_trace.h" #include "msg.h" -#include "server.h" #include "client.h" #include "net.h" #include "endian_swap.h" @@ -40,22 +39,6 @@ * super to get the address it should try and connect to. */ -#define SIN_FMT "%pIS:%u" -#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) - -#if 0 -/* - * Have a pretty aggressive keepalive timeout of around 10 seconds. The - * TCP keepalives are being processed out of task context so they should - * be responsive even when mounts are under load. We also derive the - * connect timeout from this. - */ -#define KEEPCNT 3 -#define KEEPIDLE 7 -#define KEEPINTVL 1 -#endif - - /* * Connection timeouts have to allow for enough time for servers to * reboot. Figure order minutes at the outside. @@ -77,126 +60,6 @@ struct client_info { unsigned long conn_retry_ms; }; -#if 0 -struct waiting_sender { - struct rb_node node; - struct task_struct *task; - - u64 id; - void *rx; - size_t rx_size; - int result; -}; - -static struct waiting_sender *walk_sender_tree(struct client_info *client, - u64 id, - struct waiting_sender *ins) -{ - struct rb_node **node = &client->sender_root.rb_node; - struct waiting_sender *found = NULL; - struct waiting_sender *sender; - struct rb_node *parent = NULL; - - assert_spin_locked(&client->recv_lock); - - while (*node) { - parent = *node; - sender = container_of(*node, struct waiting_sender, node); - - if (id < sender->id) { - node = &(*node)->rb_left; - } else if (id > sender->id) { - node = &(*node)->rb_right; - } else { - found = sender; - break; - } - } - - if (ins) { - /* ids are never reused and assigned under lock */ - BUG_ON(found); - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, &client->sender_root); - found = ins; - } - - return found; -} - -/* - * This work is queued once the socket is created. It blocks trying to - * receive replies to sent messages. If the sender is still around it - * receives the reply data into their buffer. If the sender has left - * then it silently drops the reply. - * - * This exits once someone shuts down the socket. If this sees a fatal - * error it shuts down the socket which causes senders to reconnect. - */ -static void scoutfs_client_recv_func(struct work_struct *work) -{ - struct client_info *client = container_of(work, struct client_info, - recv_work); - struct waiting_sender *sender; - struct scoutfs_net_header nh; - void *rx = NULL; - u16 data_len; - int ret; - - for (;;) { - /* receive the header */ - ret = scoutfs_sock_recvmsg(client->sock, &nh, sizeof(nh)); - if (ret) - break; - - data_len = le16_to_cpu(nh.data_len); - - trace_scoutfs_client_recv_reply(client->sb, - &client->sockname, - &client->peername, &nh); - - /* receive the payload */ - kfree(rx); - rx = kmalloc(data_len, GFP_NOFS); - if (!rx) { - ret = -ENOMEM; - break; - } - - /* recv failure can be server crashing, not fatal */ - ret = scoutfs_sock_recvmsg(client->sock, rx, data_len); - if (ret) { - break; - } - - /* give the payload to a sender if there is one */ - spin_lock(&client->recv_lock); - sender = walk_sender_tree(client, le64_to_cpu(nh.id), NULL); - if (sender) { - /* protocol mismatch is fatal */ - if (sender->rx_size < data_len) { - sender->result = -EIO; - } else { - memcpy(sender->rx, rx, data_len); - sender->result = 0; - } - smp_mb(); /* store result before waking */ - wake_up_process(sender->task); - } - spin_unlock(&client->recv_lock); - } - - /* make senders reconnect if we see an rx error */ - if (ret) { - /* XXX would need to break out send */ - kernel_sock_shutdown(client->sock, SHUT_RDWR); - client->recv_shutdown = true; - } - - kfree(rx); -} -#endif - static void reset_connect_timeout(struct client_info *client) { client->conn_retry_ms = CONN_RETRY_MIN_MS; @@ -208,315 +71,6 @@ static void grow_connect_timeout(struct client_info *client) CONN_RETRY_MAX_MS); } -#if 0 -/* - * Clients who try to send and don't see a connected socket call here to - * connect to the server. They get the server address and try to - * connect. - * - * Each sending client will always try to connect once. After that - * it'll sleep and retry connecting at increasing intervals. After long - * enough it will return an error. Future attempts will retry once then - * return errors. - */ -static int client_connect(struct client_info *client) -{ - struct super_block *sb = client->sb; - struct scoutfs_super_block super; - struct scoutfs_net_greeting greet; - struct sockaddr_in *sin; - struct socket *sock = NULL; - struct timeval tv; - struct kvec kv; - int retries; - int addrlen; - int optval; - int ret; - - BUG_ON(!mutex_is_locked(&client->send_mutex)); - - for(retries = 0; ; retries++) { - if (sock) { - sock_release(sock); - sock = NULL; - } - - if (retries) { - /* we tried, and we're past limit, return error */ - if (time_after(jiffies, client->conn_retry_limit_j)) { - ret = -ENOTCONN; - break; - } - - msleep_interruptible(client->conn_retry_ms); - - client->conn_retry_ms = min(client->conn_retry_ms * 2, - CONN_RETRY_MAX_MS); - } - - if (signal_pending(current)) { - ret = -ERESTARTSYS; - break; - } - - ret = scoutfs_read_super(sb, &super); - if (ret) - continue; - - if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) - continue; - - sin = &client->peername; - sin->sin_family = AF_INET; - sin->sin_addr.s_addr = le32_to_be32(super.server_addr.addr); - sin->sin_port = le16_to_be16(super.server_addr.port); - - ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, - &sock); - if (ret) - continue; - - optval = 1; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, - (char *)&optval, sizeof(optval)); - if (ret) - continue; - - /* use short timeout for connect itself */ - tv.tv_sec = 1; - tv.tv_usec = 0; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, - (char *)&tv, sizeof(tv)); - if (ret) - continue; - - client->sock = sock; - - ret = kernel_connect(sock, (struct sockaddr *)sin, - sizeof(struct sockaddr_in), 0); - if (ret) - continue; - - greet.fsid = super.id; - greet.format_hash = super.format_hash; - kv.iov_base = &greet; - kv.iov_len = sizeof(greet); - ret = scoutfs_sock_sendmsg(sock, &kv, 1); - if (ret) - continue; - - ret = scoutfs_sock_recvmsg(sock, &greet, sizeof(greet)); - if (ret) - continue; - - if (greet.fsid != super.id) { - scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&client->peername), - le64_to_cpu(greet.fsid), - le64_to_cpu(super.id)); - continue; - } - - if (greet.format_hash != super.format_hash) { - scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&client->peername), - le64_to_cpu(greet.format_hash), - le64_to_cpu(super.format_hash)); - continue; - } - - /* but use a keepalive timeout instead of send timeout */ - tv.tv_sec = 0; - tv.tv_usec = 0; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, - (char *)&tv, sizeof(tv)); - if (ret) - continue; - - optval = KEEPCNT; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, - (char *)&optval, sizeof(optval)); - if (ret) - continue; - - optval = KEEPIDLE; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, - (char *)&optval, sizeof(optval)); - if (ret) - continue; - - optval = KEEPINTVL; - ret = kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, - (char *)&optval, sizeof(optval)); - if (ret) - continue; - - optval = 1; - ret = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, - (char *)&optval, sizeof(optval)); - if (ret) - continue; - - addrlen = sizeof(struct sockaddr_in); - ret = kernel_getsockname(sock, - (struct sockaddr *)&client->sockname, - &addrlen); - if (ret) - continue; - - scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, - SIN_ARG(&client->sockname), - SIN_ARG(&client->peername)); - - client->sock_gen++; - client->recv_shutdown = false; - reset_connect_timeouts(client); - queue_work(client->recv_wq, &client->recv_work); - wake_up(&client->waitq); - ret = 0; - break; - } - - if (ret && sock) - sock_release(sock); - - return ret; -} - -/* either a sender or unmount is destroying the socket */ -static void shutdown_sock_sync(struct client_info *client) -{ - struct super_block *sb = client->sb; - struct socket *sock = client->sock; - - if (sock) { - kernel_sock_shutdown(sock, SHUT_RDWR); - cancel_work_sync(&client->recv_work); - sock_release(sock); - client->sock = NULL; - - scoutfs_info(sb, "client disconnected "SIN_FMT" -> "SIN_FMT, - SIN_ARG(&client->sockname), - SIN_ARG(&client->peername)); - } -} - -/* - * Senders sleep waiting for a reply to come down the connection out - * which they just sent a request. They need to wake up when the recv - * work has given them a reply or when it's given up and the sender - * needs to reconnect and resend. - * - * This is a condition for wait_event. The barrier orders the task - * state store before loading the sender and client fields. - */ -static int sender_should_wake(struct client_info *client, - struct waiting_sender *sender) -{ - smp_mb(); - return sender->result != -EINPROGRESS || client->recv_shutdown; -} - -/* - * Block sending a request and then waiting for the reply. All senders - * are responsible for connecting sockets and sending their requests. - * recv work blocks receiving from the socket and waking senders if - * they're reply has been copied to their buffer. If the socket sees an - * error the recv work will shutdown and wake us to reconnect. - */ -static int client_request(struct client_info *client, int type, void *data, - unsigned data_len, void *rx, size_t rx_size) -{ - struct waiting_sender sender; - struct scoutfs_net_header nh; - struct kvec kv[2]; - unsigned kv_len; - u64 sent_to_gen = ~0ULL; - int ret = 0; - - if (WARN_ON_ONCE(!data && data_len)) - return -EINVAL; - - spin_lock(&client->recv_lock); - - sender.task = current; - sender.id = client->next_id++; - sender.rx = rx; - sender.rx_size = rx_size; - sender.result = -EINPROGRESS; - - nh.id = cpu_to_le64(sender.id); - nh.data_len = cpu_to_le16(data_len); - nh.type = type; - nh.status = SCOUTFS_NET_STATUS_REQUEST; - - walk_sender_tree(client, sender.id, &sender); - - spin_unlock(&client->recv_lock); - - mutex_lock(&client->send_mutex); - - while (sender.result == -EINPROGRESS) { - - if (!client->sock) { - ret = client_connect(client); - if (ret < 0) - break; - } - - if (sent_to_gen != client->sock_gen) { - kv[0].iov_base = &nh; - kv[0].iov_len = sizeof(nh); - kv[1].iov_base = data; - kv[1].iov_len = data_len; - kv_len = data ? 2 : 1; - - trace_scoutfs_client_send_request(client->sb, - &client->sockname, - &client->peername, - &nh); - - ret = scoutfs_sock_sendmsg(client->sock, kv, kv_len); - if (ret) { - shutdown_sock_sync(client); - continue; - } - - sent_to_gen = client->sock_gen; - } - - mutex_unlock(&client->send_mutex); - - ret = wait_event_interruptible(client->waitq, - sender_should_wake(client, &sender)); - if (ret < 0 && sender.result == -EINPROGRESS) { - sender.result = ret; - ret = 0; - } - - mutex_lock(&client->send_mutex); - - /* finish tearing down the socket if recv shutdown */ - if (client->sock && client->recv_shutdown) { - shutdown_sock_sync(client); - continue; - } - } - - mutex_unlock(&client->send_mutex); - - /* only we remove senders, recv only uses senders under the lock */ - spin_lock(&client->recv_lock); - rb_erase(&sender.node, &client->sender_root); - spin_unlock(&client->recv_lock); - - if (ret == 0) - ret = sender.result; - - return ret; -} -#endif - /* * Ask for a new run of allocated inode numbers. The server can return * fewer than @count. It will success with nr == 0 if we've run out. diff --git a/kmod/src/server.c b/kmod/src/server.c index 13428ad2..1947a8bf 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -30,7 +30,6 @@ #include "compact.h" #include "scoutfs_trace.h" #include "msg.h" -#include "client.h" #include "server.h" #include "net.h" #include "endian_swap.h" @@ -46,9 +45,6 @@ * mount will have less trouble. */ -#define SIN_FMT "%pIS:%u" -#define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) - struct server_info { struct super_block *sb; @@ -81,26 +77,6 @@ struct server_info { #define DECLARE_SERVER_INFO(sb, name) \ struct server_info *name = SCOUTFS_SB(sb)->server_info -#if 0 -struct server_request { - struct server_connection *conn; - struct work_struct work; - - struct scoutfs_net_header nh; - /* data payload is allocated here, referenced as ->nh.data */ -}; - -struct server_connection { - struct server_info *server; - struct sockaddr_in sockname; - struct sockaddr_in peername; - struct list_head head; - struct socket *sock; - struct work_struct recv_work; - struct mutex send_mutex; -}; -#endif - struct commit_waiter { struct completion comp; struct llist_node node; @@ -585,57 +561,6 @@ out: trace_scoutfs_server_commit_work_exit(sb, 0, ret); } -#if 0 -/* - * Request processing synchronously sends their reply from within their - * processing work. If this fails the socket is shutdown. - */ -static int send_reply(struct server_connection *conn, u64 id, - u8 type, int error, void *data, unsigned data_len) -{ - struct scoutfs_net_header nh; - struct kvec kv[2]; - unsigned kv_len; - u8 status; - int ret; - - if (WARN_ON_ONCE(error > 0) || WARN_ON_ONCE(data && data_len == 0)) - return -EINVAL; - - kv[0].iov_base = &nh; - kv[0].iov_len = sizeof(nh); - kv_len = 1; - - /* maybe we can have better error communication to clients */ - if (error < 0) { - status = SCOUTFS_NET_STATUS_ERROR; - data = NULL; - data_len = 0; - } else { - status = SCOUTFS_NET_STATUS_SUCCESS; - if (data) { - kv[1].iov_base = data; - kv[1].iov_len = data_len; - kv_len++; - } - } - - nh.id = cpu_to_le64(id); - nh.data_len = cpu_to_le16(data_len); - nh.type = type; - nh.status = status; - - trace_scoutfs_server_send_reply(conn->server->sb, &conn->sockname, - &conn->peername, &nh); - - mutex_lock(&conn->send_mutex); - ret = scoutfs_sock_sendmsg(conn->sock, kv, kv_len); - mutex_unlock(&conn->send_mutex); - - return ret; -} -#endif - void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment) { @@ -1117,179 +1042,6 @@ int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, return ret; } -#if 0 -typedef int (*process_func_t)(struct server_connection *conn, u64 id, - u8 type, void *data, unsigned data_len); - -/* - * Each request message gets its own concurrent blocking request processing - * context. - */ -static void scoutfs_server_process_func(struct work_struct *work) -{ - struct server_request *req = container_of(work, struct server_request, - work); - struct server_connection *conn = req->conn; - static process_func_t process_funcs[] = { - [SCOUTFS_NET_ALLOC_INODES] = process_alloc_inodes, - [SCOUTFS_NET_ALLOC_EXTENT] = process_alloc_extent, - [SCOUTFS_NET_FREE_EXTENTS] = process_free_extents, - [SCOUTFS_NET_ALLOC_SEGNO] = process_alloc_segno, - [SCOUTFS_NET_RECORD_SEGMENT] = process_record_segment, - [SCOUTFS_NET_ADVANCE_SEQ] = process_advance_seq, - [SCOUTFS_NET_GET_LAST_SEQ] = process_get_last_seq, - [SCOUTFS_NET_GET_MANIFEST_ROOT] = process_get_manifest_root, - [SCOUTFS_NET_STATFS] = process_statfs, - }; - struct scoutfs_net_header *nh = &req->nh; - process_func_t func; - int ret; - - if (nh->type < ARRAY_SIZE(process_funcs)) - func = process_funcs[nh->type]; - else - func = NULL; - - if (func) - ret = func(conn, le64_to_cpu(nh->id), nh->type, nh->data, - le16_to_cpu(nh->data_len)); - else - ret = -EINVAL; - - if (ret) - kernel_sock_shutdown(conn->sock, SHUT_RDWR); - - /* process_one_work explicitly allows freeing work in its func */ - kfree(req); -} -#endif - -#if 0 -/* - * Always block receiving from the socket. This owns the socket. If - * receive fails this shuts down and frees the socket. - */ -static void scoutfs_server_recv_func(struct work_struct *work) -{ - struct server_connection *conn = container_of(work, - struct server_connection, - recv_work); - struct server_info *server = conn->server; - struct super_block *sb = server->sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct socket *sock = conn->sock; - struct workqueue_struct *req_wq; - struct scoutfs_net_greeting greet; - struct scoutfs_net_header nh; - struct server_request *req; - bool passed_greeting; - unsigned data_len; - struct kvec kv; - int ret; - - trace_scoutfs_server_recv_work_enter(sb, 0, 0); - - req_wq = alloc_workqueue("scoutfs_server_requests", - WQ_NON_REENTRANT, 0); - if (!req_wq) { - ret = -ENOMEM; - goto out; - } - - /* first bounce the greeting */ - ret = scoutfs_sock_recvmsg(sock, &greet, sizeof(greet)); - if (ret) - goto out; - - /* we'll close conn after failed greeting to let client see ours */ - passed_greeting = false; - - if (greet.fsid != super->id) { - scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(greet.fsid), - le64_to_cpu(super->id)); - } else if (greet.format_hash != super->format_hash) { - scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(greet.format_hash), - le64_to_cpu(super->format_hash)); - } else { - passed_greeting = true; - } - - greet.fsid = super->id; - greet.format_hash = super->format_hash; - kv.iov_base = &greet; - kv.iov_len = sizeof(greet); - ret = scoutfs_sock_sendmsg(sock, &kv, 1); - if (ret) - goto out; - - for (;;) { - /* receive the header */ - ret = scoutfs_sock_recvmsg(sock, &nh, sizeof(nh)); - if (ret) - break; - - if (!passed_greeting) - break; - - trace_scoutfs_server_recv_request(conn->server->sb, - &conn->sockname, - &conn->peername, &nh); - - /* XXX verify data_len isn't insane */ - /* XXX test for bad messages */ - data_len = le16_to_cpu(nh.data_len); - - req = kmalloc(sizeof(struct server_request) + data_len, - GFP_NOFS); - if (!req) { - ret = -ENOMEM; - break; - } - - ret = scoutfs_sock_recvmsg(sock, req->nh.data, data_len); - if (ret) - break; - - req->conn = conn; - INIT_WORK(&req->work, scoutfs_server_process_func); - req->nh = nh; - - queue_work(req_wq, &req->work); - /* req is freed by its work func */ - req = NULL; - } - -out: - scoutfs_info(sb, "server closing "SIN_FMT" -> "SIN_FMT, - SIN_ARG(&conn->peername), SIN_ARG(&conn->sockname)); - - /* make sure reply sending returns */ - kernel_sock_shutdown(conn->sock, SHUT_RDWR); - - /* wait for processing work to drain */ - if (req_wq) { - drain_workqueue(req_wq); - destroy_workqueue(req_wq); - } - - /* process_one_work explicitly allows freeing work in its func */ - mutex_lock(&server->mutex); - sock_release(conn->sock); - list_del_init(&conn->head); - kfree(conn); - smp_mb(); - wake_up_process(server->listen_task); - mutex_unlock(&server->mutex); - - trace_scoutfs_server_recv_work_exit(sb, 0, ret); -} -#endif - /* * This relies on the caller having read the current super and advanced * its seq so that it's dirty. This will go away when we communicate @@ -1305,16 +1057,6 @@ static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) return scoutfs_write_dirty_super(sb); } -#if 0 -static bool barrier_list_empty_careful(struct list_head *list) -{ - /* store caller's task state before loading wake condition */ - smp_mb(); - - return list_empty_careful(list); -} -#endif - static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, diff --git a/kmod/src/sock.c b/kmod/src/sock.c deleted file mode 100644 index 4783310a..00000000 --- a/kmod/src/sock.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sock.h" - -/* - * Some quick socket helper wrappers. - */ - -static struct kvec *kvec_advance(struct kvec *kv, unsigned *kv_len, - unsigned bytes) -{ - while (*kv_len && bytes) { - if (kv->iov_len <= bytes) { - bytes -= kv->iov_len; - kv++; - (*kv_len)--; - } else { - kv->iov_base += bytes; - kv->iov_len -= bytes; - bytes = 0; - } - } - - return kv; -} - -/* - * This can modify the kvec as it modifies the vec to continue after - * partial sends. - */ -int scoutfs_sock_sendmsg(struct socket *sock, struct kvec *kv, unsigned kv_len) -{ - struct msghdr msg; - int ret; - - while (kv_len) { - memset(&msg, 0, sizeof(msg)); - msg.msg_iov = (struct iovec *)kv; - msg.msg_iovlen = kv_len; - msg.msg_flags = MSG_NOSIGNAL; - - ret = kernel_sendmsg(sock, &msg, kv, kv_len, - iov_length((struct iovec *)kv, kv_len)); - if (ret <= 0) - return -ECONNABORTED; - - kv = kvec_advance(kv, &kv_len, ret); - } - - return 0; -} - -int scoutfs_sock_recvmsg(struct socket *sock, void *buf, unsigned len) -{ - struct msghdr msg; - struct kvec kv; - int ret; - - while (len) { - memset(&msg, 0, sizeof(msg)); - msg.msg_iov = (struct iovec *)&kv; - msg.msg_iovlen = 1; - msg.msg_flags = MSG_NOSIGNAL; - kv.iov_base = buf; - kv.iov_len = len; - - ret = kernel_recvmsg(sock, &msg, &kv, 1, len, msg.msg_flags); - if (ret <= 0) - return -ECONNABORTED; - - len -= ret; - buf += ret; - } - - return 0; -} diff --git a/kmod/src/sock.h b/kmod/src/sock.h deleted file mode 100644 index 5b61bea0..00000000 --- a/kmod/src/sock.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef _SCOUTFS_SOCK_H_ -#define _SCOUTFS_SOCK_H_ - -int scoutfs_sock_recvmsg(struct socket *sock, void *buf, unsigned len); -int scoutfs_sock_sendmsg(struct socket *sock, struct kvec *kv, unsigned kv_len); - -#endif From c4cb5c0651710b2d4a776bb84c77bdf9fd2404b6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jul 2018 16:26:10 -0700 Subject: [PATCH 653/920] scoutfs: add trivial seq file wrapper Add a seq file wrapper which lets callers track objects easily. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/tseq.c | 224 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/tseq.h | 32 +++++++ 3 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 kmod/src/tseq.c create mode 100644 kmod/src/tseq.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index d4d0db75..0cf70986 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -9,7 +9,7 @@ scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o lock.o \ manifest.o msg.o net.o options.o per_task.o seg.o server.o \ scoutfs_trace.o sort_priv.o super.o sysfs.o trans.o \ - triggers.o xattr.o + triggers.o tseq.o xattr.o # # The raw types aren't available in userspace headers. Make sure all diff --git a/kmod/src/tseq.c b/kmod/src/tseq.c new file mode 100644 index 00000000..781d00f3 --- /dev/null +++ b/kmod/src/tseq.c @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "tseq.h" + +/* + * This trivial seq file wrapper takes care of the details of displaying + * a set of objects in seq file output. We use an augmented rbtree to + * add new objects at the next free file position. The caller takes + * care of the object life times, only debugfs file creation can fail. + */ + +static loff_t tseq_node_total(struct rb_node *node) +{ + struct scoutfs_tseq_entry *ent; + + if (node == NULL) + return 0; + + ent = rb_entry(node, struct scoutfs_tseq_entry, node); + return ent->total; +} + +static struct scoutfs_tseq_entry *tseq_rb_next(struct scoutfs_tseq_entry *ent) +{ + struct rb_node *node = rb_next(&ent->node); + + if (node == NULL) + return NULL; + + return rb_entry(node, struct scoutfs_tseq_entry, node); +} + +static loff_t tseq_compute_total(struct scoutfs_tseq_entry *ent) +{ + return 1 + tseq_node_total(ent->node.rb_left) + + tseq_node_total(ent->node.rb_right); +} + +RB_DECLARE_CALLBACKS(static, tseq_rb_callbacks, struct scoutfs_tseq_entry, + node, loff_t, total, tseq_compute_total) + +void scoutfs_tseq_tree_init(struct scoutfs_tseq_tree *tree, + scoutfs_tseq_show_t show) +{ + spin_lock_init(&tree->lock); + tree->root = RB_ROOT; + tree->show = show; +} + +/* + * Descend towards the leaf node that should be the parent for inserting + * a new entry. + * + * We use the augmented subtree totals to see when a left subtree has + * fewer entries than the current entry's pos which tells us that there + * is a lesser free pos. + * + * If there isn't a lesser free pos then we descend to the right and set + * the minimum possible pos to the pos after the entry we're traversing. + */ +void scoutfs_tseq_add(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ins) +{ + struct scoutfs_tseq_entry *ent; + struct rb_node *parent; + struct rb_node **node; + loff_t min_pos; + + spin_lock(&tree->lock); + + node = &tree->root.rb_node; + parent = NULL; + min_pos = 0; + + while (*node) { + parent = *node; + ent = rb_entry(*node, struct scoutfs_tseq_entry, node); + + ent->total++; + + if (min_pos + tseq_node_total(ent->node.rb_left) < ent->pos) { + node = &ent->node.rb_left; + } else { + min_pos = ent->pos + 1; + node = &ent->node.rb_right; + } + } + + ins->pos = min_pos; + ins->total = 1; + rb_link_node(&ins->node, parent, node); + rb_insert_augmented(&ins->node, &tree->root, &tseq_rb_callbacks); + + spin_unlock(&tree->lock); +} + +static struct scoutfs_tseq_entry *tseq_pos_next(struct scoutfs_tseq_tree *tree, + loff_t pos) +{ + struct scoutfs_tseq_entry *next; + struct scoutfs_tseq_entry *ent; + struct rb_node *node; + + assert_spin_locked(&tree->lock); + + node = tree->root.rb_node; + next = NULL; + + while (node) { + ent = rb_entry(node, struct scoutfs_tseq_entry, node); + + if (pos < ent->pos) { + next = ent; + node = ent->node.rb_left; + } else if (pos > ent->pos) { + node = ent->node.rb_right; + } else { + return ent; + } + } + + return next; +} + +void scoutfs_tseq_del(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent) +{ + spin_lock(&tree->lock); + rb_erase_augmented(&ent->node, &tree->root, &tseq_rb_callbacks); + RB_CLEAR_NODE(&ent->node); + spin_unlock(&tree->lock); +} + +/* _stop is always called no matter what start returns */ +static void *scoutfs_tseq_seq_start(struct seq_file *m, loff_t *pos) + __acquires(tree->lock) +{ + struct scoutfs_tseq_tree *tree = m->private; + + spin_lock(&tree->lock); + + return tseq_pos_next(tree, *pos); +} + +static void *scoutfs_tseq_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct scoutfs_tseq_entry *ent = v; + + ent = tseq_rb_next(ent); + if (ent) + *pos = ent->pos; + return ent; +} + +static void scoutfs_tseq_seq_stop(struct seq_file *m, void *v) + __releases(tree->lock) +{ + struct scoutfs_tseq_tree *tree = m->private; + + spin_unlock(&tree->lock); +} + +static int scoutfs_tseq_seq_show(struct seq_file *m, void *v) +{ + struct scoutfs_tseq_tree *tree = m->private; + struct scoutfs_tseq_entry *ent = v; + + tree->show(m, ent); + return 0; +} + +static const struct seq_operations scoutfs_tseq_seq_ops = { + .start = scoutfs_tseq_seq_start, + .next = scoutfs_tseq_seq_next, + .stop = scoutfs_tseq_seq_stop, + .show = scoutfs_tseq_seq_show, +}; + +static int scoutfs_tseq_open(struct inode *inode, struct file *file) +{ + struct seq_file *m; + int ret; + + ret = seq_open(file, &scoutfs_tseq_seq_ops); + if (ret == 0) { + m = file->private_data; + m->private = inode->i_private; + } + return ret; +} + +static const struct file_operations scoutfs_tseq_fops = { + .open = scoutfs_tseq_open, + .release = seq_release, + .read = seq_read, + .llseek = seq_lseek, +}; + +/* + * This doesn't create any additional state so the returned dentry + * can be destroyed with the usual debugfs file calls. + */ +struct dentry *scoutfs_tseq_create(const char *name, struct dentry *parent, + struct scoutfs_tseq_tree *tree) +{ + return debugfs_create_file(name, S_IFREG|S_IRUSR, parent, tree, + &scoutfs_tseq_fops); +} diff --git a/kmod/src/tseq.h b/kmod/src/tseq.h new file mode 100644 index 00000000..d9b05a9e --- /dev/null +++ b/kmod/src/tseq.h @@ -0,0 +1,32 @@ +#ifndef _SCOUTFS_TSEQ_H_ +#define _SCOUTFS_TSEQ_H_ + +#include + +struct scoutfs_tseq_entry; +typedef void (*scoutfs_tseq_show_t)(struct seq_file *m, + struct scoutfs_tseq_entry *ent); + +struct scoutfs_tseq_tree { + spinlock_t lock; + struct rb_root root; + scoutfs_tseq_show_t show; +}; + +struct scoutfs_tseq_entry { + struct rb_node node; + loff_t pos; + loff_t total; +}; + +void scoutfs_tseq_tree_init(struct scoutfs_tseq_tree *tree, + scoutfs_tseq_show_t show); +void scoutfs_tseq_add(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent); +void scoutfs_tseq_del(struct scoutfs_tseq_tree *tree, + struct scoutfs_tseq_entry *ent); + +struct dentry *scoutfs_tseq_create(const char *name, struct dentry *parent, + struct scoutfs_tseq_tree *tree); + +#endif From 8ff3ef313109621958152c1397f57fdfc1fb2aac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Jul 2018 16:49:58 -0700 Subject: [PATCH 654/920] scoutfs: add trivial seq file for net connections Signed-off-by: Zach Brown --- kmod/src/net.c | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 1ab0b377..c6ba10a9 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -32,7 +32,7 @@ #include "msg.h" #include "net.h" #include "endian_swap.h" - +#include "tseq.h" /* * scoutfs networking reliably delivers requests and responses between @@ -64,6 +64,8 @@ */ struct net_info { struct workqueue_struct *shutdown_workq; + struct dentry *conn_tseq_dentry; + struct scoutfs_tseq_tree conn_tseq_tree; }; struct scoutfs_net_connection { @@ -102,6 +104,8 @@ struct scoutfs_net_connection { struct work_struct recv_work; struct work_struct shutdown_work; /* message_recv proc_work also executes in the conn workq */ + + struct scoutfs_tseq_entry tseq_entry; }; /* @@ -837,6 +841,8 @@ static void scoutfs_net_send_worker(struct work_struct *work) static void destroy_conn(struct scoutfs_net_connection *conn) { + struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *listener; struct message_send *msend; struct message_send *tmp; @@ -863,6 +869,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) } destroy_workqueue(conn->workq); + scoutfs_tseq_del(&ninf->conn_tseq_tree, &conn->tseq_entry); kfree(conn); } @@ -1114,7 +1121,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); /* connected and accepted conns print a message */ - if (conn->peername.sin_family) + if (conn->peername.sin_port != 0) scoutfs_info(sb, "%s "SIN_FMT" -> "SIN_FMT, conn->listening_conn ? "server closing" : "client disconnected", @@ -1186,6 +1193,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_down, scoutfs_net_request_t *req_funcs, char *name_suffix) { + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; /* we handle greetings, the caller shouldn't attempt to */ @@ -1210,6 +1218,8 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->notify_down = notify_down; conn->req_funcs = req_funcs; spin_lock_init(&conn->lock); + conn->sockname.sin_family = AF_INET; + conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); init_waitqueue_head(&conn->accepted_waitq); @@ -1222,6 +1232,8 @@ scoutfs_net_alloc_conn(struct super_block *sb, INIT_WORK(&conn->recv_work, scoutfs_net_recv_worker); INIT_WORK(&conn->shutdown_work, scoutfs_net_shutdown_worker); + scoutfs_tseq_add(&ninf->conn_tseq_tree, &conn->tseq_entry); + return conn; } @@ -1448,6 +1460,19 @@ int scoutfs_net_sync_request(struct super_block *sb, return ret; } +static void net_tseq_show_conn(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct scoutfs_net_connection *conn = + container_of(ent, struct scoutfs_net_connection, tseq_entry); + + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" vg %u est %u sd %u cto_ms %lu nsi %llu lpi %llu\n", + SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), + conn->valid_greeting, conn->established, + conn->shutting_down, conn->connect_timeout_ms, + conn->next_send_id, conn->last_proc_id); +} + int scoutfs_net_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1464,9 +1489,10 @@ int scoutfs_net_setup(struct super_block *sb) ret = -ENOMEM; goto out; } - sbi->net_info = ninf; + scoutfs_tseq_tree_init(&ninf->conn_tseq_tree, net_tseq_show_conn); + ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", WQ_UNBOUND, 0); if (!ninf->shutdown_workq) { @@ -1474,6 +1500,14 @@ int scoutfs_net_setup(struct super_block *sb) goto out; } + ninf->conn_tseq_dentry = scoutfs_tseq_create("connections", + sbi->debug_root, + &ninf->conn_tseq_tree); + if (!ninf->conn_tseq_dentry) { + ret = -ENOMEM; + goto out; + } + ret = 0; out: if (ret) @@ -1489,6 +1523,7 @@ void scoutfs_net_destroy(struct super_block *sb) if (ninf) { if (ninf->shutdown_workq) destroy_workqueue(ninf->shutdown_workq); + debugfs_remove(ninf->conn_tseq_dentry); kfree(ninf); sbi->net_info = NULL; } From bafa4a672083e3f87818528c0e99a38c897b72d0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 25 Jul 2018 12:16:33 -0700 Subject: [PATCH 655/920] scoutfs: add net header printk args We have macros for creating and printing trace arguments for our network header struct. Add a macro for making simple printk call args for normal formatted output callers. Signed-off-by: Zach Brown --- kmod/src/server.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/server.h b/kmod/src/server.h index 506aa546..54e61d04 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -23,6 +23,8 @@ do { \ __entry->name##_port #define SNH_FMT "id %llu data_len %u msg %u cmd %u error %u" +#define SNH_ARG(nh) le64_to_cpu((nh)->id), le16_to_cpu((nh)->data_len), \ + (nh)->msg, (nh)->cmd, (nh)->error #define snh_trace_define(name) \ __field(__u64, name##_id) \ From 07df8816e32a176900bb45d16e60eb53c83f4b8d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 25 Jul 2018 12:17:21 -0700 Subject: [PATCH 656/920] scoutfs: add trivial seq file for net messages Signed-off-by: Zach Brown --- kmod/src/net.c | 78 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index c6ba10a9..3ebc062a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -66,6 +66,8 @@ struct net_info { struct workqueue_struct *shutdown_workq; struct dentry *conn_tseq_dentry; struct scoutfs_tseq_tree conn_tseq_tree; + struct dentry *msg_tseq_dentry; + struct scoutfs_tseq_tree msg_tseq_tree; }; struct scoutfs_net_connection { @@ -122,10 +124,11 @@ struct scoutfs_net_connection { * to risk freeing messages from under the unlocked send worker. */ struct message_send { + struct scoutfs_tseq_entry tseq_entry; + unsigned long dead:1; struct list_head head; scoutfs_net_response_t resp_func; void *resp_data; - unsigned long dead:1; struct scoutfs_net_header nh; }; @@ -134,8 +137,9 @@ struct message_send { * contexts. */ struct message_recv { - struct scoutfs_net_connection *conn; + struct scoutfs_tseq_entry tseq_entry; struct work_struct proc_work; + struct scoutfs_net_connection *conn; struct scoutfs_net_header nh; }; @@ -313,6 +317,7 @@ static int submit_send(struct super_block *sb, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct message_send *msend; if (WARN_ON_ONCE(msg >= SCOUTFS_NET_MSG_UNKNOWN) || @@ -358,6 +363,8 @@ static int submit_send(struct super_block *sb, if (id_ret) *id_ret = le64_to_cpu(msend->nh.id); + scoutfs_tseq_add(&ninf->msg_tseq_tree, &msend->tseq_entry); + spin_unlock(&conn->lock); return 0; @@ -574,6 +581,7 @@ static void scoutfs_net_proc_worker(struct work_struct *work) proc_work); struct scoutfs_net_connection *conn = mrecv->conn; struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; int ret; trace_scoutfs_net_proc_work_enter(sb, 0, 0); @@ -596,6 +604,7 @@ static void scoutfs_net_proc_worker(struct work_struct *work) } /* process_one_work explicitly allows freeing work in its func */ + scoutfs_tseq_del(&ninf->msg_tseq_tree, &mrecv->tseq_entry); kfree(mrecv); /* shut down the connection if processing returns fatal errors */ @@ -671,6 +680,7 @@ static void scoutfs_net_recv_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, recv_work); struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct socket *sock = conn->sock; struct scoutfs_net_header nh; struct message_recv *mrecv; @@ -736,8 +746,11 @@ static void scoutfs_net_recv_worker(struct work_struct *work) } spin_unlock(&conn->lock); - if (mrecv) + if (mrecv) { + scoutfs_tseq_add(&ninf->msg_tseq_tree, + &mrecv->tseq_entry); queue_work(conn->workq, &mrecv->proc_work); + } } if (ret) @@ -774,6 +787,13 @@ static int sendmsg_full(struct socket *sock, void *buf, unsigned len) return 0; } +static void free_msend(struct net_info *ninf, struct message_send *msend) +{ + list_del_init(&msend->head); + scoutfs_tseq_del(&ninf->msg_tseq_tree, &msend->tseq_entry); + kfree(msend); +} + /* * Each connection has a single worker that sends queued messages down * the connection's socket. The work is queued whenever a message is @@ -789,6 +809,7 @@ static void scoutfs_net_send_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, send_work); struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct message_send *msend; int ret = 0; int len; @@ -801,8 +822,7 @@ static void scoutfs_net_send_worker(struct work_struct *work) struct message_send, head))) { if (msend->dead) { - list_del_init(&msend->head); - kfree(msend); + free_msend(ninf, msend); continue; } @@ -853,8 +873,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) /* free all messages, refactor and complete for forced unmount? */ list_splice_init(&conn->resend_queue, &conn->send_queue); list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { - list_del_init(&msend->head); - kfree(msend); + free_msend(ninf, msend); } /* accepted sockets are removed from their listener's list */ @@ -1473,6 +1492,41 @@ static void net_tseq_show_conn(struct seq_file *m, conn->next_send_id, conn->last_proc_id); } +/* + * How's this for sneaky?! We line up the structs so that the entries + * and function pointers are at the same offsets. recv's function + * pointer value is known and can't be found in send's. + */ +static bool tseq_entry_is_recv(struct scoutfs_tseq_entry *ent) +{ + struct message_recv *mrecv = + container_of(ent, struct message_recv, tseq_entry); + + BUILD_BUG_ON(offsetof(struct message_recv, tseq_entry) != + offsetof(struct message_send, tseq_entry)); + BUILD_BUG_ON(offsetof(struct message_recv, proc_work.func) != + offsetof(struct message_send, resp_func)); + + return mrecv->proc_work.func == scoutfs_net_proc_worker; +} + +static void net_tseq_show_msg(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct message_send *msend; + struct message_recv *mrecv; + + if (tseq_entry_is_recv(ent)) { + mrecv = container_of(ent, struct message_recv, tseq_entry); + + seq_printf(m, "recv "SNH_FMT"\n", SNH_ARG(&mrecv->nh)); + } else { + msend = container_of(ent, struct message_send, tseq_entry); + + seq_printf(m, "send "SNH_FMT"\n", SNH_ARG(&msend->nh)); + } +} + int scoutfs_net_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1492,6 +1546,7 @@ int scoutfs_net_setup(struct super_block *sb) sbi->net_info = ninf; scoutfs_tseq_tree_init(&ninf->conn_tseq_tree, net_tseq_show_conn); + scoutfs_tseq_tree_init(&ninf->msg_tseq_tree, net_tseq_show_msg); ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", WQ_UNBOUND, 0); @@ -1508,6 +1563,14 @@ int scoutfs_net_setup(struct super_block *sb) goto out; } + ninf->msg_tseq_dentry = scoutfs_tseq_create("messages", + sbi->debug_root, + &ninf->msg_tseq_tree); + if (!ninf->msg_tseq_dentry) { + ret = -ENOMEM; + goto out; + } + ret = 0; out: if (ret) @@ -1524,6 +1587,7 @@ void scoutfs_net_destroy(struct super_block *sb) if (ninf->shutdown_workq) destroy_workqueue(ninf->shutdown_workq); debugfs_remove(ninf->conn_tseq_dentry); + debugfs_remove(ninf->msg_tseq_dentry); kfree(ninf); sbi->net_info = NULL; } From a72b7a900155345a321a61827732fd812c8b7bbe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Jul 2018 13:35:45 -0700 Subject: [PATCH 657/920] scoutfs: convert locks seq to trivial seq Signed-off-by: Zach Brown --- kmod/src/lock.c | 111 ++++++------------------------------------------ kmod/src/lock.h | 3 ++ 2 files changed, 16 insertions(+), 98 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 20ebc225..f565be1e 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -17,8 +17,6 @@ #include #include #include -#include -#include #include #include "super.h" @@ -32,6 +30,7 @@ #include "counters.h" #include "endian_swap.h" #include "triggers.h" +#include "tseq.h" /* * scoutfs manages internode item cache consistency using the kernel's @@ -78,9 +77,9 @@ struct lock_info { unsigned long long lru_nr; struct workqueue_struct *workq; dlm_lockspace_t *lockspace; - struct dentry *debug_locks_dentry; - struct idr debug_locks_idr; atomic64_t next_refresh_gen; + struct dentry *tseq_dentry; + struct scoutfs_tseq_tree tseq_tree; }; #define DECLARE_LOCK_INFO(sb, name) \ @@ -181,8 +180,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!linfo->shutdown && lock->granted_mode != DLM_LOCK_IV); BUG_ON(delayed_work_pending(&lock->grace_work)); - if (lock->debug_locks_id) - idr_remove(&linfo->debug_locks_idr, lock->debug_locks_id); + scoutfs_tseq_del(&linfo->tseq_tree, &lock->tseq_entry); if (!RB_EMPTY_NODE(&lock->node)) rb_erase(&lock->node, &linfo->lock_tree); if (!RB_EMPTY_NODE(&lock->range_node)) @@ -202,7 +200,6 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; - int id; if (WARN_ON_ONCE(!!start != !!end)) return NULL; @@ -213,18 +210,6 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, scoutfs_inc_counter(sb, lock_alloc); - idr_preload(GFP_NOFS); - spin_lock(&linfo->lock); - id = idr_alloc(&linfo->debug_locks_idr, lock, 1, INT_MAX, GFP_NOWAIT); - if (id > 0) - lock->debug_locks_id = id; - spin_unlock(&linfo->lock); - idr_preload_end(); - if (id <= 0) { - lock_free(linfo, lock); - return NULL; - } - RB_CLEAR_NODE(&lock->node); RB_CLEAR_NODE(&lock->range_node); INIT_LIST_HEAD(&lock->lru_head); @@ -247,6 +232,7 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, lock->work_prev_mode = DLM_LOCK_IV; lock->work_mode = DLM_LOCK_IV; + scoutfs_tseq_add(&linfo->tseq_tree, &lock->tseq_entry); trace_scoutfs_lock_alloc(sb, lock); return lock; @@ -1285,50 +1271,10 @@ void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr) linfo->shrinker.shrink(&linfo->shrinker, &sc); } -/* _stop is always called no matter what start returns */ -static void *scoutfs_debug_locks_seq_start(struct seq_file *m, loff_t *pos) - __acquires(linfo->lock) +static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) { - struct super_block *sb = m->private; - DECLARE_LOCK_INFO(sb, linfo); - int id; - - spin_lock(&linfo->lock); - - if (*pos >= INT_MAX) - return NULL; - - id = *pos; - return idr_get_next(&linfo->debug_locks_idr, &id); -} - -static void *scoutfs_debug_locks_seq_next(struct seq_file *m, void *v, - loff_t *pos) -{ - struct super_block *sb = m->private; - DECLARE_LOCK_INFO(sb, linfo); - struct scoutfs_lock *lock = v; - int id; - - id = lock->debug_locks_id + 1; - lock = idr_get_next(&linfo->debug_locks_idr, &id); - if (lock) - *pos = lock->debug_locks_id; - return lock; -} - -static void scoutfs_debug_locks_seq_stop(struct seq_file *m, void *v) - __releases(linfo->lock) -{ - struct super_block *sb = m->private; - DECLARE_LOCK_INFO(sb, linfo); - - spin_unlock(&linfo->lock); -} - -static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) -{ - struct scoutfs_lock *lock = v; + struct scoutfs_lock *lock = + container_of(ent, struct scoutfs_lock, tseq_entry); seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", LN_ARG(&lock->name), SK_ARG(&lock->start), @@ -1344,37 +1290,8 @@ static int scoutfs_debug_locks_seq_show(struct seq_file *m, void *v) lock->lksb.sb_status, lock->lksb.sb_lkid, lock->lksb.sb_flags); - - return 0; } -static const struct seq_operations scoutfs_debug_locks_seq_ops = { - .start = scoutfs_debug_locks_seq_start, - .next = scoutfs_debug_locks_seq_next, - .stop = scoutfs_debug_locks_seq_stop, - .show = scoutfs_debug_locks_seq_show, -}; - -static int scoutfs_debug_locks_open(struct inode *inode, struct file *file) -{ - struct seq_file *m; - int ret; - - ret = seq_open(file, &scoutfs_debug_locks_seq_ops); - if (ret == 0) { - m = file->private_data; - m->private = inode->i_private; - } - return ret; -} - -static const struct file_operations scoutfs_debug_locks_fops = { - .open = scoutfs_debug_locks_open, - .release = seq_release, - .read = seq_read, - .llseek = seq_lseek, -}; - /* * We're going to be destroying the locks soon. We shouldn't have any * normal task holders that would have prevented unmount. We can have @@ -1472,7 +1389,7 @@ void scoutfs_lock_destroy(struct super_block *sb) } /* XXX does anything synchronize with open debugfs fds? */ - debugfs_remove(linfo->debug_locks_dentry); + debugfs_remove(linfo->tseq_dentry); /* free our stale locks that now describe released dlm locks */ spin_lock(&linfo->lock); @@ -1484,7 +1401,6 @@ void scoutfs_lock_destroy(struct super_block *sb) } spin_unlock(&linfo->lock); - idr_destroy(&linfo->debug_locks_idr); kfree(linfo); sbi->lock_info = NULL; } @@ -1515,16 +1431,15 @@ int scoutfs_lock_setup(struct super_block *sb) linfo->shrinker.seeks = DEFAULT_SEEKS; register_shrinker(&linfo->shrinker); INIT_LIST_HEAD(&linfo->lru_list); - idr_init(&linfo->debug_locks_idr); atomic64_set(&linfo->next_refresh_gen, 0); + scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show); sbi->lock_info = linfo; trace_scoutfs_lock_setup(sb, linfo); - linfo->debug_locks_dentry = debugfs_create_file("locks", - S_IFREG|S_IRUSR, sbi->debug_root, sb, - &scoutfs_debug_locks_fops); - if (!linfo->debug_locks_dentry) { + linfo->tseq_dentry = scoutfs_tseq_create("locks", sbi->debug_root, + &linfo->tseq_tree); + if (!linfo->tseq_dentry) { ret = -ENOMEM; goto out; } diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 1898a6a3..99c00bf9 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -3,6 +3,7 @@ #include #include "key.h" +#include "tseq.h" #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ #define SCOUTFS_LKF_NONBLOCK 0x02 /* only use already held locks */ @@ -40,6 +41,8 @@ struct scoutfs_lock { int work_mode; unsigned int waiters[SCOUTFS_LOCK_NR_MODES]; unsigned int users[SCOUTFS_LOCK_NR_MODES]; + + struct scoutfs_tseq_entry tseq_entry; }; struct scoutfs_lock_coverage { From a25b6324d2265e54ae1120444b75c1afe30f57f2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 13 Aug 2018 14:21:49 -0700 Subject: [PATCH 658/920] scoutfs: maintain free_blocks in one place The free_blocks counter in the super is meant to track the number of total blocks in the primary free extent index. Callers of extent manipulation were trying to keep it in sync with the extents. Segment allocation was allocating extents manually using a cursor. It forgot to update free_blocks. Segment freeing then freed the segment as an extent which did update free_blocks. This created ever accumulating free blocks over time which eventually pushed it greater than total blocks and caused df to report negative usage. This updates the free_blocks count in server extent io which is the only place we update the extent items themselves. This ensures that we'll keep the count in sync with the extent items. Callers don't have to worry about it. Signed-off-by: Zach Brown T# with '#' will be ignored, and an empty message aborts the commit. --- kmod/src/server.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 1947a8bf..2ee1be3e 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -120,6 +120,10 @@ static int init_extent_from_btree_key(struct scoutfs_extent *ext, u8 type, * This is called by the extent core on behalf of the server who holds * the appropriate locks to protect the many btree items that can be * accessed on behalf of one extent operation. + * + * The free_blocks count in the super tracks the number of blocks in + * the primary extent index. We update it here instead of expecting + * callers to remember. */ static int server_extent_io(struct super_block *sb, int op, struct scoutfs_extent *ext, void *data) @@ -192,6 +196,13 @@ static int server_extent_io(struct super_block *sb, int op, } } + if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + if (op == SEI_INSERT) + le64_add_cpu(&super->free_blocks, ext->len); + else if (op == SEI_DELETE) + le64_add_cpu(&super->free_blocks, -ext->len); + } + return ret; } @@ -209,7 +220,6 @@ static int alloc_extent(struct super_block *sb, u64 blocks, u64 *start, u64 *len) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_extent ext; int ret; @@ -244,7 +254,6 @@ static int alloc_extent(struct super_block *sb, u64 blocks, goto out; trace_scoutfs_server_alloc_extent_allocated(sb, &ext); - le64_add_cpu(&super->free_blocks, -ext.len); *start = ext.start; *len = ext.len; @@ -278,7 +287,6 @@ struct pending_free_extent { */ static int apply_pending_frees(struct super_block *sb) { - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct server_info *server = SCOUTFS_SB(sb)->server_info; struct pending_free_extent *pfe; struct pending_free_extent *tmp; @@ -298,7 +306,6 @@ static int apply_pending_frees(struct super_block *sb) break; } - le64_add_cpu(&super->free_blocks, pfe->len); list_del_init(&pfe->head); kfree(pfe); } From ed9f4b6a226a4633e00a1191515b6750ff855c5b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 Aug 2018 13:50:32 -0700 Subject: [PATCH 659/920] scoutfs: calculate and enforce segment csum We had fields in the segment header for the crc but weren't using it. This calculates the crc on write and verifies it on read. The crc covers the used bytes in the segment as indicated by the total_bytes field. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/format.h | 3 +++ kmod/src/seg.c | 62 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 94d175c2..9adc1ea2 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -113,6 +113,7 @@ EXPAND_COUNTER(net_unknown_message) \ EXPAND_COUNTER(net_unknown_request) \ EXPAND_COUNTER(seg_alloc) \ + EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ EXPAND_COUNTER(seg_shrink) \ EXPAND_COUNTER(seg_stale_read) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 6f2d46f8..5a01f57b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -271,6 +271,9 @@ struct scoutfs_segment_item { /* * Each large segment starts with a segment block that describes the * rest of the blocks that make up the segment. + * + * The crc covers the initial total_bytes of the segment but starts + * after the padding. */ struct scoutfs_segment_block { __le32 crc; diff --git a/kmod/src/seg.c b/kmod/src/seg.c index a69a226d..3b4b33a0 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -55,6 +56,9 @@ struct segment_cache { enum { SF_END_IO = 0, + SF_CALC_CRC_STARTED, + SF_CALC_CRC_DONE, + SF_INVALID_CRC, }; static void *off_ptr(struct scoutfs_segment *seg, u32 off) @@ -161,6 +165,26 @@ static void lru_check(struct segment_cache *cac, struct scoutfs_segment *seg) } } +static __le32 calc_seg_crc(struct scoutfs_segment *seg) +{ + u32 total = scoutfs_seg_total_bytes(seg); + u32 crc = ~0; + u32 off; + u32 len; + + off = offsetof(struct scoutfs_segment_block, _padding) + + FIELD_SIZEOF(struct scoutfs_segment_block, _padding); + + while (off < total) { + len = min(total - off, + SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK)); + crc = crc32c(crc, off_ptr(seg, off), len); + off += len; + } + + return cpu_to_le32(crc); +} + /* * This always inserts the segment into the rbtree. If there's already * a segment at the given seg then it is removed and returned. The @@ -346,12 +370,20 @@ struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, return seg; } +/* + * The caller has ensured that the segment won't be modified while + * it is in flight. + */ int scoutfs_seg_submit_write(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_bio_completion *comp) { + struct scoutfs_segment_block *sblk = off_ptr(seg, 0); + trace_scoutfs_seg_submit_write(sb, seg->segno); + sblk->crc = calc_seg_crc(seg); + scoutfs_bio_submit_comp(sb, WRITE, seg->pages, segno_to_blkno(seg->segno), SCOUTFS_SEGMENT_BLOCKS, comp); @@ -360,8 +392,7 @@ int scoutfs_seg_submit_write(struct super_block *sb, } /* - * Wait for IO on the segment to complete. In the cached read fast path - * the bit is already set by the reads that populated the cache. + * Wait for IO on the segment to complete. * * The caller provides the segno and seq from their segment reference to * validate that we found the version of the segment that they were @@ -370,8 +401,9 @@ int scoutfs_seg_submit_write(struct super_block *sb, * its operation. (Typically by getting a new manifest btree root and * searching for keys in the manifest.) * - * XXX drop stale segments from the cache - * XXX none of the callers perform that retry today. + * An invalid crc can be racing to read a stale segment while it's being + * written. The caller will retry and consider it corrupt if it keeps + * getting stale reads. */ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, u64 segno, u64 seq) @@ -393,10 +425,28 @@ int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, goto out; } + /* calc crc in waiting task instead of end_io */ + if (!test_bit(SF_CALC_CRC_DONE, &seg->flags) && + !test_and_set_bit(SF_CALC_CRC_STARTED, &seg->flags)) { + if (sblk->crc != calc_seg_crc(seg)) { + scoutfs_inc_counter(sb, seg_csum_error); + set_bit(SF_INVALID_CRC, &seg->flags); + } + set_bit(SF_CALC_CRC_DONE, &seg->flags); + wake_up(&cac->waitq); + } + + /* very rarely race waiting for calc to finish */ + ret = wait_event_interruptible(cac->waitq, + test_bit(SF_CALC_CRC_DONE, &seg->flags)); + if (ret) + goto out; + sblk = off_ptr(seg, 0); - if (WARN_ON_ONCE(segno != le64_to_cpu(sblk->segno)) || - WARN_ON_ONCE(seq != le64_to_cpu(sblk->seq)) || + if (test_bit(SF_INVALID_CRC, &seg->flags) || + segno != le64_to_cpu(sblk->segno) || + seq != le64_to_cpu(sblk->seq) || scoutfs_trigger(sb, SEG_STALE_READ)) { spin_lock_irqsave(&cac->lock, flags); erased = erase_seg(cac, seg); From f06b39cd7e4f380dc503fa44d9bff561a8cba295 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 23 Aug 2018 16:23:57 -0700 Subject: [PATCH 660/920] scoutfs: destroy items after locks We were destroying the item subsystem before shutting down locking. This is wrong because locking shutdown invalidates items covered by the locks. It can walk into freed memory and crash or corrupt other memory. The fix is to tear down the item subsystem after tearing down locks. Signed-off-by: Zach Brown --- kmod/src/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index d9540334..28773a3e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -129,7 +129,6 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); scoutfs_inode_destroy(sb); - scoutfs_item_destroy(sb); /* the server locks the listen address and compacts */ scoutfs_lock_shutdown(sb); @@ -138,6 +137,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); + scoutfs_item_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); debugfs_remove(sbi->debug_root); From 8b3193ea724883f5ffd4cc970ab24cf1b97ec690 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Jul 2018 15:24:45 -0700 Subject: [PATCH 661/920] scoutfs: server allocates node_id Today node_ids are randomly assigned. This adds the risk of failure from random number generation and still allows for the risk of collisions. Switch to assigning strictly advancing node_ids on the server during the initial connection greeting message exchange. This simplifies the system and allows us to derive information from the relative values of node_ids in the system. To do this we refactor the greeting code from internal to the net layer to proper client and server request and response processing. This lets the server manage persistent node_id storage and allows the client to wait for a node_id during mount. Now that net_connect is sync in the client we don't need the notify_up callback anymore. The client can perform those duties when the connect returns. The net code still has to snoop on request and response processing to see when the greetings have been exchange and allow messages to flow. Signed-off-by: Zach Brown --- kmod/src/client.c | 116 ++++++++++++++++--- kmod/src/client.h | 1 + kmod/src/format.h | 2 + kmod/src/net.c | 275 +++++++++++++++++++--------------------------- kmod/src/net.h | 11 +- kmod/src/server.c | 71 ++++++++++++ kmod/src/super.c | 7 +- 7 files changed, 295 insertions(+), 188 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index a21cbefd..81fd7fb7 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -51,6 +51,7 @@ struct client_info { struct super_block *sb; struct scoutfs_net_connection *conn; + struct completion node_id_comp; atomic_t shutting_down; struct workqueue_struct *workq; @@ -230,13 +231,78 @@ int scoutfs_client_statfs(struct super_block *sb, sizeof(struct scoutfs_net_statfs)); } +/* + * Process a greeting response in the client from the server. This is + * called for every connected socket on the connection. The first + * response will have the node_id that the server assigned the client. + */ +static int client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, int error, + void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting *gr = resp; + int ret = 0; + + if (error) { + ret = error; + goto out; + } + + if (resp_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "server sent fsid 0x%llx, client has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "server sent format 0x%llx, client has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) { + scoutfs_warn(sb, "server sent node_id %llu, client has %llu", + le64_to_cpu(gr->node_id), + sbi->node_id); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0 && gr->node_id == 0) { + scoutfs_warn(sb, "server sent node_id 0, client also has 0\n"); + ret = -EINVAL; + goto out; + } + + if (sbi->node_id == 0) { + sbi->node_id = le64_to_cpu(gr->node_id); + complete(&client->node_id_comp); + } + +out: + return ret; +} + /* * Attempt to connect to the listening address that the server wrote in * the super block. We keep trying indefinitely with an increasing * delay if we fail to either read the address or connect to it. * * We're careful to only ever have one connection attempt in flight. We - * only queue this work on mount, on error, or from the connection + * only queue this work on mount, on error, or from the notify_down * callback. */ static void scoutfs_client_connect_worker(struct work_struct *work) @@ -244,6 +310,8 @@ static void scoutfs_client_connect_worker(struct work_struct *work) struct client_info *client = container_of(work, struct client_info, connect_dwork.work); struct super_block *sb = client->sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_net_greeting greet; struct scoutfs_super_block super; struct sockaddr_in sin; int ret; @@ -262,8 +330,24 @@ static void scoutfs_client_connect_worker(struct work_struct *work) sin.sin_addr.s_addr = le32_to_be32(super.server_addr.addr); sin.sin_port = le16_to_be16(super.server_addr.port); - scoutfs_net_connect(sb, client->conn, &sin, client->conn_retry_ms); - ret = 0; + ret = scoutfs_net_connect(sb, client->conn, &sin, + client->conn_retry_ms); + if (ret) + goto out; + + reset_connect_timeout(client); + + /* send a greeting to verify endpoints of each connection */ + greet.fsid = super.id; + greet.format_hash = super.format_hash; + greet.node_id = cpu_to_le64(sbi->node_id); + + ret = scoutfs_net_submit_greeting_request(sb, client->conn, + &greet, sizeof(greet), + client_greeting, NULL); + if (ret) + scoutfs_net_shutdown(sb, client->conn); + out: if (ret && !atomic_read(&client->shutting_down)) { queue_delayed_work(client->workq, &client->connect_dwork, @@ -272,14 +356,6 @@ out: } } -static void client_notify_up(struct super_block *sb, - struct scoutfs_net_connection *conn) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - reset_connect_timeout(client); -} - /* * Called when either a connect attempt or established connection times * out and fails. @@ -296,6 +372,18 @@ static void client_notify_down(struct super_block *sb, } } +/* + * Wait for the first connected socket on the connection that assigns + * the node_id that will be used for the rest of the life time of the + * mount. + */ +int scoutfs_client_wait_node_id(struct super_block *sb) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return wait_for_completion_interruptible(&client->node_id_comp); +} + int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -310,14 +398,14 @@ int scoutfs_client_setup(struct super_block *sb) sbi->client_info = client; client->sb = sb; + init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); /* client doesn't process any incoming requests yet */ - client->conn = scoutfs_net_alloc_conn(sb, client_notify_up, - client_notify_down, NULL, - "client"); + client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, + NULL, "client"); if (!client->conn) { ret = -ENOMEM; goto out; diff --git a/kmod/src/client.h b/kmod/src/client.h index f0a8d609..410592eb 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -18,6 +18,7 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs); +int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 5a01f57b..95f732c9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -368,6 +368,7 @@ struct scoutfs_super_block { __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; + __le64 next_node_id; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; @@ -514,6 +515,7 @@ struct scoutfs_lock_name { struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; + __le64 node_id; } __packed; /* diff --git a/kmod/src/net.c b/kmod/src/net.c index 3ebc062a..775a25d7 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -50,7 +50,6 @@ * deliver messages across renumbering. * * XXX: - * - assign node_ids and validate with the greeting * - defer accepted conn destruction until reconnect timeout * - trace command and response data payloads * - checksum message contents? @@ -77,6 +76,7 @@ struct scoutfs_net_connection { scoutfs_net_request_t *req_funcs; spinlock_t lock; + wait_queue_head_t waitq; unsigned long valid_greeting:1, /* other commands can proceed */ established:1, /* added sends queue send work */ @@ -92,7 +92,6 @@ struct scoutfs_net_connection { struct list_head accepted_head; struct scoutfs_net_connection *listening_conn; struct list_head accepted_list; - wait_queue_head_t accepted_waitq; u64 next_send_id; u64 last_proc_id; @@ -371,124 +370,28 @@ static int submit_send(struct super_block *sb, } /* - * Messages can flow once we receive a valid greeting from our peer. - * Response callers are already called under the lock, request callers - * need to acquire it. + * Messages can flow once we receive and process a valid greeting from + * our peer. * - * At this point greeting request processing has queued the greeting - * response message on the send queue. All the sends waiting to be - * resent need to be added to the end of the send queue after the - * greeting response. Greeting acks are sent differently and can be - * received after resend messages. + * At this point recv processing has queued the greeting response or ack + * message on the send queue. All the sends waiting to be resent need + * to be added to the end of the send queue after the greeting message. */ static void saw_valid_greeting(struct scoutfs_net_connection *conn) { struct super_block *sb = conn->sb; - assert_spin_locked(&conn->lock); + spin_lock(&conn->lock); conn->valid_greeting = 1; if (conn->notify_up) conn->notify_up(sb, conn); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); + + spin_unlock(&conn->lock); } -static int greeting_response(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *resp, unsigned int resp_len, int error, - void *data) -{ - struct scoutfs_net_greeting *gr = resp; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (error) { - ret = error; - goto out; - } - - if (resp_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "server "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "server "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - saw_valid_greeting(conn); - -out: - return ret; -} - -/* - * Process an incoming greeting request. We try to send responses to - * failed greetings so that the sender can log some detail before - * shutting down. A failure to send a greeting response shuts down the - * connection. - */ -static int greeting_request(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - struct scoutfs_net_greeting *gr = arg; - struct scoutfs_net_greeting greet; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - int ret = 0; - - if (arg_len != sizeof(struct scoutfs_net_greeting)) { - ret = -EINVAL; - goto out; - } - - if (gr->fsid != super->id) { - scoutfs_warn(sb, "client "SIN_FMT" has fsid 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); - ret = -EINVAL; - goto out; - } - - if (gr->format_hash != super->format_hash) { - scoutfs_warn(sb, "client "SIN_FMT" has format hash 0x%llx, expected 0x%llx", - SIN_ARG(&conn->peername), - le64_to_cpu(gr->format_hash), - le64_to_cpu(super->format_hash)); - ret = -EINVAL; - goto out; - } - - greet.fsid = super->id; - greet.format_hash = super->format_hash; -out: - ret = scoutfs_net_response(sb, conn, cmd, id, ret, - &greet, sizeof(greet)); - if (ret == 0) { - spin_lock(&conn->lock); - saw_valid_greeting(conn); - spin_unlock(&conn->lock); - } - return ret; -} - - /* * Process an incoming response. The greeting should ensure that the * sender won't send us unknown commands. We return an error if we see @@ -500,20 +403,27 @@ static int process_request(struct scoutfs_net_connection *conn, struct message_recv *mrecv) { struct super_block *sb = conn->sb; - scoutfs_net_request_t req_func = NULL; + scoutfs_net_request_t req_func; + int ret; - if (conn->listening_conn != NULL && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING) { - req_func = greeting_request; - } else if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) { + if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) req_func = conn->req_funcs[mrecv->nh.cmd]; - } if (req_func == NULL) { + else + req_func = NULL; + + if (req_func == NULL) { scoutfs_inc_counter(sb, net_unknown_request); return -EINVAL; } - return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), - mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + ret = req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) + saw_valid_greeting(conn); + + return ret; } /* @@ -547,6 +457,11 @@ static int process_response(struct scoutfs_net_connection *conn, ret = submit_send(sb, conn, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, NULL, 0, NULL, NULL, NULL); + + if (!conn->valid_greeting && + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) + saw_valid_greeting(conn); + return ret; } @@ -883,7 +798,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) spin_lock(&listener->lock); list_del_init(&conn->accepted_head); if (list_empty(&listener->accepted_list)) - wake_up(&listener->accepted_waitq); + wake_up(&listener->waitq); spin_unlock(&listener->lock); } @@ -1038,8 +953,7 @@ static void scoutfs_net_connect_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, connect_work); struct super_block *sb = conn->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_net_greeting greet; + struct message_send *msend; struct socket *sock; struct timeval tv; int ret; @@ -1074,26 +988,29 @@ static void scoutfs_net_connect_worker(struct work_struct *work) if (ret) goto out; - /* greeting is about to queue send work */ - spin_lock(&conn->lock); - conn->established = 1; - spin_unlock(&conn->lock); - - queue_work(conn->workq, &conn->recv_work); - - /* queue a new updated greeting send */ - greet.fsid = super->id; - greet.format_hash = super->format_hash; - - ret = submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, 0, - &greet, sizeof(greet), greeting_response, NULL, NULL); - if (ret) - goto out; - scoutfs_info(sb, "client connected "SIN_FMT" -> "SIN_FMT, SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername)); + + spin_lock(&conn->lock); + + /* clear greeting state for next negotiation */ + conn->valid_greeting = 0; + msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_RESPONSE, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: + find_send(conn, SCOUTFS_NET_MSG_ACK, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); + if (msend) + complete_send(conn, msend, NULL, 0, 0); + + conn->established = 1; + wake_up(&conn->waitq); + + spin_unlock(&conn->lock); + + queue_work(conn->workq, &conn->recv_work); out: if (ret) shutdown_conn(conn); @@ -1135,7 +1052,6 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); struct super_block *sb = conn->sb; struct scoutfs_net_connection *acc_conn; - struct message_send *msend; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); @@ -1160,6 +1076,8 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) conn->sock = NULL; } + memset(&conn->peername, 0, sizeof(conn->peername)); + /* listening connections shut down all the connections they accepted */ spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { @@ -1168,28 +1086,16 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) spin_unlock(&acc_conn->lock); } spin_unlock(&conn->lock); - wait_event(conn->accepted_waitq, empty_accepted_list(conn)); + wait_event(conn->waitq, empty_accepted_list(conn)); spin_lock(&conn->lock); - /* all queued sends will be resent, protocol handles dupes */ list_splice_tail_init(&conn->send_queue, &conn->resend_queue); - - /* clear greeting state for next negotiation */ - conn->valid_greeting = 0; - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_RESPONSE, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_ACK, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); - if (msend) - complete_send(conn, msend, NULL, 0, 0); - + /* signal connect failure */ + memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); + wake_up(&conn->waitq); spin_unlock(&conn->lock); - memset(&conn->peername, 0, sizeof(conn->peername)); - /* tell the caller that the connection is down */ if (conn->notify_down) conn->notify_down(sb, conn); @@ -1215,11 +1121,6 @@ scoutfs_net_alloc_conn(struct super_block *sb, struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; - /* we handle greetings, the caller shouldn't attempt to */ - if (WARN_ON_ONCE(req_funcs != NULL && - req_funcs[SCOUTFS_NET_CMD_GREETING] != NULL)) - return NULL; - conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); if (!conn) return NULL; @@ -1237,11 +1138,11 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->notify_down = notify_down; conn->req_funcs = req_funcs; spin_lock_init(&conn->lock); + init_waitqueue_head(&conn->waitq); conn->sockname.sin_family = AF_INET; conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); - init_waitqueue_head(&conn->accepted_waitq); conn->next_send_id = SCOUTFS_NET_ID_GREETING + 1; INIT_LIST_HEAD(&conn->send_queue); INIT_LIST_HEAD(&conn->resend_queue); @@ -1345,22 +1246,52 @@ void scoutfs_net_listen(struct super_block *sb, } /* - * Start connecting to the given address. notify_up may be called if - * the connection completes. notify_down will be called when either the - * connection disconnects or times out. Both could be called before - * this function returns. The caller must be careful not to call - * connect again until notify_down has been called. + * Return once a connection attempt has completed either successfully + * or in error. */ -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms) +static bool connect_result(struct scoutfs_net_connection *conn, int *error) { + bool done = false; + + spin_lock(&conn->lock); + if (conn->established) { + done = true; + *error = 0; + } else if (conn->shutting_down || conn->connect_sin.sin_family == 0) { + done = true; + *error = -ESHUTDOWN; + } + spin_unlock(&conn->lock); + + return done; +} + +/* + * Connect to the given address. An error is returned if the socket was + * not connected before the given timeout. The connection isn't fully + * active until the connecting caller starts greeting negotiation by + * sending the initial greeting request. + * + * The conn notify_down callback can be called as the connection is + * shutdown before this returns. + */ +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms) +{ + int error = 0; + int ret; + spin_lock(&conn->lock); conn->connect_sin = *sin; conn->connect_timeout_ms = timeout_ms; spin_unlock(&conn->lock); queue_work(conn->workq, &conn->connect_work); + + ret = wait_event_interruptible(conn->waitq, + connect_result(conn, &error)); + return ret ?: error; } /* @@ -1379,6 +1310,20 @@ int scoutfs_net_submit_request(struct super_block *sb, arg, arg_len, resp_func, resp_data, id_ret); } +/* + * Greeting requests are special because they have a known id. + */ +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data) +{ + return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, + SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, + 0, arg, arg_len, resp_func, resp_data, NULL); +} + /* * Send a response. Responses don't get callbacks and use the request's * id so caller's don't need to get an id in return. diff --git a/kmod/src/net.h b/kmod/src/net.h index 4e270760..f81d4bf7 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -27,9 +27,9 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, scoutfs_net_request_t *req_funcs, char *name_suffix); -void scoutfs_net_connect(struct super_block *sb, - struct scoutfs_net_connection *conn, - struct sockaddr_in *sin, unsigned long timeout_ms); +int scoutfs_net_connect(struct super_block *sb, + struct scoutfs_net_connection *conn, + struct sockaddr_in *sin, unsigned long timeout_ms); int scoutfs_net_bind(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin); @@ -40,6 +40,11 @@ int scoutfs_net_submit_request(struct super_block *sb, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); +int scoutfs_net_submit_greeting_request(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data); void scoutfs_net_cancel_request(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id); diff --git a/kmod/src/server.c b/kmod/src/server.c index 2ee1be3e..2b9cd0a5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -47,6 +47,7 @@ struct server_info { struct super_block *sb; + spinlock_t lock; struct workqueue_struct *wq; struct delayed_work dwork; @@ -958,6 +959,74 @@ static int server_statfs(struct super_block *sb, &nstatfs, sizeof(nstatfs)); } +/* + * Process an incoming greeting request in the server from the client. + * We try to send responses to failed greetings so that the sender can + * log some detail before shutting down. A failure to send a greeting + * response shuts down the connection. + * + * We allocate a new node_id for the first connect attempt from a + * client. If they reconnect they'll send their initially assigned node_id + * in their greeting request. + */ +static int server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_greeting *gr = arg; + struct scoutfs_net_greeting greet; + DECLARE_SERVER_INFO(sb, server); + struct commit_waiter cw; + __le64 node_id; + int ret = 0; + + if (arg_len != sizeof(struct scoutfs_net_greeting)) { + ret = -EINVAL; + goto out; + } + + if (gr->fsid != super->id) { + scoutfs_warn(sb, "client sent fsid 0x%llx, server has 0x%llx", + le64_to_cpu(gr->fsid), + le64_to_cpu(super->id)); + ret = -EINVAL; + goto out; + } + + if (gr->format_hash != super->format_hash) { + scoutfs_warn(sb, "client sent format 0x%llx, server has 0x%llx", + le64_to_cpu(gr->format_hash), + le64_to_cpu(super->format_hash)); + ret = -EINVAL; + goto out; + } + + if (gr->node_id == 0) { + down_read(&server->commit_rwsem); + + spin_lock(&server->lock); + node_id = super->next_node_id; + le64_add_cpu(&super->next_node_id, 1); + spin_unlock(&server->lock); + + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); + if (ret) + goto out; + } else { + node_id = gr->node_id; + } + + greet.fsid = super->id; + greet.format_hash = super->format_hash; + greet.node_id = node_id; +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &greet, sizeof(greet)); +} + /* * Eventually we're going to have messages that control compaction. * Each client mount would have long-lived work that sends requests @@ -1065,6 +1134,7 @@ static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) } static scoutfs_net_request_t server_req_funcs[] = { + [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, @@ -1212,6 +1282,7 @@ int scoutfs_server_setup(struct super_block *sb) return -ENOMEM; server->sb = sb; + spin_lock_init(&server->lock); init_completion(&server->shutdown_comp); server->bind_warned = false; INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); diff --git a/kmod/src/super.c b/kmod/src/super.c index 28773a3e..15387737 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -304,12 +304,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; - /* - * XXX this is random today for initial testing, but we'll want - * it to be assigned by the server. - */ - get_random_bytes_arch(&sbi->node_id, sizeof(sbi->node_id)); - spin_lock_init(&sbi->next_ino_lock); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->trans_write_lock); @@ -338,6 +332,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_net_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: + scoutfs_client_wait_node_id(sb) ?: scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, &sbi->node_id_lock); if (ret) From 746293987c3699ab597a51a9cfc1446f505cf109 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Jul 2018 15:57:15 -0700 Subject: [PATCH 662/920] scoutfs: let server send msg to specific node_id The current sending interfaces only send a message to the peer of a given connection. For the server to send to a specific connected client it'd have to track connections itself and send to them. This adds a sending interface that uses the node_id to send to a specific connected client. The conn argument is the listening socket and its accepted sockets are searched for the destination node_id. Signed-off-by: Zach Brown --- kmod/src/net.c | 86 +++++++++++++++++++++++++++++++++++++---------- kmod/src/net.h | 6 ++++ kmod/src/server.c | 13 +++++-- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 775a25d7..42efe341 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -86,6 +86,7 @@ struct scoutfs_net_connection { unsigned long connect_timeout_ms; struct socket *sock; + u64 node_id; /* assigned during greeting */ struct sockaddr_in sockname; struct sockaddr_in peername; @@ -109,6 +110,12 @@ struct scoutfs_net_connection { struct scoutfs_tseq_entry tseq_entry; }; +/* listening and their accepting sockets have a fixed locking order */ +enum { + CONN_LOCK_LISTENER, + CONN_LOCK_ACCEPTED, +}; + /* * Messages to be sent are allocated and put on the send queue. * @@ -308,15 +315,20 @@ static void shutdown_conn(struct scoutfs_net_connection *conn) * shutting down. We only directly queue the send work if the * connection has passed the greeting and isn't being shut down. At all * other times we add new sends to the resend queue. + * + * If a non-zero node_id is specified then the conn argument is a listening + * connection and the connection to send the message down is found by + * searching for the node_id in its accepted connections. */ static int submit_send(struct super_block *sb, - struct scoutfs_net_connection *conn, + struct scoutfs_net_connection *conn, u64 node_id, u8 msg, u8 cmd, u64 id, u8 net_err, void *data, u16 data_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *acc_conn; struct message_send *msend; if (WARN_ON_ONCE(msg >= SCOUTFS_NET_MSG_UNKNOWN) || @@ -335,7 +347,25 @@ static int submit_send(struct super_block *sb, if (!msend) return -ENOMEM; - spin_lock(&conn->lock); + spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); + + if (node_id != 0) { + list_for_each_entry(acc_conn, &conn->accepted_list, + accepted_head) { + if (acc_conn->node_id == node_id) { + spin_lock_nested(&acc_conn->lock, + CONN_LOCK_ACCEPTED); + spin_unlock(&conn->lock); + conn = acc_conn; + node_id = 0; + break; + } + } + if (node_id != 0) { + spin_unlock(&conn->lock); + return -ENOTCONN; + } + } msend->resp_func = resp_func; msend->resp_data = resp_data; @@ -376,14 +406,17 @@ static int submit_send(struct super_block *sb, * At this point recv processing has queued the greeting response or ack * message on the send queue. All the sends waiting to be resent need * to be added to the end of the send queue after the greeting message. + * + * Update the conn's node_id so that servers can send to specific clients. */ -static void saw_valid_greeting(struct scoutfs_net_connection *conn) +static void saw_valid_greeting(struct scoutfs_net_connection *conn, u64 node_id) { struct super_block *sb = conn->sb; spin_lock(&conn->lock); conn->valid_greeting = 1; + conn->node_id = node_id; if (conn->notify_up) conn->notify_up(sb, conn); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); @@ -404,6 +437,7 @@ static int process_request(struct scoutfs_net_connection *conn, { struct super_block *sb = conn->sb; scoutfs_net_request_t req_func; + struct scoutfs_net_greeting *gr; int ret; if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) @@ -419,9 +453,16 @@ static int process_request(struct scoutfs_net_connection *conn, ret = req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); + /* + * Greeting response updates our *request* node_id so that + * we can consume a new allocation without callbacks. We're + * about to free the recv in the caller anyway. + */ if (!conn->valid_greeting && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) - saw_valid_greeting(conn); + mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) { + gr = (void *)mrecv->nh.data; + saw_valid_greeting(conn, le64_to_cpu(gr->node_id)); + } return ret; } @@ -454,13 +495,13 @@ static int process_response(struct scoutfs_net_connection *conn, spin_unlock(&conn->lock); if (ret == 0) - ret = submit_send(sb, conn, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, - le64_to_cpu(mrecv->nh.id), 0, NULL, 0, NULL, - NULL, NULL); + ret = submit_send(sb, conn, 0, SCOUTFS_NET_MSG_ACK, + mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, + NULL, 0, NULL, NULL, NULL); if (!conn->valid_greeting && mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) - saw_valid_greeting(conn); + saw_valid_greeting(conn, 0); return ret; } @@ -1029,12 +1070,6 @@ static bool empty_accepted_list(struct scoutfs_net_connection *conn) return empty; } -/* listening and their accepting sockets have a fixed locking order */ -enum { - CONN_LOCK_LISTENER, - CONN_LOCK_ACCEPTED, -}; - /* * Safely shut down an active connection. This can be triggered by * errors in workers or by an external call to free the connection. The @@ -1306,10 +1341,25 @@ int scoutfs_net_submit_request(struct super_block *sb, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { - return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, cmd, 0, 0, + return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_REQUEST, cmd, 0, 0, arg, arg_len, resp_func, resp_data, id_ret); } +/* + * Send a request to a specific node_id that was accepted by this listening + * connection. + */ +int scoutfs_net_submit_request_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u8 cmd, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret) +{ + return submit_send(sb, conn, node_id, SCOUTFS_NET_MSG_REQUEST, cmd, 0, + 0, arg, arg_len, resp_func, resp_data, id_ret); +} + /* * Greeting requests are special because they have a known id. */ @@ -1319,7 +1369,7 @@ int scoutfs_net_submit_greeting_request(struct super_block *sb, scoutfs_net_response_t resp_func, void *resp_data) { - return submit_send(sb, conn, SCOUTFS_NET_MSG_REQUEST, + return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_REQUEST, SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, 0, arg, arg_len, resp_func, resp_data, NULL); } @@ -1342,7 +1392,7 @@ int scoutfs_net_response(struct super_block *sb, resp_len = 0; } - return submit_send(sb, conn, SCOUTFS_NET_MSG_RESPONSE, + return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_RESPONSE, cmd, id, net_err_from_host(sb, error), resp, resp_len, NULL, NULL, NULL); } diff --git a/kmod/src/net.h b/kmod/src/net.h index f81d4bf7..7b7e0375 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -40,6 +40,12 @@ int scoutfs_net_submit_request(struct super_block *sb, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); +int scoutfs_net_submit_request_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u8 cmd, + void *arg, u16 arg_len, + scoutfs_net_response_t resp_func, + void *resp_data, u64 *id_ret); int scoutfs_net_submit_greeting_request(struct super_block *sb, struct scoutfs_net_connection *conn, void *arg, u16 arg_len, diff --git a/kmod/src/server.c b/kmod/src/server.c index 2b9cd0a5..f1d4962b 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -966,7 +966,10 @@ static int server_statfs(struct super_block *sb, * response shuts down the connection. * * We allocate a new node_id for the first connect attempt from a - * client. If they reconnect they'll send their initially assigned node_id + * client. We update the request node_id for the calling net layer to + * consume. + * + * If a client reconnects they'll send their initially assigned node_id * in their greeting request. */ static int server_greeting(struct super_block *sb, @@ -978,7 +981,7 @@ static int server_greeting(struct super_block *sb, struct scoutfs_net_greeting greet; DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; - __le64 node_id; + __le64 node_id = 0; int ret = 0; if (arg_len != sizeof(struct scoutfs_net_greeting)) { @@ -1023,8 +1026,12 @@ static int server_greeting(struct super_block *sb, greet.format_hash = super->format_hash; greet.node_id = node_id; out: - return scoutfs_net_response(sb, conn, cmd, id, ret, + ret = scoutfs_net_response(sb, conn, cmd, id, ret, &greet, sizeof(greet)); + /* give net caller client's new node_id :/ */ + if (ret == 0 && node_id != 0) + gr->node_id = node_id; + return ret; } /* From 0adbd7e439610e7a50ffbf039a04deb811971a91 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Jul 2018 16:43:06 -0700 Subject: [PATCH 663/920] scoutfs: have server track connected clients This extends the notify up and down calls to let the server keep track of connected clients. It adds the notion of per-connection info that is allocated for each connection. It's passed to the notification callbacks so that callers can have per-client storage without having to manage allocations in the callbacks. It adds the node_id argument to the notification callbacks to indicate if the call is for the listening socket itself or an accepted client connection on that listening socket. Signed-off-by: Zach Brown --- kmod/src/client.c | 5 +++-- kmod/src/net.c | 29 ++++++++++++++++++++++++----- kmod/src/net.h | 5 +++-- kmod/src/server.c | 44 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 81fd7fb7..abb3844f 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -361,7 +361,8 @@ out: * out and fails. */ static void client_notify_down(struct super_block *sb, - struct scoutfs_net_connection *conn) + struct scoutfs_net_connection *conn, void *info, + u64 node_id) { struct client_info *client = SCOUTFS_SB(sb)->client_info; @@ -404,7 +405,7 @@ int scoutfs_client_setup(struct super_block *sb) scoutfs_client_connect_worker); /* client doesn't process any incoming requests yet */ - client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, + client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, NULL, "client"); if (!client->conn) { ret = -ENOMEM; diff --git a/kmod/src/net.c b/kmod/src/net.c index 42efe341..c26c428a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -73,6 +73,7 @@ struct scoutfs_net_connection { struct super_block *sb; scoutfs_net_notify_t notify_up; scoutfs_net_notify_t notify_down; + size_t info_size; scoutfs_net_request_t *req_funcs; spinlock_t lock; @@ -108,6 +109,8 @@ struct scoutfs_net_connection { /* message_recv proc_work also executes in the conn workq */ struct scoutfs_tseq_entry tseq_entry; + + u8 info[0] __aligned(sizeof(u64)); }; /* listening and their accepting sockets have a fixed locking order */ @@ -418,7 +421,7 @@ static void saw_valid_greeting(struct scoutfs_net_connection *conn, u64 node_id) conn->valid_greeting = 1; conn->node_id = node_id; if (conn->notify_up) - conn->notify_up(sb, conn); + conn->notify_up(sb, conn, conn->info, node_id); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); @@ -947,7 +950,9 @@ static void scoutfs_net_listen_worker(struct work_struct *work) break; /* inherit accepted request funcs from listening conn */ - acc_conn = scoutfs_net_alloc_conn(sb, NULL, NULL, + acc_conn = scoutfs_net_alloc_conn(sb, conn->notify_up, + conn->notify_down, + conn->info_size, conn->req_funcs, "accepted"); if (!acc_conn) { sock_release(acc_sock); @@ -1133,7 +1138,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) /* tell the caller that the connection is down */ if (conn->notify_down) - conn->notify_down(sb, conn); + conn->notify_down(sb, conn, conn->info, conn->node_id); /* accepted conns are destroyed */ if (conn->listening_conn) { @@ -1147,16 +1152,29 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) trace_scoutfs_net_shutdown_work_exit(sb, 0, 0); } +/* + * Accepted connections inherit the callbacks from their listening + * connection. + * + * notify_up is called once a valid greeting is received. node_id is + * non-zero on accepted sockets once they've seen a valid greeting. + * Connected and listening connections have a node_id of 0. + * + * notify_down is always called as connections are shut down. It can be + * called without notify_up ever being called. The node_id is only + * non-zero for accepted connections. + */ struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, - scoutfs_net_notify_t notify_down, + scoutfs_net_notify_t notify_down, size_t info_size, scoutfs_net_request_t *req_funcs, char *name_suffix) { struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; - conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); + conn = kzalloc(offsetof(struct scoutfs_net_connection, + info[info_size]), GFP_NOFS); if (!conn) return NULL; @@ -1171,6 +1189,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->sb = sb; conn->notify_up = notify_up; conn->notify_down = notify_down; + conn->info_size = info_size; conn->req_funcs = req_funcs; spin_lock_init(&conn->lock); init_waitqueue_head(&conn->waitq); diff --git a/kmod/src/net.h b/kmod/src/net.h index 7b7e0375..88fe3a1a 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -20,12 +20,13 @@ typedef int (*scoutfs_net_response_t)(struct super_block *sb, int error, void *data); typedef void (*scoutfs_net_notify_t)(struct super_block *sb, - struct scoutfs_net_connection *conn); + struct scoutfs_net_connection *conn, + void *info, u64 node_id); struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, - scoutfs_net_notify_t notify_down, + scoutfs_net_notify_t notify_down, size_t info_size, scoutfs_net_request_t *req_funcs, char *name_suffix); int scoutfs_net_connect(struct super_block *sb, struct scoutfs_net_connection *conn, diff --git a/kmod/src/server.c b/kmod/src/server.c index f1d4962b..8f8eda8b 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -73,11 +73,21 @@ struct server_info { /* server tracks pending frees to be applied during commit */ struct rw_semaphore alloc_rwsem; struct list_head pending_frees; + + struct list_head clients; }; #define DECLARE_SERVER_INFO(sb, name) \ struct server_info *name = SCOUTFS_SB(sb)->server_info +/* + * The server tracks each connected client. + */ +struct server_client_info { + u64 node_id; + struct list_head head; +}; + struct commit_waiter { struct completion comp; struct llist_node node; @@ -1153,13 +1163,37 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_STATFS] = server_statfs, }; -static void server_notify_down(struct super_block *sb, - struct scoutfs_net_connection *conn) +static void server_notify_up(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 node_id) { + struct server_client_info *sci = info; DECLARE_SERVER_INFO(sb, server); - shutdown_server(server); + if (node_id != 0) { + sci->node_id = node_id; + spin_lock(&server->lock); + list_add_tail(&sci->head, &server->clients); + spin_unlock(&server->lock); + } } + +static void server_notify_down(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 node_id) +{ + struct server_client_info *sci = info; + DECLARE_SERVER_INFO(sb, server); + + if (node_id != 0) { + spin_lock(&server->lock); + list_del(&sci->head); + spin_unlock(&server->lock); + } else { + shutdown_server(server); + } +} + /* * This work is always running or has a delayed timer set while a super * is mounted. It tries to grab the lock to become the server. If it @@ -1193,7 +1227,8 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret) goto out; - conn = scoutfs_net_alloc_conn(sb, NULL, server_notify_down, + conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, + sizeof(struct server_client_info), server_req_funcs, "server"); if (!conn) { ret = -ENOMEM; @@ -1302,6 +1337,7 @@ int scoutfs_server_setup(struct super_block *sb) INIT_LIST_HEAD(&server->pending_seqs); init_rwsem(&server->alloc_rwsem); INIT_LIST_HEAD(&server->pending_frees); + INIT_LIST_HEAD(&server->clients); server->wq = alloc_workqueue("scoutfs_server", WQ_NON_REENTRANT, 0); if (!server->wq) { From 1ed0c6017f66a71ad3c6351696437a17bcedf8a0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 26 Jul 2018 14:24:18 -0700 Subject: [PATCH 664/920] scoutfs: remove unused keys manifest field Keys used to be variable length so the manifest struct on the wire ended in key payloads. The keys are now fixed size so that field is no longer necessary or used. It's an artifact that should have been removed when the keys were made fixed length. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 95f732c9..e06102e9 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -609,7 +609,6 @@ struct scoutfs_net_manifest_entry { struct scoutfs_key first; struct scoutfs_key last; __u8 level; - __u8 keys[0]; } __packed; struct scoutfs_net_statfs { From 00adbd31bee80ecd7605932d5500e2b5c431ff49 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 7 Aug 2018 12:53:40 -0700 Subject: [PATCH 665/920] scoutfs: add sparse bitmap library Add a quick library for maintaining a very large bitmap with sparse allocation. Signed-off-by: Zach Brown --- kmod/src/Makefile | 2 +- kmod/src/spbm.c | 153 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/spbm.h | 15 +++++ 3 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 kmod/src/spbm.c create mode 100644 kmod/src/spbm.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 0cf70986..f804c121 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -8,7 +8,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ export.o extents.o file.o inode.o ioctl.o item.o lock.o \ manifest.o msg.o net.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sort_priv.o super.o sysfs.o trans.o \ + scoutfs_trace.o sort_priv.o spbm.o super.o sysfs.o trans.o \ triggers.o tseq.o xattr.o # diff --git a/kmod/src/spbm.c b/kmod/src/spbm.c new file mode 100644 index 00000000..c26dcc2c --- /dev/null +++ b/kmod/src/spbm.c @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "spbm.h" + +#define SPBM_BITS 128 +#define SPBM_SHIFT ilog2(SPBM_BITS) +#define SPBM_MASK ((u64)SPBM_BITS - 1) +#define SPBM_LONGS (SPBM_BITS / BITS_PER_LONG) + +/* + * Maintain a sparse bitmap in an rbtree. Setting bits can allocate and + * fail but clearing will always succeed. Locking is left up to the + * caller. + */ + +struct spbm_node { + struct rb_node node; + u64 index; + unsigned long bits[SPBM_LONGS]; +}; + +void scoutfs_spbm_init(struct scoutfs_spbm *spbm) +{ + BUILD_BUG_ON(!is_power_of_2(SPBM_BITS)); + + spbm->root = RB_ROOT; +} + +enum { + /* if a node isn't found then return an allocated new node */ + SPBM_FIND_ALLOC = 0x1, +}; +static struct spbm_node *find_node(struct scoutfs_spbm *spbm, u64 index, + int flags) +{ + struct rb_node *parent; + struct rb_node **node; + struct spbm_node *sn; + + node = &spbm->root.rb_node; + parent = NULL; + sn = NULL; + while (*node) { + parent = *node; + sn = container_of(*node, struct spbm_node, node); + + if (index < sn->index) { + node = &(*node)->rb_left; + } else if (index > sn->index) { + node = &(*node)->rb_right; + } else { + break; + } + + sn = NULL; + } + + if (!sn && (flags & SPBM_FIND_ALLOC)) { + sn = kzalloc(sizeof(struct spbm_node), GFP_NOFS); + if (sn) { + sn->index = index; + rb_link_node(&sn->node, parent, node); + rb_insert_color(&sn->node, &spbm->root); + } + } + + return sn; +} + +static void calc_index_nr(u64 *index, int *nr, u64 bit) +{ + *index = bit >> SPBM_SHIFT; + *nr = bit & SPBM_MASK; +} + +int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, SPBM_FIND_ALLOC); + if (!sn) + return -ENOMEM; + + set_bit(nr, sn->bits); + + return 0; +} + +int scoutfs_spbm_test(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, 0); + if (sn) + return !!test_bit(nr, sn->bits); + + return 0; +} + +static void free_node(struct scoutfs_spbm *spbm, struct spbm_node *sn) +{ + rb_erase(&sn->node, &spbm->root); + kfree(sn); +} + +void scoutfs_spbm_clear(struct scoutfs_spbm *spbm, u64 bit) +{ + struct spbm_node *sn; + u64 index; + int nr; + + calc_index_nr(&index, &nr, bit); + + sn = find_node(spbm, index, 0); + if (sn) { + clear_bit(nr, sn->bits); + if (bitmap_empty(sn->bits, SPBM_BITS)) + free_node(spbm, sn); + } +} + +void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm) +{ + struct spbm_node *sn; + struct spbm_node *pos; + + rbtree_postorder_for_each_entry_safe(sn, pos, &spbm->root, node) + free_node(spbm, sn); +} diff --git a/kmod/src/spbm.h b/kmod/src/spbm.h new file mode 100644 index 00000000..a1a2ec5d --- /dev/null +++ b/kmod/src/spbm.h @@ -0,0 +1,15 @@ +#ifndef _SCOUTFS_SPBM_H_ +#define _SCOUTFS_SPBM_H_ + +struct scoutfs_spbm { + struct rb_root root; +}; + +void scoutfs_spbm_init(struct scoutfs_spbm *spbm); +void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm); + +int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit); +int scoutfs_spbm_test(struct scoutfs_spbm *spbm, u64 bit); +void scoutfs_spbm_clear(struct scoutfs_spbm *spbm, u64 bit); + +#endif From 30d5471e4ac4275565f2f2661b68e0e541f75e6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Aug 2018 11:33:54 -0700 Subject: [PATCH 666/920] scoutfs: call net response func outside lock Today response processing calls a requests's response callback from inside the net spinlock. This happened to work for the synchronous blocking request handler who only had to record the result and wake their waiter. It doesn't work for server compact response processing which needs to use IO to commit the result of the compaction. This lifts the call to the response function out of complete_send() and into the response processing work function. Other complete_send() callers now won't trigger the response function call and can't see errors, which they all ignored anyway. Signed-off-by: Zach Brown --- kmod/src/net.c | 52 ++++++++++++++++++++++++-------------------------- kmod/src/net.h | 2 +- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index c26c428a..c871d468 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -197,36 +197,22 @@ static struct message_send *find_send(struct scoutfs_net_connection *conn, /* * Complete a send message by moving it to the send queue and marking it - * to be freed. - * - * Request messages have their response function called. Their response - * processing can return an error if the response is invalid. The - * request message is still removed and freed in that case. + * to be freed. It won't be visible to callers trying to find sends. */ -static int complete_send(struct scoutfs_net_connection *conn, - struct message_send *msend, - void *resp, unsigned int resp_len, int error) +static void complete_send(struct scoutfs_net_connection *conn, + struct message_send *msend) { - struct super_block *sb = conn->sb; - int ret = 0; + assert_spin_locked(&conn->lock); if (WARN_ON_ONCE(msend->dead) || WARN_ON_ONCE(list_empty(&msend->head))) - return -EINVAL; + return; - assert_spin_locked(&conn->lock); - - if (msend->resp_func) - ret = msend->resp_func(sb, conn, resp, resp_len, error, - msend->resp_data); msend->dead = 1; list_move(&msend->head, &conn->send_queue); queue_work(conn->workq, &conn->send_work); - - return ret; } - /* * Translate a positive error on the wire to a negative host errno. */ @@ -482,21 +468,29 @@ static int process_response(struct scoutfs_net_connection *conn, { struct super_block *sb = conn->sb; struct message_send *msend; + scoutfs_net_response_t resp_func = NULL; + void *resp_data; int ret = 0; spin_lock(&conn->lock); msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id)); - if (msend) - ret = complete_send(conn, msend, mrecv->nh.data, - le16_to_cpu(mrecv->nh.data_len), - net_err_to_host(mrecv->nh.error)); - else + if (msend) { + resp_func = msend->resp_func; + resp_data = msend->resp_data; + complete_send(conn, msend); + } else { scoutfs_inc_counter(sb, net_dropped_response); + } spin_unlock(&conn->lock); + if (resp_func) + ret = resp_func(sb, conn, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), + net_err_to_host(mrecv->nh.error), resp_data); + if (ret == 0) ret = submit_send(sb, conn, 0, SCOUTFS_NET_MSG_ACK, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, @@ -523,7 +517,7 @@ static void process_ack(struct scoutfs_net_connection *conn, msend = find_send(conn, SCOUTFS_NET_MSG_RESPONSE, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id)); if (msend) - complete_send(conn, msend, NULL, 0, 0); + complete_send(conn, msend); else scoutfs_inc_counter(sb, net_dropped_ack); @@ -1049,7 +1043,7 @@ static void scoutfs_net_connect_worker(struct work_struct *work) find_send(conn, SCOUTFS_NET_MSG_ACK, SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); if (msend) - complete_send(conn, msend, NULL, 0, 0); + complete_send(conn, msend); conn->established = 1; wake_up(&conn->waitq); @@ -1416,6 +1410,10 @@ int scoutfs_net_response(struct super_block *sb, resp, resp_len, NULL, NULL, NULL); } +/* + * The response function that was submitted with the request is not + * called if the request is canceled here. + */ void scoutfs_net_cancel_request(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id) @@ -1425,7 +1423,7 @@ void scoutfs_net_cancel_request(struct super_block *sb, spin_lock(&conn->lock); msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, cmd, id); if (msend) - complete_send(conn, msend, NULL, 0, -ECANCELED); + complete_send(conn, msend); spin_unlock(&conn->lock); } diff --git a/kmod/src/net.h b/kmod/src/net.h index 88fe3a1a..a2c9279c 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -13,7 +13,7 @@ typedef int (*scoutfs_net_request_t)(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len); -/* These are called with a spinlock held, funcs must be fast and nonblocking */ +/* These are called in their own blocking context */ typedef int (*scoutfs_net_response_t)(struct super_block *sb, struct scoutfs_net_connection *conn, void *resp, unsigned int resp_len, From 62d6c11e3c39fa9e986c8c4c8335df5eb5428b71 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 13 Aug 2018 13:31:28 -0700 Subject: [PATCH 667/920] scoutfs: clean up workqueue flags We had gotten a bit sloppy with the workqueue flags. We needed _UNBOUND in some workqueues where we wanted concurrency by scheduling across cpus instead of waiting for the current (very long running) work on a cpu to finish. We add NON_REENTRANT out of an abundance of caution. It has gone away in modern kernels and is probably not needed here, but according to the docs we would want it so we at least document that fact by using it. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- kmod/src/lock.c | 3 ++- kmod/src/net.c | 3 ++- kmod/src/server.c | 3 ++- kmod/src/trans.c | 3 ++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 5485af0b..46b18588 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1371,7 +1371,7 @@ int scoutfs_data_setup(struct super_block *sb) INIT_WORK(&datinf->return_work, scoutfs_data_return_server_extents_worker); - datinf->workq = alloc_workqueue("scoutfs_data", 0, 1); + datinf->workq = alloc_workqueue("scoutfs_data", WQ_UNBOUND, 1); if (!datinf->workq) { kfree(datinf); return -ENOMEM; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index f565be1e..e8a6a425 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1445,7 +1445,8 @@ int scoutfs_lock_setup(struct super_block *sb) } linfo->workq = alloc_workqueue("scoutfs_lock_work", - WQ_UNBOUND|WQ_HIGHPRI, 0); + WQ_NON_REENTRANT | WQ_UNBOUND | + WQ_HIGHPRI, 0); if (!linfo->workq) { ret = -ENOMEM; goto out; diff --git a/kmod/src/net.c b/kmod/src/net.c index c871d468..27e567fb 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1561,7 +1561,8 @@ int scoutfs_net_setup(struct super_block *sb) scoutfs_tseq_tree_init(&ninf->msg_tseq_tree, net_tseq_show_msg); ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", - WQ_UNBOUND, 0); + WQ_UNBOUND | WQ_NON_REENTRANT, + 0); if (!ninf->shutdown_workq) { ret = -ENOMEM; goto out; diff --git a/kmod/src/server.c b/kmod/src/server.c index 8f8eda8b..a91cf005 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1339,7 +1339,8 @@ int scoutfs_server_setup(struct super_block *sb) INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); - server->wq = alloc_workqueue("scoutfs_server", WQ_NON_REENTRANT, 0); + server->wq = alloc_workqueue("scoutfs_server", + WQ_UNBOUND | WQ_NON_REENTRANT, 0); if (!server->wq) { kfree(server); return -ENOMEM; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 9e45b7e2..710fa6cd 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -480,7 +480,8 @@ int scoutfs_setup_trans(struct super_block *sb) spin_lock_init(&tri->lock); - sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", 0, 1); + sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", + WQ_UNBOUND, 1); if (!sbi->trans_write_workq) { kfree(tri); return -ENOMEM; From 07eec357ee1bd38fa7ef6c0dfca863510bda62b9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Aug 2018 16:26:55 -0700 Subject: [PATCH 668/920] scoutfs: simplify reliable request delivery It was a bit of an overreach to try and limit duplicate request processing in the network layer. It introduced acks and the necessity to resync last_processed_id on reconnect. In testing compaction requests we saw that request processing stopped if a client reconnected to a new server. The new server sent low request ids which the client dropped because they were lower than the ids it got from the last server. To fix this we'd need to add smarts to reset ids when connecting to new servers but not existing servers. In thinking about this, though, there's a bigger problem. Duplicate request processing protection only works up in memory in the networking connections. If the server makes persistent changes, then crashes, the client will resend the request to the new server. It will need to discover that the persistent changes have already been made. So while we protected duplicate network request processing between nodes that reconnected, we didn't protect duplicate persistent side-effects of request processing when reconnecting to a new server. Once you see that the request implementations have to take this into account then duplicate request delivery becomes a simpler instance of this same case and will be taken care of already. There's no need to implement the complexity of protecting duplicate delivery between running nodes. This removes the last_processed_id on the server. It removes resending of responses and acks. Now that ids can be processed out of order we remove the special known ID of greeting commands. They can be processed as usual. When there's only request and response packets we can differentiate them with a flag instead of a u8 message type. Signed-off-by: Zach Brown --- kmod/src/client.c | 7 +- kmod/src/counters.h | 3 - kmod/src/format.h | 26 ++--- kmod/src/net.c | 254 +++++++++++++++++--------------------------- kmod/src/net.h | 5 - kmod/src/server.h | 14 +-- 6 files changed, 113 insertions(+), 196 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index abb3844f..39d98afb 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -342,9 +342,10 @@ static void scoutfs_client_connect_worker(struct work_struct *work) greet.format_hash = super.format_hash; greet.node_id = cpu_to_le64(sbi->node_id); - ret = scoutfs_net_submit_greeting_request(sb, client->conn, - &greet, sizeof(greet), - client_greeting, NULL); + ret = scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_GREETING, + &greet, sizeof(greet), + client_greeting, NULL, NULL); if (ret) scoutfs_net_shutdown(sb, client->conn); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 9adc1ea2..3d5886c0 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -100,9 +100,7 @@ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(manifest_read_excluded_key) \ - EXPAND_COUNTER(net_dropped_ack) \ EXPAND_COUNTER(net_dropped_response) \ - EXPAND_COUNTER(net_dropped_request) \ EXPAND_COUNTER(net_send_bytes) \ EXPAND_COUNTER(net_send_error) \ EXPAND_COUNTER(net_send_messages) \ @@ -110,7 +108,6 @@ EXPAND_COUNTER(net_recv_error) \ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ - EXPAND_COUNTER(net_unknown_message) \ EXPAND_COUNTER(net_unknown_request) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index e06102e9..4d91c250 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -520,36 +520,22 @@ struct scoutfs_net_greeting { /* * This header precedes and describes all network messages sent over - * sockets. The id is set by the request and sent in the reply. The - * type is strictly redundant in the reply because the id will find the - * send but we include it in both packets to make it easier to observe - * replies without having the id from their previous request. + * sockets. The id is set by the request and sent in the response. * - * Error is only set to a translated errno on response messages and - * data_len will be 0. + * Error is only set to a translated errno and will only be found in + * response messages. */ struct scoutfs_net_header { __le64 id; __le16 data_len; - __u8 msg; __u8 cmd; + __u8 flags; __u8 error; __u8 data[0]; } __packed; -/* - * Greetings are the first messages sent down every newly established - * socket on the connection. Every other message gets a unique - * increasing id over the life time of the connection. - */ -#define SCOUTFS_NET_ID_GREETING 1 - -enum { - SCOUTFS_NET_MSG_REQUEST = 0, - SCOUTFS_NET_MSG_RESPONSE, - SCOUTFS_NET_MSG_ACK, - SCOUTFS_NET_MSG_UNKNOWN, -}; +#define SCOUTFS_NET_FLAG_RESPONSE (1 << 0) +#define SCOUTFS_NET_FLAGS_UNKNOWN (U8_MAX << 1) enum { SCOUTFS_NET_CMD_GREETING = 0, diff --git a/kmod/src/net.c b/kmod/src/net.c index 27e567fb..e889574a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -35,19 +35,35 @@ #include "tseq.h" /* - * scoutfs networking reliably delivers requests and responses between - * nodes. + * scoutfs networking delivers requests and responses between nodes. * * Nodes decide to be either a connecting client or a listening server. * Both set up a connection and specify the set of request commands they * can process. * - * The networking core maintains reliable request processing as the - * nodes reconnect. Requests are resent as connections are - * re-established until a response is received. Responses are resent - * until an ack is received. The connections are not bound to the - * addresses of the underlying socket transports and can reliably - * deliver messages across renumbering. + * Requests are tracked on a connection and sent to its peer. They're + * resent down newly established sockets on a long lived connection. + * Queued requests are removed as a response is processed or if the + * request is canceled by the sender. + * + * Request processing sends a response down the socket that received a + * connection. Processing is stopped as a socket is shutdown so + * responses are only send down sockets that received a request. + * + * Thus requests can be received multiple times as sockets are shutdown + * and reconnected. Responses are only processed once for a given + * request. It is up to request and response implementations to ensure + * that duplicate requests are safely handled. + * + * It turns out that we have to deal with duplicate request processing + * at the layer above networking anyway. Request processing can make + * persistent changes that are committed on the server before it + * crashes. The client then reconnects to before it crashes and the + * client reconnects to a server who must detect that the persistent + * work on behalf of the resent request has already been committed. If + * we have to deal with that duplicate processing we may as well + * simplify networking by allowing it between reconnecting peers as + * well. * * XXX: * - defer accepted conn destruction until reconnect timeout @@ -96,7 +112,6 @@ struct scoutfs_net_connection { struct list_head accepted_list; u64 next_send_id; - u64 last_proc_id; struct list_head send_queue; struct list_head resend_queue; @@ -122,9 +137,8 @@ enum { /* * Messages to be sent are allocated and put on the send queue. * - * Request and response messages are put on the resend queue until their - * response or ack messages are received, respectively, and they can be - * freed. + * Request messages are put on the resend queue until their response + * messages is received and they can be freed. * * The send worker is the only context that references messages while * not holding the lock. It does this while blocking sending the @@ -162,16 +176,26 @@ static int nh_bytes(unsigned int data_len) return offsetof(struct scoutfs_net_header, data[data_len]); } +static bool nh_is_response(struct scoutfs_net_header *nh) +{ + return !!(nh->flags & SCOUTFS_NET_FLAG_RESPONSE); +} + +static bool nh_is_request(struct scoutfs_net_header *nh) +{ + return !nh_is_response(nh); +} + static struct message_send *search_list(struct scoutfs_net_connection *conn, struct list_head *list, - u8 msg, u8 cmd, u64 id) + u8 cmd, u64 id) { struct message_send *msend; assert_spin_locked(&conn->lock); list_for_each_entry(msend, list, head) { - if (msend->nh.msg == msg && msend->nh.cmd == cmd && + if (nh_is_request(&msend->nh) && msend->nh.cmd == cmd && le64_to_cpu(msend->nh.id) == id) return msend; } @@ -180,16 +204,16 @@ static struct message_send *search_list(struct scoutfs_net_connection *conn, } /* - * Find an active send on the lists. It's almost certainly waiting on - * the resend queue but it could be actively being sent. + * Find an active send request on the lists. It's almost certainly + * waiting on the resend queue but it could be actively being sent. */ -static struct message_send *find_send(struct scoutfs_net_connection *conn, - u8 msg, u8 cmd, u64 id) +static struct message_send *find_request(struct scoutfs_net_connection *conn, + u8 cmd, u64 id) { struct message_send *msend; - msend = search_list(conn, &conn->resend_queue, msg, cmd, id) ?: - search_list(conn, &conn->send_queue, msg, cmd, id); + msend = search_list(conn, &conn->resend_queue, cmd, id) ?: + search_list(conn, &conn->send_queue, cmd, id); if (msend && msend->dead) msend = NULL; return msend; @@ -311,7 +335,7 @@ static void shutdown_conn(struct scoutfs_net_connection *conn) */ static int submit_send(struct super_block *sb, struct scoutfs_net_connection *conn, u64 node_id, - u8 msg, u8 cmd, u64 id, u8 net_err, + u8 cmd, u8 flags, u64 id, u8 net_err, void *data, u16 data_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) @@ -320,15 +344,13 @@ static int submit_send(struct super_block *sb, struct scoutfs_net_connection *acc_conn; struct message_send *msend; - if (WARN_ON_ONCE(msg >= SCOUTFS_NET_MSG_UNKNOWN) || - WARN_ON_ONCE(cmd >= SCOUTFS_NET_CMD_UNKNOWN) || + if (WARN_ON_ONCE(cmd >= SCOUTFS_NET_CMD_UNKNOWN) || + WARN_ON_ONCE(flags & SCOUTFS_NET_FLAGS_UNKNOWN) || WARN_ON_ONCE(net_err >= SCOUTFS_NET_ERR_UNKNOWN) || WARN_ON_ONCE(data_len > SCOUTFS_NET_MAX_DATA_LEN) || WARN_ON_ONCE(data_len && (!data || net_err)) || - WARN_ON_ONCE(net_err && (msg != SCOUTFS_NET_MSG_RESPONSE)) || - WARN_ON_ONCE(id == 0 && msg != SCOUTFS_NET_MSG_REQUEST) || - WARN_ON_ONCE((cmd == SCOUTFS_NET_CMD_GREETING) != - (id == SCOUTFS_NET_ID_GREETING))) + WARN_ON_ONCE(net_err && (!(flags & SCOUTFS_NET_FLAG_RESPONSE))) || + WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) return -EINVAL; msend = kmalloc(offsetof(struct message_send, @@ -363,8 +385,8 @@ static int submit_send(struct super_block *sb, if (id == 0) id = conn->next_send_id++; msend->nh.id = cpu_to_le64(id); - msend->nh.msg = msg; msend->nh.cmd = cmd; + msend->nh.flags = flags; msend->nh.error = net_err; msend->nh.data_len = cpu_to_le16(data_len); if (data_len) @@ -392,11 +414,13 @@ static int submit_send(struct super_block *sb, * Messages can flow once we receive and process a valid greeting from * our peer. * - * At this point recv processing has queued the greeting response or ack - * message on the send queue. All the sends waiting to be resent need - * to be added to the end of the send queue after the greeting message. + * At this point recv processing has queued the greeting response + * message on the send queue. Any request messages waiting to be resent + * need to be added to the end of the send queue after the greeting + * response. * - * Update the conn's node_id so that servers can send to specific clients. + * Update the conn's node_id so that servers can send to specific + * clients. */ static void saw_valid_greeting(struct scoutfs_net_connection *conn, u64 node_id) { @@ -458,10 +482,10 @@ static int process_request(struct scoutfs_net_connection *conn, /* * An incoming response finds the queued request and calls its response - * function. We call the function and remove it from the lists before - * trying to send the ack so that we only call the response function - * once. Future duplicate responses will just resend the ack in - * response. + * function. The response function for a given request will only be + * called once. Requests can be canceled while a response is in flight. + * It's not an error to receive a response to a request that no longer + * exists. */ static int process_response(struct scoutfs_net_connection *conn, struct message_recv *mrecv) @@ -474,8 +498,7 @@ static int process_response(struct scoutfs_net_connection *conn, spin_lock(&conn->lock); - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, mrecv->nh.cmd, - le64_to_cpu(mrecv->nh.id)); + msend = find_request(conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id)); if (msend) { resp_func = msend->resp_func; resp_data = msend->resp_data; @@ -491,11 +514,6 @@ static int process_response(struct scoutfs_net_connection *conn, le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error), resp_data); - if (ret == 0) - ret = submit_send(sb, conn, 0, SCOUTFS_NET_MSG_ACK, - mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), 0, - NULL, 0, NULL, NULL, NULL); - if (!conn->valid_greeting && mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) saw_valid_greeting(conn, 0); @@ -503,27 +521,6 @@ static int process_response(struct scoutfs_net_connection *conn, return ret; } -/* - * An incoming ack frees the pending response. - */ -static void process_ack(struct scoutfs_net_connection *conn, - struct message_recv *mrecv) -{ - struct super_block *sb = conn->sb; - struct message_send *msend; - - spin_lock(&conn->lock); - - msend = find_send(conn, SCOUTFS_NET_MSG_RESPONSE, mrecv->nh.cmd, - le64_to_cpu(mrecv->nh.id)); - if (msend) - complete_send(conn, msend); - else - scoutfs_inc_counter(sb, net_dropped_ack); - - spin_unlock(&conn->lock); -} - /* * Process an incoming received message in its own concurrent blocking * work context. @@ -539,22 +536,10 @@ static void scoutfs_net_proc_worker(struct work_struct *work) trace_scoutfs_net_proc_work_enter(sb, 0, 0); - switch (mrecv->nh.msg) { - case SCOUTFS_NET_MSG_REQUEST: - ret = process_request(conn, mrecv); - break; - case SCOUTFS_NET_MSG_RESPONSE: - ret = process_response(conn, mrecv); - break; - case SCOUTFS_NET_MSG_ACK: - process_ack(conn, mrecv); - ret = 0; - break; - default: - scoutfs_inc_counter(sb, net_unknown_message); - ret = -ENOMSG; - break; - } + if (nh_is_request(&mrecv->nh)) + ret = process_request(conn, mrecv); + else + ret = process_response(conn, mrecv); /* process_one_work explicitly allows freeing work in its func */ scoutfs_tseq_del(&ninf->msg_tseq_tree, &mrecv->tseq_entry); @@ -598,14 +583,9 @@ static bool invalid_message(struct scoutfs_net_header *nh) if (nh->id == 0) return true; - /* greeting messages must have the greeting id */ - if ((nh->cmd == SCOUTFS_NET_CMD_GREETING) != - (le64_to_cpu(nh->id) == SCOUTFS_NET_ID_GREETING)) - return true; - /* greeting should negotiate understood protocol */ - if (nh->msg >= SCOUTFS_NET_MSG_UNKNOWN || - nh->cmd >= SCOUTFS_NET_CMD_UNKNOWN || + if (nh->cmd >= SCOUTFS_NET_CMD_UNKNOWN || + (nh->flags & SCOUTFS_NET_FLAGS_UNKNOWN) || nh->error >= SCOUTFS_NET_ERR_UNKNOWN) return true; @@ -618,8 +598,7 @@ static bool invalid_message(struct scoutfs_net_header *nh) return true; /* only responses can carry errors */ - if (nh->error != SCOUTFS_NET_ERR_NONE && - nh->msg != SCOUTFS_NET_MSG_RESPONSE) + if (nh_is_request(nh) && nh->error != SCOUTFS_NET_ERR_NONE) return true; return false; @@ -681,29 +660,8 @@ static void scoutfs_net_recv_worker(struct work_struct *work) break; } - /* - * Check and maintain the last processed id for - * non-greeting requests before introducing reordering - * by queueing concurrent work. - */ - spin_lock(&conn->lock); - if (mrecv->nh.msg == SCOUTFS_NET_MSG_REQUEST && - mrecv->nh.cmd != SCOUTFS_NET_CMD_GREETING) { - if (le64_to_cpu(mrecv->nh.id) <= conn->last_proc_id) { - scoutfs_inc_counter(sb, net_dropped_request); - kfree(mrecv); - mrecv = NULL; - } else { - conn->last_proc_id = le64_to_cpu(mrecv->nh.id); - } - } - spin_unlock(&conn->lock); - - if (mrecv) { - scoutfs_tseq_add(&ninf->msg_tseq_tree, - &mrecv->tseq_entry); - queue_work(conn->workq, &mrecv->proc_work); - } + scoutfs_tseq_add(&ninf->msg_tseq_tree, &mrecv->tseq_entry); + queue_work(conn->workq, &mrecv->proc_work); } if (ret) @@ -795,11 +753,11 @@ static void scoutfs_net_send_worker(struct work_struct *work) if (ret) break; - /* acks are always freed, others will be resent if not dead */ - if (msend->nh.msg == SCOUTFS_NET_MSG_ACK) - msend->dead = 1; - else if (!msend->dead) + /* active requests are resent, everything else is freed */ + if (nh_is_request(&msend->nh) && !msend->dead) list_move_tail(&msend->head, &conn->resend_queue); + else + msend->dead = 1; } spin_unlock(&conn->lock); @@ -993,7 +951,6 @@ static void scoutfs_net_connect_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, connect_work); struct super_block *sb = conn->sb; - struct message_send *msend; struct socket *sock; struct timeval tv; int ret; @@ -1036,15 +993,6 @@ static void scoutfs_net_connect_worker(struct work_struct *work) /* clear greeting state for next negotiation */ conn->valid_greeting = 0; - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_RESPONSE, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING) ?: - find_send(conn, SCOUTFS_NET_MSG_ACK, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING); - if (msend) - complete_send(conn, msend); - conn->established = 1; wake_up(&conn->waitq); @@ -1074,18 +1022,15 @@ static bool empty_accepted_list(struct scoutfs_net_connection *conn) * errors in workers or by an external call to free the connection. The * shutting down flag ensures that this only executes once for each live * socket. - * - * Our reliability guarantee requires request processing to make forward - * progress once we've received and recorded a request id. We wait for - * processing work that is in flight and its sends will be queued for - * resending because the connection is not established while it's - * shutting down. */ static void scoutfs_net_shutdown_worker(struct work_struct *work) { DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); struct super_block *sb = conn->sb; + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *acc_conn; + struct message_send *msend; + struct message_send *tmp; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); @@ -1123,8 +1068,15 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) wait_event(conn->waitq, empty_accepted_list(conn)); spin_lock(&conn->lock); - /* all queued sends will be resent, protocol handles dupes */ + + /* resend any pending requests, drop responses or greetings */ list_splice_tail_init(&conn->send_queue, &conn->resend_queue); + list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head) { + if (nh_is_response(&msend->nh) || + msend->nh.cmd == SCOUTFS_NET_CMD_GREETING) + free_msend(ninf, msend); + } + /* signal connect failure */ memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); wake_up(&conn->waitq); @@ -1191,7 +1143,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); - conn->next_send_id = SCOUTFS_NET_ID_GREETING + 1; + conn->next_send_id = 1; INIT_LIST_HEAD(&conn->send_queue); INIT_LIST_HEAD(&conn->resend_queue); INIT_WORK(&conn->listen_work, scoutfs_net_listen_worker); @@ -1354,8 +1306,8 @@ int scoutfs_net_submit_request(struct super_block *sb, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { - return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_REQUEST, cmd, 0, 0, - arg, arg_len, resp_func, resp_data, id_ret); + return submit_send(sb, conn, 0, cmd, 0, 0, 0, arg, arg_len, + resp_func, resp_data, id_ret); } /* @@ -1369,22 +1321,8 @@ int scoutfs_net_submit_request_node(struct super_block *sb, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { - return submit_send(sb, conn, node_id, SCOUTFS_NET_MSG_REQUEST, cmd, 0, - 0, arg, arg_len, resp_func, resp_data, id_ret); -} - -/* - * Greeting requests are special because they have a known id. - */ -int scoutfs_net_submit_greeting_request(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *arg, u16 arg_len, - scoutfs_net_response_t resp_func, - void *resp_data) -{ - return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_REQUEST, - SCOUTFS_NET_CMD_GREETING, SCOUTFS_NET_ID_GREETING, - 0, arg, arg_len, resp_func, resp_data, NULL); + return submit_send(sb, conn, node_id, cmd, 0, 0, 0, arg, arg_len, + resp_func, resp_data, id_ret); } /* @@ -1405,9 +1343,9 @@ int scoutfs_net_response(struct super_block *sb, resp_len = 0; } - return submit_send(sb, conn, 0, SCOUTFS_NET_MSG_RESPONSE, - cmd, id, net_err_from_host(sb, error), - resp, resp_len, NULL, NULL, NULL); + return submit_send(sb, conn, 0, cmd, SCOUTFS_NET_FLAG_RESPONSE, id, + net_err_from_host(sb, error), resp, resp_len, + NULL, NULL, NULL); } /* @@ -1421,7 +1359,7 @@ void scoutfs_net_cancel_request(struct super_block *sb, struct message_send *msend; spin_lock(&conn->lock); - msend = find_send(conn, SCOUTFS_NET_MSG_REQUEST, cmd, id); + msend = find_request(conn, cmd, id); if (msend) complete_send(conn, msend); spin_unlock(&conn->lock); @@ -1497,11 +1435,11 @@ static void net_tseq_show_conn(struct seq_file *m, struct scoutfs_net_connection *conn = container_of(ent, struct scoutfs_net_connection, tseq_entry); - seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" vg %u est %u sd %u cto_ms %lu nsi %llu lpi %llu\n", + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" vg %u est %u sd %u cto_ms %lu nsi %llu\n", SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), conn->valid_greeting, conn->established, conn->shutting_down, conn->connect_timeout_ms, - conn->next_send_id, conn->last_proc_id); + conn->next_send_id); } /* diff --git a/kmod/src/net.h b/kmod/src/net.h index a2c9279c..f318b2bf 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -47,11 +47,6 @@ int scoutfs_net_submit_request_node(struct super_block *sb, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); -int scoutfs_net_submit_greeting_request(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *arg, u16 arg_len, - scoutfs_net_response_t resp_func, - void *resp_data); void scoutfs_net_cancel_request(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id); diff --git a/kmod/src/server.h b/kmod/src/server.h index 54e61d04..185f6c0b 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -22,15 +22,15 @@ do { \ __entry->name##_addr & 255, \ __entry->name##_port -#define SNH_FMT "id %llu data_len %u msg %u cmd %u error %u" +#define SNH_FMT "id %llu data_len %u cmd %u flags 0x%x error %u" #define SNH_ARG(nh) le64_to_cpu((nh)->id), le16_to_cpu((nh)->data_len), \ - (nh)->msg, (nh)->cmd, (nh)->error + (nh)->cmd, (nh)->flags, (nh)->error #define snh_trace_define(name) \ __field(__u64, name##_id) \ __field(__u16, name##_data_len) \ - __field(__u8, name##_msg) \ __field(__u8, name##_cmd) \ + __field(__u8, name##_flags) \ __field(__u8, name##_error) #define snh_trace_assign(name, nh) \ @@ -39,14 +39,14 @@ do { \ \ __entry->name##_id = le64_to_cpu(_nh->id); \ __entry->name##_data_len = le16_to_cpu(_nh->data_len); \ - __entry->name##_msg = _nh->msg; \ __entry->name##_cmd = _nh->cmd; \ + __entry->name##_flags = _nh->flags; \ __entry->name##_error = _nh->error; \ } while (0) -#define snh_trace_args(name) \ - __entry->name##_id, __entry->name##_data_len, __entry->name##_msg, \ - __entry->name##_cmd, __entry->name##_error +#define snh_trace_args(name) \ + __entry->name##_id, __entry->name##_data_len, __entry->name##_cmd, \ + __entry->name##_flags, __entry->name##_error void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment); From 2cc990406a1c1c8380ccbc618d19edaf96fd8d85 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 Jul 2018 09:47:57 -0700 Subject: [PATCH 669/920] scoutfs: compact using net requests Currently compaction is only performed by one thread running in the server. Total metadata throughput of the system is limited by only having one compaction operation in flight at a time. This refactors the compaction code to have the server send compaction requests to clients who then perform the compaction and send responses to the server. This spreads compaction load out amongst all the clients and greatly increases total compaction throughput. The manifest keeps track of compactions that are in flight at a given level so that we maintain segment count invariants with multiple compactions in flight. It also uses the sparse bitmap to lock down segments that are being used as inputs to avoid duplicating items across two concurrent compactions. A server thread still coordinates which segments are compacted. The search for a candidate compaction operation is largely unchanged. It now has to deal with being unable to process a compaction because its segments are busy. We add some logic to keep searching in a level until we find a compaction that doesn't intersect with current compaction requests. If there are none at the level we move up to the next level. The server will only issue a given number of compaction requests to a client at a time. When it needs to send a compaction request it rotates through the current clients until it finds one that doesn't have the max in flight. If a client disconnects the server forgets the compactions it had sent to that client. If those compactions still need to be processed they'll be sent to the next client. The segnos that are allocated for compaction are not reclaimed if a client disconnects or the server crashes. This is a known deficiency that will be addressed with the broader work to add crash recovery to the multiple points in the protocol where the server and client trade ownership of persistent state. The server needs to block as it does work for compaction in the notify_up and response callbacks. We move them out from under spin locks. The server needs to clean up allocated segnos for a compaction request that fails. We let the client send a data payload along with an error response so that it can give the server the id of the compaction that failed. Signed-off-by: Zach Brown --- kmod/src/client.c | 60 +++- kmod/src/compact.c | 397 +++++++++++----------- kmod/src/compact.h | 14 +- kmod/src/counters.h | 2 + kmod/src/format.h | 52 ++- kmod/src/manifest.c | 246 ++++++++++++-- kmod/src/manifest.h | 6 +- kmod/src/net.c | 11 +- kmod/src/scoutfs_trace.h | 169 +++++++++- kmod/src/seg.c | 9 - kmod/src/seg.h | 1 - kmod/src/server.c | 704 +++++++++++++++++++++++++++++++++++---- kmod/src/server.h | 5 - 13 files changed, 1331 insertions(+), 345 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 39d98afb..55def453 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -357,6 +357,63 @@ out: } } +/* + * Perform a compaction in the client as requested by the server. The + * server has protected the input segments and allocated the output + * segnos for us. This executes in work queued by the client's net + * connection. It only reads and write segments. The server will + * update the manifest and allocators while processing the response. An + * error response includes the compaction id so that the server can + * clean it up. + * + * If we get duplicate requests across a reconnected socket we can have + * two workers performing the same compaction simultaneously. This + * isn't particularly efficient but it's rare and won't corrupt the + * output. Our response can be lost if the socket is shutdown while + * it's in flight, the server deals with this. + */ +static int client_compact(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_net_compact_response *resp = NULL; + struct scoutfs_net_compact_request *req; + int ret; + + if (arg_len != sizeof(struct scoutfs_net_compact_request)) { + ret = -EINVAL; + goto out; + } + req = arg; + + trace_scoutfs_client_compact_start(sb, le64_to_cpu(req->id), + req->last_level, req->flags); + + resp = kzalloc(sizeof(struct scoutfs_net_compact_response), GFP_NOFS); + if (!resp) { + ret = -ENOMEM; + } else { + resp->id = req->id; + ret = scoutfs_compact(sb, req, resp); + } + + trace_scoutfs_client_compact_stop(sb, le64_to_cpu(req->id), ret); + + if (ret < 0) + ret = scoutfs_net_response(sb, conn, cmd, id, ret, + &req->id, sizeof(req->id)); + else + ret = scoutfs_net_response(sb, conn, cmd, id, 0, + resp, sizeof(*resp)); + kfree(resp); +out: + return ret; +} + +static scoutfs_net_request_t client_req_funcs[] = { + [SCOUTFS_NET_CMD_COMPACT] = client_compact, +}; + /* * Called when either a connect attempt or established connection times * out and fails. @@ -405,9 +462,8 @@ int scoutfs_client_setup(struct super_block *sb) INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); - /* client doesn't process any incoming requests yet */ client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, - NULL, "client"); + client_req_funcs, "client"); if (!client->conn) { ret = -ENOMEM; goto out; diff --git a/kmod/src/compact.c b/kmod/src/compact.c index b18b11d6..cfca5fd8 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -30,10 +31,6 @@ * segments in each level of the lsm tree and is what merges duplicate * and deletion keys. * - * When the manifest is modified in a way that requires compaction it - * kicks the compaction thread. The compaction thread calls into the - * manifest to find the segments that need to be compaction. - * * The compaction operation itself always involves a single "upper" * segment at a given level and a limited number of "lower" segments at * the next higher level whose key range intersects with the upper @@ -42,23 +39,10 @@ * Compaction proceeds by iterating over the items in the upper segment * and items in each of the lower segments in sort order. The items * from the two input segments are copied into new output segments in - * sorted order. Item space is reclaimed as duplicate or deletion items - * are removed. - * - * Once the compaction is completed the manifest is updated to remove - * the input segments and add the output segments. Here segment space - * is reclaimed when the input items fit in fewer output segments. + * sorted order. Space is reclaimed as duplicate or deletion items are + * removed and fewer segments are written than were read. */ -struct compact_info { - struct super_block *sb; - struct workqueue_struct *workq; - struct work_struct work; -}; - -#define DECLARE_COMPACT_INFO(sb, name) \ - struct compact_info *name = SCOUTFS_SB(sb)->compact_info - struct compact_seg { struct list_head entry; @@ -72,16 +56,12 @@ struct compact_seg { bool part_of_move; }; -/* - * A compaction request. It's filled up in scoutfs_compact_add() as - * the manifest is wlaked and it finds segments involved in the compaction. - */ struct compact_cursor { struct list_head csegs; /* buffer holds allocations and our returning them */ - u64 segnos[SCOUTFS_COMPACTION_MAX_UPDATE]; - unsigned nr_segnos; + u64 segnos[SCOUTFS_COMPACTION_MAX_OUTPUT]; + unsigned int nr_segnos; u8 lower_level; u8 last_level; @@ -371,6 +351,12 @@ static int compact_segments(struct super_block *sb, break; } + /* didn't get enough segnos */ + if (next_segno >= curs->nr_segnos) { + ret = -ENOSPC; + break; + } + cseg->segno = curs->segnos[next_segno]; curs->segnos[next_segno] = 0; next_segno++; @@ -435,153 +421,216 @@ static int compact_segments(struct super_block *sb, } /* - * Manifest walking is providing the details of the overall compaction - * operation. + * We want all the non-zero segnos sorted at the front of the array + * and the empty segnos all packed at the end. This is easily done by + * subtracting one from both then comparing as usual. All relations hold + * except that 0 becomes the greatest instead of the least. */ -void scoutfs_compact_describe(struct super_block *sb, void *data, - u8 upper_level, u8 last_level, bool sticky) +static int sort_cmp_segnos(const void *A, const void *B) { - struct compact_cursor *curs = data; + const u64 a = *(const u64 *)A - 1; + const u64 b = *(const u64 *)B - 1; - curs->lower_level = upper_level + 1; - curs->last_level = last_level; - curs->sticky = sticky; + return a < b ? -1 : a > b ? 1 : 0; } -/* - * Add a segment involved in the compaction operation. - * - * XXX Today we know that the caller is always adding only one upper segment - * and is then possibly adding all the lower overlapping segments. - */ -int scoutfs_compact_add(struct super_block *sb, void *data, - struct scoutfs_manifest_entry *ment) +static void sort_swap_segnos(void *A, void *B, int size) { - struct compact_cursor *curs = data; - struct compact_seg *cseg; - int ret; + u64 *a = A; + u64 *b = B; - cseg = alloc_cseg(sb, &ment->first, &ment->last); - if (!cseg) { - ret = -ENOMEM; + swap(*a, *b); +} + +static int verify_request(struct super_block *sb, + struct scoutfs_net_compact_request *req) +{ + int ret = -EINVAL; + int nr_segnos; + int nr_ents; + int i; + + /* no unknown flags */ + if (req->flags & ~SCOUTFS_NET_COMPACT_FLAG_STICKY) goto out; + + /* find the number of segments and entries */ + for (i = 0; i < ARRAY_SIZE(req->segnos); i++) { + if (req->segnos[i] == 0) + break; + } + nr_segnos = i; + + for (i = 0; i < ARRAY_SIZE(req->ents); i++) { + if (req->ents[i].segno == 0) + break; + } + nr_ents = i; + + /* must have at least an upper */ + if (nr_ents == 0) + goto out; + + sort(req->segnos, nr_segnos, sizeof(req->segnos[i]), + sort_cmp_segnos, sort_swap_segnos); + + /* segnos must be unique */ + for (i = 1; i < nr_segnos; i++) { + if (req->segnos[i] == req->segnos[i - 1]) + goto out; } - list_add_tail(&cseg->entry, &curs->csegs); + /* if we have a lower it must be under upper */ + if (nr_ents > 1 && (req->ents[1].level != req->ents[0].level + 1)) + goto out; - cseg->segno = ment->segno; - cseg->seq = ment->seq; - cseg->level = ment->level; + /* make sure lower ents are on the same level */ + for (i = 2; i < nr_ents; i++) { + if (req->ents[i].level != req->ents[i - 1].level) + goto out; + } - if (!curs->upper) - curs->upper = cseg; - else if (!curs->lower) - curs->lower = cseg; - if (curs->lower) - curs->last_lower = cseg; + for (i = 1; i < nr_ents; i++) { + /* lowers must overlap with upper */ + if (scoutfs_key_compare_ranges(&req->ents[0].first, + &req->ents[0].last, + &req->ents[i].first, + &req->ents[i].last) != 0) + goto out; + + /* lowers must be on the level below upper */ + if (req->ents[i].level != req->ents[0].level + 1) + goto out; + } + + /* last level must include lowest level */ + if (req->last_level < req->ents[nr_ents - 1].level) + goto out; + + for (i = 2; i < nr_ents; i++) { + /* lowers must be sorted by first key */ + if (scoutfs_key_compare(&req->ents[i].first, + &req->ents[i - 1].first) <= 0) + goto out; + + /* lowers must not overlap with each other */ + if (scoutfs_key_compare_ranges(&req->ents[i].first, + &req->ents[i].last, + &req->ents[i - 1].first, + &req->ents[i - 1].last) == 0) + goto out; + } ret = 0; out: + if (WARN_ON_ONCE(ret < 0)) { + scoutfs_inc_counter(sb, compact_invalid_request); + printk("id %llu last_level %u flags 0x%x\n", + le64_to_cpu(req->id), req->last_level, req->flags); + printk("segnos: "); + for (i = 0; i < ARRAY_SIZE(req->segnos); i++) + printk("%llu ", le64_to_cpu(req->segnos[i])); + printk("\n"); + printk("entries: "); + for (i = 0; i < ARRAY_SIZE(req->ents); i++) { + printk(" [%u] segno %llu seq %llu level %u first "SK_FMT" last "SK_FMT"\n", + i, le64_to_cpu(req->ents[i].segno), + le64_to_cpu(req->ents[i].seq), + req->ents[i].level, + SK_ARG(&req->ents[i].first), + SK_ARG(&req->ents[i].last)); + } + printk("\n"); + } + return ret; } /* - * Give the compaction cursor a segno to allocate from. - */ -void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno) -{ - struct compact_cursor *curs = data; - - curs->segnos[curs->nr_segnos++] = segno; -} - -/* - * Commit the result of a compaction based on the state of the cursor. - * The server caller stops the manifest from being written while we're - * making changes. We lock the manifest to atomically make our changes. + * Translate the compaction request into our native structs that we use + * to perform the compaction. The caller has verified that the request + * satisfies our constraints. * - * The erorr handling is sketchy here because calling the manifest from - * here is temporary. We should be sending a message to the server - * instead of calling the allocator and manifest. + * If we return an error the caller will clean up a partially prepared + * cursor. */ -int scoutfs_compact_commit(struct super_block *sb, void *c, void *r) +static int prepare_curs(struct super_block *sb, struct compact_cursor *curs, + struct scoutfs_net_compact_request *req) { struct scoutfs_manifest_entry ment; - struct compact_cursor *curs = c; - struct list_head *results = r; struct compact_seg *cseg; - int ret; + int ret = 0; int i; - /* free unused segnos that were allocated for the compaction */ - for (i = 0; i < curs->nr_segnos; i++) { - if (curs->segnos[i]) { - ret = scoutfs_server_free_segno(sb, curs->segnos[i]); - BUG_ON(ret); - } + curs->lower_level = req->ents[0].level + 1; + curs->last_level = req->last_level; + curs->sticky = !!(req->flags & SCOUTFS_NET_COMPACT_FLAG_STICKY); + + for (i = 0; i < ARRAY_SIZE(req->segnos); i++) { + if (req->segnos[i] == 0) + break; + curs->segnos[i] = le64_to_cpu(req->segnos[i]); } + curs->nr_segnos = i; - scoutfs_manifest_lock(sb); + for (i = 0; i < ARRAY_SIZE(req->ents); i++) { + if (req->ents[i].segno == 0) + break; - /* delete input segments, probably freeing their segnos */ - list_for_each_entry(cseg, &curs->csegs, entry) { - if (!cseg->part_of_move) { - ret = scoutfs_server_free_segno(sb, cseg->segno); - BUG_ON(ret); + scoutfs_init_ment_from_net(&ment, &req->ents[i]); + + cseg = alloc_cseg(sb, &ment.first, &ment.last); + if (!cseg) { + ret = -ENOMEM; + break; } - scoutfs_manifest_init_entry(&ment, cseg->level, 0, cseg->seq, - &cseg->first, NULL); - ret = scoutfs_manifest_del(sb, &ment); - BUG_ON(ret); + list_add_tail(&cseg->entry, &curs->csegs); + + cseg->segno = ment.segno; + cseg->seq = ment.seq; + cseg->level = ment.level; + + if (!curs->upper) + curs->upper = cseg; + else if (!curs->lower) + curs->lower = cseg; + if (curs->lower) + curs->last_lower = cseg; } - /* add output entries */ - list_for_each_entry(cseg, results, entry) { - /* XXX moved upper segments won't have read the segment :P */ - if (cseg->seg) - scoutfs_seg_init_ment(&ment, cseg->level, cseg->seg); - else - scoutfs_manifest_init_entry(&ment, cseg->level, - cseg->segno, cseg->seq, - &cseg->first, &cseg->last); - ret = scoutfs_manifest_add(sb, &ment); - BUG_ON(ret); - } - - scoutfs_manifest_unlock(sb); - - return 0; + return ret; } /* - * The compaction worker tries to make forward progress with compaction - * every time its kicked. It pretends to send a message requesting - * compaction parameters but in reality the net request function there - * is calling directly into the manifest and back into our compaction - * add routines. + * Perform a compaction by translating the incoming request into our + * working state, iterating over input segments and write output + * segments, then generating the response that describes the output + * segments. * - * We always try to clean up everything on errors. + * The server will either commit our response or cleanup the request + * if we return an error that the caller sends in response. */ -static void scoutfs_compact_func(struct work_struct *work) +int scoutfs_compact(struct super_block *sb, + struct scoutfs_net_compact_request *req, + struct scoutfs_net_compact_response *resp) { - struct compact_info *ci = container_of(work, struct compact_info, work); - struct super_block *sb = ci->sb; struct compact_cursor curs = {{NULL,}}; + struct scoutfs_manifest_entry ment; struct scoutfs_bio_completion comp; struct compact_seg *cseg; LIST_HEAD(results); int ret; int err; + int nr; INIT_LIST_HEAD(&curs.csegs); scoutfs_bio_init_comp(&comp); - ret = scoutfs_client_get_compaction(sb, (void *)&curs); - - /* short circuit no compaction work to do */ - if (ret == 0 && list_empty(&curs.csegs)) - return; + ret = verify_request(sb, req) ?: + prepare_curs(sb, &curs, req); + if (ret) + goto out; /* trace compaction ranges */ list_for_each_entry(cseg, &curs.csegs, entry) { @@ -590,84 +639,40 @@ static void scoutfs_compact_func(struct work_struct *work) &cseg->last); } - if (ret == 0 && !list_empty(&curs.csegs)) { - ret = compact_segments(sb, &curs, &comp, &results); + ret = compact_segments(sb, &curs, &comp, &results); - /* always wait for io completion */ - err = scoutfs_bio_wait_comp(sb, &comp); - if (!ret && err) - ret = err; - } - - /* don't update manifest on error, just free segnos */ - if (ret) { - list_for_each_entry(cseg, &results, entry) { - if (!cseg->part_of_move) - curs.segnos[curs.nr_segnos++] = cseg->segno; - } - free_cseg_list(sb, &curs.csegs); - free_cseg_list(sb, &results); - } - - err = scoutfs_client_finish_compaction(sb, &curs, &results); + /* always wait for io completion */ + err = scoutfs_bio_wait_comp(sb, &comp); if (!ret && err) ret = err; + if (ret) + goto out; + + /* fill entries for written output segments */ + nr = 0; + list_for_each_entry(cseg, &results, entry) { + /* XXX moved upper segments won't have read the segment :P */ + if (cseg->seg) + scoutfs_seg_init_ment(&ment, cseg->level, cseg->seg); + else + scoutfs_manifest_init_entry(&ment, cseg->level, + cseg->segno, cseg->seq, + &cseg->first, &cseg->last); + + trace_scoutfs_compact_output(sb, ment.level, ment.segno, + ment.seq, &ment.first, + &ment.last); + + scoutfs_init_ment_to_net(&resp->ents[nr++], &ment); + } + + ret = 0; +out: + if (ret == -ESTALE) + scoutfs_inc_counter(sb, compact_stale_error); free_cseg_list(sb, &curs.csegs); free_cseg_list(sb, &results); - if (ret == -ESTALE) - scoutfs_inc_counter(sb, compact_stale_error); - - WARN_ON_ONCE(ret && ret != -ESTALE); - trace_scoutfs_compact_func(sb, ret); -} - -void scoutfs_compact_kick(struct super_block *sb) -{ - DECLARE_COMPACT_INFO(sb, ci); - - queue_work(ci->workq, &ci->work); -} - -int scoutfs_compact_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct compact_info *ci; - - ci = kzalloc(sizeof(struct compact_info), GFP_KERNEL); - if (!ci) - return -ENOMEM; - - ci->sb = sb; - INIT_WORK(&ci->work, scoutfs_compact_func); - - ci->workq = alloc_workqueue("scoutfs_compact", 0, 1); - if (!ci->workq) { - kfree(ci); - return -ENOMEM; - } - - sbi->compact_info = ci; - - return 0; -} - -/* - * The system should be idle, there should not be any more manifest - * modification which would kick compaction. - */ -void scoutfs_compact_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - DECLARE_COMPACT_INFO(sb, ci); - - if (ci) { - /* stop compaction from requeueing itself */ - cancel_work_sync(&ci->work); - destroy_workqueue(ci->workq); - sbi->compact_info = NULL; - - kfree(ci); - } + return ret; } diff --git a/kmod/src/compact.h b/kmod/src/compact.h index c163ce56..788cae2c 100644 --- a/kmod/src/compact.h +++ b/kmod/src/compact.h @@ -1,16 +1,8 @@ #ifndef _SCOUTFS_COMPACT_H_ #define _SCOUTFS_COMPACT_H_ -void scoutfs_compact_kick(struct super_block *sb); - -void scoutfs_compact_describe(struct super_block *sb, void *data, - u8 upper_level, u8 last_level, bool sticky); -int scoutfs_compact_add(struct super_block *sb, void *data, - struct scoutfs_manifest_entry *ment); -void scoutfs_compact_add_segno(struct super_block *sb, void *data, u64 segno); -int scoutfs_compact_commit(struct super_block *sb, void *c, void *r); - -int scoutfs_compact_setup(struct super_block *sb); -void scoutfs_compact_destroy(struct super_block *sb); +int scoutfs_compact(struct super_block *sb, + struct scoutfs_net_compact_request *req, + struct scoutfs_net_compact_response *resp); #endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 3d5886c0..82bead6b 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -15,7 +15,9 @@ EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(btree_write_error) \ + EXPAND_COUNTER(compact_invalid_request) \ EXPAND_COUNTER(compact_operations) \ + EXPAND_COUNTER(compact_segment_busy) \ EXPAND_COUNTER(compact_segment_moved) \ EXPAND_COUNTER(compact_segment_read) \ EXPAND_COUNTER(compact_segment_write_bytes) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 4d91c250..d7736a38 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -369,6 +369,7 @@ struct scoutfs_super_block { struct scoutfs_btree_ring bring; __le64 next_seg_seq; __le64 next_node_id; + __le64 next_compact_id; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; @@ -548,6 +549,7 @@ enum { SCOUTFS_NET_CMD_GET_LAST_SEQ, SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, SCOUTFS_NET_CMD_STATFS, + SCOUTFS_NET_CMD_COMPACT, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -623,15 +625,49 @@ struct scoutfs_net_extent_list { /* arbitrarily makes a nice ~1k extent list payload */ #define SCOUTFS_NET_EXTENT_LIST_MAX_NR 64 -/* XXX eventually we'll have net compaction and will need agents to agree */ - /* one upper segment and fanout lower segments */ -#define SCOUTFS_COMPACTION_MAX_INPUT (1 + SCOUTFS_MANIFEST_FANOUT) -/* sticky can add one, and so can item page alignment */ -#define SCOUTFS_COMPACTION_SLOP 2 -/* delete all inputs and insert all outputs (same goes for alloc|free segnos) */ -#define SCOUTFS_COMPACTION_MAX_UPDATE \ - (2 * (SCOUTFS_COMPACTION_MAX_INPUT + SCOUTFS_COMPACTION_SLOP)) +#define SCOUTFS_COMPACTION_MAX_INPUT (1 + SCOUTFS_MANIFEST_FANOUT) +/* sticky can split the input and item alignment padding can add a lower */ +#define SCOUTFS_COMPACTION_SEGNO_OVERHEAD 2 +#define SCOUTFS_COMPACTION_MAX_OUTPUT \ + (SCOUTFS_COMPACTION_MAX_INPUT + SCOUTFS_COMPACTION_SEGNO_OVERHEAD) + +/* + * A compact request is sent by the server to the client. It provides + * the input segments and enough allocated segnos to write the results. + * The id uniquely identifies this compaction request and is included in + * the response to clean up its allocated resources. + */ +struct scoutfs_net_compact_request { + __le64 id; + __u8 last_level; + __u8 flags; + __le64 segnos[SCOUTFS_COMPACTION_MAX_OUTPUT]; + struct scoutfs_net_manifest_entry ents[SCOUTFS_COMPACTION_MAX_INPUT]; +} __packed; + +/* + * A sticky compaction has more lower level segments that overlap with + * the end of the upper after the last lower level segment included in + * the compaction. Items left in the upper segment after the last lower + * need to be written to the upper level instead of the lower. The + * upper segment "sticks" in place instead of moving down to the lower + * level. + */ +#define SCOUTFS_NET_COMPACT_FLAG_STICKY (1 << 0) + +/* + * A compact response is sent by the client to the server. It describes + * the written output segments that need to be added to the manifest. + * The server compares the response to the request to free unused + * allocated segnos and input manifest entries. An empty response is + * valid and can happen if, say, the upper input segment completely + * deleted all the items in a single overlapping lower segment. + */ +struct scoutfs_net_compact_response { + __le64 id; + struct scoutfs_net_manifest_entry ents[SCOUTFS_COMPACTION_MAX_OUTPUT]; +} __packed; /* * Scoutfs file handle structure - this can be copied out to userspace diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 5ecd9f61..1fa6158a 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -29,6 +29,7 @@ #include "counters.h" #include "triggers.h" #include "client.h" +#include "spbm.h" #include "scoutfs_trace.h" /* @@ -47,6 +48,8 @@ struct manifest { /* calculated on mount, const thereafter */ u64 level_limits[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + u64 compacts_pending[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; + struct scoutfs_spbm segno_busy; unsigned long flags; @@ -884,6 +887,121 @@ out: return ret; } +static bool level_should_compact(struct super_block *sb, int level) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + BUG_ON(!rwsem_is_locked(&mani->rwsem)); + + return ((s64)le64_to_cpu(super->manifest.level_counts[level]) - + (s64)mani->compacts_pending[level]) > mani->level_limits[level]; +} + +int scoutfs_manifest_should_compact(struct super_block *sb) +{ + DECLARE_MANIFEST(sb, mani); + bool should = false; + int level; + + down_read(&mani->rwsem); + for (level = mani->nr_levels - 1; level >= 0; level--) { + if (level_should_compact(sb, level)) { + should = true; + break; + } + } + up_read(&mani->rwsem); + + return should; +} + +/* + * Record that a compaction operation is in flight. We mark the segnos + * involved so that we don't use them as inputs for other compactions + * and assume that the compaction will delete a segment from its upper + * level when deciding what level to compact. + */ +static int start_compact_request(struct super_block *sb, + struct scoutfs_net_compact_request *req) +{ + DECLARE_MANIFEST(sb, mani); + int level; + int ret = 0; + int i; + + BUG_ON(!rwsem_is_locked(&mani->rwsem)); + + for (i = 0; i < ARRAY_SIZE(req->ents); i++) { + if (req->ents[i].segno == 0) + break; + + ret = scoutfs_spbm_set(&mani->segno_busy, + le64_to_cpu(req->ents[i].segno)); + if (ret) { + while (i-- > 0) + scoutfs_spbm_clear(&mani->segno_busy, + le64_to_cpu(req->ents[i].segno)); + break; + } + } + + if (ret == 0) { + level = req->ents[0].level; + mani->compacts_pending[level]++; + } + + return ret; +} + +/* + * A compaction request has completed. No longer account for it in the + * level pending counts and stop tracking all its segments. + * + * This can be called in error paths with an empty zeroed request and it + * will do nothing. + */ +void scoutfs_manifest_compact_done(struct super_block *sb, + struct scoutfs_net_compact_request *req) +{ + DECLARE_MANIFEST(sb, mani); + int level; + int i; + + down_write(&mani->rwsem); + + for (i = 0; i < ARRAY_SIZE(req->ents); i++) { + if (req->ents[i].segno == 0) + break; + + scoutfs_spbm_clear(&mani->segno_busy, + le64_to_cpu(req->ents[i].segno)); + } + + if (i > 0) { + level = req->ents[0].level; + mani->compacts_pending[level]--; + } + + up_write(&mani->rwsem); +} + +static int add_entry_unless_busy(struct super_block *sb, + struct scoutfs_net_compact_request *req, + unsigned int ind, + struct scoutfs_manifest_entry *ment) +{ + DECLARE_MANIFEST(sb, mani); + + if (scoutfs_spbm_test(&mani->segno_busy, ment->segno)) { + scoutfs_inc_counter(sb, compact_segment_busy); + return -EAGAIN; + } + + scoutfs_init_ment_to_net(&req->ents[ind], ment); + return 0; +} + /* * Give the caller the segments that will be involved in the next * compaction. @@ -898,15 +1016,19 @@ out: * We add all the segments to the compaction caller's data and let it do * its thing. It'll allocate and free segments and update the manifest. * - * Returns the number of input segments or -errno. + * Returns: + * 0: no compactions were needed at the given level + * > 0: number of total imput segments in the compaction + * -EAGAIN: segments were already in a pending compaction + * -errno: fatal error * - * XXX this will get a lot more clever: - * - ensuring concurrent compactions don't overlap + * XXX this could be more clever: * - prioritize segments with deletion or incremental records * - prioritize partial segments * - maybe compact segments by age in a given level */ -int scoutfs_manifest_next_compact(struct super_block *sb, void *data) +static int next_compact_req(struct super_block *sb, int level, + struct scoutfs_net_compact_request *req) { DECLARE_MANIFEST(sb, mani); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -918,30 +1040,17 @@ int scoutfs_manifest_next_compact(struct super_block *sb, void *data) SCOUTFS_BTREE_ITEM_REF(iref); SCOUTFS_BTREE_ITEM_REF(over_iref); SCOUTFS_BTREE_ITEM_REF(prev); - struct scoutfs_key zeros; + static struct scoutfs_key zeros; bool wrapped; bool sticky; - int level; int ret; int nr = 0; int i; scoutfs_key_set_zeros(&zeros); + memset(req, 0, sizeof(*req)); - down_write(&mani->rwsem); - - for (level = mani->nr_levels - 1; level >= 0; level--) { - if (le64_to_cpu(super->manifest.level_counts[level]) > - mani->level_limits[level]) - break; - } - - trace_scoutfs_manifest_next_compact(sb, level); - - if (level < 0) { - ret = 0; - goto out; - } + BUG_ON(!rwsem_is_locked(&mani->rwsem)); /* fill ment and ret == 0 if we find an entry at the level */ if (level == 0) { @@ -1003,9 +1112,9 @@ again: } /* add the upper input segment */ - ret = scoutfs_compact_add(sb, data, &ment); + ret = add_entry_unless_busy(sb, req, nr, &ment); if (ret) - goto out; + goto skip; nr++; /* and add a fanout's worth of lower overlapping segments */ @@ -1029,7 +1138,7 @@ again: break; } - ret = scoutfs_compact_add(sb, data, &over); + ret = add_entry_unless_busy(sb, req, nr, &over); if (ret) goto out; nr++; @@ -1042,16 +1151,20 @@ again: if (ret < 0 && ret != -ENOENT) goto out; - scoutfs_compact_describe(sb, data, level, mani->nr_levels - 1, sticky); + req->last_level = mani->nr_levels - 1; + if (sticky) + req->flags |= SCOUTFS_NET_COMPACT_FLAG_STICKY; + ret = start_compact_request(sb, req); + if (ret) + goto out; + + ret = 0; +skip: /* record the next key to start from */ mani->compact_keys[level] = ment.last; scoutfs_key_inc(&mani->compact_keys[level]); - - ret = 0; out: - up_write(&mani->rwsem); - scoutfs_btree_put_iref(&iref); scoutfs_btree_put_iref(&over_iref); scoutfs_btree_put_iref(&prev); @@ -1059,6 +1172,81 @@ out: return ret ?: nr; } +/* + * Find the next segment to compact into its lower overlapping segments. + * Fill out the callers request describing all the segments involved in + * the operation. + * + * First we search for a level to compact. A level needs compaction if + * it has more segments than its limit. We search from the bottom up + * because segments are written at the top when there's space. By + * compacting from the bottom we pull new segments down until there's + * space. If we compacted from the top down then we could create an + * imbalanced top-heavy structure. + * + * At each level we find the segment from a cursor and try to compact it + * into its lower segments. Any of the segments involved could already + * be part of a pending compaction and need to be skipped. In that case + * we move to the next segment at the level. All the segments at the + * level could be busy so we detect when we skip to the first value we + * skipped to and move on. + * + * If we return a filled compact request then we've tracked it. We + * assume it will delete an upper segment and have marked all its segnos + * as busy so they won't be used by future compaction requests. The + * caller must call complete_done when the compact operation completes. + */ +int scoutfs_manifest_next_compact(struct super_block *sb, + struct scoutfs_net_compact_request *req) +{ + DECLARE_MANIFEST(sb, mani); + struct scoutfs_key key = {0,}; + bool first; + int level; + int ret = 0; + + memset(req, 0, sizeof(*req)); + + down_write(&mani->rwsem); + + for (level = mani->nr_levels - 1; level >= 0; level--) { + if (!level_should_compact(sb, level)) + continue; + + first = true; + + for (;;) { + ret = next_compact_req(sb, level, req); + if (ret > 0 || (ret < 0 && ret != -EAGAIN)) + goto out; + if (ret == 0) + break; + + /* remember first skip and keep going */ + if (first) { + first = false; + key = mani->compact_keys[level]; + continue; + } + + /* bail if we looped around */ + if (!scoutfs_key_compare(&key, + &mani->compact_keys[level])) { + ret = 0; + break; + } + } + /* continue to next level */ + } + +out: + up_write(&mani->rwsem); + + trace_scoutfs_manifest_next_compact(sb, level, ret); + + return ret; +} + int scoutfs_manifest_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -1071,6 +1259,7 @@ int scoutfs_manifest_setup(struct super_block *sb) return -ENOMEM; init_rwsem(&mani->rwsem); + scoutfs_spbm_init(&mani->segno_busy); for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) scoutfs_key_set_zeros(&mani->compact_keys[i]); @@ -1101,6 +1290,7 @@ void scoutfs_manifest_destroy(struct super_block *sb) struct manifest *mani = sbi->manifest; if (mani) { + scoutfs_spbm_destroy(&mani->segno_busy); kfree(mani); sbi->manifest = NULL; } diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h index cd1a095d..b3ffdf54 100644 --- a/kmod/src/manifest.h +++ b/kmod/src/manifest.h @@ -38,7 +38,11 @@ int scoutfs_manifest_read_items(struct super_block *sb, int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next_key); -int scoutfs_manifest_next_compact(struct super_block *sb, void *data); +int scoutfs_manifest_should_compact(struct super_block *sb); +int scoutfs_manifest_next_compact(struct super_block *sb, + struct scoutfs_net_compact_request *req); +void scoutfs_manifest_compact_done(struct super_block *sb, + struct scoutfs_net_compact_request *req); bool scoutfs_manifest_level0_full(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index e889574a..da97665e 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -348,7 +348,7 @@ static int submit_send(struct super_block *sb, WARN_ON_ONCE(flags & SCOUTFS_NET_FLAGS_UNKNOWN) || WARN_ON_ONCE(net_err >= SCOUTFS_NET_ERR_UNKNOWN) || WARN_ON_ONCE(data_len > SCOUTFS_NET_MAX_DATA_LEN) || - WARN_ON_ONCE(data_len && (!data || net_err)) || + WARN_ON_ONCE(data_len && data == NULL) || WARN_ON_ONCE(net_err && (!(flags & SCOUTFS_NET_FLAG_RESPONSE))) || WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) return -EINVAL; @@ -430,12 +430,13 @@ static void saw_valid_greeting(struct scoutfs_net_connection *conn, u64 node_id) conn->valid_greeting = 1; conn->node_id = node_id; - if (conn->notify_up) - conn->notify_up(sb, conn, conn->info, node_id); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); spin_unlock(&conn->lock); + + if (conn->notify_up) + conn->notify_up(sb, conn, conn->info, node_id); } /* @@ -589,10 +590,6 @@ static bool invalid_message(struct scoutfs_net_header *nh) nh->error >= SCOUTFS_NET_ERR_UNKNOWN) return true; - /* errors can't have payloads */ - if (nh->data_len != 0 && nh->error != SCOUTFS_NET_ERR_NONE) - return true; - /* payloads have a limit */ if (le16_to_cpu(nh->data_len) > SCOUTFS_NET_MAX_DATA_LEN) return true; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 962bfca7..132c94f1 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -998,21 +998,24 @@ TRACE_EVENT(scoutfs_xattr_set, ); TRACE_EVENT(scoutfs_manifest_next_compact, - TP_PROTO(struct super_block *sb, int level), + TP_PROTO(struct super_block *sb, int level, int ret), - TP_ARGS(sb, level), + TP_ARGS(sb, level, ret), TP_STRUCT__entry( __field(__u64, fsid) __field(int, level) + __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); __entry->level = level; + __entry->ret = ret; ), - TP_printk(FSID_FMT" level %d", __entry->fsid, __entry->level) + TP_printk(FSID_FMT" level %d ret %d", __entry->fsid, __entry->level, + __entry->ret) ); TRACE_EVENT(scoutfs_advance_dirty_super, @@ -1069,22 +1072,127 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, __entry->found_dir_ino, __entry->name_len) ); -TRACE_EVENT(scoutfs_compact_func, - TP_PROTO(struct super_block *sb, int ret), +TRACE_EVENT(scoutfs_client_compact_start, + TP_PROTO(struct super_block *sb, u64 id, u8 last_level, u8 flags), - TP_ARGS(sb, ret), + TP_ARGS(sb, id, last_level, flags), TP_STRUCT__entry( __field(__u64, fsid) + __field(__u64, id) + __field(__u8, last_level) + __field(__u8, flags) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->id = id; + __entry->last_level = last_level; + __entry->flags = flags; + ), + + TP_printk("fsid "FSID_FMT" id %llu last_level %u flags 0x%x", + __entry->fsid, __entry->id, __entry->last_level, + __entry->flags) +); + +TRACE_EVENT(scoutfs_client_compact_stop, + TP_PROTO(struct super_block *sb, u64 id, int ret), + + TP_ARGS(sb, id, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, id) __field(int, ret) ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); + __entry->id = id; __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk("fsid "FSID_FMT" id %llu ret %d", + __entry->fsid, __entry->id, __entry->ret) +); + +TRACE_EVENT(scoutfs_server_compact_start, + TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 node_id, + unsigned long client_nr, unsigned long server_nr, + unsigned long per_client), + + TP_ARGS(sb, id, level, node_id, client_nr, server_nr, per_client), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, id) + __field(__u8, level) + __field(__u64, node_id) + __field(unsigned long, client_nr) + __field(unsigned long, server_nr) + __field(unsigned long, per_client) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->id = id; + __entry->level = level; + __entry->node_id = node_id; + __entry->client_nr = client_nr; + __entry->server_nr = server_nr; + __entry->per_client = per_client; + ), + + TP_printk("fsid "FSID_FMT" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu", + __entry->fsid, __entry->id, __entry->level, __entry->node_id, + __entry->client_nr, __entry->server_nr, __entry->per_client) +); + +TRACE_EVENT(scoutfs_server_compact_done, + TP_PROTO(struct super_block *sb, u64 id, u64 node_id, + unsigned long server_nr), + + TP_ARGS(sb, id, node_id, server_nr), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, id) + __field(__u64, node_id) + __field(unsigned long, server_nr) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->id = id; + __entry->node_id = node_id; + __entry->server_nr = server_nr; + ), + + TP_printk("fsid "FSID_FMT" id %llu node_id %llu server_nr %lu", + __entry->fsid, __entry->id, __entry->node_id, + __entry->server_nr) +); + +TRACE_EVENT(scoutfs_server_compact_response, + TP_PROTO(struct super_block *sb, u64 id, int error), + + TP_ARGS(sb, id, error), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, id) + __field(int, error) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->id = id; + __entry->error = error; + ), + + TP_printk("fsid "FSID_FMT" id %llu error %d", + __entry->fsid, __entry->id, __entry->error) ); TRACE_EVENT(scoutfs_write_begin, @@ -1277,6 +1385,12 @@ DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_input, TP_ARGS(sb, level, segno, seq, first, last) ); +DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_output, + TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, + struct scoutfs_key *first, struct scoutfs_key *last), + TP_ARGS(sb, level, segno, seq, first, last) +); + DEFINE_EVENT(scoutfs_manifest_class, scoutfs_read_item_segment, TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, struct scoutfs_key *first, struct scoutfs_key *last), @@ -1694,6 +1808,14 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_compact_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_server_compact_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) @@ -2267,6 +2389,39 @@ DEFINE_EVENT(scoutfs_segno_class, scoutfs_free_segno, TP_PROTO(struct super_block *sb, u64 segno), TP_ARGS(sb, segno) ); +DEFINE_EVENT(scoutfs_segno_class, scoutfs_remove_segno, + TP_PROTO(struct super_block *sb, u64 segno), + TP_ARGS(sb, segno) +); + +DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, + TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), + + TP_ARGS(sb, node_id, nr_clients), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__s64, node_id) + __field(unsigned long, nr_clients) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->node_id = node_id; + __entry->nr_clients = nr_clients; + ), + + TP_printk("fsid "FSID_FMT" node_id %llu nr_clients %lu", + __entry->fsid, __entry->node_id, __entry->nr_clients) +); +DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up, + TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), + TP_ARGS(sb, node_id, nr_clients) +); +DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, + TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), + TP_ARGS(sb, node_id, nr_clients) +); #endif /* _TRACE_SCOUTFS_H */ diff --git a/kmod/src/seg.c b/kmod/src/seg.c index 3b4b33a0..49523b90 100644 --- a/kmod/src/seg.c +++ b/kmod/src/seg.c @@ -315,15 +315,6 @@ out: } -/* - * This just frees the segno for the given seg. It's gross but - * symmetrical with only being able to allocate segnos by allocating a - * seg. We'll probably have to do better. - */ -int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg) -{ - return scoutfs_server_free_segno(sb, seg->segno); -} /* * The bios submitted by this don't have page references themselves. If diff --git a/kmod/src/seg.h b/kmod/src/seg.h index 74f7f961..58c4a64a 100644 --- a/kmod/src/seg.h +++ b/kmod/src/seg.h @@ -34,7 +34,6 @@ void scoutfs_seg_put(struct scoutfs_segment *seg); int scoutfs_seg_alloc(struct super_block *sb, u64 segno, struct scoutfs_segment **seg_ret); -int scoutfs_seg_free_segno(struct super_block *sb, struct scoutfs_segment *seg); bool scoutfs_seg_fits_single(u32 nr_items, u32 val_bytes); bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, struct scoutfs_key *key, struct kvec *val, diff --git a/kmod/src/server.c b/kmod/src/server.c index a91cf005..556229b4 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -34,6 +34,11 @@ #include "net.h" #include "endian_swap.h" +/* + * XXX pre commit: + * - comments + */ + /* * Every active mount can act as the server that listens on a net * connection and accepts connections from all the other mounts acting @@ -48,20 +53,19 @@ struct server_info { struct super_block *sb; spinlock_t lock; + wait_queue_head_t waitq; struct workqueue_struct *wq; struct delayed_work dwork; struct completion shutdown_comp; bool bind_warned; + struct scoutfs_net_connection *conn; /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; struct work_struct commit_work; - /* adding new segments can have to wait for compaction */ - wait_queue_head_t compaction_waitq; - /* server remembers the stable manifest root for clients */ seqcount_t stable_seqcount; struct scoutfs_btree_root stable_manifest_root; @@ -75,6 +79,13 @@ struct server_info { struct list_head pending_frees; struct list_head clients; + unsigned long nr_clients; + + /* track compaction in flight */ + unsigned long compacts_per_client; + unsigned long nr_compacts; + struct list_head compacts; + struct work_struct compact_work; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -86,6 +97,7 @@ struct server_info { struct server_client_info { u64 node_id; struct list_head head; + unsigned long nr_compacts; }; struct commit_waiter { @@ -391,7 +403,7 @@ static int free_extent(struct super_block *sb, u64 start, u64 len) * This is called by the compaction code which is running in the server. * The server caller has held all the locks, etc. */ -int scoutfs_server_free_segno(struct super_block *sb, u64 segno) +static int free_segno(struct super_block *sb, u64 segno) { scoutfs_inc_counter(sb, server_free_segno); trace_scoutfs_free_segno(sb, segno); @@ -459,11 +471,51 @@ out: return ret; } +/* + * "allocating" a segno removes an unknown segment from the allocator + * and returns it, "removing" a segno removes a specific segno from the + * allocator. + */ +static int remove_segno(struct super_block *sb, u64 segno) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct scoutfs_extent ext; + int ret; + + trace_scoutfs_remove_segno(sb, segno); + + scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, + segno << SCOUTFS_SEGMENT_BLOCK_SHIFT, + SCOUTFS_SEGMENT_BLOCKS, 0, 0); + + down_write(&server->alloc_rwsem); + ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); + up_write(&server->alloc_rwsem); + return ret; +} + static void shutdown_server(struct server_info *server) { complete(&server->shutdown_comp); } +/* + * Queue compaction work if clients have capacity for processing + * requests and the manifest knows of levels with too many segments. + */ +static void try_queue_compact(struct server_info *server) +{ + struct super_block *sb = server->sb; + bool can_request; + + spin_lock(&server->lock); + can_request = server->nr_compacts < + (server->nr_clients * server->compacts_per_client); + spin_unlock(&server->lock); + if (can_request && scoutfs_manifest_should_compact(sb)) + queue_work(server->wq, &server->compact_work); +} + /* * This is called while still holding the rwsem that prevents commits so * that the caller can be sure to be woken by the next commit after they @@ -787,8 +839,7 @@ retry: scoutfs_manifest_unlock(sb); up_read(&server->commit_rwsem); /* XXX waits indefinitely? io errors? */ - wait_event(server->compaction_waitq, - !scoutfs_manifest_level0_full(sb)); + wait_event(server->waitq, !scoutfs_manifest_level0_full(sb)); goto retry; } @@ -804,7 +855,7 @@ retry: if (ret == 0) { ret = wait_for_commit(&cw); if (ret == 0) - scoutfs_compact_kick(sb); + try_queue_compact(server); } out: @@ -1044,97 +1095,597 @@ out: return ret; } +/* requests sent to clients are tracked so we can free resources */ +struct compact_request { + struct list_head head; + u64 node_id; + struct scoutfs_net_compact_request req; +}; + /* - * Eventually we're going to have messages that control compaction. - * Each client mount would have long-lived work that sends requests - * which are stuck in processing until there's work to do. They'd get - * their entries, perform the compaction, and send a reply. But we're - * not there yet. - * - * This is a short circuit that's called directly by a work function - * that's only queued on the server. It makes compaction work inside - * the commit consistency mechanics inside request processing and - * demonstrates the moving pieces that we'd need to cut up into a series - * of messages and replies. - * - * The compaction work caller cleans up everything on errors. + * Find a node that can process our compaction request. Return a + * node_id if we found a client and added the compaction to the client + * and server counts. Returns 0 if no suitable clients were found. */ -int scoutfs_client_get_compaction(struct super_block *sb, void *curs) +static u64 compact_request_start(struct super_block *sb, + struct compact_request *cr) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct commit_waiter cw; - u64 segno; - int ret = 0; - int nr; - int i; + struct server_client_info *last; + struct server_client_info *sci; + u64 node_id = 0; - down_read(&server->commit_rwsem); + spin_lock(&server->lock); - nr = scoutfs_manifest_next_compact(sb, curs); - if (nr <= 0) { - up_read(&server->commit_rwsem); - return nr; - } + /* XXX no last_entry_or_null? :( */ + if (!list_empty(&server->clients)) + last = list_last_entry(&server->clients, + struct server_client_info, head); + else + last = NULL; - /* allow for expansion slop from sticky and alignment */ - for (i = 0; i < nr + SCOUTFS_COMPACTION_SLOP; i++) { - ret = alloc_segno(sb, &segno); - if (ret < 0) + while ((sci = list_first_entry_or_null(&server->clients, + struct server_client_info, + head)) != NULL) { + list_move_tail(&sci->head, &server->clients); + if (sci->nr_compacts < server->compacts_per_client) { + list_add(&cr->head, &server->compacts); + server->nr_compacts++; + sci->nr_compacts++; + node_id = sci->node_id; + cr->node_id = node_id; + break; + } + if (sci == last) break; - scoutfs_compact_add_segno(sb, curs, segno); } - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); + trace_scoutfs_server_compact_start(sb, le64_to_cpu(cr->req.id), + cr->req.ents[0].level, node_id, + node_id ? sci->nr_compacts : 0, + server->nr_compacts, + server->compacts_per_client); - if (ret == 0) - ret = wait_for_commit(&cw); + spin_unlock(&server->lock); + + return node_id; +} + +/* + * Find a tracked compact request for the compaction id, remove it from + * the server and client counts, and return it to the caller. + */ +static struct compact_request *compact_request_done(struct super_block *sb, + u64 id) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct compact_request *ret = NULL; + struct server_client_info *sci; + struct compact_request *cr; + + spin_lock(&server->lock); + + list_for_each_entry(cr, &server->compacts, head) { + if (le64_to_cpu(cr->req.id) != id) + continue; + + list_for_each_entry(sci, &server->clients, head) { + if (sci->node_id == cr->node_id) { + sci->nr_compacts--; + break; + } + } + + server->nr_compacts--; + list_del_init(&cr->head); + ret = cr; + break; + } + + trace_scoutfs_server_compact_done(sb, id, ret ? ret->node_id : 0, + server->nr_compacts); + + spin_unlock(&server->lock); return ret; } /* - * This is a stub for recording the results of a compaction. We just - * call back into compaction to have it call the manifest and allocator - * updates. + * When a client disconnects we forget the compactions that they had + * in flight so that we have capacity to send compaction requests to the + * remaining clients. * - * In the future we'd encode the manifest and segnos in requests sent to - * the server who'd update the manifest and allocator in request - * processing. - * - * As we finish a compaction we wait level0 writers if it opened up - * space in level 0. + * XXX we do not free their allocated segnos because they could still be + * running and writing to those blocks. To do this safely we'd need + * full recovery procedures with fencing to ensure that they're not able + * to write to those blocks anymore. */ -int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, - void *list) +static void forget_client_compacts(struct super_block *sb, + struct server_client_info *sci) { struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct compact_request *cr; + struct compact_request *pos; + LIST_HEAD(forget); + + spin_lock(&server->lock); + list_for_each_entry_safe(cr, pos, &server->compacts, head) { + if (cr->node_id == sci->node_id) { + sci->nr_compacts--; + server->nr_compacts--; + list_move(&cr->head, &forget); + } + } + spin_unlock(&server->lock); + + list_for_each_entry_safe(cr, pos, &forget, head) { + scoutfs_manifest_compact_done(sb, &cr->req); + list_del_init(&cr->head); + kfree(cr); + } +} + +static int segno_in_ents(__le64 segno, struct scoutfs_net_manifest_entry *ents, + unsigned int nr) +{ + int i; + + for (i = 0; i < nr; i++) { + if (ents[i].segno == 0) + break; + if (segno == ents[i].segno) + return 1; + } + + return 0; +} + +static int remove_segnos(struct super_block *sb, __le64 * __packed segnos, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup); + +/* + * Free segnos if they're not found in the unless entries. If this + * returns an error then we've cleaned up partial frees on error. This + * panics if it sees an error and can't cleanup on error. + * + * There are variants of this for lots of add/del, alloc/remove data + * structurs. + */ +static int free_segnos(struct super_block *sb, __le64 * __packed segnos, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup) + +{ + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (segnos[i] == 0) + break; + if (segno_in_ents(segnos[i], unless, nr_unless)) + continue; + + ret = free_segno(sb, le64_to_cpu(segnos[i])); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + remove_segnos(sb, segnos, i, unless, nr_unless, false); + break; + } + } + + return ret; +} + +static int alloc_segnos(struct super_block *sb, __le64 * __packed segnos, + unsigned int nr) + +{ + u64 segno; + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + ret = alloc_segno(sb, &segno); + if (ret < 0) { + free_segnos(sb, segnos, i, NULL, 0, false); + break; + } + segnos[i] = cpu_to_le64(segno); + } + + return ret; +} + +static int remove_segnos(struct super_block *sb, __le64 * __packed segnos, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup) + +{ + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (segnos[i] == 0) + break; + if (segno_in_ents(segnos[i], unless, nr_unless)) + continue; + + ret = remove_segno(sb, le64_to_cpu(segnos[i])); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + free_segnos(sb, segnos, i, unless, nr_unless, false); + break; + } + } + + return ret; +} + + +static int remove_entry_segnos(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup); + +static int free_entry_segnos(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup) +{ + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (ents[i].segno == 0) + break; + if (segno_in_ents(ents[i].segno, unless, nr_unless)) + continue; + + ret = free_segno(sb, le64_to_cpu(ents[i].segno)); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + remove_entry_segnos(sb, ents, i, unless, nr_unless, + false); + break; + } + } + + return ret; +} + +static int remove_entry_segnos(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, + struct scoutfs_net_manifest_entry *unless, + unsigned int nr_unless, bool cleanup) +{ + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (ents[i].segno == 0) + break; + if (segno_in_ents(ents[i].segno, unless, nr_unless)) + continue; + + ret = remove_segno(sb, le64_to_cpu(ents[i].segno)); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + free_entry_segnos(sb, ents, i, unless, nr_unless, + false); + break; + } + } + + return ret; +} + +static int del_manifest_entries(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, bool cleanup); + +static int add_manifest_entries(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, bool cleanup) +{ + struct scoutfs_manifest_entry ment; + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (ents[i].segno == 0) + break; + + scoutfs_init_ment_from_net(&ment, &ents[i]); + + ret = scoutfs_manifest_add(sb, &ment); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + del_manifest_entries(sb, ents, i, false); + break; + } + } + + return ret; +} + +static int del_manifest_entries(struct super_block *sb, + struct scoutfs_net_manifest_entry *ents, + unsigned int nr, bool cleanup) +{ + struct scoutfs_manifest_entry ment; + int ret = 0; + int i; + + for (i = 0; i < nr; i++) { + if (ents[i].segno == 0) + break; + + scoutfs_init_ment_from_net(&ment, &ents[i]); + + ret = scoutfs_manifest_del(sb, &ment); + BUG_ON(ret < 0 && !cleanup); + if (ret < 0) { + add_manifest_entries(sb, ents, i, false); + break; + } + } + + return ret; +} + +/* + * Process a received compaction response. This is called in concurrent + * processing work context so it's racing with other compaction + * responses and new compaction requests being built and sent. + * + * If the compaction failed then we only have to free the allocated + * output segnos sent in the request. + * + * If the compaction succeeded then we need to delete the input manifest + * entries, add any new output manifest entries, and free allocated + * segnos and input manifest segnos that aren't found in output segnos. + * + * And finally we always remove the compaction from the runtime client + * accounting + * + * As we finish a compaction we wake level0 writers if there's now space + * in level 0 for a new segment. + * + * Errors in processing are taken as an indication that this server is + * no longer able to do its job. We return hard errors which shut down + * the server in the hopes that another healthy server will start up. + * We may want to revisit this. + */ +static int compact_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct scoutfs_net_compact_response *cresp = NULL; + struct compact_request *cr = NULL; + bool level0_was_full = false; + bool add_ents = false; + bool del_ents = false; + bool rem_segnos = false; struct commit_waiter cw; - bool level0_was_full; + __le64 id; int ret; + if (error) { + /* an error response without an id is fatal */ + if (resp_len != sizeof(__le64)) { + ret = -EINVAL; + goto out; + } + + memcpy(&id, resp, resp_len); + + } else { + if (resp_len != sizeof(struct scoutfs_net_compact_response)) { + ret = -EINVAL; + goto out; + } + + cresp = resp; + id = cresp->id; + } + + trace_scoutfs_server_compact_response(sb, le64_to_cpu(id), error); + + /* XXX we only free tracked requests on responses, must still exist */ + cr = compact_request_done(sb, le64_to_cpu(id)); + if (WARN_ON_ONCE(cr == NULL)) { + ret = -ENOENT; + goto out; + } + down_read(&server->commit_rwsem); + scoutfs_manifest_lock(sb); level0_was_full = scoutfs_manifest_level0_full(sb); - ret = scoutfs_compact_commit(sb, curs, list); - if (ret == 0) { - queue_commit_work(server, &cw); - if (level0_was_full && !scoutfs_manifest_level0_full(sb)) - wake_up(&server->compaction_waitq); + if (error) { + ret = 0; + goto cleanup; } - up_read(&server->commit_rwsem); + /* delete old manifest entries */ + ret = del_manifest_entries(sb, cr->req.ents, ARRAY_SIZE(cr->req.ents), + true); + if (ret) + goto cleanup; + add_ents = true; + + /* add new manifest entries */ + ret = add_manifest_entries(sb, cresp->ents, ARRAY_SIZE(cresp->ents), + true); + if (ret) + goto cleanup; + del_ents = true; + + /* free allocated segnos not found in new entries */ + ret = free_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), + cresp->ents, ARRAY_SIZE(cresp->ents), true); + if (ret) + goto cleanup; + rem_segnos = true; + + /* free input segnos not found in new entries */ + ret = free_entry_segnos(sb, cr->req.ents, ARRAY_SIZE(cr->req.ents), + cresp->ents, ARRAY_SIZE(cresp->ents), true); +cleanup: + /* cleanup partial commits on errors */ + if (ret < 0 && rem_segnos) + remove_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), + cresp->ents, ARRAY_SIZE(cresp->ents), false); + if (ret < 0 && del_ents) + del_manifest_entries(sb, cresp->ents, ARRAY_SIZE(cresp->ents), + false); + if (ret < 0 && add_ents) + add_manifest_entries(sb, cr->req.ents, + ARRAY_SIZE(cr->req.ents), false); + + /* free all the allocated output segnos if compaction failed */ + if ((error || ret < 0) && cr != NULL) + free_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), + NULL, 0, false); + + if (ret == 0 && level0_was_full && !scoutfs_manifest_level0_full(sb)) + wake_up(&server->waitq); if (ret == 0) + queue_commit_work(server, &cw); + scoutfs_manifest_unlock(sb); + up_read(&server->commit_rwsem); + + if (cr) { + scoutfs_manifest_compact_done(sb, &cr->req); + kfree(cr); + } + + if (ret == 0) { ret = wait_for_commit(&cw); + if (ret == 0) + try_queue_compact(server); + } - scoutfs_compact_kick(sb); - +out: return ret; } +/* + * The compaction worker executes as the manifest is updated and we see + * that a level has too many segments and clients aren't processing all + * their max number of compaction requests. Only one compaction worker + * executes. + * + * We have the manifest build us a compaction request, find a client to + * send it too, and record it for later completion processing. + * + * The manifest tracks pending compactions and won't use the same + * segments as inputs to multiple compactions. We track the number of + * compactions in flight to each client to keep them balanced. + */ +static void scoutfs_server_compact_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + compact_work); + struct super_block *sb = server->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_net_compact_request *req; + struct compact_request *cr; + struct commit_waiter cw; + int nr_segnos = 0; + u64 node_id; + __le64 id; + int ret; + + trace_scoutfs_server_compact_work_enter(sb, 0, 0); + + cr = kzalloc(sizeof(struct compact_request), GFP_NOFS); + if (!cr) { + ret = -ENOMEM; + goto out; + } + req = &cr->req; + + /* get the input manifest entries */ + ret = scoutfs_manifest_next_compact(sb, req); + if (ret <= 0) + goto out; + + nr_segnos = ret + SCOUTFS_COMPACTION_SEGNO_OVERHEAD; + + /* get the next id and allocate possible output segnos */ + down_read(&server->commit_rwsem); + + spin_lock(&server->lock); + id = super->next_compact_id; + le64_add_cpu(&super->next_compact_id, 1); + spin_unlock(&server->lock); + + ret = alloc_segnos(sb, req->segnos, nr_segnos); + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + if (ret) + goto out; + + /* try to send to a node with capacity, they can disconnect */ +retry: + req->id = id; + node_id = compact_request_start(sb, cr); + if (node_id == 0) { + ret = 0; + goto out; + } + + /* response processing can complete compaction before this returns */ + ret = scoutfs_net_submit_request_node(sb, server->conn, node_id, + SCOUTFS_NET_CMD_COMPACT, + req, sizeof(*req), + compact_response, NULL, NULL); + if (ret < 0) { + cr = compact_request_done(sb, le64_to_cpu(id)); + BUG_ON(cr == NULL); /* must still be there, no node cleanup */ + } + if (ret == -ENOTCONN) + goto retry; + if (ret < 0) + goto out; + + /* cr is now owned by response processing */ + cr = NULL; + ret = 1; + +out: + if (ret <= 0 && cr != NULL) { + scoutfs_manifest_compact_done(sb, req); + + /* don't need to wait for commit when freeing in cleanup */ + down_read(&server->commit_rwsem); + free_segnos(sb, req->segnos, nr_segnos, NULL, 0, false); + up_read(&server->commit_rwsem); + + kfree(cr); + } + + if (ret > 0) + try_queue_compact(server); + + trace_scoutfs_server_compact_work_exit(sb, 0, ret); +} + /* * This relies on the caller having read the current super and advanced * its seq so that it's dirty. This will go away when we communicate @@ -1172,9 +1723,14 @@ static void server_notify_up(struct super_block *sb, if (node_id != 0) { sci->node_id = node_id; + sci->nr_compacts = 0; spin_lock(&server->lock); list_add_tail(&sci->head, &server->clients); + server->nr_clients++; + trace_scoutfs_server_client_up(sb, node_id, server->nr_clients); spin_unlock(&server->lock); + + try_queue_compact(server); } } @@ -1187,8 +1743,14 @@ static void server_notify_down(struct super_block *sb, if (node_id != 0) { spin_lock(&server->lock); - list_del(&sci->head); + list_del_init(&sci->head); + server->nr_clients--; + trace_scoutfs_server_client_down(sb, node_id, + server->nr_clients); spin_unlock(&server->lock); + + forget_client_compacts(sb, sci); + try_queue_compact(server); } else { shutdown_server(server); } @@ -1264,8 +1826,7 @@ static void scoutfs_server_worker(struct work_struct *work) /* start up the server subsystems before accepting */ ret = scoutfs_btree_setup(sb) ?: - scoutfs_manifest_setup(sb) ?: - scoutfs_compact_setup(sb); + scoutfs_manifest_setup(sb); if (ret) goto shutdown; @@ -1275,6 +1836,7 @@ static void scoutfs_server_worker(struct work_struct *work) scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); /* start accepting connections and processing work */ + server->conn = conn; scoutfs_net_listen(sb, conn); /* wait for listening down or umount, conn can still be live */ @@ -1285,13 +1847,12 @@ static void scoutfs_server_worker(struct work_struct *work) shutdown: /* wait for request processing */ scoutfs_net_shutdown(sb, conn); + /* drain compact work queued by responses */ + cancel_work_sync(&server->compact_work); /* wait for commit queued by request processing */ flush_work(&server->commit_work); + server->conn = NULL; - /* shut down all the server subsystems */ - scoutfs_compact_destroy(sb); - /* (wait for possible double commit work queued by compaction) */ - flush_work(&server->commit_work); destroy_pending_frees(sb); scoutfs_manifest_destroy(sb); scoutfs_btree_destroy(sb); @@ -1325,19 +1886,22 @@ int scoutfs_server_setup(struct super_block *sb) server->sb = sb; spin_lock_init(&server->lock); + init_waitqueue_head(&server->waitq); init_completion(&server->shutdown_comp); server->bind_warned = false; INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); init_rwsem(&server->commit_rwsem); init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); - init_waitqueue_head(&server->compaction_waitq); seqcount_init(&server->stable_seqcount); spin_lock_init(&server->seq_lock); INIT_LIST_HEAD(&server->pending_seqs); init_rwsem(&server->alloc_rwsem); INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); + server->compacts_per_client = 2; + INIT_LIST_HEAD(&server->compacts); + INIT_WORK(&server->compact_work, scoutfs_server_compact_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/server.h b/kmod/src/server.h index 185f6c0b..365469b1 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -53,11 +53,6 @@ void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment); -int scoutfs_client_get_compaction(struct super_block *sb, void *curs); -int scoutfs_client_finish_compaction(struct super_block *sb, void *curs, - void *list); -int scoutfs_server_free_segno(struct super_block *sb, u64 segno); - int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); From 7e9d40d65a830ee044417dbdfb2c4a313a62160f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Sep 2018 15:37:45 -0700 Subject: [PATCH 670/920] scoutfs: init ret when freeing zero extents The server forgot to initialize ret to 0 and might return undefined errnos if a client asked it to free zero extents. Signed-off-by: Zach Brown --- kmod/src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 556229b4..b2e006f2 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -746,7 +746,7 @@ static int server_free_extents(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct scoutfs_net_extent_list *nexl; struct commit_waiter cw; - int ret; + int ret = 0; int err; u64 i; From 56161750418648f060389fe6d632ef01cd1a7050 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Sep 2018 15:06:15 -0700 Subject: [PATCH 671/920] scoutfs: update rpm building infrastructure Update the makefile and spec to our current method of building rpms. Signed-off-by: Zach Brown --- kmod/.gitignore | 1 + kmod/Makefile | 41 ++++++-------------------- kmod/build_rpms.sh | 18 ------------ kmod/scoutfs-kmod.spec.in | 60 ++++++++++++++++++++++----------------- 4 files changed, 43 insertions(+), 77 deletions(-) delete mode 100755 kmod/build_rpms.sh diff --git a/kmod/.gitignore b/kmod/.gitignore index a1fd5abc..ad8b4ca1 100644 --- a/kmod/.gitignore +++ b/kmod/.gitignore @@ -13,3 +13,4 @@ rpmbuild/ rpms/ scoutfs-*.git*/ +scoutfs-kmod-*.tar diff --git a/kmod/Makefile b/kmod/Makefile index c4a40dc6..e6d5c979 100644 --- a/kmod/Makefile +++ b/kmod/Makefile @@ -14,7 +14,7 @@ endif SCOUTFS_GIT_DESCRIBE := \ $(shell git describe --all --abbrev=6 --long 2>/dev/null || \ - echo not-in-a-git-repository) + echo no-git) SCOUTFS_FORMAT_HASH := \ $(shell cat src/format.h src/ioctl.h | md5sum | cut -b1-16) @@ -24,14 +24,11 @@ SCOUTFS_ARGS := SCOUTFS_GIT_DESCRIBE=$(SCOUTFS_GIT_DESCRIBE) \ CONFIG_SCOUTFS_FS=m -C $(SK_KSRC) M=$(CURDIR)/src \ EXTRA_CFLAGS="-Werror" -# move damage locally, this will also help make it easier to cleanup after the build -RPM_DIR = $(shell pwd)/rpmbuild - # - We use the git describe from tags to set up the RPM versioning RPM_VERSION := $(shell git describe --long --tags | awk -F '-' '{gsub(/^v/,""); print $$1}') -RPM_RELEASE := $(shell git describe --long --tags | awk -F '-' '{print $$2"."$$3}') -FULL_VERSION := $(RPM_VERSION).$(RPM_RELEASE) -TARFILE = $(RPM_DIR)/SOURCES/scoutfs-kmod-$(FULL_VERSION).tar +RPM_GITHASH := $(shell git rev-parse --short HEAD) +TARFILE = scoutfs-kmod-$(RPM_VERSION).tar + .PHONY: .FORCE @@ -46,37 +43,15 @@ modules_install: make $(SCOUTFS_ARGS) modules_install -# remake this each time.. -$(RPM_DIR): .FORCE - @echo "## Clean up on isle $(RPM_DIR)..." - rm -frv $(RPM_DIR)/{BUILD,RPMS,SOURCES,SPECS,SRPMS} - mkdir -p $(RPM_DIR)/{BUILD,RPMS,SOURCES,SPECS,SRPMS} - - %.spec: %.spec.in .FORCE sed -e 's/@@VERSION@@/$(RPM_VERSION)/g' \ - -e s'/@@TAR_VERSION@@/$(FULL_VERSION)/g' \ - -e s'/@@RELEASE@@/$(RPM_RELEASE)/g' < $< > $@+ + -e 's/@@GITHASH@@/$(RPM_GITHASH)/g' < $< > $@+ mv $@+ $@ -# NOTE: Both tar & rpm are capable of being built natively on Linux, provided -# you have a local install of rpmbuild.sh for the rpm target. -# Normal exection is to use docker, as that pulls our canned image and tooling for the user. -# ./indocker.sh make tar -# ./indocker.sh make rpm -# -tar: $(RPM_DIR) scoutfs-kmod.spec - git archive --format=tar --prefix scoutfs-$(FULL_VERSION)/ HEAD^{tree} > $(TARFILE) - @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-$(FULL_VERSION)/\1@" scoutfs-kmod.spec - gzip -f -9 $(TARFILE) - - -$(TARFILE).gz: tar - -rpm: $(TARFILE).gz scoutfs-kmod.spec - rpmbuild.sh $(TARFILE).gz - +dist: scoutfs-kmod.spec + git archive --format=tar --prefix scoutfs-kmod-$(RPM_VERSION)/ HEAD^{tree} > $(TARFILE) + @ tar rf $(TARFILE) --transform="s@\(.*\)@scoutfs-$(RPM_VERSION)/\1@" scoutfs-kmod.spec clean: make $(SCOUTFS_ARGS) clean diff --git a/kmod/build_rpms.sh b/kmod/build_rpms.sh deleted file mode 100755 index c9d5c479..00000000 --- a/kmod/build_rpms.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -# Build RPMs, populating a directory structure that indicates the OS release. -# NOTE: This expects to run in a CentOS or RHEL environment, preferrably one of the versity -# rpm-build Docker containers. - -OS_RELEASE=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+' /etc/redhat-release) -echo "OS RELEASE: $OS_RELEASE" - -make rpm - -rpm_dist="rpms/$OS_RELEASE" -rm -fvr "$rpm_dist" -mkdir -p "$rpm_dist" - -cp -v rpmbuild/RPMS/x86_64/kmod-scoutfs*.rpm "$rpm_dist/" diff --git a/kmod/scoutfs-kmod.spec.in b/kmod/scoutfs-kmod.spec.in index 72ec10ea..ee1acf54 100644 --- a/kmod/scoutfs-kmod.spec.in +++ b/kmod/scoutfs-kmod.spec.in @@ -1,40 +1,51 @@ %define kmod_name scoutfs -#%%trace +%define kmod_version @@VERSION@@ +%define kmod_git_hash @@GITHASH@@ +%define pkg_date %(date +%%Y%%m%%d) -%define _tar_version @@TAR_VERSION@@ -# official builds set this to 1, we use 0 for internal/dev-test -%{!?_release: %global _release 0} +# take kernel version or default to uname -r +%{!?kversion: %global kversion %(uname -r)} +%global kernel_version %{kversion} + +%global kernel_source() /usr/src/kernels/%{kernel_version}.$(arch) +%global kernel_release() %{kversion} + +%{!?_release: %global _release 0.%{pkg_date}git%{kmod_git_hash}} Name: %{kmod_name} Summary: %{kmod_name} kernel module -Version: @@VERSION@@ -Release: %{_release}.@@RELEASE@@%{?dist} +Version: %{kmod_version} +Release: %{_release}%{?dist} License: GPLv2 Group: System/Kernel -URL: http://versity.com +URL: http://scoutfs.org/ + +BuildRequires: %{kernel_module_package_buildreqs} +BuildRequires: git +BuildRequires: kernel-devel-uname-r = %{kernel_version} +BuildRequires: module-init-tools -BuildRequires: %kernel_module_package_buildreqs ExclusiveArch: x86_64 -# Sources. -Source0: scoutfs-kmod-%{_tar_version}.tar.gz +Source: %{kmod_name}-kmod-%{kmod_version}.tar # Build only for standard kernel variant(s); for debug packages, append "debug" # after "default" (separated by space) %kernel_module_package default - # Disable the building of the debug package(s). %define debug_package %{nil} +%global install_mod_dir extra/%{name} + + %description %{kmod_name} - kernel module %prep +%setup -q -n %{kmod_name}-kmod-%{kmod_version} - -%setup -q -n %{kmod_name}-%{_tar_version} set -- * mkdir source mv "$@" source/ @@ -43,30 +54,27 @@ mkdir obj %build echo "Building for kernel: %{kernel_version} flavors: '%{flavors_to_build}'" -echo "Build var: kmodtool = %{kmodtool}" -echo "Build var: kverrel = %{kverrel}" for flavor in %flavors_to_build; do rm -rf obj/$flavor cp -r source obj/$flavor make SK_KSRC=%{kernel_source $flavor} -C obj/$flavor module done - %install export INSTALL_MOD_PATH=$RPM_BUILD_ROOT -export INSTALL_MOD_DIR=extra/%{name} -for flavor in %flavors_to_build ; do - # TODO add Makefile rule - #make SK_KSRC=%{kernel_source $flavor} -C obj/$flavor modules_install - make -C %{kernel_source $flavor} modules_install \ - M=$PWD/obj/$flavor/src +export INSTALL_MOD_DIR=%{install_mod_dir} +mkdir -p %{install_mod_dir} +for flavor in %{flavors_to_build}; do + export KSRC=%{kernel_source $flavor} + export KVERSION=%{kernel_release $KSRC} + install -d $INSTALL_MOD_PATH/lib/modules/$KVERSION/%{install_mod_dir} + cp $PWD/obj/$flavor/src/scoutfs.ko $INSTALL_MOD_PATH/lib/modules/$KVERSION/%{install_mod_dir}/ done +# mark modules executable so that strip-to-file can strip them +find %{buildroot} -type f -name \*.ko -exec %{__chmod} u+x \{\} \; + %clean rm -rf %{buildroot} - -%changelog -* Fri Nov 17 2017 Nic Henke - 1.0 -- Initial version. From f8d1489415e921f8c1c64bbb25f3e63e750f9138 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Sep 2018 15:18:27 -0700 Subject: [PATCH 672/920] scoutfs: add README.md Add a README.md for github. Signed-off-by: Zach Brown --- kmod/README.md | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 kmod/README.md diff --git a/kmod/README.md b/kmod/README.md new file mode 100644 index 00000000..9b72a2e2 --- /dev/null +++ b/kmod/README.md @@ -0,0 +1,153 @@ +# Introduction + +scoutfs is a clustered in-kernel Linux filesystem designed and built +from the ground up to support large archival systems. + +Its key differentiating features are: + + - Integrated consistent indexing to accelerate archival maintenance operations + - Shared LSM index structure to scale metadata rates with storage bandwidth + - Decoupled logical locking from serialized device writes to reduce contention + +It meets best of breed expectations: + + * Fully consistent POSIX semantics between nodes + * Rich metadata to ensure the integrity of metadata references + * Atomic transactions to maintain consistent persistent structures + * First class kernel implementation for high performance and low latency + * Open GPLv2 implementation + +# Current Status + +**Initial Alpha Open Source Release** + +scoutfs is under heavy active development. We're releasing before it's +completely polished to give the community an opportunity to affect the +design and implementation. Nothing is cast in stone. + +The core architectural design elements are in place. Much surrounding +functionality hasn't been implemented. It's appropriate for early +adopters and interested developers, not for production use. + +In that vein, expect significant incompatible changes to both the format +of network messages and persistent structures. To avoid mistakes the +implementation currently calculates a hash of the format and ioctl +header files in the source tree. The kernel module will refuse to mount +a volume created by userspace utilities with a mismatched hash, and it +will refuse to connect to a remote node with a mismatched hash. This +means having to unmount, mkfs, and remount everything across many +functional changes. Once the format is nailed down we'll wire up +forward and back compat machinery and remove this temporary safety +measure. + +The current kernel module is developed against the RHEL/CentOS 7.x +kernel to minimize the friction of developing and testing with partners' +existing infrastructure. Once we're happy with the design we'll shift +development to the upstream kernel while maintaining distro +compatibility branches. + +# Community Mailing List + +Please join us on the open scoutfs-devel@scoutfs.org [mailing list +hosted on Google Groups](https://groups.google.com/a/scoutfs.org/forum/#!forum/scoutfs-devel) +for all discussion of scoutfs. + +# Quick Start + +**This following a very rough example of the procedure to get up and +running, experience will be needed to fill in the gaps. We're happy to +help on the mailing list.** + +__Some software components (pacemaker?) may be packaged seperately by +distributions.__ + +The requirements for running scoutfs on a small cluster are: + + 1. One or more nodes running x86-64 CentOS/RHEL 7.4 (or 7.3) + 2. Access to a single shared block device + 3. IPv4 connectivity between the nodes + +The steps for getting scoutfs mounted and operational are: + + 1. Configure pacemaker clustering and the kernel DLM for locking + 2. Get the kernel module running on the nodes + 3. Make a new filesystem on the device with the userspace utilities + 4. Mount the device on all the nodes + +In this example we run all of these commands on two nodes. The block +device name is the same on all the nodes. The listen= mount option is +given the local IP address of each node. + +1. Configure and Start the DLM + + + ```shell + yum install pcs pacemaker fence-agents-all + firewall-cmd --permanent --add-service=high-availability + firewall-cmd --add-service=high-availability + passwd hacluster + systemctl start pcsd.service + systemctl enable pcsd.service + pcs cluster auth node1 node2 + pcs cluster setup --start --name scoutfs node1 node2 + pcs cluster enable + + yum install dlm + systemctl enable dlm + systemctl start dlm + ``` + +2. Get the Kernel Module and Userspace Binaries + + * Either use snapshot RPMs built from git by Versity: + + ```shell + rpm -i https://scoutfs.s3-us-west-2.amazonaws.com/scoutfs-repo-0.0.1-1.el7_4.noarch.rpm + yum install scoutfs-utils kmod-scoutfs + ``` + + * Or use the binaries built from checked out git repositories: + + ```shell + yum install kernel-devel + git clone git@github.com:versity/scoutfs-kmod-dev.git + make -C scoutfs-kmod-dev module + modprobe libcrc32c + insmod scoutfs-kmod-dev/src/scoutfs.ko + + git clone git@github.com:versity/scoutfs-utils-dev.git + make -C scoutfs-utils-dev + alias scoutfs=$PWD/scoutfs-utils-dev/src/scoutfs + + ``` + +3. Make a New Filesystem (**destroys contents, no questions asked**) + + ```shell + scoutfs mkfs /dev/shared_block_device + ``` + + +4. Mount the Filesystem + + ```shell + mkdir /mnt/scoutfs + mount -t scoutfs -o cluster=scoutfs,listen=node_ip_address \ + /dev/shared_block_device /mnt/scoutfs + + ``` + +5. For Kicks, Observe the Metadata Change Index + + The `meta_seq` index tracks the inodes that are changed in each + transaction. + + ```shell + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/one; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/two; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + touch /mnt/scoutfs/one; sync + scoutfs walk-inodes meta_seq 0 -1 /mnt/scoutfs + ``` From 9bb0c60c63c48276efa60b2786a6a3cc0380d585 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Mon, 17 Sep 2018 13:43:57 -0700 Subject: [PATCH 673/920] README: add whitepaper link The white paper is helpful and not linked from the Github README which will be a primary landing spot for folks discovering the project. --- kmod/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/README.md b/kmod/README.md index 9b72a2e2..7cb77e01 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -16,6 +16,8 @@ It meets best of breed expectations: * Atomic transactions to maintain consistent persistent structures * First class kernel implementation for high performance and low latency * Open GPLv2 implementation + +Learn more in the [white paper](https://docs.wixstatic.com/ugd/aaa89b_88a5cc84be0b4d1a90f60d8900834d28.pdf). # Current Status From 91d190622df574fd29a62c439bed3208037abeed Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 25 Sep 2018 12:58:36 -0700 Subject: [PATCH 674/920] scoutfs: remove scoutfs.md file The current plan is to maintain a nice paper describing the system in the scoutfs-utils repository. Signed-off-by: Zach Brown --- kmod/Documentation/scoutfs.md | 354 ---------------------------------- 1 file changed, 354 deletions(-) delete mode 100644 kmod/Documentation/scoutfs.md diff --git a/kmod/Documentation/scoutfs.md b/kmod/Documentation/scoutfs.md deleted file mode 100644 index e4387e7a..00000000 --- a/kmod/Documentation/scoutfs.md +++ /dev/null @@ -1,354 +0,0 @@ - -# scoutfs Engineering Compendium - ------ - -## Document Overview - -This document is intended to be a relatively unstructured but thorough -coverage of the design, implementation, and deployment of scoutfs. - -*Not Yet Discussed: repair, dump/restore, remote namespace -synchronization, compression, encryption, trim, dedup, hole punching, -SMR, iops v. bw, range locking, sorting keys by type/inode, enospc, -compaction priority, manifest server, manifest network protocol, inode -allocation, clustered open-unlink, seq queries, offline data, LSM, -forward/back compat.* - -## Raison D'être - -scoutfs is an archival posix file system. It's built to provide a posix -interface to petabytes of data in trillions of files through thousands -of nodes. - -scoutfs uses log-structured merge trees to achieve high operation -throughput with low device command rates. It uses ranged locking to -maintain consistent POSIX semantics amongst clustered nodes with minimum -synchronization overhead. It offers additional metadata indexing and -data residency interfaces for efficiently executing archival policies. -It is deployed on a shared block fabric for high bandwidth and low -latency. - -## Super Block - -The super block is the anchor of all the persistent storage in the block -device. It contains volume-wide configuration information and -references to the current stable versions of persistent data structures -in the rest of the block device. The super block is stored in two 4KB -blocks at a known location at the start of the device. - -To read the current super block both block locations are read. The -valid super block with the most recent sequence number is used. Either -of the super blocks can be corrupt because they're overwritten in place -and a crash during a write could scramble the block. - -Each new version of the super block is written to the block that doesn't -contain the current super block. If this new super block write fails -then the old super block can still be used and no data is lost. - -The super block, and indeed all file system data, doesn't touch a few -blocks at the start of the device to avoid corrupting blocks that are -used by host platforms that store data inside devices to manage them. - -## Inodes - -Inodes are stored in items identified by the inode number. - - key = struct scoutfs_inode_key { - .type = SCOUTFS_INODE_KEY, - .ino, - } - - val = struct scoutfs_inode { - size, nlink, uid, gid, atime, mtime, ..., - } - -The variable length value that stores the item struct gives us dense -inode packing without having to predefine an inode storage size when the -file system is created and gives us a future expansion mechanism that -uses the item length to determine the version of the inode struct that -is written. - -Inode numbers are 64bit and are never re-used. By never re-using inode -numbers we don't need to manage an inode number allocator that would -need to be consistent across nodes. We can grant large ranges of -numbers to mount clients for allocation. Each inode number uniquely -identify the lifetime of a file and avoids having to store a seperate -generation number for each inode number. - -## Extended Attributes - -Extended attributes are stored in items on the inode at the full name of -the attribute. The attribute name is limited to 255 bytes and the -attribute values is limited to 64KB. The max xattr value size is larger -than our max item size so we can store an xattr in multiple items, but -in the common case a single xattr is efficiently stored in a single -item. - - key = struct scoutfs_xattr_key { - .type = SCOUTFS_XATTR_KEY, - .ino, - .name, - struct scoutfs_xattr_key_footer { - .null = '\0', - .part, - } - } - -Storing the null after the attribute name, which can't be found in any -name, lets us accurately locate a given name in the presence of other -names that share partial prefixes. The part identifies each key's -position in the set of keys that make up the large value. Storing the -full name in each key ensures that all the keys that make up an -attribute are stored adjacent to each other. - -Each item's value starts with a header which describes portion of the -attribute value stored in the item. - - val = struct scoutfs_xattr_val_header { - .part_len, - .last_part, - .data, - } - -The result of all this is that operations on xattrs iterate over keys -starting with the name and part 0 and stop when they hit the final part -(or error on corruption if the parts aren't consistent.) - -## Directory Entries - -Directory entry items store the target inode number referred to by a -given entry name in a parent directory. The name is limited to 255 -non-null bytes. The large keys supported by our items let us store -directory entries in items indexed by the full entry name itself. - - key = struct scoutfs_dirent_key { - .type = SCOUTFS_DIRENT_KEY, - .ino, - .name, - } - - val = struct scoutfs_dirent { - .ino, - .readdir_pos, - .type, - } - -These full precision items let us work on each item for a given name -directly rather than scrambling their sorting by storing them at a hash -value of their name. Storing at a hash value not only adds the -complexity of collisions, it critically causes entry lock attempts in a -directory between mounts to be perfectly randomly distributed and -constantly conflicting with each other. Storing and range locking the -directory entries at their full name preserves non-overlapping patterns -between mounts and gives them a chance to efficiently operate on -disjoint sets of names. - -We index the directory entry items by the full name of the entry so -there is no limit imposed on the number of entries in a directory. The -system will run out of blocks to store entries long before the index is -incapable of storing them. - -While we can satisfy lookups with a full precision index, readdir -doesn't use a full precision iterator. It forces us to describe each -entry with a small scalar directory position. We use a separate item -that's indexed by this readdir position instead of the file name. - - key = struct scoutfs_readdir_key { - .type = SCOUTFS_DIRENT_KEY, - .ino, - .readdir_pos, - } - - val = struct scoutfs_dirent { - .ino, - .readdir_pos, - .type, - .name, - } - -The key's position is allocated as each entry is created. This results -in readdir returning entries ordered by creation time. Like inode -numbers, readdir positions are never re-used so that we don't have to -risk contention by maintaining a consistent free position index across -nodes. - -## Directory Entry Link Backrefs - -The third and final item used by each directory entry is an item that is -stored at the target inode instead of in the parent directory. These -backref items can be traversed to find the full paths from the root -inode to all the entries that link to the target inode. - - key = struct scoutfs_link_backref_key { - .type = SCOUTFS_LINK_BACKREF_KEY, - .ino, - .dir_ino, - .name, - } - - /* no value */ - -Iterating over these items for a given target ino yields the parent -dir_ino and full file name of every entry that references the target -inode. The entry items in the parent dir are stored at the full file -name so the only way for us to reference them is with another copy of -the file name, brining the total to three full copies of the name stored -for every directory entry. - -Because we store the full name for these backref items they do not -impose a limit on the number of hard links to an inode. - -## Regular File Data Extents - -scoutfs stores file data in block extents at 4KB granularity. Items -describe the extents of 4KB blocks that map logical file offsets to -physical block extents in the device: - - key = struct scoutfs_extent_key { - .type = SCOUTFS_EXTENT_KEY, - .ino, - .iblock, - .blkno, - .count, - .flags, - } - - /* no value */ - -The flags field indicates the state of the extent, for example it can be -preallocated but unwritten or offline. If the extent is offline then -the blkno is unused and should be zero. - -Checksums of file data are contained in items at the physical block -offset of the checksumed blocks. Each item contains a fixed number of -checksums for a given group of blocks. - - key = struct scoutfs_checksum_key { - .type = SCOUTFS_CHECKSUM_KEY, - .blkno, - } - - val = { - .crcs[8], - } - -The checksum items are keyed by the physical block number instead of the -logical file position so that the checksum items are only written as new -data is written. The checksum items are left alone as the file data -references change: truncate, unlink, hole punching, and cloning don't -have to modify checksum items. - -With these structures in place the file read and write paths in scoutfs -look very much like most other block file systems in Linux. The generic -buffer_head support code is used and our get_blocks callback reads and -writes the extent items that reference block extents. Write and sync -patterns, with the help of delalloc, preallocation, and fallocate, -determine the physical contiguity of extent allocations. Buffered -read-ahead and O_DIRECT reads walk the extent items and build large -efficient bios if the extents are physically contiguous. - -## Allocating Regular File Data Extents - -The primary persistent allocator for blocks on the device uses an -efficient bitmap with a bit for each 1MB segment. File data allocation -wants to track extents at 4KB granularity and also index them by the -size of the free extent, neither of which the segment bitmap allocator -supports. - -We have free extent items that track free block extents in the device at -the finer 4K granularity. There are two keys for each free extent: one -indexed by the block location and one by the size of the free extent. -Modifying a free extent can thus modify three different positions in the -key namespace: the block location, the old size location, and the new -size location. LSM lets us generate and merge these disjoint items -across different mounts efficiently. - -To avoid the prohibitively expensive lock contention of modifying these -items from multiple mounts, we first create groups of free extents and -assign a given mount to a group for the lifetime of its mount. - - key = struct scoutfs_free_extent_loc_key - .type = SCOUTFS_FREE_EXTENT_LOC_KEY, - .group, - .blkno, - .count, - } - - key = struct scoutfs_free_extent_len_key - .type = SCOUTFS_FREE_EXTENT_LEN_KEY, - .group, - .count, - .blkno, - } - -Mounts are responsible for mangement of the free extent items. They're -populated with the result from requests from the manifest server for -free segment blocks. They're consumed as file data is written and -logical extents are allocated. They're repopulated as file data is -truncated and its extents are freed. They're returned to the segment -allocator when they contain aligned 1MB free extents. - -Like all persistent filesystem items, the free extent items are -protected by range locks. In the common case a single mount will be -operating on its group and having all the lock operations satisfied by -range matches. Any mount can modify any group's extents by acquiring -the right locks, but this should be limited to rare attempts to -defragment or migrate free extents between groups. - -The manifest server is responsible for tracking the assigment of mounts -to groups as mounts come and go through clean mounts and unclean crashes -and recovery. Free extents can get stranded in groups that don't have -an assigned mount. A mount scrambling to find free space in other -groups would need a mechanism to discover other groups, perhaps with a -set of keys that record the presence of extents in each group. - -## Indexing Inodes by Modification Time - -As files are modified archival agents need to find these modified files -so that the archive can be updated. As inode counts explode it becomes -infeasible to scan the entire inode population and meet archival -deadlines. - -scoutfs maintains an index of inodes by modification time. An ioctl is -offered which iterates over the inodes in the order that they were -modified. The ioctl takes a timespec cursor from which to walk. It -fills a buffer with inodes and the time they were modified, sorted by -time. - -The ioctl results are inherently racey. There's nothing to stop an -inode from being modified and moved in the index between when the call -returns and the caller operates on the inode. - -This index is maintained by having time fields in the inode and -modification time items at those time values. The item key sorts the -items by time for the ioctl to iterate over. The items have no value. - - .type = SCOUTFS_MODTIME_KEY, - .ino = inode, - .ts.tv_sec = seconds, - .ts.tv_nsec = nanoseconds, - -As inodes are modified deletion items are created for the old time and -new items are inserted. LSM's ability to let us create items without -strictly locking their key value keeps these items from creating -unacceptable lock contention. If the modifying task has sufficient -locking on the inode it can modify these items and LSM will eventually -merge them into place. - -The index is keyed on real world time so that we don't have to create -our own consistent advancing clock. The clock only needs to be as -accurate as the users of the index require (this often doesn't add -unreasonable requirements, it's often already the case that arhicval -policies involve time and motivate a reasonably synchronized clock -across the cluster.) - -As inodes are deleted their modification items are deleted. - -> *XXX Need to figure out how to resolve multiple items created by -> concurrent writers. We want concurrent parallel writers, say, and -> they'll all way to create their own items at their write times. We'd -> need to be able to find those to delete them during future -> modification or deletion. Sort of sounds like we want -> per-node-identity backrefs for each to maintain and to purge as nodes -> leave the cluster. From 8fedfef1ccb6fc467fa0399d86ec2eb797f35a93 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 31 Oct 2018 12:23:35 -0700 Subject: [PATCH 675/920] scoutfs: remove stale net response data comment There was a time when responding with an error wouldn't include the caller's data payload. That hasn't been the case since we added compaction network requests which include a reference to the compaction operation with the error response. Signed-off-by: Zach Brown --- kmod/src/net.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index da97665e..d67c2db1 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1326,9 +1326,6 @@ int scoutfs_net_submit_request_node(struct super_block *sb, * Send a response. Responses don't get callbacks and use the request's * id so caller's don't need to get an id in return. * - * The data payload is ignored if an error is sent so that callers have - * simple processing exit paths. - * * An error is returned if the response could not be sent. */ int scoutfs_net_response(struct super_block *sb, From e9f6e79d6725dde5f3eec238725e0ebc178365fb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 25 Mar 2019 13:53:16 -0700 Subject: [PATCH 676/920] scoutfs: add uniq_name mount option Each mount is getting a specified unique name. This can be used to identify a reconnecting mount that indicates that an old instance of the same unique name can no longer exist and doesn't need to be fenced. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/options.c | 12 ++++++++++++ kmod/src/options.h | 2 ++ 3 files changed, 15 insertions(+) diff --git a/kmod/src/format.h b/kmod/src/format.h index d7736a38..7c8589bb 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -349,6 +349,7 @@ struct scoutfs_betimespec { #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) #define SCOUTFS_UUID_BYTES 16 +#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ /* XXX ipv6 */ struct scoutfs_inet_addr { diff --git a/kmod/src/options.c b/kmod/src/options.c index f4b7e472..2f1bef8d 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -29,6 +29,7 @@ static const match_table_t tokens = { {Opt_listen, "listen=%s"}, {Opt_cluster, "cluster=%s"}, + {Opt_uniq_name, "uniq_name=%s"}, {Opt_err, NULL} }; @@ -86,6 +87,12 @@ int scoutfs_parse_options(struct super_block *sb, char *options, match_strlcpy(parsed->cluster_name, args, MAX_CLUSTER_NAME_LEN); break; + case Opt_uniq_name: + len = match_strlcpy(parsed->uniq_name, args, + SCOUTFS_UNIQUE_NAME_MAX_BYTES); + if (len == 0 || len > SCOUTFS_UNIQUE_NAME_MAX_BYTES) + return -EINVAL; + break; default: scoutfs_err(sb, "Unknown or malformed option, \"%s\"\n", p); @@ -93,6 +100,11 @@ int scoutfs_parse_options(struct super_block *sb, char *options, } } + if (parsed->uniq_name[0] == '\0') { + scoutfs_err(sb, "must provide a uniq_name option"); + return -EINVAL; + } + return 0; } diff --git a/kmod/src/options.h b/kmod/src/options.h index 0c038c92..a1fffacf 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -12,6 +12,7 @@ enum { * the number of items in each block as though the blocks were tiny. */ Opt_btree_force_tiny_blocks, + Opt_uniq_name, Opt_err, }; @@ -20,6 +21,7 @@ struct mount_options { struct scoutfs_inet_addr listen_addr; char cluster_name[MAX_CLUSTER_NAME_LEN]; + char uniq_name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; }; int scoutfs_parse_options(struct super_block *sb, char *options, From 6caa87458b34b08c444b3e215aa1af5c53cf3e85 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Nov 2018 15:40:22 -0800 Subject: [PATCH 677/920] scoutfs: add scoutfs_net_client_node_id() Some upcoming network request processing paths need access to the connected client's node_id. We could add it to the arguments but that'd be a lot of churn so we'll add an accessor function for now. Signed-off-by: Zach Brown --- kmod/src/net.c | 11 +++++++++++ kmod/src/net.h | 1 + 2 files changed, 12 insertions(+) diff --git a/kmod/src/net.c b/kmod/src/net.c index d67c2db1..a217e19c 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1154,6 +1154,17 @@ scoutfs_net_alloc_conn(struct super_block *sb, return conn; } +/* + * Give the caller the client node_id of the connection. This used by + * rare server processing callers who want to send async responses after + * request processing has returned. We didn't want to plumb the + * requesting node_id into all the request handlers but that'd work too. + */ +u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn) +{ + return conn->node_id; +} + /* * Shutdown the connection. Once this returns no network traffic * or work will be executing. The caller can then connect or bind and diff --git a/kmod/src/net.h b/kmod/src/net.h index f318b2bf..cbbd29bc 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -28,6 +28,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, size_t info_size, scoutfs_net_request_t *req_funcs, char *name_suffix); +u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn); int scoutfs_net_connect(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin, unsigned long timeout_ms); From f75e1e132292e82b9228ab46866bbb127abff92b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Jan 2019 10:40:54 -0800 Subject: [PATCH 678/920] scoutfs: reformat Makefile to one object per line Reformat the scoutfs-y object list so that there's one object per line. Diffs now clearly demonstrate what is changing instead of having word wrapping constantly obscuring changes in the built objects. (Did everyone spot the scoutfs_trace sorting mistake? Another reason not to mash everything into wrapped lines :)). Signed-off-by: Zach Brown --- kmod/src/Makefile | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f804c121..24cf93fe 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -5,11 +5,38 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += bio.o block.o btree.o client.o compact.o counters.o data.o dir.o \ - export.o extents.o file.o inode.o ioctl.o item.o lock.o \ - manifest.o msg.o net.o options.o per_task.o seg.o server.o \ - scoutfs_trace.o sort_priv.o spbm.o super.o sysfs.o trans.o \ - triggers.o tseq.o xattr.o +scoutfs-y += \ + bio.o \ + block.o \ + btree.o \ + client.o \ + compact.o \ + counters.o \ + data.o \ + dir.o \ + export.o \ + extents.o \ + file.o \ + inode.o \ + ioctl.o \ + item.o \ + lock.o \ + manifest.o \ + msg.o \ + net.o \ + options.o \ + per_task.o \ + scoutfs_trace.o \ + seg.o \ + server.o \ + sort_priv.o \ + spbm.o \ + super.o \ + sysfs.o \ + trans.o \ + triggers.o \ + tseq.o \ + xattr.o # # The raw types aren't available in userspace headers. Make sure all From d57b8232eefadd2a783981da0edee73b70b22e06 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 25 Mar 2019 14:56:55 -0700 Subject: [PATCH 679/920] scoutfs: move base types in format.h We had scattered some base types throughout the format file which made them annoying to reference in higher level structs. Let's put them at the top so we can use them without declarations or moving things around in unrelated commits. Signed-off-by: Zach Brown --- kmod/src/format.h | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 7c8589bb..f577b4bf 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -38,6 +38,25 @@ */ #define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +/* + * Base types used by other structures. + */ +struct scoutfs_timespec { + __le64 sec; + __le32 nsec; +} __packed; + +struct scoutfs_betimespec { + __be64 sec; + __be32 nsec; +} __packed; + +/* XXX ipv6 */ +struct scoutfs_inet_addr { + __le32 addr; + __le16 port; +} __packed; + /* * This header is stored at the start of btree blocks and the super * block for verification. The crc is calculated by zeroing the crc and @@ -340,10 +359,6 @@ struct scoutfs_xattr { __u8 name[0]; } __packed; -struct scoutfs_betimespec { - __be64 sec; - __be32 nsec; -} __packed; /* XXX does this exist upstream somewhere? */ #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) @@ -351,12 +366,6 @@ struct scoutfs_betimespec { #define SCOUTFS_UUID_BYTES 16 #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ -/* XXX ipv6 */ -struct scoutfs_inet_addr { - __le32 addr; - __le16 port; -} __packed; - struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -378,10 +387,6 @@ struct scoutfs_super_block { #define SCOUTFS_ROOT_INO 1 -struct scoutfs_timespec { - __le64 sec; - __le32 nsec; -} __packed; /* * @meta_seq: advanced the first time an inode is updated in a given From c34dd452a715705583849a687ef792c4a84460b5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 25 Mar 2019 14:57:19 -0700 Subject: [PATCH 680/920] scoutfs: add quorum voting Add a quorum election implementation. The mounts that can participate in the election are specified in a quorum config array in the super block. Each configured participant is assigned a preallocated block that it can write to. All mounts read the quorum blocks to find the member who was elected the leader and should be running the server. The voting mounts loop reading voting blocks and writing their vote block until someone is elected with a amjority. Nothing calls this code yet, this adds the initial implementation and format. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/counters.h | 12 + kmod/src/format.h | 68 ++++ kmod/src/quorum.c | 718 +++++++++++++++++++++++++++++++++++++++ kmod/src/quorum.h | 19 ++ kmod/src/scoutfs_trace.h | 44 +++ 6 files changed, 862 insertions(+) create mode 100644 kmod/src/quorum.c create mode 100644 kmod/src/quorum.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 24cf93fe..fd79053d 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -26,6 +26,7 @@ scoutfs-y += \ net.o \ options.o \ per_task.o \ + quorum.o \ scoutfs_trace.o \ seg.o \ server.o \ diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 82bead6b..da993bc8 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -111,6 +111,18 @@ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ EXPAND_COUNTER(net_unknown_request) \ + EXPAND_COUNTER(quorum_elected) \ + EXPAND_COUNTER(quorum_election_error) \ + EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(quorum_found_leader) \ + EXPAND_COUNTER(quorum_no_leader) \ + EXPAND_COUNTER(quorum_read_block) \ + EXPAND_COUNTER(quorum_read_block_error) \ + EXPAND_COUNTER(quorum_read_invalid_block) \ + EXPAND_COUNTER(quorum_read_invalid_config) \ + EXPAND_COUNTER(quorum_waited) \ + EXPAND_COUNTER(quorum_write_block) \ + EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index f577b4bf..6d1bd222 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -38,6 +38,14 @@ */ #define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +/* + * A reasonably large region of aligned quorum blocks follow the super + * block. + */ +#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS + /* * Base types used by other structures. */ @@ -366,6 +374,65 @@ struct scoutfs_xattr { #define SCOUTFS_UUID_BYTES 16 #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ +/* + * During each quorum voting interval the fabric has to process 2 reads + * and a write for each voting mount. The only reason we limit the + * number of active quorum mounts is to limit the number of IOs per + * interval. We use a pretty conservative interval given that IOs will + * generally be faster than our constant and we'll have fewer active + * than the max. + */ +#define SCOUTFS_QUORUM_MAX_ACTIVE 7 +#define SCOUTFS_QUORUM_IO_LATENCY_MS 10 +#define SCOUTFS_QUORUM_INTERVAL_MS \ + (SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS) + +/* + * Each mount that is found in the quorum config in the super block can + * write to quorum blocks indicating which mount they vote for as + * the leader. + * + * @config_gen: references the config gen in the super block + * @write_nr: incremented for every write, only 0 when never written + * @elected_nr: incremented when elected, 0 otherwise + * @vote_slot: the active config slot that the writer is voting for + */ +struct scoutfs_quorum_block { + __le64 fsid; + __le64 blkno; + __le64 config_gen; + __le64 write_nr; + __le64 elected_nr; + __le32 crc; + __u8 vote_slot; +} __packed; + +#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS + +/* + * Each quorum voter is described by a slot which corresponds to the + * block that the voter will write to. + * + * The stale flag is used to support config migration. A new + * configuration is written in free slots and the old configuration is + * marked stale. Stale slots can only be reclaimed once we have + * evidence that the named mount won't try and write to it by seeing it + * write to other slots or connect with the new gen. + */ +struct scoutfs_quorum_config { + __le64 gen; + struct scoutfs_quorum_slot { + __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + struct scoutfs_inet_addr addr; + __u8 vote_priority; + __u8 flags; + } __packed slots[SCOUTFS_QUORUM_MAX_SLOTS]; +} __packed; + +#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0) +#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1) +#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2) + struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; @@ -383,6 +450,7 @@ struct scoutfs_super_block { struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_inet_addr server_addr; + struct scoutfs_quorum_config quorum_config; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c new file mode 100644 index 00000000..112e2da9 --- /dev/null +++ b/kmod/src/quorum.c @@ -0,0 +1,718 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "format.h" +#include "msg.h" +#include "counters.h" +#include "quorum.h" +#include "server.h" +#include "net.h" +#include "scoutfs_trace.h" + +/* + * scoutfs mounts use a region of statically allocated blocks in the + * shared metadata device to elect a leader mount who runs the server + * that the rest of the mounts of the filesystem connect to. + * + * Mounts that should participate in the election are configured in an + * array in the super block. Their position in the array determines the + * preallocated block that they'll be writing to. Mounts that aren't + * participating in the election only read the blocks to discover the + * outcome of the election. + * + * During the election each participating mount reads all the quorum + * blocks that all the mounts wrote, sees which are active and chooses + * which to vote for, and writes a new version of their block that + * includes their vote. Mounts vote for the mount with the highest + * priority in the config that is seen actively writing voting blocks + * over time. + * + * Once a mount receives a majority of votes from its peers then it + * writes its block with an indication that it has been elected. Only + * after reading that block, and seeing no other blocks that indicate + * more recently elected leaders, will it consider itself elected and + * try to fence any other previously elected leaders before starting the + * server. This ensures that racing elected leaders will always result + * in fencing all but the most recent. + * + * XXX: + * - actually fence + * - add temporary priority for choosing a specific mount as a leader + * - add config rotation (write new config, reclaim stale slots) + */ + +static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) +{ + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); + sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); +} + +/* active slots are sorted to the front for validation */ +static int cmp_slot_active(const struct scoutfs_quorum_slot *a, + const struct scoutfs_quorum_slot *b) +{ + int a_active = !!(a->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); + int b_active = !!(b->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); + + return b_active - a_active; +} + +/* slot validation has ensured that the names are null terminated */ +static int cmp_slot_names(const void *A, const void *B) +{ + const struct scoutfs_quorum_slot *a = A; + const struct scoutfs_quorum_slot *b = B; + + return cmp_slot_active(a, b) ?: + strcmp(a->name, b->name); +} + +static int cmp_slot_addrs(const void *A, const void *B) +{ + const struct scoutfs_quorum_slot *a = A; + const struct scoutfs_quorum_slot *b = B; + + return cmp_slot_active(a, b) ?: + memcmp(&a->addr, &b->addr, sizeof(a->addr)); +} + +static void swap_slots(void *A, void *B, int size) +{ + struct scoutfs_quorum_slot *a = A; + struct scoutfs_quorum_slot *b = B; + + swap(*a, *b); +} + +/* + * We'll set the callers our_slot to the slot that contains the their. + * If the name isn't found then it'll be set to -1. + */ +static int read_quorum_config(struct super_block *sb, + struct scoutfs_super_block *super, + char *our_name, int *our_slot_ret, + int *nr_active_ret) +{ + struct scoutfs_quorum_slot *sorted = NULL; + struct scoutfs_quorum_slot *slot; + struct scoutfs_quorum_config *conf; + struct sockaddr_in sin; + int nr_active = 0; + int our_slot = -1; + int ret; + int i; + + sorted = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), GFP_NOFS); + if (sorted == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + conf = &super->quorum_config; + + ret = -EINVAL; + + if (conf->gen == 0) { + scoutfs_err(sb, "invalid zero quorum config gen"); + goto out; + } + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + slot = &conf->slots[i]; + + if (slot->flags & SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN) { + scoutfs_err(sb, "quorum slot ind %u unknown flags 0x%02x", + i, slot->flags); + goto out; + } + + if ((slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) && + (slot->flags & SCOUTFS_QUORUM_SLOT_STALE)) { + scoutfs_err(sb, "quorum slot ind %u is both active and stale", + i); + goto out; + } + + if (!(slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE)) + continue; + + nr_active++; + + if (slot->name[0] == '\0') { + scoutfs_err(sb, "quorum slot ind %u name is null", i); + goto out; + } + + if (slot->name[SCOUTFS_UNIQUE_NAME_MAX_BYTES - 1] != '\0') { + scoutfs_err(sb, "quorum slot ind %u name isn't null terminated", + i); + goto out; + } + + if (our_name && strcmp(our_name, slot->name) == 0) + our_slot = i; + + addr_to_sin(&sin, &slot->addr); + + if (ipv4_is_multicast(sin.sin_addr.s_addr) || + ipv4_is_lbcast(sin.sin_addr.s_addr) || + ipv4_is_zeronet(sin.sin_addr.s_addr) || + ipv4_is_local_multicast(sin.sin_addr.s_addr) || + ntohs(sin.sin_port) == 0 || + ntohs(sin.sin_port) == U16_MAX) { + scoutfs_err(sb, "quorum slot ind %u has invalid addr %pIS:%u", + i, &sin, ntohs(sin.sin_port)); + goto out; + } + } + + if (nr_active == 0) { + scoutfs_err(sb, "quorum config has no active slots"); + goto out; + } + + if (nr_active > SCOUTFS_QUORUM_MAX_ACTIVE) { + scoutfs_err(sb, "quorum config has %u active slots, can have at most %u ", + nr_active, SCOUTFS_QUORUM_MAX_ACTIVE); + goto out; + } + + memcpy(sorted, conf->slots, + SCOUTFS_QUORUM_MAX_SLOTS * sizeof(sorted[0])); + + sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), + cmp_slot_names, swap_slots); + + for (i = 1; i < nr_active; i++) { + if (strcmp(sorted[i].name, sorted[i - 1].name) == 0) { + scoutfs_err(sb, "multiple quorum slots have the same name '%s'", + sorted[i].name); + goto out; + } + } + + sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), + cmp_slot_addrs, swap_slots); + + for (i = 1; i < nr_active; i++) { + if (memcmp(&sorted[i].addr, &sorted[i - 1].addr, + sizeof(sorted[i].addr)) == 0) { + addr_to_sin(&sin, &sorted[i].addr); + scoutfs_err(sb, "multiple quorum slots have the same address %pIS:%u", + &sin, ntohs(sin.sin_port)); + goto out; + } + } + + ret = 0; + if (our_slot_ret) + *our_slot_ret = our_slot; + if (nr_active_ret) + *nr_active_ret = nr_active; +out: + if (ret) + scoutfs_inc_counter(sb, quorum_read_invalid_config); + kfree(sorted); + return ret; +} + +/* + * The caller is about to read the current version of a set of quorum + * blocks. We invalidate all the quorum blocks in the cache and + * populate the cache with all the blocks with one large contiguous + * read. The caller then uses simple sync bh methods to access + * whichever blocks it needs. I'm not a huge fan of the plug but I + * couldn't get the individual readahead requests merged without it. + */ +static void readahead_quorum_blocks(struct super_block *sb) +{ + struct buffer_head *bh; + struct blk_plug plug; + int i; + + blk_start_plug(&plug); + + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + i); + if (!bh) + continue; + + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + + ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); + brelse(bh); + } + + blk_finish_plug(&plug); +} + +/* + * Callers don't mind us clobbering the crc temporarily. + */ +static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) +{ + __le32 calc_crc; + __le32 blk_crc; + + blk_crc = blk->crc; + blk->crc = 0; + calc_crc = cpu_to_le32(crc32c(~0, blk, sizeof(*blk))); + blk->crc = blk_crc; + + return calc_crc; +} + +static bool invalid_quorum_block(struct scoutfs_super_block *super, + struct buffer_head *bh, + struct scoutfs_quorum_block *blk) +{ + return quorum_block_crc(blk) != blk->crc || + blk->fsid != super->hdr.fsid || + le64_to_cpu(blk->blkno) != bh->b_blocknr || + blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS; +} + +/* + * Give the caller the most recently updated version of the quorum + * block. Returns 0 and fills the callers block struct on success. + * Returns -ENOENT and zeros the caller's block if we couldn't read a + * valid block. We don't consider the config gen, that's up to the + * caller. + */ +static int read_quorum_block(struct super_block *sb, + struct scoutfs_super_block *super, int slot, + struct scoutfs_quorum_block *blk_ret) +{ + struct scoutfs_quorum_block *blk; + struct buffer_head *bh; + int ret; + + /* code strongly assumes that slots and blocks are directly mapped */ + BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS != SCOUTFS_QUORUM_MAX_SLOTS); + + ret = -ENOENT; + + bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + slot); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + goto out; + } + blk = (void *)(bh->b_data); + + /* ignore unwritten blocks */ + if (blk->write_nr == 0) + goto out; + + trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + + if (invalid_quorum_block(super, bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + goto out; + } + + *blk_ret = *blk; + scoutfs_inc_counter(sb, quorum_read_block); + ret = 0; +out: + if (ret < 0) + memset(blk_ret, 0, sizeof(struct scoutfs_quorum_block)); + brelse(bh); + return ret; +} + +/* + * Iterate over config slots from the given index and return the first + * slot that has any of the given flags set. + */ +static inline int first_slot_flags(struct scoutfs_quorum_config *conf, + int i, u8 flags) +{ + for (; i < ARRAY_SIZE(conf->slots); i++) { + if (conf->slots[i].flags & flags) + break; + } + return i; +} + +/* + * Execute the loop body with the read block for each slot that's + * configured and active. If we can't read the block for whatever + * reason then the loop will execute with the blk struct zeroed. + */ +#define for_each_active_block(sb, super, conf, hists, hi, blk, slot, i) \ + for (i = first_slot_flags(conf, 0, SCOUTFS_QUORUM_SLOT_ACTIVE); \ + (i < ARRAY_SIZE(conf->slots)) && \ + (slot = &conf->slots[i], \ + hi = &hists[i], \ + read_quorum_block(sb, super, i, blk), 1); \ + i = first_slot_flags(conf, i + 1, SCOUTFS_QUORUM_SLOT_ACTIVE)) + +/* + * Iterate over every possible block, regardless of config. A lot of these + * will be zero. + */ +#define for_each_block(sb, super, i, blk) \ + for (i = 0; \ + (i < SCOUTFS_QUORUM_BLOCKS) && \ + (read_quorum_block(sb, super, i, blk), 1); \ + i++) + +/* + * Synchronously write a single quorum block. The caller has provided + * the meaningful fields for the write. We fill in the rest that are + * consistent for every write and zero the rest of the block. + */ +static int write_quorum_block(struct super_block *sb, __le64 fsid, + __le64 config_gen, u8 our_slot, __le64 write_nr, + u64 elected_nr, u8 vote_slot) +{ + struct scoutfs_quorum_block *blk; + struct buffer_head *bh; + int ret; + + BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); + + if (WARN_ON_ONCE(our_slot >= SCOUTFS_QUORUM_MAX_SLOTS) || + WARN_ON_ONCE(vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS)) + return -EINVAL; + + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + our_slot); + if (bh == NULL) { + ret = -EIO; + goto out; + } + blk = (void *)bh->b_data; + + blk->fsid = fsid; + blk->blkno = cpu_to_le64(bh->b_blocknr); + blk->config_gen = config_gen; + blk->write_nr = write_nr; + blk->elected_nr = cpu_to_le64(elected_nr); + blk->vote_slot = vote_slot; + + blk->crc = quorum_block_crc(blk); + + lock_buffer(bh); + set_buffer_mapped(bh); + bh->b_end_io = end_buffer_write_sync; + get_bh(bh); + submit_bh(WRITE_SYNC | REQ_META | REQ_PRIO, bh); + + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + else + ret = 0; + + if (ret == 0) { + trace_scoutfs_quorum_write_block(sb, bh->b_blocknr, blk); + scoutfs_inc_counter(sb, quorum_write_block); + } +out: + if (ret) + scoutfs_inc_counter(sb, quorum_write_block_error); + brelse(bh); + return ret; +} + +/* + * The caller read their quorum block which indicated that they were + * elected. We have to fence all other previously elected leaders so + * that we're running the only instance of the server. + * + * Time can pass between all phases of this: reading that we're elected, + * fencing, and writing the quorum block that clears the elected flag of + * those we fenced. + * + * This is always safe because we either have exclusive access to the + * device having fenced others or someone else would have fenced us + * before they write. + */ +static int fence_other_elected(struct super_block *sb, + struct scoutfs_super_block *super, + int our_slot, u64 elected_nr) +{ + struct scoutfs_quorum_config *conf = &super->quorum_config; + struct scoutfs_quorum_block blk; + int ret; + int i; + + for_each_block(sb, super, i, &blk) { + if (i != our_slot && + le64_to_cpu(blk.elected_nr) > 0 && + le64_to_cpu(blk.elected_nr) <= elected_nr) { + scoutfs_err(sb, "would have fenced"); + scoutfs_inc_counter(sb, quorum_fenced); + + ret = write_quorum_block(sb, super->hdr.fsid, + conf->gen, i, blk.write_nr, + 0, i); + if (ret) + break; + } + } + + return ret; +} + +struct quorum_block_history { + __le64 write_nr; + u8 writing; +}; + +/* + * The caller couldn't connect to a server. Read the quorum blocks + * until we see an elected leader and give their address to the caller. + * If we're configured as part of the quorum then we participate in the + * electing by writing our vote to our quorum block. + * + * Voting members read the blocks at regular intervals and update their + * quorum block with their vote for the elected leader. new leader. + * When a mount receives enough votes it marks its vote in the block as + * elected, fences other elected leaders, and returns to the caller who + * starts up the server for others to connect to. + * + * The calling client may have never seen a server before, or could have + * failed to connect to a valid server, or might have tried to connect + * to a dead server. They pass in an existing elected_nr if they want + * us to ignore old servers and they pass in a timeout so that they can + * return to retrying to connect to whatever address we find. + * + * When we return success we update the caller's elected info with the + * most recent elected leader we found, which may well be long gone. We + * return -ENOENT if we didn't find any elected leaders. + */ +int scoutfs_quorum_election(struct super_block *sb, char *our_name, + u64 old_elected_nr, ktime_t timeout_abs, + struct scoutfs_quorum_elected_info *qei) +{ + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_config *conf; + struct scoutfs_quorum_slot *slot; + struct scoutfs_quorum_block blk; + struct quorum_block_history *hist; + struct quorum_block_history *hi; + ktime_t expires; + ktime_t now; + __le64 write_nr = 0; + u64 elected_nr = 0; + int vote_streak = 0; + int vote_slot; + int our_slot; + int vote_prio; + int nr_active; + int nr_votes; + int majority; + int ret; + int i; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + hist = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(hist[0]), GFP_NOFS); + if (!super || !hist) { + ret = -ENOMEM; + goto out; + } + + for (;;) { + now = ktime_get(); + expires = ktime_add_ms(now, SCOUTFS_QUORUM_INTERVAL_MS); + + ret = read_quorum_config(sb, super, our_name, &our_slot, + &nr_active); + if (ret) + goto out; + conf = &super->quorum_config; + + /* allow a single vote majority when 1 or 2 active */ + if (nr_active <= 2) + majority = 1; + else if (nr_active & 1) + majority = (nr_active + 1) / 2; + else + majority = (nr_active / 2) + 1; + + readahead_quorum_blocks(sb); + + /* default to voting for ourselves, but at min prio */ + vote_slot = our_slot; + vote_prio = -1; + memset(qei, 0, sizeof(*qei)); + + for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ + /* determine which mounts are writing */ + if (blk.config_gen == conf->gen && + blk.write_nr != 0 && + blk.write_nr != hi->write_nr) + hi->writing = min(hi->writing + 1, 2); + else + hi->writing = 0; + hi->write_nr = blk.write_nr; + + /* vote for first highest priority writing block */ + if (hi->writing >= 2 && + slot->vote_priority > vote_prio) { + vote_slot = i; + vote_prio = slot->vote_priority; + } + + /* find the most recently elected leader */ + if ((blk.config_gen == conf->gen) && + (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ + addr_to_sin(&qei->sin, &slot->addr); + qei->config_gen = blk.config_gen; + qei->write_nr = blk.write_nr; + qei->elected_nr = le64_to_cpu(blk.elected_nr); + qei->config_slot = i; + } + } + + /* + * After writing a block indicating that we were elected + * we make sure that we can read it and that we're still + * the most recent elected leader. If we are then we + * try to fence. If we can't read it, or we're not the + * most recent, or we couldn't fence, then we fall back + * to participating in the election. + */ + if (elected_nr != 0) { + if (qei->write_nr == write_nr && + qei->elected_nr == elected_nr && + qei->config_slot == our_slot) { + ret = fence_other_elected(sb, super, our_slot, + elected_nr); + if (ret == 0) { + qei->run_server = true; + goto out; + } + + memset(qei, 0, sizeof(*qei)); + } + + vote_streak = 0; + } + + /* return if we found a new leader or ran out of time */ + if (qei->elected_nr > old_elected_nr || + ktime_after(now, timeout_abs)) { + if (qei->elected_nr > 0) { + scoutfs_inc_counter(sb, quorum_found_leader); + ret = 0; + } else { + scoutfs_inc_counter(sb, quorum_no_leader); + ret = -ENOENT; + } + goto out; + } + + /* wait for the next cycle if we're not in the voting config */ + if (our_slot < 0) + continue; + + nr_votes = 0; + write_nr = cpu_to_le64(1); + elected_nr = 0; + + for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ + /* count our votes (maybe including from us) */ + if (hi->writing >= 2 && blk.vote_slot == our_slot) + nr_votes++; + + /* sample existing fields for our write */ + if (i == our_slot) { + write_nr = blk.write_nr; + le64_add_cpu(&write_nr, 1); + } + elected_nr = max(elected_nr, + le64_to_cpu(blk.elected_nr)); + } + + + /* elected after sufficient cycles with a majority vote */ + if (nr_votes >= majority) + vote_streak = min(vote_streak + 1, 2); + else + vote_streak = 0; + + if (vote_streak >= 2) + elected_nr++; + else + elected_nr = 0; + + write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, + write_nr, elected_nr, vote_slot); + + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + scoutfs_inc_counter(sb, quorum_waited); + } + +out: + kfree(super); + kfree(hist); + + if (ret) { + memset(qei, 0, sizeof(*qei)); + scoutfs_inc_counter(sb, quorum_election_error); + } + + return ret; +} + +/* + * The calling server is shutting down and has finished modifying + * persistent state. We clear elected_nr from our quorum block so that + * mounts won't try to connect and so that the next next leader won't + * try to fence. + * + * By definition nothing has written to the slot since we wrote our + * elected_nr and the slot could not have been reclaimed. To reclaim + * the slot would have required proving that we were gone or fencing + * us. + * + * If this fails then the mount is in trouble because it'll probably be + * fenced by the next elected leader. + * + * XXX I think there's an interesting race here. If the server is + * running in an old config then the server's slot can be reclaimed if + * the server sees a connection from the current gen. If the server is + * taking a client connection as an indication that the slot won't be + * written then the client needs to shut down the server before trying + * to connect with a new gen. + */ +int scoutfs_quorum_clear_elected(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, + qei->config_slot, qei->write_nr, 0, + qei->config_slot); +} diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h new file mode 100644 index 00000000..82e6708e --- /dev/null +++ b/kmod/src/quorum.h @@ -0,0 +1,19 @@ +#ifndef _SCOUTFS_QUORUM_H_ +#define _SCOUTFS_QUORUM_H_ + +struct scoutfs_quorum_elected_info { + struct sockaddr_in sin; + __le64 config_gen; + __le64 write_nr; + u64 elected_nr; + unsigned int config_slot; + bool run_server; +}; + +int scoutfs_quorum_election(struct super_block *sb, char *our_name, + u64 old_elected_nr, ktime_t timeout_abs, + struct scoutfs_quorum_elected_info *qei); +int scoutfs_quorum_clear_elected(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 132c94f1..3afe9059 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2423,6 +2423,50 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, TP_ARGS(sb, node_id, nr_clients) ); +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, io_blkno, blk), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, io_blkno) + __field(__u64, hdr_blkno) + __field(__u64, config_gen) + __field(__u64, write_nr) + __field(__u64, elected_nr) + __field(__u32, crc) + __field(__u8, vote_slot) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->io_blkno = io_blkno; + __entry->hdr_blkno = le64_to_cpu(blk->blkno); + __entry->config_gen = le64_to_cpu(blk->config_gen); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->elected_nr = le64_to_cpu(blk->elected_nr); + __entry->crc = le32_to_cpu(blk->crc); + __entry->vote_slot = blk->vote_slot; + ), + + TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu crc 0x%08x vote_slot %u", + __entry->fsid, __entry->io_blkno, __entry->hdr_blkno, + __entry->config_gen, __entry->write_nr, __entry->elected_nr, + __entry->crc, __entry->vote_slot) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + TP_ARGS(sb, io_blkno, blk) +); +DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, + TP_PROTO(struct super_block *sb, u64 io_blkno, + struct scoutfs_quorum_block *blk), + TP_ARGS(sb, io_blkno, blk) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From f472c0bc87b388c20300a05b0691bf7cd1d61da7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 9 Jan 2019 16:43:45 -0800 Subject: [PATCH 681/920] scoutfs: add scoutfs_net_response_node() Today all responses can only be sent down the connection that sent the response while the request is being processed. We'll be adding subsystems that need to send responses asynchronously after initial request processing. Give them a call to send a response to a node id instead of to a node's connection. Signed-off-by: Zach Brown --- kmod/src/net.c | 15 +++++++++++++++ kmod/src/net.h | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/kmod/src/net.c b/kmod/src/net.c index a217e19c..14c763a9 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1353,6 +1353,21 @@ int scoutfs_net_response(struct super_block *sb, NULL, NULL, NULL); } +int scoutfs_net_response_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u8 cmd, u64 id, int error, + void *resp, u16 resp_len) +{ + if (error) { + resp = NULL; + resp_len = 0; + } + + return submit_send(sb, conn, node_id, cmd, SCOUTFS_NET_FLAG_RESPONSE, + id, net_err_from_host(sb, error), resp, resp_len, + NULL, NULL, NULL); +} + /* * The response function that was submitted with the request is not * called if the request is canceled here. diff --git a/kmod/src/net.h b/kmod/src/net.h index cbbd29bc..b59db537 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -58,6 +58,10 @@ int scoutfs_net_sync_request(struct super_block *sb, int scoutfs_net_response(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, int error, void *resp, u16 resp_len); +int scoutfs_net_response_node(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u8 cmd, u64 id, int error, + void *resp, u16 resp_len); void scoutfs_net_shutdown(struct super_block *sb, struct scoutfs_net_connection *conn); void scoutfs_net_free_conn(struct super_block *sb, From 34b8950bcaddc95deb9aaf4f994c40656225b9be Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 10 Jan 2019 12:43:48 -0800 Subject: [PATCH 682/920] scoutfs: initial lock server core Add the core lock server code for providing a lock service from our server. The lock messages are wired up but nothing calls them. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/format.h | 37 +++ kmod/src/lock_server.c | 630 +++++++++++++++++++++++++++++++++++++++ kmod/src/lock_server.h | 12 + kmod/src/scoutfs_trace.h | 46 +++ kmod/src/server.c | 52 +++- kmod/src/server.h | 5 + kmod/src/super.h | 2 + 8 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 kmod/src/lock_server.c create mode 100644 kmod/src/lock_server.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index fd79053d..468f6a61 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -21,6 +21,7 @@ scoutfs-y += \ ioctl.o \ item.o \ lock.o \ + lock_server.o \ manifest.o \ msg.o \ net.o \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 6d1bd222..ebce65fd 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -624,6 +624,7 @@ enum { SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, SCOUTFS_NET_CMD_STATFS, SCOUTFS_NET_CMD_COMPACT, + SCOUTFS_NET_CMD_LOCK, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -743,6 +744,42 @@ struct scoutfs_net_compact_response { struct scoutfs_net_manifest_entry ents[SCOUTFS_COMPACTION_MAX_OUTPUT]; } __packed; +struct scoutfs_net_lock { + struct scoutfs_key key; + __u8 old_mode; + __u8 new_mode; +} __packed; + +/* some enums for tracing */ +enum { + SLT_CLIENT, + SLT_SERVER, + SLT_GRANT, + SLT_INVALIDATE, + SLT_REQUEST, + SLT_RESPONSE, +}; + +/* + * Read and write locks operate as you'd expect. Multiple readers can + * hold read locks while writers are excluded. A single writer can hold + * a write lock which excludes other readers and writers. Writers can + * read while holding a write lock. + * + * Multiple writers can hold write only locks but they can not read, + * they can only generate dirty items. It's used when the system has + * other means of knowing that it's safe to overwrite items. + * + * The null mode provides no access and is used to destroy locks. + */ +enum { + SCOUTFS_LOCK_NULL = 0, + SCOUTFS_LOCK_READ, + SCOUTFS_LOCK_WRITE, + SCOUTFS_LOCK_WRITE_ONLY, + SCOUTFS_LOCK_INVALID, +}; + /* * Scoutfs file handle structure - this can be copied out to userspace * via open by handle or put on the wire from NFS. diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c new file mode 100644 index 00000000..bb5f4ed3 --- /dev/null +++ b/kmod/src/lock_server.c @@ -0,0 +1,630 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include + +#include "format.h" +#include "counters.h" +#include "net.h" +#include "tseq.h" +#include "scoutfs_trace.h" +#include "lock_server.h" + +/* + * The scoutfs server implements a simple lock service. Client mounts + * request access to locks identified by a key. The server ensures that + * access mode exclusion is properly enforced. + * + * The server processing paths are implemented in network message + * receive processing callbacks. We're receiving either a grant request + * or an invalidation response. These processing callbacks are fully + * concurrent. Our grant responses and invalidation requests are sent + * from these contexts. + * + * We separate the locking of the global index of tracked locks from the + * locking of a lock's state. This allows concurrent work on unrelated + * locks and lets processing block sending responses to unresponsive + * clients without affecting other locks. + * + * Correctness of the protocol relies on the client and server each only + * sending one request at a time for a given lock. The server won't + * process a request from a client until its outstanding invalidation + * requests for the lock to other clients have been completed. The + * server specifies both the old mode and new mode when sending messages + * to the client. This lets the client resolve possible reordering when + * processing incoming grant responses and invalidation requests. The + * server doesn't use the modes specified by the clients but they're + * provided to add context. + * + * The server relies on the node_id allocation and reliable messaging + * layers of the system. Each client has a node_id that is unique for + * its life time. Message requests and responses are reliably + * delivered in order across reconnection. + * + * The server maintains a persistent record of connected clients. A new + * server instance discovers these and waits for previously connected + * clients to reconnect and recover their state before proceeding. If + * clients don't reconnect they are forcefully prevented from unsafely + * accessing the shared persistent storage. (fenced, according to the + * rules of the platform.. could range from being powered off to having + * their switch port disabled to having their local block device set + * read-only.) + * + * The lock server doesn't respond to memory pressure. The only way + * locks are freed is if they are invalidated to null on behalf of a + * conflicting request, clients specifically request a null mode, or the + * server shuts down. + */ + +struct lock_server_info { + spinlock_t lock; + struct rb_root locks_root; + + struct scoutfs_tseq_tree tseq_tree; + struct dentry *tseq_dentry; +}; + +#define DECLARE_LOCK_SERVER_INFO(sb, name) \ + struct lock_server_info *name = SCOUTFS_SB(sb)->lock_server_info + +/* + * The state of a lock on the server is a function of the state of the + * locks on all clients. + * + * @granted: + * granted or trigger invalidation of previously granted. + * The state of a lock on the server is a function of messages that have + * been sent and received from clients on behalf of a given lock. + * + * While the invalidated list has entries, which means invalidation + * messages are still in flight, no more requests will be processed. + */ +struct server_lock_node { + atomic_t refcount; + struct mutex mutex; + struct rb_node node; + struct scoutfs_key key; + + struct list_head granted; + struct list_head requested; + struct list_head invalidated; +}; + +enum { + CLE_GRANTED, + CLE_REQUESTED, + CLE_INVALIDATED, +}; + +/* + * Interactions with the client are tracked with these little mode + * wrappers. + * + * @entry: The client mode's entry on one of the server lock lists indicating + * that the mode is actively granted, a pending request from the client, + * or a pending invalidation sent to the client. + * + * @node_id: The client's node_id used to send messages and tear down + * state as client's exit. + * + * @net_id: The id of a client's request used to send grant responses. The + * id of invalidation requests sent to clients that could be used to cancel + * the message. + * + * @mode: the mode that is granted to the client, that the client + * requested, or that the server is asserting with a pending + * invalidation request message. + */ +struct client_lock_entry { + struct list_head head; + u64 node_id; + u64 net_id; + u8 mode; + + struct server_lock_node *snode; + struct scoutfs_tseq_entry tseq_entry; + u8 on_list; +}; + +enum { + OL_GRANTED = 0, + OL_REQUESTED, + OL_INVALIDATED, +}; + +/* + * Put an entry on a server lock's list while being careful to move or + * add the list head and while maintaining debugging info. + */ +static void add_client_entry(struct server_lock_node *snode, + struct list_head *list, + struct client_lock_entry *clent) +{ + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + if (list_empty(&clent->head)) + list_add_tail(&clent->head, list); + else + list_move_tail(&clent->head, list); + + clent->on_list = list == &snode->granted ? OL_GRANTED : + list == &snode->requested ? OL_REQUESTED : + OL_INVALIDATED; +} + +static void free_client_entry(struct lock_server_info *inf, + struct server_lock_node *snode, + struct client_lock_entry *clent) +{ + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + if (!list_empty(&clent->head)) + list_del_init(&clent->head); + scoutfs_tseq_del(&inf->tseq_tree, &clent->tseq_entry); + kfree(clent); +} + +static bool invalid_mode(u8 mode) +{ + return mode >= SCOUTFS_LOCK_INVALID; +} + +/* + * Return the mode that we should invalidate a granted lock down to + * given an incompatible requested mode. Usually we completely + * invalidate the items because incompatible requests have to be writers + * and our cache will then be stale, but the single exception is + * invalidating down to a read lock having held a write lock because the + * cache is still valid for reads after being written out. + */ +static u8 invalidation_mode(u8 granted, u8 requested) +{ + if (granted == SCOUTFS_LOCK_WRITE && requested == SCOUTFS_LOCK_READ) + return SCOUTFS_LOCK_READ; + + return SCOUTFS_LOCK_NULL; +} + +/* + * Return true of the client lock instances described by the entries can + * be granted at the same time. Typically this only means they're both + * modes that are compatible between nodes. In addition there's the + * special case where a read lock on a client is compatible with a write + * lock on the same client because the client's cache covered by the + * read lock is still valid if they get a write lock. + */ +static bool client_entries_compatible(struct client_lock_entry *granted, + struct client_lock_entry *requested) +{ + return (granted->mode == requested->mode && + (granted->mode == SCOUTFS_LOCK_READ || + granted->mode == SCOUTFS_LOCK_WRITE_ONLY)) || + (granted->node_id == requested->node_id && + granted->mode == SCOUTFS_LOCK_READ && + requested->mode == SCOUTFS_LOCK_WRITE); +} + +/* + * Get a locked server lock, possibly inserting the caller's allocated + * lock if we don't find one for the given key. The server lock's mutex + * is held on return and the caller must put the lock when they're done. + */ +static struct server_lock_node *get_server_lock(struct lock_server_info *inf, + struct scoutfs_key *key, + struct server_lock_node *ins) +{ + struct rb_root *root = &inf->locks_root; + struct server_lock_node *ret = NULL; + struct server_lock_node *snode; + struct rb_node *parent = NULL; + struct rb_node **node; + int cmp; + + spin_lock(&inf->lock); + + node = &root->rb_node; + while (*node) { + parent = *node; + snode = container_of(*node, struct server_lock_node, node); + + cmp = scoutfs_key_compare(key, &snode->key); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = snode; + break; + } + } + + if (ret == NULL && ins) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + ret = ins; + } + + if (ret) + atomic_inc(&ret->refcount); + + spin_unlock(&inf->lock); + + if (ret) + mutex_lock(&ret->mutex); + + return ret; +} + +/* + * Finish with a server lock which has the mutex held, freeing it if + * it's empty and unused. + */ +static void put_server_lock(struct lock_server_info *inf, + struct server_lock_node *snode) +{ + bool should_free = false; + + BUG_ON(!mutex_is_locked(&snode->mutex)); + + if (atomic_dec_and_test(&snode->refcount) && + list_empty(&snode->granted) && + list_empty(&snode->requested) && + list_empty(&snode->invalidated)) { + spin_lock(&inf->lock); + rb_erase(&snode->node, &inf->locks_root); + spin_unlock(&inf->lock); + should_free = true; + } + + mutex_unlock(&snode->mutex); + + if (should_free) + kfree(snode); +} + +static struct client_lock_entry *find_entry(struct server_lock_node *snode, + struct list_head *list, + u64 node_id) +{ + struct client_lock_entry *clent; + + WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); + + list_for_each_entry(clent, list, head) { + if (clent->node_id == node_id) + return clent; + } + + return NULL; +} + +static int process_waiting_requests(struct super_block *sb, + struct server_lock_node *snode); + +/* + * The server is receiving an incoming request from a client. We queue + * it on the lock and process it. + * + * XXX shut down if we get enomem? + */ +int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, + u64 net_id, struct scoutfs_net_lock *nl) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *clent; + struct server_lock_node *snode; + struct server_lock_node *ins; + int ret; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST, + node_id, net_id, nl); + + if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { + ret = -EINVAL; + goto out; + } + + clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS); + if (!clent) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&clent->head); + clent->node_id = node_id; + clent->net_id = net_id; + clent->mode = nl->new_mode; + + snode = get_server_lock(inf, &nl->key, NULL); + if (snode == NULL) { + ins = kzalloc(sizeof(struct server_lock_node), GFP_NOFS); + if (ins == NULL) { + kfree(clent); + ret = -ENOMEM; + goto out; + } + + atomic_set(&ins->refcount, 0); + mutex_init(&ins->mutex); + ins->key = nl->key; + INIT_LIST_HEAD(&ins->granted); + INIT_LIST_HEAD(&ins->requested); + INIT_LIST_HEAD(&ins->invalidated); + + snode = get_server_lock(inf, &nl->key, ins); + if (snode != ins) + kfree(ins); + } + + clent->snode = snode; + add_client_entry(snode, &snode->requested, clent); + scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry); + + ret = process_waiting_requests(sb, snode); +out: + return ret; +} + +/* + * The server is receiving an invalidation response from the client. + * Find the client's entry on the server lock's invalidation list and + * free it so that request processing might be able to make forward + * progress. + * + * XXX what to do with errors? kick the client? + */ +int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock *nl) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *clent; + struct server_lock_node *snode; + int ret; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_RESPONSE, + node_id, 0, nl); + + if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { + ret = -EINVAL; + goto out; + } + + /* XXX should always have a server lock here? recovery? */ + snode = get_server_lock(inf, &nl->key, NULL); + if (!snode) { + ret = -EINVAL; + goto out; + } + + clent = find_entry(snode, &snode->invalidated, node_id); + if (!clent) { + put_server_lock(inf, snode); + ret = -EINVAL; + goto out; + } + + if (nl->new_mode == SCOUTFS_LOCK_NULL) { + free_client_entry(inf, snode, clent); + } else { + clent->mode = nl->new_mode; + add_client_entry(snode, &snode->granted, clent); + } + + ret = process_waiting_requests(sb, snode); +out: + return ret; +} + +/* + * Make forward progress on a lock by checking each waiting request in + * the order that they were received. If the next request is compatible + * with all the clients' grants then the request is granted and a + * response is sent. + * + * Invalidation requests are sent for every client grant that is + * incompatible with the next request. We won't process the next + * request again until we receive all the invalidation responses. Once + * they're all received then the request can be processed and will be + * compatible with the remaining grants. + * + * This is called with the snode mutex held. This can free the snode if + * it's empty. The caller can't reference the snode once this returns + * so we unlock the snode mutex. + */ +static int process_waiting_requests(struct super_block *sb, + struct server_lock_node *snode) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_net_lock nl; + struct client_lock_entry *req; + struct client_lock_entry *req_tmp; + struct client_lock_entry *gr; + struct client_lock_entry *gr_tmp; + int ret; + + BUG_ON(!mutex_is_locked(&snode->mutex)); + + /* request processing waits for all invalidation responses */ + if (!list_empty(&snode->invalidated)) { + ret = 0; + goto out; + } + + /* walk through pending requests in order received */ + list_for_each_entry_safe(req, req_tmp, &snode->requested, head) { + + /* send invalidation to any incompatible grants */ + list_for_each_entry_safe(gr, gr_tmp, &snode->granted, head) { + if (client_entries_compatible(gr, req)) + continue; + + nl.key = snode->key; + nl.old_mode = gr->mode; + nl.new_mode = invalidation_mode(gr->mode, req->mode); + + ret = scoutfs_server_lock_request(sb, gr->node_id, &nl); + if (ret) + goto out; + + trace_scoutfs_lock_message(sb, SLT_SERVER, + SLT_INVALIDATE, SLT_REQUEST, + gr->node_id, 0, &nl); + + add_client_entry(snode, &snode->invalidated, gr); + } + + /* wait for any newly sent invalidations */ + if (!list_empty(&snode->invalidated)) + break; + + nl.key = snode->key; + nl.new_mode = req->mode; + + /* see if there's an existing compatible grant to replace */ + gr = find_entry(snode, &snode->granted, req->node_id); + if (gr) { + nl.old_mode = gr->mode; + free_client_entry(inf, snode, gr); + } else { + nl.old_mode = SCOUTFS_LOCK_NULL; + } + + ret = scoutfs_server_lock_response(sb, req->node_id, + req->net_id, &nl); + if (ret) + goto out; + + trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, + SLT_RESPONSE, req->node_id, + req->net_id, &nl); + + /* don't track null client locks, track all else */ + if (req->mode == SCOUTFS_LOCK_NULL) + free_client_entry(inf, snode, req); + else + add_client_entry(snode, &snode->granted, req); + } + + ret = 0; +out: + put_server_lock(inf, snode); + + return ret; +} + +static char *lock_mode_string(u8 mode) +{ + static char *mode_strings[] = { + [SCOUTFS_LOCK_NULL] = "null", + [SCOUTFS_LOCK_READ] = "read", + [SCOUTFS_LOCK_WRITE] = "write", + [SCOUTFS_LOCK_WRITE_ONLY] = "write_only", + }; + + if (mode < ARRAY_SIZE(mode_strings) && mode_strings[mode]) + return mode_strings[mode]; + + return "unknown"; +} + +static char *lock_on_list_string(u8 on_list) +{ + static char *on_list_strings[] = { + [OL_GRANTED] = "granted", + [OL_REQUESTED] = "requested", + [OL_INVALIDATED] = "invalidated", + }; + + if (on_list < ARRAY_SIZE(on_list_strings) && on_list_strings[on_list]) + return on_list_strings[on_list]; + + return "unknown"; +} + +static void lock_server_tseq_show(struct seq_file *m, + struct scoutfs_tseq_entry *ent) +{ + struct client_lock_entry *clent = container_of(ent, + struct client_lock_entry, + tseq_entry); + struct server_lock_node *snode = clent->snode; + + seq_printf(m, SK_FMT" %s %s node_id %llu net_id %llu\n", + SK_ARG(&snode->key), lock_mode_string(clent->mode), + lock_on_list_string(clent->on_list), clent->node_id, + clent->net_id); +} + +int scoutfs_lock_server_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct lock_server_info *inf; + + inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); + if (!inf) + return -ENOMEM; + + spin_lock_init(&inf->lock); + inf->locks_root = RB_ROOT; + scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); + + inf->tseq_dentry = scoutfs_tseq_create("server_locks", sbi->debug_root, + &inf->tseq_tree); + if (!inf->tseq_dentry) { + kfree(inf); + return -ENOMEM; + } + + sbi->lock_server_info = inf; + + return 0; +} + +/* + * The server will have shut down networking before stopping us so we + * don't have to worry about message processing calls while we free. + */ +void scoutfs_lock_server_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct server_lock_node *snode; + struct server_lock_node *stmp; + struct client_lock_entry *clent; + struct client_lock_entry *ctmp; + LIST_HEAD(list); + + if (inf) { + debugfs_remove(inf->tseq_dentry); + + rbtree_postorder_for_each_entry_safe(snode, stmp, + &inf->locks_root, node) { + + list_splice_init(&snode->granted, &list); + list_splice_init(&snode->requested, &list); + list_splice_init(&snode->invalidated, &list); + + mutex_lock(&snode->mutex); + list_for_each_entry_safe(clent, ctmp, &list, head) { + free_client_entry(inf, snode, clent); + } + mutex_unlock(&snode->mutex); + + kfree(snode); + } + + kfree(inf); + sbi->lock_server_info = NULL; + } +} diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h new file mode 100644 index 00000000..2b6f4b1f --- /dev/null +++ b/kmod/src/lock_server.h @@ -0,0 +1,12 @@ +#ifndef _SCOUTFS_LOCK_SERVER_H_ +#define _SCOUTFS_LOCK_SERVER_H_ + +int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, + u64 net_id, struct scoutfs_net_lock *nl); +int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock *nl); + +int scoutfs_lock_server_setup(struct super_block *sb); +void scoutfs_lock_server_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3afe9059..3dadf912 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2423,6 +2423,52 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, TP_ARGS(sb, node_id, nr_clients) ); +#define slt_symbolic(mode) \ + __print_symbolic(mode, \ + { SLT_CLIENT, "client" }, \ + { SLT_SERVER, "server" }, \ + { SLT_GRANT, "grant" }, \ + { SLT_INVALIDATE, "invalidate" }, \ + { SLT_REQUEST, "request" }, \ + { SLT_RESPONSE, "response" }) + +TRACE_EVENT(scoutfs_lock_message, + TP_PROTO(struct super_block *sb, int who, int what, int dir, + u64 node_id, u64 net_id, struct scoutfs_net_lock *nl), + + TP_ARGS(sb, who, what, dir, node_id, net_id, nl), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(int, who) + __field(int, what) + __field(int, dir) + __field(__u64, node_id) + __field(__u64, net_id) + sk_trace_define(key) + __field(__u8, old_mode) + __field(__u8, new_mode) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->who = who; + __entry->what = what; + __entry->dir = dir; + __entry->node_id = node_id; + __entry->net_id = net_id; + sk_trace_assign(key, &nl->key); + __entry->old_mode = nl->old_mode; + __entry->new_mode = nl->new_mode; + ), + + TP_printk("fsid "FSID_FMT" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u", + __entry->fsid, slt_symbolic(__entry->who), + slt_symbolic(__entry->what), slt_symbolic(__entry->dir), + __entry->node_id, __entry->net_id, sk_trace_args(key), + __entry->old_mode, __entry->new_mode) +); + DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, TP_PROTO(struct super_block *sb, u64 io_blkno, struct scoutfs_quorum_block *blk), diff --git a/kmod/src/server.c b/kmod/src/server.c index b2e006f2..bf4047be 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -32,6 +32,7 @@ #include "msg.h" #include "server.h" #include "net.h" +#include "lock_server.h" #include "endian_swap.h" /* @@ -1020,6 +1021,52 @@ static int server_statfs(struct super_block *sb, &nstatfs, sizeof(nstatfs)); } +static int server_lock(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + u64 node_id = scoutfs_net_client_node_id(conn); + + if (arg_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + return scoutfs_lock_server_request(sb, node_id, id, arg); +} + +static int lock_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + u64 node_id = scoutfs_net_client_node_id(conn); + + if (resp_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + return scoutfs_lock_server_response(sb, node_id, resp); +} + +int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock *nl) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, node_id, + SCOUTFS_NET_CMD_LOCK, + nl, sizeof(*nl), + lock_response, NULL, NULL); +} + +int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, + u64 id, struct scoutfs_net_lock *nl) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_response_node(sb, server->conn, node_id, + SCOUTFS_NET_CMD_LOCK, id, 0, + nl, sizeof(*nl)); +} + /* * Process an incoming greeting request in the server from the client. * We try to send responses to failed greetings so that the sender can @@ -1712,6 +1759,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, [SCOUTFS_NET_CMD_GET_MANIFEST_ROOT] = server_get_manifest_root, [SCOUTFS_NET_CMD_STATFS] = server_statfs, + [SCOUTFS_NET_CMD_LOCK] = server_lock, }; static void server_notify_up(struct super_block *sb, @@ -1826,7 +1874,8 @@ static void scoutfs_server_worker(struct work_struct *work) /* start up the server subsystems before accepting */ ret = scoutfs_btree_setup(sb) ?: - scoutfs_manifest_setup(sb); + scoutfs_manifest_setup(sb) ?: + scoutfs_lock_server_setup(sb); if (ret) goto shutdown; @@ -1856,6 +1905,7 @@ shutdown: destroy_pending_frees(sb); scoutfs_manifest_destroy(sb); scoutfs_btree_destroy(sb); + scoutfs_lock_server_destroy(sb); /* XXX these should be persistent and reclaimed during recovery */ list_for_each_entry_safe(ps, ps_tmp, &server->pending_seqs, head) { diff --git a/kmod/src/server.h b/kmod/src/server.h index 365469b1..f2c9b8e7 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -53,6 +53,11 @@ void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment); +int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock *nl); +int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, + u64 id, struct scoutfs_net_lock *nl); + int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); diff --git a/kmod/src/super.h b/kmod/src/super.h index 99944c8d..99671dab 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -16,6 +16,7 @@ struct compact_info; struct data_info; struct trans_info; struct lock_info; +struct lock_server_info; struct client_info; struct server_info; struct inode_sb_info; @@ -59,6 +60,7 @@ struct scoutfs_sb_info { struct trans_info *trans_info; struct lock_info *lock_info; + struct lock_server_info *lock_server_info; struct client_info *client_info; struct server_info *server_info; struct sysfs_info *sfsinfo; From 7c8383eddd56b69c70eb2ea14242683764189cca Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Jan 2019 10:48:41 -0800 Subject: [PATCH 683/920] scoutfs: add scoutfs_lock_rename() Add a specific lock method for locking the global rename lock instead of having the caller specify it as a global lock. We're getting rid of the notion of lock scopes and requiring all locks to be related to keys. The rename lock will use magic keys at the end of the volume. Signed-off-by: Zach Brown --- kmod/src/dir.c | 4 +--- kmod/src/lock.c | 6 +++--- kmod/src/lock.h | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 292df2d5..e8fe3e79 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -1524,9 +1524,7 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, /* if dirs are different make sure ancestor relationships are valid */ if (old_dir != new_dir) { - ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, - SCOUTFS_LOCK_TYPE_GLOBAL_RENAME, - &rename_lock); + ret = scoutfs_lock_rename(sb, DLM_LOCK_EX, 0, &rename_lock); if (ret) return ret; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e8a6a425..aba98783 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1019,16 +1019,16 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, } /* - * Acquire a cluster lock with a global scope in the lock space. + * The rename lock is magical because it's global. */ -int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, +int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, struct scoutfs_lock **lock) { struct scoutfs_lock_name name; memset(&name, 0, sizeof(name)); name.scope = SCOUTFS_LOCK_SCOPE_GLOBAL; - name.type = type; + name.type = SCOUTFS_LOCK_TYPE_GLOBAL_RENAME; return lock_name_keys(sb, mode, flags, &name, NULL, NULL, lock); } diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 99c00bf9..89d161e8 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -66,7 +66,7 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *b, struct scoutfs_lock **b_lock, struct inode *c, struct scoutfs_lock **c_lock, struct inode *d, struct scoutfs_lock **D_lock); -int scoutfs_lock_global(struct super_block *sb, int mode, int flags, int type, +int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, struct scoutfs_lock **lock); int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock); From 08a140c8b0cfa3ce59cb49820e22c201fb86a37c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 14 Jan 2019 14:40:01 -0800 Subject: [PATCH 684/920] scoutfs: use our locking service Convert client locking to call the server's lock service instead of using a fs/dlm lockspace. The client code gets some shims to send and receive lock messages to and from the server. Callers use our lock mode constants instead of the DLM's. Locks are now identified by their starting key instead of an additional scoped lock name so that we don't have more mapping structures to track. The global rename lock uses keys that are defined by the format as only used for locking. The biggest change is in the client lock state machine. Instead of calling the dlm and getting callbacks we send messages to our server and get called from incoming message processing. We don't have everything come through a per-lock work queue. Instead we send requests either from the blocking lock caller or from a shrink work queue. Incoming messages are called in the net layer's blocking work contexts so we don't need to do any more work to defer to other contexts. The different processing contexts leads to a slightly different lock life cycle. We refactor and seperate allocation and freeing from tracking and removing locks in data structures. We add a _get and _put to track active use of locks and then async references to locks by holders and requests are tracked seperately. Our lock service's rules are a bit simpler in that we'll only ever send one request at a time and the server will only ever send one request at a time. We do have to do a bit of work to make sure we process back to back grant reponses and invalidation requests from the server. As of this change the lock setup and destruction paths are a little wobbly. They'll be shored up as we add lock recovery between the client and server. Signed-off-by: Zach Brown --- kmod/src/client.c | 50 ++ kmod/src/client.h | 4 + kmod/src/counters.h | 23 +- kmod/src/data.c | 28 +- kmod/src/dir.c | 74 ++- kmod/src/file.c | 22 +- kmod/src/format.h | 24 +- kmod/src/inode.c | 28 +- kmod/src/ioctl.c | 33 +- kmod/src/item.c | 28 +- kmod/src/lock.c | 1277 +++++++++++++++++++------------------- kmod/src/lock.h | 24 +- kmod/src/scoutfs_trace.h | 88 ++- kmod/src/super.c | 4 +- kmod/src/xattr.c | 14 +- 15 files changed, 884 insertions(+), 837 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 55def453..9ef4c247 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -231,6 +231,55 @@ int scoutfs_client_statfs(struct super_block *sb, sizeof(struct scoutfs_net_statfs)); } +/* process an incoming grant response from the server */ +static int client_lock_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + if (resp_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_grant_response(sb, resp); +} + +/* Send a lock request to the server. */ +int scoutfs_client_lock_request(struct super_block *sb, + struct scoutfs_net_lock *nl) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_LOCK, + nl, sizeof(*nl), + client_lock_response, NULL, NULL); +} + +/* Send a lock response to the server. */ +int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_response(sb, client->conn, SCOUTFS_NET_CMD_LOCK, + net_id, 0, nl, sizeof(*nl)); +} + +/* The client is receiving a invalidation request from the server */ +static int client_lock(struct super_block *sb, + struct scoutfs_net_connection *conn, u8 cmd, u64 id, + void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_net_lock)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_invalidate_request(sb, id, arg); +} + /* * Process a greeting response in the client from the server. This is * called for every connected socket on the connection. The first @@ -412,6 +461,7 @@ out: static scoutfs_net_request_t client_req_funcs[] = { [SCOUTFS_NET_CMD_COMPACT] = client_compact, + [SCOUTFS_NET_CMD_LOCK] = client_lock, }; /* diff --git a/kmod/src/client.h b/kmod/src/client.h index 410592eb..ca244f9d 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -17,6 +17,10 @@ int scoutfs_client_get_manifest_root(struct super_block *sb, struct scoutfs_btree_root *root); int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs); +int scoutfs_client_lock_request(struct super_block *sb, + struct scoutfs_net_lock *nl); +int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl); int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index da993bc8..55a7ea14 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -82,23 +82,26 @@ EXPAND_COUNTER(item_shrink_small_split) \ EXPAND_COUNTER(item_shrink_split_range) \ EXPAND_COUNTER(lock_alloc) \ - EXPAND_COUNTER(lock_ast) \ - EXPAND_COUNTER(lock_ast_edeadlk) \ - EXPAND_COUNTER(lock_ast_error) \ - EXPAND_COUNTER(lock_bast) \ - EXPAND_COUNTER(lock_dlm_call) \ - EXPAND_COUNTER(lock_dlm_call_error) \ EXPAND_COUNTER(lock_free) \ - EXPAND_COUNTER(lock_grace_enforced) \ - EXPAND_COUNTER(lock_grace_expired) \ + EXPAND_COUNTER(lock_grace_elapsed) \ EXPAND_COUNTER(lock_grace_extended) \ + EXPAND_COUNTER(lock_grace_set) \ + EXPAND_COUNTER(lock_grace_wait) \ + EXPAND_COUNTER(lock_grant_request) \ + EXPAND_COUNTER(lock_grant_response) \ EXPAND_COUNTER(lock_invalidate_clean_item) \ + EXPAND_COUNTER(lock_invalidate_coverage) \ + EXPAND_COUNTER(lock_invalidate_inode) \ + EXPAND_COUNTER(lock_invalidate_request) \ + EXPAND_COUNTER(lock_invalidate_response) \ EXPAND_COUNTER(lock_lock) \ EXPAND_COUNTER(lock_lock_error) \ EXPAND_COUNTER(lock_nonblock_eagain) \ - EXPAND_COUNTER(lock_shrink) \ - EXPAND_COUNTER(lock_write_dirty_item) \ + EXPAND_COUNTER(lock_shrink_queued) \ + EXPAND_COUNTER(lock_shrink_request_aborted) \ EXPAND_COUNTER(lock_unlock) \ + EXPAND_COUNTER(lock_wait) \ + EXPAND_COUNTER(lock_write_dirty_item) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(manifest_read_excluded_key) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 46b18588..cee53511 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -792,15 +792,17 @@ static int scoutfs_readpage(struct file *file, struct page *page) int ret; flags = SCOUTFS_LKF_REFRESH_INODE | SCOUTFS_LKF_NONBLOCK; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, flags, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, inode, + &inode_lock); if (ret < 0) { unlock_page(page); if (ret == -EAGAIN) { flags &= ~SCOUTFS_LKF_NONBLOCK; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, flags, inode, - &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, flags, + inode, &inode_lock); if (ret == 0) { - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, + SCOUTFS_LOCK_READ); ret = AOP_TRUNCATED_PAGE; } } @@ -808,7 +810,7 @@ static int scoutfs_readpage(struct file *file, struct page *page) } ret = mpage_readpage(page, scoutfs_get_block); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); return ret; } @@ -820,14 +822,14 @@ static int scoutfs_readpages(struct file *file, struct address_space *mapping, struct scoutfs_lock *inode_lock = NULL; int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) return ret; ret = mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); return ret; } @@ -1076,8 +1078,8 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) goto out; } - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) goto out; @@ -1166,7 +1168,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) } out: - scoutfs_unlock(sb, lock, DLM_LOCK_EX); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); mutex_unlock(&inode->i_mutex); trace_scoutfs_data_fallocate(sb, ino, mode, offset, len, ret); @@ -1199,7 +1201,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &inode_lock); if (ret) goto out; @@ -1240,7 +1242,7 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, blk_off = ext.start + ext.len; } - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); out: mutex_unlock(&inode->i_mutex); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index e8fe3e79..9140a1cf 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -335,7 +335,7 @@ static int scoutfs_d_revalidate(struct dentry *dentry, unsigned int flags) } dir = parent->d_inode; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, dir, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, dir, &lock); if (ret) goto out; @@ -368,7 +368,7 @@ out: trace_scoutfs_d_revalidate(sb, dentry, flags, parent, is_covered, ret); dput(parent); - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); if (ret < 0 && ret != -ECHILD) scoutfs_inc_counter(sb, dentry_revalidate_error); @@ -409,7 +409,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, if (ret) goto out; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, dir, &dir_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, dir, &dir_lock); if (ret) goto out; @@ -423,7 +423,7 @@ static struct dentry *scoutfs_lookup(struct inode *dir, struct dentry *dentry, update_dentry_info(sb, dentry, le64_to_cpu(dent.hash), le64_to_cpu(dent.pos), dir_lock); } - scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_READ); out: if (ret < 0) @@ -491,7 +491,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) SCOUTFS_DIRENT_LAST_POS, 0); kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &dir_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &dir_lock); if (ret) goto out; @@ -529,7 +529,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) } out: - scoutfs_unlock(sb, dir_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_READ); kfree(dent); return ret; @@ -665,15 +665,17 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, return ERR_PTR(ret); if (ino < scoutfs_ino(dir)) { - ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, inode_lock) ?: - scoutfs_lock_inode(sb, DLM_LOCK_EX, + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, + inode_lock) ?: + scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, dir, dir_lock); } else { - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, dir, dir_lock) ?: - scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, inode_lock); + scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, + inode_lock); } if (ret) goto out_unlock; @@ -701,8 +703,8 @@ out: out_unlock: if (ret) { scoutfs_inode_index_unlock(sb, ind_locks); - scoutfs_unlock(sb, *dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, *inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, *dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, *inode_lock, SCOUTFS_LOCK_WRITE); *dir_lock = NULL; *inode_lock = NULL; @@ -763,8 +765,8 @@ static int scoutfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, out: scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); /* XXX delete the inode item here */ if (ret && !IS_ERR_OR_NULL(inode)) @@ -803,7 +805,8 @@ static int scoutfs_link(struct dentry *old_dentry, if (dentry->d_name.len > SCOUTFS_NAME_LEN) return -ENAMETOOLONG; - ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, dir, &dir_lock, inode, &inode_lock, NULL, NULL, NULL, NULL); if (ret) @@ -858,8 +861,8 @@ out: scoutfs_release_trans(sb); out_unlock: scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); return ret; } @@ -889,7 +892,8 @@ static int scoutfs_unlink(struct inode *dir, struct dentry *dentry) u64 ind_seq; int ret = 0; - ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, dir, &dir_lock, inode, &inode_lock, NULL, NULL, NULL, NULL); if (ret) @@ -946,8 +950,8 @@ out: scoutfs_release_trans(sb); unlock: scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); return ret; } @@ -1034,8 +1038,8 @@ static void *scoutfs_follow_link(struct dentry *dentry, struct nameidata *nd) loff_t size; int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) return ERR_PTR(ret); @@ -1087,7 +1091,7 @@ out: } else { nd_set_link(nd, path); } - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); return path; } @@ -1184,8 +1188,8 @@ out: scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); return ret; } @@ -1239,12 +1243,12 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, U64_MAX); kvec_init(&val, &ent->dent, dirent_bytes(SCOUTFS_NAME_LEN)); - ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, ino, &lock); if (ret) goto out; ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); lock = NULL; if (ret < 0) goto out; @@ -1524,7 +1528,8 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, /* if dirs are different make sure ancestor relationships are valid */ if (old_dir != new_dir) { - ret = scoutfs_lock_rename(sb, DLM_LOCK_EX, 0, &rename_lock); + ret = scoutfs_lock_rename(sb, SCOUTFS_LOCK_WRITE, 0, + &rename_lock); if (ret) return ret; @@ -1537,7 +1542,8 @@ static int scoutfs_rename(struct inode *old_dir, struct dentry *old_dentry, } /* lock all the inodes */ - ret = scoutfs_lock_inodes(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, + ret = scoutfs_lock_inodes(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, old_dir, &old_dir_lock, new_dir, &new_dir_lock, old_inode, &old_inode_lock, @@ -1722,11 +1728,11 @@ out: out_unlock: scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, old_inode_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, new_inode_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, old_dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, new_dir_lock, DLM_LOCK_EX); - scoutfs_unlock(sb, rename_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, old_inode_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, new_inode_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, old_dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, new_dir_lock, SCOUTFS_LOCK_WRITE); + scoutfs_unlock(sb, rename_lock, SCOUTFS_LOCK_WRITE); return ret; } diff --git a/kmod/src/file.c b/kmod/src/file.c index 9382d96a..f78e5721 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -41,13 +41,13 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret == 0) { scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); ret = generic_file_aio_read(iocb, iov, nr_segs, pos); scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); } return ret; @@ -68,8 +68,8 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, return 0; mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) goto out; @@ -84,7 +84,7 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); out: scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); mutex_unlock(&inode->i_mutex); if (ret > 0 || ret == -EIOCBQUEUED) { @@ -107,14 +107,14 @@ int scoutfs_permission(struct inode *inode, int mask) if (mask & MAY_NOT_BLOCK) return -ECHILD; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, - inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) return ret; ret = generic_permission(inode, mask); - scoutfs_unlock(sb, inode_lock, DLM_LOCK_PR); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); return ret; } @@ -138,7 +138,7 @@ loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) * items instead of relying on generic_file_llseek() * trickery. */ - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, SCOUTFS_LKF_REFRESH_INODE, inode, &lock); case SEEK_SET: @@ -152,7 +152,7 @@ loff_t scoutfs_file_llseek(struct file *file, loff_t offset, int whence) if (ret == 0) offset = generic_file_llseek(file, offset, whence); - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); return ret ? ret : offset; } diff --git a/kmod/src/format.h b/kmod/src/format.h index ebce65fd..86cfe48d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -320,7 +320,8 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_ZONE 1 #define SCOUTFS_NODE_ZONE 2 #define SCOUTFS_FS_ZONE 3 -#define SCOUTFS_MAX_ZONE 4 /* power of 2 is efficient */ +#define SCOUTFS_LOCK_ZONE 4 +#define SCOUTFS_MAX_ZONE 8 /* power of 2 is efficient */ /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -341,6 +342,9 @@ struct scoutfs_segment_block { #define SCOUTFS_FILE_EXTENT_TYPE 7 #define SCOUTFS_ORPHAN_TYPE 8 +/* lock zone, only ever found in lock ranges, never in persistent items */ +#define SCOUTFS_RENAME_TYPE 1 + #define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ /* @@ -557,26 +561,8 @@ enum { #define SCOUTFS_MAX_VAL_SIZE SCOUTFS_XATTR_MAX_PART_SIZE -/* - * structures used by dlm - */ -#define SCOUTFS_LOCK_SCOPE_GLOBAL 1 -#define SCOUTFS_LOCK_SCOPE_FS_ITEMS 2 - -#define SCOUTFS_LOCK_TYPE_GLOBAL_RENAME 1 -#define SCOUTFS_LOCK_TYPE_GLOBAL_SERVER 2 - -struct scoutfs_lock_name { - __u8 scope; - __u8 zone; - __u8 type; - __le64 first; - __le64 second; -} __packed; - #define SCOUTFS_LOCK_INODE_GROUP_NR 1024 #define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) - #define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) /* diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 55400260..b9e6fcf9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -315,11 +315,11 @@ int scoutfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct scoutfs_lock *lock = NULL; int ret; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, SCOUTFS_LKF_REFRESH_INODE, - inode, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret == 0) { generic_fillattr(inode, stat); - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); } return ret; } @@ -406,8 +406,8 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) trace_scoutfs_setattr(dentry, attr); - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) return ret; @@ -452,7 +452,7 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); out: - scoutfs_unlock(sb, lock, DLM_LOCK_EX); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); return ret; } @@ -612,7 +612,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) struct inode *inode; int ret; - ret = scoutfs_lock_ino(sb, DLM_LOCK_PR, 0, ino, &lock); + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, ino, &lock); if (ret) return ERR_PTR(ret); @@ -641,7 +641,7 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) } out: - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); return inode; } @@ -1141,9 +1141,9 @@ int scoutfs_inode_index_try_lock_hold(struct super_block *sb, list_sort(NULL, list, cmp_index_lock); list_for_each_entry(ind_lock, list, head) { - ret = scoutfs_lock_inode_index(sb, DLM_LOCK_CW, ind_lock->type, - ind_lock->major, ind_lock->ino, - &ind_lock->lock); + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_WRITE_ONLY, + ind_lock->type, ind_lock->major, + ind_lock->ino, &ind_lock->lock); if (ret) goto out; } @@ -1188,7 +1188,7 @@ void scoutfs_inode_index_unlock(struct super_block *sb, struct list_head *list) struct index_lock *tmp; list_for_each_entry_safe(ind_lock, tmp, list, head) { - scoutfs_unlock(sb, ind_lock->lock, DLM_LOCK_CW); + scoutfs_unlock(sb, ind_lock->lock, SCOUTFS_LOCK_WRITE_ONLY); list_del_init(&ind_lock->head); kfree(ind_lock); } @@ -1403,7 +1403,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) u64 size; int ret; - ret = scoutfs_lock_ino(sb, DLM_LOCK_EX, 0, ino, &lock); + ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_WRITE, 0, ino, &lock); if (ret) return ret; @@ -1472,7 +1472,7 @@ out: if (release) scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - scoutfs_unlock(sb, lock, DLM_LOCK_EX); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); return ret; } diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 31a4a612..2c1fa74e 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -43,11 +43,11 @@ * relatively cheap because reading is going to check the segments * anyway. * - * This is copying to userspace while holding a DLM lock. This is safe - * because faulting can convert the lock to a higher level while we hold - * the lower level. DLM locks don't block tasks in a node, they match - * and the tasks fall back to local locking. In this case the spin - * locks around the item cache. + * This is copying to userspace while holding a read lock. This is safe + * because faulting can send a request for a write lock while the read + * lock is being used. The cluster locks don't block tasks in a node, + * they match and the tasks fall back to local locking. In this case + * the spin locks around the item cache. */ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) { @@ -99,8 +99,9 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) /* cap nr to the max the ioctl can return to a compat task */ walk.nr_entries = min_t(u64, walk.nr_entries, INT_MAX); - ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, type, walk.first.major, - walk.first.ino, &lock); + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_READ, type, + walk.first.major, walk.first.ino, + &lock); if (ret < 0) goto out; @@ -122,7 +123,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) key = lock->end; scoutfs_key_inc(&key); - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); /* * XXX This will miss dirty items. We'd need to @@ -143,7 +144,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) key = next_key; - ret = scoutfs_lock_inode_index(sb, DLM_LOCK_PR, + ret = scoutfs_lock_inode_index(sb, SCOUTFS_LOCK_READ, key.sk_type, le64_to_cpu(key.skii_major), le64_to_cpu(key.skii_ino), @@ -170,7 +171,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) scoutfs_key_inc(&key); } - scoutfs_unlock(sb, lock, DLM_LOCK_PR); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); out: if (nr > 0) @@ -299,8 +300,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) goto out; @@ -344,7 +345,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) } out: - scoutfs_unlock(sb, lock, DLM_LOCK_EX); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); @@ -423,8 +424,8 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) goto out; @@ -464,7 +465,7 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) current->backing_dev_info = NULL; out: scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); - scoutfs_unlock(sb, lock, DLM_LOCK_EX); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); diff --git a/kmod/src/item.c b/kmod/src/item.c index 60c461fb..d8e3f318 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -791,10 +791,11 @@ restart: static bool lock_coverage(struct scoutfs_lock *lock, struct scoutfs_key *key, int op_mode) { - signed char mode = ACCESS_ONCE(lock->granted_mode); + signed char mode = ACCESS_ONCE(lock->mode); return ((op_mode == mode) || - (op_mode == DLM_LOCK_PR && mode == DLM_LOCK_EX)) && + (op_mode == SCOUTFS_LOCK_READ && + mode == SCOUTFS_LOCK_WRITE)) && scoutfs_key_compare_ranges(key, key, &lock->start, &lock->end) == 0; } @@ -816,7 +817,7 @@ int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) return -EINVAL; trace_scoutfs_item_lookup(sb, key); @@ -976,7 +977,7 @@ int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, goto out; } - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) { + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) { ret = -EINVAL; goto out; } @@ -1142,7 +1143,7 @@ int scoutfs_item_prev(struct super_block *sb, struct scoutfs_key *key, goto out; } - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_PR))) { + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) { ret = -EINVAL; goto out; } @@ -1222,7 +1223,7 @@ int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, int ret; if (invalid_key_val(key, val) || - WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { ret = -EINVAL; goto out; } @@ -1282,7 +1283,7 @@ int scoutfs_item_create_force(struct super_block *sb, if (invalid_key_val(key, val)) return -EINVAL; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE_ONLY))) return -EINVAL; item = alloc_item(sb, key, val); @@ -1428,7 +1429,7 @@ int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) return -EINVAL; do { @@ -1473,7 +1474,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, if (invalid_key_val(key, val)) return -EINVAL; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) return -EINVAL; if (val) { @@ -1534,7 +1535,7 @@ int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { ret = -EINVAL; goto out; } @@ -1581,7 +1582,7 @@ int scoutfs_item_delete_force(struct super_block *sb, unsigned long flags; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_CW))) + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE_ONLY))) return -EINVAL; item = alloc_item(sb, key, NULL); @@ -1630,7 +1631,7 @@ int scoutfs_item_delete_save(struct super_block *sb, bool was_dirty; int ret; - if (WARN_ON_ONCE(!lock_coverage(lock, key, DLM_LOCK_EX))) { + if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { ret = -EINVAL; goto out; } @@ -1704,7 +1705,8 @@ int scoutfs_item_restore(struct super_block *sb, struct list_head *list, /* make sure all the items are locked and cached */ list_for_each_entry(item, list, entry) { - mode = item_is_dirty(item) ? DLM_LOCK_EX : DLM_LOCK_PR; + mode = item_is_dirty(item) ? SCOUTFS_LOCK_WRITE : + SCOUTFS_LOCK_READ; if (WARN_ON_ONCE(!lock_coverage(lock, &item->key, mode)) || WARN_ON_ONCE(!check_range(sb, &cac->ranges, &item->key, NULL, NULL))) { diff --git a/kmod/src/lock.c b/kmod/src/lock.c index aba98783..c33c43bd 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -12,9 +12,9 @@ */ #include #include +#include /* a rhel shed.h needed preempt_offset? */ #include #include -#include #include #include #include @@ -31,37 +31,32 @@ #include "endian_swap.h" #include "triggers.h" #include "tseq.h" +#include "client.h" /* - * scoutfs manages internode item cache consistency using the kernel's - * dlm service. We map ranges of item keys to dlm locks and use each - * lock's modes to govern what we can do with the items under the lock. + * scoutfs uses a lock service to manage item cache consistency between + * nodes. We map ranges of item keys to locks and use each lock's modes + * to govern what can be done with the items under the lock. Locks are + * held by mounts who populate, write out, and invalidate their caches + * as they acquire and release locks. * - * The management of locks is based around state updates that queue work - * which then acts on the state. The work calls the dlm to change modes - * and gets completion notification in callbacks. + * The locking client in a mount sends lock requests to the server. The + * server eventually responds with a response that grants access to the + * lock. The server then sends a revoke request to the client which + * tells it the mode that it should reduce the lock to. If it removes + * all access to the lock (by revoking it down to a null mode) then the + * lock is freed. * - * We free locks that aren't actively protecting items instead of - * converting them to NL and leaving them around. It gives us fewer - * locks consuming resources and fewer locks to wade through to try and - * diagnose a problem. + * Memory pressure on the client can cause the client to request a null + * mode from the server so that once its granted the lock can be freed. * - * So far we've only needed a minimal trylock. We don't issue a NOQUEUE - * request to the dlm which can eventually return -EAGAIN if it finds - * contention. We return -EAGAIN ourselves if a user can't immediately - * match an existing granted lock. This is fine for the only rare user - * which can back out of its lock inversion and retry with a full - * blocking lock. This saves us from having to plumb per-waiter flags - * down to dlm requests. + * So far we've only needed a minimal trylock. We return -EAGAIN if a + * lock attempt can't immediately match an existing granted lock. This + * is fine for the only rare user which can back out of its lock + * inversion and retry with a full blocking lock. */ -#define GRACE_WORK_DELAY_JIFFIES msecs_to_jiffies(2) -#define GRACE_UNLOCK_DEADLINE_KT ms_to_ktime(2) - -#define LN_FMT "%u.%u.%u.%llu.%llu" -#define LN_ARG(name) \ - (name)->scope, (name)->zone, (name)->type, le64_to_cpu((name)->first),\ - le64_to_cpu((name)->second) +#define GRACE_PERIOD_KT ms_to_ktime(2) /* * allocated per-super, freed on unmount. @@ -76,7 +71,6 @@ struct lock_info { struct list_head lru_list; unsigned long long lru_nr; struct workqueue_struct *workq; - dlm_lockspace_t *lockspace; atomic64_t next_refresh_gen; struct dentry *tseq_dentry; struct scoutfs_tseq_tree tseq_tree; @@ -85,8 +79,34 @@ struct lock_info { #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info -static void scoutfs_lock_work(struct work_struct *work); -static void scoutfs_lock_grace_work(struct work_struct *work); +static void scoutfs_lock_shrink_worker(struct work_struct *work); + +static bool lock_mode_invalid(int mode) +{ + return (unsigned)mode >= SCOUTFS_LOCK_INVALID; +} + +static bool lock_mode_can_read(int mode) +{ + return mode == SCOUTFS_LOCK_READ || mode == SCOUTFS_LOCK_WRITE; +} + +static bool lock_mode_can_write(int mode) +{ + return mode == SCOUTFS_LOCK_WRITE || mode == SCOUTFS_LOCK_WRITE_ONLY; +} + +/* + * Returns true if a lock with the granted mode can satisfy a requested + * mode. This is directional. A read lock is satisfied by a write lock + * but not vice versa. + */ +static bool lock_modes_match(int granted, int requested) +{ + return (granted == requested) || + (granted == SCOUTFS_LOCK_WRITE && + requested == SCOUTFS_LOCK_READ); +} /* * invalidate cached data associated with an inode whose lock is going @@ -98,6 +118,7 @@ static void invalidate_inode(struct super_block *sb, u64 ino) inode = scoutfs_ilookup(sb, ino); if (inode) { + scoutfs_inc_counter(sb, lock_invalidate_inode); if (S_ISREG(inode->i_mode)) truncate_inode_pages(inode->i_mapping, 0); iput(inode); @@ -105,8 +126,10 @@ static void invalidate_inode(struct super_block *sb, u64 ino) } /* - * Invalidate caches associated with this lock. We're going from the - * previous mode to the next mode. + * Invalidate caches associated with this lock. Either we're + * invalidating a write to a read or we're invalidating to null. We + * always have to write out dirty items if there are any. We can only + * leave cached items behind in the case of invalidating to a read lock. */ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, int prev, int mode) @@ -118,8 +141,14 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, u64 ino, last; int ret; + trace_scoutfs_lock_invalidate(sb, lock); + + /* verify assertion made by comment above */ + BUG_ON(!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ) && + mode != SCOUTFS_LOCK_NULL); + /* any transition from a mode allowed to dirty items has to write */ - if (prev == DLM_LOCK_CW || prev == DLM_LOCK_EX) { + if (lock_mode_can_write(prev)) { ret = scoutfs_item_writeback(sb, start, end); if (ret < 0) return ret; @@ -129,12 +158,10 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, } } - /* invalidate items that we could have but won't be able to use */ - if (prev == DLM_LOCK_CW || - (prev == DLM_LOCK_PR && mode != DLM_LOCK_EX) || - (prev == DLM_LOCK_EX && mode != DLM_LOCK_PR)) { - + /* have to invalidate if we're not in the only usable case */ + if (!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ)) { retry: + /* remove cov items to tell users that their cache is stale */ spin_lock(&lock->cov_list_lock); list_for_each_entry_safe(cov, tmp, &lock->cov_list, head) { if (!spin_trylock(&cov->cov_lock)) { @@ -145,12 +172,13 @@ retry: list_del_init(&cov->head); cov->lock = NULL; spin_unlock(&cov->cov_lock); + scoutfs_inc_counter(sb, lock_invalidate_coverage); } spin_unlock(&lock->cov_list_lock); - if (lock->name.zone == SCOUTFS_FS_ZONE) { - ino = le64_to_cpu(lock->name.first); - last = ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1; + if (lock->start.sk_zone == SCOUTFS_FS_ZONE) { + ino = le64_to_cpu(lock->start.ski_ino); + last = le64_to_cpu(lock->end.ski_ino); while (ino <= last) { invalidate_inode(sb, ino); ino++; @@ -177,31 +205,32 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) trace_scoutfs_lock_free(sb, lock); scoutfs_inc_counter(sb, lock_free); - BUG_ON(!linfo->shutdown && lock->granted_mode != DLM_LOCK_IV); - BUG_ON(delayed_work_pending(&lock->grace_work)); + /* manually checking lock_idle gives identifying line numbers */ + BUG_ON(lock->request_pending); + BUG_ON(lock->invalidate_pending); + BUG_ON(lock->waiters[SCOUTFS_LOCK_READ]); + BUG_ON(lock->waiters[SCOUTFS_LOCK_WRITE]); + BUG_ON(lock->waiters[SCOUTFS_LOCK_WRITE_ONLY]); + BUG_ON(lock->users[SCOUTFS_LOCK_READ]); + BUG_ON(lock->users[SCOUTFS_LOCK_WRITE]); + BUG_ON(lock->users[SCOUTFS_LOCK_WRITE_ONLY]); + BUG_ON(!linfo->shutdown && lock->mode != SCOUTFS_LOCK_NULL); + BUG_ON(!RB_EMPTY_NODE(&lock->node)); + BUG_ON(!RB_EMPTY_NODE(&lock->range_node)); + BUG_ON(!list_empty(&lock->lru_head)); + BUG_ON(!list_empty(&lock->cov_list)); - scoutfs_tseq_del(&linfo->tseq_tree, &lock->tseq_entry); - if (!RB_EMPTY_NODE(&lock->node)) - rb_erase(&lock->node, &linfo->lock_tree); - if (!RB_EMPTY_NODE(&lock->range_node)) - rb_erase(&lock->range_node, &linfo->lock_range_tree); - if (!list_empty(&lock->lru_head)) { - list_del(&lock->lru_head); - linfo->lru_nr--; - } kfree(lock); } static struct scoutfs_lock *lock_alloc(struct super_block *sb, - struct scoutfs_lock_name *name, struct scoutfs_key *start, struct scoutfs_key *end) { - DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; - if (WARN_ON_ONCE(!!start != !!end)) + if (WARN_ON_ONCE(!start || !end)) return NULL; lock = kzalloc(sizeof(struct scoutfs_lock), GFP_NOFS); @@ -217,22 +246,13 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, spin_lock_init(&lock->cov_list_lock); INIT_LIST_HEAD(&lock->cov_list); - if (start) { - lock->start = *start; - lock->end = *end; - } - + lock->start = *start; + lock->end = *end; lock->sb = sb; - lock->name = *name; init_waitqueue_head(&lock->waitq); - INIT_WORK(&lock->work, scoutfs_lock_work); - INIT_DELAYED_WORK(&lock->grace_work, scoutfs_lock_grace_work); - lock->granted_mode = DLM_LOCK_IV; - lock->bast_mode = DLM_LOCK_IV; - lock->work_prev_mode = DLM_LOCK_IV; - lock->work_mode = DLM_LOCK_IV; + INIT_WORK(&lock->shrink_work, scoutfs_lock_shrink_worker); + lock->mode = SCOUTFS_LOCK_NULL; - scoutfs_tseq_add(&linfo->tseq_tree, &lock->tseq_entry); trace_scoutfs_lock_alloc(sb, lock); return lock; @@ -250,33 +270,6 @@ static void lock_dec_count(unsigned int *counts, int mode) counts[mode]--; } -/* only PR and EX modes read items to populate the cache. */ -static bool lock_mode_can_read(int mode) -{ - return mode == DLM_LOCK_PR || mode == DLM_LOCK_EX; -} - -/* - * Returns true if a given user mode can be satisfied by a lock with the - * given granted mode. This is directional. A PR user is satisfied by - * an EX grant but not vice versa. - */ -static bool lock_modes_match(int granted, int user) -{ - return (granted == user) || - (granted == DLM_LOCK_EX && user == DLM_LOCK_PR); -} - -/* - * This isn't strictly the same as being compatible.. callers - * need to be very careful to understand the combinations of - * modes that are possible for them to attempt. - */ -static bool lock_mode_valid_and_greater(int mode, int other) -{ - return mode != DLM_LOCK_IV && mode > other; -} - /* * Returns true if all the actively used modes are satisfied by a lock * of the given granted mode. @@ -294,13 +287,31 @@ static bool lock_counts_match(int granted, unsigned int *counts) } /* - * An idle lock has nothing going on and could be safely unlocked and freed. + * Returns true if there are any mode counts that match with the desired + * mode. There can be other non-matching counts as well but we're only + * testing for the existence of any matching counts. + */ +static bool lock_count_match_exists(int desired, unsigned int *counts) +{ + int mode; + + for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { + if (counts[mode] && lock_modes_match(desired, mode)) + return true; + } + + return false; +} + +/* + * An idle lock has nothing going on. It can be present in the lru and + * can be freed by the final put when it has a null mode. */ static bool lock_idle(struct scoutfs_lock *lock) { int mode; - if (lock->work_mode >= 0 || lock->grace_pending) + if (lock->request_pending || lock->invalidate_pending) return false; for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { @@ -311,137 +322,6 @@ static bool lock_idle(struct scoutfs_lock *lock) return true; } -/* - * Ensure forward progress on the lock after the caller has changed the lock. - * - * This is the core of the state transition engine that makes locking - * safe. Each transition has to consider the users of the lock, pending - * bast transitions, it's current mode, what mode it should be, and what - * mode to leave it in during the transition. - * - * This can free the lock if it's idle! Callers must not reference the - * lock after calling this. - */ -static void lock_process(struct lock_info *linfo, struct scoutfs_lock *lock) -{ - bool idle; - int mode; - - assert_spin_locked(&linfo->lock); - - /* nothing to do if we're shutting down, stops rearming */ - if (linfo->shutdown) - return; - - /* errored locks are torn down */ - if (lock->error) { - wake_up(&lock->waitq); - goto out; - } - - /* - * Try to down convert a lock in response to a bast once users - * are done with it. We may have to wait for a grace period - * to expire after an unlock. - */ - if (lock->work_mode < 0 && - lock->granted_mode >= 0 && - lock->bast_mode >= 0 && - lock_counts_match(lock->bast_mode, lock->users) && - !lock->grace_pending) { - - if (ktime_before(ktime_get(), lock->grace_deadline)) { - scoutfs_inc_counter(linfo->sb, lock_grace_enforced); - queue_delayed_work(linfo->workq, &lock->grace_work, - GRACE_WORK_DELAY_JIFFIES); - lock->grace_pending = true; - } else { - lock->work_prev_mode = lock->granted_mode; - lock->work_mode = lock->bast_mode; - lock->granted_mode = lock->bast_mode; - lock->bast_mode = DLM_LOCK_IV; - queue_work(linfo->workq, &lock->work); - } - } - - /* - * Convert on behalf of waiters who aren't satisfied by the - * current mode when it won't conflict with specific waiters, - * matching users, or pending bast conversions. The new mode - * may or may not match the current granted mode so we may or - * may not need to block users during the transition. - * - * Remember that the presence of waiters doesn't necessarily - * mean that they're blocked. Multiple lock attempts naturally - * line up to add themselves to the waiters count before each - * calls lock_wait() and is transitioned to a user. - */ - for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { - if (lock->work_mode < 0 && - lock->waiters[mode] && - (lock->granted_mode < 0 || - !lock->waiters[lock->granted_mode]) && - !lock_modes_match(lock->granted_mode, mode) && - lock_counts_match(mode, lock->users) && - (lock->bast_mode < 0 || - lock_modes_match(lock->bast_mode, mode))) { - - lock->work_prev_mode = lock->granted_mode; - lock->work_mode = mode; - if (!lock_modes_match(mode, lock->granted_mode)) - lock->granted_mode = DLM_LOCK_NL; - queue_work(linfo->workq, &lock->work); - break; - } - } - - /* - * Wake any waiters who might be able to use the lock now. - * Notice that this ignores the presence of basts! This lets us - * recursively acquire locks in one task without having to track - * per-task lock references. It comes at the cost of fairness. - * Spinning overlapping users can delay a bast down conversion - * indefinitely. - */ - for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { - if (lock->waiters[mode] && - lock_modes_match(lock->granted_mode, mode)) { - wake_up(&lock->waitq); - break; - } - } - -out: - /* only idle locks are on the lru */ - idle = lock_idle(lock); - if (list_empty(&lock->lru_head) && idle) { - list_add_tail(&lock->lru_head, &linfo->lru_list); - linfo->lru_nr++; - - } else if (!list_empty(&lock->lru_head) && !idle) { - list_del_init(&lock->lru_head); - linfo->lru_nr--; - } - - /* - * We can free the lock once it's idle and it's either never - * been initially locked or has been unlocked, both of which we - * indicate with IV. - */ - if (idle && lock->granted_mode == DLM_LOCK_IV) - lock_free(linfo, lock); -} - -static int cmp_lock_names(struct scoutfs_lock_name *a, - struct scoutfs_lock_name *b) -{ - return ((int)a->scope - (int)b->scope) ?: - ((int)a->zone - (int)b->zone) ?: - ((int)a->type - (int)b->type) ?: - scoutfs_cmp_u64s(le64_to_cpu(a->first), le64_to_cpu(b->first)) ?: - scoutfs_cmp_u64s(le64_to_cpu(a->second), le64_to_cpu(b->second)); -} - static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) { DECLARE_LOCK_INFO(sb, linfo); @@ -458,10 +338,8 @@ static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, &lock->start, &lock->end); if (WARN_ON_ONCE(cmp == 0)) { - scoutfs_warn(sb, "inserting lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT" overlaps with existing lock %p name "LN_FMT" start "SK_FMT" end "SK_FMT"\n", - ins, LN_ARG(&ins->name), + scoutfs_warn(sb, "inserting lock start "SK_FMT" end "SK_FMT" overlaps with existing lock start "SK_FMT" end "SK_FMT"\n", SK_ARG(&ins->start), SK_ARG(&ins->end), - lock, LN_ARG(&lock->name), SK_ARG(&lock->start), SK_ARG(&lock->end)); return false; } @@ -479,12 +357,10 @@ static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) return true; } -static struct scoutfs_lock *lock_rb_walk(struct super_block *sb, - struct scoutfs_lock_name *name, - struct scoutfs_lock *ins) +/* returns true if the lock was inserted at its start key */ +static bool lock_insert(struct super_block *sb, struct scoutfs_lock *ins) { DECLARE_LOCK_INFO(sb, linfo); - struct scoutfs_lock *found; struct scoutfs_lock *lock; struct rb_node *parent; struct rb_node **node; @@ -494,298 +370,406 @@ static struct scoutfs_lock *lock_rb_walk(struct super_block *sb, node = &linfo->lock_tree.rb_node; parent = NULL; - found = NULL; while (*node) { parent = *node; lock = container_of(*node, struct scoutfs_lock, node); - cmp = cmp_lock_names(name, &lock->name); - if (cmp < 0) { + cmp = scoutfs_key_compare(&ins->start, &lock->start); + if (cmp < 0) node = &(*node)->rb_left; - } else if (cmp > 0) { + else if (cmp > 0) node = &(*node)->rb_right; - } else { - found = lock; - break; - } - lock = NULL; + else + return false; } - if (!found && ins) { - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, &linfo->lock_tree); - found = ins; + if (!insert_range_node(sb, ins)) + return false; + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &linfo->lock_tree); + + scoutfs_tseq_add(&linfo->tseq_tree, &ins->tseq_entry); + + return true; +} + +static void lock_remove(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); + + rb_erase(&lock->node, &linfo->lock_tree); + RB_CLEAR_NODE(&lock->node); + rb_erase(&lock->range_node, &linfo->lock_range_tree); + RB_CLEAR_NODE(&lock->range_node); + + scoutfs_tseq_del(&linfo->tseq_tree, &lock->tseq_entry); +} + +static struct scoutfs_lock *lock_lookup(struct super_block *sb, + struct scoutfs_key *start) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct rb_node *node = linfo->lock_tree.rb_node; + struct scoutfs_lock *lock; + int cmp; + + assert_spin_locked(&linfo->lock); + + while (node) { + lock = container_of(node, struct scoutfs_lock, node); + + cmp = scoutfs_key_compare(start, &lock->start); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else + return lock; } - return found; + return NULL; +} + +static void __lock_del_lru(struct lock_info *linfo, struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&lock->lru_head)) { + list_del_init(&lock->lru_head); + linfo->lru_nr--; + } } /* - * A dlm lock, conversion, or unlock call has finished. We don't - * strictly serialize the arrival of basts and our dlm calls. It's - * possible and safe for us to get a deadlock notification because we - * tried to convert in conflict with a received bast. We ignore the - * result of the deadlock conversion and processing will retry and this - * time prefer the bast. + * Get a lock and remove it from the lru. The caller must set state on + * the lock that indicates that it's busy before dropping the lock. + * Then later they call add_lru_or_free once they've cleared that state. */ -static void scoutfs_lock_ast(void *arg) +static struct scoutfs_lock *get_lock(struct super_block *sb, + struct scoutfs_key *start) { - struct scoutfs_lock *lock = arg; - struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - int status = lock->lksb.sb_status; - bool cached = false; - bool dirty = false; + struct scoutfs_lock *lock; - scoutfs_inc_counter(sb, lock_ast); + assert_spin_locked(&linfo->lock); - spin_lock(&linfo->lock); + lock = lock_lookup(sb, start); + if (lock) + __lock_del_lru(linfo, lock); - if (status == 0) { - if (lock_mode_can_read(lock->work_mode) && - !lock_mode_can_read(lock->work_prev_mode)) { - lock->refresh_gen = - atomic64_inc_return(&linfo->next_refresh_gen); + return lock; +} + +/* + * Get a lock, creating it if it doesn't exist. The caller must treat + * the lock like it came from get lock (mark sate, drop lock, clear + * state, put lock). Allocated locks aren't on the lru. + */ +static struct scoutfs_lock *create_lock(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_lock *lock; + + assert_spin_locked(&linfo->lock); + + lock = get_lock(sb, start); + if (!lock) { + spin_unlock(&linfo->lock); + lock = lock_alloc(sb, start, end); + spin_lock(&linfo->lock); + + if (lock) { + if (!lock_insert(sb, lock)) { + lock_free(linfo, lock); + lock = get_lock(sb, start); + } } - lock->granted_mode = lock->work_mode; - - } else if (status == -DLM_EUNLOCK) { - lock->granted_mode = DLM_LOCK_IV; - - } else if (status == -EDEADLK) { - /* dlm request conflicted with racing bast, try again */ - scoutfs_inc_counter(sb, lock_ast_edeadlk); - - } else if (!lock->error) { - scoutfs_inc_counter(sb, lock_ast_error); - lock->error = status; } - lock->work_prev_mode = DLM_LOCK_IV; - lock->work_mode = DLM_LOCK_IV; + return lock; +} - trace_scoutfs_lock_ast(sb, lock); +/* + * The caller is done using a lock and has cleared state that used to + * indicate that the lock wasn't idle. If it really is idle then we + * either free it if it's null or put it back on the lru. + */ +static void put_lock(struct lock_info *linfo,struct scoutfs_lock *lock) +{ + assert_spin_locked(&linfo->lock); - /* - * Catch lock modes with cached items that violate the item - * cache consistency rules. - * - * We can never have dirty items if we're calling the dlm and - * changing lock modes. We can't have cached items if we're not - * in the two modes that allow caching. - */ - if (!RB_EMPTY_NODE(&lock->range_node)) { - cached = scoutfs_item_range_cached(sb, &lock->start, - &lock->end, false); - dirty = scoutfs_item_range_cached(sb, &lock->start, &lock->end, - true); + if (lock_idle(lock)) { + if (lock->mode != SCOUTFS_LOCK_NULL) { + list_add_tail(&lock->lru_head, &linfo->lru_list); + linfo->lru_nr++; + } else { + lock_remove(linfo, lock); + lock_free(linfo, lock); + } } +} - if (WARN_ON_ONCE(dirty || - (cached && lock->granted_mode != DLM_LOCK_PR && - lock->granted_mode != DLM_LOCK_EX))) { - scoutfs_err(sb, "lock item cache consistency violation, cached %u dirty %u: name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", - cached, dirty, - LN_ARG(&lock->name), SK_ARG(&lock->start), - SK_ARG(&lock->end), lock->refresh_gen, lock->error, - lock->granted_mode, lock->bast_mode, - lock->work_prev_mode, lock->work_mode, - lock->waiters[DLM_LOCK_PR], - lock->waiters[DLM_LOCK_EX], - lock->waiters[DLM_LOCK_CW], - lock->users[DLM_LOCK_PR], - lock->users[DLM_LOCK_EX], - lock->users[DLM_LOCK_CW], - lock->lksb.sb_status, - lock->lksb.sb_lkid, - lock->lksb.sb_flags); +/* + * Locks have a grace period that extends after activity and prevents + * invalidation. It's intended to let nodes do reasonable batches of + * work as locks ping pong between nodes that are doing conflicting + * work. + */ +static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) +{ + ktime_t now = ktime_get(); + + if (ktime_after(now, lock->grace_deadline)) + scoutfs_inc_counter(sb, lock_grace_set); + else + scoutfs_inc_counter(sb, lock_grace_extended); + + lock->grace_deadline = ktime_add(now, GRACE_PERIOD_KT); +} + +/* + * The given lock is processing a received a grant response. Trigger a + * bug if the cache is inconsistent. + * + * We only have two modes that can create dirty items. We can't have + * dirty items when transitioning from write_only to write because the + * writer can't trust the cached items in the cache for reading. And we + * don't currently transition directly from write to write_only, we + * first go through null. So if we have dirty items as we're granted a + * mode it's always incorrect. + * + * And we can't have cached items that we're going to use for reading if + * the previous mode didn't allow reading. + * + * Inconsistencies have come from all sorts of bugs: invalidation missed + * items, the cache was populated outside of locking coverage, lock + * holders performed the wrong item operations under their lock, + * overlapping locks, out of order granting or invalidating, etc. + */ +static void bug_on_inconsistent_grant_cache(struct super_block *sb, + struct scoutfs_lock *lock, + int old_mode, int new_mode) +{ + bool cached = scoutfs_item_range_cached(sb, &lock->start, &lock->end, + false); + bool dirty = scoutfs_item_range_cached(sb, &lock->start, &lock->end, + true); + + if (dirty || + (cached && (!lock_mode_can_read(old_mode) || !lock_mode_can_read(new_mode)))) { + scoutfs_err(sb, "granted lock item cache inconsistency, cached %u dirty %u old_mode %d new_mode %d: start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %u waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u", + cached, dirty, old_mode, new_mode, SK_ARG(&lock->start), + SK_ARG(&lock->end), lock->refresh_gen, lock->mode, + lock->waiters[SCOUTFS_LOCK_READ], + lock->waiters[SCOUTFS_LOCK_WRITE], + lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], + lock->users[SCOUTFS_LOCK_READ], + lock->users[SCOUTFS_LOCK_WRITE], + lock->users[SCOUTFS_LOCK_WRITE_ONLY]); BUG(); } - - lock_process(linfo, lock); - spin_unlock(&linfo->lock); } /* - * A lock on this node has blocked a lock request on another node. We - * translate the dlm's communication of the blocking mode to the mode - * that we should convert our lock to. We can only either downconvert - * to a matching PR or unlock. + * The client is receiving a lock response message from the server. + * This can be reordered with incoming invlidation requests from the + * server so we have to be careful to only set the new mode once the old + * mode matches. * - * These are truly asynchronous and can arrive multiple times, at any - * time. We're careful to only set the lock's bast mode here if the - * mode that's required conflicts with the lock's current mode, the mode - * the work might be converting to, or the next mode from a previous - * bast. This stops us from setting the lock's bast mode when it isn't - * needed a confusing the state machine, like for a lock that's being - * unlocked. + * We extend the grace period as we grant the lock if there is a waiting + * locker who can use the lock. This stops invalidation from pulling + * the granted lock out from under the requester, resulting in a lot of + * churn with no forward progress. Using the grace period avoids having + * to identify a specific waiter and give it an acquired lock. It's + * also very similar to waking up the locker and having it win the race + * against the invalidation. In that case they'd extend the grace + * period anyway as they unlock. */ -static void scoutfs_lock_bast(void *arg, int blocked_mode) +int scoutfs_lock_grant_response(struct super_block *sb, + struct scoutfs_net_lock *nl) { - struct scoutfs_lock *lock = arg; - struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - int bast_mode; + struct scoutfs_lock *lock; - scoutfs_inc_counter(sb, lock_bast); + scoutfs_inc_counter(sb, lock_grant_response); spin_lock(&linfo->lock); - if (lock->granted_mode == DLM_LOCK_EX && blocked_mode == DLM_LOCK_PR) - bast_mode = DLM_LOCK_PR; - else - bast_mode = DLM_LOCK_NL; + /* lock must already be busy with request_pending */ + lock = lock_lookup(sb, &nl->key); + BUG_ON(!lock); + BUG_ON(!lock->request_pending); - /* greater is safe, only try nl < all or pr < ex */ - if (lock_mode_valid_and_greater(lock->granted_mode, bast_mode) || - lock_mode_valid_and_greater(lock->work_mode, bast_mode) || - lock_mode_valid_and_greater(lock->bast_mode, bast_mode)) - lock->bast_mode = bast_mode; + trace_scoutfs_lock_grant_response(sb, lock); - trace_scoutfs_lock_bast(sb, lock); - lock_process(linfo, lock); + /* resolve unlikely work reordering with invalidation request */ + while (lock->mode != nl->old_mode) { + spin_unlock(&linfo->lock); + /* implicit read barrier from waitq locks */ + wait_event(lock->waitq, lock->mode == nl->old_mode); + spin_lock(&linfo->lock); + } + + bug_on_inconsistent_grant_cache(sb, lock, nl->old_mode, nl->new_mode); + + if (!lock_mode_can_read(nl->old_mode) && + lock_mode_can_read(nl->new_mode)) { + lock->refresh_gen = + atomic64_inc_return(&linfo->next_refresh_gen); + } + + lock->request_pending = 0; + lock->mode = nl->new_mode; + + if (lock_count_match_exists(nl->new_mode, lock->waiters)) + extend_grace(sb, lock); + + trace_scoutfs_lock_granted(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); spin_unlock(&linfo->lock); + + return 0; } /* - * The actual work of sending lock requests to the dlm. There's only - * one of these per lock and the work_mode ensures that there's only one - * transition in flight at a time. + * Invalidation waits until the old mode indicates that we've resolved + * unlikely races with reordered grant responses from the server and + * until the new mode satisfies active users. + * + * Once it's safe to proceed we set the lock mode here under the lock to + * prevent additional users of the old mode while we're invalidating. */ -static void scoutfs_lock_work(struct work_struct *work) +static bool lock_invalidate_safe(struct lock_info *linfo, + struct scoutfs_lock *lock, + int old_mode, int new_mode) +{ + bool safe; + + spin_lock(&linfo->lock); + safe = (lock->mode == old_mode) && + lock_counts_match(new_mode, lock->users); + if (safe) + lock->mode = new_mode; + spin_unlock(&linfo->lock); + + return safe; +} + +/* + * The client is receiving a lock invalidation request from the server + * which specifies a new mode for the lock. The server will only send + * one invalidation request at a time. This is executing in a blocking + * net receive work context. + * + * This is an unsolicited request from the server so it can arrive at + * any time after we make the server aware of the lock by initially + * requesting it. We wait for users of the current mode to unlock + * before invalidating. + * + * This can arrive on behalf of our request for a mode that conflicts + * with our current mode. We have to proceed while we have a request + * pending. We can also be racing with shrink requests being sent while + * we're invalidating. + * + * This can be processed concurrently and experience reordering with a + * grant response sent back-to-back from the server. We carefully only + * invalidate once the lock mode matches what the server told us to + * invalidate. + * + * We delay invalidation processing until a grace period has elapsed since + * the last unlock. The intent is to let users do a reasonable batch of + * work before dropping the lock. Continuous unlocking can continuously + * extend the deadline. + */ +int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl) { - struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - work); - struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - int dlm_flags; - int prev; - int mode; + struct scoutfs_lock *lock; + ktime_t deadline; + bool grace_waited = false; int ret; + scoutfs_inc_counter(sb, lock_invalidate_request); + spin_lock(&linfo->lock); - - /* don't try to call a released lockspace during shutdown */ - if (linfo->shutdown) { - spin_unlock(&linfo->lock); - return; + lock = get_lock(sb, &nl->key); + if (lock) { + BUG_ON(lock->invalidate_pending); /* XXX trusting server :/ */ + lock->invalidate_pending = 1; + deadline = lock->grace_deadline; + trace_scoutfs_lock_invalidate_request(sb, lock); } - - trace_scoutfs_lock_work(sb, lock); - prev = lock->work_prev_mode; - mode = lock->work_mode; - spin_unlock(&linfo->lock); - if (!RB_EMPTY_NODE(&lock->range_node)) { - ret = lock_invalidate(sb, lock, prev, mode); - BUG_ON(ret); + BUG_ON(!lock); + + /* wait for a grace period after the most recent unlock */ + while (ktime_before(ktime_get(), deadline)) { + grace_waited = true; + scoutfs_inc_counter(linfo->sb, lock_grace_wait); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&deadline, HRTIMER_MODE_ABS); + + spin_lock(&linfo->lock); + deadline = lock->grace_deadline; + spin_unlock(&linfo->lock); } - scoutfs_inc_counter(sb, lock_dlm_call); + if (grace_waited) + scoutfs_inc_counter(linfo->sb, lock_grace_elapsed); - if (mode == DLM_LOCK_NL) { - ret = dlm_unlock(linfo->lockspace, lock->lksb.sb_lkid, 0, - &lock->lksb, lock); - } else { - dlm_flags = DLM_LKF_NOORDER; - if (prev >= 0) - dlm_flags |= DLM_LKF_CONVERT; - ret = dlm_lock(linfo->lockspace, mode, &lock->lksb, dlm_flags, - &lock->name, sizeof(lock->name), 0, - scoutfs_lock_ast, lock, scoutfs_lock_bast); - } - /* - * I don't think the lock error handling is correct yet. It - * probably doesn't try to unlock a lock that saw an error. - */ - if (ret) - scoutfs_inc_counter(sb, lock_dlm_call_error); + /* sets the lock mode to prevent use of old mode during invalidate */ + wait_event(lock->waitq, lock_invalidate_safe(linfo, lock, nl->old_mode, + nl->new_mode)); + + ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); BUG_ON(ret); + /* respond with the key and modes from the request */ + ret = scoutfs_client_lock_response(sb, net_id, nl); + BUG_ON(ret); + + scoutfs_inc_counter(sb, lock_invalidate_response); + spin_lock(&linfo->lock); - if (ret < 0) { - if (!lock->error) - lock->error = ret; - lock->work_prev_mode = DLM_LOCK_IV; - lock->work_mode = DLM_LOCK_IV; - lock_process(linfo, lock); - } + lock->invalidate_pending = 0; + + trace_scoutfs_lock_invalidated(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); spin_unlock(&linfo->lock); + + return 0; } -/* - * The grace period has elapsed since a down conversion attempt too soon - * after an unlock. It can now be down converted. - */ -static void scoutfs_lock_grace_work(struct work_struct *work) +static bool lock_wait_cond(struct super_block *sb, struct scoutfs_lock *lock, + int mode) { - struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - grace_work.work); - struct super_block *sb = lock->sb; DECLARE_LOCK_INFO(sb, linfo); - - BUG_ON(lock->grace_pending == false); + bool wake; spin_lock(&linfo->lock); - trace_scoutfs_lock_grace_work(sb, lock); - scoutfs_inc_counter(linfo->sb, lock_grace_expired); - lock->grace_pending = false; - lock_process(linfo, lock); + wake = linfo->shutdown || lock_modes_match(lock->mode, mode) || + !lock->request_pending; spin_unlock(&linfo->lock); + + if (!wake) + scoutfs_inc_counter(sb, lock_wait); + + return wake; } -/* - * Wait for a lock attempt to be resolved. We return as an active user - * once our mode is satisfied by the lock or we can return errors. - */ -static bool lock_wait(struct lock_info *linfo, struct scoutfs_lock *lock, - int mode, int flags, int *ret) +static bool lock_flags_invalid(int flags) { - struct super_block *sb = linfo->sb; - bool done; - - spin_lock(&linfo->lock); - - trace_scoutfs_lock_wait(sb, lock); - - if (lock_modes_match(lock->granted_mode, mode)) { - /* the fast path where we can use the granted mode */ - lock_dec_count(lock->waiters, mode); - lock_inc_count(lock->users, mode); - *ret = 0; - done = true; - - } else if (linfo->shutdown) { - /* locking is going away */ - *ret = -ESHUTDOWN; - done = true; - - } else if (lock->error) { - /* something horrible has happened */ - *ret = lock->error; - done = true; - - } else if (flags & SCOUTFS_LKF_NONBLOCK) { - /* never wait for "nonblocking" callers */ - scoutfs_inc_counter(sb, lock_nonblock_eagain); - *ret = -EAGAIN; - done = true; - - } else { - /* still waiting :/ */ - *ret = 0; - done = false; - } - - lock_process(linfo, lock); - - spin_unlock(&linfo->lock); - - return done; + return flags & SCOUTFS_LKF_INVALID; } /* @@ -794,121 +778,145 @@ static bool lock_wait(struct lock_info *linfo, struct scoutfs_lock *lock, * holding the lock the cache won't be invalidated and other conflicting * lock users will be serialized. The item cache can be invalidated * once the lock is unlocked. + * + * If we don't have a granted lock then we send a request for our + * desired mode if there isn't one in flight already. This can be + * racing with an invalidation request from the server. The server + * won't process our request until it receives our invalidation + * response. */ -static int lock_name_keys(struct super_block *sb, int mode, int flags, - struct scoutfs_lock_name *name, +static int lock_key_range(struct super_block *sb, int mode, int flags, struct scoutfs_key *start, struct scoutfs_key *end, struct scoutfs_lock **ret_lock) { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; - struct scoutfs_lock *ins; - int wait_ret; + struct scoutfs_net_lock nl; + bool should_send; int ret; scoutfs_inc_counter(sb, lock_lock); *ret_lock = NULL; - /* maybe catch _setup() order mistakes */ - if (WARN_ON_ONCE(!linfo || linfo->lockspace == NULL)) + if (WARN_ON_ONCE(!start || !end) || + WARN_ON_ONCE(lock_mode_invalid(mode)) || + WARN_ON_ONCE(lock_flags_invalid(flags))) + return -EINVAL; + + /* maybe catch _setup() and _shutdown order mistakes */ + if (WARN_ON_ONCE(!linfo || linfo->shutdown)) return -ENOLCK; /* have to lock before entering transactions */ if (WARN_ON_ONCE(scoutfs_trans_held())) return -EDEADLK; - ins = NULL; -retry: spin_lock(&linfo->lock); - /* don't create locks once we're shutdown */ - if (linfo->shutdown) { - spin_unlock(&linfo->lock); - ret = -ESHUTDOWN; - goto out; - } - - lock = lock_rb_walk(sb, name, ins); + /* drops and re-acquires lock if it allocates */ + lock = create_lock(sb, start, end); if (!lock) { - spin_unlock(&linfo->lock); - ins = lock_alloc(sb, name, start, end); - if (!ins) { - ret = -ENOMEM; - goto out; - } - goto retry; - - } else if (lock == ins) { - if (start && !insert_range_node(sb, ins)) { - lock_free(linfo, ins); - spin_unlock(&linfo->lock); - ret = -EINVAL; - goto out; - } - - } else if (ins) { - lock_free(linfo, ins); + ret = -ENOMEM; + goto out_unlock; } + /* the waiters count is only used by debugging output */ lock_inc_count(lock->waiters, mode); + + for (;;) { + if (linfo->shutdown) { + ret = -ESHUTDOWN; + break; + } + + /* the fast path where we can use the granted mode */ + if (lock_modes_match(lock->mode, mode)) { + lock_inc_count(lock->users, mode); + *ret_lock = lock; + ret = 0; + break; + } + + /* non-blocking callers don't wait or send requests */ + if (flags & SCOUTFS_LKF_NONBLOCK) { + scoutfs_inc_counter(sb, lock_nonblock_eagain); + ret = -EAGAIN; + break; + } + + if (!lock->request_pending) { + lock->request_pending = 1; + should_send = true; + } else { + should_send = false; + } + + spin_unlock(&linfo->lock); + + if (should_send) { + nl.key = lock->start; + nl.old_mode = lock->mode; + nl.new_mode = mode; + + ret = scoutfs_client_lock_request(sb, &nl); + if (ret) { + spin_lock(&linfo->lock); + lock->request_pending = 0; + break; + } + scoutfs_inc_counter(sb, lock_grant_request); + } + + trace_scoutfs_lock_wait(sb, lock); + + ret = wait_event_interruptible(lock->waitq, + lock_wait_cond(sb, lock, mode)); + spin_lock(&linfo->lock); + if (ret) + break; + } + + lock_dec_count(lock->waiters, mode); + + if (ret == 0) + trace_scoutfs_lock_locked(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); + +out_unlock: spin_unlock(&linfo->lock); - ret = wait_event_interruptible(lock->waitq, - lock_wait(linfo, lock, mode, flags, - &wait_ret)); - if (ret == 0) - ret = wait_ret; - if (ret) { + if (ret && ret != -EAGAIN && ret != -ERESTARTSYS) scoutfs_inc_counter(sb, lock_lock_error); - spin_lock(&linfo->lock); - lock_dec_count(lock->waiters, mode); - lock_process(linfo, lock); - spin_unlock(&linfo->lock); - } else { - *ret_lock = lock; - } -out: + return ret; } int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, struct scoutfs_lock **ret_lock) { - struct scoutfs_lock_name name; struct scoutfs_key start; struct scoutfs_key end; - ino &= ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK; + scoutfs_key_set_zeros(&start); + start.sk_zone = SCOUTFS_FS_ZONE; + start.ski_ino = cpu_to_le64(ino & ~(u64)SCOUTFS_LOCK_INODE_GROUP_MASK); - name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - name.zone = SCOUTFS_FS_ZONE; - name.type = SCOUTFS_INODE_TYPE; - name.first = cpu_to_le64(ino); - name.second = 0; + scoutfs_key_set_ones(&end); + end.sk_zone = SCOUTFS_FS_ZONE; + end.ski_ino = cpu_to_le64(ino | SCOUTFS_LOCK_INODE_GROUP_MASK); - start = (struct scoutfs_key) { - .sk_zone = SCOUTFS_FS_ZONE, - .ski_ino = cpu_to_le64(ino), - .sk_type = 0, - }; - - end = (struct scoutfs_key) { - .sk_zone = SCOUTFS_FS_ZONE, - .ski_ino = cpu_to_le64(ino + SCOUTFS_LOCK_INODE_GROUP_NR - 1), - .sk_type = U8_MAX, - }; - - return lock_name_keys(sb, mode, flags, &name, &start, &end, ret_lock); + return lock_key_range(sb, mode, flags, &start, &end, ret_lock); } /* * Acquire a lock on an inode. * * _REFRESH_INODE indicates that the caller needs to have the vfs inode - * fields current with respect to lock coverage. dlmglue increases the - * lock's refresh_gen once every time its mode is changed from a mode - * that couldn't have the inode cached to one that could. + * fields current with respect to lock coverage. The lock's refresh_gen + * is incremented as new locks are acquired and then indicates that an + * old inode with a smaller refresh_gen needs to be refreshed. */ int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **lock) @@ -1024,13 +1032,12 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, struct scoutfs_lock **lock) { - struct scoutfs_lock_name name; + struct scoutfs_key key = { + .sk_zone = SCOUTFS_LOCK_ZONE, + .sk_type = SCOUTFS_RENAME_TYPE, + }; - memset(&name, 0, sizeof(name)); - name.scope = SCOUTFS_LOCK_SCOPE_GLOBAL; - name.type = SCOUTFS_LOCK_TYPE_GLOBAL_RENAME; - - return lock_name_keys(sb, mode, flags, &name, NULL, NULL, lock); + return lock_key_range(sb, mode, flags, &key, &key, lock); } /* @@ -1066,28 +1073,19 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, } /* - * Lock the given index item. We use the index masks to name a reasonable - * batch of logical items to lock and calculate the start and end - * key values that are covered by the lock. - * + * Lock the given index item. We use the index masks to calculate the + * start and end key values that are covered by the lock. */ int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock) { - struct scoutfs_lock_name name; struct scoutfs_key start; struct scoutfs_key end; scoutfs_lock_get_index_item_range(type, major, ino, &start, &end); - name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - name.zone = start.sk_zone; - name.type = start.sk_type; - name.first = start.skii_major; - name.second = start.skii_ino; - - return lock_name_keys(sb, mode, 0, &name, &start, &end, ret_lock); + return lock_key_range(sb, mode, 0, &start, &end, ret_lock); } /* @@ -1104,35 +1102,23 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock) { - struct scoutfs_lock_name name; struct scoutfs_key start; struct scoutfs_key end; - name.scope = SCOUTFS_LOCK_SCOPE_FS_ITEMS; - name.zone = SCOUTFS_NODE_ZONE; - name.type = 0; - name.first = cpu_to_le64(node_id); - name.second = 0; + scoutfs_key_set_zeros(&start); + start.sk_zone = SCOUTFS_NODE_ZONE; + start.sko_node_id = cpu_to_le64(node_id); - start = (struct scoutfs_key) { - .sk_zone = SCOUTFS_NODE_ZONE, - .sko_node_id = cpu_to_le64(node_id), - .sk_type = 0, - }; + scoutfs_key_set_ones(&end); + end.sk_zone = SCOUTFS_NODE_ZONE; + end.sko_node_id = cpu_to_le64(node_id); - end = (struct scoutfs_key) { - .sk_zone = SCOUTFS_NODE_ZONE, - .sko_node_id = cpu_to_le64(node_id), - .sk_type = U8_MAX, - }; - - return lock_name_keys(sb, mode, flags, &name, &start, &end, lock); + return lock_key_range(sb, mode, flags, &start, &end, lock); } /* - * As we unlock we start a grace period. If a bast arrives before the - * grace period we'll wait for another full grace period we downconvert - * and invalidate the lock. Each unlock resets the downconvert delay. + * As we unlock we always extend the grace period to give the caller + * another pass at the lock before its invalidated. */ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) { @@ -1144,17 +1130,14 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) scoutfs_inc_counter(sb, lock_unlock); spin_lock(&linfo->lock); - trace_scoutfs_lock_unlock(sb, lock); lock_dec_count(lock->users, mode); - lock->grace_deadline = ktime_add(ktime_get(), GRACE_UNLOCK_DEADLINE_KT); - if (cancel_delayed_work(&lock->grace_work)) { - scoutfs_inc_counter(linfo->sb, lock_grace_extended); - queue_delayed_work(linfo->workq, &lock->grace_work, - GRACE_WORK_DELAY_JIFFIES); - } + extend_grace(sb, lock); + + trace_scoutfs_lock_unlock(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); - lock_process(linfo, lock); spin_unlock(&linfo->lock); } @@ -1217,6 +1200,54 @@ void scoutfs_lock_del_coverage(struct super_block *sb, spin_unlock(&cov->cov_lock); } +/* + * The shrink callback got the lock, marked it request_pending, and + * handed it off to us. We kick off a null request and the lock will + * be freed by the response once all users drain. If this races with + * invalidation then the server will only send the grant response once + * the invalidation is finished. + */ +static void scoutfs_lock_shrink_worker(struct work_struct *work) +{ + struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, + shrink_work); + struct super_block *sb = lock->sb; + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_net_lock nl; + int ret; + + /* unlocked lock access, but should be stable since we queued */ + nl.key = lock->start; + nl.old_mode = lock->mode; + nl.new_mode = SCOUTFS_LOCK_NULL; + + ret = scoutfs_client_lock_request(sb, &nl); + if (ret) { + /* oh well, not freeing */ + scoutfs_inc_counter(sb, lock_shrink_request_aborted); + + spin_lock(&linfo->lock); + + lock->request_pending = 0; + wake_up(&lock->waitq); + put_lock(linfo, lock); + + spin_unlock(&linfo->lock); + } +} + +/* + * Start the shrinking process for locks on the lru. If a lock is on + * the lru then it can't have any active users. We don't want to block + * or allocate here so all we do is get the lock, mark it request + * pending, and kick off the work. The work sends a null request and + * eventually the lock is freed by its response. + * + * Only a racing lock attempt that isn't matched can prevent the lock + * from being freed. It'll block waiting to send its request for its + * mode which will prevent the lock from being freed when the null + * response arrives. + */ static int scoutfs_lock_shrink(struct shrinker *shrink, struct shrink_control *sc) { @@ -1234,24 +1265,27 @@ static int scoutfs_lock_shrink(struct shrinker *shrink, spin_lock(&linfo->lock); +restart: list_for_each_entry_safe(lock, tmp, &linfo->lru_list, lru_head) { + BUG_ON(!lock_idle(lock)); + BUG_ON(lock->mode == SCOUTFS_LOCK_NULL); + if (nr-- == 0) break; + __lock_del_lru(linfo, lock); + lock->request_pending = 1; + queue_work(linfo->workq, &lock->shrink_work); + + scoutfs_inc_counter(sb, lock_shrink_queued); trace_scoutfs_lock_shrink(sb, lock); - scoutfs_inc_counter(sb, lock_shrink); - WARN_ON_ONCE(!lock_idle(lock)); - - lock->work_prev_mode = lock->granted_mode; - lock->work_mode = DLM_LOCK_NL; - lock->granted_mode = DLM_LOCK_NL; - queue_work(linfo->workq, &lock->work); - - list_del_init(&lock->lru_head); - linfo->lru_nr--; + /* could have bazillions of idle locks */ + if (cond_resched_lock(&linfo->lock)) + goto restart; } + spin_unlock(&linfo->lock); out: @@ -1276,20 +1310,15 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) struct scoutfs_lock *lock = container_of(ent, struct scoutfs_lock, tseq_entry); - seq_printf(m, "name "LN_FMT" start "SK_FMT" end "SK_FMT" refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u dlmlksb: status %d lkid 0x%x flags 0x%x\n", - LN_ARG(&lock->name), SK_ARG(&lock->start), - SK_ARG(&lock->end), lock->refresh_gen, lock->error, - lock->granted_mode, lock->bast_mode, - lock->work_prev_mode, lock->work_mode, - lock->waiters[DLM_LOCK_PR], - lock->waiters[DLM_LOCK_EX], - lock->waiters[DLM_LOCK_CW], - lock->users[DLM_LOCK_PR], - lock->users[DLM_LOCK_EX], - lock->users[DLM_LOCK_CW], - lock->lksb.sb_status, - lock->lksb.sb_lkid, - lock->lksb.sb_flags); + seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u\n", + SK_ARG(&lock->start), SK_ARG(&lock->end), + lock->refresh_gen, lock->mode, + lock->waiters[SCOUTFS_LOCK_READ], + lock->waiters[SCOUTFS_LOCK_WRITE], + lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], + lock->users[SCOUTFS_LOCK_READ], + lock->users[SCOUTFS_LOCK_WRITE], + lock->users[SCOUTFS_LOCK_WRITE_ONLY]); } /* @@ -1326,7 +1355,11 @@ void scoutfs_lock_shutdown(struct super_block *sb) * There should be no active users of locks and all future lock calls * should fail. * - * Our job is to make sure nothing references the locks and free them. + * The client networking connection will have been shutdown so we don't + * get any request or response processing calls. + * + * Our job is to make sure nothing references the remaining locks and + * free them. */ void scoutfs_lock_destroy(struct super_block *sb) { @@ -1335,7 +1368,6 @@ void scoutfs_lock_destroy(struct super_block *sb) struct scoutfs_lock *lock; struct rb_node *node; int mode; - int ret; if (!linfo) return; @@ -1352,35 +1384,15 @@ void scoutfs_lock_destroy(struct super_block *sb) for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { if (lock->waiters[mode] || lock->users[mode]) { - scoutfs_warn(sb, "lock name "LN_FMT" start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", - LN_ARG(&lock->name), + scoutfs_warn(sb, "lock start "SK_FMT" end "SK_FMT" has mode %d user after shutdown", SK_ARG(&lock->start), SK_ARG(&lock->end), mode); break; } } - - if (cancel_delayed_work(&lock->grace_work)) - lock->grace_pending = false; - } spin_unlock(&linfo->lock); - /* stop the dlm from calling our asts or basts to queue work */ - if (linfo->lockspace) { - /* - * fs/dlm has a harmless but unannotated inversion between their - * connection and socket locking that triggers during shutdown - * and disables lockdep. - */ - lockdep_off(); - ret = dlm_release_lockspace(linfo->lockspace, 2); - lockdep_on(); - if (ret) - scoutfs_warn(sb, "dlm lockspace leave failure: %d", - ret); - } - if (linfo->workq) { /* pending grace work queues normal work */ flush_workqueue(linfo->workq); @@ -1391,12 +1403,18 @@ void scoutfs_lock_destroy(struct super_block *sb) /* XXX does anything synchronize with open debugfs fds? */ debugfs_remove(linfo->tseq_dentry); - /* free our stale locks that now describe released dlm locks */ + /* + * This is very clumsy and brute force. This will be cleaned up + * as we add proper lock recovery. + */ spin_lock(&linfo->lock); node = rb_first(&linfo->lock_tree); while (node) { lock = rb_entry(node, struct scoutfs_lock, node); node = rb_next(node); + if (!list_empty(&lock->lru_head)) + __lock_del_lru(linfo, lock); + lock_remove(linfo, lock); lock_free(linfo, lock); } spin_unlock(&linfo->lock); @@ -1408,17 +1426,9 @@ void scoutfs_lock_destroy(struct super_block *sb) int scoutfs_lock_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - char name[DLM_LOCKSPACE_LEN]; struct lock_info *linfo; int ret; - /* we use >= 0 to test iv and use modes as an array index */ - BUILD_BUG_ON(DLM_LOCK_IV >= 0); - BUILD_BUG_ON(DLM_LOCK_NL >= SCOUTFS_LOCK_NR_MODES); - BUILD_BUG_ON(DLM_LOCK_PR >= SCOUTFS_LOCK_NR_MODES); - BUILD_BUG_ON(DLM_LOCK_EX >= SCOUTFS_LOCK_NR_MODES); - BUILD_BUG_ON(DLM_LOCK_CW >= SCOUTFS_LOCK_NR_MODES); - linfo = kzalloc(sizeof(struct lock_info), GFP_KERNEL); if (!linfo) return -ENOMEM; @@ -1437,14 +1447,15 @@ int scoutfs_lock_setup(struct super_block *sb) sbi->lock_info = linfo; trace_scoutfs_lock_setup(sb, linfo); - linfo->tseq_dentry = scoutfs_tseq_create("locks", sbi->debug_root, + linfo->tseq_dentry = scoutfs_tseq_create("client_locks", + sbi->debug_root, &linfo->tseq_tree); if (!linfo->tseq_dentry) { ret = -ENOMEM; goto out; } - linfo->workq = alloc_workqueue("scoutfs_lock_work", + linfo->workq = alloc_workqueue("scoutfs_lock_client_work", WQ_NON_REENTRANT | WQ_UNBOUND | WQ_HIGHPRI, 0); if (!linfo->workq) { @@ -1452,15 +1463,7 @@ int scoutfs_lock_setup(struct super_block *sb) goto out; } - snprintf(name, DLM_LOCKSPACE_LEN, "scoutfs_fsid_%llx", - le64_to_cpu(sbi->super.hdr.fsid)); - - ret = dlm_new_lockspace(name, sbi->opts.cluster_name, - DLM_LSFL_FS | DLM_LSFL_NEWEXCL, 8, - NULL, NULL, NULL, &linfo->lockspace); - if (ret) - scoutfs_warn(sb, "dlm lockspace [%s, %s] join failure: %d", - sbi->opts.cluster_name, name, ret); + ret = 0; out: if (ret) scoutfs_lock_destroy(sb); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 89d161e8..b3920ed3 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -1,14 +1,14 @@ #ifndef _SCOUTFS_LOCK_H_ #define _SCOUTFS_LOCK_H_ -#include #include "key.h" #include "tseq.h" #define SCOUTFS_LKF_REFRESH_INODE 0x01 /* update stale inode from item */ #define SCOUTFS_LKF_NONBLOCK 0x02 /* only use already held locks */ +#define SCOUTFS_LKF_INVALID (~((SCOUTFS_LKF_NONBLOCK << 1) - 1)) -#define SCOUTFS_LOCK_NR_MODES (DLM_LOCK_EX + 1) +#define SCOUTFS_LOCK_NR_MODES SCOUTFS_LOCK_INVALID /* * A few fields (start, end, refresh_gen, granted_mode) are referenced @@ -16,29 +16,22 @@ */ struct scoutfs_lock { struct super_block *sb; - struct scoutfs_lock_name name; struct scoutfs_key start; struct scoutfs_key end; struct rb_node node; struct rb_node range_node; - unsigned int debug_locks_id; u64 refresh_gen; struct list_head lru_head; wait_queue_head_t waitq; - struct work_struct work; - struct dlm_lksb lksb; + struct work_struct shrink_work; ktime_t grace_deadline; - struct delayed_work grace_work; - bool grace_pending; + unsigned long request_pending:1, + invalidate_pending:1; spinlock_t cov_list_lock; struct list_head cov_list; - int error; - int granted_mode; - int bast_mode; - int work_prev_mode; - int work_mode; + int mode; unsigned int waiters[SCOUTFS_LOCK_NR_MODES]; unsigned int users[SCOUTFS_LOCK_NR_MODES]; @@ -51,6 +44,11 @@ struct scoutfs_lock_coverage { struct list_head head; }; +int scoutfs_lock_grant_response(struct super_block *sb, + struct scoutfs_net_lock *nl); +int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock *nl); + int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3dadf912..26e244ba 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1578,32 +1578,24 @@ DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_end, TP_ARGS(sb, rng, start, end) ); -#define lock_mode(mode) \ - __print_symbolic(mode, \ - { DLM_LOCK_IV, "IV" }, \ - { DLM_LOCK_NL, "NL" }, \ - { DLM_LOCK_CR, "CR" }, \ - { DLM_LOCK_CW, "CW" }, \ - { DLM_LOCK_PR, "PR" }, \ - { DLM_LOCK_PW, "PW" }, \ - { DLM_LOCK_EX, "EX" }) +#define lock_mode(mode) \ + __print_symbolic(mode, \ + { SCOUTFS_LOCK_NULL, "NULL" }, \ + { SCOUTFS_LOCK_READ, "READ" }, \ + { SCOUTFS_LOCK_WRITE, "WRITE" }, \ + { SCOUTFS_LOCK_WRITE_ONLY, "WRITE_ONLY" }) DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck), TP_STRUCT__entry( __field(__u64, fsid) - __field(u8, name_scope) - __field(u8, name_zone) - __field(u8, name_type) - __field(u64, name_first) - __field(u64, name_second) + sk_trace_define(start) + sk_trace_define(end) __field(u64, refresh_gen) - __field(int, error) - __field(int, granted_mode) - __field(int, bast_mode) - __field(int, work_prev_mode) - __field(int, work_mode) + __field(unsigned char, request_pending) + __field(unsigned char, invalidate_pending) + __field(int, mode) __field(unsigned int, waiters_cw) __field(unsigned int, waiters_pr) __field(unsigned int, waiters_ex) @@ -1613,33 +1605,29 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, ), TP_fast_assign( __entry->fsid = FSID_ARG(sb); - __entry->name_scope = lck->name.scope; - __entry->name_zone = lck->name.zone; - __entry->name_type = lck->name.type; - __entry->name_first = le64_to_cpu(lck->name.first); - __entry->name_second = le64_to_cpu(lck->name.second); - + sk_trace_assign(start, &lck->start); + sk_trace_assign(end, &lck->end); __entry->refresh_gen = lck->refresh_gen; - __entry->error = lck->error; - __entry->granted_mode = lck->granted_mode; - __entry->bast_mode = lck->bast_mode; - __entry->work_prev_mode = lck->work_prev_mode; - __entry->work_mode = lck->work_mode; - __entry->waiters_pr = lck->waiters[DLM_LOCK_PR]; - __entry->waiters_ex = lck->waiters[DLM_LOCK_EX]; - __entry->waiters_cw = lck->waiters[DLM_LOCK_CW]; - __entry->users_pr = lck->users[DLM_LOCK_PR]; - __entry->users_ex = lck->users[DLM_LOCK_EX]; - __entry->users_cw = lck->users[DLM_LOCK_CW]; + __entry->request_pending = lck->request_pending; + __entry->invalidate_pending = lck->invalidate_pending; + __entry->mode = lck->mode; + __entry->waiters_pr = lck->waiters[SCOUTFS_LOCK_READ]; + __entry->waiters_ex = lck->waiters[SCOUTFS_LOCK_WRITE]; + __entry->waiters_cw = lck->waiters[SCOUTFS_LOCK_WRITE_ONLY]; + __entry->users_pr = lck->users[SCOUTFS_LOCK_READ]; + __entry->users_ex = lck->users[SCOUTFS_LOCK_WRITE]; + __entry->users_cw = lck->users[SCOUTFS_LOCK_WRITE_ONLY]; ), - TP_printk("fsid "FSID_FMT" name %u.%u.%u.%llu.%llu refresh_gen %llu error %d granted %d bast %d prev %d work %d waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", - __entry->fsid, __entry->name_scope, __entry->name_zone, - __entry->name_type, __entry->name_first, __entry->name_second, - __entry->refresh_gen, __entry->error, __entry->granted_mode, - __entry->bast_mode, __entry->work_prev_mode, - __entry->work_mode, __entry->waiters_pr, - __entry->waiters_ex, __entry->waiters_cw, __entry->users_pr, - __entry->users_ex, __entry->users_cw) + TP_printk("fsid "FSID_FMT" start "SK_FMT" end "SK_FMT" mode %u reqpnd %u invpnd %u rfrgen %llu waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", + __entry->fsid, sk_trace_args(start), sk_trace_args(end), + __entry->mode, __entry->request_pending, + __entry->invalidate_pending, __entry->refresh_gen, + __entry->waiters_pr, __entry->waiters_ex, __entry->waiters_cw, + __entry->users_pr, __entry->users_ex, __entry->users_cw) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) ); DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_free, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), @@ -1649,19 +1637,23 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_alloc, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_ast, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_grant_response, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_bast, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_granted, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_work, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidate_request, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); -DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_grace_work, +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_invalidated, + TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), + TP_ARGS(sb, lck) +); +DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_locked, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck) ); diff --git a/kmod/src/super.c b/kmod/src/super.c index 15387737..2e9e6875 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -123,7 +123,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_data_destroy(sb); - scoutfs_unlock(sb, sbi->node_id_lock, DLM_LOCK_EX); + scoutfs_unlock(sb, sbi->node_id_lock, SCOUTFS_LOCK_WRITE); sbi->node_id_lock = NULL; scoutfs_shutdown_trans(sb); @@ -333,7 +333,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: scoutfs_client_wait_node_id(sb) ?: - scoutfs_lock_node_id(sb, DLM_LOCK_EX, 0, sbi->node_id, + scoutfs_lock_node_id(sb, SCOUTFS_LOCK_WRITE, 0, sbi->node_id, &sbi->node_id_lock); if (ret) goto out; diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 877626a4..9fe08579 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -291,7 +291,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, if (!xat) return -ENOMEM; - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lck); if (ret) goto out; @@ -301,7 +301,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, name, name_len, 0, 0, lck); up_read(&si->xattr_rwsem); - scoutfs_unlock(sb, lck, DLM_LOCK_PR); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ); if (ret < 0) { if (ret == -ENOENT) @@ -385,8 +385,8 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, goto out; } - ret = scoutfs_lock_inode(sb, DLM_LOCK_EX, SCOUTFS_LKF_REFRESH_INODE, - inode, &lck); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lck); if (ret) goto out; @@ -466,7 +466,7 @@ release: scoutfs_inode_index_unlock(sb, &ind_locks); unlock: up_write(&si->xattr_rwsem); - scoutfs_unlock(sb, lck, DLM_LOCK_EX); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE); out: kfree(xat); @@ -509,7 +509,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) goto out; } - ret = scoutfs_lock_inode(sb, DLM_LOCK_PR, 0, inode, &lck); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lck); if (ret) goto out; @@ -546,7 +546,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) } up_read(&si->xattr_rwsem); - scoutfs_unlock(sb, lck, DLM_LOCK_PR); + scoutfs_unlock(sb, lck, SCOUTFS_LOCK_READ); out: kfree(xat); From 288d78164570a48ffd0aab9320b76be4f7f028bc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 11 Oct 2018 15:09:12 -0700 Subject: [PATCH 685/920] scoutfs: start and stop server with quorum Currently all mounts try to get a dlm lock which gives them exclusive access to become the server for the filesystem. That isn't going to work if we're moving to locking provided by the server. This uses quorum election to determine who should run the server. We switch from long running server work blocked trying to get a lock to calls which start and stop the server. Signed-off-by: Zach Brown --- kmod/src/client.c | 158 ++++++++++++++++++++++++++------------------- kmod/src/format.h | 1 - kmod/src/options.c | 21 ------ kmod/src/options.h | 8 +-- kmod/src/server.c | 136 +++++++++++++++++--------------------- kmod/src/server.h | 7 ++ 6 files changed, 158 insertions(+), 173 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 9ef4c247..81603385 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -33,19 +33,17 @@ #include "client.h" #include "net.h" #include "endian_swap.h" +#include "quorum.h" /* - * The client always maintains a connection to the server. It reads the - * super to get the address it should try and connect to. + * The client is responsible for maintaining a connection to the server. + * This includes managing quorum elections that determine which client + * should run the server that all the clients connect to. */ -/* - * Connection timeouts have to allow for enough time for servers to - * reboot. Figure order minutes at the outside. - */ -#define CONN_RETRY_MIN_MS 10UL -#define CONN_RETRY_MAX_MS (5UL * MSEC_PER_SEC) -#define CONN_RETRY_LIMIT_J (5 * 60 * HZ) +#define CLIENT_CONNECT_DELAY_MS (MSEC_PER_SEC / 10) +#define CLIENT_CONNECT_TIMEOUT_MS (1 * MSEC_PER_SEC) +#define CLIENT_QUORUM_TIMEOUT_MS (5 * MSEC_PER_SEC) struct client_info { struct super_block *sb; @@ -55,23 +53,12 @@ struct client_info { atomic_t shutting_down; struct workqueue_struct *workq; - struct delayed_work connect_dwork; + struct work_struct connect_work; - /* connection timeouts are tracked across attempts */ - unsigned long conn_retry_ms; + struct scoutfs_quorum_elected_info qei; + u64 old_elected_nr; }; -static void reset_connect_timeout(struct client_info *client) -{ - client->conn_retry_ms = CONN_RETRY_MIN_MS; -} - -static void grow_connect_timeout(struct client_info *client) -{ - client->conn_retry_ms = min(client->conn_retry_ms * 2, - CONN_RETRY_MAX_MS); -} - /* * Ask for a new run of allocated inode numbers. The server can return * fewer than @count. It will success with nr == 0 if we've run out. @@ -346,49 +333,96 @@ out: } /* - * Attempt to connect to the listening address that the server wrote in - * the super block. We keep trying indefinitely with an increasing - * delay if we fail to either read the address or connect to it. + * If the previous election told us to start the server then stop it + * and wipe the old election info. If we're not fast enough to clear + * the election block then the next server might fence us. Should + * be very unlikely as election requires multiple RMW cycles. + */ +static void stop_our_server(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei) +{ + if (qei->run_server) { + scoutfs_server_stop(sb); + scoutfs_quorum_clear_elected(sb, qei); + memset(qei, 0, sizeof(*qei)); + } +} + +/* + * This work is responsible for managing leader elections, running the + * server, and connecting clients to the server. * - * We're careful to only ever have one connection attempt in flight. We - * only queue this work on mount, on error, or from the notify_down - * callback. + * In the typical case a mount reads the quorum blocks and finds the + * address of the currently running server and connects to it. + * + * More rarely clients who aren't connected and are configured to + * participate in quorum need to elect the new leader. The elected info + * filled by quorum tells us if we were elected to run the server. + * + * This leads to the possibility that the mount who is running the + * server had its mount disconnect. This is only weirdly different from + * other clients disconnecting and trying to reconnect because of the + * way quorum slots are reconfigured and reclaimed. If we connect to a + * server with the new quorum config then we can't have any old servers + * running in the stale old quorum slot. The simplest way to do this is + * to *always* stop the server if we're running it and we got + * disconnected. It's a big hammer, but it's reliable, and arguably if + * *we* couldn't' use *our* server then something bad is happening and + * someone else should be the server. + * + * This only executes on mount, error, or as a connection disconnects + * and there's only ever one executing. */ static void scoutfs_client_connect_worker(struct work_struct *work) { struct client_info *client = container_of(work, struct client_info, - connect_dwork.work); + connect_work); struct super_block *sb = client->sb; + struct scoutfs_quorum_elected_info *qei = &client->qei; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct mount_options *opts = &sbi->opts; struct scoutfs_net_greeting greet; - struct scoutfs_super_block super; - struct sockaddr_in sin; + ktime_t timeout_abs; int ret; - ret = scoutfs_read_super(sb, &super); + /* don't try quorum and connecting while our mount runs a server */ + stop_our_server(sb, qei); + + timeout_abs = ktime_add_ms(ktime_get(), CLIENT_QUORUM_TIMEOUT_MS); + + ret = scoutfs_quorum_election(sb, opts->uniq_name, + client->old_elected_nr, + timeout_abs, qei); if (ret) goto out; - if (super.server_addr.addr == cpu_to_le32(INADDR_ANY)) { - ret = -EADDRNOTAVAIL; + if (qei->run_server) { + ret = scoutfs_server_start(sb, &qei->sin); + if (ret) { + /* forget that we tried to start the server */ + memset(qei, 0, sizeof(*qei)); + goto out; + } + } + + /* always give the server some time before connecting */ + msleep(CLIENT_CONNECT_DELAY_MS); + + ret = scoutfs_net_connect(sb, client->conn, &qei->sin, + CLIENT_CONNECT_TIMEOUT_MS); + if (ret) { + /* we couldn't connect, try electing a new server */ + client->old_elected_nr = qei->elected_nr; goto out; } - memset(&sin, 0, sizeof(sin)); - sin.sin_family = AF_INET; - sin.sin_addr.s_addr = le32_to_be32(super.server_addr.addr); - sin.sin_port = le16_to_be16(super.server_addr.port); - - ret = scoutfs_net_connect(sb, client->conn, &sin, - client->conn_retry_ms); - if (ret) - goto out; - - reset_connect_timeout(client); + /* trust this server again if it's still around after we disconnect */ + client->old_elected_nr = 0; /* send a greeting to verify endpoints of each connection */ - greet.fsid = super.id; - greet.format_hash = super.format_hash; + greet.fsid = super->id; + greet.format_hash = super->format_hash; greet.node_id = cpu_to_le64(sbi->node_id); ret = scoutfs_net_submit_request(sb, client->conn, @@ -397,13 +431,9 @@ static void scoutfs_client_connect_worker(struct work_struct *work) client_greeting, NULL, NULL); if (ret) scoutfs_net_shutdown(sb, client->conn); - out: - if (ret && !atomic_read(&client->shutting_down)) { - queue_delayed_work(client->workq, &client->connect_dwork, - msecs_to_jiffies(client->conn_retry_ms)); - grow_connect_timeout(client); - } + if (ret && !atomic_read(&client->shutting_down)) + queue_work(client->workq, &client->connect_work); } /* @@ -474,11 +504,8 @@ static void client_notify_down(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; - if (!atomic_read(&client->shutting_down)) { - queue_delayed_work(client->workq, &client->connect_dwork, - msecs_to_jiffies(client->conn_retry_ms)); - grow_connect_timeout(client); - } + if (!atomic_read(&client->shutting_down)) + queue_work(client->workq, &client->connect_work); } /* @@ -509,8 +536,7 @@ int scoutfs_client_setup(struct super_block *sb) client->sb = sb; init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); - INIT_DELAYED_WORK(&client->connect_dwork, - scoutfs_client_connect_worker); + INIT_WORK(&client->connect_work, scoutfs_client_connect_worker); client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, client_req_funcs, "client"); @@ -525,10 +551,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - reset_connect_timeout(client); - /* delay initial connect to give a local server some time to setup */ - queue_delayed_work(client->workq, &client->connect_dwork, - msecs_to_jiffies(client->conn_retry_ms)); + queue_work(client->workq, &client->connect_work); ret = 0; out: @@ -552,13 +575,16 @@ void scoutfs_client_destroy(struct super_block *sb) atomic_set(&client->shutting_down, 1); /* make sure worker isn't using the conn */ - cancel_delayed_work_sync(&client->connect_dwork); + cancel_work_sync(&client->connect_work); /* make racing conn use explode */ conn = client->conn; client->conn = NULL; scoutfs_net_free_conn(sb, conn); + /* stop running the server if we were, harmless otherwise */ + stop_our_server(sb, &client->qei); + if (client->workq) destroy_workqueue(client->workq); kfree(client); diff --git a/kmod/src/format.h b/kmod/src/format.h index 86cfe48d..a8ebb465 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -453,7 +453,6 @@ struct scoutfs_super_block { __le64 next_compact_id; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; - struct scoutfs_inet_addr server_addr; struct scoutfs_quorum_config quorum_config; } __packed; diff --git a/kmod/src/options.c b/kmod/src/options.c index 2f1bef8d..8fa7ab50 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -27,8 +27,6 @@ #include "super.h" static const match_table_t tokens = { - {Opt_listen, "listen=%s"}, - {Opt_cluster, "cluster=%s"}, {Opt_uniq_name, "uniq_name=%s"}, {Opt_err, NULL} }; @@ -55,15 +53,12 @@ u32 scoutfs_option_u32(struct super_block *sb, int token) int scoutfs_parse_options(struct super_block *sb, char *options, struct mount_options *parsed) { - char ipstr[INET_ADDRSTRLEN + 1]; substring_t args[MAX_OPT_ARGS]; int token, len; - __be32 addr; char *p; /* Set defaults */ memset(parsed, 0, sizeof(*parsed)); - strcpy(parsed->cluster_name, "scoutfs"); while ((p = strsep(&options, ",")) != NULL) { if (!*p) @@ -71,22 +66,6 @@ int scoutfs_parse_options(struct super_block *sb, char *options, token = match_token(p, tokens, args); switch (token) { - case Opt_listen: - match_strlcpy(ipstr, args, ARRAY_SIZE(ipstr)); - addr = in_aton(ipstr); - if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) || - ipv4_is_zeronet(addr) || ipv4_is_local_multicast(addr)) - return -EINVAL; - parsed->listen_addr.addr = - cpu_to_le32(be32_to_cpu(addr)); - break; - case Opt_cluster: - len = args[0].to - args[0].from; - if (len == 0 || len > (MAX_CLUSTER_NAME_LEN - 1)) - return -EINVAL; - match_strlcpy(parsed->cluster_name, args, - MAX_CLUSTER_NAME_LEN); - break; case Opt_uniq_name: len = match_strlcpy(parsed->uniq_name, args, SCOUTFS_UNIQUE_NAME_MAX_BYTES); diff --git a/kmod/src/options.h b/kmod/src/options.h index a1fffacf..a26df0e9 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -5,8 +5,6 @@ #include "format.h" enum { - Opt_listen = 0, - Opt_cluster, /* * For debugging we can quickly create huge trees by limiting * the number of items in each block as though the blocks were tiny. @@ -16,11 +14,7 @@ enum { Opt_err, }; -#define MAX_CLUSTER_NAME_LEN 17 -struct mount_options -{ - struct scoutfs_inet_addr listen_addr; - char cluster_name[MAX_CLUSTER_NAME_LEN]; +struct mount_options { char uniq_name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; }; diff --git a/kmod/src/server.c b/kmod/src/server.c index bf4047be..fba4f9eb 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -35,20 +35,14 @@ #include "lock_server.h" #include "endian_swap.h" -/* - * XXX pre commit: - * - comments - */ - /* * Every active mount can act as the server that listens on a net * connection and accepts connections from all the other mounts acting * as clients. * - * It queues long-lived work that blocks trying to acquire a lock. If - * it acquires the lock it listens on a socket and serves requests. If + * The server is started when raft elects the mount as the leader. If * it sees errors it shuts down the server in the hopes that another - * mount will have less trouble. + * mount will become the leader and have less trouble. */ struct server_info { @@ -57,9 +51,11 @@ struct server_info { wait_queue_head_t waitq; struct workqueue_struct *wq; - struct delayed_work dwork; - struct completion shutdown_comp; - bool bind_warned; + struct work_struct work; + int err; + bool shutting_down; + struct completion start_comp; + struct sockaddr_in listen_sin; struct scoutfs_net_connection *conn; /* request processing coordinates committing manifest and alloc */ @@ -495,9 +491,11 @@ static int remove_segno(struct super_block *sb, u64 segno) return ret; } -static void shutdown_server(struct server_info *server) +static void stop_server(struct server_info *server) { - complete(&server->shutdown_comp); + /* wait_event/wake_up provide barriers */ + server->shutting_down = true; + wake_up(&server->waitq); } /* @@ -1733,21 +1731,6 @@ out: trace_scoutfs_server_compact_work_exit(sb, 0, ret); } -/* - * This relies on the caller having read the current super and advanced - * its seq so that it's dirty. This will go away when we communicate - * the server address in a lock lvb. - */ -static int write_server_addr(struct super_block *sb, struct sockaddr_in *sin) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - super->server_addr.addr = be32_to_le32(sin->sin_addr.s_addr); - super->server_addr.port = be16_to_le16(sin->sin_port); - - return scoutfs_write_dirty_super(sb); -} - static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, @@ -1800,27 +1783,18 @@ static void server_notify_down(struct super_block *sb, forget_client_compacts(sb, sci); try_queue_compact(server); } else { - shutdown_server(server); + stop_server(server); } } -/* - * This work is always running or has a delayed timer set while a super - * is mounted. It tries to grab the lock to become the server. If it - * succeeds it publishes its address and accepts connections. If - * anything goes wrong it releases the lock and sets a timer to try to - * become the server all over again. - */ static void scoutfs_server_worker(struct work_struct *work) { struct server_info *server = container_of(work, struct server_info, - dwork.work); + work); struct super_block *sb = server->sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_connection *conn = NULL; - static struct sockaddr_in zeros = {0,}; - struct scoutfs_lock *lock = NULL; struct pending_seq *ps; struct pending_seq *ps_tmp; DECLARE_WAIT_QUEUE_HEAD(waitq); @@ -1830,13 +1804,6 @@ static void scoutfs_server_worker(struct work_struct *work) trace_scoutfs_server_work_enter(sb, 0, 0); - init_completion(&server->shutdown_comp); - - ret = scoutfs_lock_global(sb, DLM_LOCK_EX, 0, - SCOUTFS_LOCK_TYPE_GLOBAL_SERVER, &lock); - if (ret) - goto out; - conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, sizeof(struct server_client_info), server_req_funcs, "server"); @@ -1845,33 +1812,21 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - sin.sin_family = AF_INET; - sin.sin_addr.s_addr = le32_to_be32(sbi->opts.listen_addr.addr); - sin.sin_port = le16_to_be16(sbi->opts.listen_addr.port); + sin = server->listen_sin; - /* get the address of our listening socket */ ret = scoutfs_net_bind(sb, conn, &sin); if (ret) { - if (!server->bind_warned) { - scoutfs_err(sb, "server failed to bind to "SIN_FMT", errno %d%s. Retrying indefinitely..", - SIN_ARG(&sin), ret, - ret == -EADDRNOTAVAIL ? " (Bad address?)" - : ""); - server->bind_warned = true; - } + scoutfs_err(sb, "server failed to bind to "SIN_FMT", err %d%s", + SIN_ARG(&sin), ret, + ret == -EADDRNOTAVAIL ? " (Bad address?)" + : ""); goto out; } - /* publish the address for clients to connect to */ ret = scoutfs_read_super(sb, super); if (ret) goto out; - scoutfs_advance_dirty_super(sb); - ret = write_server_addr(sb, &sin); - if (ret) - goto out; - /* start up the server subsystems before accepting */ ret = scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: @@ -1879,6 +1834,8 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret) goto shutdown; + complete(&server->start_comp); + scoutfs_advance_dirty_super(sb); server->stable_manifest_root = super->manifest.root; @@ -1888,8 +1845,8 @@ static void scoutfs_server_worker(struct work_struct *work) server->conn = conn; scoutfs_net_listen(sb, conn); - /* wait for listening down or umount, conn can still be live */ - wait_for_completion_interruptible(&server->shutdown_comp); + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); @@ -1913,16 +1870,39 @@ shutdown: kfree(ps); } - write_server_addr(sb, &zeros); - out: scoutfs_net_free_conn(sb, conn); - scoutfs_unlock(sb, lock, DLM_LOCK_EX); - - /* always requeues, cancel_delayed_work_sync cancels on shutdown */ - queue_delayed_work(server->wq, &server->dwork, HZ / 2); trace_scoutfs_server_work_exit(sb, 0, ret); + + server->err = ret; + complete(&server->start_comp); +} + +/* XXX can we call start multiple times? */ +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin) +{ + DECLARE_SERVER_INFO(sb, server); + + server->err = 0; + server->shutting_down = false; + server->listen_sin = *sin; + init_completion(&server->start_comp); + + queue_work(server->wq, &server->work); + + wait_for_completion(&server->start_comp); + return server->err; +} + +void scoutfs_server_stop(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + stop_server(server); + /* XXX not sure both are needed */ + cancel_work_sync(&server->work); + cancel_work_sync(&server->commit_work); } int scoutfs_server_setup(struct super_block *sb) @@ -1937,9 +1917,7 @@ int scoutfs_server_setup(struct super_block *sb) server->sb = sb; spin_lock_init(&server->lock); init_waitqueue_head(&server->waitq); - init_completion(&server->shutdown_comp); - server->bind_warned = false; - INIT_DELAYED_WORK(&server->dwork, scoutfs_server_worker); + INIT_WORK(&server->work, scoutfs_server_worker); init_rwsem(&server->commit_rwsem); init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); @@ -1960,22 +1938,24 @@ int scoutfs_server_setup(struct super_block *sb) return -ENOMEM; } - queue_delayed_work(server->wq, &server->dwork, 0); - sbi->server_info = server; return 0; } +/* + * The caller should have already stopped but we do the same just in + * case. + */ void scoutfs_server_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct server_info *server = sbi->server_info; if (server) { - shutdown_server(server); + stop_server(server); /* wait for server work to wait for everything to shut down */ - cancel_delayed_work_sync(&server->dwork); + cancel_work_sync(&server->work); /* recv work/compaction could have left commit_work queued */ cancel_work_sync(&server->commit_work); diff --git a/kmod/src/server.h b/kmod/src/server.h index f2c9b8e7..01aed49b 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -48,6 +48,9 @@ do { \ __entry->name##_id, __entry->name##_data_len, __entry->name##_cmd, \ __entry->name##_flags, __entry->name##_error +struct scoutfs_net_manifest_entry; +struct scoutfs_manifest_entry; + void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, struct scoutfs_manifest_entry *ment); void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, @@ -58,6 +61,10 @@ int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, u64 id, struct scoutfs_net_lock *nl); +struct sockaddr_in; +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin); +void scoutfs_server_stop(struct super_block *sb); + int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); From 675275fbf1693522d050bda97f91fbdd3f68ff2f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Nov 2018 10:06:50 -0800 Subject: [PATCH 686/920] scoutfs: use hdr.fsid in greeting instead of id The network greeting exchange was mistakenly using the global super block magic number instead of the per-volume fsid to identify the volumes that the endpoints are working with. This prevented the check from doing its only job: to fail when clients in one volume try to connect to a server in another. Signed-off-by: Zach Brown --- kmod/src/client.c | 6 +++--- kmod/src/server.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 81603385..11bd1fe7 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -293,10 +293,10 @@ static int client_greeting(struct super_block *sb, goto out; } - if (gr->fsid != super->id) { + if (gr->fsid != super->hdr.fsid) { scoutfs_warn(sb, "server sent fsid 0x%llx, client has 0x%llx", le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); + le64_to_cpu(super->hdr.fsid)); ret = -EINVAL; goto out; } @@ -421,7 +421,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work) client->old_elected_nr = 0; /* send a greeting to verify endpoints of each connection */ - greet.fsid = super->id; + greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.node_id = cpu_to_le64(sbi->node_id); diff --git a/kmod/src/server.c b/kmod/src/server.c index fba4f9eb..e8d4d42f 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1095,10 +1095,10 @@ static int server_greeting(struct super_block *sb, goto out; } - if (gr->fsid != super->id) { + if (gr->fsid != super->hdr.fsid) { scoutfs_warn(sb, "client sent fsid 0x%llx, server has 0x%llx", le64_to_cpu(gr->fsid), - le64_to_cpu(super->id)); + le64_to_cpu(super->hdr.fsid)); ret = -EINVAL; goto out; } @@ -1128,7 +1128,7 @@ static int server_greeting(struct super_block *sb, node_id = gr->node_id; } - greet.fsid = super->id; + greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.node_id = node_id; out: From 20f4e1c33808b063f808b314479bf8abe480655f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 20 Nov 2018 15:15:55 -0800 Subject: [PATCH 687/920] scoutfs: put magic value in block header The super block had a magic value that was used to identify that the block should contain our data structure. But it was called an 'id' which was confused with the header fsid in the past. Also, the btree blocks aren't using a similar magic value at all. This moves the magic value in to the header and creates values for the super block and btree blocks. Both are written but the btree block reads don't check the value. Signed-off-by: Zach Brown --- kmod/src/btree.c | 2 +- kmod/src/format.h | 12 +++++++----- kmod/src/super.c | 8 ++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index eee9c547..a7b0f130 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -781,6 +781,7 @@ retry: bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); } + bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE); bt->hdr.blkno = cpu_to_le64(blkno); bt->hdr.seq = cpu_to_le64(seq); if (ref) { @@ -1646,7 +1647,6 @@ int scoutfs_btree_write_dirty(struct super_block *sb) /* checksum everything to reduce time between io submission merging */ for_each_dirty_bh(bti, bh, tmp) { bt = (void *)bh->b_data; - bt->hdr._pad = 0; bt->hdr.crc = scoutfs_block_calc_crc(&bt->hdr); } diff --git a/kmod/src/format.h b/kmod/src/format.h index a8ebb465..e2bfecfa 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -3,8 +3,10 @@ /* statfs(2) f_type */ #define SCOUTFS_SUPER_MAGIC 0x554f4353 /* "SCOU" */ -/* super block id */ -#define SCOUTFS_SUPER_ID 0x2e736674756f6373ULL /* "scoutfs." */ + +/* block header magic values, chosen at random */ +#define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b +#define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d /* * The super block and btree blocks are fixed 4k. @@ -67,12 +69,12 @@ struct scoutfs_inet_addr { /* * This header is stored at the start of btree blocks and the super - * block for verification. The crc is calculated by zeroing the crc and - * padding so the buffer is large and aligned. + * block for verification. The crc field is not included in the + * calculation of the crc. */ struct scoutfs_block_header { __le32 crc; - __le32 _pad; + __le32 magic; __le64 fsid; __le64 seq; __le64 blkno; diff --git a/kmod/src/super.c b/kmod/src/super.c index 2e9e6875..8149abd1 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -191,7 +191,7 @@ int scoutfs_write_dirty_super(struct super_block *sb) super = page_address(page); memcpy(super, &sbi->super, sizeof(*super)); - super->hdr._pad = 0; + super->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER); super->hdr.crc = scoutfs_block_calc_crc(&super->hdr); ret = scoutfs_bio_write(sb, &page, le64_to_cpu(super->hdr.blkno), 1); @@ -226,9 +226,9 @@ int scoutfs_read_super(struct super_block *sb, super = scoutfs_page_block_address(&page, 0); - if (super->id != cpu_to_le64(SCOUTFS_SUPER_ID)) { - scoutfs_err(sb, "super block has invalid id %llx", - le64_to_cpu(super->id)); + if (super->hdr.magic != cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER)) { + scoutfs_err(sb, "super block has invalid magic value 0x%08x", + le32_to_cpu(super->hdr.magic)); ret = -EINVAL; goto out; } From 74366f0df159c748d16630007a46ca448583cf79 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 4 Feb 2019 14:29:10 -0800 Subject: [PATCH 688/920] scoutfs: make networking more reliable The current networking code has loose reliability guarantees. If a connection between the client and server is broken then the client reconnects as though its an entirely new connection. The client resends requests but no responses are resent. A client's requests could be processed twice on the same server. The server throws away disconnected client state. This was fine, sort of, for the simple requests we had implemented so far. It's not good enough for the locking service which would prefer to let networking worry about reliable message delivery so it doesn't have to track and replay partial state across reconnection between the same client and server. This adds the infrastructure to ensure that requests and responses between a given client and server will be delivered across reconnected sockets and will only be processed once. The server keeps track of disconnected clients and restores state if the same client reconnects. This required some work around the greetings so that clients and servers can recognize each other. Now that the server remembers disconnected clients we add a farewell request so that servers can forget about clients that are shutting down and won't be reconnecting. Now that connections between the client and server are preserved we can resend responses across reconnection. We add outgoing message sequence numbers which are used to drop duplicates and communicate the received sequence back to the sender to free responses once they're received. When the client is reconnecting to a new server it resets its receive state that was dependent on the old server and it drops responses which were being sent to a server instance which no longer exists. This stronger reliable messaging guarantee will make it much easier to implement lock recovery which can now rewind state relative to requests that are in flight and replay existing state on a new server instance. Signed-off-by: Zach Brown --- kmod/src/client.c | 112 ++++++-- kmod/src/counters.h | 2 + kmod/src/format.h | 37 ++- kmod/src/net.c | 536 +++++++++++++++++++++++++++++++-------- kmod/src/net.h | 13 + kmod/src/scoutfs_trace.h | 8 + kmod/src/server.c | 63 ++++- kmod/src/server.h | 23 +- 8 files changed, 656 insertions(+), 138 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 11bd1fe7..85a5a729 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -57,6 +57,12 @@ struct client_info { struct scoutfs_quorum_elected_info qei; u64 old_elected_nr; + + u64 server_term; + + bool sending_farewell; + int farewell_error; + struct completion farewell_comp; }; /* @@ -281,7 +287,8 @@ static int client_greeting(struct super_block *sb, struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_net_greeting *gr = resp; - int ret = 0; + bool new_server; + int ret; if (error) { ret = error; @@ -328,6 +335,11 @@ static int client_greeting(struct super_block *sb, complete(&client->node_id_comp); } + new_server = le64_to_cpu(gr->server_term) != client->server_term; + scoutfs_net_client_greeting(sb, conn, new_server); + + client->server_term = le64_to_cpu(gr->server_term); + ret = 0; out: return ret; } @@ -398,7 +410,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work) goto out; if (qei->run_server) { - ret = scoutfs_server_start(sb, &qei->sin); + ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr); if (ret) { /* forget that we tried to start the server */ memset(qei, 0, sizeof(*qei)); @@ -423,7 +435,11 @@ static void scoutfs_client_connect_worker(struct work_struct *work) /* send a greeting to verify endpoints of each connection */ greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; + greet.server_term = cpu_to_le64(client->server_term); greet.node_id = cpu_to_le64(sbi->node_id); + greet.flags = 0; + if (client->sending_farewell) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_GREETING, @@ -537,6 +553,7 @@ int scoutfs_client_setup(struct super_block *sb) init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_WORK(&client->connect_work, scoutfs_client_connect_worker); + init_completion(&client->farewell_comp); client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, client_req_funcs, "client"); @@ -560,34 +577,89 @@ out: return ret; } +/* Once we get a response from the server we can shut down */ +static int client_farewell_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + if (resp_len != 0) + return -EINVAL; + + client->farewell_error = error; + complete(&client->farewell_comp); + + return 0; +} + /* * There must be no more callers to the client request functions by the * time we get here. + * + * If we've connected to a server then we send them a farewell request + * so that they don't wait for us to reconnect and trigger a timeout. + * + * This decision is a little racy. The server considers us connected + * when it assigns us a node_id as it processes the greeting. We can + * disconnect before receiving the response and leave without sending a + * farewell. So given that awkward initial race, we also have a bit of + * a race where we just test the server_term to see if we've ever gotten + * a greeting reply from any server. We don't try to synchronize with + * pending connection attempts. + * + * The consequences of aborting a mount at just the wrong time and + * disconnecting without the farewell handshake depend on what the + * server does to timed out clients. At best it'll spit out a warning + * message that a client disconnected but it won't fence us if we didn't + * have any persistent state. */ void scoutfs_client_destroy(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct client_info *client = SCOUTFS_SB(sb)->client_info; struct scoutfs_net_connection *conn; + int ret; - if (client) { - /* stop notify_down from queueing connect work */ - atomic_set(&client->shutting_down, 1); + if (client == NULL) + return; - /* make sure worker isn't using the conn */ - cancel_work_sync(&client->connect_work); - - /* make racing conn use explode */ - conn = client->conn; - client->conn = NULL; - scoutfs_net_free_conn(sb, conn); - - /* stop running the server if we were, harmless otherwise */ - stop_our_server(sb, &client->qei); - - if (client->workq) - destroy_workqueue(client->workq); - kfree(client); - sbi->client_info = NULL; + if (client->server_term != 0) { + client->sending_farewell = true; + ret = scoutfs_net_submit_request(sb, client->conn, + SCOUTFS_NET_CMD_FAREWELL, + NULL, 0, + client_farewell_response, + NULL, NULL); + if (ret == 0) { + ret = wait_for_completion_interruptible( + &client->farewell_comp); + if (ret == 0) + ret = client->farewell_error; + } + if (ret) { + scoutfs_inc_counter(sb, client_farewell_error); + scoutfs_warn(sb, "client saw farewell error %d, server might see client connection time out\n", ret); + } } + + /* stop notify_down from queueing connect work */ + atomic_set(&client->shutting_down, 1); + + /* make sure worker isn't using the conn */ + cancel_work_sync(&client->connect_work); + + /* make racing conn use explode */ + conn = client->conn; + client->conn = NULL; + scoutfs_net_free_conn(sb, conn); + + /* stop running the server if we were, harmless otherwise */ + stop_our_server(sb, &client->qei); + + if (client->workq) + destroy_workqueue(client->workq); + kfree(client); + sbi->client_info = NULL; } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 55a7ea14..e997fe31 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -15,6 +15,7 @@ EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(btree_write_error) \ + EXPAND_COUNTER(client_farewell_error) \ EXPAND_COUNTER(compact_invalid_request) \ EXPAND_COUNTER(compact_operations) \ EXPAND_COUNTER(compact_segment_busy) \ @@ -110,6 +111,7 @@ EXPAND_COUNTER(net_send_error) \ EXPAND_COUNTER(net_send_messages) \ EXPAND_COUNTER(net_recv_bytes) \ + EXPAND_COUNTER(net_recv_dropped_duplicate) \ EXPAND_COUNTER(net_recv_error) \ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index e2bfecfa..d46b2195 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -571,23 +571,53 @@ enum { */ /* - * Greetings verify identity of communicating nodes. The sender - * sends their credentials and the receiver verifies them. + * Greetings verify identity of communicating nodes. The sender sends + * their credentials and the receiver verifies them. + * + * @server_term: The raft term that elected the server. Initially 0 + * from the client, sent by the server, then sent by the client as it + * tries to reconnect. Used to identify a client reconnecting to a + * server that has timed out its connection. + * + * @node_id: The id of the client. Initially 0 from the client, + * assigned by the server, and sent by the client as it reconnects. + * Used by the server to identify reconnecting clients whose existing + * state must be dealt with. */ struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; + __le64 server_term; __le64 node_id; + __le64 flags; } __packed; +#define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 1) + /* * This header precedes and describes all network messages sent over - * sockets. The id is set by the request and sent in the response. + * sockets. + * + * @seq: A sequence number that is increased for each message queued for + * send on the sender. The sender will never reorder messages in the + * send queue so this will always increase in recv on the receiver. The + * receiver can use this to drop messages that arrived twice after being + * resent across a newly connected socket for a given connection. + * + * @recv_seq: The sequence number of the last received message. The + * receiver is sending this to the sender in every message. The sender + * uses them to drop responses which have been delivered. + * + * @id: An increasing identifier that is set in each request. Responses + * specify the request that they're responding to. * * Error is only set to a translated errno and will only be found in * response messages. */ struct scoutfs_net_header { + __le64 seq; + __le64 recv_seq; __le64 id; __le16 data_len; __u8 cmd; @@ -612,6 +642,7 @@ enum { SCOUTFS_NET_CMD_STATFS, SCOUTFS_NET_CMD_COMPACT, SCOUTFS_NET_CMD_LOCK, + SCOUTFS_NET_CMD_FAREWELL, SCOUTFS_NET_CMD_UNKNOWN, }; diff --git a/kmod/src/net.c b/kmod/src/net.c index 14c763a9..41aff95e 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -41,38 +41,39 @@ * Both set up a connection and specify the set of request commands they * can process. * - * Requests are tracked on a connection and sent to its peer. They're + * Request and response messages are queued on a connection. They're * resent down newly established sockets on a long lived connection. * Queued requests are removed as a response is processed or if the - * request is canceled by the sender. + * request is canceled by the sender. Queued responses are removed as + * the receiver acknowledges their delivery. * - * Request processing sends a response down the socket that received a - * connection. Processing is stopped as a socket is shutdown so - * responses are only send down sockets that received a request. + * Request and response resending is asymmetrical because of the + * client/server relationship. If a client connects to a new server it + * drops responses because the new server doesn't have any requests + * pending. If a server times out a client it drops everything because + * that client is never coming back. * - * Thus requests can be received multiple times as sockets are shutdown - * and reconnected. Responses are only processed once for a given - * request. It is up to request and response implementations to ensure - * that duplicate requests are safely handled. + * Requests and responses are only processed once for a given client and + * server pair. Callers have to deal with the possibility that two + * servers might both process the same client request, even though the + * client may only see the most recent response. * - * It turns out that we have to deal with duplicate request processing - * at the layer above networking anyway. Request processing can make - * persistent changes that are committed on the server before it - * crashes. The client then reconnects to before it crashes and the - * client reconnects to a server who must detect that the persistent - * work on behalf of the resent request has already been committed. If - * we have to deal with that duplicate processing we may as well - * simplify networking by allowing it between reconnecting peers as - * well. + * The functional core of this implementation is solid, but boy are the + * interface boundaries getting fuzzy. The core knows too much about + * clients and servers and the communications across the net interface + * boundary are questionable. We probably want to pull more client and + * server specific behaviour up into the client and server and turn the + * "net" code into more passive shared framing helpers. * * XXX: - * - defer accepted conn destruction until reconnect timeout * - trace command and response data payloads * - checksum message contents? - * - explicit shutdown message to free accepted, timeout and fence otherwise * - shutdown server if accept can't alloc resources for new conn? */ +/* reasonable multiple of max client reconnect attempt interval */ +#define CLIENT_RECONNECT_TIMEOUT_MS (20 * MSEC_PER_SEC) + /* * A connection's shutdown work executes in its own workqueue so that the * work can free the connection's workq. @@ -97,13 +98,19 @@ struct scoutfs_net_connection { unsigned long valid_greeting:1, /* other commands can proceed */ established:1, /* added sends queue send work */ - shutting_down:1; /* shutdown work has been queued */ + shutting_down:1, /* shutdown work has been queued */ + saw_greeting:1, /* saw greeting on this sock */ + saw_farewell:1, /* saw farewell request from client */ + reconn_wait:1, /* shutdown, waiting for reconnect */ + reconn_freeing:1; /* waiting done, setter frees */ + unsigned long reconn_deadline; struct sockaddr_in connect_sin; unsigned long connect_timeout_ms; struct socket *sock; u64 node_id; /* assigned during greeting */ + u64 greeting_id; struct sockaddr_in sockname; struct sockaddr_in peername; @@ -111,21 +118,25 @@ struct scoutfs_net_connection { struct scoutfs_net_connection *listening_conn; struct list_head accepted_list; + u64 next_send_seq; u64 next_send_id; struct list_head send_queue; struct list_head resend_queue; + atomic64_t recv_seq; + struct workqueue_struct *workq; struct work_struct listen_work; struct work_struct connect_work; struct work_struct send_work; struct work_struct recv_work; struct work_struct shutdown_work; + struct delayed_work reconn_free_dwork; /* message_recv proc_work also executes in the conn workq */ struct scoutfs_tseq_entry tseq_entry; - u8 info[0] __aligned(sizeof(u64)); + void *info; }; /* listening and their accepting sockets have a fixed locking order */ @@ -186,6 +197,10 @@ static bool nh_is_request(struct scoutfs_net_header *nh) return !nh_is_response(nh); } +/* + * We return dead requests so that the caller can stop searching other + * lists for the dead request that we found. + */ static struct message_send *search_list(struct scoutfs_net_connection *conn, struct list_head *list, u8 cmd, u64 id) @@ -343,6 +358,7 @@ static int submit_send(struct super_block *sb, struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *acc_conn; struct message_send *msend; + u64 seq; if (WARN_ON_ONCE(cmd >= SCOUTFS_NET_CMD_UNKNOWN) || WARN_ON_ONCE(flags & SCOUTFS_NET_FLAGS_UNKNOWN) || @@ -378,12 +394,16 @@ static int submit_send(struct super_block *sb, } } + seq = conn->next_send_seq++; + if (id == 0) + id = conn->next_send_id++; + msend->resp_func = resp_func; msend->resp_data = resp_data; msend->dead = 0; - if (id == 0) - id = conn->next_send_id++; + msend->nh.seq = cpu_to_le64(seq); + msend->nh.recv_seq = 0; /* set when sent, not when queued */ msend->nh.id = cpu_to_le64(id); msend->nh.cmd = cmd; msend->nh.flags = flags; @@ -411,36 +431,7 @@ static int submit_send(struct super_block *sb, } /* - * Messages can flow once we receive and process a valid greeting from - * our peer. - * - * At this point recv processing has queued the greeting response - * message on the send queue. Any request messages waiting to be resent - * need to be added to the end of the send queue after the greeting - * response. - * - * Update the conn's node_id so that servers can send to specific - * clients. - */ -static void saw_valid_greeting(struct scoutfs_net_connection *conn, u64 node_id) -{ - struct super_block *sb = conn->sb; - - spin_lock(&conn->lock); - - conn->valid_greeting = 1; - conn->node_id = node_id; - list_splice_tail_init(&conn->resend_queue, &conn->send_queue); - queue_work(conn->workq, &conn->send_work); - - spin_unlock(&conn->lock); - - if (conn->notify_up) - conn->notify_up(sb, conn, conn->info, node_id); -} - -/* - * Process an incoming response. The greeting should ensure that the + * Process an incoming request. The greeting should ensure that the * sender won't send us unknown commands. We return an error if we see * an unknown command because the greeting should agree on an understood * protocol. The request function sends a response and returns an error @@ -451,8 +442,6 @@ static int process_request(struct scoutfs_net_connection *conn, { struct super_block *sb = conn->sb; scoutfs_net_request_t req_func; - struct scoutfs_net_greeting *gr; - int ret; if (mrecv->nh.cmd < SCOUTFS_NET_CMD_UNKNOWN) req_func = conn->req_funcs[mrecv->nh.cmd]; @@ -464,21 +453,8 @@ static int process_request(struct scoutfs_net_connection *conn, return -EINVAL; } - ret = req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), - mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); - - /* - * Greeting response updates our *request* node_id so that - * we can consume a new allocation without callbacks. We're - * about to free the recv in the caller anyway. - */ - if (!conn->valid_greeting && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && ret == 0) { - gr = (void *)mrecv->nh.data; - saw_valid_greeting(conn, le64_to_cpu(gr->node_id)); - } - - return ret; + return req_func(sb, conn, mrecv->nh.cmd, le64_to_cpu(mrecv->nh.id), + mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); } /* @@ -514,11 +490,6 @@ static int process_response(struct scoutfs_net_connection *conn, ret = resp_func(sb, conn, mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error), resp_data); - - if (!conn->valid_greeting && - mrecv->nh.cmd == SCOUTFS_NET_CMD_GREETING && msend && ret == 0) - saw_valid_greeting(conn, 0); - return ret; } @@ -553,6 +524,49 @@ static void scoutfs_net_proc_worker(struct work_struct *work) trace_scoutfs_net_proc_work_exit(sb, 0, ret); } +/* + * Free live responses up to and including the seq by marking them dead + * and moving them to the send queue to be freed. + */ +static int move_acked_responses(struct scoutfs_net_connection *conn, + struct list_head *list, u64 seq) +{ + struct message_send *msend; + struct message_send *tmp; + int ret = 0; + + assert_spin_locked(&conn->lock); + + list_for_each_entry_safe(msend, tmp, list, head) { + if (le64_to_cpu(msend->nh.seq) > seq) + break; + if (!nh_is_response(&msend->nh) || msend->dead) + continue; + + msend->dead = 1; + list_move(&msend->head, &conn->send_queue); + ret = 1; + } + + return ret; +} + +/* acks are processed inline in the recv worker */ +static void free_acked_responses(struct scoutfs_net_connection *conn, u64 seq) +{ + int moved; + + spin_lock(&conn->lock); + + moved = move_acked_responses(conn, &conn->send_queue, seq) + + move_acked_responses(conn, &conn->resend_queue, seq); + + spin_unlock(&conn->lock); + + if (moved) + queue_work(conn->workq, &conn->send_work); +} + static int recvmsg_full(struct socket *sock, void *buf, unsigned len) { struct msghdr msg; @@ -578,10 +592,11 @@ static int recvmsg_full(struct socket *sock, void *buf, unsigned len) return 0; } -static bool invalid_message(struct scoutfs_net_header *nh) +static bool invalid_message(struct scoutfs_net_connection *conn, + struct scoutfs_net_header *nh) { - /* ids must be non-zero */ - if (nh->id == 0) + /* seq and id must be non-zero */ + if (nh->seq == 0 || nh->id == 0) return true; /* greeting should negotiate understood protocol */ @@ -598,6 +613,16 @@ static bool invalid_message(struct scoutfs_net_header *nh) if (nh_is_request(nh) && nh->error != SCOUTFS_NET_ERR_NONE) return true; + if (nh->cmd == SCOUTFS_NET_CMD_GREETING) { + /* each endpoint can only receive one greeting per socket */ + if (conn->saw_greeting) + return true; + + /* servers get greeting requests, clients get responses */ + if (!!conn->listening_conn != !!nh_is_request(nh)) + return true; + } + return false; } @@ -625,7 +650,7 @@ static void scoutfs_net_recv_worker(struct work_struct *work) break; /* receiving an invalid message breaks the connection */ - if (invalid_message(&nh)) { + if (invalid_message(conn, &nh)) { scoutfs_inc_counter(sb, net_recv_invalid_message); ret = -EBADMSG; break; @@ -657,8 +682,31 @@ static void scoutfs_net_recv_worker(struct work_struct *work) break; } + if (nh.cmd == SCOUTFS_NET_CMD_GREETING) { + /* greetings are out of band, no seq mechanics */ + conn->saw_greeting = 1; + + } else if (le64_to_cpu(nh.seq) <= + atomic64_read(&conn->recv_seq)) { + /* drop any resent duplicated messages */ + scoutfs_inc_counter(sb, net_recv_dropped_duplicate); + kfree(mrecv); + continue; + + } else { + /* record that we've received sender's seq */ + atomic64_set(&conn->recv_seq, le64_to_cpu(nh.seq)); + /* and free our responses that sender has received */ + free_acked_responses(conn, le64_to_cpu(nh.recv_seq)); + } + scoutfs_tseq_add(&ninf->msg_tseq_tree, &mrecv->tseq_entry); - queue_work(conn->workq, &mrecv->proc_work); + + /* synchronously process greeting before next recvmsg */ + if (nh.cmd == SCOUTFS_NET_CMD_GREETING) + scoutfs_net_proc_worker(&mrecv->proc_work); + else + queue_work(conn->workq, &mrecv->proc_work); } if (ret) @@ -712,6 +760,10 @@ static void free_msend(struct net_info *ninf, struct message_send *msend) * The worker is responsible for freeing messages so that other contexts * don't have to worry about freeing a message while we're blocked * sending it without the lock held. + * + * We set the current recv_seq on every outgoing frame as it represents + * the current connection state, not the state back when each message + * was first queued. */ static void scoutfs_net_send_worker(struct work_struct *work) { @@ -734,6 +786,9 @@ static void scoutfs_net_send_worker(struct work_struct *work) continue; } + msend->nh.recv_seq = + cpu_to_le64(atomic64_read(&conn->recv_seq)); + spin_unlock(&conn->lock); len = nh_bytes(le16_to_cpu(msend->nh.data_len)); @@ -747,14 +802,14 @@ static void scoutfs_net_send_worker(struct work_struct *work) spin_lock(&conn->lock); + msend->nh.recv_seq = 0; + if (ret) break; - /* active requests are resent, everything else is freed */ - if (nh_is_request(&msend->nh) && !msend->dead) + /* resend if it wasn't freed while we sent */ + if (!msend->dead) list_move_tail(&msend->head, &conn->resend_queue); - else - msend->dead = 1; } spin_unlock(&conn->lock); @@ -778,6 +833,10 @@ static void destroy_conn(struct scoutfs_net_connection *conn) WARN_ON_ONCE(conn->sock != NULL); WARN_ON_ONCE(!list_empty(&conn->accepted_list)); + /* tell callers that accepted connection finally done */ + if (conn->listening_conn && conn->notify_down) + conn->notify_down(sb, conn, conn->info, conn->node_id); + /* free all messages, refactor and complete for forced unmount? */ list_splice_init(&conn->resend_queue, &conn->send_queue); list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { @@ -797,6 +856,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) destroy_workqueue(conn->workq); scoutfs_tseq_del(&ninf->conn_tseq_tree, &conn->tseq_entry); + kfree(conn->info); kfree(conn); } @@ -1025,9 +1085,11 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) DEFINE_CONN_FROM_WORK(conn, work, shutdown_work); struct super_block *sb = conn->sb; struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct scoutfs_net_connection *listener; struct scoutfs_net_connection *acc_conn; struct message_send *msend; struct message_send *tmp; + unsigned long delay; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); @@ -1062,39 +1124,119 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) spin_unlock(&acc_conn->lock); } spin_unlock(&conn->lock); + + /* free any conns waiting for reconnection */ + cancel_delayed_work_sync(&conn->reconn_free_dwork); + queue_delayed_work(conn->workq, &conn->reconn_free_dwork, 0); + /* relies on delay 0 scheduling immediately so no timer to cancel */ + flush_delayed_work(&conn->reconn_free_dwork); + + /* and wait for accepted conn shutdown work to finish */ wait_event(conn->waitq, empty_accepted_list(conn)); spin_lock(&conn->lock); - /* resend any pending requests, drop responses or greetings */ + /* greetings aren't resent across sockets */ list_splice_tail_init(&conn->send_queue, &conn->resend_queue); list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head) { - if (nh_is_response(&msend->nh) || - msend->nh.cmd == SCOUTFS_NET_CMD_GREETING) + if (msend->nh.cmd == SCOUTFS_NET_CMD_GREETING) free_msend(ninf, msend); } + conn->saw_greeting = 0; + /* signal connect failure */ memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); wake_up(&conn->waitq); - spin_unlock(&conn->lock); - /* tell the caller that the connection is down */ - if (conn->notify_down) - conn->notify_down(sb, conn, conn->info, conn->node_id); + /* resolve racing with listener shutdown with locked shutting_down */ + if (conn->listening_conn && + (conn->listening_conn->shutting_down || conn->saw_farewell)) { - /* accepted conns are destroyed */ - if (conn->listening_conn) { + /* free accepted sockets after farewell or listener shutdown */ + spin_unlock(&conn->lock); destroy_conn(conn); + } else { - spin_lock(&conn->lock); - conn->shutting_down = 0; + + if (conn->listening_conn) { + /* server accepted sockets wait for reconnect */ + listener = conn->listening_conn; + delay = msecs_to_jiffies(CLIENT_RECONNECT_TIMEOUT_MS); + conn->reconn_wait = 1; + conn->reconn_deadline = jiffies + delay; + queue_delayed_work(listener->workq, + &listener->reconn_free_dwork, delay); + } else { + /* clients and listeners can retry */ + conn->shutting_down = 0; + if (conn->notify_down) + conn->notify_down(sb, conn, conn->info, + conn->node_id); + } + spin_unlock(&conn->lock); } trace_scoutfs_net_shutdown_work_exit(sb, 0, 0); } +/* + * Free any connections that have been shutdown for too long without the + * client reconnecting. This runs in work on the listening connection. + * It's racing with connection attempts searching for shutdown + * connections to steal state from. Shutdown cancels the work and waits + * for it to finish. + * + * Connections are currently freed without the lock held so this walks + * the entire list every time it frees a connection. This is irritating + * but timed out connections are rare and client counts are relatively + * low given a cpu's ability to burn through the list. + */ +static void scoutfs_net_reconn_free_worker(struct work_struct *work) +{ + DEFINE_CONN_FROM_WORK(conn, work, reconn_free_dwork.work); + struct super_block *sb = conn->sb; + struct scoutfs_net_connection *acc; + unsigned long now = jiffies; + unsigned long deadline = 0; + bool requeue = false; + + trace_scoutfs_net_reconn_free_work_enter(sb, 0, 0); + +restart: + spin_lock(&conn->lock); + list_for_each_entry(acc, &conn->accepted_list, accepted_head) { + + if (acc->reconn_wait && !acc->reconn_freeing && + (conn->shutting_down || + time_after_eq(now, acc->reconn_deadline))) { + acc->reconn_freeing = 1; + spin_unlock(&conn->lock); + if (!conn->shutting_down) + scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect", + SIN_ARG(&acc->sockname), + SIN_ARG(&acc->peername)); + destroy_conn(acc); + goto restart; + } + + /* calc delay of next work, can drift a bit */ + if (acc->reconn_wait && !acc->reconn_freeing && + (!requeue || time_before(now, deadline))) { + requeue = true; + deadline = acc->reconn_deadline; + } + } + spin_unlock(&conn->lock); + + if (requeue) + queue_delayed_work(conn->workq, &conn->reconn_free_dwork, + deadline - now); + + trace_scoutfs_net_reconn_free_work_exit(sb, 0, 0); +} + /* * Accepted connections inherit the callbacks from their listening * connection. @@ -1116,15 +1258,21 @@ scoutfs_net_alloc_conn(struct super_block *sb, struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *conn; - conn = kzalloc(offsetof(struct scoutfs_net_connection, - info[info_size]), GFP_NOFS); + conn = kzalloc(sizeof(struct scoutfs_net_connection), GFP_NOFS); if (!conn) return NULL; + conn->info = kzalloc(info_size, GFP_NOFS); + if (!conn->info) { + kfree(conn); + return NULL; + } + conn->workq = alloc_workqueue("scoutfs_net_%s", WQ_UNBOUND | WQ_NON_REENTRANT, 0, name_suffix); if (!conn->workq) { + kfree(conn->info); kfree(conn); return NULL; } @@ -1140,7 +1288,9 @@ scoutfs_net_alloc_conn(struct super_block *sb, conn->peername.sin_family = AF_INET; INIT_LIST_HEAD(&conn->accepted_head); INIT_LIST_HEAD(&conn->accepted_list); + conn->next_send_seq = 1; conn->next_send_id = 1; + atomic64_set(&conn->recv_seq, 0); INIT_LIST_HEAD(&conn->send_queue); INIT_LIST_HEAD(&conn->resend_queue); INIT_WORK(&conn->listen_work, scoutfs_net_listen_worker); @@ -1148,6 +1298,8 @@ scoutfs_net_alloc_conn(struct super_block *sb, INIT_WORK(&conn->send_work, scoutfs_net_send_worker); INIT_WORK(&conn->recv_work, scoutfs_net_recv_worker); INIT_WORK(&conn->shutdown_work, scoutfs_net_shutdown_worker); + INIT_DELAYED_WORK(&conn->reconn_free_dwork, + scoutfs_net_reconn_free_worker); scoutfs_tseq_add(&ninf->conn_tseq_tree, &conn->tseq_entry); @@ -1302,6 +1454,185 @@ int scoutfs_net_connect(struct super_block *sb, return ret ?: error; } +static void set_valid_greeting(struct scoutfs_net_connection *conn) +{ + assert_spin_locked(&conn->lock); + + /* recv should have dropped invalid duplicate greeting messages */ + BUG_ON(conn->valid_greeting); + + conn->valid_greeting = 1; + list_splice_tail_init(&conn->resend_queue, &conn->send_queue); + queue_work(conn->workq, &conn->send_work); +} + +/* + * The client has received a valid greeting from the server. Send + * can proceed and we might need to reset our recv state if we reconnected + * to a new server. + */ +void scoutfs_net_client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + bool new_server) +{ + struct net_info *ninf = SCOUTFS_SB(sb)->net_info; + struct message_send *msend; + struct message_send *tmp; + + /* only called on client connections :/ */ + BUG_ON(conn->listening_conn); + + spin_lock(&conn->lock); + + if (new_server) { + atomic64_set(&conn->recv_seq, 0); + list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head){ + if (nh_is_response(&msend->nh)) + free_msend(ninf, msend); + } + } + + set_valid_greeting(conn); + + spin_unlock(&conn->lock); + + /* client up/down drives reconnect */ + if (conn->notify_up) + conn->notify_up(sb, conn, conn->info, 0); +} + +/* + * The calling server has received a valid greeting from a client. If + * the server is reconnecting to us then we need to find its old + * connection that held its state and transfer it to this connection + * (connection and socket life cycles make this easier than migrating + * the socket between the connections). + * + * The previous connection that holds the client's state might still be + * in active use depending on network failure and work processing races. + * We shut it down before migrating its message state. We can be + * processing greetings from multiple reconnecting sockets that are all + * referring to the same original connection. We use the increasing + * greeting id to have the most recent connection attempt win. + * + * A node can be reconnecting to us for the first time. In that case we + * just trust its node_id. It will notice the new server term and take + * steps to recover. + * + * A client can be reconnecting to us after we've destroyed their state. + * This is fatal for the client if they just took too long to reconnect. + * But this can also happen if something disconnects the socket after + * we've sent a farewell response before the client received it. In + * this case we let the client reconnect so we can resend the farewell + * response and they can disconnect cleanly. + * + * At this point our connection is idle except for send submissions and + * shutdown being queued. Once we shut down a We completely own a We + * have exclusive access to a previous conn once its shutdown and we set + * _freeing. + */ +void scoutfs_net_server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u64 greeting_id, + bool sent_node_id, bool first_contact, + bool farewell) +{ + struct scoutfs_net_connection *listener; + struct scoutfs_net_connection *reconn; + struct scoutfs_net_connection *acc; + + /* only called on accepted server connections :/ */ + BUG_ON(!conn->listening_conn); + + /* see if we have a previous conn for the client's sent node_id */ + reconn = NULL; + if (sent_node_id) { + listener = conn->listening_conn; +restart: + spin_lock_nested(&listener->lock, CONN_LOCK_LISTENER); + list_for_each_entry(acc, &listener->accepted_list, + accepted_head) { + if (acc->node_id != node_id || + acc->greeting_id >= greeting_id || + acc->reconn_freeing) + continue; + + if (!acc->reconn_wait) { + spin_lock_nested(&acc->lock, + CONN_LOCK_ACCEPTED); + shutdown_conn_locked(acc); + spin_unlock(&acc->lock); + spin_unlock(&listener->lock); + msleep(10); /* XXX might be freed :/ */ + goto restart; + } + + reconn = acc; + reconn->reconn_freeing = 1; + break; + } + spin_unlock(&listener->lock); + } + + /* drop a connection if we can't find its necessary old conn */ + if (sent_node_id && !reconn && !first_contact && !farewell) { + shutdown_conn(conn); + return; + } + + /* migrate state from previous conn for this reconnecting node_id */ + if (reconn) { + spin_lock(&conn->lock); + + conn->saw_farewell = reconn->saw_farewell; + conn->next_send_seq = reconn->next_send_seq; + conn->next_send_id = reconn->next_send_id; + atomic64_set(&conn->recv_seq, atomic64_read(&reconn->recv_seq)); + + /* greeting response/ack will be on conn send queue */ + BUG_ON(!list_empty(&reconn->send_queue)); + BUG_ON(!list_empty(&conn->resend_queue)); + list_splice_init(&reconn->resend_queue, &conn->resend_queue); + + /* new conn info is unused, swap, old won't call down */ + swap(conn->info, reconn->info); + reconn->notify_down = NULL; + + spin_unlock(&conn->lock); + + /* we set _freeing */ + destroy_conn(reconn); + } + + spin_lock(&conn->lock); + + conn->node_id = node_id; + conn->greeting_id = greeting_id; + set_valid_greeting(conn); + + spin_unlock(&conn->lock); + + /* only call notify_up the first time we see the node_id */ + if (conn->notify_up && first_contact) + conn->notify_up(sb, conn, conn->info, node_id); +} + +/* + * The server has received a farewell message and is sending a response. + * All we do is mark the connection so that it is freed the next time it + * is shutdown, presumably as the client disconnects after receiving the + * response. The server caller has cleaned up all the state it had + * associated with the client. + */ +void scoutfs_net_server_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn) +{ + spin_lock(&conn->lock); + conn->saw_farewell = 1; + spin_unlock(&conn->lock); +} + + /* * Submit a request down the connection. It's up to the caller to * ensure that the conn is allocated. Sends submitted when the @@ -1455,10 +1786,13 @@ static void net_tseq_show_conn(struct seq_file *m, struct scoutfs_net_connection *conn = container_of(ent, struct scoutfs_net_connection, tseq_entry); - seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" vg %u est %u sd %u cto_ms %lu nsi %llu\n", + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" node_id %llu greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), - conn->valid_greeting, conn->established, - conn->shutting_down, conn->connect_timeout_ms, + conn->node_id, conn->greeting_id, conn->valid_greeting, + conn->established, conn->shutting_down, conn->saw_greeting, + conn->saw_farewell, conn->reconn_wait, conn->reconn_freeing, + conn->connect_timeout_ms, conn->reconn_deadline, + conn->next_send_seq, (u64)atomic64_read(&conn->recv_seq), conn->next_send_id); } diff --git a/kmod/src/net.h b/kmod/src/net.h index b59db537..82da64bf 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -67,6 +67,19 @@ void scoutfs_net_shutdown(struct super_block *sb, void scoutfs_net_free_conn(struct super_block *sb, struct scoutfs_net_connection *conn); +void scoutfs_net_client_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + bool new_server); +void scoutfs_net_server_greeting(struct super_block *sb, + struct scoutfs_net_connection *conn, + u64 node_id, u64 greeting_id, + bool sent_node_id, bool first_contact, + bool farewell); +void scoutfs_net_server_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn); +void scoutfs_net_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn); + int scoutfs_net_setup(struct super_block *sb); void scoutfs_net_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 26e244ba..0693f1ce 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1840,6 +1840,14 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_reconn_free_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_reconn_free_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); DEFINE_EVENT(scoutfs_work_class, scoutfs_net_send_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) diff --git a/kmod/src/server.c b/kmod/src/server.c index e8d4d42f..1077afe5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -56,6 +56,7 @@ struct server_info { bool shutting_down; struct completion start_comp; struct sockaddr_in listen_sin; + u64 term; struct scoutfs_net_connection *conn; /* request processing coordinates committing manifest and alloc */ @@ -1072,11 +1073,16 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, * response shuts down the connection. * * We allocate a new node_id for the first connect attempt from a - * client. We update the request node_id for the calling net layer to - * consume. + * client. * * If a client reconnects they'll send their initially assigned node_id * in their greeting request. + * + * XXX We can lose allocated node_ids here as we record the node_id as + * live as we send a valid greeting response. The client might + * disconnect before they receive the response and resent and initial + * blank greeting. We could use a client uuid to associate with + * allocated node_ids. */ static int server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -1088,6 +1094,9 @@ static int server_greeting(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; __le64 node_id = 0; + bool sent_node_id; + bool first_contact; + bool farewell; int ret = 0; if (arg_len != sizeof(struct scoutfs_net_greeting)) { @@ -1122,24 +1131,61 @@ static int server_greeting(struct super_block *sb, queue_commit_work(server, &cw); up_read(&server->commit_rwsem); ret = wait_for_commit(&cw); - if (ret) + if (ret) { + node_id = 0; goto out; + } } else { node_id = gr->node_id; } greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; + greet.server_term = cpu_to_le64(server->term); greet.node_id = node_id; + greet.flags = 0; out: ret = scoutfs_net_response(sb, conn, cmd, id, ret, &greet, sizeof(greet)); - /* give net caller client's new node_id :/ */ - if (ret == 0 && node_id != 0) - gr->node_id = node_id; + if (node_id != 0 && ret == 0) { + sent_node_id = gr->node_id != 0; + first_contact = le64_to_cpu(gr->server_term) != server->term; + if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) + farewell = true; + else + farewell = false; + + scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id, + sent_node_id, first_contact, + farewell); + } + return ret; } +/* + * The server is receiving a farewell message from a client that is + * unmounting. It won't send any more requests and once it receives our + * response it will not reconnect. + * + * XXX we should make sure that all our requests to the client have finished + * before we respond. Locking will have its own messaging for orderly + * shutdown. That leaves compaction which will be addressed as part of + * the larger work of recovering compactions that were in flight when + * a client crashed. + */ +static int server_farewell(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + if (arg_len != 0) + return -EINVAL; + + scoutfs_net_server_farewell(sb, conn); + + return scoutfs_net_response(sb, conn, cmd, id, 0, NULL, 0); +} + /* requests sent to clients are tracked so we can free resources */ struct compact_request { struct list_head head; @@ -1743,6 +1789,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GET_MANIFEST_ROOT] = server_get_manifest_root, [SCOUTFS_NET_CMD_STATFS] = server_statfs, [SCOUTFS_NET_CMD_LOCK] = server_lock, + [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, }; static void server_notify_up(struct super_block *sb, @@ -1880,13 +1927,15 @@ out: } /* XXX can we call start multiple times? */ -int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin) +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, + u64 term) { DECLARE_SERVER_INFO(sb, server); server->err = 0; server->shutting_down = false; server->listen_sin = *sin; + server->term = term; init_completion(&server->start_comp); queue_work(server->wq, &server->work); diff --git a/kmod/src/server.h b/kmod/src/server.h index 01aed49b..a843e437 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -22,11 +22,16 @@ do { \ __entry->name##_addr & 255, \ __entry->name##_port -#define SNH_FMT "id %llu data_len %u cmd %u flags 0x%x error %u" -#define SNH_ARG(nh) le64_to_cpu((nh)->id), le16_to_cpu((nh)->data_len), \ - (nh)->cmd, (nh)->flags, (nh)->error +#define SNH_FMT \ + "seq %llu recv_seq %llu id %llu data_len %u cmd %u flags 0x%x error %u" +#define SNH_ARG(nh) \ + le64_to_cpu((nh)->seq), le64_to_cpu((nh)->recv_seq), \ + le64_to_cpu((nh)->id), le16_to_cpu((nh)->data_len), (nh)->cmd, \ + (nh)->flags, (nh)->error #define snh_trace_define(name) \ + __field(__u64, name##_seq) \ + __field(__u64, name##_recv_seq) \ __field(__u64, name##_id) \ __field(__u16, name##_data_len) \ __field(__u8, name##_cmd) \ @@ -37,6 +42,8 @@ do { \ do { \ __typeof__(nh) _nh = (nh); \ \ + __entry->name##_seq = le64_to_cpu(_nh->seq); \ + __entry->name##_recv_seq = le64_to_cpu(_nh->recv_seq); \ __entry->name##_id = le64_to_cpu(_nh->id); \ __entry->name##_data_len = le16_to_cpu(_nh->data_len); \ __entry->name##_cmd = _nh->cmd; \ @@ -44,9 +51,10 @@ do { \ __entry->name##_error = _nh->error; \ } while (0) -#define snh_trace_args(name) \ - __entry->name##_id, __entry->name##_data_len, __entry->name##_cmd, \ - __entry->name##_flags, __entry->name##_error +#define snh_trace_args(name) \ + __entry->name##_seq, __entry->name##_recv_seq, __entry->name##_id, \ + __entry->name##_data_len, __entry->name##_cmd, __entry->name##_flags, \ + __entry->name##_error struct scoutfs_net_manifest_entry; struct scoutfs_manifest_entry; @@ -62,7 +70,8 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, u64 id, struct scoutfs_net_lock *nl); struct sockaddr_in; -int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin); +int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, + u64 term); void scoutfs_server_stop(struct super_block *sb); int scoutfs_server_setup(struct super_block *sb); From 801f6ad9be44af1bc5a661aa5683adf71a98c958 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 Feb 2019 09:20:12 -0800 Subject: [PATCH 689/920] scoutfs: add scoutfs_spbm_empty() Add a quick function that determines if a sparse bitmap has no bits set. Signed-off-by: Zach Brown --- kmod/src/spbm.c | 5 +++++ kmod/src/spbm.h | 1 + 2 files changed, 6 insertions(+) diff --git a/kmod/src/spbm.c b/kmod/src/spbm.c index c26dcc2c..6960f65b 100644 --- a/kmod/src/spbm.c +++ b/kmod/src/spbm.c @@ -42,6 +42,11 @@ void scoutfs_spbm_init(struct scoutfs_spbm *spbm) spbm->root = RB_ROOT; } +bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm) +{ + return RB_EMPTY_ROOT(&spbm->root); +} + enum { /* if a node isn't found then return an allocated new node */ SPBM_FIND_ALLOC = 0x1, diff --git a/kmod/src/spbm.h b/kmod/src/spbm.h index a1a2ec5d..7d9dca21 100644 --- a/kmod/src/spbm.h +++ b/kmod/src/spbm.h @@ -6,6 +6,7 @@ struct scoutfs_spbm { }; void scoutfs_spbm_init(struct scoutfs_spbm *spbm); +bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm); void scoutfs_spbm_destroy(struct scoutfs_spbm *spbm); int scoutfs_spbm_set(struct scoutfs_spbm *spbm, u64 bit); From ec0fb5380a254d8849c2ce43d37c765892716039 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Feb 2019 13:19:30 -0800 Subject: [PATCH 690/920] scoutfs: implement lock recovery When a server crashes all the connected clients still have operational locks and can be using them to protect IO. As a new server starts up its lock service needs to account for those outstanding locks before granting new locks to clients. This implements lock recovery by having the lock service recover locks from clients as it starts up. First the lock service stores records of connected clients in a btree off the super block. Records are added as the server receives their greeting and are removed as the server receives their farewell. Then the server checks for existing persistent records as it starts up. If it finds any it enters recovery and waits for all the old clients to reconnect before resuming normal processing. We add lock recover request and response messages that are used to communicate locks from the clients to the server. Signed-off-by: Zach Brown --- kmod/src/btree.c | 1 + kmod/src/client.c | 27 +++ kmod/src/client.h | 2 + kmod/src/counters.h | 1 + kmod/src/format.h | 19 ++ kmod/src/lock.c | 78 ++++++- kmod/src/lock.h | 2 + kmod/src/lock_server.c | 453 ++++++++++++++++++++++++++++++++++++++--- kmod/src/lock_server.h | 5 + kmod/src/server.c | 120 +++++++++-- kmod/src/server.h | 2 + 11 files changed, 659 insertions(+), 51 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index a7b0f130..c4ef6921 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -348,6 +348,7 @@ static void advance_to_next_half(struct scoutfs_btree_ring *bring) static size_t super_root_offsets[] = { offsetof(struct scoutfs_super_block, alloc_root), offsetof(struct scoutfs_super_block, manifest.root), + offsetof(struct scoutfs_super_block, lock_clients), }; #define for_each_super_root(super, i, root) \ diff --git a/kmod/src/client.c b/kmod/src/client.c index 85a5a729..d0d0325f 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -260,6 +260,19 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, net_id, 0, nl, sizeof(*nl)); } +/* Send a lock recover response to the server. */ +int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock_recover *nlr) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + u16 bytes = offsetof(struct scoutfs_net_lock_recover, + locks[le16_to_cpu(nlr->nr)]); + + return scoutfs_net_response(sb, client->conn, + SCOUTFS_NET_CMD_LOCK_RECOVER, + net_id, 0, nlr, bytes); +} + /* The client is receiving a invalidation request from the server */ static int client_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, @@ -273,6 +286,19 @@ static int client_lock(struct super_block *sb, return scoutfs_lock_invalidate_request(sb, id, arg); } +/* The server is asking us for the client's locks starting with the given key */ +static int client_lock_recover(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + if (arg_len != sizeof(struct scoutfs_key)) + return -EINVAL; + + /* XXX error? */ + + return scoutfs_lock_recover_request(sb, id, arg); +} + /* * Process a greeting response in the client from the server. This is * called for every connected socket on the connection. The first @@ -508,6 +534,7 @@ out: static scoutfs_net_request_t client_req_funcs[] = { [SCOUTFS_NET_CMD_COMPACT] = client_compact, [SCOUTFS_NET_CMD_LOCK] = client_lock, + [SCOUTFS_NET_CMD_LOCK_RECOVER] = client_lock_recover, }; /* diff --git a/kmod/src/client.h b/kmod/src/client.h index ca244f9d..dd0c2eae 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -21,6 +21,8 @@ int scoutfs_client_lock_request(struct super_block *sb, struct scoutfs_net_lock *nl); int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock *nl); +int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, + struct scoutfs_net_lock_recover *nlr); int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e997fe31..a35df40b 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -98,6 +98,7 @@ EXPAND_COUNTER(lock_lock) \ EXPAND_COUNTER(lock_lock_error) \ EXPAND_COUNTER(lock_nonblock_eagain) \ + EXPAND_COUNTER(lock_recover_request) \ EXPAND_COUNTER(lock_shrink_queued) \ EXPAND_COUNTER(lock_shrink_request_aborted) \ EXPAND_COUNTER(lock_unlock) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index d46b2195..be2931dc 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -272,6 +272,14 @@ struct scoutfs_extent_btree_key { __be64 minor; } __packed; +/* + * The lock server keeps a persistent record of connected clients so that + * server failover knows who to wait for before resuming operations. + */ +struct scoutfs_lock_client_btree_key { + __be64 node_id; +} __packed; + /* * The max number of links defines the max number of entries that we can * index in o(log n) and the static list head storage size in the @@ -456,6 +464,7 @@ struct scoutfs_super_block { struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_quorum_config quorum_config; + struct scoutfs_btree_root lock_clients; } __packed; #define SCOUTFS_ROOT_INO 1 @@ -642,6 +651,7 @@ enum { SCOUTFS_NET_CMD_STATFS, SCOUTFS_NET_CMD_COMPACT, SCOUTFS_NET_CMD_LOCK, + SCOUTFS_NET_CMD_LOCK_RECOVER, SCOUTFS_NET_CMD_FAREWELL, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -768,6 +778,15 @@ struct scoutfs_net_lock { __u8 new_mode; } __packed; +struct scoutfs_net_lock_recover { + __le16 nr; + struct scoutfs_net_lock locks[0]; +} __packed; + +#define SCOUTFS_NET_LOCK_MAX_RECOVER_NR \ + ((SCOUTFS_NET_MAX_DATA_LEN - sizeof(struct scoutfs_net_lock_recover)) /\ + sizeof(struct scoutfs_net_lock)) + /* some enums for tracing */ enum { SLT_CLIENT, diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c33c43bd..fa490b7c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -54,6 +54,13 @@ * lock attempt can't immediately match an existing granted lock. This * is fine for the only rare user which can back out of its lock * inversion and retry with a full blocking lock. + * + * Lock recovery is initiated by the server when it recognizes that + * we're reconnecting to it while a previous server left a persistenr + * record of us. We resend all our pending requests which are deferred + * until recovery finishes. The server sends us a recovery request and + * we respond with all our locks. Our resent requests are processed + * relative to that lock state we resend. */ #define GRACE_PERIOD_KT ms_to_ktime(2) @@ -407,7 +414,8 @@ static void lock_remove(struct lock_info *linfo, struct scoutfs_lock *lock) } static struct scoutfs_lock *lock_lookup(struct super_block *sb, - struct scoutfs_key *start) + struct scoutfs_key *start, + struct scoutfs_lock **next) { DECLARE_LOCK_INFO(sb, linfo); struct rb_node *node = linfo->lock_tree.rb_node; @@ -416,16 +424,22 @@ static struct scoutfs_lock *lock_lookup(struct super_block *sb, assert_spin_locked(&linfo->lock); + if (next) + *next = NULL; + while (node) { lock = container_of(node, struct scoutfs_lock, node); cmp = scoutfs_key_compare(start, &lock->start); - if (cmp < 0) + if (cmp < 0) { + if (next) + *next = lock; node = node->rb_left; - else if (cmp > 0) + } else if (cmp > 0) { node = node->rb_right; - else + } else { return lock; + } } return NULL; @@ -454,7 +468,7 @@ static struct scoutfs_lock *get_lock(struct super_block *sb, assert_spin_locked(&linfo->lock); - lock = lock_lookup(sb, start); + lock = lock_lookup(sb, start, NULL); if (lock) __lock_del_lru(linfo, lock); @@ -599,7 +613,7 @@ int scoutfs_lock_grant_response(struct super_block *sb, spin_lock(&linfo->lock); /* lock must already be busy with request_pending */ - lock = lock_lookup(sb, &nl->key); + lock = lock_lookup(sb, &nl->key, NULL); BUG_ON(!lock); BUG_ON(!lock->request_pending); @@ -750,6 +764,58 @@ int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, return 0; } +/* + * The server is asking us to send them as many locks as we can starting + * with the given key. We'll send a response with 0 locks to indicate + * that we've sent all our locks. This is called in client processing + * so the client won't try to reconnect to another server until we + * return. + */ +int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, + struct scoutfs_key *key) +{ + DECLARE_LOCK_INFO(sb, linfo); + struct scoutfs_net_lock_recover *nlr; + struct scoutfs_lock *lock; + struct scoutfs_lock *next; + struct rb_node *node; + int ret; + int i; + + scoutfs_inc_counter(sb, lock_recover_request); + + nlr = kmalloc(offsetof(struct scoutfs_net_lock_recover, + locks[SCOUTFS_NET_LOCK_MAX_RECOVER_NR]), + GFP_NOFS); + if (!nlr) + return -ENOMEM; + + spin_lock(&linfo->lock); + + lock = lock_lookup(sb, key, &next) ?: next; + + for (i = 0; lock && i < SCOUTFS_NET_LOCK_MAX_RECOVER_NR; i++) { + + nlr->locks[i].key = lock->start; + nlr->locks[i].old_mode = lock->mode; + nlr->locks[i].new_mode = lock->mode; + + node = rb_next(&lock->node); + if (node) + lock = rb_entry(node, struct scoutfs_lock, node); + else + lock = NULL; + } + + nlr->nr = cpu_to_le16(i); + + spin_unlock(&linfo->lock); + + ret = scoutfs_client_lock_recover_response(sb, net_id, nlr); + kfree(nlr); + return ret; +} + static bool lock_wait_cond(struct super_block *sb, struct scoutfs_lock *lock, int mode) { diff --git a/kmod/src/lock.h b/kmod/src/lock.h index b3920ed3..b6cbf767 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -48,6 +48,8 @@ int scoutfs_lock_grant_response(struct super_block *sb, struct scoutfs_net_lock *nl); int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, struct scoutfs_net_lock *nl); +int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, + struct scoutfs_key *key); int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index bb5f4ed3..5c55362c 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -18,6 +18,9 @@ #include "counters.h" #include "net.h" #include "tseq.h" +#include "spbm.h" +#include "btree.h" +#include "msg.h" #include "scoutfs_trace.h" #include "lock_server.h" @@ -67,10 +70,18 @@ * server shuts down. */ +#define LOCK_SERVER_RECOVERY_MS (10 * MSEC_PER_SEC) + struct lock_server_info { + struct super_block *sb; + spinlock_t lock; + struct mutex mutex; struct rb_root locks_root; + struct scoutfs_spbm recovery_pending; + struct delayed_work recovery_dwork; + struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; }; @@ -222,10 +233,12 @@ static bool client_entries_compatible(struct client_lock_entry *granted, */ static struct server_lock_node *get_server_lock(struct lock_server_info *inf, struct scoutfs_key *key, - struct server_lock_node *ins) + struct server_lock_node *ins, + bool or_next) { struct rb_root *root = &inf->locks_root; struct server_lock_node *ret = NULL; + struct server_lock_node *next = NULL; struct server_lock_node *snode; struct rb_node *parent = NULL; struct rb_node **node; @@ -240,6 +253,8 @@ static struct server_lock_node *get_server_lock(struct lock_server_info *inf, cmp = scoutfs_key_compare(key, &snode->key); if (cmp < 0) { + if (or_next) + next = snode; node = &(*node)->rb_left; } else if (cmp > 0) { node = &(*node)->rb_right; @@ -255,6 +270,9 @@ static struct server_lock_node *get_server_lock(struct lock_server_info *inf, ret = ins; } + if (ret == NULL && or_next && next) + ret = next; + if (ret) atomic_inc(&ret->refcount); @@ -266,6 +284,33 @@ static struct server_lock_node *get_server_lock(struct lock_server_info *inf, return ret; } +/* Get a server lock node, allocating if one doesn't exist. Caller must put. */ +static struct server_lock_node *alloc_server_lock(struct lock_server_info *inf, + struct scoutfs_key *key) +{ + struct server_lock_node *snode; + struct server_lock_node *ins; + + snode = get_server_lock(inf, key, NULL, false); + if (snode == NULL) { + ins = kzalloc(sizeof(struct server_lock_node), GFP_NOFS); + if (ins) { + atomic_set(&ins->refcount, 0); + mutex_init(&ins->mutex); + ins->key = *key; + INIT_LIST_HEAD(&ins->granted); + INIT_LIST_HEAD(&ins->requested); + INIT_LIST_HEAD(&ins->invalidated); + + snode = get_server_lock(inf, key, ins, false); + if (snode != ins) + kfree(ins); + } + } + + return snode; +} + /* * Finish with a server lock which has the mutex held, freeing it if * it's empty and unused. @@ -324,7 +369,6 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, DECLARE_LOCK_SERVER_INFO(sb, inf); struct client_lock_entry *clent; struct server_lock_node *snode; - struct server_lock_node *ins; int ret; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST, @@ -346,25 +390,11 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, clent->net_id = net_id; clent->mode = nl->new_mode; - snode = get_server_lock(inf, &nl->key, NULL); + snode = alloc_server_lock(inf, &nl->key); if (snode == NULL) { - ins = kzalloc(sizeof(struct server_lock_node), GFP_NOFS); - if (ins == NULL) { - kfree(clent); - ret = -ENOMEM; - goto out; - } - - atomic_set(&ins->refcount, 0); - mutex_init(&ins->mutex); - ins->key = nl->key; - INIT_LIST_HEAD(&ins->granted); - INIT_LIST_HEAD(&ins->requested); - INIT_LIST_HEAD(&ins->invalidated); - - snode = get_server_lock(inf, &nl->key, ins); - if (snode != ins) - kfree(ins); + kfree(clent); + ret = -ENOMEM; + goto out; } clent->snode = snode; @@ -401,7 +431,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, } /* XXX should always have a server lock here? recovery? */ - snode = get_server_lock(inf, &nl->key, NULL); + snode = get_server_lock(inf, &nl->key, NULL, false); if (!snode) { ret = -EINVAL; goto out; @@ -441,6 +471,14 @@ out: * This is called with the snode mutex held. This can free the snode if * it's empty. The caller can't reference the snode once this returns * so we unlock the snode mutex. + * + * All progress must wait for all clients to finish with recovery + * because we don't know which locks they'll hold. The unlocked + * recovery_pending test here is OK. It's filled by setup before + * anything runs. It's emptied by recovery completion. We can get a + * false nonempty result if we race with recovery completion, but that's + * OK because recovery completion processes all the locks that have + * requests after emptying, including the unlikely loser of that race. */ static int process_waiting_requests(struct super_block *sb, struct server_lock_node *snode) @@ -455,8 +493,9 @@ static int process_waiting_requests(struct super_block *sb, BUG_ON(!mutex_is_locked(&snode->mutex)); - /* request processing waits for all invalidation responses */ - if (!list_empty(&snode->invalidated)) { + /* processing waits for all invalidation responses or recovery */ + if (!list_empty(&snode->invalidated) || + !scoutfs_spbm_empty(&inf->recovery_pending)) { ret = 0; goto out; } @@ -523,6 +562,320 @@ out: return ret; } +/* + * The server received a greeting from a client for the first time. If + * the client had already talked to the server then we must find an + * existing record for it and should begin recovery. If it doesn't have + * a record then its timed out and we can't allow it to reconnect. If + * its connecting for the first time then we insert a new record. If + * + * This is running in concurrent client greeting processing contexts. + */ +int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, + bool should_exist) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_lock_client_btree_key cbk; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + cbk.node_id = cpu_to_be64(node_id); + + mutex_lock(&inf->mutex); + if (should_exist) { + ret = scoutfs_btree_lookup(sb, &super->lock_clients, + &cbk, sizeof(cbk), &iref); + if (ret == 0) + scoutfs_btree_put_iref(&iref); + } else { + ret = scoutfs_btree_insert(sb, &super->lock_clients, + &cbk, sizeof(cbk), NULL, 0); + } + mutex_unlock(&inf->mutex); + + if (should_exist && ret == 0) { + scoutfs_key_set_zeros(&key); + ret = scoutfs_server_lock_recover_request(sb, node_id, &key); + if (ret) + goto out; + } + +out: + return ret; +} + +/* + * A client sent their last recovery response and can exit recovery. If + * they were the last client in recovery then we can process all the + * server locks that had requests. + */ +static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct server_lock_node *snode; + struct scoutfs_key key; + bool still_pending; + int ret = 0; + + spin_lock(&inf->lock); + scoutfs_spbm_clear(&inf->recovery_pending, node_id); + still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); + spin_unlock(&inf->lock); + if (still_pending) + return 0; + + if (cancel) + cancel_delayed_work_sync(&inf->recovery_dwork); + + scoutfs_key_set_zeros(&key); + + while ((snode = get_server_lock(inf, &key, NULL, true))) { + + key = snode->key; + scoutfs_key_inc(&key); + + if (!list_empty(&snode->requested)) { + ret = process_waiting_requests(sb, snode); + if (ret) + break; + } else { + put_server_lock(inf, snode); + } + } + + return ret; +} + +/* + * We sent a lock recover request to the client when we received its + * greeting while in recovery. Here we instantiate all the locks it + * gave us in response and send another request from the next key. + * We're done once we receive an empty response. + */ +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock_recover *nlr) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct client_lock_entry *existing; + struct client_lock_entry *clent; + struct server_lock_node *snode; + struct scoutfs_key key; + int ret = 0; + int i; + + /* client must be in recovery */ + spin_lock(&inf->lock); + if (!scoutfs_spbm_test(&inf->recovery_pending, node_id)) + ret = -EINVAL; + spin_unlock(&inf->lock); + if (ret) + goto out; + + /* client has sent us all their locks */ + if (nlr->nr == 0) { + ret = finished_recovery(sb, node_id, true); + goto out; + } + + for (i = 0; i < le16_to_cpu(nlr->nr); i++) { + clent = kzalloc(sizeof(struct client_lock_entry), GFP_NOFS); + if (!clent) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&clent->head); + clent->node_id = node_id; + clent->net_id = 0; + clent->mode = nlr->locks[i].new_mode; + + snode = alloc_server_lock(inf, &nlr->locks[i].key); + if (snode == NULL) { + kfree(clent); + ret = -ENOMEM; + goto out; + } + + existing = find_entry(snode, &snode->granted, node_id); + if (existing) { + kfree(clent); + put_server_lock(inf, snode); + ret = -EEXIST; + goto out; + } + + clent->snode = snode; + add_client_entry(snode, &snode->granted, clent); + scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry); + + put_server_lock(inf, snode); + } + + /* send request for next batch of keys */ + key = nlr->locks[le16_to_cpu(nlr->nr) - 1].key; + scoutfs_key_inc(&key); + + ret = scoutfs_server_lock_recover_request(sb, node_id, &key); +out: + return ret; +} + +static int node_id_and_put_iref(struct scoutfs_btree_item_ref *iref, + u64 *node_id) +{ + struct scoutfs_lock_client_btree_key *cbk; + int ret; + + if (iref->key_len == sizeof(*cbk) && iref->val_len == 0) { + cbk = iref->key; + *node_id = be64_to_cpu(cbk->node_id); + ret = 0; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(iref); + return ret; +} + +/* + * This work executes if enough time passes without all of the clients + * finishing with recovery and canceling the work. We walk through the + * client records and find any that still have their recovery pending. + */ +static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) +{ + struct lock_server_info *inf = container_of(work, + struct lock_server_info, + recovery_dwork.work); + struct super_block *sb = inf->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_lock_client_btree_key cbk; + SCOUTFS_BTREE_ITEM_REF(iref); + bool timed_out; + u64 node_id; + int ret; + + /* we enter recovery if there are any client records */ + for (node_id = 0; ; node_id++) { + cbk.node_id = cpu_to_be64(node_id); + ret = scoutfs_btree_next(sb, &super->lock_clients, + &cbk, sizeof(cbk), &iref); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret == 0) + ret = node_id_and_put_iref(&iref, &node_id); + if (ret < 0) + break; + + spin_lock(&inf->lock); + if (scoutfs_spbm_test(&inf->recovery_pending, node_id)) { + scoutfs_spbm_clear(&inf->recovery_pending, node_id); + timed_out = true; + } else { + timed_out = false; + } + spin_unlock(&inf->lock); + + if (!timed_out) + continue; + + scoutfs_err(sb, "client node_id %llu lock recovery timed out", + node_id); + + /* XXX these aren't immediately committed */ + cbk.node_id = cpu_to_be64(node_id); + ret = scoutfs_btree_delete(sb, &super->lock_clients, + &cbk, sizeof(cbk)); + if (ret) + break; + } + + /* force processing all pending lock requests */ + if (ret == 0) + ret = finished_recovery(sb, 0, false); + + if (ret < 0) { + scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret); + scoutfs_server_stop(sb); + } +} + +/* + * A client is leaving the lock service. They aren't using locks and + * won't send any more requests. We tear down all the state we had for + * them. This can be called multiple times for a given client as their + * farewell is resent to new servers. It's OK to not find any state. + * If we fail to delete a persistent entry then we have to shut down and + * hope that the next server has more luck. + */ +int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) +{ + DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_lock_client_btree_key cli; + struct client_lock_entry *clent; + struct client_lock_entry *tmp; + struct server_lock_node *snode; + struct scoutfs_key key; + struct list_head *list; + bool freed; + int ret = 0; + + cli.node_id = cpu_to_be64(node_id); + mutex_lock(&inf->mutex); + ret = scoutfs_btree_delete(sb, &super->lock_clients, &cli, sizeof(cli)); + mutex_unlock(&inf->mutex); + if (ret == -ENOENT) { + ret = 0; + goto out; + } + if (ret < 0) + goto out; + + scoutfs_key_set_zeros(&key); + + while ((snode = get_server_lock(inf, &key, NULL, true))) { + + freed = false; + for (list = &snode->granted; list != NULL; + list = (list == &snode->granted) ? &snode->requested : + (list == &snode->requested) ? &snode->invalidated : + NULL) { + + list_for_each_entry_safe(clent, tmp, list, head) { + if (clent->node_id == node_id) { + free_client_entry(inf, snode, clent); + freed = true; + } + } + } + + key = snode->key; + scoutfs_key_inc(&key); + + if (freed) { + ret = process_waiting_requests(sb, snode); + if (ret) + goto out; + } else { + put_server_lock(inf, snode); + } + } + ret = 0; + +out: + if (ret < 0) { + scoutfs_err(sb, "lock server err %d during node %llu farewell, shutting down", ret, node_id); + scoutfs_server_stop(sb); + } + + return ret; +} + static char *lock_mode_string(u8 mode) { static char *mode_strings[] = { @@ -566,17 +919,35 @@ static void lock_server_tseq_show(struct seq_file *m, clent->net_id); } +/* + * Setup the lock server. This is called before networking can deliver + * requests. If we find existing client records then we enter recovery. + * Lock request processing is deferred until recovery is resolved for + * all the existing clients, either they reconnect and replay locks or + * we time them out. + */ int scoutfs_lock_server_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct lock_server_info *inf; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_lock_client_btree_key cbk; + unsigned int nr; + u64 node_id; + int ret; inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); if (!inf) return -ENOMEM; + inf->sb = sb; spin_lock_init(&inf->lock); + mutex_init(&inf->mutex); inf->locks_root = RB_ROOT; + scoutfs_spbm_init(&inf->recovery_pending); + INIT_DELAYED_WORK(&inf->recovery_dwork, + scoutfs_lock_server_recovery_timeout); scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); inf->tseq_dentry = scoutfs_tseq_create("server_locks", sbi->debug_root, @@ -588,7 +959,37 @@ int scoutfs_lock_server_setup(struct super_block *sb) sbi->lock_server_info = inf; - return 0; + /* we enter recovery if there are any client records */ + nr = 0; + for (node_id = 0; ; node_id++) { + cbk.node_id = cpu_to_be64(node_id); + ret = scoutfs_btree_next(sb, &super->lock_clients, + &cbk, sizeof(cbk), &iref); + if (ret == -ENOENT) + break; + if (ret == 0) + ret = node_id_and_put_iref(&iref, &node_id); + if (ret < 0) + goto out; + + ret = scoutfs_spbm_set(&inf->recovery_pending, node_id); + if (ret) + goto out; + nr++; + + if (node_id == U64_MAX) + break; + } + ret = 0; + + if (nr) { + schedule_delayed_work(&inf->recovery_dwork, + msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS)); + scoutfs_warn(sb, "waiting for %u lock clients to connect", nr); + } + +out: + return ret; } /* @@ -606,6 +1007,8 @@ void scoutfs_lock_server_destroy(struct super_block *sb) LIST_HEAD(list); if (inf) { + cancel_delayed_work_sync(&inf->recovery_dwork); + debugfs_remove(inf->tseq_dentry); rbtree_postorder_for_each_entry_safe(snode, stmp, @@ -624,6 +1027,8 @@ void scoutfs_lock_server_destroy(struct super_block *sb) kfree(snode); } + scoutfs_spbm_destroy(&inf->recovery_pending); + kfree(inf); sbi->lock_server_info = NULL; } diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 2b6f4b1f..cc0606a8 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -1,10 +1,15 @@ #ifndef _SCOUTFS_LOCK_SERVER_H_ #define _SCOUTFS_LOCK_SERVER_H_ +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, + struct scoutfs_net_lock_recover *nlr); int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, u64 net_id, struct scoutfs_net_lock *nl); +int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, + bool should_exist); int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, struct scoutfs_net_lock *nl); +int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id); int scoutfs_lock_server_setup(struct super_block *sb); void scoutfs_lock_server_destroy(struct super_block *sb); diff --git a/kmod/src/server.c b/kmod/src/server.c index 1077afe5..4f76a737 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -1066,6 +1066,39 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, nl, sizeof(*nl)); } +static bool invalid_recover(struct scoutfs_net_lock_recover *nlr, + unsigned long bytes) +{ + return ((bytes < sizeof(*nlr)) || + (bytes != offsetof(struct scoutfs_net_lock_recover, + locks[le16_to_cpu(nlr->nr)]))); +} + +static int lock_recover_response(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data) +{ + u64 node_id = scoutfs_net_client_node_id(conn); + + if (invalid_recover(resp, resp_len)) + return -EINVAL; + + return scoutfs_lock_server_recover_response(sb, node_id, resp); +} + +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, + struct scoutfs_key *key) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + + return scoutfs_net_submit_request_node(sb, server->conn, node_id, + SCOUTFS_NET_CMD_LOCK_RECOVER, + key, sizeof(*key), + lock_recover_response, + NULL, NULL); +} + /* * Process an incoming greeting request in the server from the client. * We try to send responses to failed greetings so that the sender can @@ -1083,6 +1116,14 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, * disconnect before they receive the response and resent and initial * blank greeting. We could use a client uuid to associate with * allocated node_ids. + * + * XXX The logic of this has gotten convoluted. The lock server can + * send a recovery request so it needs to be called after the core net + * greeting call enables messages. But we want the greeting reply to be + * sent first, so we currently queue it on the send queue before + * enabling messages. That means that a lot of errors that happen after + * the reply can't be sent to the client. They'll just see a disconnect + * and won't know what's happened. This all needs to be refactored. */ static int server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -1098,10 +1139,11 @@ static int server_greeting(struct super_block *sb, bool first_contact; bool farewell; int ret = 0; + int err; if (arg_len != sizeof(struct scoutfs_net_greeting)) { ret = -EINVAL; - goto out; + goto send_err; } if (gr->fsid != super->hdr.fsid) { @@ -1109,7 +1151,7 @@ static int server_greeting(struct super_block *sb, le64_to_cpu(gr->fsid), le64_to_cpu(super->hdr.fsid)); ret = -EINVAL; - goto out; + goto send_err; } if (gr->format_hash != super->format_hash) { @@ -1117,7 +1159,7 @@ static int server_greeting(struct super_block *sb, le64_to_cpu(gr->format_hash), le64_to_cpu(super->format_hash)); ret = -EINVAL; - goto out; + goto send_err; } if (gr->node_id == 0) { @@ -1131,35 +1173,58 @@ static int server_greeting(struct super_block *sb, queue_commit_work(server, &cw); up_read(&server->commit_rwsem); ret = wait_for_commit(&cw); - if (ret) { - node_id = 0; - goto out; - } } else { node_id = gr->node_id; } +send_err: + err = ret; + if (err) + node_id = 0; + greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); greet.node_id = node_id; greet.flags = 0; -out: - ret = scoutfs_net_response(sb, conn, cmd, id, ret, - &greet, sizeof(greet)); - if (node_id != 0 && ret == 0) { - sent_node_id = gr->node_id != 0; - first_contact = le64_to_cpu(gr->server_term) != server->term; - if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) - farewell = true; - else - farewell = false; - scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id, - sent_node_id, first_contact, - farewell); + /* queue greeting response to be sent first once messaging enabled */ + ret = scoutfs_net_response(sb, conn, cmd, id, err, + &greet, sizeof(greet)); + if (ret == 0 && err) + ret = err; + if (ret) + goto out; + + /* have the net core enable messaging and resend */ + sent_node_id = gr->node_id != 0; + first_contact = le64_to_cpu(gr->server_term) != server->term; + if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) + farewell = true; + else + farewell = false; + + scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id, + sent_node_id, first_contact, farewell); + + /* lock server might send recovery request */ + if (le64_to_cpu(gr->server_term) != server->term) { + + /* we're now doing two commits per greeting, not great */ + down_read(&server->commit_rwsem); + + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(node_id), + gr->server_term != 0); + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + if (ret) + goto out; } +out: return ret; } @@ -1178,12 +1243,25 @@ static int server_farewell(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) { + struct server_info *server = SCOUTFS_SB(sb)->server_info; + u64 node_id = scoutfs_net_client_node_id(conn); + struct commit_waiter cw; + int ret; + if (arg_len != 0) return -EINVAL; scoutfs_net_server_farewell(sb, conn); - return scoutfs_net_response(sb, conn, cmd, id, 0, NULL, 0); + down_read(&server->commit_rwsem); + ret = scoutfs_lock_server_farewell(sb, node_id); + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } /* requests sent to clients are tracked so we can free resources */ diff --git a/kmod/src/server.h b/kmod/src/server.h index a843e437..9d646813 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -68,6 +68,8 @@ int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, struct scoutfs_net_lock *nl); int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, u64 id, struct scoutfs_net_lock *nl); +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, + struct scoutfs_key *key); struct sockaddr_in; int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, From a546bd0aab0edbfcb31b9a65fb3c05b3ba0ea5d3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Feb 2019 17:15:23 -0800 Subject: [PATCH 691/920] scoutfs: check for newlines in msg.h wrappers The message formatter adds a newline so callers don't have to. But sometimes they do and we get double newlines. Add a build check that the format string doesn't end in a newline so that we stop adding these. And fix up all the current offenders. Signed-off-by: Zach Brown --- kmod/src/client.c | 4 ++-- kmod/src/lock.c | 2 +- kmod/src/msg.h | 12 +++++++++--- kmod/src/net.c | 2 +- kmod/src/options.c | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index d0d0325f..8149213d 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -351,7 +351,7 @@ static int client_greeting(struct super_block *sb, } if (sbi->node_id == 0 && gr->node_id == 0) { - scoutfs_warn(sb, "server sent node_id 0, client also has 0\n"); + scoutfs_warn(sb, "server sent node_id 0, client also has 0"); ret = -EINVAL; goto out; } @@ -667,7 +667,7 @@ void scoutfs_client_destroy(struct super_block *sb) } if (ret) { scoutfs_inc_counter(sb, client_farewell_error); - scoutfs_warn(sb, "client saw farewell error %d, server might see client connection time out\n", ret); + scoutfs_warn(sb, "client saw farewell error %d, server might see client connection time out", ret); } } diff --git a/kmod/src/lock.c b/kmod/src/lock.c index fa490b7c..d268dd82 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -345,7 +345,7 @@ static bool insert_range_node(struct super_block *sb, struct scoutfs_lock *ins) cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, &lock->start, &lock->end); if (WARN_ON_ONCE(cmp == 0)) { - scoutfs_warn(sb, "inserting lock start "SK_FMT" end "SK_FMT" overlaps with existing lock start "SK_FMT" end "SK_FMT"\n", + scoutfs_warn(sb, "inserting lock start "SK_FMT" end "SK_FMT" overlaps with existing lock start "SK_FMT" end "SK_FMT, SK_ARG(&ins->start), SK_ARG(&ins->end), SK_ARG(&lock->start), SK_ARG(&lock->end)); return false; diff --git a/kmod/src/msg.h b/kmod/src/msg.h index 8c7cea53..dbd33fb2 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -8,14 +8,20 @@ void __printf(4, 5) scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, const char *fmt, ...); +#define scoutfs_msg_check(sb, pref, str, fmt, args...) \ +do { \ + BUILD_BUG_ON(fmt[sizeof(fmt) - 2] == '\n'); \ + scoutfs_msg(sb, pref, str, fmt, ##args); \ +} while (0) + #define scoutfs_err(sb, fmt, args...) \ - scoutfs_msg(sb, KERN_ERR, " error", fmt, ##args) + scoutfs_msg_check(sb, KERN_ERR, " error", fmt, ##args) #define scoutfs_warn(sb, fmt, args...) \ - scoutfs_msg(sb, KERN_WARNING, " warning", fmt, ##args) + scoutfs_msg_check(sb, KERN_WARNING, " warning", fmt, ##args) #define scoutfs_info(sb, fmt, args...) \ - scoutfs_msg(sb, KERN_INFO, "", fmt, ##args) + scoutfs_msg_check(sb, KERN_INFO, "", fmt, ##args) #define scoutfs_bug_on(sb, cond, fmt, args...) \ do { \ diff --git a/kmod/src/net.c b/kmod/src/net.c index 41aff95e..6ea3ccd3 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -296,7 +296,7 @@ static inline u8 net_err_from_host(struct super_block *sb, int error) static bool warned; if (!warned) { warned = 1; - scoutfs_warn(sb, "host errno %d sent as EINVAL\n", + scoutfs_warn(sb, "host errno %d sent as EINVAL", error); } diff --git a/kmod/src/options.c b/kmod/src/options.c index 8fa7ab50..5c6ee18c 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -73,7 +73,7 @@ int scoutfs_parse_options(struct super_block *sb, char *options, return -EINVAL; break; default: - scoutfs_err(sb, "Unknown or malformed option, \"%s\"\n", + scoutfs_err(sb, "Unknown or malformed option, \"%s\"", p); break; } From 0bc0ff9300397a04f44bc913d73d95de9f028db0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 19 Feb 2019 14:38:12 -0800 Subject: [PATCH 692/920] scoutfs: add clock sync trace events Generate unique trace events on the send and recv side of each message sent between nodes. This can be used to reasonbly efficiently synchronize the monotonic clock in trace events between nodes given only their captured trace events. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/net.c | 5 +++++ kmod/src/scoutfs_trace.h | 26 ++++++++++++++++++++++++++ kmod/src/super.c | 32 ++++++++++++++++++++++++++++++++ kmod/src/super.h | 2 ++ 5 files changed, 66 insertions(+) diff --git a/kmod/src/format.h b/kmod/src/format.h index be2931dc..cf73e6be 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -625,6 +625,7 @@ struct scoutfs_net_greeting { * response messages. */ struct scoutfs_net_header { + __le64 clock_sync_id; __le64 seq; __le64 recv_seq; __le64 id; diff --git a/kmod/src/net.c b/kmod/src/net.c index 6ea3ccd3..98aae7b4 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -656,6 +656,8 @@ static void scoutfs_net_recv_worker(struct work_struct *work) break; } + trace_scoutfs_recv_clock_sync(nh.clock_sync_id); + data_len = le16_to_cpu(nh.data_len); scoutfs_inc_counter(sb, net_recv_messages); @@ -798,6 +800,9 @@ static void scoutfs_net_send_worker(struct work_struct *work) trace_scoutfs_net_send_message(sb, &conn->sockname, &conn->peername, &msend->nh); + msend->nh.clock_sync_id = scoutfs_clock_sync_id(); + trace_scoutfs_send_clock_sync(msend->nh.clock_sync_id); + ret = sendmsg_full(conn->sock, &msend->nh, len); spin_lock(&conn->lock); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0693f1ce..8834d196 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2513,6 +2513,32 @@ DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, TP_ARGS(sb, io_blkno, blk) ); +/* + * We can emit trace events to make it easier to synchronize the + * monotonic clocks in trace logs between nodes. By looking at the send + * and recv times of many messages flowing between nodes we can get + * surprisingly good estimates of the clock offset between them. + */ +DECLARE_EVENT_CLASS(scoutfs_clock_sync_class, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id), + TP_STRUCT__entry( + __field(__u64, clock_sync_id) + ), + TP_fast_assign( + __entry->clock_sync_id = le64_to_cpu(clock_sync_id); + ), + TP_printk("clock_sync_id %016llx", __entry->clock_sync_id) +); +DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_send_clock_sync, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id) +); +DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync, + TP_PROTO(__le64 clock_sync_id), + TP_ARGS(clock_sync_id) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/super.c b/kmod/src/super.c index 8149abd1..6e18e219 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "super.h" #include "block.h" @@ -47,6 +48,37 @@ static struct dentry *scoutfs_debugfs_root; +static DEFINE_PER_CPU(u64, clock_sync_ids) = 0; + +/* + * Give the caller a unique clock sync id for a message they're about to + * send. We make the ids reasonably globally unique by using randomly + * initialized per-cpu 64bit counters. + */ +__le64 scoutfs_clock_sync_id(void) +{ + u64 rnd = 0; + u64 ret; + u64 *id; + +retry: + preempt_disable(); + id = this_cpu_ptr(&clock_sync_ids); + if (*id == 0) { + if (rnd == 0) { + preempt_enable(); + get_random_bytes(&rnd, sizeof(rnd)); + goto retry; + } + *id = rnd; + } + + ret = ++(*id); + preempt_enable(); + + return cpu_to_le64(ret); +} + /* * Ask the server for the current statfs fields. The message is very * cheap so we're not worrying about spinning in statfs flooding the diff --git a/kmod/src/super.h b/kmod/src/super.h index 99671dab..e24f4d9a 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -91,4 +91,6 @@ int scoutfs_write_dirty_super(struct super_block *sb); /* to keep this out of the ioctl.h public interface definition */ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +__le64 scoutfs_clock_sync_id(void); + #endif From fa3e0a31c7d1566569bc75b199858cfa0f45ab45 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 24 Feb 2019 12:04:33 -0800 Subject: [PATCH 693/920] scoutfs: use SO_REUSEADDR for server socket The server's listening address is fixed by the raft config in the super block. If it shuts down and rapidly starts back up it needs to bind to the currently lingering address. Signed-off-by: Zach Brown --- kmod/src/net.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/net.c b/kmod/src/net.c index 98aae7b4..fae167f8 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -1360,6 +1360,7 @@ int scoutfs_net_bind(struct super_block *sb, { struct socket *sock = NULL; int addrlen; + int optval; int ret; /* caller state machine shouldn't let this happen */ @@ -1370,6 +1371,12 @@ int scoutfs_net_bind(struct super_block *sb, if (ret) goto out; + optval = 1; + ret = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&optval, sizeof(optval)); + if (ret) + goto out; + addrlen = sizeof(struct sockaddr_in); ret = kernel_bind(sock, (struct sockaddr *)sin, addrlen); if (ret) From 3d82dd3a4610fd44f7705854cfed08139dde933e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 24 Feb 2019 13:37:37 -0800 Subject: [PATCH 694/920] scoutfs: fix bad octet in tracing ipv4 address The macro for producing trace args for an ipv4 address had a typo when shifting the third octet down before masking. Signed-off-by: Zach Brown --- kmod/src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/server.h b/kmod/src/server.h index 9d646813..2162816d 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -18,7 +18,7 @@ do { \ #define si4_trace_args(name) \ (__entry->name##_addr >> 24), \ (__entry->name##_addr >> 16) & 255, \ - (__entry->name##_addr >> 0) & 255, \ + (__entry->name##_addr >> 8) & 255, \ __entry->name##_addr & 255, \ __entry->name##_port From e88b5732ad6731e0f8e19500f4b844be6f4ace7b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 Feb 2019 14:32:58 -0800 Subject: [PATCH 695/920] scoutfs: track trans seq in btree Currently the server tracks the outstanding transaction sequence numbers that clients have open in a simple list in memory. It's not properly cleaned up if a client unmounts and a new server that takes over after a crash won't know about open transaction sequence numbers. This stores open transaction sequence numbers in a shared persistent btree instead of in memory. It removes tracking for clients as they send their farewell during unmount. A new server that starts up will see existing entries for clients that were created by old servers. This fixes a bug where a client who unmounts could leave behind a pending sequence number that would never be cleaned up and would indefinitely limit the visibility of index items that came after it. Signed-off-by: Zach Brown --- kmod/src/btree.c | 1 + kmod/src/format.h | 12 ++- kmod/src/scoutfs_trace.h | 67 ++++++++++++++ kmod/src/server.c | 192 ++++++++++++++++++++++++++------------- 4 files changed, 210 insertions(+), 62 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index c4ef6921..bf6675e9 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -349,6 +349,7 @@ static size_t super_root_offsets[] = { offsetof(struct scoutfs_super_block, alloc_root), offsetof(struct scoutfs_super_block, manifest.root), offsetof(struct scoutfs_super_block, lock_clients), + offsetof(struct scoutfs_super_block, trans_seqs), }; #define for_each_super_root(super, i, root) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index cf73e6be..6918bd82 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -280,6 +280,15 @@ struct scoutfs_lock_client_btree_key { __be64 node_id; } __packed; +/* + * The server tracks transaction sequence numbers that clients have + * open. This limits results that can be returned from the seq indices. + */ +struct scoutfs_trans_seq_btree_key { + __be64 trans_seq; + __be64 node_id; +} __packed; + /* * The max number of links defines the max number of entries that we can * index in o(log n) and the static list head storage size in the @@ -453,7 +462,7 @@ struct scoutfs_super_block { __le64 format_hash; __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; - __le64 next_seq; + __le64 next_trans_seq; __le64 total_blocks; __le64 free_blocks; __le64 alloc_cursor; @@ -465,6 +474,7 @@ struct scoutfs_super_block { struct scoutfs_manifest manifest; struct scoutfs_quorum_config quorum_config; struct scoutfs_btree_root lock_clients; + struct scoutfs_btree_root trans_seqs; } __packed; #define SCOUTFS_ROOT_INO 1 diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 8834d196..3991ece4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2539,6 +2539,73 @@ DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync, TP_ARGS(clock_sync_id) ); +TRACE_EVENT(scoutfs_trans_seq_advance, + TP_PROTO(struct super_block *sb, u64 node_id, u64 prev_seq, + u64 next_seq), + + TP_ARGS(sb, node_id, prev_seq, next_seq), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, node_id) + __field(__u64, prev_seq) + __field(__u64, next_seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->node_id = node_id; + __entry->prev_seq = prev_seq; + __entry->next_seq = next_seq; + ), + + TP_printk("fsid "FSID_FMT" node_id %llu prev_seq %llu next_seq %llu", + __entry->fsid, __entry->node_id, __entry->prev_seq, + __entry->next_seq) +); + +TRACE_EVENT(scoutfs_trans_seq_farewell, + TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + + TP_ARGS(sb, node_id, trans_seq), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, node_id) + __field(__u64, trans_seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->node_id = node_id; + __entry->trans_seq = trans_seq; + ), + + TP_printk("fsid "FSID_FMT" node_id %llu trans_seq %llu", + __entry->fsid, __entry->node_id, __entry->trans_seq) +); + +TRACE_EVENT(scoutfs_trans_seq_last, + TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + + TP_ARGS(sb, node_id, trans_seq), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, node_id) + __field(__u64, trans_seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->node_id = node_id; + __entry->trans_seq = trans_seq; + ), + + TP_printk("fsid "FSID_FMT" node_id %llu trans_seq %llu", + __entry->fsid, __entry->node_id, __entry->trans_seq) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ diff --git a/kmod/src/server.c b/kmod/src/server.c index 4f76a737..4fbdce87 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -69,8 +69,7 @@ struct server_info { struct scoutfs_btree_root stable_manifest_root; /* server tracks seq use */ - spinlock_t seq_lock; - struct list_head pending_seqs; + struct rw_semaphore seq_rwsem; /* server tracks pending frees to be applied during commit */ struct rw_semaphore alloc_rwsem; @@ -862,22 +861,20 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } -struct pending_seq { - struct list_head head; - u64 seq; -}; - /* - * Give the client the next seq for it to use in items in its - * transaction. They tell us the seq they just used so we can remove it - * from pending tracking and possibly include it in get_last_seq - * replies. + * Give the client the next sequence number for their transaction. They + * provide their previous transaction sequence number that they've + * committed. * - * The list walk is O(clients) and the message processing rate goes from - * every committed segment to every sync deadline interval. + * We track the sequence numbers of transactions that clients have open. + * This limits the transaction sequence numbers that can be returned in + * the index of inodes by meta and data transaction numbers. We + * communicate the largest possible sequence number to clients via an + * rpc. * - * XXX The pending seq tracking should be persistent so that it survives - * server failover. + * The transaction sequence tracking is stored in a btree so it is + * shared across servers. Final entries are removed when processing a + * client's farewell or when it's removed. */ static int server_advance_seq(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -886,50 +883,113 @@ static int server_advance_seq(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct pending_seq *next_ps; - struct pending_seq *ps; struct commit_waiter cw; - __le64 * __packed their_seq = arg; + __le64 their_seq; __le64 next_seq; + struct scoutfs_trans_seq_btree_key tsk; + u64 node_id = scoutfs_net_client_node_id(conn); int ret; if (arg_len != sizeof(__le64)) { ret = -EINVAL; goto out; } - - next_ps = kmalloc(sizeof(struct pending_seq), GFP_NOFS); - if (!next_ps) { - ret = -ENOMEM; - goto out; - } + memcpy(&their_seq, arg, sizeof(their_seq)); down_read(&server->commit_rwsem); - spin_lock(&server->seq_lock); + down_write(&server->seq_rwsem); - list_for_each_entry(ps, &server->pending_seqs, head) { - if (ps->seq == le64_to_cpup(their_seq)) { - list_del_init(&ps->head); - kfree(ps); - break; - } + if (their_seq != 0) { + tsk.trans_seq = le64_to_be64(their_seq); + tsk.node_id = cpu_to_be64(node_id); + + ret = scoutfs_btree_delete(sb, &super->trans_seqs, + &tsk, sizeof(tsk)); + if (ret < 0 && ret != -ENOENT) + goto out; } - next_seq = super->next_seq; - le64_add_cpu(&super->next_seq, 1); + next_seq = super->next_trans_seq; + le64_add_cpu(&super->next_trans_seq, 1); - next_ps->seq = le64_to_cpu(next_seq); - list_add_tail(&next_ps->head, &server->pending_seqs); + trace_scoutfs_trans_seq_advance(sb, node_id, le64_to_cpu(their_seq), + le64_to_cpu(next_seq)); - spin_unlock(&server->seq_lock); - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - ret = wait_for_commit(&cw); + tsk.trans_seq = le64_to_be64(next_seq); + tsk.node_id = cpu_to_be64(node_id); + + ret = scoutfs_btree_insert(sb, &super->trans_seqs, + &tsk, sizeof(tsk), NULL, 0); out: + up_write(&server->seq_rwsem); + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + return scoutfs_net_response(sb, conn, cmd, id, ret, &next_seq, sizeof(next_seq)); } +/* + * Remove any transaction sequences owned by the client. They must have + * committed any final transaction by the time they get here via sending + * their farewell message. This can be called multiple times as the + * client's farewell is retransmitted so it's OK to not find any + * entries. This is called with the server commit rwsem held. + */ +static int remove_trans_seq(struct super_block *sb, u64 node_id) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_trans_seq_btree_key tsk; + SCOUTFS_BTREE_ITEM_REF(iref); + int ret = 0; + + down_write(&server->seq_rwsem); + + tsk.trans_seq = 0; + tsk.node_id = 0; + + for (;;) { + ret = scoutfs_btree_next(sb, &super->trans_seqs, + &tsk, sizeof(tsk), &iref); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + memcpy(&tsk, iref.key, iref.key_len); + scoutfs_btree_put_iref(&iref); + + if (be64_to_cpu(tsk.node_id) == node_id) { + trace_scoutfs_trans_seq_farewell(sb, node_id, + be64_to_cpu(tsk.trans_seq)); + ret = scoutfs_btree_delete(sb, &super->trans_seqs, + &tsk, sizeof(tsk)); + break; + } + + be64_add_cpu(&tsk.trans_seq, 1); + tsk.node_id = 0; + } + + up_write(&server->seq_rwsem); + + return ret; +} + +/* + * Give the calling client the last valid trans_seq that it can return + * in results from the indices of trans seqs to inodes. These indices + * promise to only advance so we can't return results past those that + * are still outstanding and not yet visible in the indices. If there + * are no outstanding transactions (what? how?) we give them the max + * possible sequence. + */ static int server_get_last_seq(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) @@ -937,8 +997,10 @@ static int server_get_last_seq(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct pending_seq *ps; - __le64 last_seq; + struct scoutfs_trans_seq_btree_key tsk; + SCOUTFS_BTREE_ITEM_REF(iref); + u64 node_id = scoutfs_net_client_node_id(conn); + __le64 last_seq = 0; int ret; if (arg_len != 0) { @@ -946,17 +1008,31 @@ static int server_get_last_seq(struct super_block *sb, goto out; } - spin_lock(&server->seq_lock); - ps = list_first_entry_or_null(&server->pending_seqs, - struct pending_seq, head); - if (ps) { - last_seq = cpu_to_le64(ps->seq - 1); - } else { - last_seq = super->next_seq; + down_read(&server->seq_rwsem); + + tsk.trans_seq = 0; + tsk.node_id = 0; + + ret = scoutfs_btree_next(sb, &super->trans_seqs, + &tsk, sizeof(tsk), &iref); + if (ret == 0) { + if (iref.key_len != sizeof(tsk)) { + ret = -EINVAL; + } else { + memcpy(&tsk, iref.key, iref.key_len); + last_seq = cpu_to_le64(be64_to_cpu(tsk.trans_seq) - 1); + } + scoutfs_btree_put_iref(&iref); + + } else if (ret == -ENOENT) { + last_seq = super->next_trans_seq; le64_add_cpu(&last_seq, -1ULL); + ret = 0; } - spin_unlock(&server->seq_lock); - ret = 0; + + trace_scoutfs_trans_seq_last(sb, node_id, le64_to_cpu(last_seq)); + + up_read(&server->seq_rwsem); out: return scoutfs_net_response(sb, conn, cmd, id, ret, &last_seq, sizeof(last_seq)); @@ -1254,9 +1330,12 @@ static int server_farewell(struct super_block *sb, scoutfs_net_server_farewell(sb, conn); down_read(&server->commit_rwsem); - ret = scoutfs_lock_server_farewell(sb, node_id); + + ret = scoutfs_lock_server_farewell(sb, node_id) ?: + remove_trans_seq(sb, node_id); if (ret == 0) queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); if (ret == 0) ret = wait_for_commit(&cw); @@ -1920,8 +1999,6 @@ static void scoutfs_server_worker(struct work_struct *work) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_connection *conn = NULL; - struct pending_seq *ps; - struct pending_seq *ps_tmp; DECLARE_WAIT_QUEUE_HEAD(waitq); struct sockaddr_in sin; LIST_HEAD(conn_list); @@ -1989,12 +2066,6 @@ shutdown: scoutfs_btree_destroy(sb); scoutfs_lock_server_destroy(sb); - /* XXX these should be persistent and reclaimed during recovery */ - list_for_each_entry_safe(ps, ps_tmp, &server->pending_seqs, head) { - list_del_init(&ps->head); - kfree(ps); - } - out: scoutfs_net_free_conn(sb, conn); @@ -2049,8 +2120,7 @@ int scoutfs_server_setup(struct super_block *sb) init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); seqcount_init(&server->stable_seqcount); - spin_lock_init(&server->seq_lock); - INIT_LIST_HEAD(&server->pending_seqs); + init_rwsem(&server->seq_rwsem); init_rwsem(&server->alloc_rwsem); INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); From fe63b566c9f01bc6c8a98539be4251ef9c2ae705 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 25 Mar 2019 17:09:41 -0700 Subject: [PATCH 696/920] scoutfs: use _unaligned instead of __packed We were relying on a cute (and probably broken) trick of defining pointers to unaligned base types with __packed. Modern versions of gcc warn about this. Instead we either directly access unaligned types with get_ and put_unaligned, or we copy unaligned data into aligned copies before working with it. Signed-off-by: Zach Brown --- kmod/src/server.c | 46 +++++++++++++++++++++++++++------------------- kmod/src/super.c | 5 +++-- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 4fbdce87..f18c1d81 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "format.h" #include "counters.h" @@ -1473,7 +1474,7 @@ static void forget_client_compacts(struct super_block *sb, } } -static int segno_in_ents(__le64 segno, struct scoutfs_net_manifest_entry *ents, +static int segno_in_ents(u64 segno, struct scoutfs_net_manifest_entry *ents, unsigned int nr) { int i; @@ -1481,42 +1482,44 @@ static int segno_in_ents(__le64 segno, struct scoutfs_net_manifest_entry *ents, for (i = 0; i < nr; i++) { if (ents[i].segno == 0) break; - if (segno == ents[i].segno) + if (segno == le64_to_cpu(ents[i].segno)) return 1; } return 0; } -static int remove_segnos(struct super_block *sb, __le64 * __packed segnos, +static int remove_segnos(struct super_block *sb, __le64 *segnos, unsigned int nr, struct scoutfs_net_manifest_entry *unless, unsigned int nr_unless, bool cleanup); /* - * Free segnos if they're not found in the unless entries. If this - * returns an error then we've cleaned up partial frees on error. This - * panics if it sees an error and can't cleanup on error. + * Free (unaligned) segnos if they're not found in the unless entries. + * If this returns an error then we've cleaned up partial frees on + * error. This panics if it sees an error and can't cleanup on error. * * There are variants of this for lots of add/del, alloc/remove data * structurs. */ -static int free_segnos(struct super_block *sb, __le64 * __packed segnos, +static int free_segnos(struct super_block *sb, __le64 *segnos, unsigned int nr, struct scoutfs_net_manifest_entry *unless, unsigned int nr_unless, bool cleanup) { + u64 segno; int ret = 0; int i; for (i = 0; i < nr; i++) { - if (segnos[i] == 0) + segno = le64_to_cpu(get_unaligned(&segnos[i])); + if (segno == 0) break; - if (segno_in_ents(segnos[i], unless, nr_unless)) + if (segno_in_ents(segno, unless, nr_unless)) continue; - ret = free_segno(sb, le64_to_cpu(segnos[i])); + ret = free_segno(sb, segno); BUG_ON(ret < 0 && !cleanup); if (ret < 0) { remove_segnos(sb, segnos, i, unless, nr_unless, false); @@ -1527,8 +1530,9 @@ static int free_segnos(struct super_block *sb, __le64 * __packed segnos, return ret; } -static int alloc_segnos(struct super_block *sb, __le64 * __packed segnos, - unsigned int nr) +/* the segno array can be unaligned */ +static int alloc_segnos(struct super_block *sb, __le64 * segnos, + unsigned int nr) { u64 segno; @@ -1541,28 +1545,30 @@ static int alloc_segnos(struct super_block *sb, __le64 * __packed segnos, free_segnos(sb, segnos, i, NULL, 0, false); break; } - segnos[i] = cpu_to_le64(segno); + put_unaligned(cpu_to_le64(segno), &segnos[i]); } return ret; } -static int remove_segnos(struct super_block *sb, __le64 * __packed segnos, +static int remove_segnos(struct super_block *sb, __le64 *segnos, unsigned int nr, struct scoutfs_net_manifest_entry *unless, unsigned int nr_unless, bool cleanup) { + u64 segno; int ret = 0; int i; for (i = 0; i < nr; i++) { - if (segnos[i] == 0) + segno = le64_to_cpu(get_unaligned(&segnos[i])); + if (segno == 0) break; - if (segno_in_ents(segnos[i], unless, nr_unless)) + if (segno_in_ents(segno, unless, nr_unless)) continue; - ret = remove_segno(sb, le64_to_cpu(segnos[i])); + ret = remove_segno(sb, segno); BUG_ON(ret < 0 && !cleanup); if (ret < 0) { free_segnos(sb, segnos, i, unless, nr_unless, false); @@ -1592,7 +1598,8 @@ static int free_entry_segnos(struct super_block *sb, for (i = 0; i < nr; i++) { if (ents[i].segno == 0) break; - if (segno_in_ents(ents[i].segno, unless, nr_unless)) + if (segno_in_ents(le64_to_cpu(ents[i].segno), + unless, nr_unless)) continue; ret = free_segno(sb, le64_to_cpu(ents[i].segno)); @@ -1619,7 +1626,8 @@ static int remove_entry_segnos(struct super_block *sb, for (i = 0; i < nr; i++) { if (ents[i].segno == 0) break; - if (segno_in_ents(ents[i].segno, unless, nr_unless)) + if (segno_in_ents(le64_to_cpu(ents[i].segno), + unless, nr_unless)) continue; ret = remove_segno(sb, le64_to_cpu(ents[i].segno)); diff --git a/kmod/src/super.c b/kmod/src/super.c index 6e18e219..500ffaa3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -97,7 +97,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) { struct super_block *sb = dentry->d_inode->i_sb; struct scoutfs_net_statfs nstatfs; - __le32 * __packed uuid; + __le32 uuid[4]; int ret; ret = scoutfs_client_statfs(sb, &nstatfs); @@ -113,7 +113,8 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_ffree = kst->f_bfree * 16; kst->f_files = kst->f_ffree + le64_to_cpu(nstatfs.next_ino); - uuid = (void *)nstatfs.uuid; + BUILD_BUG_ON(sizeof(uuid) != sizeof(nstatfs.uuid)); + memcpy(uuid, &nstatfs, sizeof(uuid)); kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); kst->f_namelen = SCOUTFS_NAME_LEN; From 36b0df336b98e123d484d8691f14fb45a9ba3fdf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 8 Apr 2019 12:44:03 -0700 Subject: [PATCH 697/920] scoutfs: add unmount barrier Now that a mount's client is responsible for electing and starting a server we need to be careful about coordinating unmount. We can't let unmounting clients leave the remaining mounted clients without quorum. The server carefully tracks who is mounted and who is unmounting while it is processing farewell requests. It only sends responses to voting mounts while quorum remains or once all the voting clients are all trying to unmount. We use a field in the quorum blocks to communicate to the final set of unmounting voters that their farewells have been processed and that they can finish unmounting without trying to restablish quorum. The commit introduces and maintains the unmount_barrier field in the quorum blocks. It is passed to the server from the election, the server sends it to the client and writes new versions, and the client compares what it received with what it sees in quorum blocks. The commit then has the clients send their unique name to the server who stores it in persistent mounted client records and compares the names to the quorum config when deciding which farewell reqeusts can be responded to. Now that farewell response processing can block for a very long time it is moved off into async work so that it doesn't prevent net connections from being shutdown and re-established. This also makes it easier to make global decisions based on the count of pending farewell requests. Signed-off-by: Zach Brown --- kmod/src/btree.c | 1 + kmod/src/client.c | 32 +++- kmod/src/format.h | 28 +++- kmod/src/lock_server.c | 4 +- kmod/src/net.c | 23 +-- kmod/src/net.h | 2 - kmod/src/quorum.c | 99 ++++++++++-- kmod/src/quorum.h | 10 ++ kmod/src/scoutfs_trace.h | 6 +- kmod/src/server.c | 336 +++++++++++++++++++++++++++++++++++++-- kmod/src/server.h | 7 +- 11 files changed, 488 insertions(+), 60 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index bf6675e9..006eacd3 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -350,6 +350,7 @@ static size_t super_root_offsets[] = { offsetof(struct scoutfs_super_block, manifest.root), offsetof(struct scoutfs_super_block, lock_clients), offsetof(struct scoutfs_super_block, trans_seqs), + offsetof(struct scoutfs_super_block, mounted_clients), }; #define for_each_super_root(super, i, root) \ diff --git a/kmod/src/client.c b/kmod/src/client.c index 8149213d..8d3bcee6 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -59,6 +59,7 @@ struct client_info { u64 old_elected_nr; u64 server_term; + u64 greeting_umb; bool sending_farewell; int farewell_error; @@ -365,22 +366,27 @@ static int client_greeting(struct super_block *sb, scoutfs_net_client_greeting(sb, conn, new_server); client->server_term = le64_to_cpu(gr->server_term); + client->greeting_umb = le64_to_cpu(gr->unmount_barrier); ret = 0; out: return ret; } /* - * If the previous election told us to start the server then stop it - * and wipe the old election info. If we're not fast enough to clear - * the election block then the next server might fence us. Should - * be very unlikely as election requires multiple RMW cycles. + * If the previous election told us to start the server then stop it and + * clear the indication that we were elected. We get the current + * version of the election info from the server because they might have + * modified it while they were running. the old election info. + * + * If we're not fast enough to clear the election from the quorum block + * then the next server might fence us. Should be very unlikely as + * election requires multiple RMW cycles. */ static void stop_our_server(struct super_block *sb, struct scoutfs_quorum_elected_info *qei) { if (qei->run_server) { - scoutfs_server_stop(sb); + scoutfs_server_stop(sb, qei); scoutfs_quorum_clear_elected(sb, qei); memset(qei, 0, sizeof(*qei)); } @@ -431,12 +437,22 @@ static void scoutfs_client_connect_worker(struct work_struct *work) ret = scoutfs_quorum_election(sb, opts->uniq_name, client->old_elected_nr, - timeout_abs, qei); + timeout_abs, client->sending_farewell, + client->greeting_umb, qei); if (ret) goto out; + /* we saw that the server wrote a new unmount barrier */ + if (client->sending_farewell && qei->elected_nr == 0 && + qei->unmount_barrier > client->greeting_umb) { + client->farewell_error = 0; + complete(&client->farewell_comp); + ret = 0; + goto out; + } + if (qei->run_server) { - ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr); + ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr, qei); if (ret) { /* forget that we tried to start the server */ memset(qei, 0, sizeof(*qei)); @@ -459,9 +475,11 @@ static void scoutfs_client_connect_worker(struct work_struct *work) client->old_elected_nr = 0; /* send a greeting to verify endpoints of each connection */ + memcpy(greet.name, opts->uniq_name, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); + greet.unmount_barrier = 0; greet.node_id = cpu_to_le64(sbi->node_id); greet.flags = 0; if (client->sending_farewell) diff --git a/kmod/src/format.h b/kmod/src/format.h index 6918bd82..b5ced5d4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -48,6 +48,8 @@ #define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS +#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ + /* * Base types used by other structures. */ @@ -289,6 +291,18 @@ struct scoutfs_trans_seq_btree_key { __be64 node_id; } __packed; +/* + * The server keeps a persistent record of mounted clients. + */ +struct scoutfs_mounted_client_btree_key { + __be64 node_id; +} __packed; + +struct scoutfs_mounted_client_btree_val { + __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; +} __packed; + + /* * The max number of links defines the max number of entries that we can * index in o(log n) and the static list head storage size in the @@ -395,7 +409,6 @@ struct scoutfs_xattr { #define member_sizeof(TYPE, MEMBER) (sizeof(((TYPE *)0)->MEMBER)) #define SCOUTFS_UUID_BYTES 16 -#define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ /* * During each quorum voting interval the fabric has to process 2 reads @@ -418,6 +431,7 @@ struct scoutfs_xattr { * @config_gen: references the config gen in the super block * @write_nr: incremented for every write, only 0 when never written * @elected_nr: incremented when elected, 0 otherwise + * @unmount_barrier: incremented by servers when all members have unmounted * @vote_slot: the active config slot that the writer is voting for */ struct scoutfs_quorum_block { @@ -426,6 +440,7 @@ struct scoutfs_quorum_block { __le64 config_gen; __le64 write_nr; __le64 elected_nr; + __le64 unmount_barrier; __le32 crc; __u8 vote_slot; } __packed; @@ -475,6 +490,7 @@ struct scoutfs_super_block { struct scoutfs_quorum_config quorum_config; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; + struct scoutfs_btree_root mounted_clients; } __packed; #define SCOUTFS_ROOT_INO 1 @@ -593,20 +609,30 @@ enum { * Greetings verify identity of communicating nodes. The sender sends * their credentials and the receiver verifies them. * + * @name: The client sends its unique name to the server. + * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it * tries to reconnect. Used to identify a client reconnecting to a * server that has timed out its connection. * + * @unmount_barrier: Incremented every time the remaining majority of + * quorum members all agree to leave. The server tells a quorum member + * the value that it's connecting under so that if the client sees the + * value increase in a quorum block it knows that the server has + * processed its farewell and can safely unmount. + * * @node_id: The id of the client. Initially 0 from the client, * assigned by the server, and sent by the client as it reconnects. * Used by the server to identify reconnecting clients whose existing * state must be dealt with. */ struct scoutfs_net_greeting { + __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; __le64 fsid; __le64 format_hash; __le64 server_term; + __le64 unmount_barrier; __le64 node_id; __le64 flags; } __packed; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 5c55362c..da7e1062 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -800,7 +800,7 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) if (ret < 0) { scoutfs_err(sb, "lock server saw err %d while timing out clients, shutting down", ret); - scoutfs_server_stop(sb); + scoutfs_server_abort(sb); } } @@ -870,7 +870,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) out: if (ret < 0) { scoutfs_err(sb, "lock server err %d during node %llu farewell, shutting down", ret, node_id); - scoutfs_server_stop(sb); + scoutfs_server_abort(sb); } return ret; diff --git a/kmod/src/net.c b/kmod/src/net.c index fae167f8..8449ce56 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -100,7 +100,7 @@ struct scoutfs_net_connection { established:1, /* added sends queue send work */ shutting_down:1, /* shutdown work has been queued */ saw_greeting:1, /* saw greeting on this sock */ - saw_farewell:1, /* saw farewell request from client */ + saw_farewell:1, /* saw farewell response to client */ reconn_wait:1, /* shutdown, waiting for reconnect */ reconn_freeing:1; /* waiting done, setter frees */ unsigned long reconn_deadline; @@ -788,6 +788,11 @@ static void scoutfs_net_send_worker(struct work_struct *work) continue; } + if ((msend->nh.cmd == SCOUTFS_NET_CMD_FAREWELL) && + nh_is_response(&msend->nh)) { + conn->saw_farewell = 1; + } + msend->nh.recv_seq = cpu_to_le64(atomic64_read(&conn->recv_seq)); @@ -1629,22 +1634,6 @@ restart: conn->notify_up(sb, conn, conn->info, node_id); } -/* - * The server has received a farewell message and is sending a response. - * All we do is mark the connection so that it is freed the next time it - * is shutdown, presumably as the client disconnects after receiving the - * response. The server caller has cleaned up all the state it had - * associated with the client. - */ -void scoutfs_net_server_farewell(struct super_block *sb, - struct scoutfs_net_connection *conn) -{ - spin_lock(&conn->lock); - conn->saw_farewell = 1; - spin_unlock(&conn->lock); -} - - /* * Submit a request down the connection. It's up to the caller to * ensure that the conn is allocated. Sends submitted when the diff --git a/kmod/src/net.h b/kmod/src/net.h index 82da64bf..c144c69e 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -75,8 +75,6 @@ void scoutfs_net_server_greeting(struct super_block *sb, u64 node_id, u64 greeting_id, bool sent_node_id, bool first_contact, bool farewell); -void scoutfs_net_server_farewell(struct super_block *sb, - struct scoutfs_net_connection *conn); void scoutfs_net_farewell(struct super_block *sb, struct scoutfs_net_connection *conn); diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 112e2da9..7f90e407 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -392,7 +392,8 @@ static inline int first_slot_flags(struct scoutfs_quorum_config *conf, */ static int write_quorum_block(struct super_block *sb, __le64 fsid, __le64 config_gen, u8 our_slot, __le64 write_nr, - u64 elected_nr, u8 vote_slot) + u64 elected_nr, u64 unmount_barrier, + u8 vote_slot) { struct scoutfs_quorum_block *blk; struct buffer_head *bh; @@ -416,6 +417,7 @@ static int write_quorum_block(struct super_block *sb, __le64 fsid, blk->config_gen = config_gen; blk->write_nr = write_nr; blk->elected_nr = cpu_to_le64(elected_nr); + blk->unmount_barrier = cpu_to_le64(unmount_barrier); blk->vote_slot = vote_slot; blk->crc = quorum_block_crc(blk); @@ -473,8 +475,8 @@ static int fence_other_elected(struct super_block *sb, scoutfs_inc_counter(sb, quorum_fenced); ret = write_quorum_block(sb, super->hdr.fsid, - conf->gen, i, blk.write_nr, - 0, i); + conf->gen, i, blk.write_nr, 0, + le64_to_cpu(blk.unmount_barrier), i); if (ret) break; } @@ -509,9 +511,13 @@ struct quorum_block_history { * When we return success we update the caller's elected info with the * most recent elected leader we found, which may well be long gone. We * return -ENOENT if we didn't find any elected leaders. + * + * If we return success because we saw a larger unmount barrier we set + * elected_nr to 0 and fill the unmount_barrier. */ int scoutfs_quorum_election(struct super_block *sb, char *our_name, u64 old_elected_nr, ktime_t timeout_abs, + bool unmounting, u64 our_umb, struct scoutfs_quorum_elected_info *qei) { struct scoutfs_super_block *super = NULL; @@ -524,6 +530,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, ktime_t now; __le64 write_nr = 0; u64 elected_nr = 0; + u64 unmount_barrier = 0; int vote_streak = 0; int vote_slot; int our_slot; @@ -551,13 +558,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, goto out; conf = &super->quorum_config; - /* allow a single vote majority when 1 or 2 active */ - if (nr_active <= 2) - majority = 1; - else if (nr_active & 1) - majority = (nr_active + 1) / 2; - else - majority = (nr_active / 2) + 1; + majority = scoutfs_quorum_majority(sb, conf); readahead_quorum_blocks(sb); @@ -590,6 +591,8 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, qei->config_gen = blk.config_gen; qei->write_nr = blk.write_nr; qei->elected_nr = le64_to_cpu(blk.elected_nr); + qei->unmount_barrier = + le64_to_cpu(blk.unmount_barrier); qei->config_slot = i; } } @@ -639,12 +642,23 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, nr_votes = 0; write_nr = cpu_to_le64(1); elected_nr = 0; + unmount_barrier = 0; for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ /* count our votes (maybe including from us) */ if (hi->writing >= 2 && blk.vote_slot == our_slot) nr_votes++; + /* can finish unmounting if members all left */ + if (unmounting && + le64_to_cpu(blk.unmount_barrier) > our_umb) { + qei->elected_nr = 0; + qei->unmount_barrier = + le64_to_cpu(blk.unmount_barrier); + ret = 0; + goto out; + } + /* sample existing fields for our write */ if (i == our_slot) { write_nr = blk.write_nr; @@ -652,6 +666,8 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, } elected_nr = max(elected_nr, le64_to_cpu(blk.elected_nr)); + unmount_barrier = max(unmount_barrier, + le64_to_cpu(blk.unmount_barrier)); } @@ -667,7 +683,8 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, elected_nr = 0; write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, - write_nr, elected_nr, vote_slot); + write_nr, elected_nr, unmount_barrier, + vote_slot); set_current_state(TASK_UNINTERRUPTIBLE); schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); @@ -714,5 +731,65 @@ int scoutfs_quorum_clear_elected(struct super_block *sb, return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, qei->config_slot, qei->write_nr, 0, + qei->unmount_barrier, qei->config_slot); +} + +int scoutfs_quorum_update_barrier(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei, + u64 unmount_barrier) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + qei->unmount_barrier = unmount_barrier; + + return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, + qei->config_slot, qei->write_nr, + qei->elected_nr, qei->unmount_barrier, qei->config_slot); } + +/* + * If there's only one or two active slots then a single vote is sufficient + * for a majority. + */ +int scoutfs_quorum_majority(struct super_block *sb, + struct scoutfs_quorum_config *conf) +{ + struct scoutfs_quorum_slot *slot; + int nr_active = 0; + int majority; + int i; + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + slot = &conf->slots[i]; + + if (slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) + nr_active++; + } + + if (nr_active <= 2) + majority = 1; + else if (nr_active & 1) + majority = (nr_active + 1) / 2; + else + majority = (nr_active / 2) + 1; + + return majority; +} + +bool scoutfs_quorum_voting_member(struct super_block *sb, + struct scoutfs_quorum_config *conf, + char *name) +{ + struct scoutfs_quorum_slot *slot; + int i; + + for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { + slot = &conf->slots[i]; + + if (strcmp(slot->name, name) == 0) + return true; + } + + return false; +} diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index 82e6708e..c40e9d48 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -6,14 +6,24 @@ struct scoutfs_quorum_elected_info { __le64 config_gen; __le64 write_nr; u64 elected_nr; + u64 unmount_barrier; unsigned int config_slot; bool run_server; }; int scoutfs_quorum_election(struct super_block *sb, char *our_name, u64 old_elected_nr, ktime_t timeout_abs, + bool unmounting, u64 our_umb, struct scoutfs_quorum_elected_info *qei); int scoutfs_quorum_clear_elected(struct super_block *sb, struct scoutfs_quorum_elected_info *qei); +int scoutfs_quorum_update_barrier(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei, + u64 unmount_barrier); +int scoutfs_quorum_majority(struct super_block *sb, + struct scoutfs_quorum_config *conf); +bool scoutfs_quorum_voting_member(struct super_block *sb, + struct scoutfs_quorum_config *conf, + char *name); #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 3991ece4..48040606 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2482,6 +2482,7 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, __field(__u64, config_gen) __field(__u64, write_nr) __field(__u64, elected_nr) + __field(__u64, unmount_barrier) __field(__u32, crc) __field(__u8, vote_slot) ), @@ -2493,14 +2494,15 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, __entry->config_gen = le64_to_cpu(blk->config_gen); __entry->write_nr = le64_to_cpu(blk->write_nr); __entry->elected_nr = le64_to_cpu(blk->elected_nr); + __entry->unmount_barrier = le64_to_cpu(blk->unmount_barrier); __entry->crc = le32_to_cpu(blk->crc); __entry->vote_slot = blk->vote_slot; ), - TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu crc 0x%08x vote_slot %u", + TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u", __entry->fsid, __entry->io_blkno, __entry->hdr_blkno, __entry->config_gen, __entry->write_nr, __entry->elected_nr, - __entry->crc, __entry->vote_slot) + __entry->unmount_barrier, __entry->crc, __entry->vote_slot) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, TP_PROTO(struct super_block *sb, u64 io_blkno, diff --git a/kmod/src/server.c b/kmod/src/server.c index f18c1d81..fd7cda3d 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -35,6 +35,7 @@ #include "net.h" #include "lock_server.h" #include "endian_swap.h" +#include "quorum.h" /* * Every active mount can act as the server that listens on a net @@ -60,6 +61,8 @@ struct server_info { u64 term; struct scoutfs_net_connection *conn; + struct scoutfs_quorum_elected_info qei; + /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; @@ -84,6 +87,11 @@ struct server_info { unsigned long nr_compacts; struct list_head compacts; struct work_struct compact_work; + + /* track clients waiting in unmmount for farewell response */ + struct mutex farewell_mutex; + struct list_head farewell_requests; + struct work_struct farewell_work; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -1176,6 +1184,46 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, NULL, NULL); } +static int insert_mounted_client(struct super_block *sb, u64 node_id, + char *name) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_key mck; + struct scoutfs_mounted_client_btree_val mcv; + + mck.node_id = cpu_to_be64(node_id); + strncpy(mcv.name, name, sizeof(mcv.name)); + + return scoutfs_btree_insert(sb, &super->mounted_clients, + &mck, sizeof(mck), &mcv, sizeof(mcv)); +} + +/* + * Remove the record of a mounted client. The record can already be + * removed if we're processing a farewell on behalf of a client that + * already had a previous server process its farewell. + * + * When we remove the last mounted client that's voting we write a new + * quorum block with the updated unmount_barrier. + * + * The caller has to serialize with farewell processing. + */ +static int delete_mounted_client(struct super_block *sb, u64 node_id) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_mounted_client_btree_key mck; + int ret; + + mck.node_id = cpu_to_be64(node_id); + + ret = scoutfs_btree_delete(sb, &super->mounted_clients, + &mck, sizeof(mck)); + if (ret == -ENOENT) + ret = 0; + + return ret; +} + /* * Process an incoming greeting request in the server from the client. * We try to send responses to failed greetings so that the sender can @@ -1247,9 +1295,17 @@ static int server_greeting(struct super_block *sb, le64_add_cpu(&super->next_node_id, 1); spin_unlock(&server->lock); - queue_commit_work(server, &cw); + mutex_lock(&server->farewell_mutex); + ret = insert_mounted_client(sb, le64_to_cpu(node_id), gr->name); + mutex_unlock(&server->farewell_mutex); + + if (ret == 0) + queue_commit_work(server, &cw); up_read(&server->commit_rwsem); - ret = wait_for_commit(&cw); + if (ret == 0) { + ret = wait_for_commit(&cw); + queue_work(server->wq, &server->farewell_work); + } } else { node_id = gr->node_id; } @@ -1259,9 +1315,11 @@ send_err: if (err) node_id = 0; + memset(greet.name, 0, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); + greet.unmount_barrier = cpu_to_le64(server->qei.unmount_barrier); greet.node_id = node_id; greet.flags = 0; @@ -1305,6 +1363,223 @@ out: return ret; } +struct farewell_request { + struct list_head entry; + u64 net_id; + u64 node_id; +}; + +static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) +{ + return (iref->key_len != + sizeof(struct scoutfs_mounted_client_btree_key)) || + (iref->val_len != + sizeof(struct scoutfs_mounted_client_btree_val)); +} + +/* + * This work processes farewell requests asynchronously. Requests from + * voting quorum members can be held until they're no longer needed to + * vote for quorum and elect a server to process farewell requests. + * + * This will hold farewell requests from voting clients until either it + * isn't needed for quorum because a majority remains without it, or it + * won't be needed for quorum because all the remaining mounted clients + * are voting and waiting for farewell. + * + * When we remove the last mounted client record for the last voting + * client then we increase the unmount_barrier and write it to the + * server's quorum block. If voting clients don't get their farewell + * response they'll attempt to form quorum again to start the server for + * their farewell response but will find the increased umount_barrier. + * The'll know that their farewell has been processed and they can exit + * without forming quorum. + * + * Responses that are waiting for clients who aren't voting are + * immediately sent. Clients that don't have a mounted client record + * have already had their farewell processed by another server and can + * proceed. + * + * This can trust the quorum config found in the super that was read + * when the server started. Only the current server can rewrite the + * working config. + * + * Farewell responses are unique in that sending them causes the server + * to shutdown the connection to the client next time the socket + * disconnects. If the socket is destroyed before the client gets the + * response they'll reconnect and we'll see them as a brand new client + * who immediately sends a farewell. It'll be processed and it all + * works out. + * + * If this worker sees an error it assumes that this sever is done for + * and that another had better take its place. + */ +static void farewell_worker(struct work_struct *work) +{ + struct server_info *server = container_of(work, struct server_info, + farewell_work); + struct super_block *sb = server->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_quorum_config *conf = &super->quorum_config; + struct scoutfs_mounted_client_btree_key mck; + struct scoutfs_mounted_client_btree_val *mcv; + struct farewell_request *tmp; + struct farewell_request *fw; + SCOUTFS_BTREE_ITEM_REF(iref); + struct commit_waiter cw; + unsigned int nr_unmounting = 0; + unsigned int nr_mounted = 0; + unsigned int majority; + LIST_HEAD(reqs); + LIST_HEAD(send); + bool deleted = false; + bool voting; + bool more_reqs; + int ret; + + majority = scoutfs_quorum_majority(sb, conf); + + /* grab all the requests that are waiting */ + mutex_lock(&server->farewell_mutex); + list_splice_init(&server->farewell_requests, &reqs); + mutex_unlock(&server->farewell_mutex); + + /* count how many reqs requests are from voting clients */ + nr_unmounting = 0; + list_for_each_entry_safe(fw, tmp, &reqs, entry) { + mck.node_id = cpu_to_be64(fw->node_id); + ret = scoutfs_btree_lookup(sb, &super->mounted_clients, + &mck, sizeof(mck), &iref); + if (ret == 0 && invalid_mounted_client_item(&iref)) { + scoutfs_btree_put_iref(&iref); + ret = -EIO; + } + if (ret < 0) { + if (ret == -ENOENT) { + list_move_tail(&fw->entry, &send); + continue; + } + goto out; + } + + mcv = iref.val; + voting = scoutfs_quorum_voting_member(sb, conf, mcv->name); + scoutfs_btree_put_iref(&iref); + + if (!voting) { + list_move_tail(&fw->entry, &send); + continue; + } + + nr_unmounting++; + } + + /* see how many mounted clients could vote for quorum */ + memset(&mck, 0, sizeof(mck)); + for (;;) { + ret = scoutfs_btree_next(sb, &super->mounted_clients, + &mck, sizeof(mck), &iref); + if (ret == 0 && invalid_mounted_client_item(&iref)) { + scoutfs_btree_put_iref(&iref); + ret = -EIO; + } + if (ret != 0) { + if (ret == -ENOENT) + break; + goto out; + } + + memcpy(&mck, iref.key, sizeof(mck)); + mcv = iref.val; + + if (scoutfs_quorum_voting_member(sb, conf, mcv->name)) + nr_mounted++; + + scoutfs_btree_put_iref(&iref); + be64_add_cpu(&mck.node_id, 1); + + } + + /* send as many responses as we can to maintain quorum */ + while ((fw = list_first_entry_or_null(&reqs, struct farewell_request, + entry)) && + (nr_mounted > majority || nr_unmounting >= nr_mounted)) { + + list_move_tail(&fw->entry, &send); + nr_mounted--; + nr_unmounting--; + deleted = true; + } + + /* process and send farewell responses */ + list_for_each_entry_safe(fw, tmp, &send, entry) { + + down_read(&server->commit_rwsem); + + ret = scoutfs_lock_server_farewell(sb, fw->node_id) ?: + remove_trans_seq(sb, fw->node_id) ?: + delete_mounted_client(sb, fw->node_id); + if (ret == 0) + queue_commit_work(server, &cw); + + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + if (ret) + goto out; + } + + /* update the unmount barrier the first time we delete all mounted */ + if (deleted && nr_mounted == 0) { + ret = scoutfs_quorum_update_barrier(sb, &server->qei, + server->qei.unmount_barrier + 1); + if (ret) + goto out; + } + + /* and finally send all the responses */ + list_for_each_entry_safe(fw, tmp, &send, entry) { + + ret = scoutfs_net_response_node(sb, server->conn, fw->node_id, + SCOUTFS_NET_CMD_FAREWELL, + fw->net_id, 0, NULL, 0); + if (ret) + break; + + list_del_init(&fw->entry); + kfree(fw); + } + + ret = 0; +out: + mutex_lock(&server->farewell_mutex); + more_reqs = !list_empty(&server->farewell_requests); + list_splice_init(&reqs, &server->farewell_requests); + list_splice_init(&send, &server->farewell_requests); + mutex_unlock(&server->farewell_mutex); + + if (ret < 0) + stop_server(server); + else if (more_reqs && !server->shutting_down) + queue_work(server->wq, &server->farewell_work); +} + +static void free_farewell_requests(struct super_block *sb, u64 node_id) +{ + struct server_info *server = SCOUTFS_SB(sb)->server_info; + struct farewell_request *tmp; + struct farewell_request *fw; + + mutex_lock(&server->farewell_mutex); + list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) { + if (node_id == 0 || fw->node_id == node_id) { + list_del_init(&fw->entry); + kfree(fw); + } + } + mutex_unlock(&server->farewell_mutex); +} + /* * The server is receiving a farewell message from a client that is * unmounting. It won't send any more requests and once it receives our @@ -1322,26 +1597,28 @@ static int server_farewell(struct super_block *sb, { struct server_info *server = SCOUTFS_SB(sb)->server_info; u64 node_id = scoutfs_net_client_node_id(conn); - struct commit_waiter cw; - int ret; + struct farewell_request *fw; if (arg_len != 0) return -EINVAL; - scoutfs_net_server_farewell(sb, conn); + /* XXX tear down if we fence, or if we shut down */ - down_read(&server->commit_rwsem); + fw = kmalloc(sizeof(struct farewell_request), GFP_NOFS); + if (fw == NULL) + return -ENOMEM; - ret = scoutfs_lock_server_farewell(sb, node_id) ?: - remove_trans_seq(sb, node_id); - if (ret == 0) - queue_commit_work(server, &cw); + fw->node_id = node_id; + fw->net_id = id; - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); + mutex_lock(&server->farewell_mutex); + list_add_tail(&fw->entry, &server->farewell_requests); + mutex_unlock(&server->farewell_mutex); - return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); + queue_work(server->wq, &server->farewell_work); + + /* response will be sent later */ + return 0; } /* requests sent to clients are tracked so we can free resources */ @@ -1992,6 +2269,8 @@ static void server_notify_down(struct super_block *sb, server->nr_clients); spin_unlock(&server->lock); + free_farewell_requests(sb, node_id); + forget_client_compacts(sb, sci); try_queue_compact(server); } else { @@ -2085,7 +2364,7 @@ out: /* XXX can we call start multiple times? */ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term) + u64 term, struct scoutfs_quorum_elected_info *qei) { DECLARE_SERVER_INFO(sb, server); @@ -2093,6 +2372,7 @@ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, server->shutting_down = false; server->listen_sin = *sin; server->term = term; + server->qei = *qei; init_completion(&server->start_comp); queue_work(server->wq, &server->work); @@ -2101,7 +2381,22 @@ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, return server->err; } -void scoutfs_server_stop(struct super_block *sb) +/* + * Start shutdown on the server but don't want for it to finish. + */ +void scoutfs_server_abort(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + + stop_server(server); +} + +/* + * Once the server is stopped we give the caller our election info + * which might have been modified while we were running. + */ +void scoutfs_server_stop(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei) { DECLARE_SERVER_INFO(sb, server); @@ -2109,6 +2404,8 @@ void scoutfs_server_stop(struct super_block *sb) /* XXX not sure both are needed */ cancel_work_sync(&server->work); cancel_work_sync(&server->commit_work); + + *qei = server->qei; } int scoutfs_server_setup(struct super_block *sb) @@ -2135,6 +2432,9 @@ int scoutfs_server_setup(struct super_block *sb) server->compacts_per_client = 2; INIT_LIST_HEAD(&server->compacts); INIT_WORK(&server->compact_work, scoutfs_server_compact_worker); + mutex_init(&server->farewell_mutex); + INIT_LIST_HEAD(&server->farewell_requests); + INIT_WORK(&server->farewell_work, farewell_worker); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); @@ -2164,6 +2464,10 @@ void scoutfs_server_destroy(struct super_block *sb) /* recv work/compaction could have left commit_work queued */ cancel_work_sync(&server->commit_work); + /* pending farewell requests are another server's problem */ + cancel_work_sync(&server->farewell_work); + free_farewell_requests(sb, 0); + trace_scoutfs_server_workqueue_destroy(sb, 0, 0); destroy_workqueue(server->wq); diff --git a/kmod/src/server.h b/kmod/src/server.h index 2162816d..fee6ac0e 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -72,9 +72,12 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, struct scoutfs_key *key); struct sockaddr_in; +struct scoutfs_quorum_elected_info; int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term); -void scoutfs_server_stop(struct super_block *sb); + u64 term, struct scoutfs_quorum_elected_info *qei); +void scoutfs_server_abort(struct super_block *sb); +void scoutfs_server_stop(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei); int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); From b5133bfc986840102f166bad87f18f7803cdc1bc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 10 Apr 2019 14:51:23 -0700 Subject: [PATCH 698/920] scoutfs: add elected flag to quorum block It was a mistake to use a non-zero elected_nr as the indication that a slot is considered actively elected. Zeroing it as the server shuts down wipes the elected_nr and means that it doesn't advance as each server is elected. This then causes a client connecting to a new server to be confused for a client reconnecting to a server after the server has timed it out and destroyed its state. This caused reconnection after shutting down a server to fail and clients to loop reconnecting indefinitely. This instead adds flags to the quorum block and assigns a flag to indicate that the slot should be considered active. It's cleared by fencing and by the client as the server shuts down. Signed-off-by: Zach Brown --- kmod/src/format.h | 4 ++++ kmod/src/quorum.c | 52 +++++++++++++++++++++++++--------------- kmod/src/quorum.h | 1 + kmod/src/scoutfs_trace.h | 7 ++++-- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index b5ced5d4..936b0717 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -443,8 +443,12 @@ struct scoutfs_quorum_block { __le64 unmount_barrier; __le32 crc; __u8 vote_slot; + __u8 flags; } __packed; +#define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0) +#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 1) + #define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS /* diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 7f90e407..87cdd186 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -297,7 +297,8 @@ static bool invalid_quorum_block(struct scoutfs_super_block *super, return quorum_block_crc(blk) != blk->crc || blk->fsid != super->hdr.fsid || le64_to_cpu(blk->blkno) != bh->b_blocknr || - blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS; + blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS || + (blk->flags & SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN); } /* @@ -393,7 +394,7 @@ static inline int first_slot_flags(struct scoutfs_quorum_config *conf, static int write_quorum_block(struct super_block *sb, __le64 fsid, __le64 config_gen, u8 our_slot, __le64 write_nr, u64 elected_nr, u64 unmount_barrier, - u8 vote_slot) + u8 vote_slot, u8 flags) { struct scoutfs_quorum_block *blk; struct buffer_head *bh; @@ -419,6 +420,7 @@ static int write_quorum_block(struct super_block *sb, __le64 fsid, blk->elected_nr = cpu_to_le64(elected_nr); blk->unmount_barrier = cpu_to_le64(unmount_barrier); blk->vote_slot = vote_slot; + blk->flags = flags; blk->crc = quorum_block_crc(blk); @@ -464,19 +466,24 @@ static int fence_other_elected(struct super_block *sb, { struct scoutfs_quorum_config *conf = &super->quorum_config; struct scoutfs_quorum_block blk; + u8 flags; int ret; int i; for_each_block(sb, super, i, &blk) { if (i != our_slot && - le64_to_cpu(blk.elected_nr) > 0 && + (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && le64_to_cpu(blk.elected_nr) <= elected_nr) { scoutfs_err(sb, "would have fenced"); scoutfs_inc_counter(sb, quorum_fenced); + flags = blk.flags & ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; + ret = write_quorum_block(sb, super->hdr.fsid, - conf->gen, i, blk.write_nr, 0, - le64_to_cpu(blk.unmount_barrier), i); + conf->gen, i, blk.write_nr, + le64_to_cpu(blk.elected_nr), + le64_to_cpu(blk.unmount_barrier), i, + flags); if (ret) break; } @@ -531,6 +538,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, __le64 write_nr = 0; u64 elected_nr = 0; u64 unmount_barrier = 0; + u8 flags = 0; int vote_streak = 0; int vote_slot; int our_slot; @@ -586,6 +594,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, /* find the most recently elected leader */ if ((blk.config_gen == conf->gen) && + (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ addr_to_sin(&qei->sin, &slot->addr); qei->config_gen = blk.config_gen; @@ -594,6 +603,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, qei->unmount_barrier = le64_to_cpu(blk.unmount_barrier); qei->config_slot = i; + qei->flags = blk.flags; } } @@ -605,7 +615,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, * most recent, or we couldn't fence, then we fall back * to participating in the election. */ - if (elected_nr != 0) { + if (flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) { if (qei->write_nr == write_nr && qei->elected_nr == elected_nr && qei->config_slot == our_slot) { @@ -643,6 +653,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, write_nr = cpu_to_le64(1); elected_nr = 0; unmount_barrier = 0; + flags = 0; for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ /* count our votes (maybe including from us) */ @@ -677,14 +688,14 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, else vote_streak = 0; - if (vote_streak >= 2) + if (vote_streak >= 2) { + flags |= SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; elected_nr++; - else - elected_nr = 0; + } write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, write_nr, elected_nr, unmount_barrier, - vote_slot); + vote_slot, flags); set_current_state(TASK_UNINTERRUPTIBLE); schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); @@ -705,14 +716,14 @@ out: /* * The calling server is shutting down and has finished modifying - * persistent state. We clear elected_nr from our quorum block so that - * mounts won't try to connect and so that the next next leader won't - * try to fence. + * persistent state. We clear the elected flag from our quorum block so + * that mounts won't try to connect and so that the next next leader + * won't try to fence. * * By definition nothing has written to the slot since we wrote our - * elected_nr and the slot could not have been reclaimed. To reclaim - * the slot would have required proving that we were gone or fencing - * us. + * elected quorum block and the slot could not have been reclaimed. To + * reclaim the slot would have required proving that we were gone or + * fencing us. * * If this fails then the mount is in trouble because it'll probably be * fenced by the next elected leader. @@ -729,9 +740,12 @@ int scoutfs_quorum_clear_elected(struct super_block *sb, { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + qei->flags &= ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; + return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, 0, - qei->unmount_barrier, qei->config_slot); + qei->config_slot, qei->write_nr, + qei->elected_nr, qei->unmount_barrier, + qei->config_slot, qei->flags); } int scoutfs_quorum_update_barrier(struct super_block *sb, @@ -745,7 +759,7 @@ int scoutfs_quorum_update_barrier(struct super_block *sb, return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, qei->config_slot, qei->write_nr, qei->elected_nr, qei->unmount_barrier, - qei->config_slot); + qei->config_slot, qei->flags); } /* diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index c40e9d48..26a45b5b 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -9,6 +9,7 @@ struct scoutfs_quorum_elected_info { u64 unmount_barrier; unsigned int config_slot; bool run_server; + u8 flags; }; int scoutfs_quorum_election(struct super_block *sb, char *our_name, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 48040606..5e85305e 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2485,6 +2485,7 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, __field(__u64, unmount_barrier) __field(__u32, crc) __field(__u8, vote_slot) + __field(__u8, flags) ), TP_fast_assign( @@ -2497,12 +2498,14 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, __entry->unmount_barrier = le64_to_cpu(blk->unmount_barrier); __entry->crc = le32_to_cpu(blk->crc); __entry->vote_slot = blk->vote_slot; + __entry->flags = blk->flags; ), - TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u", + TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", __entry->fsid, __entry->io_blkno, __entry->hdr_blkno, __entry->config_gen, __entry->write_nr, __entry->elected_nr, - __entry->unmount_barrier, __entry->crc, __entry->vote_slot) + __entry->unmount_barrier, __entry->crc, __entry->vote_slot, + __entry->flags) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, TP_PROTO(struct super_block *sb, u64 io_blkno, From 6342bd5679b12bfe4f18a50fd92238e0bb9dcd76 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 11 Apr 2019 12:42:29 -0700 Subject: [PATCH 699/920] scoutfs: update README.md for quorum Update the github README to describe the recent addition of integrated quorum voting and locking. Signed-off-by: Zach Brown --- kmod/README.md | 60 ++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 39 deletions(-) diff --git a/kmod/README.md b/kmod/README.md index 7cb77e01..542b4ecb 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -21,11 +21,11 @@ Learn more in the [white paper](https://docs.wixstatic.com/ugd/aaa89b_88a5cc84be # Current Status -**Initial Alpha Open Source Release** +**Alpha Open Source Development** -scoutfs is under heavy active development. We're releasing before it's -completely polished to give the community an opportunity to affect the -design and implementation. Nothing is cast in stone. +scoutfs is under heavy active development. We're developing it in the +open to give the community an opportunity to affect the design and +implementation. The core architectural design elements are in place. Much surrounding functionality hasn't been implemented. It's appropriate for early @@ -60,9 +60,6 @@ for all discussion of scoutfs. running, experience will be needed to fill in the gaps. We're happy to help on the mailing list.** -__Some software components (pacemaker?) may be packaged seperately by -distributions.__ - The requirements for running scoutfs on a small cluster are: 1. One or more nodes running x86-64 CentOS/RHEL 7.4 (or 7.3) @@ -71,35 +68,14 @@ The requirements for running scoutfs on a small cluster are: The steps for getting scoutfs mounted and operational are: - 1. Configure pacemaker clustering and the kernel DLM for locking - 2. Get the kernel module running on the nodes - 3. Make a new filesystem on the device with the userspace utilities - 4. Mount the device on all the nodes + 1. Get the kernel module running on the nodes + 2. Make a new filesystem on the device with the userspace utilities + 3. Mount the device on all the nodes In this example we run all of these commands on two nodes. The block -device name is the same on all the nodes. The listen= mount option is -given the local IP address of each node. +device name is the same on all the nodes. -1. Configure and Start the DLM - - - ```shell - yum install pcs pacemaker fence-agents-all - firewall-cmd --permanent --add-service=high-availability - firewall-cmd --add-service=high-availability - passwd hacluster - systemctl start pcsd.service - systemctl enable pcsd.service - pcs cluster auth node1 node2 - pcs cluster setup --start --name scoutfs node1 node2 - pcs cluster enable - - yum install dlm - systemctl enable dlm - systemctl start dlm - ``` - -2. Get the Kernel Module and Userspace Binaries +1. Get the Kernel Module and Userspace Binaries * Either use snapshot RPMs built from git by Versity: @@ -123,23 +99,29 @@ given the local IP address of each node. ``` -3. Make a New Filesystem (**destroys contents, no questions asked**) +2. Make a New Filesystem (**destroys contents, no questions asked**) + + We specify that every node will participate in quorum voting by + configuring each in the super block with options to mkfs. ```shell - scoutfs mkfs /dev/shared_block_device + scoutfs mkfs -o quorum_slot node1:0:172.16.1.1 \ + -o quorum_slot node2:0:172.16.1.2 /dev/shared_block_device ``` -4. Mount the Filesystem +3. Mount the Filesystem + + Each mounting node provides the name that was given to the + quorum\_slot option to mkfs. ```shell mkdir /mnt/scoutfs - mount -t scoutfs -o cluster=scoutfs,listen=node_ip_address \ - /dev/shared_block_device /mnt/scoutfs + mount -t scoutfs -o uniq_name=$NODENAME /dev/shared_block_device /mnt/scoutfs ``` -5. For Kicks, Observe the Metadata Change Index +4. For Kicks, Observe the Metadata Change Index The `meta_seq` index tracks the inodes that are changed in each transaction. From 7097d545cfb95b0bc19d1081a5fa5ae1b6c37211 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Apr 2019 12:19:35 -0700 Subject: [PATCH 700/920] scoutfs: make sure to set the sb blocksize Since fill_super was originally written we've added use of buffer_head IO by the btree and quorum voting. We forgot to set the block size so devices that didn't have the common 4k default, matching our block size, would see errors. Explicitly set it. Signed-off-by: Zach Brown --- kmod/src/super.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 500ffaa3..02f38934 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -343,13 +343,19 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); - ret = scoutfs_parse_options(sb, data, &opts); if (ret) goto out; sbi->opts = opts; + ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE); + if (ret != SCOUTFS_BLOCK_SIZE) { + scoutfs_err(sb, "failed to set blocksize, returned %d", ret); + ret = -EIO; + goto out; + } + ret = scoutfs_setup_sysfs(sb) ?: scoutfs_setup_counters(sb) ?: scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: From 3a6392aee6be1bc1609519099cc1cbc2833a2e5b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 Apr 2019 10:58:58 -0700 Subject: [PATCH 701/920] scoutfs: remove scoutfs_unlock_flags() prototype There was an old prototype for an unlock variant that hasn't been around for a while. Remove it. Signed-off-by: Zach Brown --- kmod/src/lock.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kmod/src/lock.h b/kmod/src/lock.h index b6cbf767..c3230316 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -72,8 +72,6 @@ int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, u64 node_id, struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); -void scoutfs_unlock_flags(struct super_block *sb, struct scoutfs_lock *lock, - int level, int flags); void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov); void scoutfs_lock_add_coverage(struct super_block *sb, From cfa563a4a4b04303d788302fa6827bfdc841582f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 Apr 2019 11:00:02 -0700 Subject: [PATCH 702/920] scoutfs: expand the per_task API This adds some minor functionality to the per_task API for use by the upcoming offline waiting work. Add scoutfs_per_task_add_excl() so that a caller can tell if their task was already put on a per-task list by their caller. Make scoutfs_per_task_del() return a bool to indicate if the entry was found on a list and was in fact deleted, or not. Add scoutfs_per_task_init_entry() for initializing entries that aren't declared on the stack. Signed-off-by: Zach Brown --- kmod/src/per_task.c | 32 +++++++++++++++++++++++++++++++- kmod/src/per_task.h | 5 ++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/kmod/src/per_task.c b/kmod/src/per_task.c index 24b7f412..c0424e4d 100644 --- a/kmod/src/per_task.c +++ b/kmod/src/per_task.c @@ -62,7 +62,27 @@ void scoutfs_per_task_add(struct scoutfs_per_task *pt, spin_unlock(&pt->lock); } -void scoutfs_per_task_del(struct scoutfs_per_task *pt, +/* + * Add the entry to the per-task list if the task didn't already have an + * entry on the list. Returns true if the entry was added, false if it + * wasn't. + */ +bool scoutfs_per_task_add_excl(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr) +{ + if (!scoutfs_per_task_get(pt)) { + scoutfs_per_task_add(pt, ent, ptr); + return true; + } + + return false; +} + +/* + * Return true if the entry was found on the list and was deleted, + * returns false if the entry wasn't present on a list. + */ +bool scoutfs_per_task_del(struct scoutfs_per_task *pt, struct scoutfs_per_task_entry *ent) { BUG_ON(!list_empty(&ent->head) && ent->task != current); @@ -71,7 +91,10 @@ void scoutfs_per_task_del(struct scoutfs_per_task *pt, spin_lock(&pt->lock); list_del_init(&ent->head); spin_unlock(&pt->lock); + return true; } + + return false; } void scoutfs_per_task_init(struct scoutfs_per_task *pt) @@ -79,3 +102,10 @@ void scoutfs_per_task_init(struct scoutfs_per_task *pt) spin_lock_init(&pt->lock); INIT_LIST_HEAD(&pt->list); } + +void scoutfs_per_task_init_entry(struct scoutfs_per_task_entry *ent) +{ + INIT_LIST_HEAD(&ent->head); + ent->task = NULL; + ent->ptr = NULL; +} diff --git a/kmod/src/per_task.h b/kmod/src/per_task.h index 6a055391..38616616 100644 --- a/kmod/src/per_task.h +++ b/kmod/src/per_task.h @@ -21,8 +21,11 @@ struct scoutfs_per_task_entry { void *scoutfs_per_task_get(struct scoutfs_per_task *pt); void scoutfs_per_task_add(struct scoutfs_per_task *pt, struct scoutfs_per_task_entry *ent, void *ptr); -void scoutfs_per_task_del(struct scoutfs_per_task *pt, +bool scoutfs_per_task_add_excl(struct scoutfs_per_task *pt, + struct scoutfs_per_task_entry *ent, void *ptr); +bool scoutfs_per_task_del(struct scoutfs_per_task *pt, struct scoutfs_per_task_entry *ent); void scoutfs_per_task_init(struct scoutfs_per_task *pt); +void scoutfs_per_task_init_entry(struct scoutfs_per_task_entry *ent); #endif From a6782fc03ff3959d065232b6c9a762721783f4d2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Apr 2019 09:38:34 -0700 Subject: [PATCH 703/920] scoutfs: add data waiting One of the core features of scoutfs is the ability to transparently migrate file contents to and from an archive tier. For this to be transparent we need file system operations to trigger staging the file contents back into the file system as needed. This adds the infrastructure which operations use to wait for offline extents to come online and which provides userspace with a list of blocks that the operations are waiting for. We add some waiting infrastructure that callers use to lock, check for offline extents, and unlock and wait before checking again to see if they're still offline. We add these checks and waiting to data io operations that could encounter offline extents. This has to be done carefully so that we don't wait while holding locks that would prevent staging. We use per-task structures to discover when we are the first user of a cluster lock on an inode, indicating that it's safe for us to wait because we don't hold any locks. And while we're waiting our operation is tracked and reported to userspace through an ioctl. This is a non-blocking ioctl, it's up to userspace to decide how often to check and how large a region to stage. Waiters are woken up when the file contents could have changed, not specifically when we know that the extent has come online. This lets us wake waiters when their lock is revoked so that they can block waiting to reacquire the lock and test the extents again. It lets us provide coherent demand staging across the cluster without fine grained waiting protocols sent betwen the nodes. It may result in some spurious wakeups and work but hopefully it won't, and it's a very simple and functional first pass. Signed-off-by: Zach Brown --- kmod/src/data.c | 311 ++++++++++++++++++++++++++++++++++++++- kmod/src/data.h | 50 +++++++ kmod/src/file.c | 54 ++++++- kmod/src/format.h | 5 +- kmod/src/inode.c | 43 ++++++ kmod/src/inode.h | 3 + kmod/src/ioctl.c | 52 +++++++ kmod/src/ioctl.h | 24 +++ kmod/src/lock.c | 5 +- kmod/src/scoutfs_trace.h | 39 +++++ kmod/src/super.c | 2 + kmod/src/super.h | 4 + 12 files changed, 576 insertions(+), 16 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index cee53511..ce31e642 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -732,9 +732,9 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, if (ext.len) trace_scoutfs_data_get_block_intersection(sb, &ext); - /* fail read and write if it's offline and we're not staging */ - if ((ext.flags & SEF_OFFLINE) && !si->staging) { - ret = -EINVAL; + /* non-staging callers should have waited on offline blocks */ + if (WARN_ON_ONCE((ext.flags & SEF_OFFLINE) && !si->staging)) { + ret = -EIO; goto out; } @@ -780,14 +780,28 @@ out: /* * This is almost never used. We can't block on a cluster lock while * holding the page lock because lock invalidation gets the page lock - * while blocking locks. If we can't use an existing lock then we drop - * the page lock and try again. + * while blocking locks. If a non blocking lock attempt fails we unlock + * the page and block acquiring the lock. We unlocked the page so it + * could have been truncated away, or whatever, so we return + * AOP_TRUNCATED_PAGE to have the caller try again. + * + * A similar process happens if we try to read from an offline extent + * that a caller hasn't already waited for. Instead of blocking + * acquiring the lock we block waiting for the offline extent. The page + * lock protects the page from release while we're checking and + * reading the extent. + * + * We can return errors from locking and checking offline extents. The + * page is unlocked if we return an error. */ static int scoutfs_readpage(struct file *file, struct page *page) { struct inode *inode = file->f_inode; + struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); int flags; int ret; @@ -809,27 +823,77 @@ static int scoutfs_readpage(struct file *file, struct page *page) return ret; } + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + ret = scoutfs_data_wait_check(inode, page_offset(page), + PAGE_CACHE_SIZE, SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, &dw, + inode_lock); + if (ret != 0) { + unlock_page(page); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + } + if (ret > 0) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + ret = AOP_TRUNCATED_PAGE; + } + if (ret != 0) + return ret; + } + ret = mpage_readpage(page, scoutfs_get_block); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); + return ret; } +/* + * This is used for opportunistic read-ahead which can throw the pages + * away if it needs to. If the caller didn't deal with offline extents + * then we drop those pages rather than trying to wait. Whoever is + * staging offline extents should be doing it in enormous chunks so that + * read-ahead can ramp up within each staged region. The check for + * offline extents is cheap when the inode has no offline extents. + */ static int scoutfs_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { struct inode *inode = file->f_inode; struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; + struct page *page; + struct page *tmp; int ret; ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); if (ret) - return ret; + goto out; + + list_for_each_entry_safe(page, tmp, pages, lru) { + ret = scoutfs_data_wait_check(inode, page_offset(page), + PAGE_CACHE_SIZE, SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, NULL, + inode_lock); + if (ret < 0) + goto out; + if (ret > 0) { + list_del(&page->lru); + page_cache_release(page); + if (--nr_pages == 0) { + ret = 0; + goto out; + } + } + } ret = mpage_readpages(mapping, pages, nr_pages, scoutfs_get_block); - +out: scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + BUG_ON(!list_empty(pages)); return ret; } @@ -1249,6 +1313,239 @@ out: return ret; } +/* + * Insert a new waiter. This supports multiple tasks waiting for the + * same ino and iblock by also comparing waiters by their addresses. + */ +static void insert_offline_waiting(struct rb_root *root, + struct scoutfs_data_wait *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_data_wait *dw; + int cmp; + + while (*node) { + parent = *node; + dw = rb_entry(*node, struct scoutfs_data_wait, node); + + cmp = scoutfs_cmp_u64s(ins->ino, dw->ino) ?: + scoutfs_cmp_u64s(ins->iblock, dw->iblock) ?: + scoutfs_cmp(ins, dw); + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); +} + +static struct scoutfs_data_wait *next_data_wait(struct rb_root *root, u64 ino, + u64 iblock) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct scoutfs_data_wait *next = NULL; + struct scoutfs_data_wait *dw; + int cmp; + + while (*node) { + parent = *node; + dw = rb_entry(*node, struct scoutfs_data_wait, node); + + /* go left when ino/iblock are equal to get first task */ + cmp = scoutfs_cmp_u64s(ino, dw->ino) ?: + scoutfs_cmp_u64s(iblock, dw->iblock); + if (cmp <= 0) { + node = &(*node)->rb_left; + next = dw; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } + } + + return next; +} + +static struct scoutfs_data_wait *dw_next(struct scoutfs_data_wait *dw) +{ + struct rb_node *node = rb_next(&dw->node); + if (node) + return container_of(node, struct scoutfs_data_wait, node); + return NULL; +} + +/* + * Check if we should wait by looking for extents whose flags match. + * Returns 0 if no extents were found or any error encountered. + * + * The caller must have locked the extents before calling, both across + * mounts and within this mount. + * + * Returns 1 if any file extents in the caller's region matched. If the + * wait struct is provided then it is initialized to be woken when the + * extents change after the caller unlocks after the check. The caller + * must come through _data_wait() to clean up the wait struct if we set + * it up. + */ +int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, + u8 sef, u8 op, struct scoutfs_data_wait *dw, + struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + DECLARE_DATA_WAIT_ROOT(sb, rt); + DECLARE_DATA_WAITQ(inode, wq); + struct scoutfs_extent ext = {0,}; + u64 iblock; + u64 last_block; + u64 on; + u64 off; + int ret = 0; + + if (WARN_ON_ONCE(sef & SEF_UNKNOWN) || + WARN_ON_ONCE(op & SCOUTFS_IOC_DWO_UNKNOWN) || + WARN_ON_ONCE(dw && !RB_EMPTY_NODE(&dw->node)) || + WARN_ON_ONCE(pos + len < pos)) { + ret = -EINVAL; + goto out; + } + + if ((sef & SEF_OFFLINE)) { + scoutfs_inode_get_onoff(inode, &on, &off); + if (off == 0) { + ret = 0; + goto out; + } + } + + iblock = pos >> SCOUTFS_BLOCK_SHIFT; + last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SHIFT; + + while(iblock <= last_block) { + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, + scoutfs_ino(inode), iblock, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + if (ext.start > last_block) + break; + + if (sef & ext.flags) { + if (dw) { + dw->chg = atomic64_read(&wq->changed); + dw->ino = scoutfs_ino(inode); + dw->iblock = max(iblock, ext.start); + dw->op = op; + + spin_lock(&rt->lock); + insert_offline_waiting(&rt->root, dw); + spin_unlock(&rt->lock); + } + + ret = 1; + break; + } + + iblock = ext.start + ext.len; + } + +out: + trace_scoutfs_data_wait_check(sb, scoutfs_ino(inode), pos, len, + sef, op, ext.start, ext.len, ext.flags, + ret); + return ret; +} + +bool scoutfs_data_wait_found(struct scoutfs_data_wait *dw) +{ + return !RB_EMPTY_NODE(&dw->node); +} + +int scoutfs_data_wait_check_iov(struct inode *inode, const struct iovec *iov, + unsigned long nr_segs, loff_t pos, u8 sef, + u8 op, struct scoutfs_data_wait *dw, + struct scoutfs_lock *lock) +{ + unsigned long i; + int ret = 0; + + for (i = 0; i < nr_segs; i++) { + if (iov[i].iov_len == 0) + continue; + + ret = scoutfs_data_wait_check(inode, pos, iov[i].iov_len, sef, + op, dw, lock); + if (ret != 0) + break; + + pos += iov[i].iov_len; + } + + return ret; +} + +int scoutfs_data_wait(struct inode *inode, struct scoutfs_data_wait *dw) +{ + DECLARE_DATA_WAIT_ROOT(inode->i_sb, rt); + DECLARE_DATA_WAITQ(inode, wq); + int ret; + + ret = wait_event_interruptible(wq->waitq, + atomic64_read(&wq->changed) != dw->chg); + + spin_lock(&rt->lock); + rb_erase(&dw->node, &rt->root); + RB_CLEAR_NODE(&dw->node); + spin_unlock(&rt->lock); + + return ret; +} + +void scoutfs_data_wait_changed(struct inode *inode) +{ + DECLARE_DATA_WAITQ(inode, wq); + + atomic64_inc(&wq->changed); + wake_up(&wq->waitq); +} + +int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, + struct scoutfs_ioctl_data_waiting_entry *dwe, + unsigned int nr) +{ + DECLARE_DATA_WAIT_ROOT(sb, rt); + struct scoutfs_data_wait *dw; + int ret = 0; + + spin_lock(&rt->lock); + + dw = next_data_wait(&rt->root, ino, iblock); + while (dw && ret < nr) { + + dwe->ino = dw->ino; + dwe->iblock = dw->iblock; + dwe->op = dw->op; + + while ((dw = dw_next(dw)) && + (dw->ino == dwe->ino && dw->iblock == dwe->iblock)) { + dwe->op |= dw->op; + } + + dwe++; + ret++; + } + + spin_unlock(&rt->lock); + + return ret; +} + const struct address_space_operations scoutfs_file_aops = { .readpage = scoutfs_readpage, .readpages = scoutfs_readpages, diff --git a/kmod/src/data.h b/kmod/src/data.h index bd9f84fa..d45114da 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -1,6 +1,41 @@ #ifndef _SCOUTFS_FILERW_H_ #define _SCOUTFS_FILERW_H_ +struct scoutfs_lock; +struct scoutfs_ioctl_data_waiting_entry; + +struct scoutfs_data_wait_root { + spinlock_t lock; + struct rb_root root; +}; + +#define DECLARE_DATA_WAIT_ROOT(sb, nm) \ + struct scoutfs_data_wait_root *nm = &SCOUTFS_SB(sb)->data_wait_root + +struct scoutfs_data_waitq { + atomic64_t changed; + wait_queue_head_t waitq; +}; + +#define DECLARE_DATA_WAITQ(in, nm) \ + struct scoutfs_data_waitq *nm = &SCOUTFS_I(in)->data_waitq + +/* + * Tasks can wait for data extents. + */ +struct scoutfs_data_wait { + struct rb_node node; + u64 chg; + u64 ino; + u64 iblock; + u8 op; +}; + +#define DECLARE_DATA_WAIT(nm) \ + struct scoutfs_data_wait nm = { \ + .node.__rb_parent_color = (unsigned long)(&nm.node), \ + } + extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; @@ -11,6 +46,21 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); +int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, + u8 sef, u8 op, struct scoutfs_data_wait *ow, + struct scoutfs_lock *lock); +int scoutfs_data_wait_check_iov(struct inode *inode, const struct iovec *iov, + unsigned long nr_segs, loff_t pos, u8 sef, + u8 op, struct scoutfs_data_wait *ow, + struct scoutfs_lock *lock); +bool scoutfs_data_wait_found(struct scoutfs_data_wait *ow); +int scoutfs_data_wait(struct inode *inode, + struct scoutfs_data_wait *ow); +void scoutfs_data_wait_changed(struct inode *inode); +int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, + struct scoutfs_ioctl_data_waiting_entry *dwe, + unsigned int nr); + int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/file.c b/kmod/src/file.c index f78e5721..765e5f1e 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -39,15 +39,40 @@ ssize_t scoutfs_file_aio_read(struct kiocb *iocb, const struct iovec *iov, struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); int ret; +retry: ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); - if (ret == 0) { - scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); - ret = generic_file_aio_read(iocb, iov, nr_segs, pos); - scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + if (ret) + goto out; + + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + /* protect checked extents from stage/release */ + mutex_lock(&inode->i_mutex); + atomic_inc(&inode->i_dio_count); + mutex_unlock(&inode->i_mutex); + + ret = scoutfs_data_wait_check_iov(inode, iov, nr_segs, pos, + SEF_OFFLINE, + SCOUTFS_IOC_DWO_READ, + &dw, inode_lock); + if (ret != 0) + goto out; + } + + ret = generic_file_aio_read(iocb, iov, nr_segs, pos); + +out: + if (scoutfs_per_task_del(&si->pt_data_lock, &pt_ent)) + inode_dio_done(inode); + scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + + if (scoutfs_data_wait_found(&dw)) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; } return ret; @@ -62,11 +87,13 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, struct super_block *sb = inode->i_sb; struct scoutfs_lock *inode_lock = NULL; SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + DECLARE_DATA_WAIT(dw); int ret; if (iocb->ki_left == 0) /* Does this even happen? */ return 0; +retry: mutex_lock(&inode->i_mutex); ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, inode, &inode_lock); @@ -77,16 +104,31 @@ ssize_t scoutfs_file_aio_write(struct kiocb *iocb, const struct iovec *iov, if (ret) goto out; - scoutfs_per_task_add(&si->pt_data_lock, &pt_ent, inode_lock); + if (scoutfs_per_task_add_excl(&si->pt_data_lock, &pt_ent, inode_lock)) { + /* data_version is per inode, whole file must be online */ + ret = scoutfs_data_wait_check(inode, 0, i_size_read(inode), + SEF_OFFLINE, + SCOUTFS_IOC_DWO_WRITE, + &dw, inode_lock); + if (ret != 0) + goto out; + } /* XXX: remove SUID bit */ ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); + out: scoutfs_per_task_del(&si->pt_data_lock, &pt_ent); scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_WRITE); mutex_unlock(&inode->i_mutex); + if (scoutfs_data_wait_found(&dw)) { + ret = scoutfs_data_wait(inode, &dw); + if (ret == 0) + goto retry; + } + if (ret > 0 || ret == -EIOCBQUEUED) { ssize_t err; diff --git a/kmod/src/format.h b/kmod/src/format.h index 936b0717..9fcbc082 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -390,8 +390,9 @@ struct scoutfs_file_extent { __u8 flags; } __packed; -#define SEF_OFFLINE 0x1 -#define SEF_UNWRITTEN 0x2 +#define SEF_OFFLINE (1 << 0) +#define SEF_UNWRITTEN (1 << 1) +#define SEF_UNKNOWN (U8_MAX << 2) /* * The first xattr part item has a header that describes the xattr. The diff --git a/kmod/src/inode.c b/kmod/src/inode.c index b9e6fcf9..3bdc756a 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -70,6 +70,8 @@ static void scoutfs_inode_ctor(void *obj) seqcount_init(&ci->seqcount); ci->staging = false; scoutfs_per_task_init(&ci->pt_data_lock); + atomic64_set(&ci->data_waitq.changed, 0); + init_waitqueue_head(&ci->data_waitq.waitq); init_rwsem(&ci->xattr_rwsem); RB_CLEAR_NODE(&ci->writeback_node); spin_lock_init(&ci->ino_alloc.lock); @@ -340,6 +342,9 @@ static int set_inode_size(struct inode *inode, struct scoutfs_lock *lock, if (ret) return ret; + if (new_size != i_size_read(inode)) + scoutfs_inode_inc_data_version(inode); + truncate_setsize(inode, new_size); inode->i_ctime = inode->i_mtime = CURRENT_TIME; if (truncate) @@ -394,11 +399,22 @@ int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) return ret ? ret : err; } +/* + * If we're changing the file size than the contents of the file are + * changing and we increment the data_version. This would prevent + * staging because the data_version is per-inode today, not per-extent. + * So if there are any offline extents within the new size then we need + * to stage them before we truncate. And this is called with the + * i_mutex held which would prevent staging so we release it and + * re-acquire it. Ideally we'd fix this so that we can acquire the lock + * instead of the caller. + */ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) { struct inode *inode = dentry->d_inode; struct super_block *sb = inode->i_sb; struct scoutfs_lock *lock = NULL; + DECLARE_DATA_WAIT(dw); LIST_HEAD(ind_locks); bool truncate = false; u64 attr_size; @@ -406,6 +422,7 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) trace_scoutfs_setattr(dentry, attr); +retry: ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, SCOUTFS_LKF_REFRESH_INODE, inode, &lock); if (ret) @@ -427,6 +444,28 @@ int scoutfs_setattr(struct dentry *dentry, struct iattr *attr) if (ret) goto out; + /* data_version is per inode, all must be online */ + if (attr_size > 0 && attr_size != i_size_read(inode)) { + ret = scoutfs_data_wait_check(inode, 0, attr_size, + SEF_OFFLINE, + SCOUTFS_IOC_DWO_CHANGE_SIZE, + &dw, lock); + if (ret < 0) + goto out; + if (scoutfs_data_wait_found(&dw)) { + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + + /* XXX callee locks instead? */ + mutex_unlock(&inode->i_mutex); + ret = scoutfs_data_wait(inode, &dw); + mutex_lock(&inode->i_mutex); + + if (ret == 0) + goto retry; + goto out; + } + } + /* truncating to current size truncates extents past size */ truncate = i_size_read(inode) >= attr_size; @@ -532,6 +571,10 @@ void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) write_seqcount_end(&si->seqcount); preempt_enable(); } + + /* any time offline extents decreased we try and wake waiters */ + if (inode && off < 0) + scoutfs_data_wait_changed(inode); } static u64 read_seqcount_u64(struct inode *inode, u64 *val) diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 7ae34de8..0ccd0184 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -6,6 +6,7 @@ #include "per_task.h" #include "count.h" #include "format.h" +#include "data.h" struct scoutfs_lock; @@ -48,8 +49,10 @@ struct scoutfs_inode_info { seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ struct scoutfs_per_task pt_data_lock; + struct scoutfs_data_waitq data_waitq; struct rw_semaphore xattr_rwsem; struct rb_node writeback_node; + struct inode inode; }; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 2c1fa74e..738173e9 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -541,6 +541,56 @@ static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) return ret ?: total; } +static bool inc_wrapped(u64 *ino, u64 *iblock) +{ + return (++(*iblock) == 0) && (++(*ino) == 0); +} + +static long scoutfs_ioc_data_waiting(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_data_waiting idw; + struct scoutfs_ioctl_data_waiting_entry __user *udwe; + struct scoutfs_ioctl_data_waiting_entry dwe[16]; + unsigned int nr; + int total; + int ret; + + if (copy_from_user(&idw, (void __user *)arg, sizeof(idw))) + return -EFAULT; + + if (idw.flags & SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN) + return -EINVAL; + + udwe = (void __user *)(long)idw.ents_ptr; + total = 0; + ret = 0; + while (idw.ents_nr && !inc_wrapped(&idw.after_ino, &idw.after_iblock)) { + nr = min_t(size_t, idw.ents_nr, ARRAY_SIZE(dwe)); + + ret = scoutfs_data_waiting(sb, idw.after_ino, idw.after_iblock, + dwe, nr); + BUG_ON(ret > nr); /* stack overflow \o/ */ + if (ret <= 0) + break; + + if (copy_to_user(udwe, dwe, ret * sizeof(dwe[0]))) { + ret = -EFAULT; + break; + } + + idw.after_ino = dwe[ret - 1].ino; + idw.after_iblock = dwe[ret - 1].iblock; + + udwe += ret; + idw.ents_nr -= ret; + total += ret; + ret = 0; + } + + return ret ?: total; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -556,6 +606,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_stat_more(file, arg); case SCOUTFS_IOC_ITEM_CACHE_KEYS: return scoutfs_ioc_item_cache_keys(file, arg); + case SCOUTFS_IOC_DATA_WAITING: + return scoutfs_ioc_data_waiting(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 915a130b..1b592522 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -229,4 +229,28 @@ enum { #define SCOUTFS_IOC_ITEM_CACHE_KEYS _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ struct scoutfs_ioctl_item_cache_keys) +struct scoutfs_ioctl_data_waiting_entry { + __u64 ino; + __u64 iblock; + __u8 op; +} __packed; + +#define SCOUTFS_IOC_DWO_READ (1 << 0) +#define SCOUTFS_IOC_DWO_WRITE (1 << 1) +#define SCOUTFS_IOC_DWO_CHANGE_SIZE (1 << 2) +#define SCOUTFS_IOC_DWO_UNKNOWN (U8_MAX << 3) + +struct scoutfs_ioctl_data_waiting { + __u64 flags; + __u64 after_ino; + __u64 after_iblock; + __u64 ents_ptr; + __u16 ents_nr; +} __packed; + +#define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) + +#define SCOUTFS_IOC_DATA_WAITING _IOW(SCOUTFS_IOCTL_MAGIC, 9, \ + struct scoutfs_ioctl_data_waiting) + #endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c index d268dd82..6697ec66 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -32,6 +32,7 @@ #include "triggers.h" #include "tseq.h" #include "client.h" +#include "data.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -126,8 +127,10 @@ static void invalidate_inode(struct super_block *sb, u64 ino) inode = scoutfs_ilookup(sb, ino); if (inode) { scoutfs_inc_counter(sb, lock_invalidate_inode); - if (S_ISREG(inode->i_mode)) + if (S_ISREG(inode->i_mode)) { truncate_inode_pages(inode->i_mapping, 0); + scoutfs_data_wait_changed(inode); + } iput(inode); } } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5e85305e..08d4572b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -514,6 +514,45 @@ TRACE_EVENT(scoutfs_data_truncate_items, __entry->iblock, __entry->last, __entry->offline) ); +TRACE_EVENT(scoutfs_data_wait_check, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 pos, __u64 len, + __u8 sef, __u8 op, __u64 ext_start, __u64 ext_len, + __u8 ext_flags, int ret), + + TP_ARGS(sb, ino, pos, len, sef, op, ext_start, ext_len, ext_flags, ret), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, ino) + __field(__u64, pos) + __field(__u64, len) + __field(__u8, sef) + __field(__u8, op) + __field(__u64, ext_start) + __field(__u64, ext_len) + __field(__u8, ext_flags) + __field(int, ret) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->ino = ino; + __entry->pos = pos; + __entry->len = len; + __entry->sef = sef; + __entry->op = op; + __entry->ext_start = ext_start; + __entry->ext_len = ext_len; + __entry->ext_flags = ext_flags; + __entry->ret = ret; + ), + + TP_printk(FSID_FMT" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext_start %llu ext_len %llu ext_flags 0x%x ret %d", + __entry->fsid, __entry->ino, __entry->pos, __entry->len, + __entry->sef, __entry->op, __entry->ext_start, + __entry->ext_len, __entry->ext_flags, __entry->ret) +); + TRACE_EVENT(scoutfs_sync_fs, TP_PROTO(struct super_block *sb, int wait), diff --git a/kmod/src/super.c b/kmod/src/super.c index 02f38934..ff39a6fb 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -339,6 +339,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->next_ino_lock); init_waitqueue_head(&sbi->trans_hold_wq); + spin_lock_init(&sbi->data_wait_root.lock); + sbi->data_wait_root.root = RB_ROOT; spin_lock_init(&sbi->trans_write_lock); INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); diff --git a/kmod/src/super.h b/kmod/src/super.h index e24f4d9a..6dd03ac6 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -6,6 +6,7 @@ #include "format.h" #include "options.h" +#include "data.h" struct scoutfs_counters; struct scoutfs_triggers; @@ -49,6 +50,9 @@ struct scoutfs_sb_info { wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; + /* tracks tasks waiting for data extents */ + struct scoutfs_data_wait_root data_wait_root; + spinlock_t trans_write_lock; u64 trans_write_count; u64 trans_seq; From 806ac0d8e670e3b5b87e0a170aaec93717b76f3b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 14 May 2019 14:27:50 -0700 Subject: [PATCH 704/920] scoutfs: fix mkfs option in README Fix a quick option typo in the mkfs invocations in the readme. Signed-off-by: Zach Brown --- kmod/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/README.md b/kmod/README.md index 542b4ecb..aed9631d 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -105,8 +105,8 @@ device name is the same on all the nodes. configuring each in the super block with options to mkfs. ```shell - scoutfs mkfs -o quorum_slot node1:0:172.16.1.1 \ - -o quorum_slot node2:0:172.16.1.2 /dev/shared_block_device + scoutfs mkfs --quorum_slot node1:0:172.16.1.1 \ + --quorum_slot node2:0:172.16.1.2 /dev/shared_block_device ``` From e150ebc8d2970bd6d6e0f4382d4f8d4035b5bd0f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 May 2019 16:14:32 -0700 Subject: [PATCH 705/920] scoutfs: trace btree dirty blocks Signed-off-by: Zach Brown --- kmod/src/btree.c | 6 ++++++ kmod/src/scoutfs_trace.h | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 006eacd3..1ba1c132 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -754,6 +754,12 @@ retry: if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) bring->next_block = 0; + trace_scoutfs_btree_dirty_block(sb, blkno, seq, + le64_to_cpu(bring->next_block), le64_to_cpu(bring->next_seq), + bti->cur_dirtied, bti->old_dirtied, + bt ? le64_to_cpu(bt->hdr.blkno) : 0, + bt ? le64_to_cpu(bt->hdr.seq) : 0); + /* force advancing if migration's done and we didn't just wrap */ if (all_roots_migrated(super) && !first_block_in_half(bring) && scoutfs_trigger(sb, BTREE_ADVANCE_RING_HALF)) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 08d4572b..a9c6cfe3 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2258,6 +2258,44 @@ TRACE_EVENT(scoutfs_btree_read_error, __entry->fsid, __entry->blkno, __entry->seq) ); +TRACE_EVENT(scoutfs_btree_dirty_block, + TP_PROTO(struct super_block *sb, u64 blkno, u64 seq, u64 next_block, + u64 next_seq, unsigned long cur_dirtied, + unsigned long old_dirtied, u64 bt_blkno, u64 bt_seq), + + TP_ARGS(sb, blkno, seq, next_block, next_seq, cur_dirtied, old_dirtied, + bt_blkno, bt_seq), + + TP_STRUCT__entry( + __field(__u64, fsid) + __field(__u64, blkno) + __field(__u64, seq) + __field(__u64, next_block) + __field(__u64, next_seq) + __field(unsigned long, cur_dirtied) + __field(unsigned long, old_dirtied) + __field(__u64, bt_blkno) + __field(__u64, bt_seq) + ), + + TP_fast_assign( + __entry->fsid = FSID_ARG(sb); + __entry->blkno = blkno; + __entry->seq = seq; + __entry->next_block = next_block; + __entry->next_seq = next_seq; + __entry->cur_dirtied = cur_dirtied; + __entry->old_dirtied = old_dirtied; + __entry->bt_blkno = bt_blkno; + __entry->bt_seq = bt_seq; + ), + + TP_printk("fsid "FSID_FMT" blkno %llu seq %llu next_block %llu next_seq %llu cur_dirtied %lu old_dirtied %lu bt_blkno %llu bt_seq %llu", + __entry->fsid, __entry->blkno, __entry->seq, + __entry->next_block, __entry->next_seq, __entry->cur_dirtied, + __entry->old_dirtied, __entry->bt_blkno, __entry->bt_seq) +); + DECLARE_EVENT_CLASS(scoutfs_extent_class, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), From e10033b34d3f90a14d1a82d7cd2f8d852d27440b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 May 2019 13:21:00 -0700 Subject: [PATCH 706/920] scoutfs: migrate dirty btree blocks during wrap We were seeing ring btree corruption that manifest as the server seeing stale btree blocks as it tried to read all the btrees to migrate blocks during a write. A block it tried to read didn't match its reference. It turned out that block wasn't being migrated. It would get stuck at a position in the ring. Eventually new block writes would overwrite it and then the next read would see corruption. It wasn't being migrated because the block reading function didn't realize that it had to migrate a dirty block. The block was written in a transaction at the end of the ring. The ring wrapped during the transaction and then migration tried to migrate the dirty block. It wouldn't be dirtied, and thus be migrated, because it was already dirty in the transaction. The fix is to add more cases to the dirtying decision which takes migration specifically into account. We'll no longer short circuit dirtying blocks for migration when they're in the old half of the ring even though they're dirty. Signed-off-by: Zach Brown --- kmod/src/btree.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 1ba1c132..079d4c83 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -700,9 +700,17 @@ retry: goto out; } - /* done if not dirtying or already dirty */ + /* + * We don't need to cow the exiting block if we're not + * dirtying the block, or we're not migrating and it's + * already dirty in this transaction, or we're + * migrating and it's already in the current half. + */ if (!(flags & BTW_DIRTY) || - (le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq)) { + (!(flags & BTW_MIGRATE) && + (le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq)) || + ((flags & BTW_MIGRATE) && + blkno_is_current(bring, le64_to_cpu(ref->blkno)))) { ret = 0; goto out; } From 0988cbe1e9fa3bc6c7a0eca46e234b971ee3c29b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 May 2019 13:25:22 -0700 Subject: [PATCH 707/920] scoutfs: track old and cur dirty btree blocks To avoid overwriting live btree blocks we have to migrate them between halves of the ring. Each time we cross into a new half of the ring we start migration all over again. The intent was to slowly migrate the blocks over time. We'd track dirty blocks that came from the old and current halves and keep them in balance. This would keep the overhead of the migration low and spread out through all at the start of the half that include migration. But the calculation of current blocks was completely wrong. It checked the newly allocated block which is always in the current half. It never thought it was dirtying old blocks so it'd constantly migrate trying to find them. We'd effectively migrate every btree block during the first transaction in each half. This calculates if we're dirtying old or new blocks by the source of the cow operation. We now recognize when we dirty old blocks and will stop migrating once we've migrated at least as many old blocks as we've written new blocks. Signed-off-by: Zach Brown --- kmod/src/btree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 079d4c83..7cda3178 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -750,10 +750,10 @@ retry: if (!bti->first_dirty_bh) bti->first_dirty_bh = bh; - if (blkno_is_current(bring, blkno)) - bti->cur_dirtied++; - else + if (ref && !blkno_is_current(bring, le64_to_cpu(ref->blkno))) bti->old_dirtied++; + else + bti->cur_dirtied++; /* wrap next block and increase next seq */ le64_add_cpu(&bring->next_block, 1); From 0b6bc8789cdf8f412b24f34b46b898824accaa1b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 May 2019 13:29:08 -0700 Subject: [PATCH 708/920] scoutfs: don't leak btree block refs Somewhere in the mists of time (around when we removed path tracking which held refs to blocks?) walking blocks to migrate started leaking btree block references. It was providing a pointer so the walk gave it the block it found but the caller was never dropping that ref. It wasn't doing anything with the result of the walk so we just don't provide a block pointer and the walk will drop the ref for us. This will stop leaking refs, effectively pinning the ring in memory. Signed-off-by: Zach Brown --- kmod/src/btree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 7cda3178..c680579b 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1646,7 +1646,7 @@ int scoutfs_btree_write_dirty(struct super_block *sb) ret = btree_walk(sb, root, BTW_DIRTY | BTW_NEXT | BTW_MIGRATE, - walk_key, walk_len, 0, &bt, + walk_key, walk_len, 0, NULL, iter_key, &iter_len); if (ret < 0) goto out; From c010afa8fff89bddcacb68e6b72b292c38161614 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 24 May 2019 10:09:24 -0700 Subject: [PATCH 709/920] scoutfs: add setattr_more ioctl Add an ioctl that can be used by userspace to restore a file to its offline state. To do that it needs to set inode fields that are otherwise not exposed and create an offline extent. Signed-off-by: Zach Brown --- kmod/src/count.h | 15 ++++++++ kmod/src/data.c | 43 ++++++++++++++++++++++ kmod/src/data.h | 2 + kmod/src/inode.c | 11 ++++++ kmod/src/inode.h | 1 + kmod/src/ioctl.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 18 +++++++++ 7 files changed, 185 insertions(+) diff --git a/kmod/src/count.h b/kmod/src/count.h index db35ebec..bf95cc44 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -313,4 +313,19 @@ static inline const struct scoutfs_item_count SIC_FALLOCATE_ONE(void) return cnt; } +/* + * ioc_setattr_more can dirty the inode and add a single offline extent. + */ +static inline const struct scoutfs_item_count SIC_SETATTR_MORE(void) +{ + struct scoutfs_item_count cnt = {0,}; + + __count_dirty_inode(&cnt); + + cnt.items++; + cnt.vals += sizeof(struct scoutfs_file_extent); + + return cnt; +} + #endif diff --git a/kmod/src/data.c b/kmod/src/data.c index ce31e642..f0b284ae 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1239,6 +1239,49 @@ out: return ret; } +/* + * A special case of initialzing a single large offline extent. This + * chooses not to deal with any existing extents. It can only be used + * on regular files with no data extents. It's used to restore a file + * with an offline extent which can then trigger staging. + * + * The caller has taken care of locking and holding a transaction. + * + * This could be an fallocate mode. + */ +int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, + struct scoutfs_lock *lock) + +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_extent ext; + u64 ino = scoutfs_ino(inode); + u64 len; + int ret; + + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + goto out; + } + + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, 0, 1, 0, 0); + ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + if (ret != -ENOENT) { + if (ret == 0) + ret = -EINVAL; + goto out; + } + + len = (size + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; + scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, + 0, len, 0, SEF_OFFLINE); + ret = scoutfs_extent_add(sb, data_extent_io, &ext, lock); + if (ret == 0) + scoutfs_inode_add_onoff(inode, 0, len); +out: + return ret; +} + /* * Return all the file's extents whose blocks overlap with the caller's diff --git a/kmod/src/data.h b/kmod/src/data.h index d45114da..21deeac4 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -45,6 +45,8 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len); long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); +int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, + struct scoutfs_lock *lock); int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, u8 sef, u8 op, struct scoutfs_data_wait *ow, diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 3bdc756a..afcd67aa 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -540,6 +540,17 @@ void scoutfs_inode_inc_data_version(struct inode *inode) preempt_enable(); } +void scoutfs_inode_set_data_version(struct inode *inode, u64 data_version) +{ + struct scoutfs_inode_info *si = SCOUTFS_I(inode); + + preempt_disable(); + write_seqcount_begin(&si->seqcount); + si->data_version = data_version; + write_seqcount_end(&si->seqcount); + preempt_enable(); +} + void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) { struct scoutfs_inode_info *si; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 0ccd0184..719fb391 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -103,6 +103,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, void scoutfs_inode_set_meta_seq(struct inode *inode); void scoutfs_inode_set_data_seq(struct inode *inode); void scoutfs_inode_inc_data_version(struct inode *inode); +void scoutfs_inode_set_data_version(struct inode *inode, u64 data_version); void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off); u64 scoutfs_inode_meta_seq(struct inode *inode); u64 scoutfs_inode_data_seq(struct inode *inode); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 738173e9..26ea6512 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -32,6 +32,7 @@ #include "client.h" #include "lock.h" #include "manifest.h" +#include "trans.h" #include "scoutfs_trace.h" /* @@ -591,6 +592,98 @@ static long scoutfs_ioc_data_waiting(struct file *file, unsigned long arg) return ret ?: total; } +/* + * This is used when restoring files, it lets the caller set all the + * inode attributes which are otherwise unreachable. Changing the file + * size can only be done for regular files with a data_version of 0. + */ +static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) +{ + struct inode *inode = file->f_inode; + struct super_block *sb = inode->i_sb; + struct scoutfs_ioctl_setattr_more __user *usm = (void __user *)arg; + struct scoutfs_ioctl_setattr_more sm; + struct scoutfs_lock *lock = NULL; + LIST_HEAD(ind_locks); + bool set_data_seq; + int ret; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EBADF; + goto out; + } + + if (copy_from_user(&sm, usm, sizeof(sm))) { + ret = -EFAULT; + goto out; + } + + if ((sm.i_size > 0 && sm.data_version == 0) || + ((sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) && !sm.i_size) || + (sm.flags & SCOUTFS_IOC_SETATTR_MORE_UNKNOWN)) { + ret = -EINVAL; + goto out; + } + + ret = mnt_want_write_file(file); + if (ret) + goto out; + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_WRITE, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto unlock; + + /* can only change size/dv on untouched regular files */ + if ((sm.i_size != 0 || sm.data_version != 0) && + ((!S_ISREG(inode->i_mode) || + scoutfs_inode_data_version(inode) != 0))) { + ret = -EINVAL; + goto unlock; + } + + /* setting only so we don't see 0 data seq with nonzero data_version */ + set_data_seq = sm.data_version != 0 ? true : false; + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, set_data_seq, + SIC_SETATTR_MORE()); + if (ret) + goto unlock; + + if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { + ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); + if (ret) + goto release; + } + + if (sm.data_version) + scoutfs_inode_set_data_version(inode, sm.data_version); + if (sm.i_size) + i_size_write(inode, sm.i_size); + inode->i_ctime.tv_sec = le64_to_cpu(sm.ctime.sec); + inode->i_ctime.tv_nsec = le32_to_cpu(sm.ctime.nsec); + + scoutfs_update_inode_item(inode, lock, &ind_locks); + ret = 0; + +release: + scoutfs_release_trans(sb); +unlock: + scoutfs_inode_index_unlock(sb, &ind_locks); + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE); + mutex_unlock(&inode->i_mutex); + mnt_drop_write_file(file); +out: + + return ret; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -608,6 +701,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_item_cache_keys(file, arg); case SCOUTFS_IOC_DATA_WAITING: return scoutfs_ioc_data_waiting(file, arg); + case SCOUTFS_IOC_SETATTR_MORE: + return scoutfs_ioc_setattr_more(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 1b592522..e49116b4 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -253,4 +253,22 @@ struct scoutfs_ioctl_data_waiting { #define SCOUTFS_IOC_DATA_WAITING _IOW(SCOUTFS_IOCTL_MAGIC, 9, \ struct scoutfs_ioctl_data_waiting) +/* + * If i_size is set then data_version must be non-zero. If the offline + * flag is set then i_size must be set and a offline extent will be + * created from offset 0 to i_size. + */ +struct scoutfs_ioctl_setattr_more { + __u64 data_version; + __u64 i_size; + __u64 flags; + struct scoutfs_timespec ctime; +} __packed; + +#define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) +#define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) + +#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 10, \ + struct scoutfs_ioctl_setattr_more) + #endif From 2cc4f89ad5a03088732200e519f499c6f8525691 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 28 May 2019 16:12:43 -0700 Subject: [PATCH 710/920] scoutfs: add sysfs attrs wrappers Add some helpers to manage the lifetime of groups of attributes in sysfs. We can wait until the sysfs files are no longer in use before tearing down the data that they rely on. Signed-off-by: Zach Brown --- kmod/src/sysfs.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/sysfs.h | 37 +++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index f043481e..7848e039 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -103,6 +103,89 @@ static void kobj_del_put_wait(struct kobject *kobj, struct completion *comp) #define shutdown_kobj(sfinfo, _name) \ kobj_del_put_wait(&sfsinfo->_name##_kobj, &sfsinfo->_name##_comp) +static void scoutfs_sysfs_release(struct kobject *kobj) +{ + DECLARE_SCOUTFS_SYSFS_ATTRS(ssa, kobj); + + complete(&ssa->comp); +} + +void scoutfs_sysfs_init_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa) +{ + ssa->name = NULL; +} + +/* + * If this returns success then the file will be visible and show can + * be called until unmount. + */ +int scoutfs_sysfs_create_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...) +{ + va_list args; + size_t name_len; + size_t size; + int ret; + + /* ssa should have seen init or destroy */ + if (WARN_ON_ONCE(ssa->name != NULL)) + return -EINVAL; + + ssa->sb = sb; + init_completion(&ssa->comp); + ssa->ktype.default_attrs = attrs; + ssa->ktype.sysfs_ops = &kobj_sysfs_ops; + ssa->ktype.release = scoutfs_sysfs_release; + + va_start(args, fmt); + name_len = vsnprintf(NULL, 0, fmt, args); + va_end(args); + if (WARN_ON_ONCE(name_len < 1 || name_len > NAME_MAX)) { + ret = -EINVAL; + goto out; + } + + size = name_len + 1; /* with null */ + + ssa->name = kmalloc(size, GFP_KERNEL); + if (!ssa->name) { + ret = -ENOMEM; + goto out; + } + + va_start(args, fmt); + ret = vsnprintf(ssa->name, size, fmt, args); + va_end(args); + if (ret != name_len) { + ret = -EINVAL; + goto out; + } + + ret = kobject_init_and_add(&ssa->kobj, &ssa->ktype, + scoutfs_sysfs_sb_dir(sb), "%s", ssa->name); +out: + if (ret) { + kfree(ssa->name); + ssa->name = NULL; + } + + return ret; +} + +void scoutfs_sysfs_destroy_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa) +{ + if (ssa->name) { + kobject_del(&ssa->kobj); + kobject_put(&ssa->kobj); + wait_for_completion(&ssa->comp); + kfree(ssa->name); + ssa->name = NULL; + } +} + /* * Only the return from kobj_init_and_add() tells us if the kobj needs * to be cleaned up or not. This must manually clean up the kobjs and diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h index 0d94b3a4..2f8f1087 100644 --- a/kmod/src/sysfs.h +++ b/kmod/src/sysfs.h @@ -1,6 +1,43 @@ #ifndef _SCOUTFS_SYSFS_H_ #define _SCOUTFS_SYSFS_H_ +/* + * We have some light wrappers around sysfs attributes to make it safe + * to tear down the attributes before freeing the data they describe. + */ + +#define SCOUTFS_ATTR_RO(_name) \ + static struct kobj_attribute scoutfs_attr_##_name = __ATTR_RO(_name) + +#define SCOUTFS_ATTR_PTR(_name) \ + &scoutfs_attr_##_name.attr + +struct scoutfs_sysfs_attrs { + struct super_block *sb; + char *name; + struct completion comp; + + struct kobject kobj; + struct kobj_type ktype; +}; + +#define SCOUTFS_SYSFS_ATTRS(kobj) \ + container_of(kobj, struct scoutfs_sysfs_attrs, kobj) + +#define SCOUTFS_SYSFS_ATTRS_SB(kobj) \ + (SCOUTFS_SYSFS_ATTRS(kobj)->sb) + +#define DECLARE_SCOUTFS_SYSFS_ATTRS(name, kobj) \ + struct scoutfs_sysfs_attrs *ssa = SCOUTFS_SYSFS_ATTRS(kobj) + +void scoutfs_sysfs_init_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa); +int scoutfs_sysfs_create_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa, + struct attribute **attrs, char *fmt, ...); +void scoutfs_sysfs_destroy_attrs(struct super_block *sb, + struct scoutfs_sysfs_attrs *ssa); + struct kobject *scoutfs_sysfs_sb_dir(struct super_block *sb); int scoutfs_setup_sysfs(struct super_block *sb); From 4df35efbc003e5417b4aa667cdcffdf00bbcb7c9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 28 May 2019 16:16:51 -0700 Subject: [PATCH 711/920] scoutfs: show quorum state in sysfs Add some sysfs files which show quorum state. We store the state in quorum_info off the super which is updates as we participate in elections. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/quorum.h | 2 + kmod/src/super.c | 3 ++ kmod/src/super.h | 1 + 4 files changed, 104 insertions(+) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 87cdd186..f24f4e03 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -29,6 +29,7 @@ #include "quorum.h" #include "server.h" #include "net.h" +#include "sysfs.h" #include "scoutfs_trace.h" /* @@ -63,6 +64,19 @@ * - add config rotation (write new config, reclaim stale slots) */ +struct quorum_info { + struct scoutfs_sysfs_attrs ssa; + + bool is_leader; + struct sockaddr_in conf_addr; + u16 conf_port; +}; + +#define DECLARE_QUORUM_INFO(sb, name) \ + struct quorum_info *name = SCOUTFS_SB(sb)->quorum_info +#define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \ + DECLARE_QUORUM_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name) + static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) { sin->sin_family = AF_INET; @@ -527,6 +541,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, bool unmounting, u64 our_umb, struct scoutfs_quorum_elected_info *qei) { + DECLARE_QUORUM_INFO(sb, qinf); struct scoutfs_super_block *super = NULL; struct scoutfs_quorum_config *conf; struct scoutfs_quorum_slot *slot; @@ -566,6 +581,16 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, goto out; conf = &super->quorum_config; + /* update sysfs with most recently seen config */ + if (our_slot >= 0) { + slot = &conf->slots[our_slot]; + addr_to_sin(&qinf->conf_addr, &slot->addr); + qinf->conf_port = le16_to_cpu(slot->addr.port); + } else { + memset(&qinf->conf_addr, 0, sizeof(qinf->conf_addr)); + qinf->conf_port = 0; + } + majority = scoutfs_quorum_majority(sb, conf); readahead_quorum_blocks(sb); @@ -623,6 +648,7 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, elected_nr); if (ret == 0) { qei->run_server = true; + qinf->is_leader = true; goto out; } @@ -739,8 +765,10 @@ int scoutfs_quorum_clear_elected(struct super_block *sb, struct scoutfs_quorum_elected_info *qei) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_QUORUM_INFO(sb, qinf); qei->flags &= ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; + qinf->is_leader = false; return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, qei->config_slot, qei->write_nr, @@ -807,3 +835,73 @@ bool scoutfs_quorum_voting_member(struct super_block *sb, return false; } + +static ssize_t is_leader_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); + + return snprintf(buf, PAGE_SIZE, "%u", !!qinf->is_leader); +} +SCOUTFS_ATTR_RO(is_leader); + +static ssize_t ipv4_addr_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); + + return snprintf(buf, PAGE_SIZE, "%pIS", &qinf->conf_addr); +} +SCOUTFS_ATTR_RO(ipv4_addr); + +static ssize_t ipv4_port_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); + + return snprintf(buf, PAGE_SIZE, "%u", qinf->conf_port); +} +SCOUTFS_ATTR_RO(ipv4_port); + +static struct attribute *quorum_attrs[] = { + SCOUTFS_ATTR_PTR(is_leader), + SCOUTFS_ATTR_PTR(ipv4_addr), + SCOUTFS_ATTR_PTR(ipv4_port), + NULL, +}; + +int scoutfs_quorum_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct quorum_info *qinf; + int ret; + + qinf = kzalloc(sizeof(struct quorum_info), GFP_KERNEL); + if (!qinf) { + ret = -ENOMEM; + goto out; + } + scoutfs_sysfs_init_attrs(sb, &qinf->ssa); + + sbi->quorum_info = qinf; + + ret = scoutfs_sysfs_create_attrs(sb, &qinf->ssa, quorum_attrs, + "quorum"); +out: + if (ret) + scoutfs_quorum_destroy(sb); + + return 0; +} + +void scoutfs_quorum_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct quorum_info *qinf = SCOUTFS_SB(sb)->quorum_info; + + if (qinf) { + scoutfs_sysfs_destroy_attrs(sb, &qinf->ssa); + kfree(qinf); + sbi->quorum_info = NULL; + } +} diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index 26a45b5b..e558d326 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -27,4 +27,6 @@ bool scoutfs_quorum_voting_member(struct super_block *sb, struct scoutfs_quorum_config *conf, char *name); +int scoutfs_quorum_setup(struct super_block *sb); +void scoutfs_quorum_destroy(struct super_block *sb); #endif diff --git a/kmod/src/super.c b/kmod/src/super.c index ff39a6fb..73e6bcb3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -44,6 +44,7 @@ #include "server.h" #include "options.h" #include "sysfs.h" +#include "quorum.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -161,6 +162,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); + scoutfs_quorum_destroy(sb); scoutfs_inode_destroy(sb); /* the server locks the listen address and compacts */ @@ -371,6 +373,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_setup_trans(sb) ?: scoutfs_lock_setup(sb) ?: scoutfs_net_setup(sb) ?: + scoutfs_quorum_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: scoutfs_client_wait_node_id(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 6dd03ac6..d34f4844 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -46,6 +46,7 @@ struct scoutfs_sb_info { struct inode_sb_info *inode_sb_info; struct btree_info *btree_info; struct net_info *net_info; + struct quorum_info *quorum_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From abd7ffc247aee8cf08c8a18938eb8a21a4fac671 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 30 May 2019 14:03:39 -0700 Subject: [PATCH 712/920] scoutfs: only trace read qourum blocks after io We have trace points as blocks are read, but the reads are cached as buffer heads. The iteration helpers are used to referenced cached blocks a few times in each voting cycle and we end up tracing cached read blocks multiple times. This uses a bit on the buffer_head to only trace a cached block the first time it's read. Signed-off-by: Zach Brown --- kmod/src/quorum.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index f24f4e03..5804a3a9 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -256,6 +256,13 @@ out: return ret; } +enum { + BH_ScoutfsTraced = BH_PrivateStart, +}; + +BUFFER_FNS(ScoutfsTraced, scoutfs_traced) /* has been traced */ +TAS_BUFFER_FNS(ScoutfsTraced, scoutfs_traced) + /* * The caller is about to read the current version of a set of quorum * blocks. We invalidate all the quorum blocks in the cache and @@ -279,6 +286,7 @@ static void readahead_quorum_blocks(struct super_block *sb) lock_buffer(bh); clear_buffer_uptodate(bh); + clear_buffer_scoutfs_traced(bh); unlock_buffer(bh); ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); @@ -346,7 +354,8 @@ static int read_quorum_block(struct super_block *sb, if (blk->write_nr == 0) goto out; - trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + if (!test_set_buffer_scoutfs_traced(bh)) + trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); if (invalid_quorum_block(super, bh, blk)) { scoutfs_inc_counter(sb, quorum_read_invalid_block); From c061ada671ae9dfa3282e044b128d84d249aa612 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 30 May 2019 14:28:11 -0700 Subject: [PATCH 713/920] scoutfs: mounts connect once server is listening An elected leader writes a quorum block showing that it's elected before it assumes exclusive access to the device and starts bringing up the server. This lets another later elected leader find and fence it if something happens. Other mounts were trying to connect to the server once this elected quorum block was written and before the server was listening. They'd get conection refused, decide to elect a new leader, and try to fence the server that's still running. Now, they should have tried much harder to connect to the elected leader instead of taking a single failed attempt as fatal. But that's a problem for another day that involves more work in balancing timeouts and retries. But mounts should not have tried try to connect to the server until its listening. That's easy to signal by adding a simple listening flag to the quorum block. Now mounts will only try to connect once they see the listening flag and don't see these racey refused connections. Signed-off-by: Zach Brown --- kmod/src/format.h | 3 ++- kmod/src/quorum.c | 34 +++++++++++++++++++++++++++++++--- kmod/src/quorum.h | 2 ++ kmod/src/server.c | 8 ++++++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 9fcbc082..79317691 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -448,7 +448,8 @@ struct scoutfs_quorum_block { } __packed; #define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0) -#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 1) +#define SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING (1 << 1) +#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 2) #define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 5804a3a9..7664aef6 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -58,6 +58,13 @@ * server. This ensures that racing elected leaders will always result * in fencing all but the most recent. * + * Once the elected leader verifies its written elected block it tries + * to start up the server. Once it's listening it writes another quorum + * block that indicates that it's listening. Once mounts see that + * they'll try to connect. If the server takes too long to write its + * listening flag the mounts may decide that the leader has died and try + * to elect a new leader. + * * XXX: * - actually fence * - add temporary priority for choosing a specific mount as a leader @@ -667,9 +674,10 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, vote_streak = 0; } - /* return if we found a new leader or ran out of time */ - if (qei->elected_nr > old_elected_nr || - ktime_after(now, timeout_abs)) { + /* return if we found a new listening leader or timed out */ + if (((qei->elected_nr > old_elected_nr) && + (qei->flags & SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING)) || + ktime_after(now, timeout_abs)) { if (qei->elected_nr > 0) { scoutfs_inc_counter(sb, quorum_found_leader); ret = 0; @@ -749,6 +757,26 @@ out: return ret; } +/* + * The calling server has successfully started and is listening for + * collections. It writes a new block to communicate to the other + * mounts that they should now try to connect. We do increase the write_nr + * here to still indicate that we're alive. + */ +int scoutfs_quorum_set_listening(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + qei->flags |= SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING; + le64_add_cpu(&qei->write_nr, 1); + + return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, + qei->config_slot, qei->write_nr, + qei->elected_nr, qei->unmount_barrier, + qei->config_slot, qei->flags); +} + /* * The calling server is shutting down and has finished modifying * persistent state. We clear the elected flag from our quorum block so diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index e558d326..cea55525 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -16,6 +16,8 @@ int scoutfs_quorum_election(struct super_block *sb, char *our_name, u64 old_elected_nr, ktime_t timeout_abs, bool unmounting, u64 our_umb, struct scoutfs_quorum_elected_info *qei); +int scoutfs_quorum_set_listening(struct super_block *sb, + struct scoutfs_quorum_elected_info *qei); int scoutfs_quorum_clear_elected(struct super_block *sb, struct scoutfs_quorum_elected_info *qei); int scoutfs_quorum_update_barrier(struct super_block *sb, diff --git a/kmod/src/server.c b/kmod/src/server.c index fd7cda3d..fc07a1a5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -2334,8 +2334,12 @@ static void scoutfs_server_worker(struct work_struct *work) server->conn = conn; scoutfs_net_listen(sb, conn); - /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); + ret = scoutfs_quorum_set_listening(sb, &server->qei); + + if (ret == 0) { + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); + } scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); From 7d56d8f34f17ffc44f5ecf3ab004c827516c4682 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 5 Jun 2019 14:11:09 -0700 Subject: [PATCH 714/920] scoutfs: add .show_options Add the vfs callback that prints mount options in /proc files. Signed-off-by: Zach Brown --- kmod/src/super.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index 73e6bcb3..d9417f72 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -133,6 +133,16 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) return 0; } +static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + seq_printf(seq, ",uniq_name=%s", opts->uniq_name); + + return 0; +} + static int scoutfs_sync_fs(struct super_block *sb, int wait) { trace_scoutfs_sync_fs(sb, wait); @@ -190,6 +200,7 @@ static const struct super_operations scoutfs_super_ops = { .destroy_inode = scoutfs_destroy_inode, .sync_fs = scoutfs_sync_fs, .statfs = scoutfs_statfs, + .show_options = scoutfs_show_options, .put_super = scoutfs_put_super, }; From a239f6093d0c52cb2b66ea6c2afb358b40da16c2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 5 Jun 2019 14:22:50 -0700 Subject: [PATCH 715/920] scoutfs: add mount_options/ sysfs dir Add a directory per mount that shows the values of all the mount options. Signed-off-by: Zach Brown --- kmod/src/super.c | 19 +++++++++++++++++++ kmod/src/super.h | 2 ++ kmod/src/sysfs.h | 2 ++ 3 files changed, 23 insertions(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index d9417f72..7ebaa178 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -143,6 +143,21 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) return 0; } +static ssize_t uniq_name_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + return snprintf(buf, PAGE_SIZE, "%s\n", opts->uniq_name); +} +SCOUTFS_ATTR_RO(uniq_name); + +static struct attribute *mount_options_attrs[] = { + SCOUTFS_ATTR_PTR(uniq_name), + NULL, +}; + static int scoutfs_sync_fs(struct super_block *sb, int wait) { trace_scoutfs_sync_fs(sb, wait); @@ -185,6 +200,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_item_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); + scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa); debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); scoutfs_destroy_sysfs(sb); @@ -357,6 +373,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) spin_lock_init(&sbi->trans_write_lock); INIT_DELAYED_WORK(&sbi->trans_write_work, scoutfs_trans_write_func); init_waitqueue_head(&sbi->trans_write_wq); + scoutfs_sysfs_init_attrs(sb, &sbi->mopts_ssa); ret = scoutfs_parse_options(sb, data, &opts); if (ret) @@ -376,6 +393,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: scoutfs_options_setup(sb) ?: + scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, + mount_options_attrs, "mount_options") ?: scoutfs_setup_triggers(sb) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index d34f4844..0ffd6c76 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -7,6 +7,7 @@ #include "format.h" #include "options.h" #include "data.h" +#include "sysfs.h" struct scoutfs_counters; struct scoutfs_triggers; @@ -75,6 +76,7 @@ struct scoutfs_sb_info { struct mount_options opts; struct options_sb_info *options; + struct scoutfs_sysfs_attrs mopts_ssa; struct dentry *debug_root; diff --git a/kmod/src/sysfs.h b/kmod/src/sysfs.h index 2f8f1087..73788c00 100644 --- a/kmod/src/sysfs.h +++ b/kmod/src/sysfs.h @@ -1,6 +1,8 @@ #ifndef _SCOUTFS_SYSFS_H_ #define _SCOUTFS_SYSFS_H_ +#include + /* * We have some light wrappers around sysfs attributes to make it safe * to tear down the attributes before freeing the data they describe. From 019b5f6d6b71e4ef56bd289eb5a67a806a1930be Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Jun 2019 14:32:21 -0700 Subject: [PATCH 716/920] scoutfs: add scoutfs xattr prefix and name tags Add a scoutfs. xattr prefix which then defines a series of following tags which can change the behaviour of the xattr. We start with .hide. which stops the xattr from showing up in listxattr. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 69 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 9fe08579..23aca918 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -81,12 +81,51 @@ static void init_xattr_key(struct scoutfs_key *key, u64 ino, u32 name_hash, }; } +#define SCOUTFS_XATTR_PREFIX "scoutfs." +#define SCOUTFS_XATTR_PREFIX_LEN (sizeof(SCOUTFS_XATTR_PREFIX) - 1) + static int unknown_prefix(const char *name) { return strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) && strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN) && - strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); + strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN)&& + strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN); +} + +struct prefix_tags { + unsigned long hide:1; +}; + +#define HIDE_TAG "hide." +#define HIDE_TAG_LEN (sizeof(HIDE_TAG) - 1) + +static int parse_tags(const char *name, struct prefix_tags *tgs) +{ + bool found; + + memset(tgs, 0, sizeof(struct prefix_tags)); + + if (strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN)) + return 0; + name += SCOUTFS_XATTR_PREFIX_LEN; + + found = false; + for (;;) { + if (!strncmp(name, HIDE_TAG, HIDE_TAG_LEN)) { + if (++tgs->hide == 0) + return -EINVAL; + name += HIDE_TAG_LEN; + } else { + /* only reason to use scoutfs. is tags */ + if (!found) + return -EINVAL; + break; + } + found = true; + } + + return 0; } /* @@ -355,6 +394,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_lock *lck = NULL; size_t name_len = strlen(name); struct scoutfs_key key; + struct prefix_tags tgs; LIST_HEAD(ind_locks); LIST_HEAD(saved); u8 found_parts; @@ -378,6 +418,12 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (unknown_prefix(name)) return -EOPNOTSUPP; + if (parse_tags(name, &tgs) != 0) + return -EINVAL; + + if (tgs.hide && !capable(CAP_SYS_ADMIN)) + return -EPERM; + bytes = sizeof(struct scoutfs_xattr) + name_len + size; xat = kmalloc(bytes, GFP_NOFS); if (!xat) { @@ -495,6 +541,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; struct scoutfs_key key; + struct prefix_tags tgs; unsigned int bytes; ssize_t total; u32 name_hash; @@ -528,17 +575,19 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) break; } - total += xat->name_len + 1; + if (parse_tags(xat->name, &tgs) != 0 || !tgs.hide) { + total += xat->name_len + 1; - if (size) { - if (total > size) { - ret = -ERANGE; - break; + if (size) { + if (total > size) { + ret = -ERANGE; + break; + } + + memcpy(buffer, xat->name, xat->name_len); + buffer += xat->name_len; + *(buffer++) = '\0'; } - - memcpy(buffer, xat->name, xat->name_len); - buffer += xat->name_len; - *(buffer++) = '\0'; } name_hash = le64_to_cpu(key.skx_name_hash); From a7fef3d7dd652288eb91dc0176f01a13f8dc0885 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 14 Jun 2019 14:45:39 -0700 Subject: [PATCH 717/920] scoutfs: add listxattr_raw ioctl Add an ioctl which can be used to iterate over the keys for all the xattrs on an inode. It is privileged, can see hidden inodes, and has an iteration cursor so that it can make its way through very large numbers of xattrs. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 10 ++++++++ kmod/src/xattr.c | 45 +++++++++++++++++++++++---------- kmod/src/xattr.h | 3 +++ 4 files changed, 110 insertions(+), 13 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 26ea6512..16bcc0f8 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -33,6 +33,7 @@ #include "lock.h" #include "manifest.h" #include "trans.h" +#include "xattr.h" #include "scoutfs_trace.h" /* @@ -684,6 +685,68 @@ out: return ret; } +static long scoutfs_ioc_listxattr_raw(struct file *file, unsigned long arg) +{ + struct inode *inode = file->f_inode; + struct scoutfs_ioctl_listxattr_raw __user *ulxr = (void __user *)arg; + struct scoutfs_ioctl_listxattr_raw lxr; + struct page *page = NULL; + unsigned int bytes; + int total = 0; + int ret; + + if (!(file->f_mode & FMODE_READ)) { + ret = -EBADF; + goto out; + } + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EBADF; + goto out; + } + + if (copy_from_user(&lxr, ulxr, sizeof(lxr))) { + ret = -EFAULT; + goto out; + } + + page = alloc_page(GFP_KERNEL); + if (!page) { + ret = -ENOMEM; + goto out; + } + + while (lxr.buf_bytes) { + bytes = min_t(int, lxr.buf_bytes, PAGE_SIZE); + ret = scoutfs_list_xattrs(inode, page_address(page), bytes, + &lxr.hash_pos, &lxr.id_pos, + false, true); + if (ret <= 0) + break; + + if (copy_to_user((void __user *)lxr.buf_ptr, + page_address(page), ret)) { + ret = -EFAULT; + break; + } + + lxr.buf_ptr += ret; + lxr.buf_bytes -= ret; + total += ret; + ret = 0; + } + +out: + if (page) + __free_page(page); + + if (ret == 0 && (__put_user(lxr.hash_pos, &ulxr->hash_pos) || + __put_user(lxr.id_pos, &ulxr->id_pos))) + ret = -EFAULT; + + return ret ?: total; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -703,6 +766,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_data_waiting(file, arg); case SCOUTFS_IOC_SETATTR_MORE: return scoutfs_ioc_setattr_more(file, arg); + case SCOUTFS_IOC_LISTXATTR_RAW: + return scoutfs_ioc_listxattr_raw(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index e49116b4..1c03c25e 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -271,4 +271,14 @@ struct scoutfs_ioctl_setattr_more { #define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 10, \ struct scoutfs_ioctl_setattr_more) +struct scoutfs_ioctl_listxattr_raw { + __u64 id_pos; + __u64 buf_ptr; + __u32 buf_bytes; + __u32 hash_pos; +} __packed; + +#define SCOUTFS_IOC_LISTXATTR_RAW _IOW(SCOUTFS_IOCTL_MAGIC, 11, \ + struct scoutfs_ioctl_listxattr_raw) + #endif diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 23aca918..62ad6a70 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -533,9 +533,10 @@ int scoutfs_removexattr(struct dentry *dentry, const char *name) return scoutfs_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE); } -ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, + size_t size, __u32 *hash_pos, __u64 *id_pos, + bool e_range, bool hidden) { - struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; struct scoutfs_xattr *xat = NULL; @@ -543,11 +544,16 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) struct scoutfs_key key; struct prefix_tags tgs; unsigned int bytes; - ssize_t total; - u32 name_hash; - u64 id; + ssize_t total = 0; + u32 name_hash = 0; + u64 id = 0; int ret; + if (hash_pos) + name_hash = *hash_pos; + if (id_pos) + id = *id_pos; + /* need a buffer large enough for all possible names */ bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN; xat = kmalloc(bytes, GFP_NOFS); @@ -562,10 +568,6 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) down_read(&si->xattr_rwsem); - name_hash = 0; - id = 0; - total = 0; - for (;;) { ret = get_next_xattr(inode, &key, xat, bytes, NULL, 0, name_hash, id, lck); @@ -575,12 +577,14 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) break; } - if (parse_tags(xat->name, &tgs) != 0 || !tgs.hide) { - total += xat->name_len + 1; + if (hidden || parse_tags(xat->name, &tgs) != 0 || !tgs.hide) { if (size) { - if (total > size) { - ret = -ERANGE; + if ((total + xat->name_len + 1) > size) { + if (e_range) + ret = -ERANGE; + else + ret = total; break; } @@ -588,6 +592,8 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) buffer += xat->name_len; *(buffer++) = '\0'; } + + total += xat->name_len + 1; } name_hash = le64_to_cpu(key.skx_name_hash); @@ -599,9 +605,22 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) out: kfree(xat); + if (hash_pos) + *hash_pos = name_hash; + if (id_pos) + *id_pos = id; + return ret; } +ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = dentry->d_inode; + + return scoutfs_list_xattrs(inode, buffer, size, + NULL, NULL, true, false); +} + /* * Delete all the xattr items associated with this inode. The inode is * dead so we don't need the xattr rwsem. diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index 6c205358..ca4fb7fc 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -7,6 +7,9 @@ int scoutfs_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int scoutfs_removexattr(struct dentry *dentry, const char *name); ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, + size_t size, __u32 *hash_pos, __u64 *id_pos, + bool e_range, bool hidden); int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock); From aee017903b5c463fa3ad7c4ed352b74d0efd2d01 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Jun 2019 16:41:26 -0700 Subject: [PATCH 718/920] scoutfs: add hash helper Add a quick header which calculates 64bit hashes by calculating the crc of the two halves of a byte region. Signed-off-by: Zach Brown --- kmod/src/hash.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 kmod/src/hash.h diff --git a/kmod/src/hash.h b/kmod/src/hash.h new file mode 100644 index 00000000..3ad3de09 --- /dev/null +++ b/kmod/src/hash.h @@ -0,0 +1,15 @@ +#ifndef _SCOUTFS_HASH_H_ +#define _SCOUTFS_HASH_H_ + +#include + +/* XXX replace with xxhash */ +static inline u64 scoutfs_hash64(const void *data, unsigned int len) +{ + unsigned int half = (len + 1) / 2; + + return crc32c(~0, data, half) | + ((u64)crc32c(~0, data + len - half, half) << 32); +} + +#endif From 7dfbd3950f3099844b0c9ad62152550b3bf1a677 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Jun 2019 16:40:11 -0700 Subject: [PATCH 719/920] scoutfs: add index of inodes by xattr names Add a .indx. xattr tag which adds the inode to an index of inodes keyed by the hash of xattr names. An ioctl is added which then returns all the inodes which may contain an xattr of the given name. Dropping all xattrs now has to parse the name to find out if it also has to delete an index item. Signed-off-by: Zach Brown --- kmod/src/count.h | 8 ++- kmod/src/format.h | 15 ++++- kmod/src/ioctl.c | 97 +++++++++++++++++++++++++++++ kmod/src/ioctl.h | 30 +++++++++ kmod/src/lock.c | 19 ++++++ kmod/src/lock.h | 2 + kmod/src/xattr.c | 155 +++++++++++++++++++++++++++++++++++++--------- kmod/src/xattr.h | 3 + 8 files changed, 294 insertions(+), 35 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index bf95cc44..25e4fd0c 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -205,12 +205,14 @@ static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, * item with the header and name. Any previously existing items are * deleted which dirties their key but removes their value. The two * sets of items are indexed by different ids so their items don't - * overlap. + * overlap. If the xattr name is indexed then we modify one xattr index + * item. */ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, bool creating, unsigned name_len, - unsigned size) + unsigned size, + bool indexed) { struct scoutfs_item_count cnt = {0,}; unsigned int new_parts; @@ -219,6 +221,8 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, if (old_parts) cnt.items += old_parts; + if (indexed) + cnt.items++; if (creating) { new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size) diff --git a/kmod/src/format.h b/kmod/src/format.h index 79317691..19990161 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -108,6 +108,11 @@ struct scoutfs_key { #define skii_major _sk_second #define skii_ino _sk_third +/* xattr index */ +#define skxi_hash _sk_first +#define skxi_ino _sk_second +#define skxi_id _sk_third + /* node free extent */ #define sknf_node_id _sk_first #define sknf_major _sk_second @@ -351,9 +356,10 @@ struct scoutfs_segment_block { * Keys are first sorted by major key zones. */ #define SCOUTFS_INODE_INDEX_ZONE 1 -#define SCOUTFS_NODE_ZONE 2 -#define SCOUTFS_FS_ZONE 3 -#define SCOUTFS_LOCK_ZONE 4 +#define SCOUTFS_XATTR_INDEX_ZONE 2 +#define SCOUTFS_NODE_ZONE 3 +#define SCOUTFS_FS_ZONE 4 +#define SCOUTFS_LOCK_ZONE 5 #define SCOUTFS_MAX_ZONE 8 /* power of 2 is efficient */ /* inode index zone */ @@ -361,6 +367,9 @@ struct scoutfs_segment_block { #define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 #define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ +/* xattr index zone */ +#define SCOUTFS_XATTR_INDEX_NAME_TYPE 1 + /* node zone (also used in server alloc btree) */ #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 #define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 16bcc0f8..4818ba98 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -34,6 +34,7 @@ #include "manifest.h" #include "trans.h" #include "xattr.h" +#include "hash.h" #include "scoutfs_trace.h" /* @@ -747,6 +748,100 @@ out: return ret ?: total; } +/* + * Return the inode numbers of inodes which might contain the given + * named xattr. This will only find scoutfs xattrs with the index tag + * but we don't check that the callers xattr name contains the tag and + * search for it regardless. + */ +static long scoutfs_ioc_find_xattrs(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_find_xattrs __user *ufx = (void __user *)arg; + struct scoutfs_ioctl_find_xattrs fx; + struct scoutfs_lock *lock = NULL; + struct scoutfs_key last; + struct scoutfs_key key; + char *name = NULL; + int total = 0; + u64 hash; + u64 ino; + int ret; + + if (!(file->f_mode & FMODE_READ)) { + ret = -EBADF; + goto out; + } + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + + if (copy_from_user(&fx, ufx, sizeof(fx))) { + ret = -EFAULT; + goto out; + } + + if (fx.name_bytes > SCOUTFS_XATTR_MAX_NAME_LEN) { + ret = -EINVAL; + goto out; + } + + name = kmalloc(fx.name_bytes, GFP_KERNEL); + if (!name) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(name, (void __user *)fx.name_ptr, fx.name_bytes)) { + ret = -EFAULT; + goto out; + } + + hash = scoutfs_hash64(name, fx.name_bytes); + scoutfs_xattr_index_key(&key, hash, fx.next_ino, 0); + scoutfs_xattr_index_key(&last, hash, U64_MAX, U64_MAX); + ino = 0; + + ret = scoutfs_lock_xattr_index(sb, SCOUTFS_LOCK_READ, 0, hash, &lock); + if (ret < 0) + goto out; + + while (fx.nr_inodes) { + + ret = scoutfs_item_next(sb, &key, &last, NULL, lock); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* xattrs hashes can collide and add multiple entries */ + if (le64_to_cpu(key.skxi_ino) != ino) { + ino = le64_to_cpu(key.skxi_ino); + if (put_user(ino, (u64 __user *)fx.inodes_ptr)) { + ret = -EFAULT; + break; + } + + fx.inodes_ptr += sizeof(u64); + fx.nr_inodes--; + total++; + ret = 0; + } + + scoutfs_key_inc(&key); + } + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + +out: + kfree(name); + + return ret ?: total; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -768,6 +863,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_setattr_more(file, arg); case SCOUTFS_IOC_LISTXATTR_RAW: return scoutfs_ioc_listxattr_raw(file, arg); + case SCOUTFS_IOC_FIND_XATTRS: + return scoutfs_ioc_find_xattrs(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 1c03c25e..b9966c68 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -281,4 +281,34 @@ struct scoutfs_ioctl_listxattr_raw { #define SCOUTFS_IOC_LISTXATTR_RAW _IOW(SCOUTFS_IOCTL_MAGIC, 11, \ struct scoutfs_ioctl_listxattr_raw) +/* + * Return the inode numbers of inodes which might contain the given + * named xattr. The inode may not have a set xattr with that name, the + * caller must check the returned inodes to see if they match. + * + * @next_ino: The next inode number that could be returned. Initialized + * to 0 when first searching and set to one past the last inode number + * returned to continue searching. + * @name_ptr: The address of the name of the xattr to search for. It does + * not need to be null terminated. + * @inodes_ptr: The address of the array of uint64_t inode numbers in which + * to store inode numbers that may contain the xattr. EFAULT may be returned + * if this address is not naturally aligned. + * @name_bytes: The number of non-null bytes found in the name at name_ptr. + * @nr_inodes: The number of elements in the array found at inodes_ptr. + * + * This requires the CAP_SYS_ADMIN capability and will return -EPERM if + * it's not granted. + */ +struct scoutfs_ioctl_find_xattrs { + __u64 next_ino; + __u64 name_ptr; + __u64 inodes_ptr; + __u16 name_bytes; + __u16 nr_inodes; +} __packed; + +#define SCOUTFS_IOC_FIND_XATTRS _IOW(SCOUTFS_IOCTL_MAGIC, 12, \ + struct scoutfs_ioctl_find_xattrs) + #endif diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 6697ec66..97fae82d 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -33,6 +33,7 @@ #include "tseq.h" #include "client.h" #include "data.h" +#include "xattr.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -1157,6 +1158,24 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, return lock_key_range(sb, mode, 0, &start, &end, ret_lock); } +/* + * Today we lock a hash value entirely. If we went to finer grained ino + * locking as well we'd need to check the manifest to find the next + * possible ino to lock so that we didn't try to iterate over all of + * them. + */ +int scoutfs_lock_xattr_index(struct super_block *sb, int mode, int flags, + u64 hash, struct scoutfs_lock **ret_lock) +{ + struct scoutfs_key start; + struct scoutfs_key end; + + scoutfs_xattr_index_key(&start, hash, 0, 0); + scoutfs_xattr_index_key(&end, hash, U64_MAX, U64_MAX); + + return lock_key_range(sb, mode, flags, &start, &end, ret_lock); +} + /* * The node_id lock protects a mount's private persistent items in the * node_id zone. It's held for the duration of the mount. It lets the diff --git a/kmod/src/lock.h b/kmod/src/lock.h index c3230316..da24ee8f 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -61,6 +61,8 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); +int scoutfs_lock_xattr_index(struct super_block *sb, int mode, int flags, + u64 hash, struct scoutfs_lock **ret_lock); int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *a, struct scoutfs_lock **a_lock, struct inode *b, struct scoutfs_lock **b_lock, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 62ad6a70..a877d0ec 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -25,6 +25,7 @@ #include "trans.h" #include "xattr.h" #include "lock.h" +#include "hash.h" #include "scoutfs_trace.h" /* @@ -94,40 +95,58 @@ static int unknown_prefix(const char *name) } struct prefix_tags { - unsigned long hide:1; + unsigned long hide:1, + indx:1; }; #define HIDE_TAG "hide." -#define HIDE_TAG_LEN (sizeof(HIDE_TAG) - 1) +#define INDX_TAG "indx." +#define TAG_LEN (sizeof(HIDE_TAG) - 1) -static int parse_tags(const char *name, struct prefix_tags *tgs) +static int parse_tags(const char *name, unsigned int name_len, + struct prefix_tags *tgs) { bool found; memset(tgs, 0, sizeof(struct prefix_tags)); - if (strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN)) + if ((name_len < (SCOUTFS_XATTR_PREFIX_LEN + TAG_LEN + 1)) || + strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN)) return 0; name += SCOUTFS_XATTR_PREFIX_LEN; found = false; for (;;) { - if (!strncmp(name, HIDE_TAG, HIDE_TAG_LEN)) { + if (!strncmp(name, HIDE_TAG, TAG_LEN)) { if (++tgs->hide == 0) return -EINVAL; - name += HIDE_TAG_LEN; + } else if (!strncmp(name, INDX_TAG, TAG_LEN)) { + if (++tgs->indx == 0) + return -EINVAL; } else { /* only reason to use scoutfs. is tags */ if (!found) return -EINVAL; break; } + name += TAG_LEN; found = true; } return 0; } +void scoutfs_xattr_index_key(struct scoutfs_key *key, + u64 hash, u64 ino, u64 id) +{ + scoutfs_key_set_zeros(key); + key->sk_zone = SCOUTFS_XATTR_INDEX_ZONE; + key->skxi_hash = cpu_to_le64(hash); + key->sk_type = SCOUTFS_XATTR_INDEX_NAME_TYPE; + key->skxi_ino = cpu_to_le64(ino); + key->skxi_id = cpu_to_le64(id); +} + /* * Find the next xattr and copy the key, xattr header, and as much of * the name and value into the callers buffer as we can. Returns the @@ -390,18 +409,24 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct inode *inode = dentry->d_inode; struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); struct scoutfs_xattr *xat = NULL; + struct scoutfs_lock *indx_lock = NULL; struct scoutfs_lock *lck = NULL; size_t name_len = strlen(name); + struct scoutfs_key indx_key; struct scoutfs_key key; struct prefix_tags tgs; + bool undo_indx = false; LIST_HEAD(ind_locks); LIST_HEAD(saved); u8 found_parts; unsigned int bytes; u64 ind_seq; - u64 id; + u64 hash; + u64 id = 0; int ret; + int err; trace_scoutfs_xattr_set(sb, name_len, value, size, flags); @@ -418,10 +443,10 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (unknown_prefix(name)) return -EOPNOTSUPP; - if (parse_tags(name, &tgs) != 0) + if (parse_tags(name, name_len, &tgs) != 0) return -EINVAL; - if (tgs.hide && !capable(CAP_SYS_ADMIN)) + if ((tgs.hide || tgs.indx) && !capable(CAP_SYS_ADMIN)) return -EPERM; bytes = sizeof(struct scoutfs_xattr) + name_len + size; @@ -472,13 +497,22 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, memcpy(&xat->name[xat->name_len], value, size); } + if (tgs.indx && !(found_parts && value)) { + hash = scoutfs_hash64(name, name_len); + ret = scoutfs_lock_xattr_index(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, + hash, &indx_lock); + if (ret < 0) + goto unlock; + } + retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_XATTR_SET(found_parts, value != NULL, - name_len, size)); + name_len, size, + tgs.indx)); if (ret > 0) goto retry; if (ret) @@ -488,6 +522,22 @@ retry: if (ret < 0) goto release; + if (tgs.indx && !(found_parts && value)) { + if (found_parts) + id = le64_to_cpu(key.skx_id); + hash = scoutfs_hash64(name, name_len); + scoutfs_xattr_index_key(&indx_key, hash, ino, id); + if (value) + ret = scoutfs_item_create_force(sb, &indx_key, NULL, + indx_lock); + else + ret = scoutfs_item_delete_force(sb, &indx_key, + indx_lock); + if (ret < 0) + goto release; + undo_indx = true; + } + ret = 0; if (found_parts) ret = delete_xattr_items(inode, le64_to_cpu(key.skx_name_hash), @@ -508,10 +558,21 @@ retry: ret = 0; release: + if (ret < 0 && undo_indx) { + if (value) + err = scoutfs_item_delete_force(sb, &indx_key, + indx_lock); + else + err = scoutfs_item_create_force(sb, &indx_key, NULL, + indx_lock); + BUG_ON(err); + } + scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); unlock: up_write(&si->xattr_rwsem); + scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE); out: kfree(xat); @@ -577,7 +638,9 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, break; } - if (hidden || parse_tags(xat->name, &tgs) != 0 || !tgs.hide) { + if (hidden || + parse_tags(xat->name, xat->name_len, &tgs) != 0 || + !tgs.hide) { if (size) { if ((total + xat->name_len + 1) > size) { @@ -624,54 +687,86 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) /* * Delete all the xattr items associated with this inode. The inode is * dead so we don't need the xattr rwsem. - * - * XXX This isn't great because it reads in all the items so that it can - * create deletion items for each. It would be better to have the - * caller create range deletion items for all the items covered by the - * inode. That wouldn't require reading at all. */ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock) { + struct scoutfs_lock *indx_lock = NULL; + struct scoutfs_xattr *xat = NULL; + struct scoutfs_key indx_key; struct scoutfs_key last; struct scoutfs_key key; - unsigned int items = 16; - bool holding = false; + struct prefix_tags tgs; + bool release = false; + unsigned int bytes; + struct kvec val; + u64 hash; int ret; + /* need a buffer large enough for all possible names */ + bytes = sizeof(struct scoutfs_xattr) + SCOUTFS_XATTR_MAX_NAME_LEN; + xat = kmalloc(bytes, GFP_NOFS); + if (!xat) { + ret = -ENOMEM; + goto out; + } + init_xattr_key(&key, ino, 0, 0); init_xattr_key(&last, ino, U32_MAX, U64_MAX); for (;;) { - ret = scoutfs_item_next(sb, &key, &last, NULL, lock); + kvec_init(&val, (void *)xat, bytes); + ret = scoutfs_item_next(sb, &key, &last, &val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - if (!holding) { - ret = scoutfs_hold_trans(sb, SIC_EXACT(items, 0)); - if (ret) + if (key.skx_part != 0 || + parse_tags(xat->name, xat->name_len, &tgs) != 0) + memset(&tgs, 0, sizeof(tgs)); + + if (tgs.indx) { + hash = scoutfs_hash64(xat->name, xat->name_len); + scoutfs_xattr_index_key(&indx_key, hash, ino, + le64_to_cpu(key.skx_id)); + ret = scoutfs_lock_xattr_index(sb, + SCOUTFS_LOCK_WRITE_ONLY, + 0, hash, &indx_lock); + if (ret < 0) break; - holding = true; } + ret = scoutfs_hold_trans(sb, SIC_EXACT(2, 0)); + if (ret < 0) + break; + release = true; + ret = scoutfs_item_delete(sb, &key, lock); - if (ret) + if (ret < 0) break; - if (--items == 0) { - scoutfs_release_trans(sb); - holding = false; - items = 16; + if (tgs.indx) { + ret = scoutfs_item_delete_force(sb, &indx_key, + indx_lock); + if (ret < 0) + break; } + scoutfs_release_trans(sb); + release = false; + + scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); + indx_lock = NULL; + /* don't need to inc, next won't see deleted item */ } - if (holding) + if (release) scoutfs_release_trans(sb); - + scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); + kfree(xat); +out: return ret; } diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index ca4fb7fc..efcbc62a 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -14,4 +14,7 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock); +void scoutfs_xattr_index_key(struct scoutfs_key *key, + u64 hash, u64 ino, u64 id); + #endif From 4a29cb5888a30779165f43e8bcbf49881ed3bdbb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 25 Jun 2019 11:28:48 -0700 Subject: [PATCH 720/920] scoutfs: naturally align ioctl structs Order the ioctl struct field definitions and add padding so that runtimes with different word dizes don't add different padding. Userspace is spared having to deal with packing and we don't have to worry about compat translation in the kernel. We had two persistent structures that crossed the ioctl, a key and a timespec, so we explicitly translate to and from their persistent types in the ioctl. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 25 ++++++++++------- kmod/src/ioctl.h | 70 +++++++++++++++++++++++++++++++++++++----------- kmod/src/key.h | 18 +++++++++++++ 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 4818ba98..e5e7f5a7 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -500,11 +500,14 @@ static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; struct scoutfs_ioctl_item_cache_keys ick; - struct scoutfs_key __user *ukeys; + struct scoutfs_ioctl_key __user *ukeys; + struct scoutfs_ioctl_key ikeys[16]; struct scoutfs_key keys[16]; + struct scoutfs_key key; unsigned int nr; int total; int ret; + int i; if (copy_from_user(&ick, (void __user *)arg, sizeof(ick))) return -EFAULT; @@ -512,6 +515,8 @@ static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) if (ick.which > SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES) return -EINVAL; + scoutfs_key_copy_types(&key, &ick.ikey); + ukeys = (void __user *)(long)ick.buf_ptr; total = 0; ret = 0; @@ -519,21 +524,23 @@ static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) nr = min_t(size_t, ick.buf_nr, ARRAY_SIZE(keys)); if (ick.which == SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS) - ret = scoutfs_item_copy_keys(sb, &ick.key, keys, nr); + ret = scoutfs_item_copy_keys(sb, &key, keys, nr); else - ret = scoutfs_item_copy_range_keys(sb, &ick.key, keys, - nr); + ret = scoutfs_item_copy_range_keys(sb, &key, keys, nr); BUG_ON(ret > nr); /* stack overflow \o/ */ if (ret <= 0) break; - if (copy_to_user(ukeys, keys, ret * sizeof(keys[0]))) { + for (i = 0; i < ret; i++) + scoutfs_key_copy_types(&ikeys[i], &keys[i]); + + if (copy_to_user(ukeys, ikeys, ret * sizeof(ikeys[0]))) { ret = -EFAULT; break; } - ick.key = keys[ret - 1]; - scoutfs_key_inc(&ick.key); + key = keys[ret - 1]; + scoutfs_key_inc(&key); ukeys += ret; ick.buf_nr -= ret; @@ -668,8 +675,8 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) scoutfs_inode_set_data_version(inode, sm.data_version); if (sm.i_size) i_size_write(inode, sm.i_size); - inode->i_ctime.tv_sec = le64_to_cpu(sm.ctime.sec); - inode->i_ctime.tv_nsec = le32_to_cpu(sm.ctime.nsec); + inode->i_ctime.tv_sec = sm.ctime_sec; + inode->i_ctime.tv_nsec = sm.ctime_nsec; scoutfs_update_inode_item(inode, lock, &ind_locks); ret = 0; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index b9966c68..0397b634 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -1,14 +1,41 @@ #ifndef _SCOUTFS_IOCTL_H_ #define _SCOUTFS_IOCTL_H_ +/* + * We naturally align explicit width fields in the ioctl structs so that + * userspace doesn't need to deal with padding or unaligned packing and + * we don't have to deal with 32/64 compat. It makes it a little + * awkward to communicate persistent packed structs through the ioctls + * but that happens very rarely. An interesting special case are + * 0length arrays that follow the structs. We make those start at the + * next aligned offset of the struct to be safe. + * + * This is enforced by pahole scripting in external build environments. + */ + /* XXX I have no idea how these are chosen. */ #define SCOUTFS_IOCTL_MAGIC 's' +/* + * Packed scoutfs keys rarely cross the ioctl boundary so we have a + * translation struct. + */ +struct scoutfs_ioctl_key { + __le64 _sk_first; + __le64 _sk_second; + __le64 _sk_third; + __u8 _sk_fourth; + __u8 sk_type; + __u8 sk_zone; + __u8 _pad[5]; +}; + struct scoutfs_ioctl_walk_inodes_entry { __u64 major; - __u32 minor; __u64 ino; -} __packed; + __u32 minor; + __u8 _pad[4]; +}; /* * Walk inodes in an index that is sorted by one of their fields. @@ -48,7 +75,8 @@ struct scoutfs_ioctl_walk_inodes { __u64 entries_ptr; __u32 nr_entries; __u8 index; -} __packed; + __u8 _pad[11]; /* padded to align walk_inodes_entry total size */ +}; enum { SCOUTFS_IOC_WALK_INODES_META_SEQ = 0, @@ -127,14 +155,16 @@ struct scoutfs_ioctl_ino_path { __u64 dir_pos; __u64 result_ptr; __u16 result_bytes; -} __packed; + __u8 _pad[6]; +}; struct scoutfs_ioctl_ino_path_result { __u64 dir_ino; __u64 dir_pos; __u16 path_bytes; + __u8 _pad[6]; __u8 path[0]; -} __packed; +}; /* Get a single path from the root to the given inode number */ #define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ @@ -168,7 +198,7 @@ struct scoutfs_ioctl_release { __u64 block; __u64 count; __u64 data_version; -} __packed; +}; #define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ struct scoutfs_ioctl_release) @@ -178,7 +208,8 @@ struct scoutfs_ioctl_stage { __u64 buf_ptr; __u64 offset; __s32 count; -} __packed; + __u32 _pad; +}; #define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 6, \ struct scoutfs_ioctl_stage) @@ -203,11 +234,12 @@ struct scoutfs_ioctl_stat_more { __u64 data_version; __u64 online_blocks; __u64 offline_blocks; -} __packed; +}; #define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_stat_more) + /* * Fills the buffer with either the keys for the cached items or the * keys for the cached ranges found starting with the given key. The @@ -215,11 +247,12 @@ struct scoutfs_ioctl_stat_more { * keys the returned number will always be a multiple of two. */ struct scoutfs_ioctl_item_cache_keys { - struct scoutfs_key key; + struct scoutfs_ioctl_key ikey; __u64 buf_ptr; __u16 buf_nr; __u8 which; -} __packed; + __u8 _pad[21]; /* padded to align _ioctl_key total size */ +}; enum { SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS = 0, @@ -233,7 +266,8 @@ struct scoutfs_ioctl_data_waiting_entry { __u64 ino; __u64 iblock; __u8 op; -} __packed; + __u8 _pad[7]; +}; #define SCOUTFS_IOC_DWO_READ (1 << 0) #define SCOUTFS_IOC_DWO_WRITE (1 << 1) @@ -246,7 +280,8 @@ struct scoutfs_ioctl_data_waiting { __u64 after_iblock; __u64 ents_ptr; __u16 ents_nr; -} __packed; + __u8 _pad[6]; +}; #define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) @@ -262,8 +297,10 @@ struct scoutfs_ioctl_setattr_more { __u64 data_version; __u64 i_size; __u64 flags; - struct scoutfs_timespec ctime; -} __packed; + __u64 ctime_sec; + __u32 ctime_nsec; + __u8 _pad[4]; +}; #define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) #define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) @@ -276,7 +313,7 @@ struct scoutfs_ioctl_listxattr_raw { __u64 buf_ptr; __u32 buf_bytes; __u32 hash_pos; -} __packed; +}; #define SCOUTFS_IOC_LISTXATTR_RAW _IOW(SCOUTFS_IOCTL_MAGIC, 11, \ struct scoutfs_ioctl_listxattr_raw) @@ -306,7 +343,8 @@ struct scoutfs_ioctl_find_xattrs { __u64 inodes_ptr; __u16 name_bytes; __u16 nr_inodes; -} __packed; + __u8 _pad[4]; +}; #define SCOUTFS_IOC_FIND_XATTRS _IOW(SCOUTFS_IOCTL_MAGIC, 12, \ struct scoutfs_ioctl_find_xattrs) diff --git a/kmod/src/key.h b/kmod/src/key.h index abe89ee0..7709438d 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -52,6 +52,24 @@ do { \ __entry->name##_zone, __entry->name##_first, __entry->name##_type, \ __entry->name##_second, __entry->name##_third, __entry->name##_fourth +/* + * copy fields between keys with the same fields but different types. + * The destination type might have internal padding so we zero it. + */ +#define scoutfs_key_copy_types(a, b) \ +do { \ + __typeof__(a) _to = (a); \ + __typeof__(b) _from = (b); \ + \ + memset(_to, 0, sizeof(*_to)); \ + _to->sk_zone = _from->sk_zone; \ + _to->_sk_first = _from->_sk_first; \ + _to->sk_type = _from->sk_type; \ + _to->_sk_second = _from->_sk_second; \ + _to->_sk_third = _from->_sk_third; \ + _to->_sk_fourth = _from->_sk_fourth; \ +} while (0) + static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) { key->sk_zone = 0; From 663ce531091d287bef046b2d46fad5adfa15ef97 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 28 Jun 2019 09:56:22 -0700 Subject: [PATCH 721/920] scoutfs: clean up _IO ioctl macro usage Accurately set the direction bits, pack down the used numbers, and remove stale old ioctl definitions. Signed-off-by: Zach Brown --- kmod/src/ioctl.h | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 0397b634..4a62efe9 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -88,7 +88,7 @@ enum { * Adds entries to the user's buffer for each inode that is found in the * given index between the first and last positions. */ -#define SCOUTFS_IOC_WALK_INODES _IOW(SCOUTFS_IOCTL_MAGIC, 1, \ +#define SCOUTFS_IOC_WALK_INODES _IOR(SCOUTFS_IOCTL_MAGIC, 1, \ struct scoutfs_ioctl_walk_inodes) /* @@ -167,10 +167,8 @@ struct scoutfs_ioctl_ino_path_result { }; /* Get a single path from the root to the given inode number */ -#define SCOUTFS_IOC_INO_PATH _IOW(SCOUTFS_IOCTL_MAGIC, 2, \ - struct scoutfs_ioctl_ino_path) - -#define SCOUTFS_IOC_DATA_VERSION _IOW(SCOUTFS_IOCTL_MAGIC, 4, __u64) +#define SCOUTFS_IOC_INO_PATH _IOR(SCOUTFS_IOCTL_MAGIC, 2, \ + struct scoutfs_ioctl_ino_path) /* * "Release" a contiguous range of logical blocks of file data. @@ -200,8 +198,8 @@ struct scoutfs_ioctl_release { __u64 data_version; }; -#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 5, \ - struct scoutfs_ioctl_release) +#define SCOUTFS_IOC_RELEASE _IOW(SCOUTFS_IOCTL_MAGIC, 3, \ + struct scoutfs_ioctl_release) struct scoutfs_ioctl_stage { __u64 data_version; @@ -211,7 +209,7 @@ struct scoutfs_ioctl_stage { __u32 _pad; }; -#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 6, \ +#define SCOUTFS_IOC_STAGE _IOW(SCOUTFS_IOCTL_MAGIC, 4, \ struct scoutfs_ioctl_stage) /* @@ -236,7 +234,7 @@ struct scoutfs_ioctl_stat_more { __u64 offline_blocks; }; -#define SCOUTFS_IOC_STAT_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ +#define SCOUTFS_IOC_STAT_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 5, \ struct scoutfs_ioctl_stat_more) @@ -259,7 +257,7 @@ enum { SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES, }; -#define SCOUTFS_IOC_ITEM_CACHE_KEYS _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ +#define SCOUTFS_IOC_ITEM_CACHE_KEYS _IOR(SCOUTFS_IOCTL_MAGIC, 6, \ struct scoutfs_ioctl_item_cache_keys) struct scoutfs_ioctl_data_waiting_entry { @@ -285,7 +283,7 @@ struct scoutfs_ioctl_data_waiting { #define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) -#define SCOUTFS_IOC_DATA_WAITING _IOW(SCOUTFS_IOCTL_MAGIC, 9, \ +#define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_data_waiting) /* @@ -305,7 +303,7 @@ struct scoutfs_ioctl_setattr_more { #define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) #define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) -#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 10, \ +#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ struct scoutfs_ioctl_setattr_more) struct scoutfs_ioctl_listxattr_raw { @@ -315,7 +313,7 @@ struct scoutfs_ioctl_listxattr_raw { __u32 hash_pos; }; -#define SCOUTFS_IOC_LISTXATTR_RAW _IOW(SCOUTFS_IOCTL_MAGIC, 11, \ +#define SCOUTFS_IOC_LISTXATTR_RAW _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ struct scoutfs_ioctl_listxattr_raw) /* @@ -346,7 +344,7 @@ struct scoutfs_ioctl_find_xattrs { __u8 _pad[4]; }; -#define SCOUTFS_IOC_FIND_XATTRS _IOW(SCOUTFS_IOCTL_MAGIC, 12, \ - struct scoutfs_ioctl_find_xattrs) +#define SCOUTFS_IOC_FIND_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ + struct scoutfs_ioctl_find_xattrs) #endif From d8bc962fc58a2467a50b128d8586ec2dfcb5b69f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 28 Jun 2019 10:06:25 -0700 Subject: [PATCH 722/920] scoutfs: unpriv listxattr_hidden only shows .hide. Our hidden attributes are hidden so that they don't leak out of the system when archiving tools transfer xattrs from listxattr along with the file. They're not intended to be secret, in fact users want to see their contents like they want to see other fs metadata that they can't update which describes the system. Make our listxattr ioctl only return hidden xattrs and allow anyone to see the results if they can read the file. Rename it to more accurately describe its intended use. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 43 +++++++++++++++++++++---------------------- kmod/src/ioctl.h | 6 +++--- kmod/src/xattr.c | 9 +++++---- kmod/src/xattr.h | 2 +- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index e5e7f5a7..bee29df7 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -693,27 +693,26 @@ out: return ret; } -static long scoutfs_ioc_listxattr_raw(struct file *file, unsigned long arg) +/* + * This lists .hide. attributes on the inode. It doesn't include normal + * xattrs that are visible to listxattr because we don't perform as + * rigorous security access checks as normal vfs listxattr does. + */ +static long scoutfs_ioc_listxattr_hidden(struct file *file, unsigned long arg) { struct inode *inode = file->f_inode; - struct scoutfs_ioctl_listxattr_raw __user *ulxr = (void __user *)arg; - struct scoutfs_ioctl_listxattr_raw lxr; + struct scoutfs_ioctl_listxattr_hidden __user *ulxr = (void __user *)arg; + struct scoutfs_ioctl_listxattr_hidden lxh; struct page *page = NULL; unsigned int bytes; int total = 0; int ret; - if (!(file->f_mode & FMODE_READ)) { - ret = -EBADF; + ret = inode_permission(inode, MAY_READ); + if (ret < 0) goto out; - } - if (!capable(CAP_SYS_ADMIN)) { - ret = -EBADF; - goto out; - } - - if (copy_from_user(&lxr, ulxr, sizeof(lxr))) { + if (copy_from_user(&lxh, ulxr, sizeof(lxh))) { ret = -EFAULT; goto out; } @@ -724,22 +723,22 @@ static long scoutfs_ioc_listxattr_raw(struct file *file, unsigned long arg) goto out; } - while (lxr.buf_bytes) { - bytes = min_t(int, lxr.buf_bytes, PAGE_SIZE); + while (lxh.buf_bytes) { + bytes = min_t(int, lxh.buf_bytes, PAGE_SIZE); ret = scoutfs_list_xattrs(inode, page_address(page), bytes, - &lxr.hash_pos, &lxr.id_pos, + &lxh.hash_pos, &lxh.id_pos, false, true); if (ret <= 0) break; - if (copy_to_user((void __user *)lxr.buf_ptr, + if (copy_to_user((void __user *)lxh.buf_ptr, page_address(page), ret)) { ret = -EFAULT; break; } - lxr.buf_ptr += ret; - lxr.buf_bytes -= ret; + lxh.buf_ptr += ret; + lxh.buf_bytes -= ret; total += ret; ret = 0; } @@ -748,8 +747,8 @@ out: if (page) __free_page(page); - if (ret == 0 && (__put_user(lxr.hash_pos, &ulxr->hash_pos) || - __put_user(lxr.id_pos, &ulxr->id_pos))) + if (ret == 0 && (__put_user(lxh.hash_pos, &ulxr->hash_pos) || + __put_user(lxh.id_pos, &ulxr->id_pos))) ret = -EFAULT; return ret ?: total; @@ -868,8 +867,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_data_waiting(file, arg); case SCOUTFS_IOC_SETATTR_MORE: return scoutfs_ioc_setattr_more(file, arg); - case SCOUTFS_IOC_LISTXATTR_RAW: - return scoutfs_ioc_listxattr_raw(file, arg); + case SCOUTFS_IOC_LISTXATTR_HIDDEN: + return scoutfs_ioc_listxattr_hidden(file, arg); case SCOUTFS_IOC_FIND_XATTRS: return scoutfs_ioc_find_xattrs(file, arg); } diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 4a62efe9..5aa057c0 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -306,15 +306,15 @@ struct scoutfs_ioctl_setattr_more { #define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ struct scoutfs_ioctl_setattr_more) -struct scoutfs_ioctl_listxattr_raw { +struct scoutfs_ioctl_listxattr_hidden { __u64 id_pos; __u64 buf_ptr; __u32 buf_bytes; __u32 hash_pos; }; -#define SCOUTFS_IOC_LISTXATTR_RAW _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ - struct scoutfs_ioctl_listxattr_raw) +#define SCOUTFS_IOC_LISTXATTR_HIDDEN _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ + struct scoutfs_ioctl_listxattr_hidden) /* * Return the inode numbers of inodes which might contain the given diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index a877d0ec..7196114d 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -596,7 +596,7 @@ int scoutfs_removexattr(struct dentry *dentry, const char *name) ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, size_t size, __u32 *hash_pos, __u64 *id_pos, - bool e_range, bool hidden) + bool e_range, bool show_hidden) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; @@ -607,6 +607,7 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, unsigned int bytes; ssize_t total = 0; u32 name_hash = 0; + bool is_hidden; u64 id = 0; int ret; @@ -638,10 +639,10 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, break; } - if (hidden || - parse_tags(xat->name, xat->name_len, &tgs) != 0 || - !tgs.hide) { + is_hidden = parse_tags(xat->name, xat->name_len, &tgs) == 0 && + tgs.hide; + if (show_hidden == is_hidden) { if (size) { if ((total + xat->name_len + 1) > size) { if (e_range) diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index efcbc62a..4c78e323 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -9,7 +9,7 @@ int scoutfs_removexattr(struct dentry *dentry, const char *name); ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size); ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, size_t size, __u32 *hash_pos, __u64 *id_pos, - bool e_range, bool hidden); + bool e_range, bool show_hidden); int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock); From b147d74967ce0590bf96cd50a477df9f5186dd13 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Jun 2019 15:06:05 -0700 Subject: [PATCH 723/920] scoutfs: add per-mount random id Calculate a random id which identifies the life of a particular mount. This will be visible in messages and tracing and will replace the server-assigned node_id in persistent structures and protocols. Signed-off-by: Zach Brown --- kmod/src/super.c | 22 ++++++++++++++++++++++ kmod/src/super.h | 1 + 2 files changed, 23 insertions(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index 7ebaa178..0f644c4f 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -343,6 +343,24 @@ static int scoutfs_debugfs_setup(struct super_block *sb) return 0; } +/* + * Calculate a random id for the mount very early, it's used in tracing + * and message output. The system assumes that a rid of 0 can't exist. We're + * also paranoid and avoid rids that are likely the result of bad rng. + */ +static int assign_random_id(struct scoutfs_sb_info *sbi) +{ + unsigned int attempts = 0; + + do { + if (++attempts == 100) + return -EIO; + get_random_bytes(&sbi->rid, sizeof(sbi->rid)); + } while (sbi->rid == 0 || sbi->rid == ~0ULL); + + return 0; +} + static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; @@ -366,6 +384,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) if (!sbi) return -ENOMEM; + ret = assign_random_id(sbi); + if (ret < 0) + return ret; + spin_lock_init(&sbi->next_ino_lock); init_waitqueue_head(&sbi->trans_hold_wq); spin_lock_init(&sbi->data_wait_root.lock); diff --git a/kmod/src/super.h b/kmod/src/super.h index 0ffd6c76..6f88c878 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -31,6 +31,7 @@ struct scoutfs_sb_info { struct super_block *sb; /* assigned once at the start of each mount, read-only */ + u64 rid; u64 node_id; struct scoutfs_lock *node_id_lock; From 7acbf4cc8bc7a7da26a093988a1f35a0219eee17 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Jun 2019 15:13:16 -0700 Subject: [PATCH 724/920] scoutfs: add super block format and args Add macros which provide printk format and args for a little string which identifies a specific mount. This will be used in kernel logs and trace messages. Signed-off-by: Zach Brown --- kmod/src/super.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/kmod/src/super.h b/kmod/src/super.h index 6f88c878..6cd9bb85 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -91,6 +91,40 @@ static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb) return sb->s_fs_info; } +static inline bool SCOUTFS_HAS_SBI(struct super_block *sb) +{ + return (sb != NULL) && (SCOUTFS_SB(sb) != NULL); +} + +/* + * A small string embedded in messages that's used to identify a + * specific mount. It's the three most significant bytes of the fsid + * and the rid. That gives us a strong chance of avoiding collisions + * with typical numbers of mounts. We give it a bit of structure to + * make it searchable and to be able to identify format changes, should + * we need to. The fsid will be 0 until the super has been read and the + * fsid discovered. + */ +#define SCSBF "f.%.06x.r.%.06x" +#define SCSB_SHIFT (64 - (8 * 3)) +#define SCSB_LEFR_ARGS(fsid, rid) \ + (int)(le64_to_cpu(fsid) >> SCSB_SHIFT), \ + (int)(le64_to_cpu(rid) >> SCSB_SHIFT) +#define SCSB_ARGS(sb) \ + (int)(le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) >> SCSB_SHIFT), \ + (int)(SCOUTFS_SB(sb)->rid >> SCSB_SHIFT) +#define SCSB_TRACE_FIELDS \ + __field(__u64, fsid) \ + __field(__u64, rid) +#define SCSB_TRACE_ASSIGN(sb) \ + __entry->fsid = SCOUTFS_HAS_SBI(sb) ? \ + le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) : 0;\ + __entry->rid = SCOUTFS_HAS_SBI(sb) ? \ + SCOUTFS_SB(sb)->rid : 0; +#define SCSB_TRACE_ARGS \ + (int)(__entry->fsid >> SCSB_SHIFT), \ + (int)(__entry->rid >> SCSB_SHIFT) + int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super_res); void scoutfs_advance_dirty_super(struct super_block *sb); From 754ce95f5c3786d1b5ac2a6dde4a764e50cfd84e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Jun 2019 15:19:46 -0700 Subject: [PATCH 725/920] scoutfs: use rid in console messages Change the console message output to show the fsid:rid mount identity instead of the block device name and device major and minor numbers. Signed-off-by: Zach Brown --- kmod/src/msg.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kmod/src/msg.c b/kmod/src/msg.c index f2ebc81f..1d268de3 100644 --- a/kmod/src/msg.c +++ b/kmod/src/msg.c @@ -18,9 +18,7 @@ void scoutfs_msg(struct super_block *sb, const char *prefix, const char *str, vaf.fmt = fmt; vaf.va = &args; - printk("%sscoutfs (%s %u:%u)%s: %pV\n", prefix, - sb->s_id, MAJOR(sb->s_bdev->bd_dev), MINOR(sb->s_bdev->bd_dev), - str, &vaf); + printk("%sscoutfs "SCSBF"%s: %pV\n", prefix, SCSB_ARGS(sb), str, &vaf); va_end(args); } From 7a36a289d252a0a1bbd6db870c7737a269f24e09 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Jun 2019 15:52:21 -0700 Subject: [PATCH 726/920] scoutfs: add rid to trace messages Add the mount rid to traces which included the fsid by converting them to use the super block message format and args. Signed-off-by: Zach Brown --- kmod/src/scoutfs_trace.h | 530 +++++++++++++++++++-------------------- 1 file changed, 264 insertions(+), 266 deletions(-) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a9c6cfe3..20fc1282 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -41,16 +41,13 @@ struct lock_info; -#define FSID_ARG(sb) le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid) -#define FSID_FMT "%llx" - TRACE_EVENT(scoutfs_setattr, TP_PROTO(struct dentry *dentry, struct iattr *attr), TP_ARGS(dentry, attr), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned int, d_len) __string(d_name, dentry->d_name.name) @@ -61,7 +58,7 @@ TRACE_EVENT(scoutfs_setattr, ), TP_fast_assign( - __entry->fsid = FSID_ARG(dentry->d_inode->i_sb); + SCSB_TRACE_ASSIGN(dentry->d_inode->i_sb); __entry->ino = scoutfs_ino(dentry->d_inode); __entry->d_len = dentry->d_name.len; __assign_str(d_name, dentry->d_name.name); @@ -71,8 +68,8 @@ TRACE_EVENT(scoutfs_setattr, __entry->i_size = i_size_read(dentry->d_inode); ), - TP_printk(FSID_FMT" %s ino %llu ia_valid 0x%x size change %d ia_size " - "%llu i_size %llu", __entry->fsid, __get_str(d_name), + TP_printk(SCSBF" %s ino %llu ia_valid 0x%x size change %d ia_size " + "%llu i_size %llu", SCSB_TRACE_ARGS, __get_str(d_name), __entry->ino, __entry->ia_valid, __entry->size_change, __entry->ia_size, __entry->i_size) ); @@ -83,21 +80,21 @@ TRACE_EVENT(scoutfs_complete_truncate, TP_ARGS(inode, flags), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, i_size) __field(__u32, flags) ), TP_fast_assign( - __entry->fsid = FSID_ARG(inode->i_sb); + SCSB_TRACE_ASSIGN(inode->i_sb); __entry->ino = scoutfs_ino(inode); __entry->i_size = i_size_read(inode); __entry->flags = flags; ), - TP_printk(FSID_FMT" ino %llu i_size %llu flags 0x%x", - __entry->fsid, __entry->ino, __entry->i_size, + TP_printk(SCSBF" ino %llu i_size %llu flags 0x%x", + SCSB_TRACE_ARGS, __entry->ino, __entry->i_size, __entry->flags) ); @@ -107,18 +104,18 @@ DECLARE_EVENT_CLASS(scoutfs_comp_class, TP_ARGS(sb, comp), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(struct scoutfs_bio_completion *, comp) __field(int, pending) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->comp = comp; __entry->pending = atomic_read(&comp->pending); ), - TP_printk(FSID_FMT" comp %p pending before %d", __entry->fsid, + TP_printk(SCSBF" comp %p pending before %d", SCSB_TRACE_ARGS, __entry->comp, __entry->pending) ); DEFINE_EVENT(scoutfs_comp_class, comp_end_io, @@ -156,20 +153,20 @@ DECLARE_EVENT_CLASS(scoutfs_bio_class, TP_ARGS(sb, bio, args, in_flight), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, bio) __field(void *, args) __field(int, in_flight) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->bio = bio; __entry->args = args; __entry->in_flight = in_flight; ), - TP_printk(FSID_FMT" bio %p args %p in_flight %d", __entry->fsid, + TP_printk(SCSBF" bio %p args %p in_flight %d", SCSB_TRACE_ARGS, __entry->bio, __entry->args, __entry->in_flight) ); @@ -189,20 +186,20 @@ TRACE_EVENT(scoutfs_bio_end_io, TP_ARGS(sb, bio, size, err), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, bio) __field(int, size) __field(int, err) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->bio = bio; __entry->size = size; __entry->err = err; ), - TP_printk(FSID_FMT" bio %p size %u err %d", __entry->fsid, + TP_printk(SCSBF" bio %p size %u err %d", SCSB_TRACE_ARGS, __entry->bio, __entry->size, __entry->err) ); @@ -212,20 +209,20 @@ TRACE_EVENT(scoutfs_dec_end_io, TP_ARGS(sb, args, in_flight, err), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, args) __field(int, in_flight) __field(int, err) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->args = args; __entry->in_flight = in_flight; __entry->err = err; ), - TP_printk(FSID_FMT" args %p in_flight %d err %d", __entry->fsid, + TP_printk(SCSBF" args %p in_flight %d err %d", SCSB_TRACE_ARGS, __entry->args, __entry->in_flight, __entry->err) ); @@ -235,19 +232,19 @@ DECLARE_EVENT_CLASS(scoutfs_key_ret_class, TP_ARGS(sb, key, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS sk_trace_define(key) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); sk_trace_assign(key, key); __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" key "SK_FMT" ret %d", - __entry->fsid, sk_trace_args(key), __entry->ret) + TP_printk(SCSBF" key "SK_FMT" ret %d", + SCSB_TRACE_ARGS, sk_trace_args(key), __entry->ret) ); DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_create, @@ -269,16 +266,16 @@ TRACE_EVENT(scoutfs_item_dirty_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); TRACE_EVENT(scoutfs_item_update_ret, @@ -287,16 +284,16 @@ TRACE_EVENT(scoutfs_item_update_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); TRACE_EVENT(scoutfs_item_next_ret, @@ -305,16 +302,16 @@ TRACE_EVENT(scoutfs_item_next_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); TRACE_EVENT(scoutfs_item_prev_ret, @@ -323,16 +320,16 @@ TRACE_EVENT(scoutfs_item_prev_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); TRACE_EVENT(scoutfs_erase_item, @@ -341,16 +338,16 @@ TRACE_EVENT(scoutfs_erase_item, TP_ARGS(sb, item), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, item) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->item = item; ), - TP_printk(FSID_FMT" erasing item %p", __entry->fsid, __entry->item) + TP_printk(SCSBF" erasing item %p", SCSB_TRACE_ARGS, __entry->item) ); TRACE_EVENT(scoutfs_data_fallocate, @@ -360,7 +357,7 @@ TRACE_EVENT(scoutfs_data_fallocate, TP_ARGS(sb, ino, mode, offset, len, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(int, mode) __field(__u64, offset) @@ -369,7 +366,7 @@ TRACE_EVENT(scoutfs_data_fallocate, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->mode = mode; __entry->offset = offset; @@ -377,8 +374,8 @@ TRACE_EVENT(scoutfs_data_fallocate, __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" ino %llu mode 0x%x offset %llu len %llu ret %d", - __entry->fsid, __entry->ino, __entry->mode, __entry->offset, + TP_printk(SCSBF" ino %llu mode 0x%x offset %llu len %llu ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->mode, __entry->offset, __entry->len, __entry->ret) ); @@ -389,20 +386,20 @@ TRACE_EVENT(scoutfs_data_fiemap, TP_ARGS(sb, off, i, blkno), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, off) __field(int, i) __field(__u64, blkno) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->off = off; __entry->i = i; __entry->blkno = blkno; ), - TP_printk(FSID_FMT" blk_off %llu i %u blkno %llu", __entry->fsid, + TP_printk(SCSBF" blk_off %llu i %u blkno %llu", SCSB_TRACE_ARGS, __entry->off, __entry->i, __entry->blkno) ); @@ -413,7 +410,7 @@ TRACE_EVENT(scoutfs_get_block, TP_ARGS(sb, ino, iblock, create, ret, blkno, size), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, iblock) __field(int, create) @@ -423,7 +420,7 @@ TRACE_EVENT(scoutfs_get_block, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->iblock = iblock; __entry->create = create; @@ -432,8 +429,8 @@ TRACE_EVENT(scoutfs_get_block, __entry->size = size; ), - TP_printk(FSID_FMT" ino %llu iblock %llu create %d ret %d bnr %llu " - "size %zu", __entry->fsid, __entry->ino, __entry->iblock, + TP_printk(SCSBF" ino %llu iblock %llu create %d ret %d bnr %llu " + "size %zu", SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, __entry->create, __entry->ret, __entry->blkno, __entry->size) ); @@ -445,7 +442,7 @@ TRACE_EVENT(scoutfs_data_alloc_block, TP_ARGS(sb, inode, ext, iblock, len, online_blocks, offline_blocks), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) se_trace_define(ext) __field(__u64, iblock) @@ -455,7 +452,7 @@ TRACE_EVENT(scoutfs_data_alloc_block, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = scoutfs_ino(inode); se_trace_assign(ext, ext); __entry->iblock = iblock; @@ -464,8 +461,8 @@ TRACE_EVENT(scoutfs_data_alloc_block, __entry->offline_blocks = offline_blocks; ), - TP_printk("fsid "FSID_FMT" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", - __entry->fsid, __entry->ino, se_trace_args(ext), + TP_printk(SCSBF" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", + SCSB_TRACE_ARGS, __entry->ino, se_trace_args(ext), __entry->iblock, __entry->len, __entry->online_blocks, __entry->offline_blocks) ); @@ -476,18 +473,18 @@ TRACE_EVENT(scoutfs_data_alloc_block_ret, TP_ARGS(sb, ext, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS se_trace_define(ext) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); se_trace_assign(ext, ext); __entry->ret = ret; ), - TP_printk(FSID_FMT" ext "SE_FMT" ret %d", __entry->fsid, + TP_printk(SCSBF" ext "SE_FMT" ret %d", SCSB_TRACE_ARGS, se_trace_args(ext), __entry->ret) ); @@ -497,20 +494,20 @@ TRACE_EVENT(scoutfs_data_truncate_items, TP_ARGS(sb, iblock, last, offline), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, iblock) __field(__u64, last) __field(int, offline) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->iblock = iblock; __entry->last = last; __entry->offline = offline; ), - TP_printk(FSID_FMT" iblock %llu last %llu offline %u", __entry->fsid, + TP_printk(SCSBF" iblock %llu last %llu offline %u", SCSB_TRACE_ARGS, __entry->iblock, __entry->last, __entry->offline) ); @@ -522,7 +519,7 @@ TRACE_EVENT(scoutfs_data_wait_check, TP_ARGS(sb, ino, pos, len, sef, op, ext_start, ext_len, ext_flags, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, pos) __field(__u64, len) @@ -535,7 +532,7 @@ TRACE_EVENT(scoutfs_data_wait_check, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->pos = pos; __entry->len = len; @@ -547,10 +544,10 @@ TRACE_EVENT(scoutfs_data_wait_check, __entry->ret = ret; ), - TP_printk(FSID_FMT" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext_start %llu ext_len %llu ext_flags 0x%x ret %d", - __entry->fsid, __entry->ino, __entry->pos, __entry->len, - __entry->sef, __entry->op, __entry->ext_start, - __entry->ext_len, __entry->ext_flags, __entry->ret) + TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext_start %llu ext_len %llu ext_flags 0x%x ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->pos, __entry->len, + __entry->sef, __entry->op, __entry->ext_start, + __entry->ext_len, __entry->ext_flags, __entry->ret) ); TRACE_EVENT(scoutfs_sync_fs, @@ -559,16 +556,16 @@ TRACE_EVENT(scoutfs_sync_fs, TP_ARGS(sb, wait), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, wait) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->wait = wait; ), - TP_printk(FSID_FMT" wait %d", __entry->fsid, __entry->wait) + TP_printk(SCSBF" wait %d", SCSB_TRACE_ARGS, __entry->wait) ); TRACE_EVENT(scoutfs_trans_write_func, @@ -577,16 +574,16 @@ TRACE_EVENT(scoutfs_trans_write_func, TP_ARGS(sb, dirty), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, dirty) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->dirty = dirty; ), - TP_printk(FSID_FMT" dirty %d", __entry->fsid, __entry->dirty) + TP_printk(SCSBF" dirty %d", SCSB_TRACE_ARGS, __entry->dirty) ); TRACE_EVENT(scoutfs_release_trans, @@ -600,7 +597,7 @@ TRACE_EVENT(scoutfs_release_trans, tri_items, tri_vals), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, rsv) __field(unsigned int, rsv_holders) __field(int, res_items) @@ -614,7 +611,7 @@ TRACE_EVENT(scoutfs_release_trans, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->rsv = rsv; __entry->rsv_holders = rsv_holders; __entry->res_items = res->items; @@ -627,9 +624,9 @@ TRACE_EVENT(scoutfs_release_trans, __entry->tri_vals = tri_vals; ), - TP_printk(FSID_FMT" rsv %p holders %u reserved %u.%u actual " + TP_printk(SCSBF" rsv %p holders %u reserved %u.%u actual " "%d.%d, trans holders %u writing %u reserved " - "%u.%u", __entry->fsid, __entry->rsv, __entry->rsv_holders, + "%u.%u", SCSB_TRACE_ARGS, __entry->rsv, __entry->rsv_holders, __entry->res_items, __entry->res_vals, __entry->act_items, __entry->act_vals, __entry->tri_holders, __entry->tri_writing, __entry->tri_items, __entry->tri_vals) @@ -647,7 +644,7 @@ TRACE_EVENT(scoutfs_trans_acquired_hold, tri_items, tri_vals), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, cnt_items) __field(int, cnt_vals) __field(void *, rsv) @@ -663,7 +660,7 @@ TRACE_EVENT(scoutfs_trans_acquired_hold, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->cnt_items = cnt->items; __entry->cnt_vals = cnt->vals; __entry->rsv = rsv; @@ -678,9 +675,9 @@ TRACE_EVENT(scoutfs_trans_acquired_hold, __entry->tri_vals = tri_vals; ), - TP_printk(FSID_FMT" cnt %u.%u, rsv %p holders %u reserved %u.%u " + TP_printk(SCSBF" cnt %u.%u, rsv %p holders %u reserved %u.%u " "actual %d.%d, trans holders %u writing %u reserved " - "%u.%u", __entry->fsid, __entry->cnt_items, + "%u.%u", SCSB_TRACE_ARGS, __entry->cnt_items, __entry->cnt_vals, __entry->rsv, __entry->rsv_holders, __entry->res_items, __entry->res_vals, __entry->act_items, __entry->act_vals, __entry->tri_holders, __entry->tri_writing, @@ -695,7 +692,7 @@ TRACE_EVENT(scoutfs_trans_track_item, res_vals), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, delta_items) __field(int, delta_vals) __field(int, act_items) @@ -705,7 +702,7 @@ TRACE_EVENT(scoutfs_trans_track_item, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->delta_items = delta_items; __entry->delta_vals = delta_vals; __entry->act_items = act_items; @@ -714,8 +711,8 @@ TRACE_EVENT(scoutfs_trans_track_item, __entry->res_vals = res_vals; ), - TP_printk("fsid "FSID_FMT" delta_items %d delta_vals %d act_items %d act_vals %d res_items %d res_vals %d", - __entry->fsid, __entry->delta_items, __entry->delta_vals, + TP_printk(SCSBF" delta_items %d delta_vals %d act_items %d act_vals %d res_items %d res_vals %d", + SCSB_TRACE_ARGS, __entry->delta_items, __entry->delta_vals, __entry->act_items, __entry->act_vals, __entry->res_items, __entry->res_vals) ); @@ -726,16 +723,16 @@ TRACE_EVENT(scoutfs_ioc_release_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); TRACE_EVENT(scoutfs_ioc_release, @@ -744,20 +741,20 @@ TRACE_EVENT(scoutfs_ioc_release, TP_ARGS(sb, args), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, block) __field(__u64, count) __field(__u64, vers) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->block = args->block; __entry->count = args->count; __entry->vers = args->data_version; ), - TP_printk(FSID_FMT" block %llu count %llu vers %llu", __entry->fsid, + TP_printk(SCSBF" block %llu count %llu vers %llu", SCSB_TRACE_ARGS, __entry->block, __entry->count, __entry->vers) ); @@ -767,7 +764,7 @@ TRACE_EVENT(scoutfs_ioc_walk_inodes, TP_ARGS(sb, walk), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, index) __field(__u64, first_major) __field(__u32, first_minor) @@ -778,7 +775,7 @@ TRACE_EVENT(scoutfs_ioc_walk_inodes, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->index = walk->index; __entry->first_major = walk->first.major; __entry->first_minor = walk->first.minor; @@ -788,8 +785,8 @@ TRACE_EVENT(scoutfs_ioc_walk_inodes, __entry->last_ino = walk->last.ino; ), - TP_printk(FSID_FMT" index %u first %llu.%u.%llu last %llu.%u.%llu", - __entry->fsid, __entry->index, __entry->first_major, + TP_printk(SCSBF" index %u first %llu.%u.%llu last %llu.%u.%llu", + SCSB_TRACE_ARGS, __entry->index, __entry->first_major, __entry->first_minor, __entry->first_ino, __entry->last_major, __entry->last_minor, __entry->last_ino) ); @@ -818,7 +815,7 @@ DECLARE_EVENT_CLASS(scoutfs_index_item_class, TP_ARGS(sb, type, major, minor, ino), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u8, type) __field(__u64, major) __field(__u32, minor) @@ -826,16 +823,16 @@ DECLARE_EVENT_CLASS(scoutfs_index_item_class, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->type = type; __entry->major = major; __entry->minor = minor; __entry->ino = ino; ), - TP_printk("fsid "FSID_FMT" type %u major %llu minor %u ino %llu", - __entry->fsid, __entry->type, __entry->major, __entry->minor, - __entry->ino) + TP_printk(SCSBF" type %u major %llu minor %u ino %llu", + SCSB_TRACE_ARGS, __entry->type, __entry->major, + __entry->minor, __entry->ino) ); DEFINE_EVENT(scoutfs_index_item_class, scoutfs_create_index_item, @@ -857,7 +854,7 @@ TRACE_EVENT(scoutfs_alloc_ino, TP_ARGS(sb, ret, ino, next_ino, next_nr), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) __field(__u64, ino) __field(__u64, next_ino) @@ -865,16 +862,16 @@ TRACE_EVENT(scoutfs_alloc_ino, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; __entry->ino = ino; __entry->next_ino = next_ino; __entry->next_nr = next_nr; ), - TP_printk(FSID_FMT" ret %d ino %llu next_ino %llu next_nr %llu", - __entry->fsid, __entry->ret, __entry->ino, __entry->next_ino, - __entry->next_nr) + TP_printk(SCSBF" ret %d ino %llu next_ino %llu next_nr %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->ino, + __entry->next_ino, __entry->next_nr) ); TRACE_EVENT(scoutfs_evict_inode, @@ -884,20 +881,20 @@ TRACE_EVENT(scoutfs_evict_inode, TP_ARGS(sb, ino, nlink, is_bad_ino), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned int, nlink) __field(unsigned int, is_bad_ino) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->nlink = nlink; __entry->is_bad_ino = is_bad_ino; ), - TP_printk(FSID_FMT" ino %llu nlink %u bad %d", __entry->fsid, + TP_printk(SCSBF" ino %llu nlink %u bad %d", SCSB_TRACE_ARGS, __entry->ino, __entry->nlink, __entry->is_bad_ino) ); @@ -908,20 +905,20 @@ TRACE_EVENT(scoutfs_drop_inode, TP_ARGS(sb, ino, nlink, unhashed), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned int, nlink) __field(unsigned int, unhashed) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->nlink = nlink; __entry->unhashed = unhashed; ), - TP_printk(FSID_FMT" ino %llu nlink %u unhashed %d", __entry->fsid, + TP_printk(SCSBF" ino %llu nlink %u unhashed %d", SCSB_TRACE_ARGS, __entry->ino, __entry->nlink, __entry->unhashed) ); @@ -931,20 +928,20 @@ TRACE_EVENT(scoutfs_inode_walk_writeback, TP_ARGS(sb, ino, write, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(int, write) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->write = write; __entry->ret = ret; ), - TP_printk(FSID_FMT" ino %llu write %d ret %d", __entry->fsid, + TP_printk(SCSBF" ino %llu write %d ret %d", SCSB_TRACE_ARGS, __entry->ino, __entry->write, __entry->ret) ); @@ -954,16 +951,16 @@ DECLARE_EVENT_CLASS(scoutfs_segment_class, TP_ARGS(sb, segno), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, segno) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->segno = segno; ), - TP_printk(FSID_FMT" segno %llu", __entry->fsid, __entry->segno) + TP_printk(SCSBF" segno %llu", SCSB_TRACE_ARGS, __entry->segno) ); DEFINE_EVENT(scoutfs_segment_class, scoutfs_seg_submit_read, @@ -982,16 +979,16 @@ DECLARE_EVENT_CLASS(scoutfs_lock_info_class, TP_ARGS(sb, linfo), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(struct lock_info *, linfo) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->linfo = linfo; ), - TP_printk(FSID_FMT" linfo %p", __entry->fsid, __entry->linfo) + TP_printk(SCSBF" linfo %p", SCSB_TRACE_ARGS, __entry->linfo) ); DEFINE_EVENT(scoutfs_lock_info_class, scoutfs_lock_setup, @@ -1016,7 +1013,7 @@ TRACE_EVENT(scoutfs_xattr_set, TP_ARGS(sb, name_len, value, size, flags), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(size_t, name_len) __field(const void *, value) __field(size_t, size) @@ -1024,15 +1021,15 @@ TRACE_EVENT(scoutfs_xattr_set, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->name_len = name_len; __entry->value = value; __entry->size = size; __entry->flags = flags; ), - TP_printk(FSID_FMT" name_len %zu value %p size %zu flags 0x%x", - __entry->fsid, __entry->name_len, __entry->value, + TP_printk(SCSBF" name_len %zu value %p size %zu flags 0x%x", + SCSB_TRACE_ARGS, __entry->name_len, __entry->value, __entry->size, __entry->flags) ); @@ -1042,18 +1039,18 @@ TRACE_EVENT(scoutfs_manifest_next_compact, TP_ARGS(sb, level, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, level) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->level = level; __entry->ret = ret; ), - TP_printk(FSID_FMT" level %d ret %d", __entry->fsid, __entry->level, + TP_printk(SCSBF" level %d ret %d", SCSB_TRACE_ARGS, __entry->level, __entry->ret) ); @@ -1063,16 +1060,16 @@ TRACE_EVENT(scoutfs_advance_dirty_super, TP_ARGS(sb, seq), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, seq) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->seq = seq; ), - TP_printk(FSID_FMT" super seq now %llu", __entry->fsid, __entry->seq) + TP_printk(SCSBF" super seq now %llu", SCSB_TRACE_ARGS, __entry->seq) ); TRACE_EVENT(scoutfs_dir_add_next_linkref, @@ -1084,7 +1081,7 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, name_len), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, dir_ino) __field(__u64, dir_pos) @@ -1095,7 +1092,7 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->dir_ino = dir_ino; __entry->dir_pos = dir_pos; @@ -1105,8 +1102,8 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, __entry->name_len = name_len; ), - TP_printk("fsid "FSID_FMT" ino %llu dir_ino %llu dir_pos %llu ret %d found_dir_ino %llu found_dir_pos %llu name_len %u", - __entry->fsid, __entry->ino, __entry->dir_pos, + TP_printk(SCSBF" ino %llu dir_ino %llu dir_pos %llu ret %d found_dir_ino %llu found_dir_pos %llu name_len %u", + SCSB_TRACE_ARGS, __entry->ino, __entry->dir_pos, __entry->dir_ino, __entry->ret, __entry->found_dir_pos, __entry->found_dir_ino, __entry->name_len) ); @@ -1117,21 +1114,21 @@ TRACE_EVENT(scoutfs_client_compact_start, TP_ARGS(sb, id, last_level, flags), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, id) __field(__u8, last_level) __field(__u8, flags) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->last_level = last_level; __entry->flags = flags; ), - TP_printk("fsid "FSID_FMT" id %llu last_level %u flags 0x%x", - __entry->fsid, __entry->id, __entry->last_level, + TP_printk(SCSBF" id %llu last_level %u flags 0x%x", + SCSB_TRACE_ARGS, __entry->id, __entry->last_level, __entry->flags) ); @@ -1141,19 +1138,19 @@ TRACE_EVENT(scoutfs_client_compact_stop, TP_ARGS(sb, id, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, id) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" id %llu ret %d", - __entry->fsid, __entry->id, __entry->ret) + TP_printk(SCSBF" id %llu ret %d", + SCSB_TRACE_ARGS, __entry->id, __entry->ret) ); TRACE_EVENT(scoutfs_server_compact_start, @@ -1164,7 +1161,7 @@ TRACE_EVENT(scoutfs_server_compact_start, TP_ARGS(sb, id, level, node_id, client_nr, server_nr, per_client), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, id) __field(__u8, level) __field(__u64, node_id) @@ -1174,7 +1171,7 @@ TRACE_EVENT(scoutfs_server_compact_start, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->level = level; __entry->node_id = node_id; @@ -1183,9 +1180,10 @@ TRACE_EVENT(scoutfs_server_compact_start, __entry->per_client = per_client; ), - TP_printk("fsid "FSID_FMT" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu", - __entry->fsid, __entry->id, __entry->level, __entry->node_id, - __entry->client_nr, __entry->server_nr, __entry->per_client) + TP_printk(SCSBF" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu", + SCSB_TRACE_ARGS, __entry->id, __entry->level, + __entry->node_id, __entry->client_nr, __entry->server_nr, + __entry->per_client) ); TRACE_EVENT(scoutfs_server_compact_done, @@ -1195,21 +1193,21 @@ TRACE_EVENT(scoutfs_server_compact_done, TP_ARGS(sb, id, node_id, server_nr), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, id) __field(__u64, node_id) __field(unsigned long, server_nr) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->node_id = node_id; __entry->server_nr = server_nr; ), - TP_printk("fsid "FSID_FMT" id %llu node_id %llu server_nr %lu", - __entry->fsid, __entry->id, __entry->node_id, + TP_printk(SCSBF" id %llu node_id %llu server_nr %lu", + SCSB_TRACE_ARGS, __entry->id, __entry->node_id, __entry->server_nr) ); @@ -1219,19 +1217,19 @@ TRACE_EVENT(scoutfs_server_compact_response, TP_ARGS(sb, id, error), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, id) __field(int, error) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->error = error; ), - TP_printk("fsid "FSID_FMT" id %llu error %d", - __entry->fsid, __entry->id, __entry->error) + TP_printk(SCSBF" id %llu error %d", + SCSB_TRACE_ARGS, __entry->id, __entry->error) ); TRACE_EVENT(scoutfs_write_begin, @@ -1240,20 +1238,20 @@ TRACE_EVENT(scoutfs_write_begin, TP_ARGS(sb, ino, pos, len), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, inode) __field(__u64, pos) __field(__u32, len) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->inode = ino; __entry->pos = pos; __entry->len = len; ), - TP_printk(FSID_FMT" ino %llu pos %llu len %u", __entry->fsid, + TP_printk(SCSBF" ino %llu pos %llu len %u", SCSB_TRACE_ARGS, __entry->inode, __entry->pos, __entry->len) ); @@ -1264,7 +1262,7 @@ TRACE_EVENT(scoutfs_write_end, TP_ARGS(sb, ino, idx, pos, len, copied), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(unsigned long, idx) __field(__u64, pos) @@ -1273,7 +1271,7 @@ TRACE_EVENT(scoutfs_write_end, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; __entry->idx = idx; __entry->pos = pos; @@ -1281,8 +1279,8 @@ TRACE_EVENT(scoutfs_write_end, __entry->copied = copied; ), - TP_printk(FSID_FMT" ino %llu pgind %lu pos %llu len %u copied %d", - __entry->fsid, __entry->ino, __entry->idx, __entry->pos, + TP_printk(SCSBF" ino %llu pgind %lu pos %llu len %u copied %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->idx, __entry->pos, __entry->len, __entry->copied) ); @@ -1445,7 +1443,7 @@ TRACE_EVENT(scoutfs_read_item_keys, struct scoutfs_key *seg_end), TP_ARGS(sb, key, start, end, seg_start, seg_end), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS sk_trace_define(key) sk_trace_define(start) sk_trace_define(end) @@ -1453,15 +1451,15 @@ TRACE_EVENT(scoutfs_read_item_keys, sk_trace_define(seg_end) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); sk_trace_assign(key, key); sk_trace_assign(start, start); sk_trace_assign(end, end); sk_trace_assign(seg_start, seg_start); sk_trace_assign(seg_end, seg_end); ), - TP_printk("fsid "FSID_FMT" key "SK_FMT" start "SK_FMT" end "SK_FMT" seg_start "SK_FMT" seg_end "SK_FMT"", - __entry->fsid, sk_trace_args(key), sk_trace_args(start), + TP_printk(SCSBF" key "SK_FMT" start "SK_FMT" end "SK_FMT" seg_start "SK_FMT" seg_end "SK_FMT"", + SCSB_TRACE_ARGS, sk_trace_args(key), sk_trace_args(start), sk_trace_args(end), sk_trace_args(seg_start), sk_trace_args(seg_end)) ); @@ -1470,14 +1468,14 @@ DECLARE_EVENT_CLASS(scoutfs_key_class, TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS sk_trace_define(key) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); sk_trace_assign(key, key); ), - TP_printk(FSID_FMT" key "SK_FMT, __entry->fsid, sk_trace_args(key)) + TP_printk(SCSBF" key "SK_FMT, SCSB_TRACE_ARGS, sk_trace_args(key)) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, @@ -1491,16 +1489,16 @@ TRACE_EVENT(scoutfs_item_lookup_ret, TP_ARGS(sb, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->ret = ret; ), - TP_printk(FSID_FMT" ret %d", __entry->fsid, __entry->ret) + TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) ); DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, @@ -1523,17 +1521,17 @@ DECLARE_EVENT_CLASS(scoutfs_range_class, struct scoutfs_key *end), TP_ARGS(sb, start, end), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS sk_trace_define(start) sk_trace_define(end) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); sk_trace_assign(start, start); sk_trace_assign(end, end); ), - TP_printk("fsid "FSID_FMT" start "SK_FMT" end "SK_FMT, - __entry->fsid, sk_trace_args(start), sk_trace_args(end)) + TP_printk(SCSBF" start "SK_FMT" end "SK_FMT, + SCSB_TRACE_ARGS, sk_trace_args(start), sk_trace_args(end)) ); DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, @@ -1553,19 +1551,19 @@ DECLARE_EVENT_CLASS(scoutfs_cached_range_class, struct scoutfs_key *start, struct scoutfs_key *end), TP_ARGS(sb, rng, start, end), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, rng) sk_trace_define(start) sk_trace_define(end) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->rng = rng; sk_trace_assign(start, start); sk_trace_assign(end, end); ), - TP_printk("fsid "FSID_FMT" rng %p start "SK_FMT" end "SK_FMT, - __entry->fsid, __entry->rng, sk_trace_args(start), + TP_printk(SCSBF" rng %p start "SK_FMT" end "SK_FMT, + SCSB_TRACE_ARGS, __entry->rng, sk_trace_args(start), sk_trace_args(end)) ); @@ -1628,7 +1626,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, TP_PROTO(struct super_block *sb, struct scoutfs_lock *lck), TP_ARGS(sb, lck), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS sk_trace_define(start) sk_trace_define(end) __field(u64, refresh_gen) @@ -1643,7 +1641,7 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __field(unsigned int, users_ex) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); sk_trace_assign(start, &lck->start); sk_trace_assign(end, &lck->end); __entry->refresh_gen = lck->refresh_gen; @@ -1657,8 +1655,8 @@ DECLARE_EVENT_CLASS(scoutfs_lock_class, __entry->users_ex = lck->users[SCOUTFS_LOCK_WRITE]; __entry->users_cw = lck->users[SCOUTFS_LOCK_WRITE_ONLY]; ), - TP_printk("fsid "FSID_FMT" start "SK_FMT" end "SK_FMT" mode %u reqpnd %u invpnd %u rfrgen %llu waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", - __entry->fsid, sk_trace_args(start), sk_trace_args(end), + TP_printk(SCSBF" start "SK_FMT" end "SK_FMT" mode %u reqpnd %u invpnd %u rfrgen %llu waiters: pr %u ex %u cw %u users: pr %u ex %u cw %u", + SCSB_TRACE_ARGS, sk_trace_args(start), sk_trace_args(end), __entry->mode, __entry->request_pending, __entry->invalidate_pending, __entry->refresh_gen, __entry->waiters_pr, __entry->waiters_ex, __entry->waiters_cw, @@ -1757,7 +1755,7 @@ TRACE_EVENT(scoutfs_seg_append_item, TP_ARGS(sb, segno, seq, nr_items, total_bytes, key, val_len), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, segno) __field(__u64, seq) __field(__u32, nr_items) @@ -1767,7 +1765,7 @@ TRACE_EVENT(scoutfs_seg_append_item, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->segno = segno; __entry->seq = seq; __entry->nr_items = nr_items; @@ -1776,8 +1774,8 @@ TRACE_EVENT(scoutfs_seg_append_item, __entry->val_len = val_len; ), - TP_printk("fsid "FSID_FMT" segno %llu seq %llu nr_items %u total_bytes %u key "SK_FMT" val_len %u", - __entry->fsid, __entry->segno, __entry->seq, + TP_printk(SCSBF" segno %llu seq %llu nr_items %u total_bytes %u key "SK_FMT" val_len %u", + SCSB_TRACE_ARGS, __entry->segno, __entry->seq, __entry->nr_items, __entry->total_bytes, sk_trace_args(key), __entry->val_len) ); @@ -1787,19 +1785,19 @@ DECLARE_EVENT_CLASS(scoutfs_net_class, struct sockaddr_in *peer, struct scoutfs_net_header *nh), TP_ARGS(sb, name, peer, nh), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS si4_trace_define(name) si4_trace_define(peer) snh_trace_define(nh) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); si4_trace_assign(name, name); si4_trace_assign(peer, peer); snh_trace_assign(nh, nh); ), - TP_printk("fsid "FSID_FMT" name "SI4_FMT" peer "SI4_FMT" nh "SNH_FMT, - __entry->fsid, si4_trace_args(name), si4_trace_args(peer), + TP_printk(SCSBF" name "SI4_FMT" peer "SI4_FMT" nh "SNH_FMT, + SCSB_TRACE_ARGS, si4_trace_args(name), si4_trace_args(peer), snh_trace_args(nh)) ); @@ -1819,17 +1817,17 @@ DECLARE_EVENT_CLASS(scoutfs_work_class, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, data) __field(int, ret) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->data = data; __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" data %llu ret %d", - __entry->fsid, __entry->data, __entry->ret) + TP_printk(SCSBF" data %llu ret %d", + SCSB_TRACE_ARGS, __entry->data, __entry->ret) ); DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), @@ -2058,7 +2056,7 @@ TRACE_EVENT(scoutfs_rename, TP_ARGS(sb, old_dir, old_dentry, new_dir, new_dentry), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, old_dir_ino) __string(old_name, old_dentry->d_name.name) __field(__u64, new_dir_ino) @@ -2067,7 +2065,7 @@ TRACE_EVENT(scoutfs_rename, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->old_dir_ino = scoutfs_ino(old_dir); __assign_str(old_name, old_dentry->d_name.name) __entry->new_dir_ino = scoutfs_ino(new_dir); @@ -2076,8 +2074,8 @@ TRACE_EVENT(scoutfs_rename, scoutfs_ino(new_dentry->d_inode) : 0; ), - TP_printk("fsid "FSID_FMT" old_dir_ino %llu old_name %s new_dir_ino %llu new_name %s new_inode_ino %llu", - __entry->fsid, __entry->old_dir_ino, __get_str(old_name), + TP_printk(SCSBF" old_dir_ino %llu old_name %s new_dir_ino %llu new_name %s new_inode_ino %llu", + SCSB_TRACE_ARGS, __entry->old_dir_ino, __get_str(old_name), __entry->new_dir_ino, __get_str(new_name), __entry->new_inode_ino) ); @@ -2090,7 +2088,7 @@ TRACE_EVENT(scoutfs_d_revalidate, TP_ARGS(sb, dentry, flags, parent, is_covered, ret), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __string(name, dentry->d_name.name) __field(__u64, ino) __field(__u64, parent_ino) @@ -2101,7 +2099,7 @@ TRACE_EVENT(scoutfs_d_revalidate, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __assign_str(name, dentry->d_name.name) __entry->ino = dentry->d_inode ? scoutfs_ino(dentry->d_inode) : 0; @@ -2113,8 +2111,8 @@ TRACE_EVENT(scoutfs_d_revalidate, __entry->ret = ret; ), - TP_printk("fsid "FSID_FMT" name %s ino %llu parent_ino %llu flags 0x%x s_root %u is_covered %u ret %d", - __entry->fsid, __get_str(name), __entry->ino, + TP_printk(SCSBF" name %s ino %llu parent_ino %llu flags 0x%x s_root %u is_covered %u ret %d", + SCSB_TRACE_ARGS, __get_str(name), __entry->ino, __entry->parent_ino, __entry->flags, __entry->is_root, __entry->is_covered, @@ -2125,19 +2123,19 @@ DECLARE_EVENT_CLASS(scoutfs_super_lifecycle_class, TP_PROTO(struct super_block *sb), TP_ARGS(sb), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(void *, sb) __field(void *, sbi) __field(void *, s_root) ), TP_fast_assign( - __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + SCSB_TRACE_ASSIGN(sb); __entry->sb = sb; __entry->sbi = SCOUTFS_SB(sb); __entry->s_root = sb->s_root; ), - TP_printk("fsid "FSID_FMT" sb %p sbi %p s_root %p", - __entry->fsid, __entry->sb, __entry->sbi, __entry->s_root) + TP_printk(SCSBF" sb %p sbi %p s_root %p", + SCSB_TRACE_ARGS, __entry->sb, __entry->sbi, __entry->s_root) ); DEFINE_EVENT(scoutfs_super_lifecycle_class, scoutfs_fill_super, @@ -2159,20 +2157,20 @@ DECLARE_EVENT_CLASS(scoutfs_fileid_class, TP_PROTO(struct super_block *sb, int fh_type, struct scoutfs_fid *fid), TP_ARGS(sb, fh_type, fid), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, fh_type) __field(u64, ino) __field(u64, parent_ino) ), TP_fast_assign( - __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + SCSB_TRACE_ASSIGN(sb); __entry->fh_type = fh_type; __entry->ino = le64_to_cpu(fid->ino); __entry->parent_ino = fh_type == FILEID_SCOUTFS_WITH_PARENT ? le64_to_cpu(fid->parent_ino) : 0ULL; ), - TP_printk("fsid "FSID_FMT" type %d ino %llu parent %llu", - __entry->fsid, __entry->fh_type, __entry->ino, + TP_printk(SCSBF" type %d ino %llu parent %llu", + SCSB_TRACE_ARGS, __entry->fh_type, __entry->ino, __entry->parent_ino) ); @@ -2197,19 +2195,19 @@ TRACE_EVENT(scoutfs_get_parent, TP_ARGS(sb, inode, parent), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, parent) ), TP_fast_assign( - __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + SCSB_TRACE_ASSIGN(sb); __entry->ino = scoutfs_ino(inode); __entry->parent = parent; ), - TP_printk("fsid "FSID_FMT" child %llu parent %llu", - __entry->fsid, __entry->ino, __entry->parent) + TP_printk(SCSBF" child %llu parent %llu", + SCSB_TRACE_ARGS, __entry->ino, __entry->parent) ); TRACE_EVENT(scoutfs_get_name, @@ -2219,21 +2217,21 @@ TRACE_EVENT(scoutfs_get_name, TP_ARGS(sb, parent, child, name), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, parent_ino) __field(__u64, child_ino) __string(name, name) ), TP_fast_assign( - __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + SCSB_TRACE_ASSIGN(sb); __entry->parent_ino = scoutfs_ino(parent); __entry->child_ino = scoutfs_ino(child); __assign_str(name, name); ), - TP_printk("fsid "FSID_FMT" parent %llu child %llu name: %s", - __entry->fsid, __entry->parent_ino, __entry->child_ino, + TP_printk(SCSBF" parent %llu child %llu name: %s", + SCSB_TRACE_ARGS, __entry->parent_ino, __entry->child_ino, __get_str(name)) ); @@ -2243,19 +2241,19 @@ TRACE_EVENT(scoutfs_btree_read_error, TP_ARGS(sb, ref), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, blkno) __field(__u64, seq) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->blkno = le64_to_cpu(ref->blkno); __entry->seq = le64_to_cpu(ref->seq); ), - TP_printk("fsid "FSID_FMT" blkno %llu seq %llu", - __entry->fsid, __entry->blkno, __entry->seq) + TP_printk(SCSBF" blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->blkno, __entry->seq) ); TRACE_EVENT(scoutfs_btree_dirty_block, @@ -2267,7 +2265,7 @@ TRACE_EVENT(scoutfs_btree_dirty_block, bt_blkno, bt_seq), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, blkno) __field(__u64, seq) __field(__u64, next_block) @@ -2279,7 +2277,7 @@ TRACE_EVENT(scoutfs_btree_dirty_block, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->blkno = blkno; __entry->seq = seq; __entry->next_block = next_block; @@ -2290,8 +2288,8 @@ TRACE_EVENT(scoutfs_btree_dirty_block, __entry->bt_seq = bt_seq; ), - TP_printk("fsid "FSID_FMT" blkno %llu seq %llu next_block %llu next_seq %llu cur_dirtied %lu old_dirtied %lu bt_blkno %llu bt_seq %llu", - __entry->fsid, __entry->blkno, __entry->seq, + TP_printk(SCSBF" blkno %llu seq %llu next_block %llu next_seq %llu cur_dirtied %lu old_dirtied %lu bt_blkno %llu bt_seq %llu", + SCSB_TRACE_ARGS, __entry->blkno, __entry->seq, __entry->next_block, __entry->next_seq, __entry->cur_dirtied, __entry->old_dirtied, __entry->bt_blkno, __entry->bt_seq) ); @@ -2302,17 +2300,17 @@ DECLARE_EVENT_CLASS(scoutfs_extent_class, TP_ARGS(sb, ext), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS se_trace_define(ext) ), TP_fast_assign( - __entry->fsid = SCOUTFS_SB(sb) ? FSID_ARG(sb) : 0; + SCSB_TRACE_ASSIGN(sb); se_trace_assign(ext, ext); ), - TP_printk("fsid "FSID_FMT" ext "SE_FMT, - __entry->fsid, se_trace_args(ext)) + TP_printk(SCSBF" ext "SE_FMT, + SCSB_TRACE_ARGS, se_trace_args(ext)) ); DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_insert, @@ -2420,7 +2418,7 @@ TRACE_EVENT(scoutfs_online_offline_blocks, TP_ARGS(inode, on_delta, off_delta, on_now, off_now), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__s64, on_delta) __field(__s64, off_delta) __field(__u64, on_now) @@ -2428,15 +2426,15 @@ TRACE_EVENT(scoutfs_online_offline_blocks, ), TP_fast_assign( - __entry->fsid = FSID_ARG(inode->i_sb); + SCSB_TRACE_ASSIGN(inode->i_sb); __entry->on_delta = on_delta; __entry->off_delta = off_delta; __entry->on_now = on_now; __entry->off_now = off_now; ), - TP_printk("fsid "FSID_FMT" on_delta %lld off_delta %lld on_now %llu off_now %llu ", - __entry->fsid, __entry->on_delta, __entry->off_delta, + TP_printk(SCSBF" on_delta %lld off_delta %lld on_now %llu off_now %llu ", + SCSB_TRACE_ARGS, __entry->on_delta, __entry->off_delta, __entry->on_now, __entry->off_now) ); @@ -2446,17 +2444,17 @@ DECLARE_EVENT_CLASS(scoutfs_segno_class, TP_ARGS(sb, segno), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__s64, segno) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->segno = segno; ), - TP_printk("fsid "FSID_FMT" segno %llu", - __entry->fsid, __entry->segno) + TP_printk(SCSBF" segno %llu", + SCSB_TRACE_ARGS, __entry->segno) ); DEFINE_EVENT(scoutfs_segno_class, scoutfs_alloc_segno, TP_PROTO(struct super_block *sb, u64 segno), @@ -2477,19 +2475,19 @@ DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, TP_ARGS(sb, node_id, nr_clients), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__s64, node_id) __field(unsigned long, nr_clients) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->node_id = node_id; __entry->nr_clients = nr_clients; ), - TP_printk("fsid "FSID_FMT" node_id %llu nr_clients %lu", - __entry->fsid, __entry->node_id, __entry->nr_clients) + TP_printk(SCSBF" node_id %llu nr_clients %lu", + SCSB_TRACE_ARGS, __entry->node_id, __entry->nr_clients) ); DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up, TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), @@ -2516,7 +2514,7 @@ TRACE_EVENT(scoutfs_lock_message, TP_ARGS(sb, who, what, dir, node_id, net_id, nl), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(int, who) __field(int, what) __field(int, dir) @@ -2528,7 +2526,7 @@ TRACE_EVENT(scoutfs_lock_message, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->who = who; __entry->what = what; __entry->dir = dir; @@ -2539,8 +2537,8 @@ TRACE_EVENT(scoutfs_lock_message, __entry->new_mode = nl->new_mode; ), - TP_printk("fsid "FSID_FMT" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u", - __entry->fsid, slt_symbolic(__entry->who), + TP_printk(SCSBF" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u", + SCSB_TRACE_ARGS, slt_symbolic(__entry->who), slt_symbolic(__entry->what), slt_symbolic(__entry->dir), __entry->node_id, __entry->net_id, sk_trace_args(key), __entry->old_mode, __entry->new_mode) @@ -2553,7 +2551,7 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, TP_ARGS(sb, io_blkno, blk), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, io_blkno) __field(__u64, hdr_blkno) __field(__u64, config_gen) @@ -2566,7 +2564,7 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->io_blkno = io_blkno; __entry->hdr_blkno = le64_to_cpu(blk->blkno); __entry->config_gen = le64_to_cpu(blk->config_gen); @@ -2578,8 +2576,8 @@ DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, __entry->flags = blk->flags; ), - TP_printk("fsid "FSID_FMT" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", - __entry->fsid, __entry->io_blkno, __entry->hdr_blkno, + TP_printk(SCSBF" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", + SCSB_TRACE_ARGS, __entry->io_blkno, __entry->hdr_blkno, __entry->config_gen, __entry->write_nr, __entry->elected_nr, __entry->unmount_barrier, __entry->crc, __entry->vote_slot, __entry->flags) @@ -2628,21 +2626,21 @@ TRACE_EVENT(scoutfs_trans_seq_advance, TP_ARGS(sb, node_id, prev_seq, next_seq), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, node_id) __field(__u64, prev_seq) __field(__u64, next_seq) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->node_id = node_id; __entry->prev_seq = prev_seq; __entry->next_seq = next_seq; ), - TP_printk("fsid "FSID_FMT" node_id %llu prev_seq %llu next_seq %llu", - __entry->fsid, __entry->node_id, __entry->prev_seq, + TP_printk(SCSBF" node_id %llu prev_seq %llu next_seq %llu", + SCSB_TRACE_ARGS, __entry->node_id, __entry->prev_seq, __entry->next_seq) ); @@ -2652,19 +2650,19 @@ TRACE_EVENT(scoutfs_trans_seq_farewell, TP_ARGS(sb, node_id, trans_seq), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, node_id) __field(__u64, trans_seq) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->node_id = node_id; __entry->trans_seq = trans_seq; ), - TP_printk("fsid "FSID_FMT" node_id %llu trans_seq %llu", - __entry->fsid, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" node_id %llu trans_seq %llu", + SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) ); TRACE_EVENT(scoutfs_trans_seq_last, @@ -2673,19 +2671,19 @@ TRACE_EVENT(scoutfs_trans_seq_last, TP_ARGS(sb, node_id, trans_seq), TP_STRUCT__entry( - __field(__u64, fsid) + SCSB_TRACE_FIELDS __field(__u64, node_id) __field(__u64, trans_seq) ), TP_fast_assign( - __entry->fsid = FSID_ARG(sb); + SCSB_TRACE_ASSIGN(sb); __entry->node_id = node_id; __entry->trans_seq = trans_seq; ), - TP_printk("fsid "FSID_FMT" node_id %llu trans_seq %llu", - __entry->fsid, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" node_id %llu trans_seq %llu", + SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) ); #endif /* _TRACE_SCOUTFS_H */ From bd7a7fe97ea8b645d5bf9b7d96a3129f841e98e4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Jun 2019 11:19:59 -0700 Subject: [PATCH 727/920] scoutfs: use fr identity in pseudo fs paths Use the fr mount identity string in the sysfs/fs/ and debugfs paths we register for each mount. Signed-off-by: Zach Brown --- kmod/src/super.c | 16 ++++++++-------- kmod/src/sysfs.c | 3 +-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 0f644c4f..ea282ce8 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -325,16 +325,16 @@ out: return ret; } +/* + * This needs to be setup after reading the super because it uses the + * fsid found in the super block. + */ static int scoutfs_debugfs_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); char name[32]; - /* - * XXX: Move the name variable to sbi and use it in - * init_lock_info as well. - */ - snprintf(name, 32, "%llx", le64_to_cpu(sbi->super.hdr.fsid)); + snprintf(name, ARRAY_SIZE(name), SCSBF, SCSB_ARGS(sb)); sbi->debug_root = debugfs_create_dir(name, scoutfs_debugfs_root); if (!sbi->debug_root) @@ -410,10 +410,10 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) goto out; } - ret = scoutfs_setup_sysfs(sb) ?: - scoutfs_setup_counters(sb) ?: - scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: + ret = scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: scoutfs_debugfs_setup(sb) ?: + scoutfs_setup_sysfs(sb) ?: + scoutfs_setup_counters(sb) ?: scoutfs_options_setup(sb) ?: scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, mount_options_attrs, "mount_options") ?: diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index 7848e039..2c056c5c 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -204,10 +204,9 @@ int scoutfs_setup_sysfs(struct super_block *sb) sfsinfo->sb = sb; sbi->sfsinfo = sfsinfo; - /* XXX can have multiple mounts of a device, need mount id */ init_completion(&sfsinfo->sb_id_comp); ret = kobject_init_and_add(&sfsinfo->sb_id_kobj, &sb_id_ktype, - &scoutfs_kset->kobj, "%s", sb->s_id); + &scoutfs_kset->kobj, SCSBF, SCSB_ARGS(sb)); if (ret) kfree(sfsinfo); From 6f5cfd8cc23ef012a98f5d9eedb2a1cae04e74ae Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Jun 2019 11:39:17 -0700 Subject: [PATCH 728/920] scoutfs: use rid instead of node_id in items Use the mount's generated random id in persistent items and the lock that protects them instead of the assigned node_id. Signed-off-by: Zach Brown --- kmod/src/data.c | 46 +++++++++++++++++++++++----------------------- kmod/src/format.h | 12 ++++++------ kmod/src/inode.c | 20 ++++++++++---------- kmod/src/lock.c | 20 ++++++++++---------- kmod/src/lock.h | 4 ++-- kmod/src/super.c | 8 ++++---- kmod/src/super.h | 2 +- 7 files changed, 56 insertions(+), 56 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index f0b284ae..4402c7aa 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -63,8 +63,8 @@ /* * The largest extent that we'll store in a single item. This will - * determine the granularity of interleaved concurrent allocations on a - * node. Sequential max length allocations could still see contiguous + * determine the granularity of interleaved concurrent allocations in a + * mount. Sequential max length allocations could still see contiguous * physical extent allocations. It limits the amount of IO needed to * invalidate a lock. And it determines the granularity of parallel * writes to a file between nodes. @@ -100,12 +100,12 @@ static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) }; } -static void init_free_extent_key(struct scoutfs_key *key, u8 type, u64 node_id, +static void init_free_extent_key(struct scoutfs_key *key, u8 type, u64 rid, u64 major, u64 minor) { *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_NODE_ZONE, - .sknf_node_id = cpu_to_le64(node_id), + .sk_zone = SCOUTFS_RID_ZONE, + .sknf_rid = cpu_to_le64(rid), .sk_type = type, .sknf_major = cpu_to_le64(major), .sknf_minor = cpu_to_le64(minor), @@ -135,7 +135,7 @@ static int init_extent_from_item(struct scoutfs_extent *ext, flags = fex->flags; } else { - owner = le64_to_cpu(key->sknf_node_id); + owner = le64_to_cpu(key->sknf_rid); start = le64_to_cpu(key->sknf_major); len = le64_to_cpu(key->sknf_minor); if (key->sk_type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) @@ -159,7 +159,7 @@ static int init_extent_from_item(struct scoutfs_extent *ext, * keeping their _BLOCKS_ item in sync with the primary _BLKNO_ item * that callers operate on. * - * The count of free blocks stored in node items is kept consistent by + * The count of free blocks stored in items is kept consistent by * updating the count every time we create or delete items. Updated * extents are deleted and then recreated so the count can bounce around * a bit, but it's OK for it to be imprecise at the margins. @@ -315,9 +315,9 @@ static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, /* free an allocated mapping */ if (rem.map) { scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, rem.map, rem.len, 0, 0); + sbi->rid, rem.map, rem.len, 0, 0); ret = scoutfs_extent_add(sb, data_extent_io, &fr, - sbi->node_id_lock); + sbi->rid_lock); if (ret) goto out; rem_fr = true; @@ -360,7 +360,7 @@ out: SC_DATA_EXTENT_TRUNC_CLEANUP, corrupt_data_extent_trunc_cleanup, &rem); scoutfs_extent_cleanup(ret < 0 && rem_fr, scoutfs_extent_remove, sb, - data_extent_io, &fr, sbi->node_id_lock, + data_extent_io, &fr, sbi->rid_lock, SC_DATA_EXTENT_TRUNC_CLEANUP, corrupt_data_extent_trunc_cleanup, &rem); @@ -457,9 +457,9 @@ static int get_server_extent(struct super_block *sb) goto out; scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, start, len, 0, 0); + sbi->rid, start, len, 0, 0); trace_scoutfs_data_get_server_extent(sb, &ext); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, sbi->node_id_lock); + ret = scoutfs_extent_add(sb, data_extent_io, &ext, sbi->rid_lock); /* XXX don't free extent on error, crash recovery with server */ out: @@ -488,14 +488,14 @@ static int find_free_extent(struct super_block *sb, u64 len, for (;;) { /* first try to find the first sufficient extent */ scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, len, 0, 0); + sbi->rid, 0, len, 0, 0); ret = scoutfs_extent_next(sb, data_extent_io, ext, - sbi->node_id_lock); + sbi->rid_lock); /* if none big enough, look for last largest smaller */ if (ret == -ENOENT && len > 1) ret = scoutfs_extent_prev(sb, data_extent_io, ext, - sbi->node_id_lock); + sbi->rid_lock); /* ask the server for more if we think it'll help */ if (ret == -ENOENT || ext->len < len) { @@ -510,7 +510,7 @@ static int find_free_extent(struct super_block *sb, u64 len, if (ret == 0) scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->node_id, ext->start, + sbi->rid, ext->start, min(ext->len, len), 0, 0); trace_scoutfs_data_find_free_extent(sb, ext); @@ -587,7 +587,7 @@ static int alloc_block(struct super_block *sb, struct inode *inode, iblock, 1, fr.start, 0); /* remove the free extent that we're allocating */ - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); + ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); if (ret) goto out; add_fr = true; @@ -631,7 +631,7 @@ out: SC_DATA_EXTENT_ALLOC_CLEANUP, corrupt_data_extent_alloc_cleanup, &blk); scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->node_id_lock, + data_extent_io, &fr, sbi->rid_lock, SC_DATA_EXTENT_ALLOC_CLEANUP, corrupt_data_extent_alloc_cleanup, &blk); @@ -1069,7 +1069,7 @@ static s64 fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, if (WARN_ON_ONCE(ret)) goto out; - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->node_id_lock); + ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); if (ret) goto out; add_fr = true; @@ -1093,7 +1093,7 @@ out: SC_DATA_EXTENT_FALLOCATE_CLEANUP, corrupt_data_extent_fallocate_cleanup, &fal); scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->node_id_lock, + data_extent_io, &fr, sbi->rid_lock, SC_DATA_EXTENT_FALLOCATE_CLEANUP, corrupt_data_extent_alloc_cleanup, &fal); return ret; @@ -1649,9 +1649,9 @@ static void scoutfs_data_return_server_extents_worker(struct work_struct *work) free > NODE_FREE_HIGH_WATER_BLOCKS) { scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->node_id, 0, 1, 0, 0); + sbi->rid, 0, 1, 0, 0); ret = scoutfs_extent_next(sb, data_extent_io, &ext, - sbi->node_id_lock); + sbi->rid_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1664,7 +1664,7 @@ static void scoutfs_data_return_server_extents_worker(struct work_struct *work) ext.len = min(ext.len, free - NODE_FREE_HIGH_WATER_BLOCKS); ret = scoutfs_extent_remove(sb, data_extent_io, &ext, - sbi->node_id_lock); + sbi->rid_lock); if (ret) break; diff --git a/kmod/src/format.h b/kmod/src/format.h index 19990161..68396c19 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -114,12 +114,12 @@ struct scoutfs_key { #define skxi_id _sk_third /* node free extent */ -#define sknf_node_id _sk_first +#define sknf_rid _sk_first #define sknf_major _sk_second #define sknf_minor _sk_third /* node orphan inode */ -#define sko_node_id _sk_first +#define sko_rid _sk_first #define sko_ino _sk_second /* inode */ @@ -357,7 +357,7 @@ struct scoutfs_segment_block { */ #define SCOUTFS_INODE_INDEX_ZONE 1 #define SCOUTFS_XATTR_INDEX_ZONE 2 -#define SCOUTFS_NODE_ZONE 3 +#define SCOUTFS_RID_ZONE 3 #define SCOUTFS_FS_ZONE 4 #define SCOUTFS_LOCK_ZONE 5 #define SCOUTFS_MAX_ZONE 8 /* power of 2 is efficient */ @@ -370,9 +370,10 @@ struct scoutfs_segment_block { /* xattr index zone */ #define SCOUTFS_XATTR_INDEX_NAME_TYPE 1 -/* node zone (also used in server alloc btree) */ +/* rid zone (also used in server alloc btree) */ #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 #define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 +#define SCOUTFS_ORPHAN_TYPE 3 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -382,12 +383,11 @@ struct scoutfs_segment_block { #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 #define SCOUTFS_FILE_EXTENT_TYPE 7 -#define SCOUTFS_ORPHAN_TYPE 8 /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 -#define SCOUTFS_MAX_TYPE 16 /* power of 2 is efficient */ +#define SCOUTFS_MAX_TYPE 8 /* power of 2 is efficient */ /* * File extents have more data than easily fits in the key so we move diff --git a/kmod/src/inode.c b/kmod/src/inode.c index afcd67aa..6fb70014 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -1410,11 +1410,11 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, return inode; } -static void init_orphan_key(struct scoutfs_key *key, u64 node_id, u64 ino) +static void init_orphan_key(struct scoutfs_key *key, u64 rid, u64 ino) { *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_NODE_ZONE, - .sko_node_id = cpu_to_le64(node_id), + .sk_zone = SCOUTFS_RID_ZONE, + .sko_rid = cpu_to_le64(rid), .sk_type = SCOUTFS_ORPHAN_TYPE, .sko_ino = cpu_to_le64(ino), }; @@ -1423,11 +1423,11 @@ static void init_orphan_key(struct scoutfs_key *key, u64 node_id, u64 ino) static int remove_orphan_item(struct super_block *sb, u64 ino) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; + struct scoutfs_lock *lock = sbi->rid_lock; struct scoutfs_key key; int ret; - init_orphan_key(&key, sbi->node_id, ino); + init_orphan_key(&key, sbi->rid, ino); ret = scoutfs_item_delete(sb, &key, lock); if (ret == -ENOENT) @@ -1574,7 +1574,7 @@ int scoutfs_drop_inode(struct inode *inode) int scoutfs_scan_orphans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; + struct scoutfs_lock *lock = sbi->rid_lock; struct scoutfs_key key; struct scoutfs_key last; int err = 0; @@ -1582,8 +1582,8 @@ int scoutfs_scan_orphans(struct super_block *sb) trace_scoutfs_scan_orphans(sb); - init_orphan_key(&key, sbi->node_id, 0); - init_orphan_key(&last, sbi->node_id, ~0ULL); + init_orphan_key(&key, sbi->rid, 0); + init_orphan_key(&last, sbi->rid, ~0ULL); while (1) { ret = scoutfs_item_next(sb, &key, &last, NULL, lock); @@ -1612,13 +1612,13 @@ int scoutfs_orphan_inode(struct inode *inode) { struct super_block *sb = inode->i_sb; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_lock *lock = sbi->node_id_lock; + struct scoutfs_lock *lock = sbi->rid_lock; struct scoutfs_key key; int ret; trace_scoutfs_orphan_inode(sb, inode); - init_orphan_key(&key, sbi->node_id, scoutfs_ino(inode)); + init_orphan_key(&key, sbi->rid, scoutfs_ino(inode)); ret = scoutfs_item_create(sb, &key, NULL, lock); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 97fae82d..9536376a 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1177,29 +1177,29 @@ int scoutfs_lock_xattr_index(struct super_block *sb, int mode, int flags, } /* - * The node_id lock protects a mount's private persistent items in the - * node_id zone. It's held for the duration of the mount. It lets the - * mount modify the node_id items at will and signals to other mounts - * that we're still alive and our node_id items shouldn't be reclaimed. + * The rid lock protects a mount's private persistent items in the rid + * zone. It's held for the duration of the mount. It lets the mount + * modify the rid items at will and signals to other mounts that we're + * still alive and our rid items shouldn't be reclaimed. * * Being held for the entire mount prevents other nodes from reclaiming * our items, like free blocks, when it would make sense for them to be * able to. Maybe we have a bunch free and they're trying to allocate * and are getting ENOSPC. */ -int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, - u64 node_id, struct scoutfs_lock **lock) +int scoutfs_lock_rid(struct super_block *sb, int mode, int flags, + u64 rid, struct scoutfs_lock **lock) { struct scoutfs_key start; struct scoutfs_key end; scoutfs_key_set_zeros(&start); - start.sk_zone = SCOUTFS_NODE_ZONE; - start.sko_node_id = cpu_to_le64(node_id); + start.sk_zone = SCOUTFS_RID_ZONE; + start.sko_rid = cpu_to_le64(rid); scoutfs_key_set_ones(&end); - end.sk_zone = SCOUTFS_NODE_ZONE; - end.sko_node_id = cpu_to_le64(node_id); + end.sk_zone = SCOUTFS_RID_ZONE; + end.sko_rid = cpu_to_le64(rid); return lock_key_range(sb, mode, flags, &start, &end, lock); } diff --git a/kmod/src/lock.h b/kmod/src/lock.h index da24ee8f..febd5ff5 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -70,8 +70,8 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *d, struct scoutfs_lock **D_lock); int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, struct scoutfs_lock **lock); -int scoutfs_lock_node_id(struct super_block *sb, int mode, int flags, - u64 node_id, struct scoutfs_lock **lock); +int scoutfs_lock_rid(struct super_block *sb, int mode, int flags, + u64 rid, struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int level); diff --git a/kmod/src/super.c b/kmod/src/super.c index ea282ce8..cc6b37c3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -182,8 +182,8 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_data_destroy(sb); - scoutfs_unlock(sb, sbi->node_id_lock, SCOUTFS_LOCK_WRITE); - sbi->node_id_lock = NULL; + scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); + sbi->rid_lock = NULL; scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); @@ -429,8 +429,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: scoutfs_client_wait_node_id(sb) ?: - scoutfs_lock_node_id(sb, SCOUTFS_LOCK_WRITE, 0, sbi->node_id, - &sbi->node_id_lock); + scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, + &sbi->rid_lock); if (ret) goto out; diff --git a/kmod/src/super.h b/kmod/src/super.h index 6cd9bb85..08004731 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -33,7 +33,7 @@ struct scoutfs_sb_info { /* assigned once at the start of each mount, read-only */ u64 rid; u64 node_id; - struct scoutfs_lock *node_id_lock; + struct scoutfs_lock *rid_lock; struct scoutfs_super_block super; From 1ea75a9d5427da3bc7cbb0410567e0469cdea2df Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jul 2019 16:19:23 -0700 Subject: [PATCH 729/920] scoutfs: add scoutfs_addr sin conversion functions Add some quick functions that let us convert between our persistent packed inet addr struct and native sockaddr_in structs. Signed-off-by: Zach Brown --- kmod/src/net.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kmod/src/net.h b/kmod/src/net.h index c144c69e..a15e005a 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -2,10 +2,26 @@ #define _SCOUTFS_NET_H_ #include +#include "endian_swap.h" #define SIN_FMT "%pIS:%u" #define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) +static inline void scoutfs_addr_to_sin(struct sockaddr_in *sin, + struct scoutfs_inet_addr *addr) +{ + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); + sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); +} + +static inline void scoutfs_addr_from_sin(struct scoutfs_inet_addr *addr, + struct sockaddr_in *sin) +{ + addr->addr = be32_to_le32(sin->sin_addr.s_addr); + addr->port = be16_to_le16(sin->sin_port); +} + struct scoutfs_net_connection; /* These are called in their own blocking context */ From 532256271cf3e693ce2963c3b6398310d0a917b6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jul 2019 16:28:54 -0700 Subject: [PATCH 730/920] scoutfs: simplify scoutfs_write_super() The pattern of advancing and writing a "dirty super" comes from the time when the format had two persistent super blocks. One was kept in memory and modified as changes were made. Advancing it changed which of the two supers would be eventually written. This no longer makes sense now that we only have one super block. Remove the idea of advancing and writing an implicit dirty super block that's stored in the super block info. Instead use a single scoutfs_write_super() which takes the super block struct to write. We still store and increment the hdr.gen in the super block. It used to be used to tell which of the two super blocks are more recent, now it is just some information that can tell us something about the life of the super block. Signed-off-by: Zach Brown --- kmod/src/server.c | 5 ++--- kmod/src/super.c | 28 ++++++++-------------------- kmod/src/super.h | 4 ++-- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index fc07a1a5..d077d858 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -584,6 +584,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) struct server_info *server = container_of(work, struct server_info, commit_work); struct super_block *sb = server->sb; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct commit_waiter *cw; struct commit_waiter *pos; struct llist_node *node; @@ -611,7 +612,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - ret = scoutfs_write_dirty_super(sb); + ret = scoutfs_write_super(sb, super); if (ret) { scoutfs_err(sb, "server error writing super block: %d", ret); goto out; @@ -623,7 +624,6 @@ static void scoutfs_server_commit_func(struct work_struct *work) server->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; write_seqcount_end(&server->stable_seqcount); - scoutfs_advance_dirty_super(sb); ret = 0; out: @@ -2325,7 +2325,6 @@ static void scoutfs_server_worker(struct work_struct *work) complete(&server->start_comp); - scoutfs_advance_dirty_super(sb); server->stable_manifest_root = super->manifest.root; scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); diff --git a/kmod/src/super.c b/kmod/src/super.c index cc6b37c3..b4dcf658 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -221,28 +221,15 @@ static const struct super_operations scoutfs_super_ops = { }; /* - * The caller advances the sequence number in the super block header - * every time it wants to dirty it and eventually write it to reference - * dirty data that's been written. - */ -void scoutfs_advance_dirty_super(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - - le64_add_cpu(&super->hdr.seq, 1); - trace_scoutfs_advance_dirty_super(sb, le64_to_cpu(super->hdr.seq)); -} - -/* - * The caller is responsible for setting the super header's blkno - * and seq to something reasonable. + * Write the caller's super. The caller has always read a valid super + * before modifying and writing it. The caller's super is modified + * to reflect the write. * * XXX it'd be pretty easy to preallocate to avoid failure here. */ -int scoutfs_write_dirty_super(struct super_block *sb) +int scoutfs_write_super(struct super_block *sb, + struct scoutfs_super_block *caller) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super; struct page *page; int ret; @@ -251,9 +238,10 @@ int scoutfs_write_dirty_super(struct super_block *sb) if (!page) return -ENOMEM; + le64_add_cpu(&caller->hdr.seq, 1); + super = page_address(page); - memcpy(super, &sbi->super, sizeof(*super)); - super->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER); + memcpy(super, caller, sizeof(*super)); super->hdr.crc = scoutfs_block_calc_crc(&super->hdr); ret = scoutfs_bio_write(sb, &page, le64_to_cpu(super->hdr.blkno), 1); diff --git a/kmod/src/super.h b/kmod/src/super.h index 08004731..741f2ddc 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -127,8 +127,8 @@ static inline bool SCOUTFS_HAS_SBI(struct super_block *sb) int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super_res); -void scoutfs_advance_dirty_super(struct super_block *sb); -int scoutfs_write_dirty_super(struct super_block *sb); +int scoutfs_write_super(struct super_block *sb, + struct scoutfs_super_block *super); /* to keep this out of the ioctl.h public interface definition */ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); From 5929a3674738771c57c473db6c6ceae90bb1b013 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jul 2019 16:41:51 -0700 Subject: [PATCH 731/920] scoutfs: add server_addr mount option Add a server_addr mount option that takes an ipv4 address. This will be used by the upcoming changes to quorum voting to indicate that a mount should participate in voting and to specify the address that its server should listen on. Signed-off-by: Zach Brown --- kmod/src/options.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/options.h | 3 +++ kmod/src/super.c | 13 ++++++++++++ 3 files changed, 66 insertions(+) diff --git a/kmod/src/options.c b/kmod/src/options.c index 5c6ee18c..362a5792 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -27,6 +27,7 @@ #include "super.h" static const match_table_t tokens = { + {Opt_server_addr, "server_addr=%s"}, {Opt_uniq_name, "uniq_name=%s"}, {Opt_err, NULL} }; @@ -50,12 +51,54 @@ u32 scoutfs_option_u32(struct super_block *sb, int token) return 0; } +/* The caller's string is null terminted and can be clobbered */ +static int parse_ipv4(struct super_block *sb, char *str, + struct sockaddr_in *sin) +{ + unsigned long port = 0; + __be32 addr; + char *c; + int ret; + + /* null term port, if specified */ + c = strchr(str, ':'); + if (c) + *c = '\0'; + + /* parse addr */ + addr = in_aton(str); + if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) || + ipv4_is_zeronet(addr) || + ipv4_is_local_multicast(addr)) { + scoutfs_err(sb, "invalid unicast ipv4 address: %s", str); + return -EINVAL; + } + + /* parse port, if specified */ + if (c) { + c++; + ret = kstrtoul(c, 0, &port); + if (ret != 0 || port == 0 || port >= U16_MAX) { + scoutfs_err(sb, "invalid port in ipv4 address: %s", c); + return -EINVAL; + } + } + + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = addr; + sin->sin_port = cpu_to_be16(port); + + return 0; +} + int scoutfs_parse_options(struct super_block *sb, char *options, struct mount_options *parsed) { + char ipstr[INET_ADDRSTRLEN + 1]; substring_t args[MAX_OPT_ARGS]; int token, len; char *p; + int ret; /* Set defaults */ memset(parsed, 0, sizeof(*parsed)); @@ -66,6 +109,13 @@ int scoutfs_parse_options(struct super_block *sb, char *options, token = match_token(p, tokens, args); switch (token) { + case Opt_server_addr: + + match_strlcpy(ipstr, args, ARRAY_SIZE(ipstr)); + ret = parse_ipv4(sb, ipstr, &parsed->server_addr); + if (ret < 0) + return ret; + break; case Opt_uniq_name: len = match_strlcpy(parsed->uniq_name, args, SCOUTFS_UNIQUE_NAME_MAX_BYTES); diff --git a/kmod/src/options.h b/kmod/src/options.h index a26df0e9..74ae5fa4 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -2,6 +2,7 @@ #define _SCOUTFS_OPTIONS_H_ #include +#include #include "format.h" enum { @@ -10,11 +11,13 @@ enum { * the number of items in each block as though the blocks were tiny. */ Opt_btree_force_tiny_blocks, + Opt_server_addr, Opt_uniq_name, Opt_err, }; struct mount_options { + struct sockaddr_in server_addr; char uniq_name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; }; diff --git a/kmod/src/super.c b/kmod/src/super.c index b4dcf658..e65f0c2a 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -138,11 +138,23 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) struct super_block *sb = root->d_sb; struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + seq_printf(seq, ",server_addr="SIN_FMT, SIN_ARG(&opts->server_addr)); seq_printf(seq, ",uniq_name=%s", opts->uniq_name); return 0; } +static ssize_t server_addr_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + return snprintf(buf, PAGE_SIZE, SIN_FMT"\n", + SIN_ARG(&opts->server_addr)); +} +SCOUTFS_ATTR_RO(server_addr); + static ssize_t uniq_name_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -154,6 +166,7 @@ static ssize_t uniq_name_show(struct kobject *kobj, SCOUTFS_ATTR_RO(uniq_name); static struct attribute *mount_options_attrs[] = { + SCOUTFS_ATTR_PTR(server_addr), SCOUTFS_ATTR_PTR(uniq_name), NULL, }; From 5b258cee3b6d94b6ffaf6b856635b857e333accc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jul 2019 17:04:09 -0700 Subject: [PATCH 732/920] scoutfs: refine quorum voting The current quorum voting implementatoin had some rough edges that increased the complexity of the system and introduced undesirable failure modes. We can keep the same basic pattern but move functionality around a few places, and rethink the quorum voting, to end up with a meaningfully simpler system. The motivation for this work was to remove the need to provide a uniq_name option for every mount instance. The first big change is to remove the idea of static configuration slots for mounts. This removes the use of uniq_name. Mounts now simply have a server_addr mount option instead of using their uniq_name to find their address in the configuration. The server can't check the configuration to see if a given connected client's name is found in the quorum config. Clients can set a flag in their sent greeting which indicates that they're a voter. This removes the uniq_name from the greeting and mounted client records. Without a static configuration mounts no longer have dedicated block locations to write to. We increase the size of the region of quorum blocks and have voters simply write to a random block. Overwriting vote blocks is OK because we move from heartbeating design patterns to a protocol strongly based on raft's election. We're using quorum blocks to communicate votes instead of network messages and overwriting blocks is analagous to lossy networks droping vote messages in the raft election protocol. We were using the dedicated per-mount quorum blocks to track mounts that had been elected and needed to be fenced. We no longer have that storage so instead we add the idea of an election log that is stored in every voting block. Readers merge the logs from all the blocks they read and write the resulting merged log in their block. With no static quorum configuration we no longer have to worry about the complexity of changing the slot configurations while they're in use. The only persistent configuration is the number of votes a candidate needs to be elected by a quorum. It was a mistake to use quorum voting blocks to communicate state between the server and the quorum voters. We can easily move the unmount_barrier, server address, and fencing state from the quorum blocks into the super block. The server no longer needs the quorum election info struct to be able to later write its quorum block. It instead writes a few fields in the super. There's only one place where clients need to look to find out who they should connect to or if they can finish unmount. Signed-off-by: Zach Brown --- kmod/src/client.c | 158 +++-- kmod/src/counters.h | 15 +- kmod/src/format.h | 113 ++-- kmod/src/quorum.c | 1269 +++++++++++++++++--------------------- kmod/src/quorum.h | 30 +- kmod/src/scoutfs_trace.h | 131 +++- kmod/src/server.c | 136 ++-- kmod/src/server.h | 5 +- kmod/src/super.c | 14 +- 9 files changed, 874 insertions(+), 997 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 8d3bcee6..8ef58589 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -53,10 +53,7 @@ struct client_info { atomic_t shutting_down; struct workqueue_struct *workq; - struct work_struct connect_work; - - struct scoutfs_quorum_elected_info qei; - u64 old_elected_nr; + struct delayed_work connect_dwork; u64 server_term; u64 greeting_umb; @@ -373,117 +370,108 @@ out: } /* - * If the previous election told us to start the server then stop it and - * clear the indication that we were elected. We get the current - * version of the election info from the server because they might have - * modified it while they were running. the old election info. + * This work is responsible for maintaining a connection from the client + * to the server. It's queued on mount and disconnect and we requeue + * the work if the work fails and we're not shutting down. * - * If we're not fast enough to clear the election from the quorum block - * then the next server might fence us. Should be very unlikely as - * election requires multiple RMW cycles. - */ -static void stop_our_server(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - if (qei->run_server) { - scoutfs_server_stop(sb, qei); - scoutfs_quorum_clear_elected(sb, qei); - memset(qei, 0, sizeof(*qei)); - } -} - -/* - * This work is responsible for managing leader elections, running the - * server, and connecting clients to the server. - * - * In the typical case a mount reads the quorum blocks and finds the + * In the typical case a mount reads the super blocks and finds the * address of the currently running server and connects to it. + * Non-voting clients who can't connect will keep trying alternating + * reading the address and getting connect timeouts. * - * More rarely clients who aren't connected and are configured to - * participate in quorum need to elect the new leader. The elected info - * filled by quorum tells us if we were elected to run the server. + * Voting mounts will try to elect a leader if they can't connect to the + * server. When a quorum can't connect and are able to elect a leader + * then a new server is started. The new server will write its address + * in the super and everyone will be able to connect. * - * This leads to the possibility that the mount who is running the - * server had its mount disconnect. This is only weirdly different from - * other clients disconnecting and trying to reconnect because of the - * way quorum slots are reconfigured and reclaimed. If we connect to a - * server with the new quorum config then we can't have any old servers - * running in the stale old quorum slot. The simplest way to do this is - * to *always* stop the server if we're running it and we got - * disconnected. It's a big hammer, but it's reliable, and arguably if - * *we* couldn't' use *our* server then something bad is happening and - * someone else should be the server. - * - * This only executes on mount, error, or as a connection disconnects - * and there's only ever one executing. + * There's a tricky bit of coordination required to safely unmount. + * Clients need to tell the server that they won't be coming back with a + * farewell request. Once a client receives its farewell response it + * can exit. But a majority of clients need to stick around to elect a + * server to process all their farewell requests. This is coordinated + * by having the greeting tell the server that a client is a voter. The + * server then holds on to farewell requests from voters until only + * requests from the final quorum remain. These farewell responses are + * only sent after updating an unmount barrier in the super to indicate + * to the final quorum that they can safely exit without having received + * a farewell response over the network. */ static void scoutfs_client_connect_worker(struct work_struct *work) { struct client_info *client = container_of(work, struct client_info, - connect_work); + connect_dwork.work); struct super_block *sb = client->sb; - struct scoutfs_quorum_elected_info *qei = &client->qei; struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_super_block *super = NULL; struct mount_options *opts = &sbi->opts; + const bool am_voter = opts->server_addr.sin_addr.s_addr != 0; struct scoutfs_net_greeting greet; + struct sockaddr_in sin; ktime_t timeout_abs; + u64 elected_term; int ret; - /* don't try quorum and connecting while our mount runs a server */ - stop_our_server(sb, qei); + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } - timeout_abs = ktime_add_ms(ktime_get(), CLIENT_QUORUM_TIMEOUT_MS); - - ret = scoutfs_quorum_election(sb, opts->uniq_name, - client->old_elected_nr, - timeout_abs, client->sending_farewell, - client->greeting_umb, qei); + ret = scoutfs_read_super(sb, super); if (ret) goto out; - /* we saw that the server wrote a new unmount barrier */ - if (client->sending_farewell && qei->elected_nr == 0 && - qei->unmount_barrier > client->greeting_umb) { + /* can safely unmount if we see that server processed our farewell */ + if (am_voter && client->sending_farewell && + (le64_to_cpu(super->unmount_barrier) > client->greeting_umb)) { client->farewell_error = 0; complete(&client->farewell_comp); ret = 0; goto out; } - if (qei->run_server) { - ret = scoutfs_server_start(sb, &qei->sin, qei->elected_nr, qei); - if (ret) { - /* forget that we tried to start the server */ - memset(qei, 0, sizeof(*qei)); + /* try to connect to the super's server address */ + scoutfs_addr_to_sin(&sin, &super->server_addr); + if (sin.sin_addr.s_addr != 0 && sin.sin_port != 0) + ret = scoutfs_net_connect(sb, client->conn, &sin, + CLIENT_CONNECT_TIMEOUT_MS); + else + ret = -ENOTCONN; + + /* voters try to elect a leader if they couldn't connect */ + if (ret < 0) { + /* non-voters will keep retrying */ + if (!am_voter) goto out; - } - } - /* always give the server some time before connecting */ - msleep(CLIENT_CONNECT_DELAY_MS); + /* make sure local server isn't writing super during votes */ + scoutfs_server_stop(sb); - ret = scoutfs_net_connect(sb, client->conn, &qei->sin, - CLIENT_CONNECT_TIMEOUT_MS); - if (ret) { - /* we couldn't connect, try electing a new server */ - client->old_elected_nr = qei->elected_nr; + timeout_abs = ktime_add_ms(ktime_get(), + CLIENT_QUORUM_TIMEOUT_MS); + + ret = scoutfs_quorum_election(sb, timeout_abs, + le64_to_cpu(super->quorum_server_term), + &elected_term); + /* start the server if we were asked to */ + if (elected_term > 0) + ret = scoutfs_server_start(sb, &opts->server_addr, + elected_term); + ret = -ENOTCONN; goto out; } - /* trust this server again if it's still around after we disconnect */ - client->old_elected_nr = 0; - /* send a greeting to verify endpoints of each connection */ - memcpy(greet.name, opts->uniq_name, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); - greet.unmount_barrier = 0; + greet.unmount_barrier = cpu_to_le64(client->greeting_umb); greet.node_id = cpu_to_le64(sbi->node_id); greet.flags = 0; if (client->sending_farewell) greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); + if (am_voter) + greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_VOTER); ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_GREETING, @@ -492,8 +480,12 @@ static void scoutfs_client_connect_worker(struct work_struct *work) if (ret) scoutfs_net_shutdown(sb, client->conn); out: + kfree(super); + + /* always have a small delay before retrying to avoid storms */ if (ret && !atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, + msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); } /* @@ -566,7 +558,7 @@ static void client_notify_down(struct super_block *sb, struct client_info *client = SCOUTFS_SB(sb)->client_info; if (!atomic_read(&client->shutting_down)) - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); } /* @@ -597,7 +589,8 @@ int scoutfs_client_setup(struct super_block *sb) client->sb = sb; init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); - INIT_WORK(&client->connect_work, scoutfs_client_connect_worker); + INIT_DELAYED_WORK(&client->connect_dwork, + scoutfs_client_connect_worker); init_completion(&client->farewell_comp); client->conn = scoutfs_net_alloc_conn(sb, NULL, client_notify_down, 0, @@ -613,7 +606,7 @@ int scoutfs_client_setup(struct super_block *sb) goto out; } - queue_work(client->workq, &client->connect_work); + queue_delayed_work(client->workq, &client->connect_dwork, 0); ret = 0; out: @@ -693,16 +686,13 @@ void scoutfs_client_destroy(struct super_block *sb) atomic_set(&client->shutting_down, 1); /* make sure worker isn't using the conn */ - cancel_work_sync(&client->connect_work); + cancel_delayed_work_sync(&client->connect_dwork); /* make racing conn use explode */ conn = client->conn; client->conn = NULL; scoutfs_net_free_conn(sb, conn); - /* stop running the server if we were, harmless otherwise */ - stop_our_server(sb, &client->qei); - if (client->workq) destroy_workqueue(client->workq); kfree(client); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a35df40b..60ea6ee9 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -117,18 +117,19 @@ EXPAND_COUNTER(net_recv_invalid_message) \ EXPAND_COUNTER(net_recv_messages) \ EXPAND_COUNTER(net_unknown_request) \ - EXPAND_COUNTER(quorum_elected) \ - EXPAND_COUNTER(quorum_election_error) \ - EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(quorum_found_leader) \ - EXPAND_COUNTER(quorum_no_leader) \ + EXPAND_COUNTER(quorum_cycle) \ + EXPAND_COUNTER(quorum_elected_leader) \ + EXPAND_COUNTER(quorum_election_timeout) \ + EXPAND_COUNTER(quorum_failure) \ + EXPAND_COUNTER(quorum_new_leader) \ EXPAND_COUNTER(quorum_read_block) \ EXPAND_COUNTER(quorum_read_block_error) \ EXPAND_COUNTER(quorum_read_invalid_block) \ - EXPAND_COUNTER(quorum_read_invalid_config) \ - EXPAND_COUNTER(quorum_waited) \ + EXPAND_COUNTER(quorum_saw_super_leader) \ + EXPAND_COUNTER(quorum_timedout) \ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ + EXPAND_COUNTER(quorum_fenced) \ EXPAND_COUNTER(seg_alloc) \ EXPAND_COUNTER(seg_csum_error) \ EXPAND_COUNTER(seg_free) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 68396c19..c2f2fa15 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -42,11 +42,14 @@ /* * A reasonably large region of aligned quorum blocks follow the super - * block. + * block. Each voting cycle reads the entire region so we don't want it + * to be too enormous. 256K seems like a reasonably chunky single IO. + * The number of blocks in the region also determines the number of + * mounts that have a reasonable probability of not overwriting each + * other's random block locations. */ -#define SCOUTFS_QUORUM_BLKNO ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_BLOCKS ((128ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ @@ -304,9 +307,10 @@ struct scoutfs_mounted_client_btree_key { } __packed; struct scoutfs_mounted_client_btree_val { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 flags; } __packed; +#define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) /* * The max number of links defines the max number of entries that we can @@ -421,70 +425,47 @@ struct scoutfs_xattr { #define SCOUTFS_UUID_BYTES 16 /* - * During each quorum voting interval the fabric has to process 2 reads - * and a write for each voting mount. The only reason we limit the - * number of active quorum mounts is to limit the number of IOs per - * interval. We use a pretty conservative interval given that IOs will - * generally be faster than our constant and we'll have fewer active - * than the max. + * Mounts read all the quorum blocks and write to one random quorum + * block during a cycle. The min cycle time limits the per-mount iop + * load during elections. The random cycle delay makes it less likely + * that mounts will read and write at the same time and miss each + * other's writes. An election only completes if a quorum of mounts + * vote for a leader before any of their elections timeout. This is + * made less likely by the probability that mounts will overwrite each + * others random block locations. The max quorum count limits that + * probability. 9 mounts only have a 55% chance of writing to unique 4k + * blocks in a 256k region. The election timeout is set to include + * enough cycles to usually complete the election. Once a leader is + * elected it spends a number of cycles writing out blocks with itself + * logged as a leader. This reduces the possibility that servers + * will have their log entries overwritten and not be fenced. */ -#define SCOUTFS_QUORUM_MAX_ACTIVE 7 -#define SCOUTFS_QUORUM_IO_LATENCY_MS 10 -#define SCOUTFS_QUORUM_INTERVAL_MS \ - (SCOUTFS_QUORUM_MAX_ACTIVE * 3 * SCOUTFS_QUORUM_IO_LATENCY_MS) +#define SCOUTFS_QUORUM_MAX_COUNT 9 +#define SCOUTFS_QUORUM_CYCLE_LO_MS 10 +#define SCOUTFS_QUORUM_CYCLE_HI_MS 20 +#define SCOUTFS_QUORUM_TERM_LO_MS 250 +#define SCOUTFS_QUORUM_TERM_HI_MS 500 +#define SCOUTFS_QUORUM_ELECTED_LOG_CYCLES 10 -/* - * Each mount that is found in the quorum config in the super block can - * write to quorum blocks indicating which mount they vote for as - * the leader. - * - * @config_gen: references the config gen in the super block - * @write_nr: incremented for every write, only 0 when never written - * @elected_nr: incremented when elected, 0 otherwise - * @unmount_barrier: incremented by servers when all members have unmounted - * @vote_slot: the active config slot that the writer is voting for - */ struct scoutfs_quorum_block { __le64 fsid; __le64 blkno; - __le64 config_gen; + __le64 term; __le64 write_nr; - __le64 elected_nr; - __le64 unmount_barrier; + __le64 voter_rid; + __le64 vote_for_rid; __le32 crc; - __u8 vote_slot; - __u8 flags; -} __packed; - -#define SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED (1 << 0) -#define SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING (1 << 1) -#define SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN (U8_MAX << 2) - -#define SCOUTFS_QUORUM_MAX_SLOTS SCOUTFS_QUORUM_BLOCKS - -/* - * Each quorum voter is described by a slot which corresponds to the - * block that the voter will write to. - * - * The stale flag is used to support config migration. A new - * configuration is written in free slots and the old configuration is - * marked stale. Stale slots can only be reclaimed once we have - * evidence that the named mount won't try and write to it by seeing it - * write to other slots or connect with the new gen. - */ -struct scoutfs_quorum_config { - __le64 gen; - struct scoutfs_quorum_slot { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; + __u8 log_nr; + struct scoutfs_quorum_log { + __le64 term; + __le64 rid; struct scoutfs_inet_addr addr; - __u8 vote_priority; - __u8 flags; - } __packed slots[SCOUTFS_QUORUM_MAX_SLOTS]; + } __packed log[0]; } __packed; -#define SCOUTFS_QUORUM_SLOT_ACTIVE (1 << 0) -#define SCOUTFS_QUORUM_SLOT_STALE (1 << 1) -#define SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN (U8_MAX << 2) +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_quorum_block)) / \ + sizeof(struct scoutfs_quorum_log)) struct scoutfs_super_block { struct scoutfs_block_header hdr; @@ -500,9 +481,13 @@ struct scoutfs_super_block { __le64 next_seg_seq; __le64 next_node_id; __le64 next_compact_id; + __le64 quorum_fenced_term; + __le64 quorum_server_term; + __le64 unmount_barrier; + __u8 quorum_count; + struct scoutfs_inet_addr server_addr; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; - struct scoutfs_quorum_config quorum_config; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; @@ -624,8 +609,6 @@ enum { * Greetings verify identity of communicating nodes. The sender sends * their credentials and the receiver verifies them. * - * @name: The client sends its unique name to the server. - * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it * tries to reconnect. Used to identify a client reconnecting to a @@ -634,7 +617,7 @@ enum { * @unmount_barrier: Incremented every time the remaining majority of * quorum members all agree to leave. The server tells a quorum member * the value that it's connecting under so that if the client sees the - * value increase in a quorum block it knows that the server has + * value increase in the super block then it knows that the server has * processed its farewell and can safely unmount. * * @node_id: The id of the client. Initially 0 from the client, @@ -643,7 +626,6 @@ enum { * state must be dealt with. */ struct scoutfs_net_greeting { - __u8 name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; __le64 fsid; __le64 format_hash; __le64 server_term; @@ -653,7 +635,8 @@ struct scoutfs_net_greeting { } __packed; #define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) -#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 1) +#define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) +#define SCOUTFS_NET_GREETING_FLAG_INVALID (~(__u64)0 << 2) /* * This header precedes and describes all network messages sent over diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index 7664aef6..a3e7e6d2 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -33,50 +33,66 @@ #include "scoutfs_trace.h" /* - * scoutfs mounts use a region of statically allocated blocks in the - * shared metadata device to elect a leader mount who runs the server - * that the rest of the mounts of the filesystem connect to. + * scoutfs mounts communicate through a region of preallocated blocks to + * elect a leader who starts the server. Mounts which have been + * configured with a server address and which can't connect to a server + * attempt to form a quorum to elect a new leader who starts a new + * server. * - * Mounts that should participate in the election are configured in an - * array in the super block. Their position in the array determines the - * preallocated block that they'll be writing to. Mounts that aren't - * participating in the election only read the blocks to discover the - * outcome of the election. + * The mounts participating in the election use a variant of the raft + * election protocol to establish quorum and elect a leader. We use + * block reads and writes instead of network messages. Mounts read all + * the blocks looking for messages to receive. Mounts write their vote + * to a random block in the region to send a message to all other + * mounts. Unlikely collisions are analogous to lossy networks losing + * messages and are handled by the protocol. * - * During the election each participating mount reads all the quorum - * blocks that all the mounts wrote, sees which are active and chooses - * which to vote for, and writes a new version of their block that - * includes their vote. Mounts vote for the mount with the highest - * priority in the config that is seen actively writing voting blocks - * over time. + * We allow a "majority" of 1 voter when there are less than three + * possible voters. This lets a simple network establish quorum. If + * the raft quorum timeouts align to leaders could both elect themselves + * and race to fence each other. In the worst case they could continue + * to do this indefinitely but it's unlikely as it would require a + * sequence of identical random raft timeouts. * - * Once a mount receives a majority of votes from its peers then it - * writes its block with an indication that it has been elected. Only - * after reading that block, and seeing no other blocks that indicate - * more recently elected leaders, will it consider itself elected and - * try to fence any other previously elected leaders before starting the - * server. This ensures that racing elected leaders will always result - * in fencing all but the most recent. + * One of the reasons we use block reads and writes as the quorum + * communication medium is that it lets us leave behind a shared + * persistent log of previous election results. This then lets a newly + * elected leader fence all previously elected leaders that haven't + * shutdown so that they can safely assume exclusive access to the + * shared device. Every written block includes a log of election + * results. Every voter merges the log from every block it reads the + * block it writes. A leader doesn't attempt to fence until it's spent + * a few cycles writing blocks with itself as the log entry. This gives + * other voters time to migrate the log entry through the blocks. * - * Once the elected leader verifies its written elected block it tries - * to start up the server. Once it's listening it writes another quorum - * block that indicates that it's listening. Once mounts see that - * they'll try to connect. If the server takes too long to write its - * listening flag the mounts may decide that the leader has died and try - * to elect a new leader. + * Once a leader is elected it fences any previously elected leaders + * still present in the log it merged while reading all the voting + * blocks. Once they've fenced they update the super block record of + * the latest term that has been fenced. This trims the log over time + * and keeps from attempting to fence the same mounts multiple times. + * As the server later shuts down it writes its term into the super to + * stop it from being fenced. * - * XXX: - * - actually fence - * - add temporary priority for choosing a specific mount as a leader - * - add config rotation (write new config, reclaim stale slots) + * The final complication comes during unmount. Clients exit after the + * server responds to their farewell request. But a majority of clients + * need to be present to elect a server to process farewell requests. + * The server knows which clients will attempt to vote for quorum and + * only responds to their farewell requests once they're no longer + * needed to elect a server -- either there's still quorum remaining of + * other mounts or the only mounts remaining are all quorum voters that + * have sent farewell requests. Before sending these final responses + * the server updates an unmount_barrier field in the super. If clients + * that are waiting for a farewell response see the unmount barrier + * increment they know that their farewell has been processed and they + * can assume a successful farewell response and exit cleanly. + * + * XXX: - actually fence */ struct quorum_info { struct scoutfs_sysfs_attrs ssa; bool is_leader; - struct sockaddr_in conf_addr; - u16 conf_port; }; #define DECLARE_QUORUM_INFO(sb, name) \ @@ -84,199 +100,21 @@ struct quorum_info { #define DECLARE_QUORUM_INFO_KOBJ(kobj, name) \ DECLARE_QUORUM_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name) -static void addr_to_sin(struct sockaddr_in *sin, struct scoutfs_inet_addr *addr) -{ - sin->sin_family = AF_INET; - sin->sin_addr.s_addr = cpu_to_be32(le32_to_cpu(addr->addr)); - sin->sin_port = cpu_to_be16(le16_to_cpu(addr->port)); -} - -/* active slots are sorted to the front for validation */ -static int cmp_slot_active(const struct scoutfs_quorum_slot *a, - const struct scoutfs_quorum_slot *b) -{ - int a_active = !!(a->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - int b_active = !!(b->flags & SCOUTFS_QUORUM_SLOT_ACTIVE); - - return b_active - a_active; -} - -/* slot validation has ensured that the names are null terminated */ -static int cmp_slot_names(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - strcmp(a->name, b->name); -} - -static int cmp_slot_addrs(const void *A, const void *B) -{ - const struct scoutfs_quorum_slot *a = A; - const struct scoutfs_quorum_slot *b = B; - - return cmp_slot_active(a, b) ?: - memcmp(&a->addr, &b->addr, sizeof(a->addr)); -} - -static void swap_slots(void *A, void *B, int size) -{ - struct scoutfs_quorum_slot *a = A; - struct scoutfs_quorum_slot *b = B; - - swap(*a, *b); -} - /* - * We'll set the callers our_slot to the slot that contains the their. - * If the name isn't found then it'll be set to -1. + * Return an absolute ktime timeout expires value in the future after a + * random duration between hi and lo where both limits are possible. */ -static int read_quorum_config(struct super_block *sb, - struct scoutfs_super_block *super, - char *our_name, int *our_slot_ret, - int *nr_active_ret) +static ktime_t random_to(u32 lo, u32 hi) { - struct scoutfs_quorum_slot *sorted = NULL; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_config *conf; - struct sockaddr_in sin; - int nr_active = 0; - int our_slot = -1; - int ret; - int i; - - sorted = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), GFP_NOFS); - if (sorted == NULL) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_read_super(sb, super); - if (ret) - goto out; - conf = &super->quorum_config; - - ret = -EINVAL; - - if (conf->gen == 0) { - scoutfs_err(sb, "invalid zero quorum config gen"); - goto out; - } - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_FLAGS_UNKNOWN) { - scoutfs_err(sb, "quorum slot ind %u unknown flags 0x%02x", - i, slot->flags); - goto out; - } - - if ((slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) && - (slot->flags & SCOUTFS_QUORUM_SLOT_STALE)) { - scoutfs_err(sb, "quorum slot ind %u is both active and stale", - i); - goto out; - } - - if (!(slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE)) - continue; - - nr_active++; - - if (slot->name[0] == '\0') { - scoutfs_err(sb, "quorum slot ind %u name is null", i); - goto out; - } - - if (slot->name[SCOUTFS_UNIQUE_NAME_MAX_BYTES - 1] != '\0') { - scoutfs_err(sb, "quorum slot ind %u name isn't null terminated", - i); - goto out; - } - - if (our_name && strcmp(our_name, slot->name) == 0) - our_slot = i; - - addr_to_sin(&sin, &slot->addr); - - if (ipv4_is_multicast(sin.sin_addr.s_addr) || - ipv4_is_lbcast(sin.sin_addr.s_addr) || - ipv4_is_zeronet(sin.sin_addr.s_addr) || - ipv4_is_local_multicast(sin.sin_addr.s_addr) || - ntohs(sin.sin_port) == 0 || - ntohs(sin.sin_port) == U16_MAX) { - scoutfs_err(sb, "quorum slot ind %u has invalid addr %pIS:%u", - i, &sin, ntohs(sin.sin_port)); - goto out; - } - } - - if (nr_active == 0) { - scoutfs_err(sb, "quorum config has no active slots"); - goto out; - } - - if (nr_active > SCOUTFS_QUORUM_MAX_ACTIVE) { - scoutfs_err(sb, "quorum config has %u active slots, can have at most %u ", - nr_active, SCOUTFS_QUORUM_MAX_ACTIVE); - goto out; - } - - memcpy(sorted, conf->slots, - SCOUTFS_QUORUM_MAX_SLOTS * sizeof(sorted[0])); - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_names, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (strcmp(sorted[i].name, sorted[i - 1].name) == 0) { - scoutfs_err(sb, "multiple quorum slots have the same name '%s'", - sorted[i].name); - goto out; - } - } - - sort(sorted, SCOUTFS_QUORUM_MAX_SLOTS, sizeof(sorted[0]), - cmp_slot_addrs, swap_slots); - - for (i = 1; i < nr_active; i++) { - if (memcmp(&sorted[i].addr, &sorted[i - 1].addr, - sizeof(sorted[i].addr)) == 0) { - addr_to_sin(&sin, &sorted[i].addr); - scoutfs_err(sb, "multiple quorum slots have the same address %pIS:%u", - &sin, ntohs(sin.sin_port)); - goto out; - } - } - - ret = 0; - if (our_slot_ret) - *our_slot_ret = our_slot; - if (nr_active_ret) - *nr_active_ret = nr_active; -out: - if (ret) - scoutfs_inc_counter(sb, quorum_read_invalid_config); - kfree(sorted); - return ret; + return ktime_add_ms(ktime_get(), lo + prandom_u32_max((hi + 1) - lo)); } -enum { - BH_ScoutfsTraced = BH_PrivateStart, -}; - -BUFFER_FNS(ScoutfsTraced, scoutfs_traced) /* has been traced */ -TAS_BUFFER_FNS(ScoutfsTraced, scoutfs_traced) - /* - * The caller is about to read the current version of a set of quorum - * blocks. We invalidate all the quorum blocks in the cache and - * populate the cache with all the blocks with one large contiguous - * read. The caller then uses simple sync bh methods to access - * whichever blocks it needs. I'm not a huge fan of the plug but I - * couldn't get the individual readahead requests merged without it. + * The caller is about to read all the quorum blocks. We invalidate any + * cached blocks and issue one large contiguous read to repopulate the + * cache. The caller then uses normal sb_bread to read each block. I'm + * not a huge fan of the plug but I couldn't get the individual + * readahead requests merged without it. */ static void readahead_quorum_blocks(struct super_block *sb) { @@ -293,7 +131,6 @@ static void readahead_quorum_blocks(struct super_block *sb) lock_buffer(bh); clear_buffer_uptodate(bh); - clear_buffer_scoutfs_traced(bh); unlock_buffer(bh); ll_rw_block(READA | REQ_META | REQ_PRIO, 1, &bh); @@ -303,6 +140,25 @@ static void readahead_quorum_blocks(struct super_block *sb) blk_finish_plug(&plug); } +struct quorum_block_head { + struct list_head head; + union { + struct scoutfs_quorum_block blk; + u8 bytes[SCOUTFS_BLOCK_SIZE]; + }; +}; + +static void free_quorum_blocks(struct list_head *blocks) +{ + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + + list_for_each_entry_safe(qbh, tmp, blocks, head) { + list_del_init(&qbh->head); + kfree(qbh); + } +} + /* * Callers don't mind us clobbering the crc temporarily. */ @@ -319,139 +175,149 @@ static __le32 quorum_block_crc(struct scoutfs_quorum_block *blk) return calc_crc; } -static bool invalid_quorum_block(struct scoutfs_super_block *super, - struct buffer_head *bh, +static size_t quorum_block_bytes(struct scoutfs_quorum_block *blk) +{ + return offsetof(struct scoutfs_quorum_block, + log[blk->log_nr]); +} + +static bool invalid_quorum_block(struct buffer_head *bh, struct scoutfs_quorum_block *blk) { - return quorum_block_crc(blk) != blk->crc || - blk->fsid != super->hdr.fsid || + return bh->b_size != SCOUTFS_BLOCK_SIZE || + sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE || + quorum_block_crc(blk) != blk->crc || le64_to_cpu(blk->blkno) != bh->b_blocknr || - blk->vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS || - (blk->flags & SCOUTFS_QUORUM_BLOCK_FLAGS_UNKNOWN); + blk->term == 0 || + blk->log_nr > SCOUTFS_QUORUM_LOG_MAX || + quorum_block_bytes(blk) > SCOUTFS_BLOCK_SIZE; +} + +/* true if a is stale and should be ignored */ +static bool stale_quorum_block(struct scoutfs_quorum_block *a, + struct scoutfs_quorum_block *b) +{ + if (le64_to_cpu(a->term) < le64_to_cpu(b->term)) + return true; + + if (le64_to_cpu(a->voter_rid) == le64_to_cpu(b->voter_rid) && + le64_to_cpu(a->write_nr) <= le64_to_cpu(b->write_nr)) + return true; + + return false; } /* - * Give the caller the most recently updated version of the quorum - * block. Returns 0 and fills the callers block struct on success. - * Returns -ENOENT and zeros the caller's block if we couldn't read a - * valid block. We don't consider the config gen, that's up to the - * caller. + * Get the most recent blocks from all the voters for the most recent term. + * We ignore any corrupt blocks, blocks not for our fsid, previous terms, + * and previous writes from a rid in the current term. */ -static int read_quorum_block(struct super_block *sb, - struct scoutfs_super_block *super, int slot, - struct scoutfs_quorum_block *blk_ret) +static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct quorum_block_head *qbh; + struct quorum_block_head *tmp; + struct buffer_head *bh = NULL; + LIST_HEAD(stale); int ret; + int i; - /* code strongly assumes that slots and blocks are directly mapped */ - BUILD_BUG_ON(SCOUTFS_QUORUM_BLOCKS != SCOUTFS_QUORUM_MAX_SLOTS); + readahead_quorum_blocks(sb); - ret = -ENOENT; + for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { + brelse(bh); + bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + i); + if (!bh) { + scoutfs_inc_counter(sb, quorum_read_block_error); + ret = -EIO; + goto out; + } + blk = (void *)(bh->b_data); - bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + slot); - if (!bh) { - scoutfs_inc_counter(sb, quorum_read_block_error); - goto out; - } - blk = (void *)(bh->b_data); + /* ignore unwritten blocks or blocks for other filesystems */ + if (blk->voter_rid == 0 || blk->fsid != super->hdr.fsid) + continue; - /* ignore unwritten blocks */ - if (blk->write_nr == 0) - goto out; + if (invalid_quorum_block(bh, blk)) { + scoutfs_inc_counter(sb, quorum_read_invalid_block); + continue; + } - if (!test_set_buffer_scoutfs_traced(bh)) - trace_scoutfs_quorum_read_block(sb, bh->b_blocknr, blk); + list_for_each_entry_safe(qbh, tmp, blocks, head) { + if (stale_quorum_block(blk, &qbh->blk)) { + blk = NULL; + break; + } - if (invalid_quorum_block(super, bh, blk)) { - scoutfs_inc_counter(sb, quorum_read_invalid_block); - goto out; + if (stale_quorum_block(&qbh->blk, blk)) + list_move(&qbh->head, &stale); + } + free_quorum_blocks(&stale); + + if (!blk) + continue; + + qbh = kmalloc(sizeof(struct quorum_block_head), + GFP_NOFS); + if (!qbh) { + ret = -ENOMEM; + goto out; + } + + memcpy(&qbh->blk, blk, quorum_block_bytes(blk)); + list_add_tail(&qbh->head, blocks); + } + + list_for_each_entry(qbh, blocks, head) { + trace_scoutfs_quorum_read_block(sb, &qbh->blk); + scoutfs_inc_counter(sb, quorum_read_block); } - *blk_ret = *blk; - scoutfs_inc_counter(sb, quorum_read_block); ret = 0; out: - if (ret < 0) - memset(blk_ret, 0, sizeof(struct scoutfs_quorum_block)); brelse(bh); + if (ret < 0) + free_quorum_blocks(blocks); return ret; } -/* - * Iterate over config slots from the given index and return the first - * slot that has any of the given flags set. - */ -static inline int first_slot_flags(struct scoutfs_quorum_config *conf, - int i, u8 flags) -{ - for (; i < ARRAY_SIZE(conf->slots); i++) { - if (conf->slots[i].flags & flags) - break; - } - return i; -} - -/* - * Execute the loop body with the read block for each slot that's - * configured and active. If we can't read the block for whatever - * reason then the loop will execute with the blk struct zeroed. - */ -#define for_each_active_block(sb, super, conf, hists, hi, blk, slot, i) \ - for (i = first_slot_flags(conf, 0, SCOUTFS_QUORUM_SLOT_ACTIVE); \ - (i < ARRAY_SIZE(conf->slots)) && \ - (slot = &conf->slots[i], \ - hi = &hists[i], \ - read_quorum_block(sb, super, i, blk), 1); \ - i = first_slot_flags(conf, i + 1, SCOUTFS_QUORUM_SLOT_ACTIVE)) - -/* - * Iterate over every possible block, regardless of config. A lot of these - * will be zero. - */ -#define for_each_block(sb, super, i, blk) \ - for (i = 0; \ - (i < SCOUTFS_QUORUM_BLOCKS) && \ - (read_quorum_block(sb, super, i, blk), 1); \ - i++) - /* * Synchronously write a single quorum block. The caller has provided - * the meaningful fields for the write. We fill in the rest that are - * consistent for every write and zero the rest of the block. + * the meaningful fields for the write. We fill in the fsid, blkno, and + * crc for every write and zero the rest of the block. */ -static int write_quorum_block(struct super_block *sb, __le64 fsid, - __le64 config_gen, u8 our_slot, __le64 write_nr, - u64 elected_nr, u64 unmount_barrier, - u8 vote_slot, u8 flags) +static int write_quorum_block(struct super_block *sb, + struct scoutfs_quorum_block *our_blk) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_quorum_block *blk; - struct buffer_head *bh; + struct buffer_head *bh = NULL; + size_t size; int ret; BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); - if (WARN_ON_ONCE(our_slot >= SCOUTFS_QUORUM_MAX_SLOTS) || - WARN_ON_ONCE(vote_slot >= SCOUTFS_QUORUM_MAX_SLOTS)) - return -EINVAL; - - bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + our_slot); + bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS)); if (bh == NULL) { ret = -EIO; goto out; } + + size = quorum_block_bytes(our_blk); + if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SIZE || + size > bh->b_size)) { + ret = -EIO; + goto out; + } + blk = (void *)bh->b_data; + memset(blk, 0, bh->b_size); + memcpy(blk, our_blk, size); - blk->fsid = fsid; + blk->fsid = super->hdr.fsid; blk->blkno = cpu_to_le64(bh->b_blocknr); - blk->config_gen = config_gen; - blk->write_nr = write_nr; - blk->elected_nr = cpu_to_le64(elected_nr); - blk->unmount_barrier = cpu_to_le64(unmount_barrier); - blk->vote_slot = vote_slot; - blk->flags = flags; - blk->crc = quorum_block_crc(blk); lock_buffer(bh); @@ -467,7 +333,7 @@ static int write_quorum_block(struct super_block *sb, __le64 fsid, ret = 0; if (ret == 0) { - trace_scoutfs_quorum_write_block(sb, bh->b_blocknr, blk); + trace_scoutfs_quorum_write_block(sb, blk); scoutfs_inc_counter(sb, quorum_write_block); } out: @@ -478,401 +344,384 @@ out: } /* - * The caller read their quorum block which indicated that they were - * elected. We have to fence all other previously elected leaders so - * that we're running the only instance of the server. - * - * Time can pass between all phases of this: reading that we're elected, - * fencing, and writing the quorum block that clears the elected flag of - * those we fenced. - * - * This is always safe because we either have exclusive access to the - * device having fenced others or someone else would have fenced us - * before they write. + * Returns true if there's an entry for the given election. */ -static int fence_other_elected(struct super_block *sb, - struct scoutfs_super_block *super, - int our_slot, u64 elected_nr) +static bool log_contains(struct scoutfs_quorum_block *blk, u64 term, u64 rid) { - struct scoutfs_quorum_config *conf = &super->quorum_config; - struct scoutfs_quorum_block blk; - u8 flags; - int ret; int i; - for_each_block(sb, super, i, &blk) { - if (i != our_slot && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - le64_to_cpu(blk.elected_nr) <= elected_nr) { - scoutfs_err(sb, "would have fenced"); - scoutfs_inc_counter(sb, quorum_fenced); - - flags = blk.flags & ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - - ret = write_quorum_block(sb, super->hdr.fsid, - conf->gen, i, blk.write_nr, - le64_to_cpu(blk.elected_nr), - le64_to_cpu(blk.unmount_barrier), i, - flags); - if (ret) - break; - } - } - - return ret; -} - -struct quorum_block_history { - __le64 write_nr; - u8 writing; -}; - -/* - * The caller couldn't connect to a server. Read the quorum blocks - * until we see an elected leader and give their address to the caller. - * If we're configured as part of the quorum then we participate in the - * electing by writing our vote to our quorum block. - * - * Voting members read the blocks at regular intervals and update their - * quorum block with their vote for the elected leader. new leader. - * When a mount receives enough votes it marks its vote in the block as - * elected, fences other elected leaders, and returns to the caller who - * starts up the server for others to connect to. - * - * The calling client may have never seen a server before, or could have - * failed to connect to a valid server, or might have tried to connect - * to a dead server. They pass in an existing elected_nr if they want - * us to ignore old servers and they pass in a timeout so that they can - * return to retrying to connect to whatever address we find. - * - * When we return success we update the caller's elected info with the - * most recent elected leader we found, which may well be long gone. We - * return -ENOENT if we didn't find any elected leaders. - * - * If we return success because we saw a larger unmount barrier we set - * elected_nr to 0 and fill the unmount_barrier. - */ -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei) -{ - DECLARE_QUORUM_INFO(sb, qinf); - struct scoutfs_super_block *super = NULL; - struct scoutfs_quorum_config *conf; - struct scoutfs_quorum_slot *slot; - struct scoutfs_quorum_block blk; - struct quorum_block_history *hist; - struct quorum_block_history *hi; - ktime_t expires; - ktime_t now; - __le64 write_nr = 0; - u64 elected_nr = 0; - u64 unmount_barrier = 0; - u8 flags = 0; - int vote_streak = 0; - int vote_slot; - int our_slot; - int vote_prio; - int nr_active; - int nr_votes; - int majority; - int ret; - int i; - - super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); - hist = kcalloc(SCOUTFS_QUORUM_MAX_SLOTS, sizeof(hist[0]), GFP_NOFS); - if (!super || !hist) { - ret = -ENOMEM; - goto out; - } - - for (;;) { - now = ktime_get(); - expires = ktime_add_ms(now, SCOUTFS_QUORUM_INTERVAL_MS); - - ret = read_quorum_config(sb, super, our_name, &our_slot, - &nr_active); - if (ret) - goto out; - conf = &super->quorum_config; - - /* update sysfs with most recently seen config */ - if (our_slot >= 0) { - slot = &conf->slots[our_slot]; - addr_to_sin(&qinf->conf_addr, &slot->addr); - qinf->conf_port = le16_to_cpu(slot->addr.port); - } else { - memset(&qinf->conf_addr, 0, sizeof(qinf->conf_addr)); - qinf->conf_port = 0; - } - - majority = scoutfs_quorum_majority(sb, conf); - - readahead_quorum_blocks(sb); - - /* default to voting for ourselves, but at min prio */ - vote_slot = our_slot; - vote_prio = -1; - memset(qei, 0, sizeof(*qei)); - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* determine which mounts are writing */ - if (blk.config_gen == conf->gen && - blk.write_nr != 0 && - blk.write_nr != hi->write_nr) - hi->writing = min(hi->writing + 1, 2); - else - hi->writing = 0; - hi->write_nr = blk.write_nr; - - /* vote for first highest priority writing block */ - if (hi->writing >= 2 && - slot->vote_priority > vote_prio) { - vote_slot = i; - vote_prio = slot->vote_priority; - } - - /* find the most recently elected leader */ - if ((blk.config_gen == conf->gen) && - (blk.flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) && - (le64_to_cpu(blk.elected_nr) > qei->elected_nr)){ - addr_to_sin(&qei->sin, &slot->addr); - qei->config_gen = blk.config_gen; - qei->write_nr = blk.write_nr; - qei->elected_nr = le64_to_cpu(blk.elected_nr); - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - qei->config_slot = i; - qei->flags = blk.flags; - } - } - - /* - * After writing a block indicating that we were elected - * we make sure that we can read it and that we're still - * the most recent elected leader. If we are then we - * try to fence. If we can't read it, or we're not the - * most recent, or we couldn't fence, then we fall back - * to participating in the election. - */ - if (flags & SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED) { - if (qei->write_nr == write_nr && - qei->elected_nr == elected_nr && - qei->config_slot == our_slot) { - ret = fence_other_elected(sb, super, our_slot, - elected_nr); - if (ret == 0) { - qei->run_server = true; - qinf->is_leader = true; - goto out; - } - - memset(qei, 0, sizeof(*qei)); - } - - vote_streak = 0; - } - - /* return if we found a new listening leader or timed out */ - if (((qei->elected_nr > old_elected_nr) && - (qei->flags & SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING)) || - ktime_after(now, timeout_abs)) { - if (qei->elected_nr > 0) { - scoutfs_inc_counter(sb, quorum_found_leader); - ret = 0; - } else { - scoutfs_inc_counter(sb, quorum_no_leader); - ret = -ENOENT; - } - goto out; - } - - /* wait for the next cycle if we're not in the voting config */ - if (our_slot < 0) - continue; - - nr_votes = 0; - write_nr = cpu_to_le64(1); - elected_nr = 0; - unmount_barrier = 0; - flags = 0; - - for_each_active_block(sb, super, conf, hist, hi, &blk, slot, i){ - /* count our votes (maybe including from us) */ - if (hi->writing >= 2 && blk.vote_slot == our_slot) - nr_votes++; - - /* can finish unmounting if members all left */ - if (unmounting && - le64_to_cpu(blk.unmount_barrier) > our_umb) { - qei->elected_nr = 0; - qei->unmount_barrier = - le64_to_cpu(blk.unmount_barrier); - ret = 0; - goto out; - } - - /* sample existing fields for our write */ - if (i == our_slot) { - write_nr = blk.write_nr; - le64_add_cpu(&write_nr, 1); - } - elected_nr = max(elected_nr, - le64_to_cpu(blk.elected_nr)); - unmount_barrier = max(unmount_barrier, - le64_to_cpu(blk.unmount_barrier)); - } - - - /* elected after sufficient cycles with a majority vote */ - if (nr_votes >= majority) - vote_streak = min(vote_streak + 1, 2); - else - vote_streak = 0; - - if (vote_streak >= 2) { - flags |= SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - elected_nr++; - } - - write_quorum_block(sb, super->hdr.fsid, conf->gen, our_slot, - write_nr, elected_nr, unmount_barrier, - vote_slot, flags); - - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); - scoutfs_inc_counter(sb, quorum_waited); - } - -out: - kfree(super); - kfree(hist); - - if (ret) { - memset(qei, 0, sizeof(*qei)); - scoutfs_inc_counter(sb, quorum_election_error); - } - - return ret; -} - -/* - * The calling server has successfully started and is listening for - * collections. It writes a new block to communicate to the other - * mounts that they should now try to connect. We do increase the write_nr - * here to still indicate that we're alive. - */ -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->flags |= SCOUTFS_QUORUM_BLOCK_FLAG_LISTENING; - le64_add_cpu(&qei->write_nr, 1); - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * The calling server is shutting down and has finished modifying - * persistent state. We clear the elected flag from our quorum block so - * that mounts won't try to connect and so that the next next leader - * won't try to fence. - * - * By definition nothing has written to the slot since we wrote our - * elected quorum block and the slot could not have been reclaimed. To - * reclaim the slot would have required proving that we were gone or - * fencing us. - * - * If this fails then the mount is in trouble because it'll probably be - * fenced by the next elected leader. - * - * XXX I think there's an interesting race here. If the server is - * running in an old config then the server's slot can be reclaimed if - * the server sees a connection from the current gen. If the server is - * taking a client connection as an indication that the slot won't be - * written then the client needs to shut down the server before trying - * to connect with a new gen. - */ -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - DECLARE_QUORUM_INFO(sb, qinf); - - qei->flags &= ~SCOUTFS_QUORUM_BLOCK_FLAG_ELECTED; - qinf->is_leader = false; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - qei->unmount_barrier = unmount_barrier; - - return write_quorum_block(sb, super->hdr.fsid, qei->config_gen, - qei->config_slot, qei->write_nr, - qei->elected_nr, qei->unmount_barrier, - qei->config_slot, qei->flags); -} - -/* - * If there's only one or two active slots then a single vote is sufficient - * for a majority. - */ -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf) -{ - struct scoutfs_quorum_slot *slot; - int nr_active = 0; - int majority; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (slot->flags & SCOUTFS_QUORUM_SLOT_ACTIVE) - nr_active++; - } - - if (nr_active <= 2) - majority = 1; - else if (nr_active & 1) - majority = (nr_active + 1) / 2; - else - majority = (nr_active / 2) + 1; - - return majority; -} - -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name) -{ - struct scoutfs_quorum_slot *slot; - int i; - - for (i = 0; i < SCOUTFS_QUORUM_MAX_SLOTS; i++) { - slot = &conf->slots[i]; - - if (strcmp(slot->name, name) == 0) + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) == term && + le64_to_cpu(blk->log[i].rid) == rid) return true; } return false; } +/* add an entry to the log, returning error if it's full */ +static int log_add(struct scoutfs_quorum_block *blk, u64 term, u64 rid, + struct scoutfs_inet_addr *addr) +{ + int i; + + if (log_contains(blk, term, rid)) + return 0; + + if (blk->log_nr == SCOUTFS_QUORUM_LOG_MAX) + return -ENOSPC; + + i = blk->log_nr++; + blk->log[i].term = cpu_to_le64(term); + blk->log[i].rid = cpu_to_le64(rid); + blk->log[i].addr = *addr; + + return 0; +} + +/* migrate live log entries between blocks, returning err if full */ +static int log_merge(struct scoutfs_quorum_block *our_blk, + struct scoutfs_quorum_block *blk, + u64 fenced_term) +{ + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) > fenced_term) { + ret = log_add(our_blk, le64_to_cpu(blk->log[i].term), + le64_to_cpu(blk->log[i].rid), + &blk->log[i].addr); + if (ret < 0) + return ret; + } + } + + return 0; +} + +/* Remove old log entries for a voter before a given term. */ +static void log_purge(struct scoutfs_quorum_block *blk, u64 term, u64 rid) +{ + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].term) < term && + le64_to_cpu(blk->log[i].rid) == rid) { + if (i != blk->log_nr - 1) + swap(blk->log[i], blk->log[blk->log_nr - 1]); + blk->log_nr--; + i--; /* continue from swapped in entry */ + } + } +} + + +/* + * The caller received a majority of votes and has been elected. Before + * assuming exclusive write access to the device we fence the winners of + * any previous elections still present in the log. Once they're fenced + * we re-read the super and update the fenced_term to indicate that + * those previous elections can be ignored and purged from the log. + * + * We can be attempting this concurrently with both previous and future + * elected leaders. The leader with the greatest elected term will win + * and fence all previous elected leaders. + * + * We clobber the caller's block as we go to not fence rids multiple times. + */ +static int fence_previous(struct super_block *sb, + struct scoutfs_quorum_block *blk, + u64 our_rid, u64 fenced_term, u64 term) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct sockaddr_in their_sin; + int ret; + int i; + + for (i = 0; i < blk->log_nr; i++) { + if (le64_to_cpu(blk->log[i].rid) != our_rid && + le64_to_cpu(blk->log[i].term) > fenced_term && + le64_to_cpu(blk->log[i].term) < term) { + + scoutfs_inc_counter(sb, quorum_fenced); + scoutfs_addr_to_sin(&their_sin, &blk->log[i].addr); + scoutfs_err(sb, "fencing "SCSBF" at "SIN_FMT, + SCSB_LEFR_ARGS(super->hdr.fsid, + blk->log[i].rid), + SIN_ARG(&their_sin)); + + log_purge(blk, term, le64_to_cpu(blk->log[i].rid)); + i = -1; /* start over */ + } + } + + /* update fenced term now that we have exclusive access */ + ret = 0; + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (super) { + ret = scoutfs_read_super(sb, super); + if (ret == 0) { + super->quorum_fenced_term = cpu_to_le64(term - 1); + ret = scoutfs_write_super(sb, super); + + } + kfree(super); + } else { + ret = -ENOMEM; + } + + if (ret != 0) { + scoutfs_err(sb, "failed to update fenced_term in super, this mount will probably be fenced"); + } + + return ret; +} + + + +/* + * The calling voting mount couldn't connect to a server. Participate + * in a raft election to chose a mount to start a new server. If a + * majority of other mounts join us then one of us will be elected and + * our caller will start the server. + * + * Voting members read the blocks at regular intervals. If they see a + * new election they vote for that candidate for the remainder of the + * election. If the election timeout expires they will start a new + * election and vote for themselves. Eventually a sufficient majority + * sees a new election and all vote in the majority for that candidate. + * + * The calling client may have just failed to connect to an elected + * address in the super block. We assume that server is dead and ignore + * it when trying to elect a new leader. But we eventually return with + * a timeout because the server could actually be fine and the client + * could have had communication to the server restored. + * + * We return success if we see a new server elected. If we are elected + * we set the caller's elected_term so they know to start the server. + */ +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term) +{ + DECLARE_QUORUM_INFO(sb, qinf); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = NULL; + struct scoutfs_quorum_block *our_blk = NULL; + struct scoutfs_quorum_block *blk; + struct quorum_block_head *qbh; + struct scoutfs_inet_addr addr; + enum { VOTER, CANDIDATE }; + ktime_t cycle_to; + ktime_t term_to; + LIST_HEAD(blocks); + u64 vote_for_write_nr; + u64 vote_for_rid; + u64 write_nr; + u64 term; + int log_cycles = 0; + int votes; + int role; + int ret; + + *elected_term = 0; + + trace_scoutfs_quorum_election(sb, prev_term); + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + our_blk = kmalloc(SCOUTFS_BLOCK_SIZE, GFP_NOFS); + if (!super || !our_blk) { + ret = -ENOMEM; + goto out; + } + + /* start out as a passive voter */ + role = VOTER; + term = 0; + write_nr = 0; + vote_for_rid = 0; + vote_for_write_nr = 0; + + /* we'll become a candidate if we don't see another candidate */ + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + + for (;;) { + memset(our_blk, 0, SCOUTFS_BLOCK_SIZE); + + scoutfs_inc_counter(sb, quorum_cycle); + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + /* done if we see evidence of a new server */ + if (le64_to_cpu(super->quorum_server_term) > prev_term) { + scoutfs_inc_counter(sb, quorum_saw_super_leader); + ret = 0; + goto out; + } + + /* done if we couldn't elect anyone */ + if (ktime_after(ktime_get(), timeout_abs)) { + scoutfs_inc_counter(sb, quorum_timedout); + ret = -ETIMEDOUT; + goto out; + } + + /* become a candidate if the election times out */ + if (ktime_after(ktime_get(), term_to)) { + scoutfs_inc_counter(sb, quorum_election_timeout); + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + role = CANDIDATE; + term++; + vote_for_rid = sbi->rid; + log_cycles = 0; + } + + free_quorum_blocks(&blocks); + ret = read_quorum_blocks(sb, &blocks); + if (ret < 0) + goto out; + + votes = 0; + + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + /* + * Become a voter for a candidate the first time + * we see a new term. + * + * And also if we're a candidate and see a + * higher rid candidate in our term. This + * minimizes instability when two quorums are + * possible and race to elect two leaders. This + * is only barely reasonable when accepting the + * risk of instability in two mount + * configurations. + */ + if ((le64_to_cpu(blk->term) > term) || + (role == CANDIDATE && + le64_to_cpu(blk->term) == term && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->voter_rid) > sbi->rid)) { + role = VOTER; + term = le64_to_cpu(blk->term); + vote_for_rid = le64_to_cpu(blk->vote_for_rid); + vote_for_write_nr = 0; + votes = 0; + log_cycles = 0; + } + + /* candidate writes suppress voter election timers */ + if (role == VOTER && + blk->voter_rid == blk->vote_for_rid && + le64_to_cpu(blk->write_nr) > vote_for_write_nr) { + term_to = random_to(SCOUTFS_QUORUM_TERM_LO_MS, + SCOUTFS_QUORUM_TERM_HI_MS); + vote_for_write_nr = le64_to_cpu(blk->write_nr); + } + + /* count our votes */ + if (role == CANDIDATE && + le64_to_cpu(blk->vote_for_rid) == sbi->rid) { + votes++; + } + + /* try to write greater write_nr */ + write_nr = max(write_nr, le64_to_cpu(blk->write_nr)); + } + + trace_scoutfs_quorum_election_vote(sb, role, term, + vote_for_rid, votes, + log_cycles, + super->quorum_count); + + /* first merge logs from all votes this term */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + ret = log_merge(our_blk, blk, + le64_to_cpu(super->quorum_fenced_term)); + if (ret < 0) + goto out; + } + + /* remove logs for voters that can't be servers */ + list_for_each_entry(qbh, &blocks, head) { + blk = &qbh->blk; + + if (blk->voter_rid != blk->vote_for_rid) + log_purge(our_blk, le64_to_cpu(blk->term), + le64_to_cpu(blk->voter_rid)); + } + + /* add ourselves to the log when we see vote quorum */ + if (role == CANDIDATE && votes >= super->quorum_count) { + scoutfs_addr_from_sin(&addr, &opts->server_addr); + ret = log_add(our_blk, term, vote_for_rid, &addr); + if (ret < 0) + goto out; + log_cycles++; /* will be written *this* cycle */ + } + + /* elected candidates can proceed after their log cycles */ + if (role == CANDIDATE && + log_cycles > SCOUTFS_QUORUM_ELECTED_LOG_CYCLES) { + /* our_blk is clobbered */ + ret = fence_previous(sb, our_blk, sbi->rid, + le64_to_cpu(super->quorum_fenced_term), + term); + if (ret < 0) + goto out; + scoutfs_inc_counter(sb, quorum_elected_leader); + qinf->is_leader = true; + *elected_term = term; + goto out; + } + + /* write our block every cycle */ + if (term > 0) { + our_blk->term = cpu_to_le64(term); + write_nr++; + our_blk->write_nr = cpu_to_le64(write_nr); + our_blk->voter_rid = cpu_to_le64(sbi->rid); + our_blk->vote_for_rid = cpu_to_le64(vote_for_rid); + + ret = write_quorum_block(sb, our_blk); + if (ret < 0) + goto out; + } + + /* add a small random delay to each cycle */ + cycle_to = random_to(SCOUTFS_QUORUM_CYCLE_LO_MS, + SCOUTFS_QUORUM_CYCLE_HI_MS); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&cycle_to, HRTIMER_MODE_ABS); + } + +out: + free_quorum_blocks(&blocks); + kfree(super); + kfree(our_blk); + + trace_scoutfs_quorum_election_ret(sb, ret, *elected_term); + if (ret) + scoutfs_inc_counter(sb, quorum_failure); + + return ret; +} + +void scoutfs_quorum_clear_leader(struct super_block *sb) +{ + DECLARE_QUORUM_INFO(sb, qinf); + + qinf->is_leader = false; +} + static ssize_t is_leader_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -882,28 +731,8 @@ static ssize_t is_leader_show(struct kobject *kobj, } SCOUTFS_ATTR_RO(is_leader); -static ssize_t ipv4_addr_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%pIS", &qinf->conf_addr); -} -SCOUTFS_ATTR_RO(ipv4_addr); - -static ssize_t ipv4_port_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - DECLARE_QUORUM_INFO_KOBJ(kobj, qinf); - - return snprintf(buf, PAGE_SIZE, "%u", qinf->conf_port); -} -SCOUTFS_ATTR_RO(ipv4_port); - static struct attribute *quorum_attrs[] = { SCOUTFS_ATTR_PTR(is_leader), - SCOUTFS_ATTR_PTR(ipv4_addr), - SCOUTFS_ATTR_PTR(ipv4_port), NULL, }; diff --git a/kmod/src/quorum.h b/kmod/src/quorum.h index cea55525..96eac0e4 100644 --- a/kmod/src/quorum.h +++ b/kmod/src/quorum.h @@ -1,33 +1,9 @@ #ifndef _SCOUTFS_QUORUM_H_ #define _SCOUTFS_QUORUM_H_ -struct scoutfs_quorum_elected_info { - struct sockaddr_in sin; - __le64 config_gen; - __le64 write_nr; - u64 elected_nr; - u64 unmount_barrier; - unsigned int config_slot; - bool run_server; - u8 flags; -}; - -int scoutfs_quorum_election(struct super_block *sb, char *our_name, - u64 old_elected_nr, ktime_t timeout_abs, - bool unmounting, u64 our_umb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_set_listening(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_clear_elected(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); -int scoutfs_quorum_update_barrier(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei, - u64 unmount_barrier); -int scoutfs_quorum_majority(struct super_block *sb, - struct scoutfs_quorum_config *conf); -bool scoutfs_quorum_voting_member(struct super_block *sb, - struct scoutfs_quorum_config *conf, - char *name); +int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, + u64 prev_term, u64 *elected_term); +void scoutfs_quorum_clear_leader(struct super_block *sb); int scoutfs_quorum_setup(struct super_block *sb); void scoutfs_quorum_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 20fc1282..d87dcb2c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2544,53 +2544,118 @@ TRACE_EVENT(scoutfs_lock_message, __entry->old_mode, __entry->new_mode) ); -DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk), +TRACE_EVENT(scoutfs_quorum_election, + TP_PROTO(struct super_block *sb, u64 prev_term), + + TP_ARGS(sb, prev_term), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, io_blkno) - __field(__u64, hdr_blkno) - __field(__u64, config_gen) - __field(__u64, write_nr) - __field(__u64, elected_nr) - __field(__u64, unmount_barrier) - __field(__u32, crc) - __field(__u8, vote_slot) - __field(__u8, flags) + __field(__u64, prev_term) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->io_blkno = io_blkno; - __entry->hdr_blkno = le64_to_cpu(blk->blkno); - __entry->config_gen = le64_to_cpu(blk->config_gen); - __entry->write_nr = le64_to_cpu(blk->write_nr); - __entry->elected_nr = le64_to_cpu(blk->elected_nr); - __entry->unmount_barrier = le64_to_cpu(blk->unmount_barrier); - __entry->crc = le32_to_cpu(blk->crc); - __entry->vote_slot = blk->vote_slot; - __entry->flags = blk->flags; + __entry->prev_term = prev_term; ), - TP_printk(SCSBF" io_blkno %llu hdr_blkno %llu config_gen %llu write_nr %llu elected_nr %llu umb %llu crc 0x%08x vote_slot %u flags %02x", - SCSB_TRACE_ARGS, __entry->io_blkno, __entry->hdr_blkno, - __entry->config_gen, __entry->write_nr, __entry->elected_nr, - __entry->unmount_barrier, __entry->crc, __entry->vote_slot, - __entry->flags) + TP_printk(SCSBF" prev_term %llu", + SCSB_TRACE_ARGS, __entry->prev_term) +); + +TRACE_EVENT(scoutfs_quorum_election_ret, + TP_PROTO(struct super_block *sb, int ret, u64 elected_term), + + TP_ARGS(sb, ret, elected_term), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, ret) + __field(__u64, elected_term) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ret = ret; + __entry->elected_term = elected_term; + ), + + TP_printk(SCSBF" ret %d elected_term %llu", + SCSB_TRACE_ARGS, __entry->ret, __entry->elected_term) +); + +TRACE_EVENT(scoutfs_quorum_election_vote, + TP_PROTO(struct super_block *sb, int role, u64 term, u64 vote_for_rid, + int votes, int log_cycles, int quorum_count), + + TP_ARGS(sb, role, term, vote_for_rid, votes, log_cycles, quorum_count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(int, role) + __field(__u64, term) + __field(__u64, vote_for_rid) + __field(int, votes) + __field(int, log_cycles) + __field(int, quorum_count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->role = role; + __entry->term = term; + __entry->vote_for_rid = vote_for_rid; + __entry->votes = votes; + __entry->log_cycles = log_cycles; + __entry->quorum_count = quorum_count; + ), + + TP_printk(SCSBF" role %d term %llu vote_for_rid %016llx votes %d log_cycles %d quorum_count %d", + SCSB_TRACE_ARGS, __entry->role, __entry->term, + __entry->vote_for_rid, __entry->votes, __entry->log_cycles, + __entry->quorum_count) +); + +DECLARE_EVENT_CLASS(scoutfs_quorum_block_class, + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + + TP_ARGS(sb, blk), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, term) + __field(__u64, write_nr) + __field(__u64, voter_rid) + __field(__u64, vote_for_rid) + __field(__u32, crc) + __field(__u8, log_nr) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = le64_to_cpu(blk->blkno); + __entry->term = le64_to_cpu(blk->term); + __entry->write_nr = le64_to_cpu(blk->write_nr); + __entry->voter_rid = le64_to_cpu(blk->voter_rid); + __entry->vote_for_rid = le64_to_cpu(blk->vote_for_rid); + __entry->crc = le32_to_cpu(blk->crc); + __entry->log_nr = blk->log_nr; + ), + + TP_printk(SCSBF" blkno %llu term %llu write_nr %llu voter_rid %016llx vote_for_rid %016llx crc 0x%08x log_nr %u", + SCSB_TRACE_ARGS, __entry->blkno, __entry->term, + __entry->write_nr, __entry->voter_rid, __entry->vote_for_rid, + __entry->crc, __entry->log_nr) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_read_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); DEFINE_EVENT(scoutfs_quorum_block_class, scoutfs_quorum_write_block, - TP_PROTO(struct super_block *sb, u64 io_blkno, - struct scoutfs_quorum_block *blk), - TP_ARGS(sb, io_blkno, blk) + TP_PROTO(struct super_block *sb, struct scoutfs_quorum_block *blk), + TP_ARGS(sb, blk) ); /* diff --git a/kmod/src/server.c b/kmod/src/server.c index d077d858..291c7df5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -42,8 +42,8 @@ * connection and accepts connections from all the other mounts acting * as clients. * - * The server is started when raft elects the mount as the leader. If - * it sees errors it shuts down the server in the hopes that another + * The server is started by the mount that is elected leader by quorum. + * If it sees errors it shuts down the server in the hopes that another * mount will become the leader and have less trouble. */ @@ -61,8 +61,6 @@ struct server_info { u64 term; struct scoutfs_net_connection *conn; - struct scoutfs_quorum_elected_info qei; - /* request processing coordinates committing manifest and alloc */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; @@ -1185,14 +1183,16 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, } static int insert_mounted_client(struct super_block *sb, u64 node_id, - char *name) + u64 gr_flags) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; mck.node_id = cpu_to_be64(node_id); - strncpy(mcv.name, name, sizeof(mcv.name)); + mcv.flags = 0; + if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) + mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; return scoutfs_btree_insert(sb, &super->mounted_clients, &mck, sizeof(mck), &mcv, sizeof(mcv)); @@ -1260,6 +1260,7 @@ static int server_greeting(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; __le64 node_id = 0; + __le64 umb = 0; bool sent_node_id; bool first_contact; bool farewell; @@ -1293,10 +1294,12 @@ static int server_greeting(struct super_block *sb, spin_lock(&server->lock); node_id = super->next_node_id; le64_add_cpu(&super->next_node_id, 1); + umb = super->unmount_barrier; spin_unlock(&server->lock); mutex_lock(&server->farewell_mutex); - ret = insert_mounted_client(sb, le64_to_cpu(node_id), gr->name); + ret = insert_mounted_client(sb, le64_to_cpu(node_id), + le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); if (ret == 0) @@ -1308,6 +1311,7 @@ static int server_greeting(struct super_block *sb, } } else { node_id = gr->node_id; + umb = gr->unmount_barrier; } send_err: @@ -1315,11 +1319,10 @@ send_err: if (err) node_id = 0; - memset(greet.name, 0, sizeof(greet.name)); greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); - greet.unmount_barrier = cpu_to_le64(server->qei.unmount_barrier); + greet.unmount_barrier = umb; greet.node_id = node_id; greet.flags = 0; @@ -1379,31 +1382,20 @@ static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) /* * This work processes farewell requests asynchronously. Requests from - * voting quorum members can be held until they're no longer needed to - * vote for quorum and elect a server to process farewell requests. - * - * This will hold farewell requests from voting clients until either it - * isn't needed for quorum because a majority remains without it, or it - * won't be needed for quorum because all the remaining mounted clients - * are voting and waiting for farewell. + * voting clients can be held until only the final quorum remains and + * they've all sent farewell requests. * * When we remove the last mounted client record for the last voting - * client then we increase the unmount_barrier and write it to the - * server's quorum block. If voting clients don't get their farewell - * response they'll attempt to form quorum again to start the server for - * their farewell response but will find the increased umount_barrier. - * The'll know that their farewell has been processed and they can exit - * without forming quorum. + * client then we increase the unmount_barrier and write it to the super + * block. If voting clients don't get their farewell response they'll + * see the greater umount_barrier in the super and will know that their + * farewell has been processed and that they can exit. * * Responses that are waiting for clients who aren't voting are * immediately sent. Clients that don't have a mounted client record * have already had their farewell processed by another server and can * proceed. * - * This can trust the quorum config found in the super that was read - * when the server started. Only the current server can rewrite the - * working config. - * * Farewell responses are unique in that sending them causes the server * to shutdown the connection to the client next time the socket * disconnects. If the socket is destroyed before the client gets the @@ -1420,7 +1412,6 @@ static void farewell_worker(struct work_struct *work) farewell_work); struct super_block *sb = server->sb; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_quorum_config *conf = &super->quorum_config; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val *mcv; struct farewell_request *tmp; @@ -1429,7 +1420,6 @@ static void farewell_worker(struct work_struct *work) struct commit_waiter cw; unsigned int nr_unmounting = 0; unsigned int nr_mounted = 0; - unsigned int majority; LIST_HEAD(reqs); LIST_HEAD(send); bool deleted = false; @@ -1437,8 +1427,6 @@ static void farewell_worker(struct work_struct *work) bool more_reqs; int ret; - majority = scoutfs_quorum_majority(sb, conf); - /* grab all the requests that are waiting */ mutex_lock(&server->farewell_mutex); list_splice_init(&server->farewell_requests, &reqs); @@ -1463,7 +1451,7 @@ static void farewell_worker(struct work_struct *work) } mcv = iref.val; - voting = scoutfs_quorum_voting_member(sb, conf, mcv->name); + voting = (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) != 0; scoutfs_btree_put_iref(&iref); if (!voting) { @@ -1492,7 +1480,7 @@ static void farewell_worker(struct work_struct *work) memcpy(&mck, iref.key, sizeof(mck)); mcv = iref.val; - if (scoutfs_quorum_voting_member(sb, conf, mcv->name)) + if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) nr_mounted++; scoutfs_btree_put_iref(&iref); @@ -1503,7 +1491,8 @@ static void farewell_worker(struct work_struct *work) /* send as many responses as we can to maintain quorum */ while ((fw = list_first_entry_or_null(&reqs, struct farewell_request, entry)) && - (nr_mounted > majority || nr_unmounting >= nr_mounted)) { + (nr_mounted > super->quorum_count || + nr_unmounting >= nr_mounted)) { list_move_tail(&fw->entry, &send); nr_mounted--; @@ -1529,10 +1518,13 @@ static void farewell_worker(struct work_struct *work) goto out; } - /* update the unmount barrier the first time we delete all mounted */ + /* update the unmount barrier if we deleted all voting clients */ if (deleted && nr_mounted == 0) { - ret = scoutfs_quorum_update_barrier(sb, &server->qei, - server->qei.unmount_barrier + 1); + down_read(&server->commit_rwsem); + le64_add_cpu(&super->unmount_barrier, 1); + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + ret = wait_for_commit(&cw); if (ret) goto out; } @@ -2290,9 +2282,14 @@ static void scoutfs_server_worker(struct work_struct *work) struct sockaddr_in sin; LIST_HEAD(conn_list); int ret; + int err; trace_scoutfs_server_work_enter(sb, 0, 0); + sin = server->listen_sin; + + scoutfs_info(sb, "server setting up at "SIN_FMT, SIN_ARG(&sin)); + conn = scoutfs_net_alloc_conn(sb, server_notify_up, server_notify_down, sizeof(struct server_client_info), server_req_funcs, "server"); @@ -2301,8 +2298,6 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - sin = server->listen_sin; - ret = scoutfs_net_bind(sb, conn, &sin); if (ret) { scoutfs_err(sb, "server failed to bind to "SIN_FMT", err %d%s", @@ -2312,37 +2307,44 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; } - ret = scoutfs_read_super(sb, super); if (ret) goto out; /* start up the server subsystems before accepting */ - ret = scoutfs_btree_setup(sb) ?: + ret = scoutfs_read_super(sb, super) ?: + scoutfs_btree_setup(sb) ?: scoutfs_manifest_setup(sb) ?: scoutfs_lock_server_setup(sb); if (ret) goto shutdown; - complete(&server->start_comp); + /* + * Write our address in the super before it's possible for net + * processing to start writing the super as part of + * transactions. In theory clients could be trying to connect + * to our address without having seen it in the super (maybe + * they saw it a long time ago). + */ + scoutfs_addr_from_sin(&super->server_addr, &sin); + super->quorum_server_term = cpu_to_le64(server->term); + ret = scoutfs_write_super(sb, super); + if (ret < 0) + goto shutdown; server->stable_manifest_root = super->manifest.root; - scoutfs_info(sb, "server started on "SIN_FMT, SIN_ARG(&sin)); - /* start accepting connections and processing work */ server->conn = conn; scoutfs_net_listen(sb, conn); - ret = scoutfs_quorum_set_listening(sb, &server->qei); + scoutfs_info(sb, "server ready at "SIN_FMT, SIN_ARG(&sin)); + complete(&server->start_comp); - if (ret == 0) { - /* wait_event/wake_up provide barriers */ - wait_event_interruptible(server->waitq, server->shutting_down); - } - - scoutfs_info(sb, "server shutting down on "SIN_FMT, SIN_ARG(&sin)); + /* wait_event/wake_up provide barriers */ + wait_event_interruptible(server->waitq, server->shutting_down); shutdown: + scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); /* wait for request processing */ scoutfs_net_shutdown(sb, conn); /* drain compact work queued by responses */ @@ -2357,17 +2359,41 @@ shutdown: scoutfs_lock_server_destroy(sb); out: + scoutfs_quorum_clear_leader(sb); scoutfs_net_free_conn(sb, conn); + scoutfs_info(sb, "server stopped at "SIN_FMT, SIN_ARG(&sin)); trace_scoutfs_server_work_exit(sb, 0, ret); + /* + * Always try to clear our presence in the super so that we're + * not fenced. We do this last because other mounts will try to + * reach quorum the moment they see zero here. The later we do + * this the longer we have to finish shutdown while clients + * timeout. + */ + err = scoutfs_read_super(sb, super); + if (err == 0) { + super->quorum_fenced_term = cpu_to_le64(server->term); + memset(&super->server_addr, 0, sizeof(super->server_addr)); + err = scoutfs_write_super(sb, super); + } + if (err < 0) { + scoutfs_err(sb, "failed to clear election term %llu at "SIN_FMT", this mount could be fenced", + server->term, SIN_ARG(&sin)); + } + server->err = ret; complete(&server->start_comp); } -/* XXX can we call start multiple times? */ +/* + * Wait for the server to successfully start. If this returns error then + * the super block's fence_term has been set to the new server's term so + * that it won't be fenced. + */ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei) + u64 term) { DECLARE_SERVER_INFO(sb, server); @@ -2375,7 +2401,6 @@ int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, server->shutting_down = false; server->listen_sin = *sin; server->term = term; - server->qei = *qei; init_completion(&server->start_comp); queue_work(server->wq, &server->work); @@ -2398,8 +2423,7 @@ void scoutfs_server_abort(struct super_block *sb) * Once the server is stopped we give the caller our election info * which might have been modified while we were running. */ -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei) +void scoutfs_server_stop(struct super_block *sb) { DECLARE_SERVER_INFO(sb, server); @@ -2407,8 +2431,6 @@ void scoutfs_server_stop(struct super_block *sb, /* XXX not sure both are needed */ cancel_work_sync(&server->work); cancel_work_sync(&server->commit_work); - - *qei = server->qei; } int scoutfs_server_setup(struct super_block *sb) diff --git a/kmod/src/server.h b/kmod/src/server.h index fee6ac0e..32c6ccea 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -74,10 +74,9 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, struct sockaddr_in; struct scoutfs_quorum_elected_info; int scoutfs_server_start(struct super_block *sb, struct sockaddr_in *sin, - u64 term, struct scoutfs_quorum_elected_info *qei); + u64 term); void scoutfs_server_abort(struct super_block *sb); -void scoutfs_server_stop(struct super_block *sb, - struct scoutfs_quorum_elected_info *qei); +void scoutfs_server_stop(struct super_block *sb); int scoutfs_server_setup(struct super_block *sb); void scoutfs_server_destroy(struct super_block *sb); diff --git a/kmod/src/super.c b/kmod/src/super.c index e65f0c2a..1ac127ff 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -200,7 +200,6 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); - scoutfs_quorum_destroy(sb); scoutfs_inode_destroy(sb); /* the server locks the listen address and compacts */ @@ -210,6 +209,9 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); + /* server clears quorum leader flag during shutdown */ + scoutfs_quorum_destroy(sb); + scoutfs_item_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); @@ -319,6 +321,16 @@ int scoutfs_read_super(struct super_block *sb, goto out; } + /* XXX do we want more rigorous invalid super checking? */ + + if (super->quorum_count == 0 || + super->quorum_count > SCOUTFS_QUORUM_MAX_COUNT) { + scoutfs_err(sb, "super block has invalid quorum count %u, must be > 0 and <= %u", + super->quorum_count, SCOUTFS_QUORUM_MAX_COUNT); + ret = -EINVAL; + goto out; + } + *super_res = *super; ret = 0; out: From f0a86f05f86d77a0d28e27543164c8a55b905807 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 8 Jul 2019 09:26:06 -0700 Subject: [PATCH 733/920] scoutfs: remove unused uniq_name option Signed-off-by: Zach Brown --- kmod/src/options.c | 14 +------------- kmod/src/options.h | 2 -- kmod/src/super.c | 12 ------------ 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/kmod/src/options.c b/kmod/src/options.c index 362a5792..332b1cda 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -28,7 +28,6 @@ static const match_table_t tokens = { {Opt_server_addr, "server_addr=%s"}, - {Opt_uniq_name, "uniq_name=%s"}, {Opt_err, NULL} }; @@ -96,7 +95,7 @@ int scoutfs_parse_options(struct super_block *sb, char *options, { char ipstr[INET_ADDRSTRLEN + 1]; substring_t args[MAX_OPT_ARGS]; - int token, len; + int token; char *p; int ret; @@ -116,12 +115,6 @@ int scoutfs_parse_options(struct super_block *sb, char *options, if (ret < 0) return ret; break; - case Opt_uniq_name: - len = match_strlcpy(parsed->uniq_name, args, - SCOUTFS_UNIQUE_NAME_MAX_BYTES); - if (len == 0 || len > SCOUTFS_UNIQUE_NAME_MAX_BYTES) - return -EINVAL; - break; default: scoutfs_err(sb, "Unknown or malformed option, \"%s\"", p); @@ -129,11 +122,6 @@ int scoutfs_parse_options(struct super_block *sb, char *options, } } - if (parsed->uniq_name[0] == '\0') { - scoutfs_err(sb, "must provide a uniq_name option"); - return -EINVAL; - } - return 0; } diff --git a/kmod/src/options.h b/kmod/src/options.h index 74ae5fa4..0078dca2 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -12,13 +12,11 @@ enum { */ Opt_btree_force_tiny_blocks, Opt_server_addr, - Opt_uniq_name, Opt_err, }; struct mount_options { struct sockaddr_in server_addr; - char uniq_name[SCOUTFS_UNIQUE_NAME_MAX_BYTES]; }; int scoutfs_parse_options(struct super_block *sb, char *options, diff --git a/kmod/src/super.c b/kmod/src/super.c index 1ac127ff..c5a8aeef 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -139,7 +139,6 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) struct mount_options *opts = &SCOUTFS_SB(sb)->opts; seq_printf(seq, ",server_addr="SIN_FMT, SIN_ARG(&opts->server_addr)); - seq_printf(seq, ",uniq_name=%s", opts->uniq_name); return 0; } @@ -155,19 +154,8 @@ static ssize_t server_addr_show(struct kobject *kobj, } SCOUTFS_ATTR_RO(server_addr); -static ssize_t uniq_name_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); - struct mount_options *opts = &SCOUTFS_SB(sb)->opts; - - return snprintf(buf, PAGE_SIZE, "%s\n", opts->uniq_name); -} -SCOUTFS_ATTR_RO(uniq_name); - static struct attribute *mount_options_attrs[] = { SCOUTFS_ATTR_PTR(server_addr), - SCOUTFS_ATTR_PTR(uniq_name), NULL, }; From ab7bde9e2cb273d18cc3a7496baeda7fb09f0639 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 10 Jul 2019 14:06:18 -0700 Subject: [PATCH 734/920] scoutfs: replace node_id with rid in networking Use the client's rid in networking instead of the node_id. The node_id no longer has to be allocated by the server and sent in the greeting. Instead the client sends it to the server in its greeting. The server then uses the client's announced rid just like it used to use the its node_id. It's used to record clients in the btree and to identify clients in sending and receive processing. The use of the rid in networking calls makes its way to locking and compaction which now use the rid to identify clients intead of the node_id. Signed-off-by: Zach Brown --- kmod/src/client.c | 55 +++---------- kmod/src/client.h | 1 - kmod/src/format.h | 22 +++--- kmod/src/lock_server.c | 111 ++++++++++++++------------- kmod/src/lock_server.h | 10 +-- kmod/src/net.c | 80 +++++++++---------- kmod/src/net.h | 13 ++-- kmod/src/scoutfs_trace.h | 92 +++++++++++----------- kmod/src/server.c | 161 ++++++++++++++++++--------------------- kmod/src/server.h | 6 +- kmod/src/super.c | 1 - kmod/src/super.h | 1 - 12 files changed, 251 insertions(+), 302 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 8ef58589..e9d7d480 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -49,7 +49,6 @@ struct client_info { struct super_block *sb; struct scoutfs_net_connection *conn; - struct completion node_id_comp; atomic_t shutting_down; struct workqueue_struct *workq; @@ -299,8 +298,9 @@ static int client_lock_recover(struct super_block *sb, /* * Process a greeting response in the client from the server. This is - * called for every connected socket on the connection. The first - * response will have the node_id that the server assigned the client. + * called for every connected socket on the connection. Each response + * contains the remote server's elected term which can be used to + * identify server failover. */ static int client_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -309,7 +309,6 @@ static int client_greeting(struct super_block *sb, { struct client_info *client = SCOUTFS_SB(sb)->client_info; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_net_greeting *gr = resp; bool new_server; int ret; @@ -340,25 +339,6 @@ static int client_greeting(struct super_block *sb, goto out; } - if (sbi->node_id != 0 && le64_to_cpu(gr->node_id) != sbi->node_id) { - scoutfs_warn(sb, "server sent node_id %llu, client has %llu", - le64_to_cpu(gr->node_id), - sbi->node_id); - ret = -EINVAL; - goto out; - } - - if (sbi->node_id == 0 && gr->node_id == 0) { - scoutfs_warn(sb, "server sent node_id 0, client also has 0"); - ret = -EINVAL; - goto out; - } - - if (sbi->node_id == 0) { - sbi->node_id = le64_to_cpu(gr->node_id); - complete(&client->node_id_comp); - } - new_server = le64_to_cpu(gr->server_term) != client->server_term; scoutfs_net_client_greeting(sb, conn, new_server); @@ -466,7 +446,7 @@ static void scoutfs_client_connect_worker(struct work_struct *work) greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(client->server_term); greet.unmount_barrier = cpu_to_le64(client->greeting_umb); - greet.node_id = cpu_to_le64(sbi->node_id); + greet.rid = cpu_to_le64(sbi->rid); greet.flags = 0; if (client->sending_farewell) greet.flags |= cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL); @@ -553,7 +533,7 @@ static scoutfs_net_request_t client_req_funcs[] = { */ static void client_notify_down(struct super_block *sb, struct scoutfs_net_connection *conn, void *info, - u64 node_id) + u64 rid) { struct client_info *client = SCOUTFS_SB(sb)->client_info; @@ -561,18 +541,6 @@ static void client_notify_down(struct super_block *sb, queue_delayed_work(client->workq, &client->connect_dwork, 0); } -/* - * Wait for the first connected socket on the connection that assigns - * the node_id that will be used for the rest of the life time of the - * mount. - */ -int scoutfs_client_wait_node_id(struct super_block *sb) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - return wait_for_completion_interruptible(&client->node_id_comp); -} - int scoutfs_client_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -587,7 +555,6 @@ int scoutfs_client_setup(struct super_block *sb) sbi->client_info = client; client->sb = sb; - init_completion(&client->node_id_comp); atomic_set(&client->shutting_down, 0); INIT_DELAYED_WORK(&client->connect_dwork, scoutfs_client_connect_worker); @@ -640,12 +607,12 @@ static int client_farewell_response(struct super_block *sb, * so that they don't wait for us to reconnect and trigger a timeout. * * This decision is a little racy. The server considers us connected - * when it assigns us a node_id as it processes the greeting. We can - * disconnect before receiving the response and leave without sending a - * farewell. So given that awkward initial race, we also have a bit of - * a race where we just test the server_term to see if we've ever gotten - * a greeting reply from any server. We don't try to synchronize with - * pending connection attempts. + * when it records a persistent record of our rid as it processes our + * greeting. We can disconnect before receiving the greeting response + * and leave without sending a farewell. So given that awkward initial + * race, we also have a bit of a race where we just test the server_term + * to see if we've ever gotten a greeting reply from any server. We + * don't try to synchronize with pending connection attempts. * * The consequences of aborting a mount at just the wrong time and * disconnecting without the farewell handshake depend on what the diff --git a/kmod/src/client.h b/kmod/src/client.h index dd0c2eae..fe938306 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -24,7 +24,6 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock_recover *nlr); -int scoutfs_client_wait_node_id(struct super_block *sb); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index c2f2fa15..2534fe16 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -287,7 +287,7 @@ struct scoutfs_extent_btree_key { * server failover knows who to wait for before resuming operations. */ struct scoutfs_lock_client_btree_key { - __be64 node_id; + __be64 rid; } __packed; /* @@ -296,14 +296,14 @@ struct scoutfs_lock_client_btree_key { */ struct scoutfs_trans_seq_btree_key { __be64 trans_seq; - __be64 node_id; + __be64 rid; } __packed; /* * The server keeps a persistent record of mounted clients. */ struct scoutfs_mounted_client_btree_key { - __be64 node_id; + __be64 rid; } __packed; struct scoutfs_mounted_client_btree_val { @@ -479,7 +479,6 @@ struct scoutfs_super_block { __le64 alloc_cursor; struct scoutfs_btree_ring bring; __le64 next_seg_seq; - __le64 next_node_id; __le64 next_compact_id; __le64 quorum_fenced_term; __le64 quorum_server_term; @@ -611,8 +610,9 @@ enum { * * @server_term: The raft term that elected the server. Initially 0 * from the client, sent by the server, then sent by the client as it - * tries to reconnect. Used to identify a client reconnecting to a - * server that has timed out its connection. + * tries to reconnect. Used to identify a client reconnecting to both + * the same serer after receiving a greeting response and to a new + * server after failover. * * @unmount_barrier: Incremented every time the remaining majority of * quorum members all agree to leave. The server tells a quorum member @@ -620,17 +620,17 @@ enum { * value increase in the super block then it knows that the server has * processed its farewell and can safely unmount. * - * @node_id: The id of the client. Initially 0 from the client, - * assigned by the server, and sent by the client as it reconnects. - * Used by the server to identify reconnecting clients whose existing - * state must be dealt with. + * @rid: The client's random id that was generated once as the mount + * started up. This identifies a specific remote mount across + * connections and servers. It's set to the client's rid in both the + * request and response for consistency. */ struct scoutfs_net_greeting { __le64 fsid; __le64 format_hash; __le64 server_term; __le64 unmount_barrier; - __le64 node_id; + __le64 rid; __le64 flags; } __packed; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index da7e1062..3f93c150 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -50,10 +50,10 @@ * server doesn't use the modes specified by the clients but they're * provided to add context. * - * The server relies on the node_id allocation and reliable messaging - * layers of the system. Each client has a node_id that is unique for - * its life time. Message requests and responses are reliably - * delivered in order across reconnection. + * The server relies on the client's static rid and on reliable + * messaging. Each client has a rid that is unique for its life time. + * Message requests and responses are reliably delivered in order across + * reconnection. * * The server maintains a persistent record of connected clients. A new * server instance discovers these and waits for previously connected @@ -126,7 +126,7 @@ enum { * that the mode is actively granted, a pending request from the client, * or a pending invalidation sent to the client. * - * @node_id: The client's node_id used to send messages and tear down + * @rid: The client's rid used to send messages and tear down * state as client's exit. * * @net_id: The id of a client's request used to send grant responses. The @@ -139,7 +139,7 @@ enum { */ struct client_lock_entry { struct list_head head; - u64 node_id; + u64 rid; u64 net_id; u8 mode; @@ -221,7 +221,7 @@ static bool client_entries_compatible(struct client_lock_entry *granted, return (granted->mode == requested->mode && (granted->mode == SCOUTFS_LOCK_READ || granted->mode == SCOUTFS_LOCK_WRITE_ONLY)) || - (granted->node_id == requested->node_id && + (granted->rid == requested->rid && granted->mode == SCOUTFS_LOCK_READ && requested->mode == SCOUTFS_LOCK_WRITE); } @@ -340,14 +340,14 @@ static void put_server_lock(struct lock_server_info *inf, static struct client_lock_entry *find_entry(struct server_lock_node *snode, struct list_head *list, - u64 node_id) + u64 rid) { struct client_lock_entry *clent; WARN_ON_ONCE(!mutex_is_locked(&snode->mutex)); list_for_each_entry(clent, list, head) { - if (clent->node_id == node_id) + if (clent->rid == rid) return clent; } @@ -363,7 +363,7 @@ static int process_waiting_requests(struct super_block *sb, * * XXX shut down if we get enomem? */ -int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -372,7 +372,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, int ret; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, SLT_REQUEST, - node_id, net_id, nl); + rid, net_id, nl); if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { ret = -EINVAL; @@ -386,7 +386,7 @@ int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, } INIT_LIST_HEAD(&clent->head); - clent->node_id = node_id; + clent->rid = rid; clent->net_id = net_id; clent->mode = nl->new_mode; @@ -414,7 +414,7 @@ out: * * XXX what to do with errors? kick the client? */ -int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -423,7 +423,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, int ret; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_RESPONSE, - node_id, 0, nl); + rid, 0, nl); if (invalid_mode(nl->old_mode) || invalid_mode(nl->new_mode)) { ret = -EINVAL; @@ -437,7 +437,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, goto out; } - clent = find_entry(snode, &snode->invalidated, node_id); + clent = find_entry(snode, &snode->invalidated, rid); if (!clent) { put_server_lock(inf, snode); ret = -EINVAL; @@ -512,13 +512,13 @@ static int process_waiting_requests(struct super_block *sb, nl.old_mode = gr->mode; nl.new_mode = invalidation_mode(gr->mode, req->mode); - ret = scoutfs_server_lock_request(sb, gr->node_id, &nl); + ret = scoutfs_server_lock_request(sb, gr->rid, &nl); if (ret) goto out; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_INVALIDATE, SLT_REQUEST, - gr->node_id, 0, &nl); + gr->rid, 0, &nl); add_client_entry(snode, &snode->invalidated, gr); } @@ -531,7 +531,7 @@ static int process_waiting_requests(struct super_block *sb, nl.new_mode = req->mode; /* see if there's an existing compatible grant to replace */ - gr = find_entry(snode, &snode->granted, req->node_id); + gr = find_entry(snode, &snode->granted, req->rid); if (gr) { nl.old_mode = gr->mode; free_client_entry(inf, snode, gr); @@ -539,13 +539,13 @@ static int process_waiting_requests(struct super_block *sb, nl.old_mode = SCOUTFS_LOCK_NULL; } - ret = scoutfs_server_lock_response(sb, req->node_id, + ret = scoutfs_server_lock_response(sb, req->rid, req->net_id, &nl); if (ret) goto out; trace_scoutfs_lock_message(sb, SLT_SERVER, SLT_GRANT, - SLT_RESPONSE, req->node_id, + SLT_RESPONSE, req->rid, req->net_id, &nl); /* don't track null client locks, track all else */ @@ -571,7 +571,7 @@ out: * * This is running in concurrent client greeting processing contexts. */ -int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, bool should_exist) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -581,7 +581,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, struct scoutfs_key key; int ret; - cbk.node_id = cpu_to_be64(node_id); + cbk.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); if (should_exist) { @@ -597,7 +597,7 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, if (should_exist && ret == 0) { scoutfs_key_set_zeros(&key); - ret = scoutfs_server_lock_recover_request(sb, node_id, &key); + ret = scoutfs_server_lock_recover_request(sb, rid, &key); if (ret) goto out; } @@ -611,7 +611,7 @@ out: * they were the last client in recovery then we can process all the * server locks that had requests. */ -static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) +static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct server_lock_node *snode; @@ -620,7 +620,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) int ret = 0; spin_lock(&inf->lock); - scoutfs_spbm_clear(&inf->recovery_pending, node_id); + scoutfs_spbm_clear(&inf->recovery_pending, rid); still_pending = !scoutfs_spbm_empty(&inf->recovery_pending); spin_unlock(&inf->lock); if (still_pending) @@ -654,7 +654,7 @@ static int finished_recovery(struct super_block *sb, u64 node_id, bool cancel) * gave us in response and send another request from the next key. * We're done once we receive an empty response. */ -int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr) { DECLARE_LOCK_SERVER_INFO(sb, inf); @@ -667,7 +667,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, /* client must be in recovery */ spin_lock(&inf->lock); - if (!scoutfs_spbm_test(&inf->recovery_pending, node_id)) + if (!scoutfs_spbm_test(&inf->recovery_pending, rid)) ret = -EINVAL; spin_unlock(&inf->lock); if (ret) @@ -675,7 +675,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, /* client has sent us all their locks */ if (nlr->nr == 0) { - ret = finished_recovery(sb, node_id, true); + ret = finished_recovery(sb, rid, true); goto out; } @@ -687,7 +687,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, } INIT_LIST_HEAD(&clent->head); - clent->node_id = node_id; + clent->rid = rid; clent->net_id = 0; clent->mode = nlr->locks[i].new_mode; @@ -698,7 +698,7 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, goto out; } - existing = find_entry(snode, &snode->granted, node_id); + existing = find_entry(snode, &snode->granted, rid); if (existing) { kfree(clent); put_server_lock(inf, snode); @@ -717,20 +717,20 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, key = nlr->locks[le16_to_cpu(nlr->nr) - 1].key; scoutfs_key_inc(&key); - ret = scoutfs_server_lock_recover_request(sb, node_id, &key); + ret = scoutfs_server_lock_recover_request(sb, rid, &key); out: return ret; } -static int node_id_and_put_iref(struct scoutfs_btree_item_ref *iref, - u64 *node_id) +static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, + u64 *rid) { struct scoutfs_lock_client_btree_key *cbk; int ret; if (iref->key_len == sizeof(*cbk) && iref->val_len == 0) { cbk = iref->key; - *node_id = be64_to_cpu(cbk->node_id); + *rid = be64_to_cpu(cbk->rid); ret = 0; } else { ret = -EIO; @@ -754,12 +754,12 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) struct scoutfs_lock_client_btree_key cbk; SCOUTFS_BTREE_ITEM_REF(iref); bool timed_out; - u64 node_id; + u64 rid; int ret; /* we enter recovery if there are any client records */ - for (node_id = 0; ; node_id++) { - cbk.node_id = cpu_to_be64(node_id); + for (rid = 0; ; rid++) { + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_next(sb, &super->lock_clients, &cbk, sizeof(cbk), &iref); if (ret == -ENOENT) { @@ -767,13 +767,13 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) break; } if (ret == 0) - ret = node_id_and_put_iref(&iref, &node_id); + ret = get_rid_and_put_ref(&iref, &rid); if (ret < 0) break; spin_lock(&inf->lock); - if (scoutfs_spbm_test(&inf->recovery_pending, node_id)) { - scoutfs_spbm_clear(&inf->recovery_pending, node_id); + if (scoutfs_spbm_test(&inf->recovery_pending, rid)) { + scoutfs_spbm_clear(&inf->recovery_pending, rid); timed_out = true; } else { timed_out = false; @@ -783,11 +783,11 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) if (!timed_out) continue; - scoutfs_err(sb, "client node_id %llu lock recovery timed out", - node_id); + scoutfs_err(sb, "client rid %016llx lock recovery timed out", + rid); /* XXX these aren't immediately committed */ - cbk.node_id = cpu_to_be64(node_id); + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->lock_clients, &cbk, sizeof(cbk)); if (ret) @@ -812,7 +812,7 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) * If we fail to delete a persistent entry then we have to shut down and * hope that the next server has more luck. */ -int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -825,7 +825,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) bool freed; int ret = 0; - cli.node_id = cpu_to_be64(node_id); + cli.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); ret = scoutfs_btree_delete(sb, &super->lock_clients, &cli, sizeof(cli)); mutex_unlock(&inf->mutex); @@ -847,7 +847,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) NULL) { list_for_each_entry_safe(clent, tmp, list, head) { - if (clent->node_id == node_id) { + if (clent->rid == rid) { free_client_entry(inf, snode, clent); freed = true; } @@ -869,7 +869,8 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id) out: if (ret < 0) { - scoutfs_err(sb, "lock server err %d during node %llu farewell, shutting down", ret, node_id); + scoutfs_err(sb, "lock server err %d during client rid %016llx farewell, shutting down", + ret, rid); scoutfs_server_abort(sb); } @@ -913,9 +914,9 @@ static void lock_server_tseq_show(struct seq_file *m, tseq_entry); struct server_lock_node *snode = clent->snode; - seq_printf(m, SK_FMT" %s %s node_id %llu net_id %llu\n", + seq_printf(m, SK_FMT" %s %s rid %016llx net_id %llu\n", SK_ARG(&snode->key), lock_mode_string(clent->mode), - lock_on_list_string(clent->on_list), clent->node_id, + lock_on_list_string(clent->on_list), clent->rid, clent->net_id); } @@ -934,7 +935,7 @@ int scoutfs_lock_server_setup(struct super_block *sb) SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_lock_client_btree_key cbk; unsigned int nr; - u64 node_id; + u64 rid; int ret; inf = kzalloc(sizeof(struct lock_server_info), GFP_KERNEL); @@ -961,23 +962,23 @@ int scoutfs_lock_server_setup(struct super_block *sb) /* we enter recovery if there are any client records */ nr = 0; - for (node_id = 0; ; node_id++) { - cbk.node_id = cpu_to_be64(node_id); + for (rid = 0; ; rid++) { + cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_next(sb, &super->lock_clients, &cbk, sizeof(cbk), &iref); if (ret == -ENOENT) break; if (ret == 0) - ret = node_id_and_put_iref(&iref, &node_id); + ret = get_rid_and_put_ref(&iref, &rid); if (ret < 0) goto out; - ret = scoutfs_spbm_set(&inf->recovery_pending, node_id); + ret = scoutfs_spbm_set(&inf->recovery_pending, rid); if (ret) goto out; nr++; - if (node_id == U64_MAX) + if (rid == U64_MAX) break; } ret = 0; diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index cc0606a8..0ac2f772 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -1,15 +1,15 @@ #ifndef _SCOUTFS_LOCK_SERVER_H_ #define _SCOUTFS_LOCK_SERVER_H_ -int scoutfs_lock_server_recover_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock_recover *nlr); -int scoutfs_lock_server_request(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_request(struct super_block *sb, u64 rid, u64 net_id, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_greeting(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, bool should_exist); -int scoutfs_lock_server_response(struct super_block *sb, u64 node_id, +int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); -int scoutfs_lock_server_farewell(struct super_block *sb, u64 node_id); +int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); int scoutfs_lock_server_setup(struct super_block *sb); void scoutfs_lock_server_destroy(struct super_block *sb); diff --git a/kmod/src/net.c b/kmod/src/net.c index 8449ce56..1177336d 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -109,7 +109,7 @@ struct scoutfs_net_connection { unsigned long connect_timeout_ms; struct socket *sock; - u64 node_id; /* assigned during greeting */ + u64 rid; u64 greeting_id; struct sockaddr_in sockname; struct sockaddr_in peername; @@ -344,12 +344,12 @@ static void shutdown_conn(struct scoutfs_net_connection *conn) * connection has passed the greeting and isn't being shut down. At all * other times we add new sends to the resend queue. * - * If a non-zero node_id is specified then the conn argument is a listening + * If a non-zero rid is specified then the conn argument is a listening * connection and the connection to send the message down is found by - * searching for the node_id in its accepted connections. + * searching for the rid in its accepted connections. */ static int submit_send(struct super_block *sb, - struct scoutfs_net_connection *conn, u64 node_id, + struct scoutfs_net_connection *conn, u64 rid, u8 cmd, u8 flags, u64 id, u8 net_err, void *data, u16 data_len, scoutfs_net_response_t resp_func, void *resp_data, @@ -376,19 +376,19 @@ static int submit_send(struct super_block *sb, spin_lock_nested(&conn->lock, CONN_LOCK_LISTENER); - if (node_id != 0) { + if (rid != 0) { list_for_each_entry(acc_conn, &conn->accepted_list, accepted_head) { - if (acc_conn->node_id == node_id) { + if (acc_conn->rid == rid) { spin_lock_nested(&acc_conn->lock, CONN_LOCK_ACCEPTED); spin_unlock(&conn->lock); conn = acc_conn; - node_id = 0; + rid = 0; break; } } - if (node_id != 0) { + if (rid != 0) { spin_unlock(&conn->lock); return -ENOTCONN; } @@ -845,7 +845,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) /* tell callers that accepted connection finally done */ if (conn->listening_conn && conn->notify_down) - conn->notify_down(sb, conn, conn->info, conn->node_id); + conn->notify_down(sb, conn, conn->info, conn->rid); /* free all messages, refactor and complete for forced unmount? */ list_splice_init(&conn->resend_queue, &conn->send_queue); @@ -1182,7 +1182,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) conn->shutting_down = 0; if (conn->notify_down) conn->notify_down(sb, conn, conn->info, - conn->node_id); + conn->rid); } spin_unlock(&conn->lock); @@ -1251,12 +1251,12 @@ restart: * Accepted connections inherit the callbacks from their listening * connection. * - * notify_up is called once a valid greeting is received. node_id is + * notify_up is called once a valid greeting is received. rid is * non-zero on accepted sockets once they've seen a valid greeting. - * Connected and listening connections have a node_id of 0. + * Connected and listening connections have a rid of 0. * * notify_down is always called as connections are shut down. It can be - * called without notify_up ever being called. The node_id is only + * called without notify_up ever being called. The rid is only * non-zero for accepted connections. */ struct scoutfs_net_connection * @@ -1317,14 +1317,15 @@ scoutfs_net_alloc_conn(struct super_block *sb, } /* - * Give the caller the client node_id of the connection. This used by - * rare server processing callers who want to send async responses after - * request processing has returned. We didn't want to plumb the - * requesting node_id into all the request handlers but that'd work too. + * Give the caller the client rid of the connection. This used by rare + * server processing callers who want to send async responses after + * request processing has returned. We didn't want the churn of + * providing the requesting rid to all the request handlers, but we + * probably should. */ -u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn) +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn) { - return conn->node_id; + return conn->rid; } /* @@ -1520,7 +1521,7 @@ void scoutfs_net_client_greeting(struct super_block *sb, /* * The calling server has received a valid greeting from a client. If - * the server is reconnecting to us then we need to find its old + * the client is reconnecting to us then we need to find its old * connection that held its state and transfer it to this connection * (connection and socket life cycles make this easier than migrating * the socket between the connections). @@ -1532,9 +1533,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, * referring to the same original connection. We use the increasing * greeting id to have the most recent connection attempt win. * - * A node can be reconnecting to us for the first time. In that case we - * just trust its node_id. It will notice the new server term and take - * steps to recover. + * A node can be reconnecting to us for the first time. It will notice + * the new server term and take steps to recover. * * A client can be reconnecting to us after we've destroyed their state. * This is fatal for the client if they just took too long to reconnect. @@ -1550,8 +1550,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, */ void scoutfs_net_server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u64 greeting_id, - bool sent_node_id, bool first_contact, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, bool farewell) { struct scoutfs_net_connection *listener; @@ -1561,15 +1561,15 @@ void scoutfs_net_server_greeting(struct super_block *sb, /* only called on accepted server connections :/ */ BUG_ON(!conn->listening_conn); - /* see if we have a previous conn for the client's sent node_id */ + /* see if we have a previous conn for the client's sent rid */ reconn = NULL; - if (sent_node_id) { + if (reconnecting) { listener = conn->listening_conn; restart: spin_lock_nested(&listener->lock, CONN_LOCK_LISTENER); list_for_each_entry(acc, &listener->accepted_list, accepted_head) { - if (acc->node_id != node_id || + if (acc->rid != rid || acc->greeting_id >= greeting_id || acc->reconn_freeing) continue; @@ -1592,12 +1592,12 @@ restart: } /* drop a connection if we can't find its necessary old conn */ - if (sent_node_id && !reconn && !first_contact && !farewell) { + if (reconnecting && !reconn && !first_contact && !farewell) { shutdown_conn(conn); return; } - /* migrate state from previous conn for this reconnecting node_id */ + /* migrate state from previous conn for this reconnecting rid */ if (reconn) { spin_lock(&conn->lock); @@ -1623,15 +1623,15 @@ restart: spin_lock(&conn->lock); - conn->node_id = node_id; + conn->rid = rid; conn->greeting_id = greeting_id; set_valid_greeting(conn); spin_unlock(&conn->lock); - /* only call notify_up the first time we see the node_id */ + /* only call notify_up the first time we see the rid */ if (conn->notify_up && first_contact) - conn->notify_up(sb, conn, conn->info, node_id); + conn->notify_up(sb, conn, conn->info, rid); } /* @@ -1651,17 +1651,17 @@ int scoutfs_net_submit_request(struct super_block *sb, } /* - * Send a request to a specific node_id that was accepted by this listening + * Send a request to a specific rid that was accepted by this listening * connection. */ int scoutfs_net_submit_request_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, + u64 rid, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret) { - return submit_send(sb, conn, node_id, cmd, 0, 0, 0, arg, arg_len, + return submit_send(sb, conn, rid, cmd, 0, 0, 0, arg, arg_len, resp_func, resp_data, id_ret); } @@ -1687,7 +1687,7 @@ int scoutfs_net_response(struct super_block *sb, int scoutfs_net_response_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, u64 id, int error, + u64 rid, u8 cmd, u64 id, int error, void *resp, u16 resp_len) { if (error) { @@ -1695,7 +1695,7 @@ int scoutfs_net_response_node(struct super_block *sb, resp_len = 0; } - return submit_send(sb, conn, node_id, cmd, SCOUTFS_NET_FLAG_RESPONSE, + return submit_send(sb, conn, rid, cmd, SCOUTFS_NET_FLAG_RESPONSE, id, net_err_from_host(sb, error), resp, resp_len, NULL, NULL, NULL); } @@ -1787,9 +1787,9 @@ static void net_tseq_show_conn(struct seq_file *m, struct scoutfs_net_connection *conn = container_of(ent, struct scoutfs_net_connection, tseq_entry); - seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" node_id %llu greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", + seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" rid %016llx greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), - conn->node_id, conn->greeting_id, conn->valid_greeting, + conn->rid, conn->greeting_id, conn->valid_greeting, conn->established, conn->shutting_down, conn->saw_greeting, conn->saw_farewell, conn->reconn_wait, conn->reconn_freeing, conn->connect_timeout_ms, conn->reconn_deadline, diff --git a/kmod/src/net.h b/kmod/src/net.h index a15e005a..0113c16c 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -37,14 +37,14 @@ typedef int (*scoutfs_net_response_t)(struct super_block *sb, typedef void (*scoutfs_net_notify_t)(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id); + void *info, u64 rid); struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, scoutfs_net_notify_t notify_down, size_t info_size, scoutfs_net_request_t *req_funcs, char *name_suffix); -u64 scoutfs_net_client_node_id(struct scoutfs_net_connection *conn); +u64 scoutfs_net_client_rid(struct scoutfs_net_connection *conn); int scoutfs_net_connect(struct super_block *sb, struct scoutfs_net_connection *conn, struct sockaddr_in *sin, unsigned long timeout_ms); @@ -60,8 +60,7 @@ int scoutfs_net_submit_request(struct super_block *sb, void *resp_data, u64 *id_ret); int scoutfs_net_submit_request_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, - void *arg, u16 arg_len, + u64 rid, u8 cmd, void *arg, u16 arg_len, scoutfs_net_response_t resp_func, void *resp_data, u64 *id_ret); void scoutfs_net_cancel_request(struct super_block *sb, @@ -76,7 +75,7 @@ int scoutfs_net_response(struct super_block *sb, u8 cmd, u64 id, int error, void *resp, u16 resp_len); int scoutfs_net_response_node(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u8 cmd, u64 id, int error, + u64 rid, u8 cmd, u64 id, int error, void *resp, u16 resp_len); void scoutfs_net_shutdown(struct super_block *sb, struct scoutfs_net_connection *conn); @@ -88,8 +87,8 @@ void scoutfs_net_client_greeting(struct super_block *sb, bool new_server); void scoutfs_net_server_greeting(struct super_block *sb, struct scoutfs_net_connection *conn, - u64 node_id, u64 greeting_id, - bool sent_node_id, bool first_contact, + u64 rid, u64 greeting_id, + bool reconnecting, bool first_contact, bool farewell); void scoutfs_net_farewell(struct super_block *sb, struct scoutfs_net_connection *conn); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index d87dcb2c..b452620d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1154,17 +1154,17 @@ TRACE_EVENT(scoutfs_client_compact_stop, ); TRACE_EVENT(scoutfs_server_compact_start, - TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 node_id, + TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 rid, unsigned long client_nr, unsigned long server_nr, unsigned long per_client), - TP_ARGS(sb, id, level, node_id, client_nr, server_nr, per_client), + TP_ARGS(sb, id, level, rid, client_nr, server_nr, per_client), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, id) __field(__u8, level) - __field(__u64, node_id) + __field(__u64, c_rid) __field(unsigned long, client_nr) __field(unsigned long, server_nr) __field(unsigned long, per_client) @@ -1174,40 +1174,40 @@ TRACE_EVENT(scoutfs_server_compact_start, SCSB_TRACE_ASSIGN(sb); __entry->id = id; __entry->level = level; - __entry->node_id = node_id; + __entry->c_rid = rid; __entry->client_nr = client_nr; __entry->server_nr = server_nr; __entry->per_client = per_client; ), - TP_printk(SCSBF" id %llu level %u node_id %llu client_nr %lu server_nr %lu per_client %lu", + TP_printk(SCSBF" id %llu level %u rid %016llx client_nr %lu server_nr %lu per_client %lu", SCSB_TRACE_ARGS, __entry->id, __entry->level, - __entry->node_id, __entry->client_nr, __entry->server_nr, + __entry->c_rid, __entry->client_nr, __entry->server_nr, __entry->per_client) ); TRACE_EVENT(scoutfs_server_compact_done, - TP_PROTO(struct super_block *sb, u64 id, u64 node_id, + TP_PROTO(struct super_block *sb, u64 id, u64 rid, unsigned long server_nr), - TP_ARGS(sb, id, node_id, server_nr), + TP_ARGS(sb, id, rid, server_nr), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, id) - __field(__u64, node_id) + __field(__u64, c_rid) __field(unsigned long, server_nr) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->id = id; - __entry->node_id = node_id; + __entry->rid = rid; __entry->server_nr = server_nr; ), - TP_printk(SCSBF" id %llu node_id %llu server_nr %lu", - SCSB_TRACE_ARGS, __entry->id, __entry->node_id, + TP_printk(SCSBF" id %llu rid %016llx server_nr %lu", + SCSB_TRACE_ARGS, __entry->id, __entry->c_rid, __entry->server_nr) ); @@ -2470,32 +2470,32 @@ DEFINE_EVENT(scoutfs_segno_class, scoutfs_remove_segno, ); DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients), + TP_ARGS(sb, rid, nr_clients), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__s64, node_id) + __field(__s64, c_rid) __field(unsigned long, nr_clients) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->c_rid = rid; __entry->nr_clients = nr_clients; ), - TP_printk(SCSBF" node_id %llu nr_clients %lu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->nr_clients) + TP_printk(SCSBF" rid %016llx nr_clients %lu", + SCSB_TRACE_ARGS, __entry->c_rid, __entry->nr_clients) ); DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_up, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients) + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) ); DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, - TP_PROTO(struct super_block *sb, u64 node_id, unsigned long nr_clients), - TP_ARGS(sb, node_id, nr_clients) + TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), + TP_ARGS(sb, rid, nr_clients) ); #define slt_symbolic(mode) \ @@ -2509,16 +2509,16 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down, TRACE_EVENT(scoutfs_lock_message, TP_PROTO(struct super_block *sb, int who, int what, int dir, - u64 node_id, u64 net_id, struct scoutfs_net_lock *nl), + u64 rid, u64 net_id, struct scoutfs_net_lock *nl), - TP_ARGS(sb, who, what, dir, node_id, net_id, nl), + TP_ARGS(sb, who, what, dir, rid, net_id, nl), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(int, who) __field(int, what) __field(int, dir) - __field(__u64, node_id) + __field(__u64, m_rid) __field(__u64, net_id) sk_trace_define(key) __field(__u8, old_mode) @@ -2530,17 +2530,17 @@ TRACE_EVENT(scoutfs_lock_message, __entry->who = who; __entry->what = what; __entry->dir = dir; - __entry->node_id = node_id; + __entry->m_rid = rid; __entry->net_id = net_id; sk_trace_assign(key, &nl->key); __entry->old_mode = nl->old_mode; __entry->new_mode = nl->new_mode; ), - TP_printk(SCSBF" %s %s %s node_id %llu net_id %llu key "SK_FMT" old_mode %u new_mode %u", + TP_printk(SCSBF" %s %s %s rid %016llx net_id %llu key "SK_FMT" old_mode %u new_mode %u", SCSB_TRACE_ARGS, slt_symbolic(__entry->who), slt_symbolic(__entry->what), slt_symbolic(__entry->dir), - __entry->node_id, __entry->net_id, sk_trace_args(key), + __entry->m_rid, __entry->net_id, sk_trace_args(key), __entry->old_mode, __entry->new_mode) ); @@ -2685,70 +2685,70 @@ DEFINE_EVENT(scoutfs_clock_sync_class, scoutfs_recv_clock_sync, ); TRACE_EVENT(scoutfs_trans_seq_advance, - TP_PROTO(struct super_block *sb, u64 node_id, u64 prev_seq, + TP_PROTO(struct super_block *sb, u64 rid, u64 prev_seq, u64 next_seq), - TP_ARGS(sb, node_id, prev_seq, next_seq), + TP_ARGS(sb, rid, prev_seq, next_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, prev_seq) __field(__u64, next_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->prev_seq = prev_seq; __entry->next_seq = next_seq; ), - TP_printk(SCSBF" node_id %llu prev_seq %llu next_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->prev_seq, + TP_printk(SCSBF" rid %016llx prev_seq %llu next_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->prev_seq, __entry->next_seq) ); TRACE_EVENT(scoutfs_trans_seq_farewell, - TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), - TP_ARGS(sb, node_id, trans_seq), + TP_ARGS(sb, rid, trans_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, trans_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->trans_seq = trans_seq; ), - TP_printk(SCSBF" node_id %llu trans_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) ); TRACE_EVENT(scoutfs_trans_seq_last, - TP_PROTO(struct super_block *sb, u64 node_id, u64 trans_seq), + TP_PROTO(struct super_block *sb, u64 rid, u64 trans_seq), - TP_ARGS(sb, node_id, trans_seq), + TP_ARGS(sb, rid, trans_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, node_id) + __field(__u64, s_rid) __field(__u64, trans_seq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->node_id = node_id; + __entry->s_rid = rid; __entry->trans_seq = trans_seq; ), - TP_printk(SCSBF" node_id %llu trans_seq %llu", - SCSB_TRACE_ARGS, __entry->node_id, __entry->trans_seq) + TP_printk(SCSBF" rid %016llx trans_seq %llu", + SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) ); #endif /* _TRACE_SCOUTFS_H */ diff --git a/kmod/src/server.c b/kmod/src/server.c index 291c7df5..4fabfff6 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -99,7 +99,7 @@ struct server_info { * The server tracks each connected client. */ struct server_client_info { - u64 node_id; + u64 rid; struct list_head head; unsigned long nr_compacts; }; @@ -894,7 +894,7 @@ static int server_advance_seq(struct super_block *sb, __le64 their_seq; __le64 next_seq; struct scoutfs_trans_seq_btree_key tsk; - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); int ret; if (arg_len != sizeof(__le64)) { @@ -908,7 +908,7 @@ static int server_advance_seq(struct super_block *sb, if (their_seq != 0) { tsk.trans_seq = le64_to_be64(their_seq); - tsk.node_id = cpu_to_be64(node_id); + tsk.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->trans_seqs, &tsk, sizeof(tsk)); @@ -919,11 +919,11 @@ static int server_advance_seq(struct super_block *sb, next_seq = super->next_trans_seq; le64_add_cpu(&super->next_trans_seq, 1); - trace_scoutfs_trans_seq_advance(sb, node_id, le64_to_cpu(their_seq), + trace_scoutfs_trans_seq_advance(sb, rid, le64_to_cpu(their_seq), le64_to_cpu(next_seq)); tsk.trans_seq = le64_to_be64(next_seq); - tsk.node_id = cpu_to_be64(node_id); + tsk.rid = cpu_to_be64(rid); ret = scoutfs_btree_insert(sb, &super->trans_seqs, &tsk, sizeof(tsk), NULL, 0); @@ -946,7 +946,7 @@ out: * client's farewell is retransmitted so it's OK to not find any * entries. This is called with the server commit rwsem held. */ -static int remove_trans_seq(struct super_block *sb, u64 node_id) +static int remove_trans_seq(struct super_block *sb, u64 rid) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); @@ -958,7 +958,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) down_write(&server->seq_rwsem); tsk.trans_seq = 0; - tsk.node_id = 0; + tsk.rid = 0; for (;;) { ret = scoutfs_btree_next(sb, &super->trans_seqs, @@ -972,8 +972,8 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) memcpy(&tsk, iref.key, iref.key_len); scoutfs_btree_put_iref(&iref); - if (be64_to_cpu(tsk.node_id) == node_id) { - trace_scoutfs_trans_seq_farewell(sb, node_id, + if (be64_to_cpu(tsk.rid) == rid) { + trace_scoutfs_trans_seq_farewell(sb, rid, be64_to_cpu(tsk.trans_seq)); ret = scoutfs_btree_delete(sb, &super->trans_seqs, &tsk, sizeof(tsk)); @@ -981,7 +981,7 @@ static int remove_trans_seq(struct super_block *sb, u64 node_id) } be64_add_cpu(&tsk.trans_seq, 1); - tsk.node_id = 0; + tsk.rid = 0; } up_write(&server->seq_rwsem); @@ -1006,7 +1006,7 @@ static int server_get_last_seq(struct super_block *sb, struct scoutfs_super_block *super = &sbi->super; struct scoutfs_trans_seq_btree_key tsk; SCOUTFS_BTREE_ITEM_REF(iref); - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); __le64 last_seq = 0; int ret; @@ -1018,7 +1018,7 @@ static int server_get_last_seq(struct super_block *sb, down_read(&server->seq_rwsem); tsk.trans_seq = 0; - tsk.node_id = 0; + tsk.rid = 0; ret = scoutfs_btree_next(sb, &super->trans_seqs, &tsk, sizeof(tsk), &iref); @@ -1037,7 +1037,7 @@ static int server_get_last_seq(struct super_block *sb, ret = 0; } - trace_scoutfs_trans_seq_last(sb, node_id, le64_to_cpu(last_seq)); + trace_scoutfs_trans_seq_last(sb, rid, le64_to_cpu(last_seq)); up_read(&server->seq_rwsem); out: @@ -1107,12 +1107,12 @@ static int server_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (arg_len != sizeof(struct scoutfs_net_lock)) return -EINVAL; - return scoutfs_lock_server_request(sb, node_id, id, arg); + return scoutfs_lock_server_request(sb, rid, id, arg); } static int lock_response(struct super_block *sb, @@ -1120,31 +1120,31 @@ static int lock_response(struct super_block *sb, void *resp, unsigned int resp_len, int error, void *data) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (resp_len != sizeof(struct scoutfs_net_lock)) return -EINVAL; - return scoutfs_lock_server_response(sb, node_id, resp); + return scoutfs_lock_server_response(sb, rid, resp); } -int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_submit_request_node(sb, server->conn, node_id, + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK, nl, sizeof(*nl), lock_response, NULL, NULL); } -int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock *nl) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_response_node(sb, server->conn, node_id, + return scoutfs_net_response_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK, id, 0, nl, sizeof(*nl)); } @@ -1162,34 +1162,34 @@ static int lock_recover_response(struct super_block *sb, void *resp, unsigned int resp_len, int error, void *data) { - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); if (invalid_recover(resp, resp_len)) return -EINVAL; - return scoutfs_lock_server_recover_response(sb, node_id, resp); + return scoutfs_lock_server_recover_response(sb, rid, resp); } -int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - return scoutfs_net_submit_request_node(sb, server->conn, node_id, + return scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK_RECOVER, key, sizeof(*key), lock_recover_response, NULL, NULL); } -static int insert_mounted_client(struct super_block *sb, u64 node_id, +static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; - mck.node_id = cpu_to_be64(node_id); + mck.rid = cpu_to_be64(rid); mcv.flags = 0; if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; @@ -1208,13 +1208,13 @@ static int insert_mounted_client(struct super_block *sb, u64 node_id, * * The caller has to serialize with farewell processing. */ -static int delete_mounted_client(struct super_block *sb, u64 node_id) +static int delete_mounted_client(struct super_block *sb, u64 rid) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; int ret; - mck.node_id = cpu_to_be64(node_id); + mck.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, &super->mounted_clients, &mck, sizeof(mck)); @@ -1230,17 +1230,8 @@ static int delete_mounted_client(struct super_block *sb, u64 node_id) * log some detail before shutting down. A failure to send a greeting * response shuts down the connection. * - * We allocate a new node_id for the first connect attempt from a - * client. - * - * If a client reconnects they'll send their initially assigned node_id - * in their greeting request. - * - * XXX We can lose allocated node_ids here as we record the node_id as - * live as we send a valid greeting response. The client might - * disconnect before they receive the response and resent and initial - * blank greeting. We could use a client uuid to associate with - * allocated node_ids. + * If a client reconnects they'll send their previously received + * serer_term in their greeting request. * * XXX The logic of this has gotten convoluted. The lock server can * send a recovery request so it needs to be called after the core net @@ -1259,9 +1250,8 @@ static int server_greeting(struct super_block *sb, struct scoutfs_net_greeting greet; DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; - __le64 node_id = 0; __le64 umb = 0; - bool sent_node_id; + bool reconnecting; bool first_contact; bool farewell; int ret = 0; @@ -1288,17 +1278,15 @@ static int server_greeting(struct super_block *sb, goto send_err; } - if (gr->node_id == 0) { + if (gr->server_term == 0) { down_read(&server->commit_rwsem); spin_lock(&server->lock); - node_id = super->next_node_id; - le64_add_cpu(&super->next_node_id, 1); umb = super->unmount_barrier; spin_unlock(&server->lock); mutex_lock(&server->farewell_mutex); - ret = insert_mounted_client(sb, le64_to_cpu(node_id), + ret = insert_mounted_client(sb, le64_to_cpu(gr->rid), le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); @@ -1310,20 +1298,17 @@ static int server_greeting(struct super_block *sb, queue_work(server->wq, &server->farewell_work); } } else { - node_id = gr->node_id; umb = gr->unmount_barrier; } send_err: err = ret; - if (err) - node_id = 0; greet.fsid = super->hdr.fsid; greet.format_hash = super->format_hash; greet.server_term = cpu_to_le64(server->term); greet.unmount_barrier = umb; - greet.node_id = node_id; + greet.rid = gr->rid; greet.flags = 0; /* queue greeting response to be sent first once messaging enabled */ @@ -1335,15 +1320,15 @@ send_err: goto out; /* have the net core enable messaging and resend */ - sent_node_id = gr->node_id != 0; + reconnecting = gr->server_term != 0; first_contact = le64_to_cpu(gr->server_term) != server->term; if (gr->flags & cpu_to_le64(SCOUTFS_NET_GREETING_FLAG_FAREWELL)) farewell = true; else farewell = false; - scoutfs_net_server_greeting(sb, conn, le64_to_cpu(node_id), id, - sent_node_id, first_contact, farewell); + scoutfs_net_server_greeting(sb, conn, le64_to_cpu(gr->rid), id, + reconnecting, first_contact, farewell); /* lock server might send recovery request */ if (le64_to_cpu(gr->server_term) != server->term) { @@ -1351,7 +1336,7 @@ send_err: /* we're now doing two commits per greeting, not great */ down_read(&server->commit_rwsem); - ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(node_id), + ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), gr->server_term != 0); if (ret == 0) queue_commit_work(server, &cw); @@ -1369,7 +1354,7 @@ out: struct farewell_request { struct list_head entry; u64 net_id; - u64 node_id; + u64 rid; }; static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) @@ -1435,7 +1420,7 @@ static void farewell_worker(struct work_struct *work) /* count how many reqs requests are from voting clients */ nr_unmounting = 0; list_for_each_entry_safe(fw, tmp, &reqs, entry) { - mck.node_id = cpu_to_be64(fw->node_id); + mck.rid = cpu_to_be64(fw->rid); ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &mck, sizeof(mck), &iref); if (ret == 0 && invalid_mounted_client_item(&iref)) { @@ -1484,7 +1469,7 @@ static void farewell_worker(struct work_struct *work) nr_mounted++; scoutfs_btree_put_iref(&iref); - be64_add_cpu(&mck.node_id, 1); + be64_add_cpu(&mck.rid, 1); } @@ -1505,9 +1490,9 @@ static void farewell_worker(struct work_struct *work) down_read(&server->commit_rwsem); - ret = scoutfs_lock_server_farewell(sb, fw->node_id) ?: - remove_trans_seq(sb, fw->node_id) ?: - delete_mounted_client(sb, fw->node_id); + ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: + remove_trans_seq(sb, fw->rid) ?: + delete_mounted_client(sb, fw->rid); if (ret == 0) queue_commit_work(server, &cw); @@ -1532,7 +1517,7 @@ static void farewell_worker(struct work_struct *work) /* and finally send all the responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = scoutfs_net_response_node(sb, server->conn, fw->node_id, + ret = scoutfs_net_response_node(sb, server->conn, fw->rid, SCOUTFS_NET_CMD_FAREWELL, fw->net_id, 0, NULL, 0); if (ret) @@ -1556,7 +1541,7 @@ out: queue_work(server->wq, &server->farewell_work); } -static void free_farewell_requests(struct super_block *sb, u64 node_id) +static void free_farewell_requests(struct super_block *sb, u64 rid) { struct server_info *server = SCOUTFS_SB(sb)->server_info; struct farewell_request *tmp; @@ -1564,7 +1549,7 @@ static void free_farewell_requests(struct super_block *sb, u64 node_id) mutex_lock(&server->farewell_mutex); list_for_each_entry_safe(fw, tmp, &server->farewell_requests, entry) { - if (node_id == 0 || fw->node_id == node_id) { + if (rid == 0 || fw->rid == rid) { list_del_init(&fw->entry); kfree(fw); } @@ -1588,7 +1573,7 @@ static int server_farewell(struct super_block *sb, u8 cmd, u64 id, void *arg, u16 arg_len) { struct server_info *server = SCOUTFS_SB(sb)->server_info; - u64 node_id = scoutfs_net_client_node_id(conn); + u64 rid = scoutfs_net_client_rid(conn); struct farewell_request *fw; if (arg_len != 0) @@ -1600,7 +1585,7 @@ static int server_farewell(struct super_block *sb, if (fw == NULL) return -ENOMEM; - fw->node_id = node_id; + fw->rid = rid; fw->net_id = id; mutex_lock(&server->farewell_mutex); @@ -1616,13 +1601,13 @@ static int server_farewell(struct super_block *sb, /* requests sent to clients are tracked so we can free resources */ struct compact_request { struct list_head head; - u64 node_id; + u64 rid; struct scoutfs_net_compact_request req; }; /* * Find a node that can process our compaction request. Return a - * node_id if we found a client and added the compaction to the client + * rid if we found a client and added the compaction to the client * and server counts. Returns 0 if no suitable clients were found. */ static u64 compact_request_start(struct super_block *sb, @@ -1631,7 +1616,7 @@ static u64 compact_request_start(struct super_block *sb, struct server_info *server = SCOUTFS_SB(sb)->server_info; struct server_client_info *last; struct server_client_info *sci; - u64 node_id = 0; + u64 rid = 0; spin_lock(&server->lock); @@ -1650,8 +1635,8 @@ static u64 compact_request_start(struct super_block *sb, list_add(&cr->head, &server->compacts); server->nr_compacts++; sci->nr_compacts++; - node_id = sci->node_id; - cr->node_id = node_id; + rid = sci->rid; + cr->rid = rid; break; } if (sci == last) @@ -1659,14 +1644,14 @@ static u64 compact_request_start(struct super_block *sb, } trace_scoutfs_server_compact_start(sb, le64_to_cpu(cr->req.id), - cr->req.ents[0].level, node_id, - node_id ? sci->nr_compacts : 0, + cr->req.ents[0].level, rid, + rid ? sci->nr_compacts : 0, server->nr_compacts, server->compacts_per_client); spin_unlock(&server->lock); - return node_id; + return rid; } /* @@ -1688,7 +1673,7 @@ static struct compact_request *compact_request_done(struct super_block *sb, continue; list_for_each_entry(sci, &server->clients, head) { - if (sci->node_id == cr->node_id) { + if (sci->rid == cr->rid) { sci->nr_compacts--; break; } @@ -1700,7 +1685,7 @@ static struct compact_request *compact_request_done(struct super_block *sb, break; } - trace_scoutfs_server_compact_done(sb, id, ret ? ret->node_id : 0, + trace_scoutfs_server_compact_done(sb, id, ret ? ret->rid : 0, server->nr_compacts); spin_unlock(&server->lock); @@ -1728,7 +1713,7 @@ static void forget_client_compacts(struct super_block *sb, spin_lock(&server->lock); list_for_each_entry_safe(cr, pos, &server->compacts, head) { - if (cr->node_id == sci->node_id) { + if (cr->rid == sci->rid) { sci->nr_compacts--; server->nr_compacts--; list_move(&cr->head, &forget); @@ -2129,7 +2114,7 @@ static void scoutfs_server_compact_worker(struct work_struct *work) struct compact_request *cr; struct commit_waiter cw; int nr_segnos = 0; - u64 node_id; + u64 rid; __le64 id; int ret; @@ -2169,14 +2154,14 @@ static void scoutfs_server_compact_worker(struct work_struct *work) /* try to send to a node with capacity, they can disconnect */ retry: req->id = id; - node_id = compact_request_start(sb, cr); - if (node_id == 0) { + rid = compact_request_start(sb, cr); + if (rid == 0) { ret = 0; goto out; } /* response processing can complete compaction before this returns */ - ret = scoutfs_net_submit_request_node(sb, server->conn, node_id, + ret = scoutfs_net_submit_request_node(sb, server->conn, rid, SCOUTFS_NET_CMD_COMPACT, req, sizeof(*req), compact_response, NULL, NULL); @@ -2228,18 +2213,18 @@ static scoutfs_net_request_t server_req_funcs[] = { static void server_notify_up(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id) + void *info, u64 rid) { struct server_client_info *sci = info; DECLARE_SERVER_INFO(sb, server); - if (node_id != 0) { - sci->node_id = node_id; + if (rid != 0) { + sci->rid = rid; sci->nr_compacts = 0; spin_lock(&server->lock); list_add_tail(&sci->head, &server->clients); server->nr_clients++; - trace_scoutfs_server_client_up(sb, node_id, server->nr_clients); + trace_scoutfs_server_client_up(sb, rid, server->nr_clients); spin_unlock(&server->lock); try_queue_compact(server); @@ -2248,20 +2233,20 @@ static void server_notify_up(struct super_block *sb, static void server_notify_down(struct super_block *sb, struct scoutfs_net_connection *conn, - void *info, u64 node_id) + void *info, u64 rid) { struct server_client_info *sci = info; DECLARE_SERVER_INFO(sb, server); - if (node_id != 0) { + if (rid != 0) { spin_lock(&server->lock); list_del_init(&sci->head); server->nr_clients--; - trace_scoutfs_server_client_down(sb, node_id, + trace_scoutfs_server_client_down(sb, rid, server->nr_clients); spin_unlock(&server->lock); - free_farewell_requests(sb, node_id); + free_farewell_requests(sb, rid); forget_client_compacts(sb, sci); try_queue_compact(server); diff --git a/kmod/src/server.h b/kmod/src/server.h index 32c6ccea..83103b9e 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -64,11 +64,11 @@ void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, struct scoutfs_net_manifest_entry *net_ment); -int scoutfs_server_lock_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); -int scoutfs_server_lock_response(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock *nl); -int scoutfs_server_lock_recover_request(struct super_block *sb, u64 node_id, +int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); struct sockaddr_in; diff --git a/kmod/src/super.c b/kmod/src/super.c index c5a8aeef..f160e1a3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -429,7 +429,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_quorum_setup(sb) ?: scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: - scoutfs_client_wait_node_id(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, &sbi->rid_lock); if (ret) diff --git a/kmod/src/super.h b/kmod/src/super.h index 741f2ddc..0545b34c 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -32,7 +32,6 @@ struct scoutfs_sb_info { /* assigned once at the start of each mount, read-only */ u64 rid; - u64 node_id; struct scoutfs_lock *rid_lock; struct scoutfs_super_block super; From 97f3971dcd5708aea419bd86a73a4cf7cbb1b4ae Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 Jul 2019 16:16:30 -0700 Subject: [PATCH 735/920] scoutfs: add rid sysfs file Add a "rid" file along the "fsid" file in the per-mount sysfs dir that gives the mounts rid. Signed-off-by: Zach Brown --- kmod/src/sysfs.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kmod/src/sysfs.c b/kmod/src/sysfs.c index 2c056c5c..bea667d0 100644 --- a/kmod/src/sysfs.c +++ b/kmod/src/sysfs.c @@ -43,10 +43,20 @@ static ssize_t fsid_show(struct kobject *kobj, struct attribute *attr, struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - return snprintf(buf, PAGE_SIZE, "%llx\n", le64_to_cpu(super->hdr.fsid)); + return snprintf(buf, PAGE_SIZE, "%016llx\n", + le64_to_cpu(super->hdr.fsid)); } ATTR_FUNCS_RO(fsid); +static ssize_t rid_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct super_block *sb = KOBJ_TO_SB(kobj, sb_id_kobj); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return snprintf(buf, PAGE_SIZE, "%016llx\n", sbi->rid); +} +ATTR_FUNCS_RO(rid); + /* * ops are defined per type, not per attribute. To have attributes with * different types that want different funcs we wrap them with a struct @@ -82,6 +92,7 @@ static ssize_t attr_funcs_show(struct kobject *kobj, struct attribute *attr, static struct attribute *sb_id_attrs[] = { &fsid_attr_funcs.attr, + &rid_attr_funcs.attr, NULL, }; KTYPE(sb_id); From a7ce9f22e2f72ed59360990f2f4ec36d7cb14eb3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 19 Jul 2019 16:37:44 -0700 Subject: [PATCH 736/920] scoutfs: add statfs ioctl Add an ioctl that can fill a user struct with file system info. We're going to use this to find the fsid and rid of a mount. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 23 +++++++++++++++++++++++ kmod/src/ioctl.h | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index bee29df7..c1d64aa0 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -848,6 +848,27 @@ out: return ret ?: total; } +static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_ioctl_statfs_more sfm; + + if (get_user(sfm.valid_bytes, (__u64 __user *)arg)) + return -EFAULT; + + sfm.valid_bytes = min_t(u64, sfm.valid_bytes, + sizeof(struct scoutfs_ioctl_statfs_more)); + sfm.fsid = le64_to_cpu(super->hdr.fsid); + sfm.rid = sbi->rid; + + if (copy_to_user((void __user *)arg, &sfm, sfm.valid_bytes)) + return -EFAULT; + + return 0; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -871,6 +892,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_listxattr_hidden(file, arg); case SCOUTFS_IOC_FIND_XATTRS: return scoutfs_ioc_find_xattrs(file, arg); + case SCOUTFS_IOC_STATFS_MORE: + return scoutfs_ioc_statfs_more(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 5aa057c0..5693668e 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -347,4 +347,26 @@ struct scoutfs_ioctl_find_xattrs { #define SCOUTFS_IOC_FIND_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ struct scoutfs_ioctl_find_xattrs) +/* + * Give the user information about the filesystem. + * + * @valid_bytes stores the number of bytes that are valid in the + * structure. The caller sets this to the size of the struct that they + * understand. The kernel then fills and copies back the min of the + * size they and the user caller understand. The user can tell if a + * field is set if all of its bytes are within the valid_bytes that the + * kernel set on return. + * + * New fields are only added to the end of the struct. + */ +struct scoutfs_ioctl_statfs_more { + __u64 valid_bytes; + __u64 fsid; + __u64 rid; +} __packed; + +#define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ + struct scoutfs_ioctl_statfs_more) + + #endif From ec7f60bebb0125eb18e66f4b057ca69a8345377a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Jul 2019 16:31:08 -0700 Subject: [PATCH 737/920] scoutfs: net conn lifetime tracing Add trace events for network connections. Signed-off-by: Zach Brown --- kmod/src/net.c | 153 ++++++++++++++++++--------------------- kmod/src/net.h | 98 ++++++++++++++++++++----- kmod/src/scoutfs_trace.h | 124 +++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+), 100 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index 1177336d..f56a367a 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -86,58 +86,20 @@ struct net_info { struct scoutfs_tseq_tree msg_tseq_tree; }; -struct scoutfs_net_connection { - struct super_block *sb; - scoutfs_net_notify_t notify_up; - scoutfs_net_notify_t notify_down; - size_t info_size; - scoutfs_net_request_t *req_funcs; - - spinlock_t lock; - wait_queue_head_t waitq; - - unsigned long valid_greeting:1, /* other commands can proceed */ - established:1, /* added sends queue send work */ - shutting_down:1, /* shutdown work has been queued */ - saw_greeting:1, /* saw greeting on this sock */ - saw_farewell:1, /* saw farewell response to client */ - reconn_wait:1, /* shutdown, waiting for reconnect */ - reconn_freeing:1; /* waiting done, setter frees */ - unsigned long reconn_deadline; - - struct sockaddr_in connect_sin; - unsigned long connect_timeout_ms; - - struct socket *sock; - u64 rid; - u64 greeting_id; - struct sockaddr_in sockname; - struct sockaddr_in peername; - - struct list_head accepted_head; - struct scoutfs_net_connection *listening_conn; - struct list_head accepted_list; - - u64 next_send_seq; - u64 next_send_id; - struct list_head send_queue; - struct list_head resend_queue; - - atomic64_t recv_seq; - - struct workqueue_struct *workq; - struct work_struct listen_work; - struct work_struct connect_work; - struct work_struct send_work; - struct work_struct recv_work; - struct work_struct shutdown_work; - struct delayed_work reconn_free_dwork; - /* message_recv proc_work also executes in the conn workq */ - - struct scoutfs_tseq_entry tseq_entry; - - void *info; -}; +/* flags enum is in net.h */ +#define test_conn_fl(conn, which) (!!((conn)->flags & CONN_FL_##which)) +#define set_conn_fl(conn, which) \ +do { \ + (conn)->flags |= CONN_FL_##which; \ +} while (0) +#define clear_conn_fl(conn, which) \ +do { \ + (conn)->flags &= ~CONN_FL_##which; \ +} while (0) +#define assign_conn_fl(dst, src, which) \ +do { \ + (dst)->flags |= ((conn)->flags & CONN_FL_##which); \ +} while (0) /* listening and their accepting sockets have a fixed locking order */ enum { @@ -319,9 +281,10 @@ static void shutdown_conn_locked(struct scoutfs_net_connection *conn) assert_spin_locked(&conn->lock); - if (!conn->shutting_down) { - conn->established = 0; - conn->shutting_down = 1; + if (!test_conn_fl(conn, shutting_down)) { + clear_conn_fl(conn, established); + set_conn_fl(conn, shutting_down); + trace_scoutfs_conn_shutdown_queued(conn); queue_work(ninf->shutdown_workq, &conn->shutdown_work); } } @@ -412,8 +375,9 @@ static int submit_send(struct super_block *sb, if (data_len) memcpy(msend->nh.data, data, data_len); - if (conn->established && - (conn->valid_greeting || cmd == SCOUTFS_NET_CMD_GREETING)) { + if (test_conn_fl(conn, established) && + (test_conn_fl(conn, valid_greeting) || + cmd == SCOUTFS_NET_CMD_GREETING)) { list_add_tail(&msend->head, &conn->send_queue); queue_work(conn->workq, &conn->send_work); } else { @@ -615,7 +579,7 @@ static bool invalid_message(struct scoutfs_net_connection *conn, if (nh->cmd == SCOUTFS_NET_CMD_GREETING) { /* each endpoint can only receive one greeting per socket */ - if (conn->saw_greeting) + if (test_conn_fl(conn, saw_greeting)) return true; /* servers get greeting requests, clients get responses */ @@ -686,7 +650,7 @@ static void scoutfs_net_recv_worker(struct work_struct *work) if (nh.cmd == SCOUTFS_NET_CMD_GREETING) { /* greetings are out of band, no seq mechanics */ - conn->saw_greeting = 1; + set_conn_fl(conn, saw_greeting); } else if (le64_to_cpu(nh.seq) <= atomic64_read(&conn->recv_seq)) { @@ -790,7 +754,7 @@ static void scoutfs_net_send_worker(struct work_struct *work) if ((msend->nh.cmd == SCOUTFS_NET_CMD_FAREWELL) && nh_is_response(&msend->nh)) { - conn->saw_farewell = 1; + set_conn_fl(conn, saw_farewell); } msend->nh.recv_seq = @@ -840,6 +804,8 @@ static void destroy_conn(struct scoutfs_net_connection *conn) struct message_send *msend; struct message_send *tmp; + trace_scoutfs_conn_destroy_start(conn); + WARN_ON_ONCE(conn->sock != NULL); WARN_ON_ONCE(!list_empty(&conn->accepted_list)); @@ -867,6 +833,7 @@ static void destroy_conn(struct scoutfs_net_connection *conn) destroy_workqueue(conn->workq); scoutfs_tseq_del(&ninf->conn_tseq_tree, &conn->tseq_entry); kfree(conn->info); + trace_scoutfs_conn_destroy_free(conn); kfree(conn); } @@ -995,9 +962,11 @@ static void scoutfs_net_listen_worker(struct work_struct *work) acc_conn->sock = acc_sock; acc_conn->listening_conn = conn; - acc_conn->established = 1; + set_conn_fl(acc_conn, established); list_add_tail(&acc_conn->accepted_head, &conn->accepted_list); + trace_scoutfs_conn_accept(acc_conn); + spin_unlock(&conn->lock); queue_work(acc_conn->workq, &acc_conn->recv_work); @@ -1043,6 +1012,8 @@ static void scoutfs_net_connect_worker(struct work_struct *work) conn->sock = sock; spin_unlock(&conn->lock); + trace_scoutfs_conn_connect_start(conn); + ret = kernel_connect(sock, (struct sockaddr *)&conn->connect_sin, sizeof(struct sockaddr_in), 0); if (ret) @@ -1059,10 +1030,12 @@ static void scoutfs_net_connect_worker(struct work_struct *work) spin_lock(&conn->lock); /* clear greeting state for next negotiation */ - conn->valid_greeting = 0; - conn->established = 1; + clear_conn_fl(conn, valid_greeting); + set_conn_fl(conn, established); wake_up(&conn->waitq); + trace_scoutfs_conn_connect_complete(conn); + spin_unlock(&conn->lock); queue_work(conn->workq, &conn->recv_work); @@ -1102,6 +1075,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) unsigned long delay; trace_scoutfs_net_shutdown_work_enter(sb, 0, 0); + trace_scoutfs_conn_shutdown_start(conn); /* connected and accepted conns print a message */ if (conn->peername.sin_port != 0) @@ -1153,7 +1127,7 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) free_msend(ninf, msend); } - conn->saw_greeting = 0; + clear_conn_fl(conn, saw_greeting); /* signal connect failure */ memset(&conn->connect_sin, 0, sizeof(conn->connect_sin)); @@ -1161,7 +1135,8 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) /* resolve racing with listener shutdown with locked shutting_down */ if (conn->listening_conn && - (conn->listening_conn->shutting_down || conn->saw_farewell)) { + (test_conn_fl(conn->listening_conn, shutting_down) || + test_conn_fl(conn, saw_farewell))) { /* free accepted sockets after farewell or listener shutdown */ spin_unlock(&conn->lock); @@ -1173,18 +1148,19 @@ static void scoutfs_net_shutdown_worker(struct work_struct *work) /* server accepted sockets wait for reconnect */ listener = conn->listening_conn; delay = msecs_to_jiffies(CLIENT_RECONNECT_TIMEOUT_MS); - conn->reconn_wait = 1; + set_conn_fl(conn, reconn_wait); conn->reconn_deadline = jiffies + delay; queue_delayed_work(listener->workq, &listener->reconn_free_dwork, delay); } else { /* clients and listeners can retry */ - conn->shutting_down = 0; + clear_conn_fl(conn, shutting_down); if (conn->notify_down) conn->notify_down(sb, conn, conn->info, conn->rid); } + trace_scoutfs_conn_shutdown_complete(conn); spin_unlock(&conn->lock); } @@ -1218,12 +1194,13 @@ restart: spin_lock(&conn->lock); list_for_each_entry(acc, &conn->accepted_list, accepted_head) { - if (acc->reconn_wait && !acc->reconn_freeing && - (conn->shutting_down || + if (test_conn_fl(acc, reconn_wait) && + !test_conn_fl(acc, reconn_freeing) && + (test_conn_fl(conn, shutting_down) || time_after_eq(now, acc->reconn_deadline))) { - acc->reconn_freeing = 1; + set_conn_fl(acc, reconn_freeing); spin_unlock(&conn->lock); - if (!conn->shutting_down) + if (!test_conn_fl(conn, shutting_down)) scoutfs_info(sb, "client timed out "SIN_FMT" -> "SIN_FMT", can not reconnect", SIN_ARG(&acc->sockname), SIN_ARG(&acc->peername)); @@ -1232,7 +1209,8 @@ restart: } /* calc delay of next work, can drift a bit */ - if (acc->reconn_wait && !acc->reconn_freeing && + if (test_conn_fl(acc, reconn_wait) && + !test_conn_fl(acc, reconn_freeing) && (!requeue || time_before(now, deadline))) { requeue = true; deadline = acc->reconn_deadline; @@ -1312,6 +1290,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_reconn_free_worker); scoutfs_tseq_add(&ninf->conn_tseq_tree, &conn->tseq_entry); + trace_scoutfs_conn_alloc(conn); return conn; } @@ -1432,13 +1411,15 @@ static bool connect_result(struct scoutfs_net_connection *conn, int *error) bool done = false; spin_lock(&conn->lock); - if (conn->established) { + if (test_conn_fl(conn, established)) { done = true; *error = 0; - } else if (conn->shutting_down || conn->connect_sin.sin_family == 0) { + } else if (test_conn_fl(conn, shutting_down) || + conn->connect_sin.sin_family == 0) { done = true; *error = -ESHUTDOWN; } + trace_scoutfs_conn_connect_result(conn); spin_unlock(&conn->lock); return done; @@ -1477,9 +1458,9 @@ static void set_valid_greeting(struct scoutfs_net_connection *conn) assert_spin_locked(&conn->lock); /* recv should have dropped invalid duplicate greeting messages */ - BUG_ON(conn->valid_greeting); + BUG_ON(test_conn_fl(conn, valid_greeting)); - conn->valid_greeting = 1; + set_conn_fl(conn, valid_greeting); list_splice_tail_init(&conn->resend_queue, &conn->send_queue); queue_work(conn->workq, &conn->send_work); } @@ -1571,10 +1552,10 @@ restart: accepted_head) { if (acc->rid != rid || acc->greeting_id >= greeting_id || - acc->reconn_freeing) + test_conn_fl(acc, reconn_freeing)) continue; - if (!acc->reconn_wait) { + if (!test_conn_fl(acc, reconn_wait)) { spin_lock_nested(&acc->lock, CONN_LOCK_ACCEPTED); shutdown_conn_locked(acc); @@ -1585,7 +1566,7 @@ restart: } reconn = acc; - reconn->reconn_freeing = 1; + set_conn_fl(reconn, reconn_freeing); break; } spin_unlock(&listener->lock); @@ -1601,7 +1582,7 @@ restart: if (reconn) { spin_lock(&conn->lock); - conn->saw_farewell = reconn->saw_farewell; + assign_conn_fl(conn, reconn, saw_farewell); conn->next_send_seq = reconn->next_send_seq; conn->next_send_id = reconn->next_send_id; atomic64_set(&conn->recv_seq, atomic64_read(&reconn->recv_seq)); @@ -1615,6 +1596,7 @@ restart: swap(conn->info, reconn->info); reconn->notify_down = NULL; + trace_scoutfs_conn_reconn_migrate(conn); spin_unlock(&conn->lock); /* we set _freeing */ @@ -1789,9 +1771,14 @@ static void net_tseq_show_conn(struct seq_file *m, seq_printf(m, "name "SIN_FMT" peer "SIN_FMT" rid %016llx greeting_id %llu vg %u est %u sd %u sg %u sf %u rw %u rf %u cto_ms rdl_j %lu %lu nss %llu rs %llu nsi %llu\n", SIN_ARG(&conn->sockname), SIN_ARG(&conn->peername), - conn->rid, conn->greeting_id, conn->valid_greeting, - conn->established, conn->shutting_down, conn->saw_greeting, - conn->saw_farewell, conn->reconn_wait, conn->reconn_freeing, + conn->rid, conn->greeting_id, + test_conn_fl(conn, valid_greeting), + test_conn_fl(conn, established), + test_conn_fl(conn, shutting_down), + test_conn_fl(conn, saw_greeting), + test_conn_fl(conn, saw_farewell), + test_conn_fl(conn, reconn_wait), + test_conn_fl(conn, reconn_freeing), conn->connect_timeout_ms, conn->reconn_deadline, conn->next_send_seq, (u64)atomic64_read(&conn->recv_seq), conn->next_send_id); diff --git a/kmod/src/net.h b/kmod/src/net.h index 0113c16c..1a899d83 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -3,6 +3,87 @@ #include #include "endian_swap.h" +#include "tseq.h" + +struct scoutfs_net_connection; + +/* These are called in their own blocking context */ +typedef int (*scoutfs_net_request_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len); + +/* These are called in their own blocking context */ +typedef int (*scoutfs_net_response_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *resp, unsigned int resp_len, + int error, void *data); + +typedef void (*scoutfs_net_notify_t)(struct super_block *sb, + struct scoutfs_net_connection *conn, + void *info, u64 rid); + +/* + * The conn is only here so that tracing can get at its fields without + * having trace functions with a trillion arguments. Tracing requires + * duplicating the arguments for every event, no thanks. + */ + +struct scoutfs_net_connection { + struct super_block *sb; + scoutfs_net_notify_t notify_up; + scoutfs_net_notify_t notify_down; + size_t info_size; + scoutfs_net_request_t *req_funcs; + + spinlock_t lock; + wait_queue_head_t waitq; + + unsigned long flags; /* CONN_FL_* bitmask */ + unsigned long reconn_deadline; + + struct sockaddr_in connect_sin; + unsigned long connect_timeout_ms; + + struct socket *sock; + u64 rid; + u64 greeting_id; + struct sockaddr_in sockname; + struct sockaddr_in peername; + + struct list_head accepted_head; + struct scoutfs_net_connection *listening_conn; + struct list_head accepted_list; + + u64 next_send_seq; + u64 next_send_id; + struct list_head send_queue; + struct list_head resend_queue; + + atomic64_t recv_seq; + + struct workqueue_struct *workq; + struct work_struct listen_work; + struct work_struct connect_work; + struct work_struct send_work; + struct work_struct recv_work; + struct work_struct shutdown_work; + struct delayed_work reconn_free_dwork; + /* message_recv proc_work also executes in the conn workq */ + + struct scoutfs_tseq_entry tseq_entry; + + void *info; +}; + +enum { + CONN_FL_valid_greeting = (1UL << 0), /* other commands can proceed */ + CONN_FL_established = (1UL << 1), /* added sends queue send work */ + CONN_FL_shutting_down = (1UL << 2), /* shutdown work was queued */ + CONN_FL_saw_greeting = (1UL << 3), /* saw greeting on this sock */ + CONN_FL_saw_farewell = (1UL << 4), /* saw farewell response */ + CONN_FL_reconn_wait = (1UL << 5), /* shutdown, waiting for reconn */ + CONN_FL_reconn_freeing = (1UL << 6), /* waiting done, setter frees */ +}; #define SIN_FMT "%pIS:%u" #define SIN_ARG(sin) sin, be16_to_cpu((sin)->sin_port) @@ -22,23 +103,6 @@ static inline void scoutfs_addr_from_sin(struct scoutfs_inet_addr *addr, addr->port = be16_to_le16(sin->sin_port); } -struct scoutfs_net_connection; - -/* These are called in their own blocking context */ -typedef int (*scoutfs_net_request_t)(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len); - -/* These are called in their own blocking context */ -typedef int (*scoutfs_net_response_t)(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *resp, unsigned int resp_len, - int error, void *data); - -typedef void (*scoutfs_net_notify_t)(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *info, u64 rid); - struct scoutfs_net_connection * scoutfs_net_alloc_conn(struct super_block *sb, scoutfs_net_notify_t notify_up, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b452620d..b15126ec 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -38,6 +38,7 @@ #include "dir.h" #include "extents.h" #include "server.h" +#include "net.h" struct lock_info; @@ -1813,6 +1814,129 @@ DEFINE_EVENT(scoutfs_net_class, scoutfs_net_recv_message, TP_ARGS(sb, name, peer, nh) ); +#define conn_flag_entry(which) \ + CONN_FL_##which, __stringify(which) + +#define print_conn_flags(flags) __print_flags(flags, "|", \ + { conn_flag_entry(valid_greeting) }, \ + { conn_flag_entry(established) }, \ + { conn_flag_entry(shutting_down) }, \ + { conn_flag_entry(saw_greeting) }, \ + { conn_flag_entry(saw_farewell) }, \ + { conn_flag_entry(reconn_wait) }, \ + { conn_flag_entry(reconn_freeing) }) + +/* + * This is called from alloc and free when the caller only has safe + * access to the struct itself, be very careful not to follow any + * indirection out of the storage for the conn struct. + */ +DECLARE_EVENT_CLASS(scoutfs_net_conn_class, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(unsigned long, flags) + __field(unsigned long, reconn_deadline) + __field(unsigned long, connect_timeout_ms) + __field(void *, sock) + __field(__u64, c_rid) + __field(__u64, greeting_id) + si4_trace_define(sockname) + si4_trace_define(peername) + __field(unsigned char, e_accepted_head) + __field(void *, listening_conn) + __field(unsigned char, e_accepted_list) + __field(__u64, next_send_seq) + __field(__u64, next_send_id) + __field(unsigned char, e_send_queue) + __field(unsigned char, e_resend_queue) + __field(__u64, recv_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(conn->sb); + __entry->flags = conn->flags; + __entry->reconn_deadline = conn->reconn_deadline; + __entry->connect_timeout_ms = conn->connect_timeout_ms; + __entry->sock = conn->sock; + __entry->c_rid = conn->rid; + __entry->greeting_id = conn->greeting_id; + si4_trace_assign(sockname, &conn->sockname); + si4_trace_assign(peername, &conn->peername); + __entry->e_accepted_head = !!list_empty(&conn->accepted_head); + __entry->listening_conn = conn->listening_conn; + __entry->e_accepted_list = !!list_empty(&conn->accepted_list); + __entry->next_send_seq = conn->next_send_seq; + __entry->next_send_id = conn->next_send_id; + __entry->e_send_queue = !!list_empty(&conn->send_queue); + __entry->e_resend_queue = !!list_empty(&conn->resend_queue); + __entry->recv_seq = atomic64_read(&conn->recv_seq); + ), + TP_printk(SCSBF" flags %s rc_dl %lu cto %lu sk %p rid %llu grid %llu sn "SI4_FMT" pn "SI4_FMT" eah %u lc %p eal %u nss %llu nsi %llu esq %u erq %u rs %llu", + SCSB_TRACE_ARGS, + print_conn_flags(__entry->flags), + __entry->reconn_deadline, + __entry->connect_timeout_ms, + __entry->sock, + __entry->c_rid, + __entry->greeting_id, + si4_trace_args(sockname), + si4_trace_args(peername), + __entry->e_accepted_head, + __entry->listening_conn, + __entry->e_accepted_list, + __entry->next_send_seq, + __entry->next_send_id, + __entry->e_send_queue, + __entry->e_resend_queue, + __entry->recv_seq) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_alloc, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_result, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_connect_complete, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_accept, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_reconn_migrate, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_queued, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_shutdown_complete, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_destroy_start, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); +DEFINE_EVENT(scoutfs_net_conn_class, scoutfs_conn_destroy_free, + TP_PROTO(struct scoutfs_net_connection *conn), + TP_ARGS(conn) +); + DECLARE_EVENT_CLASS(scoutfs_work_class, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret), From 319ff86014875134131234a6ced3f9e3cc656637 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 26 Jul 2019 16:52:01 -0700 Subject: [PATCH 738/920] scoutfs: lock recovery info messages Lock recovery is perfectly normal if a server is unmounted and another is elected to take its place. Turn the lock recovery message into an info message instead of a warning and add another info message when lock recovery is complete. Signed-off-by: Zach Brown --- kmod/src/lock_server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 3f93c150..b330218e 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -631,6 +631,8 @@ static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) scoutfs_key_set_zeros(&key); + scoutfs_info(sb, "all lock clients recovered"); + while ((snode = get_server_lock(inf, &key, NULL, true))) { key = snode->key; @@ -986,7 +988,7 @@ int scoutfs_lock_server_setup(struct super_block *sb) if (nr) { schedule_delayed_work(&inf->recovery_dwork, msecs_to_jiffies(LOCK_SERVER_RECOVERY_MS)); - scoutfs_warn(sb, "waiting for %u lock clients to connect", nr); + scoutfs_info(sb, "waiting for %u lock clients to recover", nr); } out: From feaf17c3a5ca69e2b2dff864c59bd1eeb87c98f9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 30 Jul 2019 09:42:23 -0700 Subject: [PATCH 739/920] scoutfs: add conn destroy workq Lockdep gets angry when we try to destroy an accepted conn workqueue from within work in a listening conn's workqueue. It doesn't recognize that they have a hierarchical relationship that maintains a consistent order and we can't get at the workqueue lockdep_map to set subclasses. We add a destroy workqueue which will have its own class. Signed-off-by: Zach Brown --- kmod/src/net.c | 32 ++++++++++++++++++++++++++++++-- kmod/src/net.h | 1 + kmod/src/scoutfs_trace.h | 8 ++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/kmod/src/net.c b/kmod/src/net.c index f56a367a..79056204 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -80,6 +80,7 @@ */ struct net_info { struct workqueue_struct *shutdown_workq; + struct workqueue_struct *destroy_workq; struct dentry *conn_tseq_dentry; struct scoutfs_tseq_tree conn_tseq_tree; struct dentry *msg_tseq_dentry; @@ -796,14 +797,25 @@ static void scoutfs_net_send_worker(struct work_struct *work) trace_scoutfs_net_send_work_exit(sb, 0, ret); } -static void destroy_conn(struct scoutfs_net_connection *conn) +/* + * Listening conns try to destroy accepted conns. Workqueues model + * flushing work as acquiring a workqueue class lock so it thinks that + * this is a deadlock because it doesn't know about our hierarchy of + * workqueues. The workqueue lockdep_map is private so we can't set a + * subclass to differentiate between listening and accepted conn + * workqueues. Instead we queue final conn destruction off to a longer + * lived specific workqueue that has a different class. + */ +static void scoutfs_net_destroy_worker(struct work_struct *work) { + DEFINE_CONN_FROM_WORK(conn, work, destroy_work); struct super_block *sb = conn->sb; struct net_info *ninf = SCOUTFS_SB(sb)->net_info; struct scoutfs_net_connection *listener; struct message_send *msend; struct message_send *tmp; + trace_scoutfs_net_destroy_work_enter(sb, 0, 0); trace_scoutfs_conn_destroy_start(conn); WARN_ON_ONCE(conn->sock != NULL); @@ -835,6 +847,15 @@ static void destroy_conn(struct scoutfs_net_connection *conn) kfree(conn->info); trace_scoutfs_conn_destroy_free(conn); kfree(conn); + + trace_scoutfs_net_destroy_work_exit(sb, 0, 0); +} + +static void destroy_conn(struct scoutfs_net_connection *conn) +{ + struct net_info *ninf = SCOUTFS_SB(conn->sb)->net_info; + + queue_work(ninf->destroy_workq, &conn->destroy_work); } /* @@ -1286,6 +1307,7 @@ scoutfs_net_alloc_conn(struct super_block *sb, INIT_WORK(&conn->send_work, scoutfs_net_send_worker); INIT_WORK(&conn->recv_work, scoutfs_net_recv_worker); INIT_WORK(&conn->shutdown_work, scoutfs_net_shutdown_worker); + INIT_WORK(&conn->destroy_work, scoutfs_net_destroy_worker); INIT_DELAYED_WORK(&conn->reconn_free_dwork, scoutfs_net_reconn_free_worker); @@ -1317,6 +1339,7 @@ void scoutfs_net_shutdown(struct super_block *sb, { shutdown_conn(conn); flush_work(&conn->shutdown_work); + flush_work(&conn->destroy_work); } /* @@ -1843,7 +1866,10 @@ int scoutfs_net_setup(struct super_block *sb) ninf->shutdown_workq = alloc_workqueue("scoutfs_net_shutdown", WQ_UNBOUND | WQ_NON_REENTRANT, 0); - if (!ninf->shutdown_workq) { + ninf->destroy_workq = alloc_workqueue("scoutfs_net_destroy", + WQ_UNBOUND | WQ_NON_REENTRANT, + 0); + if (!ninf->shutdown_workq || !ninf->destroy_workq) { ret = -ENOMEM; goto out; } @@ -1879,6 +1905,8 @@ void scoutfs_net_destroy(struct super_block *sb) if (ninf) { if (ninf->shutdown_workq) destroy_workqueue(ninf->shutdown_workq); + if (ninf->destroy_workq) + destroy_workqueue(ninf->destroy_workq); debugfs_remove(ninf->conn_tseq_dentry); debugfs_remove(ninf->msg_tseq_dentry); kfree(ninf); diff --git a/kmod/src/net.h b/kmod/src/net.h index 1a899d83..4e2312f9 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -67,6 +67,7 @@ struct scoutfs_net_connection { struct work_struct send_work; struct work_struct recv_work; struct work_struct shutdown_work; + struct work_struct destroy_work; struct delayed_work reconn_free_dwork; /* message_recv proc_work also executes in the conn workq */ diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b15126ec..b7945b2d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2001,6 +2001,14 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_net_shutdown_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_destroy_work_enter, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); +DEFINE_EVENT(scoutfs_work_class, scoutfs_net_destroy_work_exit, + TP_PROTO(struct super_block *sb, u64 data, int ret), + TP_ARGS(sb, data, ret) +); DEFINE_EVENT(scoutfs_work_class, scoutfs_net_reconn_free_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) From 07c9edb58fe00666213579cad0877dc7a4883e7a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 2 Aug 2019 10:24:57 -0700 Subject: [PATCH 740/920] scoutfs: warn on compaction stale seg reads It's possible to trigger stale segment reads during compaction. This shouldn't be possible during regular operation because the server protects the input segments while the compaction is pending. Stale segment reads can only happen to client reads which aren't serialized with segment allocation and writes. Warn if we see a stale segment read during compaction. It means that we either have a bug in the server or someone armed a stale segment read trigger that hit compaction. Signed-off-by: Zach Brown --- kmod/src/compact.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kmod/src/compact.c b/kmod/src/compact.c index cfca5fd8..b220a771 100644 --- a/kmod/src/compact.c +++ b/kmod/src/compact.c @@ -608,8 +608,10 @@ static int prepare_curs(struct super_block *sb, struct compact_cursor *curs, * segments, then generating the response that describes the output * segments. * - * The server will either commit our response or cleanup the request - * if we return an error that the caller sends in response. + * The server will either commit our response or cleanup the request if + * we return an error that the caller sends in response. The server + * protects the input segments so they shouldn't be overwritten by other + * compactions or allocations. We shouldn't get stale segment reads. */ int scoutfs_compact(struct super_block *sb, struct scoutfs_net_compact_request *req, @@ -668,8 +670,11 @@ int scoutfs_compact(struct super_block *sb, ret = 0; out: - if (ret == -ESTALE) + /* server protects input segments, shouldn't be possible */ + if (WARN_ON_ONCE(ret == -ESTALE)) { scoutfs_inc_counter(sb, compact_stale_error); + ret = -EIO; + } free_cseg_list(sb, &curs.csegs); free_cseg_list(sb, &results); From 15a492fe57ff71e886329ef0ce3a653a081ea5f8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 5 Aug 2019 16:06:26 -0700 Subject: [PATCH 741/920] scoutfs: always dirty parents when migrating In a previous commit ("1bd094f scoutfs: migrate dirty btree blocks during wrap") we fixed a bug where we wouldn't migrate blocks from the old half of the ring because they were already dirty in memory. The fix accidentally introduced the case where we wouldn't dirty blocks when migrating if they were already in the current half. We always have to dirty parent blocks when migrating because we might need to modify them to reference the new location of child blocks that are migrated. This bug meant that we'd modify clean blocks in memory which would never make it to the persistent copy. The system could survive as long as it never read that block back from its persistent location. To see the corruption you'd either need tall btrees to be shared between mounts or you'd need one mount to evict its clean (actually modified) cached btree block under memory pressure and then try to read it back. Signed-off-by: Zach Brown --- kmod/src/btree.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index c680579b..1cf35586 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -701,16 +701,20 @@ retry: } /* - * We don't need to cow the exiting block if we're not - * dirtying the block, or we're not migrating and it's - * already dirty in this transaction, or we're - * migrating and it's already in the current half. + * We need to create a new dirty copy of the block if + * the caller asked for it. If the block is already + * dirty then we can return it if either we're not + * migrating so it doesn't matter which half it's in, or + * we're migrating and the dirty block is already in the + * second half. We can be migrating into a new half + * while blocks are still dirty in the old half. And we + * always have to dirty parent blocks in the current + * half in case we need to dirty their children. */ if (!(flags & BTW_DIRTY) || - (!(flags & BTW_MIGRATE) && - (le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq)) || - ((flags & BTW_MIGRATE) && - blkno_is_current(bring, le64_to_cpu(ref->blkno)))) { + ((le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq) && + (!(flags & BTW_MIGRATE) || + blkno_is_current(bring, le64_to_cpu(ref->blkno))))) { ret = 0; goto out; } From b1cc8b1a594b4cb9632e019f5ea52d3899f3d71f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Aug 2019 15:16:35 -0700 Subject: [PATCH 742/920] scoutfs: update README.md for server_addr Update the instructions for starting up a system with the quorum count mkfs option and server_addr mount option. Signed-off-by: Zach Brown --- kmod/README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/kmod/README.md b/kmod/README.md index aed9631d..fb5a7cb6 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -72,7 +72,7 @@ The steps for getting scoutfs mounted and operational are: 2. Make a new filesystem on the device with the userspace utilities 3. Mount the device on all the nodes -In this example we run all of these commands on two nodes. The block +In this example we run all of these commands on three nodes. The block device name is the same on all the nodes. 1. Get the Kernel Module and Userspace Binaries @@ -96,29 +96,26 @@ device name is the same on all the nodes. git clone git@github.com:versity/scoutfs-utils-dev.git make -C scoutfs-utils-dev alias scoutfs=$PWD/scoutfs-utils-dev/src/scoutfs - ``` 2. Make a New Filesystem (**destroys contents, no questions asked**) - We specify that every node will participate in quorum voting by - configuring each in the super block with options to mkfs. + We specify that two of our three nodes must be present to form a + quorum for the system to function. ```shell - scoutfs mkfs --quorum_slot node1:0:172.16.1.1 \ - --quorum_slot node2:0:172.16.1.2 /dev/shared_block_device + scoutfs mkfs -Q 2 /dev/shared_block_device ``` - 3. Mount the Filesystem - Each mounting node provides the name that was given to the - quorum\_slot option to mkfs. + Each mounting node provides its local IP address on which it will run + an internal server for the other mounts if it is elected the leader by + the quorum. ```shell mkdir /mnt/scoutfs - mount -t scoutfs -o uniq_name=$NODENAME /dev/shared_block_device /mnt/scoutfs - + mount -t scoutfs -o server_address=$NODE_ADDR /dev/shared_block_device /mnt/scoutfs ``` 4. For Kicks, Observe the Metadata Change Index From 8c631b019b849b4bf38021f6c1862cf1cd102e47 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Mon, 26 Aug 2019 19:43:44 +0800 Subject: [PATCH 743/920] scoutfs: fix wrong option example in README scoutfs f.000000.r.200d94 error: Unknown or malformed option, "server_address=192.168.31.220" Should be server_addr, fix it. Cc: Zach Brown Fixes: 10c32("scoutfs: update README.md for server_addr") Signed-off-by: Wang Shilong --- kmod/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/README.md b/kmod/README.md index fb5a7cb6..5152a124 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -115,7 +115,7 @@ device name is the same on all the nodes. ```shell mkdir /mnt/scoutfs - mount -t scoutfs -o server_address=$NODE_ADDR /dev/shared_block_device /mnt/scoutfs + mount -t scoutfs -o server_addr=$NODE_ADDR /dev/shared_block_device /mnt/scoutfs ``` 4. For Kicks, Observe the Metadata Change Index From 15becd6ef8a17ee954fa86e3890bdfa9d69b974c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 9 Sep 2019 15:13:33 -0700 Subject: [PATCH 744/920] scoutfs: force locks idle as shutdown frees Usually lock_free() is called as users finish using a lock and when its state shows that it is idle and won't be freed out from under another use. During shutdown we manually call lock_free() on all locks because shutdown promises that there will be no more lock users, including networking callbacks. But there is a case where network requests can be pending and we shutdown before waiting for their reply. This trips BUG_ON assertions in lock_free() that would otherwise catch unsafe calls of lock_free(). This is easiest to reproduce by interrupting a mount (which is waiting on a lock to read the root inode). The fix is to update each lock's state during shutdown to reflect the promise made by shutdown. Requests aren't actually pending because we've shutdown networking befrore getting here. Signed-off-by: Zach Brown --- kmod/src/lock.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 9536376a..290439dd 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1492,14 +1492,19 @@ void scoutfs_lock_destroy(struct super_block *sb) debugfs_remove(linfo->tseq_dentry); /* - * This is very clumsy and brute force. This will be cleaned up - * as we add proper lock recovery. + * Usually lock_free is only called once locks are idle but all + * locks are idle by definition during shutdown. We need to + * manually update the lock's state to reflect that we've given + * up on pending work that would otherwise prevent free from + * being called (and would trip assertions in our manual calling + * of free). */ spin_lock(&linfo->lock); node = rb_first(&linfo->lock_tree); while (node) { lock = rb_entry(node, struct scoutfs_lock, node); node = rb_next(node); + lock->request_pending = 0; if (!list_empty(&lock->lru_head)) __lock_del_lru(linfo, lock); lock_remove(linfo, lock); From 2a6d2098549c138a291b35507e786a72c5dd8c01 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Nov 2019 11:02:16 -0800 Subject: [PATCH 745/920] scoutfs: add kernelcompat files Add files that we'll use to detect and work around incompatibilities between kernel versions. Signed-off-by: Zach Brown --- kmod/src/Makefile | 3 +++ kmod/src/Makefile.kernelcompat | 7 +++++++ kmod/src/kernelcompat.h | 4 ++++ 3 files changed, 14 insertions(+) create mode 100644 kmod/src/Makefile.kernelcompat create mode 100644 kmod/src/kernelcompat.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 468f6a61..8048bfd9 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -4,6 +4,9 @@ CFLAGS_super.o = -DSCOUTFS_GIT_DESCRIBE=\"$(SCOUTFS_GIT_DESCRIBE)\" \ -DSCOUTFS_FORMAT_HASH=0x$(SCOUTFS_FORMAT_HASH)LLU CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include + +# add EXTRA_CFLAGS defines for kernel compat +-include $(src)/Makefile.kernelcompat scoutfs-y += \ bio.o \ diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat new file mode 100644 index 00000000..3108c527 --- /dev/null +++ b/kmod/src/Makefile.kernelcompat @@ -0,0 +1,7 @@ +# +# We try to detect the specific api incompatibilities with simple tests +# because distros regularly backport features without changing the +# version. +# + +ccflags-y += -include $(src)/kernelcompat.h diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h new file mode 100644 index 00000000..dbf8b8f3 --- /dev/null +++ b/kmod/src/kernelcompat.h @@ -0,0 +1,4 @@ +#ifndef _SCOUTFS_KERNELCOMPAT_H_ +#define _SCOUTFS_KERNELCOMPAT_H_ + +#endif From ddd1a4ef5a4f3d769138fd640a4816ef8aa0095d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Nov 2019 14:30:32 -0800 Subject: [PATCH 746/920] scoutfs: support newer ->iterate readdir The modern upstream kernel has a ->iterate() readdir file_operattions method which takes a context and calls dir_emit(). We add some kernelcompat helpers to juggle the various function definitions, types, and arguments to support both the old ->readdir(filldir) and the new ->iterate(ctx) interfaces. Signed-off-by: Zach Brown --- kmod/src/Makefile.kernelcompat | 29 ++++++++++++++++++ kmod/src/dir.c | 54 +++++++++++++++------------------- kmod/src/kernelcompat.h | 45 ++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 30 deletions(-) diff --git a/kmod/src/Makefile.kernelcompat b/kmod/src/Makefile.kernelcompat index 3108c527..bd236c43 100644 --- a/kmod/src/Makefile.kernelcompat +++ b/kmod/src/Makefile.kernelcompat @@ -5,3 +5,32 @@ # ccflags-y += -include $(src)/kernelcompat.h + +# +# v3.10-rc6-21-gbb6f619b3a49 +# +# _readdir changes from fop->readdir() to fop->iterate() and from +# filldir(dirent) to dir_emit(ctx). +# +ifneq (,$(shell grep 'iterate.*dir_context' include/linux/fs.h)) +ccflags-y += -DKC_ITERATE_DIR_CONTEXT +endif + +# +# v3.10-rc6-23-g5f99f4e79abc +# +# Helpers including dir_emit_dots() are added in the process of +# switching dcache_readdir() from fop->readdir() to fop->iterate() +# +ifneq (,$(shell grep 'dir_emit_dots' include/linux/fs.h)) +ccflags-y += -DKC_DIR_EMIT_DOTS +endif + +# +# RHEL extended the fop struct so to use it we have to set +# a flag to indicate that the struct is large enough and +# contains the pointer. +# +ifneq (,$(shell grep 'FMODE_KABI_ITERATE' include/linux/fs.h)) +ccflags-y += -DKC_FMODE_KABI_ITERATE +endif diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 9140a1cf..41f2ab82 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -436,28 +436,6 @@ out: return d_splice_alias(inode, dentry); } -/* this exists upstream so we can just delete it in a forward port */ -static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) -{ - struct dentry *dentry = file->f_path.dentry; - struct inode *inode = dentry->d_inode; - struct inode *parent = dentry->d_parent->d_inode; - - if (file->f_pos == 0) { - if (filldir(dirent, ".", 1, 1, scoutfs_ino(inode), DT_DIR)) - return 0; - file->f_pos = 1; - } - - if (file->f_pos == 1) { - if (filldir(dirent, "..", 2, 1, scoutfs_ino(parent), DT_DIR)) - return 0; - file->f_pos = 2; - } - - return 1; -} - /* * readdir simply iterates over the dirent items for the dir inode and * uses their offset as the readdir position. @@ -465,7 +443,8 @@ static int dir_emit_dots(struct file *file, void *dirent, filldir_t filldir) * It will need to be careful not to read past the region of the dirent * hash offset keys that it has access to. */ -static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) +static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, + void *dirent, kc_readdir_ctx_t ctx) { struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; @@ -478,7 +457,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) u64 pos; int ret; - if (!dir_emit_dots(file, dirent, filldir)) + if (!kc_dir_emit_dots(file, dirent, ctx)) return 0; dent = alloc_dirent(SCOUTFS_NAME_LEN); @@ -497,7 +476,7 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) for (;;) { init_dirent_key(&key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), - file->f_pos, 0); + kc_readdir_pos(file, ctx), 0); ret = scoutfs_item_next(sb, &key, &last_key, &val, dir_lock); if (ret < 0) { @@ -511,21 +490,24 @@ static int scoutfs_readdir(struct file *file, void *dirent, filldir_t filldir) scoutfs_corruption(sb, SC_DIRENT_READDIR_NAME_LEN, corrupt_dirent_readdir_name_len, "dir_ino %llu pos %llu key "SK_FMT" len %d", - scoutfs_ino(inode), file->f_pos, + scoutfs_ino(inode), + kc_readdir_pos(file, ctx), SK_ARG(&key), name_len); ret = -EIO; goto out; } pos = le64_to_cpu(key.skd_major); + kc_readdir_pos(file, ctx) = pos; - if (filldir(dirent, dent->name, name_len, pos, - le64_to_cpu(dent->ino), dentry_type(dent->type))) { + if (!kc_dir_emit(ctx, dent, dent->name, name_len, pos, + le64_to_cpu(dent->ino), + dentry_type(dent->type))) { ret = 0; break; } - file->f_pos = pos + 1; + kc_readdir_pos(file, ctx) = pos + 1; } out: @@ -1737,8 +1719,20 @@ out_unlock: return ret; } +#ifdef KC_FMODE_KABI_ITERATE +/* we only need this to set the iterate flag for kabi :/ */ +static int scoutfs_dir_open(struct inode *inode, struct file *file) +{ + file->f_mode |= FMODE_KABI_ITERATE; + return 0; +} +#endif + const struct file_operations scoutfs_dir_fops = { - .readdir = scoutfs_readdir, + .KC_FOP_READDIR = scoutfs_readdir, +#ifdef KC_FMODE_KABI_ITERATE + .open = scoutfs_dir_open, +#endif .unlocked_ioctl = scoutfs_ioctl, .fsync = scoutfs_file_fsync, .llseek = generic_file_llseek, diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index dbf8b8f3..1ee16022 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -1,4 +1,49 @@ #ifndef _SCOUTFS_KERNELCOMPAT_H_ #define _SCOUTFS_KERNELCOMPAT_H_ +#ifndef KC_ITERATE_DIR_CONTEXT +#include +typedef filldir_t kc_readdir_ctx_t; +#define KC_DECLARE_READDIR(name, file, dirent, ctx) name(file, dirent, ctx) +#define KC_FOP_READDIR readdir +#define kc_readdir_pos(filp, ctx) (filp)->f_pos +#define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, dirent, ctx) +#define kc_dir_emit(ctx, dentry, name, name_len, pos, ino, dt) \ + (ctx(dentry, name, name_len, pos, ino, dt) == 0) +#else +typedef struct dir_context * kc_readdir_ctx_t; +#define KC_DECLARE_READDIR(name, file, dirent, ctx) name(file, ctx) +#define KC_FOP_READDIR iterate +#define kc_readdir_pos(filp, ctx) (ctx)->pos +#define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, ctx) +#define kc_dir_emit(ctx, dentry, name, name_len, pos, ino, dt) \ + dir_emit(ctx, name, name_len, ino, dt) +#endif + +#ifndef KC_DIR_EMIT_DOTS +/* + * Kernels before ->iterate and don't have dir_emit_dots so we give them + * one that works with the ->readdir() filldir() method. + */ +static inline int dir_emit_dots(struct file *file, void *dirent, + filldir_t filldir) +{ + if (file->f_pos == 0) { + if (filldir(dirent, ".", 1, 1, + file->f_path.dentry->d_inode->i_ino, DT_DIR)) + return 0; + file->f_pos = 1; + } + + if (file->f_pos == 1) { + if (filldir(dirent, "..", 2, 1, + parent_ino(file->f_path.dentry), DT_DIR)) + return 0; + file->f_pos = 2; + } + + return 1; +} +#endif + #endif From ac2d00629c7bb8c5e27a4e3b57ffa680770aa288 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 Sep 2019 15:58:02 -0700 Subject: [PATCH 747/920] scoutfs: add scoutfs_lock_protected() The item code had a manual comparison of lock modes when testing if a given access was protected by a held lock. Let's offer a proper interface from the lock code. Signed-off-by: Zach Brown --- kmod/src/lock.c | 19 +++++++++++++++++++ kmod/src/lock.h | 2 ++ 2 files changed, 21 insertions(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 290439dd..aa3d6b04 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1288,6 +1288,25 @@ void scoutfs_lock_del_coverage(struct super_block *sb, spin_unlock(&cov->cov_lock); } +/* + * Returns true if the given lock protects the given access of the given + * key. The lock must have a current granted mode that is compatible + * with the access mode and the access key must be in the lock's key + * range. + * + * This is called by lock holders who's use of the lock must be preventing + * the mode and keys from changing. + */ +bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, + int mode) +{ + signed char lock_mode = ACCESS_ONCE(lock->mode); + + return lock_modes_match(lock_mode, mode) && + scoutfs_key_compare_ranges(key, key, + &lock->start, &lock->end) == 0; +} + /* * The shrink callback got the lock, marked it request_pending, and * handed it off to us. We kick off a null request and the lock will diff --git a/kmod/src/lock.h b/kmod/src/lock.h index febd5ff5..8a665c10 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -83,6 +83,8 @@ bool scoutfs_lock_is_covered(struct super_block *sb, struct scoutfs_lock_coverage *cov); void scoutfs_lock_del_coverage(struct super_block *sb, struct scoutfs_lock_coverage *cov); +bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, + int mode); void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); From 4265ecedb0e98773e3785627c73de67b726f88fc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 11:45:02 -0700 Subject: [PATCH 748/920] scoutfs: increase max btree value length Now that we're storing fs items in the btree we need a larger max value length. Signed-off-by: Zach Brown --- kmod/src/format.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 2534fe16..f189f623 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -160,9 +160,10 @@ struct scoutfs_key_be { __u8 _sk_fourth; }__packed; -/* chose reasonable max key and value lens that have room for some u64s */ +/* chose reasonable max key lens that have room for some u64s */ #define SCOUTFS_BTREE_MAX_KEY_LEN 40 -#define SCOUTFS_BTREE_MAX_VAL_LEN 64 +/* when we split we want to have multiple items on each side */ +#define SCOUTFS_BTREE_MAX_VAL_LEN (SCOUTFS_BLOCK_SIZE / 8) /* * The min number of free bytes we must leave in a parent as we descend From f3a8a5110ee044cc5ad6e5c2291af6af223954ee Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 29 Sep 2019 22:12:52 -0700 Subject: [PATCH 749/920] scoutfs: allow btree update with different lengths The previous _btree_update implementation required that the new value be the same length as the old value. This change allows a new updated item to be a different length. It performs the btree walk assuming that the item will be larger so that there's room for the difference. It doesn't search for the size of the existing item so it doesn't know if the new item is smaller. It might leave the dirty leaf under the low water mark, which is fine. Signed-off-by: Zach Brown --- kmod/src/btree.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 1cf35586..b9e56585 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1356,10 +1356,17 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root } /* - * Update a btree item. The key and value must be of the same length (though - * it would be easy enough for us to change that if a caller cared). + * Update a btree item. -ENOENT is returned if the item didn't exist. + * + * We don't know the existing item's value length as we first descend. + * We assume that the new value is longer and try to split so that we + * can insert if that's true. If the new value is shorter than the + * existing then the leaf might fall under the minimum watermark, but at + * least we can do that while we simply can't insert a new longer value + * which doesn't fit. */ -int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len) { @@ -1372,19 +1379,14 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root if (invalid_item(key, key_len, val_len)) return -EINVAL; - ret = btree_walk(sb, root, BTW_DIRTY, key, key_len, 0, &bt, NULL, NULL); + ret = btree_walk(sb, root, BTW_DIRTY | BTW_INSERT, key, key_len, + val_len, &bt, NULL, NULL); if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { item = pos_item(bt, pos); - if (item_key_len(item) != key_len || - item_val_len(item) != val_len) { - ret = -EINVAL; - } else { - memcpy(item_key(item), key, key_len); - memcpy(item_val(item), val, val_len); - ret = 0; - } + delete_item(bt, pos); + create_item(bt, pos, key, key_len, val, val_len); ret = 0; } else { ret = -ENOENT; From 42b311c5beae441ef98bbb9cb4a189a1dd4901a7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Oct 2019 09:54:09 -0700 Subject: [PATCH 750/920] scoutfs: memmove deleted btree items It turns out that the sorting performed by btree block item compaction was pretty expensive. It's cheaper to keep the items packed at the end of the block by moving earlier items towards the back of the block as interior items are deleted. When the items are always packed at the end of the block we no longer need to track fragmented free space and can remove the 'free_reclaim' btree block field. This brought the bulk empty file create rate up by about 20%. Signed-off-by: Zach Brown --- kmod/src/btree.c | 178 ++++++++++++++-------------------------------- kmod/src/format.h | 1 - 2 files changed, 55 insertions(+), 124 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index b9e56585..dadafc0f 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -23,7 +23,6 @@ #include "format.h" #include "key.h" #include "btree.h" -#include "sort_priv.h" #include "counters.h" #include "triggers.h" #include "options.h" @@ -200,8 +199,8 @@ static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) le16_to_cpu(item->val_len)); } -/* number of contig free bytes between last item header and first item */ -static inline unsigned int contig_free(struct scoutfs_btree_block *bt) +/* number of free bytes between last item header and first item */ +static inline unsigned int free_bytes(struct scoutfs_btree_block *bt) { unsigned int nr = le16_to_cpu(bt->nr_items); @@ -209,17 +208,11 @@ static inline unsigned int contig_free(struct scoutfs_btree_block *bt) offsetof(struct scoutfs_btree_block, item_hdrs[nr]); } -/* number of contig bytes free after reclaiming free amongst items */ -static inline unsigned int reclaimable_free(struct scoutfs_btree_block *bt) -{ - return contig_free(bt) + le16_to_cpu(bt->free_reclaim); -} - /* all bytes used by item offsets, headers, and values */ static inline unsigned int used_total(struct scoutfs_btree_block *bt) { return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - - reclaimable_free(bt); + free_bytes(bt); } static inline struct scoutfs_btree_item * @@ -371,102 +364,13 @@ static bool all_roots_migrated(struct scoutfs_super_block *super) return true; } -static int cmp_hdr_item_key(void *priv, const void *a_ptr, const void *b_ptr) -{ - struct scoutfs_btree_block *bt = priv; - const struct scoutfs_btree_item_header *a_hdr = a_ptr; - const struct scoutfs_btree_item_header *b_hdr = b_ptr; - struct scoutfs_btree_item *a_item = off_item(bt, a_hdr->off); - struct scoutfs_btree_item *b_item = off_item(bt, b_hdr->off); - - return cmp_keys(item_key(a_item), item_key_len(a_item), - item_key(b_item), item_key_len(b_item)); -} - -static int cmp_hdr_off(void *priv, const void *a_ptr, const void *b_ptr) -{ - const struct scoutfs_btree_item_header *a_hdr = a_ptr; - const struct scoutfs_btree_item_header *b_hdr = b_ptr; - - return (int)le16_to_cpu(a_hdr->off) - (int)le16_to_cpu(b_hdr->off); -} - -static void swap_hdr(void *priv, void *a_ptr, void *b_ptr, int size) -{ - struct scoutfs_btree_item_header *a_hdr = a_ptr; - struct scoutfs_btree_item_header *b_hdr = b_ptr; - - swap(*a_hdr, *b_hdr); -} - -/* - * As items are deleted they create fragmented free space. Even if we - * indexed free space in the block it could still get sufficiently - * fragmented to force a split on insertion even though the two - * resulting blocks would have less than the minimum space consumed by - * items. - * - * We don't bother implementing free space indexing and addressing that - * corner case. Instead we track the number of bytes that could be - * reclaimed if we compacted the item space after the free_end offset. - * If this additional free space would satisfy an insertion then we - * compact the items instead of splitting the block. - * - * We move the free space to the center of the block by walking - * backwards through the items in offset order and packing them towards - * the end of the block. - * - * We don't have specific metadata to either walk the items in offset - * order or to update the item offsets as we move items. We sort the - * item offset array to achieve both ends. First we sort it by offset - * so we can walk in reverse order. As we move items we update their - * offset and then sort by keys once we're done. - */ -static void compact_items(struct scoutfs_btree_block *bt) -{ - unsigned int nr = le16_to_cpu(bt->nr_items); - struct scoutfs_btree_item *from; - struct scoutfs_btree_item *to; - unsigned int bytes; - __le16 end; - int i; - - sort_priv(bt, bt->item_hdrs, nr, sizeof(bt->item_hdrs[0]), - cmp_hdr_off, swap_hdr); - - end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); - - for (i = nr - 1; i >= 0; i--) { - from = pos_item(bt, i); - - bytes = item_bytes(from); - le16_add_cpu(&end, -bytes); - to = off_item(bt, end); - bt->item_hdrs[i].off = end; - - if (from != to) - memmove(to, from, bytes); - } - - bt->free_end = end; - bt->free_reclaim = 0; - - sort_priv(bt, bt->item_hdrs, nr, sizeof(bt->item_hdrs[0]), - cmp_hdr_item_key, swap_hdr); -} - /* move a number of contigous elements from the src index to the dst index */ #define memmove_arr(arr, dst, src, nr) \ memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) /* * Insert a new item into the block. The caller has made sure that - * there's space for the item and its metadata but we might have to - * compact the block to make that space contiguous. - * - * The possibility of compaction means that callers *can not* hold item, - * key, or value pointers across item creation. An easy way to verify - * this is to audit pos_item() callers. + * there's space for the item and its metadata. */ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, void *key, unsigned key_len, void *val, @@ -477,10 +381,7 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, unsigned all_bytes; all_bytes = all_len_bytes(key_len, val_len); - if (contig_free(bt) < all_bytes) { - BUG_ON(reclaimable_free(bt) < all_bytes); - compact_items(bt); - } + BUG_ON(free_bytes(bt) < all_bytes); if (pos < nr) memmove_arr(bt->item_hdrs, pos + 1, pos, nr - pos); @@ -503,24 +404,59 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, } /* - * Delete an item from a btree block. We record the amount of space it - * frees to later decide if we can satisfy an insertion by compaction - * instead of splitting. + * Delete an item from a btree block. + * + * This moves all the headers after the item (in sort order) towards the + * start of the header array. It moves all the items before the removed + * item towards the end of the block. The items that have to be moved + * can be anywhere in the sort order. We first move the item region + * and then walk the headers looking for offsets that need to be updated. + * + * The item motion means that callers can not hold item references + * across item deletion. */ static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) { - struct scoutfs_btree_item *item = pos_item(bt, pos); unsigned int nr = le16_to_cpu(bt->nr_items); + unsigned int updated; + unsigned int total; + unsigned int first; + unsigned int bytes; + unsigned int last; + unsigned int off; + int i; + + /* calculate region of items to move */ + first = le16_to_cpu(bt->free_end); + last = le16_to_cpu(bt->item_hdrs[pos].off); + total = last - first; + bytes = item_bytes(pos_item(bt, pos)); + + /* move items before deleted to the back of the block */ + if (total > 0) { + /* update headers before memove overwrites deleted item */ + for (i = 0, updated = 0; i < nr && updated < total; i++) { + off = le16_to_cpu(bt->item_hdrs[i].off); + if (off >= first && off < last) { + updated += item_bytes(pos_item(bt, i)); + le16_add_cpu(&bt->item_hdrs[i].off, bytes); + } + } + BUG_ON(updated != total); + + memmove(off_item(bt, cpu_to_le16(first + bytes)), + off_item(bt, cpu_to_le16(first)), total); + + } + + /* wipe deleted bytes to avoid leaking data */ + memset(off_item(bt, cpu_to_le16(first)), 0, bytes); if (pos < (nr - 1)) memmove_arr(bt->item_hdrs, pos, pos + 1, nr - 1 - pos); - le16_add_cpu(&bt->free_reclaim, item_bytes(item)); - nr--; - bt->nr_items = cpu_to_le16(nr); - - /* wipe deleted items to avoid leaking data */ - memset(item, 0, item_bytes(item)); + le16_add_cpu(&bt->free_end, bytes); + le16_add_cpu(&bt->nr_items, -1); } /* @@ -884,7 +820,7 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, else all_bytes = all_len_bytes(key_len, val_len); - if (reclaimable_free(right) >= all_bytes) + if (free_bytes(right) >= all_bytes) return 0; /* alloc split neighbour first to avoid unwinding tree growth */ @@ -1007,7 +943,7 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, static int verify_btree_block(struct scoutfs_btree_block *bt, int level) { struct scoutfs_btree_item *item; - struct scoutfs_btree_item *prev; + struct scoutfs_btree_item *prev = NULL; unsigned int bytes = 0; unsigned int after_off = sizeof(struct scoutfs_btree_block); unsigned int first_off; @@ -1048,17 +984,15 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) if (first_off < le16_to_cpu(bt->free_end)) goto out; - if ((le16_to_cpu(bt->free_end) + bytes + - le16_to_cpu(bt->free_reclaim)) != SCOUTFS_BLOCK_SIZE) + if ((le16_to_cpu(bt->free_end) + bytes) != SCOUTFS_BLOCK_SIZE) goto out; bad = 0; out: if (bad) { - printk("bt %p blkno %llu level %d end %u reclaim %u nr %u (after %u bytes %u)\n", + printk("bt %p blkno %llu level %d end %u nr %u (after %u bytes %u)\n", bt, le64_to_cpu(bt->hdr.blkno), level, - le16_to_cpu(bt->free_end), - le16_to_cpu(bt->free_reclaim), le16_to_cpu(bt->nr_items), + le16_to_cpu(bt->free_end), le16_to_cpu(bt->nr_items), after_off, bytes); for (i = 0; i < nr; i++) { item = pos_item(bt, i); @@ -1370,7 +1304,6 @@ int scoutfs_btree_update(struct super_block *sb, void *key, unsigned key_len, void *val, unsigned val_len) { - struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; int pos; int cmp; @@ -1384,7 +1317,6 @@ int scoutfs_btree_update(struct super_block *sb, if (ret == 0) { pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { - item = pos_item(bt, pos); delete_item(bt, pos); create_item(bt, pos, key, key_len, val, val_len); ret = 0; diff --git a/kmod/src/format.h b/kmod/src/format.h index f189f623..340f1c57 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -223,7 +223,6 @@ struct scoutfs_btree_item { struct scoutfs_btree_block { struct scoutfs_block_header hdr; __le16 free_end; - __le16 free_reclaim; __le16 nr_items; __u8 level; struct scoutfs_btree_item_header item_hdrs[0]; From e444c2b8c218587764c7bcf83a4d88ff21eaf49f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Oct 2019 13:26:27 -0700 Subject: [PATCH 751/920] scoutfs: remove sort_priv The only user was item compaction in the btree and it has been removed. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 - kmod/src/sort_priv.c | 71 -------------------------------------------- kmod/src/sort_priv.h | 8 ----- 3 files changed, 80 deletions(-) delete mode 100644 kmod/src/sort_priv.c delete mode 100644 kmod/src/sort_priv.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 8048bfd9..9b776a71 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -34,7 +34,6 @@ scoutfs-y += \ scoutfs_trace.o \ seg.o \ server.o \ - sort_priv.o \ spbm.o \ super.o \ sysfs.o \ diff --git a/kmod/src/sort_priv.c b/kmod/src/sort_priv.c deleted file mode 100644 index 2acc0802..00000000 --- a/kmod/src/sort_priv.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * A copy of sort() from upstream with a priv argument that's passed - * to comparison, like list_sort(). - */ - -/* ------------------------ */ - -/* - * A fast, small, non-recursive O(nlog n) sort for the Linux kernel - * - * Jan 23 2005 Matt Mackall - */ - -#include -#include -#include -#include -#include "sort_priv.h" - -/** - * sort - sort an array of elements - * @priv: caller's pointer to pass to comparison and swap functions - * @base: pointer to data to sort - * @num: number of elements - * @size: size of each element - * @cmp_func: pointer to comparison function - * @swap_func: pointer to swap function or NULL - * - * This function does a heapsort on the given array. You may provide a - * swap_func function optimized to your element type. - * - * Sorting time is O(n log n) both on average and worst-case. While - * qsort is about 20% faster on average, it suffers from exploitable - * O(n*n) worst-case behavior and extra memory requirements that make - * it less suitable for kernel use. - */ - -void sort_priv(void *priv, void *base, size_t num, size_t size, - int (*cmp_func)(void *priv, const void *, const void *), - void (*swap_func)(void *priv, void *, void *, int size)) -{ - /* pre-scale counters for performance */ - int i = (num/2 - 1) * size, n = num * size, c, r; - - /* heapify */ - for ( ; i >= 0; i -= size) { - for (r = i; r * 2 + size < n; r = c) { - c = r * 2 + size; - if (c < n - size && - cmp_func(priv, base + c, base + c + size) < 0) - c += size; - if (cmp_func(priv, base + r, base + c) >= 0) - break; - swap_func(priv, base + r, base + c, size); - } - } - - /* sort */ - for (i = n - size; i > 0; i -= size) { - swap_func(priv, base, base + i, size); - for (r = 0; r * 2 + size < i; r = c) { - c = r * 2 + size; - if (c < i - size && - cmp_func(priv, base + c, base + c + size) < 0) - c += size; - if (cmp_func(priv, base + r, base + c) >= 0) - break; - swap_func(priv, base + r, base + c, size); - } - } -} diff --git a/kmod/src/sort_priv.h b/kmod/src/sort_priv.h deleted file mode 100644 index c5fde547..00000000 --- a/kmod/src/sort_priv.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SCOUTFS_SORT_PRIV_H_ -#define _SCOUTFS_SORT_PRIV_H_ - -void sort_priv(void *priv, void *base, size_t num, size_t size, - int (*cmp_func)(void *priv, const void *, const void *), - void (*swap_func)(void *priv, void *, void *, int size)); - -#endif From 9456eda5837ba3a4859d290eff2fdac5d64971a8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Oct 2019 15:01:02 -0700 Subject: [PATCH 752/920] scoutfs: support larger btree block sizes The btree block header had some aggressively small values that limited the largest block size that could be supported. Use larger 32bit values so that we can support larger block sizes. Signed-off-by: Zach Brown --- kmod/src/btree.c | 76 +++++++++++++++++++++++------------------------ kmod/src/format.h | 6 ++-- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index dadafc0f..674c7e50 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -151,7 +151,7 @@ static char max_key[SCOUTFS_BTREE_MAX_KEY_LEN] = { }; /* number of contiguous bytes used by the item header, key, and value */ -static inline unsigned len_bytes(unsigned key_len, unsigned val_len) +static inline unsigned int len_bytes(unsigned key_len, unsigned val_len) { return sizeof(struct scoutfs_btree_item) + key_len + val_len; } @@ -202,9 +202,9 @@ static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) /* number of free bytes between last item header and first item */ static inline unsigned int free_bytes(struct scoutfs_btree_block *bt) { - unsigned int nr = le16_to_cpu(bt->nr_items); + unsigned int nr = le32_to_cpu(bt->nr_items); - return le16_to_cpu(bt->free_end) - + return le32_to_cpu(bt->free_end) - offsetof(struct scoutfs_btree_block, item_hdrs[nr]); } @@ -216,9 +216,9 @@ static inline unsigned int used_total(struct scoutfs_btree_block *bt) } static inline struct scoutfs_btree_item * -off_item(struct scoutfs_btree_block *bt, __le16 off) +off_item(struct scoutfs_btree_block *bt, __le32 off) { - return (void *)bt + le16_to_cpu(off); + return (void *)bt + le32_to_cpu(off); } static inline struct scoutfs_btree_item * @@ -230,7 +230,7 @@ pos_item(struct scoutfs_btree_block *bt, unsigned int pos) static inline struct scoutfs_btree_item * last_item(struct scoutfs_btree_block *bt) { - return pos_item(bt, le16_to_cpu(bt->nr_items) - 1); + return pos_item(bt, le32_to_cpu(bt->nr_items) - 1); } static inline void *item_key(struct scoutfs_btree_item *item) @@ -275,7 +275,7 @@ static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, { struct scoutfs_btree_item *item; unsigned int start = 0; - unsigned int end = le16_to_cpu(bt->nr_items); + unsigned int end = le32_to_cpu(bt->nr_items); unsigned int pos = 0; *cmp = -1; @@ -376,7 +376,7 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, void *key, unsigned key_len, void *val, unsigned val_len) { - unsigned nr = le16_to_cpu(bt->nr_items); + unsigned int nr = le32_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; unsigned all_bytes; @@ -386,12 +386,12 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, if (pos < nr) memmove_arr(bt->item_hdrs, pos + 1, pos, nr - pos); - le16_add_cpu(&bt->free_end, -len_bytes(key_len, val_len)); + le32_add_cpu(&bt->free_end, -len_bytes(key_len, val_len)); bt->item_hdrs[pos].off = bt->free_end; nr++; - bt->nr_items = cpu_to_le16(nr); + bt->nr_items = cpu_to_le32(nr); - BUG_ON(le16_to_cpu(bt->free_end) < + BUG_ON(le32_to_cpu(bt->free_end) < offsetof(struct scoutfs_btree_block, item_hdrs[nr])); item = pos_item(bt, pos); @@ -417,7 +417,7 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, */ static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) { - unsigned int nr = le16_to_cpu(bt->nr_items); + unsigned int nr = le32_to_cpu(bt->nr_items); unsigned int updated; unsigned int total; unsigned int first; @@ -427,8 +427,8 @@ static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) int i; /* calculate region of items to move */ - first = le16_to_cpu(bt->free_end); - last = le16_to_cpu(bt->item_hdrs[pos].off); + first = le32_to_cpu(bt->free_end); + last = le32_to_cpu(bt->item_hdrs[pos].off); total = last - first; bytes = item_bytes(pos_item(bt, pos)); @@ -436,27 +436,27 @@ static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) if (total > 0) { /* update headers before memove overwrites deleted item */ for (i = 0, updated = 0; i < nr && updated < total; i++) { - off = le16_to_cpu(bt->item_hdrs[i].off); + off = le32_to_cpu(bt->item_hdrs[i].off); if (off >= first && off < last) { updated += item_bytes(pos_item(bt, i)); - le16_add_cpu(&bt->item_hdrs[i].off, bytes); + le32_add_cpu(&bt->item_hdrs[i].off, bytes); } } BUG_ON(updated != total); - memmove(off_item(bt, cpu_to_le16(first + bytes)), - off_item(bt, cpu_to_le16(first)), total); + memmove(off_item(bt, cpu_to_le32(first + bytes)), + off_item(bt, cpu_to_le32(first)), total); } /* wipe deleted bytes to avoid leaking data */ - memset(off_item(bt, cpu_to_le16(first)), 0, bytes); + memset(off_item(bt, cpu_to_le32(first)), 0, bytes); if (pos < (nr - 1)) memmove_arr(bt->item_hdrs, pos, pos + 1, nr - 1 - pos); - le16_add_cpu(&bt->free_end, bytes); - le16_add_cpu(&bt->nr_items, -1); + le32_add_cpu(&bt->free_end, bytes); + le32_add_cpu(&bt->nr_items, -1); } /* @@ -474,14 +474,14 @@ static void move_items(struct scoutfs_btree_block *dst, unsigned int f; if (move_right) { - f = le16_to_cpu(src->nr_items) - 1; + f = le32_to_cpu(src->nr_items) - 1; t = 0; } else { f = 0; - t = le16_to_cpu(dst->nr_items); + t = le32_to_cpu(dst->nr_items); } - while (f < le16_to_cpu(src->nr_items) && to_move > 0) { + while (f < le32_to_cpu(src->nr_items) && to_move > 0) { from = pos_item(src, f); create_item(dst, t, item_key(from), item_key_len(from), @@ -735,7 +735,7 @@ retry: new = NULL; memset(bt, 0, SCOUTFS_BLOCK_SIZE); bt->hdr.fsid = super->hdr.fsid; - bt->free_end = cpu_to_le16(SCOUTFS_BLOCK_SIZE); + bt->free_end = cpu_to_le32(SCOUTFS_BLOCK_SIZE); } bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE); @@ -918,13 +918,13 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, update_parent_item(bring, parent, pos, bt); /* update or delete sibling's parent item */ - if (le16_to_cpu(sib->nr_items) == 0) + if (le32_to_cpu(sib->nr_items) == 0) delete_item(parent, sib_pos); else if (move_right) update_parent_item(bring, parent, sib_pos, sib); /* and finally shrink the tree if our parent is the root with 1 */ - if (le16_to_cpu(parent->nr_items) == 1) { + if (le32_to_cpu(parent->nr_items) == 1) { root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; @@ -952,7 +952,7 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) unsigned int i = 0; int bad = 1; - nr = le16_to_cpu(bt->nr_items); + nr = le32_to_cpu(bt->nr_items); if (nr == 0) goto out; @@ -965,7 +965,7 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) } for (i = 0; i < nr; i++) { - off = le16_to_cpu(bt->item_hdrs[i].off); + off = le32_to_cpu(bt->item_hdrs[i].off); if (off >= SCOUTFS_BLOCK_SIZE || off < after_off) goto out; @@ -981,10 +981,10 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) prev = item; } - if (first_off < le16_to_cpu(bt->free_end)) + if (first_off < le32_to_cpu(bt->free_end)) goto out; - if ((le16_to_cpu(bt->free_end) + bytes) != SCOUTFS_BLOCK_SIZE) + if ((le32_to_cpu(bt->free_end) + bytes) != SCOUTFS_BLOCK_SIZE) goto out; bad = 0; @@ -992,12 +992,12 @@ out: if (bad) { printk("bt %p blkno %llu level %d end %u nr %u (after %u bytes %u)\n", bt, le64_to_cpu(bt->hdr.blkno), level, - le16_to_cpu(bt->free_end), le16_to_cpu(bt->nr_items), + le32_to_cpu(bt->free_end), le32_to_cpu(bt->nr_items), after_off, bytes); for (i = 0; i < nr; i++) { item = pos_item(bt, i); printk(" [%u] off %u key_len %u val_len %u\n", - i, le16_to_cpu(bt->item_hdrs[i].off), + i, le32_to_cpu(bt->item_hdrs[i].off), item_key_len(item), item_val_len(item)); } BUG_ON(bad); @@ -1049,9 +1049,9 @@ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_item *item; struct scoutfs_btree_ref *ref; - unsigned level; - unsigned pos; - unsigned nr; + unsigned int level; + unsigned int pos; + unsigned int nr; int cmp; int ret; @@ -1139,7 +1139,7 @@ restart: if (level == 0) break; - nr = le16_to_cpu(bt->nr_items); + nr = le32_to_cpu(bt->nr_items); /* Find the next child block for the search key. */ pos = find_pos(bt, key, key_len, &cmp); @@ -1423,7 +1423,7 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, pos--; /* found the next item in this leaf */ - if (pos >= 0 && pos < le16_to_cpu(bt->nr_items)) { + if (pos >= 0 && pos < le32_to_cpu(bt->nr_items)) { item = pos_item(bt, pos); init_item_ref(iref, item); ret = 0; diff --git a/kmod/src/format.h b/kmod/src/format.h index 340f1c57..785a441e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -211,7 +211,7 @@ struct scoutfs_btree_root { } __packed; struct scoutfs_btree_item_header { - __le16 off; + __le32 off; } __packed; struct scoutfs_btree_item { @@ -222,8 +222,8 @@ struct scoutfs_btree_item { struct scoutfs_btree_block { struct scoutfs_block_header hdr; - __le16 free_end; - __le16 nr_items; + __le32 free_end; + __le32 nr_items; __u8 level; struct scoutfs_btree_item_header item_hdrs[0]; } __packed; From d20c950c17a495b4bd8c1d1d5a0ac03ce37ccd37 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 18 Oct 2019 14:33:42 -0700 Subject: [PATCH 753/920] scoutfs: restore our block cache Previous versions of the system had a simple block cache. This brings it back with support for blocks that are larger than page size, a more efficient LRU, and an explicit writer context. Signed-off-by: Zach Brown --- kmod/src/block.c | 821 +++++++++++++++++++++++++++++++++++++++++++- kmod/src/block.h | 41 +++ kmod/src/counters.h | 12 +- kmod/src/super.c | 2 + kmod/src/super.h | 2 + 5 files changed, 876 insertions(+), 2 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index bfe07e16..ecc62c21 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Versity Software, Inc. All rights reserved. + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -12,11 +12,87 @@ */ #include #include +#include #include +#include +#include +#include +#include +#include #include "format.h" #include "super.h" #include "block.h" +#include "counters.h" +#include "msg.h" + +/* + * The scoutfs block cache manages metadata blocks that can be larger + * than the page size. Callers can have their own contexts for tracking + * dirty blocks that are written together. We pin dirty blocks in + * memory and only checksum them all as they're all written. + * + * An LRU is maintained so the VM can reclaim the oldest presumably + * unlikely to be used blocks. But we don't maintain a perfect record + * of access order. We only move accessed blocks to the tail of the rcu + * if they weren't in the most recently moved fraction of the total + * population. This means that reclaim will walk through waves of that + * fraction of the population. It's close enough and removes lru + * maintenance locking from the fast path. + */ + +struct block_info { + struct super_block *sb; + spinlock_t lock; + struct radix_tree_root radix; + struct list_head lru_list; + u64 lru_nr; + u64 lru_move_counter; + wait_queue_head_t waitq; + struct shrinker shrinker; + struct work_struct free_work; + struct llist_head free_llist; +}; + +#define DECLARE_BLOCK_INFO(sb, name) \ + struct block_info *name = SCOUTFS_SB(sb)->block_info + +enum { + BLOCK_BIT_UPTODATE = 0, /* contents consistent with media */ + BLOCK_BIT_NEW, /* newly allocated, contents undefined */ + BLOCK_BIT_DIRTY, /* dirty, writer will write */ + BLOCK_BIT_ERROR, /* saw IO error */ + BLOCK_BIT_DELETED, /* has been deleted from radix tree */ + BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ + BLOCK_BIT_VIRT, /* mapped virt allocation */ + BLOCK_BIT_CRC_VALID, /* crc has been verified */ +}; + +struct block_private { + struct scoutfs_block bl; + struct super_block *sb; + atomic_t refcount; + union { + struct list_head lru_entry; + struct llist_node free_node; + }; + u64 lru_moved; + struct list_head dirty_entry; + unsigned long bits; + atomic_t io_count; + union { + struct page *page; + void *virt; + }; +}; + +#define BLOCK_PRIVATE(_bl) \ + container_of((_bl), struct block_private, bl) + +/* + * These _block_header helpers are from a previous generation and may + * be refactored away. + */ __le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr) { @@ -41,3 +117,746 @@ bool scoutfs_block_valid_ref(struct super_block *sb, return hdr->fsid == super->hdr.fsid && hdr->seq == seq && hdr->blkno == blkno; } + +static struct block_private *block_alloc(struct super_block *sb, u64 blkno) +{ + struct block_private *bp; + + /* + * If we had multiple blocks per page we'd need to be a little + * more careful with a partial page allocator when allocating + * blocks and would make the lru per-page instead of per-block. + */ + BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); + + bp = kzalloc(sizeof(struct block_private), GFP_NOFS); + if (!bp) + goto out; + + bp->page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); + if (bp->page) { + scoutfs_inc_counter(sb, block_cache_alloc_page_order); + set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); + bp->bl.data = page_address(bp->page); + } else { + bp->virt = __vmalloc(SCOUTFS_BLOCK_SIZE, + GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); + if (!bp->virt) { + kfree(bp); + bp = NULL; + goto out; + } + + scoutfs_inc_counter(sb, block_cache_alloc_virt); + set_bit(BLOCK_BIT_VIRT, &bp->bits); + bp->bl.data = bp->virt; + } + + bp->bl.blkno = blkno; + bp->sb = sb; + atomic_set(&bp->refcount, 1); + INIT_LIST_HEAD(&bp->lru_entry); + INIT_LIST_HEAD(&bp->dirty_entry); + set_bit(BLOCK_BIT_NEW, &bp->bits); + atomic_set(&bp->io_count, 0); + +out: + if (!bp) + scoutfs_inc_counter(sb, block_cache_alloc_failure); + return bp; +} + +static void block_free(struct super_block *sb, struct block_private *bp) +{ + scoutfs_inc_counter(sb, block_cache_free); + + if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) + __free_pages(bp->page, SCOUTFS_BLOCK_PAGE_ORDER); + else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) + vfree(bp->virt); + else + BUG(); + + /* lru_entry could have been clobbered by union member free_node */ + WARN_ON_ONCE(!list_empty(&bp->dirty_entry)); + WARN_ON_ONCE(atomic_read(&bp->refcount)); + WARN_ON_ONCE(atomic_read(&bp->io_count)); + kfree(bp); +} + +/* + * We free blocks in task context so we can free kernel virtual mappings. + */ +static void block_free_work(struct work_struct *work) +{ + struct block_info *binf = container_of(work, struct block_info, + free_work); + struct super_block *sb = binf->sb; + struct block_private *bp; + struct llist_node *deleted; + + deleted = llist_del_all(&binf->free_llist); + + llist_for_each_entry(bp, deleted, free_node) { + block_free(sb, bp); + } +} + +/* + * After we've dropped the final ref kick off the final free in task + * context. This happens in the relatively rare cases of IO errors, + * stale cached data, memory pressure, and unmount. + */ +static void block_put(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + + if (!IS_ERR_OR_NULL(bp) && atomic_dec_and_test(&bp->refcount)) { + WARN_ON_ONCE(!list_empty(&bp->lru_entry)); + llist_add(&bp->free_node, &binf->free_llist); + schedule_work(&binf->free_work); + } +} + +/* + * Add a new block into the cache. The caller holds the lock and has + * preloaded the radix. + */ +static void block_insert(struct super_block *sb, struct block_private *bp, + u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + + assert_spin_locked(&binf->lock); + BUG_ON(!list_empty(&bp->lru_entry)); + + atomic_inc(&bp->refcount); + radix_tree_insert(&binf->radix, blkno, bp); + list_add_tail(&bp->lru_entry, &binf->lru_list); + bp->lru_moved = ++binf->lru_move_counter; + binf->lru_nr++; +} + +/* + * Only move the block to the tail of the LRU if it's outside of the + * small fraction of the lru population that has been most recently + * used. This gives us a reasonable number of most recently accessed + * blocks which will be reclaimed after the rest of the least recently + * used blocks while reducing per-access locking overhead of maintaining + * the LRU. We don't care about unlikely non-atomic u64 accesses racing + * and messing up LRU position. + * + * This can race with blocks being removed from the cache (shrinking, + * stale, errors) so we're careful to only move the entry if it's still + * on the list after we acquire the lock. We still hold a reference so it's + * lru_entry hasn't transitioned to being used as the free_node. + */ +static void block_accessed(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + u64 recent = binf->lru_nr >> 3; + + scoutfs_inc_counter(sb, block_cache_access); + + if (bp->lru_moved < (binf->lru_move_counter - recent)) { + spin_lock(&binf->lock); + if (!list_empty(&bp->lru_entry)) { + list_move_tail(&bp->lru_entry, &binf->lru_list); + bp->lru_moved = ++binf->lru_move_counter; + scoutfs_inc_counter(sb, block_cache_lru_move); + } + spin_unlock(&binf->lock); + } +} + +/* + * Remove a block from the cache and drop its reference. We only remove + * the block once as the deleted bit is first set. + */ +static void block_remove(struct super_block *sb, struct block_private *bp) +{ + DECLARE_BLOCK_INFO(sb, binf); + + assert_spin_locked(&binf->lock); + + if (!test_and_set_bit(BLOCK_BIT_DELETED, &bp->bits)) { + BUG_ON(list_empty(&bp->lru_entry)); + radix_tree_delete(&binf->radix, bp->bl.blkno); + list_del_init(&bp->lru_entry); + binf->lru_nr--; + block_put(sb, bp); + } +} + +/* + * Called during shutdown with no other users. + */ +static void block_remove_all(struct super_block *sb) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp; + + spin_lock(&binf->lock); + + while (radix_tree_gang_lookup(&binf->radix, (void **)&bp, 0, 1) == 1) { + wait_event(binf->waitq, atomic_read(&bp->io_count) == 0); + block_remove(sb, bp); + } + + spin_unlock(&binf->lock); + + WARN_ON_ONCE(!list_empty(&binf->lru_list)); + WARN_ON_ONCE(binf->lru_nr != 0); + WARN_ON_ONCE(binf->radix.rnode != NULL); +} + +/* + * XXX The io_count and sb fields in the block_private are only used + * during IO. We don't need to have them sitting around for the entire + * lifetime of each cached block. + * + * This is happening in interrupt context so we do as little work as + * possible. Final freeing, verifying checksums, and unlinking errored + * blocks are all done by future users of the blocks. + */ +static void block_end_io(struct super_block *sb, int rw, + struct block_private *bp, int err) +{ + DECLARE_BLOCK_INFO(sb, binf); + bool is_read = !(rw & WRITE); + + if (err) { + scoutfs_inc_counter(sb, block_cache_end_io_error); + set_bit(BLOCK_BIT_ERROR, &bp->bits); + } + + /* update bits before waiters see io_count == 0 */ + if (atomic_read(&bp->io_count) == 1) { + if (is_read && !test_bit(BLOCK_BIT_ERROR, &bp->bits)) + set_bit(BLOCK_BIT_UPTODATE, &bp->bits); + } + + /* make sure bits are visible to woken */ + smp_mb__after_atomic(); + + /* then wake */ + if (atomic_dec_and_test(&bp->io_count)) + wake_up(&binf->waitq); +} + +static void block_bio_end_io(struct bio *bio, int err) +{ + struct block_private *bp = bio->bi_private; + struct super_block *sb = bp->sb; + + block_end_io(sb, bio->bi_rw, bp, err); + bio_put(bio); + block_put(sb, bp); +} + +/* + * Kick off IO for a single block. + */ +static int block_submit_bio(struct super_block *sb, struct block_private *bp, + int rw) +{ + struct bio *bio = NULL; + struct blk_plug plug; + struct page *page; + unsigned long off; + sector_t sector; + int ret = 0; + + sector = bp->bl.blkno << (SCOUTFS_BLOCK_SHIFT - 9); + + WARN_ON_ONCE(bp->bl.blkno == U64_MAX); + WARN_ON_ONCE(sector == U64_MAX || sector == 0); + + /* don't let racing end_io during submission think block is complete */ + atomic_inc(&bp->io_count); + + blk_start_plug(&plug); + + for (off = 0; off < SCOUTFS_BLOCK_SIZE; off += PAGE_SIZE) { + if (!bio) { + bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); + if (!bio) { + ret = -ENOMEM; + break; + } + + bio->bi_sector = sector + (off >> 9); + bio->bi_bdev = sb->s_bdev; + bio->bi_end_io = block_bio_end_io; + bio->bi_private = bp; + + atomic_inc(&bp->refcount); + atomic_inc(&bp->io_count); + } + + if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) + page = virt_to_page((char *)bp->bl.data + off); + else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) + page = vmalloc_to_page((char *)bp->bl.data + off); + else + BUG(); + + if (!bio_add_page(bio, page, PAGE_SIZE, 0)) { + submit_bio(rw, bio); + bio = NULL; + } + } + + if (bio) + submit_bio(rw, bio); + + blk_finish_plug(&plug); + + /* let racing end_io know we're done */ + block_end_io(sb, rw, bp, ret); + + return ret; +} + +/* + * Return a reference to a cached block in the system, allocating a new + * block if one isn't found in the radix. Its contents are undefined if + * it's newly allocated. + */ +static struct block_private *block_get(struct super_block *sb, u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *found; + struct block_private *bp; + int ret; + + rcu_read_lock(); + bp = radix_tree_lookup(&binf->radix, blkno); + if (bp) + atomic_inc(&bp->refcount); + rcu_read_unlock(); + + /* drop failed reads that interrupted waiters abandoned */ + if (bp && (test_bit(BLOCK_BIT_ERROR, &bp->bits) && + !test_bit(BLOCK_BIT_DIRTY, &bp->bits))) { + spin_lock(&binf->lock); + block_remove(sb, bp); + spin_unlock(&binf->lock); + block_put(sb, bp); + bp = NULL; + } + + if (!bp) { + bp = block_alloc(sb, blkno); + if (bp == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + /* could use slot instead of lookup/insert */ + spin_lock(&binf->lock); + found = radix_tree_lookup(&binf->radix, blkno); + if (found) { + atomic_inc(&found->refcount); + } else { + block_insert(sb, bp, blkno); + } + spin_unlock(&binf->lock); + radix_tree_preload_end(); + + if (found) { + block_put(sb, bp); + bp = found; + } + } + + block_accessed(sb, bp); + ret = 0; + +out: + if (ret < 0) { + block_put(sb, bp); + return ERR_PTR(ret); + } + + return bp; +} + +/* + * Return a cached block or a newly allocated block whose contents are + * undefined. The caller is going to initialize the block contents. + */ +struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno) +{ + struct block_private *bp; + + bp = block_get(sb, blkno); + if (IS_ERR(bp)) + return ERR_CAST(bp); + + set_bit(BLOCK_BIT_UPTODATE, &bp->bits); + set_bit(BLOCK_BIT_CRC_VALID, &bp->bits); + + return &bp->bl; +} + +static bool uptodate_or_error(struct block_private *bp) +{ + smp_rmb(); /* test after adding to wait queue */ + return test_bit(BLOCK_BIT_UPTODATE, &bp->bits) || + test_bit(BLOCK_BIT_ERROR, &bp->bits); +} + +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp = NULL; + int ret; + + bp = block_get(sb, blkno); + if (IS_ERR(bp)) { + ret = PTR_ERR(bp); + goto out; + } + + if (!test_bit(BLOCK_BIT_UPTODATE, &bp->bits) && + test_and_clear_bit(BLOCK_BIT_NEW, &bp->bits)) { + ret = block_submit_bio(sb, bp, READ); + if (ret < 0) + goto out; + } + + ret = wait_event_interruptible(binf->waitq, uptodate_or_error(bp)); + if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bp->bits)) + ret = -EIO; + +out: + if (ret < 0) { + block_put(sb, bp); + return ERR_PTR(ret); + } + + return &bp->bl; +} + +/* + * Drop a stale cached read block from the cache. A future read will + * re-read the block from the device. This doesn't drop the caller's reference, + * they still have to call _put. + */ +void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (!WARN_ON_ONCE(test_bit(BLOCK_BIT_DIRTY, &bp->bits))) { + scoutfs_inc_counter(sb, block_cache_invalidate); + spin_lock(&binf->lock); + block_remove(sb, bp); + spin_unlock(&binf->lock); + } +} + +bool scoutfs_block_consistent_ref(struct super_block *sb, + struct scoutfs_block *bl, + __le64 seq, __le64 blkno, u32 magic) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct block_private *bp = BLOCK_PRIVATE(bl); + struct scoutfs_block_header *hdr = bl->data; + + if (!test_bit(BLOCK_BIT_CRC_VALID, &bp->bits)) { + if (hdr->crc != scoutfs_block_calc_crc(hdr)) + return false; + set_bit(BLOCK_BIT_CRC_VALID, &bp->bits); + } + + return hdr->magic == cpu_to_le32(magic) && + hdr->fsid == super->hdr.fsid && + hdr->seq == seq && + hdr->blkno == blkno; +} + +void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl) +{ + if (!IS_ERR_OR_NULL(bl)) + block_put(sb, BLOCK_PRIVATE(bl)); +} + +void scoutfs_block_writer_init(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + spin_lock_init(&wri->lock); + INIT_LIST_HEAD(&wri->dirty_list); + wri->nr_dirty_blocks = 0; +} + +/* + * Mark a given block dirty. The caller serializes all dirtying calls + * with writer write calls. As it happens we dirty in allocation order + * and allocate with an advancing cursor so we always dirty in block + * offset order and can walk our list to submit nice ordered IO. + */ +void scoutfs_block_writer_mark_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (!test_and_set_bit(BLOCK_BIT_DIRTY, &bp->bits)) { + BUG_ON(!list_empty(&bp->dirty_entry)); + atomic_inc(&bp->refcount); + spin_lock(&wri->lock); + list_add_tail(&bp->dirty_entry, &wri->dirty_list); + wri->nr_dirty_blocks++; + spin_unlock(&wri->lock); + } +} + +bool scoutfs_block_writer_is_dirty(struct super_block *sb, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + return test_bit(BLOCK_BIT_DIRTY, &bp->bits) != 0; +} + +/* + * Submit writes for all the dirty blocks in the writer's dirty list and + * wait for them to complete. The caller must serialize this with + * attempts to dirty blocks in the writer. If we return an error then + * all the blocks will still be considered dirty. This can be called + * again to attempt to write all the blocks again. + */ +int scoutfs_block_writer_write(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct scoutfs_block_header *hdr; + struct block_private *bp; + struct blk_plug plug; + int ret = 0; + + if (wri->nr_dirty_blocks == 0) + return 0; + + /* checksum everything to reduce time between io submission merging */ + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + hdr = bp->bl.data; + hdr->crc = scoutfs_block_calc_crc(hdr); + } + + blk_start_plug(&plug); + + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + /* retry previous write errors */ + clear_bit(BLOCK_BIT_ERROR, &bp->bits); + + ret = block_submit_bio(sb, bp, WRITE); + if (ret < 0) + break; + } + + blk_finish_plug(&plug); + + list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { + /* XXX should this be interruptible? */ + wait_event(binf->waitq, atomic_read(&bp->io_count) == 0); + if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bp->bits)) { + clear_bit(BLOCK_BIT_ERROR, &bp->bits); + ret = -EIO; + } + } + + if (ret == 0) + scoutfs_block_writer_forget_all(sb, wri); + + return ret; +} + +static void block_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct block_private *bp) +{ + assert_spin_locked(&wri->lock); + + clear_bit(BLOCK_BIT_DIRTY, &bp->bits); + list_del_init(&bp->dirty_entry); + wri->nr_dirty_blocks--; + block_put(sb, bp); +} + +/* + * Clear the dirty status of all the blocks in the writer. The blocks + * remain clean in cache but can be freed by reclaim and then re-read + * from disk, losing whatever modifications made them dirty. + */ +void scoutfs_block_writer_forget_all(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + struct block_private *tmp; + struct block_private *bp; + + spin_lock(&wri->lock); + + list_for_each_entry_safe(bp, tmp, &wri->dirty_list, dirty_entry) + block_forget(sb, wri, bp); + + spin_unlock(&wri->lock); +} + +/* + * Forget that the given block was dirty. It won't be written in the + * future. Its contents remain in the cache. This is typically used + * as a block is freed. If it is allocated and re-used then its contents + * will be re-initialized. + * + * The caller should ensure that we don't try and mark and forget the + * same block, but this is racing with marking and forgetting other + * blocks. + */ +void scoutfs_block_writer_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) { + scoutfs_inc_counter(sb, block_cache_forget); + spin_lock(&wri->lock); + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) + block_forget(sb, wri, bp); + spin_unlock(&wri->lock); + } +} + +/* + * The caller has ensured that no more dirtying will take place. This + * helps the caller avoid doing a bunch of work before calling into the + * writer to write dirty blocks that didn't exist. + */ +bool scoutfs_block_writer_has_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + return wri->nr_dirty_blocks != 0; +} + +/* + * This is a best-effort guess. It's only used for heuristics so it's OK + * if it goes a little bonkers sometimes. + */ +u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, + struct scoutfs_block_writer *wri) +{ + return wri->nr_dirty_blocks * SCOUTFS_BLOCK_SIZE; +} + +/* + * Remove a number of least recently accessed blocks and free them. We + * don't take locking hit of removing blocks from the lru as they're + * used so this is racing with accesses holding an elevated refcount. + * We check the refcount to attempt to not free a block that snuck in + * and is being accessed while the block is still at the head of the + * LRU. + * + * Dirty blocks will always have an elevated refcount (and will be + * likely be towards the tail of the LRU). Even if we do remove them + * from the LRU their dirty refcount will keep them live until IO + * completes and their dirty refcount is dropped. + */ +static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) +{ + struct block_info *binf = container_of(shrink, struct block_info, + shrinker); + struct super_block *sb = binf->sb; + struct block_private *tmp; + struct block_private *bp; + unsigned long nr; + LIST_HEAD(list); + + nr = sc->nr_to_scan; + if (!nr) + goto out; + + spin_lock(&binf->lock); + + list_for_each_entry_safe(bp, tmp, &binf->lru_list, lru_entry) { + + if (atomic_read(&bp->refcount) > 1) + continue; + + if (nr-- == 0) + break; + + scoutfs_inc_counter(sb, block_cache_shrink); + block_remove(sb, bp); + } + + spin_unlock(&binf->lock); + +out: + return min_t(u64, binf->lru_nr * SCOUTFS_PAGES_PER_BLOCK, INT_MAX); +} + +int scoutfs_block_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct block_info *binf; + loff_t size; + int ret; + + /* we store blknos in longs in the radix */ + size = i_size_read(sb->s_bdev->bd_inode); + if ((size >> SCOUTFS_BLOCK_SHIFT) >= LONG_MAX) { + scoutfs_err(sb, "Cant reference all blocks in %llu byte device with %u bit long radix tree indexes", + size, BITS_PER_LONG); + return -EINVAL; + } + + binf = kzalloc(sizeof(struct block_info), GFP_KERNEL); + if (!binf) { + ret = -ENOMEM; + goto out; + } + + binf->sb = sb; + spin_lock_init(&binf->lock); + INIT_RADIX_TREE(&binf->radix, GFP_ATOMIC); /* insertion preloads */ + INIT_LIST_HEAD(&binf->lru_list); + init_waitqueue_head(&binf->waitq); + binf->shrinker.shrink = block_shrink; + binf->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&binf->shrinker); + INIT_WORK(&binf->free_work, block_free_work); + init_llist_head(&binf->free_llist); + + sbi->block_info = binf; + + ret = 0; +out: + if (ret) + scoutfs_block_destroy(sb); + + return 0; +} + +void scoutfs_block_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct block_info *binf = SCOUTFS_SB(sb)->block_info; + + if (binf) { + unregister_shrinker(&binf->shrinker); + block_remove_all(sb); + flush_work(&binf->free_work); + + WARN_ON_ONCE(!llist_empty(&binf->free_llist)); + kfree(binf); + + sbi->block_info = NULL; + } +} diff --git a/kmod/src/block.h b/kmod/src/block.h index 5bf42331..dbb0d54c 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -1,10 +1,51 @@ #ifndef _SCOUTFS_BLOCK_H_ #define _SCOUTFS_BLOCK_H_ +struct scoutfs_block_writer { + spinlock_t lock; + struct list_head dirty_list; + u64 nr_dirty_blocks; +}; + +struct scoutfs_block { + u64 blkno; + void *data; +}; + __le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr); bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr); bool scoutfs_block_valid_ref(struct super_block *sb, struct scoutfs_block_header *hdr, __le64 seq, __le64 blkno); +struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno); +struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); +void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl); +bool scoutfs_block_consistent_ref(struct super_block *sb, + struct scoutfs_block *bl, + __le64 seq, __le64 blkno, u32 magic); +void scoutfs_block_put(struct super_block *sb, struct scoutfs_block *bl); + +void scoutfs_block_writer_init(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_mark_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl); +bool scoutfs_block_writer_is_dirty(struct super_block *sb, + struct scoutfs_block *bl); +int scoutfs_block_writer_write(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_forget_all(struct super_block *sb, + struct scoutfs_block_writer *wri); +void scoutfs_block_writer_forget(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl); +bool scoutfs_block_writer_has_dirty(struct super_block *sb, + struct scoutfs_block_writer *wri); +u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, + struct scoutfs_block_writer *wri); + +int scoutfs_block_setup(struct super_block *sb); +void scoutfs_block_destroy(struct super_block *sb); + #endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 60ea6ee9..22bf8645 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,6 +12,16 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(block_cache_access) \ + EXPAND_COUNTER(block_cache_alloc_failure) \ + EXPAND_COUNTER(block_cache_alloc_page_order) \ + EXPAND_COUNTER(block_cache_alloc_virt) \ + EXPAND_COUNTER(block_cache_end_io_error) \ + EXPAND_COUNTER(block_cache_forget) \ + EXPAND_COUNTER(block_cache_free) \ + EXPAND_COUNTER(block_cache_invalidate) \ + EXPAND_COUNTER(block_cache_lru_move) \ + EXPAND_COUNTER(block_cache_shrink) \ EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(btree_write_error) \ @@ -152,7 +162,7 @@ EXPAND_COUNTER(trans_write_item) \ EXPAND_COUNTER(trans_write_deletion_item) -#define FIRST_COUNTER btree_read_error +#define FIRST_COUNTER block_cache_access #define LAST_COUNTER trans_write_deletion_item #undef EXPAND_COUNTER diff --git a/kmod/src/super.c b/kmod/src/super.c index f160e1a3..5237c4b1 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -201,6 +201,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_quorum_destroy(sb); scoutfs_item_destroy(sb); + scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); scoutfs_sysfs_destroy_attrs(sb, &sbi->mopts_ssa); @@ -421,6 +422,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_setup_triggers(sb) ?: scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: + scoutfs_block_setup(sb) ?: scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 0545b34c..80700ee6 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -26,6 +26,7 @@ struct btree_info; struct sysfs_info; struct options_sb_info; struct net_info; +struct block_info; struct scoutfs_sb_info { struct super_block *sb; @@ -48,6 +49,7 @@ struct scoutfs_sb_info { struct btree_info *btree_info; struct net_info *net_info; struct quorum_info *quorum_info; + struct block_info *block_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From bdafa6ede6a620d208939f0f855c5b312a13199e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 16 Oct 2019 10:39:11 -0700 Subject: [PATCH 754/920] scoutfs: add block allocator Add our block allocator core. It'll be used shortly. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/balloc.c | 630 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/balloc.h | 36 +++ kmod/src/format.h | 22 ++ 4 files changed, 689 insertions(+) create mode 100644 kmod/src/balloc.c create mode 100644 kmod/src/balloc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 9b776a71..ec6611c1 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -10,6 +10,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += \ bio.o \ + balloc.o \ block.o \ btree.o \ client.o \ diff --git a/kmod/src/balloc.c b/kmod/src/balloc.c new file mode 100644 index 00000000..bcd03bd6 --- /dev/null +++ b/kmod/src/balloc.c @@ -0,0 +1,630 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "key.h" +#include "counters.h" +#include "msg.h" +#include "block.h" +#include "btree.h" +#include "per_task.h" +#include "balloc.h" + +#include "scoutfs_trace.h" + +/* + * scoutfs tracks free metadata blocks in bitmap items in allocation + * btrees. Most of the free metadata is operated on by the server and + * tracked in large core trees rooted in the super block. The server + * moves free items from the core trees to private trees for mounts. + * + * Allocation is performed by btrees which are performing cow updates. + * We can't write to stable blocks during a transaction, we can only + * write into free space in the previous stable fs image. This means + * that we can't satisfy dirty block allocations with frees of + * previously stable blocks in this transaction. We implement this by + * allocating from one tree and freeing into another. They're merged as + * the free blocks are committed and can be safely written to in the + * next transaction. + * + * We're allocating and freeing blocks on behalf of btree ops by calling + * btree ops. This would deadlock if we always called btree ops from + * the allocator directly, but instead we recognize recursion and have + * the called allocator hand blknos back to its calling allocator to + * store into btrees on its behalf. + * + * We use explicit allocation and writing contexts because both the + * client and server are working on independent allocation and item + * trees. + */ + +struct item_modification { + struct list_head entry; + u64 blkno; + u64 count; + int op; + struct scoutfs_balloc_root *root; + struct scoutfs_balloc_root *src; +}; + +static bool add_item_mod(struct list_head *list, int op, u64 blkno, u64 count, + struct scoutfs_balloc_root *root, + struct scoutfs_balloc_root *src) +{ + struct item_modification *im = kmalloc(sizeof(struct item_modification), + GFP_NOFS); + if (im) { + im->blkno = blkno; + im->count = count; + im->op = op; + im->root = root; + im->src = src; + list_add_tail(&im->entry, list); + return true; + } + + return false; +} + +/* make room to dirty two trees of an absurdly large height */ +#define MAX_BLKNOS (2 * ((32 * 2) + 1)) + +struct blkno_fifo { + int first; + int nr; + u64 blknos[MAX_BLKNOS]; +}; + +static inline void blkno_fifo_init(struct blkno_fifo *bf) +{ + bf->first = 0; + bf->nr = 0; +} + +static inline int blkno_fifo_nr(struct blkno_fifo *bf) +{ + BUG_ON(bf->nr < 0 || bf->nr > MAX_BLKNOS); + return bf->nr; +} + +static inline u64 blkno_fifo_out(struct blkno_fifo *bf) +{ + BUG_ON(blkno_fifo_nr(bf) == 0); + bf->nr--; + return bf->blknos[bf->first++]; +} + +static inline void blkno_fifo_in(struct blkno_fifo *bf, u64 blkno) +{ + unsigned int end = (bf->first + bf->nr) % MAX_BLKNOS; + + BUG_ON(blkno_fifo_nr(bf) == MAX_BLKNOS); + bf->blknos[end] = blkno; + bf->nr++; +} + +struct caller_blknos { + struct blkno_fifo free; + struct blkno_fifo alloced; + struct blkno_fifo freed; +}; + +/* + * Find a number of next free blknos from a starting point. We can land + * in the end of an empty item. If this returns 0 then nr_found have + * been found. + */ +static int find_next_free(struct super_block *sb, + struct scoutfs_balloc_root *root, u64 from, + u64 *found, unsigned int nr_found) +{ + struct scoutfs_balloc_item_key bik; + struct scoutfs_balloc_item_val biv; + SCOUTFS_BTREE_ITEM_REF(iref); + unsigned int f = 0; + unsigned int bit; + u64 base; + int ret = 0; + + while (f < nr_found) { + base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; + bit = from & SCOUTFS_BALLOC_ITEM_BIT_MASK; + bik.base = cpu_to_be64(base); + + ret = scoutfs_btree_next(sb, &root->root, + &bik, sizeof(bik), &iref); + if (ret < 0) /* including ENOENT */ + break; + + if (iref.key_len == sizeof(bik) && + iref.val_len == sizeof(biv)) { + memcpy(&bik, iref.key, iref.key_len); + memcpy(&biv, iref.val, iref.val_len); + + /* start from first bit in next whole item */ + if (be64_to_cpu(bik.base) != base) + bit = 0; + + while (f < nr_found) { + bit = find_next_bit_le(biv.bits, + SCOUTFS_BALLOC_ITEM_BITS, bit); + if (bit >= SCOUTFS_BALLOC_ITEM_BITS) + break; + + found[f++] = (be64_to_cpu(bik.base) << + SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + + bit; + bit++; + } + + from = (be64_to_cpu(bik.base) << + SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + bit; + ret = 0; + + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + break; + } + + return ret; +} + +/* + * Return the first blkno in the next item. Because from can land in an + * item we can return a blkno that is less than from. + */ +static int find_next_item(struct super_block *sb, + struct scoutfs_balloc_root *root, u64 from, + u64 *found) +{ + struct scoutfs_balloc_item_key bik; + SCOUTFS_BTREE_ITEM_REF(iref); + u64 base; + int ret; + + base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; + bik.base = cpu_to_be64(base); + + ret = scoutfs_btree_next(sb, &root->root, &bik, sizeof(bik), &iref); + if (ret < 0) /* including ENOENT */ + goto out; + + if (iref.key_len == sizeof(struct scoutfs_balloc_item_key) && + iref.val_len == sizeof(struct scoutfs_balloc_item_val)) { + memcpy(&bik, iref.key, iref.key_len); + *found = be64_to_cpu(bik.base) << + SCOUTFS_BALLOC_ITEM_BASE_SHIFT; + ret = 0; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); +out: + return ret; +} + +enum { + IM_OP_SET, + IM_OP_SET_BULK, + IM_OP_CLEAR, + IM_OP_MOVE, +}; + +static int copy_item_bits(struct scoutfs_balloc_item_val *biv, + struct scoutfs_btree_item_ref *iref, int ret, + bool *existed) +{ + if (ret < 0) { + if (ret == -ENOENT) { + memset(biv, 0, sizeof(struct scoutfs_balloc_item_val)); + if (existed) + *existed = false; + ret = 0; + } + } else { + if (iref->key_len == sizeof(struct scoutfs_balloc_item_key) && + iref->val_len == sizeof(struct scoutfs_balloc_item_val)) { + memcpy(biv, iref->val, iref->val_len); + if (existed) + *existed = true; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(iref); + } + + return ret; +} + +/* + * We can use native longs to set aligned 64bit regions, but have to use + * individual _le calls on leading and trailing partial regions. + */ +static void bitmap_set_le(__le64 *map, int start, int nr) +{ + unsigned int full; + + while (start & 63 && nr-- > 0) + set_bit_le(start++, map); + + if (nr > 64) { + full = round_down(nr, 64); + bitmap_set((long *)map, start, full); + start += full; + nr -= full; + + } + + while (nr-- > 0) + set_bit_le(start++, map); +} + +/* + * Modify allocation item bits in service of the caller's operation. + * This has to be done very carefully so that we don't deadlock in + * recursion as btree dirtying calls back in to block allocation. + * + * A given btree operation can need to allocate blknos for dirty blocks + * and free the old clean blknos. The btree code will attempt to call + * balloc again. We add a per_task record of allocated and freed blknos + * which those allocation calls use instead of calling more btree ops. + * They then return to us and we perform the btree ops to satisfy those + * allocations and frees that were recorded. + * + * Each op that cows btree blocks generates more ops to records those + * allocations and frees. Eventually the ops hit existing dirty blocks + * and we can return. + */ +static int modify_items(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + int op, u64 blkno, u64 count, + struct scoutfs_balloc_root *root, + struct scoutfs_balloc_root *src, u64 next_free) +{ + SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); + struct scoutfs_balloc_item_key bik; + struct scoutfs_balloc_item_val biv; + struct scoutfs_balloc_item_val tmp; + struct item_modification *im; + SCOUTFS_BTREE_ITEM_REF(iref); + struct caller_blknos *cb; + unsigned int need_free; + unsigned int nr; + LIST_HEAD(mods); + u64 nexts[16]; + bool existed; + u64 base; + int bit; + int ret; + int i; + + /* doing native long ops on stack bits */ + BUILD_BUG_ON(offsetof(struct scoutfs_balloc_item_val, bits) % + (BITS_PER_LONG / 8)); + + cb = kmalloc(sizeof(struct caller_blknos), GFP_NOFS); + if (!cb) { + ret = -ENOMEM; + goto out; + } + + blkno_fifo_init(&cb->free); + blkno_fifo_init(&cb->alloced); + blkno_fifo_init(&cb->freed); + + scoutfs_per_task_add(&alloc->pt_caller_blknos, &pt_ent, cb); + + if (!add_item_mod(&mods, op, blkno, count, root, src)) { + ret = -ENOENT; + goto out; + } + + while ((im = list_first_entry_or_null(&mods, struct item_modification, + entry))) { + + base = im->blkno >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; + bik.base = cpu_to_be64(base); + existed = false; + + if (im->op != IM_OP_SET_BULK) { + /* get the current item to modify */ + ret = scoutfs_btree_lookup(sb, &im->root->root, + &bik, sizeof(bik), &iref); + ret = copy_item_bits(&biv, &iref, ret, &existed); + if (ret < 0) + goto out; + /* XXX corruption */ + BUG_ON(im->op == IM_OP_CLEAR && !existed); + } + + /* modify the item's bit */ + bit = im->blkno & SCOUTFS_BALLOC_ITEM_BIT_MASK; + if (im->op == IM_OP_SET) { + set_bit_le(bit, &biv.bits); + } else if (im->op == IM_OP_SET_BULK) { + memset(&biv, 0, sizeof(biv)); + bitmap_set_le(biv.bits, 0, im->count); + } else if (im->op == IM_OP_CLEAR) { + clear_bit_le(bit, &biv.bits); + } + + /* move just read the destination item, or in src item bits */ + if (im->op == IM_OP_MOVE) { + ret = scoutfs_btree_lookup(sb, &im->src->root, &bik, + sizeof(bik), &iref); + ret = copy_item_bits(&tmp, &iref, ret, NULL); + if (ret < 0) + goto out; + + /* shouldn't have free in both places */ + if (bitmap_intersects((long *)biv.bits, + (long *)tmp.bits, + SCOUTFS_BALLOC_ITEM_BITS)) { + ret = -EIO; + goto out; + } + bitmap_or((long *)biv.bits, (long *)biv.bits, + (long *)tmp.bits, SCOUTFS_BALLOC_ITEM_BITS); + } + + /* make sure we have enough free blocks for btree dirtying */ + need_free = (im->root->root.height * 2) + 1; + if (im->op == IM_OP_MOVE) + need_free += (im->src->root.height * 2) + 1; + + /* fill free fifo for potential dirtying */ + while (blkno_fifo_nr(&cb->free) < need_free) { + nr = min_t(int, need_free - blkno_fifo_nr(&cb->free), + ARRAY_SIZE(nexts)); + ret = find_next_free(sb, &alloc->alloc_root, next_free, + nexts, nr); + if (ret < 0) + goto out; + + next_free = nexts[nr - 1] + 1; + for (i = 0; i < nr; i++) + blkno_fifo_in(&cb->free, nexts[i]); + } + + /* + * Perform the op's item modifications, we go to do the + * trouble of differentiating between update and + * insertion instead of just using force so that we + * don't split when we don't need to. + */ + if (im->op == IM_OP_CLEAR && + bitmap_empty((long *)biv.bits, SCOUTFS_BALLOC_ITEM_BITS)) + ret = scoutfs_btree_delete(sb, alloc, wri, + &im->root->root, + &bik, sizeof(bik)); + else if (im->op == IM_OP_SET_BULK || + (im->op == IM_OP_SET && !existed)) + ret = scoutfs_btree_insert(sb, alloc, wri, + &im->root->root, + &bik, sizeof(bik), + &biv, sizeof(biv)); + else if (im->op == IM_OP_MOVE && existed) + ret = scoutfs_btree_delete(sb, alloc, wri, + &im->src->root, + &bik, sizeof(bik)) ?: + scoutfs_btree_update(sb, alloc, wri, + &im->root->root, + &bik, sizeof(bik), + &biv, sizeof(biv)); + else if (im->op == IM_OP_MOVE && !existed) + ret = scoutfs_btree_delete(sb, alloc, wri, + &im->src->root, + &bik, sizeof(bik)) ?: + scoutfs_btree_insert(sb, alloc, wri, + &im->root->root, + &bik, sizeof(bik), + &biv, sizeof(biv)); + else + ret = scoutfs_btree_update(sb, alloc, wri, + &im->root->root, + &bik, sizeof(bik), + &biv, sizeof(biv)); + if (ret < 0) + goto out; + + /* update bit counts to reflect op */ + if (im->op == IM_OP_SET) { + le64_add_cpu(&root->total_free, 1); + } else if (im->op == IM_OP_SET_BULK) { + le64_add_cpu(&root->total_free, im->count); + } else if (im->op == IM_OP_CLEAR) { + le64_add_cpu(&root->total_free, -1); + } else if (im->op == IM_OP_MOVE) { + nr = bitmap_weight((long *)biv.bits, + SCOUTFS_BALLOC_ITEM_BITS); + le64_add_cpu(&root->total_free, nr); + le64_add_cpu(&src->total_free, -nr); + } + + list_del(&im->entry); + kfree(im); + + /* and queue new modifications needed from btree ops */ + + while (blkno_fifo_nr(&cb->alloced)) { + if (!add_item_mod(&mods, IM_OP_CLEAR, + blkno_fifo_out(&cb->alloced), 0, + &alloc->alloc_root, NULL)) { + ret = -ENOENT; + goto out; + } + } + + while (blkno_fifo_nr(&cb->freed)) { + if (!add_item_mod(&mods, IM_OP_SET, + blkno_fifo_out(&cb->freed), 0, + &alloc->free_root, NULL)) { + ret = -ENOENT; + goto out; + } + } + } + + ret = 0; +out: + scoutfs_per_task_del(&alloc->pt_caller_blknos, &pt_ent); + BUG_ON(ret < 0); /* dirty block refs and bits are inconsistent */ + BUG_ON(!list_empty(&mods)); /* reminder to clean up */ + kfree(cb); + return ret; +} + +void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, + struct scoutfs_balloc_root *alloc_root, + struct scoutfs_balloc_root *free_root) +{ + mutex_init(&alloc->mutex); + scoutfs_per_task_init(&alloc->pt_caller_blknos); + alloc->alloc_root = *alloc_root; + alloc->free_root = *free_root; +} + +/* + * Add alloc items for a contiugous regions of blknos. The starting + * blkno must be aligned to the start of a bitmap item. Once these are + * added they can be used by the current transaction so the caller must + * be very careful that they're free. + */ +int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + u64 blkno, u64 count) +{ + u64 nr; + int ret = 0; + + mutex_lock(&alloc->mutex); + while (count > 0) { + nr = min_t(u64, count, SCOUTFS_BALLOC_ITEM_BITS), + ret = modify_items(sb, alloc, wri, IM_OP_SET_BULK, blkno, nr, + &alloc->alloc_root, NULL, 0); + if (ret < 0) + break; + blkno += nr; + count -= nr; + } + mutex_unlock(&alloc->mutex); + + return ret; +} + +int scoutfs_balloc_alloc(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, u64 *blkno_ret) +{ + struct caller_blknos *cb; + u64 next; + int ret; + + /* if we're called by balloc then the caller works for us */ + cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); + if (cb) { + *blkno_ret = blkno_fifo_out(&cb->free); + blkno_fifo_in(&cb->alloced, *blkno_ret); + return 0; + } + + mutex_lock(&alloc->mutex); + ret = find_next_free(sb, &alloc->alloc_root, 0, &next, 1) ?: + modify_items(sb, alloc, wri, IM_OP_CLEAR, next, 0, + &alloc->alloc_root, NULL, next + 1); + mutex_unlock(&alloc->mutex); + + if (ret == 0) + *blkno_ret = next; + + return ret; +} + +int scoutfs_balloc_free(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + u64 blkno) +{ + struct caller_blknos *cb; + int ret; + + /* if we're called by balloc then the caller works for us */ + cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); + if (cb) { + blkno_fifo_in(&cb->freed, blkno); + return 0; + } + + mutex_lock(&alloc->mutex); + ret = modify_items(sb, alloc, wri, IM_OP_SET, blkno, 0, + &alloc->free_root, NULL, 0); + mutex_unlock(&alloc->mutex); + + return ret; +} + +/* + * Move full items from the source to destination tree, moving at least + * the given number of blocks but likely more. + * + * This has to be done very carefully because we don't want to allocate + * dirty btree blocks from blknos in the source item that is moving. We + * find the first blkno in the next free item in the source tree so that + * we can start allocating dirty btree blocks after that item. + * + * This will not wrap the starting from blkno if it doesn't start at 0 + * and runs out of items. The caller is expected to deal with this. + */ +int scoutfs_balloc_move(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + u64 from, u64 at_least, u64 *next_past) +{ + u64 target; + u64 next; + int ret = 0; + + mutex_lock(&alloc->mutex); + + target = le64_to_cpu(dst->total_free) + at_least; + + while (le64_to_cpu(dst->total_free) < target && + le64_to_cpu(src->total_free) > 0) { + ret = find_next_item(sb, src, from, &next) ?: + modify_items(sb, alloc, wri, IM_OP_MOVE, next, 0, + dst, src, + next + SCOUTFS_BALLOC_ITEM_BITS); + if (ret < 0) + break; + + from = next + SCOUTFS_BALLOC_ITEM_BITS; + *next_past = from; + } + + mutex_unlock(&alloc->mutex); + + return ret; +} diff --git a/kmod/src/balloc.h b/kmod/src/balloc.h new file mode 100644 index 00000000..246a42a6 --- /dev/null +++ b/kmod/src/balloc.h @@ -0,0 +1,36 @@ +#ifndef _SCOUTFS_BALLOC_H_ +#define _SCOUTFS_BALLOC_H_ + +#include "per_task.h" + +struct scoutfs_block_writer; + +struct scoutfs_balloc_allocator { + struct mutex mutex; + struct scoutfs_per_task pt_caller_blknos; + struct scoutfs_balloc_root alloc_root; + struct scoutfs_balloc_root free_root; +}; + +void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, + struct scoutfs_balloc_root *alloc_root, + struct scoutfs_balloc_root *free_root); +int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + u64 blkno, u64 count); +int scoutfs_balloc_alloc(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, u64 *blkno_ret); +int scoutfs_balloc_free(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + u64 blkno); +int scoutfs_balloc_move(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + u64 from, u64 at_least, u64 *next_past); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 785a441e..58a73b9a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -269,6 +269,28 @@ struct scoutfs_manifest_btree_val { struct scoutfs_key last_key; } __packed; +/* + * Free metadata blocks are tracked by block allocator items. + */ +struct scoutfs_balloc_root { + struct scoutfs_btree_root root; + __le64 total_free; +} __packed; +struct scoutfs_balloc_item_key { + __be64 base; +} __packed; + +#define SCOUTFS_BALLOC_ITEM_BYTES 256 +#define SCOUTFS_BALLOC_ITEM_U64S (SCOUTFS_BALLOC_ITEM_BYTES / \ + sizeof(__u64)) +#define SCOUTFS_BALLOC_ITEM_BITS (SCOUTFS_BALLOC_ITEM_BYTES * 8) +#define SCOUTFS_BALLOC_ITEM_BASE_SHIFT ilog2(SCOUTFS_BALLOC_ITEM_BITS) +#define SCOUTFS_BALLOC_ITEM_BIT_MASK (SCOUTFS_BALLOC_ITEM_BITS - 1) + +struct scoutfs_balloc_item_val { + __le64 bits[SCOUTFS_BALLOC_ITEM_U64S]; +} __packed; + /* * Free extents are stored in the server in an allocation btree. The * type differentiates whether start or length is in stored in the major From 8775826d7e1089daf53a6201950b5c0b7ffb382d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 29 Sep 2019 14:42:58 -0700 Subject: [PATCH 755/920] scoutfs: have btree use blocks, allocator, writer Convert the btree to use our block cache, block allocation, and the caller's explicit dirty block tracking writer context instead of the ring. This is in preparation for the btree forest format where there are concurrent multiple writers of independent dynamically sized btrees instead of only the server writing one btree with a fixed maximum size. All the machinery for tracking dirty blocks in the ring and migrating is no longer needed. Signed-off-by: Zach Brown --- kmod/src/btree.c | 719 ++++++++++----------------------------- kmod/src/btree.h | 34 +- kmod/src/format.h | 5 - kmod/src/scoutfs_trace.h | 21 +- 4 files changed, 201 insertions(+), 578 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 674c7e50..c8919c52 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -14,10 +14,9 @@ #include #include #include -#include #include #include -#include +#include #include "super.h" #include "format.h" @@ -28,12 +27,12 @@ #include "options.h" #include "msg.h" #include "block.h" +#include "balloc.h" #include "scoutfs_trace.h" /* - * scoutfs uses a cow btree in a ring of preallocated blocks to index - * the manifest (and allocator, but mostly the manifest). + * scoutfs uses a cow btree to index fs metadata. * * Using a cow btree lets nodes determine the validity of cached blocks * based on a single root ref (blkno, seq) that is communicated through @@ -41,48 +40,12 @@ * overwritten in the ring they can continue to use those cached blocks * as the newer cowed blocks continue to reference them. * - * New blocks written to the btree are allocated from the tail of the - * preallocated ring. This avoids a fine grained persistent record of - * free btree blocks. It also gathers all dirty btree blocks into one - * contiguous write. - * - * To ensure that newly written blocks don't overwrite previously valid - * existing blocks in the ring we take two preventative measures. First - * we ensure that there are 4x the number of preallocated blocks that - * would be needed to store the btrees. Then, second, for every set of - * blocks written to the current half of the ring we ensure that at - * least half of the written blocks are cow copies of valid blocks that - * were stored in the old half of the ring. This ensures that the - * current half of the ring will contain all the valid referenced btree - * blocks by the time it fills up and wraps around to start overwriting - * the old half of the ring. - * - * To find the blocks in the old half of the ring we store a migration - * key in the super. Whenever we need to dirty old blocks we sweep leaf - * blocks from that key dirtying old blocks we find. - * - * Blocks are of a fixed size and are set to 4k to avoid multi-page - * blocks. This means they can be smaller than the page size and we can - * need to pin dirty blocks and invalidate and re-read stable blocks - * that could fall in the same page. We use buffer heads to track - * sub-page block state for us. We abuse knowledge of the page cache - * and buffer heads to cast between pointers to the blocks and the - * buffer heads that contain reference counts of the block contents. - * - * We store modified blocks in a list on b_private instead of marking - * the blocks dirty. We don't want them written out (and possibly - * reclaimed and re-read) before we have a chance to update their - * checksums. We hold an elevated bh count to avoid the buffers from - * being removed from the pages while we have them in the list. - * * Today callers provide all the locking. They serialize readers and * writers and writers and committing all the dirty blocks. * * Btree items are stored in each block as a small header with the key * followed by the value. New items are allocated from the back of the - * block towards the front. Deleted items can be reclaimed by packing - * items towards the back of the block by walking them in reverse offset - * order. + * block towards the front. * * A dense array of item headers after the btree block header stores the * offsets of the items and is kept sorted by the item's keys. The @@ -109,25 +72,6 @@ * - validate structures on read? */ -/* - * There's one physical ring that stores the blocks for all btrees. We - * track the state of the ring and all its dirty blocks in this one - * btree_info per mount/super. - */ -struct btree_info { - struct mutex mutex; - - unsigned long cur_dirtied; - unsigned long old_dirtied; - struct buffer_head *first_dirty_bh; - struct buffer_head *last_dirty_bh; - u64 first_dirty_blkno; - u64 first_dirty_seq; -}; - -#define DECLARE_BTREE_INFO(sb, name) \ - struct btree_info *name = SCOUTFS_SB(sb)->btree_info - /* btree walking has a bunch of behavioural bit flags */ enum { BTW_NEXT = (1 << 0), /* return >= key */ @@ -138,7 +82,6 @@ enum { BTW_ALLOC = (1 << 5), /* allocate a new block for 0 ref */ BTW_INSERT = (1 << 6), /* walking to insert, try splitting */ BTW_DELETE = (1 << 7), /* walking to delete, try merging */ - BTW_MIGRATE = (1 << 8), /* don't dirty old leaf blocks */ }; /* @@ -298,72 +241,6 @@ static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, return pos; } -/* - * A block is current if it's in the same half of the ring as the next - * dirty block in the transaction. - */ -static bool blkno_is_current(struct scoutfs_btree_ring *bring, u64 blkno) -{ - u64 half_blkno = le64_to_cpu(bring->first_blkno) + - (le64_to_cpu(bring->nr_blocks) / 2); - u64 next_blkno = le64_to_cpu(bring->first_blkno) + - le64_to_cpu(bring->next_block); - - return (blkno < half_blkno) == (next_blkno < half_blkno); -} - -static bool first_block_in_half(struct scoutfs_btree_ring *bring) -{ - u64 block = le64_to_cpu(bring->next_block); - - return block == 0 || block == (le64_to_cpu(bring->nr_blocks) / 2); -} - -/* Set next_block to the start of the other half */ -static void advance_to_next_half(struct scoutfs_btree_ring *bring) -{ - u64 block = le64_to_cpu(bring->next_block); - u64 half = le64_to_cpu(bring->nr_blocks) / 2; - u64 offset; - - if (block >= half) { - offset = le64_to_cpu(bring->nr_blocks) - block; - block = 0; - } else { - offset = half - block; - block = half; - } - - bring->next_block = cpu_to_le64(block); - le64_add_cpu(&bring->next_seq, offset); -} - -static size_t super_root_offsets[] = { - offsetof(struct scoutfs_super_block, alloc_root), - offsetof(struct scoutfs_super_block, manifest.root), - offsetof(struct scoutfs_super_block, lock_clients), - offsetof(struct scoutfs_super_block, trans_seqs), - offsetof(struct scoutfs_super_block, mounted_clients), -}; - -#define for_each_super_root(super, i, root) \ - for (i = 0; i < ARRAY_SIZE(super_root_offsets) && \ - (root = ((void *)super + super_root_offsets[i]), 1);\ - i++) - -static bool all_roots_migrated(struct scoutfs_super_block *super) -{ - struct scoutfs_btree_root *root; - int i; - - for_each_super_root(super, i, root) { - if (root->migration_key_len) - return false; - } - - return true; -} - /* move a number of contigous elements from the src index to the dst index */ #define memmove_arr(arr, dst, src, nr) \ memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) @@ -497,76 +374,6 @@ static void move_items(struct scoutfs_btree_block *dst, } } -/* - * This is only used after we've elevated bh reference counts. Until we - * drop the counts the bhs won't be removed from the page. This lets us - * use pointers to the block contents in the api and not have to litter - * it with redundant containers. - */ -static struct buffer_head *virt_to_bh(void *kaddr) -{ - struct buffer_head *bh; - struct page *page; - long off; - - page = virt_to_page((unsigned long)kaddr); - BUG_ON(!page_has_buffers(page)); - bh = page_buffers(page); - BUG_ON((unsigned long)bh->b_data != - ((unsigned long)kaddr & PAGE_CACHE_MASK)); - - off = (unsigned long)kaddr & ~PAGE_CACHE_MASK; - while (off >= SCOUTFS_BLOCK_SIZE) { - bh = bh->b_this_page; - off -= SCOUTFS_BLOCK_SIZE; - } - - return bh; -} - -static void put_btree_block(void *ptr) -{ - if (!IS_ERR_OR_NULL(ptr)) - put_bh(virt_to_bh(ptr)); -} - -enum { - BH_ScoutfsChecked = BH_PrivateStart, - BH_ScoutfsValidCrc, -}; - -BUFFER_FNS(ScoutfsChecked, scoutfs_checked) /* has had crc checked */ -BUFFER_FNS(ScoutfsValidCrc, scoutfs_valid_crc) /* crc matched */ - - -/* - * Make sure that we've found a valid block and that it's the block that - * we're looking for. - */ -static bool valid_referenced_block(struct super_block *sb, - struct scoutfs_btree_ref *ref, - struct scoutfs_btree_block *bt, - struct buffer_head *bh) -{ - smp_rmb(); /* load checked before crc */ - if (!buffer_scoutfs_checked(bh)) { - lock_buffer(bh); - if (!buffer_scoutfs_checked(bh)) { - if (scoutfs_block_valid_crc(&bt->hdr)) - set_buffer_scoutfs_valid_crc(bh); - else - clear_buffer_scoutfs_valid_crc(bh); - - smp_wmb(); /* store crc before checked */ - set_buffer_scoutfs_checked(bh); - } - unlock_buffer(bh); - } - - return buffer_scoutfs_valid_crc(bh) && - scoutfs_block_valid_ref(sb, &bt->hdr, ref->seq, ref->blkno); -} - /* * This is used to lookup cached blocks, read blocks, cow blocks for * dirtying, and allocate new blocks. @@ -578,54 +385,45 @@ static bool valid_referenced_block(struct super_block *sb, * returning -ESTALE if it still looks wrong. The caller can retry the * read from a more current root or decide that this is a persistent * error. - * - * btree callers serialize concurrent writers in a btree but not between - * btrees. We have to lock around the shared btree_info. Callers do - * lock between all btree writers and writing dirty blocks. We don't - * have to lock around the bti fields that are only changed by commits. */ -static int get_ref_block(struct super_block *sb, int flags, +static int get_ref_block(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, int flags, struct scoutfs_btree_ref *ref, - struct scoutfs_btree_block **bt_ret) + struct scoutfs_block **bl_ret) { - DECLARE_BTREE_INFO(sb, bti); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_btree_ring *bring = &super->bring; - struct scoutfs_btree_root *root; struct scoutfs_btree_block *bt = NULL; struct scoutfs_btree_block *new; - struct buffer_head *bh; + struct scoutfs_block *new_bl = NULL; + struct scoutfs_block *bl = NULL; bool retried = false; u64 blkno; u64 seq; int ret; - int i; /* always get the current block, either to return or cow from */ if (ref && ref->blkno) { retry: - bh = sb_bread(sb, le64_to_cpu(ref->blkno)); - if (!bh) { + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl)) { trace_scoutfs_btree_read_error(sb, ref); scoutfs_inc_counter(sb, btree_read_error); - ret = -EIO; + ret = PTR_ERR(bl); goto out; } - bt = (void *)bh->b_data; + bt = (void *)bl->data; - if (!valid_referenced_block(sb, ref, bt, bh) || + if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_BTREE) || scoutfs_trigger(sb, BTREE_STALE_READ)) { scoutfs_inc_counter(sb, btree_stale_read); - lock_buffer(bh); - clear_buffer_uptodate(bh); - clear_buffer_scoutfs_valid_crc(bh); - smp_wmb(); /* store crc before checked */ - clear_buffer_scoutfs_checked(bh); - unlock_buffer(bh); - put_bh(bh); - bt = NULL; + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = NULL; if (!retried) { retried = true; @@ -639,18 +437,10 @@ retry: /* * We need to create a new dirty copy of the block if * the caller asked for it. If the block is already - * dirty then we can return it if either we're not - * migrating so it doesn't matter which half it's in, or - * we're migrating and the dirty block is already in the - * second half. We can be migrating into a new half - * while blocks are still dirty in the old half. And we - * always have to dirty parent blocks in the current - * half in case we need to dirty their children. + * dirty then we can return it. */ if (!(flags & BTW_DIRTY) || - ((le64_to_cpu(bt->hdr.seq) >= bti->first_dirty_seq) && - (!(flags & BTW_MIGRATE) || - blkno_is_current(bring, le64_to_cpu(ref->blkno))))) { + scoutfs_block_writer_is_dirty(sb, bl)) { ret = 0; goto out; } @@ -660,83 +450,39 @@ retry: goto out; } - mutex_lock(&bti->mutex); + ret = scoutfs_balloc_alloc(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; - blkno = le64_to_cpu(bring->first_blkno) + le64_to_cpu(bring->next_block); - seq = le64_to_cpu(bring->next_seq); + prandom_bytes(&seq, sizeof(seq)); - bh = sb_getblk(sb, blkno); - if (!bh) { - ret = -ENOMEM; - mutex_unlock(&bti->mutex); + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + ret = scoutfs_balloc_free(sb, alloc, wri, blkno); + BUG_ON(ret); /* radix should have been dirty */ + ret = PTR_ERR(new_bl); goto out; } - new = (void *)bh->b_data; + new = (void *)new_bl->data; - set_buffer_uptodate(bh); - set_buffer_scoutfs_checked(bh); - set_buffer_scoutfs_valid_crc(bh); - - /* - * Track our contiguous dirty blocks by holding a ref and putting - * them in a list. We don't want them marked dirty or else they - * can be written out before we're ready. - */ - get_bh(bh); - bh->b_private = NULL; - if (bti->last_dirty_bh) - bti->last_dirty_bh->b_private = bh; - bti->last_dirty_bh = bh; - if (!bti->first_dirty_bh) - bti->first_dirty_bh = bh; - - if (ref && !blkno_is_current(bring, le64_to_cpu(ref->blkno))) - bti->old_dirtied++; - else - bti->cur_dirtied++; - - /* wrap next block and increase next seq */ - le64_add_cpu(&bring->next_block, 1); - le64_add_cpu(&bring->next_seq, 1); - - if (le64_to_cpu(bring->next_block) == le64_to_cpu(bring->nr_blocks)) - bring->next_block = 0; + scoutfs_block_writer_mark_dirty(sb, wri, new_bl); trace_scoutfs_btree_dirty_block(sb, blkno, seq, - le64_to_cpu(bring->next_block), le64_to_cpu(bring->next_seq), - bti->cur_dirtied, bti->old_dirtied, - bt ? le64_to_cpu(bt->hdr.blkno) : 0, - bt ? le64_to_cpu(bt->hdr.seq) : 0); - - /* force advancing if migration's done and we didn't just wrap */ - if (all_roots_migrated(super) && !first_block_in_half(bring) && - scoutfs_trigger(sb, BTREE_ADVANCE_RING_HALF)) - advance_to_next_half(bring); - - /* reset the migration keys if we've just entered a new half */ - if (first_block_in_half(bring)) { - for_each_super_root(super, i, root) { - memset(root->migration_key, 0, - sizeof(root->migration_key)); - root->migration_key_len = cpu_to_le16(1); - } - } - - mutex_unlock(&bti->mutex); + bt ? le64_to_cpu(bt->hdr.blkno) : 0, + bt ? le64_to_cpu(bt->hdr.seq) : 0); if (bt) { /* returning a cow of an existing block */ memcpy(new, bt, SCOUTFS_BLOCK_SIZE); - put_btree_block(bt); - bt = new; + scoutfs_block_put(sb, bl); } else { /* returning a newly allocated block */ - bt = new; - new = NULL; - memset(bt, 0, SCOUTFS_BLOCK_SIZE); - bt->hdr.fsid = super->hdr.fsid; - bt->free_end = cpu_to_le32(SCOUTFS_BLOCK_SIZE); + memset(new, 0, SCOUTFS_BLOCK_SIZE); + new->hdr.fsid = super->hdr.fsid; + new->free_end = cpu_to_le32(SCOUTFS_BLOCK_SIZE); } + bl = new_bl; + bt = new; bt->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BTREE); bt->hdr.blkno = cpu_to_le64(blkno); @@ -749,11 +495,11 @@ retry: out: if (ret) { - put_btree_block(bt); - bt = NULL; + scoutfs_block_put(sb, bl); + bl = NULL; } - *bt_ret = bt; + *bl_ret = bl; return ret; } @@ -761,8 +507,7 @@ out: * Create a new item in the parent which references the child. The caller * specifies the key in the item that describes the items in the child. */ -static void create_parent_item(struct scoutfs_btree_ring *bring, - struct scoutfs_btree_block *parent, +static void create_parent_item(struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *child, void *key, unsigned key_len) { @@ -779,14 +524,13 @@ static void create_parent_item(struct scoutfs_btree_ring *bring, * recreating it. Descent should have ensured that there was always * room for a maximal key in parents. */ -static void update_parent_item(struct scoutfs_btree_ring *bring, - struct scoutfs_btree_block *parent, +static void update_parent_item(struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *child) { struct scoutfs_btree_item *item = last_item(child); delete_item(parent, pos); - create_parent_item(bring, parent, pos, child, + create_parent_item(parent, pos, child, item_key(item), item_key_len(item)); } @@ -801,17 +545,21 @@ static void update_parent_item(struct scoutfs_btree_ring *bring, * * Returns -errno, 0 if nothing done, or 1 if we split. */ -static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, +static int try_split(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len, unsigned val_len, struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *right) { - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; - struct scoutfs_btree_block *left = NULL; + struct scoutfs_block *left_bl = NULL; + struct scoutfs_block *par_bl = NULL; + struct scoutfs_btree_block *left; struct scoutfs_btree_item *item; unsigned int all_bytes; - bool put_parent = false; int ret; + int err; if (scoutfs_option_bool(sb, Opt_btree_force_tiny_blocks)) all_bytes = SCOUTFS_BLOCK_SIZE - SCOUTFS_BTREE_TINY_BLOCK_SIZE; @@ -824,18 +572,23 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, return 0; /* alloc split neighbour first to avoid unwinding tree growth */ - ret = get_ref_block(sb, BTW_ALLOC, NULL, &left); + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &left_bl); if (ret) return ret; + left = left_bl->data; + left->level = right->level; if (!parent) { - ret = get_ref_block(sb, BTW_ALLOC, NULL, &parent); + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl); if (ret) { - put_btree_block(left); + err = scoutfs_balloc_free(sb, alloc, wri, + le64_to_cpu(left->hdr.blkno)); + BUG_ON(err); /* radix should have been dirty */ + scoutfs_block_put(sb, left_bl); return ret; } - put_parent = true; + parent = par_bl->data; parent->level = root->height; root->height++; @@ -843,19 +596,18 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, root->ref.seq = parent->hdr.seq; pos = 0; - create_parent_item(bring, parent, pos, right, + create_parent_item(parent, pos, right, &max_key, sizeof(max_key)); } move_items(left, right, false, used_total(right) / 2); item = last_item(left); - create_parent_item(bring, parent, pos, left, + create_parent_item(parent, pos, left, item_key(item), item_key_len(item)); - put_btree_block(left); - if (put_parent) - put_btree_block(parent); + scoutfs_block_put(sb, left_bl); + scoutfs_block_put(sb, par_bl); return 1; } @@ -869,12 +621,15 @@ static int try_split(struct super_block *sb, struct scoutfs_btree_root *root, * * XXX this could more cleverly chose a merge candidate sibling */ -static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, +static int try_merge(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *bt) { - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; struct scoutfs_btree_block *sib; + struct scoutfs_block *sib_bl; struct scoutfs_btree_ref *ref; unsigned int min_used; unsigned int sib_pos; @@ -902,9 +657,10 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, } ref = item_val(pos_item(parent, sib_pos)); - ret = get_ref_block(sb, BTW_DIRTY, ref, &sib); + ret = get_ref_block(sb, alloc, wri, BTW_DIRTY, ref, &sib_bl); if (ret) return ret; + sib = sib_bl->data; if (used_total(sib) < min_used) to_move = used_total(sib); @@ -915,22 +671,30 @@ static int try_merge(struct super_block *sb, struct scoutfs_btree_root *root, /* update our parent's item */ if (!move_right) - update_parent_item(bring, parent, pos, bt); + update_parent_item(parent, pos, bt); /* update or delete sibling's parent item */ - if (le32_to_cpu(sib->nr_items) == 0) + if (le32_to_cpu(sib->nr_items) == 0) { delete_item(parent, sib_pos); - else if (move_right) - update_parent_item(bring, parent, sib_pos, sib); + ret = scoutfs_balloc_free(sb, alloc, wri, + le64_to_cpu(sib->hdr.blkno)); + BUG_ON(ret); /* could have dirtied alloc to avoid error */ + + } else if (move_right) { + update_parent_item(parent, sib_pos, sib); + } /* and finally shrink the tree if our parent is the root with 1 */ if (le32_to_cpu(parent->nr_items) == 1) { root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; + ret = scoutfs_balloc_free(sb, alloc, wri, + le64_to_cpu(parent->hdr.blkno)); + BUG_ON(ret); /* could have dirtied alloc to avoid error */ } - put_btree_block(sib); + scoutfs_block_put(sb, sib_bl); return 1; } @@ -1038,15 +802,19 @@ static void inc_key(u8 *bytes, unsigned *len) * dirtying old leaf blocks and isn't actually doing anything with the * blocks themselves. */ -static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, +static int btree_walk(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, int flags, void *key, unsigned key_len, unsigned int val_len, - struct scoutfs_btree_block **bt_ret, void *iter_key, + struct scoutfs_block **bl_ret, void *iter_key, unsigned *iter_len) { - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; + struct scoutfs_block *par_bl = NULL; + struct scoutfs_block *bl = NULL; struct scoutfs_btree_block *parent = NULL; - struct scoutfs_btree_block *bt = NULL; + struct scoutfs_btree_block *bt; struct scoutfs_btree_item *item; struct scoutfs_btree_ref *ref; unsigned int level; @@ -1055,13 +823,16 @@ static int btree_walk(struct super_block *sb, struct scoutfs_btree_root *root, int cmp; int ret; - if (WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL)) + if (WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL) || + WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; restart: - put_btree_block(parent); + scoutfs_block_put(sb, par_bl); + par_bl = NULL; parent = NULL; - put_btree_block(bt); + scoutfs_block_put(sb, bl); + bl = NULL; bt = NULL; level = root->height; if (iter_len) @@ -1073,8 +844,10 @@ restart: if (!(flags & BTW_INSERT)) { ret = -ENOENT; } else { - ret = get_ref_block(sb, BTW_ALLOC, &root->ref, &bt); + ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, + &root->ref, &bl); if (ret == 0) { + bt = bl->data; bt->level = 0; root->height = 1; } @@ -1085,16 +858,10 @@ restart: ref = &root->ref; while(level-- > 0) { - /* no point in dirtying current leaf blocks for migration */ - if ((flags & BTW_MIGRATE) && level == 0 && - blkno_is_current(bring, le64_to_cpu(ref->blkno))) { - ret = 0; - break; - } - - ret = get_ref_block(sb, flags, ref, &bt); + ret = get_ref_block(sb, alloc, wri, flags, ref, &bl); if (ret) break; + bt = bl->data; /* XXX it'd be nice to make this tunable */ ret = 0 && verify_btree_block(bt, level); @@ -1126,10 +893,10 @@ restart: */ ret = 0; if (flags & (BTW_INSERT | BTW_DELETE)) - ret = try_split(sb, root, key, key_len, val_len, - parent, pos, bt); + ret = try_split(sb, alloc, wri, root, key, key_len, + val_len, parent, pos, bt); if (ret == 0 && (flags & BTW_DELETE) && parent) - ret = try_merge(sb, root, parent, pos, bt); + ret = try_merge(sb, alloc, wri, root, parent, pos, bt); if (ret > 0) goto restart; else if (ret < 0) @@ -1170,31 +937,37 @@ restart: memcpy(iter_key, item_key(item), *iter_len); } - put_btree_block(parent); + scoutfs_block_put(sb, par_bl); + par_bl = bl; parent = bt; + bl = NULL; bt = NULL; ref = item_val(pos_item(parent, pos)); } out: - put_btree_block(parent); + scoutfs_block_put(sb, par_bl); if (ret) { - put_btree_block(bt); - bt = NULL; + scoutfs_block_put(sb, bl); + bl = NULL; } - if (bt_ret) - *bt_ret = bt; + if (bl_ret) + *bl_ret = bl; else - put_btree_block(bt); + scoutfs_block_put(sb, bl); return ret; } static void init_item_ref(struct scoutfs_btree_item_ref *iref, + struct super_block *sb, + struct scoutfs_block *bl, struct scoutfs_btree_item *item) { + iref->sb = sb; + iref->bl = bl; iref->key = item_key(item); iref->key_len = le16_to_cpu(item->key_len); iref->val = item_val(item); @@ -1203,8 +976,8 @@ static void init_item_ref(struct scoutfs_btree_item_ref *iref, void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref) { - if (!IS_ERR_OR_NULL(iref) && !IS_ERR_OR_NULL(iref->key)) { - put_btree_block(iref->key); + if (!IS_ERR_OR_NULL(iref) && !IS_ERR_OR_NULL(iref->bl)) { + scoutfs_block_put(iref->sb, iref->bl); memset(iref, 0, sizeof(struct scoutfs_btree_item_ref)); } } @@ -1220,6 +993,7 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; unsigned int pos; int cmp; int ret; @@ -1227,15 +1001,17 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root if (WARN_ON_ONCE(iref->key)) return -EINVAL; - ret = btree_walk(sb, root, 0, key, key_len, 0, &bt, NULL, NULL); + ret = btree_walk(sb, NULL, NULL, root, 0, key, key_len, 0, &bl, + NULL, NULL); if (ret == 0) { + bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { item = pos_item(bt, pos); - init_item_ref(iref, item); + init_item_ref(iref, sb, bl, item); ret = 0; } else { - put_btree_block(bt); + scoutfs_block_put(sb, bl); ret = -ENOENT; } @@ -1260,11 +1036,15 @@ static bool invalid_item(void *key, unsigned key_len, unsigned val_len) * If no value pointer is given then the item is created with a zero * length value. */ -int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len) { struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; int pos; int cmp; int ret; @@ -1272,9 +1052,10 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root if (invalid_item(key, key_len, val_len)) return -EINVAL; - ret = btree_walk(sb, root, BTW_DIRTY | BTW_INSERT, key, key_len, - val_len, &bt, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, + key, key_len, val_len, &bl, NULL, NULL); if (ret == 0) { + bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); if (cmp) { create_item(bt, pos, key, key_len, val, val_len); @@ -1283,7 +1064,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root ret = -EEXIST; } - put_btree_block(bt); + scoutfs_block_put(sb, bl); } return ret; @@ -1300,11 +1081,14 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root * which doesn't fit. */ int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len) { struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; int pos; int cmp; int ret; @@ -1312,9 +1096,10 @@ int scoutfs_btree_update(struct super_block *sb, if (invalid_item(key, key_len, val_len)) return -EINVAL; - ret = btree_walk(sb, root, BTW_DIRTY | BTW_INSERT, key, key_len, - val_len, &bt, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, + key, key_len, val_len, &bl, NULL, NULL); if (ret == 0) { + bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { delete_item(bt, pos); @@ -1324,7 +1109,7 @@ int scoutfs_btree_update(struct super_block *sb, ret = -ENOENT; } - put_btree_block(bt); + scoutfs_block_put(sb, bl); } return ret; @@ -1334,17 +1119,22 @@ int scoutfs_btree_update(struct super_block *sb, * Delete an item from the tree. -ENOENT is returned if the key isn't * found. */ -int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len) { struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; int pos; int cmp; int ret; - ret = btree_walk(sb, root, BTW_DELETE | BTW_DIRTY, key, key_len, 0, - &bt, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, + key, key_len, 0, &bl, NULL, NULL); if (ret == 0) { + bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { delete_item(bt, pos); @@ -1360,7 +1150,7 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root ret = -ENOENT; } - put_btree_block(bt); + scoutfs_block_put(sb, bl); } return ret; @@ -1378,12 +1168,13 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root * it lets the tree shape change between each walk and allows empty * blocks. */ -static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, +static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, int flags, void *key, unsigned key_len, struct scoutfs_btree_item_ref *iref) { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; unsigned iter_len; unsigned walk_len; void *iter_key; @@ -1407,10 +1198,11 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, walk_len = key_len; for (;;) { - ret = btree_walk(sb, root, flags, walk_key, walk_len, 0, &bt, - iter_key, &iter_len); + ret = btree_walk(sb, NULL, NULL, root, flags, walk_key, + walk_len, 0, &bl, iter_key, &iter_len); if (ret < 0) break; + bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); @@ -1425,12 +1217,12 @@ static int btree_iter(struct super_block *sb, struct scoutfs_btree_root *root, /* found the next item in this leaf */ if (pos >= 0 && pos < le32_to_cpu(bt->nr_items)) { item = pos_item(bt, pos); - init_item_ref(iref, item); + init_item_ref(iref, sb, bl, item); ret = 0; break; } - put_btree_block(bt); + scoutfs_block_put(sb, bl); /* nothing in this leaf, walk gave us a key */ if (iter_len > 0) { @@ -1485,194 +1277,29 @@ int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root * * <0 is returned on error, including -ENOENT if the key isn't present. */ -int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len) { struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; int cmp; int ret; - ret = btree_walk(sb, root, BTW_DIRTY, key, key_len, 0, &bt, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, key_len, 0, &bl, + NULL, NULL); if (ret == 0) { + bt = bl->data; find_pos(bt, key, key_len, &cmp); if (cmp == 0) ret = 0; else ret = -ENOENT; - put_btree_block(bt); + + scoutfs_block_put(sb, bl); } return ret; } - -/* - * This initializes all our tracking info based on the super. Called - * before dirtying anything after having read the super or finished - * writing dirty blocks. - */ -static int btree_prepare_write(struct super_block *sb) -{ - struct scoutfs_btree_ring *bring = &SCOUTFS_SB(sb)->super.bring; - DECLARE_BTREE_INFO(sb, bti); - - bti->cur_dirtied = 0; - bti->old_dirtied = 0; - bti->first_dirty_bh = NULL; - bti->last_dirty_bh = NULL; - bti->first_dirty_blkno = le64_to_cpu(bring->first_blkno) + - le64_to_cpu(bring->next_block); - bti->first_dirty_seq = le64_to_cpu(bring->next_seq); - - return 0; -} - -/* - * The caller is serializing btree item dirtying and dirty block writing. - */ -bool scoutfs_btree_has_dirty(struct super_block *sb) -{ - DECLARE_BTREE_INFO(sb, bti); - - return bti->first_dirty_bh != NULL; -} - -/* dirty block allocation built this list */ -#define for_each_dirty_bh(bti, bh, tmp) \ - for (bh = bti->first_dirty_bh; bh && (tmp = bh->b_private, 1); bh = tmp) - -/* - * Write the dirty region of blocks to the ring. The caller still has - * to write the super after we're done. That could fail and we could - * be asked to write the blocks all over again. - * - * We're the only writer. - */ -int scoutfs_btree_write_dirty(struct super_block *sb) -{ - DECLARE_BTREE_INFO(sb, bti); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_btree_root *root; - struct scoutfs_btree_block *bt; - struct buffer_head *tmp; - struct buffer_head *bh; - struct blk_plug plug; - unsigned int walk_len; - unsigned int iter_len; - bool progress; - void *walk_key; - void *iter_key; - int ret; - int i; - - if (bti->first_dirty_bh == NULL) - return 0; - - iter_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); - if (!iter_key) - return -ENOMEM; - - progress = true; - while (progress && bti->old_dirtied < bti->cur_dirtied) { - progress = false; - - for_each_super_root(super, i, root) { - walk_key = root->migration_key; - walk_len = le16_to_cpu(root->migration_key_len); - if (walk_len == 0) - continue; - - ret = btree_walk(sb, root, - BTW_DIRTY | BTW_NEXT | BTW_MIGRATE, - walk_key, walk_len, 0, NULL, - iter_key, &iter_len); - if (ret < 0) - goto out; - - root->migration_key_len = cpu_to_le16(iter_len); - if (iter_len) { - memcpy(walk_key, iter_key, iter_len); - progress = true; - } else { - memset(walk_key, 0, SCOUTFS_BTREE_MAX_KEY_LEN); - } - } - } - - /* checksum everything to reduce time between io submission merging */ - for_each_dirty_bh(bti, bh, tmp) { - bt = (void *)bh->b_data; - bt->hdr.crc = scoutfs_block_calc_crc(&bt->hdr); - } - - blk_start_plug(&plug); - - for_each_dirty_bh(bti, bh, tmp) { - lock_buffer(bh); - set_buffer_mapped(bh); - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - /* XXX should be more careful with flags */ - submit_bh(WRITE_SYNC | REQ_META | REQ_PRIO, bh); - } - - blk_finish_plug(&plug); - - ret = 0; - for_each_dirty_bh(bti, bh, tmp) { - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { - scoutfs_inc_counter(sb, btree_write_error); - ret = -EIO; - } - } - -out: - kfree(iter_key); - return ret; -} - -/* - * The dirty blocks and their super reference have been successfully written. - * Remove them from the dirty list and drop their references and prepare - * for the next write. - */ -void scoutfs_btree_write_complete(struct super_block *sb) -{ - DECLARE_BTREE_INFO(sb, bti); - struct buffer_head *bh; - struct buffer_head *tmp; - - for_each_dirty_bh(bti, bh, tmp) { - bh->b_private = NULL; - put_bh(bh); - } - - btree_prepare_write(sb); -} - -int scoutfs_btree_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct btree_info *bti; - - bti = kzalloc(sizeof(struct btree_info), GFP_KERNEL); - if (!bti) - return -ENOMEM; - - mutex_init(&bti->mutex); - - sbi->btree_info = bti; - - btree_prepare_write(sb); - - return 0; -} - -void scoutfs_btree_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - - kfree(sbi->btree_info); - sbi->btree_info = NULL; -} diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 860923a2..d56b2f3e 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -3,7 +3,13 @@ #include +struct scoutfs_balloc_allocator; +struct scoutfs_block_writer; +struct scoutfs_block; + struct scoutfs_btree_item_ref { + struct super_block *sb; + struct scoutfs_block *bl; void *key; unsigned key_len; void *val; @@ -13,16 +19,26 @@ struct scoutfs_btree_item_ref { #define SCOUTFS_BTREE_ITEM_REF(name) \ struct scoutfs_btree_item_ref name = {NULL,} + int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len, struct scoutfs_btree_item_ref *iref); -int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_insert(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); -int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_update(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); -int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_delete(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len); int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len, @@ -36,16 +52,12 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len, struct scoutfs_btree_item_ref *iref); -int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, +int scoutfs_btree_dirty(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, void *key, unsigned key_len); void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref); -bool scoutfs_btree_has_dirty(struct super_block *sb); -int scoutfs_btree_write_dirty(struct super_block *sb); -void scoutfs_btree_write_complete(struct super_block *sb); - -int scoutfs_btree_setup(struct super_block *sb); -void scoutfs_btree_destroy(struct super_block *sb); - #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 58a73b9a..cb177d18 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -199,15 +199,10 @@ struct scoutfs_btree_ref { /* * A height of X means that the first block read will have level X-1 and * the leaves will have level 0. - * - * The migration key is used to walk the tree finding old blocks to migrate - * into the current half of the ring. */ struct scoutfs_btree_root { struct scoutfs_btree_ref ref; __u8 height; - __le16 migration_key_len; - __u8 migration_key[SCOUTFS_BTREE_MAX_KEY_LEN]; } __packed; struct scoutfs_btree_item_header { diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b7945b2d..4c2f9a4d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2389,21 +2389,15 @@ TRACE_EVENT(scoutfs_btree_read_error, ); TRACE_EVENT(scoutfs_btree_dirty_block, - TP_PROTO(struct super_block *sb, u64 blkno, u64 seq, u64 next_block, - u64 next_seq, unsigned long cur_dirtied, - unsigned long old_dirtied, u64 bt_blkno, u64 bt_seq), + TP_PROTO(struct super_block *sb, u64 blkno, u64 seq, + u64 bt_blkno, u64 bt_seq), - TP_ARGS(sb, blkno, seq, next_block, next_seq, cur_dirtied, old_dirtied, - bt_blkno, bt_seq), + TP_ARGS(sb, blkno, seq, bt_blkno, bt_seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, blkno) __field(__u64, seq) - __field(__u64, next_block) - __field(__u64, next_seq) - __field(unsigned long, cur_dirtied) - __field(unsigned long, old_dirtied) __field(__u64, bt_blkno) __field(__u64, bt_seq) ), @@ -2412,18 +2406,13 @@ TRACE_EVENT(scoutfs_btree_dirty_block, SCSB_TRACE_ASSIGN(sb); __entry->blkno = blkno; __entry->seq = seq; - __entry->next_block = next_block; - __entry->next_seq = next_seq; - __entry->cur_dirtied = cur_dirtied; - __entry->old_dirtied = old_dirtied; __entry->bt_blkno = bt_blkno; __entry->bt_seq = bt_seq; ), - TP_printk(SCSBF" blkno %llu seq %llu next_block %llu next_seq %llu cur_dirtied %lu old_dirtied %lu bt_blkno %llu bt_seq %llu", + TP_printk(SCSBF" blkno %llu seq %llu bt_blkno %llu bt_seq %llu", SCSB_TRACE_ARGS, __entry->blkno, __entry->seq, - __entry->next_block, __entry->next_seq, __entry->cur_dirtied, - __entry->old_dirtied, __entry->bt_blkno, __entry->bt_seq) + __entry->bt_blkno, __entry->bt_seq) ); DECLARE_EVENT_CLASS(scoutfs_extent_class, From 0f83dfd51274391e729d6031300c524c5c7db4f9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 08:15:02 -0700 Subject: [PATCH 756/920] scoutfs: update block btree interfaces in server Teach the server to maintain and use its block allocator and writer contexts when operating on its btrees. The manifest tree operations aren't updated because they're about to be removed. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 ++ kmod/src/lock_server.c | 20 ++++++++++++--- kmod/src/lock_server.h | 4 ++- kmod/src/manifest.c | 4 +-- kmod/src/server.c | 57 +++++++++++++++++++++++++++--------------- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index cb177d18..e667d550 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -502,6 +502,8 @@ struct scoutfs_super_block { __le64 unmount_barrier; __u8 quorum_count; struct scoutfs_inet_addr server_addr; + struct scoutfs_balloc_root core_balloc_alloc; + struct scoutfs_balloc_root core_balloc_free; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; struct scoutfs_btree_root lock_clients; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index b330218e..5c393038 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -19,6 +19,8 @@ #include "net.h" #include "tseq.h" #include "spbm.h" +#include "block.h" +#include "balloc.h" #include "btree.h" #include "msg.h" #include "scoutfs_trace.h" @@ -84,6 +86,9 @@ struct lock_server_info { struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; + + struct scoutfs_balloc_allocator *alloc; + struct scoutfs_block_writer *wri; }; #define DECLARE_LOCK_SERVER_INFO(sb, name) \ @@ -590,7 +595,8 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, if (ret == 0) scoutfs_btree_put_iref(&iref); } else { - ret = scoutfs_btree_insert(sb, &super->lock_clients, + ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri, + &super->lock_clients, &cbk, sizeof(cbk), NULL, 0); } mutex_unlock(&inf->mutex); @@ -790,7 +796,8 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) /* XXX these aren't immediately committed */ cbk.rid = cpu_to_be64(rid); - ret = scoutfs_btree_delete(sb, &super->lock_clients, + ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, + &super->lock_clients, &cbk, sizeof(cbk)); if (ret) break; @@ -829,7 +836,8 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) cli.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); - ret = scoutfs_btree_delete(sb, &super->lock_clients, &cli, sizeof(cli)); + ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, + &super->lock_clients, &cli, sizeof(cli)); mutex_unlock(&inf->mutex); if (ret == -ENOENT) { ret = 0; @@ -929,7 +937,9 @@ static void lock_server_tseq_show(struct seq_file *m, * all the existing clients, either they reconnect and replay locks or * we time them out. */ -int scoutfs_lock_server_setup(struct super_block *sb) +int scoutfs_lock_server_setup(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -952,6 +962,8 @@ int scoutfs_lock_server_setup(struct super_block *sb) INIT_DELAYED_WORK(&inf->recovery_dwork, scoutfs_lock_server_recovery_timeout); scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); + inf->alloc = alloc; + inf->wri = wri; inf->tseq_dentry = scoutfs_tseq_create("server_locks", sbi->debug_root, &inf->tseq_tree); diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 0ac2f772..784b2575 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -11,7 +11,9 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); -int scoutfs_lock_server_setup(struct super_block *sb); +int scoutfs_lock_server_setup(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri); void scoutfs_lock_server_destroy(struct super_block *sb); #endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c index 1fa6158a..74a250e8 100644 --- a/kmod/src/manifest.c +++ b/kmod/src/manifest.c @@ -191,7 +191,7 @@ int scoutfs_manifest_add(struct super_block *sb, trace_scoutfs_manifest_add(sb, ment->level, ment->segno, ment->seq, &ment->first, &ment->last); - ret = scoutfs_btree_insert(sb, &super->manifest.root, + ret = scoutfs_btree_insert(sb, NULL, &super->manifest.root, &mkey, sizeof(mkey), &mval, sizeof(mval)); if (ret == 0) { mani->nr_levels = max_t(u8, mani->nr_levels, ment->level + 1); @@ -223,7 +223,7 @@ int scoutfs_manifest_del(struct super_block *sb, init_btree_key(&mkey, ment->level, ment->seq, &ment->first); - ret = scoutfs_btree_delete(sb, &super->manifest.root, + ret = scoutfs_btree_delete(sb, NULL, &super->manifest.root, &mkey, sizeof(mkey)); if (ret == 0) add_level_count(sb, ment->level, -1ULL); diff --git a/kmod/src/server.c b/kmod/src/server.c index 4fabfff6..534aec66 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -25,6 +25,8 @@ #include "format.h" #include "counters.h" #include "inode.h" +#include "block.h" +#include "balloc.h" #include "btree.h" #include "manifest.h" #include "seg.h" @@ -90,6 +92,9 @@ struct server_info { struct mutex farewell_mutex; struct list_head farewell_requests; struct work_struct farewell_work; + + struct scoutfs_balloc_allocator alloc; + struct scoutfs_block_writer wri; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -155,6 +160,7 @@ static int init_extent_from_btree_key(struct scoutfs_extent *ext, u8 type, static int server_extent_io(struct super_block *sb, int op, struct scoutfs_extent *ext, void *data) { + DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_extent_btree_key ebk; SCOUTFS_BTREE_ITEM_REF(iref); @@ -197,11 +203,13 @@ static int server_extent_io(struct super_block *sb, int op, } } else if (op == SEI_INSERT) { - ret = scoutfs_btree_insert(sb, &super->alloc_root, + ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, + &super->alloc_root, &ebk, sizeof(ebk), NULL, 0); } else if (op == SEI_DELETE) { - ret = scoutfs_btree_delete(sb, &super->alloc_root, + ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, + &super->alloc_root, &ebk, sizeof(ebk)); } else { @@ -599,25 +607,21 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - if (!scoutfs_btree_has_dirty(sb)) { - ret = 0; - goto out; - } - - ret = scoutfs_btree_write_dirty(sb); + ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); goto out; } + super->core_balloc_alloc = server->alloc.alloc_root; + super->core_balloc_free = server->alloc.free_root; + ret = scoutfs_write_super(sb, super); if (ret) { scoutfs_err(sb, "server error writing super block: %d", ret); goto out; } - scoutfs_btree_write_complete(sb); - write_seqcount_begin(&server->stable_seqcount); server->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; write_seqcount_end(&server->stable_seqcount); @@ -910,7 +914,8 @@ static int server_advance_seq(struct super_block *sb, tsk.trans_seq = le64_to_be64(their_seq); tsk.rid = cpu_to_be64(rid); - ret = scoutfs_btree_delete(sb, &super->trans_seqs, + ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, + &super->trans_seqs, &tsk, sizeof(tsk)); if (ret < 0 && ret != -ENOENT) goto out; @@ -925,7 +930,8 @@ static int server_advance_seq(struct super_block *sb, tsk.trans_seq = le64_to_be64(next_seq); tsk.rid = cpu_to_be64(rid); - ret = scoutfs_btree_insert(sb, &super->trans_seqs, + ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, + &super->trans_seqs, &tsk, sizeof(tsk), NULL, 0); out: up_write(&server->seq_rwsem); @@ -975,7 +981,9 @@ static int remove_trans_seq(struct super_block *sb, u64 rid) if (be64_to_cpu(tsk.rid) == rid) { trace_scoutfs_trans_seq_farewell(sb, rid, be64_to_cpu(tsk.trans_seq)); - ret = scoutfs_btree_delete(sb, &super->trans_seqs, + ret = scoutfs_btree_delete(sb, &server->alloc, + &server->wri, + &super->trans_seqs, &tsk, sizeof(tsk)); break; } @@ -1185,6 +1193,7 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags) { + DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; @@ -1194,7 +1203,8 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; - return scoutfs_btree_insert(sb, &super->mounted_clients, + return scoutfs_btree_insert(sb, &server->alloc, &server->wri, + &super->mounted_clients, &mck, sizeof(mck), &mcv, sizeof(mcv)); } @@ -1210,13 +1220,15 @@ static int insert_mounted_client(struct super_block *sb, u64 rid, */ static int delete_mounted_client(struct super_block *sb, u64 rid) { + DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_mounted_client_btree_key mck; int ret; mck.rid = cpu_to_be64(rid); - ret = scoutfs_btree_delete(sb, &super->mounted_clients, + ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, + &super->mounted_clients, &mck, sizeof(mck)); if (ret == -ENOENT) ret = 0; @@ -2296,10 +2308,16 @@ static void scoutfs_server_worker(struct work_struct *work) goto out; /* start up the server subsystems before accepting */ - ret = scoutfs_read_super(sb, super) ?: - scoutfs_btree_setup(sb) ?: - scoutfs_manifest_setup(sb) ?: - scoutfs_lock_server_setup(sb); + ret = scoutfs_read_super(sb, super); + if (ret < 0) + goto shutdown; + + scoutfs_balloc_init(&server->alloc, &super->core_balloc_alloc, + &super->core_balloc_free); + scoutfs_block_writer_init(sb, &server->wri); + + ret = scoutfs_manifest_setup(sb) ?: + scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); if (ret) goto shutdown; @@ -2340,7 +2358,6 @@ shutdown: destroy_pending_frees(sb); scoutfs_manifest_destroy(sb); - scoutfs_btree_destroy(sb); scoutfs_lock_server_destroy(sb); out: From e6af174c792e15abe00489e420a4ade6521d3066 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 29 Sep 2019 22:38:00 -0700 Subject: [PATCH 757/920] scoutfs: add commit btree net command Add a simple start of a command that the client will use to commit its dirty trees. This'll be expanded in the future to include more trees and block allocation. Signed-off-by: Zach Brown --- kmod/src/client.c | 20 +++++ kmod/src/client.h | 6 ++ kmod/src/format.h | 6 ++ kmod/src/server.c | 222 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+) diff --git a/kmod/src/client.c b/kmod/src/client.c index e9d7d480..ff740cf8 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -167,6 +167,26 @@ int scoutfs_client_record_segment(struct super_block *sb, &net_ment, sizeof(net_ment), NULL, 0); } +int scoutfs_client_get_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_LOG_TREES, + NULL, 0, lt, sizeof(*lt)); +} + +int scoutfs_client_commit_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_COMMIT_LOG_TREES, + lt, sizeof(*lt), NULL, 0); +} + int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) { struct client_info *client = SCOUTFS_SB(sb)->client_info; diff --git a/kmod/src/client.h b/kmod/src/client.h index fe938306..73bc69ee 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -1,6 +1,8 @@ #ifndef _SCOUTFS_CLIENT_H_ #define _SCOUTFS_CLIENT_H_ +struct scoutfs_segment; + int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, @@ -10,6 +12,10 @@ int scoutfs_client_free_extents(struct super_block *sb, int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); int scoutfs_client_record_segment(struct super_block *sb, struct scoutfs_segment *seg, u8 level); +int scoutfs_client_get_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt); +int scoutfs_client_commit_log_trees(struct super_block *sb, + struct scoutfs_log_trees *lt); u64 *scoutfs_client_bulk_alloc(struct super_block *sb); int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); diff --git a/kmod/src/format.h b/kmod/src/format.h index e667d550..03afe418 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -492,6 +492,8 @@ struct scoutfs_super_block { __le64 next_ino; __le64 next_trans_seq; __le64 total_blocks; + __le64 next_uninit_free_block; + __le64 core_balloc_cursor; __le64 free_blocks; __le64 alloc_cursor; struct scoutfs_btree_ring bring; @@ -506,6 +508,8 @@ struct scoutfs_super_block { struct scoutfs_balloc_root core_balloc_free; struct scoutfs_btree_root alloc_root; struct scoutfs_manifest manifest; + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; @@ -699,6 +703,8 @@ enum { SCOUTFS_NET_CMD_FREE_EXTENTS, SCOUTFS_NET_CMD_ALLOC_SEGNO, SCOUTFS_NET_CMD_RECORD_SEGMENT, + SCOUTFS_NET_CMD_GET_LOG_TREES, + SCOUTFS_NET_CMD_COMMIT_LOG_TREES, SCOUTFS_NET_CMD_ADVANCE_SEQ, SCOUTFS_NET_CMD_GET_LAST_SEQ, SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, diff --git a/kmod/src/server.c b/kmod/src/server.c index 534aec66..578d94d2 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -95,6 +95,8 @@ struct server_info { struct scoutfs_balloc_allocator alloc; struct scoutfs_block_writer wri; + + struct mutex logs_mutex; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -566,6 +568,40 @@ static inline int wait_for_commit(struct commit_waiter *cw) return cw->ret; } +/* + * Add newly initialized free metadata block allocator items to the core + * block allocator. This is called as we commit transactions in the + * server. It adds many more free blocks than is ever consumed by a + * transaction so this will stay ahead of the server's block allocation. + * The intent is to have a low constant overhead to initializing block + * allocators over time instead of requiring a large amount of IO during + * mkfs. + */ +static int add_uninit_balloc_items(struct super_block *sb, + struct server_info *server, + struct scoutfs_super_block *super) +{ + u64 next = le64_to_cpu(super->next_uninit_free_block); + u64 total = le64_to_cpu(super->total_blocks); + u64 nr; + int ret; + + /* next_uninit should always start a new item */ + if (WARN_ON_ONCE(next & SCOUTFS_BALLOC_ITEM_BIT_MASK)) + return -EIO; + + nr = min_t(u64, total - next, + round_up(512 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE, + SCOUTFS_BALLOC_ITEM_BITS)); + + ret = scoutfs_balloc_add_alloc_bulk(sb, &server->alloc, &server->wri, + next, nr); + if (ret == 0) + le64_add_cpu(&super->next_uninit_free_block, nr); + + return ret; +} + /* * A core function of request processing is to modify the manifest and * allocator. Often the processing needs to make the modifications @@ -607,6 +643,10 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } + /* XXX not sure what to do about failure here */ + ret = add_uninit_balloc_items(sb, server, super); + BUG_ON(ret); + ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); @@ -872,6 +912,185 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* + * Give the client references to stable persistent trees that they'll + * use to write their next transaction. + */ +static int server_get_log_trees(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + u64 rid = scoutfs_net_client_rid(conn); + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees lt; + struct commit_waiter cw; + u64 next_past; + u64 at_least; + u64 target; + u64 from; + int ret; + + if (arg_len != 0) { + ret = -EINVAL; + goto out; + } + + down_read(&server->commit_rwsem); + + mutex_lock(&server->logs_mutex); + + memset(<k, 0, sizeof(ltk)); + ltk.rid = cpu_to_be64(rid); + ltk.nr = cpu_to_be64(U64_MAX); + + ret = scoutfs_btree_prev(sb, &super->logs_root, + <k, sizeof(ltk), &iref); + if (ret < 0 && ret != -ENOENT) + goto unlock; + if (ret == 0) { + if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && + iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + memcpy(<k, iref.key, iref.key_len); + memcpy(<v, iref.val, iref.val_len); + if (be64_to_cpu(ltk.rid) != rid) + ret = -ENOENT; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret == -EIO) + goto unlock; + } + + /* initialize new roots if we don't have any */ + if (ret == -ENOENT) { + ltk.rid = cpu_to_be64(rid); + ltk.nr = cpu_to_be64(1); + memset(<v, 0, sizeof(ltv)); + } + + target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; + + /* XXX arbitrarily give client enough metadata for a transaction */ + while (le64_to_cpu(ltv.alloc_root.total_free) < target) { + from = le64_to_cpu(super->core_balloc_cursor); + at_least = target - le64_to_cpu(ltv.alloc_root.total_free); + + ret = scoutfs_balloc_move(sb, &server->alloc, &server->wri, + <v.alloc_root, + &server->alloc.alloc_root, + from, at_least, &next_past); + if (ret == -ENOENT && from != 0) { + super->core_balloc_cursor = 0; + continue; + } + if (ret < 0) + goto unlock; + + super->core_balloc_cursor = cpu_to_le64(next_past); + + } + + /* update client's log tree's item */ + ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, + &super->logs_root, <k, sizeof(ltk), + <v, sizeof(ltv)); +unlock: + mutex_unlock(&server->logs_mutex); + + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); + + if (ret == 0) { + lt.alloc_root = ltv.alloc_root; + lt.free_root = ltv.free_root; + lt.item_root = ltv.item_root; + lt.bloom_ref = ltv.bloom_ref; + lt.rid = be64_to_le64(ltk.rid); + lt.nr = be64_to_le64(ltk.nr); + } + +out: + WARN_ON_ONCE(ret < 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, <, sizeof(lt)); +} + +/* + * The client is sending the roots of all the btree blocks that they + * wrote to their free space for their transaction. Make it persistent + * by referencing the roots from their log item in the logs root and + * committing. + */ +static int server_commit_log_trees(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees *lt; + struct commit_waiter cw; + int ret; + + if (arg_len != sizeof(struct scoutfs_log_trees)) { + ret = -EINVAL; + goto out; + } + lt = arg; + + down_read(&server->commit_rwsem); + mutex_lock(&server->logs_mutex); + + /* find the client's existing item */ + memset(<k, 0, sizeof(ltk)); + ltk.rid = le64_to_be64(lt->rid); + ltk.nr = le64_to_be64(lt->nr); + ret = scoutfs_btree_lookup(sb, &super->logs_root, + <k, sizeof(ltk), &iref); + if (ret < 0 && ret != -ENOENT) + goto unlock; + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + memcpy(<v, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto unlock; + } + + ltv.alloc_root = lt->alloc_root; + ltv.free_root = lt->free_root; + ltv.item_root = lt->item_root; + ltv.bloom_ref = lt->bloom_ref; + + ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, + &super->logs_root, <k, sizeof(ltk), + <v, sizeof(ltv)); + +unlock: + mutex_unlock(&server->logs_mutex); + + if (ret == 0) + queue_commit_work(server, &cw); + up_read(&server->commit_rwsem); + if (ret == 0) + ret = wait_for_commit(&cw); +out: + WARN_ON_ONCE(ret < 0); + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); +} + /* * Give the client the next sequence number for their transaction. They * provide their previous transaction sequence number that they've @@ -2215,6 +2434,8 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, [SCOUTFS_NET_CMD_ALLOC_SEGNO] = server_alloc_segno, [SCOUTFS_NET_CMD_RECORD_SEGMENT] = server_record_segment, + [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, + [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, [SCOUTFS_NET_CMD_GET_MANIFEST_ROOT] = server_get_manifest_root, @@ -2462,6 +2683,7 @@ int scoutfs_server_setup(struct super_block *sb) mutex_init(&server->farewell_mutex); INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); + mutex_init(&server->logs_mutex); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); From 858dad1d51c45c5b6a3daf68d18845a6135fad4b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 27 Sep 2019 15:59:44 -0700 Subject: [PATCH 758/920] scoutfs: add forest subsystem The forest code presents a consistent item interface that's implemented on top of a forest of persistent btrees. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/forest.c | 1389 +++++++++++++++++++++++++++++++++++++++++++++ kmod/src/forest.h | 51 ++ kmod/src/format.h | 62 +- kmod/src/lock.h | 3 + kmod/src/super.h | 2 + 6 files changed, 1505 insertions(+), 3 deletions(-) create mode 100644 kmod/src/forest.c create mode 100644 kmod/src/forest.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index ec6611c1..f90c22e3 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -21,6 +21,7 @@ scoutfs-y += \ export.o \ extents.o \ file.o \ + forest.o \ inode.o \ ioctl.o \ item.o \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c new file mode 100644 index 00000000..cf1bbf1d --- /dev/null +++ b/kmod/src/forest.c @@ -0,0 +1,1389 @@ +/* + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "lock.h" +#include "btree.h" +#include "client.h" +#include "balloc.h" +#include "block.h" +#include "forest.h" + +/* + * scoutfs items are stored in a forest of btrees. Each mount writes + * items into its own relatively small log btree. Each mount can also + * have a few finalized log btrees sitting around that it is no longer + * writing to. Finally a much larger core fs btree is the final home + * for metadata. + * + * The log btrees are modified by multiple transactions over time so + * there is no consistent ordering relationship between the items in + * different btrees. Each item in a log btree stores a version number + * for the item. Readers check log btrees for the most recent version + * that it should use. + * + * From a mount's perspective, the only btree whose blocks are actively + * changing is the mount's own log btree in memory. Every other btree + * it reads is stable (but could be stale) on disk. They don't need to + * be locked, but we might have to retry reads if we hit blocks that + * have been overwritten. + * + * Log btrees are typically very sparse. It would be wasteful for + * readers to read every log btree looking for an item. Each log btree + * contains a bloom filter keyed on the starting key of locks. This + * lets lock holders quickly eliminate log trees that cannot contain + * keys protected by their lock and it caches the btrees to search in + * the lock for the duration of its use. + */ + +/* + * todo: + * - when we adopt a new bloom root we'd need to reset bloom bits in locks + * - add a bunch of counters so we can see bloom/tree ops/etc + */ + +struct forest_info { + struct rw_semaphore rwsem; + struct scoutfs_balloc_allocator alloc; + struct scoutfs_block_writer wri; + struct scoutfs_log_trees our_log; +}; + +#define DECLARE_FOREST_INFO(sb, name) \ + struct forest_info *name = SCOUTFS_SB(sb)->forest_info + +struct forest_root { + struct list_head entry; + struct scoutfs_btree_root item_root; + u64 rid; + u64 nr; +}; + +struct forest_super_refs { + struct scoutfs_btree_ref fs_ref; + struct scoutfs_btree_ref logs_ref; +} __packed; + +struct forest_bloom_nrs { + unsigned int nrs[SCOUTFS_FOREST_BLOOM_NRS]; +}; + +/* + * We have static forest_root entries for the fs and our log btrees so + * that we can iterate over them along with all the discovered and + * allocated log btrees. + */ +struct forest_lock_private { + u64 last_refreshed; + struct rw_semaphore rwsem; + struct list_head roots; + struct forest_root fs_root; + struct forest_root our_log_root; + unsigned long flags; +}; + +enum { + LPRIV_FLAG_ALL_BLOOM_BITS = 0, +}; + +static inline void set_lpriv_flag(struct forest_lock_private *lpriv, int flag) +{ + set_bit(flag, &lpriv->flags); +} +static inline int test_lpriv_flag(struct forest_lock_private *lpriv, int flag) +{ + return test_bit(flag, &lpriv->flags); +} + +static bool is_fs_root(struct forest_lock_private *lpriv, + struct forest_root *fr) +{ + return fr == &lpriv->fs_root; +} + +static bool is_our_log_root(struct forest_lock_private *lpriv, + struct forest_root *fr) +{ + return fr == &lpriv->our_log_root; +} + +static struct forest_lock_private *get_lock_private(struct scoutfs_lock *lock) +{ + struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); + + if (lpriv == NULL) { + lpriv = kzalloc(sizeof(struct forest_lock_private), GFP_NOFS); + if (lpriv) { + init_rwsem(&lpriv->rwsem); + INIT_LIST_HEAD(&lpriv->roots); + INIT_LIST_HEAD(&lpriv->fs_root.entry); + INIT_LIST_HEAD(&lpriv->our_log_root.entry); + + if (cmpxchg(&lock->forest_private, NULL, lpriv) != NULL) + kfree(lpriv); + lpriv = lock->forest_private; + } + } + + return lpriv; +} + +/* + * We can tell if an item is currently dirty in our transaction's log + * root if its lock is held for writing and the item's version matches + * the lock's write version. + */ +static bool is_our_dirty_item(struct scoutfs_lock *lock, + struct forest_root *fr, u64 vers) +{ + struct forest_lock_private *lpriv = get_lock_private(lock); + + return is_our_log_root(lpriv, fr) && + lock->mode == SCOUTFS_LOCK_WRITE && + vers == lock->write_version; +} + +static void clear_roots(struct forest_lock_private *lpriv) +{ + struct forest_root *fr; + struct forest_root *tmp; + + list_for_each_entry_safe(fr, tmp, &lpriv->roots, entry) { + list_del_init(&fr->entry); + if (!is_fs_root(lpriv, fr) && !is_our_log_root(lpriv, fr)) + kfree(fr); + } +} + +/* + * Make sure that our log btree will be at the head of the list of + * btrees to read. This can be racing with clearing the list to check + * the bloom blocks again. We want the addition of the log btree to + * persist across clearing the lists so we set the rid/nr which causes + * the root to be added to the list after the bloom blocks are checked. + */ +static void add_our_log_root(struct forest_info *finf, + struct forest_lock_private *lpriv) +{ + struct forest_root *fr = &lpriv->our_log_root; + + BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); + + if (fr->rid == 0) { + fr->rid = le64_to_cpu(finf->our_log.rid); + fr->nr = le64_to_cpu(finf->our_log.nr); + } + + if (list_empty(&fr->entry)) + list_add(&fr->entry, &lpriv->roots); +} + +/* + * This is called by the locking code while it's excluding users of the + * lock. + */ +void scoutfs_forest_clear_lock(struct super_block *sb, + struct scoutfs_lock *lock) +{ + struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); + + if (lpriv) { + clear_roots(lpriv); + kfree(lpriv); + } +} + +/* + * All the btrees we read are stable and read-only except for our log + * btree which is being actively modified in memory by locked writers. + * Once we lock it we need to get the current version of the root. + */ +static void read_lock_forest_root(struct forest_info *finf, + struct forest_lock_private *lpriv, + struct forest_root *fr) +{ + if (is_our_log_root(lpriv, fr)) { + down_read(&finf->rwsem); + fr->item_root = finf->our_log.item_root; + } +} + +static void read_unlock_forest_root(struct forest_info *finf, + struct forest_lock_private *lpriv, + struct forest_root *fr) +{ + if (is_our_log_root(lpriv, fr)) { + up_read(&finf->rwsem); + } +} + +/* + * XXX need something better. + */ +static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, + struct scoutfs_key *key) +{ + u32 crc = ~0; + int i; + + for (i = 0; i < ARRAY_SIZE(bloom->nrs); i++) { + crc = crc32c(crc, key, sizeof(struct scoutfs_key)); + bloom->nrs[i] = crc % SCOUTFS_FOREST_BLOOM_BITS; + } +} + +/* + * Empty the list of btrees currently stored in the lock and walk the + * current fs image looking for btrees whose bloom filters indicate that + * the btree may contain items covered by the lock. + * + * We ensure that the our log btree is always first and that the fs + * btree is always last because those positions offer short-circuiting + * optimizations. + * + * This doesn't deal with rereading stale blocks itself.. it returns + * ESTALE to the caller who already has to deal with retrying stale + * blocks from their btree reads. We give them the super refs we read + * so that they can identify persistent stale block errors that come + * from corruption. + * + * Because we're starting all the reads from a stable read super this + * will not see any dirty blocks we have in memory. We don't have to + * lock any of the btree reads. It also won't find the currently dirty + * version of our log btree. Writers mark our static log btree in lpriv + * to indicate that we should include our dirty log btree in reads. + * We'll also naturally add it if we see a persistent version on disk + * with all of the bloom bits set. + */ +static int refresh_bloom_roots(struct super_block *sb, + struct scoutfs_lock *lock, + struct forest_super_refs *srefs) +{ + DECLARE_FOREST_INFO(sb, finf); + struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct forest_bloom_nrs bloom; + struct scoutfs_super_block super; + struct forest_root *fr = NULL; + struct scoutfs_bloom_block *bb; + struct scoutfs_block *bl; + int ret; + int i; + + memset(srefs, 0, sizeof(*srefs)); + + /* empty the list so no one iterates until someone's added */ + clear_roots(lpriv); + + ret = scoutfs_read_super(sb, &super); + if (ret) + goto out; + + srefs->fs_ref = super.fs_root.ref; + srefs->logs_ref = super.logs_root.ref; + + calc_bloom_nrs(&bloom, &lock->start); + + memset(<k, 0, sizeof(ltk)); + for (;; be64_add_cpu(<k.nr, 1)) { + + ret = scoutfs_btree_next(sb, &super.logs_root, + <k, sizeof(ltk), &iref); + if (ret == -ENOENT) { + ret = 0; + break; + } + if (ret < 0) + goto out; + + if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && + iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + memcpy(<k, iref.key, iref.key_len); + memcpy(<v, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + if (ltv.bloom_ref.blkno == 0) + continue; + + bl = scoutfs_block_read(sb, le64_to_cpu(ltv.bloom_ref.blkno)); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + bb = bl->data; + + for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { + if (!test_bit_le(bloom.nrs[i], bb->bits)) + break; + } + + scoutfs_block_put(sb, bl); + + /* one of the bloom bits wasn't set */ + if (i != ARRAY_SIZE(bloom.nrs)) + continue; + + /* add our current log tree if we see its bloom */ + if (be64_to_cpu(ltk.rid) == le64_to_cpu(finf->our_log.rid) && + be64_to_cpu(ltk.nr) == le64_to_cpu(finf->our_log.nr)) { + add_our_log_root(finf, lpriv); + continue; + } + + /* all bloom bits set, add to the list */ + fr = kzalloc(sizeof(struct forest_root), GFP_NOFS); + if (fr == NULL) { + ret = -ENOMEM; + goto out; + } + + fr->item_root = ltv.item_root; + fr->rid = be64_to_cpu(ltk.rid); + fr->nr = be64_to_cpu(ltk.nr); + + list_add_tail(&fr->entry, &lpriv->roots); + } + + /* add our current log root if a locked writer added it */ + if (lpriv->our_log_root.rid != 0) + add_our_log_root(finf, lpriv); + + /* always add the fs root at the tail */ + fr = &lpriv->fs_root; + fr->item_root = super.fs_root; + fr->rid = 0; + fr->nr = 0; + list_add_tail(&fr->entry, &lpriv->roots); + + lpriv->last_refreshed = lock->refresh_gen; + + ret = 0; + +out: + if (ret < 0) + clear_roots(lpriv); + return ret; +} + +/* initialize some super refs that initially aren't equal */ +#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ + struct forest_super_refs a = {{cpu_to_le64(0),}}; \ + struct forest_super_refs b = {{cpu_to_le64(1),}} + + +/* + * The caller saw stale blocks. If they're seeing the same root refs + * and are still getting stale then it's consistent corruption and we + * return an error. Otherwise we refresh the bloom roots and try again. + * If this returns 0 then the caller is going to retry. If *we* saw + * stale blocks trying to refresh the bloom then we return 0 to have the + * caller remember the root refs and try again. + */ +static int refresh_check_stale(struct super_block *sb, + struct scoutfs_lock *lock, + struct forest_super_refs *prev_srefs, + struct forest_super_refs *srefs) +{ + struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); + int ret; + + if (memcmp(prev_srefs, srefs, sizeof(*srefs)) == 0) + return -EIO; + *prev_srefs = *srefs; + + down_write(&lpriv->rwsem); + ret = refresh_bloom_roots(sb, lock, srefs); + up_write(&lpriv->rwsem); + if (ret == -ESTALE) + ret = 0; + + return ret; +} + +/* + * Iterate over all the roots that could contain items covered by the + * caller's lock. The caller starts iteration by passing in a NULL fr. + * We return -ESTALE if the caller needs to refresh the bloom roots. We + * use the lock's refresh gen to find out when the lock was invalidated + * and the contents of the trees could have changed. + */ +static int for_each_forest_root(struct scoutfs_lock *lock, + struct forest_lock_private *lpriv, + struct forest_root **fr) +{ + if (WARN_ON_ONCE(!rwsem_is_locked(&lpriv->rwsem))) + return -EIO; + + if (list_empty(&lpriv->roots) || + lock->refresh_gen != lpriv->last_refreshed) + return -ESTALE; + + if (*fr == NULL) + *fr = list_prepare_entry((*fr), &lpriv->roots, entry); + + list_for_each_entry_continue((*fr), &lpriv->roots, entry) + return 0; + + *fr = NULL; + return 0; +} + +/* + * We fake 1 as the version for the fs items. The least valid log item + * version is also 1, but we guarantee that we check the log trees first + * so they'll always be found before the fs items. + */ +static u64 item_vers(struct forest_lock_private *lpriv, + struct forest_root *fr, void *val) +{ + struct scoutfs_log_item_value *liv; + + if (is_fs_root(lpriv, fr)) + return 1; + + liv = val; + return le64_to_cpu(liv->vers); +} + +static bool item_flags(struct forest_lock_private *lpriv, + struct forest_root *fr, void *val) +{ + struct scoutfs_log_item_value *liv; + + if (is_fs_root(lpriv, fr)) + return 0; + + liv = val; + return liv->flags; +} + +static bool item_is_deletion(struct forest_lock_private *lpriv, + struct forest_root *fr, void *val) +{ + return item_flags(lpriv, fr, val) & SCOUTFS_LOG_ITEM_FLAG_DELETION; +} + +/* just a little helper to slim down all the call sites */ +static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, + int mode) +{ + if (WARN_ON_ONCE(!scoutfs_lock_protected(lock, key, mode))) + return -EINVAL; + else + return 0; +} + +/* + * Copy the cached item's value into the caller's single value vector. + * The number of bytes that fit in the vec and were copied is returned. + * A null val returns 0. Items in log trees have a value header that + * needs to be skipped. + */ +static int copy_val(struct forest_lock_private *lpriv, struct forest_root *fr, + struct kvec *val, struct scoutfs_btree_item_ref *iref) +{ + void *val_start = iref->val; + unsigned int val_len = iref->val_len; + int ret; + + if (!is_fs_root(lpriv, fr)) { + val_start += sizeof(struct scoutfs_log_item_value); + val_len -= sizeof(struct scoutfs_log_item_value); + } + + if (val) { + ret = min_t(size_t, val_len, val->iov_len); + memcpy(val->iov_base, val_start, ret); + } else { + ret = 0; + } + + return ret; +} + +int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock) +{ + DECLARE_FOREST_INFO(sb, finf); + DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); + struct forest_lock_private *lpriv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key_be kbe; + struct forest_root *fr; + u64 found_vers; + u64 vers; + int ret; + int err; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) + goto out; + + lpriv = get_lock_private(lock); + if (!lpriv) { + ret = -ENOMEM; + goto out; + } + + scoutfs_key_to_be(&kbe, key); + +retry: + down_read(&lpriv->rwsem); + + found_vers = 0; + ret = -ENOENT; + fr = NULL; + + while (!(err = for_each_forest_root(lock, lpriv, &fr)) && fr) { + + /* done if we found log items before fs root */ + if (found_vers > 0 && is_fs_root(lpriv, fr)) + break; + + read_lock_forest_root(finf, lpriv, fr); + err = scoutfs_btree_lookup(sb, &fr->item_root, + &kbe, sizeof(kbe), &iref); + if (err < 0) + read_unlock_forest_root(finf, lpriv, fr); + if (err == -ENOENT) + continue; + if (err < 0) + break; + + vers = item_vers(lpriv, fr, iref.val); + + if (vers > found_vers) { + found_vers = vers; + + if (item_is_deletion(lpriv, fr, iref.val)) + ret = -ENOENT; + else + ret = copy_val(lpriv, fr, val, &iref); + } + scoutfs_btree_put_iref(&iref); + read_unlock_forest_root(finf, lpriv, fr); + + /* done if we have the most recent locked dirty version */ + if (is_our_dirty_item(lock, fr, vers)) + break; + } + + up_read(&lpriv->rwsem); + + if (err == -ESTALE) { + err = refresh_check_stale(sb, lock, &prev_srefs, &srefs); + if (err == 0) + goto retry; + ret = err; + } +out: + return ret; +} + +int scoutfs_forest_lookup_exact(struct super_block *sb, + struct scoutfs_key *key, struct kvec *val, + struct scoutfs_lock *lock) +{ + int ret; + + ret = scoutfs_forest_lookup(sb, key, val, lock); + if (ret == val->iov_len) + ret = 0; + else if (ret >= 0) + ret = -EIO; + + return ret; +} + +static inline void forest_iter_set_max(struct scoutfs_key *key, bool forward) +{ + if (forward) + scoutfs_key_set_ones(key); + else + scoutfs_key_set_zeros(key); +} + +static inline void forest_iter_set_min(struct scoutfs_key *key, bool forward) +{ + return forest_iter_set_max(key, !forward); +} + +static inline void forest_iter_key_advance(struct scoutfs_key *key, bool forward) +{ + if (forward) + scoutfs_key_inc(key); + else + scoutfs_key_dec(key); +} + +/* returns true if a is before b in the direction of iteration */ +static inline bool forest_iter_key_before(struct scoutfs_key *a, + struct scoutfs_key *b, bool forward) +{ + int cmp = scoutfs_key_compare(a, b); + + return forward ? cmp < 0 : cmp > 0; +} + +/* returns true if a is before or equal to b in the direction of iteration */ +static inline bool forest_iter_key_within(struct scoutfs_key *a, + struct scoutfs_key *b, bool forward) +{ + int cmp = scoutfs_key_compare(a, b); + + return forward ? cmp <= 0 : cmp >= 0; +} + +static inline int forest_iter_btree_search(struct super_block *sb, + struct scoutfs_btree_root *root, + void *key, unsigned key_len, + struct scoutfs_btree_item_ref *iref, + bool forward) +{ + if (forward) + return scoutfs_btree_next(sb, root, key, key_len, iref); + else + return scoutfs_btree_prev(sb, root, key, key_len, iref); +} + +struct forest_iter_pos { + struct list_head entry; + struct forest_root *fr; + struct scoutfs_key pos; +}; + +/* + * Iterate over items in all the roots looking for the next least + * non-deletion item in the direction of iteration. The roots can have + * any mix of deletion items and item versions. As we iterate we record + * the non-deletion item we've seen with the earliest key and the + * greatest version of that specific key. We record iteration positions + * in all btrees and we know we've finished once we've found a possible + * item to return and all the btrees have checked up to that position. + */ +static int forest_iter(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *end, struct kvec *val, + struct scoutfs_lock *lock, bool fwd) +{ + DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); + struct forest_lock_private *lpriv; + DECLARE_FOREST_INFO(sb, finf); + SCOUTFS_BTREE_ITEM_REF(iref); + struct forest_iter_pos *tmp; + struct forest_iter_pos *ip; + struct scoutfs_key_be kbe; + struct scoutfs_key found; + struct forest_root *fr; + LIST_HEAD(list); + int found_copied; + u64 found_vers; + u64 vers; + int ret; + + scoutfs_key_set_zeros(&found); + found_copied = 0; + found_vers = 0; + ret = 0; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) + goto out; + + /* use the end key as the end key if it's closer to reduce compares */ + if (forest_iter_key_before(&lock->end, end, fwd)) + end = &lock->end; + + /* convenience to avoid searching if caller iterates past their end */ + if (!forest_iter_key_within(key, end, fwd)) { + ret = -ENOENT; + goto out; + } + + lpriv = get_lock_private(lock); + if (!lpriv) { + ret = -ENOMEM; + goto out; + } + +retry: + down_read(&lpriv->rwsem); + + /* track iteration position in each btree */ + fr = NULL; + while (!(ret = for_each_forest_root(lock, lpriv, &fr)) && fr) { + ip = kmalloc(sizeof(struct forest_iter_pos), GFP_NOFS); + if (!ip) { + ret = -ENOMEM; + goto unlock; + } + + ip->fr = fr; + forest_iter_set_min(&ip->pos, fwd); + list_add_tail(&ip->entry, &list); + } + if (ret < 0) + goto unlock; + + forest_iter_set_max(&found, fwd); + found_vers = 0; + found_copied = 0; + + /* check each tree until they've all searched up to found */ + while (!list_empty(&list)) { + list_for_each_entry_safe(ip, tmp, &list, entry) { + fr = ip->fr; + + /* remove once we can't contain any more items */ + if (!forest_iter_key_before(&ip->pos, &found, fwd) || + !forest_iter_key_within(&ip->pos, end, fwd)) { + list_del(&ip->entry); + kfree(ip); + continue; + } + + /* iter pos key is only set after searching */ + if (forest_iter_key_before(&ip->pos, key, fwd)) + scoutfs_key_to_be(&kbe, key); + else + scoutfs_key_to_be(&kbe, &ip->pos); + + read_lock_forest_root(finf, lpriv, fr); + ret = forest_iter_btree_search(sb, &fr->item_root, + &kbe, sizeof(kbe), + &iref, fwd); + if (ret < 0) + read_unlock_forest_root(finf, lpriv, fr); + if (ret == -ENOENT) { + forest_iter_set_max(&ip->pos, fwd); + continue; + } + if (ret < 0) + goto unlock; + + scoutfs_key_from_be(&ip->pos, iref.key); + vers = item_vers(lpriv, fr, iref.val); + + /* record next earliest item and copy to caller */ + if (!item_is_deletion(lpriv, fr, iref.val) && + forest_iter_key_within(&ip->pos, end, fwd) && + (forest_iter_key_before(&ip->pos, &found, fwd) || + (scoutfs_key_compare(&ip->pos, &found) == 0 && + vers > found_vers))) { + + found = ip->pos; + found_vers = vers; + found_copied = copy_val(lpriv, fr, val, &iref); + } + scoutfs_btree_put_iref(&iref); + read_unlock_forest_root(finf, lpriv, fr); + + forest_iter_key_advance(&ip->pos, fwd); + } + } + + ret = 0; +unlock: + up_read(&lpriv->rwsem); + + list_for_each_entry_safe(ip, tmp, &list, entry) { + list_del(&ip->entry); + kfree(ip); + } + + if (ret == -ESTALE) { + ret = refresh_check_stale(sb, lock, &prev_srefs, &srefs); + if (ret == 0) + goto retry; + } + +out: + if (ret == 0) { + /* _next/_prev interfaces modify caller's key :/ */ + if (found_vers > 0) { + *key = found; + ret = found_copied; + } else { + ret = -ENOENT; + } + } + + return ret; +} + +int scoutfs_forest_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, struct kvec *val, + struct scoutfs_lock *lock) +{ + return forest_iter(sb, key, last, val, lock, true); +} + +int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *first, struct kvec *val, + struct scoutfs_lock *lock) +{ + return forest_iter(sb, key, first, val, lock, false); +} + +/* + * This is an unlocked iteration across all the btrees to find a hint at + * the next key that the caller could read. It's used to find out what + * next key range to lock, presuming you're allowed to only see items + * that have been synced. We read the super every time to get the most + * recent trees. + * + * We don't bother skipping deletion or reservation items here. They're + * unlikely. The caller will iterate them over safely and call again to + * find the next hint after them. + * + * We're reading from stable persistent trees so we don't need to lock + * against writers, their writes are cow into free blocks. + */ +int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next) +{ + DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); + struct scoutfs_super_block super; + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key_be kbe; + struct scoutfs_key found; + bool have_next; + int ret; + +retry: + ret = scoutfs_read_super(sb, &super); + if (ret) + goto out; + + srefs.fs_ref = super.fs_root.ref; + srefs.logs_ref = super.logs_root.ref; + + memset(<k, 0, sizeof(ltk)); + have_next = false; + + for (;; be64_add_cpu(<k.nr, 1)) { + + ret = scoutfs_btree_next(sb, &super.logs_root, + <k, sizeof(ltk), &iref); + if (ret == -ENOENT) { + if (have_next) + ret = 0; + break; + } + if (ret == -ESTALE) + break; + if (ret < 0) + goto out; + + if (iref.key_len == sizeof(ltk) && + iref.val_len == sizeof(ltv)) { + memcpy(<k, iref.key, iref.key_len); + memcpy(<v, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + scoutfs_key_to_be(&kbe, key); + ret = scoutfs_btree_next(sb, <v.item_root, + &kbe, sizeof(kbe), &iref); + if (ret == -ENOENT) + continue; + if (ret == -ESTALE) + break; + if (ret < 0) + goto out; + + if (iref.key_len == sizeof(kbe)) + scoutfs_key_from_be(&found, iref.key); + else + ret = -EIO; + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + if (!have_next || scoutfs_key_compare(&found, next) < 0) { + have_next = true; + *next = found; + } + } + + if (ret == -ESTALE) { + if (memcmp(&prev_srefs, &srefs, sizeof(srefs)) == 0) + return -EIO; + prev_srefs = srefs; + goto retry; + } +out: + + return ret; +} + + +/* + * Make sure that the bloom bits for the lock's start value are all set + * in the bloom block. We record the bits being set in the lock so that + * we only dirty the bloom block once per lock acquisition per log + * btree. + * + * If all the bloom bits weren't set then our log btree won't have been + * found by the search for log btrees to read under the lock. The + * caller is about to insert an item into the log tree that future + * readers must find so we make sure that the log root is added to the + * lock's list of roots. + * + * This can be racing with itself and readers in any stages of checking + * the forest trees and bloom blocks. + */ +static int set_lock_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock) +{ + DECLARE_FOREST_INFO(sb, finf); + struct forest_lock_private *lpriv; + struct scoutfs_block *new_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_bloom_block *bb; + struct scoutfs_btree_ref *ref; + struct forest_bloom_nrs bloom; + u64 blkno; + int ret; + int err; + int i; + + lpriv = get_lock_private(lock); + if (!lpriv) { + ret = -ENOMEM; + goto out; + } + + if (test_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS)) { + ret = 0; + goto out; + } + + calc_bloom_nrs(&bloom, &lock->start); + + down_write(&finf->rwsem); + + ref = &finf->our_log.bloom_ref; + + if (ref->blkno) { + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto unlock; + } + bb = bl->data; + } + + if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { + + ret = scoutfs_balloc_alloc(sb, &finf->alloc, &finf->wri, + &blkno); + if (ret < 0) + goto unlock; + + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + blkno); + BUG_ON(err); /* could have dirtied */ + ret = PTR_ERR(new_bl); + goto unlock; + } + + if (bl) { + err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + le64_to_cpu(ref->blkno)); + BUG_ON(err); /* could have dirtied */ + memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); + } else { + memset(new_bl->data, 0, SCOUTFS_BLOCK_SIZE); + } + + scoutfs_block_writer_mark_dirty(sb, &finf->wri, new_bl); + + scoutfs_block_put(sb, bl); + bl = new_bl; + bb = bl->data; + new_bl = NULL; + + bb->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BLOOM); + bb->hdr.blkno = cpu_to_le64(blkno); + prandom_bytes(&bb->hdr.seq, sizeof(bb->hdr.seq)); + ref->blkno = bb->hdr.blkno; + ref->seq = bb->hdr.seq; + } + + for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { + if (!test_and_set_bit_le(bloom.nrs[i], bb->bits)) { + le64_add_cpu(&bb->total_set, 1); + } + } + + ret = 0; +unlock: + up_write(&finf->rwsem); + + if (ret == 0) { + down_write(&lpriv->rwsem); + add_our_log_root(finf, lpriv); + up_write(&lpriv->rwsem); + set_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS); + } + +out: + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * The btree code takes a single value buffer. When we're working with + * the log btrees we want to add a log item value metadata header. In + * the interest of expedience we're just allocating a new contiguous + * buffer that prepends the header. We could make the btree ops take + * vectored values or we could make all btree items have the metadata. + */ +static struct kvec *alloc_log_item_value(struct kvec *val, __u8 flags, + struct scoutfs_lock *lock) +{ + struct scoutfs_log_item_value *liv; + struct kvec *kv; + unsigned int val_len = val ? val->iov_len : 0; + + kv = kmalloc(sizeof(*kv) + sizeof(*liv) + val_len, GFP_NOFS); + if (kv) { + liv = (void *)kv + sizeof(*kv); + + kv->iov_base = liv; + kv->iov_len = sizeof(*liv) + val_len; + + liv->vers = cpu_to_le64(lock->write_version); + liv->flags = flags; + if (val) + memcpy(liv->data, val->iov_base, val->iov_len); + } + + return kv; +} + +/* + * Create a new dirty item. Can return -EEXIST if the item already + * exists or will just force createion the caller's item, overwriting + * any existing item. We can be overwriting an existing deletion item + * in our log root. + */ +static int forest_insert(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock, + bool check_eexist, bool check_enoent) +{ + DECLARE_FOREST_INFO(sb, finf); + struct scoutfs_key_be kbe; + struct kvec *iv = NULL; + int ret; + + if (check_eexist || check_enoent) { + ret = scoutfs_forest_lookup(sb, key, NULL, lock); + if (ret == 0 && check_eexist) { + ret = -EEXIST; + goto out; + } + if (ret == -ENOENT) { + if (check_enoent) + goto out; + ret = 0; + } + if (ret < 0) + goto out; + } + + ret = set_lock_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + iv = alloc_log_item_value(val, 0, lock); + if (iv == NULL) { + ret = -ENOMEM; + goto out; + } + + scoutfs_key_to_be(&kbe, key); + + down_write(&finf->rwsem); + ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + &finf->our_log.item_root, &kbe, sizeof(kbe), + iv->iov_base, iv->iov_len); + up_write(&finf->rwsem); + kfree(iv); +out: + return ret; +} + +/* + * Insert an item, returning -EEXIST if it already exists. + */ +int scoutfs_forest_create(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock) +{ + int ret; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) + return ret; + + return forest_insert(sb, key, val, lock, true, false); +} + +/* + * Insert an item, ignoring whether it exists or not. + */ +int scoutfs_forest_create_force(struct super_block *sb, + struct scoutfs_key *key, struct kvec *val, + struct scoutfs_lock *lock) +{ + int ret; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) + return ret; + + return forest_insert(sb, key, val, lock, false, false); +} + +/* + * Overwrite an existing item, possibly changing its value length, + * returning -ENOENT if it didn't already exist. + */ +int scoutfs_forest_update(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock) +{ + int ret; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) + return ret; + + return forest_insert(sb, key, val, lock, false, true); +} + +/* XXX not yet supported, idea is btree op that only uses dirty blocks */ +int scoutfs_forest_delete_dirty(struct super_block *sb, + struct scoutfs_key *key) +{ + BUG(); + return 0; +} + +static int forest_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock, bool check_enoent) +{ + DECLARE_FOREST_INFO(sb, finf); + struct scoutfs_log_item_value liv; + struct scoutfs_key_be kbe; + int ret; + + if (check_enoent) { + ret = scoutfs_forest_lookup(sb, key, NULL, lock); + if (ret < 0) + goto out; + } + + ret = set_lock_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + scoutfs_key_to_be(&kbe, key); + liv.vers = cpu_to_le64(lock->write_version); + liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; + + down_write(&finf->rwsem); + ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + &finf->our_log.item_root, + &kbe, sizeof(kbe), &liv, sizeof(liv)); + up_write(&finf->rwsem); +out: + return ret; +} + +/* + * Delete an item from the forest of btrees. This interface returns + * -ENOENT if the item doesn't exist (may already be deleted). We have + * to first read from the forest to see if it exists. If we get -ENOENT + * it might be because it exists in our log tree. We force our deletion + * item regardless of the current state of the item in our log tree. + */ +int scoutfs_forest_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + int ret; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) + return ret; + + return forest_delete(sb, key, lock, true); +} + +/* + * Like deletion, but we don't have to read the current item to return + * -ENOENT. We just force a deletion item. + */ +int scoutfs_forest_delete_force(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + int ret; + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) + return ret; + + return forest_delete(sb, key, lock, false); +} + +/* XXX not supported, just for initial demo */ +int scoutfs_forest_delete_save(struct super_block *sb, + struct scoutfs_key *key, + struct list_head *list, + struct scoutfs_lock *lock) +{ + int ret = scoutfs_forest_delete(sb, key, lock); + BUG_ON(ret != 0); + return ret; +} + +/* XXX not supported, just for initial demo */ +int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, + struct scoutfs_lock *lock) +{ + BUG(); + return 0; +} + +/* XXX not supported, just for initial demo */ +void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) +{ +} + + +/* + * This is called from transactions as a new transaction opens and is + * serialized with all writers. Get the tree roots that we'll need + * for the transaction. + */ +int scoutfs_forest_get_log_trees(struct super_block *sb) +{ + DECLARE_FOREST_INFO(sb, finf); + struct scoutfs_log_trees lt; + int ret; + + ret = scoutfs_client_get_log_trees(sb, <); + if (ret) + goto out; + + down_write(&finf->rwsem); + scoutfs_balloc_init(&finf->alloc, <.alloc_root, <.free_root); + scoutfs_block_writer_init(sb, &finf->wri); + finf->our_log = lt; + up_write(&finf->rwsem); + + ret = 0; +out: + return ret; +} + +bool scoutfs_forest_has_dirty(struct super_block *sb) +{ + DECLARE_FOREST_INFO(sb, finf); + + return scoutfs_block_writer_has_dirty(sb, &finf->wri); +} + +unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb) +{ + DECLARE_FOREST_INFO(sb, finf); + + return scoutfs_block_writer_dirty_bytes(sb, &finf->wri); +} + +int scoutfs_forest_write(struct super_block *sb) +{ + DECLARE_FOREST_INFO(sb, finf); + + return scoutfs_block_writer_write(sb, &finf->wri); +} + +/* + * This is called during transaction commit which excludes forest writer + * calls. The caller has already written all the dirty blocks that the + * forest roots reference. + */ +int scoutfs_forest_commit(struct super_block *sb) +{ + DECLARE_FOREST_INFO(sb, finf); + struct scoutfs_log_trees lt = { + .alloc_root = finf->alloc.alloc_root, + .free_root = finf->alloc.free_root, + .item_root = finf->our_log.item_root, + .bloom_ref = finf->our_log.bloom_ref, + .rid = finf->our_log.rid, + .nr = finf->our_log.nr, + }; + + return scoutfs_client_commit_log_trees(sb, <); +} + +int scoutfs_forest_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct forest_info *finf; + int ret; + + finf = kzalloc(sizeof(struct forest_info), GFP_KERNEL); + if (!finf) { + ret = -ENOMEM; + goto out; + } + + /* the finf fields will be setup as we open a transaction */ + init_rwsem(&finf->rwsem); + + sbi->forest_info = finf; + ret = 0; +out: + if (ret) + scoutfs_forest_destroy(sb); + + return 0; +} + +void scoutfs_forest_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct forest_info *finf = SCOUTFS_SB(sb)->forest_info; + + if (finf) { + scoutfs_block_writer_forget_all(sb, &finf->wri); + kfree(finf); + sbi->forest_info = NULL; + } +} diff --git a/kmod/src/forest.h b/kmod/src/forest.h new file mode 100644 index 00000000..1b1d5641 --- /dev/null +++ b/kmod/src/forest.h @@ -0,0 +1,51 @@ +#ifndef _SCOUTFS_FOREST_H_ +#define _SCOUTFS_FOREST_H_ + +int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock); +int scoutfs_forest_lookup_exact(struct super_block *sb, + struct scoutfs_key *key, struct kvec *val, + struct scoutfs_lock *lock); +int scoutfs_forest_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, struct kvec *val, + struct scoutfs_lock *lock); +int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *next); +int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *first, struct kvec *val, + struct scoutfs_lock *lock); +int scoutfs_forest_create(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock); +int scoutfs_forest_create_force(struct super_block *sb, + struct scoutfs_key *key, struct kvec *val, + struct scoutfs_lock *lock); +int scoutfs_forest_update(struct super_block *sb, struct scoutfs_key *key, + struct kvec *val, struct scoutfs_lock *lock); +int scoutfs_forest_delete_dirty(struct super_block *sb, + struct scoutfs_key *key); +int scoutfs_forest_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_forest_delete_force(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_forest_delete_save(struct super_block *sb, + struct scoutfs_key *key, + struct list_head *list, + struct scoutfs_lock *lock); +int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, + struct scoutfs_lock *lock); +void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); + +int scoutfs_forest_get_log_trees(struct super_block *sb); +bool scoutfs_forest_has_dirty(struct super_block *sb); +unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb); +int scoutfs_forest_write(struct super_block *sb); +int scoutfs_forest_commit(struct super_block *sb); + +void scoutfs_forest_clear_lock(struct super_block *sb, + struct scoutfs_lock *lock); + +int scoutfs_forest_setup(struct super_block *sb); +void scoutfs_forest_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 03afe418..834a1196 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -7,6 +7,7 @@ /* block header magic values, chosen at random */ #define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b #define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d +#define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 /* * The super block and btree blocks are fixed 4k. @@ -329,6 +330,61 @@ struct scoutfs_mounted_client_btree_val { #define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) +struct scoutfs_log_trees { + struct scoutfs_balloc_root alloc_root; + struct scoutfs_balloc_root free_root; + struct scoutfs_btree_root item_root; + struct scoutfs_btree_ref bloom_ref; + __le64 rid; + __le64 nr; +} __packed; + +struct scoutfs_log_trees_key { + __be64 rid; + __be64 nr; +} __packed; + +struct scoutfs_log_trees_val { + struct scoutfs_balloc_root alloc_root; + struct scoutfs_balloc_root free_root; + struct scoutfs_btree_root item_root; + struct scoutfs_btree_ref bloom_ref; +} __packed; + +struct scoutfs_log_item_value { + __le64 vers; + __u8 flags; + __u8 data[0]; +} __packed; + +/* + * FS items are limited by the max btree value length with the log item + * value header. + */ +#define SCOUTFS_MAX_VAL_SIZE \ + (SCOUTFS_BTREE_MAX_VAL_LEN - sizeof(struct scoutfs_log_item_value)) + +#define SCOUTFS_LOG_ITEM_FLAG_DELETION (1 << 0) + +struct scoutfs_bloom_block { + struct scoutfs_block_header hdr; + __le64 total_set; + __le64 bits[0]; +} __packed; + +/* + * Log trees include a tree of items that make up a fixed size bloom + * filter. Just a few megs worth of items lets us test for the presence + * of locks that cover billions of files with a .1% chance of false + * positives. The log trees should be finalized and merged long before + * the bloom filters fill up and start returning excessive false positives. + */ +#define SCOUTFS_FOREST_BLOOM_NRS 7 +#define SCOUTFS_FOREST_BLOOM_BITS \ + (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_bloom_block)) / \ + member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ + member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) \ + /* * The max number of links defines the max number of entries that we can * index in o(log n) and the static list head storage size in the @@ -496,6 +552,8 @@ struct scoutfs_super_block { __le64 core_balloc_cursor; __le64 free_blocks; __le64 alloc_cursor; + __le64 first_fs_blkno; + __le64 last_fs_blkno; struct scoutfs_btree_ring bring; __le64 next_seg_seq; __le64 next_compact_id; @@ -507,8 +565,8 @@ struct scoutfs_super_block { struct scoutfs_balloc_root core_balloc_alloc; struct scoutfs_balloc_root core_balloc_free; struct scoutfs_btree_root alloc_root; - struct scoutfs_manifest manifest; struct scoutfs_btree_root fs_root; + struct scoutfs_manifest manifest; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; @@ -617,8 +675,6 @@ enum { DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ SCOUTFS_XATTR_MAX_PART_SIZE); -#define SCOUTFS_MAX_VAL_SIZE SCOUTFS_XATTR_MAX_PART_SIZE - #define SCOUTFS_LOCK_INODE_GROUP_NR 1024 #define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) #define SCOUTFS_LOCK_SEQ_GROUP_MASK ((1ULL << 10) - 1) diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 8a665c10..f61bbe70 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -36,6 +36,9 @@ struct scoutfs_lock { unsigned int users[SCOUTFS_LOCK_NR_MODES]; struct scoutfs_tseq_entry tseq_entry; + + /* the forest btree code stores data per lock */ + struct forest_lock_private *forest_private; }; struct scoutfs_lock_coverage { diff --git a/kmod/src/super.h b/kmod/src/super.h index 80700ee6..4f9fcf24 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -27,6 +27,7 @@ struct sysfs_info; struct options_sb_info; struct net_info; struct block_info; +struct forest_info; struct scoutfs_sb_info { struct super_block *sb; @@ -50,6 +51,7 @@ struct scoutfs_sb_info { struct net_info *net_info; struct quorum_info *quorum_info; struct block_info *block_info; + struct forest_info *forest_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From 48448d392694805e24306d0ab290a69dafef6009 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 29 Sep 2019 22:19:57 -0700 Subject: [PATCH 759/920] scoutfs: convert fs callers to forest Convert fs callers to work with the btree forest calls instead of the lsm item cache calls. This is mostly a mechanical syntax conversion. The inode dirtying path does now update the item rather than simply dirtying it. Signed-off-by: Zach Brown --- kmod/src/data.c | 10 +++++----- kmod/src/dir.c | 38 +++++++++++++++++++------------------- kmod/src/inode.c | 36 ++++++++++++++++++------------------ kmod/src/ioctl.c | 14 ++++---------- kmod/src/super.c | 3 +++ kmod/src/xattr.c | 38 +++++++++++++++++++------------------- 6 files changed, 68 insertions(+), 71 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 4402c7aa..b2deb16b 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -32,7 +32,7 @@ #include "trans.h" #include "counters.h" #include "scoutfs_trace.h" -#include "item.h" +#include "forest.h" #include "ioctl.h" #include "client.h" #include "lock.h" @@ -218,19 +218,19 @@ static int data_extent_io(struct super_block *sb, int op, expected = val.iov_len; if (op == SEI_NEXT) - ret = scoutfs_item_next(sb, &key, &last, &val, lock); + ret = scoutfs_forest_next(sb, &key, &last, &val, lock); else - ret = scoutfs_item_prev(sb, &key, &first, &val, lock); + ret = scoutfs_forest_prev(sb, &key, &first, &val, lock); if (ret >= 0 && ret != expected) ret = -EIO; if (ret == expected) ret = init_extent_from_item(ext, &key, &fex); } else if (op == SEI_INSERT) { - ret = scoutfs_item_create(sb, &key, &val, lock); + ret = scoutfs_forest_create(sb, &key, &val, lock); } else if (op == SEI_DELETE) { - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_forest_delete(sb, &key, lock); } else { ret = WARN_ON_ONCE(-EINVAL); diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 41f2ab82..709fc40a 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -29,7 +29,7 @@ #include "trans.h" #include "xattr.h" #include "kvec.h" -#include "item.h" +#include "forest.h" #include "lock.h" #include "counters.h" #include "scoutfs_trace.h" @@ -253,7 +253,7 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); for (;;) { - ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); + ret = scoutfs_forest_next(sb, &key, &last_key, &val, lock); if (ret < 0) break; @@ -478,7 +478,7 @@ static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, init_dirent_key(&key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), kc_readdir_pos(file, ctx), 0); - ret = scoutfs_item_next(sb, &key, &last_key, &val, dir_lock); + ret = scoutfs_forest_next(sb, &key, &last_key, &val, dir_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -556,23 +556,23 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); kvec_init(&val, dent, dirent_bytes(name_len)); - ret = scoutfs_item_create(sb, &ent_key, &val, dir_lock); + ret = scoutfs_forest_create(sb, &ent_key, &val, dir_lock); if (ret) goto out; del_ent = true; - ret = scoutfs_item_create(sb, &rdir_key, &val, dir_lock); + ret = scoutfs_forest_create(sb, &rdir_key, &val, dir_lock); if (ret) goto out; del_rdir = true; - ret = scoutfs_item_create(sb, &lb_key, &val, inode_lock); + ret = scoutfs_forest_create(sb, &lb_key, &val, inode_lock); out: if (ret < 0) { if (del_ent) - scoutfs_item_delete_dirty(sb, &ent_key); + scoutfs_forest_delete_dirty(sb, &ent_key); if (del_rdir) - scoutfs_item_delete_dirty(sb, &rdir_key); + scoutfs_forest_delete_dirty(sb, &rdir_key); } kfree(dent); @@ -602,15 +602,15 @@ static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); - ret = scoutfs_item_delete_save(sb, &ent_key, &dir_saved, dir_lock) ?: - scoutfs_item_delete_save(sb, &rdir_key, &dir_saved, dir_lock) ?: - scoutfs_item_delete_save(sb, &lb_key, &inode_saved, inode_lock); + ret = scoutfs_forest_delete_save(sb, &ent_key, &dir_saved, dir_lock) ?: + scoutfs_forest_delete_save(sb, &rdir_key, &dir_saved, dir_lock) ?: + scoutfs_forest_delete_save(sb, &lb_key, &inode_saved, inode_lock); if (ret < 0) { - scoutfs_item_restore(sb, &dir_saved, dir_lock); - scoutfs_item_restore(sb, &inode_saved, inode_lock); + scoutfs_forest_restore(sb, &dir_saved, dir_lock); + scoutfs_forest_restore(sb, &inode_saved, inode_lock); } else { - scoutfs_item_free_batch(sb, &dir_saved); - scoutfs_item_free_batch(sb, &inode_saved); + scoutfs_forest_free_batch(sb, &dir_saved); + scoutfs_forest_free_batch(sb, &inode_saved); } return ret; @@ -988,11 +988,11 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, kvec_init(&val, (void *)target, bytes); if (op == SYM_CREATE) - ret = scoutfs_item_create(sb, &key, &val, lock); + ret = scoutfs_forest_create(sb, &key, &val, lock); else if (op == SYM_LOOKUP) - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); else if (op == SYM_DELETE) - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_forest_delete(sb, &key, lock); if (ret) break; @@ -1229,7 +1229,7 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, if (ret) goto out; - ret = scoutfs_item_next(sb, &key, &last_key, &val, lock); + ret = scoutfs_forest_next(sb, &key, &last_key, &val, lock); scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); lock = NULL; if (ret < 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 6fb70014..32ed084e 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -31,7 +31,7 @@ #include "trans.h" #include "msg.h" #include "kvec.h" -#include "item.h" +#include "forest.h" #include "client.h" #include "cmp.h" @@ -296,7 +296,7 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, mutex_lock(&si->item_mutex); if (atomic64_read(&si->last_refreshed) < refresh_gen) { - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); @@ -741,9 +741,7 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) * have to update it now with the current inode contents. * * Callers don't delete these dirty items on errors. They're still - * valid and will be merged with the current item eventually. They can - * be found in the dirty block to avoid future dirtying (say repeated - * creations in a directory). + * valid and will be merged with the current item eventually. * * The caller has to prevent sync between dirtying and updating the * inodes. @@ -753,15 +751,17 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_key key; struct scoutfs_inode sinode; + struct scoutfs_key key; + struct kvec val; int ret; store_inode(&sinode, inode); + kvec_init(&val, &sinode, sizeof(sinode)); init_inode_key(&key, scoutfs_ino(inode)); - ret = scoutfs_item_dirty(sb, &key, lock); + ret = scoutfs_forest_update(sb, &key, &val, lock); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -893,7 +893,7 @@ static int update_index_items(struct super_block *sb, scoutfs_inode_init_index_key(&ins, type, major, minor, ino); ins_lock = find_index_lock(lock_list, type, major, minor, ino); - ret = scoutfs_item_create_force(sb, &ins, NULL, ins_lock); + ret = scoutfs_forest_create_force(sb, &ins, NULL, ins_lock); if (ret || !will_del_index(si, type, major, minor)) return ret; @@ -905,9 +905,9 @@ static int update_index_items(struct super_block *sb, del_lock = find_index_lock(lock_list, type, si->item_majors[type], si->item_minors[type], ino); - ret = scoutfs_item_delete_force(sb, &del, del_lock); + ret = scoutfs_forest_delete_force(sb, &del, del_lock); if (ret) { - err = scoutfs_item_delete(sb, &ins, ins_lock); + err = scoutfs_forest_delete(sb, &ins, ins_lock); BUG_ON(err); } @@ -984,7 +984,7 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, init_inode_key(&key, ino); kvec_init(&val, &sinode, sizeof(sinode)); - err = scoutfs_item_update(sb, &key, &val, lock); + err = scoutfs_forest_update(sb, &key, &val, lock); if (err) { scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); @@ -1259,7 +1259,7 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, scoutfs_inode_init_index_key(&key, type, major, minor, ino); lock = find_index_lock(ind_locks, type, major, minor, ino); - ret = scoutfs_item_delete_force(sb, &key, lock); + ret = scoutfs_forest_delete_force(sb, &key, lock); if (ret == -ENOENT) ret = 0; return ret; @@ -1401,7 +1401,7 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, init_inode_key(&key, scoutfs_ino(inode)); kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_item_create(sb, &key, &val, lock); + ret = scoutfs_forest_create(sb, &key, &val, lock); if (ret) { iput(inode); return ERR_PTR(ret); @@ -1429,7 +1429,7 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) init_orphan_key(&key, sbi->rid, ino); - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_forest_delete(sb, &key, lock); if (ret == -ENOENT) ret = 0; @@ -1464,7 +1464,7 @@ static int delete_inode_items(struct super_block *sb, u64 ino) init_inode_key(&key, ino); kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_item_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1517,7 +1517,7 @@ retry: goto out; } - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_forest_delete(sb, &key, lock); if (ret) goto out; @@ -1586,7 +1586,7 @@ int scoutfs_scan_orphans(struct super_block *sb) init_orphan_key(&last, sbi->rid, ~0ULL); while (1) { - ret = scoutfs_item_next(sb, &key, &last, NULL, lock); + ret = scoutfs_forest_next(sb, &key, &last, NULL, lock); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) @@ -1620,7 +1620,7 @@ int scoutfs_orphan_inode(struct inode *inode) init_orphan_key(&key, sbi->rid, scoutfs_ino(inode)); - ret = scoutfs_item_create(sb, &key, NULL, lock); + ret = scoutfs_forest_create(sb, &key, NULL, lock); return ret; } diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index c1d64aa0..4c271816 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -27,6 +27,7 @@ #include "ioctl.h" #include "super.h" #include "inode.h" +#include "forest.h" #include "item.h" #include "data.h" #include "client.h" @@ -110,7 +111,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) for (nr = 0; nr < walk.nr_entries; ) { - ret = scoutfs_item_next(sb, &key, &last_key, NULL, lock); + ret = scoutfs_forest_next(sb, &key, &last_key, NULL, lock); if (ret < 0 && ret != -ENOENT) break; @@ -128,14 +129,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); - /* - * XXX This will miss dirty items. We'd need to - * force writeouts of dirty items in our - * zone|type and get the manifest root for that. - * It'd mean adding a lock to the inode index - * items which isn't quite there yet. - */ - ret = scoutfs_manifest_next_key(sb, &key, &next_key); + ret = scoutfs_forest_next_hint(sb, &key, &next_key); if (ret < 0 && ret != -ENOENT) goto out; @@ -816,7 +810,7 @@ static long scoutfs_ioc_find_xattrs(struct file *file, unsigned long arg) while (fx.nr_inodes) { - ret = scoutfs_item_next(sb, &key, &last, NULL, lock); + ret = scoutfs_forest_next(sb, &key, &last, NULL, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; diff --git a/kmod/src/super.c b/kmod/src/super.c index 5237c4b1..b3b5370d 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -45,6 +45,7 @@ #include "options.h" #include "sysfs.h" #include "quorum.h" +#include "forest.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -189,6 +190,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); scoutfs_inode_destroy(sb); + scoutfs_forest_destroy(sb); /* the server locks the listen address and compacts */ scoutfs_lock_shutdown(sb); @@ -423,6 +425,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_seg_setup(sb) ?: scoutfs_item_setup(sb) ?: scoutfs_block_setup(sb) ?: + scoutfs_forest_setup(sb) ?: scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 7196114d..da109367 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -21,7 +21,7 @@ #include "key.h" #include "super.h" #include "kvec.h" -#include "item.h" +#include "forest.h" #include "trans.h" #include "xattr.h" #include "lock.h" @@ -195,7 +195,7 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key, for (;;) { key->skx_part = part; kvec_init(&val, (void *)xat + total, bytes - total); - ret = scoutfs_item_next(sb, key, &last, &val, lock); + ret = scoutfs_forest_next(sb, key, &last, &val, lock); if (ret < 0) { /* XXX corruption, ran out of parts */ if (ret == -ENOENT && part > 0) @@ -283,10 +283,10 @@ static int create_xattr_items(struct inode *inode, u64 id, part_bytes = min(bytes - total, SCOUTFS_XATTR_MAX_PART_SIZE); kvec_init(&val, (void *)xat + total, part_bytes); - ret = scoutfs_item_create(sb, &key, &val, lock); + ret = scoutfs_forest_create(sb, &key, &val, lock); if (ret) { while (key.skx_part-- > 0) - scoutfs_item_delete_dirty(sb, &key); + scoutfs_forest_delete_dirty(sb, &key); break; } @@ -313,7 +313,7 @@ static int delete_xattr_items(struct inode *inode, u32 name_hash, u64 id, init_xattr_key(&key, scoutfs_ino(inode), name_hash, id); do { - ret = scoutfs_item_delete_save(sb, &key, list, lock); + ret = scoutfs_forest_delete_save(sb, &key, list, lock); } while (ret == 0 && ++key.skx_part < nr_parts); return ret; @@ -528,11 +528,11 @@ retry: hash = scoutfs_hash64(name, name_len); scoutfs_xattr_index_key(&indx_key, hash, ino, id); if (value) - ret = scoutfs_item_create_force(sb, &indx_key, NULL, - indx_lock); + ret = scoutfs_forest_create_force(sb, &indx_key, NULL, + indx_lock); else - ret = scoutfs_item_delete_force(sb, &indx_key, - indx_lock); + ret = scoutfs_forest_delete_force(sb, &indx_key, + indx_lock); if (ret < 0) goto release; undo_indx = true; @@ -546,10 +546,10 @@ retry: if (value && ret == 0) ret = create_xattr_items(inode, id, xat, bytes, lck); if (ret < 0) { - scoutfs_item_restore(sb, &saved, lck); + scoutfs_forest_restore(sb, &saved, lck); goto release; } - scoutfs_item_free_batch(sb, &saved); + scoutfs_forest_free_batch(sb, &saved); /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); @@ -560,11 +560,11 @@ retry: release: if (ret < 0 && undo_indx) { if (value) - err = scoutfs_item_delete_force(sb, &indx_key, - indx_lock); + err = scoutfs_forest_delete_force(sb, &indx_key, + indx_lock); else - err = scoutfs_item_create_force(sb, &indx_key, NULL, - indx_lock); + err = scoutfs_forest_create_force(sb, &indx_key, NULL, + indx_lock); BUG_ON(err); } @@ -717,7 +717,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, for (;;) { kvec_init(&val, (void *)xat, bytes); - ret = scoutfs_item_next(sb, &key, &last, &val, lock); + ret = scoutfs_forest_next(sb, &key, &last, &val, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -744,13 +744,13 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, break; release = true; - ret = scoutfs_item_delete(sb, &key, lock); + ret = scoutfs_forest_delete(sb, &key, lock); if (ret < 0) break; if (tgs.indx) { - ret = scoutfs_item_delete_force(sb, &indx_key, - indx_lock); + ret = scoutfs_forest_delete_force(sb, &indx_key, + indx_lock); if (ret < 0) break; } From 58f062a2c168ec51c02c7a4a69dcf910e3273a83 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 08:20:44 -0700 Subject: [PATCH 760/920] scoutfs: use forest in locking and transaction Transaction commit now has to ask the forest to write the btrees during a transaction commit instead of writing dirty items in segments. It also determines if holds fit in the dirty transaction by looking at dirty btree blocks instead of item counts. Locking no longer has to invalidate a private item cache because the forest paths use the btree block cache where inconsistency is discovered and invalidated as blocks are read. Signed-off-by: Zach Brown --- kmod/src/counters.h | 5 +--- kmod/src/lock.c | 64 ++++------------------------------------ kmod/src/scoutfs_trace.h | 6 ++-- kmod/src/super.c | 3 +- kmod/src/trans.c | 41 ++++++++----------------- 5 files changed, 23 insertions(+), 96 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 22bf8645..ef808252 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -100,7 +100,7 @@ EXPAND_COUNTER(lock_grace_wait) \ EXPAND_COUNTER(lock_grant_request) \ EXPAND_COUNTER(lock_grant_response) \ - EXPAND_COUNTER(lock_invalidate_clean_item) \ + EXPAND_COUNTER(lock_invalidate_commit) \ EXPAND_COUNTER(lock_invalidate_coverage) \ EXPAND_COUNTER(lock_invalidate_inode) \ EXPAND_COUNTER(lock_invalidate_request) \ @@ -113,7 +113,6 @@ EXPAND_COUNTER(lock_shrink_request_aborted) \ EXPAND_COUNTER(lock_unlock) \ EXPAND_COUNTER(lock_wait) \ - EXPAND_COUNTER(lock_write_dirty_item) \ EXPAND_COUNTER(manifest_compact_migrate) \ EXPAND_COUNTER(manifest_hard_stale_error) \ EXPAND_COUNTER(manifest_read_excluded_key) \ @@ -157,8 +156,6 @@ EXPAND_COUNTER(trans_commit_item_flush) \ EXPAND_COUNTER(trans_commit_sync_fs) \ EXPAND_COUNTER(trans_commit_timer) \ - EXPAND_COUNTER(trans_level0_seg_write_bytes) \ - EXPAND_COUNTER(trans_level0_seg_writes) \ EXPAND_COUNTER(trans_write_item) \ EXPAND_COUNTER(trans_write_deletion_item) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index aa3d6b04..6397d573 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -21,7 +21,7 @@ #include "super.h" #include "lock.h" -#include "item.h" +#include "forest.h" #include "scoutfs_trace.h" #include "msg.h" #include "cmp.h" @@ -145,12 +145,10 @@ static void invalidate_inode(struct super_block *sb, u64 ino) static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, int prev, int mode) { - struct scoutfs_key *start = &lock->start; - struct scoutfs_key *end = &lock->end; struct scoutfs_lock_coverage *cov; struct scoutfs_lock_coverage *tmp; u64 ino, last; - int ret; + int ret = 0; trace_scoutfs_lock_invalidate(sb, lock); @@ -159,12 +157,12 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, mode != SCOUTFS_LOCK_NULL); /* any transition from a mode allowed to dirty items has to write */ - if (lock_mode_can_write(prev)) { - ret = scoutfs_item_writeback(sb, start, end); + if (lock_mode_can_write(prev) && scoutfs_forest_has_dirty(sb)) { + ret = scoutfs_trans_sync(sb, 1); if (ret < 0) return ret; if (ret > 0) { - scoutfs_add_counter(sb, lock_write_dirty_item, ret); + scoutfs_add_counter(sb, lock_invalidate_commit, ret); ret = 0; } } @@ -195,13 +193,6 @@ retry: ino++; } } - - ret = scoutfs_item_invalidate(sb, start, end); - if (ret > 0) { - scoutfs_add_counter(sb, lock_invalidate_clean_item, - ret); - ret = 0; - } } return ret; @@ -548,49 +539,6 @@ static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) lock->grace_deadline = ktime_add(now, GRACE_PERIOD_KT); } -/* - * The given lock is processing a received a grant response. Trigger a - * bug if the cache is inconsistent. - * - * We only have two modes that can create dirty items. We can't have - * dirty items when transitioning from write_only to write because the - * writer can't trust the cached items in the cache for reading. And we - * don't currently transition directly from write to write_only, we - * first go through null. So if we have dirty items as we're granted a - * mode it's always incorrect. - * - * And we can't have cached items that we're going to use for reading if - * the previous mode didn't allow reading. - * - * Inconsistencies have come from all sorts of bugs: invalidation missed - * items, the cache was populated outside of locking coverage, lock - * holders performed the wrong item operations under their lock, - * overlapping locks, out of order granting or invalidating, etc. - */ -static void bug_on_inconsistent_grant_cache(struct super_block *sb, - struct scoutfs_lock *lock, - int old_mode, int new_mode) -{ - bool cached = scoutfs_item_range_cached(sb, &lock->start, &lock->end, - false); - bool dirty = scoutfs_item_range_cached(sb, &lock->start, &lock->end, - true); - - if (dirty || - (cached && (!lock_mode_can_read(old_mode) || !lock_mode_can_read(new_mode)))) { - scoutfs_err(sb, "granted lock item cache inconsistency, cached %u dirty %u old_mode %d new_mode %d: start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %u waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u", - cached, dirty, old_mode, new_mode, SK_ARG(&lock->start), - SK_ARG(&lock->end), lock->refresh_gen, lock->mode, - lock->waiters[SCOUTFS_LOCK_READ], - lock->waiters[SCOUTFS_LOCK_WRITE], - lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], - lock->users[SCOUTFS_LOCK_READ], - lock->users[SCOUTFS_LOCK_WRITE], - lock->users[SCOUTFS_LOCK_WRITE_ONLY]); - BUG(); - } -} - /* * The client is receiving a lock response message from the server. * This can be reordered with incoming invlidation requests from the @@ -631,8 +579,6 @@ int scoutfs_lock_grant_response(struct super_block *sb, spin_lock(&linfo->lock); } - bug_on_inconsistent_grant_cache(sb, lock, nl->old_mode, nl->new_mode); - if (!lock_mode_can_read(nl->old_mode) && lock_mode_can_read(nl->new_mode)) { lock->refresh_gen = diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 4c2f9a4d..bcdfaf5c 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -570,13 +570,13 @@ TRACE_EVENT(scoutfs_sync_fs, ); TRACE_EVENT(scoutfs_trans_write_func, - TP_PROTO(struct super_block *sb, int dirty), + TP_PROTO(struct super_block *sb, unsigned long dirty), TP_ARGS(sb, dirty), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(int, dirty) + __field(unsigned long, dirty) ), TP_fast_assign( @@ -584,7 +584,7 @@ TRACE_EVENT(scoutfs_trans_write_func, __entry->dirty = dirty; ), - TP_printk(SCSBF" dirty %d", SCSB_TRACE_ARGS, __entry->dirty) + TP_printk(SCSBF" dirty %lu", SCSB_TRACE_ARGS, __entry->dirty) ); TRACE_EVENT(scoutfs_release_trans, diff --git a/kmod/src/super.c b/kmod/src/super.c index b3b5370d..34f9f8df 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -435,7 +435,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_server_setup(sb) ?: scoutfs_client_setup(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, - &sbi->rid_lock); + &sbi->rid_lock) ?: + scoutfs_forest_get_log_trees(sb); if (ret) goto out; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 710fa6cd..aa7930d6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -23,6 +23,7 @@ #include "data.h" #include "bio.h" #include "item.h" +#include "forest.h" #include "manifest.h" #include "seg.h" #include "counters.h" @@ -110,44 +111,27 @@ void scoutfs_trans_write_func(struct work_struct *work) trans_write_work.work); struct super_block *sb = sbi->sb; DECLARE_TRANS_INFO(sb, tri); - struct scoutfs_bio_completion comp; - struct scoutfs_segment *seg = NULL; - u64 segno; int ret = 0; - scoutfs_bio_init_comp(&comp); sbi->trans_task = current; wait_event(sbi->trans_hold_wq, drained_holders(tri)); - trace_scoutfs_trans_write_func(sb, scoutfs_item_has_dirty(sb)); + trace_scoutfs_trans_write_func(sb, scoutfs_forest_dirty_bytes(sb)); - - if (scoutfs_item_has_dirty(sb)) { + if (scoutfs_forest_has_dirty(sb)) { if (sbi->trans_deadline_expired) scoutfs_inc_counter(sb, trans_commit_timer); - /* - * XXX only straight pass through, we're not worrying - * about leaking segnos nor duplicate manifest entries - * on crashes between us and the server. - */ + ret = scoutfs_inode_walk_writeback(sb, true) ?: - scoutfs_client_alloc_segno(sb, &segno) ?: - scoutfs_seg_alloc(sb, segno, &seg) ?: - scoutfs_item_dirty_seg(sb, seg) ?: - scoutfs_seg_submit_write(sb, seg, &comp) ?: + scoutfs_forest_write(sb) ?: scoutfs_inode_walk_writeback(sb, false) ?: - scoutfs_bio_wait_comp(sb, &comp) ?: - scoutfs_client_record_segment(sb, seg, 0) ?: - scoutfs_client_advance_seq(sb, &sbi->trans_seq); - scoutfs_seg_put(seg); + scoutfs_forest_commit(sb) ?: + scoutfs_client_advance_seq(sb, &sbi->trans_seq) ?: + scoutfs_forest_get_log_trees(sb); if (ret) goto out; - scoutfs_inc_counter(sb, trans_level0_seg_writes); - scoutfs_add_counter(sb, trans_level0_seg_write_bytes, - scoutfs_seg_total_bytes(seg)); - } else if (sbi->trans_deadline_expired) { /* * If we're not writing data then we only advance the @@ -295,7 +279,6 @@ static bool acquired_hold(struct super_block *sb, bool acquired = false; unsigned items; unsigned vals; - bool fits; spin_lock(&tri->lock); @@ -316,8 +299,9 @@ static bool acquired_hold(struct super_block *sb, /* see if we can reserve space for our item count */ items = tri->reserved_items + cnt->items; vals = tri->reserved_vals + cnt->vals; - fits = scoutfs_item_dirty_fits_single(sb, items, vals); - if (!fits) { + + /* XXX just limit to 256K transactions */ + if (scoutfs_forest_dirty_bytes(sb) >= (256 * 1024)) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; @@ -352,8 +336,7 @@ int scoutfs_hold_trans(struct super_block *sb, * Caller shouldn't provide garbage counts, nor counts that * can't fit in segments by themselves. */ - if (WARN_ON_ONCE(cnt.items <= 0 || cnt.vals < 0) || - WARN_ON_ONCE(!scoutfs_seg_fits_single(cnt.items, cnt.vals))) + if (WARN_ON_ONCE(cnt.items <= 0 || cnt.vals < 0)) return -EINVAL; if (current == sbi->trans_task) From 43f451d015a890bd04cbd5bd183c3d1844e5dae6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 09:55:04 -0700 Subject: [PATCH 761/920] scoutfs: read and write super with buffer_head Use simple buffer_heads to read and write the super. After getting rid of the lsm code this would be the last user of our bio helpers. With this converted we can remove the bio helpers along with the rest of the lsm code. Signed-off-by: Zach Brown --- kmod/src/super.c | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 34f9f8df..12f7c199 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "super.h" #include "block.h" @@ -237,23 +238,32 @@ int scoutfs_write_super(struct super_block *sb, struct scoutfs_super_block *caller) { struct scoutfs_super_block *super; - struct page *page; + struct buffer_head *bh; int ret; - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) + bh = sb_getblk(sb, SCOUTFS_SUPER_BLKNO); + if (!bh) return -ENOMEM; le64_add_cpu(&caller->hdr.seq, 1); - super = page_address(page); + memset(bh->b_data, 0, bh->b_size); + super = (void *)bh->b_data; memcpy(super, caller, sizeof(*super)); super->hdr.crc = scoutfs_block_calc_crc(&super->hdr); - ret = scoutfs_bio_write(sb, &page, le64_to_cpu(super->hdr.blkno), 1); - WARN_ON_ONCE(ret); + lock_buffer(bh); + set_buffer_mapped(bh); + set_buffer_dirty(bh); + unlock_buffer(bh); - __free_page(page); + ll_rw_block(WRITE, 1, &bh); + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + else + ret = 0; + brelse(bh); return ret; } @@ -266,21 +276,25 @@ int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super_res) { struct scoutfs_super_block *super; - struct page *page; + struct buffer_head *bh = NULL; __le32 calc; int ret; - page = alloc_page(GFP_KERNEL); - if (!page) - return -ENOMEM; - - ret = scoutfs_bio_read(sb, &page, SCOUTFS_SUPER_BLKNO, 1); - if (ret) { + bh = sb_getblk(sb, SCOUTFS_SUPER_BLKNO); + if (bh) { + lock_buffer(bh); + clear_buffer_uptodate(bh); + unlock_buffer(bh); + brelse(bh); + } + bh = sb_bread(sb, SCOUTFS_SUPER_BLKNO); + if (!bh) { + ret = -EIO; scoutfs_err(sb, "error reading super block: %d", ret); goto out; } - super = scoutfs_page_block_address(&page, 0); + super = (void *)(bh->b_data); if (super->hdr.magic != cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER)) { scoutfs_err(sb, "super block has invalid magic value 0x%08x", @@ -325,7 +339,7 @@ int scoutfs_read_super(struct super_block *sb, *super_res = *super; ret = 0; out: - __free_page(page); + brelse(bh); return ret; } From edd8fe075cf3917e3014cade96b0642ee3ac5b89 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 09:56:27 -0700 Subject: [PATCH 762/920] scoutfs: remove lsm code Remove all the now unused code that deals with lsm: segment IO, the item cache, and the manifest. Signed-off-by: Zach Brown --- kmod/src/Makefile | 5 - kmod/src/bio.c | 223 ---- kmod/src/bio.h | 43 - kmod/src/client.c | 102 -- kmod/src/client.h | 7 - kmod/src/compact.c | 683 ----------- kmod/src/compact.h | 8 - kmod/src/counters.h | 48 +- kmod/src/file.c | 1 - kmod/src/format.h | 164 --- kmod/src/ioctl.c | 59 - kmod/src/ioctl.h | 32 +- kmod/src/item.c | 2498 -------------------------------------- kmod/src/item.h | 75 -- kmod/src/manifest.c | 1297 -------------------- kmod/src/manifest.h | 52 - kmod/src/net.c | 3 - kmod/src/scoutfs_trace.h | 866 ------------- kmod/src/seg.c | 868 ------------- kmod/src/seg.h | 51 - kmod/src/server.c | 891 +------------- kmod/src/server.h | 8 - kmod/src/super.c | 9 - kmod/src/super.h | 8 - kmod/src/trans.c | 4 - 25 files changed, 10 insertions(+), 7995 deletions(-) delete mode 100644 kmod/src/bio.c delete mode 100644 kmod/src/bio.h delete mode 100644 kmod/src/compact.c delete mode 100644 kmod/src/compact.h delete mode 100644 kmod/src/item.c delete mode 100644 kmod/src/item.h delete mode 100644 kmod/src/manifest.c delete mode 100644 kmod/src/manifest.h delete mode 100644 kmod/src/seg.c delete mode 100644 kmod/src/seg.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f90c22e3..8c9d0fdb 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -9,12 +9,10 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -include $(src)/Makefile.kernelcompat scoutfs-y += \ - bio.o \ balloc.o \ block.o \ btree.o \ client.o \ - compact.o \ counters.o \ data.o \ dir.o \ @@ -24,17 +22,14 @@ scoutfs-y += \ forest.o \ inode.o \ ioctl.o \ - item.o \ lock.o \ lock_server.o \ - manifest.o \ msg.o \ net.o \ options.o \ per_task.o \ quorum.o \ scoutfs_trace.o \ - seg.o \ server.o \ spbm.o \ super.o \ diff --git a/kmod/src/bio.c b/kmod/src/bio.c deleted file mode 100644 index f47ed7b2..00000000 --- a/kmod/src/bio.c +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "bio.h" -#include "scoutfs_trace.h" - -struct bio_end_io_args { - struct super_block *sb; - atomic_t in_flight; - int err; - scoutfs_bio_end_io_t end_io; - void *data; -}; - -static void dec_end_io(struct bio_end_io_args *args, int err) -{ - if (err && !args->err) - args->err = err; - - trace_scoutfs_dec_end_io(args->sb, args, atomic_read(&args->in_flight), - err); - - if (atomic_dec_and_test(&args->in_flight)) { - args->end_io(args->sb, args->data, args->err); - kfree(args); - } -} - -static void bio_end_io(struct bio *bio, int err) -{ - struct bio_end_io_args *args = bio->bi_private; - - trace_scoutfs_bio_end_io(args->sb, bio, bio->bi_size, err); - - dec_end_io(args, err); - bio_put(bio); -} - -/* - * Read or write the given number of 4k blocks from the front of the - * pages provided by the caller. We translate the block count into a - * page count and fill bios a page at a time. - * - * The caller is responsible for ensuring that the pages aren't freed - * while bios are in flight. - * - * The end_io function is always called once with the error result of - * the IO. It can be called before _submit returns. - */ -void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, - u64 blkno, unsigned int nr_blocks, - scoutfs_bio_end_io_t end_io, void *data) -{ - unsigned int nr_pages = DIV_ROUND_UP(nr_blocks, - SCOUTFS_BLOCKS_PER_PAGE); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct bio_end_io_args *args; - struct blk_plug plug; - unsigned int bytes; - struct page *page; - struct bio *bio = NULL; - int ret = 0; - int i; - - if (super->total_blocks && - WARN_ON_ONCE(blkno >= le64_to_cpu(super->total_blocks))) { - end_io(sb, data, -EIO); - return; - } - - args = kmalloc(sizeof(struct bio_end_io_args), GFP_NOFS); - if (!args) { - end_io(sb, data, -ENOMEM); - return; - } - - args->sb = sb; - atomic_set(&args->in_flight, 1); - args->err = 0; - args->end_io = end_io; - args->data = data; - - blk_start_plug(&plug); - - for (i = 0; i < nr_pages; i++) { - page = pages[i]; - - if (!bio) { - bio = bio_alloc(GFP_NOFS, nr_pages - i); - if (!bio) - bio = bio_alloc(GFP_NOFS, 1); - if (!bio) { - ret = -ENOMEM; - break; - } - - bio->bi_sector = blkno << (SCOUTFS_BLOCK_SHIFT - 9); - bio->bi_bdev = sb->s_bdev; - bio->bi_end_io = bio_end_io; - bio->bi_private = args; - } - - bytes = min_t(int, nr_blocks << SCOUTFS_BLOCK_SHIFT, PAGE_SIZE); - - if (bio_add_page(bio, page, bytes, 0) != bytes) { - /* submit the full bio and retry this page */ - atomic_inc(&args->in_flight); - trace_scoutfs_bio_submit(sb, bio, args, - atomic_read(&args->in_flight)); - submit_bio(rw, bio); - bio = NULL; - i--; - continue; - } - - blkno += SCOUTFS_BLOCKS_PER_PAGE; - nr_blocks -= SCOUTFS_BLOCKS_PER_PAGE; - } - - if (bio) { - atomic_inc(&args->in_flight); - trace_scoutfs_bio_submit_partial(sb, bio, args, - atomic_read(&args->in_flight)); - submit_bio(rw, bio); - } - - blk_finish_plug(&plug); - dec_end_io(args, ret); -} - -void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp) -{ - /* this initial pending is dropped by wait */ - atomic_set(&comp->pending, 1); - init_completion(&comp->comp); - comp->err = 0; - trace_scoutfs_bio_init_comp(comp); -} - -static void comp_end_io(struct super_block *sb, void *data, int err) -{ - struct scoutfs_bio_completion *comp = data; - - if (err && !comp->err) - comp->err = err; - - trace_comp_end_io(sb, comp); - - if (atomic_dec_and_test(&comp->pending)) - complete(&comp->comp); -} - -void scoutfs_bio_submit_comp(struct super_block *sb, int rw, - struct page **pages, u64 blkno, - unsigned int nr_blocks, - struct scoutfs_bio_completion *comp) -{ - atomic_inc(&comp->pending); - trace_scoutfs_bio_submit_comp(sb, comp); - - scoutfs_bio_submit(sb, rw, pages, blkno, nr_blocks, comp_end_io, comp); -} - -int scoutfs_bio_wait_comp(struct super_block *sb, - struct scoutfs_bio_completion *comp) -{ - comp_end_io(sb, comp, 0); - trace_scoutfs_bio_wait_comp(sb, comp); - wait_for_completion(&comp->comp); - return comp->err; -} - -/* - * A synchronous read of the given blocks. - * - * XXX we could make this interruptible. - */ -int scoutfs_bio_read(struct super_block *sb, struct page **pages, - u64 blkno, unsigned int nr_blocks) -{ - struct scoutfs_bio_completion comp; - - scoutfs_bio_init_comp(&comp); - scoutfs_bio_submit_comp(sb, READ, pages, blkno, nr_blocks, &comp); - return scoutfs_bio_wait_comp(sb, &comp); -} - -int scoutfs_bio_write(struct super_block *sb, struct page **pages, - u64 blkno, unsigned int nr_blocks) -{ - struct scoutfs_bio_completion comp; - - scoutfs_bio_init_comp(&comp); - scoutfs_bio_submit_comp(sb, WRITE, pages, blkno, nr_blocks, &comp); - - return scoutfs_bio_wait_comp(sb, &comp); -} - -/* return pointer to the blk 4k block offset amongst the pages */ -void *scoutfs_page_block_address(struct page **pages, unsigned int blk) -{ - unsigned int i = blk / SCOUTFS_BLOCKS_PER_PAGE; - unsigned int off = (blk % SCOUTFS_BLOCKS_PER_PAGE) << - SCOUTFS_BLOCK_SHIFT; - - return page_address(pages[i]) + off; -} diff --git a/kmod/src/bio.h b/kmod/src/bio.h deleted file mode 100644 index 93439775..00000000 --- a/kmod/src/bio.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef _SCOUTFS_BIO_H_ -#define _SCOUTFS_BIO_H_ - -/* - * Our little block IO wrapper is just a convenience wrapper that takes - * our block size units and handles tracks multiple bios per larger io. - * - * If bios could hold an unlimited number of pages instead of - * BIO_MAX_PAGES then this would just use a single bio directly. - */ - -/* - * Track aggregate IO completion for multiple multi-bio submissions. - */ -struct scoutfs_bio_completion { - atomic_t pending; - struct completion comp; - long err; -}; - -typedef void (*scoutfs_bio_end_io_t)(struct super_block *sb, void *data, - int err); - -void scoutfs_bio_submit(struct super_block *sb, int rw, struct page **pages, - u64 blkno, unsigned int nr_blocks, - scoutfs_bio_end_io_t end_io, void *data); - -void scoutfs_bio_init_comp(struct scoutfs_bio_completion *comp); -void scoutfs_bio_submit_comp(struct super_block *sb, int rw, - struct page **pages, u64 blkno, - unsigned int nr_blocks, - struct scoutfs_bio_completion *comp); -int scoutfs_bio_wait_comp(struct super_block *sb, - struct scoutfs_bio_completion *comp); - -int scoutfs_bio_read(struct super_block *sb, struct page **pages, - u64 blkno, unsigned int nr_blocks); -int scoutfs_bio_write(struct super_block *sb, struct page **pages, - u64 blkno, unsigned int nr_blocks); - -void *scoutfs_page_block_address(struct page **pages, unsigned int blk); - -#endif diff --git a/kmod/src/client.c b/kmod/src/client.c index ff740cf8..dd8abf03 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -25,9 +25,6 @@ #include "counters.h" #include "inode.h" #include "btree.h" -#include "manifest.h" -#include "seg.h" -#include "compact.h" #include "scoutfs_trace.h" #include "msg.h" #include "client.h" @@ -133,40 +130,6 @@ int scoutfs_client_free_extents(struct super_block *sb, nexl, bytes, NULL, 0); } -int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - __le64 lesegno; - int ret; - - ret = scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_ALLOC_SEGNO, - NULL, 0, &lesegno, sizeof(lesegno)); - if (ret == 0) { - if (lesegno == 0) - ret = -ENOSPC; - else - *segno = le64_to_cpu(lesegno); - } - - return ret; -} - -int scoutfs_client_record_segment(struct super_block *sb, - struct scoutfs_segment *seg, u8 level) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - struct scoutfs_net_manifest_entry net_ment; - struct scoutfs_manifest_entry ment; - - scoutfs_seg_init_ment(&ment, level, seg); - scoutfs_init_ment_to_net(&net_ment, &ment); - - return scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_RECORD_SEGMENT, - &net_ment, sizeof(net_ment), NULL, 0); -} - int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt) { @@ -219,17 +182,6 @@ int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq) return ret; } -int scoutfs_client_get_manifest_root(struct super_block *sb, - struct scoutfs_btree_root *root) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - return scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, - NULL, 0, root, - sizeof(struct scoutfs_btree_root)); -} - int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs) { @@ -488,61 +440,7 @@ out: msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); } -/* - * Perform a compaction in the client as requested by the server. The - * server has protected the input segments and allocated the output - * segnos for us. This executes in work queued by the client's net - * connection. It only reads and write segments. The server will - * update the manifest and allocators while processing the response. An - * error response includes the compaction id so that the server can - * clean it up. - * - * If we get duplicate requests across a reconnected socket we can have - * two workers performing the same compaction simultaneously. This - * isn't particularly efficient but it's rare and won't corrupt the - * output. Our response can be lost if the socket is shutdown while - * it's in flight, the server deals with this. - */ -static int client_compact(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - struct scoutfs_net_compact_response *resp = NULL; - struct scoutfs_net_compact_request *req; - int ret; - - if (arg_len != sizeof(struct scoutfs_net_compact_request)) { - ret = -EINVAL; - goto out; - } - req = arg; - - trace_scoutfs_client_compact_start(sb, le64_to_cpu(req->id), - req->last_level, req->flags); - - resp = kzalloc(sizeof(struct scoutfs_net_compact_response), GFP_NOFS); - if (!resp) { - ret = -ENOMEM; - } else { - resp->id = req->id; - ret = scoutfs_compact(sb, req, resp); - } - - trace_scoutfs_client_compact_stop(sb, le64_to_cpu(req->id), ret); - - if (ret < 0) - ret = scoutfs_net_response(sb, conn, cmd, id, ret, - &req->id, sizeof(req->id)); - else - ret = scoutfs_net_response(sb, conn, cmd, id, 0, - resp, sizeof(*resp)); - kfree(resp); -out: - return ret; -} - static scoutfs_net_request_t client_req_funcs[] = { - [SCOUTFS_NET_CMD_COMPACT] = client_compact, [SCOUTFS_NET_CMD_LOCK] = client_lock, [SCOUTFS_NET_CMD_LOCK_RECOVER] = client_lock_recover, }; diff --git a/kmod/src/client.h b/kmod/src/client.h index 73bc69ee..9ee65a25 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -1,17 +1,12 @@ #ifndef _SCOUTFS_CLIENT_H_ #define _SCOUTFS_CLIENT_H_ -struct scoutfs_segment; - int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, u64 *len); int scoutfs_client_free_extents(struct super_block *sb, struct scoutfs_net_extent_list *nexl); -int scoutfs_client_alloc_segno(struct super_block *sb, u64 *segno); -int scoutfs_client_record_segment(struct super_block *sb, - struct scoutfs_segment *seg, u8 level); int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); int scoutfs_client_commit_log_trees(struct super_block *sb, @@ -19,8 +14,6 @@ int scoutfs_client_commit_log_trees(struct super_block *sb, u64 *scoutfs_client_bulk_alloc(struct super_block *sb); int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); -int scoutfs_client_get_manifest_root(struct super_block *sb, - struct scoutfs_btree_root *root); int scoutfs_client_statfs(struct super_block *sb, struct scoutfs_net_statfs *nstatfs); int scoutfs_client_lock_request(struct super_block *sb, diff --git a/kmod/src/compact.c b/kmod/src/compact.c deleted file mode 100644 index b220a771..00000000 --- a/kmod/src/compact.c +++ /dev/null @@ -1,683 +0,0 @@ -/* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "seg.h" -#include "bio.h" -#include "cmp.h" -#include "compact.h" -#include "manifest.h" -#include "counters.h" -#include "server.h" -#include "scoutfs_trace.h" - -/* - * Compaction is what maintains the exponentially increasing number of - * segments in each level of the lsm tree and is what merges duplicate - * and deletion keys. - * - * The compaction operation itself always involves a single "upper" - * segment at a given level and a limited number of "lower" segments at - * the next higher level whose key range intersects with the upper - * segment. - * - * Compaction proceeds by iterating over the items in the upper segment - * and items in each of the lower segments in sort order. The items - * from the two input segments are copied into new output segments in - * sorted order. Space is reclaimed as duplicate or deletion items are - * removed and fewer segments are written than were read. - */ - -struct compact_seg { - struct list_head entry; - - u64 segno; - u64 seq; - u8 level; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_segment *seg; - int off; - bool part_of_move; -}; - -struct compact_cursor { - struct list_head csegs; - - /* buffer holds allocations and our returning them */ - u64 segnos[SCOUTFS_COMPACTION_MAX_OUTPUT]; - unsigned int nr_segnos; - - u8 lower_level; - u8 last_level; - - struct compact_seg *upper; - struct compact_seg *lower; - - bool sticky; - struct compact_seg *last_lower; - - __le32 *links[SCOUTFS_MAX_SKIP_LINKS]; -}; - -static void free_cseg(struct super_block *sb, struct compact_seg *cseg) -{ - WARN_ON_ONCE(!list_empty(&cseg->entry)); - - scoutfs_seg_put(cseg->seg); - kfree(cseg); -} - -static struct compact_seg *alloc_cseg(struct super_block *sb, - struct scoutfs_key *first, - struct scoutfs_key *last) -{ - struct compact_seg *cseg; - - cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); - if (cseg) { - INIT_LIST_HEAD(&cseg->entry); - cseg->first = *first; - cseg->last = *last; - } - - return cseg; -} - -static void free_cseg_list(struct super_block *sb, struct list_head *list) -{ - struct compact_seg *cseg; - struct compact_seg *tmp; - - list_for_each_entry_safe(cseg, tmp, list, entry) { - list_del_init(&cseg->entry); - free_cseg(sb, cseg); - } -} - -static int read_segment(struct super_block *sb, struct compact_seg *cseg) -{ - struct scoutfs_segment *seg; - int ret; - - if (cseg == NULL || cseg->seg) - return 0; - - seg = scoutfs_seg_submit_read(sb, cseg->segno); - if (IS_ERR(seg)) { - ret = PTR_ERR(seg); - } else { - cseg->seg = seg; - scoutfs_inc_counter(sb, compact_segment_read); - ret = scoutfs_seg_wait(sb, cseg->seg, cseg->segno, cseg->seq); - } - - /* XXX verify read segment metadata */ - - return ret; -} - -static struct compact_seg *next_spos(struct compact_cursor *curs, - struct compact_seg *cseg) -{ - if (cseg->entry.next == &curs->csegs) - return NULL; - - return list_next_entry(cseg, entry); -} - -/* - * Point the caller's key and value kvecs at the next item that should - * be copied from the upper or lower segments. We use the item that has - * the lowest key or the upper if they're the same. We advance the - * cursor past the item that is returned. - * - * XXX this will get fancier as we get range deletion items and - * incremental update items. - */ -static int next_item(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_key *item_key, struct kvec *item_val, - u8 *item_flags) -{ - struct compact_seg *upper = curs->upper; - struct compact_seg *lower = curs->lower; - struct scoutfs_key lower_key; - struct kvec lower_val; - u8 lower_flags; - int cmp; - int ret; - -retry: - if (upper) { - ret = scoutfs_seg_get_item(upper->seg, upper->off, - item_key, item_val, item_flags); - if (ret < 0) - upper = NULL; - } - - while (lower) { - ret = read_segment(sb, lower); - if (ret) - goto out; - - ret = scoutfs_seg_get_item(lower->seg, lower->off, - &lower_key, &lower_val, - &lower_flags); - if (ret == 0) - break; - lower = next_spos(curs, lower); - } - - /* we're done if all are empty */ - if (!upper && !lower) { - ret = 0; - goto out; - } - - /* - * < 0: return upper, advance upper - * == 0: return upper, advance both - * > 0: return lower, advance lower - */ - if (upper && lower) - cmp = scoutfs_key_compare(item_key, &lower_key); - else if (upper) - cmp = -1; - else - cmp = 1; - - if (cmp > 0) { - *item_key = lower_key; - *item_val = lower_val; - *item_flags = lower_flags; - } - - /* - * If we have a sticky compaction then we can't mix items from - * the upper level past the last lower key into the lower level. - * The caller will notice when they're emptying the final upper - * level in a sticky merge and leave it at the upper level. - */ - if (curs->sticky && curs->lower && - (!lower || lower == curs->last_lower) && - scoutfs_key_compare(item_key, &curs->last_lower->last) > 0) { - ret = 0; - goto out; - } - - if (cmp <= 0) - upper->off = scoutfs_seg_next_off(upper->seg, upper->off); - if (cmp >= 0) - lower->off = scoutfs_seg_next_off(lower->seg, lower->off); - - /* - * Deletion items make their way down all the levels, replacing - * all the duplicate items that they find. When we're - * compacting to the last level we can remove them by retrying - * the search after we've advanced past them. - * - * If we're filling the remaining items in a sticky merge into - * the upper level then we have to preserve the deletion items. - */ - if ((curs->lower_level == curs->last_level) && - (!curs->sticky || lower) && - ((*item_flags) & SCOUTFS_ITEM_FLAG_DELETION)) - goto retry; - - ret = 1; -out: - curs->upper = upper; - curs->lower = lower; - - return ret; -} - -static int compact_segments(struct super_block *sb, - struct compact_cursor *curs, - struct scoutfs_bio_completion *comp, - struct list_head *results) -{ - struct scoutfs_key item_key; - struct scoutfs_segment *seg; - struct compact_seg *cseg; - struct compact_seg *upper; - struct compact_seg *lower; - unsigned next_segno = 0; - bool append_filled = false; - struct kvec item_val; - int ret = 0; - u8 flags; - - scoutfs_inc_counter(sb, compact_operations); - if (curs->sticky) - scoutfs_inc_counter(sb, compact_sticky_upper); - - while (curs->upper || curs->lower) { - - upper = curs->upper; - lower = curs->lower; - - /* - * If we're at the start of the upper segment and - * there's no lower segment then we might as well just - * move the segment in the manifest. We can't do this - * if we're moving to the last level because we might - * need to drop any deletion items. - * - * XXX We should have metadata in the manifest to tell - * us that there's no deletion items in the segment. - */ - if (upper && upper->off == 0 && !lower && !curs->sticky && - ((upper->level + 1) < curs->last_level)) { - - /* - * XXX blah! these csegs are getting - * ridiculous. We should have a robust manifest - * entry iterator that reading and compacting - * can use. - */ - cseg = alloc_cseg(sb, &upper->first, &upper->last); - if (!cseg) { - ret = -ENOMEM; - break; - } - - cseg->segno = upper->segno; - cseg->seq = upper->seq; - cseg->level = upper->level + 1; - cseg->seg = upper->seg; - if (cseg->seg) - scoutfs_seg_get(cseg->seg); - list_add_tail(&cseg->entry, results); - - /* don't mess with its segno */ - upper->part_of_move = true; - cseg->part_of_move = true; - - curs->upper = NULL; - - scoutfs_inc_counter(sb, compact_segment_moved); - break; - } - - /* we're going to need its next key */ - ret = read_segment(sb, upper); - if (ret) - break; - - /* - * XXX we could intelligently skip reading and merging - * lower segments here. The lower segment won't change - * if: - * - the lower segment is entirely before the upper - * - the lower segment is full - * - * We don't have the metadata to determine that it's - * full today so we want to read lower segments that don't - * overlap so that we can merge partial lowers with - * its neighbours. - */ - - ret = read_segment(sb, lower); - if (ret) - break; - - if (!append_filled) - ret = next_item(sb, curs, &item_key, &item_val, &flags); - else - ret = 1; - if (ret <= 0) - break; - - /* no cseg keys, manifest update uses seg item keys */ - cseg = kzalloc(sizeof(struct compact_seg), GFP_NOFS); - if (!cseg) { - ret = -ENOMEM; - break; - } - - /* didn't get enough segnos */ - if (next_segno >= curs->nr_segnos) { - ret = -ENOSPC; - break; - } - - cseg->segno = curs->segnos[next_segno]; - curs->segnos[next_segno] = 0; - next_segno++; - - /* - * Compaction can free all the remaining items resulting - * in an empty output segment. We just free it in that - * case. - */ - ret = scoutfs_seg_alloc(sb, cseg->segno, &seg); - if (ret < 0) { - next_segno--; - curs->segnos[next_segno] = cseg->segno; - kfree(cseg); - scoutfs_seg_put(seg); - break; - } - - /* - * The remaining upper items in a sticky merge have to - * be written into the upper level. - */ - if (curs->sticky && !lower) { - cseg->level = curs->lower_level - 1; - scoutfs_inc_counter(sb, compact_sticky_written); - } else { - cseg->level = curs->lower_level; - } - - /* csegs will be claned up once they're on the list */ - cseg->seg = seg; - list_add_tail(&cseg->entry, results); - - for (;;) { - if (!scoutfs_seg_append_item(sb, seg, &item_key, - &item_val, flags, - curs->links)) { - append_filled = true; - ret = 0; - break; - } - ret = next_item(sb, curs, &item_key, &item_val, &flags); - if (ret <= 0) { - append_filled = false; - break; - } - } - if (ret < 0) - break; - - /* start a complete segment write now, we'll wait later */ - ret = scoutfs_seg_submit_write(sb, seg, comp); - if (ret) - break; - - scoutfs_inc_counter(sb, compact_segment_writes); - scoutfs_add_counter(sb, compact_segment_write_bytes, - scoutfs_seg_total_bytes(seg)); - } - - return ret; -} - -/* - * We want all the non-zero segnos sorted at the front of the array - * and the empty segnos all packed at the end. This is easily done by - * subtracting one from both then comparing as usual. All relations hold - * except that 0 becomes the greatest instead of the least. - */ -static int sort_cmp_segnos(const void *A, const void *B) -{ - const u64 a = *(const u64 *)A - 1; - const u64 b = *(const u64 *)B - 1; - - return a < b ? -1 : a > b ? 1 : 0; -} - -static void sort_swap_segnos(void *A, void *B, int size) -{ - u64 *a = A; - u64 *b = B; - - swap(*a, *b); -} - -static int verify_request(struct super_block *sb, - struct scoutfs_net_compact_request *req) -{ - int ret = -EINVAL; - int nr_segnos; - int nr_ents; - int i; - - /* no unknown flags */ - if (req->flags & ~SCOUTFS_NET_COMPACT_FLAG_STICKY) - goto out; - - /* find the number of segments and entries */ - for (i = 0; i < ARRAY_SIZE(req->segnos); i++) { - if (req->segnos[i] == 0) - break; - } - nr_segnos = i; - - for (i = 0; i < ARRAY_SIZE(req->ents); i++) { - if (req->ents[i].segno == 0) - break; - } - nr_ents = i; - - /* must have at least an upper */ - if (nr_ents == 0) - goto out; - - sort(req->segnos, nr_segnos, sizeof(req->segnos[i]), - sort_cmp_segnos, sort_swap_segnos); - - /* segnos must be unique */ - for (i = 1; i < nr_segnos; i++) { - if (req->segnos[i] == req->segnos[i - 1]) - goto out; - } - - /* if we have a lower it must be under upper */ - if (nr_ents > 1 && (req->ents[1].level != req->ents[0].level + 1)) - goto out; - - /* make sure lower ents are on the same level */ - for (i = 2; i < nr_ents; i++) { - if (req->ents[i].level != req->ents[i - 1].level) - goto out; - } - - for (i = 1; i < nr_ents; i++) { - /* lowers must overlap with upper */ - if (scoutfs_key_compare_ranges(&req->ents[0].first, - &req->ents[0].last, - &req->ents[i].first, - &req->ents[i].last) != 0) - goto out; - - /* lowers must be on the level below upper */ - if (req->ents[i].level != req->ents[0].level + 1) - goto out; - } - - /* last level must include lowest level */ - if (req->last_level < req->ents[nr_ents - 1].level) - goto out; - - for (i = 2; i < nr_ents; i++) { - /* lowers must be sorted by first key */ - if (scoutfs_key_compare(&req->ents[i].first, - &req->ents[i - 1].first) <= 0) - goto out; - - /* lowers must not overlap with each other */ - if (scoutfs_key_compare_ranges(&req->ents[i].first, - &req->ents[i].last, - &req->ents[i - 1].first, - &req->ents[i - 1].last) == 0) - goto out; - } - - ret = 0; -out: - if (WARN_ON_ONCE(ret < 0)) { - scoutfs_inc_counter(sb, compact_invalid_request); - printk("id %llu last_level %u flags 0x%x\n", - le64_to_cpu(req->id), req->last_level, req->flags); - printk("segnos: "); - for (i = 0; i < ARRAY_SIZE(req->segnos); i++) - printk("%llu ", le64_to_cpu(req->segnos[i])); - printk("\n"); - printk("entries: "); - for (i = 0; i < ARRAY_SIZE(req->ents); i++) { - printk(" [%u] segno %llu seq %llu level %u first "SK_FMT" last "SK_FMT"\n", - i, le64_to_cpu(req->ents[i].segno), - le64_to_cpu(req->ents[i].seq), - req->ents[i].level, - SK_ARG(&req->ents[i].first), - SK_ARG(&req->ents[i].last)); - } - printk("\n"); - } - - return ret; -} - -/* - * Translate the compaction request into our native structs that we use - * to perform the compaction. The caller has verified that the request - * satisfies our constraints. - * - * If we return an error the caller will clean up a partially prepared - * cursor. - */ -static int prepare_curs(struct super_block *sb, struct compact_cursor *curs, - struct scoutfs_net_compact_request *req) -{ - struct scoutfs_manifest_entry ment; - struct compact_seg *cseg; - int ret = 0; - int i; - - curs->lower_level = req->ents[0].level + 1; - curs->last_level = req->last_level; - curs->sticky = !!(req->flags & SCOUTFS_NET_COMPACT_FLAG_STICKY); - - for (i = 0; i < ARRAY_SIZE(req->segnos); i++) { - if (req->segnos[i] == 0) - break; - curs->segnos[i] = le64_to_cpu(req->segnos[i]); - } - curs->nr_segnos = i; - - for (i = 0; i < ARRAY_SIZE(req->ents); i++) { - if (req->ents[i].segno == 0) - break; - - scoutfs_init_ment_from_net(&ment, &req->ents[i]); - - cseg = alloc_cseg(sb, &ment.first, &ment.last); - if (!cseg) { - ret = -ENOMEM; - break; - } - - list_add_tail(&cseg->entry, &curs->csegs); - - cseg->segno = ment.segno; - cseg->seq = ment.seq; - cseg->level = ment.level; - - if (!curs->upper) - curs->upper = cseg; - else if (!curs->lower) - curs->lower = cseg; - if (curs->lower) - curs->last_lower = cseg; - } - - return ret; -} - -/* - * Perform a compaction by translating the incoming request into our - * working state, iterating over input segments and write output - * segments, then generating the response that describes the output - * segments. - * - * The server will either commit our response or cleanup the request if - * we return an error that the caller sends in response. The server - * protects the input segments so they shouldn't be overwritten by other - * compactions or allocations. We shouldn't get stale segment reads. - */ -int scoutfs_compact(struct super_block *sb, - struct scoutfs_net_compact_request *req, - struct scoutfs_net_compact_response *resp) -{ - struct compact_cursor curs = {{NULL,}}; - struct scoutfs_manifest_entry ment; - struct scoutfs_bio_completion comp; - struct compact_seg *cseg; - LIST_HEAD(results); - int ret; - int err; - int nr; - - INIT_LIST_HEAD(&curs.csegs); - scoutfs_bio_init_comp(&comp); - - ret = verify_request(sb, req) ?: - prepare_curs(sb, &curs, req); - if (ret) - goto out; - - /* trace compaction ranges */ - list_for_each_entry(cseg, &curs.csegs, entry) { - trace_scoutfs_compact_input(sb, cseg->level, cseg->segno, - cseg->seq, &cseg->first, - &cseg->last); - } - - ret = compact_segments(sb, &curs, &comp, &results); - - /* always wait for io completion */ - err = scoutfs_bio_wait_comp(sb, &comp); - if (!ret && err) - ret = err; - if (ret) - goto out; - - /* fill entries for written output segments */ - nr = 0; - list_for_each_entry(cseg, &results, entry) { - /* XXX moved upper segments won't have read the segment :P */ - if (cseg->seg) - scoutfs_seg_init_ment(&ment, cseg->level, cseg->seg); - else - scoutfs_manifest_init_entry(&ment, cseg->level, - cseg->segno, cseg->seq, - &cseg->first, &cseg->last); - - trace_scoutfs_compact_output(sb, ment.level, ment.segno, - ment.seq, &ment.first, - &ment.last); - - scoutfs_init_ment_to_net(&resp->ents[nr++], &ment); - } - - ret = 0; -out: - /* server protects input segments, shouldn't be possible */ - if (WARN_ON_ONCE(ret == -ESTALE)) { - scoutfs_inc_counter(sb, compact_stale_error); - ret = -EIO; - } - - free_cseg_list(sb, &curs.csegs); - free_cseg_list(sb, &results); - - return ret; -} diff --git a/kmod/src/compact.h b/kmod/src/compact.h deleted file mode 100644 index 788cae2c..00000000 --- a/kmod/src/compact.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SCOUTFS_COMPACT_H_ -#define _SCOUTFS_COMPACT_H_ - -int scoutfs_compact(struct super_block *sb, - struct scoutfs_net_compact_request *req, - struct scoutfs_net_compact_response *resp); - -#endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index ef808252..628d0493 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -26,16 +26,6 @@ EXPAND_COUNTER(btree_stale_read) \ EXPAND_COUNTER(btree_write_error) \ EXPAND_COUNTER(client_farewell_error) \ - EXPAND_COUNTER(compact_invalid_request) \ - EXPAND_COUNTER(compact_operations) \ - EXPAND_COUNTER(compact_segment_busy) \ - EXPAND_COUNTER(compact_segment_moved) \ - EXPAND_COUNTER(compact_segment_read) \ - EXPAND_COUNTER(compact_segment_write_bytes) \ - EXPAND_COUNTER(compact_segment_writes) \ - EXPAND_COUNTER(compact_stale_error) \ - EXPAND_COUNTER(compact_sticky_upper) \ - EXPAND_COUNTER(compact_sticky_written) \ EXPAND_COUNTER(corrupt_btree_block_level) \ EXPAND_COUNTER(corrupt_btree_no_child_ref) \ EXPAND_COUNTER(corrupt_data_extent_trunc_cleanup) \ @@ -71,27 +61,6 @@ EXPAND_COUNTER(extent_next) \ EXPAND_COUNTER(extent_prev) \ EXPAND_COUNTER(extent_remove) \ - EXPAND_COUNTER(item_alloc) \ - EXPAND_COUNTER(item_batch_duplicate) \ - EXPAND_COUNTER(item_batch_inserted) \ - EXPAND_COUNTER(item_create) \ - EXPAND_COUNTER(item_delete) \ - EXPAND_COUNTER(item_free) \ - EXPAND_COUNTER(item_lookup_hit) \ - EXPAND_COUNTER(item_lookup_miss) \ - EXPAND_COUNTER(item_range_alloc) \ - EXPAND_COUNTER(item_range_free) \ - EXPAND_COUNTER(item_range_hit) \ - EXPAND_COUNTER(item_range_insert) \ - EXPAND_COUNTER(item_range_miss) \ - EXPAND_COUNTER(item_shrink) \ - EXPAND_COUNTER(item_shrink_alone) \ - EXPAND_COUNTER(item_shrink_empty_range) \ - EXPAND_COUNTER(item_shrink_next_dirty) \ - EXPAND_COUNTER(item_shrink_outside) \ - EXPAND_COUNTER(item_shrink_range_end) \ - EXPAND_COUNTER(item_shrink_small_split) \ - EXPAND_COUNTER(item_shrink_split_range) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grace_elapsed) \ @@ -113,9 +82,6 @@ EXPAND_COUNTER(lock_shrink_request_aborted) \ EXPAND_COUNTER(lock_unlock) \ EXPAND_COUNTER(lock_wait) \ - EXPAND_COUNTER(manifest_compact_migrate) \ - EXPAND_COUNTER(manifest_hard_stale_error) \ - EXPAND_COUNTER(manifest_read_excluded_key) \ EXPAND_COUNTER(net_dropped_response) \ EXPAND_COUNTER(net_send_bytes) \ EXPAND_COUNTER(net_send_error) \ @@ -139,28 +105,18 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(seg_alloc) \ - EXPAND_COUNTER(seg_csum_error) \ - EXPAND_COUNTER(seg_free) \ - EXPAND_COUNTER(seg_shrink) \ - EXPAND_COUNTER(seg_stale_read) \ - EXPAND_COUNTER(server_alloc_segno) \ EXPAND_COUNTER(server_extent_alloc) \ EXPAND_COUNTER(server_extent_alloc_error) \ EXPAND_COUNTER(server_free_extent) \ EXPAND_COUNTER(server_free_pending_extent) \ EXPAND_COUNTER(server_free_pending_error) \ - EXPAND_COUNTER(server_free_segno) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ - EXPAND_COUNTER(trans_commit_item_flush) \ EXPAND_COUNTER(trans_commit_sync_fs) \ - EXPAND_COUNTER(trans_commit_timer) \ - EXPAND_COUNTER(trans_write_item) \ - EXPAND_COUNTER(trans_write_deletion_item) + EXPAND_COUNTER(trans_commit_timer) #define FIRST_COUNTER block_cache_access -#define LAST_COUNTER trans_write_deletion_item +#define LAST_COUNTER trans_commit_timer #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/file.c b/kmod/src/file.c index 765e5f1e..f35ef039 100644 --- a/kmod/src/file.c +++ b/kmod/src/file.c @@ -23,7 +23,6 @@ #include "super.h" #include "data.h" #include "scoutfs_trace.h" -#include "item.h" #include "lock.h" #include "file.h" #include "inode.h" diff --git a/kmod/src/format.h b/kmod/src/format.h index 834a1196..a167c5cc 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -20,18 +20,6 @@ #define SCOUTFS_BLOCK_SECTORS (1 << SCOUTFS_BLOCK_SECTOR_SHIFT) #define SCOUTFS_BLOCK_MAX (U64_MAX >> SCOUTFS_BLOCK_SHIFT) -/* - * FS data is stored in segments, for now they're fixed size. They'll - * be dynamic. - */ -#define SCOUTFS_SEGMENT_SHIFT 20 -#define SCOUTFS_SEGMENT_SIZE (1 << SCOUTFS_SEGMENT_SHIFT) -#define SCOUTFS_SEGMENT_MASK (SCOUTFS_SEGMENT_SIZE - 1) -#define SCOUTFS_SEGMENT_PAGES (SCOUTFS_SEGMENT_SIZE / PAGE_SIZE) -#define SCOUTFS_SEGMENT_BLOCKS (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) -#define SCOUTFS_SEGMENT_BLOCK_SHIFT \ - (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT) - #define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) #define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) @@ -224,47 +212,6 @@ struct scoutfs_btree_block { struct scoutfs_btree_item_header item_hdrs[0]; } __packed; -struct scoutfs_btree_ring { - __le64 first_blkno; - __le64 nr_blocks; - __le64 next_block; - __le64 next_seq; -} __packed; - -/* - * This is absurdly huge. If there was only ever 1 item per segment and - * 2^64 items the tree could get this deep. - */ -#define SCOUTFS_MANIFEST_MAX_LEVEL 20 - -#define SCOUTFS_MANIFEST_FANOUT 10 - -struct scoutfs_manifest { - struct scoutfs_btree_root root; - __le64 level_counts[SCOUTFS_MANIFEST_MAX_LEVEL]; -} __packed; - -/* - * Manifest entries are split across btree keys and values. Putting - * some entry fields in the value keeps the key smaller and increases - * the fanout of the btree which keeps the tree smaller and reduces - * block IO. - * - * The key is made up of the level, first key, and seq. At level 0 - * segments can completely overlap and have identical key ranges but we - * avoid duplicate btree keys by including the unique seq. - */ -struct scoutfs_manifest_btree_key { - __u8 level; - struct scoutfs_key_be first_key; - __be64 seq; -} __packed; - -struct scoutfs_manifest_btree_val { - __le64 segno; - struct scoutfs_key last_key; -} __packed; - /* * Free metadata blocks are tracked by block allocator items. */ @@ -385,50 +332,6 @@ struct scoutfs_bloom_block { member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) \ -/* - * The max number of links defines the max number of entries that we can - * index in o(log n) and the static list head storage size in the - * segment block. We always pay the static storage cost, which is tiny, - * and we can look at the number of items to know the greatest number of - * links and skip most of the initial 0 links. - */ -#define SCOUTFS_MAX_SKIP_LINKS 32 - -/* - * Items are packed into segments and linked together in a skip list. - * Each item's header, links, key, and value are stored contiguously. - * They're not allowed to cross a block boundary. - */ -struct scoutfs_segment_item { - struct scoutfs_key key; - __le16 val_len; - __u8 flags; - __u8 nr_links; - __le32 skip_links[0]; - /* __u8 val_bytes[val_len] */ -} __packed; - -#define SCOUTFS_ITEM_FLAG_DELETION (1 << 0) - -/* - * Each large segment starts with a segment block that describes the - * rest of the blocks that make up the segment. - * - * The crc covers the initial total_bytes of the segment but starts - * after the padding. - */ -struct scoutfs_segment_block { - __le32 crc; - __le32 _padding; - __le64 segno; - __le64 seq; - __le32 last_item_off; - __le32 total_bytes; - __le32 nr_items; - __le32 skip_links[SCOUTFS_MAX_SKIP_LINKS]; - /* packed items */ -} __packed; - /* * Keys are first sorted by major key zones. */ @@ -551,12 +454,8 @@ struct scoutfs_super_block { __le64 next_uninit_free_block; __le64 core_balloc_cursor; __le64 free_blocks; - __le64 alloc_cursor; __le64 first_fs_blkno; __le64 last_fs_blkno; - struct scoutfs_btree_ring bring; - __le64 next_seg_seq; - __le64 next_compact_id; __le64 quorum_fenced_term; __le64 quorum_server_term; __le64 unmount_barrier; @@ -566,7 +465,6 @@ struct scoutfs_super_block { struct scoutfs_balloc_root core_balloc_free; struct scoutfs_btree_root alloc_root; struct scoutfs_btree_root fs_root; - struct scoutfs_manifest manifest; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; @@ -757,15 +655,11 @@ enum { SCOUTFS_NET_CMD_ALLOC_INODES, SCOUTFS_NET_CMD_ALLOC_EXTENT, SCOUTFS_NET_CMD_FREE_EXTENTS, - SCOUTFS_NET_CMD_ALLOC_SEGNO, - SCOUTFS_NET_CMD_RECORD_SEGMENT, SCOUTFS_NET_CMD_GET_LOG_TREES, SCOUTFS_NET_CMD_COMMIT_LOG_TREES, SCOUTFS_NET_CMD_ADVANCE_SEQ, SCOUTFS_NET_CMD_GET_LAST_SEQ, - SCOUTFS_NET_CMD_GET_MANIFEST_ROOT, SCOUTFS_NET_CMD_STATFS, - SCOUTFS_NET_CMD_COMPACT, SCOUTFS_NET_CMD_LOCK, SCOUTFS_NET_CMD_LOCK_RECOVER, SCOUTFS_NET_CMD_FAREWELL, @@ -804,20 +698,6 @@ struct scoutfs_net_inode_alloc { __le64 nr; } __packed; -struct scoutfs_net_key_range { - __le16 start_len; - __le16 end_len; - __u8 key_bytes[0]; -} __packed; - -struct scoutfs_net_manifest_entry { - __le64 segno; - __le64 seq; - struct scoutfs_key first; - struct scoutfs_key last; - __u8 level; -} __packed; - struct scoutfs_net_statfs { __le64 total_blocks; /* total blocks in device */ __le64 next_ino; /* next unused inode number */ @@ -844,50 +724,6 @@ struct scoutfs_net_extent_list { /* arbitrarily makes a nice ~1k extent list payload */ #define SCOUTFS_NET_EXTENT_LIST_MAX_NR 64 -/* one upper segment and fanout lower segments */ -#define SCOUTFS_COMPACTION_MAX_INPUT (1 + SCOUTFS_MANIFEST_FANOUT) -/* sticky can split the input and item alignment padding can add a lower */ -#define SCOUTFS_COMPACTION_SEGNO_OVERHEAD 2 -#define SCOUTFS_COMPACTION_MAX_OUTPUT \ - (SCOUTFS_COMPACTION_MAX_INPUT + SCOUTFS_COMPACTION_SEGNO_OVERHEAD) - -/* - * A compact request is sent by the server to the client. It provides - * the input segments and enough allocated segnos to write the results. - * The id uniquely identifies this compaction request and is included in - * the response to clean up its allocated resources. - */ -struct scoutfs_net_compact_request { - __le64 id; - __u8 last_level; - __u8 flags; - __le64 segnos[SCOUTFS_COMPACTION_MAX_OUTPUT]; - struct scoutfs_net_manifest_entry ents[SCOUTFS_COMPACTION_MAX_INPUT]; -} __packed; - -/* - * A sticky compaction has more lower level segments that overlap with - * the end of the upper after the last lower level segment included in - * the compaction. Items left in the upper segment after the last lower - * need to be written to the upper level instead of the lower. The - * upper segment "sticks" in place instead of moving down to the lower - * level. - */ -#define SCOUTFS_NET_COMPACT_FLAG_STICKY (1 << 0) - -/* - * A compact response is sent by the client to the server. It describes - * the written output segments that need to be added to the manifest. - * The server compares the response to the request to free unused - * allocated segnos and input manifest entries. An empty response is - * valid and can happen if, say, the upper input segment completely - * deleted all the items in a single overlapping lower segment. - */ -struct scoutfs_net_compact_response { - __le64 id; - struct scoutfs_net_manifest_entry ents[SCOUTFS_COMPACTION_MAX_OUTPUT]; -} __packed; - struct scoutfs_net_lock { struct scoutfs_key key; __u8 old_mode; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 4c271816..10196df4 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -28,11 +28,9 @@ #include "super.h" #include "inode.h" #include "forest.h" -#include "item.h" #include "data.h" #include "client.h" #include "lock.h" -#include "manifest.h" #include "trans.h" #include "xattr.h" #include "hash.h" @@ -490,61 +488,6 @@ static long scoutfs_ioc_stat_more(struct file *file, unsigned long arg) return 0; } -static long scoutfs_ioc_item_cache_keys(struct file *file, unsigned long arg) -{ - struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_item_cache_keys ick; - struct scoutfs_ioctl_key __user *ukeys; - struct scoutfs_ioctl_key ikeys[16]; - struct scoutfs_key keys[16]; - struct scoutfs_key key; - unsigned int nr; - int total; - int ret; - int i; - - if (copy_from_user(&ick, (void __user *)arg, sizeof(ick))) - return -EFAULT; - - if (ick.which > SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES) - return -EINVAL; - - scoutfs_key_copy_types(&key, &ick.ikey); - - ukeys = (void __user *)(long)ick.buf_ptr; - total = 0; - ret = 0; - while (ick.buf_nr) { - nr = min_t(size_t, ick.buf_nr, ARRAY_SIZE(keys)); - - if (ick.which == SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS) - ret = scoutfs_item_copy_keys(sb, &key, keys, nr); - else - ret = scoutfs_item_copy_range_keys(sb, &key, keys, nr); - BUG_ON(ret > nr); /* stack overflow \o/ */ - if (ret <= 0) - break; - - for (i = 0; i < ret; i++) - scoutfs_key_copy_types(&ikeys[i], &keys[i]); - - if (copy_to_user(ukeys, ikeys, ret * sizeof(ikeys[0]))) { - ret = -EFAULT; - break; - } - - key = keys[ret - 1]; - scoutfs_key_inc(&key); - - ukeys += ret; - ick.buf_nr -= ret; - total += ret; - ret = 0; - } - - return ret ?: total; -} - static bool inc_wrapped(u64 *ino, u64 *iblock) { return (++(*iblock) == 0) && (++(*ino) == 0); @@ -876,8 +819,6 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_stage(file, arg); case SCOUTFS_IOC_STAT_MORE: return scoutfs_ioc_stat_more(file, arg); - case SCOUTFS_IOC_ITEM_CACHE_KEYS: - return scoutfs_ioc_item_cache_keys(file, arg); case SCOUTFS_IOC_DATA_WAITING: return scoutfs_ioc_data_waiting(file, arg); case SCOUTFS_IOC_SETATTR_MORE: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 5693668e..df0c1b54 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -238,28 +238,6 @@ struct scoutfs_ioctl_stat_more { struct scoutfs_ioctl_stat_more) -/* - * Fills the buffer with either the keys for the cached items or the - * keys for the cached ranges found starting with the given key. The - * number of keys filled in the buffer is returned. When filling range - * keys the returned number will always be a multiple of two. - */ -struct scoutfs_ioctl_item_cache_keys { - struct scoutfs_ioctl_key ikey; - __u64 buf_ptr; - __u16 buf_nr; - __u8 which; - __u8 _pad[21]; /* padded to align _ioctl_key total size */ -}; - -enum { - SCOUTFS_IOC_ITEM_CACHE_KEYS_ITEMS = 0, - SCOUTFS_IOC_ITEM_CACHE_KEYS_RANGES, -}; - -#define SCOUTFS_IOC_ITEM_CACHE_KEYS _IOR(SCOUTFS_IOCTL_MAGIC, 6, \ - struct scoutfs_ioctl_item_cache_keys) - struct scoutfs_ioctl_data_waiting_entry { __u64 ino; __u64 iblock; @@ -283,7 +261,7 @@ struct scoutfs_ioctl_data_waiting { #define SCOUTFS_IOC_DATA_WAITING_FLAGS_UNKNOWN (U8_MAX << 0) -#define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 7, \ +#define SCOUTFS_IOC_DATA_WAITING _IOR(SCOUTFS_IOCTL_MAGIC, 6, \ struct scoutfs_ioctl_data_waiting) /* @@ -303,7 +281,7 @@ struct scoutfs_ioctl_setattr_more { #define SCOUTFS_IOC_SETATTR_MORE_OFFLINE (1 << 0) #define SCOUTFS_IOC_SETATTR_MORE_UNKNOWN (U8_MAX << 1) -#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 8, \ +#define SCOUTFS_IOC_SETATTR_MORE _IOW(SCOUTFS_IOCTL_MAGIC, 7, \ struct scoutfs_ioctl_setattr_more) struct scoutfs_ioctl_listxattr_hidden { @@ -313,7 +291,7 @@ struct scoutfs_ioctl_listxattr_hidden { __u32 hash_pos; }; -#define SCOUTFS_IOC_LISTXATTR_HIDDEN _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ +#define SCOUTFS_IOC_LISTXATTR_HIDDEN _IOR(SCOUTFS_IOCTL_MAGIC, 8, \ struct scoutfs_ioctl_listxattr_hidden) /* @@ -344,7 +322,7 @@ struct scoutfs_ioctl_find_xattrs { __u8 _pad[4]; }; -#define SCOUTFS_IOC_FIND_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ +#define SCOUTFS_IOC_FIND_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ struct scoutfs_ioctl_find_xattrs) /* @@ -365,7 +343,7 @@ struct scoutfs_ioctl_statfs_more { __u64 rid; } __packed; -#define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ +#define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ struct scoutfs_ioctl_statfs_more) diff --git a/kmod/src/item.c b/kmod/src/item.c deleted file mode 100644 index d8e3f318..00000000 --- a/kmod/src/item.c +++ /dev/null @@ -1,2498 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "kvec.h" -#include "manifest.h" -#include "item.h" -#include "seg.h" -#include "counters.h" -#include "scoutfs_trace.h" -#include "trans.h" - -/* - * A simple rbtree of cached items isolates the item API callers from - * the relatively expensive segment searches. - * - * The item cache uses an rbtree of key ranges to record regions of keys - * that are completely described by the items. This lets it return - * negative lookups cache hits for items that don't exist without having - * to constantly perform expensive segment searches. - * - * Deletions of persistent items are recorded with items in the rbtree - * which record the key of the deletion. They're removed once they're - * written to a level0 segment. While they're present in the cache we - * have to be careful to clobber them in creation and skip them in - * lookups. We only need deletion items for keys that exist in - * segments. We can immediately free non-persistent items when they're - * deleted. - */ - -static bool invalid_key_val(struct scoutfs_key *key, struct kvec *val) -{ - return WARN_ON_ONCE(val && (val->iov_len > SCOUTFS_MAX_VAL_SIZE)); -} - -struct item_cache { - struct super_block *sb; - - spinlock_t lock; - struct rb_root items; - struct rb_root ranges; - - long nr_dirty_items; - long dirty_val_bytes; - - struct shrinker shrinker; - struct list_head lru_list; - unsigned long lru_nr; -}; - -/* - * The dirty bits track if the given item is dirty and if its child - * subtrees contain any dirty items. - * - * The entry list_head typically stores clean items on an lru for shrinking. - * It's also briefly used to track items in a batch after they're - * allocated but before they're inserted for the first time. - * - * The persistent bit indicates that the item's key is present in - * segments. If we delete persistent items we have to write a deletion - * item to delete to remove the existing item. We can free deleted items - * that aren't persistent without writing them. - */ -struct cached_item { - struct rb_node node; - struct list_head entry; - - long dirty; - unsigned deletion:1, - persistent:1; - - struct scoutfs_key key; - void *val; - unsigned int val_len; -}; - -struct cached_range { - struct rb_node node; - - struct scoutfs_key start; - struct scoutfs_key end; -}; - -#define trace_range(which, sb, rng) \ - trace_scoutfs_item_range_##which(sb, (rng), &(rng)->start, &(rng)->end) - -static u8 item_flags(struct cached_item *item) -{ - return item->deletion ? SCOUTFS_ITEM_FLAG_DELETION : 0; -} - -static void free_item(struct super_block *sb, struct cached_item *item) -{ - if (!IS_ERR_OR_NULL(item)) { - scoutfs_inc_counter(sb, item_free); - WARN_ON_ONCE(!list_empty(&item->entry)); - WARN_ON_ONCE(!RB_EMPTY_NODE(&item->node)); - kfree(item->val); - kfree(item); - } -} - -/* - * The value vec may be null if the item has no value. Values are - * allocated separately so that we can free them when deleting or swap - * them in place when updating items. - */ -static struct cached_item *alloc_item(struct super_block *sb, - struct scoutfs_key *key, - struct kvec *val) -{ - struct cached_item *item; - - item = kzalloc(sizeof(struct cached_item), GFP_NOFS); - if (!item) - goto out; - - item->key = *key; - RB_CLEAR_NODE(&item->node); - INIT_LIST_HEAD(&item->entry); - - if (val) { - item->val = kmalloc(val->iov_len, GFP_NOFS); - if (!item->val) { - free_item(sb, item); - item = NULL; - goto out; - } - item->val_len = val->iov_len; - memcpy(item->val, val->iov_base, val->iov_len); - } - - scoutfs_inc_counter(sb, item_alloc); -out: - return item; -} - -/* - * Copy the cached item's value into the caller's single value vector. - * The number of bytes that fit in the vec and were copied is returned. - * A null val returns 0. - */ -static int copy_item_val(struct kvec *val, struct cached_item *item) -{ - int ret; - - if (val) { - ret = min_t(size_t, item->val_len, val->iov_len); - memcpy(val->iov_base, item->val, ret); - } else { - ret = 0; - } - - return ret; -} - -/* - * Walk the item rbtree and return the item found and the next and - * prev items. - */ -static struct cached_item *walk_items(struct rb_root *root, - struct scoutfs_key *key, - struct cached_item **prev, - struct cached_item **next) -{ - struct rb_node *node = root->rb_node; - struct cached_item *item; - int cmp; - - *prev = NULL; - *next = NULL; - - while (node) { - item = container_of(node, struct cached_item, node); - - cmp = scoutfs_key_compare(key, &item->key); - if (cmp < 0) { - *next = item; - node = node->rb_left; - } else if (cmp > 0) { - *prev = item; - node = node->rb_right; - } else { - return item; - } - } - - return NULL; -} - -/* - * Look for the item with the given key. Callers of this are looking - * for existing items. They would just return -ENOENT from a deletion - * item if we gave it to them so we return null for deletion items. - * Callers that would remove a deletion item before inserting a new - * version of the item do so by having insert_item() replace existing - * deleted items on their behalf. - */ -static struct cached_item *find_item(struct super_block *sb, - struct rb_root *root, - struct scoutfs_key *key) -{ - struct cached_item *prev; - struct cached_item *next; - struct cached_item *item; - - item = walk_items(root, key, &prev, &next); - - if (item && item->deletion) - item = NULL; - - if (item) - scoutfs_inc_counter(sb, item_lookup_hit); - else - scoutfs_inc_counter(sb, item_lookup_miss); - - return item; -} - -static struct cached_item *next_item(struct rb_root *root, - struct scoutfs_key *key) -{ - struct cached_item *prev; - struct cached_item *next; - - return walk_items(root, key, &prev, &next) ?: next; -} - -static struct cached_item *prev_item(struct rb_root *root, - struct scoutfs_key *key) -{ - struct cached_item *prev; - struct cached_item *next; - - return walk_items(root, key, &prev, &next) ?: prev; -} - -/* - * We store the dirty bits in a single value so that the simple - * augmented rbtree implementation gets a single scalar value to compare - * and store. - */ -#define ITEM_DIRTY 0x1 -#define LEFT_DIRTY 0x2 -#define RIGHT_DIRTY 0x4 - -static bool item_is_dirty(struct cached_item *item) -{ - return (item->dirty & ITEM_DIRTY) != 0; -} - -/* - * Return the given dirty bit if the item with the given node is dirty - * or has dirty children. - */ -static long node_dirty_bit(struct rb_node *node, long dirty) -{ - struct cached_item *item; - - if (node) { - item = container_of(node, struct cached_item, node); - if (item->dirty) - return dirty; - } - - return 0; -} - -static long compute_item_dirty(struct cached_item *item) -{ - return (item->dirty & ITEM_DIRTY) | - node_dirty_bit(item->node.rb_left, LEFT_DIRTY) | - node_dirty_bit(item->node.rb_right, RIGHT_DIRTY); -} - -static void scoutfs_item_rb_propagate(struct rb_node *node, - struct rb_node *stop) -{ - struct cached_item *item; - long dirty; - - while (node != stop) { - item = container_of(node, struct cached_item, node); - dirty = compute_item_dirty(item); - - if (item->dirty == dirty) - break; - - item->dirty = dirty; - node = rb_parent(&item->node); - } -} - -static void scoutfs_item_rb_copy(struct rb_node *old, struct rb_node *new) -{ - struct cached_item *n = container_of(new, struct cached_item, node); - - n->dirty = compute_item_dirty(n); -} - -/* calculate the new parent last as it depends on the old parent */ -static void scoutfs_item_rb_rotate(struct rb_node *old, struct rb_node *new) -{ - struct cached_item *o = container_of(old, struct cached_item, node); - struct cached_item *n = container_of(new, struct cached_item, node); - - o->dirty = compute_item_dirty(o); - n->dirty = compute_item_dirty(n); -} - -/* - * The generic RB_DECLARE_CALLBACKS() helpers are built for augmented - * values that are simple commutative function of the left and right - * children's augmented values. During rotation the new parent just - * gets the old parent's augmented value and then the old parent's value - * is calculated. - * - * Our dirty bits don't work that way. They are not just an or of the - * child's bits, the bits depend on the left and right children - * specifically. During rotation both parents need to be specifically - * recalculated. (They could be masked and asigned based on the - * direction of the rotation but that's annoying, let's just - * recalculate.) - */ -static const struct rb_augment_callbacks scoutfs_item_rb_cb = { - .propagate = scoutfs_item_rb_propagate, - .copy = scoutfs_item_rb_copy, - .rotate = scoutfs_item_rb_rotate, -}; - -/* - * The caller has changed an item's dirty bit. Its child dirty bits are - * still consistent. But its parent's bits might need to be updated. - * Its bits are consistent so we don't propagate from the node itself - * because it would immediately terminate. - */ -static void update_dirty_parents(struct cached_item *item) -{ - scoutfs_item_rb_propagate(rb_parent(&item->node), NULL); -} - -static void update_dirty_item_counts(struct super_block *sb, signed items, - signed vals) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - - cac->nr_dirty_items += items; - cac->dirty_val_bytes += vals; - - scoutfs_trans_track_item(sb, items, vals); -} - -static void mark_item_dirty(struct super_block *sb, struct item_cache *cac, - struct cached_item *item) -{ - if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) - return; - - if (item_is_dirty(item)) - return; - - item->dirty |= ITEM_DIRTY; - list_del_init(&item->entry); - cac->lru_nr--; - - update_dirty_item_counts(sb, 1, item->val_len); - update_dirty_parents(item); -} - -static void clear_item_dirty(struct super_block *sb, struct item_cache *cac, - struct cached_item *item) -{ - if (WARN_ON_ONCE(RB_EMPTY_NODE(&item->node))) - return; - - if (!item_is_dirty(item)) - return; - - item->dirty &= ~ITEM_DIRTY; - list_add_tail(&item->entry, &cac->lru_list); - cac->lru_nr++; - - update_dirty_item_counts(sb, -1, -item->val_len); - - WARN_ON_ONCE(cac->nr_dirty_items < 0 || cac->dirty_val_bytes < 0); - - update_dirty_parents(item); -} - -static void item_referenced(struct item_cache *cac, struct cached_item *item) -{ - if (!item->dirty) - list_move_tail(&item->entry, &cac->lru_list); -} - -/* remove the item from its tracking data structures */ -static void unlink_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *item) -{ - clear_item_dirty(sb, cac, item); - rb_erase_augmented(&item->node, &cac->items, &scoutfs_item_rb_cb); - RB_CLEAR_NODE(&item->node); - if (!list_empty(&item->entry)) { - list_del_init(&item->entry); - cac->lru_nr--; - } -} - -/* - * Safely erase an item from the tree. Make sure to remove its dirty - * accounting, use the augmented erase, and free it. - */ -static void erase_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *item) -{ - trace_scoutfs_erase_item(sb, item); - - unlink_item(sb, cac, item); - free_item(sb, item); -} - -/* - * Delete an item from the cache. If it wasn't persistent we can just - * free the item. The caller must not try to use the item after calling - * this. - * - * If it was persistent we have to write a deletion item so that - * compaction will remove the old item. We only need the key for the - * deletion item so we can free the value. - */ -static void delete_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *item) -{ - if (!item->persistent) { - erase_item(sb, cac, item); - return; - } - - /* uses val_len to update item accounting */ - clear_item_dirty(sb, cac, item); - - kfree(item->val); - item->val = NULL; - item->val_len = 0; - - item->deletion = 1; - mark_item_dirty(sb, cac, item); - scoutfs_inc_counter(sb, item_delete); -} - -/* - * Try to add an item to the cache. The caller is responsible for - * marking the newly inserted item dirty. - * - * We distinguish between callers seeing trying to insert a new logical - * item and others trying to populate the cache. - * - * New logical item creators have made sure the items are participating - * in consistent locking. It's safe for them to clobber dirty deletion - * items with a new version of the item. The newly inserted item needs - * to retain the persistence of the item it replaces so that if it is later - * deleted it will still write a deletion item. - * - * Cache readers can only populate items that weren't present already. - * In particular, they absolutely cannot replace dirty old inode index items - * with the old version that was just deleted (outside of range caching and - * locking consistency). - */ -static int insert_item(struct super_block *sb, struct item_cache *cac, - struct cached_item *ins, bool logical_overwrite, - bool cache_populate) -{ - struct rb_root *root = &cac->items; - struct cached_item *item; - struct rb_node *parent; - struct rb_node **node; - int cmp; - -restart: - node = &root->rb_node; - parent = NULL; - while (*node) { - parent = *node; - item = container_of(*node, struct cached_item, node); - - cmp = scoutfs_key_compare(&ins->key, &item->key); - if (cmp < 0) { - if (ins->dirty) - item->dirty |= LEFT_DIRTY; - node = &(*node)->rb_left; - } else if (cmp > 0) { - if (ins->dirty) - item->dirty |= RIGHT_DIRTY; - node = &(*node)->rb_right; - } else { - if (cache_populate || - (!item->deletion && !logical_overwrite)) - return -EEXIST; - - /* sadly there's no augmented replace */ - erase_item(sb, cac, item); - if (item->persistent) - ins->persistent = 1; - goto restart; - } - } - - trace_scoutfs_item_insertion(sb, &ins->key); - - rb_link_node(&ins->node, parent, node); - rb_insert_augmented(&ins->node, root, &scoutfs_item_rb_cb); - - BUG_ON(item_is_dirty(ins)); - list_add_tail(&ins->entry, &cac->lru_list); - cac->lru_nr++; - - return 0; -} - -static struct cached_range *rb_first_rng(struct rb_root *root) -{ - struct rb_node *node; - - if ((node = rb_first(root))) - return container_of(node, struct cached_range, node); - - return NULL; -} - -static struct cached_range *rb_next_rng(struct cached_range *rng) -{ - struct rb_node *node; - - if (rng && (node = rb_next(&rng->node))) - return container_of(node, struct cached_range, node); - - return NULL; -} - -static struct cached_range *walk_ranges(struct rb_root *root, - struct scoutfs_key *key, - struct cached_range **prev, - struct cached_range **next) -{ - struct rb_node *node = root->rb_node; - struct cached_range *rng; - int cmp; - - if (prev) - *prev = NULL; - if (next) - *next = NULL; - - while (node) { - rng = container_of(node, struct cached_range, node); - - cmp = scoutfs_key_compare_ranges(key, key, - &rng->start, &rng->end); - if (cmp < 0) { - if (next) - *next = rng; - node = node->rb_left; - } else if (cmp > 0) { - if (prev) - *prev = rng; - node = node->rb_right; - } else { - return rng; - } - } - - return NULL; -} - -/* - * Return true if the given key is covered by a cached range. start and - * end are set to the existing cached range. - * - * Return false if the key is not covered by a range. start and end are - * set to zero. (Nothing uses these today, this is to avoid tracing - * uninitialized keys in this case.) - */ -static bool check_range(struct super_block *sb, struct rb_root *root, - struct scoutfs_key *key, struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_range *rng; - - rng = walk_ranges(&cac->ranges, key, NULL, NULL); - if (rng) { - scoutfs_inc_counter(sb, item_range_hit); - if (start) - *start = rng->start; - if (end) - *end = rng->end; - return true; - } - - if (start) - scoutfs_key_set_zeros(start); - if (end) - scoutfs_key_set_zeros(end); - - scoutfs_inc_counter(sb, item_range_miss); - return false; -} - -static void free_range(struct super_block *sb, struct cached_range *rng) -{ - if (!IS_ERR_OR_NULL(rng)) { - scoutfs_inc_counter(sb, item_range_free); - trace_range(free, sb, rng); - kfree(rng); - } -} - -/* - * Insert a new cached range. It might overlap with any number of - * existing cached ranges. As we descend we combine with and free any - * overlapping ranges before restarting the descent. - * - * We're responsible for the ins allocation. We free it if we don't - * insert it in the tree. - */ -static void insert_range(struct super_block *sb, struct rb_root *root, - struct cached_range *ins) -{ - struct cached_range *rng; - struct rb_node *parent; - struct rb_node **node; - int start_cmp; - int end_cmp; - int cmp; - - scoutfs_inc_counter(sb, item_range_insert); - -restart: - parent = NULL; - node = &root->rb_node; - while (*node) { - parent = *node; - rng = container_of(*node, struct cached_range, node); - - cmp = scoutfs_key_compare_ranges(&ins->start, &ins->end, - &rng->start, &rng->end); - /* simple iteration until we overlap */ - if (cmp < 0) { - node = &(*node)->rb_left; - continue; - } else if (cmp > 0) { - node = &(*node)->rb_right; - continue; - } - - start_cmp = scoutfs_key_compare(&ins->start, &rng->start); - end_cmp = scoutfs_key_compare(&ins->end, &rng->end); - - /* free our insertion if we're entirely within an existing */ - if (start_cmp >= 0 && end_cmp <= 0) { - free_range(sb, ins); - return; - } - - /* expand to cover partial overlap before freeing */ - if (start_cmp < 0 && end_cmp < 0) - swap(ins->end, rng->end); - else if (start_cmp > 0 && end_cmp > 0) - swap(ins->start, rng->start); - - /* remove and free all overlaps and restart the descent */ - rb_erase(&rng->node, root); - free_range(sb, rng); - goto restart; - } - - trace_range(ins_rb_insert, sb, ins); - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); -} - -/* - * Remove a given cached range. The caller has already removed all the - * items that fell within the range. There can be any number of - * existing cached ranges that overlap with the range that should be - * removed. - * - * The caller's range has full precision keys that specify the endpoints - * that will not be considered cached. If we use them to set the new - * bounds of existing ranges then we have to dec/inc them into the range - * to have them represent the last/first valid key, not the first/last - * key to be removed. - * - * Like insert_, we're responsible for freeing the caller's range. We - * might insert it into the tree to track the other half of a range - * that's split by the removal. - */ -static void remove_range(struct super_block *sb, struct rb_root *root, - struct cached_range *rem) -{ - struct cached_range *rng; - struct rb_node *parent; - struct rb_node **node; - bool insert = false; - int start_cmp; - int end_cmp; - int cmp; - -restart: - parent = NULL; - node = &root->rb_node; - while (*node) { - parent = *node; - rng = container_of(*node, struct cached_range, node); - - cmp = scoutfs_key_compare_ranges(&rem->start, &rem->end, - &rng->start, &rng->end); - /* simple iteration until we overlap */ - if (cmp < 0) { - node = &(*node)->rb_left; - continue; - } else if (cmp > 0) { - node = &(*node)->rb_right; - continue; - } - - start_cmp = scoutfs_key_compare(&rem->start, &rng->start); - end_cmp = scoutfs_key_compare(&rem->end, &rng->end); - - /* remove the middle of an existing range, insert other half */ - if (start_cmp > 0 && end_cmp < 0) { - swap(rng->end, rem->start); - scoutfs_key_dec(&rng->end); - trace_range(remove_mid_left, sb, rng); - - swap(rem->start, rem->end); - scoutfs_key_inc(&rem->start); - insert = true; - goto restart; - } - - /* remove partial overlap from existing */ - if (start_cmp < 0 && end_cmp < 0) { - swap(rem->end, rng->start); - scoutfs_key_inc(&rng->start); - trace_range(remove_start, sb, rng); - continue; - } - - if (start_cmp > 0 && end_cmp > 0) { - swap(rem->start, rng->end); - scoutfs_key_dec(&rng->end); - trace_range(remove_end, sb, rng); - continue; - } - - /* erase and free existing surrounded by removal */ - rb_erase(&rng->node, root); - free_range(sb, rng); - goto restart; - } - - if (insert) { - trace_range(rem_rb_insert, sb, rem); - rb_link_node(&rem->node, parent, node); - rb_insert_color(&rem->node, root); - } else { - free_range(sb, rem); - } -} - -/* Return true if the lock protects the use of the key. */ -static bool lock_coverage(struct scoutfs_lock *lock, - struct scoutfs_key *key, int op_mode) -{ - signed char mode = ACCESS_ONCE(lock->mode); - - return ((op_mode == mode) || - (op_mode == SCOUTFS_LOCK_READ && - mode == SCOUTFS_LOCK_WRITE)) && - scoutfs_key_compare_ranges(key, key, - &lock->start, &lock->end) == 0; -} - -/* - * Find an item with the given key and copy its value into the caller's - * value vector. The amount of bytes copied is returned which can be 0 - * or truncated if the caller's buffer isn't big enough or if val is null. - * - * The end key limits how many keys after the search key can be read - * and inserted into the cache. - */ -int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) - return -EINVAL; - - trace_scoutfs_item_lookup(sb, key); - - do { - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - item_referenced(cac, item); - if (val) - ret = copy_item_val(val, item); - else - ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENOENT; - } else { - ret = -ENODATA; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); - - trace_scoutfs_item_lookup_ret(sb, ret); - return ret; -} - -/* - * This requires that the item at the specified key has a value of the - * same length as the caller's value buffer. Callers are asserting that - * mismatched size are corruption so it returns -EIO if the sizes don't - * match. This isn't the fast path so we don't mind the copying - * overhead that comes from only detecting the size mismatch after the - * copy by reusing the more permissive _lookup(). - * - * The end key limits how many keys after the search key can be read and - * inserted into the cache. - * - * Returns 0 or -errno. - */ -int scoutfs_item_lookup_exact(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock) -{ - int ret; - - ret = scoutfs_item_lookup(sb, key, val, lock); - if (ret == val->iov_len) - ret = 0; - else if (ret >= 0) - ret = -EIO; - - return ret; -} - -/* - * Return the next linked node in the tree that isn't a deletion item - * and which is still within the last allowed key value. - */ -static struct cached_item *next_item_node(struct rb_root *root, - struct cached_item *item, - struct scoutfs_key *last) -{ - struct rb_node *node; - - while (item) { - node = rb_next(&item->node); - if (!node) { - item = NULL; - break; - } - - item = container_of(node, struct cached_item, node); - - if (scoutfs_key_compare(&item->key, last) > 0) { - item = NULL; - break; - } - - if (!item->deletion) - break; - } - - return item; -} - -/* - * Find the next item to return from the "_next" item interface. It's the - * next item from the key that isn't a deletion item and is within the - * bounds of the end of the cache and the caller's last key. - */ -static struct cached_item *item_for_next(struct rb_root *root, - struct scoutfs_key *key, - struct scoutfs_key *range_end, - struct scoutfs_key *last) -{ - struct cached_item *item; - - /* limit by the lesser of the two */ - if (range_end && scoutfs_key_compare(range_end, last) < 0) - last = range_end; - - item = next_item(root, key); - if (item) { - if (scoutfs_key_compare(&item->key, last) > 0) - item = NULL; - else if (item->deletion) - item = next_item_node(root, item, last); - } - - return item; -} - -/* - * Return the next item starting with the given key and returning the - * last key at most. - * - * The range covered by the lock also limits the last item that can be - * returned. -ENOENT can be returned when there are no next items - * covered by the lock but there are still items before the last key - * outside of the lock. The caller needs to know to reacquire the next - * lock to continue iteration. - * - * -ENOENT is returned if there are no items between the given and last - * keys inside the range covered by the lock. - * - * The next item's key is copied to the caller's key. The caller is - * responsible for dealing with key lengths and truncation. - * - * The next item's value is copied into the callers value. The number - * of value bytes copied is returned. The copied value can be truncated - * by the caller's value buffer length. - */ -int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *last, struct kvec *val, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct scoutfs_key pos; - struct scoutfs_key range_end; - struct cached_item *item; - unsigned long flags; - bool cached; - int ret; - - /* use the end key as the last key if it's closer to reduce compares */ - if (scoutfs_key_compare(&lock->end, last) < 0) - last = &lock->end; - - /* convenience to avoid searching if caller iterates past their last */ - if (scoutfs_key_compare(key, last) > 0) { - ret = -ENOENT; - goto out; - } - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) { - ret = -EINVAL; - goto out; - } - - pos = *key; - - spin_lock_irqsave(&cac->lock, flags); - - for(;;) { - /* see if we have cache coverage of our iterator pos */ - cached = check_range(sb, &cac->ranges, &pos, NULL, &range_end); - - trace_scoutfs_item_next_range_check(sb, !!cached, key, - &pos, last, &lock->end, - &range_end); - - if (!cached) { - /* populate missing cached range starting at pos */ - spin_unlock_irqrestore(&cac->lock, flags); - - ret = scoutfs_manifest_read_items(sb, &pos, - &lock->start, - &lock->end); - - spin_lock_irqsave(&cac->lock, flags); - if (ret) - break; - else - continue; - } - - /* see if there's an item in the cached range from pos */ - item = item_for_next(&cac->items, &pos, &range_end, last); - if (!item) { - if (scoutfs_key_compare(&range_end, last) < 0) { - /* keep searching after empty cached range */ - pos = range_end; - scoutfs_key_inc(&pos); - continue; - } - - /* no item and cache covers last, done */ - ret = -ENOENT; - break; - } - - /* we have a next item inside the cached range, done */ - *key = item->key; - if (val) { - item_referenced(cac, item); - ret = copy_item_val(val, item); - } else { - ret = 0; - } - break; - } - - spin_unlock_irqrestore(&cac->lock, flags); -out: - - trace_scoutfs_item_next_ret(sb, ret); - return ret; -} - -/* - * Return the prev linked node in the tree that isn't a deletion item - * and which is still within the first allowed key value. - */ -static struct cached_item *prev_item_node(struct rb_root *root, - struct cached_item *item, - struct scoutfs_key *first) -{ - struct rb_node *node; - - while (item) { - node = rb_prev(&item->node); - if (!node) { - item = NULL; - break; - } - - item = container_of(node, struct cached_item, node); - - if (scoutfs_key_compare(&item->key, first) < 0) { - item = NULL; - break; - } - - if (!item->deletion) - break; - } - - return item; -} - -/* - * Find the prev item to return from the "_prev" item interface. It's the - * prev item from the key that isn't a deletion item and is within the - * bounds of the start of the cache and the caller's first key. - */ -static struct cached_item *item_for_prev(struct rb_root *root, - struct scoutfs_key *key, - struct scoutfs_key *range_start, - struct scoutfs_key *first) -{ - struct cached_item *item; - - /* limit by the greater of the two */ - if (range_start && scoutfs_key_compare(range_start, first) > 0) - first = range_start; - - item = prev_item(root, key); - if (item) { - if (scoutfs_key_compare(&item->key, first) < 0) - item = NULL; - else if (item->deletion) - item = prev_item_node(root, item, first); - } - - return item; -} - -/* - * Return the prev item starting with the given key and returning the - * first key at least. - * - * The range covered by the lock also limits the first item that can be - * returned. -ENOENT can be returned when there are no prev items - * covered by the lock but there are still items after the first key - * outside of the lock. The caller needs to know to reacquire the next - * lock to continue iteration. - * - * -ENOENT is returned if there are no items between the given and - * first key inside the range covered by the lock. - * - * The prev item's key is copied to the caller's key. The caller is - * responsible for dealing with key lengths and truncation. - * - * The prev item's value is copied into the callers value. The number - * of value bytes copied is returned. The copied value can be truncated - * by the caller's value buffer length. - */ -int scoutfs_item_prev(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *first, struct kvec *val, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct scoutfs_key range_start; - struct scoutfs_key pos; - struct cached_item *item; - unsigned long flags; - bool cached; - int ret; - - /* use the start key as the first key if it's closer */ - if (scoutfs_key_compare(&lock->start, first) > 0) - first = &lock->start; - - /* convenience to avoid searching if caller iterates past their last */ - if (scoutfs_key_compare(key, first) < 0) { - ret = -ENOENT; - goto out; - } - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_READ))) { - ret = -EINVAL; - goto out; - } - - pos = *key; - - spin_lock_irqsave(&cac->lock, flags); - - for(;;) { - /* see if we have cache coverage of our iterator pos */ - cached = check_range(sb, &cac->ranges, &pos, - &range_start, NULL); - - trace_scoutfs_item_prev_range_check(sb, !!cached, key, - &pos, first, &lock->start, - &range_start); - - if (!cached) { - /* populate missing cached range starting at pos */ - spin_unlock_irqrestore(&cac->lock, flags); - - ret = scoutfs_manifest_read_items(sb, &pos, - &lock->start, - &lock->end); - - spin_lock_irqsave(&cac->lock, flags); - if (ret) - break; - else - continue; - } - - /* see if there's an item in the cached range from pos */ - item = item_for_prev(&cac->items, &pos, &range_start, first); - if (!item) { - if (scoutfs_key_compare(&range_start, first) > 0) { - /* keep searching before empty cached range */ - pos = range_start; - scoutfs_key_dec(&pos); - continue; - } - - /* no item and cache covers first, done */ - ret = -ENOENT; - break; - } - - /* we have a prev item inside the cached range, done */ - *key = item->key; - if (val) { - item_referenced(cac, item); - ret = copy_item_val(val, item); - } else { - ret = 0; - } - break; - } - - spin_unlock_irqrestore(&cac->lock, flags); -out: - - trace_scoutfs_item_prev_ret(sb, ret); - return ret; -} - -/* - * Create a new dirty item in the cache. Returns -EEXIST if an item - * already exists with the given key. - */ -int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (invalid_key_val(key, val) || - WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { - ret = -EINVAL; - goto out; - } - - item = alloc_item(sb, key, val); - if (!item) { - ret = -ENOMEM; - goto out; - } - - do { - spin_lock_irqsave(&cac->lock, flags); - - if (!check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENODATA; - } else { - ret = insert_item(sb, cac, item, false, false); - if (!ret) { - scoutfs_inc_counter(sb, item_create); - mark_item_dirty(sb, cac, item); - } - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); - - if (ret) - free_item(sb, item); - -out: - trace_scoutfs_item_create(sb, key, ret); - return ret; -} - -/* - * "force" an item creation without first reading to see if the item - * exist. The caller is asserting that they know it's correct to - * overwrite a possibly existing item with this newly created item. - * - * Because this can be overwriting an existing item we need to be sure - * that we write a deletion item if it's deleted so we force its - * persistent flag. - */ -int scoutfs_item_create_force(struct super_block *sb, - struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (invalid_key_val(key, val)) - return -EINVAL; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE_ONLY))) - return -EINVAL; - - item = alloc_item(sb, key, val); - if (!item) - return -ENOMEM; - - item->persistent = 1; - - spin_lock_irqsave(&cac->lock, flags); - - ret = insert_item(sb, cac, item, true, false); - if (ret) { - printk(KERN_EMERG "Scoutfs: corrupted item cache found while" - " creating item "SK_FMT" on fs %llu\n", SK_ARG(key), - le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); - BUG_ON(ret); - } - scoutfs_inc_counter(sb, item_create); - mark_item_dirty(sb, cac, item); - - spin_unlock_irqrestore(&cac->lock, flags); - - if (ret) - free_item(sb, item); - - return ret; -} - -/* - * Allocate an item with the key and value and add it to the list of - * items to be inserted as a batch later. The caller adds in sort order - * and we add with _tail to maintain that order. - */ -int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key *key, struct kvec *val) -{ - struct cached_item *item; - int ret; - - if (invalid_key_val(key, val)) - return -EINVAL; - - item = alloc_item(sb, key, val); - if (item) { - list_add_tail(&item->entry, list); - ret = 0; - } else { - ret = -ENOMEM; - } - - return ret; -} - - -/* - * Insert a batch of clean read items from segments into the item cache. - * - * The caller hasn't been locked so the cached items could have changed - * since they were asked to read. If there are duplicates in the item - * cache they might be newer than what was read so we must drop them on - * the floor. - * - * The batch atomically adds the items and updates the cached range to - * include the callers range that covers the items. - * - * It's safe to re-add items to the batch list after they aren't - * inserted because _safe iteration will always be past the head entry - * that will be inserted. - */ -int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_range *rng; - struct cached_item *item; - struct cached_item *tmp; - unsigned long flags; - int ret; - - trace_scoutfs_item_insert_batch(sb, start, end); - - if (WARN_ON_ONCE(scoutfs_key_compare(start, end) > 0)) - return -EINVAL; - - scoutfs_inc_counter(sb, item_range_alloc); - rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); - if (!rng) { - free_range(sb, rng); - ret = -ENOMEM; - goto out; - } - - rng->start = *start; - rng->end = *end; - - spin_lock_irqsave(&cac->lock, flags); - - insert_range(sb, &cac->ranges, rng); - - list_for_each_entry_safe(item, tmp, list, entry) { - list_del_init(&item->entry); - item->persistent = 1; - if (insert_item(sb, cac, item, false, true)) { - scoutfs_inc_counter(sb, item_batch_duplicate); - list_add(&item->entry, list); - } else { - scoutfs_inc_counter(sb, item_batch_inserted); - } - } - - spin_unlock_irqrestore(&cac->lock, flags); - - ret = 0; -out: - scoutfs_item_free_batch(sb, list); - return ret; -} - -void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list) -{ - struct cached_item *item; - struct cached_item *tmp; - - list_for_each_entry_safe(item, tmp, list, entry) { - list_del_init(&item->entry); - free_item(sb, item); - } -} - - -/* - * If the item exists make sure it's dirty and pinned. It can be read - * if it wasn't cached. -ENOENT is returned if the item doesn't exist. - */ -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) - return -EINVAL; - - do { - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - mark_item_dirty(sb, cac, item); - ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENOENT; - } else { - ret = -ENODATA; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); - - trace_scoutfs_item_dirty_ret(sb, ret); - return ret; -} - -/* - * Set the value of an existing item in the tree. The item is marked dirty - * and the previous value is freed. The provided value may be null. - * - * Returns -ENOENT if the item doesn't exist. - */ -int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - void *up_val = NULL; - int ret; - - if (invalid_key_val(key, val)) - return -EINVAL; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) - return -EINVAL; - - if (val) { - up_val = kmalloc(val->iov_len, GFP_NOFS); - if (!up_val) { - ret = -ENOMEM; - goto out; - } - memcpy(up_val, val->iov_base, val->iov_len); - } - - do { - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - clear_item_dirty(sb, cac, item); - swap(up_val, item->val); - item->val_len = val ? val->iov_len : 0; - mark_item_dirty(sb, cac, item); - ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENOENT; - } else { - ret = -ENODATA; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); -out: - kfree(up_val); - - trace_scoutfs_item_update_ret(sb, ret); - return ret; -} - -/* - * Delete an existing item with the given key. - * - * If a non-deletion item is present then we mark it dirty and deleted - * and free it's value. - * - * Returns -ENOENT if an item doesn't exist at the key. This forces us - * to read the item before creating a deletion item for it. XXX If we - * relaxed this we'd need to see if callers make use of -ENOENT and if - * there are any ways for userspace to overwhelm the system with - * deletion items for items that didn't exist in the first place. - */ -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { - ret = -EINVAL; - goto out; - } - - do { - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - delete_item(sb, cac, item); - ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENOENT; - } else { - ret = -ENODATA; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); - -out: - trace_scoutfs_item_delete(sb, key, ret); - return ret; -} - -/* - * "force" a deletion by creating a deletion item without first reading - * the existing item. - * - * The caller knows that there is an existing item but doesn't want to - * pay the cost of reading it before writing a deletion item. We mark - * the allocated deletion item persistent to ensure that it's written. - */ -int scoutfs_item_delete_force(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - int ret; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE_ONLY))) - return -EINVAL; - - item = alloc_item(sb, key, NULL); - if (!item) - return -ENOMEM; - - item->persistent = 1; - - spin_lock_irqsave(&cac->lock, flags); - ret = insert_item(sb, cac, item, true, false); - if (ret) { - printk(KERN_EMERG "Scoutfs: corrupted item cache found while" - " deleting item "SK_FMT" on fs %llu\n", SK_ARG(key), - le64_to_cpu(SCOUTFS_SB(sb)->super.hdr.fsid)); - BUG_ON(ret); - } - scoutfs_inc_counter(sb, item_create); - mark_item_dirty(sb, cac, item); - - delete_item(sb, cac, item); - spin_unlock_irqrestore(&cac->lock, flags); - - return ret; -} - -/* - * Delete an item and give it to the caller so that they can restore it - * later. - * - * The deleted items can be dirty or not.. we maintain an accurate dirty - * count as we remove the deleted items and leave their dirty flag set - * so that restore can mark them dirty again. - * - * Returns -ENOENT if the item didn't exist and couldn't be deleted. - */ -int scoutfs_item_delete_save(struct super_block *sb, - struct scoutfs_key *key, - struct list_head *list, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - struct cached_item *del; - unsigned long flags; - bool was_dirty; - int ret; - - if (WARN_ON_ONCE(!lock_coverage(lock, key, SCOUTFS_LOCK_WRITE))) { - ret = -EINVAL; - goto out; - } - - del = alloc_item(sb, key, NULL); - if (!del) { - ret = -ENOMEM; - goto out; - } - - do { - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) { - was_dirty = item_is_dirty(item); - unlink_item(sb, cac, item); - list_add_tail(&item->entry, list); - if (was_dirty) - item->dirty |= ITEM_DIRTY; - - del->persistent = item->persistent; - ret = insert_item(sb, cac, del, false, false); - BUG_ON(ret); - delete_item(sb, cac, del); - del = NULL; - ret = 0; - } else if (check_range(sb, &cac->ranges, key, NULL, NULL)) { - ret = -ENOENT; - } else { - ret = -ENODATA; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - } while (ret == -ENODATA && - (ret = scoutfs_manifest_read_items(sb, key, &lock->start, - &lock->end)) == 0); - - free_item(sb, del); -out: - trace_scoutfs_item_delete_save(sb, key, ret); - return ret; -} - -/* - * Restore a set of previousl saved items. They're returned to the - * cached and marked dirty if they were dirty when they were saved. - * Restored items completely overwrite any existing cached items. - * - * The caller must have held locks covering the save and restore so that - * the cached ranges still exist. - */ -int scoutfs_item_restore(struct super_block *sb, struct list_head *list, - struct scoutfs_lock *lock) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *existing; - struct cached_item *item; - struct cached_item *tmp; - unsigned long flags; - bool was_dirty; - int mode; - int ret; - - if (list_empty(list)) - return 0; - - spin_lock_irqsave(&cac->lock, flags); - - /* make sure all the items are locked and cached */ - list_for_each_entry(item, list, entry) { - mode = item_is_dirty(item) ? SCOUTFS_LOCK_WRITE : - SCOUTFS_LOCK_READ; - if (WARN_ON_ONCE(!lock_coverage(lock, &item->key, mode)) || - WARN_ON_ONCE(!check_range(sb, &cac->ranges, &item->key, - NULL, NULL))) { - ret = -EINVAL; - goto out; - } - } - - list_for_each_entry_safe(item, tmp, list, entry) { - was_dirty = item_is_dirty(item); - item->dirty &= ~ITEM_DIRTY; - list_del_init(&item->entry); - - existing = find_item(sb, &cac->items, &item->key); - if (existing) - erase_item(sb, cac, existing); - insert_item(sb, cac, item, false, false); - if (was_dirty) - mark_item_dirty(sb, cac, item); - } - - ret = 0; -out: - spin_unlock_irqrestore(&cac->lock, flags); - - return ret; -} - -/* - * Delete an item that the caller knows must be dirty because they hold - * locks and the transaction and have created or dirtied it. This can't - * fail. - */ -void scoutfs_item_delete_dirty(struct super_block *sb, - struct scoutfs_key *key) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - if (item) - delete_item(sb, cac, item); - - spin_unlock_irqrestore(&cac->lock, flags); -} - -/* - * Copy the callers value into the dirty item and truncate its value if - * the existing value is longer. The caller must have ensured that the - * item was dirty and had a large enough value. If the updated value is - * smaller then it will sit in the larger item allocation until the - * value is eventually freed along with the item. - */ -void scoutfs_item_update_dirty(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - unsigned long flags; - unsigned int new_len = val ? val->iov_len : 0; - signed delta; - - spin_lock_irqsave(&cac->lock, flags); - - item = find_item(sb, &cac->items, key); - - BUG_ON(!item || !item_is_dirty(item) || new_len > item->val_len); - - delta = new_len - item->val_len; - if (val) - memcpy(item->val, val->iov_base, new_len); - item->val_len = new_len; - update_dirty_item_counts(sb, 0, delta); - - spin_unlock_irqrestore(&cac->lock, flags); -} - -/* - * Return the first dirty node in the subtree starting at the given node. - */ -static struct cached_item *first_dirty(struct rb_node *node) -{ - struct cached_item *ret = NULL; - struct cached_item *item; - - while (node) { - item = container_of(node, struct cached_item, node); - - if (item->dirty & LEFT_DIRTY) { - node = item->node.rb_left; - } else if (item_is_dirty(item)) { - ret = item; - break; - } else if (item->dirty & RIGHT_DIRTY) { - node = item->node.rb_right; - } else { - break; - } - } - - return ret; -} - -/* - * Find the next dirty item after a given item. First we see if we have - * a dirty item in our right subtree. If not we ascend through parents - * skipping those that are less than us. If we find a parent that's - * greater than us then we see if it's dirty, if not we start the search - * all over again by checking its right subtree then ascending. - */ -static struct cached_item *next_dirty(struct cached_item *item) -{ - struct rb_node *parent; - struct rb_node *node; - - while (item) { - if (item->dirty & RIGHT_DIRTY) - return first_dirty(item->node.rb_right); - - /* find next greatest parent */ - node = &item->node; - while ((parent = rb_parent(node)) && parent->rb_right == node) - node = parent; - if (!parent) - break; - - /* done if our next greatest parent itself is dirty */ - item = container_of(parent, struct cached_item, node); - if (item_is_dirty(item)) - return item; - - /* continue to check right subtree */ - } - - return NULL; -} - -static bool dirty_item_within(struct rb_root *root, - struct scoutfs_key *from, - struct scoutfs_key *end) -{ - struct cached_item *item; - - item = next_item(root, from); - if (item && !item_is_dirty(item)) - item = next_dirty(item); - - return item && scoutfs_key_compare(&item->key, end) <= 0; -} - -bool scoutfs_item_has_dirty(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - unsigned long flags; - bool has; - - spin_lock_irqsave(&cac->lock, flags); - has = cac->nr_dirty_items != 0; - spin_unlock_irqrestore(&cac->lock, flags); - - return has; -} - -/* - * Return true if the item cache covers the given range. If dirty is - * provided then we only return true if there are dirty items in the - * range. - * - * If the start of the query range doesn't overlap a cached range then - * we see if the next cached range starts before the end of the query range. - */ -bool scoutfs_item_range_cached(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end, bool dirty) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_range *next; - struct cached_range *rng; - unsigned long flags; - bool cached = false; - - spin_lock_irqsave(&cac->lock, flags); - - if (dirty) { - if (dirty_item_within(&cac->items, start, end)) - cached = true; - } else { - rng = walk_ranges(&cac->ranges, start, NULL, &next); - if (rng || - (next && scoutfs_key_compare(&next->start, end) <= 0)) - cached = true; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - return cached; -} - -/* - * Returns true if adding more items with the given count, keys, and values - * still fits in a single item along with the current dirty items. - */ -bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, - u32 val_bytes) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - unsigned long flags; - bool fits; - - spin_lock_irqsave(&cac->lock, flags); - fits = scoutfs_seg_fits_single(nr_items + cac->nr_dirty_items, - val_bytes + cac->dirty_val_bytes); - spin_unlock_irqrestore(&cac->lock, flags); - - return fits; -} - -/* - * Fill the given segment with sorted dirty items. - * - * The caller is responsible for the consistency of the dirty items once - * they're in its seg. We can consider them clean once we store them. - * - * XXX this first/append pattern will go away once we can write a stream - * of items to a segment without needing to know the item count to - * find the starting key and value offsets. - */ -int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - __le32 *links[SCOUTFS_MAX_SKIP_LINKS]; - struct cached_item *item = NULL; - struct cached_item *del; - unsigned long flags; - struct kvec val; - bool appended; - - spin_lock_irqsave(&cac->lock, flags); - - item = first_dirty(cac->items.rb_node); - while (item) { - kvec_init(&val, item->val, item->val_len); - appended = scoutfs_seg_append_item(sb, seg, &item->key, &val, - item_flags(item), links); - /* trans reservation should have limited dirty */ - BUG_ON(!appended); - - if (item->deletion) - scoutfs_inc_counter(sb, trans_write_deletion_item); - else - scoutfs_inc_counter(sb, trans_write_item); - - /* non-persistent should have been freed (safe to write) */ - WARN_ON_ONCE(item->deletion && !item->persistent); - - clear_item_dirty(sb, cac, item); - item->persistent = 1; - - del = item; - item = next_dirty(item); - - if (del->deletion) - erase_item(sb, cac, del); - } - - spin_unlock_irqrestore(&cac->lock, flags); - - return 0; -} - -/* - * The caller wants us to write out any dirty items within the given - * range. We look for any dirty items within the range and if we find - * any we issue a sync which writes out all the dirty items. - * - * Returns a sync error or the number of dirty items written. - */ -int scoutfs_item_writeback(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - unsigned long flags; - bool sync = false; - int count = 0; - int ret = 0; - - /* XXX think about racing with trans write */ - - spin_lock_irqsave(&cac->lock, flags); - - if (cac->nr_dirty_items && dirty_item_within(&cac->items, start, end)) { - sync = true; - count = cac->nr_dirty_items; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - if (sync) { - scoutfs_inc_counter(sb, trans_commit_item_flush); - ret = scoutfs_trans_sync(sb, 1); - } - - return ret ?: count; -} - -/* - * The caller wants us to drop any items within the range on the floor. - * They should have ensured that items in this range won't be dirty. - * - * Returns errors or the count of the items invalidated. - */ -int scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_range *rng; - struct cached_item *next; - struct cached_item *item; - struct rb_node *node; - unsigned long flags; - int count = 0; - int ret; - - trace_scoutfs_item_invalidate_range(sb, start, end); - - /* XXX think about racing with trans write */ - - scoutfs_inc_counter(sb, item_range_alloc); - rng = kzalloc(sizeof(struct cached_range), GFP_NOFS); - if (!rng) { - free_range(sb, rng); - ret = -ENOMEM; - goto out; - } - - rng->start = *start; - rng->end = *end; - - spin_lock_irqsave(&cac->lock, flags); - - for (item = next_item(&cac->items, start); - item && scoutfs_key_compare(&item->key, end) <= 0; - item = next) { - - /* XXX seems like this should be a helper? */ - node = rb_next(&item->node); - if (node) - next = container_of(node, struct cached_item, node); - else - next = NULL; - - WARN_ON_ONCE(item_is_dirty(item)); - erase_item(sb, cac, item); - count++; - } - - remove_range(sb, &cac->ranges, rng); - - spin_unlock_irqrestore(&cac->lock, flags); - - ret = 0; -out: - return ret ?: count; -} - -static struct cached_item *rb_next_item(struct cached_item *item) -{ - struct rb_node *node; - - if (item && (node = rb_next(&item->node))) - return container_of(node, struct cached_item, node); - - return NULL; -} - -static struct cached_item *rb_prev_item(struct cached_item *item) -{ - struct rb_node *node; - - if (item && (node = rb_prev(&item->node))) - return container_of(node, struct cached_item, node); - - return NULL; -} - -/* - * Find the bounds of an item cache shrinking operation. Starting from - * an item, walk through either next items to the right or prev items to - * the left. Record items that are valid final shrinking points because - * using their key for a new range end doesn't cross the remaining - * existing item. We stop if we check enough items, hit a dirty item, - * or run out of items in the range. - * - * We only can't use an item as a new range end point if moving its key - * crosses the next item in the cache. This only happens when smaller - * items share a prefix with the next larger item. This only happens - * for item populations with names (dirents, xattrs) that share - * prefixes. We really don't want to be unable to reclaim so we - * aggressively try to walk past all of them. - */ -#define BOUNDARY_MIN 32 -#define BOUNDARY_MAX 300 -static struct cached_item *shrink_boundary(struct super_block *sb, - struct cached_item *item, - struct cached_item **next_ret, - struct scoutfs_key *end, - bool right) -{ - struct cached_item *found = NULL; - struct cached_item *next; - bool cmp; - int i; - - *next_ret = NULL; - - for (i = 0; i < BOUNDARY_MAX; i++) { - if (right) - next = rb_next_item(item); - else - next = rb_prev_item(item); - - if (next) { - if (right) - cmp = scoutfs_key_compare(&next->key, end) > 0; - else - cmp = scoutfs_key_compare(&next->key, end) < 0; - } else { - cmp = true; - } - if (cmp) { - scoutfs_inc_counter(sb, item_shrink_range_end); - found = item; - *next_ret = NULL; - break; - } - - if (right) { - scoutfs_key_inc(&item->key); - cmp = scoutfs_key_compare(&item->key, &next->key) <= 0; - scoutfs_key_dec(&item->key); - } else { - scoutfs_key_dec(&item->key); - cmp = scoutfs_key_compare(&item->key, &next->key) >= 0; - scoutfs_key_inc(&item->key); - } - if (cmp) { - found = item; - *next_ret = next; - if (i >= BOUNDARY_MIN) - break; - } - - if (item_is_dirty(next)) { - scoutfs_inc_counter(sb, item_shrink_next_dirty); - break; - } - - item = next; - } - - return found; -} - -/* - * The caller found an item in the lru and the range it falls within. - * This frees items around the item. After finding the boundaries we - * have to either update the ranges if items remain or free the item. - * - * We're in the context of a shrinker so we can't allocate. If we - * remove items from the middle of a range we use the memory from some - * removed items to store the new split range. - */ -static int shrink_around(struct super_block *sb, struct cached_range *rng, - struct cached_item *item) -{ - struct item_cache *cac = SCOUTFS_SB(sb)->item_cache; - struct scoutfs_key rng_end; - struct scoutfs_key key; - struct cached_range *new_rng; - struct cached_item *first; - struct cached_item *last; - struct cached_item *prev; - struct cached_item *next; - int nr = 0; - - /* we're re-using item memory as ranges :P */ - BUILD_BUG_ON(sizeof(struct cached_item) < sizeof(struct cached_range)); - - first = shrink_boundary(sb, item, &prev, &rng->start, false); - last = shrink_boundary(sb, item, &next, &rng->end, true); - - trace_scoutfs_item_shrink_around(sb, &rng->start, &rng->end, &item->key, - prev ? &prev->key : NULL, - first ? &first->key : NULL, - last ? &last->key : NULL, - next ? &next->key : NULL); - - /* can't shrink if we can't use neighbours */ - if (!first || !last) { - scoutfs_inc_counter(sb, item_shrink_alone); - return 0; - } - - /* can't split if we don't have an item to use for the range */ - if (next && prev && (first == last)) { - scoutfs_inc_counter(sb, item_shrink_small_split); - return 0; - } - - /* set end of remaining existing range, save old for split or freeing */ - if (prev) { - rng_end = rng->end; - rng->end = first->key; - scoutfs_key_dec(&rng->end); - trace_range(shrink_end, sb, rng); - } - - /* set start of remaining existing range */ - if (next && !prev) { - rng->start = last->key; - scoutfs_key_inc(&rng->start); - trace_range(shrink_start, sb, rng); - } - - /* add new range, stealing existing end */ - if (next && prev) { - item = last; - last = rb_prev_item(last); - - unlink_item(sb, cac, item); - key = item->key; - kfree(item->val); - nr++; - - new_rng = (void *)item; - item = NULL; - memset(new_rng, 0, sizeof(struct cached_range)); - - new_rng->end = rng_end; - new_rng->start = key; - scoutfs_key_inc(&new_rng->start); - insert_range(sb, &cac->ranges, new_rng); - - scoutfs_inc_counter(sb, item_shrink_split_range); - } - - /* totally emptied the range */ - if (!prev && !next) { - rb_erase(&rng->node, &cac->ranges); - free_range(sb, rng); - } - - /* and finally shrink all the surrounding items */ - for (item = first; - item && (next = item == last ? NULL : rb_next_item(item), 1); - item = next) { - trace_scoutfs_item_shrink(sb, &item->key); - scoutfs_inc_counter(sb, item_shrink); - erase_item(sb, cac, item); - nr++; - } - - return nr; -} - -/* - * Shrink the item cache. - * - * Unfortunately this is complicated by the rbtree of ranges that track - * the validity of the cache. If we free items we have to make sure - * they're not covered by ranges or else they'd be considered a valid - * negative cache hit. We aggressively try to free items because if we - * have a structural pattern of keys that we can't free then those build - * up and fill memory. - * - * We can also hit items in the lru which aren't covered by ranges, we - * free those immediately. - */ -static int item_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) -{ - struct item_cache *cac = container_of(shrink, struct item_cache, - shrinker); - struct super_block *sb = cac->sb; - struct cached_range *rng; - struct cached_item *item; - struct cached_item *first_moved = NULL; - unsigned long flags; - unsigned long nr; - int ret; - - nr = sc->nr_to_scan; - if (nr == 0) - goto out; - - spin_lock_irqsave(&cac->lock, flags); - - while (nr && - (item = list_first_entry_or_null(&cac->lru_list, - struct cached_item, entry))) { - - /* can't have dirty items on the lru */ - BUG_ON(item_is_dirty(item)); - - /* if we're not in a range just shrink the item */ - rng = walk_ranges(&cac->ranges, &item->key, NULL, NULL); - if (!rng) { - scoutfs_inc_counter(sb, item_shrink_outside); - erase_item(sb, cac, item); - nr--; - continue; - } - - ret = shrink_around(sb, rng, item); - if (ret == 0) { - if (first_moved && first_moved == item) - break; - else if (!first_moved) - first_moved = item; - list_move_tail(&item->entry, &cac->lru_list); - continue; - } - - nr -= min_t(unsigned long, nr, ret); - } - - /* always try to free empty ranges */ - while (RB_EMPTY_ROOT(&cac->items) && - (rng = rb_first_rng(&cac->ranges))) { - scoutfs_inc_counter(sb, item_shrink_empty_range); - rb_erase(&rng->node, &cac->ranges); - free_range(sb, rng); - } - - spin_unlock_irqrestore(&cac->lock, flags); - -out: - ret = min_t(unsigned long, cac->lru_nr, INT_MAX); - trace_scoutfs_item_shrink_exit(sb, sc->nr_to_scan, ret); - return ret; -} - -/* - * Copy the keys of the sorted cached ranges starting with the search - * key into the caller's key array. The number of copied range keys is - * returned which will always be a multiple of two. - */ -int scoutfs_item_copy_range_keys(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_key *keys, unsigned nr) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct rb_node *node = cac->ranges.rb_node; - struct cached_range *next = NULL; - struct cached_range *rng; - unsigned long flags; - int ret = 0; - int cmp; - - spin_lock_irqsave(&cac->lock, flags); - - while (node) { - rng = container_of(node, struct cached_range, node); - - cmp = scoutfs_key_compare_ranges(key, key, - &rng->start, &rng->end); - if (cmp < 0) { - next = rng; - node = node->rb_left; - } else if (cmp > 0) { - node = node->rb_right; - } else { - next = rng; - break; - } - } - - for (rng = next; rng; rng = rb_next_rng(rng)) { - if (ret + 2 > nr) - break; - - keys[ret++] = rng->start; - keys[ret++] = rng->end; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - return ret; -} - -/* - * Copy keys for the sorted cached items starting with the search key - * into the caller's key array. The number of copied keys is returned. - */ -int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *keys, unsigned nr) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item = NULL; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&cac->lock, flags); - - for (item = next_item(&cac->items, key); item; - item = rb_next_item(item)) { - - if (ret == nr) - break; - - if (item->deletion) - continue; - - keys[ret++] = item->key; - } - - spin_unlock_irqrestore(&cac->lock, flags); - - return ret; -} - -int scoutfs_item_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac; - - cac = kzalloc(sizeof(struct item_cache), GFP_KERNEL); - if (!cac) - return -ENOMEM; - sbi->item_cache = cac; - - cac->sb = sb; - spin_lock_init(&cac->lock); - cac->items = RB_ROOT; - cac->ranges = RB_ROOT; - cac->shrinker.shrink = item_lru_shrink; - cac->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&cac->shrinker); - INIT_LIST_HEAD(&cac->lru_list); - - return 0; -} - -/* - * There's no more users of the items and ranges at this point. We can - * destroy them without locking and ignoring augmentation. - */ -void scoutfs_item_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct item_cache *cac = sbi->item_cache; - struct cached_item *item; - struct cached_item *pos_item; - struct cached_range *rng; - struct cached_range *pos_rng; - - if (cac) { - if (cac->shrinker.shrink == item_lru_shrink) - unregister_shrinker(&cac->shrinker); - - rbtree_postorder_for_each_entry_safe(item, pos_item, - &cac->items, node) { - RB_CLEAR_NODE(&item->node); - INIT_LIST_HEAD(&item->entry); - free_item(sb, item); - } - - rbtree_postorder_for_each_entry_safe(rng, pos_rng, - &cac->ranges, node) { - free_range(sb, rng); - } - - kfree(cac); - } -} diff --git a/kmod/src/item.h b/kmod/src/item.h deleted file mode 100644 index c281b895..00000000 --- a/kmod/src/item.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef _SCOUTFS_ITEM_H_ -#define _SCOUTFS_ITEM_H_ - -#include - -struct scoutfs_segment; -struct scoutfs_key; - -int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_lookup_exact(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *last, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_item_prev(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *first, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_create_force(struct super_block *sb, - struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock); -int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -void scoutfs_item_delete_dirty(struct super_block *sb, - struct scoutfs_key *key); -void scoutfs_item_update_dirty(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val); -int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock); -int scoutfs_item_delete_force(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_lock *lock); -int scoutfs_item_delete_save(struct super_block *sb, - struct scoutfs_key *key, - struct list_head *list, - struct scoutfs_lock *lock); -int scoutfs_item_restore(struct super_block *sb, struct list_head *list, - struct scoutfs_lock *lock); - -int scoutfs_item_add_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key *key, struct kvec *val); -int scoutfs_item_insert_batch(struct super_block *sb, struct list_head *list, - struct scoutfs_key *start, - struct scoutfs_key *end); -void scoutfs_item_free_batch(struct super_block *sb, struct list_head *list); - -bool scoutfs_item_has_dirty(struct super_block *sb); -bool scoutfs_item_range_cached(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end, bool dirty); -bool scoutfs_item_dirty_fits_single(struct super_block *sb, u32 nr_items, - u32 val_bytes); -int scoutfs_item_dirty_seg(struct super_block *sb, struct scoutfs_segment *seg); -int scoutfs_item_writeback(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end); -int scoutfs_item_invalidate(struct super_block *sb, - struct scoutfs_key *start, - struct scoutfs_key *end); - -int scoutfs_item_copy_range_keys(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_key *keys, unsigned nr); -int scoutfs_item_copy_keys(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *keys, unsigned nr); - -int scoutfs_item_setup(struct super_block *sb); -void scoutfs_item_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/manifest.c b/kmod/src/manifest.c deleted file mode 100644 index 74a250e8..00000000 --- a/kmod/src/manifest.c +++ /dev/null @@ -1,1297 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "kvec.h" -#include "seg.h" -#include "item.h" -#include "btree.h" -#include "cmp.h" -#include "compact.h" -#include "manifest.h" -#include "trans.h" -#include "counters.h" -#include "triggers.h" -#include "client.h" -#include "spbm.h" -#include "scoutfs_trace.h" - -/* - * Manifest entries are stored in the cow btrees in the persistently - * allocated ring of blocks in the shared device. This lets clients - * read consistent old versions of the manifest when it's safe to do so. - * - * Manifest entries are sorted first by level then by their first key. - * This enables the primary searches based on key value for looking up - * items in segments via the manifest. - */ - -struct manifest { - struct rw_semaphore rwsem; - u8 nr_levels; - - /* calculated on mount, const thereafter */ - u64 level_limits[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; - u64 compacts_pending[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; - struct scoutfs_spbm segno_busy; - - unsigned long flags; - - struct scoutfs_key compact_keys[SCOUTFS_MANIFEST_MAX_LEVEL + 1]; -}; - -#define MANI_FLAG_LEVEL0_FULL (1 << 0) - -#define DECLARE_MANIFEST(sb, name) \ - struct manifest *name = SCOUTFS_SB(sb)->manifest - -/* - * A reader uses references to segments copied from a walk of the - * manifest. The references are a point in time sample of the manifest. - * The manifest and segments can change while the reader uses their - * references. Locking ensures that the items they're reading will be - * stable while the manifest and segments change, and the segment - * allocator gives readers time to use immutable stale segments before - * their reallocated and reused. - */ -struct manifest_ref { - struct list_head entry; - - u64 segno; - u64 seq; - struct scoutfs_segment *seg; - int found_ctr; - int off; - u8 level; - bool retried; - - struct scoutfs_key first; - struct scoutfs_key last; -}; - -/* - * Change the level count under the manifest lock. We then maintain a - * bit that can be tested outside the lock to determine if the caller - * should wait for level 0 segments to drain. - */ -static void add_level_count(struct super_block *sb, int level, s64 val) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - __le64 count; - int full; - - le64_add_cpu(&super->manifest.level_counts[level], val); - - if (level == 0) { - count = super->manifest.level_counts[level]; - full = test_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); - if (count && !full) - set_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); - else if (!count && full) - clear_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); - } -} - -/* - * Return whether or not level 0 segments are full. It's safe to use - * this as a wait_event condition because it doesn't block. - * - * Callers rely on on the spin locks in wait queues to synchronize - * testing this as a sleeping condition with addition to the wait queue - * and waking of the waitqueue. - */ -bool scoutfs_manifest_level0_full(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - - return test_bit(MANI_FLAG_LEVEL0_FULL, &mani->flags); -} - -void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, - u64 level, u64 segno, u64 seq, - struct scoutfs_key *first, - struct scoutfs_key *last) -{ - ment->level = level; - ment->segno = segno; - ment->seq = seq; - scoutfs_key_copy_or_zeros(&ment->first, first); - scoutfs_key_copy_or_zeros(&ment->last, last); -} - -static void init_btree_key(struct scoutfs_manifest_btree_key *mkey, - u8 level, u64 seq, struct scoutfs_key *first) -{ - mkey->level = level; - scoutfs_key_to_be(&mkey->first_key, first); - mkey->seq = cpu_to_be64(seq); -} - -static void init_btree_val(struct scoutfs_manifest_btree_val *mval, - u64 segno, struct scoutfs_key *last) -{ - mval->segno = cpu_to_le64(segno); - mval->last_key = *last; -} - -/* initialize a native manifest entry to point to the btree key and value */ -static void init_ment_iref(struct scoutfs_manifest_entry *ment, - struct scoutfs_btree_item_ref *iref) -{ - struct scoutfs_manifest_btree_key *mkey = iref->key; - struct scoutfs_manifest_btree_val *mval = iref->val; - - ment->level = mkey->level; - scoutfs_key_from_be(&ment->first, &mkey->first_key); - ment->seq = be64_to_cpu(mkey->seq); - ment->segno = le64_to_cpu(mval->segno); - ment->last = mval->last_key; -} - - -/* - * Insert a new manifest entry in the ring. The ring allocates a new - * node for us and we fill it. - * - * This must be called with the manifest lock held. - */ -int scoutfs_manifest_add(struct super_block *sb, - struct scoutfs_manifest_entry *ment) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_btree_key mkey; - struct scoutfs_manifest_btree_val mval; - int ret; - - lockdep_assert_held(&mani->rwsem); - - init_btree_key(&mkey, ment->level, ment->seq, &ment->first); - init_btree_val(&mval, ment->segno, &ment->last); - - trace_scoutfs_manifest_add(sb, ment->level, ment->segno, ment->seq, - &ment->first, &ment->last); - - ret = scoutfs_btree_insert(sb, NULL, &super->manifest.root, - &mkey, sizeof(mkey), &mval, sizeof(mval)); - if (ret == 0) { - mani->nr_levels = max_t(u8, mani->nr_levels, ment->level + 1); - add_level_count(sb, ment->level, 1); - } - - return ret; -} - -/* - * This must be called with the manifest lock held. - * - * When this is called from the network we can take the keys directly as - * they were sent from the clients. - */ -int scoutfs_manifest_del(struct super_block *sb, - struct scoutfs_manifest_entry *ment) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_btree_key mkey; - int ret; - - trace_scoutfs_manifest_delete(sb, ment->level, ment->segno, ment->seq, - &ment->first, &ment->last); - - lockdep_assert_held(&mani->rwsem); - - init_btree_key(&mkey, ment->level, ment->seq, &ment->first); - - ret = scoutfs_btree_delete(sb, NULL, &super->manifest.root, - &mkey, sizeof(mkey)); - if (ret == 0) - add_level_count(sb, ment->level, -1ULL); - - return ret; -} - -/* - * XXX This feels pretty gross, but it's a simple way to give compaction - * atomic updates. It'll go away once compactions go to the trouble of - * communicating their atomic results in a message instead of a series - * of function calls. - */ -int scoutfs_manifest_lock(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - - down_write(&mani->rwsem); - - return 0; -} - -int scoutfs_manifest_unlock(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - - up_write(&mani->rwsem); - - return 0; -} - -static void free_ref(struct super_block *sb, struct manifest_ref *ref) -{ - if (!IS_ERR_OR_NULL(ref)) { - WARN_ON_ONCE(!list_empty(&ref->entry)); - scoutfs_seg_put(ref->seg); - kfree(ref); - } -} - -/* - * Allocate a reading manifest ref so that we can work with segments - * described by the callers manifest entry. - */ -static int alloc_manifest_ref(struct super_block *sb, struct list_head *ref_list, - struct scoutfs_manifest_entry *ment) -{ - struct manifest_ref *ref; - - ref = kzalloc(sizeof(struct manifest_ref), GFP_NOFS); - if (!ref) - return -ENOMEM; - - ref->first = ment->first; - ref->last = ment->last; - ref->level = ment->level; - ref->segno = ment->segno; - ref->seq = ment->seq; - - list_add_tail(&ref->entry, ref_list); - - return 0; -} - -/* - * Give the caller the next entry that overlaps with the given key at th - * egiven level. We first check the previous entry before the key to - * see if it overlaps. If it does then we return it. If it doesn't - * then we return the raw next entry after the key. The caller has to - * test it. - * - * If a start key is provided then the caller is working with cache - * ranges. If we find a previous entry that doesn't contain the key - * then we see if we should shrink the range to make sure that it - * doesn't include this segment whose items we're not using. - * - * Returns 0 with the iref pointing to the btree item with the entry, - * callers has to put the iref when they're done. - */ -static int btree_prev_overlap_or_next(struct super_block *sb, - struct scoutfs_btree_root *root, - void *bkey, unsigned bkey_len, - struct scoutfs_key *key, - struct scoutfs_key *start, u8 level, - struct scoutfs_btree_item_ref *iref) -{ - struct scoutfs_manifest_entry ment; - int ret; - - ret = scoutfs_btree_prev(sb, root, bkey, bkey_len, iref); - if (ret < 0 && ret != -ENOENT) - return ret; - - if (ret == 0) { - init_ment_iref(&ment, iref); - - /* shrink range so it doesn't cover skipped prev */ - if (start && ment.level == level && - scoutfs_key_compare(&ment.last, key) < 0 && - scoutfs_key_compare(&ment.last, start) >= 0) { - *start = ment.last; - scoutfs_key_inc(start); - } - - /* skip prev that doesn't contain the key */ - if (ment.level != level || - scoutfs_key_compare(&ment.last, key) < 0) - ret = -ENOENT; - } - if (ret == -ENOENT) { - scoutfs_btree_put_iref(iref); - ret = scoutfs_btree_next(sb, root, bkey, bkey_len, iref); - } - - return ret; -} - -/* - * Get references to all the level 0 segments whose item ranges - * intersect with the callers range. The entries are sorted by their - * first key so we can stop searching once our end key can only keep - * being less than the increasing start key. - * - * This can return -ESTALE if it reads through stale btree blocks. - */ -static int get_zero_refs(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *start, - struct scoutfs_key *end, - struct list_head *ref_list) -{ - struct scoutfs_manifest_btree_key mkey; - struct scoutfs_manifest_entry ment; - SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key zeros; - int cmp; - int ret; - - scoutfs_key_set_zeros(&zeros); - init_btree_key(&mkey, 0, 0, &zeros); - - for (;;) { - ret = scoutfs_btree_next(sb, root, &mkey, sizeof(mkey), &iref); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - init_ment_iref(&ment, &iref); - scoutfs_btree_put_iref(&iref); - - /* done if we went past level 0 */ - if (ment.level > 0) { - ret = 0; - break; - } - - cmp = scoutfs_key_compare_ranges(start, end, &ment.first, - &ment.last); - /* done if all the ments will be greater */ - if (cmp < 0) { - ret = 0; - break; - } - - if (cmp == 0) { - ret = alloc_manifest_ref(sb, ref_list, &ment); - if (ret) - break; - } - - scoutfs_key_inc(&ment.first); - init_btree_key(&mkey, ment.level, ment.seq, &ment.first); - } - - return ret; -} - -/* - * Get references to all segments in non-zero levels that contain the - * caller's key. The item ranges of segments at each non-zero level - * don't overlap so we can iterate through the key space in each segment - * starting with the search key. In each level we need the first - * existing segment that intersects with the range, even if it doesn't - * contain the key. The key might fall between segments at that level. - * - * The caller can provide the range of items that they're going to - * consider authoritative for the range of segments that we give them. - * We have to shrink this range if we give them segments that don't - * cover the range. This includes implicitly negative cached space - * that's created by using the segment after the hole between segments. - * If a segment is entirely outside of the caller's range then we can't - * trust its contents. - * - * This can return -ESTALE if it reads through stale btree blocks. - */ -static int get_nonzero_refs(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_key *start, - struct scoutfs_key *end, - struct list_head *ref_list) -{ - struct scoutfs_manifest_btree_key mkey; - struct scoutfs_manifest_entry ment; - SCOUTFS_BTREE_ITEM_REF(iref); - int ret; - int i; - - if (WARN_ON_ONCE(!!start != !!end) || - WARN_ON_ONCE(start && scoutfs_key_compare(start, end) > 0)) - return -EINVAL; - - for (i = 1; ; i++) { - init_btree_key(&mkey, i, 0, key); - - ret = btree_prev_overlap_or_next(sb, root, &mkey, sizeof(mkey), - key, start, i, &iref); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - init_ment_iref(&ment, &iref); - scoutfs_btree_put_iref(&iref); - - if (ment.level != i || - (end && scoutfs_key_compare(&ment.first, end) > 0)) - continue; - - ret = alloc_manifest_ref(sb, ref_list, &ment); - if (ret) - break; - - if (start && scoutfs_key_compare(&ment.first, start) > 0 && - scoutfs_key_compare(&ment.first, key) <= 0) - *start = ment.first; - - if (end && scoutfs_key_compare(&ment.last, end) < 0 && - scoutfs_key_compare(&ment.last, key) >= 0) - *end = ment.last; - } - - return ret; -} - -/* - * If we saw persistent stale blocks or segment reads while walking the - * manifest then we might be trying to read through an old stale root - * that has been overwritten. We can ask for a new root and try again. - * If we don't get a new root and the errors persist then the we've hit - * corruption. - */ -static int handle_stale_btree(struct super_block *sb, - struct scoutfs_btree_root *root, - __le64 last_root_seq, int ret) -{ - bool force_hard = scoutfs_trigger(sb, HARD_STALE_ERROR); - - if (ret == -ESTALE || force_hard) { - if ((last_root_seq != root->ref.seq) && !force_hard) - return -EAGAIN; - - scoutfs_inc_counter(sb, manifest_hard_stale_error); - return -EIO; - } - - return ret; -} - -static int cmp_ment_ref_segno(void *priv, struct list_head *A, - struct list_head *B) -{ - struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); - struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); - - return scoutfs_cmp_u64s(a->segno, b->segno); -} - -/* - * Sort by from most to least recent item contents.. from lowest to higest - * level and from highest to loweset seq in level 0. - */ -static int cmp_ment_ref_level_seq(void *priv, struct list_head *A, - struct list_head *B) -{ - struct manifest_ref *a = list_entry(A, struct manifest_ref, entry); - struct manifest_ref *b = list_entry(B, struct manifest_ref, entry); - - if (a->level == 0 && b->level == 0) - return -scoutfs_cmp_u64s(a->seq, b->seq); - - return a->level < b->level ? -1 : a->level > b->level ? 1 : 0; -} - -/* - * The caller found a hole in the item cache that they'd like populated. - * We can only trust items in the segments within their range (they hold - * a lock) and they're going to keep calling ("He'll keep calling me, - * he'll keep calling me") until we insert a range into the cache that - * contains the search key. - * - * We search the manifest for all the non-zero segments that contain the - * key. We adjust the search range if the segments don't cover the - * whole locked range. We have to be careful not to shrink the range - * past the key, it could be outside the segments and we still want to - * negatively cache it. Once we have the search range we get the level - * zero segments that overlap. - * - * Once we have the segments we iterate over them and allocate the items - * to insert into the cache. We find the next item in each segment, - * ignore deletion items, prefer more recent segments, and advance past - * the items that we used. - * - * Returns 0 if we successfully inserted items. - * - * Returns -errno if we failed to make any change in the cache. - * - * This is asking the seg code to read each entire segment. The seg - * code could give it it helpers to submit and wait on blocks within the - * segment so that we don't have wild bandwidth amplification for cold - * random reads. - * - * The segments are immutable at this point so we can use their contents - * as long as we hold refs. - */ -int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_key *start, - struct scoutfs_key *end) -{ - struct scoutfs_key item_key; - struct scoutfs_key found_key; - struct scoutfs_key batch_end; - struct scoutfs_key seg_start; - struct scoutfs_key seg_end; - struct scoutfs_btree_root root; - struct scoutfs_segment *seg; - struct manifest_ref *ref; - struct manifest_ref *tmp; - __le64 last_root_seq; - struct kvec found_val; - struct kvec item_val; - LIST_HEAD(ref_list); - LIST_HEAD(batch); - u8 found_flags = 0; - u8 item_flags; - int found_ctr; - bool found; - bool added; - int ret = 0; - int err; - int cmp; - - /* - * Ask the manifest server which manifest root to read from. Lock - * holding callers will be responsible for this in the future. They'll - * either get a manifest ref in the lvb of their lock or they'll - * ask the server the first time the system sees the lock. - */ - last_root_seq = 0; -retry_stale: - - seg_start = *start; - seg_end = *end; - - ret = scoutfs_client_get_manifest_root(sb, &root); - if (ret) - goto out; - - /* get non-zero segments that intersect with the key, shrinks range */ - ret = get_nonzero_refs(sb, &root, key, &seg_start, &seg_end, &ref_list); - if (ret) - goto out; - - trace_scoutfs_read_item_keys(sb, key, start, end, &seg_start, &seg_end); - - /* then get level 0s that intersect with our search range */ - ret = get_zero_refs(sb, &root, &seg_start, &seg_end, &ref_list); - if (ret) - goto out; - - /* sort by segment to issue advancing reads */ - list_sort(NULL, &ref_list, cmp_ment_ref_segno); - -resubmit: - /* submit reads for all the segments */ - list_for_each_entry(ref, &ref_list, entry) { - /* don't resubmit if we've read */ - if (ref->seg) - continue; - - trace_scoutfs_read_item_segment(sb, ref->level, ref->segno, - ref->seq, &ref->first, - &ref->last); - - seg = scoutfs_seg_submit_read(sb, ref->segno); - if (IS_ERR(seg)) { - ret = PTR_ERR(seg); - break; - } - - ref->seg = seg; - } - - /* always wait for submitted segments */ - list_for_each_entry(ref, &ref_list, entry) { - if (!ref->seg) - continue; - - err = scoutfs_seg_wait(sb, ref->seg, ref->segno, ref->seq); - if (err == -ESTALE && !ref->retried) { - ref->retried = true; - err = 0; - scoutfs_seg_put(ref->seg); - ref->seg = NULL; - goto resubmit; - } - if (err && !ret) - ret = err; - } - if (ret) - goto out; - - /* now sort refs by item age */ - list_sort(NULL, &ref_list, cmp_ment_ref_level_seq); - - /* walk items from the start of our range */ - list_for_each_entry(ref, &ref_list, entry) - ref->off = scoutfs_seg_find_off(ref->seg, &seg_start); - - found_ctr = 0; - - added = false; - for (;;) { - found = false; - found_ctr++; - - /* find the next least key from the off in each segment */ - list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - if (ref->off < 0) - continue; - - /* - * Check the next item in the segment. We're - * done with the segment if there are no more - * items or if the next item is past the keys - * that our segments can see. - */ - ret = scoutfs_seg_get_item(ref->seg, ref->off, - &item_key, &item_val, - &item_flags); - if (ret < 0 || - scoutfs_key_compare(&item_key, &seg_end) > 0) { - ref->off = -1; - continue; - } - - /* see if it's the new least item */ - if (found) { - cmp = scoutfs_key_compare(&item_key, - &found_key); - if (cmp >= 0) { - if (cmp == 0) - ref->found_ctr = found_ctr; - continue; - } - } - - /* remember new least key */ - found_key = item_key; - found_val = item_val; - found_flags = item_flags; - ref->found_ctr = ++found_ctr; - found = true; - } - - /* ran out of keys in segs, range extends to seg end */ - if (!found) { - batch_end = seg_end; - ret = 0; - break; - } - - /* - * Add the next found item to the batch if it's not a - * deletion item. We still need to use their key to - * remember the end of the batch for negative caching. - * - * If we fail to add an item we're done. If we already - * have items it's not a failure and the end of the - * cached range is the last successfully added item. - */ - if (!(found_flags & SCOUTFS_ITEM_FLAG_DELETION)) { - ret = scoutfs_item_add_batch(sb, &batch, &found_key, - &found_val); - if (ret) { - if (added) - ret = 0; - break; - } - added = true; - } - - /* the last successful key determines range end until run out */ - batch_end = found_key; - - /* if we just saw the end key then we're done */ - if (scoutfs_key_compare(&found_key, &seg_end) == 0) { - ret = 0; - break; - } - - /* advance all the positions that had the found key */ - list_for_each_entry(ref, &ref_list, entry) { - if (ref->found_ctr == found_ctr) - ref->off = scoutfs_seg_next_off(ref->seg, - ref->off); - } - - ret = 0; - } - - if (ret < 0) { - scoutfs_item_free_batch(sb, &batch); - } else { - if (scoutfs_key_compare(key, &batch_end) > 0) - scoutfs_inc_counter(sb, manifest_read_excluded_key); - ret = scoutfs_item_insert_batch(sb, &batch, &seg_start, - &batch_end); - } -out: - list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - list_del_init(&ref->entry); - free_ref(sb, ref); - } - - ret = handle_stale_btree(sb, &root, last_root_seq, ret); - if (ret == -EAGAIN) { - last_root_seq = root.ref.seq; - goto retry_stale; - } - - return ret; -} - -/* - * Give the caller a hint to the next key that they'll find after their - * search key. - * - * We read the segments that intersect the key and return either the - * next item we see or the nearest segment limit. - * - * This is a hint because we can return deleted items or the next - * nearest segment limit can be well before the next items in the next - * segments. The caller needs to very carefully iterate using the next - * key we return. - * - * Returns 0 if it set next_key and -ENOENT if the key was after all the - * segments in the manifest. - */ -int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *next_key) -{ - struct scoutfs_key item_key; - struct scoutfs_btree_root root; - struct scoutfs_segment *seg; - struct manifest_ref *ref; - struct manifest_ref *tmp; - __le64 last_root_seq; - LIST_HEAD(ref_list); - bool found; - int ret; - int err; - - last_root_seq = 0; -retry_stale: - ret = scoutfs_client_get_manifest_root(sb, &root); - if (ret) - goto out; - - ret = get_zero_refs(sb, &root, key, key, &ref_list) ?: - get_nonzero_refs(sb, &root, key, NULL, NULL, &ref_list); - if (ret) - goto out; - - if (list_empty(&ref_list)) { - ret = -ENOENT; - goto out; - } - - list_sort(NULL, &ref_list, cmp_ment_ref_segno); - - list_for_each_entry(ref, &ref_list, entry) { - seg = scoutfs_seg_submit_read(sb, ref->segno); - if (IS_ERR(seg)) { - ret = PTR_ERR(seg); - break; - } - - ref->seg = seg; - } - - list_for_each_entry(ref, &ref_list, entry) { - if (!ref->seg) - break; - - err = scoutfs_seg_wait(sb, ref->seg, ref->segno, ref->seq); - if (err && !ret) - ret = err; - } - if (ret) - goto out; - - list_sort(NULL, &ref_list, cmp_ment_ref_level_seq); - - /* default to returning the nearest segment limit and find offsets */ - found = false; - list_for_each_entry(ref, &ref_list, entry) { - if (ref->level > 0 && - (!found || - scoutfs_key_compare(&ref->last, next_key) < 0)) { - *next_key = ref->last; - found = true; - } - - ref->off = scoutfs_seg_find_off(ref->seg, key); - } - - /* return the nearest item in the segments */ - list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - if (ref->off < 0) - continue; - - ret = scoutfs_seg_get_item(ref->seg, ref->off, &item_key, - NULL, NULL); - if (ret < 0) - continue; - - if (!found || scoutfs_key_compare(&item_key, next_key) < 0) { - *next_key = item_key; - found = true; - } - } - - ret = 0; -out: - list_for_each_entry_safe(ref, tmp, &ref_list, entry) { - list_del_init(&ref->entry); - free_ref(sb, ref); - } - - ret = handle_stale_btree(sb, &root, last_root_seq, ret); - if (ret == -EAGAIN) { - last_root_seq = root.ref.seq; - goto retry_stale; - } - - return ret; -} - -static bool level_should_compact(struct super_block *sb, int level) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - BUG_ON(!rwsem_is_locked(&mani->rwsem)); - - return ((s64)le64_to_cpu(super->manifest.level_counts[level]) - - (s64)mani->compacts_pending[level]) > mani->level_limits[level]; -} - -int scoutfs_manifest_should_compact(struct super_block *sb) -{ - DECLARE_MANIFEST(sb, mani); - bool should = false; - int level; - - down_read(&mani->rwsem); - for (level = mani->nr_levels - 1; level >= 0; level--) { - if (level_should_compact(sb, level)) { - should = true; - break; - } - } - up_read(&mani->rwsem); - - return should; -} - -/* - * Record that a compaction operation is in flight. We mark the segnos - * involved so that we don't use them as inputs for other compactions - * and assume that the compaction will delete a segment from its upper - * level when deciding what level to compact. - */ -static int start_compact_request(struct super_block *sb, - struct scoutfs_net_compact_request *req) -{ - DECLARE_MANIFEST(sb, mani); - int level; - int ret = 0; - int i; - - BUG_ON(!rwsem_is_locked(&mani->rwsem)); - - for (i = 0; i < ARRAY_SIZE(req->ents); i++) { - if (req->ents[i].segno == 0) - break; - - ret = scoutfs_spbm_set(&mani->segno_busy, - le64_to_cpu(req->ents[i].segno)); - if (ret) { - while (i-- > 0) - scoutfs_spbm_clear(&mani->segno_busy, - le64_to_cpu(req->ents[i].segno)); - break; - } - } - - if (ret == 0) { - level = req->ents[0].level; - mani->compacts_pending[level]++; - } - - return ret; -} - -/* - * A compaction request has completed. No longer account for it in the - * level pending counts and stop tracking all its segments. - * - * This can be called in error paths with an empty zeroed request and it - * will do nothing. - */ -void scoutfs_manifest_compact_done(struct super_block *sb, - struct scoutfs_net_compact_request *req) -{ - DECLARE_MANIFEST(sb, mani); - int level; - int i; - - down_write(&mani->rwsem); - - for (i = 0; i < ARRAY_SIZE(req->ents); i++) { - if (req->ents[i].segno == 0) - break; - - scoutfs_spbm_clear(&mani->segno_busy, - le64_to_cpu(req->ents[i].segno)); - } - - if (i > 0) { - level = req->ents[0].level; - mani->compacts_pending[level]--; - } - - up_write(&mani->rwsem); -} - -static int add_entry_unless_busy(struct super_block *sb, - struct scoutfs_net_compact_request *req, - unsigned int ind, - struct scoutfs_manifest_entry *ment) -{ - DECLARE_MANIFEST(sb, mani); - - if (scoutfs_spbm_test(&mani->segno_busy, ment->segno)) { - scoutfs_inc_counter(sb, compact_segment_busy); - return -EAGAIN; - } - - scoutfs_init_ment_to_net(&req->ents[ind], ment); - return 0; -} - -/* - * Give the caller the segments that will be involved in the next - * compaction. - * - * For now we have a simple candidate search. We only initiate - * compaction when a level has exceeded its exponentially increasing - * limit on the number of segments. Once we have a level we use keys at - * each level to chose the next segment. This results in a pattern - * where clock hands sweep through each level. The hands wrap much - * faster on the higher levels. - * - * We add all the segments to the compaction caller's data and let it do - * its thing. It'll allocate and free segments and update the manifest. - * - * Returns: - * 0: no compactions were needed at the given level - * > 0: number of total imput segments in the compaction - * -EAGAIN: segments were already in a pending compaction - * -errno: fatal error - * - * XXX this could be more clever: - * - prioritize segments with deletion or incremental records - * - prioritize partial segments - * - maybe compact segments by age in a given level - */ -static int next_compact_req(struct super_block *sb, int level, - struct scoutfs_net_compact_request *req) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_manifest_btree_key mkey; - struct scoutfs_manifest_entry next; - struct scoutfs_manifest_entry ment; - struct scoutfs_manifest_entry over; - SCOUTFS_BTREE_ITEM_REF(iref); - SCOUTFS_BTREE_ITEM_REF(over_iref); - SCOUTFS_BTREE_ITEM_REF(prev); - static struct scoutfs_key zeros; - bool wrapped; - bool sticky; - int ret; - int nr = 0; - int i; - - scoutfs_key_set_zeros(&zeros); - memset(req, 0, sizeof(*req)); - - BUG_ON(!rwsem_is_locked(&mani->rwsem)); - - /* fill ment and ret == 0 if we find an entry at the level */ - if (level == 0) { - - /* find the oldest level 0 */ - init_btree_key(&mkey, 0, 0, &zeros); - ment.seq = U64_MAX; - - for (;;) { - ret = scoutfs_btree_next(sb, &super->manifest.root, - &mkey, sizeof(mkey), &iref); - if (ret < 0) { - if (ret == -ENOENT && ment.seq != U64_MAX) - ret = 0; - break; - } - - init_ment_iref(&next, &iref); - scoutfs_btree_put_iref(&iref); - - if (next.level > 0) { - if (ment.seq == U64_MAX) - ret = -ENOENT; - break; - } - - if (next.seq < ment.seq) - ment = next; - - init_btree_key(&mkey, next.level, next.seq + 1, - &next.first); - } - - } else { - /* find the next segment after the compaction at this level */ - init_btree_key(&mkey, level, 0, &mani->compact_keys[level]); - wrapped = false; -again: - ret = scoutfs_btree_next(sb, &super->manifest.root, - &mkey, sizeof(mkey), &iref); - if (ret == 0) { - init_ment_iref(&ment, &iref); - scoutfs_btree_put_iref(&iref); - if (ment.level != level) - ret = -ENOENT; - } - /* try again if we wrapped */ - if (ret == -ENOENT && !wrapped) { - init_btree_key(&mkey, level, 0, &zeros); - wrapped = true; - goto again; - } - } - - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - goto out; - } - - /* add the upper input segment */ - ret = add_entry_unless_busy(sb, req, nr, &ment); - if (ret) - goto skip; - nr++; - - /* and add a fanout's worth of lower overlapping segments */ - init_btree_key(&mkey, level + 1, 0, &ment.first); - ret = btree_prev_overlap_or_next(sb, &super->manifest.root, - &mkey, sizeof(mkey), &ment.first, - NULL, level + 1, &over_iref); - sticky = false; - for (i = 0; ret == 0 && i < SCOUTFS_MANIFEST_FANOUT + 1; i++) { - init_ment_iref(&over, &over_iref); - if (over.level != level + 1) - break; - - if (scoutfs_key_compare_ranges(&ment.first, &ment.last, - &over.first, &over.last) != 0) - break; - - /* upper level has to stay around when more than fanout */ - if (i == SCOUTFS_MANIFEST_FANOUT) { - sticky = true; - break; - } - - ret = add_entry_unless_busy(sb, req, nr, &over); - if (ret) - goto out; - nr++; - - swap(prev, over_iref); - ret = scoutfs_btree_after(sb, &super->manifest.root, - prev.key, prev.key_len, &over_iref); - scoutfs_btree_put_iref(&prev); - } - if (ret < 0 && ret != -ENOENT) - goto out; - - req->last_level = mani->nr_levels - 1; - if (sticky) - req->flags |= SCOUTFS_NET_COMPACT_FLAG_STICKY; - - ret = start_compact_request(sb, req); - if (ret) - goto out; - - ret = 0; -skip: - /* record the next key to start from */ - mani->compact_keys[level] = ment.last; - scoutfs_key_inc(&mani->compact_keys[level]); -out: - scoutfs_btree_put_iref(&iref); - scoutfs_btree_put_iref(&over_iref); - scoutfs_btree_put_iref(&prev); - - return ret ?: nr; -} - -/* - * Find the next segment to compact into its lower overlapping segments. - * Fill out the callers request describing all the segments involved in - * the operation. - * - * First we search for a level to compact. A level needs compaction if - * it has more segments than its limit. We search from the bottom up - * because segments are written at the top when there's space. By - * compacting from the bottom we pull new segments down until there's - * space. If we compacted from the top down then we could create an - * imbalanced top-heavy structure. - * - * At each level we find the segment from a cursor and try to compact it - * into its lower segments. Any of the segments involved could already - * be part of a pending compaction and need to be skipped. In that case - * we move to the next segment at the level. All the segments at the - * level could be busy so we detect when we skip to the first value we - * skipped to and move on. - * - * If we return a filled compact request then we've tracked it. We - * assume it will delete an upper segment and have marked all its segnos - * as busy so they won't be used by future compaction requests. The - * caller must call complete_done when the compact operation completes. - */ -int scoutfs_manifest_next_compact(struct super_block *sb, - struct scoutfs_net_compact_request *req) -{ - DECLARE_MANIFEST(sb, mani); - struct scoutfs_key key = {0,}; - bool first; - int level; - int ret = 0; - - memset(req, 0, sizeof(*req)); - - down_write(&mani->rwsem); - - for (level = mani->nr_levels - 1; level >= 0; level--) { - if (!level_should_compact(sb, level)) - continue; - - first = true; - - for (;;) { - ret = next_compact_req(sb, level, req); - if (ret > 0 || (ret < 0 && ret != -EAGAIN)) - goto out; - if (ret == 0) - break; - - /* remember first skip and keep going */ - if (first) { - first = false; - key = mani->compact_keys[level]; - continue; - } - - /* bail if we looped around */ - if (!scoutfs_key_compare(&key, - &mani->compact_keys[level])) { - ret = 0; - break; - } - } - /* continue to next level */ - } - -out: - up_write(&mani->rwsem); - - trace_scoutfs_manifest_next_compact(sb, level, ret); - - return ret; -} - -int scoutfs_manifest_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct manifest *mani; - int i; - - mani = kzalloc(sizeof(struct manifest), GFP_KERNEL); - if (!mani) - return -ENOMEM; - - init_rwsem(&mani->rwsem); - scoutfs_spbm_init(&mani->segno_busy); - - for (i = 0; i < ARRAY_SIZE(mani->compact_keys); i++) - scoutfs_key_set_zeros(&mani->compact_keys[i]); - - for (i = ARRAY_SIZE(super->manifest.level_counts) - 1; i >= 0; i--) { - if (super->manifest.level_counts[i]) { - mani->nr_levels = i + 1; - break; - } - } - - /* always trigger a compaction if there's a single l0 segment? */ - mani->level_limits[0] = 0; - mani->level_limits[1] = SCOUTFS_MANIFEST_FANOUT; - for (i = 2; i < ARRAY_SIZE(mani->level_limits); i++) { - mani->level_limits[i] = mani->level_limits[i - 1] * - SCOUTFS_MANIFEST_FANOUT; - } - - sbi->manifest = mani; - - return 0; -} - -void scoutfs_manifest_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct manifest *mani = sbi->manifest; - - if (mani) { - scoutfs_spbm_destroy(&mani->segno_busy); - kfree(mani); - sbi->manifest = NULL; - } -} diff --git a/kmod/src/manifest.h b/kmod/src/manifest.h deleted file mode 100644 index b3ffdf54..00000000 --- a/kmod/src/manifest.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef _SCOUTFS_MANIFEST_H_ -#define _SCOUTFS_MANIFEST_H_ - -#include "key.h" - -struct scoutfs_bio_completion; - -/* - * This native manifest entry references the physical storage of a - * manifest entry which can exist in a segment header and its edge keys, - * a network transmission of a packed entry and its keys, or in btree - * blocks spread between an item key and value. - */ -struct scoutfs_manifest_entry { - u8 level; - u64 segno; - u64 seq; - struct scoutfs_key first; - struct scoutfs_key last; -}; - -void scoutfs_manifest_init_entry(struct scoutfs_manifest_entry *ment, - u64 level, u64 segno, u64 seq, - struct scoutfs_key *first, - struct scoutfs_key *last); -int scoutfs_manifest_add(struct super_block *sb, - struct scoutfs_manifest_entry *ment); -int scoutfs_manifest_del(struct super_block *sb, - struct scoutfs_manifest_entry *ment); - -int scoutfs_manifest_lock(struct super_block *sb); -int scoutfs_manifest_unlock(struct super_block *sb); - -int scoutfs_manifest_read_items(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_key *start, - struct scoutfs_key *end); -int scoutfs_manifest_next_key(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *next_key); - -int scoutfs_manifest_should_compact(struct super_block *sb); -int scoutfs_manifest_next_compact(struct super_block *sb, - struct scoutfs_net_compact_request *req); -void scoutfs_manifest_compact_done(struct super_block *sb, - struct scoutfs_net_compact_request *req); - -bool scoutfs_manifest_level0_full(struct super_block *sb); - -int scoutfs_manifest_setup(struct super_block *sb); -void scoutfs_manifest_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/net.c b/kmod/src/net.c index 79056204..9d9d9145 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -25,9 +25,6 @@ #include "counters.h" #include "inode.h" #include "btree.h" -#include "manifest.h" -#include "seg.h" -#include "compact.h" #include "scoutfs_trace.h" #include "msg.h" #include "net.h" diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index bcdfaf5c..968cfebb 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -29,11 +29,9 @@ #include "key.h" #include "format.h" #include "lock.h" -#include "seg.h" #include "super.h" #include "ioctl.h" #include "count.h" -#include "bio.h" #include "export.h" #include "dir.h" #include "extents.h" @@ -99,258 +97,6 @@ TRACE_EVENT(scoutfs_complete_truncate, __entry->flags) ); -DECLARE_EVENT_CLASS(scoutfs_comp_class, - TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), - - TP_ARGS(sb, comp), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(struct scoutfs_bio_completion *, comp) - __field(int, pending) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->comp = comp; - __entry->pending = atomic_read(&comp->pending); - ), - - TP_printk(SCSBF" comp %p pending before %d", SCSB_TRACE_ARGS, - __entry->comp, __entry->pending) -); -DEFINE_EVENT(scoutfs_comp_class, comp_end_io, - TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), - TP_ARGS(sb, comp) -); -DEFINE_EVENT(scoutfs_comp_class, scoutfs_bio_submit_comp, - TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), - TP_ARGS(sb, comp) -); -DEFINE_EVENT(scoutfs_comp_class, scoutfs_bio_wait_comp, - TP_PROTO(struct super_block *sb, struct scoutfs_bio_completion *comp), - TP_ARGS(sb, comp) -); - -TRACE_EVENT(scoutfs_bio_init_comp, - TP_PROTO(void *comp), - - TP_ARGS(comp), - - TP_STRUCT__entry( - __field(void *, comp) - ), - - TP_fast_assign( - __entry->comp = comp; - ), - - TP_printk("initing comp %p", __entry->comp) -); - -DECLARE_EVENT_CLASS(scoutfs_bio_class, - TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), - - TP_ARGS(sb, bio, args, in_flight), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(void *, bio) - __field(void *, args) - __field(int, in_flight) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->bio = bio; - __entry->args = args; - __entry->in_flight = in_flight; - ), - - TP_printk(SCSBF" bio %p args %p in_flight %d", SCSB_TRACE_ARGS, - __entry->bio, __entry->args, __entry->in_flight) -); - -DEFINE_EVENT(scoutfs_bio_class, scoutfs_bio_submit, - TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), - TP_ARGS(sb, bio, args, in_flight) -); - -DEFINE_EVENT(scoutfs_bio_class, scoutfs_bio_submit_partial, - TP_PROTO(struct super_block *sb, void *bio, void *args, int in_flight), - TP_ARGS(sb, bio, args, in_flight) -); - -TRACE_EVENT(scoutfs_bio_end_io, - TP_PROTO(struct super_block *sb, void *bio, int size, int err), - - TP_ARGS(sb, bio, size, err), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(void *, bio) - __field(int, size) - __field(int, err) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->bio = bio; - __entry->size = size; - __entry->err = err; - ), - - TP_printk(SCSBF" bio %p size %u err %d", SCSB_TRACE_ARGS, - __entry->bio, __entry->size, __entry->err) -); - -TRACE_EVENT(scoutfs_dec_end_io, - TP_PROTO(struct super_block *sb, void *args, int in_flight, int err), - - TP_ARGS(sb, args, in_flight, err), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(void *, args) - __field(int, in_flight) - __field(int, err) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->args = args; - __entry->in_flight = in_flight; - __entry->err = err; - ), - - TP_printk(SCSBF" args %p in_flight %d err %d", SCSB_TRACE_ARGS, - __entry->args, __entry->in_flight, __entry->err) -); - -DECLARE_EVENT_CLASS(scoutfs_key_ret_class, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), - - TP_ARGS(sb, key, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - __entry->ret = ret; - ), - - TP_printk(SCSBF" key "SK_FMT" ret %d", - SCSB_TRACE_ARGS, sk_trace_args(key), __entry->ret) -); - -DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_create, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), - TP_ARGS(sb, key, ret) -); -DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_delete, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), - TP_ARGS(sb, key, ret) -); -DEFINE_EVENT(scoutfs_key_ret_class, scoutfs_item_delete_save, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, int ret), - TP_ARGS(sb, key, ret) -); - -TRACE_EVENT(scoutfs_item_dirty_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - -TRACE_EVENT(scoutfs_item_update_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - -TRACE_EVENT(scoutfs_item_next_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - -TRACE_EVENT(scoutfs_item_prev_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - -TRACE_EVENT(scoutfs_erase_item, - TP_PROTO(struct super_block *sb, void *item), - - TP_ARGS(sb, item), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(void *, item) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->item = item; - ), - - TP_printk(SCSBF" erasing item %p", SCSB_TRACE_ARGS, __entry->item) -); - TRACE_EVENT(scoutfs_data_fallocate, TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, loff_t len, int ret), @@ -946,34 +692,6 @@ TRACE_EVENT(scoutfs_inode_walk_writeback, __entry->ino, __entry->write, __entry->ret) ); -DECLARE_EVENT_CLASS(scoutfs_segment_class, - TP_PROTO(struct super_block *sb, __u64 segno), - - TP_ARGS(sb, segno), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, segno) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->segno = segno; - ), - - TP_printk(SCSBF" segno %llu", SCSB_TRACE_ARGS, __entry->segno) -); - -DEFINE_EVENT(scoutfs_segment_class, scoutfs_seg_submit_read, - TP_PROTO(struct super_block *sb, __u64 segno), - TP_ARGS(sb, segno) -); - -DEFINE_EVENT(scoutfs_segment_class, scoutfs_seg_submit_write, - TP_PROTO(struct super_block *sb, __u64 segno), - TP_ARGS(sb, segno) -); - DECLARE_EVENT_CLASS(scoutfs_lock_info_class, TP_PROTO(struct super_block *sb, struct lock_info *linfo), @@ -1034,27 +752,6 @@ TRACE_EVENT(scoutfs_xattr_set, __entry->size, __entry->flags) ); -TRACE_EVENT(scoutfs_manifest_next_compact, - TP_PROTO(struct super_block *sb, int level, int ret), - - TP_ARGS(sb, level, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, level) - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->level = level; - __entry->ret = ret; - ), - - TP_printk(SCSBF" level %d ret %d", SCSB_TRACE_ARGS, __entry->level, - __entry->ret) -); - TRACE_EVENT(scoutfs_advance_dirty_super, TP_PROTO(struct super_block *sb, __u64 seq), @@ -1109,130 +806,6 @@ TRACE_EVENT(scoutfs_dir_add_next_linkref, __entry->found_dir_ino, __entry->name_len) ); -TRACE_EVENT(scoutfs_client_compact_start, - TP_PROTO(struct super_block *sb, u64 id, u8 last_level, u8 flags), - - TP_ARGS(sb, id, last_level, flags), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, id) - __field(__u8, last_level) - __field(__u8, flags) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->id = id; - __entry->last_level = last_level; - __entry->flags = flags; - ), - - TP_printk(SCSBF" id %llu last_level %u flags 0x%x", - SCSB_TRACE_ARGS, __entry->id, __entry->last_level, - __entry->flags) -); - -TRACE_EVENT(scoutfs_client_compact_stop, - TP_PROTO(struct super_block *sb, u64 id, int ret), - - TP_ARGS(sb, id, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, id) - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->id = id; - __entry->ret = ret; - ), - - TP_printk(SCSBF" id %llu ret %d", - SCSB_TRACE_ARGS, __entry->id, __entry->ret) -); - -TRACE_EVENT(scoutfs_server_compact_start, - TP_PROTO(struct super_block *sb, u64 id, u8 level, u64 rid, - unsigned long client_nr, unsigned long server_nr, - unsigned long per_client), - - TP_ARGS(sb, id, level, rid, client_nr, server_nr, per_client), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, id) - __field(__u8, level) - __field(__u64, c_rid) - __field(unsigned long, client_nr) - __field(unsigned long, server_nr) - __field(unsigned long, per_client) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->id = id; - __entry->level = level; - __entry->c_rid = rid; - __entry->client_nr = client_nr; - __entry->server_nr = server_nr; - __entry->per_client = per_client; - ), - - TP_printk(SCSBF" id %llu level %u rid %016llx client_nr %lu server_nr %lu per_client %lu", - SCSB_TRACE_ARGS, __entry->id, __entry->level, - __entry->c_rid, __entry->client_nr, __entry->server_nr, - __entry->per_client) -); - -TRACE_EVENT(scoutfs_server_compact_done, - TP_PROTO(struct super_block *sb, u64 id, u64 rid, - unsigned long server_nr), - - TP_ARGS(sb, id, rid, server_nr), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, id) - __field(__u64, c_rid) - __field(unsigned long, server_nr) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->id = id; - __entry->rid = rid; - __entry->server_nr = server_nr; - ), - - TP_printk(SCSBF" id %llu rid %016llx server_nr %lu", - SCSB_TRACE_ARGS, __entry->id, __entry->c_rid, - __entry->server_nr) -); - -TRACE_EVENT(scoutfs_server_compact_response, - TP_PROTO(struct super_block *sb, u64 id, int error), - - TP_ARGS(sb, id, error), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, id) - __field(int, error) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->id = id; - __entry->error = error; - ), - - TP_printk(SCSBF" id %llu error %d", - SCSB_TRACE_ARGS, __entry->id, __entry->error) -); - TRACE_EVENT(scoutfs_write_begin, TP_PROTO(struct super_block *sb, u64 ino, loff_t pos, unsigned len), @@ -1382,89 +955,6 @@ TRACE_EVENT(scoutfs_scan_orphans, TP_printk("dev %d,%d", MAJOR(__entry->dev), MINOR(__entry->dev)) ); -DECLARE_EVENT_CLASS(scoutfs_manifest_class, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last), - TP_STRUCT__entry( - __field(u8, level) - __field(u64, segno) - __field(u64, seq) - sk_trace_define(first) - sk_trace_define(last) - ), - TP_fast_assign( - __entry->level = level; - __entry->segno = segno; - __entry->seq = seq; - sk_trace_assign(first, first); - sk_trace_assign(last, last); - ), - TP_printk("level %u segno %llu seq %llu first "SK_FMT" last "SK_FMT, - __entry->level, __entry->segno, __entry->seq, - sk_trace_args(first), sk_trace_args(last)) -); - -DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_add, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last) -); - -DEFINE_EVENT(scoutfs_manifest_class, scoutfs_manifest_delete, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last) -); - -DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_input, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last) -); - -DEFINE_EVENT(scoutfs_manifest_class, scoutfs_compact_output, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last) -); - -DEFINE_EVENT(scoutfs_manifest_class, scoutfs_read_item_segment, - TP_PROTO(struct super_block *sb, u8 level, u64 segno, u64 seq, - struct scoutfs_key *first, struct scoutfs_key *last), - TP_ARGS(sb, level, segno, seq, first, last) -); - -TRACE_EVENT(scoutfs_read_item_keys, - TP_PROTO(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_key *start, - struct scoutfs_key *end, - struct scoutfs_key *seg_start, - struct scoutfs_key *seg_end), - TP_ARGS(sb, key, start, end, seg_start, seg_end), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - sk_trace_define(start) - sk_trace_define(end) - sk_trace_define(seg_start) - sk_trace_define(seg_end) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - sk_trace_assign(start, start); - sk_trace_assign(end, end); - sk_trace_assign(seg_start, seg_start); - sk_trace_assign(seg_end, seg_end); - ), - TP_printk(SCSBF" key "SK_FMT" start "SK_FMT" end "SK_FMT" seg_start "SK_FMT" seg_end "SK_FMT"", - SCSB_TRACE_ARGS, sk_trace_args(key), sk_trace_args(start), - sk_trace_args(end), sk_trace_args(seg_start), - sk_trace_args(seg_end)) -); - DECLARE_EVENT_CLASS(scoutfs_key_class, TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key), @@ -1479,143 +969,11 @@ DECLARE_EVENT_CLASS(scoutfs_key_class, TP_printk(SCSBF" key "SK_FMT, SCSB_TRACE_ARGS, sk_trace_args(key)) ); -DEFINE_EVENT(scoutfs_key_class, scoutfs_item_lookup, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key), - TP_ARGS(sb, key) -); - -TRACE_EVENT(scoutfs_item_lookup_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - -DEFINE_EVENT(scoutfs_key_class, scoutfs_item_insertion, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key), - TP_ARGS(sb, key) -); - -DEFINE_EVENT(scoutfs_key_class, scoutfs_item_shrink, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key), - TP_ARGS(sb, key) -); - DEFINE_EVENT(scoutfs_key_class, scoutfs_xattr_get_next_key, TP_PROTO(struct super_block *sb, struct scoutfs_key *key), TP_ARGS(sb, key) ); -DECLARE_EVENT_CLASS(scoutfs_range_class, - TP_PROTO(struct super_block *sb, struct scoutfs_key *start, - struct scoutfs_key *end), - TP_ARGS(sb, start, end), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(start) - sk_trace_define(end) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(start, start); - sk_trace_assign(end, end); - ), - TP_printk(SCSBF" start "SK_FMT" end "SK_FMT, - SCSB_TRACE_ARGS, sk_trace_args(start), sk_trace_args(end)) -); - -DEFINE_EVENT(scoutfs_range_class, scoutfs_item_insert_batch, - TP_PROTO(struct super_block *sb, struct scoutfs_key *start, - struct scoutfs_key *end), - TP_ARGS(sb, start, end) -); - -DEFINE_EVENT(scoutfs_range_class, scoutfs_item_invalidate_range, - TP_PROTO(struct super_block *sb, struct scoutfs_key *start, - struct scoutfs_key *end), - TP_ARGS(sb, start, end) -); - -DECLARE_EVENT_CLASS(scoutfs_cached_range_class, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(void *, rng) - sk_trace_define(start) - sk_trace_define(end) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->rng = rng; - sk_trace_assign(start, start); - sk_trace_assign(end, end); - ), - TP_printk(SCSBF" rng %p start "SK_FMT" end "SK_FMT, - SCSB_TRACE_ARGS, __entry->rng, sk_trace_args(start), - sk_trace_args(end)) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_free, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_ins_rb_insert, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_mid_left, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_start, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_remove_end, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_rem_rb_insert, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_start, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - -DEFINE_EVENT(scoutfs_cached_range_class, scoutfs_item_range_shrink_end, - TP_PROTO(struct super_block *sb, void *rng, - struct scoutfs_key *start, struct scoutfs_key *end), - TP_ARGS(sb, rng, start, end) -); - #define lock_mode(mode) \ __print_symbolic(mode, \ { SCOUTFS_LOCK_NULL, "NULL" }, \ @@ -1708,79 +1066,6 @@ DEFINE_EVENT(scoutfs_lock_class, scoutfs_lock_shrink, TP_ARGS(sb, lck) ); -DECLARE_EVENT_CLASS(scoutfs_seg_class, - TP_PROTO(struct scoutfs_segment *seg), - TP_ARGS(seg), - TP_STRUCT__entry( - __field(unsigned int, major) - __field(unsigned int, minor) - __field(struct scoutfs_segment *, seg) - __field(int, refcount) - __field(u64, segno) - __field(unsigned long, flags) - __field(int, err) - ), - TP_fast_assign( - __entry->major = MAJOR(seg->sb->s_bdev->bd_dev); - __entry->minor = MINOR(seg->sb->s_bdev->bd_dev); - __entry->seg = seg; - __entry->refcount = atomic_read(&seg->refcount); - __entry->segno = seg->segno; - __entry->flags = seg->flags; - __entry->err = seg->err; - ), - TP_printk("dev %u:%u seg %p refcount %d segno %llu flags %lx err %d", - __entry->major, __entry->minor, __entry->seg, __entry->refcount, - __entry->segno, __entry->flags, __entry->err) -); - -DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_alloc, - TP_PROTO(struct scoutfs_segment *seg), - TP_ARGS(seg) -); - -DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_shrink, - TP_PROTO(struct scoutfs_segment *seg), - TP_ARGS(seg) -); - -DEFINE_EVENT(scoutfs_seg_class, scoutfs_seg_free, - TP_PROTO(struct scoutfs_segment *seg), - TP_ARGS(seg) -); - -TRACE_EVENT(scoutfs_seg_append_item, - TP_PROTO(struct super_block *sb, u64 segno, u64 seq, u32 nr_items, - u32 total_bytes, struct scoutfs_key *key, u16 val_len), - - TP_ARGS(sb, segno, seq, nr_items, total_bytes, key, val_len), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, segno) - __field(__u64, seq) - __field(__u32, nr_items) - __field(__u32, total_bytes) - sk_trace_define(key) - __field(__u16, val_len) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->segno = segno; - __entry->seq = seq; - __entry->nr_items = nr_items; - __entry->total_bytes = total_bytes; - sk_trace_assign(key, key); - __entry->val_len = val_len; - ), - - TP_printk(SCSBF" segno %llu seq %llu nr_items %u total_bytes %u key "SK_FMT" val_len %u", - SCSB_TRACE_ARGS, __entry->segno, __entry->seq, - __entry->nr_items, __entry->total_bytes, - sk_trace_args(key), __entry->val_len) -); - DECLARE_EVENT_CLASS(scoutfs_net_class, TP_PROTO(struct super_block *sb, struct sockaddr_in *name, struct sockaddr_in *peer, struct scoutfs_net_header *nh), @@ -1961,14 +1246,6 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_server_commit_work_exit, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) ); -DEFINE_EVENT(scoutfs_work_class, scoutfs_server_compact_work_enter, - TP_PROTO(struct super_block *sb, u64 data, int ret), - TP_ARGS(sb, data, ret) -); -DEFINE_EVENT(scoutfs_work_class, scoutfs_server_compact_work_exit, - TP_PROTO(struct super_block *sb, u64 data, int ret), - TP_ARGS(sb, data, ret) -); DEFINE_EVENT(scoutfs_work_class, scoutfs_net_proc_work_enter, TP_PROTO(struct super_block *sb, u64 data, int ret), TP_ARGS(sb, data, ret) @@ -2054,66 +1331,6 @@ DEFINE_EVENT(scoutfs_work_class, scoutfs_data_return_server_extents_exit, TP_ARGS(sb, data, ret) ); -TRACE_EVENT(scoutfs_item_next_range_check, - TP_PROTO(struct super_block *sb, int cached, - struct scoutfs_key *key, struct scoutfs_key *pos, - struct scoutfs_key *last, struct scoutfs_key *end, - struct scoutfs_key *range_end), - TP_ARGS(sb, cached, key, pos, last, end, range_end), - TP_STRUCT__entry( - __field(void *, sb) - __field(int, cached) - sk_trace_define(key) - sk_trace_define(pos) - sk_trace_define(last) - sk_trace_define(end) - sk_trace_define(range_end) - ), - TP_fast_assign( - __entry->sb = sb; - __entry->cached = cached; - sk_trace_assign(key, key); - sk_trace_assign(pos, pos); - sk_trace_assign(last, last); - sk_trace_assign(end, end); - sk_trace_assign(range_end, range_end); - ), - TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" last "SK_FMT" end "SK_FMT" range_end "SK_FMT, - __entry->sb, __entry->cached, sk_trace_args(key), - sk_trace_args(pos), sk_trace_args(last), - sk_trace_args(end), sk_trace_args(range_end)) -); - -TRACE_EVENT(scoutfs_item_prev_range_check, - TP_PROTO(struct super_block *sb, int cached, - struct scoutfs_key *key, struct scoutfs_key *pos, - struct scoutfs_key *first, struct scoutfs_key *start, - struct scoutfs_key *range_start), - TP_ARGS(sb, cached, key, pos, first, start, range_start), - TP_STRUCT__entry( - __field(void *, sb) - __field(int, cached) - sk_trace_define(key) - sk_trace_define(pos) - sk_trace_define(first) - sk_trace_define(start) - sk_trace_define(range_start) - ), - TP_fast_assign( - __entry->sb = sb; - __entry->cached = cached; - sk_trace_assign(key, key); - sk_trace_assign(pos, pos); - sk_trace_assign(first, first); - sk_trace_assign(start, start); - sk_trace_assign(range_start, range_start); - ), - TP_printk("sb %p cached %d key "SK_FMT" pos "SK_FMT" first "SK_FMT" start "SK_FMT" range_start "SK_FMT, - __entry->sb, __entry->cached, sk_trace_args(key), - sk_trace_args(pos), sk_trace_args(first), - sk_trace_args(start), sk_trace_args(range_start)) -); - DECLARE_EVENT_CLASS(scoutfs_shrink_exit_class, TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), TP_ARGS(sb, nr_to_scan, ret), @@ -2136,50 +1353,6 @@ DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_lock_shrink_exit, TP_ARGS(sb, nr_to_scan, ret) ); -DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_seg_shrink_exit, - TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), - TP_ARGS(sb, nr_to_scan, ret) -); - -DEFINE_EVENT(scoutfs_shrink_exit_class, scoutfs_item_shrink_exit, - TP_PROTO(struct super_block *sb, unsigned long nr_to_scan, int ret), - TP_ARGS(sb, nr_to_scan, ret) -); - -TRACE_EVENT(scoutfs_item_shrink_around, - TP_PROTO(struct super_block *sb, - struct scoutfs_key *rng_start, - struct scoutfs_key *rng_end, struct scoutfs_key *item, - struct scoutfs_key *prev, struct scoutfs_key *first, - struct scoutfs_key *last, struct scoutfs_key *next), - TP_ARGS(sb, rng_start, rng_end, item, prev, first, last, next), - TP_STRUCT__entry( - __field(void *, sb) - sk_trace_define(rng_start) - sk_trace_define(rng_end) - sk_trace_define(item) - sk_trace_define(prev) - sk_trace_define(first) - sk_trace_define(last) - sk_trace_define(next) - ), - TP_fast_assign( - __entry->sb = sb; - sk_trace_assign(rng_start, rng_start); - sk_trace_assign(rng_end, rng_end); - sk_trace_assign(item, item); - sk_trace_assign(prev, prev); - sk_trace_assign(first, first); - sk_trace_assign(last, last); - sk_trace_assign(next, next); - ), - TP_printk("sb %p rng_start "SK_FMT" rng_end "SK_FMT" item "SK_FMT" prev "SK_FMT" first "SK_FMT" last "SK_FMT" next "SK_FMT, - __entry->sb, sk_trace_args(rng_start), - sk_trace_args(rng_end), sk_trace_args(item), - sk_trace_args(prev), sk_trace_args(first), - sk_trace_args(last), sk_trace_args(next)) -); - TRACE_EVENT(scoutfs_rename, TP_PROTO(struct super_block *sb, struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, @@ -2515,14 +1688,6 @@ DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_allocated, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) ); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_segno_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_segno_allocated, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_free_pending_extent, TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), TP_ARGS(sb, ext) @@ -2559,37 +1724,6 @@ TRACE_EVENT(scoutfs_online_offline_blocks, __entry->on_now, __entry->off_now) ); -DECLARE_EVENT_CLASS(scoutfs_segno_class, - TP_PROTO(struct super_block *sb, u64 segno), - - TP_ARGS(sb, segno), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__s64, segno) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->segno = segno; - ), - - TP_printk(SCSBF" segno %llu", - SCSB_TRACE_ARGS, __entry->segno) -); -DEFINE_EVENT(scoutfs_segno_class, scoutfs_alloc_segno, - TP_PROTO(struct super_block *sb, u64 segno), - TP_ARGS(sb, segno) -); -DEFINE_EVENT(scoutfs_segno_class, scoutfs_free_segno, - TP_PROTO(struct super_block *sb, u64 segno), - TP_ARGS(sb, segno) -); -DEFINE_EVENT(scoutfs_segno_class, scoutfs_remove_segno, - TP_PROTO(struct super_block *sb, u64 segno), - TP_ARGS(sb, segno) -); - DECLARE_EVENT_CLASS(scoutfs_server_client_count_class, TP_PROTO(struct super_block *sb, u64 rid, unsigned long nr_clients), diff --git a/kmod/src/seg.c b/kmod/src/seg.c deleted file mode 100644 index 49523b90..00000000 --- a/kmod/src/seg.c +++ /dev/null @@ -1,868 +0,0 @@ -/* - * Copyright (C) 2016 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "seg.h" -#include "bio.h" -#include "kvec.h" -#include "cmp.h" -#include "manifest.h" -#include "key.h" -#include "counters.h" -#include "triggers.h" -#include "msg.h" -#include "server.h" -#include "scoutfs_trace.h" - -/* - * seg.c should just be about the cache and io, and maybe - * iteration and stuff. - * - * XXX: - * - lru and shrinker - * - verify csum - * - make sure item headers don't cross page boundaries - * - just wait on pages instead of weird flags? - */ - -struct segment_cache { - struct super_block *sb; - spinlock_t lock; - struct rb_root root; - wait_queue_head_t waitq; - - struct shrinker shrinker; - struct list_head lru_list; - unsigned long lru_nr; -}; - - -enum { - SF_END_IO = 0, - SF_CALC_CRC_STARTED, - SF_CALC_CRC_DONE, - SF_INVALID_CRC, -}; - -static void *off_ptr(struct scoutfs_segment *seg, u32 off) -{ - unsigned int pg = off >> PAGE_SHIFT; - unsigned int pg_off = off & ~PAGE_MASK; - - return page_address(seg->pages[pg]) + pg_off; -} - -static struct scoutfs_segment *alloc_seg(struct super_block *sb, u64 segno) -{ - struct scoutfs_segment *seg; - struct page *page; - int i; - - /* don't waste the tail of pages */ - BUILD_BUG_ON(SCOUTFS_SEGMENT_SIZE % PAGE_SIZE); - - seg = kzalloc(sizeof(struct scoutfs_segment), GFP_NOFS); - if (!seg) - return seg; - - seg->sb = sb; - RB_CLEAR_NODE(&seg->node); - INIT_LIST_HEAD(&seg->lru_entry); - atomic_set(&seg->refcount, 1); - seg->segno = segno; - - for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) { - page = alloc_page(GFP_NOFS); - if (!page) { - scoutfs_seg_put(seg); - return ERR_PTR(-ENOMEM); - } - - seg->pages[i] = page; - } - - trace_scoutfs_seg_alloc(seg); - scoutfs_inc_counter(sb, seg_alloc); - - return seg; -} - -void scoutfs_seg_get(struct scoutfs_segment *seg) -{ - atomic_inc(&seg->refcount); -} - -void scoutfs_seg_put(struct scoutfs_segment *seg) -{ - int i; - - if (!IS_ERR_OR_NULL(seg) && atomic_dec_and_test(&seg->refcount)) { - trace_scoutfs_seg_free(seg); - scoutfs_inc_counter(seg->sb, seg_free); - WARN_ON_ONCE(!RB_EMPTY_NODE(&seg->node)); - WARN_ON_ONCE(!list_empty(&seg->lru_entry)); - for (i = 0; i < SCOUTFS_SEGMENT_PAGES; i++) - if (seg->pages[i]) - __free_page(seg->pages[i]); - kfree(seg); - } -} - -static struct scoutfs_segment *find_seg(struct rb_root *root, u64 segno) -{ - struct rb_node *node = root->rb_node; - struct rb_node *parent = NULL; - struct scoutfs_segment *seg; - int cmp; - - while (node) { - parent = node; - seg = container_of(node, struct scoutfs_segment, node); - - cmp = scoutfs_cmp_u64s(segno, seg->segno); - if (cmp < 0) - node = node->rb_left; - else if (cmp > 0) - node = node->rb_right; - else - return seg; - } - - return NULL; -} - -static void lru_check(struct segment_cache *cac, struct scoutfs_segment *seg) -{ - if (RB_EMPTY_NODE(&seg->node)) { - if (!list_empty(&seg->lru_entry)) { - list_del_init(&seg->lru_entry); - cac->lru_nr--; - } - } else { - if (list_empty(&seg->lru_entry)) { - list_add_tail(&seg->lru_entry, &cac->lru_list); - cac->lru_nr++; - } else { - list_move_tail(&seg->lru_entry, &cac->lru_list); - } - } -} - -static __le32 calc_seg_crc(struct scoutfs_segment *seg) -{ - u32 total = scoutfs_seg_total_bytes(seg); - u32 crc = ~0; - u32 off; - u32 len; - - off = offsetof(struct scoutfs_segment_block, _padding) + - FIELD_SIZEOF(struct scoutfs_segment_block, _padding); - - while (off < total) { - len = min(total - off, - SCOUTFS_BLOCK_SIZE - (off & SCOUTFS_BLOCK_MASK)); - crc = crc32c(crc, off_ptr(seg, off), len); - off += len; - } - - return cpu_to_le32(crc); -} - -/* - * This always inserts the segment into the rbtree. If there's already - * a segment at the given seg then it is removed and returned. The - * caller doesn't have to erase it from the tree if it's returned but it - * does have to put the reference that it's given. - */ -static struct scoutfs_segment *replace_seg(struct segment_cache *cac, - struct scoutfs_segment *ins) -{ - struct rb_root *root = &cac->root; - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct scoutfs_segment *seg; - struct scoutfs_segment *found = NULL; - int cmp; - - while (*node) { - parent = *node; - seg = container_of(*node, struct scoutfs_segment, node); - - cmp = scoutfs_cmp_u64s(ins->segno, seg->segno); - if (cmp < 0) { - node = &(*node)->rb_left; - } else if (cmp > 0) { - node = &(*node)->rb_right; - } else { - rb_replace_node(&seg->node, &ins->node, root); - RB_CLEAR_NODE(&seg->node); - lru_check(cac, seg); - lru_check(cac, ins); - found = seg; - break; - } - } - - if (!found) { - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); - lru_check(cac, ins); - } - - return found; -} - -static bool erase_seg(struct segment_cache *cac, struct scoutfs_segment *seg) -{ - if (!RB_EMPTY_NODE(&seg->node)) { - rb_erase(&seg->node, &cac->root); - RB_CLEAR_NODE(&seg->node); - lru_check(cac, seg); - return true; - } - - return false; -} - -static void seg_end_io(struct super_block *sb, void *data, int err) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac = sbi->segment_cache; - struct scoutfs_segment *seg = data; - unsigned long flags; - bool erased = false; - - spin_lock_irqsave(&cac->lock, flags); - - set_bit(SF_END_IO, &seg->flags); - - if (err) { - seg->err = err; - erased = erase_seg(cac, seg); - } else { - lru_check(cac, seg); - } - - spin_unlock_irqrestore(&cac->lock, flags); - - smp_mb__after_atomic(); - if (waitqueue_active(&cac->waitq)) - wake_up(&cac->waitq); - - if (erased) - scoutfs_seg_put(seg); - scoutfs_seg_put(seg); -} - -static u64 segno_to_blkno(u64 blkno) -{ - return blkno << (SCOUTFS_SEGMENT_SHIFT - SCOUTFS_BLOCK_SHIFT); -} - -int scoutfs_seg_alloc(struct super_block *sb, u64 segno, - struct scoutfs_segment **seg_ret) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac = sbi->segment_cache; - struct scoutfs_segment *existing; - struct scoutfs_segment *seg; - unsigned long flags; - int ret; - - seg = alloc_seg(sb, segno); - if (!seg) { - ret = -ENOMEM; - goto out; - } - - /* reads shouldn't wait for this */ - set_bit(SF_END_IO, &seg->flags); - - /* zero the block header so the caller knows to initialize */ - memset(page_address(seg->pages[0]), 0, - sizeof(struct scoutfs_segment_block)); - - /* XXX always remove existing segs, is that necessary? */ - spin_lock_irqsave(&cac->lock, flags); - - atomic_inc(&seg->refcount); - existing = replace_seg(cac, seg); - spin_unlock_irqrestore(&cac->lock, flags); - if (existing) - scoutfs_seg_put(existing); - - ret = 0; -out: - *seg_ret = seg; - return ret; - -} - - -/* - * The bios submitted by this don't have page references themselves. If - * this succeeds then the caller must call _wait before putting their - * seg ref. - */ -struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, - u64 segno) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac = sbi->segment_cache; - struct scoutfs_segment *existing; - struct scoutfs_segment *seg; - unsigned long flags; - - trace_scoutfs_seg_submit_read(sb, segno); - - spin_lock_irqsave(&cac->lock, flags); - seg = find_seg(&cac->root, segno); - if (seg) { - lru_check(cac, seg); - atomic_inc(&seg->refcount); - } - spin_unlock_irqrestore(&cac->lock, flags); - if (seg) - return seg; - - seg = alloc_seg(sb, segno); - if (IS_ERR(seg)) - return seg; - - /* always drop existing segs, could compare seqs */ - spin_lock_irqsave(&cac->lock, flags); - atomic_inc(&seg->refcount); - existing = replace_seg(cac, seg); - spin_unlock_irqrestore(&cac->lock, flags); - if (existing) - scoutfs_seg_put(existing); - - atomic_inc(&seg->refcount); - scoutfs_bio_submit(sb, READ, seg->pages, segno_to_blkno(seg->segno), - SCOUTFS_SEGMENT_BLOCKS, seg_end_io, seg); - - return seg; -} - -/* - * The caller has ensured that the segment won't be modified while - * it is in flight. - */ -int scoutfs_seg_submit_write(struct super_block *sb, - struct scoutfs_segment *seg, - struct scoutfs_bio_completion *comp) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - - trace_scoutfs_seg_submit_write(sb, seg->segno); - - sblk->crc = calc_seg_crc(seg); - - scoutfs_bio_submit_comp(sb, WRITE, seg->pages, - segno_to_blkno(seg->segno), - SCOUTFS_SEGMENT_BLOCKS, comp); - - return 0; -} - -/* - * Wait for IO on the segment to complete. - * - * The caller provides the segno and seq from their segment reference to - * validate that we found the version of the segment that they were - * looking for. If we find an old cached version we return -ESTALE and - * the caller has to retry its reference to find the current segment for - * its operation. (Typically by getting a new manifest btree root and - * searching for keys in the manifest.) - * - * An invalid crc can be racing to read a stale segment while it's being - * written. The caller will retry and consider it corrupt if it keeps - * getting stale reads. - */ -int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, - u64 segno, u64 seq) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac = sbi->segment_cache; - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - unsigned long flags; - bool erased; - int ret; - - ret = wait_event_interruptible(cac->waitq, - test_bit(SF_END_IO, &seg->flags)); - if (ret) - goto out; - - if (seg->err) { - ret = seg->err; - goto out; - } - - /* calc crc in waiting task instead of end_io */ - if (!test_bit(SF_CALC_CRC_DONE, &seg->flags) && - !test_and_set_bit(SF_CALC_CRC_STARTED, &seg->flags)) { - if (sblk->crc != calc_seg_crc(seg)) { - scoutfs_inc_counter(sb, seg_csum_error); - set_bit(SF_INVALID_CRC, &seg->flags); - } - set_bit(SF_CALC_CRC_DONE, &seg->flags); - wake_up(&cac->waitq); - } - - /* very rarely race waiting for calc to finish */ - ret = wait_event_interruptible(cac->waitq, - test_bit(SF_CALC_CRC_DONE, &seg->flags)); - if (ret) - goto out; - - sblk = off_ptr(seg, 0); - - if (test_bit(SF_INVALID_CRC, &seg->flags) || - segno != le64_to_cpu(sblk->segno) || - seq != le64_to_cpu(sblk->seq) || - scoutfs_trigger(sb, SEG_STALE_READ)) { - spin_lock_irqsave(&cac->lock, flags); - erased = erase_seg(cac, seg); - spin_unlock_irqrestore(&cac->lock, flags); - if (erased) - scoutfs_seg_put(seg); - - scoutfs_inc_counter(sb, seg_stale_read); - ret = -ESTALE; - } -out: - return ret; -} - -static u32 item_bytes(u8 nr_links, u16 val_len) -{ - return offsetof(struct scoutfs_segment_item, skip_links[nr_links]) + - val_len; -} - -static inline void *item_val_ptr(struct scoutfs_segment_item *item) -{ - return (void *)item + item_bytes(item->nr_links, 0); -} - -/* copy the item key into the caller's key and init their val to ref the val */ -static void get_item_key_val(struct scoutfs_segment *seg, int off, - struct scoutfs_key *key, struct kvec *val) -{ - struct scoutfs_segment_item *item = off_ptr(seg, off); - - if (key) - *key = item->key; - - if (val) - kvec_init(val, item_val_ptr(item), le16_to_cpu(item->val_len)); -} - -static void first_last_keys(struct scoutfs_segment *seg, - struct scoutfs_key *first, - struct scoutfs_key *last) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - - get_item_key_val(seg, sizeof(struct scoutfs_segment_block), - first, NULL); - get_item_key_val(seg, le32_to_cpu(sblk->last_item_off), last, NULL); -} - -static int check_caller_off(struct scoutfs_segment_block *sblk, int off) -{ - if (off >= 0 && off < sizeof(struct scoutfs_segment_block)) - off = sizeof(struct scoutfs_segment_block); - - if (off > le32_to_cpu(sblk->last_item_off)) - off = -ENOENT; - - return off; -} - -/* - * Give the caller the key and value of the item at the given offset. - * - * Negative offsets are sticky errors and offsets outside the used bytes - * in the segment return -ENOENT; - * - * All other offsets must be initial values less than the segment header - * size, notably including 0, or returned from _next_off(). - */ -int scoutfs_seg_get_item(struct scoutfs_segment *seg, int off, - struct scoutfs_key *key, struct kvec *val,u8 *flags) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; - - off = check_caller_off(sblk, off); - if (off < 0) - return off; - - get_item_key_val(seg, off, key, val); - - if (flags) { - item = off_ptr(seg, off); - *flags = item->flags; - } - - return 0; -} - -/* - * Return the number of links that the *next* added node should have. - * We're appending in order so we can use the low bits of the node count - * to get an ideal distribution of the number of links to enable (log n) - * searching: of links in each node. Half of the nodes will have 1 - * links, a quarter will have 2, an eighth will have 3, and so on. - */ -static u8 skip_next_nr(u32 nr_items) -{ - return ffs(nr_items + 1); -} - -/* The highest 1-based set bit is the max number of links any node can have */ -static u8 skip_most_nr(u32 nr_items) -{ - return fls(nr_items); -} - -/* - * Find offset of the first item in the segment whose key is greater - * than or equal to the search key. -ENOENT is returned if there's no - * item that matches. - * - * This is a standard skip list search from the segment block through - * the items. Follow high less frequent links while the key is greater - * than the items and descend down to lower more frequent links when the - * search key is less. - */ -int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key *key) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; - __le32 *links; - int cmp; - int ret; - int i; - int off; - - links = sblk->skip_links; - ret = -ENOENT; - for (i = skip_most_nr(le32_to_cpu(sblk->nr_items)) - 1; i >= 0; i--) { - if (links[i] == 0) - continue; - - off = le32_to_cpu(links[i]); - item = off_ptr(seg, off); - - cmp = scoutfs_key_compare(key, &item->key); - if (cmp == 0) { - ret = off; - break; - } - - if (cmp > 0) { - links = item->skip_links; - i++; - } else { - ret = off; - } - } - - return ret; -} - -/* - * Return the offset of the next item after the current item. The input offset - * must be a valid offset from _find_off(). - */ -int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; - - off = check_caller_off(sblk, off); - if (off > 0) { - item = off_ptr(seg, off); - off = le32_to_cpu(item->skip_links[0]); - if (off == 0) - off = -ENOENT; - } - return off; -} - -/* - * Return the count of bytes of the segment actually used. - */ -u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - - return le32_to_cpu(sblk->total_bytes); -} - -/* - * Returns true if the given item population will fit in a single - * segment. - * - * We don't have items cross block boundaries. It would be too - * expensive to maintain packing of sorted dirty items in bins. Instead - * we assume that we'll lose the worst case largest possible item on every - * block transition. This will almost never be the case. This causes us - * to lose around 15% of space for level 0 segment writes. - * - * Our pattern of item link counts ensures that there will always be fewer - * than two links per item. We assume the worst case items have the - * max number of links. - */ -bool scoutfs_seg_fits_single(u32 nr_items, u32 val_bytes) -{ - u32 header = sizeof(struct scoutfs_segment_block); - u32 items = nr_items * item_bytes(2, 0); - u32 item_pad = item_bytes(skip_most_nr(nr_items), - SCOUTFS_MAX_VAL_SIZE) - 1; - u32 padding = (SCOUTFS_SEGMENT_SIZE / SCOUTFS_BLOCK_SIZE) * item_pad; - - return (header + items + val_bytes + padding) <= SCOUTFS_SEGMENT_SIZE; -} - -static u32 align_item_off(struct scoutfs_segment *seg, u32 item_off, u32 bytes) -{ - u32 space = SCOUTFS_BLOCK_SIZE - (item_off & SCOUTFS_BLOCK_MASK); - - if (bytes > space) { - memset(off_ptr(seg, item_off), 0, space); - return item_off + space; - } - - return item_off; -} - - -/* - * Append an item to the segment. The caller always appends items that - * have been sorted by their keys. They may not know how many will fit. - * We return true if we appended and false if the segment was full. - */ -bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key *key, struct kvec *val, - u8 flags, __le32 **links) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_segment_item *item; - struct kvec item_val; - u8 nr_links; - u32 val_len; - u32 bytes; - u32 off; - int i; - - val_len = val ? val->iov_len : 0; - - /* initialize the segment and skip links as the first item is appended */ - if (sblk->nr_items == 0) { - /* XXX the segment block header is a mess, be better */ - sblk->segno = cpu_to_le64(seg->segno); - sblk->seq = super->next_seg_seq; - le64_add_cpu(&super->next_seg_seq, 1); - sblk->total_bytes = cpu_to_le32(sizeof(*sblk)); - - for (i = 0; i < SCOUTFS_MAX_SKIP_LINKS; i++) - links[i] = &sblk->skip_links[i]; - } - - trace_scoutfs_seg_append_item(sb, le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->seq), - le32_to_cpu(sblk->nr_items), - le32_to_cpu(sblk->total_bytes), - key, val_len); - - /* - * It's very bad data corruption if we write out of order items - * to a segment. It'll mislead the key search during read and - * stop it from finding its items. - */ - off = le32_to_cpu(sblk->last_item_off); - if (off) { - item = off_ptr(seg, off); - scoutfs_bug_on(sb, scoutfs_key_compare(key, &item->key) <= 0, - "key "SK_FMT" item->key "SK_FMT, - SK_ARG(key), SK_ARG(&item->key)); - } - - nr_links = skip_next_nr(le32_to_cpu(sblk->nr_items)); - bytes = item_bytes(nr_links, val_len); - off = align_item_off(seg, le32_to_cpu(sblk->total_bytes), bytes); - - if ((off + bytes) > SCOUTFS_SEGMENT_SIZE) - return false; - - sblk->last_item_off = cpu_to_le32(off); - sblk->total_bytes = cpu_to_le32(off + bytes); - le32_add_cpu(&sblk->nr_items, 1); - - item = off_ptr(seg, off); - item->key = *key; - item->val_len = cpu_to_le16(val_len); - item->flags = flags; - - /* point the previous skip links at our appended item */ - item->nr_links = nr_links; - for (i = 0; i < nr_links; i++) { - item->skip_links[i] = 0; - *links[i] = cpu_to_le32(off); - links[i] = &item->skip_links[i]; - } - - get_item_key_val(seg, off, NULL, &item_val); - if (val_len) - memcpy(item_val.iov_base, val->iov_base, val_len); - - return true; -} - -void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, - struct scoutfs_segment *seg) -{ - struct scoutfs_segment_block *sblk = off_ptr(seg, 0); - struct scoutfs_key first; - struct scoutfs_key last; - - first_last_keys(seg, &first, &last); - - scoutfs_manifest_init_entry(ment, level, le64_to_cpu(sblk->segno), - le64_to_cpu(sblk->seq), &first, &last); -} - -/* - * We maintain an LRU of segments so that the shrinker can free the - * oldest under memory pressure. Segments are only present in the LRU - * after their IO has completed and while they're in the rbtree. This - * shrink only removes them from the rbtree and drops the reference it - * held. They may be freed a bit later once all their active references - * are dropped. - * - * If this is called with nr_to_scan == 0 then it only returns the nr. - * We avoid acquiring the lock in that case. - * - * Lookup code only uses the lru entry to change position in the LRU while - * the segment is in the rbtree. Once we remove it no one else will use - * the LRU entry and we can use it to track all the segments that we're - * going to put outside of the lock. - * - * XXX: - * - are sc->nr_to_scan and our return meant to be in units of pages? - * - should we sync a transaction here? - */ -static int seg_lru_shrink(struct shrinker *shrink, struct shrink_control *sc) -{ - struct segment_cache *cac = container_of(shrink, struct segment_cache, - shrinker); - struct super_block *sb = cac->sb; - struct scoutfs_segment *seg; - struct scoutfs_segment *tmp; - unsigned long flags; - unsigned long nr; - LIST_HEAD(list); - int ret; - - nr = DIV_ROUND_UP(sc->nr_to_scan, SCOUTFS_SEGMENT_PAGES); - if (!nr) - goto out; - - spin_lock_irqsave(&cac->lock, flags); - - list_for_each_entry_safe(seg, tmp, &cac->lru_list, lru_entry) { - /* shouldn't be possible */ - if (WARN_ON_ONCE(RB_EMPTY_NODE(&seg->node))) - continue; - - if (nr-- == 0) - break; - - /* using ref that rb tree presence had */ - erase_seg(cac, seg); - list_add_tail(&seg->lru_entry, &list); - } - - spin_unlock_irqrestore(&cac->lock, flags); - - list_for_each_entry_safe(seg, tmp, &list, lru_entry) { - trace_scoutfs_seg_shrink(seg); - scoutfs_inc_counter(sb, seg_shrink); - list_del_init(&seg->lru_entry); - scoutfs_seg_put(seg); - } - -out: - ret = min_t(unsigned long, cac->lru_nr * SCOUTFS_SEGMENT_PAGES, - INT_MAX); - trace_scoutfs_seg_shrink_exit(sb, sc->nr_to_scan, ret); - return ret; -} - -int scoutfs_seg_setup(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac; - - cac = kzalloc(sizeof(struct segment_cache), GFP_KERNEL); - if (!cac) - return -ENOMEM; - sbi->segment_cache = cac; - - cac->sb = sb; - spin_lock_init(&cac->lock); - cac->root = RB_ROOT; - init_waitqueue_head(&cac->waitq); - - cac->shrinker.shrink = seg_lru_shrink; - cac->shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&cac->shrinker); - INIT_LIST_HEAD(&cac->lru_list); - - return 0; -} - -void scoutfs_seg_destroy(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct segment_cache *cac = sbi->segment_cache; - struct scoutfs_segment *seg; - struct rb_node *node; - - if (cac) { - if (cac->shrinker.shrink == seg_lru_shrink) - unregister_shrinker(&cac->shrinker); - - for (node = rb_first(&cac->root); node; ) { - seg = container_of(node, struct scoutfs_segment, node); - node = rb_next(node); - erase_seg(cac, seg); - scoutfs_seg_put(seg); - } - - kfree(cac); - } -} diff --git a/kmod/src/seg.h b/kmod/src/seg.h deleted file mode 100644 index 58c4a64a..00000000 --- a/kmod/src/seg.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef _SCOUTFS_SEG_H_ -#define _SCOUTFS_SEG_H_ - -struct scoutfs_bio_completion; -struct scoutfs_key; -struct scoutfs_manifest_entry; -struct kvec; - -/* this is only visible for trace events */ -struct scoutfs_segment { - struct super_block *sb; - struct rb_node node; - struct list_head lru_entry; - atomic_t refcount; - u64 segno; - unsigned long flags; - int err; - struct page *pages[SCOUTFS_SEGMENT_PAGES]; -}; - -struct scoutfs_segment *scoutfs_seg_submit_read(struct super_block *sb, - u64 segno); -int scoutfs_seg_wait(struct super_block *sb, struct scoutfs_segment *seg, - u64 segno, u64 seq); - -int scoutfs_seg_find_off(struct scoutfs_segment *seg, struct scoutfs_key *key); -int scoutfs_seg_next_off(struct scoutfs_segment *seg, int off); -u32 scoutfs_seg_total_bytes(struct scoutfs_segment *seg); -int scoutfs_seg_get_item(struct scoutfs_segment *seg, int off, - struct scoutfs_key *key, struct kvec *val, u8 *flags); - -void scoutfs_seg_get(struct scoutfs_segment *seg); -void scoutfs_seg_put(struct scoutfs_segment *seg); - -int scoutfs_seg_alloc(struct super_block *sb, u64 segno, - struct scoutfs_segment **seg_ret); -bool scoutfs_seg_fits_single(u32 nr_items, u32 val_bytes); -bool scoutfs_seg_append_item(struct super_block *sb, struct scoutfs_segment *seg, - struct scoutfs_key *key, struct kvec *val, - u8 flags, __le32 **links); -void scoutfs_seg_init_ment(struct scoutfs_manifest_entry *ment, int level, - struct scoutfs_segment *seg); - -int scoutfs_seg_submit_write(struct super_block *sb, - struct scoutfs_segment *seg, - struct scoutfs_bio_completion *comp); - -int scoutfs_seg_setup(struct super_block *sb); -void scoutfs_seg_destroy(struct super_block *sb); - -#endif diff --git a/kmod/src/server.c b/kmod/src/server.c index 578d94d2..ee378ea8 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -28,9 +28,6 @@ #include "block.h" #include "balloc.h" #include "btree.h" -#include "manifest.h" -#include "seg.h" -#include "compact.h" #include "scoutfs_trace.h" #include "msg.h" #include "server.h" @@ -63,15 +60,11 @@ struct server_info { u64 term; struct scoutfs_net_connection *conn; - /* request processing coordinates committing manifest and alloc */ + /* request processing coordinates shared commits */ struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; struct work_struct commit_work; - /* server remembers the stable manifest root for clients */ - seqcount_t stable_seqcount; - struct scoutfs_btree_root stable_manifest_root; - /* server tracks seq use */ struct rw_semaphore seq_rwsem; @@ -82,12 +75,6 @@ struct server_info { struct list_head clients; unsigned long nr_clients; - /* track compaction in flight */ - unsigned long compacts_per_client; - unsigned long nr_compacts; - struct list_head compacts; - struct work_struct compact_work; - /* track clients waiting in unmmount for farewell response */ struct mutex farewell_mutex; struct list_head farewell_requests; @@ -108,7 +95,6 @@ struct server_info { struct server_client_info { u64 rid; struct list_head head; - unsigned long nr_compacts; }; struct commit_waiter { @@ -245,13 +231,7 @@ static int server_extent_io(struct super_block *sb, int op, /* * Allocate an extent of the given length in the first smallest free - * extent that contains it. We allocate in multiples of segment blocks - * and expose that to callers today. - * - * This doesn't have the cursor that segment allocation does. It's - * possible that a recently freed segment can merge to form a larger - * free extent that can be very quickly allocated to a node. The hope is - * that doesn't happen very often. + * extent that contains it. */ static int alloc_extent(struct super_block *sb, u64 blocks, u64 *start, u64 *len) @@ -265,11 +245,6 @@ static int alloc_extent(struct super_block *sb, u64 blocks, down_write(&server->alloc_rwsem); - if (blocks & (SCOUTFS_SEGMENT_BLOCKS - 1)) { - ret = -EINVAL; - goto out; - } - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, 0, 0, blocks, 0, 0); ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); @@ -413,101 +388,6 @@ static int free_extent(struct super_block *sb, u64 start, u64 len) return ret; } -/* - * This is called by the compaction code which is running in the server. - * The server caller has held all the locks, etc. - */ -static int free_segno(struct super_block *sb, u64 segno) -{ - scoutfs_inc_counter(sb, server_free_segno); - trace_scoutfs_free_segno(sb, segno); - return free_extent(sb, segno << SCOUTFS_SEGMENT_BLOCK_SHIFT, - SCOUTFS_SEGMENT_BLOCKS); -} - -/* - * Allocate a segment on behalf of compaction or a node wanting to write - * a level 0 segment. It has to be aligned to the segment size because - * we address segments with aligned segment numbers instead of block - * offsets. - * - * We can use a simple cursor sweep of the index by start because all - * server extents are multiples of the segment size. Sweeping through - * the volume tries to spread out new segment writes and make it more - * rare to write to a recently freed segment which can cause a client to - * have to re-read the manifest. - */ -static int alloc_segno(struct super_block *sb, u64 *segno) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_extent ext; - u64 curs; - int ret; - - down_write(&server->alloc_rwsem); - - curs = ALIGN(le64_to_cpu(super->alloc_cursor), SCOUTFS_SEGMENT_BLOCKS); - *segno = 0; - - do { - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, - curs, 1, 0, 0); - ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); - } while (ret == -ENOENT && curs && (curs = 0, 1)); - if (ret) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - trace_scoutfs_server_alloc_segno_next(sb, &ext); - - /* use cursor if within extent, otherwise start of next extent */ - if (ext.start < curs) - ext.start = curs; - ext.len = SCOUTFS_SEGMENT_BLOCKS; - - ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); - if (ret) - goto out; - - super->alloc_cursor = cpu_to_le64(ext.start + ext.len); - - *segno = ext.start >> SCOUTFS_SEGMENT_BLOCK_SHIFT; - - trace_scoutfs_server_alloc_segno_allocated(sb, &ext); - trace_scoutfs_alloc_segno(sb, *segno); - scoutfs_inc_counter(sb, server_alloc_segno); - -out: - up_write(&server->alloc_rwsem); - return ret; -} - -/* - * "allocating" a segno removes an unknown segment from the allocator - * and returns it, "removing" a segno removes a specific segno from the - * allocator. - */ -static int remove_segno(struct super_block *sb, u64 segno) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_extent ext; - int ret; - - trace_scoutfs_remove_segno(sb, segno); - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, - segno << SCOUTFS_SEGMENT_BLOCK_SHIFT, - SCOUTFS_SEGMENT_BLOCKS, 0, 0); - - down_write(&server->alloc_rwsem); - ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); - up_write(&server->alloc_rwsem); - return ret; -} - static void stop_server(struct server_info *server) { /* wait_event/wake_up provide barriers */ @@ -515,23 +395,6 @@ static void stop_server(struct server_info *server) wake_up(&server->waitq); } -/* - * Queue compaction work if clients have capacity for processing - * requests and the manifest knows of levels with too many segments. - */ -static void try_queue_compact(struct server_info *server) -{ - struct super_block *sb = server->sb; - bool can_request; - - spin_lock(&server->lock); - can_request = server->nr_compacts < - (server->nr_clients * server->compacts_per_client); - spin_unlock(&server->lock); - if (can_request && scoutfs_manifest_should_compact(sb)) - queue_work(server->wq, &server->compact_work); -} - /* * This is called while still holding the rwsem that prevents commits so * that the caller can be sure to be woken by the next commit after they @@ -662,12 +525,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - write_seqcount_begin(&server->stable_seqcount); - server->stable_manifest_root = SCOUTFS_SB(sb)->super.manifest.root; - write_seqcount_end(&server->stable_seqcount); - ret = 0; - out: node = llist_del_all(&server->commit_waiters); @@ -681,26 +539,6 @@ out: trace_scoutfs_server_commit_work_exit(sb, 0, ret); } -void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, - struct scoutfs_manifest_entry *ment) -{ - net_ment->segno = cpu_to_le64(ment->segno); - net_ment->seq = cpu_to_le64(ment->seq); - net_ment->first = ment->first; - net_ment->last = ment->last; - net_ment->level = ment->level; -} - -void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, - struct scoutfs_net_manifest_entry *net_ment) -{ - ment->segno = le64_to_cpu(net_ment->segno); - ment->seq = le64_to_cpu(net_ment->seq); - ment->level = net_ment->level; - ment->first = net_ment->first; - ment->last = net_ment->last; -} - static int server_alloc_inodes(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) @@ -829,89 +667,6 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } -/* - * We still special case segno allocation because it's aligned and we'd - * like to keep that detail in the server. - */ -static int server_alloc_segno(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct commit_waiter cw; - __le64 lesegno = 0; - u64 segno; - int ret; - - if (arg_len != 0) { - ret = -EINVAL; - goto out; - } - - down_read(&server->commit_rwsem); - ret = alloc_segno(sb, &segno); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); - if (ret) - goto out; - - lesegno = cpu_to_le64(segno); -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, - &lesegno, sizeof(lesegno)); -} - -static int server_record_segment(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_net_manifest_entry *net_ment; - struct scoutfs_manifest_entry ment; - struct commit_waiter cw; - int ret; - - if (arg_len != sizeof(struct scoutfs_net_manifest_entry)) { - ret = -EINVAL; - goto out; - } - - net_ment = arg; - -retry: - down_read(&server->commit_rwsem); - scoutfs_manifest_lock(sb); - - if (scoutfs_manifest_level0_full(sb)) { - scoutfs_manifest_unlock(sb); - up_read(&server->commit_rwsem); - /* XXX waits indefinitely? io errors? */ - wait_event(server->waitq, !scoutfs_manifest_level0_full(sb)); - goto retry; - } - - scoutfs_init_ment_from_net(&ment, net_ment); - - ret = scoutfs_manifest_add(sb, &ment); - scoutfs_manifest_unlock(sb); - - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - - if (ret == 0) { - ret = wait_for_commit(&cw); - if (ret == 0) - try_queue_compact(server); - } - -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); -} - /* * Give the client references to stable persistent trees that they'll * use to write their next transaction. @@ -1272,29 +1027,6 @@ out: &last_seq, sizeof(last_seq)); } -static int server_get_manifest_root(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_btree_root root; - unsigned int start; - int ret; - - if (arg_len == 0) { - do { - start = read_seqcount_begin(&server->stable_seqcount); - root = server->stable_manifest_root; - } while (read_seqcount_retry(&server->stable_seqcount, start)); - ret = 0; - } else { - ret = -EINVAL; - } - - return scoutfs_net_response(sb, conn, cmd, id, ret, - &root, sizeof(root)); -} - /* * Sample the super stats that the client wants for statfs by serializing * with each component. @@ -1829,616 +1561,15 @@ static int server_farewell(struct super_block *sb, return 0; } -/* requests sent to clients are tracked so we can free resources */ -struct compact_request { - struct list_head head; - u64 rid; - struct scoutfs_net_compact_request req; -}; - -/* - * Find a node that can process our compaction request. Return a - * rid if we found a client and added the compaction to the client - * and server counts. Returns 0 if no suitable clients were found. - */ -static u64 compact_request_start(struct super_block *sb, - struct compact_request *cr) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct server_client_info *last; - struct server_client_info *sci; - u64 rid = 0; - - spin_lock(&server->lock); - - /* XXX no last_entry_or_null? :( */ - if (!list_empty(&server->clients)) - last = list_last_entry(&server->clients, - struct server_client_info, head); - else - last = NULL; - - while ((sci = list_first_entry_or_null(&server->clients, - struct server_client_info, - head)) != NULL) { - list_move_tail(&sci->head, &server->clients); - if (sci->nr_compacts < server->compacts_per_client) { - list_add(&cr->head, &server->compacts); - server->nr_compacts++; - sci->nr_compacts++; - rid = sci->rid; - cr->rid = rid; - break; - } - if (sci == last) - break; - } - - trace_scoutfs_server_compact_start(sb, le64_to_cpu(cr->req.id), - cr->req.ents[0].level, rid, - rid ? sci->nr_compacts : 0, - server->nr_compacts, - server->compacts_per_client); - - spin_unlock(&server->lock); - - return rid; -} - -/* - * Find a tracked compact request for the compaction id, remove it from - * the server and client counts, and return it to the caller. - */ -static struct compact_request *compact_request_done(struct super_block *sb, - u64 id) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct compact_request *ret = NULL; - struct server_client_info *sci; - struct compact_request *cr; - - spin_lock(&server->lock); - - list_for_each_entry(cr, &server->compacts, head) { - if (le64_to_cpu(cr->req.id) != id) - continue; - - list_for_each_entry(sci, &server->clients, head) { - if (sci->rid == cr->rid) { - sci->nr_compacts--; - break; - } - } - - server->nr_compacts--; - list_del_init(&cr->head); - ret = cr; - break; - } - - trace_scoutfs_server_compact_done(sb, id, ret ? ret->rid : 0, - server->nr_compacts); - - spin_unlock(&server->lock); - - return ret; -} - -/* - * When a client disconnects we forget the compactions that they had - * in flight so that we have capacity to send compaction requests to the - * remaining clients. - * - * XXX we do not free their allocated segnos because they could still be - * running and writing to those blocks. To do this safely we'd need - * full recovery procedures with fencing to ensure that they're not able - * to write to those blocks anymore. - */ -static void forget_client_compacts(struct super_block *sb, - struct server_client_info *sci) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct compact_request *cr; - struct compact_request *pos; - LIST_HEAD(forget); - - spin_lock(&server->lock); - list_for_each_entry_safe(cr, pos, &server->compacts, head) { - if (cr->rid == sci->rid) { - sci->nr_compacts--; - server->nr_compacts--; - list_move(&cr->head, &forget); - } - } - spin_unlock(&server->lock); - - list_for_each_entry_safe(cr, pos, &forget, head) { - scoutfs_manifest_compact_done(sb, &cr->req); - list_del_init(&cr->head); - kfree(cr); - } -} - -static int segno_in_ents(u64 segno, struct scoutfs_net_manifest_entry *ents, - unsigned int nr) -{ - int i; - - for (i = 0; i < nr; i++) { - if (ents[i].segno == 0) - break; - if (segno == le64_to_cpu(ents[i].segno)) - return 1; - } - - return 0; -} - -static int remove_segnos(struct super_block *sb, __le64 *segnos, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup); - -/* - * Free (unaligned) segnos if they're not found in the unless entries. - * If this returns an error then we've cleaned up partial frees on - * error. This panics if it sees an error and can't cleanup on error. - * - * There are variants of this for lots of add/del, alloc/remove data - * structurs. - */ -static int free_segnos(struct super_block *sb, __le64 *segnos, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup) - -{ - u64 segno; - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - segno = le64_to_cpu(get_unaligned(&segnos[i])); - if (segno == 0) - break; - if (segno_in_ents(segno, unless, nr_unless)) - continue; - - ret = free_segno(sb, segno); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - remove_segnos(sb, segnos, i, unless, nr_unless, false); - break; - } - } - - return ret; -} - -/* the segno array can be unaligned */ -static int alloc_segnos(struct super_block *sb, __le64 * segnos, - unsigned int nr) - -{ - u64 segno; - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - ret = alloc_segno(sb, &segno); - if (ret < 0) { - free_segnos(sb, segnos, i, NULL, 0, false); - break; - } - put_unaligned(cpu_to_le64(segno), &segnos[i]); - } - - return ret; -} - -static int remove_segnos(struct super_block *sb, __le64 *segnos, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup) - -{ - u64 segno; - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - segno = le64_to_cpu(get_unaligned(&segnos[i])); - if (segno == 0) - break; - if (segno_in_ents(segno, unless, nr_unless)) - continue; - - ret = remove_segno(sb, segno); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - free_segnos(sb, segnos, i, unless, nr_unless, false); - break; - } - } - - return ret; -} - - -static int remove_entry_segnos(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup); - -static int free_entry_segnos(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup) -{ - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - if (ents[i].segno == 0) - break; - if (segno_in_ents(le64_to_cpu(ents[i].segno), - unless, nr_unless)) - continue; - - ret = free_segno(sb, le64_to_cpu(ents[i].segno)); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - remove_entry_segnos(sb, ents, i, unless, nr_unless, - false); - break; - } - } - - return ret; -} - -static int remove_entry_segnos(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, - struct scoutfs_net_manifest_entry *unless, - unsigned int nr_unless, bool cleanup) -{ - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - if (ents[i].segno == 0) - break; - if (segno_in_ents(le64_to_cpu(ents[i].segno), - unless, nr_unless)) - continue; - - ret = remove_segno(sb, le64_to_cpu(ents[i].segno)); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - free_entry_segnos(sb, ents, i, unless, nr_unless, - false); - break; - } - } - - return ret; -} - -static int del_manifest_entries(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, bool cleanup); - -static int add_manifest_entries(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, bool cleanup) -{ - struct scoutfs_manifest_entry ment; - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - if (ents[i].segno == 0) - break; - - scoutfs_init_ment_from_net(&ment, &ents[i]); - - ret = scoutfs_manifest_add(sb, &ment); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - del_manifest_entries(sb, ents, i, false); - break; - } - } - - return ret; -} - -static int del_manifest_entries(struct super_block *sb, - struct scoutfs_net_manifest_entry *ents, - unsigned int nr, bool cleanup) -{ - struct scoutfs_manifest_entry ment; - int ret = 0; - int i; - - for (i = 0; i < nr; i++) { - if (ents[i].segno == 0) - break; - - scoutfs_init_ment_from_net(&ment, &ents[i]); - - ret = scoutfs_manifest_del(sb, &ment); - BUG_ON(ret < 0 && !cleanup); - if (ret < 0) { - add_manifest_entries(sb, ents, i, false); - break; - } - } - - return ret; -} - -/* - * Process a received compaction response. This is called in concurrent - * processing work context so it's racing with other compaction - * responses and new compaction requests being built and sent. - * - * If the compaction failed then we only have to free the allocated - * output segnos sent in the request. - * - * If the compaction succeeded then we need to delete the input manifest - * entries, add any new output manifest entries, and free allocated - * segnos and input manifest segnos that aren't found in output segnos. - * - * And finally we always remove the compaction from the runtime client - * accounting - * - * As we finish a compaction we wake level0 writers if there's now space - * in level 0 for a new segment. - * - * Errors in processing are taken as an indication that this server is - * no longer able to do its job. We return hard errors which shut down - * the server in the hopes that another healthy server will start up. - * We may want to revisit this. - */ -static int compact_response(struct super_block *sb, - struct scoutfs_net_connection *conn, - void *resp, unsigned int resp_len, - int error, void *data) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_net_compact_response *cresp = NULL; - struct compact_request *cr = NULL; - bool level0_was_full = false; - bool add_ents = false; - bool del_ents = false; - bool rem_segnos = false; - struct commit_waiter cw; - __le64 id; - int ret; - - if (error) { - /* an error response without an id is fatal */ - if (resp_len != sizeof(__le64)) { - ret = -EINVAL; - goto out; - } - - memcpy(&id, resp, resp_len); - - } else { - if (resp_len != sizeof(struct scoutfs_net_compact_response)) { - ret = -EINVAL; - goto out; - } - - cresp = resp; - id = cresp->id; - } - - trace_scoutfs_server_compact_response(sb, le64_to_cpu(id), error); - - /* XXX we only free tracked requests on responses, must still exist */ - cr = compact_request_done(sb, le64_to_cpu(id)); - if (WARN_ON_ONCE(cr == NULL)) { - ret = -ENOENT; - goto out; - } - - down_read(&server->commit_rwsem); - scoutfs_manifest_lock(sb); - - level0_was_full = scoutfs_manifest_level0_full(sb); - - if (error) { - ret = 0; - goto cleanup; - } - - /* delete old manifest entries */ - ret = del_manifest_entries(sb, cr->req.ents, ARRAY_SIZE(cr->req.ents), - true); - if (ret) - goto cleanup; - add_ents = true; - - /* add new manifest entries */ - ret = add_manifest_entries(sb, cresp->ents, ARRAY_SIZE(cresp->ents), - true); - if (ret) - goto cleanup; - del_ents = true; - - /* free allocated segnos not found in new entries */ - ret = free_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), - cresp->ents, ARRAY_SIZE(cresp->ents), true); - if (ret) - goto cleanup; - rem_segnos = true; - - /* free input segnos not found in new entries */ - ret = free_entry_segnos(sb, cr->req.ents, ARRAY_SIZE(cr->req.ents), - cresp->ents, ARRAY_SIZE(cresp->ents), true); -cleanup: - /* cleanup partial commits on errors */ - if (ret < 0 && rem_segnos) - remove_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), - cresp->ents, ARRAY_SIZE(cresp->ents), false); - if (ret < 0 && del_ents) - del_manifest_entries(sb, cresp->ents, ARRAY_SIZE(cresp->ents), - false); - if (ret < 0 && add_ents) - add_manifest_entries(sb, cr->req.ents, - ARRAY_SIZE(cr->req.ents), false); - - /* free all the allocated output segnos if compaction failed */ - if ((error || ret < 0) && cr != NULL) - free_segnos(sb, cr->req.segnos, ARRAY_SIZE(cr->req.segnos), - NULL, 0, false); - - if (ret == 0 && level0_was_full && !scoutfs_manifest_level0_full(sb)) - wake_up(&server->waitq); - - if (ret == 0) - queue_commit_work(server, &cw); - scoutfs_manifest_unlock(sb); - up_read(&server->commit_rwsem); - - if (cr) { - scoutfs_manifest_compact_done(sb, &cr->req); - kfree(cr); - } - - if (ret == 0) { - ret = wait_for_commit(&cw); - if (ret == 0) - try_queue_compact(server); - } - -out: - return ret; -} - -/* - * The compaction worker executes as the manifest is updated and we see - * that a level has too many segments and clients aren't processing all - * their max number of compaction requests. Only one compaction worker - * executes. - * - * We have the manifest build us a compaction request, find a client to - * send it too, and record it for later completion processing. - * - * The manifest tracks pending compactions and won't use the same - * segments as inputs to multiple compactions. We track the number of - * compactions in flight to each client to keep them balanced. - */ -static void scoutfs_server_compact_worker(struct work_struct *work) -{ - struct server_info *server = container_of(work, struct server_info, - compact_work); - struct super_block *sb = server->sb; - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_net_compact_request *req; - struct compact_request *cr; - struct commit_waiter cw; - int nr_segnos = 0; - u64 rid; - __le64 id; - int ret; - - trace_scoutfs_server_compact_work_enter(sb, 0, 0); - - cr = kzalloc(sizeof(struct compact_request), GFP_NOFS); - if (!cr) { - ret = -ENOMEM; - goto out; - } - req = &cr->req; - - /* get the input manifest entries */ - ret = scoutfs_manifest_next_compact(sb, req); - if (ret <= 0) - goto out; - - nr_segnos = ret + SCOUTFS_COMPACTION_SEGNO_OVERHEAD; - - /* get the next id and allocate possible output segnos */ - down_read(&server->commit_rwsem); - - spin_lock(&server->lock); - id = super->next_compact_id; - le64_add_cpu(&super->next_compact_id, 1); - spin_unlock(&server->lock); - - ret = alloc_segnos(sb, req->segnos, nr_segnos); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); - if (ret) - goto out; - - /* try to send to a node with capacity, they can disconnect */ -retry: - req->id = id; - rid = compact_request_start(sb, cr); - if (rid == 0) { - ret = 0; - goto out; - } - - /* response processing can complete compaction before this returns */ - ret = scoutfs_net_submit_request_node(sb, server->conn, rid, - SCOUTFS_NET_CMD_COMPACT, - req, sizeof(*req), - compact_response, NULL, NULL); - if (ret < 0) { - cr = compact_request_done(sb, le64_to_cpu(id)); - BUG_ON(cr == NULL); /* must still be there, no node cleanup */ - } - if (ret == -ENOTCONN) - goto retry; - if (ret < 0) - goto out; - - /* cr is now owned by response processing */ - cr = NULL; - ret = 1; - -out: - if (ret <= 0 && cr != NULL) { - scoutfs_manifest_compact_done(sb, req); - - /* don't need to wait for commit when freeing in cleanup */ - down_read(&server->commit_rwsem); - free_segnos(sb, req->segnos, nr_segnos, NULL, 0, false); - up_read(&server->commit_rwsem); - - kfree(cr); - } - - if (ret > 0) - try_queue_compact(server); - - trace_scoutfs_server_compact_work_exit(sb, 0, ret); -} - static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, - [SCOUTFS_NET_CMD_ALLOC_SEGNO] = server_alloc_segno, - [SCOUTFS_NET_CMD_RECORD_SEGMENT] = server_record_segment, [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, - [SCOUTFS_NET_CMD_GET_MANIFEST_ROOT] = server_get_manifest_root, [SCOUTFS_NET_CMD_STATFS] = server_statfs, [SCOUTFS_NET_CMD_LOCK] = server_lock, [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, @@ -2453,14 +1584,11 @@ static void server_notify_up(struct super_block *sb, if (rid != 0) { sci->rid = rid; - sci->nr_compacts = 0; spin_lock(&server->lock); list_add_tail(&sci->head, &server->clients); server->nr_clients++; trace_scoutfs_server_client_up(sb, rid, server->nr_clients); spin_unlock(&server->lock); - - try_queue_compact(server); } } @@ -2480,9 +1608,6 @@ static void server_notify_down(struct super_block *sb, spin_unlock(&server->lock); free_farewell_requests(sb, rid); - - forget_client_compacts(sb, sci); - try_queue_compact(server); } else { stop_server(server); } @@ -2537,8 +1662,7 @@ static void scoutfs_server_worker(struct work_struct *work) &super->core_balloc_free); scoutfs_block_writer_init(sb, &server->wri); - ret = scoutfs_manifest_setup(sb) ?: - scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); if (ret) goto shutdown; @@ -2555,8 +1679,6 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret < 0) goto shutdown; - server->stable_manifest_root = super->manifest.root; - /* start accepting connections and processing work */ server->conn = conn; scoutfs_net_listen(sb, conn); @@ -2571,14 +1693,11 @@ shutdown: scoutfs_info(sb, "server shutting down at "SIN_FMT, SIN_ARG(&sin)); /* wait for request processing */ scoutfs_net_shutdown(sb, conn); - /* drain compact work queued by responses */ - cancel_work_sync(&server->compact_work); /* wait for commit queued by request processing */ flush_work(&server->commit_work); server->conn = NULL; destroy_pending_frees(sb); - scoutfs_manifest_destroy(sb); scoutfs_lock_server_destroy(sb); out: @@ -2672,14 +1791,10 @@ int scoutfs_server_setup(struct super_block *sb) init_rwsem(&server->commit_rwsem); init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); - seqcount_init(&server->stable_seqcount); init_rwsem(&server->seq_rwsem); init_rwsem(&server->alloc_rwsem); INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); - server->compacts_per_client = 2; - INIT_LIST_HEAD(&server->compacts); - INIT_WORK(&server->compact_work, scoutfs_server_compact_worker); mutex_init(&server->farewell_mutex); INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); diff --git a/kmod/src/server.h b/kmod/src/server.h index 83103b9e..c3e82541 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -56,14 +56,6 @@ do { \ __entry->name##_data_len, __entry->name##_cmd, __entry->name##_flags, \ __entry->name##_error -struct scoutfs_net_manifest_entry; -struct scoutfs_manifest_entry; - -void scoutfs_init_ment_to_net(struct scoutfs_net_manifest_entry *net_ment, - struct scoutfs_manifest_entry *ment); -void scoutfs_init_ment_from_net(struct scoutfs_manifest_entry *ment, - struct scoutfs_net_manifest_entry *net_ment); - int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); int scoutfs_server_lock_response(struct super_block *sb, u64 rid, diff --git a/kmod/src/super.c b/kmod/src/super.c index 12f7c199..9387a4b5 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -33,11 +33,6 @@ #include "counters.h" #include "triggers.h" #include "trans.h" -#include "item.h" -#include "manifest.h" -#include "seg.h" -#include "bio.h" -#include "compact.h" #include "data.h" #include "lock.h" #include "net.h" @@ -197,13 +192,11 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_lock_shutdown(sb); scoutfs_server_destroy(sb); scoutfs_net_destroy(sb); - scoutfs_seg_destroy(sb); scoutfs_lock_destroy(sb); /* server clears quorum leader flag during shutdown */ scoutfs_quorum_destroy(sb); - scoutfs_item_destroy(sb); scoutfs_block_destroy(sb); scoutfs_destroy_triggers(sb); scoutfs_options_destroy(sb); @@ -436,8 +429,6 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_sysfs_create_attrs(sb, &sbi->mopts_ssa, mount_options_attrs, "mount_options") ?: scoutfs_setup_triggers(sb) ?: - scoutfs_seg_setup(sb) ?: - scoutfs_item_setup(sb) ?: scoutfs_block_setup(sb) ?: scoutfs_forest_setup(sb) ?: scoutfs_inode_setup(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 4f9fcf24..07d6653b 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -11,10 +11,7 @@ struct scoutfs_counters; struct scoutfs_triggers; -struct item_cache; struct manifest; -struct segment_cache; -struct compact_info; struct data_info; struct trans_info; struct lock_info; @@ -40,11 +37,6 @@ struct scoutfs_sb_info { spinlock_t next_ino_lock; - struct manifest *manifest; - struct item_cache *item_cache; - struct segment_cache *segment_cache; - struct seg_alloc *seg_alloc; - struct compact_info *compact_info; struct data_info *data_info; struct inode_sb_info *inode_sb_info; struct btree_info *btree_info; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index aa7930d6..3be2804f 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -21,11 +21,7 @@ #include "super.h" #include "trans.h" #include "data.h" -#include "bio.h" -#include "item.h" #include "forest.h" -#include "manifest.h" -#include "seg.h" #include "counters.h" #include "client.h" #include "inode.h" From 2fab3b4377035ee35169af24482cc13222f7c882 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Sep 2019 15:19:05 -0700 Subject: [PATCH 763/920] scoutfs: allow larger 8MB transactions Try using larger transactions. This will probably be tweaked over time. Signed-off-by: Zach Brown --- kmod/src/trans.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 3be2804f..deb25f20 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -296,8 +296,8 @@ static bool acquired_hold(struct super_block *sb, items = tri->reserved_items + cnt->items; vals = tri->reserved_vals + cnt->vals; - /* XXX just limit to 256K transactions */ - if (scoutfs_forest_dirty_bytes(sb) >= (256 * 1024)) { + /* XXX arbitrarily limit to 8 meg transactions */ + if (scoutfs_forest_dirty_bytes(sb) >= (8 * 1024 * 1024)) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; From 43d416003a0502513d6b50187c46e1e1b1acdb01 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 24 Oct 2019 15:54:35 -0700 Subject: [PATCH 764/920] scoutfs: add scoutfs_btree_force Add a btree_update variant which will insert the item if a previous wasn't found instead of returning -ENOENT. This saves callers from having to lookup befure updating to discover if they should call _create or _update. Signed-off-by: Zach Brown --- kmod/src/btree.c | 34 ++++++++++++++++++++++++++++++++++ kmod/src/btree.h | 6 ++++++ 2 files changed, 40 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index c8919c52..936de009 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1115,6 +1115,40 @@ int scoutfs_btree_update(struct super_block *sb, return ret; } +/* + * Create an item, overwriting any item that might exist. It's _update + * which will insert instead of returning -ENOENT. + */ +int scoutfs_btree_force(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len) +{ + struct scoutfs_btree_block *bt; + struct scoutfs_block *bl; + int pos; + int cmp; + int ret; + + if (invalid_item(key, key_len, val_len)) + return -EINVAL; + + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, + key, key_len, val_len, &bl, NULL, NULL); + if (ret == 0) { + bt = bl->data; + pos = find_pos(bt, key, key_len, &cmp); + if (cmp == 0) + delete_item(bt, pos); + create_item(bt, pos, key, key_len, val, val_len); + scoutfs_block_put(sb, bl); + } + + return ret; +} + /* * Delete an item from the tree. -ENOENT is returned if the key isn't * found. diff --git a/kmod/src/btree.h b/kmod/src/btree.h index d56b2f3e..9278cec8 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -35,6 +35,12 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); +int scoutfs_btree_force(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + void *key, unsigned key_len, + void *val, unsigned val_len); int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_balloc_allocator *alloc, struct scoutfs_block_writer *wri, From fbffad1d5196fe2d3bec7a40a1ec209218fb6bda Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 25 Oct 2019 11:34:49 -0700 Subject: [PATCH 765/920] scoutfs: add initial lock write_version We need a way to compare two items in different log btrees and learn which is the most recent. Each time we grant a new write lock we give it a larger write version. Items store the version of the lock they're written under. Readers can now easily see which item is newer. This is a trivial initial implementation which is not consistent across unmount or server failover. We'll need to recover the greatest write_version from locks during recovery and from log trees as the server starts up. Signed-off-by: Zach Brown --- kmod/src/format.h | 1 + kmod/src/lock.c | 1 + kmod/src/lock.h | 5 +++-- kmod/src/lock_server.c | 8 ++++++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index a167c5cc..cb804c4e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -726,6 +726,7 @@ struct scoutfs_net_extent_list { struct scoutfs_net_lock { struct scoutfs_key key; + __le64 write_version; __u8 old_mode; __u8 new_mode; } __packed; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 6397d573..c491ed22 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -587,6 +587,7 @@ int scoutfs_lock_grant_response(struct super_block *sb, lock->request_pending = 0; lock->mode = nl->new_mode; + lock->write_version = le64_to_cpu(nl->write_version); if (lock_count_match_exists(nl->new_mode, lock->waiters)) extend_grace(sb, lock); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index f61bbe70..a6f8a688 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -11,8 +11,8 @@ #define SCOUTFS_LOCK_NR_MODES SCOUTFS_LOCK_INVALID /* - * A few fields (start, end, refresh_gen, granted_mode) are referenced - * by code outside lock.c. + * A few fields (start, end, refresh_gen, write_version, granted_mode) + * are referenced by code outside lock.c. */ struct scoutfs_lock { struct super_block *sb; @@ -21,6 +21,7 @@ struct scoutfs_lock { struct rb_node node; struct rb_node range_node; u64 refresh_gen; + u64 write_version; struct list_head lru_head; wait_queue_head_t waitq; struct work_struct shrink_work; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 5c393038..2199bb23 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -494,6 +494,8 @@ static int process_waiting_requests(struct super_block *sb, struct client_lock_entry *req_tmp; struct client_lock_entry *gr; struct client_lock_entry *gr_tmp; + static atomic64_t write_version = ATOMIC64_INIT(0); + u64 wv; int ret; BUG_ON(!mutex_is_locked(&snode->mutex)); @@ -544,6 +546,12 @@ static int process_waiting_requests(struct super_block *sb, nl.old_mode = SCOUTFS_LOCK_NULL; } + if (nl.new_mode == SCOUTFS_LOCK_WRITE || + nl.new_mode == SCOUTFS_LOCK_WRITE_ONLY) { + wv = atomic64_inc_return(&write_version); + nl.write_version = cpu_to_le64(wv); + } + ret = scoutfs_server_lock_response(sb, req->rid, req->net_id, &nl); if (ret) From 388175fc6a9c7219dcb435e2f93ba0dac19c6c1a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 30 Oct 2019 15:02:06 -0700 Subject: [PATCH 766/920] scoutfs: add forest tracing Add some tracing events to the forest subsystem. Signed-off-by: Zach Brown --- kmod/src/forest.c | 31 +++++++++ kmod/src/scoutfs_trace.h | 139 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index cf1bbf1d..b405457f 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -24,6 +24,7 @@ #include "balloc.h" #include "block.h" #include "forest.h" +#include "scoutfs_trace.h" /* * scoutfs items are stored in a forest of btrees. Each mount writes @@ -342,6 +343,13 @@ static int refresh_bloom_roots(struct super_block *sb, scoutfs_block_put(sb, bl); + trace_scoutfs_forest_bloom_search(sb, &lock->start, + be64_to_cpu(ltk.rid), + be64_to_cpu(ltk.nr), + le64_to_cpu(ltv.bloom_ref.blkno), + le64_to_cpu(ltv.bloom_ref.seq), + i); + /* one of the bloom bits wasn't set */ if (i != ARRAY_SIZE(bloom.nrs)) continue; @@ -365,6 +373,10 @@ static int refresh_bloom_roots(struct super_block *sb, fr->nr = be64_to_cpu(ltk.nr); list_add_tail(&fr->entry, &lpriv->roots); + + trace_scoutfs_forest_add_root(sb, &lock->start, fr->rid, + fr->nr, le64_to_cpu(fr->item_root.ref.blkno), + le64_to_cpu(fr->item_root.ref.seq)); } /* add our current log root if a locked writer added it */ @@ -754,6 +766,9 @@ retry: list_for_each_entry_safe(ip, tmp, &list, entry) { fr = ip->fr; + trace_scoutfs_forest_iter_search(sb, fr->rid, fr->nr, + &ip->pos); + /* remove once we can't contain any more items */ if (!forest_iter_key_before(&ip->pos, &found, fwd) || !forest_iter_key_within(&ip->pos, end, fwd)) { @@ -784,6 +799,11 @@ retry: scoutfs_key_from_be(&ip->pos, iref.key); vers = item_vers(lpriv, fr, iref.val); + trace_scoutfs_forest_iter_found(sb, fr->rid, fr->nr, + vers, + item_flags(lpriv, fr, iref.val), + &ip->pos); + /* record next earliest item and copy to caller */ if (!item_is_deletion(lpriv, fr, iref.val) && forest_iter_key_within(&ip->pos, end, fwd) && @@ -818,6 +838,8 @@ unlock: } out: + trace_scoutfs_forest_iter_ret(sb, key, end, fwd, ret, + found_vers, found_copied, &found); if (ret == 0) { /* _next/_prev interfaces modify caller's key :/ */ if (found_vers > 0) { @@ -969,6 +991,7 @@ static int set_lock_bloom_bits(struct super_block *sb, struct scoutfs_bloom_block *bb; struct scoutfs_btree_ref *ref; struct forest_bloom_nrs bloom; + int nr_set = 0; u64 blkno; int ret; int err; @@ -1042,9 +1065,17 @@ static int set_lock_bloom_bits(struct super_block *sb, for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { if (!test_and_set_bit_le(bloom.nrs[i], bb->bits)) { le64_add_cpu(&bb->total_set, 1); + nr_set++; } } + trace_scoutfs_forest_bloom_set(sb, &lock->start, + le64_to_cpu(finf->our_log.rid), + le64_to_cpu(finf->our_log.nr), + le64_to_cpu(finf->our_log.bloom_ref.blkno), + le64_to_cpu(finf->our_log.bloom_ref.seq), + nr_set); + ret = 0; unlock: up_write(&finf->rwsem); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 968cfebb..e3ed5fd5 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2006,6 +2006,145 @@ TRACE_EVENT(scoutfs_trans_seq_last, SCSB_TRACE_ARGS, __entry->s_rid, __entry->trans_seq) ); +DECLARE_EVENT_CLASS(scoutfs_forest_bloom_class, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, blkno) + __field(__u64, seq) + __field(unsigned int, count) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->blkno = blkno; + __entry->seq = seq; + __entry->count = count; + ), + TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu blkno %llu seq %llx count %u", + SCSB_TRACE_ARGS, sk_trace_args(key), __entry->b_rid, + __entry->nr, __entry->blkno, __entry->seq, __entry->count) +); +DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_set, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count) +); +DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_search, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + u64 rid, u64 nr, u64 blkno, u64 seq, unsigned int count), + TP_ARGS(sb, key, rid, nr, blkno, seq, count) +); + +TRACE_EVENT(scoutfs_forest_add_root, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, + u64 nr, u64 blkno, u64 seq), + TP_ARGS(sb, key, rid, nr, blkno, seq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, blkno) + __field(__u64, seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->blkno = blkno; + __entry->seq = seq; + ), + TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu blkno %llu seq %llx", + SCSB_TRACE_ARGS, sk_trace_args(key), + __entry->b_rid, __entry->nr, __entry->blkno, __entry->seq) +); + +TRACE_EVENT(scoutfs_forest_iter_search, + TP_PROTO(struct super_block *sb, u64 rid, u64 nr, + struct scoutfs_key *pos), + TP_ARGS(sb, rid, nr, pos), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, b_rid) + __field(__u64, nr) + sk_trace_define(pos) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->b_rid = rid; + __entry->nr = nr; + sk_trace_assign(pos, pos); + ), + TP_printk(SCSBF" rid %016llx nr %llu pos "SK_FMT, + SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, + sk_trace_args(pos)) +); + +TRACE_EVENT(scoutfs_forest_iter_found, + TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 vers, + u8 flags, struct scoutfs_key *key), + TP_ARGS(sb, rid, nr, vers, flags, key), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, vers) + __field(__u8, flags) + sk_trace_define(key) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->vers = vers; + __entry->flags = flags; + sk_trace_assign(key, key); + ), + TP_printk(SCSBF" rid %016llx nr %llu vers %llu flags %x key "SK_FMT, + SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, + __entry->vers, __entry->flags, sk_trace_args(key)) +); + +TRACE_EVENT(scoutfs_forest_iter_ret, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *end, bool forward, int ret, + u64 found_vers, int found_copied, struct scoutfs_key *found), + TP_ARGS(sb, key, end, forward, ret, found_vers, found_copied, found), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + sk_trace_define(end) + __field(char, forward) + __field(int, ret) + __field(__u64, found_vers) + __field(int, found_copied) + sk_trace_define(found) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + sk_trace_assign(end, end); + __entry->forward = !!forward; + __entry->ret = ret; + __entry->found_vers = found_vers; + __entry->found_copied = found_copied; + sk_trace_assign(found, found); + ), + TP_printk(SCSBF" key "SK_FMT" end "SK_FMT" fwd %u ret %d fv %llu fc %d f "SK_FMT, + SCSB_TRACE_ARGS, sk_trace_args(key), sk_trace_args(end), + __entry->forward, __entry->ret, __entry->found_vers, + __entry->found_copied, sk_trace_args(found)) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 986e66d6c680a1d40a75568e75c345a154604a64 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 31 Oct 2019 15:02:46 -0700 Subject: [PATCH 767/920] scoutfs: add block tracing Add tracing of operations on our block cache. Signed-off-by: Zach Brown --- kmod/src/block.c | 27 +++++++++++++++ kmod/src/scoutfs_trace.h | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index ecc62c21..18f3b37e 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -25,6 +25,7 @@ #include "block.h" #include "counters.h" #include "msg.h" +#include "scoutfs_trace.h" /* * The scoutfs block cache manages metadata blocks that can be larger @@ -86,6 +87,15 @@ struct block_private { }; }; +#define TRACE_BLOCK(which, bp) \ +do { \ + __typeof__(bp) _bp = (bp); \ + trace_scoutfs_block_##which(_bp->sb, _bp, _bp->bl.blkno, \ + atomic_read(&_bp->refcount), \ + atomic_read(&_bp->io_count), \ + _bp->bits, _bp->lru_moved); \ +} while (0) + #define BLOCK_PRIVATE(_bl) \ container_of((_bl), struct block_private, bl) @@ -160,6 +170,8 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) set_bit(BLOCK_BIT_NEW, &bp->bits); atomic_set(&bp->io_count, 0); + TRACE_BLOCK(allocate, bp); + out: if (!bp) scoutfs_inc_counter(sb, block_cache_alloc_failure); @@ -170,6 +182,8 @@ static void block_free(struct super_block *sb, struct block_private *bp) { scoutfs_inc_counter(sb, block_cache_free); + TRACE_BLOCK(free, bp); + if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) __free_pages(bp->page, SCOUTFS_BLOCK_PAGE_ORDER); else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) @@ -235,6 +249,8 @@ static void block_insert(struct super_block *sb, struct block_private *bp, list_add_tail(&bp->lru_entry, &binf->lru_list); bp->lru_moved = ++binf->lru_move_counter; binf->lru_nr++; + + TRACE_BLOCK(insert, bp); } /* @@ -349,8 +365,10 @@ static void block_bio_end_io(struct bio *bio, int err) struct block_private *bp = bio->bi_private; struct super_block *sb = bp->sb; + block_end_io(sb, bio->bi_rw, bp, err); bio_put(bio); + TRACE_BLOCK(end_io, bp); block_put(sb, bp); } @@ -392,6 +410,8 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, atomic_inc(&bp->refcount); atomic_inc(&bp->io_count); + + TRACE_BLOCK(submit, bp); } if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) @@ -558,6 +578,7 @@ void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl) spin_lock(&binf->lock); block_remove(sb, bp); spin_unlock(&binf->lock); + TRACE_BLOCK(invalidate, bp); } } @@ -614,6 +635,8 @@ void scoutfs_block_writer_mark_dirty(struct super_block *sb, list_add_tail(&bp->dirty_entry, &wri->dirty_list); wri->nr_dirty_blocks++; spin_unlock(&wri->lock); + + TRACE_BLOCK(mark_dirty, bp); } } @@ -687,6 +710,7 @@ static void block_forget(struct super_block *sb, clear_bit(BLOCK_BIT_DIRTY, &bp->bits); list_del_init(&bp->dirty_entry); wri->nr_dirty_blocks--; + TRACE_BLOCK(forget, bp); block_put(sb, bp); } @@ -792,8 +816,11 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) if (nr-- == 0) break; + TRACE_BLOCK(shrink, bp); + scoutfs_inc_counter(sb, block_cache_shrink); block_remove(sb, bp); + } spin_unlock(&binf->lock); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index e3ed5fd5..affdd3d1 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2145,6 +2145,79 @@ TRACE_EVENT(scoutfs_forest_iter_ret, __entry->found_copied, sk_trace_args(found)) ); +DECLARE_EVENT_CLASS(scoutfs_block_class, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(void *, bp) + __field(__u64, blkno) + __field(int, refcount) + __field(int, io_count) + __field(unsigned long, bits) + __field(__u64, lru_moved) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->bp = bp; + __entry->blkno = blkno; + __entry->refcount = refcount; + __entry->io_count = io_count; + __entry->bits = bits; + __entry->lru_moved = lru_moved; + ), + TP_printk(SCSBF" bp %p blkno %llu refcount %d io_count %d bits 0x%lx lru_moved %llu", + SCSB_TRACE_ARGS, __entry->bp, __entry->blkno, + __entry->refcount, __entry->io_count, __entry->bits, + __entry->lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_allocate, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_free, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_insert, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_end_io, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_submit, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_invalidate, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_mark_dirty, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_forget, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From dee9fbcf662683a479ab1ba04d665ebd0fa106a3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 8 Nov 2019 10:24:44 -0800 Subject: [PATCH 768/920] scoutfs: use packed extents and bitmaps The btree forest item storage doesn't have as much item granular state as the item cache did. The item cache could tell if a cached item was populated from persistent storage or was created in memory. It could simply remove created items rather than leaving behind a deletion item. The cached btree blocks in the btree forest item storage mechanism can't do this. It has to create deletion items when deleting newly created items because it doesn't know if the item already exists in the persistent record or not. This created a problem with the extent storage we were using. The individual extent items were stored with a key set to the last logical block of their extent. As extents grew or shrank they often were deleted and created at different key values during a transaction. In the btree forest log trees this left a huge stream of deletion items beind, one for every previous version of the extent. Then searches for an extent covering a block would have to skip over all these deleted items before hitting the current stored extent. Streaming writes would operate on O(n) for every extent operation. It got to be out of hand. This large change solves the problem by using more coarse and stable item storage to track free blocks and blocks mapped into file data. For file data we now have large packed extent items which store packed representations of all the logical mappings of a fixed region of a file. The data code has loading and storage functions which transfer that persistent version to and from the version that is modified in memory. Free blocks are stored in bitmaps that are similarly efficiently packed into fixed size items. The client is no longer working with free extent items managed by the forest, it's working with free block bitmap btrees directly. It needs access to the client's metadata block allocator and block write contexts so we move those two out of the forest code and up into the transaction. Previously the client and server would exchange extents with network messages. Now the roots of the btrees that store the free block bitmap items are communicated along with the roots of the other trees involved in a transaction. The client doesn't need to send free extents back to the server so we can remove those tasks and rpcs. The server no longer has to manage free extents. It transfers block bitmap items between trees around commits. All of its extent manipulation can be removed. The item size portion of transaction item counts are removed because we're not using that level of granularity now that metadata transactions are dirty btree blocks instead of dirty items we pack into fixed sized segments. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 - kmod/src/client.c | 42 - kmod/src/client.h | 4 - kmod/src/count.h | 22 +- kmod/src/counters.h | 5 - kmod/src/data.c | 2444 ++++++++++++++++++++++++++------------ kmod/src/data.h | 27 + kmod/src/forest.c | 87 +- kmod/src/forest.h | 14 +- kmod/src/format.h | 121 +- kmod/src/ioctl.c | 14 +- kmod/src/lock.c | 2 +- kmod/src/scoutfs_trace.h | 232 +--- kmod/src/server.c | 534 +++------ kmod/src/super.c | 2 +- kmod/src/trans.c | 63 +- kmod/src/trans.h | 3 + 17 files changed, 2080 insertions(+), 1537 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 8c9d0fdb..5772a9fc 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -17,7 +17,6 @@ scoutfs-y += \ data.o \ dir.o \ export.o \ - extents.o \ file.o \ forest.o \ inode.o \ diff --git a/kmod/src/client.c b/kmod/src/client.c index dd8abf03..83dbacab 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -88,48 +88,6 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, return ret; } -/* - * Ask the server for an extent of at most @blocks blocks. It can return - * smaller extents. - */ -int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, - u64 *len) - -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - __le64 leblocks = cpu_to_le64(blocks); - struct scoutfs_net_extent nex; - int ret; - - ret = scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_ALLOC_EXTENT, - &leblocks, sizeof(leblocks), - &nex, sizeof(nex)); - if (ret == 0) { - if (nex.len == 0) { - ret = -ENOSPC; - } else { - *start = le64_to_cpu(nex.start); - *len = le64_to_cpu(nex.len); - } - } - - return ret; -} - -int scoutfs_client_free_extents(struct super_block *sb, - struct scoutfs_net_extent_list *nexl) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - unsigned int bytes; - - bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(le64_to_cpu(nexl->nr)); - - return scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_FREE_EXTENTS, - nexl, bytes, NULL, 0); -} - int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt) { diff --git a/kmod/src/client.h b/kmod/src/client.h index 9ee65a25..cb77c30c 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -3,10 +3,6 @@ int scoutfs_client_alloc_inodes(struct super_block *sb, u64 count, u64 *ino, u64 *nr); -int scoutfs_client_alloc_extent(struct super_block *sb, u64 blocks, u64 *start, - u64 *len); -int scoutfs_client_free_extents(struct super_block *sb, - struct scoutfs_net_extent_list *nexl); int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); int scoutfs_client_commit_log_trees(struct super_block *sb, diff --git a/kmod/src/count.h b/kmod/src/count.h index 25e4fd0c..97d56ef5 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -252,7 +252,7 @@ static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) __count_dirty_inode(&cnt); cnt.items += nr_free + nr_file; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + cnt.vals += nr_file; return cnt; } @@ -276,22 +276,7 @@ SIC_TRUNC_EXTENT(struct inode *inode) __count_dirty_inode(&cnt); cnt.items += nr_file + nr_free; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); - - return cnt; -} - -/* - * Returning extents to the server can, at most: - * - delete MAX_NR extents with indexed copies - * - create an extent for the leftovers of the last extent - */ -static inline const struct scoutfs_item_count SIC_RETURN_EXTENTS(void) -{ - struct scoutfs_item_count cnt = {0,}; - unsigned int nr = SCOUTFS_NET_EXTENT_LIST_MAX_NR + 1; - - cnt.items += (nr * 2); + cnt.vals += nr_file; return cnt; } @@ -312,7 +297,7 @@ static inline const struct scoutfs_item_count SIC_FALLOCATE_ONE(void) __count_dirty_inode(&cnt); cnt.items += nr_free + nr_file; - cnt.vals += nr_file * sizeof(struct scoutfs_file_extent); + cnt.vals += nr_file; return cnt; } @@ -327,7 +312,6 @@ static inline const struct scoutfs_item_count SIC_SETATTR_MORE(void) __count_dirty_inode(&cnt); cnt.items++; - cnt.vals += sizeof(struct scoutfs_file_extent); return cnt; } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 628d0493..932e8a4b 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -105,11 +105,6 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(server_extent_alloc) \ - EXPAND_COUNTER(server_extent_alloc_error) \ - EXPAND_COUNTER(server_free_extent) \ - EXPAND_COUNTER(server_free_pending_extent) \ - EXPAND_COUNTER(server_free_pending_error) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_sync_fs) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index b2deb16b..1d3c29e0 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Versity Software, Inc. All rights reserved. + * Copyright (C) 2019 Versity Software, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public @@ -21,7 +21,6 @@ #include #include #include -#include #include "format.h" #include "super.h" @@ -34,339 +33,1346 @@ #include "scoutfs_trace.h" #include "forest.h" #include "ioctl.h" -#include "client.h" +#include "btree.h" #include "lock.h" #include "file.h" -#include "extents.h" #include "msg.h" #include "count.h" /* - * scoutfs uses extent items to track file data block mappings and free - * blocks. + * Logical file blocks are mapped to device blocks with extents stored + * in items. Each extent item maps a fixed size logical region and can + * contain multiple extent records. Each extent record is packed to + * minimize the space it uses. The logical starting block is implicit + * so sparse extents are stored to skip unmapped blocks, and the mapped + * blkno is encoded as the difference from the previous extent and only + * its set bytes are stored. * - * Typically we'll allocate a single block in get_block if a mapping - * isn't found. + * To operate on the extents we load their item and unpack them into an + * rbtree of full extent records in memory. Once the memory extents are + * modified they can be packed back into the item. Typically there are + * very few extents that cover the region. * - * We special case extending contiguous files. In that case we'll preallocate - * an unwritten extent at the end of the file. The size of the preallocation - * is based on the file size and is capped. + * Free blocks are tracked with bitmaps that are stored in items. Again + * the bitmaps are stored in a packed form and operated on in memory in + * a native form. Only 64bit words with a mix of set and clear bits are + * stored. The bitmaps are translated into long bitmaps in memory so we + * can use the kernel's long bitmap interfaces. * - * XXX - * - truncate - * - mmap - * - better io error propagation - * - forced unmount with dirty data - * - direct IO - * - need trans around each bulk alloc + * There are two types of bitmap items: little bitmap bits track + * individual blocks and large bitmap bits track full little bitmap + * items. The logical packed extent item and little bitmap item sizes + * are chosen such that a full little bitmap represents a full packed + * extent item so we can allocate maximum size extents from the large + * bitmap bits. + * + * The client is given a tree of free block bitmap items from the server + * at the start of each transaction. The client allocates from items in + * an allocation tree and frees into items in a free tree. The server + * is responsible for filling the alloc tree and reclaiming the free + * tree as transactions are opened and committed. */ -/* - * The largest extent that we'll store in a single item. This will - * determine the granularity of interleaved concurrent allocations in a - * mount. Sequential max length allocations could still see contiguous - * physical extent allocations. It limits the amount of IO needed to - * invalidate a lock. And it determines the granularity of parallel - * writes to a file between nodes. - */ -#define MAX_EXTENT_BLOCKS (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SHIFT) -/* - * We ask for a fixed size from the server today. - */ -#define SERVER_ALLOC_BLOCKS (MAX_EXTENT_BLOCKS * 8) -/* - * Send free extents back to the server if we have plenty locally. - */ -#define NODE_FREE_HIGH_WATER_BLOCKS (SERVER_ALLOC_BLOCKS * 16) - struct data_info { struct super_block *sb; struct rw_semaphore alloc_rwsem; - atomic64_t node_free_blocks; - struct workqueue_struct *workq; - struct work_struct return_work; + struct scoutfs_balloc_allocator *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -static void init_file_extent_key(struct scoutfs_key *key, u64 ino, u64 last) +static void init_packed_extent_key(struct scoutfs_key *key, u64 ino, + u64 iblock, u8 part) { *key = (struct scoutfs_key) { .sk_zone = SCOUTFS_FS_ZONE, - .skfe_ino = cpu_to_le64(ino), - .sk_type = SCOUTFS_FILE_EXTENT_TYPE, - .skfe_last = cpu_to_le64(last), + .skpe_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_PACKED_EXTENT_TYPE, + .skpe_base = cpu_to_le64(iblock >> SCOUTFS_PACKEXT_BASE_SHIFT), + .skpe_part = part, }; } -static void init_free_extent_key(struct scoutfs_key *key, u8 type, u64 rid, - u64 major, u64 minor) -{ - *key = (struct scoutfs_key) { - .sk_zone = SCOUTFS_RID_ZONE, - .sknf_rid = cpu_to_le64(rid), - .sk_type = type, - .sknf_major = cpu_to_le64(major), - .sknf_minor = cpu_to_le64(minor), - }; -} - -static int init_extent_from_item(struct scoutfs_extent *ext, - struct scoutfs_key *key, - struct scoutfs_file_extent *fex) -{ - u64 owner; - u64 start; - u64 map; - u64 len; - u8 flags; - - if (key->sk_type != SCOUTFS_FILE_EXTENT_TYPE && - key->sk_type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - key->sk_type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - return -EIO; /* XXX corruption, unknown key type */ - - if (key->sk_type == SCOUTFS_FILE_EXTENT_TYPE) { - owner = le64_to_cpu(key->skfe_ino); - len = le64_to_cpu(fex->len); - start = le64_to_cpu(key->skfe_last) - len + 1; - map = le64_to_cpu(fex->blkno); - flags = fex->flags; - - } else { - owner = le64_to_cpu(key->sknf_rid); - start = le64_to_cpu(key->sknf_major); - len = le64_to_cpu(key->sknf_minor); - if (key->sk_type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(start, len); - start -= len - 1; - map = 0; - flags = 0; - } - - return scoutfs_extent_init(ext, key->sk_type, owner, start, len, map, - flags); -} - /* - * Read and write file extent and free extent items. - * - * File extents and free extents are indexed by the last position in the - * extent so that we can find intersections with _next. - * - * We also index free extents by their length. We implement that by - * keeping their _BLOCKS_ item in sync with the primary _BLKNO_ item - * that callers operate on. - * - * The count of free blocks stored in items is kept consistent by - * updating the count every time we create or delete items. Updated - * extents are deleted and then recreated so the count can bounce around - * a bit, but it's OK for it to be imprecise at the margins. + * Packed extents are read from items and unpacked into this structure + * in memory so they can be easily manipulated before being packed and + * stored in items. */ -static int data_extent_io(struct super_block *sb, int op, - struct scoutfs_extent *ext, void *data) +struct unpacked_extents { + u64 iblock; + struct rb_root extents; + __u8 existing_parts; + bool changed; +}; + +struct unpacked_extent { + struct rb_node node; + u64 iblock; + u64 count; + u64 blkno; + u8 flags; +}; + +static void init_traced_extent(struct scoutfs_traced_extent *te, + u64 iblock, u64 count, u64 blkno, u8 flags) { - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_lock *lock = data; - struct scoutfs_file_extent fex; - struct scoutfs_key first; - struct scoutfs_key last; - struct scoutfs_key key; - struct kvec val; - bool mirror = false; - u8 mirror_type; - u8 mirror_op = 0; - int expected; - int ret; - int err; + te->iblock = iblock; + te->count = count; + te->blkno = blkno; + te->flags = flags; +} - if (WARN_ON_ONCE(ext->type != SCOUTFS_FILE_EXTENT_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EINVAL; +static void copy_traced_extent(struct scoutfs_traced_extent *te, + struct unpacked_extent *ext) +{ + te->iblock = ext->iblock; + te->count = ext->count; + te->blkno = ext->blkno; + te->flags = ext->flags; +} - if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - (op == SEI_INSERT || op == SEI_DELETE)) { - mirror = true; - mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; - mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; - } +static u64 ext_last(struct unpacked_extent *ext) +{ + return ext->iblock + ext->count - 1; +} - if (ext->type == SCOUTFS_FILE_EXTENT_TYPE) { - init_file_extent_key(&key, ext->owner, - ext->start + ext->len - 1); - init_file_extent_key(&first, ext->owner, 0); - init_file_extent_key(&last, ext->owner, U64_MAX); - fex.blkno = cpu_to_le64(ext->map); - fex.len = cpu_to_le64(ext->len); - fex.flags = ext->flags; - kvec_init(&val, &fex, sizeof(fex)); - } else { - init_free_extent_key(&key, ext->type, ext->owner, - ext->start + ext->len - 1, ext->len); - if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(key.sknf_major, key.sknf_minor); - init_free_extent_key(&first, ext->type, ext->owner, - 0, 0); - init_free_extent_key(&last, ext->type, ext->owner, - U64_MAX, U64_MAX); - kvec_init(&val, NULL, 0); - } +static u64 bitmap_base(u64 blkno) +{ + return blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; +} - if (op == SEI_NEXT || op == SEI_PREV) { - expected = val.iov_len; +/* The first possible iblock in an item that contains the given iblock */ +static u64 first_iblock(u64 iblock) +{ + return iblock & SCOUTFS_PACKEXT_BASE_MASK; +} - if (op == SEI_NEXT) - ret = scoutfs_forest_next(sb, &key, &last, &val, lock); - else - ret = scoutfs_forest_prev(sb, &key, &first, &val, lock); - if (ret >= 0 && ret != expected) - ret = -EIO; - if (ret == expected) - ret = init_extent_from_item(ext, &key, &fex); +/* The last possible iblock in an item that contains the given iblock */ +static u64 last_iblock(u64 iblock) +{ + return iblock | ~SCOUTFS_PACKEXT_BASE_MASK; +} - } else if (op == SEI_INSERT) { - ret = scoutfs_forest_create(sb, &key, &val, lock); +/* + * Extents can merge if they're logically contiguous, have block + * mappings or not which also must be contiguous, and have matching + * flags. + * + * We also require that a given extent's allocation be from only one + * bitmap item because the block bitmap clearing functions only operate + * on one item. + */ +static bool extents_merge(struct unpacked_extent *left, + struct unpacked_extent *right) +{ + return (left->iblock + left->count == right->iblock) && + ((!left->blkno && !right->blkno) || + (left->blkno + left->count == right->blkno)) && + (left->flags == right->flags) && + (bitmap_base(left->blkno) == bitmap_base(right->blkno)); +} - } else if (op == SEI_DELETE) { - ret = scoutfs_forest_delete(sb, &key, lock); +static struct unpacked_extent *first_extent(struct unpacked_extents *unpe) +{ + return rb_entry_safe(rb_first(&unpe->extents), + struct unpacked_extent, node); +} - } else { - ret = WARN_ON_ONCE(-EINVAL); - } +static struct unpacked_extent *last_extent(struct unpacked_extents *unpe) +{ + return rb_entry_safe(rb_last(&unpe->extents), + struct unpacked_extent, node); +} - if (ret == 0 && mirror) { - swap(ext->type, mirror_type); - ret = data_extent_io(sb, op, ext, data); - swap(ext->type, mirror_type); - if (ret) { - err = data_extent_io(sb, mirror_op, ext, data); - BUG_ON(err); +static struct unpacked_extent *next_extent(struct unpacked_extent *ext) +{ + return rb_entry_safe(rb_next(&ext->node), + struct unpacked_extent, node); +} + +static struct unpacked_extent *prev_extent(struct unpacked_extent *ext) +{ + return rb_entry_safe(rb_prev(&ext->node), + struct unpacked_extent, node); +} + +/* + * Find the first extent that intersects the requested range. NULL is + * returned if no extents intersect. + */ +static struct unpacked_extent *find_extent(struct unpacked_extents *unpe, + u64 iblock, u64 last) +{ + + struct rb_node *node = unpe->extents.rb_node; + struct unpacked_extent *ret = NULL; + struct unpacked_extent *ext; + + if (iblock > last) + return NULL; + + while (node) { + ext = rb_entry(node, struct unpacked_extent, node); + + if (last < ext->iblock) { + node = node->rb_left; + } else if (iblock > ext_last(ext)) { + node = node->rb_right; + } else { + ret = ext; + node = node->rb_left; } } - if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { - if (op == SEI_INSERT) - atomic64_add(ext->len, &datinf->node_free_blocks); - else if (op == SEI_DELETE) - atomic64_sub(ext->len, &datinf->node_free_blocks); + return ret; +} + +static void track_blocks(struct unpacked_extent *ext, s64 delta, + s64 *on, s64 *off) +{ + if (ext->blkno && !(ext->flags & SEF_UNWRITTEN)) + *on += delta; + else if (ext->flags & SEF_OFFLINE) + *off += delta; +} + +static void modify_and_track_count(struct unpacked_extent *ext, u64 count, + s64 *on, s64 *off) +{ + track_blocks(ext, count - ext->count, on, off); + ext->count = count; +} + +/* + * Callers can temporarily insert extents with equal starting iblocks. + * We're careful to insert those to the left so that caller's can find + * these existing overlapping extents by iterating with next. + */ +static void insert_extent(struct unpacked_extents *unpe, + struct unpacked_extent *ins, s64 *on, s64 *off) +{ + struct rb_node **node = &unpe->extents.rb_node; + struct rb_node *parent = NULL; + struct unpacked_extent *ext; + int cmp; + + while (*node) { + parent = *node; + ext = rb_entry(*node, struct unpacked_extent, node); + + cmp = scoutfs_cmp_u64s(ins->iblock, ext->iblock); + if (cmp <= 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, &unpe->extents); + + track_blocks(ins, ins->count, on, off); +} + +static void remove_extent(struct unpacked_extents *unpe, + struct unpacked_extent *ext, s64 *on, s64 *off) +{ + rb_erase(&ext->node, &unpe->extents); + track_blocks(ext, -ext->count, on, off); + kfree(ext); +} + +static void free_unpacked_extents(struct unpacked_extents *unpe) +{ + struct unpacked_extent *ext; + struct unpacked_extent *tmp; + + if (unpe) { + rbtree_postorder_for_each_entry_safe(ext, tmp, &unpe->extents, + node) { + kfree(ext); + } + kfree(unpe); + } +} + +static int unpack_extent(struct unpacked_extent *ext, u64 iblock, + struct scoutfs_packed_extent *pe, int size, + u64 prev_blkno) +{ + __le64 lediff; + u64 blkno; + u64 diff; + + if (size < sizeof(struct scoutfs_packed_extent) || + size < (sizeof(struct scoutfs_packed_extent) + pe->diff_bytes)) + return 0; + + if (pe->diff_bytes) { + lediff = 0; + memcpy(&lediff, pe->le_blkno_diff, pe->diff_bytes); + diff = le64_to_cpu(lediff); + diff = (diff >> 1) ^ (-(diff & 1)); + blkno = prev_blkno + diff; + } else { + blkno = 0; + } + + ext->iblock = iblock; + ext->blkno = blkno; + ext->count = le16_to_cpu(pe->count); + ext->flags = pe->flags; + + return sizeof(struct scoutfs_packed_extent) + pe->diff_bytes; +} + +static int load_unpacked_extents(struct super_block *sb, u64 ino, + u64 iblock, u64 last, bool empty_enoent, + struct unpacked_extents **unpe_ret, + struct scoutfs_lock *lock) +{ + struct unpacked_extents *unpe = NULL; + struct scoutfs_packed_extent *pe; + struct unpacked_extent *ext; + struct scoutfs_key key; + struct scoutfs_key end; + struct rb_node *parent; + struct rb_node **node; + void *buf = NULL; + struct kvec val; + u64 prev_blkno; + bool saw_final; + int size; + int ret; + int p; + + *unpe_ret = NULL; + + unpe = kzalloc(sizeof(struct unpacked_extents), GFP_NOFS); + if (!unpe) { + ret = -ENOMEM; + goto out; + } + + unpe->extents = RB_ROOT; + unpe->changed = true; + /* updated later if _next gives us a greater key */ + unpe->iblock = first_iblock(iblock); + + buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + if (last > iblock) + init_packed_extent_key(&end, ino, last, 0); + + parent = NULL; + node = &unpe->extents.rb_node; + prev_blkno = 0; + saw_final = false; + + for (p = 0; !saw_final; p++) { + init_packed_extent_key(&key, ino, iblock, p); + kvec_init(&val, buf, SCOUTFS_PACKEXT_MAX_BYTES); + + /* maybe search for next initial item, lookup more parts */ + if (p == 0 && last > iblock) + ret = scoutfs_forest_next(sb, &key, &end, &val, lock); + else + ret = scoutfs_forest_lookup(sb, &key, &val, lock); + if (ret < 0) { + if (p == 0 && ret == -ENOENT && empty_enoent) + ret = 0; + goto out; + } + + if (key.skpe_part != p) { + ret = -EIO; /* corruption */ + goto out; + } + + if (p == 0) { + iblock = le64_to_cpu(key.skpe_base) << + SCOUTFS_PACKEXT_BASE_SHIFT; + unpe->iblock = iblock; + } + pe = buf; + size = ret; + + while (size > 0) { + ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + if (!ext) { + ret = -ENOMEM; + goto out; + } + + ret = unpack_extent(ext, iblock, pe, size, prev_blkno); + if (ret == 0) { /* XXX corruption? */ + kfree(ext); + ret = -EIO; + goto out; + } + + saw_final = pe->final; + pe = (void *)pe + ret; + size -= ret; + + /* sparse packed extents advance iblock */ + if (ext->flags == 0 && ext->blkno == 0) { + iblock += ext->count; + kfree(ext); + ext = NULL; + continue; + } + + iblock += ext->count; + prev_blkno = ext->blkno + ext->count - 1; + + /* building the rbtree from sorted nodes */ + rb_link_node(&ext->node, parent, node); + rb_insert_color(&ext->node, &unpe->extents); + parent = &ext->node; + node = &ext->node.rb_right; + + if (saw_final) + unpe->existing_parts = p + 1; + } + } + + ret = 0; +out: + kfree(buf); + if (ret < 0) + free_unpacked_extents(unpe); + else + *unpe_ret = unpe; + + return ret; +} + +static int pack_extent(struct scoutfs_packed_extent *pe, int size, + struct unpacked_extent *ext, + u64 prev_blkno, bool final) +{ + int diff_bytes; + __le64 lediff; + u64 diff; + int bytes; + int last; + + diff = ext->blkno - prev_blkno; + diff = (diff << 1) ^ ((s64)diff >> 63); /* shift sign extend */ + lediff = cpu_to_le64(diff); + last = fls64(diff); + diff_bytes = (last + 7) >> 3; + + bytes = offsetof(struct scoutfs_packed_extent, + le_blkno_diff[diff_bytes]); + if (size < bytes) + return 0; + + pe->count = cpu_to_le16(ext->count); + pe->diff_bytes = diff_bytes; + pe->flags = ext->flags; + pe->final = !!final; + if (diff_bytes) + memcpy(pe->le_blkno_diff, &lediff, diff_bytes); + + return bytes; +} + +static int store_packed_extents(struct super_block *sb, u64 ino, + struct unpacked_extents *unpe, + struct scoutfs_lock *lock) +{ + struct scoutfs_packed_extent *pe; + struct unpacked_extent *final; + struct unpacked_extent *ext; + struct scoutfs_key key; + struct kvec val; + void *buf = NULL; + u64 prev_blkno; + u64 iblock; + int space; + int size; + int ret; + int p; + int i; + + if (!unpe->changed) + return 0; + + if (RB_EMPTY_ROOT(&unpe->extents)) { + for (p = 0; p < unpe->existing_parts; p++) { + init_packed_extent_key(&key, ino, unpe->iblock, p); + ret = scoutfs_forest_delete(sb, &key, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + } + unpe->existing_parts = 0; + unpe->changed = false; + return 0; + } + + buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + final = last_extent(unpe); + prev_blkno = 0; + + pe = buf; + space = SCOUTFS_PACKEXT_MAX_BYTES; + size = 0; + p = 0; + iblock = unpe->iblock; + + ext = first_extent(unpe); + while (ext) { + /* encode sparse extent to advance iblock */ + if (ext->iblock > iblock && space >= sizeof(*pe)) { + pe->count = cpu_to_le16(ext->iblock - iblock); + pe->diff_bytes = 0; + pe->flags = 0; + pe->final = 0; + pe++; + space -= sizeof(*pe); + size += sizeof(*pe); + iblock = ext->iblock; + } + + /* encode actual extent */ + if (ext->iblock == iblock && + (ret = pack_extent(pe, space, ext, prev_blkno, + ext == final)) > 0) { + pe = (void *)pe + ret; + space -= ret; + size += ret; + iblock += ext->count; + prev_blkno = ext->blkno + ext->count - 1; + ext = next_extent(ext); + if (ext) + continue; + } + + /* store full item or after packing final extent */ + init_packed_extent_key(&key, ino, unpe->iblock, p); + kvec_init(&val, buf, size); + if (p < unpe->existing_parts) + ret = scoutfs_forest_update(sb, &key, &val, lock); + else + ret = scoutfs_forest_create(sb, &key, &val, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + + pe = buf; + space = SCOUTFS_PACKEXT_MAX_BYTES; + size = 0; + p++; + } + + /* delete any remaining previous part items */ + for (i = p; i < unpe->existing_parts; i++) { + init_packed_extent_key(&key, ino, unpe->iblock, i); + ret = scoutfs_forest_delete(sb, &key, lock); + BUG_ON(ret); /* XXX inconsistent between parts */ + } + + /* the next store has to know our stored parts */ + unpe->existing_parts = p; + unpe->changed = false; + ret = 0; +out: + kfree(buf); + + return ret; +} + +/* + * Set a logical extent mapping in the unpacked extents for a region of + * a file. The caller's extent is authoritative, any existing + * overlapping extents are trimmed or removed. The new extent can be + * merged with remaining adjacent and compatible extents. + * + * If the caller provides an inode struct then we'll keep the inode + * block counts in sync with flagged extents because updating the inode + * counts won't fail. The caller is expected to keep all other state + * consistent with the extents (i_size, i_blocks, allocator bitmaps). + */ +static int set_extent(struct super_block *sb, struct inode *inode, + u64 ino, struct unpacked_extents *unpe, + u64 iblock, u64 blkno, u64 count, u8 flags) +{ + struct unpacked_extent *split; + struct unpacked_extent *next; + struct unpacked_extent *prev; + struct unpacked_extent *ext; + u64 offset; + s64 on = 0; + s64 off = 0; + + /* make sure the given extent fits entirely within one item */ + if (WARN_ON_ONCE(first_iblock(iblock) != + first_iblock(iblock + count - 1))) + return -EINVAL; + + ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + split = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); + if (!ext || !split) { + kfree(ext); + kfree(split); + return -ENOMEM; + } + + unpe->changed = true; + + ext->iblock = iblock; + ext->blkno = blkno; + ext->count = count; + ext->flags = flags; + + insert_extent(unpe, ext, &on, &off); + + prev = prev_extent(ext); + + /* splitting an existing extent? */ + if (prev && ext_last(prev) > ext_last(ext)) { + split->iblock = ext_last(ext) + 1; + split->count = ext_last(prev) - split->iblock + 1; + split->blkno = prev->blkno ? + prev->blkno + prev->count - split->count : 0; + split->flags = prev->flags; + + modify_and_track_count(prev, ext->iblock - prev->iblock, + &on, &off); + + insert_extent(unpe, split, &on, &off); + next = split; + split = NULL; + } else { + next = NULL; + } + + /* trimming a prev extent? */ + if (prev && ext_last(prev) >= ext->iblock) { + modify_and_track_count(prev, ext->iblock - prev->iblock, + &on, &off); + } + + /* merging with a prev extent? */ + if (prev && extents_merge(prev, ext)) { + ext->iblock = prev->iblock; + ext->blkno = prev->blkno; + modify_and_track_count(ext, ext->count + prev->count, + &on, &off); + remove_extent(unpe, prev, &on, &off); + } + + /* if didn't split find next, removing any totally within ours */ + if (!next) { + while ((next = next_extent(ext)) && + ext_last(next) <= ext_last(ext)) { + remove_extent(unpe, next, &on, &off); + } + } + + /* trimming a next extent? */ + if (next && next->iblock <= ext_last(ext)) { + offset = (ext_last(ext) + 1) - next->iblock; + next->iblock += offset; + next->blkno = next->blkno ? next->blkno + offset : 0; + modify_and_track_count(next, next->count - offset, + &on, &off); + } + + /* merging with a next extent? */ + if (next && extents_merge(ext, next)) { + modify_and_track_count(ext, ext->count + next->count, + &on, &off); + remove_extent(unpe, next, &on, &off); + } + + /* and finally remove our extent if it was only removing others */ + if (ext->blkno == 0 && ext->flags == 0) + remove_extent(unpe, ext, &on, &off); + + if (inode) + scoutfs_inode_add_onoff(inode, on, off); + + kfree(split); + return 0; +} + +static bool block_bitmap_fits(u64 blkno, u64 count) +{ + return ((blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK) + count) <= + SCOUTFS_BLOCK_BITMAP_BITS; +} + +static void block_bitmap_bit(u64 *base, int *bit, u64 blkno, u8 type) +{ + if (type == SCOUTFS_BLOCK_BITMAP_BIG) + blkno >>= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + *bit = blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK; + *base = blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; +} + +static u64 block_bitmap_blkno(u64 base, int bit, u8 type) +{ + u64 blkno; + + blkno = (base << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT) + bit; + + if (type == SCOUTFS_BLOCK_BITMAP_BIG) + blkno <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + return blkno; +} + +struct block_bitmap { + u64 base; + u8 type; + bool exists; + unsigned long bits[DIV_ROUND_UP(SCOUTFS_BLOCK_BITMAP_BITS, + BITS_PER_LONG)]; +}; + +static inline __le64 long_bits_to_le64(unsigned long *bits, unsigned int i) +{ +#if BITS_PER_LONG == 64 + return cpu_to_le64(bits[i]); +#elif BITS_PER_LONG == 32 + i <<= 1; + return cpu_to_le64(bits[i] | ((u64)bits[i + 1] << 32)); +#else +#error "unexpected BITS_PER_LONG value?" +#endif +} + +static inline void u64_to_long_bits(unsigned long *bits, unsigned int i, u64 x) +{ +#if BITS_PER_LONG == 64 + bits[i] = x; +#else + i <<= 1; + bits[i] = x; + bits[i + 1] = x >> 32; +#endif +} + +/* + * Block bitmaps are unpacked into native long bitmaps in memory for use + * with the kernel's bitmap functions. This requires a bit of finesse + * to make sure that we translate the bits appropriately to + * architectures with different word size and endian. + */ +static int unpack_block_bitmap(struct block_bitmap *bb, + struct scoutfs_btree_item_ref *iref) +{ + struct scoutfs_block_bitmap_key *bbk; + struct scoutfs_packed_bitmap *pb; + unsigned int nr; + u64 present; + u64 set; + u64 b; + int ret; + int w; + int i; + + if (iref->key_len != sizeof(struct scoutfs_block_bitmap_key) || + iref->val_len < sizeof(struct scoutfs_packed_bitmap)) { + ret = -EIO; + goto out; + } + pb = iref->val; + + bbk = iref->key; + bb->type = bbk->type; + bb->base = be64_to_cpu(bbk->base); + + nr = hweight64(le64_to_cpu(pb->present)); + + if (iref->val_len != + offsetof(struct scoutfs_packed_bitmap, words[nr])) { + ret = -EIO; + goto out; + } + + present = le64_to_cpu(pb->present); + set = le64_to_cpu(pb->set); + w = 0; + for (i = 0, b = 1; + (present | set) && i < SCOUTFS_PACKED_BITMAP_WORDS; + i++, b <<= 1) { + if (set & b) + u64_to_long_bits(bb->bits, i, ~0ULL); + else if (present & b) + u64_to_long_bits(bb->bits, i, + le64_to_cpu(pb->words[w++])); + } + ret = 0; + +out: + return ret; +} + +static int load_block_bitmap(struct super_block *sb, + struct scoutfs_btree_root *root, + u64 blkno, u8 type, bool next, bool zero_enoent, + struct block_bitmap **bb_ret) +{ + struct scoutfs_block_bitmap_key bbk; + struct block_bitmap *bb = NULL; + SCOUTFS_BTREE_ITEM_REF(iref); + u64 base; + int bit; + int ret; + + bb = kzalloc(sizeof(struct block_bitmap), GFP_NOFS); + if (!bb) { + ret = -ENOMEM; + goto out; + } + + block_bitmap_bit(&base, &bit, blkno, type); + + bbk.type = type; + bbk.base = cpu_to_be64(base); + + if (next) + ret = scoutfs_btree_next(sb, root, &bbk, sizeof(bbk), &iref); + else + ret = scoutfs_btree_lookup(sb, root, &bbk, sizeof(bbk), &iref); + if (ret == 0) { + ret = unpack_block_bitmap(bb, &iref); + bb->exists = true; + scoutfs_btree_put_iref(&iref); + } + if (ret == -ENOENT && zero_enoent) { + bb->base = base; + bb->type = type; + ret = 0; + } + +out: + if (ret < 0) { + kfree(bb); + *bb_ret = NULL; + } else { + *bb_ret = bb; + } + return ret; +} + +/* + * Block bitmaps start with two flag words that indicate if logical + * words are all 0s, all 1s, or a mix of set and clear bits stored in + * the item payload. Typically the allocators will have long runs of + * set or clear bits so we don't store most of the bitmaps. Badly + * fragmented allocators will be (2/64 = ~3%) larger. + */ +static int pack_block_bitmap(struct scoutfs_packed_bitmap *pb, + struct block_bitmap *bb) +{ + __le64 word; + u64 present = 0; + u64 set = 0; + u64 b; + int w; + int i; + + w = 0; + for (i = 0, b = 1; i < SCOUTFS_PACKED_BITMAP_WORDS; i++, b <<= 1) { + word = long_bits_to_le64(bb->bits, i); + + if (word == cpu_to_le64(~0ULL)) { + set |= b; + } else if (word != 0) { + present |= b; + pb->words[w++] = word; + } + } + + pb->set = cpu_to_le64(set); + pb->present = cpu_to_le64(present); + + return offsetof(struct scoutfs_packed_bitmap, words[w]); +} + +static int store_block_bitmap(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct block_bitmap *bb) +{ + struct scoutfs_block_bitmap_key bbk; + struct scoutfs_packed_bitmap *pb; + int size; + int ret; + + bbk.type = bb->type; + bbk.base = cpu_to_be64(bb->base); + + if (bitmap_empty(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS)) { + if (!bb->exists) { + ret = 0; + goto out; + } + + ret = scoutfs_btree_delete(sb, alloc, wri, root, + &bbk, sizeof(bbk)); + + } else { + pb = kmalloc(SCOUTFS_PACKED_BITMAP_MAX_BYTES, GFP_NOFS); + if (!pb) { + ret = -ENOMEM; + goto out; + } + + size = pack_block_bitmap(pb, bb); + + ret = scoutfs_btree_force(sb, alloc, wri, root, + &bbk, sizeof(bbk), pb, size); + kfree(pb); + if (ret == 0) + bb->exists = true; + } +out: + return ret; +} + +/* + * Set a region of bitmaps which must fit in one item. The caller's + * blkno is translated to an item base and then the number of bits are + * set. The caller is specifying a number of bits to set, not a block + * extent. + */ +static int set_block_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u8 type, u64 blkno, + int nbits) +{ + struct block_bitmap *bb = NULL; + u64 base; + int bit; + int ret; + + if (WARN_ON_ONCE(!block_bitmap_fits(blkno, nbits))) + return -EINVAL; + + ret = load_block_bitmap(sb, root, blkno, type, false, true, &bb); + if (ret < 0) + goto out; + + block_bitmap_bit(&base, &bit, blkno, type); + + bitmap_set(bb->bits, bit, nbits); + + /* if a little bitmap is full, set it's big and delete it */ + if (type == SCOUTFS_BLOCK_BITMAP_LITTLE && + bitmap_full(bb->bits, SCOUTFS_PACKED_BITMAP_BITS)) { + ret = set_block_bits(sb, alloc, wri, root, + SCOUTFS_BLOCK_BITMAP_BIG, blkno, 1); + if (ret < 0) + goto out; + + bitmap_zero(bb->bits, SCOUTFS_PACKED_BITMAP_BITS); + } + + ret = store_block_bitmap(sb, alloc, wri, root, bb); + BUG_ON(ret < 0); /* cleared bit out of sync with existing littles */ + +out: + kfree(bb); + return ret; +} + +/* + * Find a region of free blocks for the caller. The caller can ask for + * an arbitrarily large extent but we'll only return at most a bitmap's + * worth of blocks from one allocation. + * + * Big bitmap items are stored before little items. This let's large + * allocations naturally fall back to being satisfied by little items + * when there are no more remaining big items. Small allocations first + * look for little items and then search again for big items that they + * can break up. + * + * We always simply look for the first free region. This is operating + * in the client on trees whose items are populated by the server + * between each transaction. The server is responsible for distributing + * the items such that the client tends to allocate across the device + * over time. + */ +static int alloc_blocks(struct super_block *sb, u64 count, u64 *blkno_ret, + u64 *count_ret) +{ + DECLARE_DATA_INFO(sb, datinf); + struct scoutfs_balloc_root *broot = &datinf->data_alloc; + struct block_bitmap *bb = NULL; + u64 blkno; + u8 type; + int bit; + int end; + int ret; + + if (WARN_ON_ONCE(count == 0)) + return -EINVAL; + + /* will only allocate from one block bitmap item at a time */ + count = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS); + + /* small allocations first look for little items, then check big */ + if (count < SCOUTFS_BLOCK_BITMAP_BITS) + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + else + type = SCOUTFS_BLOCK_BITMAP_BIG; + + do { + ret = load_block_bitmap(sb, &broot->root, 0, type, + true, false, &bb); + } while ((ret == -ENOENT && type == SCOUTFS_BLOCK_BITMAP_LITTLE) && + (type = SCOUTFS_BLOCK_BITMAP_BIG, 1)); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + goto out; + } + + bit = find_first_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS); + if (WARN_ON_ONCE(bit >= SCOUTFS_BLOCK_BITMAP_BITS)) { + ret = -EIO; /* stored items should have bits set */ + goto out; + } + + blkno = block_bitmap_blkno(bb->base, bit, bb->type); + + if (bb->type == SCOUTFS_BLOCK_BITMAP_BIG) { + /* set remaining little bits if using big for partial small */ + if (count != SCOUTFS_BLOCK_BITMAP_BITS) { + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, + SCOUTFS_BLOCK_BITMAP_LITTLE, + blkno + count, + SCOUTFS_BLOCK_BITMAP_BITS - count); + if (ret < 0) + goto out; + } + + clear_bit(bit, bb->bits); + + } else { + end = find_next_zero_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS, + bit + 1); + end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); /* catch > size */ + count = min_t(u64, count, end - bit); + + bitmap_clear(bb->bits, bit, count); + } + + ret = store_block_bitmap(sb, datinf->alloc, datinf->wri, + &broot->root, bb); + BUG_ON(ret < 0); /* little partial out of sync with big */ + + le64_add_cpu(&broot->total_free, -count); + + *blkno_ret = blkno; + *count_ret = count; +out: + kfree(bb); + return ret; +} + +/* + * Set free block bits in the block bitmaps and update the root's + * total_free count. The caller can specifiy the root so that this can + * be used both to free used allocations as well as to return unused + * allocations in error paths. The caller must ensure that the block + * regions fit in a single block bitmap (by for the blocks in an + * extent). + */ +static int free_blocks(struct super_block *sb, + struct scoutfs_balloc_root *broot, u64 blkno, u64 count) +{ + DECLARE_DATA_INFO(sb, datinf); + int ret; + + if (count == SCOUTFS_BLOCK_BITMAP_BITS) + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, SCOUTFS_BLOCK_BITMAP_BIG, + blkno, 1); + else + ret = set_block_bits(sb, datinf->alloc, datinf->wri, + &broot->root, SCOUTFS_BLOCK_BITMAP_LITTLE, + blkno, count); + + if (ret == 0) + le64_add_cpu(&broot->total_free, count); + + return ret; +} + +/* + * Ensure that the destination free block bitmap tree has the minimum + * total free blocks by moving bits from the source tree. It will first + * try to find big bits starting at the cursor but will fall back to + * little bits after having wrapped the cursor. + * + * This will move all the items from the source to the destination if + * that's what it takes to reach the minimum. + * + * This is called by the server which provides its writer and metadata + * allocation contexts. It has locked the two allocation trees that + * will be modified. + */ +int scoutfs_data_move_alloc_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + __le64 *cursor, u64 min_dst_total) + +{ + struct block_bitmap *sbb = NULL; + struct block_bitmap *dbb = NULL; + u64 needed; + u64 blocks; + u64 moved; + u64 blkno; + u64 base; + u64 curs; + u8 type; + int nbits; + int bit; + int end; + int ret = 0; + + /* start moving big bitmap items */ + type = SCOUTFS_BLOCK_BITMAP_BIG; + curs = le64_to_cpup(cursor); + + while (le64_to_cpu(dst->total_free) < min_dst_total) { + + /* find the next source bitmap item with bits to move */ + kfree(sbb); + ret = load_block_bitmap(sb, &src->root, curs, type, + true, false, &sbb); + if (ret == 0 && sbb->type != type) + ret = -ENOENT; + if (ret < 0) { + if (ret == -ENOENT) { + if (curs > 0) { + curs = 0; + continue; + } + if (type == SCOUTFS_BLOCK_BITMAP_BIG) { + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + curs = le64_to_cpup(cursor); + continue; + } + ret = -ENOSPC; + } + break; + } + + /* load the destination bitmap */ + blkno = block_bitmap_blkno(sbb->base, 0, type); + kfree(dbb); + ret = load_block_bitmap(sb, &dst->root, blkno, type, + false, true, &dbb); + if (ret < 0) + break; + + /* figure out how many bits to move, can overshoot */ + needed = min_dst_total - le64_to_cpu(dst->total_free); + if (type == SCOUTFS_BLOCK_BITMAP_BIG) { + needed = (needed + SCOUTFS_BLOCK_BITMAP_BITS - 1) + >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + } + + /* start searching from the cursor if within item */ + if (curs > blkno) + blkno = curs; + block_bitmap_bit(&base, &bit, blkno, type); + + moved = 0; + while (moved < needed) { + bit = find_next_bit(sbb->bits, + SCOUTFS_BLOCK_BITMAP_BITS, bit); + if (bit >= SCOUTFS_BLOCK_BITMAP_BITS) + break; + + end = find_next_zero_bit(sbb->bits, + SCOUTFS_BLOCK_BITMAP_BITS, + bit + 1); + end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); + nbits = min_t(u64, needed - moved, end - bit); + + bitmap_clear(sbb->bits, bit, nbits); + bitmap_set(dbb->bits, bit, nbits); + + curs = block_bitmap_blkno(dbb->base, bit + nbits, type); + moved += nbits; + } + + ret = store_block_bitmap(sb, alloc, wri, &dst->root, dbb); + if (ret < 0) + break; + + ret = store_block_bitmap(sb, alloc, wri, &src->root, sbb); + BUG_ON(ret); /* inconsistent src/dst, save orig src */ + + blocks = moved; + if (sbb->type == SCOUTFS_BLOCK_BITMAP_BIG) + blocks <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + + le64_add_cpu(&dst->total_free, blocks); + le64_add_cpu(&src->total_free, -blocks); + + *cursor = cpu_to_le64(curs); + } + + kfree(sbb); + kfree(dbb); + + return ret; +} + +/* + * The server caller is making their way through free data blocks + * initializing free block bitmap bits for the first time. This is the + * only mechanism that initializes free block bitmap items so we know + * that we never have to merge with existing items as long as we always + * write a full item. + * + * The caller gives us the fully extent of blknos that we could + * initialize and we figure out the size of the largest item and its + * bits which cover the start of the extent. We can set big bits if the + * extent is aligned to a small bitmap and is large enough. + */ +int scoutfs_data_add_free_blocks(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *broot, + u64 blkno, u64 count) + +{ + u64 base; + u8 type; + int nbits; + int bit; + int ret; + + type = SCOUTFS_BLOCK_BITMAP_LITTLE; + block_bitmap_bit(&base, &bit, blkno, type); + + if (bit == 0 && count >= SCOUTFS_BLOCK_BITMAP_BITS) { + type = SCOUTFS_BLOCK_BITMAP_BIG; + block_bitmap_bit(&base, &bit, blkno, type); + nbits = min_t(u64, count >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT, + SCOUTFS_BLOCK_BITMAP_BITS - bit); + count = (u64)nbits << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; + } else { + nbits = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS - bit); + count = nbits; + } + + ret = set_block_bits(sb, alloc, wri, &broot->root, type, blkno, nbits); + if (ret == 0) { + le64_add_cpu(&broot->total_free, count); + ret = count; } return ret; } /* - * Find and remove or mark offline the next extent that intersects with - * the caller's range. The caller is responsible for transactions and - * locks. + * Find and remove or mark offline the block mappings that intersect + * with the caller's range. The caller is responsible for transactions + * and locks. * * Returns: * - -errno on errors * - 0 if there are no more extents to stop iteration * - +iblock of next logical block to truncate the next block from - * - * Since our extents are block granular we can never have > S64_MAX - * iblock values. Returns -ENOENT if no extent was found and -errno on - * errors. */ -static s64 truncate_one_extent(struct super_block *sb, struct inode *inode, - u64 ino, u64 iblock, u64 last, bool offline, - struct scoutfs_lock *lock) +static s64 truncate_extents(struct super_block *sb, struct inode *inode, + u64 ino, u64 iblock, u64 last, bool offline, + struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_extent next; - struct scoutfs_extent rem; - struct scoutfs_extent fr; - struct scoutfs_extent ofl; - bool rem_fr = false; - bool add_rem = false; - s64 offline_delta = 0; - s64 online_delta = 0; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + struct scoutfs_traced_extent te; + u64 offset; + u64 blkno; + u64 count; + u8 flags; s64 ret; + int err; - scoutfs_extent_init(&next, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &next, lock); + ret = load_unpacked_extents(sb, ino, iblock, last, false, &unpe, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; goto out; } - trace_scoutfs_data_truncate_next(sb, &next); + flags = offline ? SEF_OFFLINE : 0; - scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, last - iblock + 1, 0, 0); - if (!scoutfs_extent_intersection(&rem, &next)) { - ret = 0; - goto out; + ret = 0; + ext = find_extent(unpe, iblock, last); + while (ext && ext->iblock <= last) { + + /* nothing to do when already offline and unmapped */ + if ((offline && (ext->flags & SEF_OFFLINE)) && !ext->blkno) { + ext = next_extent(ext); + continue; + } + + iblock = max(ext->iblock, iblock); + offset = iblock - ext->iblock; + blkno = ext->blkno + offset; + count = min(ext->count - offset, last - iblock + 1); + + if (ext->blkno) { + down_write(&datinf->alloc_rwsem); + err = free_blocks(sb, &datinf->data_free, blkno, count); + up_write(&datinf->alloc_rwsem); + if (err < 0) { + ret = err; + break; + } + } + + init_traced_extent(&te, iblock, count, 0, flags); + trace_scoutfs_data_extent_truncated(sb, ino, &te); + + err = set_extent(sb, inode, ino, unpe, iblock, 0, count, flags); + BUG_ON(err); /* inconsistent alloc and extents */ + + /* modifying could have merged and deleted ext, search again */ + iblock += count; + if (iblock > last) + break; + ext = find_extent(unpe, iblock, last); } - trace_scoutfs_data_truncate_remove(sb, &rem); + err = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(err); /* inconsistent alloc and extents */ - /* nothing to do if the extent's already offline and unallocated */ - if ((offline && (rem.flags & SEF_OFFLINE)) && !rem.map) { - ret = 1; - goto out; - } - - /* free an allocated mapping */ - if (rem.map) { - scoutfs_extent_init(&fr, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, rem.map, rem.len, 0, 0); - ret = scoutfs_extent_add(sb, data_extent_io, &fr, - sbi->rid_lock); - if (ret) - goto out; - rem_fr = true; - } - - /* remove the extent */ - ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); - if (ret) - goto out; - add_rem = true; - - /* add an offline extent */ - if (offline) { - scoutfs_extent_init(&ofl, SCOUTFS_FILE_EXTENT_TYPE, rem.owner, - rem.start, rem.len, 0, SEF_OFFLINE); - trace_scoutfs_data_truncate_offline(sb, &ofl); - ret = scoutfs_extent_add(sb, data_extent_io, &ofl, lock); - if (ret) - goto out; - } - - if (rem.map && !(rem.flags & SEF_UNWRITTEN)) - online_delta += -rem.len; - if (!offline && (rem.flags & SEF_OFFLINE)) - offline_delta += -rem.len; - if (offline && !(rem.flags & SEF_OFFLINE)) - offline_delta += ofl.len; - - scoutfs_inode_add_onoff(inode, online_delta, offline_delta); - - /* start returning free extents to the server after a small delay */ - if (rem.map && (atomic64_read(&datinf->node_free_blocks) > - NODE_FREE_HIGH_WATER_BLOCKS)) - queue_work(datinf->workq, &datinf->return_work); - - ret = 1; + /* continue after the packed extent item if we exhausted extents */ + if (ret == 0) + ret = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; out: - scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, - data_extent_io, &rem, lock, - SC_DATA_EXTENT_TRUNC_CLEANUP, - corrupt_data_extent_trunc_cleanup, &rem); - scoutfs_extent_cleanup(ret < 0 && rem_fr, scoutfs_extent_remove, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_TRUNC_CLEANUP, - corrupt_data_extent_trunc_cleanup, &rem); - - if (ret > 0) - ret = rem.start + rem.len; - + free_unpacked_extents(unpe); return ret; } @@ -382,7 +1388,7 @@ out: * and offline blocks. If it's not provided then the inode is being * destroyed and isn't reachable, we don't need to update it. * - * The caller is in charge of locking the inode and extents, but we may + * The caller is in charge of locking the inode and data, but we may * have to modify far more items than fit in a transaction so we're in * charge of batching updates into transactions. If the inode is * provided then we're responsible for updating its item as we go. @@ -392,7 +1398,6 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { struct scoutfs_item_count cnt = SIC_TRUNC_EXTENT(inode); - DECLARE_DATA_INFO(sb, datinf); LIST_HEAD(ind_locks); s64 ret = 0; @@ -421,11 +1426,9 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, else ret = 0; - down_write(&datinf->alloc_rwsem); if (ret == 0) - ret = truncate_one_extent(sb, inode, ino, iblock, last, - offline, lock); - up_write(&datinf->alloc_rwsem); + ret = truncate_extents(sb, inode, ino, iblock, last, + offline, lock); if (inode) scoutfs_update_inode_item(inode, lock, &ind_locks); @@ -443,94 +1446,13 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return ret; } -static int get_server_extent(struct super_block *sb) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_extent ext; - u64 start; - u64 len; - int ret; - - ret = scoutfs_client_alloc_extent(sb, SERVER_ALLOC_BLOCKS, - &start, &len); - if (ret) - goto out; - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, start, len, 0, 0); - trace_scoutfs_data_get_server_extent(sb, &ext); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, sbi->rid_lock); - /* XXX don't free extent on error, crash recovery with server */ - -out: - return ret; -} - /* - * Find a free extent to satisfy an allocation of at most @len blocks. - * - * Returns 0 and fills the caller's extent with a _BLKNO_TYPE extent if - * we found a match. It's len may be less than desired. No stored - * extents have been modified. - * - * Returns -errno on error and -ENOSPC if no free extents were found. - * - * The caller's extent is always clobbered. - */ -static int find_free_extent(struct super_block *sb, u64 len, - struct scoutfs_extent *ext) -{ - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - int ret; - - len = min(len, MAX_EXTENT_BLOCKS); - - for (;;) { - /* first try to find the first sufficient extent */ - scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->rid, 0, len, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, ext, - sbi->rid_lock); - - /* if none big enough, look for last largest smaller */ - if (ret == -ENOENT && len > 1) - ret = scoutfs_extent_prev(sb, data_extent_io, ext, - sbi->rid_lock); - - /* ask the server for more if we think it'll help */ - if (ret == -ENOENT || ext->len < len) { - ret = get_server_extent(sb); - if (ret == 0) - continue; - } - - /* use the extent we found or return errors */ - break; - } - - if (ret == 0) - scoutfs_extent_init(ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, - sbi->rid, ext->start, - min(ext->len, len), 0, 0); - - trace_scoutfs_data_find_free_extent(sb, ext); - return ret; -} - -/* - * The caller is writing to a logical block that doesn't have an + * The caller is writing to a logical iblock that doesn't have an * allocated extent. * - * We always allocate an extent starting at the logical block. The - * caller has considered overlapping and following extents and has given - * us a maximum length that we could safely allocate. Preallocation - * heuristics decide to use this length or only a single block. - * - * If the caller passes in an existing extent then we remove the - * allocated region from the existing extent. We then add a single - * block extent for the caller to write into. Then if we allocated - * multiple blocks we add an unwritten extent for the rest of the blocks - * in the extent. + * We always allocate an extent starting at the logical iblock. The + * caller has searched for an extent containing iblock. If it already + * existed then it must be unallocated and offline. * * Preallocation is used if we're strictly contiguously extending * writes. That is, if the logical block offset equals the number of @@ -542,111 +1464,99 @@ static int find_free_extent(struct super_block *sb, u64 len, * staging, sparse files, multi-node writes, etc. fallocate() is always * a better tool to use. * - * On success we update the caller's extent to the single block - * allocated extent for the logical block for use in block mapping. + * We can mangle the extents so the caller is going to search for the + * intersecting extent again if we succeed. */ static int alloc_block(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 iblock, u64 len, + struct unpacked_extents *unpe, + struct unpacked_extent *ext, u64 iblock, struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); DECLARE_DATA_INFO(sb, datinf); const u64 ino = scoutfs_ino(inode); - struct scoutfs_extent unwr; - struct scoutfs_extent old; - struct scoutfs_extent blk; - struct scoutfs_extent fr; - bool add_old = false; - bool add_fr = false; - bool rem_blk = false; - u64 offline; + struct scoutfs_traced_extent te; + u64 blkno = 0; + u64 count; u64 online; + u64 offline; + u64 last; + u8 flags; int ret; + int err; + + /* can only allocate over existing unallocated offline extent */ + if (WARN_ON_ONCE(ext && + !(iblock >= ext->iblock && iblock <= ext_last(ext) && + ext->blkno == 0 && (ext->flags & SEF_OFFLINE)))) + return -EINVAL; down_write(&datinf->alloc_rwsem); scoutfs_inode_get_onoff(inode, &online, &offline); - /* strictly contiguous extending writes will try to preallocate */ + if (ext) { + /* limit preallocation to remaining existing (offline) extent */ + count = ext->count - (iblock - ext->iblock); + flags = ext->flags; + } else { + /* otherwise alloc to next extent or end of packed item */ + last = last_iblock(iblock); + ext = find_extent(unpe, iblock, last); + if (ext) + count = ext->iblock - iblock; + else + count = last - iblock + 1; + flags = 0; + } + + /* only strictly contiguous extending writes will try to preallocate */ if (iblock > 1 && iblock == online) - len = min3(len, iblock, MAX_EXTENT_BLOCKS); + count = min(iblock, count); else - len = 1; + count = 1; - trace_scoutfs_data_alloc_block(sb, inode, ext, iblock, len, - online, offline); - - ret = find_free_extent(sb, len, &fr); + ret = alloc_blocks(sb, count, &blkno, &count); if (ret < 0) goto out; - trace_scoutfs_data_alloc_block_next(sb, &fr); - - /* initialize the new mapped block extent, referenced by cleanup */ - scoutfs_extent_init(&blk, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, 1, fr.start, 0); - - /* remove the free extent that we're allocating */ - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); - if (ret) + ret = set_extent(sb, inode, ino, unpe, iblock, blkno, 1, 0); + if (ret < 0) goto out; - add_fr = true; - /* remove an existing offline or unwritten block extent */ - if (ext->flags) { - scoutfs_extent_init(&old, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock, len, 0, ext->flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &old, lock); - if (ret) - goto out; - add_old = true; + init_traced_extent(&te, iblock, blkno, 1, 0); + trace_scoutfs_data_alloc_block(sb, ino, &te); + + if (count > 1) { + ret = set_extent(sb, inode, ino, unpe, iblock + 1, + blkno + 1, count - 1, flags | SEF_UNWRITTEN); + if (ret < 0) { + err = set_extent(sb, inode, ino, unpe, iblock, 0, 1, + flags); + BUG_ON(err); /* couldn't restore original */ + } + + init_traced_extent(&te, iblock + 1, blkno + 1, count - 1, + flags | SEF_UNWRITTEN); + trace_scoutfs_data_prealloc_unwritten(sb, ino, &te); } - /* add the block that the caller is writing */ - ret = scoutfs_extent_add(sb, data_extent_io, &blk, lock); - if (ret) - goto out; - rem_blk = true; + ret = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(ret); /* inconsistent previous extent state */ - /* and maybe add the remaining unwritten extent */ - if (len > 1) { - scoutfs_extent_init(&unwr, SCOUTFS_FILE_EXTENT_TYPE, ino, - iblock + 1, len - 1, fr.start + 1, - ext->flags | SEF_UNWRITTEN); - ret = scoutfs_extent_add(sb, data_extent_io, &unwr, lock); - if (ret) - goto out; - } - - scoutfs_inode_add_onoff(inode, 1, - (ext->flags & SEF_OFFLINE) ? -1ULL : 0); - ret = 0; out: - scoutfs_extent_cleanup(ret < 0 && rem_blk, scoutfs_extent_remove, sb, - data_extent_io, &blk, lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); - scoutfs_extent_cleanup(ret < 0 && add_old, scoutfs_extent_add, sb, - data_extent_io, &old, lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); - scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_ALLOC_CLEANUP, - corrupt_data_extent_alloc_cleanup, &blk); + if (ret < 0 && blkno > 0) { + err = free_blocks(sb, &datinf->data_alloc, blkno, count); + BUG_ON(err); /* leaked free blocks */ + } up_write(&datinf->alloc_rwsem); - trace_scoutfs_data_alloc_block_ret(sb, ext, ret); - if (ret == 0) - *ext = blk; return ret; } /* - * A caller is writing into unwritten allocated space. This can also be - * called for staging writes so we clear both the unwritten and offline - * flags. We record the extent as online as allocating writes would. + * A caller is writing into an unwritten block. This can also be called + * for staging writes so we clear both the unwritten and offline flags. * * We don't have to wait for dirty block IO to complete before clearing * the unwritten flag in metadata because we have strict synchronization @@ -655,36 +1565,30 @@ out: * is committed. */ static int convert_unwritten(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 start, u64 len, + struct unpacked_extents *unpe, + struct unpacked_extent *ext, u64 iblock, struct scoutfs_lock *lock) { - struct scoutfs_extent conv; + u64 blkno; + u8 ext_fl; int err; int ret; - if (WARN_ON_ONCE(!ext->map) || - WARN_ON_ONCE(!(ext->flags & SEF_UNWRITTEN))) - return -EINVAL; + blkno = ext->blkno + (iblock - ext->iblock); + ext_fl = ext->flags; - scoutfs_extent_init(&conv, ext->type, ext->owner, start, len, - ext->map + (start - ext->start), ext->flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &conv, lock); - if (ret) + ret = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, + blkno, 1, ext_fl & ~(SEF_OFFLINE|SEF_UNWRITTEN)); + if (ret < 0) goto out; - conv.flags &= ~(SEF_UNWRITTEN | SEF_OFFLINE); - ret = scoutfs_extent_add(sb, data_extent_io, &conv, lock); - if (ret) { - conv.flags = ext->flags; - err = scoutfs_extent_add(sb, data_extent_io, &conv, lock); - BUG_ON(err); - goto out; + ret = store_packed_extents(sb, scoutfs_ino(inode), unpe, lock); + if (ret < 0) { + err = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, + blkno, 1, ext_fl); + BUG_ON(err); /* packed and unpacked inconsistent */ } - scoutfs_inode_add_onoff(inode, len, - (ext->flags & SEF_OFFLINE) ? -len : 0); - *ext = conv; - ret = 0; out: return ret; } @@ -693,12 +1597,13 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) { struct scoutfs_inode_info *si = SCOUTFS_I(inode); + const u64 ino = scoutfs_ino(inode); struct super_block *sb = inode->i_sb; struct scoutfs_lock *lock = NULL; - struct scoutfs_extent ext; - u64 next_iblock = 0; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext = NULL; + DECLARE_TRACED_EXTENT(te); u64 offset; - u64 len; int ret; WARN_ON_ONCE(create && !mutex_is_locked(&inode->i_mutex)); @@ -711,69 +1616,53 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, goto out; } - /* look for the extent that overlaps our iblock */ - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret && ret != -ENOENT) + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); + if (ret < 0) goto out; - if (ret == 0) { - trace_scoutfs_data_get_block_next(sb, &ext); - /* remember start of next to limit preallocation */ - if (ext.start > iblock) - next_iblock = ext.start; - } - - /* didn't find an extent or it's past our iblock */ - if (ret == -ENOENT || ext.start > iblock) - memset(&ext, 0, sizeof(ext)); - - if (ext.len) - trace_scoutfs_data_get_block_intersection(sb, &ext); + ext = find_extent(unpe, iblock, iblock); /* non-staging callers should have waited on offline blocks */ - if (WARN_ON_ONCE((ext.flags & SEF_OFFLINE) && !si->staging)) { + if (WARN_ON_ONCE(ext && (ext->flags & SEF_OFFLINE) && !si->staging)) { ret = -EIO; goto out; } /* convert unwritten to written */ - if (create && (ext.flags & SEF_UNWRITTEN)) { - ret = convert_unwritten(sb, inode, &ext, iblock, 1, lock); - if (ret == 0) + if (create && ext && (ext->flags & SEF_UNWRITTEN)) { + ret = convert_unwritten(sb, inode, unpe, ext, iblock, lock); + if (ret == 0) { set_buffer_new(bh); + ext = find_extent(unpe, iblock, iblock); + } goto out; } - /* allocate an extent from our logical block */ - if (create && !ext.map) { - /* limit possible alloc to this extent, next, or logical max */ - if (ext.len > 0) - len = ext.len - (iblock - ext.start); - else if (next_iblock > iblock) - len = ext.start - iblock; - else - len = SCOUTFS_BLOCK_MAX - iblock; - - ret = alloc_block(sb, inode, &ext, iblock, len, lock); - if (ret == 0) + /* allocate and map blocks containing our logical block */ + if (create && (!ext || !ext->blkno)) { + ret = alloc_block(sb, inode, unpe, ext, iblock, lock); + if (ret == 0) { set_buffer_new(bh); + ext = find_extent(unpe, iblock, iblock); + } } else { ret = 0; } - out: /* map usable extent, else leave bh unmapped for sparse reads */ - if (ret == 0 && ext.map && !(ext.flags & SEF_UNWRITTEN)) { - offset = iblock - ext.start; - map_bh(bh, inode->i_sb, ext.map + offset); + if (ret == 0 && ext && ext->blkno && !(ext->flags & SEF_UNWRITTEN)) { + offset = iblock - ext->iblock; + map_bh(bh, inode->i_sb, ext->blkno + offset); bh->b_size = min_t(u64, bh->b_size, - (ext.len - offset) << SCOUTFS_BLOCK_SHIFT); + (ext->count - offset) << SCOUTFS_BLOCK_SHIFT); } + if (ext) + copy_traced_extent(&te, ext); + trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, - ret, bh->b_blocknr, bh->b_size); + &te, ret, bh->b_blocknr, bh->b_size); + free_unpacked_extents(unpe); return ret; } @@ -1035,67 +1924,95 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, } /* - * Allocate one extent on behalf of fallocate. The caller has given us - * the largest extent we can add, its flags, and the flags of an - * existing overlapping extent to remove. + * Try to allocate unwritten extents for any unallocated regions of the + * logical block extent from the caller. We work one packed extent item + * at a time. * - * We allocate the largest extent that we can and return its length or - * -errno. + * We return an error or the numbet of contiguous blocks starting at + * iblock that were successfully processed. */ -static s64 fallocate_one_extent(struct super_block *sb, u64 ino, u64 start, - u64 len, u8 flags, u8 rem_flags, - struct scoutfs_lock *lock) +static int fallocate_extents(struct super_block *sb, struct inode *inode, + u64 iblock, u64 last, struct scoutfs_lock *lock) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_extent fal; - struct scoutfs_extent rem; - struct scoutfs_extent fr; - bool add_rem = false; - bool add_fr = false; - s64 ret; + DECLARE_DATA_INFO(sb, datinf); + const u64 ino = scoutfs_ino(inode); + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + u8 ext_fl; + u64 blkno; + u64 count; + int done; + int ret; + int err; - if (WARN_ON_ONCE(len == 0) || - WARN_ON_ONCE(start + len < start)) { - ret = -EINVAL; - goto out; - } + /* work with the extents in one item at a time */ + last = min(last, last_iblock(iblock)); + done = 0; - ret = find_free_extent(sb, len, &fr); + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); if (ret < 0) goto out; - ret = scoutfs_extent_init(&fal, SCOUTFS_FILE_EXTENT_TYPE, ino, - start, fr.len, fr.start, flags); - if (WARN_ON_ONCE(ret)) - goto out; + ext = find_extent(unpe, iblock, last); + while (iblock <= last) { - ret = scoutfs_extent_remove(sb, data_extent_io, &fr, sbi->rid_lock); - if (ret) - goto out; - add_fr = true; + /* default to allocate to end of region */ + count = last - iblock + 1; + ext_fl = 0; - /* remove a region of the existing extent */ - if (rem_flags) { - scoutfs_extent_init(&rem, SCOUTFS_FILE_EXTENT_TYPE, ino, - fal.start, fal.len, 0, rem_flags); - ret = scoutfs_extent_remove(sb, data_extent_io, &rem, lock); - if (ret) - goto out; - add_rem = true; + if (!ext) { + /* no extent, default alloc from above */ + + } else if (ext->iblock <= iblock && ext->blkno) { + /* skip portion of allocated extent */ + count = min(count, ext->count - (iblock - ext->iblock)); + iblock += count; + done += count; + ext = next_extent(ext); + continue; + + } else if (ext->iblock <= iblock && !ext->blkno) { + /* alloc portion of unallocated extent */ + count = min(count, ext->count - (iblock - ext->iblock)); + ext_fl = ext->flags; + + } else if (iblock < ext->iblock) { + /* alloc hole until next extent */ + count = min(count, ext->iblock - iblock); + } + + down_write(&datinf->alloc_rwsem); + + ret = alloc_blocks(sb, count, &blkno, &count); + if (ret == 0) { + ret = set_extent(sb, inode, ino, unpe, iblock, blkno, + count, ext_fl | SEF_UNWRITTEN); + if (ret < 0) { + err = free_blocks(sb, &datinf->data_alloc, + blkno, count); + BUG_ON(err); /* inconsistent */ + } + } + + up_write(&datinf->alloc_rwsem); + + if (ret < 0) + break; + + iblock += count; + done += count; + ext = find_extent(unpe, iblock, last); } - ret = scoutfs_extent_add(sb, data_extent_io, &fal, lock); + ret = store_packed_extents(sb, ino, unpe, lock); + BUG_ON(ret); /* inconsistent with unpacked and alloc */ + if (ret == 0) - ret = fal.len; + ret = done; + out: - scoutfs_extent_cleanup(ret < 0 && add_rem, scoutfs_extent_add, sb, - data_extent_io, &rem, lock, - SC_DATA_EXTENT_FALLOCATE_CLEANUP, - corrupt_data_extent_fallocate_cleanup, &fal); - scoutfs_extent_cleanup(ret < 0 && add_fr, scoutfs_extent_add, sb, - data_extent_io, &fr, sbi->rid_lock, - SC_DATA_EXTENT_FALLOCATE_CLEANUP, - corrupt_data_extent_alloc_cleanup, &fal); + free_unpacked_extents(unpe); + return ret; } @@ -1105,6 +2022,9 @@ out: * * The caller has only prevented freezing by entering a fs write * context. We're responsible for all other locking and consistency. + * + * This can be used to preallocate files for staging. We find existing + * offline extents and allocate block for them and set unwritten. */ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) { @@ -1112,15 +2032,10 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); struct scoutfs_lock *lock = NULL; - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_extent ext; LIST_HEAD(ind_locks); - u64 last_block; - u64 iblock; - s64 blocks; loff_t end; - u8 rem_flags; - u8 flags; + u64 iblock; + u64 last; int ret; mutex_lock(&inode->i_mutex); @@ -1157,78 +2072,34 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) } iblock = offset >> SCOUTFS_BLOCK_SHIFT; - last_block = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; + last = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; - while(iblock <= last_block) { - - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - ino, iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret < 0 && ret != -ENOENT) - goto out; - - blocks = last_block - iblock + 1; - flags = SEF_UNWRITTEN; - rem_flags = 0; - - if (ret == -ENOENT || ext.start > last_block) { - /* no next extent or past us, all remaining blocks */ - - } else if (iblock < ext.start) { - /* sparse region until next extent */ - blocks = min_t(u64, blocks, ext.start - iblock); - - } else if (ext.map > 0) { - /* skip past an allocated extent */ - blocks = min_t(u64, blocks, - (ext.start + ext.len) - iblock); - iblock += blocks; - blocks = 0; - - } else { - /* allocating a portion of an unallocated extent */ - blocks = min_t(u64, blocks, - (ext.start + ext.len) - iblock); - flags |= ext.flags; - rem_flags = ext.flags; - /* XXX corruption; why'd we store map == flags == 0? */ - if (rem_flags == 0) { - ret = -EIO; - goto out; - } - } + while(iblock <= last) { ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, SIC_FALLOCATE_ONE()); if (ret) goto out; - if (blocks > 0) { - down_write(&datinf->alloc_rwsem); - blocks = fallocate_one_extent(sb, ino, iblock, blocks, - flags, rem_flags, lock); - up_write(&datinf->alloc_rwsem); - if (blocks < 0) - ret = blocks; - else - ret = 0; - } + ret = fallocate_extents(sb, inode, iblock, last, lock); - if (ret == 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { - end = (iblock + blocks) << SCOUTFS_BLOCK_SHIFT; - if (end == 0 || end > offset + len) + if (ret >= 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { + end = (iblock + ret) << SCOUTFS_BLOCK_SHIFT; + if (end > offset + len) end = offset + len; if (end > i_size_read(inode)) i_size_write(inode, end); - scoutfs_update_inode_item(inode, lock, &ind_locks); } + if (ret >= 0) + scoutfs_update_inode_item(inode, lock, &ind_locks); scoutfs_release_trans(sb); scoutfs_inode_index_unlock(sb, &ind_locks); - if (ret) + if (ret <= 0) goto out; - iblock += blocks; + iblock += ret; + ret = 0; } out: @@ -1240,48 +2111,106 @@ out: } /* - * A special case of initialzing a single large offline extent. This + * A special case of initializing a single large offline extent. This * chooses not to deal with any existing extents. It can only be used * on regular files with no data extents. It's used to restore a file * with an offline extent which can then trigger staging. * - * The caller has taken care of locking and holding a transaction. - * - * This could be an fallocate mode. + * The caller has taken care of locking. We're creating many packed + * extent items which may have to be written in multiple transactions. + * We create exetnts from the front of the file and use the offline + * block count to figure out where to continue from. */ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct scoutfs_extent ext; + struct unpacked_extents *unpe = NULL; u64 ino = scoutfs_ino(inode); - u64 len; + LIST_HEAD(ind_locks); + bool held = false; + u64 blocks; + u64 iblock; + u64 count; + u64 on; + u64 off; int ret; - if (!S_ISREG(inode->i_mode)) { - ret = -EINVAL; - goto out; + blocks = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); + + scoutfs_inode_get_onoff(inode, &on, &off); + iblock = off; + + while (iblock < blocks) { + /* we're updating meta_seq with offline block count */ + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_SETATTR_MORE()); + if (ret < 0) + goto out; + held = true; + + ret = scoutfs_dirty_inode_item(inode, lock); + if (ret < 0) + goto out; + + ret = load_unpacked_extents(sb, ino, iblock, iblock, true, + &unpe, lock); + if (ret < 0) + goto out; + + count = min(blocks, last_iblock(iblock) - iblock + 1); + + ret = set_extent(sb, inode, ino, unpe, iblock, 0, count, + SEF_OFFLINE); + if (ret < 0) + goto out; + + free_unpacked_extents(unpe); + unpe = NULL; + + scoutfs_update_inode_item(inode, lock, &ind_locks); + + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + held = false; + + iblock += count; } - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, 0, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); - if (ret != -ENOENT) { - if (ret == 0) - ret = -EINVAL; - goto out; - } - - len = (size + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, ino, - 0, len, 0, SEF_OFFLINE); - ret = scoutfs_extent_add(sb, data_extent_io, &ext, lock); - if (ret == 0) - scoutfs_inode_add_onoff(inode, 0, len); + ret = 0; out: + if (held) { + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); + } + free_unpacked_extents(unpe); return ret; } +/* + * This copies to userspace :/ + */ +static int fill_extent(struct fiemap_extent_info *fieinfo, + struct unpacked_extent *ext, u32 fiemap_flags) +{ + u32 flags; + + if (ext->count == 0) + return 0; + + flags = fiemap_flags; + if (ext->flags & SEF_OFFLINE) + flags |= FIEMAP_EXTENT_UNKNOWN; + else if (ext->flags & SEF_UNWRITTEN) + flags |= FIEMAP_EXTENT_UNWRITTEN; + + return fiemap_fill_next_extent(fieinfo, + ext->iblock << SCOUTFS_BLOCK_SHIFT, + ext->blkno << SCOUTFS_BLOCK_SHIFT, + ext->count << SCOUTFS_BLOCK_SHIFT, + flags); +} /* * Return all the file's extents whose blocks overlap with the caller's @@ -1292,15 +2221,20 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { struct super_block *sb = inode->i_sb; - struct scoutfs_lock *inode_lock = NULL; - struct scoutfs_extent ext; - u64 blk_off; - u64 logical = 0; - u64 phys = 0; - u64 size = 0; - u32 flags = 0; + const u64 ino = scoutfs_ino(inode); + struct scoutfs_lock *lock = NULL; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + struct unpacked_extent cur; + struct scoutfs_traced_extent te; + u32 last_flags; + u64 iblock; + u64 last; int ret; + if (len == 0) + return 0; + ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) return ret; @@ -1308,51 +2242,64 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, /* XXX overkill? */ mutex_lock(&inode->i_mutex); - ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &inode_lock); + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); if (ret) goto out; - blk_off = start >> SCOUTFS_BLOCK_SHIFT; + /* use a dummy extent to track */ + memset(&cur, 0, sizeof(cur)); + last_flags = 0; + + iblock = start >> SCOUTFS_BLOCK_SHIFT; + last = (start + len - 1) >> SCOUTFS_BLOCK_SHIFT; for (;;) { - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), blk_off, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, inode_lock); - /* fiemap will return last and stop when we see enoent */ - if (ret < 0 && ret != -ENOENT) + ret = load_unpacked_extents(sb, ino, iblock, last, false, + &unpe, lock); + if (ret < 0) { + last_flags = FIEMAP_EXTENT_LAST; break; - - if (ret == 0) - trace_scoutfs_data_fiemap_extent(sb, &ext); - - if (size) { - if (ret == -ENOENT) - flags |= FIEMAP_EXTENT_LAST; - ret = fiemap_fill_next_extent(fieinfo, logical, phys, - size, flags); - if (ret || (logical + size >= (start + len))) { - if (ret == 1) - ret = 0; - break; - } } - logical = ext.start << SCOUTFS_BLOCK_SHIFT; - phys = ext.map << SCOUTFS_BLOCK_SHIFT; - size = ext.len << SCOUTFS_BLOCK_SHIFT; - flags = 0; - if (ext.flags & SEF_OFFLINE) - flags |= FIEMAP_EXTENT_UNKNOWN; - if (ext.flags & SEF_UNWRITTEN) - flags |= FIEMAP_EXTENT_UNWRITTEN; + for (ext = find_extent(unpe, iblock, last); ext; + ext = next_extent(ext)) { - blk_off = ext.start + ext.len; + copy_traced_extent(&te, ext); + trace_scoutfs_data_fiemap_extent(sb, ino, &te); + + if (ext->iblock > last) { + /* not setting _LAST, it's for end of file */ + ret = 0; + break; + } + + if (extents_merge(&cur, ext)) { + cur.count += ext->count; + continue; + } + + ret = fill_extent(fieinfo, &cur, 0); + if (ret != 0) + goto out; + cur = *ext; + } + + iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; + free_unpacked_extents(unpe); + unpe = NULL; } - scoutfs_unlock(sb, inode_lock, SCOUTFS_LOCK_READ); + if (cur.count) + ret = fill_extent(fieinfo, &cur, last_flags); out: + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); mutex_unlock(&inode->i_mutex); + free_unpacked_extents(unpe); + + if (ret == 1) + ret = 0; + return ret; } @@ -1438,9 +2385,12 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); DECLARE_DATA_WAIT_ROOT(sb, rt); DECLARE_DATA_WAITQ(inode, wq); - struct scoutfs_extent ext = {0,}; + struct unpacked_extents *unpe = NULL; + struct unpacked_extent *ext; + DECLARE_TRACED_EXTENT(te); u64 iblock; u64 last_block; u64 on; @@ -1467,41 +2417,51 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SHIFT; while(iblock <= last_block) { - scoutfs_extent_init(&ext, SCOUTFS_FILE_EXTENT_TYPE, - scoutfs_ino(inode), iblock, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, lock); + + free_unpacked_extents(unpe); + ret = load_unpacked_extents(sb, ino, iblock, last_block, false, + &unpe, lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; - break; + goto out; } - if (ext.start > last_block) - break; + for (ext = find_extent(unpe, iblock, last_block); ext; + ext = next_extent(ext)) { - if (sef & ext.flags) { - if (dw) { - dw->chg = atomic64_read(&wq->changed); - dw->ino = scoutfs_ino(inode); - dw->iblock = max(iblock, ext.start); - dw->op = op; + if (ext->iblock > last_block) { + ret = 0; + goto out; + } - spin_lock(&rt->lock); - insert_offline_waiting(&rt->root, dw); - spin_unlock(&rt->lock); + if (sef & ext->flags) { + if (dw) { + dw->chg = atomic64_read(&wq->changed); + dw->ino = ino; + dw->iblock = max(iblock, ext->iblock); + dw->op = op; + + spin_lock(&rt->lock); + insert_offline_waiting(&rt->root, dw); + spin_unlock(&rt->lock); + } + + copy_traced_extent(&te, ext); + ret = 1; + goto out; } - ret = 1; - break; } - iblock = ext.start + ext.len; + iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; } out: - trace_scoutfs_data_wait_check(sb, scoutfs_ino(inode), pos, len, - sef, op, ext.start, ext.len, ext.flags, - ret); + trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &te, ret); + + free_unpacked_extents(unpe); + return ret; } @@ -1609,93 +2569,34 @@ const struct file_operations scoutfs_file_fops = { .fallocate = scoutfs_fallocate, }; -/* - * Return extents to the server if we're over the high water mark. Each - * work call sends one batch of extents so that the work can be easily - * canceled to stop progress during unmount. - */ -static void scoutfs_data_return_server_extents_worker(struct work_struct *work) +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) { - struct data_info *datinf = container_of(work, struct data_info, - return_work); - struct super_block *sb = datinf->sb; - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_net_extent_list *nexl; - struct scoutfs_extent ext; - u64 nr = 0; - u64 free; - int bytes; - int ret; - int err; - - trace_scoutfs_data_return_server_extents_enter(sb, 0, 0); - - bytes = SCOUTFS_NET_EXTENT_LIST_BYTES(SCOUTFS_NET_EXTENT_LIST_MAX_NR); - nexl = kmalloc(bytes, GFP_NOFS); - if (!nexl) { - ret = -ENOMEM; - goto out; - } - - ret = scoutfs_hold_trans(sb, SIC_RETURN_EXTENTS()); - if (ret) - goto out; + DECLARE_DATA_INFO(sb, datinf); down_write(&datinf->alloc_rwsem); - free = atomic64_read(&datinf->node_free_blocks); - - while (nr < SCOUTFS_NET_EXTENT_LIST_MAX_NR && - free > NODE_FREE_HIGH_WATER_BLOCKS) { - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, - sbi->rid, 0, 1, 0, 0); - ret = scoutfs_extent_next(sb, data_extent_io, &ext, - sbi->rid_lock); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - break; - } - - trace_scoutfs_data_return_server_extent(sb, &ext); - - ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - ext.len = min(ext.len, free - NODE_FREE_HIGH_WATER_BLOCKS); - - ret = scoutfs_extent_remove(sb, data_extent_io, &ext, - sbi->rid_lock); - if (ret) - break; - - nexl->extents[nr].start = cpu_to_le64(ext.start); - nexl->extents[nr].len = cpu_to_le64(ext.len); - - nr++; - free -= ext.len; - } - - nexl->nr = cpu_to_le64(nr); + datinf->alloc = alloc; + datinf->wri = wri; + datinf->data_alloc = lt->data_alloc; + datinf->data_free = lt->data_free; up_write(&datinf->alloc_rwsem); +} - if (nr > 0) { - err = scoutfs_client_free_extents(sb, nexl); - /* XXX leaked extents if free failed */ - if (ret == 0 && err < 0) - ret = err; - } +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) +{ + DECLARE_DATA_INFO(sb, datinf); - scoutfs_release_trans(sb); -out: - kfree(nexl); + down_read(&datinf->alloc_rwsem); - trace_scoutfs_data_return_server_extents_exit(sb, nr, ret); + lt->data_alloc = datinf->data_alloc; + lt->data_free = datinf->data_free; - /* keep returning if we're still over the water mark */ - if (ret == 0 && (atomic64_read(&datinf->node_free_blocks) > - NODE_FREE_HIGH_WATER_BLOCKS)) - queue_work(datinf->workq, &datinf->return_work); + up_read(&datinf->alloc_rwsem); } int scoutfs_data_setup(struct super_block *sb) @@ -1709,15 +2610,6 @@ int scoutfs_data_setup(struct super_block *sb) datinf->sb = sb; init_rwsem(&datinf->alloc_rwsem); - atomic64_set(&datinf->node_free_blocks, 0); - INIT_WORK(&datinf->return_work, - scoutfs_data_return_server_extents_worker); - - datinf->workq = alloc_workqueue("scoutfs_data", WQ_UNBOUND, 1); - if (!datinf->workq) { - kfree(datinf); - return -ENOMEM; - } sbi->data_info = datinf; return 0; @@ -1729,12 +2621,6 @@ void scoutfs_data_destroy(struct super_block *sb) struct data_info *datinf = sbi->data_info; if (datinf) { - if (datinf->workq) { - cancel_work_sync(&datinf->return_work); - destroy_workqueue(datinf->workq); - datinf->workq = NULL; - } - sbi->data_info = NULL; kfree(datinf); } diff --git a/kmod/src/data.h b/kmod/src/data.h index 21deeac4..912284d9 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -36,8 +36,17 @@ struct scoutfs_data_wait { .node.__rb_parent_color = (unsigned long)(&nm.node), \ } +struct scoutfs_traced_extent { + u64 iblock; + u64 count; + u64 blkno; + u8 flags; +}; + extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; +struct scoutfs_balloc_allocator; +struct scoutfs_block_writer; int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, u64 ino, u64 iblock, u64 last, bool offline, @@ -63,6 +72,24 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); +int scoutfs_data_move_alloc_bits(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *dst, + struct scoutfs_balloc_root *src, + __le64 *cursor, u64 min_dst_total); +int scoutfs_data_add_free_blocks(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_balloc_root *broot, + u64 blkno, u64 count); +void scoutfs_data_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_data_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); + int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/forest.c b/kmod/src/forest.c index b405457f..ac49459c 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -61,8 +61,8 @@ struct forest_info { struct rw_semaphore rwsem; - struct scoutfs_balloc_allocator alloc; - struct scoutfs_block_writer wri; + struct scoutfs_balloc_allocator *alloc; + struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; }; @@ -1025,14 +1025,14 @@ static int set_lock_bloom_bits(struct super_block *sb, if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { - ret = scoutfs_balloc_alloc(sb, &finf->alloc, &finf->wri, + ret = scoutfs_balloc_alloc(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) goto unlock; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); @@ -1040,7 +1040,7 @@ static int set_lock_bloom_bits(struct super_block *sb, } if (bl) { - err = scoutfs_balloc_free(sb, &finf->alloc, &finf->wri, + err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, le64_to_cpu(ref->blkno)); BUG_ON(err); /* could have dirtied */ memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); @@ -1048,7 +1048,7 @@ static int set_lock_bloom_bits(struct super_block *sb, memset(new_bl->data, 0, SCOUTFS_BLOCK_SIZE); } - scoutfs_block_writer_mark_dirty(sb, &finf->wri, new_bl); + scoutfs_block_writer_mark_dirty(sb, finf->wri, new_bl); scoutfs_block_put(sb, bl); bl = new_bl; @@ -1165,7 +1165,7 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, scoutfs_key_to_be(&kbe, key); down_write(&finf->rwsem); - ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, &finf->our_log.item_root, &kbe, sizeof(kbe), iv->iov_base, iv->iov_len); up_write(&finf->rwsem); @@ -1249,7 +1249,7 @@ static int forest_delete(struct super_block *sb, struct scoutfs_key *key, liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; down_write(&finf->rwsem); - ret = scoutfs_btree_force(sb, &finf->alloc, &finf->wri, + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, &finf->our_log.item_root, &kbe, sizeof(kbe), &liv, sizeof(liv)); up_write(&finf->rwsem); @@ -1318,69 +1318,41 @@ void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) /* * This is called from transactions as a new transaction opens and is - * serialized with all writers. Get the tree roots that we'll need - * for the transaction. + * serialized with all writers. */ -int scoutfs_forest_get_log_trees(struct super_block *sb) +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt) { DECLARE_FOREST_INFO(sb, finf); - struct scoutfs_log_trees lt; - int ret; - - ret = scoutfs_client_get_log_trees(sb, <); - if (ret) - goto out; down_write(&finf->rwsem); - scoutfs_balloc_init(&finf->alloc, <.alloc_root, <.free_root); - scoutfs_block_writer_init(sb, &finf->wri); - finf->our_log = lt; + + finf->alloc = alloc; + finf->wri = wri; + + /* we use the item and bloom trees */ + memset(&finf->our_log, 0, sizeof(finf->our_log)); + finf->our_log.item_root = lt->item_root; + finf->our_log.bloom_ref = lt->bloom_ref; + up_write(&finf->rwsem); - - ret = 0; -out: - return ret; -} - -bool scoutfs_forest_has_dirty(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_has_dirty(sb, &finf->wri); -} - -unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_dirty_bytes(sb, &finf->wri); -} - -int scoutfs_forest_write(struct super_block *sb) -{ - DECLARE_FOREST_INFO(sb, finf); - - return scoutfs_block_writer_write(sb, &finf->wri); } /* * This is called during transaction commit which excludes forest writer * calls. The caller has already written all the dirty blocks that the - * forest roots reference. + * forest roots reference. They're getting the roots to send to the server + * for the commit. */ -int scoutfs_forest_commit(struct super_block *sb) +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt) { DECLARE_FOREST_INFO(sb, finf); - struct scoutfs_log_trees lt = { - .alloc_root = finf->alloc.alloc_root, - .free_root = finf->alloc.free_root, - .item_root = finf->our_log.item_root, - .bloom_ref = finf->our_log.bloom_ref, - .rid = finf->our_log.rid, - .nr = finf->our_log.nr, - }; - return scoutfs_client_commit_log_trees(sb, <); + lt->item_root = finf->our_log.item_root; + lt->bloom_ref = finf->our_log.bloom_ref; } int scoutfs_forest_setup(struct super_block *sb) @@ -1395,7 +1367,7 @@ int scoutfs_forest_setup(struct super_block *sb) goto out; } - /* the finf fields will be setup as we open a transaction */ + /* the finf fields will be setup as we open a transaction */ init_rwsem(&finf->rwsem); sbi->forest_info = finf; @@ -1413,7 +1385,6 @@ void scoutfs_forest_destroy(struct super_block *sb) struct forest_info *finf = SCOUTFS_SB(sb)->forest_info; if (finf) { - scoutfs_block_writer_forget_all(sb, &finf->wri); kfree(finf); sbi->forest_info = NULL; } diff --git a/kmod/src/forest.h b/kmod/src/forest.h index 1b1d5641..7757d1f1 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -1,6 +1,9 @@ #ifndef _SCOUTFS_FOREST_H_ #define _SCOUTFS_FOREST_H_ +struct scoutfs_balloc_allocator; +struct scoutfs_block_writer; + int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); int scoutfs_forest_lookup_exact(struct super_block *sb, @@ -36,11 +39,12 @@ int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, struct scoutfs_lock *lock); void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); -int scoutfs_forest_get_log_trees(struct super_block *sb); -bool scoutfs_forest_has_dirty(struct super_block *sb); -unsigned long scoutfs_forest_dirty_bytes(struct super_block *sb); -int scoutfs_forest_write(struct super_block *sb); -int scoutfs_forest_commit(struct super_block *sb); +void scoutfs_forest_init_btrees(struct super_block *sb, + struct scoutfs_balloc_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_log_trees *lt); +void scoutfs_forest_get_btrees(struct super_block *sb, + struct scoutfs_log_trees *lt); void scoutfs_forest_clear_lock(struct super_block *sb, struct scoutfs_lock *lock); diff --git a/kmod/src/format.h b/kmod/src/format.h index cb804c4e..9b9fd06d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -105,11 +105,6 @@ struct scoutfs_key { #define skxi_ino _sk_second #define skxi_id _sk_third -/* node free extent */ -#define sknf_rid _sk_first -#define sknf_major _sk_second -#define sknf_minor _sk_third - /* node orphan inode */ #define sko_rid _sk_first #define sko_ino _sk_second @@ -132,9 +127,10 @@ struct scoutfs_key { #define sks_ino _sk_first #define sks_nr _sk_second -/* file extent */ -#define skfe_ino _sk_first -#define skfe_last _sk_second +/* packed extents */ +#define skpe_ino _sk_first +#define skpe_base _sk_second +#define skpe_part _sk_fourth /* * The btree still uses memcmp() to compare keys. We should fix that @@ -235,18 +231,32 @@ struct scoutfs_balloc_item_val { } __packed; /* - * Free extents are stored in the server in an allocation btree. The - * type differentiates whether start or length is in stored in the major - * value and is the primary sort key. 'start' is set to the final block - * in the extent so that overlaping queries can be done with next - * instead prev. + * Free data blocks are tracked in bitmaps stored in btree items. */ -struct scoutfs_extent_btree_key { +struct scoutfs_block_bitmap_key { __u8 type; - __be64 major; - __be64 minor; + __be64 base; } __packed; +#define SCOUTFS_BLOCK_BITMAP_BIG 0 +#define SCOUTFS_BLOCK_BITMAP_LITTLE 1 + +#define SCOUTFS_PACKED_BITMAP_WORDS 32 +#define SCOUTFS_PACKED_BITMAP_BITS (SCOUTFS_PACKED_BITMAP_WORDS * 64) +#define SCOUTFS_PACKED_BITMAP_MAX_BYTES \ + offsetof(struct scoutfs_packed_bitmap, \ + words[SCOUTFS_PACKED_BITMAP_WORDS]) + +#define SCOUTFS_BLOCK_BITMAP_BITS SCOUTFS_PACKED_BITMAP_BITS +#define SCOUTFS_BLOCK_BITMAP_BIT_MASK (SCOUTFS_PACKED_BITMAP_BITS - 1) +#define SCOUTFS_BLOCK_BITMAP_BASE_SHIFT (ilog2(SCOUTFS_PACKED_BITMAP_BITS)) + +struct scoutfs_packed_bitmap { + __le64 present; + __le64 set; + __le64 words[0]; +}; + /* * The lock server keeps a persistent record of connected clients so that * server failover knows who to wait for before resuming operations. @@ -277,11 +287,18 @@ struct scoutfs_mounted_client_btree_val { #define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) +/* + * XXX I imagine we should rename these now that they've evolved to track + * all the btrees that clients use during a transaction. It's not just + * about item logs, it's about clients making changes to trees. + */ struct scoutfs_log_trees { struct scoutfs_balloc_root alloc_root; struct scoutfs_balloc_root free_root; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; __le64 rid; __le64 nr; } __packed; @@ -296,6 +313,8 @@ struct scoutfs_log_trees_val { struct scoutfs_balloc_root free_root; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; + struct scoutfs_balloc_root data_alloc; + struct scoutfs_balloc_root data_free; } __packed; struct scoutfs_log_item_value { @@ -351,9 +370,7 @@ struct scoutfs_bloom_block { #define SCOUTFS_XATTR_INDEX_NAME_TYPE 1 /* rid zone (also used in server alloc btree) */ -#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 -#define SCOUTFS_FREE_EXTENT_BLOCKS_TYPE 2 -#define SCOUTFS_ORPHAN_TYPE 3 +#define SCOUTFS_ORPHAN_TYPE 1 /* fs zone */ #define SCOUTFS_INODE_TYPE 1 @@ -362,23 +379,45 @@ struct scoutfs_bloom_block { #define SCOUTFS_READDIR_TYPE 4 #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 -#define SCOUTFS_FILE_EXTENT_TYPE 7 +#define SCOUTFS_PACKED_EXTENT_TYPE 7 /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 #define SCOUTFS_MAX_TYPE 8 /* power of 2 is efficient */ + /* - * File extents have more data than easily fits in the key so we move - * the non-indexed fields into the value. + * The extents that map blocks in a fixed-size logical region of a file + * are packed and stored in item values. The packed extents are + * contiguous so the starting logical block is implicit from the length + * of previous extents. Sparse regions are represented by 0 flags and + * blkno. The blkno of a packed extent is encoded as the zigzag (lsb is + * sign bit) difference from the last blkno of the previous extent. + * This guarantees that non-sparse extents must have a blkno delta of at + * least -1/1. High zero byte aren't stored. */ -struct scoutfs_file_extent { - __le64 blkno; - __le64 len; - __u8 flags; +struct scoutfs_packed_extent { + __le16 count; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u8 diff_bytes:4, + flags:3, + final:1; +#elif defined(__BIG_ENDIAN_BITFIELD) + __u8 final:1, + flags:3, + diff_bytes:4; +#else +#error "no {BIG,LITTLE}_ENDIAN_BITFIELD defined?" +#endif + __u8 le_blkno_diff[0]; } __packed; +#define SCOUTFS_PACKEXT_BLOCKS (8 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) +#define SCOUTFS_PACKEXT_BASE_SHIFT (ilog2(SCOUTFS_PACKEXT_BLOCKS)) +#define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) +#define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE + #define SEF_OFFLINE (1 << 0) #define SEF_UNWRITTEN (1 << 1) #define SEF_UNKNOWN (U8_MAX << 2) @@ -451,8 +490,12 @@ struct scoutfs_super_block { __le64 next_ino; __le64 next_trans_seq; __le64 total_blocks; - __le64 next_uninit_free_block; + __le64 next_uninit_meta_blkno; + __le64 last_uninit_meta_blkno; + __le64 next_uninit_data_blkno; + __le64 last_uninit_data_blkno; __le64 core_balloc_cursor; + __le64 core_data_alloc_cursor; __le64 free_blocks; __le64 first_fs_blkno; __le64 last_fs_blkno; @@ -463,7 +506,8 @@ struct scoutfs_super_block { struct scoutfs_inet_addr server_addr; struct scoutfs_balloc_root core_balloc_alloc; struct scoutfs_balloc_root core_balloc_free; - struct scoutfs_btree_root alloc_root; + struct scoutfs_balloc_root core_data_alloc; + struct scoutfs_balloc_root core_data_free; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; @@ -653,8 +697,6 @@ struct scoutfs_net_header { enum { SCOUTFS_NET_CMD_GREETING = 0, SCOUTFS_NET_CMD_ALLOC_INODES, - SCOUTFS_NET_CMD_ALLOC_EXTENT, - SCOUTFS_NET_CMD_FREE_EXTENTS, SCOUTFS_NET_CMD_GET_LOG_TREES, SCOUTFS_NET_CMD_COMMIT_LOG_TREES, SCOUTFS_NET_CMD_ADVANCE_SEQ, @@ -705,25 +747,6 @@ struct scoutfs_net_statfs { __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; -struct scoutfs_net_extent { - __le64 start; - __le64 len; -} __packed; - -struct scoutfs_net_extent_list { - __le64 nr; - struct { - __le64 start; - __le64 len; - } __packed extents[0]; -} __packed; - -#define SCOUTFS_NET_EXTENT_LIST_BYTES(nr) \ - offsetof(struct scoutfs_net_extent_list, extents[nr]) - -/* arbitrarily makes a nice ~1k extent list payload */ -#define SCOUTFS_NET_EXTENT_LIST_MAX_NR 64 - struct scoutfs_net_lock { struct scoutfs_key key; __le64 write_version; diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 10196df4..eeebec8e 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -587,6 +587,13 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) if (ret) goto unlock; + /* create offline extents in potentially many transactions */ + if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { + ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); + if (ret) + goto unlock; + } + /* can only change size/dv on untouched regular files */ if ((sm.i_size != 0 || sm.data_version != 0) && ((!S_ISREG(inode->i_mode) || @@ -602,12 +609,6 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) if (ret) goto unlock; - if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { - ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); - if (ret) - goto release; - } - if (sm.data_version) scoutfs_inode_set_data_version(inode, sm.data_version); if (sm.i_size) @@ -618,7 +619,6 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) scoutfs_update_inode_item(inode, lock, &ind_locks); ret = 0; -release: scoutfs_release_trans(sb); unlock: scoutfs_inode_index_unlock(sb, &ind_locks); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index c491ed22..cdb6e561 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -157,7 +157,7 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, mode != SCOUTFS_LOCK_NULL); /* any transition from a mode allowed to dirty items has to write */ - if (lock_mode_can_write(prev) && scoutfs_forest_has_dirty(sb)) { + if (lock_mode_can_write(prev) && scoutfs_trans_has_dirty(sb)) { ret = scoutfs_trans_sync(sb, 1); if (ret < 0) return ret; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index affdd3d1..712ab69b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -34,12 +34,33 @@ #include "count.h" #include "export.h" #include "dir.h" -#include "extents.h" #include "server.h" #include "net.h" +#include "data.h" struct lock_info; +#define STE_FMT "[%llu %llu %llu 0x%x]" +#define STE_ARGS(te) (te)->iblock, (te)->count, (te)->blkno, (te)->flags +#define STE_FIELDS(pref) \ + __field(__u64, pref##_iblock) \ + __field(__u64, pref##_count) \ + __field(__u64, pref##_blkno) \ + __field(__u8, pref##_flags) +#define STE_ASSIGN(pref, te) \ + __entry->pref##_iblock = (te)->iblock; \ + __entry->pref##_count = (te)->count; \ + __entry->pref##_blkno = (te)->blkno; \ + __entry->pref##_flags = (te)->flags; +#define STE_ENTRY_ARGS(pref) \ + __entry->pref##_iblock, \ + __entry->pref##_count, \ + __entry->pref##_blkno, \ + __entry->pref##_flags + +#define DECLARE_TRACED_EXTENT(name) \ + struct scoutfs_traced_extent name = {0} + TRACE_EVENT(scoutfs_setattr, TP_PROTO(struct dentry *dentry, struct iattr *attr), @@ -152,15 +173,17 @@ TRACE_EVENT(scoutfs_data_fiemap, TRACE_EVENT(scoutfs_get_block, TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, - int create, int ret, __u64 blkno, size_t size), + int create, struct scoutfs_traced_extent *te, + int ret, __u64 blkno, size_t size), - TP_ARGS(sb, ino, iblock, create, ret, blkno, size), + TP_ARGS(sb, ino, iblock, create, te, ret, blkno, size), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) __field(__u64, iblock) __field(int, create) + STE_FIELDS(ext) __field(int, ret) __field(__u64, blkno) __field(size_t, size) @@ -171,68 +194,58 @@ TRACE_EVENT(scoutfs_get_block, __entry->ino = ino; __entry->iblock = iblock; __entry->create = create; + STE_ASSIGN(ext, te) __entry->ret = ret; __entry->blkno = blkno; __entry->size = size; ), - TP_printk(SCSBF" ino %llu iblock %llu create %d ret %d bnr %llu " - "size %zu", SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, - __entry->create, __entry->ret, __entry->blkno, __entry->size) + TP_printk(SCSBF" ino %llu iblock %llu create %d ext "STE_FMT" ret %d bnr %llu size %zu", + SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, + __entry->create, STE_ENTRY_ARGS(ext), __entry->ret, + __entry->blkno, __entry->size) ); -TRACE_EVENT(scoutfs_data_alloc_block, - TP_PROTO(struct super_block *sb, struct inode *inode, - struct scoutfs_extent *ext, u64 iblock, u64 len, - u64 online_blocks, u64 offline_blocks), +TRACE_EVENT(scoutfs_data_file_extent_class, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), - TP_ARGS(sb, inode, ext, iblock, len, online_blocks, offline_blocks), + TP_ARGS(sb, ino, te), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, ino) - se_trace_define(ext) - __field(__u64, iblock) - __field(__u64, len) - __field(__u64, online_blocks) - __field(__u64, offline_blocks) + STE_FIELDS(ext) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->ino = scoutfs_ino(inode); - se_trace_assign(ext, ext); - __entry->iblock = iblock; - __entry->len = len; - __entry->online_blocks = online_blocks; - __entry->offline_blocks = offline_blocks; + __entry->ino = ino; + STE_ASSIGN(ext, te) ), - TP_printk(SCSBF" ino %llu ext "SE_FMT" iblock %llu len %llu online_blocks %llu offline_blocks %llu", - SCSB_TRACE_ARGS, __entry->ino, se_trace_args(ext), - __entry->iblock, __entry->len, __entry->online_blocks, - __entry->offline_blocks) + TP_printk(SCSBF" ino %llu ext "STE_FMT, + SCSB_TRACE_ARGS, __entry->ino, STE_ENTRY_ARGS(ext)) ); - -TRACE_EVENT(scoutfs_data_alloc_block_ret, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext, int ret), - - TP_ARGS(sb, ext, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - se_trace_define(ext) - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - se_trace_assign(ext, ext); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ext "SE_FMT" ret %d", SCSB_TRACE_ARGS, - se_trace_args(ext), __entry->ret) +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc_block, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc_unwritten, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_extent_truncated, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_fiemap_extent, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) ); TRACE_EVENT(scoutfs_data_truncate_items, @@ -260,10 +273,9 @@ TRACE_EVENT(scoutfs_data_truncate_items, TRACE_EVENT(scoutfs_data_wait_check, TP_PROTO(struct super_block *sb, __u64 ino, __u64 pos, __u64 len, - __u8 sef, __u8 op, __u64 ext_start, __u64 ext_len, - __u8 ext_flags, int ret), + __u8 sef, __u8 op, struct scoutfs_traced_extent *te, int ret), - TP_ARGS(sb, ino, pos, len, sef, op, ext_start, ext_len, ext_flags, ret), + TP_ARGS(sb, ino, pos, len, sef, op, te, ret), TP_STRUCT__entry( SCSB_TRACE_FIELDS @@ -272,9 +284,7 @@ TRACE_EVENT(scoutfs_data_wait_check, __field(__u64, len) __field(__u8, sef) __field(__u8, op) - __field(__u64, ext_start) - __field(__u64, ext_len) - __field(__u8, ext_flags) + STE_FIELDS(ext) __field(int, ret) ), @@ -285,16 +295,13 @@ TRACE_EVENT(scoutfs_data_wait_check, __entry->len = len; __entry->sef = sef; __entry->op = op; - __entry->ext_start = ext_start; - __entry->ext_len = ext_len; - __entry->ext_flags = ext_flags; + STE_ASSIGN(ext, te) __entry->ret = ret; ), - TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext_start %llu ext_len %llu ext_flags 0x%x ret %d", + TP_printk(SCSBF" ino %llu pos %llu len %llu sef 0x%x op 0x%x ext "STE_FMT" ret %d", SCSB_TRACE_ARGS, __entry->ino, __entry->pos, __entry->len, - __entry->sef, __entry->op, __entry->ext_start, - __entry->ext_len, __entry->ext_flags, __entry->ret) + __entry->sef, __entry->op, STE_ENTRY_ARGS(ext), __entry->ret) ); TRACE_EVENT(scoutfs_sync_fs, @@ -1588,115 +1595,6 @@ TRACE_EVENT(scoutfs_btree_dirty_block, __entry->bt_blkno, __entry->bt_seq) ); -DECLARE_EVENT_CLASS(scoutfs_extent_class, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - - TP_ARGS(sb, ext), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - se_trace_define(ext) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - se_trace_assign(ext, ext); - ), - - TP_printk(SCSBF" ext "SE_FMT, - SCSB_TRACE_ARGS, se_trace_args(ext)) -); - -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_insert, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_delete, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_input, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_next_output, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_input, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_prev_output, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_add, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_extent_remove, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); - -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_remove, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_truncate_offline, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_server_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_find_free_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_alloc_block_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_get_block_intersection, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_fiemap_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_data_return_server_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_next, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_alloc_extent_allocated, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_free_pending_extent, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); -DEFINE_EVENT(scoutfs_extent_class, scoutfs_server_extent_io, - TP_PROTO(struct super_block *sb, struct scoutfs_extent *ext), - TP_ARGS(sb, ext) -); - TRACE_EVENT(scoutfs_online_offline_blocks, TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, u64 on_now, u64 off_now), diff --git a/kmod/src/server.c b/kmod/src/server.c index ee378ea8..54dee331 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -68,9 +68,7 @@ struct server_info { /* server tracks seq use */ struct rw_semaphore seq_rwsem; - /* server tracks pending frees to be applied during commit */ struct rw_semaphore alloc_rwsem; - struct list_head pending_frees; struct list_head clients; unsigned long nr_clients; @@ -103,291 +101,6 @@ struct commit_waiter { int ret; }; -static void init_extent_btree_key(struct scoutfs_extent_btree_key *ebk, - u8 type, u64 major, u64 minor) -{ - ebk->type = type; - ebk->major = cpu_to_be64(major); - ebk->minor = cpu_to_be64(minor); -} - -static int init_extent_from_btree_key(struct scoutfs_extent *ext, u8 type, - struct scoutfs_extent_btree_key *ebk, - unsigned int key_bytes) -{ - u64 start; - u64 len; - - /* btree _next doesn't have last key limit */ - if (ebk->type != type) - return -ENOENT; - - if (key_bytes != sizeof(struct scoutfs_extent_btree_key) || - (ebk->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ebk->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EIO; /* XXX corruption, bad key */ - - start = be64_to_cpu(ebk->major); - len = be64_to_cpu(ebk->minor); - if (ebk->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(start, len); - start -= len - 1; - - return scoutfs_extent_init(ext, ebk->type, 0, start, len, 0, 0); -} - -/* - * This is called by the extent core on behalf of the server who holds - * the appropriate locks to protect the many btree items that can be - * accessed on behalf of one extent operation. - * - * The free_blocks count in the super tracks the number of blocks in - * the primary extent index. We update it here instead of expecting - * callers to remember. - */ -static int server_extent_io(struct super_block *sb, int op, - struct scoutfs_extent *ext, void *data) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_extent_btree_key ebk; - SCOUTFS_BTREE_ITEM_REF(iref); - bool mirror = false; - u8 mirror_type; - u8 mirror_op = 0; - int ret; - int err; - - trace_scoutfs_server_extent_io(sb, ext); - - if (WARN_ON_ONCE(ext->type != SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - ext->type != SCOUTFS_FREE_EXTENT_BLOCKS_TYPE)) - return -EINVAL; - - if (ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE && - (op == SEI_INSERT || op == SEI_DELETE)) { - mirror = true; - mirror_type = SCOUTFS_FREE_EXTENT_BLOCKS_TYPE; - mirror_op = op == SEI_INSERT ? SEI_DELETE : SEI_INSERT; - } - - init_extent_btree_key(&ebk, ext->type, ext->start + ext->len - 1, - ext->len); - if (ext->type == SCOUTFS_FREE_EXTENT_BLOCKS_TYPE) - swap(ebk.major, ebk.minor); - - if (op == SEI_NEXT || op == SEI_PREV) { - if (op == SEI_NEXT) - ret = scoutfs_btree_next(sb, &super->alloc_root, - &ebk, sizeof(ebk), &iref); - else - ret = scoutfs_btree_prev(sb, &super->alloc_root, - &ebk, sizeof(ebk), &iref); - if (ret == 0) { - ret = init_extent_from_btree_key(ext, ext->type, - iref.key, - iref.key_len); - scoutfs_btree_put_iref(&iref); - } - - } else if (op == SEI_INSERT) { - ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, - &super->alloc_root, - &ebk, sizeof(ebk), NULL, 0); - - } else if (op == SEI_DELETE) { - ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, - &super->alloc_root, - &ebk, sizeof(ebk)); - - } else { - ret = WARN_ON_ONCE(-EINVAL); - } - - if (ret == 0 && mirror) { - swap(ext->type, mirror_type); - ret = server_extent_io(sb, op, ext, data); - swap(ext->type, mirror_type); - if (ret < 0) { - err = server_extent_io(sb, mirror_op, ext, data); - if (err) - scoutfs_corruption(sb, - SC_SERVER_EXTENT_CLEANUP, - corrupt_server_extent_cleanup, - "op %u ext "SE_FMT" ret %d", - op, SE_ARG(ext), err); - } - } - - if (ret == 0 && ext->type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { - if (op == SEI_INSERT) - le64_add_cpu(&super->free_blocks, ext->len); - else if (op == SEI_DELETE) - le64_add_cpu(&super->free_blocks, -ext->len); - } - - return ret; -} - -/* - * Allocate an extent of the given length in the first smallest free - * extent that contains it. - */ -static int alloc_extent(struct super_block *sb, u64 blocks, - u64 *start, u64 *len) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct scoutfs_extent ext; - int ret; - - *start = 0; - *len = 0; - - down_write(&server->alloc_rwsem); - - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLOCKS_TYPE, 0, - 0, blocks, 0, 0); - ret = scoutfs_extent_next(sb, server_extent_io, &ext, NULL); - if (ret == -ENOENT) - ret = scoutfs_extent_prev(sb, server_extent_io, &ext, NULL); - if (ret) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - trace_scoutfs_server_alloc_extent_next(sb, &ext); - - ext.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; - ext.len = min(blocks, ext.len); - - ret = scoutfs_extent_remove(sb, server_extent_io, &ext, NULL); - if (ret) - goto out; - - trace_scoutfs_server_alloc_extent_allocated(sb, &ext); - - *start = ext.start; - *len = ext.len; - ret = 0; - -out: - up_write(&server->alloc_rwsem); - - if (ret) - scoutfs_inc_counter(sb, server_extent_alloc_error); - else - scoutfs_inc_counter(sb, server_extent_alloc); - - return ret; -} - -struct pending_free_extent { - struct list_head head; - u64 start; - u64 len; -}; - -/* - * Now that the transaction's done we can apply all the pending frees. - * The list entries are totally unsorted so this is the first time that - * we can discover corruption from duplicated frees, etc. This can also - * fail on normal transient io or memory errors. - * - * We can't unwind if this fails. The caller can freak out or keep - * trying forever. - */ -static int apply_pending_frees(struct super_block *sb) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - struct pending_free_extent *tmp; - struct scoutfs_extent ext; - int ret; - - down_write(&server->alloc_rwsem); - - list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { - scoutfs_inc_counter(sb, server_free_pending_extent); - scoutfs_extent_init(&ext, SCOUTFS_FREE_EXTENT_BLKNO_TYPE, 0, - pfe->start, pfe->len, 0, 0); - trace_scoutfs_server_free_pending_extent(sb, &ext); - ret = scoutfs_extent_add(sb, server_extent_io, &ext, NULL); - if (ret) { - scoutfs_inc_counter(sb, server_free_pending_error); - break; - } - - list_del_init(&pfe->head); - kfree(pfe); - } - - up_write(&server->alloc_rwsem); - - return 0; -} - -/* - * If there are still pending frees to destroy it means the server didn't - * shut down cleanly and that's not well supported today so we want to - * have it holler if this happens. In the future we'd cleanly support - * forced shutdown that had been told that it's OK to throw away dirty - * state. - */ -static int destroy_pending_frees(struct super_block *sb) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - struct pending_free_extent *tmp; - - WARN_ON_ONCE(!list_empty(&server->pending_frees)); - - down_write(&server->alloc_rwsem); - - list_for_each_entry_safe(pfe, tmp, &server->pending_frees, head) { - list_del_init(&pfe->head); - kfree(pfe); - } - - up_write(&server->alloc_rwsem); - - return 0; -} - -/* - * We can't satisfy allocations with freed extents until the removed - * references to the freed extents have been committed. We add freed - * extents to a list that is only applied to the persistent indexes as - * the transaction is being committed and the current transaction won't - * try to allocate any more extents. If we didn't do this then we could - * write to referenced data as part of the commit that frees it. If the - * commit was interrupted the stable data could have been overwritten. - */ -static int free_extent(struct super_block *sb, u64 start, u64 len) -{ - struct server_info *server = SCOUTFS_SB(sb)->server_info; - struct pending_free_extent *pfe; - int ret; - - scoutfs_inc_counter(sb, server_free_extent); - - down_write(&server->alloc_rwsem); - - pfe = kmalloc(sizeof(struct pending_free_extent), GFP_NOFS); - if (!pfe) { - ret = -ENOMEM; - } else { - pfe->start = start; - pfe->len = len; - list_add_tail(&pfe->head, &server->pending_frees); - ret = 0; - } - - up_write(&server->alloc_rwsem); - - return ret; -} - static void stop_server(struct server_info *server) { /* wait_event/wake_up provide barriers */ @@ -444,23 +157,60 @@ static int add_uninit_balloc_items(struct super_block *sb, struct server_info *server, struct scoutfs_super_block *super) { - u64 next = le64_to_cpu(super->next_uninit_free_block); - u64 total = le64_to_cpu(super->total_blocks); + u64 next = le64_to_cpu(super->next_uninit_meta_blkno); + u64 last = le64_to_cpu(super->last_uninit_meta_blkno); u64 nr; int ret; + if (next > last) + return 0; + /* next_uninit should always start a new item */ if (WARN_ON_ONCE(next & SCOUTFS_BALLOC_ITEM_BIT_MASK)) return -EIO; - nr = min_t(u64, total - next, + nr = min_t(u64, last - next + 1, round_up(512 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE, SCOUTFS_BALLOC_ITEM_BITS)); ret = scoutfs_balloc_add_alloc_bulk(sb, &server->alloc, &server->wri, next, nr); if (ret == 0) - le64_add_cpu(&super->next_uninit_free_block, nr); + le64_add_cpu(&super->next_uninit_meta_blkno, nr); + + return ret; +} + +/* + * Add newly initialized free block bitmap items. + */ +static int add_uninit_data_alloc_items(struct super_block *sb, + struct server_info *server, + struct scoutfs_super_block *super) +{ + int nr = 16; + u64 next; + u64 last; + int ret; + + while (nr-- > 0) { + next = le64_to_cpu(super->next_uninit_data_blkno); + last = le64_to_cpu(super->last_uninit_data_blkno); + if (next > last) { + ret = 0; + break; + } + + ret = scoutfs_data_add_free_blocks(sb, &server->alloc, + &server->wri, + &super->core_data_alloc, + next, last - next + 1); + if (ret <= 0) + break; + + le64_add_cpu(&super->next_uninit_data_blkno, ret); + ret = 0; + } return ret; } @@ -499,17 +249,14 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); - /* try to free first which can dirty the btrees */ - ret = apply_pending_frees(sb); - if (ret) { - scoutfs_err(sb, "server error freeing extents: %d", ret); - goto out; - } - /* XXX not sure what to do about failure here */ ret = add_uninit_balloc_items(sb, server, super); BUG_ON(ret); + /* XXX not sure what to do about failure here */ + ret = add_uninit_data_alloc_items(sb, server, super); + BUG_ON(ret); + ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); @@ -579,94 +326,6 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); } -/* - * Give the client an extent allocation of len blocks. We leave the - * details to the extent allocator. - */ -static int server_alloc_extent(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct commit_waiter cw; - struct scoutfs_net_extent nex = {0,}; - __le64 leblocks; - u64 start; - u64 len; - int ret; - - if (arg_len != sizeof(leblocks)) { - ret = -EINVAL; - goto out; - } - - memcpy(&leblocks, arg, arg_len); - - down_read(&server->commit_rwsem); - ret = alloc_extent(sb, le64_to_cpu(leblocks), &start, &len); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); - if (ret) - goto out; - - nex.start = cpu_to_le64(start); - nex.len = cpu_to_le64(len); -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, &nex, sizeof(nex)); -} - -static bool invalid_net_extent_list(struct scoutfs_net_extent_list *nexl, - unsigned data_len) -{ - return (data_len < sizeof(struct scoutfs_net_extent_list)) || - (le64_to_cpu(nexl->nr) > SCOUTFS_NET_EXTENT_LIST_MAX_NR) || - (data_len != offsetof(struct scoutfs_net_extent_list, - extents[le64_to_cpu(nexl->nr)])); -} - -static int server_free_extents(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_net_extent_list *nexl; - struct commit_waiter cw; - int ret = 0; - int err; - u64 i; - - nexl = arg; - if (invalid_net_extent_list(nexl, arg_len)) { - ret = -EINVAL; - goto out; - } - - down_read(&server->commit_rwsem); - - for (i = 0; i < le64_to_cpu(nexl->nr); i++) { - ret = free_extent(sb, le64_to_cpu(nexl->extents[i].start), - le64_to_cpu(nexl->extents[i].len)); - if (ret) - break; - } - - if (i > 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - - if (i > 0) { - err = wait_for_commit(&cw); - if (ret == 0) - ret = err; - } - -out: - return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); -} - /* * Give the client references to stable persistent trees that they'll * use to write their next transaction. @@ -728,9 +387,8 @@ static int server_get_log_trees(struct super_block *sb, memset(<v, 0, sizeof(ltv)); } + /* ensure client has enough free metadata blocks for a transaction */ target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; - - /* XXX arbitrarily give client enough metadata for a transaction */ while (le64_to_cpu(ltv.alloc_root.total_free) < target) { from = le64_to_cpu(super->core_balloc_cursor); at_least = target - le64_to_cpu(ltv.alloc_root.total_free); @@ -747,9 +405,20 @@ static int server_get_log_trees(struct super_block *sb, goto unlock; super->core_balloc_cursor = cpu_to_le64(next_past); - } + /* fill client's data block allocator */ + target = (2ULL*1024*1024*1024) / SCOUTFS_BLOCK_SIZE; + down_write(&server->alloc_rwsem); + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + <v.data_alloc, + &super->core_data_alloc, + &super->core_data_alloc_cursor, + target); + up_write(&server->alloc_rwsem); + if (ret < 0) + goto unlock; + /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), @@ -768,6 +437,8 @@ unlock: lt.free_root = ltv.free_root; lt.item_root = ltv.item_root; lt.bloom_ref = ltv.bloom_ref; + lt.data_alloc = ltv.data_alloc; + lt.data_free = ltv.data_free; lt.rid = be64_to_le64(ltk.rid); lt.nr = be64_to_le64(ltk.nr); } @@ -824,10 +495,14 @@ static int server_commit_log_trees(struct super_block *sb, goto unlock; } + /* XXX probably want to merge free blocks */ + ltv.alloc_root = lt->alloc_root; ltv.free_root = lt->free_root; ltv.item_root = lt->item_root; ltv.bloom_ref = lt->bloom_ref; + ltv.data_alloc = lt->data_alloc; + ltv.data_free = lt->data_free; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), @@ -846,6 +521,82 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* + * A client is being evicted so we want to reclaim resources from their + * log tree items. The item trees and bloom refs stay around to be read + * and eventually merged and we reclaim all the allocator items. + * + * The caller holds the commit rwsem which means we do all this work + * in one server commit. We'll need to keep the total amount of blocks + * in trees in check. + * + * By the time we're evicting a client they've either synced their data + * or have been forcefully removed. The free blocks in the allocator + * roots are stable and can be merged back into allocator items for use + * without risking overwriting stable data. + */ +static int reclaim_log_trees(struct super_block *sb, u64 rid) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + DECLARE_SERVER_INFO(sb, server); + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_log_trees_key ltk; + struct scoutfs_log_trees_val ltv; + __le64 curs; + u64 tot; + int ret; + + memset(<k, 0, sizeof(ltk)); + memset(<v, 0, sizeof(ltv)); + + mutex_lock(&server->logs_mutex); + down_write(&server->alloc_rwsem); + + /* find the client's existing item */ + ltk.rid = cpu_to_be64(rid); + ltk.nr = 0; + ret = scoutfs_btree_next(sb, &super->logs_root, + <k, sizeof(ltk), &iref); + if (ret == 0) { + if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && + iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + memcpy(<k, iref.key, iref.key_len); + memcpy(<v, iref.val, iref.val_len); + if (be64_to_cpu(ltk.rid) != rid) + ret = -ENOENT; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + goto out; + } + + tot = le64_to_cpu(super->core_data_alloc.total_free) + + le64_to_cpu(ltv.data_alloc.total_free); + curs = 0; + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + &super->core_data_alloc, + <v.data_alloc, &curs, tot); + if (ret < 0) + goto out; + + tot = le64_to_cpu(super->core_data_alloc.total_free) + + le64_to_cpu(ltv.data_free.total_free); + curs = 0; + ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, + &super->core_data_alloc, + <v.data_free, &curs, tot); +out: + up_write(&server->alloc_rwsem); + mutex_unlock(&server->logs_mutex); + + return ret; +} + /* * Give the client the next sequence number for their transaction. They * provide their previous transaction sequence number that they've @@ -1455,6 +1206,7 @@ static void farewell_worker(struct work_struct *work) ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: remove_trans_seq(sb, fw->rid) ?: + reclaim_log_trees(sb, fw->rid) ?: delete_mounted_client(sb, fw->rid); if (ret == 0) queue_commit_work(server, &cw); @@ -1564,8 +1316,6 @@ static int server_farewell(struct super_block *sb, static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GREETING] = server_greeting, [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, - [SCOUTFS_NET_CMD_ALLOC_EXTENT] = server_alloc_extent, - [SCOUTFS_NET_CMD_FREE_EXTENTS] = server_free_extents, [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, @@ -1697,7 +1447,6 @@ shutdown: flush_work(&server->commit_work); server->conn = NULL; - destroy_pending_frees(sb); scoutfs_lock_server_destroy(sb); out: @@ -1793,7 +1542,6 @@ int scoutfs_server_setup(struct super_block *sb) INIT_WORK(&server->commit_work, scoutfs_server_commit_func); init_rwsem(&server->seq_rwsem); init_rwsem(&server->alloc_rwsem); - INIT_LIST_HEAD(&server->pending_frees); INIT_LIST_HEAD(&server->clients); mutex_init(&server->farewell_mutex); INIT_LIST_HEAD(&server->farewell_requests); diff --git a/kmod/src/super.c b/kmod/src/super.c index 9387a4b5..69de8ebe 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -441,7 +441,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_client_setup(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, &sbi->rid_lock) ?: - scoutfs_forest_get_log_trees(sb); + scoutfs_trans_get_log_trees(sb); if (ret) goto out; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index deb25f20..679ef1b1 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -25,6 +25,8 @@ #include "counters.h" #include "client.h" #include "inode.h" +#include "balloc.h" +#include "block.h" #include "scoutfs_trace.h" /* @@ -60,6 +62,10 @@ struct trans_info { unsigned reserved_vals; unsigned holders; bool writing; + + struct scoutfs_log_trees lt; + struct scoutfs_balloc_allocator alloc; + struct scoutfs_block_writer wri; }; #define DECLARE_TRANS_INFO(sb, name) \ @@ -77,6 +83,48 @@ static bool drained_holders(struct trans_info *tri) return drained; } +static int commit_btrees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + + lt = tri->lt; + lt.alloc_root = tri->alloc.alloc_root; + lt.free_root = tri->alloc.free_root; + scoutfs_forest_get_btrees(sb, <); + scoutfs_data_get_btrees(sb, <); + + return scoutfs_client_commit_log_trees(sb, <); +} + +/* + * This gets all the resources from the server that the client will + * need during the transaction. + */ +int scoutfs_trans_get_log_trees(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + struct scoutfs_log_trees lt; + int ret = 0; + + ret = scoutfs_client_get_log_trees(sb, <); + if (ret == 0) { + tri->lt = lt; + scoutfs_balloc_init(&tri->alloc, <.alloc_root, <.free_root); + scoutfs_block_writer_init(sb, &tri->wri); + + scoutfs_forest_init_btrees(sb, &tri->alloc, &tri->wri, <); + scoutfs_data_init_btrees(sb, &tri->alloc, &tri->wri, <); + } + return ret; +} + +bool scoutfs_trans_has_dirty(struct super_block *sb) +{ + DECLARE_TRANS_INFO(sb, tri); + + return scoutfs_block_writer_has_dirty(sb, &tri->wri); +} /* * This work func is responsible for writing out all the dirty blocks * that make up the current dirty transaction. It prevents writers from @@ -113,18 +161,19 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); - trace_scoutfs_trans_write_func(sb, scoutfs_forest_dirty_bytes(sb)); + trace_scoutfs_trans_write_func(sb, + scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); - if (scoutfs_forest_has_dirty(sb)) { + if (scoutfs_block_writer_has_dirty(sb, &tri->wri)) { if (sbi->trans_deadline_expired) scoutfs_inc_counter(sb, trans_commit_timer); ret = scoutfs_inode_walk_writeback(sb, true) ?: - scoutfs_forest_write(sb) ?: + scoutfs_block_writer_write(sb, &tri->wri) ?: scoutfs_inode_walk_writeback(sb, false) ?: - scoutfs_forest_commit(sb) ?: + commit_btrees(sb) ?: scoutfs_client_advance_seq(sb, &sbi->trans_seq) ?: - scoutfs_forest_get_log_trees(sb); + scoutfs_trans_get_log_trees(sb); if (ret) goto out; @@ -297,7 +346,8 @@ static bool acquired_hold(struct super_block *sb, vals = tri->reserved_vals + cnt->vals; /* XXX arbitrarily limit to 8 meg transactions */ - if (scoutfs_forest_dirty_bytes(sb) >= (8 * 1024 * 1024)) { + if (scoutfs_block_writer_dirty_bytes(sb, &tri->wri) >= + (8 * 1024 * 1024)) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; @@ -481,6 +531,7 @@ void scoutfs_shutdown_trans(struct super_block *sb) DECLARE_TRANS_INFO(sb, tri); if (tri) { + scoutfs_block_writer_forget_all(sb, &tri->wri); if (sbi->trans_write_workq) { cancel_delayed_work_sync(&sbi->trans_write_work); destroy_workqueue(sbi->trans_write_workq); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 04e28f9c..e5f3228e 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -16,6 +16,9 @@ void scoutfs_release_trans(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, signed vals); +int scoutfs_trans_get_log_trees(struct super_block *sb); +bool scoutfs_trans_has_dirty(struct super_block *sb); + int scoutfs_setup_trans(struct super_block *sb); void scoutfs_shutdown_trans(struct super_block *sb); From 0de6cade1977d87919f75778207dfeea9890aaea Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 11 Dec 2019 10:05:47 -0800 Subject: [PATCH 769/920] scoutfs: remove generic extents storage We are no longer storing individual extents in items from multiple places and indexed in multiple ways. We can remove this extent support code. Signed-off-by: Zach Brown --- kmod/src/extents.c | 328 --------------------------------------------- kmod/src/extents.h | 94 ------------- 2 files changed, 422 deletions(-) delete mode 100644 kmod/src/extents.c delete mode 100644 kmod/src/extents.h diff --git a/kmod/src/extents.c b/kmod/src/extents.c deleted file mode 100644 index c5bacdfd..00000000 --- a/kmod/src/extents.c +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright (C) 2018 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include - -#include "extents.h" -#include "counters.h" -#include "scoutfs_trace.h" -#include "msg.h" - -/* - * These low level functions take on the fiddly details of extent - * manipulation. Callers handle serialization and storage and call in - * here to add or remove extents. This slices and dices the extents - * while dodging all the fence posts. - */ - -/* return the last logical position that is in the extent, inclusive */ -static u64 extent_end(struct scoutfs_extent *ext) -{ - return ext->start + ext->len - 1; -} - -/* returns true if the two extents overlap */ -static bool extents_overlap(struct scoutfs_extent *a, struct scoutfs_extent *b) -{ - return extent_end(a) >= b->start && a->start <= extent_end(b); -} - -/* Returns true if a is entirely within b */ -static bool extent_within(struct scoutfs_extent *a, struct scoutfs_extent *b) -{ - return a->start >= b->start && extent_end(a) <= extent_end(b); -} - -/* - * Returns true if two extents can be merged because they're adjacent, - * mapping is equally set or not, mappings are adjacent if they're set, - * and all the rest of the fields match. - */ -static bool extents_can_merge(struct scoutfs_extent *a, - struct scoutfs_extent *b) -{ - if (a->start > b->start) - swap(a, b); - - return (a->owner == b->owner) && - ((a->start + a->len) == b->start) && - (!!a->map == !!b->map) && - (!a->map || ((a->map + a->len) == b->map)) && - (a->type == b->type) && - (a->flags == b->flags); -} - -int scoutfs_extent_init(struct scoutfs_extent *ext, u8 type, u64 owner, - u64 start, u64 len, u64 map, u8 flags) -{ - /* don't allow 0 len or len wrapping map or start */ - if ((start + len <= start) || (map + len <= map)) - return -EIO; - - ext->owner = owner; - ext->start = start; - ext->len = len; - ext->map = map; - ext->type = type; - ext->flags = flags; - - return 0; -} - -/* - * Returns true if the two extents intersect and modifies a to be the - * intersection of the two extents. Callers only need to initialize a's - * start and len when probing for an intersection and we'll copy the - * rest from b. - */ -bool scoutfs_extent_intersection(struct scoutfs_extent *a, - struct scoutfs_extent *b) -{ - u64 new_start; - u64 new_end; - - if (extents_overlap(a, b)) { - new_end = min(extent_end(a), extent_end(b)); - new_start = max(a->start, b->start); - - a->owner = b->owner; - a->start = new_start; - a->len = new_end - new_start + 1; - a->map = b->map ? (new_start - b->start) + b->map: 0; - a->type = b->type; - a->flags = b->flags; - return true; - } - - return false; -} - -static int extent_insert(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ins, void *data) -{ - scoutfs_inc_counter(sb, extent_insert); - trace_scoutfs_extent_insert(sb, ins); - return iof(sb, SEI_INSERT, ins, data); -} - -static int extent_delete(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *del, void *data) -{ - scoutfs_inc_counter(sb, extent_delete); - trace_scoutfs_extent_delete(sb, del); - return iof(sb, SEI_DELETE, del, data); -} - -/* - * Find the next extent using the given extent as the starting search - * position. This just passes the extent through to the underlying key - * building and searching routines. - * - * Callers have to be very careful when building the search extent. - * Most extents are indexed by their final logical position and some - * have all the metadata in the key. So a typical pattern is to search - * for an intersection by searching from a single block extent with the - * rest of the fields set to zero. - * - * But some callers are searching indexes of free extents where both the - * length and start are meaningful. - * - * The io function is responsible for ensuring that we return next - * extents with the same type and owner as the given extent. - */ -int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ext, void *data) -{ - int ret; - - scoutfs_inc_counter(sb, extent_next); - trace_scoutfs_extent_next_input(sb, ext); - ret = iof(sb, SEI_NEXT, ext, data); - if (ret == 0) - trace_scoutfs_extent_next_output(sb, ext); - return ret; -} - -int scoutfs_extent_prev(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ext, void *data) -{ - int ret; - - scoutfs_inc_counter(sb, extent_prev); - trace_scoutfs_extent_prev_input(sb, ext); - ret = iof(sb, SEI_PREV, ext, data); - if (ret == 0) - trace_scoutfs_extent_prev_output(sb, ext); - return ret; -} - -/* - * Search for a next extent and see if we can merge it with the caller's - * extent. The caller has initialized next for us to search from. If - * we can merge then we update the callers extent, delete the old - * extent, and return 1. If we return an error or 0 then nothing will - * have changed. - */ -static int try_merge_next(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ext, - struct scoutfs_extent *next, void *data) -{ - int ret; - - ret = scoutfs_extent_next(sb, iof, next, data); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - goto out; - } - - if (extents_overlap(ext, next)) { - ret = -EIO; - goto out; - } - - if (!extents_can_merge(ext, next)) { - ret = 0; - goto out; - } - - if (next->start < ext->start) { - ext->start = next->start; - ext->map = next->map; - ext->len += next->len; - } else { - ext->len += next->len; - } - - ret = extent_delete(sb, iof, next, data); - if (ret == 0) - ret = 1; -out: - return ret; -} - -/* - * Add a new extent. It can not overlap with any existing extents. It - * may be merged with neighbouring extents. - */ -int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *add, void *data) -{ - struct scoutfs_extent right; - struct scoutfs_extent left; - struct scoutfs_extent ext; - bool ins_left = false; - bool ins_right = false; - int ret; - - scoutfs_inc_counter(sb, extent_add); - trace_scoutfs_extent_add(sb, add); - ext = *add; - - /* see if we are merging with and deleting a left neighbour */ - if (ext.start) { - scoutfs_extent_init(&left, ext.type, ext.owner, - ext.start - 1, 1, 0, 0); - ret = try_merge_next(sb, iof, &ext, &left, data); - if (ret < 0) - goto out; - if (ret > 0) - ins_left = true; - } - - /* see if we are merging with and deleting a right neighbour */ - if (ext.start + ext.len <= SCOUTFS_BLOCK_MAX) { - scoutfs_extent_init(&right, ext.type, ext.owner, - ext.start, 1, 0, 0); - ret = try_merge_next(sb, iof, &ext, &right, data); - if (ret < 0) - goto out; - if (ret > 0) - ins_right = true; - } - - /* finally insert our new (possibly merged) extent */ - ret = extent_insert(sb, iof, &ext, data); -out: - scoutfs_extent_cleanup(ret < 0 && ins_right, extent_insert, sb, iof, - &right, data, SC_EXTENT_ADD_CLEANUP, - corrupt_extent_add_cleanup, add); - scoutfs_extent_cleanup(ret < 0 && ins_left, extent_insert, sb, iof, - &left, data, SC_EXTENT_ADD_CLEANUP, - corrupt_extent_add_cleanup, add); - return ret; -} - - -/* - * Remove a region of an existing extent. The region to remove must be - * be fully within an existing extent. This creates the items left - * behind on either end of the removed region as appropriate. - */ -int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *rem, void *data) -{ - struct scoutfs_extent right; - struct scoutfs_extent left; - struct scoutfs_extent ext; - bool ins_ext = false; - bool del_left = false; - int ret; - - scoutfs_inc_counter(sb, extent_remove); - trace_scoutfs_extent_remove(sb, rem); - - scoutfs_extent_init(&ext, rem->type, rem->owner, rem->start, 1, 0, 0); - ret = scoutfs_extent_next(sb, iof, &ext, data); - if (ret < 0) - goto out; - - /* make sure they're correct */ - if (!extent_within(rem, &ext)) { - ret = -EIO; - goto out; - } - - ret = extent_delete(sb, iof, &ext, data); - if (ret) - goto out; - ins_ext = true; - - if (rem->start != ext.start) { - scoutfs_extent_init(&left, ext.type, ext.owner, - ext.start, rem->start - ext.start, - ext.map, ext.flags); - ret = extent_insert(sb, iof, &left, data); - if (ret) - goto out; - del_left = true; - } - - if (extent_end(rem) != extent_end(&ext)) { - scoutfs_extent_init(&right, ext.type, ext.owner, - rem->start + rem->len, - extent_end(&ext) - extent_end(rem), - ext.map ? rem->map + rem->len : 0, - ext.flags); - ret = extent_insert(sb, iof, &right, data); - } - -out: - scoutfs_extent_cleanup(ret < 0 && del_left, extent_delete, sb, iof, - &left, data, SC_EXTENT_REM_CLEANUP, - corrupt_extent_rem_cleanup, rem); - scoutfs_extent_cleanup(ret < 0 && ins_ext, extent_insert, sb, iof, - &ext, data, SC_EXTENT_REM_CLEANUP, - corrupt_extent_rem_cleanup, rem); - return ret; -} diff --git a/kmod/src/extents.h b/kmod/src/extents.h deleted file mode 100644 index 478892a8..00000000 --- a/kmod/src/extents.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef _SCOUTFS_EXTENTS_H_ -#define _SCOUTFS_EXTENTS_H_ - -/* - * Native storage for an extent. Read and write translates between - * these and persistent storage. - */ -struct scoutfs_extent { - u64 owner; - u64 start; - u64 len; - u64 map; - u8 type; - u8 flags; -}; - -#define SE_FMT "%llu.%llu.%llu.%llu.%u.%x" -#define SE_ARG(ext) (ext)->owner, (ext)->start, (ext)->len, (ext)->map, \ - (ext)->type, (ext)->flags - -#define se_trace_define(name) \ - __field(__u64, name##_owner) \ - __field(__u64, name##_start) \ - __field(__u64, name##_len) \ - __field(__u64, name##_map) \ - __field(__u8, name##_type) \ - __field(__u8, name##_flags) - -/* doesn't support null extent pointers */ -#define se_trace_assign(name, ext) \ -do { \ - __typeof__(ext) _ext = (ext); \ - \ - __entry->name##_owner = _ext->owner; \ - __entry->name##_start = _ext->start; \ - __entry->name##_len = _ext->len; \ - __entry->name##_map = _ext->map; \ - __entry->name##_type = _ext->type; \ - __entry->name##_flags = _ext->flags; \ -} while (0) - -#define se_trace_args(name) \ - __entry->name##_owner, __entry->name##_start, __entry->name##_len, \ - __entry->name##_map, __entry->name##_type, __entry->name##_flags - -enum { - SEI_NEXT, - SEI_PREV, - SEI_INSERT, - SEI_DELETE, -}; -typedef int (*scoutfs_extent_io_t)(struct super_block *sb, int op, - struct scoutfs_extent *ext, void *data); - -int scoutfs_extent_init(struct scoutfs_extent *ext, u8 type, u64 owner, - u64 start, u64 len, u64 map, u8 flags); -bool scoutfs_extent_intersection(struct scoutfs_extent *a, - struct scoutfs_extent *b); - -int scoutfs_extent_next(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ext, void *data); -int scoutfs_extent_prev(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *ext, void *data); -int scoutfs_extent_add(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *add, void *data); -int scoutfs_extent_remove(struct super_block *sb, scoutfs_extent_io_t iof, - struct scoutfs_extent *rem, void *data); - -/* - * The process of modifying an extent creates and deletes many - * intermediate extents. If we hit an error we need to undo the - * process. If we then hit an error we can be left with inconsistent - * extent items. - * - * We could fix this for extents that are stored in the item cache - * because it has tools for ensuring that operations can't fail. - * Extents that are stored in the btree currently can't avoid errors. - * We'd have to predirty blocks, allow deletion to fall below thresholds - * if merging saw an error, and preallocate blocks to be used for - * splitting/growth. It'd probably be worth it. - */ -#define scoutfs_extent_cleanup(cond, ext_func, sb, iof, clean, data, \ - which, ctr, ext) \ -do { \ - __typeof__(sb) _sb = (sb); \ - int _ret; \ - \ - if ((cond) && (_ret = ext_func(_sb, iof, clean, data)) < 0) \ - scoutfs_corruption(_sb, which, ctr, \ - "ext "SE_FMT" clean "SE_FMT" ret %d", \ - SE_ARG(ext), SE_ARG(clean), _ret); \ -} while (0) - -#endif From 55fa73f407535120fe7dd1d447df2c09bb3d0352 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Dec 2019 15:42:31 -0800 Subject: [PATCH 770/920] scoutfs: add packed extent and bitmap tracing Signed-off-by: Zach Brown --- kmod/src/data.c | 8 ++++- kmod/src/scoutfs_trace.h | 65 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 1d3c29e0..802e7d47 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1085,6 +1085,10 @@ static int alloc_blocks(struct super_block *sb, u64 count, u64 *blkno_ret, *blkno_ret = blkno; *count_ret = count; + + trace_scoutfs_data_alloc_blocks(sb, broot, bb->base, bb->type, bit, + blkno, count); + out: kfree(bb); return ret; @@ -1113,8 +1117,10 @@ static int free_blocks(struct super_block *sb, &broot->root, SCOUTFS_BLOCK_BITMAP_LITTLE, blkno, count); - if (ret == 0) + if (ret == 0) { le64_add_cpu(&broot->total_free, count); + trace_scoutfs_data_free_blocks(sb, broot, blkno, count); + } return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 712ab69b..33dbd9e4 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -118,6 +118,71 @@ TRACE_EVENT(scoutfs_complete_truncate, __entry->flags) ); +TRACE_EVENT(scoutfs_data_alloc_blocks, + TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, + u64 base, u8 type, int bit, u64 blkno, u64 count), + + TP_ARGS(sb, broot, base, type, bit, blkno, count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, root_seq) + __field(__u64, root_total_free) + __field(__u64, base) + __field(u8, type) + __field(int, bit) + __field(__u64, blkno) + __field(__u64, count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); + __entry->root_seq = le64_to_cpu(broot->root.ref.seq); + __entry->root_total_free = le64_to_cpu(broot->total_free); + __entry->base = base; + __entry->type = type; + __entry->bit = bit; + __entry->blkno = blkno; + __entry->count = count; + ), + + TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu base %llu type %u bit %d blkno %llu count %llu\n", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, + __entry->root_total_free, __entry->base, __entry->type, + __entry->bit, __entry->blkno, __entry->count) +); + +TRACE_EVENT(scoutfs_data_free_blocks, + TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, + u64 blkno, u64 count), + + TP_ARGS(sb, broot, blkno, count), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, root_seq) + __field(__u64, root_total_free) + __field(__u64, blkno) + __field(__u64, count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); + __entry->root_seq = le64_to_cpu(broot->root.ref.seq); + __entry->root_total_free = le64_to_cpu(broot->total_free); + __entry->blkno = blkno; + __entry->count = count; + ), + + TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu blkno %llu count %llu\n", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, + __entry->root_total_free, __entry->blkno, __entry->count) +); + TRACE_EVENT(scoutfs_data_fallocate, TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, loff_t len, int ret), From 3978bbd23f0582bf2e6775bdc855d2e42bb30c6d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Dec 2019 16:14:45 -0800 Subject: [PATCH 771/920] scoutfs: have xattr use max val size The xattr code had a static defintion of the largest part item that it would create. Change it to be a function of the largest fs item value that can be created and clean up the code a bit in the process. Signed-off-by: Zach Brown --- kmod/src/count.h | 2 +- kmod/src/format.h | 4 ++-- kmod/src/xattr.c | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 97d56ef5..b2dd8152 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -225,7 +225,7 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, cnt.items++; if (creating) { - new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size) + new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size); cnt.items += new_parts; cnt.vals += sizeof(struct scoutfs_xattr) + name_len + size; diff --git a/kmod/src/format.h b/kmod/src/format.h index 9b9fd06d..d801c40d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -611,11 +611,11 @@ enum { #define SCOUTFS_XATTR_MAX_NAME_LEN 255 #define SCOUTFS_XATTR_MAX_VAL_LEN 65535 -#define SCOUTFS_XATTR_MAX_PART_SIZE 512U +#define SCOUTFS_XATTR_MAX_PART_SIZE SCOUTFS_MAX_VAL_SIZE #define SCOUTFS_XATTR_NR_PARTS(name_len, val_len) \ DIV_ROUND_UP(sizeof(struct scoutfs_xattr) + name_len + val_len, \ - SCOUTFS_XATTR_MAX_PART_SIZE); + (unsigned int)SCOUTFS_XATTR_MAX_PART_SIZE) #define SCOUTFS_LOCK_INODE_GROUP_NR 1024 #define SCOUTFS_LOCK_INODE_GROUP_MASK (SCOUTFS_LOCK_INODE_GROUP_NR - 1) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index da109367..4dbe9900 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -270,8 +270,8 @@ static int create_xattr_items(struct inode *inode, u64 id, struct super_block *sb = inode->i_sb; struct scoutfs_key key; unsigned int part_bytes; + unsigned int total; struct kvec val; - int total; int ret; init_xattr_key(&key, scoutfs_ino(inode), @@ -280,7 +280,8 @@ static int create_xattr_items(struct inode *inode, u64 id, total = 0; ret = 0; while (total < bytes) { - part_bytes = min(bytes - total, SCOUTFS_XATTR_MAX_PART_SIZE); + part_bytes = min_t(unsigned int, bytes - total, + SCOUTFS_XATTR_MAX_PART_SIZE); kvec_init(&val, (void *)xat + total, part_bytes); ret = scoutfs_forest_create(sb, &key, &val, lock); From 587120830d43104b4a628ea05599f2af578eaf07 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 17 Dec 2019 10:21:42 -0800 Subject: [PATCH 772/920] scoutfs: initialize transaction block writer As we shut down the transaction tries to destroy any remaining dirty blocks in its writer context. The block writer context was only initialized by the client as it asked the server for the log trees. This makes sure the writer is always initialized. Signed-off-by: Zach Brown --- kmod/src/trans.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 679ef1b1..6bd4c218 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -508,6 +508,7 @@ int scoutfs_setup_trans(struct super_block *sb) return -ENOMEM; spin_lock_init(&tri->lock); + scoutfs_block_writer_init(sb, &tri->wri); sbi->trans_write_workq = alloc_workqueue("scoutfs_trans", WQ_UNBOUND, 1); From 85178efa194defa069c9904f26d5c730dbbbef2d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 18 Dec 2019 10:20:57 -0800 Subject: [PATCH 773/920] scoutfs: add more forest tracing Signed-off-by: Zach Brown --- kmod/src/forest.c | 5 +++++ kmod/src/scoutfs_trace.h | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index ac49459c..3dce5f32 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -298,6 +298,8 @@ static int refresh_bloom_roots(struct super_block *sb, if (ret) goto out; + trace_scoutfs_forest_read_super(sb, &super); + srefs->fs_ref = super.fs_root.ref; srefs->logs_ref = super.logs_root.ref; @@ -1353,6 +1355,9 @@ void scoutfs_forest_get_btrees(struct super_block *sb, lt->item_root = finf->our_log.item_root; lt->bloom_ref = finf->our_log.bloom_ref; + + trace_scoutfs_forest_prepare_commit(sb, <->item_root.ref, + <->bloom_ref); } int scoutfs_forest_setup(struct super_block *sb) diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 33dbd9e4..56e34ae2 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2006,6 +2006,53 @@ DEFINE_EVENT(scoutfs_forest_bloom_class, scoutfs_forest_bloom_search, TP_ARGS(sb, key, rid, nr, blkno, seq, count) ); +TRACE_EVENT(scoutfs_forest_prepare_commit, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_ref *item_ref, + struct scoutfs_btree_ref *bloom_ref), + TP_ARGS(sb, item_ref, bloom_ref), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, item_blkno) + __field(__u64, item_seq) + __field(__u64, bloom_blkno) + __field(__u64, bloom_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->item_blkno = le64_to_cpu(item_ref->blkno); + __entry->item_seq = le64_to_cpu(item_ref->seq); + __entry->bloom_blkno = le64_to_cpu(bloom_ref->blkno); + __entry->bloom_seq = le64_to_cpu(bloom_ref->seq); + ), + TP_printk(SCSBF" item blkno %llu seq %llu bloom blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->item_blkno, __entry->item_seq, + __entry->bloom_blkno, __entry->bloom_seq) +); + +TRACE_EVENT(scoutfs_forest_read_super, + TP_PROTO(struct super_block *sb, struct scoutfs_super_block *super), + TP_ARGS(sb, super), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, hdr_seq) + __field(__u64, fs_blkno) + __field(__u64, fs_seq) + __field(__u64, logs_blkno) + __field(__u64, logs_seq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->hdr_seq = le64_to_cpu(super->hdr.seq); + __entry->fs_blkno = le64_to_cpu(super->fs_root.ref.blkno); + __entry->fs_seq = le64_to_cpu(super->fs_root.ref.seq); + __entry->logs_blkno = le64_to_cpu(super->logs_root.ref.blkno); + __entry->logs_seq = le64_to_cpu(super->logs_root.ref.seq); + ), + TP_printk(SCSBF" hdr seq %llu fs blkno %llu seq %llu logs blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->hdr_seq, __entry->fs_blkno, + __entry->fs_seq, __entry->logs_blkno, __entry->logs_seq) +); + TRACE_EVENT(scoutfs_forest_add_root, TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, u64 nr, u64 blkno, u64 seq), From e034ffa7e9ce27cecf8c1d7c56e87a867f2ee23a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 18 Dec 2019 15:44:46 -0800 Subject: [PATCH 774/920] scoutfs: fix forest iteration The forest item iterator was missing items. Picture the following search pattern: - find a candidate item to return in a root - ignore a greater candidate to return in another root - find the first candidates item's deletion in another root The problem was that finding the deletion item didn't reset the notion that we'd found a key. The next item from the second root was never used because the found key wasn't reset and that root had already searched past the found key. The core architectural problem is that iteration can't examine each item only once given that keys and deletions can be randomly distributed across the roots. The most efficient way to solve the problem is to really sort the iteration positions in each root and then walk those in order. We get the right answer and pay some data structure overhead to perform the minimum number of btree searches. Signed-off-by: Zach Brown --- kmod/src/forest.c | 260 ++++++++++++++++++++++++++++----------- kmod/src/scoutfs_trace.h | 31 +---- 2 files changed, 191 insertions(+), 100 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 3dce5f32..f16ee31a 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -517,10 +517,10 @@ static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, * needs to be skipped. */ static int copy_val(struct forest_lock_private *lpriv, struct forest_root *fr, - struct kvec *val, struct scoutfs_btree_item_ref *iref) + struct kvec *val, void *item_val, int item_val_len) { - void *val_start = iref->val; - unsigned int val_len = iref->val_len; + void *val_start = item_val; + unsigned int val_len = item_val_len; int ret; if (!is_fs_root(lpriv, fr)) { @@ -594,7 +594,8 @@ retry: if (item_is_deletion(lpriv, fr, iref.val)) ret = -ENOENT; else - ret = copy_val(lpriv, fr, val, &iref); + ret = copy_val(lpriv, fr, val, + iref.val, iref.val_len); } scoutfs_btree_put_iref(&iref); read_unlock_forest_root(finf, lpriv, fr); @@ -652,6 +653,15 @@ static inline void forest_iter_key_advance(struct scoutfs_key *key, bool forward scoutfs_key_dec(key); } +static inline int forest_iter_key_cmp(struct scoutfs_key *a, + struct scoutfs_key *b, bool forward) +{ + int cmp = scoutfs_key_compare(a, b); + if (cmp == 0 || forward) + return cmp; + return -cmp; +} + /* returns true if a is before b in the direction of iteration */ static inline bool forest_iter_key_before(struct scoutfs_key *a, struct scoutfs_key *b, bool forward) @@ -683,19 +693,114 @@ static inline int forest_iter_btree_search(struct super_block *sb, } struct forest_iter_pos { - struct list_head entry; + struct rb_node node; struct forest_root *fr; - struct scoutfs_key pos; + struct scoutfs_key key; + u64 vers; + bool deletion; + void *val; + int val_len; }; +static struct forest_iter_pos *first_iter_pos(struct rb_root *root) +{ + return rb_entry_safe(rb_first(root), struct forest_iter_pos, node); +} + +static struct forest_iter_pos *next_iter_pos(struct forest_iter_pos *ip) +{ + return rb_entry_safe(rb_next(&ip->node), struct forest_iter_pos, node); +} + +/* + * Sort root iter positions first by missing items, then by key in the + * direction if iteration, and then by reverse version. Thus the first + * iter_pos in the rbtree is either a root that needs to check the next + * item, a deletion that removes all older versions of the key, or is + * the item that iteration should return. + */ +static int cmp_iter_pos(struct forest_iter_pos *a, struct forest_iter_pos *b, + bool fwd) +{ + int cmp; + + if (a->vers == 0) + return -1; + if (b->vers == 0) + return 1; + + cmp = forest_iter_key_cmp(&a->key, &b->key, fwd); + if (cmp) + return cmp; + + return scoutfs_cmp_u64s(b->vers, a->vers); +} + +/* + * There's a sneaky subtlety here. The fs items have a fake verison of + * 1 which can equal a log tree version of 1. We always iterate over + * the fs root last so we try to insert the fake fs item last. It will + * compare equal to the version and will be inserted to the right of the + * existing log item. + */ +static void insert_iter_pos(struct forest_iter_pos *ins, struct rb_root *root, + bool fwd) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct forest_iter_pos *ip; + int cmp; + + while (*node) { + parent = *node; + ip = container_of(*node, struct forest_iter_pos, node); + + cmp = cmp_iter_pos(ins, ip, fwd); + if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); +} + +/* + * clear the version and re-insert the iter_pos so that the next + * iteration will search for the next item in the root. + */ +static void advance_iter_pos(struct forest_iter_pos *ip, struct rb_root *root, + bool fwd) +{ + ip->vers = 0; + forest_iter_key_advance(&ip->key, fwd); + kfree(ip->val); + ip->val = NULL; + rb_erase(&ip->node, root); + insert_iter_pos(ip, root, fwd); +} + +static void destroy_iter_pos(struct forest_iter_pos *ip, struct rb_root *root) +{ + kfree(ip->val); + rb_erase(&ip->node, root); + kfree(ip); +} + /* * Iterate over items in all the roots looking for the next least * non-deletion item in the direction of iteration. The roots can have - * any mix of deletion items and item versions. As we iterate we record - * the non-deletion item we've seen with the earliest key and the - * greatest version of that specific key. We record iteration positions - * in all btrees and we know we've finished once we've found a possible - * item to return and all the btrees have checked up to that position. + * any combination of item keys, versions, and deletions so we have to + * be very careful. + * + * We store the next item in each root in a node in an rbtree. The + * nodes are sorted by needing to be read, key, then reverse version. + * The first node in the rbtree is always a root to search, a deletion + * item to remove, or the item that iteration should return. + * + * btree locking prevents us from holding references to the items in all + * the roots so we store copies of the items in the nodes. */ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *end, struct kvec *val, @@ -705,21 +810,17 @@ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, struct forest_lock_private *lpriv; DECLARE_FOREST_INFO(sb, finf); SCOUTFS_BTREE_ITEM_REF(iref); - struct forest_iter_pos *tmp; - struct forest_iter_pos *ip; + struct rb_root iter_root = RB_ROOT; + struct scoutfs_key found_key; struct scoutfs_key_be kbe; - struct scoutfs_key found; + struct forest_iter_pos *nip; + struct forest_iter_pos *ip; struct forest_root *fr; - LIST_HEAD(list); - int found_copied; - u64 found_vers; - u64 vers; + u64 found_vers = 0; + int found_ret = 0; int ret; - scoutfs_key_set_zeros(&found); - found_copied = 0; - found_vers = 0; - ret = 0; + scoutfs_key_set_zeros(&found_key); if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) goto out; @@ -743,7 +844,7 @@ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, retry: down_read(&lpriv->rwsem); - /* track iteration position in each btree */ + /* initialize iter position for each tree */ fr = NULL; while (!(ret = for_each_forest_root(lock, lpriv, &fr)) && fr) { ip = kmalloc(sizeof(struct forest_iter_pos), GFP_NOFS); @@ -753,37 +854,26 @@ retry: } ip->fr = fr; - forest_iter_set_min(&ip->pos, fwd); - list_add_tail(&ip->entry, &list); + ip->key = *key; + ip->vers = 0; + ip->deletion = false; + ip->val = NULL; + insert_iter_pos(ip, &iter_root, fwd); } if (ret < 0) goto unlock; - forest_iter_set_max(&found, fwd); + scoutfs_key_set_zeros(&found_key); found_vers = 0; - found_copied = 0; + found_ret = -ENOENT; - /* check each tree until they've all searched up to found */ - while (!list_empty(&list)) { - list_for_each_entry_safe(ip, tmp, &list, entry) { - fr = ip->fr; + /* search until we hit the end key on all roots */ + while ((ip = first_iter_pos(&iter_root))) { + fr = ip->fr; - trace_scoutfs_forest_iter_search(sb, fr->rid, fr->nr, - &ip->pos); - - /* remove once we can't contain any more items */ - if (!forest_iter_key_before(&ip->pos, &found, fwd) || - !forest_iter_key_within(&ip->pos, end, fwd)) { - list_del(&ip->entry); - kfree(ip); - continue; - } - - /* iter pos key is only set after searching */ - if (forest_iter_key_before(&ip->pos, key, fwd)) - scoutfs_key_to_be(&kbe, key); - else - scoutfs_key_to_be(&kbe, &ip->pos); + /* search for the next item in the root */ + if (ip->vers == 0) { + scoutfs_key_to_be(&kbe, &ip->key); read_lock_forest_root(finf, lpriv, fr); ret = forest_iter_btree_search(sb, &fr->item_root, @@ -792,45 +882,69 @@ retry: if (ret < 0) read_unlock_forest_root(finf, lpriv, fr); if (ret == -ENOENT) { - forest_iter_set_max(&ip->pos, fwd); + destroy_iter_pos(ip, &iter_root); continue; } if (ret < 0) goto unlock; - scoutfs_key_from_be(&ip->pos, iref.key); - vers = item_vers(lpriv, fr, iref.val); + scoutfs_key_from_be(&ip->key, iref.key); + ip->vers = item_vers(lpriv, fr, iref.val); + ip->deletion = item_is_deletion(lpriv, fr, iref.val); - trace_scoutfs_forest_iter_found(sb, fr->rid, fr->nr, - vers, + trace_scoutfs_forest_iter_search(sb, fr->rid, fr->nr, + ip->vers, item_flags(lpriv, fr, iref.val), - &ip->pos); + &ip->key); - /* record next earliest item and copy to caller */ - if (!item_is_deletion(lpriv, fr, iref.val) && - forest_iter_key_within(&ip->pos, end, fwd) && - (forest_iter_key_before(&ip->pos, &found, fwd) || - (scoutfs_key_compare(&ip->pos, &found) == 0 && - vers > found_vers))) { - - found = ip->pos; - found_vers = vers; - found_copied = copy_val(lpriv, fr, val, &iref); + if (!forest_iter_key_within(&ip->key, end, fwd)) { + /* root is done if next is past end */ + destroy_iter_pos(ip, &iter_root); + } else { + kfree(ip->val); + ip->val = kmalloc(iref.val_len, GFP_NOFS); + if (!ip->val) { + ret = -ENOMEM; + } else { + /* copy item and re-sort its node */ + memcpy(ip->val, iref.val, iref.val_len); + ip->val_len = iref.val_len; + rb_erase(&ip->node, &iter_root); + insert_iter_pos(ip, &iter_root, fwd); + } } + scoutfs_btree_put_iref(&iref); read_unlock_forest_root(finf, lpriv, fr); - forest_iter_key_advance(&ip->pos, fwd); + if (ret < 0) + goto unlock; + continue; } + + /* deletions remove all earlier versions and themselves */ + if (ip->deletion) { + while ((nip = next_iter_pos(ip)) && + !scoutfs_key_compare(&ip->key, &nip->key)) { + advance_iter_pos(nip, &iter_root, fwd); + } + advance_iter_pos(ip, &iter_root, fwd); + continue; + } + + /* use the first non-deletion across all roots */ + found_key = ip->key; + found_vers = ip->vers; + found_ret = copy_val(lpriv, ip->fr, val, ip->val, ip->val_len); + break; } ret = 0; unlock: up_read(&lpriv->rwsem); - list_for_each_entry_safe(ip, tmp, &list, entry) { - list_del(&ip->entry); - kfree(ip); + rbtree_postorder_for_each_entry_safe(ip, nip, &iter_root, node) { + destroy_iter_pos(ip, &iter_root); } if (ret == -ESTALE) { @@ -841,15 +955,13 @@ unlock: out: trace_scoutfs_forest_iter_ret(sb, key, end, fwd, ret, - found_vers, found_copied, &found); + found_vers, found_ret, &found_key); + if (ret == 0) { + ret = found_ret; /* _next/_prev interfaces modify caller's key :/ */ - if (found_vers > 0) { - *key = found; - ret = found_copied; - } else { - ret = -ENOENT; - } + if (ret >= 0) + *key = found_key; } return ret; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 56e34ae2..5f8db8a3 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2079,27 +2079,6 @@ TRACE_EVENT(scoutfs_forest_add_root, ); TRACE_EVENT(scoutfs_forest_iter_search, - TP_PROTO(struct super_block *sb, u64 rid, u64 nr, - struct scoutfs_key *pos), - TP_ARGS(sb, rid, nr, pos), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, b_rid) - __field(__u64, nr) - sk_trace_define(pos) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->b_rid = rid; - __entry->nr = nr; - sk_trace_assign(pos, pos); - ), - TP_printk(SCSBF" rid %016llx nr %llu pos "SK_FMT, - SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, - sk_trace_args(pos)) -); - -TRACE_EVENT(scoutfs_forest_iter_found, TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 vers, u8 flags, struct scoutfs_key *key), TP_ARGS(sb, rid, nr, vers, flags, key), @@ -2127,8 +2106,8 @@ TRACE_EVENT(scoutfs_forest_iter_found, TRACE_EVENT(scoutfs_forest_iter_ret, TP_PROTO(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *end, bool forward, int ret, - u64 found_vers, int found_copied, struct scoutfs_key *found), - TP_ARGS(sb, key, end, forward, ret, found_vers, found_copied, found), + u64 found_vers, int found_ret, struct scoutfs_key *found), + TP_ARGS(sb, key, end, forward, ret, found_vers, found_ret, found), TP_STRUCT__entry( SCSB_TRACE_FIELDS sk_trace_define(key) @@ -2136,7 +2115,7 @@ TRACE_EVENT(scoutfs_forest_iter_ret, __field(char, forward) __field(int, ret) __field(__u64, found_vers) - __field(int, found_copied) + __field(int, found_ret) sk_trace_define(found) ), TP_fast_assign( @@ -2146,13 +2125,13 @@ TRACE_EVENT(scoutfs_forest_iter_ret, __entry->forward = !!forward; __entry->ret = ret; __entry->found_vers = found_vers; - __entry->found_copied = found_copied; + __entry->found_ret = found_ret; sk_trace_assign(found, found); ), TP_printk(SCSBF" key "SK_FMT" end "SK_FMT" fwd %u ret %d fv %llu fc %d f "SK_FMT, SCSB_TRACE_ARGS, sk_trace_args(key), sk_trace_args(end), __entry->forward, __entry->ret, __entry->found_vers, - __entry->found_copied, sk_trace_args(found)) + __entry->found_ret, sk_trace_args(found)) ); DECLARE_EVENT_CLASS(scoutfs_block_class, From 5ed1cb3aaf4a21fa4f1c107dddffe6a0579adc65 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 17 Jan 2020 11:14:38 -0800 Subject: [PATCH 775/920] scoutfs: remove LSM from README.md Update the summary of the benefit we get from concurrent per-mount commits. Instead of describing it specifically in terms of LSM we abstract it out a bit to make it also true of writing per-mount log btrees. Signed-off-by: Zach Brown --- kmod/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kmod/README.md b/kmod/README.md index 5152a124..2b249c6f 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -5,9 +5,8 @@ from the ground up to support large archival systems. Its key differentiating features are: - - Integrated consistent indexing to accelerate archival maintenance operations - - Shared LSM index structure to scale metadata rates with storage bandwidth - - Decoupled logical locking from serialized device writes to reduce contention + - Integrated consistent indexing accelerates archival maintenance operations + - Log-structured commits allow nodes to write concurrently without contention It meets best of breed expectations: From 10fd4fcec0395839b49b828388c28e1e75046350 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Feb 2020 13:48:28 -0800 Subject: [PATCH 776/920] scoutfs: verify read bloom block ref The bloom block reading code forgot to test if the read block was stale. It would trust whatever it read. Now the read when building up roots to use can return stale and retry. Signed-off-by: Zach Brown --- kmod/src/forest.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index f16ee31a..64e4a49e 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -249,6 +249,24 @@ static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, } } +static struct scoutfs_block *read_bloom_ref(struct super_block *sb, + struct scoutfs_btree_ref *ref) +{ + struct scoutfs_block *bl; + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (IS_ERR(bl)) + return bl; + + if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_BLOOM)) { + scoutfs_block_put(sb, bl); + return ERR_PTR(-ESTALE); + } + + return bl; +} + /* * Empty the list of btrees currently stored in the lock and walk the * current fs image looking for btrees whose bloom filters indicate that @@ -331,7 +349,7 @@ static int refresh_bloom_roots(struct super_block *sb, if (ltv.bloom_ref.blkno == 0) continue; - bl = scoutfs_block_read(sb, le64_to_cpu(ltv.bloom_ref.blkno)); + bl = read_bloom_ref(sb, <v.bloom_ref); if (IS_ERR(bl)) { ret = PTR_ERR(bl); goto out; @@ -1098,6 +1116,7 @@ out: static int set_lock_bloom_bits(struct super_block *sb, struct scoutfs_lock *lock) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_FOREST_INFO(sb, finf); struct forest_lock_private *lpriv; struct scoutfs_block *new_bl = NULL; @@ -1129,7 +1148,7 @@ static int set_lock_bloom_bits(struct super_block *sb, ref = &finf->our_log.bloom_ref; if (ref->blkno) { - bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + bl = read_bloom_ref(sb, ref); if (IS_ERR(bl)) { ret = PTR_ERR(bl); goto unlock; @@ -1170,6 +1189,7 @@ static int set_lock_bloom_bits(struct super_block *sb, new_bl = NULL; bb->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_BLOOM); + bb->hdr.fsid = super->hdr.fsid; bb->hdr.blkno = cpu_to_le64(blkno); prandom_bytes(&bb->hdr.seq, sizeof(bb->hdr.seq)); ref->blkno = bb->hdr.blkno; From 05a8573054b3ef2047a2a19e2a5978c3b55e1b29 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 20 Feb 2020 13:41:03 -0800 Subject: [PATCH 777/920] scoutfs: add block visited bit Add functions for callers to maintain a visited bit in cached blocks. The radix allocator is going to use this to count the number of clean blocks it sees across paths through the radix which can share parent blocks. Signed-off-by: Zach Brown --- kmod/src/block.c | 17 +++++++++++++++++ kmod/src/block.h | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 18f3b37e..641d9c3d 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -67,6 +67,7 @@ enum { BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ BLOCK_BIT_VIRT, /* mapped virt allocation */ BLOCK_BIT_CRC_VALID, /* crc has been verified */ + BLOCK_BIT_VISITED, /* used by callers to track blocks */ }; struct block_private { @@ -128,6 +129,22 @@ bool scoutfs_block_valid_ref(struct super_block *sb, hdr->blkno == blkno; } +bool scoutfs_block_tas_visited(struct super_block *sb, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + return test_bit(BLOCK_BIT_VISITED, &bp->bits) != 0; +} + +void scoutfs_block_clear_visited(struct super_block *sb, + struct scoutfs_block *bl) +{ + struct block_private *bp = BLOCK_PRIVATE(bl); + + clear_bit(BLOCK_BIT_VISITED, &bp->bits); +} + static struct block_private *block_alloc(struct super_block *sb, u64 blkno) { struct block_private *bp; diff --git a/kmod/src/block.h b/kmod/src/block.h index dbb0d54c..abcdd440 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -17,6 +17,10 @@ bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr); bool scoutfs_block_valid_ref(struct super_block *sb, struct scoutfs_block_header *hdr, __le64 seq, __le64 blkno); +bool scoutfs_block_tas_visited(struct super_block *sb, + struct scoutfs_block *bl); +void scoutfs_block_clear_visited(struct super_block *sb, + struct scoutfs_block *bl); struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno); struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); From 809d4be58e44338487cfda8514c0f1944293680c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 20 Feb 2020 13:59:03 -0800 Subject: [PATCH 778/920] scoutfs: switch block cache to rbtree Switch the block cache from indexing blocks in a radix tree to using an rbtree. We lose the RCU lookups but we gain being able to move blocks around in the cache without allocation failure. And we no longer have the problem of not being able to index large blocks with a 32bit long radix key. Signed-off-by: Zach Brown --- kmod/src/block.c | 90 +++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 641d9c3d..8249a3d6 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "format.h" #include "super.h" @@ -45,7 +46,7 @@ struct block_info { struct super_block *sb; spinlock_t lock; - struct radix_tree_root radix; + struct rb_root root; struct list_head lru_list; u64 lru_nr; u64 lru_move_counter; @@ -63,7 +64,7 @@ enum { BLOCK_BIT_NEW, /* newly allocated, contents undefined */ BLOCK_BIT_DIRTY, /* dirty, writer will write */ BLOCK_BIT_ERROR, /* saw IO error */ - BLOCK_BIT_DELETED, /* has been deleted from radix tree */ + BLOCK_BIT_DELETED, /* has been deleted from rbtree */ BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ BLOCK_BIT_VIRT, /* mapped virt allocation */ BLOCK_BIT_CRC_VALID, /* crc has been verified */ @@ -72,6 +73,7 @@ enum { struct block_private { struct scoutfs_block bl; + struct rb_node node; struct super_block *sb; atomic_t refcount; union { @@ -180,6 +182,7 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) } bp->bl.blkno = blkno; + RB_CLEAR_NODE(&bp->node); bp->sb = sb; atomic_set(&bp->refcount, 1); INIT_LIST_HEAD(&bp->lru_entry); @@ -249,9 +252,39 @@ static void block_put(struct super_block *sb, struct block_private *bp) } } +static struct block_private *walk_block_rbtree(struct rb_root *root, + u64 blkno, + struct block_private *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct block_private *bp; + int cmp; + + while (*node) { + parent = *node; + bp = container_of(*node, struct block_private, node); + + cmp = scoutfs_cmp_u64s(bp->bl.blkno, blkno); + if (cmp == 0) + return bp; + else if (cmp < 0) + node = &(*node)->rb_left; + else + node = &(*node)->rb_right; + } + + if (ins) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, root); + return ins; + } + + return NULL; +} + /* - * Add a new block into the cache. The caller holds the lock and has - * preloaded the radix. + * Add a new block into the cache. The caller holds the lock. */ static void block_insert(struct super_block *sb, struct block_private *bp, u64 blkno) @@ -260,9 +293,10 @@ static void block_insert(struct super_block *sb, struct block_private *bp, assert_spin_locked(&binf->lock); BUG_ON(!list_empty(&bp->lru_entry)); + BUG_ON(!RB_EMPTY_NODE(&bp->node)); atomic_inc(&bp->refcount); - radix_tree_insert(&binf->radix, blkno, bp); + walk_block_rbtree(&binf->root, blkno, bp); list_add_tail(&bp->lru_entry, &binf->lru_list); bp->lru_moved = ++binf->lru_move_counter; binf->lru_nr++; @@ -310,11 +344,10 @@ static void block_remove(struct super_block *sb, struct block_private *bp) { DECLARE_BLOCK_INFO(sb, binf); - assert_spin_locked(&binf->lock); - if (!test_and_set_bit(BLOCK_BIT_DELETED, &bp->bits)) { BUG_ON(list_empty(&bp->lru_entry)); - radix_tree_delete(&binf->radix, bp->bl.blkno); + rb_erase(&bp->node, &binf->root); + RB_CLEAR_NODE(&bp->node); list_del_init(&bp->lru_entry); binf->lru_nr--; block_put(sb, bp); @@ -328,19 +361,18 @@ static void block_remove_all(struct super_block *sb) { DECLARE_BLOCK_INFO(sb, binf); struct block_private *bp; + struct rb_node *node; - spin_lock(&binf->lock); - - while (radix_tree_gang_lookup(&binf->radix, (void **)&bp, 0, 1) == 1) { + for (node = rb_first(&binf->root); node; ) { + bp = container_of(node, struct block_private, node); + node = rb_next(node); wait_event(binf->waitq, atomic_read(&bp->io_count) == 0); block_remove(sb, bp); } - spin_unlock(&binf->lock); - WARN_ON_ONCE(!list_empty(&binf->lru_list)); WARN_ON_ONCE(binf->lru_nr != 0); - WARN_ON_ONCE(binf->radix.rnode != NULL); + WARN_ON_ONCE(!RB_EMPTY_ROOT(&binf->root)); } /* @@ -457,8 +489,8 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, /* * Return a reference to a cached block in the system, allocating a new - * block if one isn't found in the radix. Its contents are undefined if - * it's newly allocated. + * block if one isn't found in the rbtree. Its contents are undefined + * if it's newly allocated. */ static struct block_private *block_get(struct super_block *sb, u64 blkno) { @@ -467,11 +499,11 @@ static struct block_private *block_get(struct super_block *sb, u64 blkno) struct block_private *bp; int ret; - rcu_read_lock(); - bp = radix_tree_lookup(&binf->radix, blkno); + spin_lock(&binf->lock); + bp = walk_block_rbtree(&binf->root, blkno, NULL); if (bp) atomic_inc(&bp->refcount); - rcu_read_unlock(); + spin_unlock(&binf->lock); /* drop failed reads that interrupted waiters abandoned */ if (bp && (test_bit(BLOCK_BIT_ERROR, &bp->bits) && @@ -490,20 +522,15 @@ static struct block_private *block_get(struct super_block *sb, u64 blkno) goto out; } - ret = radix_tree_preload(GFP_NOFS); - if (ret) - goto out; - - /* could use slot instead of lookup/insert */ + /* could refactor to insert in one walk */ spin_lock(&binf->lock); - found = radix_tree_lookup(&binf->radix, blkno); + found = walk_block_rbtree(&binf->root, blkno, NULL); if (found) { atomic_inc(&found->refcount); } else { block_insert(sb, bp, blkno); } spin_unlock(&binf->lock); - radix_tree_preload_end(); if (found) { block_put(sb, bp); @@ -850,17 +877,8 @@ int scoutfs_block_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct block_info *binf; - loff_t size; int ret; - /* we store blknos in longs in the radix */ - size = i_size_read(sb->s_bdev->bd_inode); - if ((size >> SCOUTFS_BLOCK_SHIFT) >= LONG_MAX) { - scoutfs_err(sb, "Cant reference all blocks in %llu byte device with %u bit long radix tree indexes", - size, BITS_PER_LONG); - return -EINVAL; - } - binf = kzalloc(sizeof(struct block_info), GFP_KERNEL); if (!binf) { ret = -ENOMEM; @@ -869,7 +887,7 @@ int scoutfs_block_setup(struct super_block *sb) binf->sb = sb; spin_lock_init(&binf->lock); - INIT_RADIX_TREE(&binf->radix, GFP_ATOMIC); /* insertion preloads */ + binf->root = RB_ROOT; INIT_LIST_HEAD(&binf->lru_list); init_waitqueue_head(&binf->waitq); binf->shrinker.shrink = block_shrink; From 8681f920e07fcf7a60c6dfe4db8621d796fe1f04 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 20 Feb 2020 14:01:06 -0800 Subject: [PATCH 779/920] scoutfs: add scoutfs_block_move Add a call to move a block's location in the cache without failure. The radix allocator is going to use this to dirty radix blocks while making atomic changes to multipls paths through multiple radix trees. Signed-off-by: Zach Brown --- kmod/src/block.c | 38 ++++++++++++++++++++++++++++++++++++++ kmod/src/block.h | 3 +++ kmod/src/scoutfs_trace.h | 5 +++++ 3 files changed, 46 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 8249a3d6..eb2c3b2e 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -802,6 +802,44 @@ void scoutfs_block_writer_forget(struct super_block *sb, } } +/* + * Change a cached block's location. We're careful to only change its + * position in the rbtree. If we find another block existing at the new + * location then we remove it from the cache and forget it if it was + * dirty. + */ +void scoutfs_block_move(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl, u64 blkno) +{ + DECLARE_BLOCK_INFO(sb, binf); + struct block_private *bp = BLOCK_PRIVATE(bl); + struct block_private *existing = NULL; + + spin_lock(&binf->lock); + + existing = walk_block_rbtree(&binf->root, blkno, NULL); + if (existing) { + /* only nesting of binf and wri locks */ + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) { + spin_lock(&wri->lock); + if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) + block_forget(sb, wri, bp); + spin_unlock(&wri->lock); + } + block_remove(sb, existing); + } + + rb_erase(&bp->node, &binf->root); + RB_CLEAR_NODE(&bp->node); + bp->bl.blkno = blkno; + walk_block_rbtree(&binf->root, blkno, bp); + + TRACE_BLOCK(move, bp); + + spin_unlock(&binf->lock); +} + /* * The caller has ensured that no more dirtying will take place. This * helps the caller avoid doing a bunch of work before calling into the diff --git a/kmod/src/block.h b/kmod/src/block.h index abcdd440..8a405f11 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -44,6 +44,9 @@ void scoutfs_block_writer_forget_all(struct super_block *sb, void scoutfs_block_writer_forget(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_block *bl); +void scoutfs_block_move(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_block *bl, u64 blkno); bool scoutfs_block_writer_has_dirty(struct super_block *sb, struct scoutfs_block_writer *wri); u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5f8db8a3..15fc7daa 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2191,6 +2191,11 @@ DEFINE_EVENT(scoutfs_block_class, scoutfs_block_invalidate, int refcount, int io_count, unsigned long bits, u64 lru_moved), TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) ); +DEFINE_EVENT(scoutfs_block_class, scoutfs_block_move, + TP_PROTO(struct super_block *sb, void *bp, u64 blkno, + int refcount, int io_count, unsigned long bits, u64 lru_moved), + TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) +); DEFINE_EVENT(scoutfs_block_class, scoutfs_block_mark_dirty, TP_PROTO(struct super_block *sb, void *bp, u64 blkno, int refcount, int io_count, unsigned long bits, u64 lru_moved), From 455a547e8e2f08e5e86a13aa639a472923bba911 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Feb 2020 16:00:31 -0800 Subject: [PATCH 780/920] scoutfs: add radix allocator Add the allocator that uses bits stored in the leaves of a cow radix. It'll replace two metadata and data allocators that were previously storing allocation bitmap fragments in btree items. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/format.h | 38 + kmod/src/radix.c | 1455 ++++++++++++++++++++++++++++++++++++++ kmod/src/radix.h | 43 ++ kmod/src/scoutfs_trace.h | 159 +++++ 5 files changed, 1696 insertions(+) create mode 100644 kmod/src/radix.c create mode 100644 kmod/src/radix.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 5772a9fc..4492cc52 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -28,6 +28,7 @@ scoutfs-y += \ options.o \ per_task.o \ quorum.o \ + radix.o \ scoutfs_trace.o \ server.o \ spbm.o \ diff --git a/kmod/src/format.h b/kmod/src/format.h index d801c40d..edb429b4 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -8,6 +8,7 @@ #define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b #define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d #define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 +#define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 /* * The super block and btree blocks are fixed 4k. @@ -132,6 +133,43 @@ struct scoutfs_key { #define skpe_base _sk_second #define skpe_part _sk_fourth +struct scoutfs_radix_block { + struct scoutfs_block_header hdr; + __le32 sm_first; + __le32 lg_first; + union { + struct scoutfs_radix_ref { + __le64 blkno; + __le64 seq; + __le64 sm_total; + __le64 lg_total; + } __packed refs[0]; + __le64 bits[0]; + } __packed; +} __packed; + +struct scoutfs_radix_root { + __u8 height; + __le64 next_find_bit; + struct scoutfs_radix_ref ref; +} __packed; + +#define SCOUTFS_RADIX_REFS \ + ((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_radix_block, refs[0])) /\ + sizeof(struct scoutfs_radix_ref)) + +/* 8 meg regions with 4k data blocks */ +#define SCOUTFS_RADIX_LG_SHIFT 11 +#define SCOUTFS_RADIX_LG_BITS (1 << SCOUTFS_RADIX_LG_SHIFT) +#define SCOUTFS_RADIX_LG_MASK (SCOUTFS_RADIX_LG_BITS - 1) + +/* round block bits down to a multiple of large ranges */ +#define SCOUTFS_RADIX_BITS \ + (((SCOUTFS_BLOCK_SIZE - \ + offsetof(struct scoutfs_radix_block, bits[0])) * 8) & \ + ~(__u64)SCOUTFS_RADIX_LG_MASK) +#define SCOUTFS_RADIX_BITS_BYTES (SCOUTFS_RADIX_BITS / 8) + /* * The btree still uses memcmp() to compare keys. We should fix that * before too long. diff --git a/kmod/src/radix.c b/kmod/src/radix.c new file mode 100644 index 00000000..981aae31 --- /dev/null +++ b/kmod/src/radix.c @@ -0,0 +1,1455 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "counters.h" +#include "block.h" +#include "radix.h" +#include "scoutfs_trace.h" + +/* + * scoutfs uses bitmap blocks in cow radix trees to allocate free + * blocks. We like the radix trees because their stable structure lets + * us easily build up the resources to make atomic changes, splice trees + * around, and they have a respectable storage overhead when they're + * highly fragmented. + * + * An allocator itself contains two trees: one for bits that were stable + * at the start of a transaction and are available to satisfy + * allocation, and one for bits that were freed during this transaction + * which describe stable referenced blocks and can't be re-used to + * satisfy allocations until the transaction is committed. + * + * Each allocator contains a mutex that protects its two trees. It's + * typical for callers to allocate by calling radix ops on one of the + * alloc trees while providing the allocator to manage allocation of the + * radix blocks themselves. This is safe. If the caller is operating + * on other radix trees outside of the allocator struct (data + * allocations, server manipulating client trees) it is responsible for + * locking these external trees. + * + * The trees are updated by making cow copies of modified blocks and + * writing them into free space. The system does have a use for walking + * the old stable version of a tree -- the server needs to merge stable + * freed space back into its dirty available allocator tree while + * avoiding new frees that are arriving as it cows blocks during the + * merge process. + * + * Allocations search for the next free bit from a cursor that's stored + * in the root of each tree. We track the next set parent ref or leaf + * bit in references to blocks to avoid searching entire blocks. + * + * The radix isn't always fully populated. References can contain + * blknos with 0 or ~0 to indicate that its referenced subtree is either + * entirely empty or full. The counters that describe these stubbed out + * subtrees will be correct as though all the blocks were populated. + * Traversal instantiates initialized empty or full blocks as it + * descends. This lets mkfs initialize a tree with a large contigious + * set region without having to populate all its blocks. + * + * The radix is used to allocate and free blocks when performing cow + * updates of the blocks that make up radix itself. Recursion is + * carefully avoided by building up references to all the blocks needed + * for the operation and then dirtying and modifying them all at once. + * + * Radix block references contain totals of bits set in its referenced + * subtree. This helps us balance the number of free bits stored across + * multiple trees. + * + * The radix tracks large aligned regions of set bits that are used to + * satisfy larger data extent allocations. These large regions are also + * tracked in the metadata allocator trees but aren't used. + */ + +/* + * We create temporary synthetic blocks past possible blocks to populate + * stubbed out refs that reference entirely empty or full subtrees. + * They're moved to properly allocated blknos. + */ +#define RADIX_SYNTH_BLKNO (SCOUTFS_BLOCK_MAX + 1) + +struct radix_path { + struct rb_node node; + struct list_head head; + struct list_head alloc_head; + u8 height; + struct scoutfs_radix_root *root; + u64 leaf_bit; + /* path and index arrays indexed by level, [0] is leaf */ + struct scoutfs_block **bls; + unsigned int *inds; +}; + +struct radix_change { + struct list_head paths; + struct list_head new_paths; + struct list_head alloc_paths; + struct rb_root rbroot; + u64 block_allocs; + u64 caller_allocs; + u64 alloc_bits; + u64 next_synth; +}; + +static struct radix_path *alloc_path(struct scoutfs_radix_root *root) +{ + struct radix_path *path; + u8 height = root->height; + + path = kzalloc(sizeof(struct radix_path) + + (member_sizeof(struct radix_path, inds[0]) * height) + + (member_sizeof(struct radix_path, bls[0]) * height), + GFP_NOFS); + if (path) { + RB_CLEAR_NODE(&path->node); + INIT_LIST_HEAD(&path->head); + INIT_LIST_HEAD(&path->alloc_head); + path->height = root->height; + path->root = root; + path->bls = (void *)(path + 1); + path->inds = (void *)(&path->bls[height]); + } + return path; +} + +/* Return a pointer to a reference in the path to a block at the given level. */ +static struct scoutfs_radix_ref *path_ref(struct radix_path *path, int level) +{ + struct scoutfs_radix_block *rdx; + + BUG_ON(level < 0 || level >= path->height); + + if (level == path->height - 1) { + return &path->root->ref; + } else { + rdx = path->bls[level + 1]->data; + return &rdx->refs[path->inds[level + 1]]; + } +} + +static bool paths_share_blocks(struct radix_path *a, struct radix_path *b) +{ + int i; + + for (i = 0; i < min(a->height, b->height); i++) { + if (a->bls[i] == b->bls[i]) + return true; + } + + return false; +} + +/* + * Drop a path's reference to blocks and free its memory. If we still + * have synthetic blocks then we reset their references to the original + * empty or full blknos. Ref sequence numbers aren't updated when we + * initially reference synthetic blocks. + */ +static void free_path(struct super_block *sb, struct radix_path *path) +{ + struct scoutfs_radix_ref *ref; + struct scoutfs_block *bl; + __le64 orig; + int i; + + if (!IS_ERR_OR_NULL(path)) { + for (i = 0; i < path->height; i++) { + bl = path->bls[i]; + if (bl == NULL) + continue; + + if (bl->blkno >= RADIX_SYNTH_BLKNO) { + ref = path_ref(path, i); + if (bl->blkno & 1) + orig = cpu_to_le64(U64_MAX); + else + orig = 0; + + if (ref->blkno != orig) + ref->blkno = orig; + } + scoutfs_block_put(sb, bl); + } + kfree(path); + } +} + +static struct radix_change *alloc_change(void) +{ + struct radix_change *chg; + + chg = kzalloc(sizeof(struct radix_change), GFP_NOFS); + if (chg) { + INIT_LIST_HEAD(&chg->paths); + INIT_LIST_HEAD(&chg->new_paths); + INIT_LIST_HEAD(&chg->alloc_paths); + chg->rbroot = RB_ROOT; + chg->next_synth = RADIX_SYNTH_BLKNO; + } + return chg; +} + +static void free_change(struct super_block *sb, struct radix_change *chg) +{ + struct radix_path *path; + struct radix_path *tmp; + + if (!IS_ERR_OR_NULL(chg)) { + list_splice_init(&chg->new_paths, &chg->paths); + list_for_each_entry_safe(path, tmp, &chg->paths, head) { + list_del_init(&path->head); + free_path(sb, path); + } + kfree(chg); + } +} + +/* + * We can use native longs to set full aligned regions, but we have to + * use individual _le bit calls on leading and trailing partial regions. + * + * XXX these would be more efficient if we calculated masks for the + * initial and final partial regions. + */ +static void bitmap_set_le(__le64 *map, int ind, int nbits) +{ + unsigned int full; + + while (ind & (BITS_PER_LONG - 1) && nbits-- > 0) + set_bit_le(ind++, map); + + if (nbits >= BITS_PER_LONG) { + full = round_down(nbits, BITS_PER_LONG); + bitmap_set((long *)map, ind, full); + ind += full; + nbits -= full; + } + + while (nbits-- > 0) + set_bit_le(ind++, map); +} + +static void bitmap_clear_le(__le64 *map, int ind, int nbits) +{ + unsigned int full; + + while (ind & (BITS_PER_LONG - 1) && nbits-- > 0) + clear_bit_le(ind++, map); + + if (nbits >= BITS_PER_LONG) { + full = round_down(nbits, BITS_PER_LONG); + bitmap_clear((long *)map, ind, full); + ind += full; + nbits -= full; + } + + while (nbits-- > 0) + clear_bit_le(ind++, map); +} + +/* Returns true if the given region is all 0. */ +static bool bitmap_empty_region_le(__le64 *map, int ind, int nbits) +{ + unsigned long size = ind + nbits; + + return find_next_bit_le(map, size, ind) >= size; +} + +/* Returns true if the given region is all set. */ +static bool bitmap_full_region_le(__le64 *map, int ind, int nbits) +{ + unsigned long size = ind + nbits; + + return find_next_zero_bit_le(map, size, ind) >= size; +} + +/* + * Return true if the large region containing the full precision small bit + * index is full. + */ +static bool lg_is_full(__le64 *map, int ind) +{ + return bitmap_full_region_le(map, ind & ~SCOUTFS_RADIX_LG_MASK, + SCOUTFS_RADIX_LG_BITS); +} + +/* + * Count the number of bits set in the large regions that contain the input + * bits. + */ +static u64 count_lg_bits(void *bits, int ind, int nbits) +{ + u64 count = 0; + int end; + + ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); + end = round_up(ind + nbits, SCOUTFS_RADIX_LG_BITS); + + while (ind < end) { + if (lg_is_full(bits, ind)) + count += SCOUTFS_RADIX_LG_BITS; + ind += SCOUTFS_RADIX_LG_BITS; + } + + return count; +} + +/* + * For each of the large bit regions with bits set in the input bitmap, + * count the number of bits in corresponding large regions that are + * fully set in the result bitmap. + */ +static u64 count_lg_bitmap(void *result, void *input) +{ + u64 count = 0; + int ind = 0; + + while ((ind = find_next_bit(input, SCOUTFS_RADIX_BITS, ind)) + < SCOUTFS_RADIX_BITS) { + if (lg_is_full(result, ind)) + count += SCOUTFS_RADIX_LG_BITS; + ind = round_up(ind + 1, SCOUTFS_RADIX_LG_BITS); + } + + return count; +} + + +/* ind is a small full precision bit index, not in units of large regions */ +static int find_next_lg(__le64 *map, int ind) +{ + for (ind = round_up(ind, SCOUTFS_RADIX_LG_BITS); + ind <= (SCOUTFS_RADIX_BITS - SCOUTFS_RADIX_LG_BITS); + ind += SCOUTFS_RADIX_LG_BITS) { + if (test_bit_le(ind, map) && lg_is_full(map, ind)) + return ind; + } + + return SCOUTFS_RADIX_BITS; +} + +static u64 bit_from_inds(struct radix_path *path) +{ + u64 bit = path->inds[0]; + u64 mult = SCOUTFS_RADIX_BITS; + int i; + + for (i = 1; i < path->height; i++) { + bit += (u64)path->inds[i] * mult; + mult *= SCOUTFS_RADIX_REFS; + } + + return bit; +} + +/* return the last bit that can be stored in a tree with the given height */ +static u64 last_from_height(u8 height) +{ + u64 bit = SCOUTFS_RADIX_BITS - 1; + u64 mult = SCOUTFS_RADIX_BITS; + int i; + + for (i = 1; i < U8_MAX; i++) { + bit += (u64)(SCOUTFS_RADIX_REFS - 1) * mult; + mult *= SCOUTFS_RADIX_REFS; + } + + return bit; +} + +static u8 height_from_last(u64 last) +{ + u64 bit = SCOUTFS_RADIX_BITS - 1; + u64 mult = SCOUTFS_RADIX_BITS; + int i; + + for (i = 1; i <= U8_MAX; i++) { + if (bit >= last) + return i; + + bit += (u64)(SCOUTFS_RADIX_REFS - 1) * mult; + mult *= SCOUTFS_RADIX_REFS; + } + + return U8_MAX; +} + +/* total number of bits set in a full subtree with first block at level */ +static u64 full_subtree_total(int level) +{ + u64 total = SCOUTFS_RADIX_BITS; + int i; + + for (i = 1; i <= level; i++) + total *= SCOUTFS_RADIX_REFS; + + return total; +} + +static void calc_level_inds(struct radix_path *path, u64 bit) +{ + u32 ind; + int i; + + bit = div_u64_rem(bit, SCOUTFS_RADIX_BITS, &ind); + path->inds[0] = ind; + + for (i = 1; i < path->height; i++) { + bit = div_u64_rem(bit, SCOUTFS_RADIX_REFS, &ind); + path->inds[i] = ind; + } +} + +static u64 calc_leaf_bit(u64 bit) +{ + u32 ind; + div_u64_rem(bit, SCOUTFS_RADIX_BITS, &ind); + + return bit - ind; +} + +static int compare_path(struct scoutfs_radix_root *root, u64 leaf_bit, + struct radix_path *path) +{ + return scoutfs_cmp((unsigned long)root, (unsigned long)path->root) ?: + scoutfs_cmp(leaf_bit, path->leaf_bit); +} + +static struct radix_path *walk_paths(struct rb_root *rbroot, + struct scoutfs_radix_root *root, + u64 leaf_bit, struct radix_path *ins) +{ + struct rb_node **node = &rbroot->rb_node; + struct rb_node *parent = NULL; + struct radix_path *path; + int cmp; + + while (*node) { + parent = *node; + path = container_of(*node, struct radix_path, node); + + cmp = compare_path(root, leaf_bit, path); + if (cmp < 0) + node = &(*node)->rb_left; + else if (cmp > 0) + node = &(*node)->rb_right; + else + return path; + } + + if (ins) { + rb_link_node(&ins->node, parent, node); + rb_insert_color(&ins->node, rbroot); + return ins; + } + + return NULL; +} + +/* + * Update the first tracking in a block after the caller has modified + * the block at the given index. If the modification at the index is + * populated we check to see if first should be earler. If the + * modification at the index is now empty (cleared leaf bits or parent + * ref total going to 0) we can advance the first tracker by an offset + * (number of bits in the leaf, to the next ref in parents), or set + * first to the end of the block if the entire block is now empty. + * + * The first field is only guaranteed to be before the first set region, + * if there is any. It's only advanced when the current first is + * cleared. If regions are cleared out of order then you can be left + * with a first less than the limit in a block with none set. + */ +static void update_first(__le32 *first, int ind, u32 limit, s32 offset, + bool empty_ind, bool entirely_empty) +{ + if (!empty_ind) { + if (ind < le32_to_cpup(first)) + *first = cpu_to_le32(ind); + + } else { + if (entirely_empty) + *first = cpu_to_le32(limit); + else if (ind == le32_to_cpup(first)) + le32_add_cpu(first, offset); + } +} + +/* + * The caller has changed bits in a leaf block. We update the first + * fields in the block header and the total fields in block references. + */ +static void fixup_first_total(struct super_block *sb, struct radix_path *path, + int ind, s64 sm_delta, s64 lg_delta) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *rdx_ref; + struct scoutfs_radix_ref *ref; + int level; + + for (level = 0; level < path->height; level++) { + rdx = path->bls[level]->data; + ref = path_ref(path, level); + if (level > 0) { + ind = path->inds[level]; + rdx_ref = &rdx->refs[ind]; + } + + le64_add_cpu(&ref->sm_total, sm_delta); + le64_add_cpu(&ref->lg_total, lg_delta); + + if (level == 0) { + update_first(&rdx->sm_first, ind, SCOUTFS_RADIX_BITS, + -sm_delta, sm_delta < 0, + ref->sm_total == 0); + update_first(&rdx->lg_first, + round_down(ind, SCOUTFS_RADIX_LG_BITS), + SCOUTFS_RADIX_BITS, + -lg_delta, lg_delta < 0, + ref->lg_total == 0); + } else { + update_first(&rdx->sm_first, ind, SCOUTFS_RADIX_REFS, + 1, rdx_ref->sm_total == 0, + ref->sm_total == 0); + update_first(&rdx->lg_first, + round_down(ind, SCOUTFS_RADIX_LG_BITS), + SCOUTFS_RADIX_REFS, + 1, rdx_ref->lg_total == 0, + ref->lg_total == 0); + } + } +} + +/* + * Allocate (clear and return) a region of bits from the leaf block of a + * path. The leaf walk has ensured that we have at least one block free. + * + * We always try to allocate smaller multi-block allocations from the + * start of the small region. This at least gets a single task extending + * a file one large extent. Multiple tasks extending writes will interleave. + * It'll do for now. + * + * We always search for free bits from the start of the leaf. + * This means that we can return recently freed blocks just behind the + * next free cursor. I'm not sure if that's much of a problem. + */ +static void alloc_leaf_bits(struct super_block *sb, struct radix_path *path, + int nbits, u64 *bit_ret, int *nbits_ret) +{ + struct scoutfs_radix_block *rdx = path->bls[0]->data; + struct scoutfs_radix_ref *ref = path_ref(path, 0); + s64 lg_delta; + int ind; + int end; + + if (nbits >= SCOUTFS_RADIX_LG_BITS && ref->lg_total != 0) { + /* always allocate large allocs from full large regions */ + ind = le32_to_cpu(rdx->lg_first); + ind = find_next_lg(rdx->bits, ind); + + } else { + /* otherwise alloc as much as we can from the next small */ + ind = le32_to_cpu(rdx->sm_first); + ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); + + if (nbits > 1) { + end = find_next_zero_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); + nbits = min(nbits, end - ind); + } + } + + /* callers and structures should have ensured success */ + BUG_ON(ind >= SCOUTFS_RADIX_BITS); + + lg_delta = count_lg_bits(rdx->bits, ind, nbits); + bitmap_clear_le(rdx->bits, ind, nbits); + + fixup_first_total(sb, path, ind, -nbits, -lg_delta); + + *bit_ret = path->leaf_bit + ind; + *nbits_ret = nbits; +} + +/* + * Allocate a metadata blkno for the caller from the leaves of paths + * which were stored in the change for metadata allocation. + */ +static u64 change_alloc_meta(struct super_block *sb, struct radix_change *chg) +{ + struct scoutfs_radix_ref *ref; + struct radix_path *path; + int nbits_ret; + u64 bit; + + path = list_first_entry_or_null(&chg->alloc_paths, struct radix_path, + alloc_head); + BUG_ON(!path); /* shouldn't be possible */ + + alloc_leaf_bits(sb, path, 1, &bit, &nbits_ret); + + /* remove the path from the alloc list once its empty */ + ref = path_ref(path, 0); + if (ref->sm_total == 0) + list_del_init(&path->alloc_head); + + return bit; +} + +static void set_path_leaf_bits(struct super_block *sb, struct radix_path *path, + u64 bit, int nbits) +{ + struct scoutfs_radix_block *rdx; + int ind; + + BUG_ON(nbits <= 0); + BUG_ON(calc_leaf_bit(bit) != calc_leaf_bit(bit + nbits - 1)); + BUG_ON(calc_leaf_bit(bit) != path->leaf_bit); + + rdx = path->bls[0]->data; + ind = bit - path->leaf_bit; + + /* should have returned an error if it was set while we got paths */ + BUG_ON(!bitmap_empty_region_le(rdx->bits, ind, nbits)); + bitmap_set_le(rdx->bits, ind, nbits); + + fixup_first_total(sb, path, ind, nbits, + count_lg_bits(rdx->bits, ind, nbits)); + + trace_scoutfs_radix_set(sb, path->root, path->bls[0]->blkno, + bit, ind, nbits); +} + +/* Find the path for the root and bit in the change and set the region */ +static void set_change_leaf_bits(struct super_block *sb, + struct radix_change *chg, + struct scoutfs_radix_root *root, + u64 bit, int nbits) +{ + struct radix_path *path; + + path = walk_paths(&chg->rbroot, root, calc_leaf_bit(bit), NULL); + BUG_ON(!path); /* should have gotten paths for all leaves to set */ + set_path_leaf_bits(sb, path, bit, nbits); +} + +/* + * Initialize a reference to a block at the given level. + */ +static void init_ref(struct scoutfs_radix_ref *ref, int level, bool full) +{ + u64 tot; + + if (full) { + tot = full_subtree_total(level); + + ref->blkno = cpu_to_le64(U64_MAX); + ref->seq = cpu_to_le64(0); + ref->sm_total = cpu_to_le64(tot); + ref->lg_total = cpu_to_le64(tot >> SCOUTFS_RADIX_LG_SHIFT); + } else { + + ref->blkno = cpu_to_le64(0); + ref->seq = cpu_to_le64(0); + ref->sm_total = cpu_to_le64(0); + ref->lg_total = cpu_to_le64(0); + } +} + +/* Initialize a new empty or full block at a given level. */ +static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, + u64 blkno, __le64 seq, int level, bool full) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_radix_ref ref; + u32 first = full ? 0 : level ? SCOUTFS_RADIX_REFS : SCOUTFS_RADIX_BITS; + int tail; + int i; + + /* we use native long bitmap functions on the block bitmaps */ + BUILD_BUG_ON(offsetof(struct scoutfs_radix_block, bits) & + (sizeof(long) - 1)); + + rdx->hdr.fsid = super->hdr.fsid; + rdx->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_RADIX); + rdx->hdr.blkno = cpu_to_le64(blkno); + rdx->hdr.seq = seq; + rdx->sm_first = cpu_to_le32(first); + rdx->lg_first = cpu_to_le32(first); + + if (level == 0) { + if (full) + memset(rdx->bits, 0xff, SCOUTFS_RADIX_BITS_BYTES); + else + memset(rdx->bits, 0, SCOUTFS_RADIX_BITS_BYTES); + + tail = SCOUTFS_BLOCK_SIZE - + offsetof(struct scoutfs_radix_block, bits) - + SCOUTFS_RADIX_BITS_BYTES; + } else { + init_ref(&ref, level - 1, full); + + for (i = 0; i < SCOUTFS_RADIX_REFS; i++) + memcpy(&rdx->refs[i], &ref, sizeof(ref)); + + tail = SCOUTFS_BLOCK_SIZE - + offsetof(struct scoutfs_radix_block, + refs[SCOUTFS_RADIX_REFS]); + } + + /* make sure we don't write uninitialized tail kernel memory to disk */ + if (tail) + memset((void *)rdx + SCOUTFS_BLOCK_SIZE - tail, 0, tail); +} + +/* get path flags */ +enum { + GPF_NEXT_SM = (1 << 0), + GPF_NEXT_LG = (1 << 1), +}; +/* + * Give the caller an allocated path that holds references to the blocks + * traversed to the leaf of the given root. + */ +static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, + struct radix_change *chg, int gpf, u64 bit, + struct radix_path **path_ret) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *ref; + struct radix_path *path = NULL; + struct scoutfs_block *bl; + bool saw_inconsistent = false; + u64 blkno; + u64 synth; + int level; + int ind; + int ret; + int i; + + /* can't operate outside radix until we support growing devices */ + if (WARN_ON_ONCE(root->height < height_from_last(bit)) || + WARN_ON_ONCE((gpf & GPF_NEXT_SM) && (gpf & GPF_NEXT_LG))) + return -EINVAL; + + path = alloc_path(root); + if (!path) { + ret = -ENOMEM; + goto out; + } + + /* switch to searching for small bits if no large found */ + if ((gpf & GPF_NEXT_LG) && le64_to_cpu(root->ref.lg_total) == 0) + gpf ^= GPF_NEXT_LG | GPF_NEXT_SM; + + calc_level_inds(path, bit); + + for (level = root->height - 1; level >= 0; level--) { + ref = path_ref(path, level); + + blkno = le64_to_cpu(ref->blkno); + if (blkno == U64_MAX || blkno == 0) { + synth = chg->next_synth++; + if ((blkno & 1) != (synth & 1)) + synth = chg->next_synth++; + /* careful not to go too high or wrap */ + if (synth == U64_MAX || synth < RADIX_SYNTH_BLKNO) { + ret = -ENOSPC; + goto out; + } + bl = scoutfs_block_create(sb, synth); + if (!IS_ERR_OR_NULL(bl)) { + init_block(sb, bl->data, synth, ref->seq, level, + blkno == U64_MAX); + ref->blkno = cpu_to_le64(bl->blkno); + + } + } else { + bl = scoutfs_block_read(sb, blkno); + } + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + + /* + * We can have a stale block in the cache but the tree + * shouldn't be changing under us. We don't have to + * reread a root and restart descent. If we don't get a + * consistent block after reading from the device then + * we've found corruption. + */ + if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_RADIX)) { + if (!saw_inconsistent) { + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + saw_inconsistent = true; + level++; + continue; + } + ret = -EIO; + goto out; + } + saw_inconsistent = false; + + path->bls[level] = bl; + if (level == 0) { + /* path's leaf_bit is first in the leaf block */ + path->inds[0] = 0; + break; + } + + rdx = bl->data; + ind = path->inds[level]; + + /* search for a path to a leaf with a set large region */ + while ((gpf & GPF_NEXT_LG) && ind < SCOUTFS_RADIX_REFS && + le64_to_cpu(rdx->refs[ind].lg_total) == 0) { + if (ind < le32_to_cpu(rdx->lg_first)) + ind = le32_to_cpu(rdx->lg_first); + else + ind++; + } + + /* search for a path to a leaf with a any bits set */ + while ((gpf & GPF_NEXT_SM) && ind < SCOUTFS_RADIX_REFS && + le64_to_cpu(rdx->refs[ind].sm_total) == 0) { + if (ind < le32_to_cpu(rdx->sm_first)) + ind = le32_to_cpu(rdx->sm_first); + else + ind++; + } + + if (ind >= SCOUTFS_RADIX_REFS) { + ret = -ENOENT; + goto out; + } + + /* reset all lower indices if we searched */ + if (ind != path->inds[level]) { + for (i = level - 1; i >= 0; i--) + path->inds[i] = 0; + path->inds[level] = ind; + } + } + + path->leaf_bit = bit_from_inds(path); + ret = 0; +out: + if (ret < 0) { + free_path(sb, path); + path = NULL; + } + + *path_ret = path; + return ret; +} + +/* + * Get all the paths we're going to need to dirty all the blocks in all + * the paths in the change. The caller has added their path to the leaf + * that they want to change to start the process off. + * + * For every clean block in paths we can have to set a bit in a leaf to + * free the old blkno and clear a bit in a leaf to allocate a new dirty + * blkno. We keep checking new paths for clean blocks until eventually + * all the paths only contain blocks whose blknos are in leaves that we + * already have paths to. + */ +static int get_all_paths(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct radix_change *chg) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *ref; + struct scoutfs_block *bl; + struct radix_path *path; + struct radix_path *adding; + struct radix_path *found; + bool meta_wrapped; + bool stable; + u64 start_meta; + u64 next_meta; + u64 last_meta; + u64 leaf_bit; + int ind; + int ret; + int i; + + start_meta = calc_leaf_bit(le64_to_cpu(alloc->avail.next_find_bit)); + next_meta = start_meta; + last_meta = le64_to_cpu(super->last_meta_blkno); + meta_wrapped = false; + + do { + stable = true; + + /* get paths to leaves to allocate dirty blknos from */ + if (chg->alloc_bits < chg->block_allocs + chg->caller_allocs) { + stable = false; + + if (next_meta == start_meta && meta_wrapped) { + ret = -ENOSPC; + break; + } + + ret = get_path(sb, &alloc->avail, chg, GPF_NEXT_SM, + next_meta, &adding); + if (ret < 0) { + if (ret == -ENOENT) { + if (next_meta != 0) { + next_meta = 0; + meta_wrapped = true; + continue; + } else { + ret = -ENOSPC; + } + } + break; + } + + next_meta = adding->leaf_bit + SCOUTFS_RADIX_BITS; + if (next_meta > last_meta) { + meta_wrapped = true; + next_meta = 0; + } + + /* might already have path, maybe add it to alloc */ + found = walk_paths(&chg->rbroot, adding->root, + adding->leaf_bit, adding); + if (found != adding) { + free_path(sb, adding); + adding = found; + } else { + list_add_tail(&adding->head, &chg->new_paths); + } + if (list_empty(&adding->alloc_head)) { + ref = path_ref(adding, 0); + chg->alloc_bits += le64_to_cpu(ref->sm_total); + list_add_tail(&adding->alloc_head, + &chg->alloc_paths); + } + } + + if ((path = list_first_entry_or_null(&chg->new_paths, + struct radix_path, + head))) { + list_move_tail(&path->head, &chg->paths); + stable = false; + + /* check all the blocks in all new paths */ + for (i = path->height - 1; i >= 0; i--) { + bl = path->bls[i]; + + /* dirty are done, only visit each block once */ + if (scoutfs_block_writer_is_dirty(sb, bl) || + scoutfs_block_tas_visited(sb, bl)) + continue; + + /* record the number of allocs we'll need */ + chg->block_allocs++; + + /* don't need to free synth blknos */ + if (bl->blkno >= RADIX_SYNTH_BLKNO) + continue; + + /* see if we already a path to this leaf */ + leaf_bit = calc_leaf_bit(bl->blkno); + if (walk_paths(&chg->rbroot, &alloc->freed, + leaf_bit, NULL)) + continue; + + /* get a new path to freed leaf to set */ + ret = get_path(sb, &alloc->freed, chg, 0, + bl->blkno, &adding); + if (ret < 0) + break; + + rdx = adding->bls[0]->data; + ind = bl->blkno - adding->leaf_bit; + if (test_bit_le(ind, rdx->bits)) { + /* XXX corruption, bit already set? */ + ret = -EIO; + break; + } + + walk_paths(&chg->rbroot, adding->root, + adding->leaf_bit, adding); + list_add_tail(&adding->head, &chg->new_paths); + } + } + + ret = 0; + } while (!stable); + + alloc->avail.next_find_bit = cpu_to_le64(next_meta); + + return ret; +} + +/* + * We have pinned blocks in paths to all the leaves that we need to + * modify to make a change to radix trees. Walk through the paths + * moving blocks to their new allocated blknos, freeing the old stable + * blknos. + */ +static void dirty_all_path_blocks(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct radix_change *chg) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *ref; + struct scoutfs_block *bl; + struct radix_path *path; + u64 blkno; + int level; + + BUG_ON(!list_empty(&chg->new_paths)); + + list_for_each_entry(path, &chg->paths, head) { + + for (level = path->height - 1; level >= 0; level--) { + bl = path->bls[level]; + + if (scoutfs_block_writer_is_dirty(sb, bl)) + continue; + + if (bl->blkno < RADIX_SYNTH_BLKNO) + set_change_leaf_bits(sb, chg, &alloc->freed, + bl->blkno, 1); + + blkno = change_alloc_meta(sb, chg); + scoutfs_block_clear_visited(sb, bl); + scoutfs_block_move(sb, wri, bl, blkno); + scoutfs_block_writer_mark_dirty(sb, wri, bl); + + rdx = bl->data; + rdx->hdr.blkno = cpu_to_le64(bl->blkno); + le64_add_cpu(&rdx->hdr.seq, 1); + + ref = path_ref(path, level); + ref->blkno = rdx->hdr.blkno; + ref->seq = rdx->hdr.seq; + } + } +} + +static void store_next_find_bit(struct scoutfs_radix_root *root, u64 bit) +{ + if (bit > last_from_height(root->height)) + bit = 0; + root->next_find_bit = cpu_to_le64(bit); +} + +static bool valid_free_bit_range(struct super_block *sb, bool meta, + u64 bit, int nbits) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + u64 last = bit + nbits - 1; + + return (nbits > 0) && + (last >= bit) && + (!meta || (bit >= le64_to_cpu(super->first_meta_blkno) && + last <= le64_to_cpu(super->last_meta_blkno))) && + (meta || (bit >= le64_to_cpu(super->first_data_blkno) && + last <= le64_to_cpu(super->last_data_blkno))); +} + +static int radix_free(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *root, bool meta, + u64 bit, int nbits) +{ + struct scoutfs_radix_block *rdx; + struct radix_change *chg; + struct radix_path *path; + int ind; + int ret; + + /* we only operate on one leaf */ + if (WARN_ON_ONCE(!valid_free_bit_range(sb, meta, bit, nbits)) || + WARN_ON_ONCE(calc_leaf_bit(bit) != calc_leaf_bit(bit + nbits - 1))) + return -EINVAL; + + mutex_lock(&alloc->mutex); + + chg = alloc_change(); + if (!chg) { + ret = -ENOMEM; + goto out; + } + + ret = get_path(sb, root, chg, 0, bit, &path); + if (ret < 0) + goto out; + list_add_tail(&path->head, &chg->new_paths); + + ind = bit - path->leaf_bit; + rdx = path->bls[0]->data; + if (!bitmap_empty_region_le(rdx->bits, ind, nbits)) { + /* XXX corruption, trying to free set bits */ + ret = -EIO; + goto out; + } + + ret = get_all_paths(sb, alloc, chg); + if (ret < 0) + goto out; + + dirty_all_path_blocks(sb, alloc, wri, chg); + set_path_leaf_bits(sb, path, bit, nbits); + ret = 0; +out: + free_change(sb, chg); + mutex_unlock(&alloc->mutex); + return ret; +} + +/* + * Return a single allocated metadata block for the caller. We let the change + * find a leaf in the metadata allocator for us. + */ +int scoutfs_radix_alloc(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, u64 *blkno) +{ + struct radix_change *chg; + int ret; + + mutex_lock(&alloc->mutex); + + chg = alloc_change(); + if (!chg) { + ret = -ENOMEM; + goto out; + } + + chg->caller_allocs = 1; + ret = get_all_paths(sb, alloc, chg); + if (ret < 0) + goto out; + + dirty_all_path_blocks(sb, alloc, wri, chg); + *blkno = change_alloc_meta(sb, chg); + ret = 0; +out: + free_change(sb, chg); + mutex_unlock(&alloc->mutex); + + return ret; +} + +/* + * Return an allocated data block extent by finding and clearing it from + * the caller's tree. The caller must protect access to their tree. We + * have to search in and allocate from the separate data allocator tree + * ourselves. + */ +int scoutfs_radix_alloc_data(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *root, + int count, u64 *blkno_ret, int *count_ret) +{ + struct radix_change *chg; + struct radix_path *path; + u64 bit; + int nbits; + int gpf; + int ret; + + *blkno_ret = 0; + *count_ret = 0; + + if (WARN_ON_ONCE(count <= 0 || blkno_ret == NULL || count_ret == NULL)) + return -EINVAL; + + nbits = min(count, SCOUTFS_RADIX_LG_BITS); + gpf = nbits > 1 ? GPF_NEXT_LG : GPF_NEXT_SM; + + mutex_lock(&alloc->mutex); + + chg = alloc_change(); + if (!chg) { + ret = -ENOMEM; + goto out; + } + +find_next: + bit = le64_to_cpu(root->next_find_bit); + ret = get_path(sb, root, chg, gpf, bit, &path); + if (ret) { + if (ret == -ENOENT) { + if (root->next_find_bit != 0) { + root->next_find_bit = 0; + goto find_next; + } + ret = -ENOSPC; + } + goto out; + } + list_add_tail(&path->head, &chg->new_paths); + + store_next_find_bit(root, bit); + + ret = get_all_paths(sb, alloc, chg); + if (ret < 0) + goto out; + + dirty_all_path_blocks(sb, alloc, wri, chg); + alloc_leaf_bits(sb, path, nbits, blkno_ret, count_ret); + ret = 0; +out: + free_change(sb, chg); + mutex_unlock(&alloc->mutex); + + return ret; +} + +/* + * Free a single metadata block by adding it to the allocator's freed + * tree. Callers can trust our allocator to lock. + */ +int scoutfs_radix_free(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, u64 blkno) +{ + return radix_free(sb, alloc, wri, &alloc->freed, true, blkno, 1); +} + +/* + * Free a data block extent by setting it in the caller's tree. The + * caller must protect access to their tree. + */ +int scoutfs_radix_free_data(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *root, + u64 blkno, int count) +{ + return radix_free(sb, alloc, wri, root, false, blkno, count); +} + +/* + * Move bits between the source and destination trees. The bits to move + * are found in the input tree. + * + * Typically the input and source trees are the same. We're careful to + * modify the dst first because modifying src might also be modifying + * inp. + * + * The input and source trees aren't the same when the caller is being + * careful to use a read-only input tree because the source tree is + * changing during the merge. This happens when the server tries to + * reclaim its freed tree by moving it into its avail. Because our + * dirtying actually moves clean blocks we need to be careful to not + * reference dirty blocks from the input tree walk. This is discovered + * after dirtying the blocks. The additional input walk will this time + * read the old blocks. + * + * We can also be called with a src tree that is the current allocator + * avail tree. In this case dirtying the blocks in all the paths can + * consume bits in the source tree. We notice when dirtying allocation + * empties the src block and we retry finding a new leaf to merge. + * + * The caller specifies the minimum count to move. -ENOENT will be + * returned if the source tree runs out of bits, potentially after + * having already moved bits. More than the minimum can be moved + * because whole leaves worth of bits are moved. + * + * This is pretty expensive because it fully references full leaf blocks + * a few times. It could be more efficient if it short circuited walks + * and spliced refs in parents when it finds that subtrees don't + * intersect. + */ +int scoutfs_radix_merge(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *dst, + struct scoutfs_radix_root *src, + struct scoutfs_radix_root *inp, u64 count) +{ + struct scoutfs_radix_block *inp_rdx; + struct scoutfs_radix_block *src_rdx; + struct scoutfs_radix_block *dst_rdx; + struct radix_change *chg = NULL; + struct radix_path *inp_path = NULL; + struct radix_path *src_path; + struct radix_path *dst_path; + s64 src_lg_delta; + s64 dst_lg_delta; + s64 sm_delta; + u64 bit; + int ind; + int ret; + + mutex_lock(&alloc->mutex); + + while (count > 0) { + + chg = alloc_change(); + if (!chg) { + ret = -ENOMEM; + goto out; + } + + bit = le64_to_cpu(src->next_find_bit); +wrapped: + ret = get_path(sb, inp, chg, GPF_NEXT_SM, bit, &inp_path); + if (ret < 0) { + if (ret == -ENOENT) { + if (bit != 0) { + bit = 0; + goto wrapped; + } else { + ret = -ENOSPC; + } + } + goto out; + } + /* unique input is not modified, not stored in the change */ + bit = inp_path->leaf_bit; + + ret = get_path(sb, src, chg, 0, bit, &src_path); + if (ret < 0) + goto out; + list_add_tail(&src_path->head, &chg->new_paths); + + ret = get_path(sb, dst, chg, 0, bit, &dst_path); + if (ret < 0) + goto out; + list_add_tail(&dst_path->head, &chg->new_paths); + + ret = get_all_paths(sb, alloc, chg); + if (ret < 0) + goto out; + + /* this can modify src/dst when they're alloc trees */ + dirty_all_path_blocks(sb, alloc, wri, chg); + + inp_rdx = inp_path->bls[0]->data; + src_rdx = src_path->bls[0]->data; + dst_rdx = dst_path->bls[0]->data; + + sm_delta = le64_to_cpu(path_ref(inp_path, 0)->sm_total); + ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, + le32_to_cpu(inp_rdx->sm_first)); + + /* back out and retry if no input left, or inp not ro */ + if (sm_delta == 0 || + (inp != src && paths_share_blocks(inp_path, src_path))) { + free_path(sb, inp_path); + inp_path = NULL; + free_change(sb, chg); + chg = NULL; + continue; + } + + /* make sure all input bits are set in src */ + if (inp != src && + !bitmap_subset((void *)inp_rdx->bits, + (void *)src_rdx->bits, + SCOUTFS_RADIX_BITS)) { + ret = -EIO; + goto out; + } + + /* make sure all input bits are clear in dst */ + if (bitmap_intersects((void *)dst_rdx->bits, + (void *)inp_rdx->bits, + SCOUTFS_RADIX_BITS)) { + ret = -EIO; + goto out; + } + + /* carefully modify src last, it might also be inp */ + bitmap_xor((void *)dst_rdx->bits, (void *)dst_rdx->bits, + (void *)inp_rdx->bits, SCOUTFS_RADIX_BITS); + dst_lg_delta = count_lg_bitmap(dst_rdx->bits, inp_rdx->bits); + + src_lg_delta = count_lg_bitmap(src_rdx->bits, inp_rdx->bits); + bitmap_xor((void *)src_rdx->bits, (void *)src_rdx->bits, + (void *)inp_rdx->bits, SCOUTFS_RADIX_BITS); + + fixup_first_total(sb, src_path, ind, -sm_delta, -src_lg_delta); + fixup_first_total(sb, dst_path, ind, sm_delta, dst_lg_delta); + + trace_scoutfs_radix_merge(sb, src, src_path->bls[0]->blkno, + dst, dst_path->bls[0]->blkno, count, + ind, sm_delta, src_lg_delta, + dst_lg_delta); + + free_path(sb, inp_path); + inp_path = NULL; + free_change(sb, chg); + chg = NULL; + + store_next_find_bit(src, bit + SCOUTFS_RADIX_BITS); + count -= min_t(u64, count, sm_delta); + } + + ret = 0; +out: + free_path(sb, inp_path); + free_change(sb, chg); + mutex_unlock(&alloc->mutex); + + return ret; +} + +void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, + struct scoutfs_radix_root *avail, + struct scoutfs_radix_root *freed) +{ + mutex_init(&alloc->mutex); + alloc->avail = *avail; + alloc->freed = *freed; +} + +/* + * Initialize a root with an empty ref. We set the height to the size + * of the device and descent will fill in blocks. + */ +void scoutfs_radix_root_init(struct super_block *sb, + struct scoutfs_radix_root *root, bool meta) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + u64 last; + + if (meta) + last = le64_to_cpu(super->last_meta_blkno); + else + last = le64_to_cpu(super->last_data_blkno); + + root->height = height_from_last(last); + root->next_find_bit = cpu_to_le64(0); + init_ref(&root->ref, 0, false); +} + +/* + * The first bit nr in a leaf containing the bit, used by callers to + * identify regions that span leafs and would need to be freed in + * multiple calls. + */ +u64 scoutfs_radix_bit_leaf_nr(u64 bit) +{ + return calc_leaf_bit(bit); +} diff --git a/kmod/src/radix.h b/kmod/src/radix.h new file mode 100644 index 00000000..15433bb0 --- /dev/null +++ b/kmod/src/radix.h @@ -0,0 +1,43 @@ +#ifndef _SCOUTFS_RADIX_H_ +#define _SCOUTFS_RADIX_H_ + +#include "per_task.h" + +struct scoutfs_block_writer; + +struct scoutfs_radix_allocator { + struct mutex mutex; + struct scoutfs_radix_root avail; + struct scoutfs_radix_root freed; +}; + +int scoutfs_radix_alloc(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, u64 *blkno); +int scoutfs_radix_alloc_data(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *root, + int count, u64 *blkno_ret, int *count_ret); +int scoutfs_radix_free(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, u64 blkno); +int scoutfs_radix_free_data(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *root, + u64 blkno, int count); +int scoutfs_radix_merge(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_radix_root *dst, + struct scoutfs_radix_root *src, + struct scoutfs_radix_root *inp, u64 count); +void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, + struct scoutfs_radix_root *avail, + struct scoutfs_radix_root *freed); +void scoutfs_radix_root_init(struct super_block *sb, + struct scoutfs_radix_root *root, bool meta); +u64 scoutfs_radix_bit_leaf_nr(u64 bit); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 15fc7daa..e08d15ac 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2212,6 +2212,165 @@ DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink, TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) ); +TRACE_EVENT(scoutfs_radix_dirty, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + u64 orig_blkno, u64 dirty_blkno, u64 par_blkno), + TP_ARGS(sb, root, orig_blkno, dirty_blkno, par_blkno), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, orig_blkno) + __field(__u64, dirty_blkno) + __field(__u64, par_blkno) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->orig_blkno = orig_blkno; + __entry->dirty_blkno = dirty_blkno; + __entry->par_blkno = par_blkno; + ), + TP_printk(SCSBF" root_blkno %llu orig_blkno %llu dirty_blkno %llu par_blkno %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->orig_blkno, + __entry->dirty_blkno, __entry->par_blkno) +); + +TRACE_EVENT(scoutfs_radix_walk, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + int grl, int level, u64 blkno, int ind, u64 bit, u64 next), + TP_ARGS(sb, root, grl, level, blkno, ind, bit, next), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(unsigned int, grl) + __field(__u64, blkno) + __field(int, level) + __field(int, ind) + __field(__u64, bit) + __field(__u64, next) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->grl = grl; + __entry->blkno = blkno; + __entry->level = level; + __entry->ind = ind; + __entry->bit = bit; + __entry->next = next; + ), + TP_printk(SCSBF" root_blkno %llu grl 0x%x blkno %llu level %d ind %d bit %llu next %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->grl, + __entry->blkno, __entry->level, __entry->ind, __entry->bit, + __entry->next) +); + +TRACE_EVENT(scoutfs_radix_fixup_refs, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + u32 sm_first, u64 sm_total, u16 lg_first, u64 lg_total, + u64 blkno, int level), + TP_ARGS(sb, root, sm_first, sm_total, lg_first, lg_total, blkno, level), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u32, sm_first) + __field(__u64, sm_total) + __field(__u16, lg_first) + __field(__u64, lg_total) + __field(__u64, blkno) + __field(int, level) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->sm_first = sm_first; + __entry->sm_total = sm_total; + __entry->lg_first = lg_first; + __entry->lg_total = lg_total; + __entry->blkno = blkno; + __entry->level = level; + ), + TP_printk(SCSBF" root_blkno %llu sm_first %u sm_total %llu lg_first %u lg_total %llu blkno %llu level %u", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->sm_first, + __entry->sm_total, __entry->lg_first, __entry->lg_total, + __entry->blkno, __entry->level) +); + +DECLARE_EVENT_CLASS(scoutfs_radix_bitop, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + u64 blkno, u64 bit, int ind, int nbits), + TP_ARGS(sb, root, blkno, bit, ind, nbits), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, blkno) + __field(__u64, bit) + __field(int, ind) + __field(int, nbits) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->blkno = blkno; + __entry->bit = bit; + __entry->ind = ind; + __entry->nbits = nbits; + ), + TP_printk(SCSBF" root_blkno %llu blkno %llu bit %llu ind %d nbits %d", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->blkno, + __entry->bit, __entry->ind, __entry->nbits) +); +DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_clear, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + u64 blkno, u64 bit, int ind, int nbits), + TP_ARGS(sb, root, blkno, bit, ind, nbits) +); +DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set, + TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, + u64 blkno, u64 bit, int ind, int nbits), + TP_ARGS(sb, root, blkno, bit, ind, nbits) +); + +TRACE_EVENT(scoutfs_radix_merge, + TP_PROTO(struct super_block *sb, + struct scoutfs_radix_root *src, u64 src_blkno, + struct scoutfs_radix_root *dst, u64 dst_blkno, + u64 count, int ind, int sm_delta, int src_lg_delta, + int dst_lg_delta), + TP_ARGS(sb, src, src_blkno, dst, dst_blkno, count, ind, + sm_delta, src_lg_delta, dst_lg_delta), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, src_root_blkno) + __field(__u64, src_blkno) + __field(__u64, dst_root_blkno) + __field(__u64, dst_blkno) + __field(__u64, count) + __field(int, ind) + __field(int, sm_delta) + __field(int, src_lg_delta) + __field(int, dst_lg_delta) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->src_root_blkno = le64_to_cpu(src->ref.blkno); + __entry->src_blkno = src_blkno; + __entry->dst_root_blkno = le64_to_cpu(dst->ref.blkno); + __entry->dst_blkno = dst_blkno; + __entry->count = count; + __entry->ind = ind; + __entry->sm_delta = sm_delta; + __entry->src_lg_delta = src_lg_delta; + __entry->dst_lg_delta = dst_lg_delta; + ), + TP_printk(SCSBF" src_root_blkno %llu src_blkno %llu dst_root_blkno %llu dst_blkno %llu count %llu ind %u sm_delta %d src_lg_delta %d dst_lg_delta %d", + SCSB_TRACE_ARGS, + __entry->src_root_blkno, __entry->src_blkno, + __entry->dst_root_blkno, __entry->dst_blkno, + __entry->count, __entry->ind, __entry->sm_delta, + __entry->src_lg_delta, __entry->dst_lg_delta) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 85142dcadf77fc7266eb67cc829ca20099767952 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Feb 2020 15:52:35 -0800 Subject: [PATCH 781/920] scoutfs: use radix allocator Convert metadata block and file data extent allocations to use the radix allocator. Most of this is simple transitions between types and calls. The server no longer has to initialize blocks because mkfs can write a single radix parent block with fully set parent refs to initialize a full radix. We remove the code and fields that were responsible for adding uninitialized data and metadata. The rest of the unused block allocator code is only ifdefed out. It'll be removed in a separate patch to reduce noise here. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 - kmod/src/btree.c | 36 ++++---- kmod/src/btree.h | 12 +-- kmod/src/data.c | 88 +++++++++---------- kmod/src/data.h | 6 +- kmod/src/forest.c | 15 ++-- kmod/src/forest.h | 4 +- kmod/src/format.h | 41 +++++---- kmod/src/lock_server.c | 6 +- kmod/src/lock_server.h | 2 +- kmod/src/scoutfs_trace.h | 2 + kmod/src/server.c | 184 ++++++++++----------------------------- kmod/src/trans.c | 11 +-- 13 files changed, 158 insertions(+), 250 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 4492cc52..f0c62bb4 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -9,7 +9,6 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -include $(src)/Makefile.kernelcompat scoutfs-y += \ - balloc.o \ block.o \ btree.o \ client.o \ diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 936de009..f31e44f5 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -27,7 +27,7 @@ #include "options.h" #include "msg.h" #include "block.h" -#include "balloc.h" +#include "radix.h" #include "scoutfs_trace.h" @@ -387,7 +387,7 @@ static void move_items(struct scoutfs_btree_block *dst, * error. */ static int get_ref_block(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, int flags, struct scoutfs_btree_ref *ref, struct scoutfs_block **bl_ret) @@ -450,7 +450,7 @@ retry: goto out; } - ret = scoutfs_balloc_alloc(sb, alloc, wri, &blkno); + ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); if (ret < 0) goto out; @@ -458,7 +458,7 @@ retry: new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - ret = scoutfs_balloc_free(sb, alloc, wri, blkno); + ret = scoutfs_radix_free(sb, alloc, wri, blkno); BUG_ON(ret); /* radix should have been dirty */ ret = PTR_ERR(new_bl); goto out; @@ -546,7 +546,7 @@ static void update_parent_item(struct scoutfs_btree_block *parent, * Returns -errno, 0 if nothing done, or 1 if we split. */ static int try_split(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, unsigned val_len, @@ -582,8 +582,8 @@ static int try_split(struct super_block *sb, if (!parent) { ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl); if (ret) { - err = scoutfs_balloc_free(sb, alloc, wri, - le64_to_cpu(left->hdr.blkno)); + err = scoutfs_radix_free(sb, alloc, wri, + le64_to_cpu(left->hdr.blkno)); BUG_ON(err); /* radix should have been dirty */ scoutfs_block_put(sb, left_bl); return ret; @@ -622,7 +622,7 @@ static int try_split(struct super_block *sb, * XXX this could more cleverly chose a merge candidate sibling */ static int try_merge(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, unsigned pos, @@ -676,8 +676,8 @@ static int try_merge(struct super_block *sb, /* update or delete sibling's parent item */ if (le32_to_cpu(sib->nr_items) == 0) { delete_item(parent, sib_pos); - ret = scoutfs_balloc_free(sb, alloc, wri, - le64_to_cpu(sib->hdr.blkno)); + ret = scoutfs_radix_free(sb, alloc, wri, + le64_to_cpu(sib->hdr.blkno)); BUG_ON(ret); /* could have dirtied alloc to avoid error */ } else if (move_right) { @@ -689,8 +689,8 @@ static int try_merge(struct super_block *sb, root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; - ret = scoutfs_balloc_free(sb, alloc, wri, - le64_to_cpu(parent->hdr.blkno)); + ret = scoutfs_radix_free(sb, alloc, wri, + le64_to_cpu(parent->hdr.blkno)); BUG_ON(ret); /* could have dirtied alloc to avoid error */ } @@ -803,7 +803,7 @@ static void inc_key(u8 *bytes, unsigned *len) * blocks themselves. */ static int btree_walk(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, int flags, void *key, unsigned key_len, @@ -1037,7 +1037,7 @@ static bool invalid_item(void *key, unsigned key_len, unsigned val_len) * length value. */ int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, @@ -1081,7 +1081,7 @@ int scoutfs_btree_insert(struct super_block *sb, * which doesn't fit. */ int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, @@ -1120,7 +1120,7 @@ int scoutfs_btree_update(struct super_block *sb, * which will insert instead of returning -ENOENT. */ int scoutfs_btree_force(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, @@ -1154,7 +1154,7 @@ int scoutfs_btree_force(struct super_block *sb, * found. */ int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len) @@ -1312,7 +1312,7 @@ int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root * <0 is returned on error, including -ENOENT if the key isn't present. */ int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len) diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 9278cec8..e37ab023 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -3,7 +3,7 @@ #include -struct scoutfs_balloc_allocator; +struct scoutfs_radix_allocator; struct scoutfs_block_writer; struct scoutfs_block; @@ -24,25 +24,25 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root void *key, unsigned key_len, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); int scoutfs_btree_force(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len, void *val, unsigned val_len); int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len); @@ -59,7 +59,7 @@ int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root void *key, unsigned key_len, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, void *key, unsigned key_len); diff --git a/kmod/src/data.c b/kmod/src/data.c index 802e7d47..4eaf701e 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -38,6 +38,7 @@ #include "file.h" #include "msg.h" #include "count.h" +#include "radix.h" /* * Logical file blocks are mapped to device blocks with extents stored @@ -53,33 +54,17 @@ * modified they can be packed back into the item. Typically there are * very few extents that cover the region. * - * Free blocks are tracked with bitmaps that are stored in items. Again - * the bitmaps are stored in a packed form and operated on in memory in - * a native form. Only 64bit words with a mix of set and clear bits are - * stored. The bitmaps are translated into long bitmaps in memory so we - * can use the kernel's long bitmap interfaces. - * - * There are two types of bitmap items: little bitmap bits track - * individual blocks and large bitmap bits track full little bitmap - * items. The logical packed extent item and little bitmap item sizes - * are chosen such that a full little bitmap represents a full packed - * extent item so we can allocate maximum size extents from the large - * bitmap bits. - * - * The client is given a tree of free block bitmap items from the server - * at the start of each transaction. The client allocates from items in - * an allocation tree and frees into items in a free tree. The server - * is responsible for filling the alloc tree and reclaiming the free - * tree as transactions are opened and committed. + * The client is given a radix allocator with trees for allocating + * blocks and recording frees at the start of each transaction. */ struct data_info { struct super_block *sb; struct rw_semaphore alloc_rwsem; - struct scoutfs_balloc_allocator *alloc; + struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; - struct scoutfs_balloc_root data_alloc; - struct scoutfs_balloc_root data_free; + struct scoutfs_radix_root data_avail; + struct scoutfs_radix_root data_freed; }; #define DECLARE_DATA_INFO(sb, name) \ @@ -140,11 +125,6 @@ static u64 ext_last(struct unpacked_extent *ext) return ext->iblock + ext->count - 1; } -static u64 bitmap_base(u64 blkno) -{ - return blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; -} - /* The first possible iblock in an item that contains the given iblock */ static u64 first_iblock(u64 iblock) { @@ -163,8 +143,8 @@ static u64 last_iblock(u64 iblock) * flags. * * We also require that a given extent's allocation be from only one - * bitmap item because the block bitmap clearing functions only operate - * on one item. + * radix bitmap leaf block because the radix freeing functions only + * operate on one leaf block. */ static bool extents_merge(struct unpacked_extent *left, struct unpacked_extent *right) @@ -173,7 +153,8 @@ static bool extents_merge(struct unpacked_extent *left, ((!left->blkno && !right->blkno) || (left->blkno + left->count == right->blkno)) && (left->flags == right->flags) && - (bitmap_base(left->blkno) == bitmap_base(right->blkno)); + (scoutfs_radix_bit_leaf_nr(left->blkno) == + scoutfs_radix_bit_leaf_nr(right->blkno + right->count - 1)); } static struct unpacked_extent *first_extent(struct unpacked_extents *unpe) @@ -708,6 +689,7 @@ static int set_extent(struct super_block *sb, struct inode *inode, return 0; } +#if 0 static bool block_bitmap_fits(u64 blkno, u64 count) { return ((blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK) + count) <= @@ -1298,6 +1280,7 @@ int scoutfs_data_add_free_blocks(struct super_block *sb, return ret; } +#endif /* * Find and remove or mark offline the block mappings that intersect @@ -1350,7 +1333,10 @@ static s64 truncate_extents(struct super_block *sb, struct inode *inode, if (ext->blkno) { down_write(&datinf->alloc_rwsem); - err = free_blocks(sb, &datinf->data_free, blkno, count); + err = scoutfs_radix_free_data(sb, datinf->alloc, + datinf->wri, + &datinf->data_freed, + blkno, count); up_write(&datinf->alloc_rwsem); if (err < 0) { ret = err; @@ -1482,11 +1468,11 @@ static int alloc_block(struct super_block *sb, struct inode *inode, const u64 ino = scoutfs_ino(inode); struct scoutfs_traced_extent te; u64 blkno = 0; - u64 count; u64 online; u64 offline; u64 last; u8 flags; + int count; int ret; int err; @@ -1517,11 +1503,13 @@ static int alloc_block(struct super_block *sb, struct inode *inode, /* only strictly contiguous extending writes will try to preallocate */ if (iblock > 1 && iblock == online) - count = min(iblock, count); + count = min_t(u64, iblock, count); else count = 1; - ret = alloc_blocks(sb, count, &blkno, &count); + ret = scoutfs_radix_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, count, &blkno, + &count); if (ret < 0) goto out; @@ -1551,7 +1539,9 @@ static int alloc_block(struct super_block *sb, struct inode *inode, out: if (ret < 0 && blkno > 0) { - err = free_blocks(sb, &datinf->data_alloc, blkno, count); + err = scoutfs_radix_free_data(sb, datinf->alloc, datinf->wri, + &datinf->data_freed, + blkno, count); BUG_ON(err); /* leaked free blocks */ } @@ -1946,7 +1936,7 @@ static int fallocate_extents(struct super_block *sb, struct inode *inode, struct unpacked_extent *ext; u8 ext_fl; u64 blkno; - u64 count; + int count; int done; int ret; int err; @@ -1971,7 +1961,8 @@ static int fallocate_extents(struct super_block *sb, struct inode *inode, } else if (ext->iblock <= iblock && ext->blkno) { /* skip portion of allocated extent */ - count = min(count, ext->count - (iblock - ext->iblock)); + count = min_t(u64, count, + ext->count - (iblock - ext->iblock)); iblock += count; done += count; ext = next_extent(ext); @@ -1979,23 +1970,28 @@ static int fallocate_extents(struct super_block *sb, struct inode *inode, } else if (ext->iblock <= iblock && !ext->blkno) { /* alloc portion of unallocated extent */ - count = min(count, ext->count - (iblock - ext->iblock)); + count = min_t(u64, count, + ext->count - (iblock - ext->iblock)); ext_fl = ext->flags; } else if (iblock < ext->iblock) { /* alloc hole until next extent */ - count = min(count, ext->iblock - iblock); + count = min_t(u64, count, ext->iblock - iblock); } down_write(&datinf->alloc_rwsem); - ret = alloc_blocks(sb, count, &blkno, &count); + ret = scoutfs_radix_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, count, + &blkno, &count); if (ret == 0) { ret = set_extent(sb, inode, ino, unpe, iblock, blkno, count, ext_fl | SEF_UNWRITTEN); if (ret < 0) { - err = free_blocks(sb, &datinf->data_alloc, - blkno, count); + err = scoutfs_radix_free_data(sb, datinf->alloc, + datinf->wri, + &datinf->data_avail, + blkno, count); BUG_ON(err); /* inconsistent */ } } @@ -2576,7 +2572,7 @@ const struct file_operations scoutfs_file_fops = { }; void scoutfs_data_init_btrees(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt) { @@ -2586,8 +2582,8 @@ void scoutfs_data_init_btrees(struct super_block *sb, datinf->alloc = alloc; datinf->wri = wri; - datinf->data_alloc = lt->data_alloc; - datinf->data_free = lt->data_free; + datinf->data_avail = lt->data_avail; + datinf->data_freed = lt->data_freed; up_write(&datinf->alloc_rwsem); } @@ -2599,8 +2595,8 @@ void scoutfs_data_get_btrees(struct super_block *sb, down_read(&datinf->alloc_rwsem); - lt->data_alloc = datinf->data_alloc; - lt->data_free = datinf->data_free; + lt->data_avail = datinf->data_avail; + lt->data_freed = datinf->data_freed; up_read(&datinf->alloc_rwsem); } diff --git a/kmod/src/data.h b/kmod/src/data.h index 912284d9..0da25fa1 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -45,7 +45,7 @@ struct scoutfs_traced_extent { extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; -struct scoutfs_balloc_allocator; +struct scoutfs_radix_allocator; struct scoutfs_block_writer; int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, @@ -72,6 +72,7 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); +#if 0 int scoutfs_data_move_alloc_bits(struct super_block *sb, struct scoutfs_balloc_allocator *alloc, struct scoutfs_block_writer *wri, @@ -83,8 +84,9 @@ int scoutfs_data_add_free_blocks(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_balloc_root *broot, u64 blkno, u64 count); +#endif void scoutfs_data_init_btrees(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt); void scoutfs_data_get_btrees(struct super_block *sb, diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 64e4a49e..a88d2119 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -21,7 +21,7 @@ #include "lock.h" #include "btree.h" #include "client.h" -#include "balloc.h" +#include "radix.h" #include "block.h" #include "forest.h" #include "scoutfs_trace.h" @@ -61,7 +61,7 @@ struct forest_info { struct rw_semaphore rwsem; - struct scoutfs_balloc_allocator *alloc; + struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; }; @@ -1158,22 +1158,21 @@ static int set_lock_bloom_bits(struct super_block *sb, if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { - ret = scoutfs_balloc_alloc(sb, finf->alloc, finf->wri, - &blkno); + ret = scoutfs_radix_alloc(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) goto unlock; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, - blkno); + err = scoutfs_radix_free(sb, finf->alloc, finf->wri, + blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); goto unlock; } if (bl) { - err = scoutfs_balloc_free(sb, finf->alloc, finf->wri, + err = scoutfs_radix_free(sb, finf->alloc, finf->wri, le64_to_cpu(ref->blkno)); BUG_ON(err); /* could have dirtied */ memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); @@ -1455,7 +1454,7 @@ void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) * serialized with all writers. */ void scoutfs_forest_init_btrees(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt) { diff --git a/kmod/src/forest.h b/kmod/src/forest.h index 7757d1f1..c82f1a2b 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -1,7 +1,7 @@ #ifndef _SCOUTFS_FOREST_H_ #define _SCOUTFS_FOREST_H_ -struct scoutfs_balloc_allocator; +struct scoutfs_radix_allocator; struct scoutfs_block_writer; int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, @@ -40,7 +40,7 @@ int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); void scoutfs_forest_init_btrees(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt); void scoutfs_forest_get_btrees(struct super_block *sb, diff --git a/kmod/src/format.h b/kmod/src/format.h index edb429b4..34f55252 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -246,6 +246,7 @@ struct scoutfs_btree_block { struct scoutfs_btree_item_header item_hdrs[0]; } __packed; +#if 0 /* * Free metadata blocks are tracked by block allocator items. */ @@ -294,6 +295,7 @@ struct scoutfs_packed_bitmap { __le64 set; __le64 words[0]; }; +#endif /* * The lock server keeps a persistent record of connected clients so that @@ -331,12 +333,12 @@ struct scoutfs_mounted_client_btree_val { * about item logs, it's about clients making changes to trees. */ struct scoutfs_log_trees { - struct scoutfs_balloc_root alloc_root; - struct scoutfs_balloc_root free_root; + struct scoutfs_radix_root meta_avail; + struct scoutfs_radix_root meta_freed; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; - struct scoutfs_balloc_root data_alloc; - struct scoutfs_balloc_root data_free; + struct scoutfs_radix_root data_avail; + struct scoutfs_radix_root data_freed; __le64 rid; __le64 nr; } __packed; @@ -347,12 +349,12 @@ struct scoutfs_log_trees_key { } __packed; struct scoutfs_log_trees_val { - struct scoutfs_balloc_root alloc_root; - struct scoutfs_balloc_root free_root; + struct scoutfs_radix_root meta_avail; + struct scoutfs_radix_root meta_freed; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; - struct scoutfs_balloc_root data_alloc; - struct scoutfs_balloc_root data_free; + struct scoutfs_radix_root data_avail; + struct scoutfs_radix_root data_freed; } __packed; struct scoutfs_log_item_value { @@ -527,25 +529,22 @@ struct scoutfs_super_block { __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 next_trans_seq; - __le64 total_blocks; - __le64 next_uninit_meta_blkno; - __le64 last_uninit_meta_blkno; - __le64 next_uninit_data_blkno; - __le64 last_uninit_data_blkno; - __le64 core_balloc_cursor; - __le64 core_data_alloc_cursor; + __le64 total_meta_blocks; /* both static and dynamic */ + __le64 first_meta_blkno; /* first dynamically allocated */ + __le64 last_meta_blkno; + __le64 total_data_blocks; + __le64 first_data_blkno; + __le64 last_data_blkno; __le64 free_blocks; - __le64 first_fs_blkno; - __le64 last_fs_blkno; __le64 quorum_fenced_term; __le64 quorum_server_term; __le64 unmount_barrier; __u8 quorum_count; struct scoutfs_inet_addr server_addr; - struct scoutfs_balloc_root core_balloc_alloc; - struct scoutfs_balloc_root core_balloc_free; - struct scoutfs_balloc_root core_data_alloc; - struct scoutfs_balloc_root core_data_free; + struct scoutfs_radix_root core_meta_avail; + struct scoutfs_radix_root core_meta_freed; + struct scoutfs_radix_root core_data_avail; + struct scoutfs_radix_root core_data_freed; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 2199bb23..f4bf64a0 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -20,7 +20,7 @@ #include "tseq.h" #include "spbm.h" #include "block.h" -#include "balloc.h" +#include "radix.h" #include "btree.h" #include "msg.h" #include "scoutfs_trace.h" @@ -87,7 +87,7 @@ struct lock_server_info { struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; - struct scoutfs_balloc_allocator *alloc; + struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; }; @@ -946,7 +946,7 @@ static void lock_server_tseq_show(struct seq_file *m, * we time them out. */ int scoutfs_lock_server_setup(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 784b2575..99c82b8d 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -12,7 +12,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid, int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); int scoutfs_lock_server_setup(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, + struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri); void scoutfs_lock_server_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index e08d15ac..c5b028fc 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -118,6 +118,7 @@ TRACE_EVENT(scoutfs_complete_truncate, __entry->flags) ); +#if 0 TRACE_EVENT(scoutfs_data_alloc_blocks, TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, u64 base, u8 type, int bit, u64 blkno, u64 count), @@ -182,6 +183,7 @@ TRACE_EVENT(scoutfs_data_free_blocks, SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, __entry->root_total_free, __entry->blkno, __entry->count) ); +#endif TRACE_EVENT(scoutfs_data_fallocate, TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, diff --git a/kmod/src/server.c b/kmod/src/server.c index 54dee331..93c89fa7 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -26,7 +26,7 @@ #include "counters.h" #include "inode.h" #include "block.h" -#include "balloc.h" +#include "radix.h" #include "btree.h" #include "scoutfs_trace.h" #include "msg.h" @@ -78,7 +78,7 @@ struct server_info { struct list_head farewell_requests; struct work_struct farewell_work; - struct scoutfs_balloc_allocator alloc; + struct scoutfs_radix_allocator alloc; struct scoutfs_block_writer wri; struct mutex logs_mutex; @@ -144,77 +144,6 @@ static inline int wait_for_commit(struct commit_waiter *cw) return cw->ret; } -/* - * Add newly initialized free metadata block allocator items to the core - * block allocator. This is called as we commit transactions in the - * server. It adds many more free blocks than is ever consumed by a - * transaction so this will stay ahead of the server's block allocation. - * The intent is to have a low constant overhead to initializing block - * allocators over time instead of requiring a large amount of IO during - * mkfs. - */ -static int add_uninit_balloc_items(struct super_block *sb, - struct server_info *server, - struct scoutfs_super_block *super) -{ - u64 next = le64_to_cpu(super->next_uninit_meta_blkno); - u64 last = le64_to_cpu(super->last_uninit_meta_blkno); - u64 nr; - int ret; - - if (next > last) - return 0; - - /* next_uninit should always start a new item */ - if (WARN_ON_ONCE(next & SCOUTFS_BALLOC_ITEM_BIT_MASK)) - return -EIO; - - nr = min_t(u64, last - next + 1, - round_up(512 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE, - SCOUTFS_BALLOC_ITEM_BITS)); - - ret = scoutfs_balloc_add_alloc_bulk(sb, &server->alloc, &server->wri, - next, nr); - if (ret == 0) - le64_add_cpu(&super->next_uninit_meta_blkno, nr); - - return ret; -} - -/* - * Add newly initialized free block bitmap items. - */ -static int add_uninit_data_alloc_items(struct super_block *sb, - struct server_info *server, - struct scoutfs_super_block *super) -{ - int nr = 16; - u64 next; - u64 last; - int ret; - - while (nr-- > 0) { - next = le64_to_cpu(super->next_uninit_data_blkno); - last = le64_to_cpu(super->last_uninit_data_blkno); - if (next > last) { - ret = 0; - break; - } - - ret = scoutfs_data_add_free_blocks(sb, &server->alloc, - &server->wri, - &super->core_data_alloc, - next, last - next + 1); - if (ret <= 0) - break; - - le64_add_cpu(&super->next_uninit_data_blkno, ret); - ret = 0; - } - - return ret; -} - /* * A core function of request processing is to modify the manifest and * allocator. Often the processing needs to make the modifications @@ -249,22 +178,14 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); - /* XXX not sure what to do about failure here */ - ret = add_uninit_balloc_items(sb, server, super); - BUG_ON(ret); - - /* XXX not sure what to do about failure here */ - ret = add_uninit_data_alloc_items(sb, server, super); - BUG_ON(ret); - ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); goto out; } - super->core_balloc_alloc = server->alloc.alloc_root; - super->core_balloc_free = server->alloc.free_root; + super->core_meta_avail = server->alloc.avail; + super->core_meta_freed = server->alloc.freed; ret = scoutfs_write_super(sb, super); if (ret) { @@ -342,10 +263,8 @@ static int server_get_log_trees(struct super_block *sb, struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees lt; struct commit_waiter cw; - u64 next_past; - u64 at_least; + u64 count; u64 target; - u64 from; int ret; if (arg_len != 0) { @@ -385,39 +304,37 @@ static int server_get_log_trees(struct super_block *sb, ltk.rid = cpu_to_be64(rid); ltk.nr = cpu_to_be64(1); memset(<v, 0, sizeof(ltv)); + scoutfs_radix_root_init(sb, <v.meta_avail, true); + scoutfs_radix_root_init(sb, <v.meta_freed, true); + scoutfs_radix_root_init(sb, <v.data_avail, false); + scoutfs_radix_root_init(sb, <v.data_freed, false); } /* ensure client has enough free metadata blocks for a transaction */ target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; - while (le64_to_cpu(ltv.alloc_root.total_free) < target) { - from = le64_to_cpu(super->core_balloc_cursor); - at_least = target - le64_to_cpu(ltv.alloc_root.total_free); + if (le64_to_cpu(ltv.meta_avail.ref.sm_total) < target) { + count = target - le64_to_cpu(ltv.meta_avail.ref.sm_total); - ret = scoutfs_balloc_move(sb, &server->alloc, &server->wri, - <v.alloc_root, - &server->alloc.alloc_root, - from, at_least, &next_past); - if (ret == -ENOENT && from != 0) { - super->core_balloc_cursor = 0; - continue; - } + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + <v.meta_avail, + &server->alloc.avail, + &server->alloc.avail, count); if (ret < 0) goto unlock; - - super->core_balloc_cursor = cpu_to_le64(next_past); } - /* fill client's data block allocator */ + /* ensure client has enough free data blocks for a transaction */ target = (2ULL*1024*1024*1024) / SCOUTFS_BLOCK_SIZE; - down_write(&server->alloc_rwsem); - ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, - <v.data_alloc, - &super->core_data_alloc, - &super->core_data_alloc_cursor, - target); - up_write(&server->alloc_rwsem); - if (ret < 0) - goto unlock; + if (le64_to_cpu(ltv.data_avail.ref.sm_total) < target) { + count = target - le64_to_cpu(ltv.data_avail.ref.sm_total); + + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + <v.data_avail, + &super->core_data_avail, + &super->core_data_avail, count); + if (ret < 0) + goto unlock; + } /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, @@ -433,12 +350,12 @@ unlock: ret = wait_for_commit(&cw); if (ret == 0) { - lt.alloc_root = ltv.alloc_root; - lt.free_root = ltv.free_root; + lt.meta_avail = ltv.meta_avail; + lt.meta_freed = ltv.meta_freed; lt.item_root = ltv.item_root; lt.bloom_ref = ltv.bloom_ref; - lt.data_alloc = ltv.data_alloc; - lt.data_free = ltv.data_free; + lt.data_avail = ltv.data_avail; + lt.data_freed = ltv.data_freed; lt.rid = be64_to_le64(ltk.rid); lt.nr = be64_to_le64(ltk.nr); } @@ -497,12 +414,12 @@ static int server_commit_log_trees(struct super_block *sb, /* XXX probably want to merge free blocks */ - ltv.alloc_root = lt->alloc_root; - ltv.free_root = lt->free_root; + ltv.meta_avail = lt->meta_avail; + ltv.meta_freed = lt->meta_freed; ltv.item_root = lt->item_root; ltv.bloom_ref = lt->bloom_ref; - ltv.data_alloc = lt->data_alloc; - ltv.data_free = lt->data_free; + ltv.data_avail = lt->data_avail; + ltv.data_freed = lt->data_freed; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), @@ -542,13 +459,8 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; - __le64 curs; - u64 tot; int ret; - memset(<k, 0, sizeof(ltk)); - memset(<v, 0, sizeof(ltv)); - mutex_lock(&server->logs_mutex); down_write(&server->alloc_rwsem); @@ -575,21 +487,17 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) goto out; } - tot = le64_to_cpu(super->core_data_alloc.total_free) + - le64_to_cpu(ltv.data_alloc.total_free); - curs = 0; - ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, - &super->core_data_alloc, - <v.data_alloc, &curs, tot); + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &super->core_data_avail, + <v.data_avail, <v.data_avail, + le64_to_cpu(ltv.data_avail.ref.sm_total)); if (ret < 0) goto out; - tot = le64_to_cpu(super->core_data_alloc.total_free) + - le64_to_cpu(ltv.data_free.total_free); - curs = 0; - ret = scoutfs_data_move_alloc_bits(sb, &server->alloc, &server->wri, - &super->core_data_alloc, - <v.data_free, &curs, tot); + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &super->core_data_avail, + <v.data_freed, <v.data_freed, + le64_to_cpu(ltv.data_freed.ref.sm_total)); out: up_write(&server->alloc_rwsem); mutex_unlock(&server->logs_mutex); @@ -801,7 +709,9 @@ static int server_statfs(struct super_block *sb, spin_unlock(&sbi->next_ino_lock); down_read(&server->alloc_rwsem); - nstatfs.total_blocks = super->total_blocks; + nstatfs.total_blocks = super->total_meta_blocks; + le64_add_cpu(&nstatfs.total_blocks, + le64_to_cpu(super->total_data_blocks)); nstatfs.bfree = super->free_blocks; up_read(&server->alloc_rwsem); ret = 0; @@ -1408,8 +1318,8 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret < 0) goto shutdown; - scoutfs_balloc_init(&server->alloc, &super->core_balloc_alloc, - &super->core_balloc_free); + scoutfs_radix_init_alloc(&server->alloc, &super->core_meta_avail, + &super->core_meta_freed); scoutfs_block_writer_init(sb, &server->wri); ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 6bd4c218..a5017821 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -25,7 +25,7 @@ #include "counters.h" #include "client.h" #include "inode.h" -#include "balloc.h" +#include "radix.h" #include "block.h" #include "scoutfs_trace.h" @@ -64,7 +64,7 @@ struct trans_info { bool writing; struct scoutfs_log_trees lt; - struct scoutfs_balloc_allocator alloc; + struct scoutfs_radix_allocator alloc; struct scoutfs_block_writer wri; }; @@ -89,8 +89,8 @@ static int commit_btrees(struct super_block *sb) struct scoutfs_log_trees lt; lt = tri->lt; - lt.alloc_root = tri->alloc.alloc_root; - lt.free_root = tri->alloc.free_root; + lt.meta_avail = tri->alloc.avail; + lt.meta_freed = tri->alloc.freed; scoutfs_forest_get_btrees(sb, <); scoutfs_data_get_btrees(sb, <); @@ -110,7 +110,8 @@ int scoutfs_trans_get_log_trees(struct super_block *sb) ret = scoutfs_client_get_log_trees(sb, <); if (ret == 0) { tri->lt = lt; - scoutfs_balloc_init(&tri->alloc, <.alloc_root, <.free_root); + scoutfs_radix_init_alloc(&tri->alloc, <.meta_avail, + <.meta_freed); scoutfs_block_writer_init(sb, &tri->wri); scoutfs_forest_init_btrees(sb, &tri->alloc, &tri->wri, <); From 300b7bc3ba3f7f2add66859404391376146a39a4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Feb 2020 15:59:27 -0800 Subject: [PATCH 782/920] scoutfs: remove allocators that used btree items Now that we have the allocators that use radix blocks we can remove all the code that was using btree items to store free block bitmaps. Signed-off-by: Zach Brown --- kmod/src/balloc.c | 630 --------------------------------------- kmod/src/balloc.h | 36 --- kmod/src/data.c | 593 ------------------------------------ kmod/src/data.h | 13 - kmod/src/format.h | 51 ---- kmod/src/scoutfs_trace.h | 67 ----- 6 files changed, 1390 deletions(-) delete mode 100644 kmod/src/balloc.c delete mode 100644 kmod/src/balloc.h diff --git a/kmod/src/balloc.c b/kmod/src/balloc.c deleted file mode 100644 index bcd03bd6..00000000 --- a/kmod/src/balloc.c +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Copyright (C) 2019 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "key.h" -#include "counters.h" -#include "msg.h" -#include "block.h" -#include "btree.h" -#include "per_task.h" -#include "balloc.h" - -#include "scoutfs_trace.h" - -/* - * scoutfs tracks free metadata blocks in bitmap items in allocation - * btrees. Most of the free metadata is operated on by the server and - * tracked in large core trees rooted in the super block. The server - * moves free items from the core trees to private trees for mounts. - * - * Allocation is performed by btrees which are performing cow updates. - * We can't write to stable blocks during a transaction, we can only - * write into free space in the previous stable fs image. This means - * that we can't satisfy dirty block allocations with frees of - * previously stable blocks in this transaction. We implement this by - * allocating from one tree and freeing into another. They're merged as - * the free blocks are committed and can be safely written to in the - * next transaction. - * - * We're allocating and freeing blocks on behalf of btree ops by calling - * btree ops. This would deadlock if we always called btree ops from - * the allocator directly, but instead we recognize recursion and have - * the called allocator hand blknos back to its calling allocator to - * store into btrees on its behalf. - * - * We use explicit allocation and writing contexts because both the - * client and server are working on independent allocation and item - * trees. - */ - -struct item_modification { - struct list_head entry; - u64 blkno; - u64 count; - int op; - struct scoutfs_balloc_root *root; - struct scoutfs_balloc_root *src; -}; - -static bool add_item_mod(struct list_head *list, int op, u64 blkno, u64 count, - struct scoutfs_balloc_root *root, - struct scoutfs_balloc_root *src) -{ - struct item_modification *im = kmalloc(sizeof(struct item_modification), - GFP_NOFS); - if (im) { - im->blkno = blkno; - im->count = count; - im->op = op; - im->root = root; - im->src = src; - list_add_tail(&im->entry, list); - return true; - } - - return false; -} - -/* make room to dirty two trees of an absurdly large height */ -#define MAX_BLKNOS (2 * ((32 * 2) + 1)) - -struct blkno_fifo { - int first; - int nr; - u64 blknos[MAX_BLKNOS]; -}; - -static inline void blkno_fifo_init(struct blkno_fifo *bf) -{ - bf->first = 0; - bf->nr = 0; -} - -static inline int blkno_fifo_nr(struct blkno_fifo *bf) -{ - BUG_ON(bf->nr < 0 || bf->nr > MAX_BLKNOS); - return bf->nr; -} - -static inline u64 blkno_fifo_out(struct blkno_fifo *bf) -{ - BUG_ON(blkno_fifo_nr(bf) == 0); - bf->nr--; - return bf->blknos[bf->first++]; -} - -static inline void blkno_fifo_in(struct blkno_fifo *bf, u64 blkno) -{ - unsigned int end = (bf->first + bf->nr) % MAX_BLKNOS; - - BUG_ON(blkno_fifo_nr(bf) == MAX_BLKNOS); - bf->blknos[end] = blkno; - bf->nr++; -} - -struct caller_blknos { - struct blkno_fifo free; - struct blkno_fifo alloced; - struct blkno_fifo freed; -}; - -/* - * Find a number of next free blknos from a starting point. We can land - * in the end of an empty item. If this returns 0 then nr_found have - * been found. - */ -static int find_next_free(struct super_block *sb, - struct scoutfs_balloc_root *root, u64 from, - u64 *found, unsigned int nr_found) -{ - struct scoutfs_balloc_item_key bik; - struct scoutfs_balloc_item_val biv; - SCOUTFS_BTREE_ITEM_REF(iref); - unsigned int f = 0; - unsigned int bit; - u64 base; - int ret = 0; - - while (f < nr_found) { - base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bit = from & SCOUTFS_BALLOC_ITEM_BIT_MASK; - bik.base = cpu_to_be64(base); - - ret = scoutfs_btree_next(sb, &root->root, - &bik, sizeof(bik), &iref); - if (ret < 0) /* including ENOENT */ - break; - - if (iref.key_len == sizeof(bik) && - iref.val_len == sizeof(biv)) { - memcpy(&bik, iref.key, iref.key_len); - memcpy(&biv, iref.val, iref.val_len); - - /* start from first bit in next whole item */ - if (be64_to_cpu(bik.base) != base) - bit = 0; - - while (f < nr_found) { - bit = find_next_bit_le(biv.bits, - SCOUTFS_BALLOC_ITEM_BITS, bit); - if (bit >= SCOUTFS_BALLOC_ITEM_BITS) - break; - - found[f++] = (be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + - bit; - bit++; - } - - from = (be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT) + bit; - ret = 0; - - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); - if (ret < 0) - break; - } - - return ret; -} - -/* - * Return the first blkno in the next item. Because from can land in an - * item we can return a blkno that is less than from. - */ -static int find_next_item(struct super_block *sb, - struct scoutfs_balloc_root *root, u64 from, - u64 *found) -{ - struct scoutfs_balloc_item_key bik; - SCOUTFS_BTREE_ITEM_REF(iref); - u64 base; - int ret; - - base = from >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bik.base = cpu_to_be64(base); - - ret = scoutfs_btree_next(sb, &root->root, &bik, sizeof(bik), &iref); - if (ret < 0) /* including ENOENT */ - goto out; - - if (iref.key_len == sizeof(struct scoutfs_balloc_item_key) && - iref.val_len == sizeof(struct scoutfs_balloc_item_val)) { - memcpy(&bik, iref.key, iref.key_len); - *found = be64_to_cpu(bik.base) << - SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - ret = 0; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); -out: - return ret; -} - -enum { - IM_OP_SET, - IM_OP_SET_BULK, - IM_OP_CLEAR, - IM_OP_MOVE, -}; - -static int copy_item_bits(struct scoutfs_balloc_item_val *biv, - struct scoutfs_btree_item_ref *iref, int ret, - bool *existed) -{ - if (ret < 0) { - if (ret == -ENOENT) { - memset(biv, 0, sizeof(struct scoutfs_balloc_item_val)); - if (existed) - *existed = false; - ret = 0; - } - } else { - if (iref->key_len == sizeof(struct scoutfs_balloc_item_key) && - iref->val_len == sizeof(struct scoutfs_balloc_item_val)) { - memcpy(biv, iref->val, iref->val_len); - if (existed) - *existed = true; - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(iref); - } - - return ret; -} - -/* - * We can use native longs to set aligned 64bit regions, but have to use - * individual _le calls on leading and trailing partial regions. - */ -static void bitmap_set_le(__le64 *map, int start, int nr) -{ - unsigned int full; - - while (start & 63 && nr-- > 0) - set_bit_le(start++, map); - - if (nr > 64) { - full = round_down(nr, 64); - bitmap_set((long *)map, start, full); - start += full; - nr -= full; - - } - - while (nr-- > 0) - set_bit_le(start++, map); -} - -/* - * Modify allocation item bits in service of the caller's operation. - * This has to be done very carefully so that we don't deadlock in - * recursion as btree dirtying calls back in to block allocation. - * - * A given btree operation can need to allocate blknos for dirty blocks - * and free the old clean blknos. The btree code will attempt to call - * balloc again. We add a per_task record of allocated and freed blknos - * which those allocation calls use instead of calling more btree ops. - * They then return to us and we perform the btree ops to satisfy those - * allocations and frees that were recorded. - * - * Each op that cows btree blocks generates more ops to records those - * allocations and frees. Eventually the ops hit existing dirty blocks - * and we can return. - */ -static int modify_items(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - int op, u64 blkno, u64 count, - struct scoutfs_balloc_root *root, - struct scoutfs_balloc_root *src, u64 next_free) -{ - SCOUTFS_DECLARE_PER_TASK_ENTRY(pt_ent); - struct scoutfs_balloc_item_key bik; - struct scoutfs_balloc_item_val biv; - struct scoutfs_balloc_item_val tmp; - struct item_modification *im; - SCOUTFS_BTREE_ITEM_REF(iref); - struct caller_blknos *cb; - unsigned int need_free; - unsigned int nr; - LIST_HEAD(mods); - u64 nexts[16]; - bool existed; - u64 base; - int bit; - int ret; - int i; - - /* doing native long ops on stack bits */ - BUILD_BUG_ON(offsetof(struct scoutfs_balloc_item_val, bits) % - (BITS_PER_LONG / 8)); - - cb = kmalloc(sizeof(struct caller_blknos), GFP_NOFS); - if (!cb) { - ret = -ENOMEM; - goto out; - } - - blkno_fifo_init(&cb->free); - blkno_fifo_init(&cb->alloced); - blkno_fifo_init(&cb->freed); - - scoutfs_per_task_add(&alloc->pt_caller_blknos, &pt_ent, cb); - - if (!add_item_mod(&mods, op, blkno, count, root, src)) { - ret = -ENOENT; - goto out; - } - - while ((im = list_first_entry_or_null(&mods, struct item_modification, - entry))) { - - base = im->blkno >> SCOUTFS_BALLOC_ITEM_BASE_SHIFT; - bik.base = cpu_to_be64(base); - existed = false; - - if (im->op != IM_OP_SET_BULK) { - /* get the current item to modify */ - ret = scoutfs_btree_lookup(sb, &im->root->root, - &bik, sizeof(bik), &iref); - ret = copy_item_bits(&biv, &iref, ret, &existed); - if (ret < 0) - goto out; - /* XXX corruption */ - BUG_ON(im->op == IM_OP_CLEAR && !existed); - } - - /* modify the item's bit */ - bit = im->blkno & SCOUTFS_BALLOC_ITEM_BIT_MASK; - if (im->op == IM_OP_SET) { - set_bit_le(bit, &biv.bits); - } else if (im->op == IM_OP_SET_BULK) { - memset(&biv, 0, sizeof(biv)); - bitmap_set_le(biv.bits, 0, im->count); - } else if (im->op == IM_OP_CLEAR) { - clear_bit_le(bit, &biv.bits); - } - - /* move just read the destination item, or in src item bits */ - if (im->op == IM_OP_MOVE) { - ret = scoutfs_btree_lookup(sb, &im->src->root, &bik, - sizeof(bik), &iref); - ret = copy_item_bits(&tmp, &iref, ret, NULL); - if (ret < 0) - goto out; - - /* shouldn't have free in both places */ - if (bitmap_intersects((long *)biv.bits, - (long *)tmp.bits, - SCOUTFS_BALLOC_ITEM_BITS)) { - ret = -EIO; - goto out; - } - bitmap_or((long *)biv.bits, (long *)biv.bits, - (long *)tmp.bits, SCOUTFS_BALLOC_ITEM_BITS); - } - - /* make sure we have enough free blocks for btree dirtying */ - need_free = (im->root->root.height * 2) + 1; - if (im->op == IM_OP_MOVE) - need_free += (im->src->root.height * 2) + 1; - - /* fill free fifo for potential dirtying */ - while (blkno_fifo_nr(&cb->free) < need_free) { - nr = min_t(int, need_free - blkno_fifo_nr(&cb->free), - ARRAY_SIZE(nexts)); - ret = find_next_free(sb, &alloc->alloc_root, next_free, - nexts, nr); - if (ret < 0) - goto out; - - next_free = nexts[nr - 1] + 1; - for (i = 0; i < nr; i++) - blkno_fifo_in(&cb->free, nexts[i]); - } - - /* - * Perform the op's item modifications, we go to do the - * trouble of differentiating between update and - * insertion instead of just using force so that we - * don't split when we don't need to. - */ - if (im->op == IM_OP_CLEAR && - bitmap_empty((long *)biv.bits, SCOUTFS_BALLOC_ITEM_BITS)) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik)); - else if (im->op == IM_OP_SET_BULK || - (im->op == IM_OP_SET && !existed)) - ret = scoutfs_btree_insert(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else if (im->op == IM_OP_MOVE && existed) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->src->root, - &bik, sizeof(bik)) ?: - scoutfs_btree_update(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else if (im->op == IM_OP_MOVE && !existed) - ret = scoutfs_btree_delete(sb, alloc, wri, - &im->src->root, - &bik, sizeof(bik)) ?: - scoutfs_btree_insert(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - else - ret = scoutfs_btree_update(sb, alloc, wri, - &im->root->root, - &bik, sizeof(bik), - &biv, sizeof(biv)); - if (ret < 0) - goto out; - - /* update bit counts to reflect op */ - if (im->op == IM_OP_SET) { - le64_add_cpu(&root->total_free, 1); - } else if (im->op == IM_OP_SET_BULK) { - le64_add_cpu(&root->total_free, im->count); - } else if (im->op == IM_OP_CLEAR) { - le64_add_cpu(&root->total_free, -1); - } else if (im->op == IM_OP_MOVE) { - nr = bitmap_weight((long *)biv.bits, - SCOUTFS_BALLOC_ITEM_BITS); - le64_add_cpu(&root->total_free, nr); - le64_add_cpu(&src->total_free, -nr); - } - - list_del(&im->entry); - kfree(im); - - /* and queue new modifications needed from btree ops */ - - while (blkno_fifo_nr(&cb->alloced)) { - if (!add_item_mod(&mods, IM_OP_CLEAR, - blkno_fifo_out(&cb->alloced), 0, - &alloc->alloc_root, NULL)) { - ret = -ENOENT; - goto out; - } - } - - while (blkno_fifo_nr(&cb->freed)) { - if (!add_item_mod(&mods, IM_OP_SET, - blkno_fifo_out(&cb->freed), 0, - &alloc->free_root, NULL)) { - ret = -ENOENT; - goto out; - } - } - } - - ret = 0; -out: - scoutfs_per_task_del(&alloc->pt_caller_blknos, &pt_ent); - BUG_ON(ret < 0); /* dirty block refs and bits are inconsistent */ - BUG_ON(!list_empty(&mods)); /* reminder to clean up */ - kfree(cb); - return ret; -} - -void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, - struct scoutfs_balloc_root *alloc_root, - struct scoutfs_balloc_root *free_root) -{ - mutex_init(&alloc->mutex); - scoutfs_per_task_init(&alloc->pt_caller_blknos); - alloc->alloc_root = *alloc_root; - alloc->free_root = *free_root; -} - -/* - * Add alloc items for a contiugous regions of blknos. The starting - * blkno must be aligned to the start of a bitmap item. Once these are - * added they can be used by the current transaction so the caller must - * be very careful that they're free. - */ -int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno, u64 count) -{ - u64 nr; - int ret = 0; - - mutex_lock(&alloc->mutex); - while (count > 0) { - nr = min_t(u64, count, SCOUTFS_BALLOC_ITEM_BITS), - ret = modify_items(sb, alloc, wri, IM_OP_SET_BULK, blkno, nr, - &alloc->alloc_root, NULL, 0); - if (ret < 0) - break; - blkno += nr; - count -= nr; - } - mutex_unlock(&alloc->mutex); - - return ret; -} - -int scoutfs_balloc_alloc(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno_ret) -{ - struct caller_blknos *cb; - u64 next; - int ret; - - /* if we're called by balloc then the caller works for us */ - cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); - if (cb) { - *blkno_ret = blkno_fifo_out(&cb->free); - blkno_fifo_in(&cb->alloced, *blkno_ret); - return 0; - } - - mutex_lock(&alloc->mutex); - ret = find_next_free(sb, &alloc->alloc_root, 0, &next, 1) ?: - modify_items(sb, alloc, wri, IM_OP_CLEAR, next, 0, - &alloc->alloc_root, NULL, next + 1); - mutex_unlock(&alloc->mutex); - - if (ret == 0) - *blkno_ret = next; - - return ret; -} - -int scoutfs_balloc_free(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno) -{ - struct caller_blknos *cb; - int ret; - - /* if we're called by balloc then the caller works for us */ - cb = scoutfs_per_task_get(&alloc->pt_caller_blknos); - if (cb) { - blkno_fifo_in(&cb->freed, blkno); - return 0; - } - - mutex_lock(&alloc->mutex); - ret = modify_items(sb, alloc, wri, IM_OP_SET, blkno, 0, - &alloc->free_root, NULL, 0); - mutex_unlock(&alloc->mutex); - - return ret; -} - -/* - * Move full items from the source to destination tree, moving at least - * the given number of blocks but likely more. - * - * This has to be done very carefully because we don't want to allocate - * dirty btree blocks from blknos in the source item that is moving. We - * find the first blkno in the next free item in the source tree so that - * we can start allocating dirty btree blocks after that item. - * - * This will not wrap the starting from blkno if it doesn't start at 0 - * and runs out of items. The caller is expected to deal with this. - */ -int scoutfs_balloc_move(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - u64 from, u64 at_least, u64 *next_past) -{ - u64 target; - u64 next; - int ret = 0; - - mutex_lock(&alloc->mutex); - - target = le64_to_cpu(dst->total_free) + at_least; - - while (le64_to_cpu(dst->total_free) < target && - le64_to_cpu(src->total_free) > 0) { - ret = find_next_item(sb, src, from, &next) ?: - modify_items(sb, alloc, wri, IM_OP_MOVE, next, 0, - dst, src, - next + SCOUTFS_BALLOC_ITEM_BITS); - if (ret < 0) - break; - - from = next + SCOUTFS_BALLOC_ITEM_BITS; - *next_past = from; - } - - mutex_unlock(&alloc->mutex); - - return ret; -} diff --git a/kmod/src/balloc.h b/kmod/src/balloc.h deleted file mode 100644 index 246a42a6..00000000 --- a/kmod/src/balloc.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef _SCOUTFS_BALLOC_H_ -#define _SCOUTFS_BALLOC_H_ - -#include "per_task.h" - -struct scoutfs_block_writer; - -struct scoutfs_balloc_allocator { - struct mutex mutex; - struct scoutfs_per_task pt_caller_blknos; - struct scoutfs_balloc_root alloc_root; - struct scoutfs_balloc_root free_root; -}; - -void scoutfs_balloc_init(struct scoutfs_balloc_allocator *alloc, - struct scoutfs_balloc_root *alloc_root, - struct scoutfs_balloc_root *free_root); -int scoutfs_balloc_add_alloc_bulk(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno, u64 count); -int scoutfs_balloc_alloc(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno_ret); -int scoutfs_balloc_free(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - u64 blkno); -int scoutfs_balloc_move(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - u64 from, u64 at_least, u64 *next_past); - -#endif diff --git a/kmod/src/data.c b/kmod/src/data.c index 4eaf701e..0750b441 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -689,599 +689,6 @@ static int set_extent(struct super_block *sb, struct inode *inode, return 0; } -#if 0 -static bool block_bitmap_fits(u64 blkno, u64 count) -{ - return ((blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK) + count) <= - SCOUTFS_BLOCK_BITMAP_BITS; -} - -static void block_bitmap_bit(u64 *base, int *bit, u64 blkno, u8 type) -{ - if (type == SCOUTFS_BLOCK_BITMAP_BIG) - blkno >>= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - *bit = blkno & SCOUTFS_BLOCK_BITMAP_BIT_MASK; - *base = blkno >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; -} - -static u64 block_bitmap_blkno(u64 base, int bit, u8 type) -{ - u64 blkno; - - blkno = (base << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT) + bit; - - if (type == SCOUTFS_BLOCK_BITMAP_BIG) - blkno <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - return blkno; -} - -struct block_bitmap { - u64 base; - u8 type; - bool exists; - unsigned long bits[DIV_ROUND_UP(SCOUTFS_BLOCK_BITMAP_BITS, - BITS_PER_LONG)]; -}; - -static inline __le64 long_bits_to_le64(unsigned long *bits, unsigned int i) -{ -#if BITS_PER_LONG == 64 - return cpu_to_le64(bits[i]); -#elif BITS_PER_LONG == 32 - i <<= 1; - return cpu_to_le64(bits[i] | ((u64)bits[i + 1] << 32)); -#else -#error "unexpected BITS_PER_LONG value?" -#endif -} - -static inline void u64_to_long_bits(unsigned long *bits, unsigned int i, u64 x) -{ -#if BITS_PER_LONG == 64 - bits[i] = x; -#else - i <<= 1; - bits[i] = x; - bits[i + 1] = x >> 32; -#endif -} - -/* - * Block bitmaps are unpacked into native long bitmaps in memory for use - * with the kernel's bitmap functions. This requires a bit of finesse - * to make sure that we translate the bits appropriately to - * architectures with different word size and endian. - */ -static int unpack_block_bitmap(struct block_bitmap *bb, - struct scoutfs_btree_item_ref *iref) -{ - struct scoutfs_block_bitmap_key *bbk; - struct scoutfs_packed_bitmap *pb; - unsigned int nr; - u64 present; - u64 set; - u64 b; - int ret; - int w; - int i; - - if (iref->key_len != sizeof(struct scoutfs_block_bitmap_key) || - iref->val_len < sizeof(struct scoutfs_packed_bitmap)) { - ret = -EIO; - goto out; - } - pb = iref->val; - - bbk = iref->key; - bb->type = bbk->type; - bb->base = be64_to_cpu(bbk->base); - - nr = hweight64(le64_to_cpu(pb->present)); - - if (iref->val_len != - offsetof(struct scoutfs_packed_bitmap, words[nr])) { - ret = -EIO; - goto out; - } - - present = le64_to_cpu(pb->present); - set = le64_to_cpu(pb->set); - w = 0; - for (i = 0, b = 1; - (present | set) && i < SCOUTFS_PACKED_BITMAP_WORDS; - i++, b <<= 1) { - if (set & b) - u64_to_long_bits(bb->bits, i, ~0ULL); - else if (present & b) - u64_to_long_bits(bb->bits, i, - le64_to_cpu(pb->words[w++])); - } - ret = 0; - -out: - return ret; -} - -static int load_block_bitmap(struct super_block *sb, - struct scoutfs_btree_root *root, - u64 blkno, u8 type, bool next, bool zero_enoent, - struct block_bitmap **bb_ret) -{ - struct scoutfs_block_bitmap_key bbk; - struct block_bitmap *bb = NULL; - SCOUTFS_BTREE_ITEM_REF(iref); - u64 base; - int bit; - int ret; - - bb = kzalloc(sizeof(struct block_bitmap), GFP_NOFS); - if (!bb) { - ret = -ENOMEM; - goto out; - } - - block_bitmap_bit(&base, &bit, blkno, type); - - bbk.type = type; - bbk.base = cpu_to_be64(base); - - if (next) - ret = scoutfs_btree_next(sb, root, &bbk, sizeof(bbk), &iref); - else - ret = scoutfs_btree_lookup(sb, root, &bbk, sizeof(bbk), &iref); - if (ret == 0) { - ret = unpack_block_bitmap(bb, &iref); - bb->exists = true; - scoutfs_btree_put_iref(&iref); - } - if (ret == -ENOENT && zero_enoent) { - bb->base = base; - bb->type = type; - ret = 0; - } - -out: - if (ret < 0) { - kfree(bb); - *bb_ret = NULL; - } else { - *bb_ret = bb; - } - return ret; -} - -/* - * Block bitmaps start with two flag words that indicate if logical - * words are all 0s, all 1s, or a mix of set and clear bits stored in - * the item payload. Typically the allocators will have long runs of - * set or clear bits so we don't store most of the bitmaps. Badly - * fragmented allocators will be (2/64 = ~3%) larger. - */ -static int pack_block_bitmap(struct scoutfs_packed_bitmap *pb, - struct block_bitmap *bb) -{ - __le64 word; - u64 present = 0; - u64 set = 0; - u64 b; - int w; - int i; - - w = 0; - for (i = 0, b = 1; i < SCOUTFS_PACKED_BITMAP_WORDS; i++, b <<= 1) { - word = long_bits_to_le64(bb->bits, i); - - if (word == cpu_to_le64(~0ULL)) { - set |= b; - } else if (word != 0) { - present |= b; - pb->words[w++] = word; - } - } - - pb->set = cpu_to_le64(set); - pb->present = cpu_to_le64(present); - - return offsetof(struct scoutfs_packed_bitmap, words[w]); -} - -static int store_block_bitmap(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, - struct block_bitmap *bb) -{ - struct scoutfs_block_bitmap_key bbk; - struct scoutfs_packed_bitmap *pb; - int size; - int ret; - - bbk.type = bb->type; - bbk.base = cpu_to_be64(bb->base); - - if (bitmap_empty(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS)) { - if (!bb->exists) { - ret = 0; - goto out; - } - - ret = scoutfs_btree_delete(sb, alloc, wri, root, - &bbk, sizeof(bbk)); - - } else { - pb = kmalloc(SCOUTFS_PACKED_BITMAP_MAX_BYTES, GFP_NOFS); - if (!pb) { - ret = -ENOMEM; - goto out; - } - - size = pack_block_bitmap(pb, bb); - - ret = scoutfs_btree_force(sb, alloc, wri, root, - &bbk, sizeof(bbk), pb, size); - kfree(pb); - if (ret == 0) - bb->exists = true; - } -out: - return ret; -} - -/* - * Set a region of bitmaps which must fit in one item. The caller's - * blkno is translated to an item base and then the number of bits are - * set. The caller is specifying a number of bits to set, not a block - * extent. - */ -static int set_block_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, u8 type, u64 blkno, - int nbits) -{ - struct block_bitmap *bb = NULL; - u64 base; - int bit; - int ret; - - if (WARN_ON_ONCE(!block_bitmap_fits(blkno, nbits))) - return -EINVAL; - - ret = load_block_bitmap(sb, root, blkno, type, false, true, &bb); - if (ret < 0) - goto out; - - block_bitmap_bit(&base, &bit, blkno, type); - - bitmap_set(bb->bits, bit, nbits); - - /* if a little bitmap is full, set it's big and delete it */ - if (type == SCOUTFS_BLOCK_BITMAP_LITTLE && - bitmap_full(bb->bits, SCOUTFS_PACKED_BITMAP_BITS)) { - ret = set_block_bits(sb, alloc, wri, root, - SCOUTFS_BLOCK_BITMAP_BIG, blkno, 1); - if (ret < 0) - goto out; - - bitmap_zero(bb->bits, SCOUTFS_PACKED_BITMAP_BITS); - } - - ret = store_block_bitmap(sb, alloc, wri, root, bb); - BUG_ON(ret < 0); /* cleared bit out of sync with existing littles */ - -out: - kfree(bb); - return ret; -} - -/* - * Find a region of free blocks for the caller. The caller can ask for - * an arbitrarily large extent but we'll only return at most a bitmap's - * worth of blocks from one allocation. - * - * Big bitmap items are stored before little items. This let's large - * allocations naturally fall back to being satisfied by little items - * when there are no more remaining big items. Small allocations first - * look for little items and then search again for big items that they - * can break up. - * - * We always simply look for the first free region. This is operating - * in the client on trees whose items are populated by the server - * between each transaction. The server is responsible for distributing - * the items such that the client tends to allocate across the device - * over time. - */ -static int alloc_blocks(struct super_block *sb, u64 count, u64 *blkno_ret, - u64 *count_ret) -{ - DECLARE_DATA_INFO(sb, datinf); - struct scoutfs_balloc_root *broot = &datinf->data_alloc; - struct block_bitmap *bb = NULL; - u64 blkno; - u8 type; - int bit; - int end; - int ret; - - if (WARN_ON_ONCE(count == 0)) - return -EINVAL; - - /* will only allocate from one block bitmap item at a time */ - count = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS); - - /* small allocations first look for little items, then check big */ - if (count < SCOUTFS_BLOCK_BITMAP_BITS) - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - else - type = SCOUTFS_BLOCK_BITMAP_BIG; - - do { - ret = load_block_bitmap(sb, &broot->root, 0, type, - true, false, &bb); - } while ((ret == -ENOENT && type == SCOUTFS_BLOCK_BITMAP_LITTLE) && - (type = SCOUTFS_BLOCK_BITMAP_BIG, 1)); - if (ret < 0) { - if (ret == -ENOENT) - ret = -ENOSPC; - goto out; - } - - bit = find_first_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS); - if (WARN_ON_ONCE(bit >= SCOUTFS_BLOCK_BITMAP_BITS)) { - ret = -EIO; /* stored items should have bits set */ - goto out; - } - - blkno = block_bitmap_blkno(bb->base, bit, bb->type); - - if (bb->type == SCOUTFS_BLOCK_BITMAP_BIG) { - /* set remaining little bits if using big for partial small */ - if (count != SCOUTFS_BLOCK_BITMAP_BITS) { - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, - SCOUTFS_BLOCK_BITMAP_LITTLE, - blkno + count, - SCOUTFS_BLOCK_BITMAP_BITS - count); - if (ret < 0) - goto out; - } - - clear_bit(bit, bb->bits); - - } else { - end = find_next_zero_bit(bb->bits, SCOUTFS_BLOCK_BITMAP_BITS, - bit + 1); - end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); /* catch > size */ - count = min_t(u64, count, end - bit); - - bitmap_clear(bb->bits, bit, count); - } - - ret = store_block_bitmap(sb, datinf->alloc, datinf->wri, - &broot->root, bb); - BUG_ON(ret < 0); /* little partial out of sync with big */ - - le64_add_cpu(&broot->total_free, -count); - - *blkno_ret = blkno; - *count_ret = count; - - trace_scoutfs_data_alloc_blocks(sb, broot, bb->base, bb->type, bit, - blkno, count); - -out: - kfree(bb); - return ret; -} - -/* - * Set free block bits in the block bitmaps and update the root's - * total_free count. The caller can specifiy the root so that this can - * be used both to free used allocations as well as to return unused - * allocations in error paths. The caller must ensure that the block - * regions fit in a single block bitmap (by for the blocks in an - * extent). - */ -static int free_blocks(struct super_block *sb, - struct scoutfs_balloc_root *broot, u64 blkno, u64 count) -{ - DECLARE_DATA_INFO(sb, datinf); - int ret; - - if (count == SCOUTFS_BLOCK_BITMAP_BITS) - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, SCOUTFS_BLOCK_BITMAP_BIG, - blkno, 1); - else - ret = set_block_bits(sb, datinf->alloc, datinf->wri, - &broot->root, SCOUTFS_BLOCK_BITMAP_LITTLE, - blkno, count); - - if (ret == 0) { - le64_add_cpu(&broot->total_free, count); - trace_scoutfs_data_free_blocks(sb, broot, blkno, count); - } - - return ret; -} - -/* - * Ensure that the destination free block bitmap tree has the minimum - * total free blocks by moving bits from the source tree. It will first - * try to find big bits starting at the cursor but will fall back to - * little bits after having wrapped the cursor. - * - * This will move all the items from the source to the destination if - * that's what it takes to reach the minimum. - * - * This is called by the server which provides its writer and metadata - * allocation contexts. It has locked the two allocation trees that - * will be modified. - */ -int scoutfs_data_move_alloc_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - __le64 *cursor, u64 min_dst_total) - -{ - struct block_bitmap *sbb = NULL; - struct block_bitmap *dbb = NULL; - u64 needed; - u64 blocks; - u64 moved; - u64 blkno; - u64 base; - u64 curs; - u8 type; - int nbits; - int bit; - int end; - int ret = 0; - - /* start moving big bitmap items */ - type = SCOUTFS_BLOCK_BITMAP_BIG; - curs = le64_to_cpup(cursor); - - while (le64_to_cpu(dst->total_free) < min_dst_total) { - - /* find the next source bitmap item with bits to move */ - kfree(sbb); - ret = load_block_bitmap(sb, &src->root, curs, type, - true, false, &sbb); - if (ret == 0 && sbb->type != type) - ret = -ENOENT; - if (ret < 0) { - if (ret == -ENOENT) { - if (curs > 0) { - curs = 0; - continue; - } - if (type == SCOUTFS_BLOCK_BITMAP_BIG) { - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - curs = le64_to_cpup(cursor); - continue; - } - ret = -ENOSPC; - } - break; - } - - /* load the destination bitmap */ - blkno = block_bitmap_blkno(sbb->base, 0, type); - kfree(dbb); - ret = load_block_bitmap(sb, &dst->root, blkno, type, - false, true, &dbb); - if (ret < 0) - break; - - /* figure out how many bits to move, can overshoot */ - needed = min_dst_total - le64_to_cpu(dst->total_free); - if (type == SCOUTFS_BLOCK_BITMAP_BIG) { - needed = (needed + SCOUTFS_BLOCK_BITMAP_BITS - 1) - >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - } - - /* start searching from the cursor if within item */ - if (curs > blkno) - blkno = curs; - block_bitmap_bit(&base, &bit, blkno, type); - - moved = 0; - while (moved < needed) { - bit = find_next_bit(sbb->bits, - SCOUTFS_BLOCK_BITMAP_BITS, bit); - if (bit >= SCOUTFS_BLOCK_BITMAP_BITS) - break; - - end = find_next_zero_bit(sbb->bits, - SCOUTFS_BLOCK_BITMAP_BITS, - bit + 1); - end = min(end, SCOUTFS_BLOCK_BITMAP_BITS); - nbits = min_t(u64, needed - moved, end - bit); - - bitmap_clear(sbb->bits, bit, nbits); - bitmap_set(dbb->bits, bit, nbits); - - curs = block_bitmap_blkno(dbb->base, bit + nbits, type); - moved += nbits; - } - - ret = store_block_bitmap(sb, alloc, wri, &dst->root, dbb); - if (ret < 0) - break; - - ret = store_block_bitmap(sb, alloc, wri, &src->root, sbb); - BUG_ON(ret); /* inconsistent src/dst, save orig src */ - - blocks = moved; - if (sbb->type == SCOUTFS_BLOCK_BITMAP_BIG) - blocks <<= SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - - le64_add_cpu(&dst->total_free, blocks); - le64_add_cpu(&src->total_free, -blocks); - - *cursor = cpu_to_le64(curs); - } - - kfree(sbb); - kfree(dbb); - - return ret; -} - -/* - * The server caller is making their way through free data blocks - * initializing free block bitmap bits for the first time. This is the - * only mechanism that initializes free block bitmap items so we know - * that we never have to merge with existing items as long as we always - * write a full item. - * - * The caller gives us the fully extent of blknos that we could - * initialize and we figure out the size of the largest item and its - * bits which cover the start of the extent. We can set big bits if the - * extent is aligned to a small bitmap and is large enough. - */ -int scoutfs_data_add_free_blocks(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *broot, - u64 blkno, u64 count) - -{ - u64 base; - u8 type; - int nbits; - int bit; - int ret; - - type = SCOUTFS_BLOCK_BITMAP_LITTLE; - block_bitmap_bit(&base, &bit, blkno, type); - - if (bit == 0 && count >= SCOUTFS_BLOCK_BITMAP_BITS) { - type = SCOUTFS_BLOCK_BITMAP_BIG; - block_bitmap_bit(&base, &bit, blkno, type); - nbits = min_t(u64, count >> SCOUTFS_BLOCK_BITMAP_BASE_SHIFT, - SCOUTFS_BLOCK_BITMAP_BITS - bit); - count = (u64)nbits << SCOUTFS_BLOCK_BITMAP_BASE_SHIFT; - } else { - nbits = min_t(u64, count, SCOUTFS_BLOCK_BITMAP_BITS - bit); - count = nbits; - } - - ret = set_block_bits(sb, alloc, wri, &broot->root, type, blkno, nbits); - if (ret == 0) { - le64_add_cpu(&broot->total_free, count); - ret = count; - } - - return ret; -} -#endif - /* * Find and remove or mark offline the block mappings that intersect * with the caller's range. The caller is responsible for transactions diff --git a/kmod/src/data.h b/kmod/src/data.h index 0da25fa1..895de61b 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -72,19 +72,6 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); -#if 0 -int scoutfs_data_move_alloc_bits(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *dst, - struct scoutfs_balloc_root *src, - __le64 *cursor, u64 min_dst_total); -int scoutfs_data_add_free_blocks(struct super_block *sb, - struct scoutfs_balloc_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_balloc_root *broot, - u64 blkno, u64 count); -#endif void scoutfs_data_init_btrees(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, diff --git a/kmod/src/format.h b/kmod/src/format.h index 34f55252..6cfb7322 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -246,57 +246,6 @@ struct scoutfs_btree_block { struct scoutfs_btree_item_header item_hdrs[0]; } __packed; -#if 0 -/* - * Free metadata blocks are tracked by block allocator items. - */ -struct scoutfs_balloc_root { - struct scoutfs_btree_root root; - __le64 total_free; -} __packed; -struct scoutfs_balloc_item_key { - __be64 base; -} __packed; - -#define SCOUTFS_BALLOC_ITEM_BYTES 256 -#define SCOUTFS_BALLOC_ITEM_U64S (SCOUTFS_BALLOC_ITEM_BYTES / \ - sizeof(__u64)) -#define SCOUTFS_BALLOC_ITEM_BITS (SCOUTFS_BALLOC_ITEM_BYTES * 8) -#define SCOUTFS_BALLOC_ITEM_BASE_SHIFT ilog2(SCOUTFS_BALLOC_ITEM_BITS) -#define SCOUTFS_BALLOC_ITEM_BIT_MASK (SCOUTFS_BALLOC_ITEM_BITS - 1) - -struct scoutfs_balloc_item_val { - __le64 bits[SCOUTFS_BALLOC_ITEM_U64S]; -} __packed; - -/* - * Free data blocks are tracked in bitmaps stored in btree items. - */ -struct scoutfs_block_bitmap_key { - __u8 type; - __be64 base; -} __packed; - -#define SCOUTFS_BLOCK_BITMAP_BIG 0 -#define SCOUTFS_BLOCK_BITMAP_LITTLE 1 - -#define SCOUTFS_PACKED_BITMAP_WORDS 32 -#define SCOUTFS_PACKED_BITMAP_BITS (SCOUTFS_PACKED_BITMAP_WORDS * 64) -#define SCOUTFS_PACKED_BITMAP_MAX_BYTES \ - offsetof(struct scoutfs_packed_bitmap, \ - words[SCOUTFS_PACKED_BITMAP_WORDS]) - -#define SCOUTFS_BLOCK_BITMAP_BITS SCOUTFS_PACKED_BITMAP_BITS -#define SCOUTFS_BLOCK_BITMAP_BIT_MASK (SCOUTFS_PACKED_BITMAP_BITS - 1) -#define SCOUTFS_BLOCK_BITMAP_BASE_SHIFT (ilog2(SCOUTFS_PACKED_BITMAP_BITS)) - -struct scoutfs_packed_bitmap { - __le64 present; - __le64 set; - __le64 words[0]; -}; -#endif - /* * The lock server keeps a persistent record of connected clients so that * server failover knows who to wait for before resuming operations. diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index c5b028fc..c4b4b48d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -118,73 +118,6 @@ TRACE_EVENT(scoutfs_complete_truncate, __entry->flags) ); -#if 0 -TRACE_EVENT(scoutfs_data_alloc_blocks, - TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, - u64 base, u8 type, int bit, u64 blkno, u64 count), - - TP_ARGS(sb, broot, base, type, bit, blkno, count), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(__u64, root_seq) - __field(__u64, root_total_free) - __field(__u64, base) - __field(u8, type) - __field(int, bit) - __field(__u64, blkno) - __field(__u64, count) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); - __entry->root_seq = le64_to_cpu(broot->root.ref.seq); - __entry->root_total_free = le64_to_cpu(broot->total_free); - __entry->base = base; - __entry->type = type; - __entry->bit = bit; - __entry->blkno = blkno; - __entry->count = count; - ), - - TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu base %llu type %u bit %d blkno %llu count %llu\n", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, - __entry->root_total_free, __entry->base, __entry->type, - __entry->bit, __entry->blkno, __entry->count) -); - -TRACE_EVENT(scoutfs_data_free_blocks, - TP_PROTO(struct super_block *sb, struct scoutfs_balloc_root *broot, - u64 blkno, u64 count), - - TP_ARGS(sb, broot, blkno, count), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(__u64, root_seq) - __field(__u64, root_total_free) - __field(__u64, blkno) - __field(__u64, count) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(broot->root.ref.blkno); - __entry->root_seq = le64_to_cpu(broot->root.ref.seq); - __entry->root_total_free = le64_to_cpu(broot->total_free); - __entry->blkno = blkno; - __entry->count = count; - ), - - TP_printk(SCSBF" root_blkno %llu root_seq %llu root_total_free %llu blkno %llu count %llu\n", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, - __entry->root_total_free, __entry->blkno, __entry->count) -); -#endif - TRACE_EVENT(scoutfs_data_fallocate, TP_PROTO(struct super_block *sb, u64 ino, int mode, loff_t offset, loff_t len, int ret), From 128a2c64f4ba6745c1aa68b8f18f4164c90ae836 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 23 Feb 2020 20:08:41 -0800 Subject: [PATCH 783/920] scoutfs: restore df/statfs block counts The removal of extent allocators in the server removed the tracking of total free blocks in the system as extents were allocated and freed. This restores tracking of total free blocks by observing the difference in each allocator's sm_total count as a new version is stored during a commit on the server. We change the single free_blocks counter in the super to separate counts of free metadata and data blocks to reflect the metadata and data allocators. The statfs net command is updated. Signed-off-by: Zach Brown --- kmod/src/format.h | 3 ++- kmod/src/server.c | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 6cfb7322..42eecc90 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -481,10 +481,11 @@ struct scoutfs_super_block { __le64 total_meta_blocks; /* both static and dynamic */ __le64 first_meta_blkno; /* first dynamically allocated */ __le64 last_meta_blkno; + __le64 free_meta_blocks; __le64 total_data_blocks; __le64 first_data_blkno; __le64 last_data_blkno; - __le64 free_blocks; + __le64 free_data_blocks; __le64 quorum_fenced_term; __le64 quorum_server_term; __le64 unmount_barrier; diff --git a/kmod/src/server.c b/kmod/src/server.c index 93c89fa7..b05ef812 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -144,6 +144,18 @@ static inline int wait_for_commit(struct commit_waiter *cw) return cw->ret; } +/* + * The caller is about to overwrite a ref to an alloc tree. As we do + * so we update the given super free block counter with the difference + * between the old and new allocator roots. + */ +static void update_free_blocks(__le64 *blocks, struct scoutfs_radix_root *prev, + struct scoutfs_radix_root *next) +{ + le64_add_cpu(blocks, le64_to_cpu(next->ref.sm_total) - + le64_to_cpu(prev->ref.sm_total)); +} + /* * A core function of request processing is to modify the manifest and * allocator. Often the processing needs to make the modifications @@ -184,6 +196,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } + update_free_blocks(&super->free_meta_blocks, &super->core_meta_avail, + &server->alloc.avail); + update_free_blocks(&super->free_meta_blocks, &super->core_meta_freed, + &server->alloc.freed); + super->core_meta_avail = server->alloc.avail; super->core_meta_freed = server->alloc.freed; @@ -414,6 +431,15 @@ static int server_commit_log_trees(struct super_block *sb, /* XXX probably want to merge free blocks */ + update_free_blocks(&super->free_meta_blocks, <v.meta_avail, + <->meta_avail); + update_free_blocks(&super->free_meta_blocks, <v.meta_freed, + <->meta_freed); + update_free_blocks(&super->free_data_blocks, <v.data_avail, + <->data_avail); + update_free_blocks(&super->free_data_blocks, <v.data_freed, + <->data_freed); + ltv.meta_avail = lt->meta_avail; ltv.meta_freed = lt->meta_freed; ltv.item_root = lt->item_root; @@ -712,7 +738,9 @@ static int server_statfs(struct super_block *sb, nstatfs.total_blocks = super->total_meta_blocks; le64_add_cpu(&nstatfs.total_blocks, le64_to_cpu(super->total_data_blocks)); - nstatfs.bfree = super->free_blocks; + nstatfs.bfree = super->free_meta_blocks; + le64_add_cpu(&nstatfs.bfree, + le64_to_cpu(super->free_data_blocks)); up_read(&server->alloc_rwsem); ret = 0; } else { From 5b6401b5cd32a916a92c6079a911304ab0d97a94 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sun, 23 Feb 2020 21:02:24 -0800 Subject: [PATCH 784/920] scoutfs: add missed btree block freeing The conversion of the btree to using allocators missed freeing blocks in two places. As we overwrite dirty new blocks we weren't freeing the old stable block as its reference was overwritten. And as we removed the final item in the tree we weren't freeing the final empty block as it's removed. Signed-off-by: Zach Brown --- kmod/src/btree.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index f31e44f5..0a7405b4 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -465,6 +465,19 @@ retry: } new = (void *)new_bl->data; + /* free old stable blkno we're about to overwrite */ + if (ref && ref->blkno) { + ret = scoutfs_radix_free(sb, alloc, wri, + le64_to_cpu(ref->blkno)); + if (ret) { + ret = scoutfs_radix_free(sb, alloc, wri, blkno); + BUG_ON(ret); /* radix should have been dirty */ + scoutfs_block_put(sb, new_bl); + new_bl = NULL; + goto out; + } + } + scoutfs_block_writer_mark_dirty(sb, wri, new_bl); trace_scoutfs_btree_dirty_block(sb, blkno, seq, @@ -1171,14 +1184,18 @@ int scoutfs_btree_delete(struct super_block *sb, bt = bl->data; pos = find_pos(bt, key, key_len, &cmp); if (cmp == 0) { - delete_item(bt, pos); - ret = 0; - - /* delete the final block in the tree */ - if (bt->nr_items == 0) { - root->height = 0; - root->ref.blkno = 0; - root->ref.seq = 0; + if (le32_to_cpu(bt->nr_items) == 1) { + /* remove final empty block */ + ret = scoutfs_radix_free(sb, alloc, wri, + bl->blkno); + if (ret == 0) { + root->height = 0; + root->ref.blkno = 0; + root->ref.seq = 0; + } + } else { + delete_item(bt, pos); + ret = 0; } } else { ret = -ENOENT; From ce7f7bdbd3f84694d4a14a717a7b4d237f153072 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Feb 2020 09:38:19 -0800 Subject: [PATCH 785/920] scoutfs: reclaim client log allocators The server now consistently reclaims free space in client allocator radix trees. It merges the client's freed trees as the client opens a new transaction. And it reclaims all the client's trees when it is removed. Signed-off-by: Zach Brown --- kmod/src/server.c | 59 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index b05ef812..ba6c0342 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -265,8 +265,14 @@ out: } /* - * Give the client references to stable persistent trees that they'll - * use to write their next transaction. + * Give the client roots to all the trees that they'll use to build + * their transaction. + * + * We make sure that their alloc trees have sufficient blocks to + * allocate metadata and data for the transaction. We merge their freed + * trees back into the core allocators. They're were committed with the + * previous transaction so they're stable and can now be reused, even by + * the server in this commit. */ static int server_get_log_trees(struct super_block *sb, struct scoutfs_net_connection *conn, @@ -327,6 +333,17 @@ static int server_get_log_trees(struct super_block *sb, scoutfs_radix_root_init(sb, <v.data_freed, false); } + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.avail, + <v.meta_freed, <v.meta_freed, + le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &super->core_data_avail, + <v.data_freed, <v.data_freed, + le64_to_cpu(ltv.data_freed.ref.sm_total)); + if (ret < 0) + goto unlock; + /* ensure client has enough free metadata blocks for a transaction */ target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; if (le64_to_cpu(ltv.meta_avail.ref.sm_total) < target) { @@ -429,8 +446,6 @@ static int server_commit_log_trees(struct super_block *sb, goto unlock; } - /* XXX probably want to merge free blocks */ - update_free_blocks(&super->free_meta_blocks, <v.meta_avail, <->meta_avail); update_free_blocks(&super->free_meta_blocks, <v.meta_freed, @@ -469,14 +484,17 @@ out: * log tree items. The item trees and bloom refs stay around to be read * and eventually merged and we reclaim all the allocator items. * - * The caller holds the commit rwsem which means we do all this work - * in one server commit. We'll need to keep the total amount of blocks - * in trees in check. + * The caller holds the commit rwsem which means we do all this work in + * one server commit. We'll need to keep the total amount of blocks in + * trees in check. * * By the time we're evicting a client they've either synced their data * or have been forcefully removed. The free blocks in the allocator * roots are stable and can be merged back into allocator items for use * without risking overwriting stable data. + * + * We can return an error without fully reclaiming all the log item's + * referenced data. */ static int reclaim_log_trees(struct super_block *sb, u64 rid) { @@ -486,6 +504,7 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; int ret; + int err; mutex_lock(&server->logs_mutex); down_write(&server->alloc_rwsem); @@ -513,17 +532,33 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) goto out; } + /* + * All of these can return errors after having modified the + * radix trees. We have to try and update the roots in the + * log item. + */ ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.avail, + <v.meta_avail, <v.meta_avail, + le64_to_cpu(ltv.meta_avail.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.avail, + <v.meta_freed, <v.meta_freed, + le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, &super->core_data_avail, <v.data_avail, <v.data_avail, - le64_to_cpu(ltv.data_avail.ref.sm_total)); - if (ret < 0) - goto out; - - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + le64_to_cpu(ltv.data_avail.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, &super->core_data_avail, <v.data_freed, <v.data_freed, le64_to_cpu(ltv.data_freed.ref.sm_total)); + + err = scoutfs_btree_update(sb, &server->alloc, &server->wri, + &super->logs_root, <k, sizeof(ltk), + <v, sizeof(ltv)); + BUG_ON(err != 0); /* alloc and log item roots out of sync */ + out: up_write(&server->alloc_rwsem); mutex_unlock(&server->logs_mutex); From 093f8ead58312dd0da3bd0eba0e943a8c455ff9e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Feb 2020 10:01:54 -0800 Subject: [PATCH 786/920] scoutfs: refactor server commit locking Server processing paths had open coded management of holding and applying transactions. Refactor that into hold_commit() and apply_commit() helpers. It makes the code a whole lot clearer and gives us a place in hold_commit() to add code that needs to be run before anything is modified in a commit on the server. Signed-off-by: Zach Brown --- kmod/src/server.c | 187 +++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 86 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index ba6c0342..39e38cd8 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -64,6 +64,7 @@ struct server_info { struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; struct work_struct commit_work; + bool prepared_commit; /* server tracks seq use */ struct rw_semaphore seq_rwsem; @@ -109,14 +110,44 @@ static void stop_server(struct server_info *server) } /* - * This is called while still holding the rwsem that prevents commits so - * that the caller can be sure to be woken by the next commit after they - * queue and release the lock. + * Hold the shared rwsem that lets multiple holders modify blocks in the + * current commit and prevents the commit worker from acquiring the + * exclusive write lock to write the commit. This can fail for the + * first holder failing to prepare a new commit. + */ +static int hold_commit(struct super_block *sb) +{ + DECLARE_SERVER_INFO(sb, server); + int ret = 0; + + down_read(&server->commit_rwsem); + + while (!server->prepared_commit) { + up_read(&server->commit_rwsem); + down_write(&server->commit_rwsem); + + server->prepared_commit = true; + ret = 0; + + up_write(&server->commit_rwsem); + if (ret) + break; + down_read(&server->commit_rwsem); + } + + return ret; +} + +/* + * This is called while holding the commit and returns once the commit + * is successfully written. Many holders can all wait for all holders + * to drain before their shared commit is applied and they're all woken. * - * It's important to realize that the caller's commit_waiter list node - * might be serviced by a currently running commit work while queueing - * another work run in the future. This caller can return from - * wait_for_commit() while the commit_work is still queued. + * It's important to realize that our commit_waiter list node might be + * serviced by a currently executing commit work that is blocked waiting + * for the holders to release the commit_rwsem. This caller can return + * from wait_for_commit() while another future commit_work is still + * queued. * * This could queue delayed work but we're first trying to have batching * work by having concurrent modification line up behind a commit in @@ -124,24 +155,26 @@ static void stop_server(struct server_info *server) * will race to make their changes and they'll all be applied by the * next commit after that. */ -static void queue_commit_work(struct server_info *server, - struct commit_waiter *cw) +static int apply_commit(struct super_block *sb, int err) { - lockdep_assert_held(&server->commit_rwsem); + DECLARE_SERVER_INFO(sb, server); + struct commit_waiter cw; - cw->ret = 0; - init_completion(&cw->comp); - llist_add(&cw->node, &server->commit_waiters); - queue_work(server->wq, &server->commit_work); -} + if (err == 0) { + cw.ret = 0; + init_completion(&cw.comp); + llist_add(&cw.node, &server->commit_waiters); + queue_work(server->wq, &server->commit_work); + } -/* - * Wait for a commit during request processing and return its status. - */ -static inline int wait_for_commit(struct commit_waiter *cw) -{ - wait_for_completion(&cw->comp); - return cw->ret; + up_read(&server->commit_rwsem); + + if (err == 0) { + wait_for_completion(&cw.comp); + err = cw.ret; + } + + return err; } /* @@ -157,11 +190,10 @@ static void update_free_blocks(__le64 *blocks, struct scoutfs_radix_root *prev, } /* - * A core function of request processing is to modify the manifest and - * allocator. Often the processing needs to make the modifications - * persistent before replying. We'd like to batch these commits as much - * as is reasonable so that we don't degrade to a few IO round trips per - * request. + * Concurrent request processing dirties blocks in a commit and makes + * the modifications persistent before replying. We'd like to batch + * these commits as much as is reasonable so that we don't degrade to a + * few IO round trips per request. * * Getting that batching right is bound up in the concurrency of request * processing so a clear way to implement the batched commits is to @@ -210,6 +242,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } + server->prepared_commit = false; ret = 0; out: node = llist_del_all(&server->commit_waiters); @@ -228,11 +261,9 @@ static int server_alloc_inodes(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) { - DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_net_inode_alloc ial = { 0, }; - struct commit_waiter cw; __le64 lecount; u64 ino; u64 nr; @@ -245,7 +276,9 @@ static int server_alloc_inodes(struct super_block *sb, memcpy(&lecount, arg, arg_len); - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; spin_lock(&sbi->next_ino_lock); ino = le64_to_cpu(super->next_ino); @@ -253,13 +286,11 @@ static int server_alloc_inodes(struct super_block *sb, le64_add_cpu(&super->next_ino, nr); spin_unlock(&sbi->next_ino_lock); - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - - ial.ino = cpu_to_le64(ino); - ial.nr = cpu_to_le64(nr); - - ret = wait_for_commit(&cw); + ret = apply_commit(sb, ret); + if (ret == 0) { + ial.ino = cpu_to_le64(ino); + ial.nr = cpu_to_le64(nr); + } out: return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); } @@ -285,7 +316,6 @@ static int server_get_log_trees(struct super_block *sb, struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees lt; - struct commit_waiter cw; u64 count; u64 target; int ret; @@ -295,7 +325,9 @@ static int server_get_log_trees(struct super_block *sb, goto out; } - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; mutex_lock(&server->logs_mutex); @@ -377,12 +409,7 @@ static int server_get_log_trees(struct super_block *sb, unlock: mutex_unlock(&server->logs_mutex); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); - + ret = apply_commit(sb, ret); if (ret == 0) { lt.meta_avail = ltv.meta_avail; lt.meta_freed = ltv.meta_freed; @@ -415,7 +442,6 @@ static int server_commit_log_trees(struct super_block *sb, struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees *lt; - struct commit_waiter cw; int ret; if (arg_len != sizeof(struct scoutfs_log_trees)) { @@ -424,7 +450,10 @@ static int server_commit_log_trees(struct super_block *sb, } lt = arg; - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret < 0) + goto out; + mutex_lock(&server->logs_mutex); /* find the client's existing item */ @@ -469,11 +498,7 @@ static int server_commit_log_trees(struct super_block *sb, unlock: mutex_unlock(&server->logs_mutex); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); + ret = apply_commit(sb, ret); out: WARN_ON_ONCE(ret < 0); return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); @@ -588,7 +613,6 @@ static int server_advance_seq(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct commit_waiter cw; __le64 their_seq; __le64 next_seq; struct scoutfs_trans_seq_btree_key tsk; @@ -601,7 +625,10 @@ static int server_advance_seq(struct super_block *sb, } memcpy(&their_seq, arg, sizeof(their_seq)); - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; + down_write(&server->seq_rwsem); if (their_seq != 0) { @@ -629,11 +656,7 @@ static int server_advance_seq(struct super_block *sb, &tsk, sizeof(tsk), NULL, 0); out: up_write(&server->seq_rwsem); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); + ret = apply_commit(sb, ret); return scoutfs_net_response(sb, conn, cmd, id, ret, &next_seq, sizeof(next_seq)); @@ -936,7 +959,6 @@ static int server_greeting(struct super_block *sb, struct scoutfs_net_greeting *gr = arg; struct scoutfs_net_greeting greet; DECLARE_SERVER_INFO(sb, server); - struct commit_waiter cw; __le64 umb = 0; bool reconnecting; bool first_contact; @@ -966,7 +988,9 @@ static int server_greeting(struct super_block *sb, } if (gr->server_term == 0) { - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret < 0) + goto send_err; spin_lock(&server->lock); umb = super->unmount_barrier; @@ -977,13 +1001,8 @@ static int server_greeting(struct super_block *sb, le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) { - ret = wait_for_commit(&cw); - queue_work(server->wq, &server->farewell_work); - } + ret = apply_commit(sb, ret); + queue_work(server->wq, &server->farewell_work); } else { umb = gr->unmount_barrier; } @@ -1021,15 +1040,13 @@ send_err: if (le64_to_cpu(gr->server_term) != server->term) { /* we're now doing two commits per greeting, not great */ - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), gr->server_term != 0); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); + ret = apply_commit(sb, ret); if (ret) goto out; } @@ -1089,7 +1106,6 @@ static void farewell_worker(struct work_struct *work) struct farewell_request *tmp; struct farewell_request *fw; SCOUTFS_BTREE_ITEM_REF(iref); - struct commit_waiter cw; unsigned int nr_unmounting = 0; unsigned int nr_mounted = 0; LIST_HEAD(reqs); @@ -1174,30 +1190,29 @@ static void farewell_worker(struct work_struct *work) /* process and send farewell responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: remove_trans_seq(sb, fw->rid) ?: reclaim_log_trees(sb, fw->rid) ?: delete_mounted_client(sb, fw->rid); - if (ret == 0) - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - if (ret == 0) - ret = wait_for_commit(&cw); + ret = apply_commit(sb, ret); if (ret) goto out; } /* update the unmount barrier if we deleted all voting clients */ if (deleted && nr_mounted == 0) { - down_read(&server->commit_rwsem); + ret = hold_commit(sb); + if (ret) + goto out; + le64_add_cpu(&super->unmount_barrier, 1); - queue_commit_work(server, &cw); - up_read(&server->commit_rwsem); - ret = wait_for_commit(&cw); + + ret = apply_commit(sb, ret); if (ret) goto out; } From 76ed6275480865460738c23a687a0b212d450847 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Feb 2020 10:10:09 -0800 Subject: [PATCH 787/920] scoutfs: reclaim freed metadata blocks in server Reclaim freed metadata blocks in the server by merging the stable freed tree into the allocator as a commit opens and we can trust that the stable version of the freed allocator in the super is a strict subset of the allocator's dirty freed tree. Signed-off-by: Zach Brown --- kmod/src/server.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index 39e38cd8..df6e62e8 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -114,10 +114,19 @@ static void stop_server(struct server_info *server) * current commit and prevents the commit worker from acquiring the * exclusive write lock to write the commit. This can fail for the * first holder failing to prepare a new commit. + * + * We reclaim the server's stable meta_freed blocks. This is run before + * anything has modified allocators in the server. We know that the + * stable meta_freed tree in the super contains all the stable free + * blocks which can be merged back into avail. We reference the stable + * freed tree in the super because the server allocator's freed tree is + * going to be added to as blocks are freed during the merge. */ static int hold_commit(struct super_block *sb) { + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); + u64 tot; int ret = 0; down_read(&server->commit_rwsem); @@ -126,12 +135,25 @@ static int hold_commit(struct super_block *sb) up_read(&server->commit_rwsem); down_write(&server->commit_rwsem); - server->prepared_commit = true; - ret = 0; + if (!server->prepared_commit) { + BUG_ON(scoutfs_block_writer_dirty_bytes(sb, + &server->wri)); + tot = le64_to_cpu(super->core_meta_freed.ref.sm_total); + + ret = scoutfs_radix_merge(sb, &server->alloc, + &server->wri, + &server->alloc.avail, + &server->alloc.freed, + &super->core_meta_freed, + true, tot); + if (ret == 0) + server->prepared_commit = true; + } up_write(&server->commit_rwsem); - if (ret) + if (ret < 0) break; + down_read(&server->commit_rwsem); } From 44a7e2ab56f575a6294f50955657d8554015eba4 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 24 Feb 2020 10:37:46 -0800 Subject: [PATCH 788/920] scoutfs: more carefully handle alloc cursors The first pass at the radix allocator wasn't paying a lot of attention to the allocation cursors. This more carefully manages them. They're only advanced after allocating. Previously the metadata alloc cursor was advanced as it searched through leaves that it might allocate from. We test for wrapping past the specific final allocatable bit, rather than the limit of what the radix height can store. This required pushing knoweldge of metadata or data allocs down through some of the code paths. Signed-off-by: Zach Brown --- kmod/src/radix.c | 70 +++++++++++++++++++---------------------------- kmod/src/radix.h | 2 +- kmod/src/server.c | 17 ++++++------ 3 files changed, 38 insertions(+), 51 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 981aae31..657c1936 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -357,19 +357,14 @@ static u64 bit_from_inds(struct radix_path *path) return bit; } -/* return the last bit that can be stored in a tree with the given height */ -static u64 last_from_height(u8 height) +static u64 last_from_super(struct super_block *sb, bool meta) { - u64 bit = SCOUTFS_RADIX_BITS - 1; - u64 mult = SCOUTFS_RADIX_BITS; - int i; + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - for (i = 1; i < U8_MAX; i++) { - bit += (u64)(SCOUTFS_RADIX_REFS - 1) * mult; - mult *= SCOUTFS_RADIX_REFS; - } - - return bit; + if (meta) + return le64_to_cpu(super->last_meta_blkno); + else + return le64_to_cpu(super->last_data_blkno); } static u8 height_from_last(u64 last) @@ -535,6 +530,14 @@ static void fixup_first_total(struct super_block *sb, struct radix_path *path, } } +static void store_next_find_bit(struct super_block *sb, bool meta, + struct scoutfs_radix_root *root, u64 bit) +{ + if (bit > last_from_super(sb, meta)) + bit = 0; + root->next_find_bit = cpu_to_le64(bit); +} + /* * Allocate (clear and return) a region of bits from the leaf block of a * path. The leaf walk has ensured that we have at least one block free. @@ -548,7 +551,8 @@ static void fixup_first_total(struct super_block *sb, struct radix_path *path, * This means that we can return recently freed blocks just behind the * next free cursor. I'm not sure if that's much of a problem. */ -static void alloc_leaf_bits(struct super_block *sb, struct radix_path *path, +static void alloc_leaf_bits(struct super_block *sb, bool meta, + struct radix_path *path, int nbits, u64 *bit_ret, int *nbits_ret) { struct scoutfs_radix_block *rdx = path->bls[0]->data; @@ -583,6 +587,8 @@ static void alloc_leaf_bits(struct super_block *sb, struct radix_path *path, *bit_ret = path->leaf_bit + ind; *nbits_ret = nbits; + + store_next_find_bit(sb, meta, path->root, path->leaf_bit + ind + nbits); } /* @@ -600,7 +606,7 @@ static u64 change_alloc_meta(struct super_block *sb, struct radix_change *chg) alloc_head); BUG_ON(!path); /* shouldn't be possible */ - alloc_leaf_bits(sb, path, 1, &bit, &nbits_ret); + alloc_leaf_bits(sb, true, path, 1, &bit, &nbits_ret); /* remove the path from the alloc list once its empty */ ref = path_ref(path, 0); @@ -904,7 +910,8 @@ static int get_all_paths(struct super_block *sb, if (chg->alloc_bits < chg->block_allocs + chg->caller_allocs) { stable = false; - if (next_meta == start_meta && meta_wrapped) { + /* we're not modifying as we go, check for wrapping */ + if (next_meta >= start_meta && meta_wrapped) { ret = -ENOSPC; break; } @@ -913,13 +920,9 @@ static int get_all_paths(struct super_block *sb, next_meta, &adding); if (ret < 0) { if (ret == -ENOENT) { - if (next_meta != 0) { - next_meta = 0; - meta_wrapped = true; - continue; - } else { - ret = -ENOSPC; - } + meta_wrapped = true; + next_meta = 0; + continue; } break; } @@ -998,8 +1001,6 @@ static int get_all_paths(struct super_block *sb, ret = 0; } while (!stable); - alloc->avail.next_find_bit = cpu_to_le64(next_meta); - return ret; } @@ -1051,13 +1052,6 @@ static void dirty_all_path_blocks(struct super_block *sb, } } -static void store_next_find_bit(struct scoutfs_radix_root *root, u64 bit) -{ - if (bit > last_from_height(root->height)) - bit = 0; - root->next_find_bit = cpu_to_le64(bit); -} - static bool valid_free_bit_range(struct super_block *sb, bool meta, u64 bit, int nbits) { @@ -1208,14 +1202,12 @@ find_next: } list_add_tail(&path->head, &chg->new_paths); - store_next_find_bit(root, bit); - ret = get_all_paths(sb, alloc, chg); if (ret < 0) goto out; dirty_all_path_blocks(sb, alloc, wri, chg); - alloc_leaf_bits(sb, path, nbits, blkno_ret, count_ret); + alloc_leaf_bits(sb, false, path, nbits, blkno_ret, count_ret); ret = 0; out: free_change(sb, chg); @@ -1285,7 +1277,7 @@ int scoutfs_radix_merge(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_radix_root *dst, struct scoutfs_radix_root *src, - struct scoutfs_radix_root *inp, u64 count) + struct scoutfs_radix_root *inp, bool meta, u64 count) { struct scoutfs_radix_block *inp_rdx; struct scoutfs_radix_block *src_rdx; @@ -1402,7 +1394,7 @@ wrapped: free_change(sb, chg); chg = NULL; - store_next_find_bit(src, bit + SCOUTFS_RADIX_BITS); + store_next_find_bit(sb, meta, src, bit + SCOUTFS_RADIX_BITS); count -= min_t(u64, count, sm_delta); } @@ -1431,13 +1423,7 @@ void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, void scoutfs_radix_root_init(struct super_block *sb, struct scoutfs_radix_root *root, bool meta) { - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - u64 last; - - if (meta) - last = le64_to_cpu(super->last_meta_blkno); - else - last = le64_to_cpu(super->last_data_blkno); + u64 last = last_from_super(sb, meta); root->height = height_from_last(last); root->next_find_bit = cpu_to_le64(0); diff --git a/kmod/src/radix.h b/kmod/src/radix.h index 15433bb0..982797e7 100644 --- a/kmod/src/radix.h +++ b/kmod/src/radix.h @@ -32,7 +32,7 @@ int scoutfs_radix_merge(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_radix_root *dst, struct scoutfs_radix_root *src, - struct scoutfs_radix_root *inp, u64 count); + struct scoutfs_radix_root *inp, bool meta, u64 count); void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, struct scoutfs_radix_root *avail, struct scoutfs_radix_root *freed); diff --git a/kmod/src/server.c b/kmod/src/server.c index df6e62e8..d0106c8a 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -389,11 +389,11 @@ static int server_get_log_trees(struct super_block *sb, ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, &server->alloc.avail, - <v.meta_freed, <v.meta_freed, + <v.meta_freed, <v.meta_freed, true, le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: scoutfs_radix_merge(sb, &server->alloc, &server->wri, &super->core_data_avail, - <v.data_freed, <v.data_freed, + <v.data_freed, <v.data_freed, false, le64_to_cpu(ltv.data_freed.ref.sm_total)); if (ret < 0) goto unlock; @@ -406,7 +406,7 @@ static int server_get_log_trees(struct super_block *sb, ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, <v.meta_avail, &server->alloc.avail, - &server->alloc.avail, count); + &server->alloc.avail, true, count); if (ret < 0) goto unlock; } @@ -419,7 +419,8 @@ static int server_get_log_trees(struct super_block *sb, ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, <v.data_avail, &super->core_data_avail, - &super->core_data_avail, count); + &super->core_data_avail, false, + count); if (ret < 0) goto unlock; } @@ -586,19 +587,19 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) */ ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, &server->alloc.avail, - <v.meta_avail, <v.meta_avail, + <v.meta_avail, <v.meta_avail, true, le64_to_cpu(ltv.meta_avail.ref.sm_total)) ?: scoutfs_radix_merge(sb, &server->alloc, &server->wri, &server->alloc.avail, - <v.meta_freed, <v.meta_freed, + <v.meta_freed, <v.meta_freed, true, le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: scoutfs_radix_merge(sb, &server->alloc, &server->wri, &super->core_data_avail, - <v.data_avail, <v.data_avail, + <v.data_avail, <v.data_avail, false, le64_to_cpu(ltv.data_avail.ref.sm_total)) ?: scoutfs_radix_merge(sb, &server->alloc, &server->wri, &super->core_data_avail, - <v.data_freed, <v.data_freed, + <v.data_freed, <v.data_freed, false, le64_to_cpu(ltv.data_freed.ref.sm_total)); err = scoutfs_btree_update(sb, &server->alloc, &server->wri, From 757ee855206b74dac4e2702100f5d565cfc17c76 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 25 Feb 2020 17:10:39 -0800 Subject: [PATCH 789/920] scoutfs: don't lose block wakeups The block end_io path could lose wakeups. Both the bio submission task and a bio's end_io completion could see an io_count > 1 and neither would set the block uptodate before dropping their io_count and waking. It got into this mess because readers were waiting for io_count to drop to 0. We add a io_busy bit which indicates that io is still in flight which waiters now wait for. This gives the final io_count drop a chance to do work before clearing io_busy and dropping their reference before waking. Signed-off-by: Zach Brown --- kmod/src/block.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index eb2c3b2e..9a004407 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -63,6 +63,7 @@ enum { BLOCK_BIT_UPTODATE = 0, /* contents consistent with media */ BLOCK_BIT_NEW, /* newly allocated, contents undefined */ BLOCK_BIT_DIRTY, /* dirty, writer will write */ + BLOCK_BIT_IO_BUSY, /* bios are in flight */ BLOCK_BIT_ERROR, /* saw IO error */ BLOCK_BIT_DELETED, /* has been deleted from rbtree */ BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ @@ -354,6 +355,12 @@ static void block_remove(struct super_block *sb, struct block_private *bp) } } +static bool io_busy(struct block_private *bp) +{ + smp_rmb(); /* test after adding to wait queue */ + return test_bit(BLOCK_BIT_IO_BUSY, &bp->bits); +} + /* * Called during shutdown with no other users. */ @@ -366,7 +373,7 @@ static void block_remove_all(struct super_block *sb) for (node = rb_first(&binf->root); node; ) { bp = container_of(node, struct block_private, node); node = rb_next(node); - wait_event(binf->waitq, atomic_read(&bp->io_count) == 0); + wait_event(binf->waitq, !io_busy(bp)); block_remove(sb, bp); } @@ -395,17 +402,19 @@ static void block_end_io(struct super_block *sb, int rw, set_bit(BLOCK_BIT_ERROR, &bp->bits); } - /* update bits before waiters see io_count == 0 */ - if (atomic_read(&bp->io_count) == 1) { - if (is_read && !test_bit(BLOCK_BIT_ERROR, &bp->bits)) - set_bit(BLOCK_BIT_UPTODATE, &bp->bits); - } + if (!atomic_dec_and_test(&bp->io_count)) + return; - /* make sure bits are visible to woken */ - smp_mb__after_atomic(); + if (is_read && !test_bit(BLOCK_BIT_ERROR, &bp->bits)) + set_bit(BLOCK_BIT_UPTODATE, &bp->bits); - /* then wake */ - if (atomic_dec_and_test(&bp->io_count)) + clear_bit(BLOCK_BIT_IO_BUSY, &bp->bits); + block_put(sb, bp); + + /* make sure set and cleared bits are visible to woken */ + smp_mb(); + + if (waitqueue_active(&binf->waitq)) wake_up(&binf->waitq); } @@ -414,11 +423,9 @@ static void block_bio_end_io(struct bio *bio, int err) struct block_private *bp = bio->bi_private; struct super_block *sb = bp->sb; - + TRACE_BLOCK(end_io, bp); block_end_io(sb, bio->bi_rw, bp, err); bio_put(bio); - TRACE_BLOCK(end_io, bp); - block_put(sb, bp); } /* @@ -441,6 +448,8 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, /* don't let racing end_io during submission think block is complete */ atomic_inc(&bp->io_count); + set_bit(BLOCK_BIT_IO_BUSY, &bp->bits); + atomic_inc(&bp->refcount); blk_start_plug(&plug); @@ -457,7 +466,6 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, bio->bi_end_io = block_bio_end_io; bio->bi_private = bp; - atomic_inc(&bp->refcount); atomic_inc(&bp->io_count); TRACE_BLOCK(submit, bp); @@ -732,7 +740,7 @@ int scoutfs_block_writer_write(struct super_block *sb, list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { /* XXX should this be interruptible? */ - wait_event(binf->waitq, atomic_read(&bp->io_count) == 0); + wait_event(binf->waitq, !io_busy(bp)); if (ret == 0 && test_bit(BLOCK_BIT_ERROR, &bp->bits)) { clear_bit(BLOCK_BIT_ERROR, &bp->bits); ret = -EIO; From c10c7d97482247da8517c685cae9401245661b61 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 Feb 2020 14:33:28 -0800 Subject: [PATCH 790/920] scoutfs: clean up forest lock data The client lock code forgot to call into the forest to clear its per-lock tracking before freeing the lock. This would result in a slow memory leak over time as locks were reclaimed by memory pressure. It shouldn't have affected consistency. Signed-off-by: Zach Brown --- kmod/src/lock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/lock.c b/kmod/src/lock.c index cdb6e561..1faa8df0 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -222,6 +222,7 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!list_empty(&lock->lru_head)); BUG_ON(!list_empty(&lock->cov_list)); + scoutfs_forest_clear_lock(sb, lock); kfree(lock); } From 6eac823bd31118c8ba47fd27b739b193af8c5782 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 Feb 2020 15:03:20 -0800 Subject: [PATCH 791/920] scoutfs: add radix block metadata checker Add a quick runtime check of the consistency of the radix block and reference metadata fields. Signed-off-by: Zach Brown --- kmod/src/radix.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 657c1936..13f3f170 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -456,6 +456,61 @@ static struct radix_path *walk_paths(struct rb_root *rbroot, return NULL; } +/* + * Make sure radix metadata is consistent. + */ +static void check_first_total(struct radix_path *path) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *ref; + int level; + u64 st; + u64 lt; + u32 sf; + u32 lf; + int i; + + for (level = 0; level < path->height; level++) { + rdx = path->bls[level]->data; + ref = path_ref(path, level); + + if (level == 0) { + st = bitmap_weight((long *)rdx->bits, + SCOUTFS_RADIX_BITS); + lt = count_lg_bits(rdx->bits, 0, SCOUTFS_RADIX_BITS); + + sf = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, 0); + lf = find_next_lg(rdx->bits, 0); + } else { + st = 0; + lt = 0; + sf = SCOUTFS_RADIX_REFS; + lf = SCOUTFS_RADIX_REFS; + for (i = 0; i < SCOUTFS_RADIX_REFS; i++) { + st += le64_to_cpu(rdx->refs[i].sm_total); + lt += le64_to_cpu(rdx->refs[i].lg_total); + if (rdx->refs[i].sm_total != 0 && i < sf) + sf = i; + if (rdx->refs[i].lg_total != 0 && i < lf) + lf = i; + } + } + + if (le64_to_cpu(ref->sm_total) != st || + le64_to_cpu(ref->lg_total) != lt || + le32_to_cpu(rdx->sm_first) > sf || + le32_to_cpu(rdx->lg_first) > lf) { + printk("radix inconsistency: level %u calced sf %u st %llu lf %u lt %llu, stored sf %u st %llu lf %u lt %llu\n", + level, sf, st, lf, lt, + le32_to_cpu(rdx->sm_first), + le64_to_cpu(ref->sm_total), + le32_to_cpu(rdx->lg_first), + le64_to_cpu(ref->lg_total)); + BUG(); + } + } +} + /* * Update the first tracking in a block after the caller has modified * the block at the given index. If the modification at the index is @@ -528,6 +583,9 @@ static void fixup_first_total(struct super_block *sb, struct radix_path *path, ref->lg_total == 0); } } + + if (0) /* expensive, would be nice to make conditional */ + check_first_total(path); } static void store_next_find_bit(struct super_block *sb, bool meta, From d374a7c06f83e0b8d393e295c5a605da3422833a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 26 Feb 2020 17:12:06 -0800 Subject: [PATCH 792/920] scoutfs: fix up radix block _first tracking Updating the _first tracking in leaf bits was pretty confusing because we tried to mashing all the tracking updates from all leaf modifications into one shared code path. It had a bug where merging would advance _first tracking by the number of bits merged in the leaf rather than the number of contiguous set bits after the new first. This lead to allocation failures eventually as _first was after actual set bits in the leaf. This fixes that by moving _first tracking updates into the leaf callers that modify bits and to the parent ref updating code. In the process we also fix little bugs in the support code that were found by the radix block consistency checking. Signed-off-by: Zach Brown --- kmod/src/radix.c | 134 ++++++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 71 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 13f3f170..51214eef 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -51,8 +51,9 @@ * merge process. * * Allocations search for the next free bit from a cursor that's stored - * in the root of each tree. We track the next set parent ref or leaf - * bit in references to blocks to avoid searching entire blocks. + * in the root of each tree. We track the first set parent ref or leaf + * bit in references to blocks to avoid searching entire blocks every + * time. * * The radix isn't always fully populated. References can contain * blknos with 0 or ~0 to indicate that its referenced subtree is either @@ -296,14 +297,15 @@ static u64 count_lg_bits(void *bits, int ind, int nbits) { u64 count = 0; int end; + int i; - ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); + i = round_down(ind, SCOUTFS_RADIX_LG_BITS); end = round_up(ind + nbits, SCOUTFS_RADIX_LG_BITS); - while (ind < end) { - if (lg_is_full(bits, ind)) + while (i < end) { + if (lg_is_full(bits, i)) count += SCOUTFS_RADIX_LG_BITS; - ind += SCOUTFS_RADIX_LG_BITS; + i += SCOUTFS_RADIX_LG_BITS; } return count; @@ -319,7 +321,7 @@ static u64 count_lg_bitmap(void *result, void *input) u64 count = 0; int ind = 0; - while ((ind = find_next_bit(input, SCOUTFS_RADIX_BITS, ind)) + while ((ind = find_next_bit_le(input, SCOUTFS_RADIX_BITS, ind)) < SCOUTFS_RADIX_BITS) { if (lg_is_full(result, ind)) count += SCOUTFS_RADIX_LG_BITS; @@ -511,76 +513,42 @@ static void check_first_total(struct radix_path *path) } } -/* - * Update the first tracking in a block after the caller has modified - * the block at the given index. If the modification at the index is - * populated we check to see if first should be earler. If the - * modification at the index is now empty (cleared leaf bits or parent - * ref total going to 0) we can advance the first tracker by an offset - * (number of bits in the leaf, to the next ref in parents), or set - * first to the end of the block if the entire block is now empty. - * - * The first field is only guaranteed to be before the first set region, - * if there is any. It's only advanced when the current first is - * cleared. If regions are cleared out of order then you can be left - * with a first less than the limit in a block with none set. - */ -static void update_first(__le32 *first, int ind, u32 limit, s32 offset, - bool empty_ind, bool entirely_empty) -{ - if (!empty_ind) { - if (ind < le32_to_cpup(first)) - *first = cpu_to_le32(ind); - - } else { - if (entirely_empty) - *first = cpu_to_le32(limit); - else if (ind == le32_to_cpup(first)) - le32_add_cpu(first, offset); - } -} +#define set_first_nonzero_ref(rdx, ind, first, total) \ +do { \ + int _ind = min_t(u32, le32_to_cpu(rdx->first), (ind)); \ + \ + while (_ind < SCOUTFS_RADIX_REFS && rdx->refs[_ind].total == 0) \ + _ind++; \ + \ + rdx->first = cpu_to_le32(_ind); \ +} while (0) /* - * The caller has changed bits in a leaf block. We update the first - * fields in the block header and the total fields in block references. + * The caller has changed bits in a leaf block and updated the block's + * first tracking. We update the first tracking and totals in parent + * blocks and refs up to the root ref. We do this after modifying + * leaves, instead of during descent, because we descend through clean + * blocks and then dirty all he blocks in all the paths before modifying + * leaves. */ -static void fixup_first_total(struct super_block *sb, struct radix_path *path, - int ind, s64 sm_delta, s64 lg_delta) +static void fixup_parent_refs(struct radix_path *path, + s64 sm_delta, s64 lg_delta) { struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *rdx_ref; struct scoutfs_radix_ref *ref; int level; + int ind; for (level = 0; level < path->height; level++) { rdx = path->bls[level]->data; ref = path_ref(path, level); - if (level > 0) { - ind = path->inds[level]; - rdx_ref = &rdx->refs[ind]; - } le64_add_cpu(&ref->sm_total, sm_delta); le64_add_cpu(&ref->lg_total, lg_delta); - - if (level == 0) { - update_first(&rdx->sm_first, ind, SCOUTFS_RADIX_BITS, - -sm_delta, sm_delta < 0, - ref->sm_total == 0); - update_first(&rdx->lg_first, - round_down(ind, SCOUTFS_RADIX_LG_BITS), - SCOUTFS_RADIX_BITS, - -lg_delta, lg_delta < 0, - ref->lg_total == 0); - } else { - update_first(&rdx->sm_first, ind, SCOUTFS_RADIX_REFS, - 1, rdx_ref->sm_total == 0, - ref->sm_total == 0); - update_first(&rdx->lg_first, - round_down(ind, SCOUTFS_RADIX_LG_BITS), - SCOUTFS_RADIX_REFS, - 1, rdx_ref->lg_total == 0, - ref->lg_total == 0); + if (level > 0) { + ind = path->inds[level]; + set_first_nonzero_ref(rdx, ind, sm_first, sm_total); + set_first_nonzero_ref(rdx, ind, lg_first, lg_total); } } @@ -615,7 +583,9 @@ static void alloc_leaf_bits(struct super_block *sb, bool meta, { struct scoutfs_radix_block *rdx = path->bls[0]->data; struct scoutfs_radix_ref *ref = path_ref(path, 0); - s64 lg_delta; + u32 sm_first; + u32 lg_first; + int lg_nbits; int ind; int end; @@ -623,6 +593,8 @@ static void alloc_leaf_bits(struct super_block *sb, bool meta, /* always allocate large allocs from full large regions */ ind = le32_to_cpu(rdx->lg_first); ind = find_next_lg(rdx->bits, ind); + sm_first = le32_to_cpu(rdx->sm_first); + lg_first = round_up(ind + nbits, SCOUTFS_RADIX_LG_BITS); } else { /* otherwise alloc as much as we can from the next small */ @@ -633,15 +605,21 @@ static void alloc_leaf_bits(struct super_block *sb, bool meta, end = find_next_zero_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); nbits = min(nbits, end - ind); } + + sm_first = ind + nbits; + lg_first = le32_to_cpu(rdx->lg_first); } /* callers and structures should have ensured success */ BUG_ON(ind >= SCOUTFS_RADIX_BITS); - lg_delta = count_lg_bits(rdx->bits, ind, nbits); + lg_nbits = count_lg_bits(rdx->bits, ind, nbits); bitmap_clear_le(rdx->bits, ind, nbits); - fixup_first_total(sb, path, ind, -nbits, -lg_delta); + /* always update the first we searched through */ + rdx->sm_first = cpu_to_le32(sm_first); + rdx->lg_first = cpu_to_le32(lg_first); + fixup_parent_refs(path, -nbits, -lg_nbits); *bit_ret = path->leaf_bit + ind; *nbits_ret = nbits; @@ -678,6 +656,7 @@ static void set_path_leaf_bits(struct super_block *sb, struct radix_path *path, u64 bit, int nbits) { struct scoutfs_radix_block *rdx; + int lg_ind; int ind; BUG_ON(nbits <= 0); @@ -686,13 +665,18 @@ static void set_path_leaf_bits(struct super_block *sb, struct radix_path *path, rdx = path->bls[0]->data; ind = bit - path->leaf_bit; + lg_ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); /* should have returned an error if it was set while we got paths */ BUG_ON(!bitmap_empty_region_le(rdx->bits, ind, nbits)); bitmap_set_le(rdx->bits, ind, nbits); - fixup_first_total(sb, path, ind, nbits, - count_lg_bits(rdx->bits, ind, nbits)); + if (ind < le32_to_cpu(rdx->sm_first)) + rdx->sm_first = cpu_to_le32(ind); + if (lg_ind < le32_to_cpu(rdx->lg_first) && + lg_is_full(rdx->bits, lg_ind)) + rdx->lg_first = cpu_to_le32(lg_ind); + fixup_parent_refs(path, nbits, count_lg_bits(rdx->bits, ind, nbits)); trace_scoutfs_radix_set(sb, path->root, path->bls[0]->blkno, bit, ind, nbits); @@ -724,7 +708,7 @@ static void init_ref(struct scoutfs_radix_ref *ref, int level, bool full) ref->blkno = cpu_to_le64(U64_MAX); ref->seq = cpu_to_le64(0); ref->sm_total = cpu_to_le64(tot); - ref->lg_total = cpu_to_le64(tot >> SCOUTFS_RADIX_LG_SHIFT); + ref->lg_total = cpu_to_le64(tot); } else { ref->blkno = cpu_to_le64(0); @@ -1348,6 +1332,7 @@ int scoutfs_radix_merge(struct super_block *sb, s64 dst_lg_delta; s64 sm_delta; u64 bit; + int lg_ind; int ind; int ret; @@ -1402,6 +1387,7 @@ wrapped: sm_delta = le64_to_cpu(path_ref(inp_path, 0)->sm_total); ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, le32_to_cpu(inp_rdx->sm_first)); + lg_ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); /* back out and retry if no input left, or inp not ro */ if (sm_delta == 0 || @@ -1439,8 +1425,14 @@ wrapped: bitmap_xor((void *)src_rdx->bits, (void *)src_rdx->bits, (void *)inp_rdx->bits, SCOUTFS_RADIX_BITS); - fixup_first_total(sb, src_path, ind, -sm_delta, -src_lg_delta); - fixup_first_total(sb, dst_path, ind, sm_delta, dst_lg_delta); + if (ind < le32_to_cpu(dst_rdx->sm_first)) + dst_rdx->sm_first = cpu_to_le32(ind); + /* first doesn't have to be precise, search will cleanup */ + if (lg_ind < le32_to_cpu(dst_rdx->lg_first)) + dst_rdx->lg_first = cpu_to_le32(lg_ind); + + fixup_parent_refs(src_path, -sm_delta, -src_lg_delta); + fixup_parent_refs(dst_path, sm_delta, dst_lg_delta); trace_scoutfs_radix_merge(sb, src, src_path->bls[0]->blkno, dst, dst_path->bls[0]->blkno, count, From 7cf8d01c1b92e2f679c1eff9d30dd568ea27a3b9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 27 Feb 2020 19:49:37 -0800 Subject: [PATCH 793/920] scoutfs: fix super read error race The conversion to reading the super with buffer_head IO caused racing readers to risk spurious errors. Clearing uptodate to force device access could race with a current waking reader. They could wake and find uptodate cleared and think that an IO error had occurred. The buffer_head functions generally require higher level serialization of this kind of use of the uptodate bit. We use bh_private as a counter to ensure that we don't clear uptodate while there are active readers. We then also use a private buffer_head bit to satisfy batches of waiting readers with each IO. Signed-off-by: Zach Brown --- kmod/src/super.c | 58 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 69de8ebe..1e23b89b 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -261,9 +261,22 @@ int scoutfs_write_super(struct super_block *sb, return ret; } +enum { + BH_ScoutfsReadGroup = BH_PrivateStart, +}; +BUFFER_FNS(ScoutfsReadGroup, scoutfs_read_group); + /* * Read the super block. If it's valid store it in the caller's super * struct. + * + * The caller requires a current version of the super as they call. We + * can't use a cached version from before they called. We use a bit and + * bh_private as a counter to satisfy groups of waiting readers with + * each issued read rather than have every call issue a read. We have + * to be careful to serialize clearing uptodate when forcing media + * access so that it doesn't cause blocked readers to get spurious read + * errors. */ int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super_res) @@ -271,17 +284,42 @@ int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super; struct buffer_head *bh = NULL; __le32 calc; + int group; int ret; bh = sb_getblk(sb, SCOUTFS_SUPER_BLKNO); - if (bh) { - lock_buffer(bh); - clear_buffer_uptodate(bh); - unlock_buffer(bh); - brelse(bh); + if (bh == NULL) { + ret = -ENOMEM; + scoutfs_err(sb, "error alloationg buffer for super block: %d", + ret); + goto out; } - bh = sb_bread(sb, SCOUTFS_SUPER_BLKNO); - if (!bh) { + + /* wait for current group to finish */ + group = !!buffer_scoutfs_read_group(bh); + lock_buffer(bh); + while (!!buffer_scoutfs_read_group(bh) == group && + (unsigned long)bh->b_private != 0) { + unlock_buffer(bh); + cond_resched(); + lock_buffer(bh); + } + + if ((unsigned long)(bh->b_private)++ == 0) { + /* first group locker advances group, submits, and relocks */ + if (buffer_scoutfs_read_group(bh)) + clear_buffer_scoutfs_read_group(bh); + else + set_buffer_scoutfs_read_group(bh); + clear_buffer_uptodate(bh); + bh->b_end_io = end_buffer_read_sync; + get_bh(bh); + submit_bh(READ | REQ_META | REQ_PRIO, bh); + lock_buffer(bh); + } else { + /* additional group lockers acquired after read completed */ + } + if (!buffer_uptodate(bh)) { ret = -EIO; scoutfs_err(sb, "error reading super block: %d", ret); goto out; @@ -332,7 +370,11 @@ int scoutfs_read_super(struct super_block *sb, *super_res = *super; ret = 0; out: - brelse(bh); + if (bh != NULL) { + (unsigned long)(bh->b_private)--; + unlock_buffer(bh); + brelse(bh); + } return ret; } From e9e515524b82210a90a3415e071e2f05685f3e52 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 13:48:00 -0800 Subject: [PATCH 794/920] scoutfs: remove unused corruption sources Remove a bunch of constants for sources of corruption that are no longer used in the code. Signed-off-by: Zach Brown --- kmod/src/format.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 42eecc90..8707bb3f 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -805,12 +805,6 @@ enum { SC_BTREE_BLOCK_LEVEL, SC_BTREE_NO_CHILD_REF, SC_INODE_BLOCK_COUNTS, - SC_EXTENT_ADD_CLEANUP, - SC_EXTENT_REM_CLEANUP, - SC_DATA_EXTENT_TRUNC_CLEANUP, - SC_DATA_EXTENT_ALLOC_CLEANUP, - SC_SERVER_EXTENT_CLEANUP, - SC_DATA_EXTENT_FALLOCATE_CLEANUP, SC_NR_SOURCES, }; From debac8ab061aa92f4efe94fc6ab68bb9c0b4f4d2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 14:00:57 -0800 Subject: [PATCH 795/920] scoutfs: free all forest iter pos Forest item iteration allocates iterator positions for each tree root it reads from. The postorder destruction of the iterator nodes wasn't quite right because we were balancing the nodes as they were freed. That can change parent/child relationships and cause postorder iteration to skip some nodes, leaking memory. It would have worked if we just freed the nodes without using rb_erase to balance. The fix is to actually iterate over the rbnodes while using the destroy helper which rebalances as it frees. Signed-off-by: Zach Brown --- kmod/src/forest.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index a88d2119..abb1c646 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -961,7 +961,10 @@ retry: unlock: up_read(&lpriv->rwsem); - rbtree_postorder_for_each_entry_safe(ip, nip, &iter_root, node) { + /* destroy_ rebalances so postorder traversal could skip nodes */ + for (ip = first_iter_pos(&iter_root); + ip && (nip = next_iter_pos(ip), 1); + ip = nip) { destroy_iter_pos(ip, &iter_root); } From e8b0bbc6194028aa8f01b0393501a69fce1c0454 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 14:05:51 -0800 Subject: [PATCH 796/920] scoutfs: remove unused counters Remove a bunch of unused counters which have accumulated over time as we've worked on the code and forgotten to remove counters. Signed-off-by: Zach Brown --- kmod/src/counters.h | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 932e8a4b..5fc62f19 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -24,29 +24,16 @@ EXPAND_COUNTER(block_cache_shrink) \ EXPAND_COUNTER(btree_read_error) \ EXPAND_COUNTER(btree_stale_read) \ - EXPAND_COUNTER(btree_write_error) \ EXPAND_COUNTER(client_farewell_error) \ EXPAND_COUNTER(corrupt_btree_block_level) \ EXPAND_COUNTER(corrupt_btree_no_child_ref) \ - EXPAND_COUNTER(corrupt_data_extent_trunc_cleanup) \ - EXPAND_COUNTER(corrupt_data_extent_alloc_cleanup) \ - EXPAND_COUNTER(corrupt_data_extent_fallocate_cleanup) \ EXPAND_COUNTER(corrupt_dirent_backref_name_len) \ EXPAND_COUNTER(corrupt_dirent_name_len) \ EXPAND_COUNTER(corrupt_dirent_readdir_name_len) \ EXPAND_COUNTER(corrupt_inode_block_counts) \ - EXPAND_COUNTER(corrupt_extent_add_cleanup) \ - EXPAND_COUNTER(corrupt_extent_rem_cleanup) \ - EXPAND_COUNTER(corrupt_server_extent_cleanup) \ EXPAND_COUNTER(corrupt_symlink_inode_size) \ EXPAND_COUNTER(corrupt_symlink_missing_item) \ EXPAND_COUNTER(corrupt_symlink_not_null_term) \ - EXPAND_COUNTER(data_end_writeback_page) \ - EXPAND_COUNTER(data_invalidatepage) \ - EXPAND_COUNTER(data_readpage) \ - EXPAND_COUNTER(data_write_begin) \ - EXPAND_COUNTER(data_write_end) \ - EXPAND_COUNTER(data_writepage) \ EXPAND_COUNTER(dentry_revalidate_error) \ EXPAND_COUNTER(dentry_revalidate_invalid) \ EXPAND_COUNTER(dentry_revalidate_locked) \ @@ -55,12 +42,6 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ - EXPAND_COUNTER(extent_add) \ - EXPAND_COUNTER(extent_delete) \ - EXPAND_COUNTER(extent_insert) \ - EXPAND_COUNTER(extent_next) \ - EXPAND_COUNTER(extent_prev) \ - EXPAND_COUNTER(extent_remove) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grace_elapsed) \ @@ -96,7 +77,6 @@ EXPAND_COUNTER(quorum_elected_leader) \ EXPAND_COUNTER(quorum_election_timeout) \ EXPAND_COUNTER(quorum_failure) \ - EXPAND_COUNTER(quorum_new_leader) \ EXPAND_COUNTER(quorum_read_block) \ EXPAND_COUNTER(quorum_read_block_error) \ EXPAND_COUNTER(quorum_read_invalid_block) \ From 65724c672498e07fe31e66f023afad7dd14eee70 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 14:06:28 -0800 Subject: [PATCH 797/920] scoutfs: forest comment update A quick update of the comment describing the forest's use of the bloom filter block. It used to be a tree of bloom filter items. Signed-off-by: Zach Brown --- kmod/src/format.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 8707bb3f..2d295d20 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -328,11 +328,11 @@ struct scoutfs_bloom_block { } __packed; /* - * Log trees include a tree of items that make up a fixed size bloom - * filter. Just a few megs worth of items lets us test for the presence - * of locks that cover billions of files with a .1% chance of false - * positives. The log trees should be finalized and merged long before - * the bloom filters fill up and start returning excessive false positives. + * Item log trees are accompanied by a block of bits that make up a + * bloom filter which indicate if the item log trees may contain items + * covered by a lock. The log trees should be finalized and merged long + * before the bloom filters fill up and start returning excessive false + * positives. */ #define SCOUTFS_FOREST_BLOOM_NRS 7 #define SCOUTFS_FOREST_BLOOM_BITS \ From ac3466921ac24c5de04b77cc569cd8e7ebdb7e9e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 14:20:40 -0800 Subject: [PATCH 798/920] scoutfs: invalidate stale bloom blocks We need to invalidate old stale blocks we encounter when reading old bloom block references written by other nodes. This is the same consistency mechanism used by btree blocks. Signed-off-by: Zach Brown --- kmod/src/forest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index abb1c646..2c1456ec 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -260,6 +260,7 @@ static struct scoutfs_block *read_bloom_ref(struct super_block *sb, if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, SCOUTFS_BLOCK_MAGIC_BLOOM)) { + scoutfs_block_invalidate(sb, bl); scoutfs_block_put(sb, bl); return ERR_PTR(-ESTALE); } From 462749cb87a9b8bdc391e37a5b13a80c45c40e90 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 4 Mar 2020 14:59:53 -0800 Subject: [PATCH 799/920] scoutfs: add stage and release tracing Add a bit more tracing to stage, release, and unwritten extent conversion so we can get a bit more visibility into the threads staging and releasing. Signed-off-by: Zach Brown --- kmod/src/data.c | 4 ++ kmod/src/ioctl.c | 7 ++- kmod/src/scoutfs_trace.h | 93 ++++++++++++++++++++++++++++++---------- 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 0750b441..f9666857 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -972,6 +972,7 @@ static int convert_unwritten(struct super_block *sb, struct inode *inode, struct unpacked_extent *ext, u64 iblock, struct scoutfs_lock *lock) { + struct scoutfs_traced_extent te; u64 blkno; u8 ext_fl; int err; @@ -980,6 +981,9 @@ static int convert_unwritten(struct super_block *sb, struct inode *inode, blkno = ext->blkno + (iblock - ext->iblock); ext_fl = ext->flags; + init_traced_extent(&te, iblock, 1, blkno, ext_fl); + trace_scoutfs_data_convert_unwritten(sb, scoutfs_ino(inode), &te); + ret = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, blkno, 1, ext_fl & ~(SEF_OFFLINE|SEF_UNWRITTEN)); if (ret < 0) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index eeebec8e..148e7356 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -281,7 +281,7 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; - trace_scoutfs_ioc_release(sb, &args); + trace_scoutfs_ioc_release(sb, scoutfs_ino(inode), &args); if (args.count == 0) return 0; @@ -344,7 +344,7 @@ out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); - trace_scoutfs_ioc_release_ret(sb, ret); + trace_scoutfs_ioc_release_ret(sb, scoutfs_ino(inode), ret); return ret; } @@ -395,6 +395,8 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) if (copy_from_user(&args, (void __user *)arg, sizeof(args))) return -EFAULT; + trace_scoutfs_ioc_stage(sb, scoutfs_ino(inode), &args); + end_size = args.offset + args.count; /* verify arg constraints that aren't dependent on file */ @@ -464,6 +466,7 @@ out: mutex_unlock(&inode->i_mutex); mnt_drop_write_file(file); + trace_scoutfs_ioc_stage_ret(sb, scoutfs_ino(inode), ret); return ret; } diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index c4b4b48d..7a56c831 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -61,6 +61,27 @@ struct lock_info; #define DECLARE_TRACED_EXTENT(name) \ struct scoutfs_traced_extent name = {0} +DECLARE_EVENT_CLASS(scoutfs_ino_ret_class, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + + TP_ARGS(sb, ino, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->ret = ret; + ), + + TP_printk(SCSBF" ino %llu ret %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->ret) +); + TRACE_EVENT(scoutfs_setattr, TP_PROTO(struct dentry *dentry, struct iattr *attr), @@ -232,6 +253,11 @@ DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc_block, struct scoutfs_traced_extent *te), TP_ARGS(sb, ino, te) ); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_convert_unwritten, + TP_PROTO(struct super_block *sb, __u64 ino, + struct scoutfs_traced_extent *te), + TP_ARGS(sb, ino, te) +); DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc_unwritten, TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_traced_extent *te), @@ -471,31 +497,15 @@ TRACE_EVENT(scoutfs_trans_track_item, __entry->res_vals) ); -TRACE_EVENT(scoutfs_ioc_release_ret, - TP_PROTO(struct super_block *sb, int ret), - - TP_ARGS(sb, ret), - - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(int, ret) - ), - - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->ret = ret; - ), - - TP_printk(SCSBF" ret %d", SCSB_TRACE_ARGS, __entry->ret) -); - TRACE_EVENT(scoutfs_ioc_release, - TP_PROTO(struct super_block *sb, struct scoutfs_ioctl_release *args), + TP_PROTO(struct super_block *sb, u64 ino, + struct scoutfs_ioctl_release *args), - TP_ARGS(sb, args), + TP_ARGS(sb, ino, args), TP_STRUCT__entry( SCSB_TRACE_FIELDS + __field(__u64, ino) __field(__u64, block) __field(__u64, count) __field(__u64, vers) @@ -503,13 +513,52 @@ TRACE_EVENT(scoutfs_ioc_release, TP_fast_assign( SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; __entry->block = args->block; __entry->count = args->count; __entry->vers = args->data_version; ), - TP_printk(SCSBF" block %llu count %llu vers %llu", SCSB_TRACE_ARGS, - __entry->block, __entry->count, __entry->vers) + TP_printk(SCSBF" ino %llu block %llu count %llu vers %llu", + SCSB_TRACE_ARGS, __entry->ino, __entry->block, + __entry->count, __entry->vers) +); + +DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_release_ret, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + TP_ARGS(sb, ino, ret) +); + +TRACE_EVENT(scoutfs_ioc_stage, + TP_PROTO(struct super_block *sb, u64 ino, + struct scoutfs_ioctl_stage *args), + + TP_ARGS(sb, ino, args), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, vers) + __field(__u64, offset) + __field(__s32, count) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->vers = args->data_version; + __entry->offset = args->offset; + __entry->count = args->count; + ), + + TP_printk(SCSBF" ino %llu vers %llu offset %llu count %d", + SCSB_TRACE_ARGS, __entry->ino, __entry->vers, + __entry->offset, __entry->count) +); + +DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_stage_ret, + TP_PROTO(struct super_block *sb, u64 ino, int ret), + TP_ARGS(sb, ino, ret) ); TRACE_EVENT(scoutfs_ioc_walk_inodes, From ef1dc677d05273d4ed9959e8f468658ed067ee1a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Mar 2020 10:18:19 -0700 Subject: [PATCH 800/920] scoutfs: store initialied offline unpacked extents The setattr_more ioctl has its own helper for creating uninitialized extents when we know that there can't be any other existing extents. We don't have to worry about freeing blocks they might have referenced. This helper forgot to actually store the modified extents back into packed extent items after setting extents offline. Signed-off-by: Zach Brown --- kmod/src/data.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index f9666857..17653b4f 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1579,6 +1579,10 @@ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, if (ret < 0) goto out; + ret = store_packed_extents(sb, ino, unpe, lock); + if (ret < 0) + goto out; + free_unpacked_extents(unpe); unpe = NULL; From 88422c6405a39f8c96221e5a6cf1adb8288a6433 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Mar 2020 10:22:46 -0700 Subject: [PATCH 801/920] scoutfs: fiemap with no extents returns 0 Don't return -ENOENT from fiemap on a file with no extents. The operation is supposed to succeed with no extents. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index 17653b4f..47601783 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1674,6 +1674,8 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, ret = load_unpacked_extents(sb, ino, iblock, last, false, &unpe, lock); if (ret < 0) { + if (ret == -ENOENT) + ret = 0; last_flags = FIEMAP_EXTENT_LAST; break; } From 6228f7cde7536943101bd1925a4e5cc59d04a332 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Mar 2020 15:23:29 -0700 Subject: [PATCH 802/920] scoutfs: create offline extents after arg checks With the introduction of packed extent items the setattr_more ioctl had to be careful not to try and dirty all the extent items in one transaction. But it pulled the extent creation call up to high and was doing it before some argument checks that were done after the inode was refreshed by acquiring its lock. This moves the extent creation to be done after the args are verified for the inode. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 148e7356..24317e73 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -590,13 +590,6 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) if (ret) goto unlock; - /* create offline extents in potentially many transactions */ - if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { - ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); - if (ret) - goto unlock; - } - /* can only change size/dv on untouched regular files */ if ((sm.i_size != 0 || sm.data_version != 0) && ((!S_ISREG(inode->i_mode) || @@ -605,6 +598,13 @@ static long scoutfs_ioc_setattr_more(struct file *file, unsigned long arg) goto unlock; } + /* create offline extents in potentially many transactions */ + if (sm.flags & SCOUTFS_IOC_SETATTR_MORE_OFFLINE) { + ret = scoutfs_data_init_offline_extent(inode, sm.i_size, lock); + if (ret) + goto unlock; + } + /* setting only so we don't see 0 data seq with nonzero data_version */ set_data_seq = sm.data_version != 0 ? true : false; ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, set_data_seq, From 6ae0ac936c576e5ddf684d29a883b32006d5b91b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 1 Apr 2020 09:40:15 -0700 Subject: [PATCH 803/920] scoutfs: fix setattr offline extent length We miscalculated the length of extents to create when initializing offline extents for setattr_more. We were clamping the extent length in each packed extent item by the full size of the offline extent, ignoring the iblock position that we were starting from. Signed-off-by: Zach Brown --- kmod/src/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index 47601783..f909d0f1 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1572,7 +1572,7 @@ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, if (ret < 0) goto out; - count = min(blocks, last_iblock(iblock) - iblock + 1); + count = min(blocks - iblock, last_iblock(iblock) - iblock + 1); ret = set_extent(sb, inode, ino, unpe, iblock, 0, count, SEF_OFFLINE); From 44ac668afa562f691dfae3652cae1b988586da9d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 7 Apr 2020 10:16:40 -0700 Subject: [PATCH 804/920] scoutfs: add small private block io read and write Add two quick functions which perform IO on small fixed size 4K blocks to or from the caller's buffer with privately allocated pages and bios. Callers have no interaction with each other. This matches the behaviour expected by callers of scoutfs_read_super and _write_super. Signed-off-by: Zach Brown --- kmod/src/block.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/block.h | 6 +++ 2 files changed, 107 insertions(+) diff --git a/kmod/src/block.c b/kmod/src/block.c index 9a004407..8c8f1b5b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -919,6 +919,107 @@ out: return min_t(u64, binf->lru_nr * SCOUTFS_PAGES_PER_BLOCK, INT_MAX); } +#define SCOUTFS_SM_BLOCK_SHIFT 12 +#define SCOUTFS_SM_BLOCK_SIZE (1 << SCOUTFS_SM_BLOCK_SHIFT) + +struct sm_block_completion { + struct completion comp; + int err; +}; + +static void sm_block_bio_end_io(struct bio *bio, int err) +{ + struct sm_block_completion *sbc = bio->bi_private; + + sbc->err = err; + complete(&sbc->comp); + bio_put(bio); +} + +/* + * Perform a private synchronous read or write of a small fixed size 4K + * block. We allocate a private page and bio and copy to or from the + * caller's buffer. + * + * The interface is a little weird because our blocks always start with + * a block header that contains a crc of the entire block. We're the + * only layer that sees the full block buffer so we pass the calculated + * crc to the caller for them to check in their context. + */ +static int sm_block_io(struct super_block *sb, int rw, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc) +{ + struct scoutfs_block_header *pg_hdr; + struct sm_block_completion sbc; + struct page *page; + struct bio *bio; + int ret; + + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_SM_BLOCK_SIZE); + /* block calc crc is assuming block size, they'll be different later */ + BUILD_BUG_ON(SCOUTFS_SM_BLOCK_SIZE != SCOUTFS_BLOCK_SIZE); + + if (WARN_ON_ONCE(len > SCOUTFS_SM_BLOCK_SIZE) || + WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) + return -EINVAL; + + page = alloc_page(GFP_NOFS); + if (!page) + return -ENOMEM; + + pg_hdr = page_address(page); + + if (rw & WRITE) { + memcpy(pg_hdr, hdr, len); + if (len < SCOUTFS_SM_BLOCK_SIZE) + memset((char *)pg_hdr + len, 0, + SCOUTFS_SM_BLOCK_SIZE - len); + pg_hdr->crc = scoutfs_block_calc_crc(pg_hdr); + } + + bio = bio_alloc(GFP_NOFS, 1); + if (!bio) { + ret = -ENOMEM; + goto out; + } + + bio->bi_sector = blkno << (SCOUTFS_SM_BLOCK_SHIFT - 9); + bio->bi_bdev = sb->s_bdev; + bio->bi_end_io = sm_block_bio_end_io; + bio->bi_private = &sbc; + bio_add_page(bio, page, SCOUTFS_SM_BLOCK_SIZE, 0); + + init_completion(&sbc.comp); + sbc.err = 0; + + submit_bio((rw & WRITE) ? WRITE_SYNC : READ_SYNC, bio); + + wait_for_completion(&sbc.comp); + ret = sbc.err; + + if (ret == 0 && !(rw & WRITE)) { + memcpy(hdr, pg_hdr, len); + *blk_crc = scoutfs_block_calc_crc(pg_hdr); + } +out: + __free_page(page); + return ret; +} + +int scoutfs_block_read_sm(struct super_block *sb, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc) +{ + return sm_block_io(sb, READ, blkno, hdr, len, blk_crc); +} + +int scoutfs_block_write_sm(struct super_block *sb, u64 blkno, + struct scoutfs_block_header *hdr, size_t len) +{ + return sm_block_io(sb, WRITE, blkno, hdr, len, NULL); +} + int scoutfs_block_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/block.h b/kmod/src/block.h index 8a405f11..dc62bf77 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -52,6 +52,12 @@ bool scoutfs_block_writer_has_dirty(struct super_block *sb, u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, struct scoutfs_block_writer *wri); +int scoutfs_block_read_sm(struct super_block *sb, u64 blkno, + struct scoutfs_block_header *hdr, size_t len, + __le32 *blk_crc); +int scoutfs_block_write_sm(struct super_block *sb, u64 blkno, + struct scoutfs_block_header *hdr, size_t len); + int scoutfs_block_setup(struct super_block *sb); void scoutfs_block_destroy(struct super_block *sb); From ae9e060fbfa887de3f76ca29d31596455273abb5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 7 Apr 2020 10:18:02 -0700 Subject: [PATCH 805/920] scoutfs: read and write supers as sm blocks Back in ancient LSM times these functions to read and write the super block reused the bio functions that LSM segment IO used. Each IO would be performed with privately allocated pages and bios. When we got rid of the LSM code we got rid of the bio functions. It was quick and easy to transition super read/write to use buffer_heads. This introduced sharing of the super's buffer_head between readers and writers. First we saw concurrent readers being confused by the uptodate bit and added a bunch of complexity to coordinate use of the uptodate bit. Now we're seeing the writer copy its super for writing into the buffer that readers are using, causing crc failures on read. Let's not use buffer_heads anymore (always good advice). We added quick block functions to read and write small blocks with private pages and bios. Use those here to read and write the super so that readers and writers operate on their own buffers again. Signed-off-by: Zach Brown --- kmod/src/super.c | 105 +++++++---------------------------------------- 1 file changed, 14 insertions(+), 91 deletions(-) diff --git a/kmod/src/super.c b/kmod/src/super.c index 1e23b89b..30cabd73 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -21,7 +21,6 @@ #include #include #include -#include #include "super.h" #include "block.h" @@ -224,108 +223,36 @@ static const struct super_operations scoutfs_super_ops = { * Write the caller's super. The caller has always read a valid super * before modifying and writing it. The caller's super is modified * to reflect the write. - * - * XXX it'd be pretty easy to preallocate to avoid failure here. */ int scoutfs_write_super(struct super_block *sb, - struct scoutfs_super_block *caller) + struct scoutfs_super_block *super) { - struct scoutfs_super_block *super; - struct buffer_head *bh; - int ret; + le64_add_cpu(&super->hdr.seq, 1); - bh = sb_getblk(sb, SCOUTFS_SUPER_BLKNO); - if (!bh) - return -ENOMEM; - - le64_add_cpu(&caller->hdr.seq, 1); - - memset(bh->b_data, 0, bh->b_size); - super = (void *)bh->b_data; - memcpy(super, caller, sizeof(*super)); - super->hdr.crc = scoutfs_block_calc_crc(&super->hdr); - - lock_buffer(bh); - set_buffer_mapped(bh); - set_buffer_dirty(bh); - unlock_buffer(bh); - - ll_rw_block(WRITE, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) - ret = -EIO; - else - ret = 0; - brelse(bh); - - return ret; + return scoutfs_block_write_sm(sb, SCOUTFS_SUPER_BLKNO, &super->hdr, + sizeof(struct scoutfs_super_block)); } -enum { - BH_ScoutfsReadGroup = BH_PrivateStart, -}; -BUFFER_FNS(ScoutfsReadGroup, scoutfs_read_group); - /* * Read the super block. If it's valid store it in the caller's super * struct. - * - * The caller requires a current version of the super as they call. We - * can't use a cached version from before they called. We use a bit and - * bh_private as a counter to satisfy groups of waiting readers with - * each issued read rather than have every call issue a read. We have - * to be careful to serialize clearing uptodate when forcing media - * access so that it doesn't cause blocked readers to get spurious read - * errors. */ int scoutfs_read_super(struct super_block *sb, struct scoutfs_super_block *super_res) { struct scoutfs_super_block *super; - struct buffer_head *bh = NULL; __le32 calc; - int group; int ret; - bh = sb_getblk(sb, SCOUTFS_SUPER_BLKNO); - if (bh == NULL) { - ret = -ENOMEM; - scoutfs_err(sb, "error alloationg buffer for super block: %d", - ret); + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) + return -ENOMEM; + + ret = scoutfs_block_read_sm(sb, SCOUTFS_SUPER_BLKNO, &super->hdr, + sizeof(struct scoutfs_super_block), + &calc); + if (ret < 0) goto out; - } - - /* wait for current group to finish */ - group = !!buffer_scoutfs_read_group(bh); - lock_buffer(bh); - while (!!buffer_scoutfs_read_group(bh) == group && - (unsigned long)bh->b_private != 0) { - unlock_buffer(bh); - cond_resched(); - lock_buffer(bh); - } - - if ((unsigned long)(bh->b_private)++ == 0) { - /* first group locker advances group, submits, and relocks */ - if (buffer_scoutfs_read_group(bh)) - clear_buffer_scoutfs_read_group(bh); - else - set_buffer_scoutfs_read_group(bh); - clear_buffer_uptodate(bh); - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(READ | REQ_META | REQ_PRIO, bh); - lock_buffer(bh); - } else { - /* additional group lockers acquired after read completed */ - } - if (!buffer_uptodate(bh)) { - ret = -EIO; - scoutfs_err(sb, "error reading super block: %d", ret); - goto out; - } - - super = (void *)(bh->b_data); if (super->hdr.magic != cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SUPER)) { scoutfs_err(sb, "super block has invalid magic value 0x%08x", @@ -334,11 +261,11 @@ int scoutfs_read_super(struct super_block *sb, goto out; } - calc = scoutfs_block_calc_crc(&super->hdr); if (calc != super->hdr.crc) { scoutfs_err(sb, "super block has invalid crc 0x%08x, calculated 0x%08x", le32_to_cpu(super->hdr.crc), le32_to_cpu(calc)); ret = -EINVAL; + goto out; } if (le64_to_cpu(super->hdr.blkno) != SCOUTFS_SUPER_BLKNO) { @@ -370,11 +297,7 @@ int scoutfs_read_super(struct super_block *sb, *super_res = *super; ret = 0; out: - if (bh != NULL) { - (unsigned long)(bh->b_private)--; - unlock_buffer(bh); - brelse(bh); - } + kfree(super); return ret; } From 192453e717de52d3ef7297612523e63dae0e591d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Apr 2020 10:58:23 -0700 Subject: [PATCH 806/920] scoutfs: add server error messages Add specific error messages for failures that can happen as the server commits log trees from the client. These are severe enough that we'd like to know about them. Signed-off-by: Zach Brown --- kmod/src/server.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index d0106c8a..0bdbedd3 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -474,8 +474,10 @@ static int server_commit_log_trees(struct super_block *sb, lt = arg; ret = hold_commit(sb); - if (ret < 0) + if (ret < 0) { + scoutfs_err(sb, "server error preparing commit: %d", ret); goto out; + } mutex_lock(&server->logs_mutex); @@ -485,13 +487,17 @@ static int server_commit_log_trees(struct super_block *sb, ltk.nr = le64_to_be64(lt->nr); ret = scoutfs_btree_lookup(sb, &super->logs_root, <k, sizeof(ltk), &iref); - if (ret < 0 && ret != -ENOENT) + if (ret < 0 && ret != -ENOENT) { + scoutfs_err(sb, "server error finding client logs: %d", ret); goto unlock; + } if (ret == 0) { if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { memcpy(<v, iref.val, iref.val_len); } else { ret = -EIO; + scoutfs_err(sb, "server error, invalid log item: %d", + ret); } scoutfs_btree_put_iref(&iref); if (ret < 0) @@ -517,11 +523,15 @@ static int server_commit_log_trees(struct super_block *sb, ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, <k, sizeof(ltk), <v, sizeof(ltv)); + if (ret < 0) + scoutfs_err(sb, "server error updating client logs: %d", ret); unlock: mutex_unlock(&server->logs_mutex); ret = apply_commit(sb, ret); + if (ret < 0) + scoutfs_err(sb, "server error commiting client logs: %d", ret); out: WARN_ON_ONCE(ret < 0); return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); From 66f8b3814cf8ca78c59d8df24cfb3cbdec5dc40d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 10 Apr 2020 11:15:24 -0700 Subject: [PATCH 807/920] scoutfs: remove warning on reading while staging An incorrect warning condition was added as fallocate was implemented. It tried to warn against trying to read from the staging ioctl. But the staging boolean is set on the inode when the staging ioctl has the inode mutex. It protects against writes, but page reading doesn't use the mutex. It's perfectly acceptable for reads to be attempted while the staging ioctl is busy. We rely on it for a large read to consume staging being written. The warning caused reads to fail while the stager ioctl was working. Typically this would hit read-ahead and just force sync reads. But it could hit sync reads and cause EIO. Signed-off-by: Zach Brown --- kmod/src/data.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index f909d0f1..39f368ce 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1017,8 +1017,7 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, /* make sure caller holds a cluster lock */ lock = scoutfs_per_task_get(&si->pt_data_lock); - if (WARN_ON_ONCE(!lock) || - WARN_ON_ONCE(!create && si->staging)) { + if (WARN_ON_ONCE(!lock)) { ret = -EINVAL; goto out; } From 4c1f78afd446062fa6f74021128e7909b9f3e22d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 14 Apr 2020 10:50:59 -0700 Subject: [PATCH 808/920] scoutfs: use our own _le bitmap xor helper We were using bitmap_xor() to set and clear blocks of allocator bits at a time. bitmap_xor() is a ternary function with two const input pointers and we were providing the changing destination as a const input pointer. That doesn't seem wise. Signed-off-by: Zach Brown --- kmod/src/radix.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 51214eef..d25fa397 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -245,6 +245,23 @@ static void bitmap_set_le(__le64 *map, int ind, int nbits) set_bit_le(ind++, map); } +/* + * xor the destination bitmap with the source. bitmap_xor() requires 2 + * const inputs so I'm not comfortable giving it the changing + * destination pointer as one of the const input pointers. + */ +static void bitmap_xor_bitmap_le(__le64 *dst, __le64 *src, int nbits) +{ + int i; + + BUG_ON((unsigned long)src & 7); + BUG_ON((unsigned long)dst & 7); + BUG_ON(nbits & 63); + + for (i = 0; i < nbits; i += 64) + *(dst++) ^= *(src++); +} + static void bitmap_clear_le(__le64 *map, int ind, int nbits) { unsigned int full; @@ -1417,13 +1434,13 @@ wrapped: } /* carefully modify src last, it might also be inp */ - bitmap_xor((void *)dst_rdx->bits, (void *)dst_rdx->bits, - (void *)inp_rdx->bits, SCOUTFS_RADIX_BITS); + bitmap_xor_bitmap_le(dst_rdx->bits, inp_rdx->bits, + SCOUTFS_RADIX_BITS); dst_lg_delta = count_lg_bitmap(dst_rdx->bits, inp_rdx->bits); src_lg_delta = count_lg_bitmap(src_rdx->bits, inp_rdx->bits); - bitmap_xor((void *)src_rdx->bits, (void *)src_rdx->bits, - (void *)inp_rdx->bits, SCOUTFS_RADIX_BITS); + bitmap_xor_bitmap_le(src_rdx->bits, inp_rdx->bits, + SCOUTFS_RADIX_BITS); if (ind < le32_to_cpu(dst_rdx->sm_first)) dst_rdx->sm_first = cpu_to_le32(ind); From 968e719a9af1ef4ebfd3508bbf9ed07e33eda0f6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 14 Apr 2020 11:55:31 -0700 Subject: [PATCH 809/920] scoutfs: check for bad radix merge count When we're merging bits that are set in a read-only input tree then we can't try to merge more bits than exist in the input tree. That'll cause us to loop around and double-free bits. Signed-off-by: Zach Brown --- kmod/src/radix.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index d25fa397..c390850a 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1355,6 +1355,13 @@ int scoutfs_radix_merge(struct super_block *sb, mutex_lock(&alloc->mutex); + /* can't try to free too much when inp is read-only */ + if (inp != src && + WARN_ON_ONCE(count > le64_to_cpu(inp->ref.sm_total))) { + ret = -EINVAL; + goto out; + } + while (count > 0) { chg = alloc_change(); From 2478d124dd839b5a7a1d8fe68a197c49299d284e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Apr 2020 11:04:57 -0700 Subject: [PATCH 810/920] scoutfs: use random radix block ref seqs The seq portion of radix block references is intended to differentiate versions of a given block location over time. The current method of incrementing the existing value as the block is dirtied is risky. It means that every lineage of a block has the same sequence number progression. Different trees referencing the same block over time could get confused. It's more robust to have large random numbers. The collision window is then evenly distributed over the 64bit space rather than being bunched up all in in the initial seq values. Signed-off-by: Zach Brown --- kmod/src/radix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index c390850a..429b867c 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "super.h" #include "format.h" @@ -1102,7 +1103,7 @@ static void dirty_all_path_blocks(struct super_block *sb, rdx = bl->data; rdx->hdr.blkno = cpu_to_le64(bl->blkno); - le64_add_cpu(&rdx->hdr.seq, 1); + prandom_bytes(&rdx->hdr.seq, sizeof(rdx->hdr.seq)); ref = path_ref(path, level); ref->blkno = rdx->hdr.blkno; From 2c5e3aa5513621752d6eb2436bd3f80c73363733 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Apr 2020 11:48:23 -0700 Subject: [PATCH 811/920] scoutfs: trace radix merge input root and leaf bit Add a bit more detail to the radix merge trace. It was missing the input block and leaf bit. Also use abbreviations of the fields in the trace output so that it's slightly less enormous. Signed-off-by: Zach Brown --- kmod/src/radix.c | 5 +++-- kmod/src/scoutfs_trace.h | 24 ++++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 429b867c..09ab5c4f 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1459,9 +1459,10 @@ wrapped: fixup_parent_refs(src_path, -sm_delta, -src_lg_delta); fixup_parent_refs(dst_path, sm_delta, dst_lg_delta); - trace_scoutfs_radix_merge(sb, src, src_path->bls[0]->blkno, + trace_scoutfs_radix_merge(sb, inp, inp_path->bls[0]->blkno, + src, src_path->bls[0]->blkno, dst, dst_path->bls[0]->blkno, count, - ind, sm_delta, src_lg_delta, + bit, ind, sm_delta, src_lg_delta, dst_lg_delta); free_path(sb, inp_path); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 7a56c831..66867318 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2317,19 +2317,23 @@ DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set, TRACE_EVENT(scoutfs_radix_merge, TP_PROTO(struct super_block *sb, + struct scoutfs_radix_root *inp, u64 inp_blkno, struct scoutfs_radix_root *src, u64 src_blkno, struct scoutfs_radix_root *dst, u64 dst_blkno, - u64 count, int ind, int sm_delta, int src_lg_delta, - int dst_lg_delta), - TP_ARGS(sb, src, src_blkno, dst, dst_blkno, count, ind, - sm_delta, src_lg_delta, dst_lg_delta), + u64 count, u64 leaf_bit, int ind, int sm_delta, + int src_lg_delta, int dst_lg_delta), + TP_ARGS(sb, inp, inp_blkno, src, src_blkno, dst, dst_blkno, count, + leaf_bit, ind, sm_delta, src_lg_delta, dst_lg_delta), TP_STRUCT__entry( SCSB_TRACE_FIELDS + __field(__u64, inp_root_blkno) + __field(__u64, inp_blkno) __field(__u64, src_root_blkno) __field(__u64, src_blkno) __field(__u64, dst_root_blkno) __field(__u64, dst_blkno) __field(__u64, count) + __field(__u64, leaf_bit) __field(int, ind) __field(int, sm_delta) __field(int, src_lg_delta) @@ -2337,22 +2341,26 @@ TRACE_EVENT(scoutfs_radix_merge, ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); + __entry->inp_root_blkno = le64_to_cpu(inp->ref.blkno); + __entry->inp_blkno = inp_blkno; __entry->src_root_blkno = le64_to_cpu(src->ref.blkno); __entry->src_blkno = src_blkno; __entry->dst_root_blkno = le64_to_cpu(dst->ref.blkno); __entry->dst_blkno = dst_blkno; __entry->count = count; + __entry->leaf_bit = leaf_bit; __entry->ind = ind; __entry->sm_delta = sm_delta; __entry->src_lg_delta = src_lg_delta; __entry->dst_lg_delta = dst_lg_delta; ), - TP_printk(SCSBF" src_root_blkno %llu src_blkno %llu dst_root_blkno %llu dst_blkno %llu count %llu ind %u sm_delta %d src_lg_delta %d dst_lg_delta %d", - SCSB_TRACE_ARGS, + TP_printk(SCSBF" irb %llu ib %llu srb %llu sb %llu drb %llu db %llu cnt %llu lb %llu ind %u smd %d sld %d dld %d", + SCSB_TRACE_ARGS, __entry->inp_root_blkno, __entry->inp_blkno, __entry->src_root_blkno, __entry->src_blkno, __entry->dst_root_blkno, __entry->dst_blkno, - __entry->count, __entry->ind, __entry->sm_delta, - __entry->src_lg_delta, __entry->dst_lg_delta) + __entry->count, __entry->leaf_bit, __entry->ind, + __entry->sm_delta, __entry->src_lg_delta, + __entry->dst_lg_delta) ); #endif /* _TRACE_SCOUTFS_H */ From d2a15ea5068f34880e056469b3fa9817a0a7dc90 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Apr 2020 16:24:51 -0700 Subject: [PATCH 812/920] scoutfs: fix depth-first radix next bit search The radix block next bit search could return a spurious -ENOENT if it ran out of references in a parent block further down the tree. It needs to bubble up to try the next ref in its parent so that it keeps performing a depth-first search of the entire tree. This lead to an assertion being tripped in _radix_merge. Getting an early -ENOENT caused it to start searching from 0 again. When it's iterating over a read-only input it could find the same leaf and try to clear source bits that were already cleared. Signed-off-by: Zach Brown --- kmod/src/radix.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 09ab5c4f..c9d51a73 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -901,9 +901,29 @@ static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, ind++; } + /* + * Didn't find a ref in the rest of the block at + * this level. If we're the root block there's no + * more next bits to return. If we're further down + * we bubble up a level and continue on a depth-first + * search. We check the next ref from our parent and reset + * all the child inds to the left spine of the new + * subtree. + */ if (ind >= SCOUTFS_RADIX_REFS) { - ret = -ENOENT; - goto out; + if (level == root->height - 1) { + ret = -ENOENT; + goto out; + } + path->inds[level + 1]++; + for (i = level; i >= 0; i--) + path->inds[i] = 0; + for (i = level; i <= level + 1; i++) { + scoutfs_block_put(sb, path->bls[i]); + path->bls[i] = NULL; + } + level += 2; + continue; } /* reset all lower indices if we searched */ From 495358996c421462b71fa39cfd96466869af1ec8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 21 Apr 2020 15:36:43 -0700 Subject: [PATCH 813/920] scoutfs: fix older kc readdir emit When we added the kernelcompat layer around the old and new readdir interfaces there was some confusion in the old readdir interface filldir arguments. We were passing in our scoutfs dent item struct pointer instead of the filldir callback buf pointer. This prevented readdir from working in older kernels because filldir would immediately see a corrupt buf and return an error. This renames the emit compat macro arguments to make them consistent with the other calls and readdir now provides the correct pointer to the emit wrapper. Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 +- kmod/src/kernelcompat.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 709fc40a..18164384 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -500,7 +500,7 @@ static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, pos = le64_to_cpu(key.skd_major); kc_readdir_pos(file, ctx) = pos; - if (!kc_dir_emit(ctx, dent, dent->name, name_len, pos, + if (!kc_dir_emit(ctx, dirent, dent->name, name_len, pos, le64_to_cpu(dent->ino), dentry_type(dent->type))) { ret = 0; diff --git a/kmod/src/kernelcompat.h b/kmod/src/kernelcompat.h index 1ee16022..6899f684 100644 --- a/kmod/src/kernelcompat.h +++ b/kmod/src/kernelcompat.h @@ -8,15 +8,15 @@ typedef filldir_t kc_readdir_ctx_t; #define KC_FOP_READDIR readdir #define kc_readdir_pos(filp, ctx) (filp)->f_pos #define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, dirent, ctx) -#define kc_dir_emit(ctx, dentry, name, name_len, pos, ino, dt) \ - (ctx(dentry, name, name_len, pos, ino, dt) == 0) +#define kc_dir_emit(ctx, dirent, name, name_len, pos, ino, dt) \ + (ctx(dirent, name, name_len, pos, ino, dt) == 0) #else typedef struct dir_context * kc_readdir_ctx_t; #define KC_DECLARE_READDIR(name, file, dirent, ctx) name(file, ctx) #define KC_FOP_READDIR iterate #define kc_readdir_pos(filp, ctx) (ctx)->pos #define kc_dir_emit_dots(file, dirent, ctx) dir_emit_dots(file, ctx) -#define kc_dir_emit(ctx, dentry, name, name_len, pos, ino, dt) \ +#define kc_dir_emit(ctx, dirent, name, name_len, pos, ino, dt) \ dir_emit(ctx, name, name_len, ino, dt) #endif From 7da8ddb8a1dd7ec0d37f0aa47f2831e725595d55 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Apr 2020 13:48:55 -0700 Subject: [PATCH 814/920] scoutfs: fix data.h include guard The identifier for data.h's include guard was brought over from an old file and still had the old name. Update it to reflect it's use in data, not filerw. Signed-off-by: Zach Brown --- kmod/src/data.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/data.h b/kmod/src/data.h index 895de61b..8b58d4f6 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -1,5 +1,5 @@ -#ifndef _SCOUTFS_FILERW_H_ -#define _SCOUTFS_FILERW_H_ +#ifndef _SCOUTFS_DATA_H_ +#define _SCOUTFS_DATA_H_ struct scoutfs_lock; struct scoutfs_ioctl_data_waiting_entry; From 9ad86d4d291ad305088333742d2caece2ffcd93f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Apr 2020 14:04:04 -0700 Subject: [PATCH 815/920] scoutfs: commit trans before premature enospc File data allocations come from radix allocators which are populated by the server before each client transation. It's possible to fully consume the data allocator within one transaction if the number of dirty metadata blocks is kept low. This could result in premature ENOSPC. This was happening to the archive-light-cycle test. If the transactions performed by previous tests lined up just right then the creation of the initial test files could see ENOSPC and cause all sorts of nonsense in the rest of the test, culminating in cmp commands stuck in offline waits. This introduces high and low data allocator water marks for transactions. The server tries to fill data allocators for each transaction to the high water mark and the client forces the commit of a transaction if its data allocator falls below the low water mark. The archive-light-cycle test now passes easily and we see the trans_commit_data_alloc_low counter increasing during the test. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/data.c | 10 ++++++++++ kmod/src/data.h | 1 + kmod/src/radix.c | 6 ++++++ kmod/src/radix.h | 2 ++ kmod/src/server.c | 3 ++- kmod/src/trans.c | 14 ++++++++++++++ kmod/src/trans.h | 5 +++++ 8 files changed, 41 insertions(+), 1 deletion(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 5fc62f19..8859e646 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -85,6 +85,7 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_sync_fs) \ diff --git a/kmod/src/data.c b/kmod/src/data.c index 39f368ce..f83089e9 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -2017,6 +2017,16 @@ void scoutfs_data_get_btrees(struct super_block *sb, up_read(&datinf->alloc_rwsem); } +/* + * This isn't serializing with allocators so it can be a bit racey. + */ +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb) +{ + DECLARE_DATA_INFO(sb, datinf); + + return scoutfs_radix_root_free_bytes(sb, &datinf->data_avail); +} + int scoutfs_data_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/data.h b/kmod/src/data.h index 8b58d4f6..9e0a9c87 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -78,6 +78,7 @@ void scoutfs_data_init_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); void scoutfs_data_get_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); +u64 scoutfs_data_alloc_free_bytes(struct super_block *sb); int scoutfs_data_setup(struct super_block *sb); void scoutfs_data_destroy(struct super_block *sb); diff --git a/kmod/src/radix.c b/kmod/src/radix.c index c9d51a73..15d80e19 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1526,6 +1526,12 @@ void scoutfs_radix_root_init(struct super_block *sb, init_ref(&root->ref, 0, false); } +u64 scoutfs_radix_root_free_bytes(struct super_block *sb, + struct scoutfs_radix_root *root) +{ + return le64_to_cpu(root->ref.sm_total) << SCOUTFS_BLOCK_SHIFT; +} + /* * The first bit nr in a leaf containing the bit, used by callers to * identify regions that span leafs and would need to be freed in diff --git a/kmod/src/radix.h b/kmod/src/radix.h index 982797e7..0ca79431 100644 --- a/kmod/src/radix.h +++ b/kmod/src/radix.h @@ -38,6 +38,8 @@ void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, struct scoutfs_radix_root *freed); void scoutfs_radix_root_init(struct super_block *sb, struct scoutfs_radix_root *root, bool meta); +u64 scoutfs_radix_root_free_bytes(struct super_block *sb, + struct scoutfs_radix_root *root); u64 scoutfs_radix_bit_leaf_nr(u64 bit); #endif diff --git a/kmod/src/server.c b/kmod/src/server.c index 0bdbedd3..a371748f 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -35,6 +35,7 @@ #include "lock_server.h" #include "endian_swap.h" #include "quorum.h" +#include "trans.h" /* * Every active mount can act as the server that listens on a net @@ -412,7 +413,7 @@ static int server_get_log_trees(struct super_block *sb, } /* ensure client has enough free data blocks for a transaction */ - target = (2ULL*1024*1024*1024) / SCOUTFS_BLOCK_SIZE; + target = SCOUTFS_TRANS_DATA_ALLOC_HWM / SCOUTFS_BLOCK_SIZE; if (le64_to_cpu(ltv.data_avail.ref.sm_total) < target) { count = target - le64_to_cpu(ltv.data_avail.ref.sm_total); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index a5017821..a3467aa6 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -315,6 +315,13 @@ struct scoutfs_reservation { * we piggy back on their hold. We wait if the writer is trying to * write out the transation. And if our items won't fit then we kick off * a write. + * + * This is called as a condition for wait_event. It is very limited in + * the locking (blocking) it can do because the caller has set the task + * state before testing the condition safely race with waking after + * setting the condition. Our checking the amount of dirty metadata + * blocks and free data blocks is racy, but we don't mind the risk of + * delaying or prematurely forcing commits. */ static bool acquired_hold(struct super_block *sb, struct scoutfs_reservation *rsv, @@ -354,6 +361,13 @@ static bool acquired_hold(struct super_block *sb, goto out; } + /* Try to refill data allocator before premature enospc */ + if (scoutfs_data_alloc_free_bytes(sb) <= SCOUTFS_TRANS_DATA_ALLOC_LWM) { + scoutfs_inc_counter(sb, trans_commit_data_alloc_low); + queue_trans_work(sbi); + goto out; + } + tri->reserved_items = items; tri->reserved_vals = vals; diff --git a/kmod/src/trans.h b/kmod/src/trans.h index e5f3228e..014a35e8 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -1,6 +1,11 @@ #ifndef _SCOUTFS_TRANS_H_ #define _SCOUTFS_TRANS_H_ +/* the server will attempt to fill data allocs for each trans */ +#define SCOUTFS_TRANS_DATA_ALLOC_HWM (2ULL * 1024 * 1024 * 1024) +/* the client will force commits if data allocators get too low */ +#define SCOUTFS_TRANS_DATA_ALLOC_LWM (256ULL * 1024 * 1024) + #include "count.h" void scoutfs_trans_write_func(struct work_struct *work); From e3b1f2e2b0f84dd16febc8c6dc2771791f37732d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Apr 2020 14:56:06 -0700 Subject: [PATCH 816/920] scoutfs: add counters for radix enospc Add counters for the various sources of ENOSPC from the radix block allocator. Signed-off-by: Zach Brown --- kmod/src/counters.h | 3 +++ kmod/src/radix.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 8859e646..ab4a9c15 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -85,6 +85,9 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(radix_enospc_data) \ + EXPAND_COUNTER(radix_enospc_paths) \ + EXPAND_COUNTER(radix_enospc_synth) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 15d80e19..e8e39b42 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -834,6 +834,7 @@ static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, synth = chg->next_synth++; /* careful not to go too high or wrap */ if (synth == U64_MAX || synth < RADIX_SYNTH_BLKNO) { + scoutfs_inc_counter(sb, radix_enospc_synth); ret = -ENOSPC; goto out; } @@ -992,6 +993,7 @@ static int get_all_paths(struct super_block *sb, /* we're not modifying as we go, check for wrapping */ if (next_meta >= start_meta && meta_wrapped) { + scoutfs_inc_counter(sb, radix_enospc_paths); ret = -ENOSPC; break; } @@ -1276,6 +1278,7 @@ find_next: root->next_find_bit = 0; goto find_next; } + scoutfs_inc_counter(sb, radix_enospc_data); ret = -ENOSPC; } goto out; From d16b18562de44f555f1b9f752fef3e3be19b3217 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Apr 2020 10:32:01 -0700 Subject: [PATCH 817/920] scoutfs: make sure forest sees dirty log tree Item writes are first stored in dirty blocks in the private version of the mount's log tree. Local readers need to be sure to check the dirty version of the mount's log tree to make sure that they see the result of writes. Usually trees are found by walking the log tree items stored in another btree in the super. The private dirty version of a mount's log tree hasn't been committed yet and isn't visible in these items. The forest uses its lock private data to track which lock has seen items written and so should always check the local dirty log tree when reading. The intent was to use the per-lock static forest_root for the log tree to record that it had been marked by a write and was then always used for reads. We used storing the forest info's rid and testing for a non-zero forest_root rid as the mechanism for always testing the dirty log root during read. But we weren't setting the forest info rid as each transaction opened. It was always 0 so readers never added the dirty log tree for reading. The fix is to use the more reliable indication that the log root has items for us by testing the flag that all the bits have been set. Then we're also sure to always set the rid/nr of the forest_info record of our log tree, and the per-lock forest_root copy of it whenever we use it. This fixed spurious errors we were seeing as creates tried to read the item they just wrote as memory reclaim freed locks. Signed-off-by: Zach Brown --- kmod/src/forest.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 2c1456ec..bcb7b809 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -174,10 +174,10 @@ static void clear_roots(struct forest_lock_private *lpriv) /* * Make sure that our log btree will be at the head of the list of - * btrees to read. This can be racing with clearing the list to check - * the bloom blocks again. We want the addition of the log btree to - * persist across clearing the lists so we set the rid/nr which causes - * the root to be added to the list after the bloom blocks are checked. + * btrees to read. We update the forest_root to refer to the most + * recent version of our log root before we try and use it instead of + * updating every instance of the forest_roots on locks as commits give + * us new versions of the same log tree. */ static void add_our_log_root(struct forest_info *finf, struct forest_lock_private *lpriv) @@ -186,13 +186,11 @@ static void add_our_log_root(struct forest_info *finf, BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); - if (fr->rid == 0) { + if (list_empty(&fr->entry)) { fr->rid = le64_to_cpu(finf->our_log.rid); fr->nr = le64_to_cpu(finf->our_log.nr); - } - - if (list_empty(&fr->entry)) list_add(&fr->entry, &lpriv->roots); + } } /* @@ -214,6 +212,10 @@ void scoutfs_forest_clear_lock(struct super_block *sb, * All the btrees we read are stable and read-only except for our log * btree which is being actively modified in memory by locked writers. * Once we lock it we need to get the current version of the root. + * + * The finf rwsem protects updates of the finf root fields, the first + * caller here will change the fr fields and the rest will overwrite + * them with the same values. */ static void read_lock_forest_root(struct forest_info *finf, struct forest_lock_private *lpriv, @@ -222,6 +224,8 @@ static void read_lock_forest_root(struct forest_info *finf, if (is_our_log_root(lpriv, fr)) { down_read(&finf->rwsem); fr->item_root = finf->our_log.item_root; + fr->rid = le64_to_cpu(finf->our_log.rid); + fr->nr = le64_to_cpu(finf->our_log.nr); } } @@ -375,7 +379,7 @@ static int refresh_bloom_roots(struct super_block *sb, if (i != ARRAY_SIZE(bloom.nrs)) continue; - /* add our current log tree if we see its bloom */ + /* use our dirty log instead of the old committed version */ if (be64_to_cpu(ltk.rid) == le64_to_cpu(finf->our_log.rid) && be64_to_cpu(ltk.nr) == le64_to_cpu(finf->our_log.nr)) { add_our_log_root(finf, lpriv); @@ -400,8 +404,8 @@ static int refresh_bloom_roots(struct super_block *sb, le64_to_cpu(fr->item_root.ref.seq)); } - /* add our current log root if a locked writer added it */ - if (lpriv->our_log_root.rid != 0) + /* make sure readers search our dirty log after writers set bloom */ + if (test_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS)) add_our_log_root(finf, lpriv); /* always add the fs root at the tail */ @@ -1469,10 +1473,12 @@ void scoutfs_forest_init_btrees(struct super_block *sb, finf->alloc = alloc; finf->wri = wri; - /* we use the item and bloom trees */ + /* the lt allocator fields have been used by the caller */ memset(&finf->our_log, 0, sizeof(finf->our_log)); finf->our_log.item_root = lt->item_root; finf->our_log.bloom_ref = lt->bloom_ref; + finf->our_log.rid = lt->rid; + finf->our_log.nr = lt->nr; up_write(&finf->rwsem); } From f5863142be65b3cffa9a771fe3210b25ad1b7a6a Mon Sep 17 00:00:00 2001 From: Benjamin LaHaise Date: Wed, 20 May 2020 16:19:03 -0400 Subject: [PATCH 818/920] scoutfs: add data_wait_err for reporting errors Add support for reporting errors to data waiters via a new SCOUTFS_IOC_DATA_WAIT_ERR ioctl. This allows waiters to return an error to readers when staging fails. Signed-off-by: Benjamin LaHaise [zab: renamed to data_wait_err, took ino arg] Signed-off-by: Zach Brown --- kmod/src/data.c | 32 +++++++++++++++++++++ kmod/src/data.h | 4 +++ kmod/src/ioctl.c | 61 ++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 17 +++++++++++ kmod/src/scoutfs_trace.h | 31 ++++++++++++++++++++ 5 files changed, 145 insertions(+) diff --git a/kmod/src/data.c b/kmod/src/data.c index f83089e9..b129aa42 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -1923,6 +1923,8 @@ int scoutfs_data_wait(struct inode *inode, struct scoutfs_data_wait *dw) spin_lock(&rt->lock); rb_erase(&dw->node, &rt->root); RB_CLEAR_NODE(&dw->node); + if (!ret && dw->err) + ret = dw->err; spin_unlock(&rt->lock); return ret; @@ -1936,6 +1938,36 @@ void scoutfs_data_wait_changed(struct inode *inode) wake_up(&wq->waitq); } +long scoutfs_data_wait_err(struct inode *inode, u64 sblock, u64 eblock, + u64 op, long err) +{ + struct super_block *sb = inode->i_sb; + const u64 ino = scoutfs_ino(inode); + DECLARE_DATA_WAIT_ROOT(sb, rt); + struct scoutfs_data_wait *dw; + long nr = 0; + + if (!err) + return 0; + + spin_lock(&rt->lock); + + for (dw = next_data_wait(&rt->root, ino, sblock); + dw; dw = dw_next(dw)) { + if (dw->ino != ino || dw->iblock > eblock) + break; + if ((dw->op & op) && !dw->err) { + dw->err = err; + nr++; + } + } + + spin_unlock(&rt->lock); + if (nr) + scoutfs_data_wait_changed(inode); + return nr; +} + int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr) diff --git a/kmod/src/data.h b/kmod/src/data.h index 9e0a9c87..b4ee7344 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -28,12 +28,14 @@ struct scoutfs_data_wait { u64 chg; u64 ino; u64 iblock; + long err; u8 op; }; #define DECLARE_DATA_WAIT(nm) \ struct scoutfs_data_wait nm = { \ .node.__rb_parent_color = (unsigned long)(&nm.node), \ + .err = 0, \ } struct scoutfs_traced_extent { @@ -68,6 +70,8 @@ bool scoutfs_data_wait_found(struct scoutfs_data_wait *ow); int scoutfs_data_wait(struct inode *inode, struct scoutfs_data_wait *ow); void scoutfs_data_wait_changed(struct inode *inode); +long scoutfs_data_wait_err(struct inode *inode, u64 sblock, u64 eblock, u64 op, + long err); int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, struct scoutfs_ioctl_data_waiting_entry *dwe, unsigned int nr); diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 24317e73..a321a1db 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -348,6 +348,65 @@ out: return ret; } +static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_data_wait_err args; + struct scoutfs_lock *lock = NULL; + struct inode *inode = NULL; + u64 sblock; + u64 eblock; + long ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&args, (void __user *)arg, sizeof(args))) + return -EFAULT; + if (args.count == 0) + return 0; + if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err)) + return -EINVAL; + if ((args.op & SCOUTFS_IOC_DWO_UNKNOWN) || !IS_ERR_VALUE(args.err)) + return -EINVAL; + + trace_scoutfs_ioc_data_wait_err(sb, &args); + + sblock = args.offset >> SCOUTFS_BLOCK_SHIFT; + eblock = (args.offset + args.count - 1) >> SCOUTFS_BLOCK_SHIFT; + + if (sblock > eblock) + return -EINVAL; + + inode = scoutfs_ilookup(sb, args.ino); + if (!inode) { + ret = -ESTALE; + goto out; + } + + mutex_lock(&inode->i_mutex); + + ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, + SCOUTFS_LKF_REFRESH_INODE, inode, &lock); + if (ret) + goto unlock; + + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + } else if (scoutfs_inode_data_version(inode) != args.data_version) { + ret = -ESTALE; + } else { + ret = scoutfs_data_wait_err(inode, sblock, eblock, args.op, + args.err); + } + + scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); +unlock: + mutex_unlock(&inode->i_mutex); + iput(inode); +out: + return ret; +} + /* * Write the archived contents of the file back if the data_version * still matches. @@ -832,6 +891,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_find_xattrs(file, arg); case SCOUTFS_IOC_STATFS_MORE: return scoutfs_ioc_statfs_more(file, arg); + case SCOUTFS_IOC_DATA_WAIT_ERR: + return scoutfs_ioc_data_wait_err(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index df0c1b54..4b635f88 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -346,5 +346,22 @@ struct scoutfs_ioctl_statfs_more { #define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ struct scoutfs_ioctl_statfs_more) +/* + * Cause matching waiters to return an error. + * + * Find current waiters that match the inode, op, and block range to wake + * up and return an error. + */ +struct scoutfs_ioctl_data_wait_err { + __u64 ino; + __u64 data_version; + __u64 offset; + __u64 count; + __u64 op; + __s64 err; +}; + +#define SCOUTFS_IOC_DATA_WAIT_ERR _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ + struct scoutfs_ioctl_data_wait_err) #endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 66867318..b87a5443 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -556,6 +556,37 @@ TRACE_EVENT(scoutfs_ioc_stage, __entry->offset, __entry->count) ); +TRACE_EVENT(scoutfs_ioc_data_wait_err, + TP_PROTO(struct super_block *sb, + struct scoutfs_ioctl_data_wait_err *args), + + TP_ARGS(sb, args), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, vers) + __field(__u64, offset) + __field(__u64, count) + __field(__u64, op) + __field(__s64, err) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = args->ino; + __entry->vers = args->data_version; + __entry->offset = args->offset; + __entry->count = args->count; + __entry->op = args->op; + __entry->err = args->err; + ), + + TP_printk(SCSBF" ino %llu vers %llu offset %llu count %llu op %llx err %lld", + SCSB_TRACE_ARGS, __entry->ino, __entry->vers, + __entry->offset, __entry->count, __entry->op, __entry->err) +); + DEFINE_EVENT(scoutfs_ino_ret_class, scoutfs_ioc_stage_ret, TP_PROTO(struct super_block *sb, u64 ino, int ret), TP_ARGS(sb, ino, ret) From ff9386faba06dd8f756a7a3321c3d84a0ede41dc Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 May 2020 13:59:37 -0700 Subject: [PATCH 819/920] scoutfs: export server commit holds The calls for holding and applying commits in the server are currently private. The lock server is a server component that has been seperated out into its own file. Most of the time the server calls it during commits so the btree changes made in the lock server are protected by the commits. But there are btree calls in the lock server that happen outside of calls from the server. Exporting these calls will let the lock server make all its btree changes in server commits. Signed-off-by: Zach Brown --- kmod/src/server.c | 40 ++++++++++++++++++++++------------------ kmod/src/server.h | 2 ++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index a371748f..7cd47f18 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -122,8 +122,12 @@ static void stop_server(struct server_info *server) * blocks which can be merged back into avail. We reference the stable * freed tree in the super because the server allocator's freed tree is * going to be added to as blocks are freed during the merge. + * + * This is exported for server components isolated in their own files + * (lock_server) and which are not called directly by the server core + * (async timeout work). */ -static int hold_commit(struct super_block *sb) +int scoutfs_server_hold_commit(struct super_block *sb) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); @@ -178,7 +182,7 @@ static int hold_commit(struct super_block *sb) * will race to make their changes and they'll all be applied by the * next commit after that. */ -static int apply_commit(struct super_block *sb, int err) +int scoutfs_server_apply_commit(struct super_block *sb, int err) { DECLARE_SERVER_INFO(sb, server); struct commit_waiter cw; @@ -299,7 +303,7 @@ static int server_alloc_inodes(struct super_block *sb, memcpy(&lecount, arg, arg_len); - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; @@ -309,7 +313,7 @@ static int server_alloc_inodes(struct super_block *sb, le64_add_cpu(&super->next_ino, nr); spin_unlock(&sbi->next_ino_lock); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret == 0) { ial.ino = cpu_to_le64(ino); ial.nr = cpu_to_le64(nr); @@ -348,7 +352,7 @@ static int server_get_log_trees(struct super_block *sb, goto out; } - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; @@ -433,7 +437,7 @@ static int server_get_log_trees(struct super_block *sb, unlock: mutex_unlock(&server->logs_mutex); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret == 0) { lt.meta_avail = ltv.meta_avail; lt.meta_freed = ltv.meta_freed; @@ -474,7 +478,7 @@ static int server_commit_log_trees(struct super_block *sb, } lt = arg; - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret < 0) { scoutfs_err(sb, "server error preparing commit: %d", ret); goto out; @@ -530,7 +534,7 @@ static int server_commit_log_trees(struct super_block *sb, unlock: mutex_unlock(&server->logs_mutex); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret < 0) scoutfs_err(sb, "server error commiting client logs: %d", ret); out: @@ -659,7 +663,7 @@ static int server_advance_seq(struct super_block *sb, } memcpy(&their_seq, arg, sizeof(their_seq)); - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; @@ -690,7 +694,7 @@ static int server_advance_seq(struct super_block *sb, &tsk, sizeof(tsk), NULL, 0); out: up_write(&server->seq_rwsem); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); return scoutfs_net_response(sb, conn, cmd, id, ret, &next_seq, sizeof(next_seq)); @@ -1022,7 +1026,7 @@ static int server_greeting(struct super_block *sb, } if (gr->server_term == 0) { - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret < 0) goto send_err; @@ -1035,7 +1039,7 @@ static int server_greeting(struct super_block *sb, le64_to_cpu(gr->flags)); mutex_unlock(&server->farewell_mutex); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); queue_work(server->wq, &server->farewell_work); } else { umb = gr->unmount_barrier; @@ -1074,13 +1078,13 @@ send_err: if (le64_to_cpu(gr->server_term) != server->term) { /* we're now doing two commits per greeting, not great */ - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; ret = scoutfs_lock_server_greeting(sb, le64_to_cpu(gr->rid), gr->server_term != 0); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret) goto out; } @@ -1224,7 +1228,7 @@ static void farewell_worker(struct work_struct *work) /* process and send farewell responses */ list_for_each_entry_safe(fw, tmp, &send, entry) { - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; @@ -1233,20 +1237,20 @@ static void farewell_worker(struct work_struct *work) reclaim_log_trees(sb, fw->rid) ?: delete_mounted_client(sb, fw->rid); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret) goto out; } /* update the unmount barrier if we deleted all voting clients */ if (deleted && nr_mounted == 0) { - ret = hold_commit(sb); + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; le64_add_cpu(&super->unmount_barrier, 1); - ret = apply_commit(sb, ret); + ret = scoutfs_server_apply_commit(sb, ret); if (ret) goto out; } diff --git a/kmod/src/server.h b/kmod/src/server.h index c3e82541..0bc92ab8 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -62,6 +62,8 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock *nl); int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); +int scoutfs_server_hold_commit(struct super_block *sb); +int scoutfs_server_apply_commit(struct super_block *sb, int err); struct sockaddr_in; struct scoutfs_quorum_elected_info; From c98e75006efc28df0dac016d16e5bbb274512d7e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 28 May 2020 14:04:41 -0700 Subject: [PATCH 820/920] scoutfs: remove lock_client entries in commit The lock server maintains some items in btrees in the server. It is usually called by the server core during a commit so it doesn't need to worry about managing commits. But the lock recovery timeout code happens in its own async context. It needs to protect the lock_client item removals with a commit. This was causing failures during xfstests that simulate node crashes by unmounting with dm-flakey. Lock recovery would dirty blocks in the btree writer outside of a commit. The first server commit holder would find dirty blocks and throw an assertion indicating that someone modified blocks without holding a commit. Signed-off-by: Zach Brown --- kmod/src/lock_server.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index f4bf64a0..03460ef3 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -773,6 +773,10 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) u64 rid; int ret; + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + /* we enter recovery if there are any client records */ for (rid = 0; ; rid++) { cbk.rid = cpu_to_be64(rid); @@ -802,7 +806,6 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) scoutfs_err(sb, "client rid %016llx lock recovery timed out", rid); - /* XXX these aren't immediately committed */ cbk.rid = cpu_to_be64(rid); ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, &super->lock_clients, @@ -811,6 +814,8 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) break; } + ret = scoutfs_server_apply_commit(sb, ret); +out: /* force processing all pending lock requests */ if (ret == 0) ret = finished_recovery(sb, 0, false); From 22716c0389768b2268c29161a0ac27b2f1197510 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Apr 2020 17:51:56 -0700 Subject: [PATCH 821/920] scoutfs: add scoutfs_key_is_zeros() Add a little function for testing if a given scoutfs key is all zeros. Signed-off-by: Zach Brown --- kmod/src/key.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/key.h b/kmod/src/key.h index 7709438d..9427f2e4 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -80,6 +80,13 @@ static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) key->_sk_fourth = 0; } +static inline bool scoutfs_key_is_zeros(struct scoutfs_key *key) +{ + return key->sk_zone == 0 && key->_sk_first == 0 && key->sk_type == 0 && + key->_sk_second == 0 && key->_sk_third == 0 && + key->_sk_fourth == 0; +} + static inline void scoutfs_key_copy_or_zeros(struct scoutfs_key *dst, struct scoutfs_key *src) { From f9df3ada6cc55f08edfa27ccb7ccf2183b94f8fb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 22 Apr 2020 17:52:15 -0700 Subject: [PATCH 822/920] scoutfs: remove MAX key TYPE and ZONE These were used for constructing arrays of string mappings of key fields. We don't print keys with symbolic strings anymore so we don't need to maintain these values anymore. Signed-off-by: Zach Brown --- kmod/src/format.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 2d295d20..493a4ae3 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -348,7 +348,6 @@ struct scoutfs_bloom_block { #define SCOUTFS_RID_ZONE 3 #define SCOUTFS_FS_ZONE 4 #define SCOUTFS_LOCK_ZONE 5 -#define SCOUTFS_MAX_ZONE 8 /* power of 2 is efficient */ /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -373,9 +372,6 @@ struct scoutfs_bloom_block { /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 -#define SCOUTFS_MAX_TYPE 8 /* power of 2 is efficient */ - - /* * The extents that map blocks in a fixed-size logical region of a file * are packed and stored in item values. The packed extents are From ad99636af8582c9774c4c98bc11e2c3a6b185987 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 23 Apr 2020 10:29:23 -0700 Subject: [PATCH 823/920] scoutfs: use scoutfs_key as btree key The btree currently uses variable length big-endian buffers that are compared with memcmp() as keys. This is a historical relic of the time when keys could be very large. We had dirent keys that included the name and manifest entries that included those fs keys. But now all the btree callers are jumping through hoops to translate their fs keys into big-endian btree keys. And the memcmp() of the keys is showing up in profiles. This makes the btree take native scoutfs_key structs as its key. The forest callers which are working with fs keys can just pass their keys straight through. The server btree callers with their private btrees get key fields definied for their use instead of having individual big-endian key structs. A nice side-effect of this is that splitting parents doesn't have to assume that a maximal key will be inserted by a child split. We can have more keys in parents and wider trees. Signed-off-by: Zach Brown --- kmod/src/btree.c | 247 ++++++++++++++++------------------------- kmod/src/btree.h | 29 ++--- kmod/src/forest.c | 83 +++++--------- kmod/src/format.h | 75 ++++--------- kmod/src/key.h | 32 ++---- kmod/src/lock_server.c | 50 ++++----- kmod/src/server.c | 168 +++++++++++++--------------- 7 files changed, 273 insertions(+), 411 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 0a7405b4..0aa2edd6 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -84,32 +84,22 @@ enum { BTW_DELETE = (1 << 7), /* walking to delete, try merging */ }; -/* - * This greatest key value is stored down the right spine of the tree - * and has to be sorted by memcmp() greater than all possible keys in - * all btrees. - */ -static char max_key[SCOUTFS_BTREE_MAX_KEY_LEN] = { - [0 ... (SCOUTFS_BTREE_MAX_KEY_LEN - 1)] = 0xff, -}; - -/* number of contiguous bytes used by the item header, key, and value */ -static inline unsigned int len_bytes(unsigned key_len, unsigned val_len) +/* number of contiguous bytes used by the item and it's value */ +static inline unsigned int len_bytes(unsigned val_len) { - return sizeof(struct scoutfs_btree_item) + key_len + val_len; + return sizeof(struct scoutfs_btree_item) + val_len; } /* number of contiguous bytes used an existing item */ static inline unsigned int item_bytes(struct scoutfs_btree_item *item) { - return len_bytes(le16_to_cpu(item->key_len), le16_to_cpu(item->val_len)); + return len_bytes(le16_to_cpu(item->val_len)); } /* total block bytes used by an item: header, item, key, value */ -static inline unsigned int all_len_bytes(unsigned key_len, unsigned val_len) +static inline unsigned int all_len_bytes(unsigned val_len) { - return sizeof(struct scoutfs_btree_item_header) + - len_bytes(key_len, val_len); + return sizeof(struct scoutfs_btree_item_header) + len_bytes(val_len); } /* @@ -138,8 +128,7 @@ static inline int min_used_bytes(int block_size) /* total block bytes used by an existing item */ static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) { - return all_len_bytes(le16_to_cpu(item->key_len), - le16_to_cpu(item->val_len)); + return all_len_bytes(le16_to_cpu(item->val_len)); } /* number of free bytes between last item header and first item */ @@ -176,19 +165,14 @@ last_item(struct scoutfs_btree_block *bt) return pos_item(bt, le32_to_cpu(bt->nr_items) - 1); } -static inline void *item_key(struct scoutfs_btree_item *item) +static inline struct scoutfs_key *item_key(struct scoutfs_btree_item *item) { - return item->data; -} - -static inline unsigned item_key_len(struct scoutfs_btree_item *item) -{ - return le16_to_cpu(item->key_len); + return &item->key; } static inline void *item_val(struct scoutfs_btree_item *item) { - return item_key(item) + le16_to_cpu(item->key_len); + return item->val; } static inline unsigned item_val_len(struct scoutfs_btree_item *item) @@ -196,12 +180,6 @@ static inline unsigned item_val_len(struct scoutfs_btree_item *item) return le16_to_cpu(item->val_len); } -static inline int cmp_keys(void *a, unsigned a_len, void *b, unsigned b_len) -{ - return memcmp(a, b, min(a_len, b_len)) ?: - a_len < b_len ? -1 : a_len > b_len ? 1 : 0; -} - /* * Returns the sorted item position that an item with the given key * should occupy. @@ -213,7 +191,7 @@ static inline int cmp_keys(void *a, unsigned a_len, void *b, unsigned b_len) * If the given key is greater then all items' keys then the number of * items can be returned. */ -static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, +static int find_pos(struct scoutfs_btree_block *bt, struct scoutfs_key *key, int *cmp) { struct scoutfs_btree_item *item; @@ -227,7 +205,7 @@ static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, pos = start + (end - start) / 2; item = pos_item(bt, pos); - *cmp = cmp_keys(key, key_len, item_key(item), item_key_len(item)); + *cmp = scoutfs_key_compare(key, item_key(item)); if (*cmp < 0) { end = pos; } else if (*cmp > 0) { @@ -250,20 +228,19 @@ static int find_pos(struct scoutfs_btree_block *bt, void *key, unsigned key_len, * there's space for the item and its metadata. */ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, - void *key, unsigned key_len, void *val, - unsigned val_len) + struct scoutfs_key *key, void *val, unsigned val_len) { unsigned int nr = le32_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; unsigned all_bytes; - all_bytes = all_len_bytes(key_len, val_len); + all_bytes = all_len_bytes(val_len); BUG_ON(free_bytes(bt) < all_bytes); if (pos < nr) memmove_arr(bt->item_hdrs, pos + 1, pos, nr - pos); - le32_add_cpu(&bt->free_end, -len_bytes(key_len, val_len)); + le32_add_cpu(&bt->free_end, -len_bytes(val_len)); bt->item_hdrs[pos].off = bt->free_end; nr++; bt->nr_items = cpu_to_le32(nr); @@ -272,10 +249,9 @@ static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, offsetof(struct scoutfs_btree_block, item_hdrs[nr])); item = pos_item(bt, pos); - item->key_len = cpu_to_le16(key_len); + *item_key(item) = *key; item->val_len = cpu_to_le16(val_len); - memcpy(item_key(item), key, key_len); if (val_len) memcpy(item_val(item), val, val_len); } @@ -361,8 +337,8 @@ static void move_items(struct scoutfs_btree_block *dst, while (f < le32_to_cpu(src->nr_items) && to_move > 0) { from = pos_item(src, f); - create_item(dst, t, item_key(from), item_key_len(from), - item_val(from), item_val_len(from)); + create_item(dst, t, item_key(from), item_val(from), + item_val_len(from)); to_move -= all_item_bytes(from); @@ -522,14 +498,14 @@ out: */ static void create_parent_item(struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *child, - void *key, unsigned key_len) + struct scoutfs_key *key) { struct scoutfs_btree_ref ref = { .blkno = child->hdr.blkno, .seq = child->hdr.seq, }; - create_item(parent, pos, key, key_len, &ref, sizeof(ref)); + create_item(parent, pos, key, &ref, sizeof(ref)); } /* @@ -543,8 +519,7 @@ static void update_parent_item(struct scoutfs_btree_block *parent, struct scoutfs_btree_item *item = last_item(child); delete_item(parent, pos); - create_parent_item(parent, pos, child, - item_key(item), item_key_len(item)); + create_parent_item(parent, pos, child, item_key(item)); } /* @@ -562,7 +537,7 @@ static int try_split(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, unsigned val_len, + struct scoutfs_key *key, unsigned val_len, struct scoutfs_btree_block *parent, unsigned pos, struct scoutfs_btree_block *right) { @@ -570,6 +545,7 @@ static int try_split(struct super_block *sb, struct scoutfs_block *par_bl = NULL; struct scoutfs_btree_block *left; struct scoutfs_btree_item *item; + struct scoutfs_key max_key; unsigned int all_bytes; int ret; int err; @@ -579,7 +555,7 @@ static int try_split(struct super_block *sb, else if (right->level) all_bytes = SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES; else - all_bytes = all_len_bytes(key_len, val_len); + all_bytes = all_len_bytes(val_len); if (free_bytes(right) >= all_bytes) return 0; @@ -608,16 +584,16 @@ static int try_split(struct super_block *sb, root->ref.blkno = parent->hdr.blkno; root->ref.seq = parent->hdr.seq; + scoutfs_key_set_ones(&max_key); + pos = 0; - create_parent_item(parent, pos, right, - &max_key, sizeof(max_key)); + create_parent_item(parent, pos, right, &max_key); } move_items(left, right, false, used_total(right) / 2); item = last_item(left); - create_parent_item(parent, pos, left, - item_key(item), item_key_len(item)); + create_parent_item(parent, pos, left, item_key(item)); scoutfs_block_put(sb, left_bl); scoutfs_block_put(sb, par_bl); @@ -751,8 +727,8 @@ static int verify_btree_block(struct scoutfs_btree_block *bt, int level) item = pos_item(bt, i); bytes += item_bytes(item); - if (i > 0 && cmp_keys(item_key(item), item_key_len(item), - item_key(prev), item_key_len(prev)) <= 0) + if (i > 0 && scoutfs_key_compare(item_key(item), + item_key(prev)) <= 0) goto out; prev = item; @@ -773,9 +749,9 @@ out: after_off, bytes); for (i = 0; i < nr; i++) { item = pos_item(bt, i); - printk(" [%u] off %u key_len %u val_len %u\n", + printk(" [%u] off %u val_len %u\n", i, le32_to_cpu(bt->item_hdrs[i].off), - item_key_len(item), item_val_len(item)); + item_val_len(item)); } BUG_ON(bad); } @@ -783,22 +759,6 @@ out: return 0; } -/* XXX bleh, this should probably share code with the key_buf equivalent */ -static void inc_key(u8 *bytes, unsigned *len) -{ - int i; - - if (*len < SCOUTFS_BTREE_MAX_KEY_LEN) { - memset(bytes + *len, 0, SCOUTFS_BTREE_MAX_KEY_LEN - *len); - *len = SCOUTFS_BTREE_MAX_KEY_LEN; - } - - for (i = *len - 1; i >= 0; i--) { - if (++bytes[i] != 0) - break; - } -} - /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their @@ -819,10 +779,10 @@ static int btree_walk(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - int flags, void *key, unsigned key_len, + int flags, struct scoutfs_key *key, unsigned int val_len, - struct scoutfs_block **bl_ret, void *iter_key, - unsigned *iter_len) + struct scoutfs_block **bl_ret, + struct scoutfs_key *iter_key) { struct scoutfs_block *par_bl = NULL; struct scoutfs_block *bl = NULL; @@ -848,8 +808,6 @@ restart: bl = NULL; bt = NULL; level = root->height; - if (iter_len) - *iter_len = 0; pos = 0; ret = 0; @@ -906,8 +864,8 @@ restart: */ ret = 0; if (flags & (BTW_INSERT | BTW_DELETE)) - ret = try_split(sb, alloc, wri, root, key, key_len, - val_len, parent, pos, bt); + ret = try_split(sb, alloc, wri, root, key, val_len, + parent, pos, bt); if (ret == 0 && (flags & BTW_DELETE) && parent) ret = try_merge(sb, alloc, wri, root, parent, pos, bt); if (ret > 0) @@ -922,7 +880,7 @@ restart: nr = le32_to_cpu(bt->nr_items); /* Find the next child block for the search key. */ - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (pos >= nr) { scoutfs_corruption(sb, SC_BTREE_NO_CHILD_REF, corrupt_btree_block_level, @@ -940,14 +898,12 @@ restart: /* give the caller the next key to iterate towards */ if (iter_key && (flags & BTW_NEXT) && (pos < (nr - 1))) { item = pos_item(bt, pos); - *iter_len = item_key_len(item); - memcpy(iter_key, item_key(item), *iter_len); - inc_key(iter_key, iter_len); + *iter_key = *item_key(item); + scoutfs_key_inc(iter_key); } else if (iter_key && (flags & BTW_PREV) && (pos > 0)) { item = pos_item(bt, pos - 1); - *iter_len = item_key_len(item); - memcpy(iter_key, item_key(item), *iter_len); + *iter_key = *item_key(item); } scoutfs_block_put(sb, par_bl); @@ -982,7 +938,6 @@ static void init_item_ref(struct scoutfs_btree_item_ref *iref, iref->sb = sb; iref->bl = bl; iref->key = item_key(item); - iref->key_len = le16_to_cpu(item->key_len); iref->val = item_val(item); iref->val_len = le16_to_cpu(item->val_len); } @@ -1000,8 +955,9 @@ void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref) * item ref. They're given a reference to the block that they'll drop * when they're done. */ -int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { struct scoutfs_btree_item *item; @@ -1014,11 +970,10 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root if (WARN_ON_ONCE(iref->key)) return -EINVAL; - ret = btree_walk(sb, NULL, NULL, root, 0, key, key_len, 0, &bl, - NULL, NULL); + ret = btree_walk(sb, NULL, NULL, root, 0, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (cmp == 0) { item = pos_item(bt, pos); init_item_ref(iref, sb, bl, item); @@ -1033,11 +988,9 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root return ret; } -static bool invalid_item(void *key, unsigned key_len, unsigned val_len) +static bool invalid_item(unsigned val_len) { - return WARN_ON_ONCE(key_len == 0) || - WARN_ON_ONCE(key_len > SCOUTFS_BTREE_MAX_KEY_LEN) || - WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN); + return WARN_ON_ONCE(val_len > SCOUTFS_BTREE_MAX_VAL_LEN); } /* @@ -1053,7 +1006,7 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len) { struct scoutfs_btree_block *bt; @@ -1062,16 +1015,16 @@ int scoutfs_btree_insert(struct super_block *sb, int cmp; int ret; - if (invalid_item(key, key_len, val_len)) + if (invalid_item(val_len)) return -EINVAL; - ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, - key, key_len, val_len, &bl, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (cmp) { - create_item(bt, pos, key, key_len, val, val_len); + create_item(bt, pos, key, val, val_len); ret = 0; } else { ret = -EEXIST; @@ -1097,7 +1050,7 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len) { struct scoutfs_btree_block *bt; @@ -1106,17 +1059,17 @@ int scoutfs_btree_update(struct super_block *sb, int cmp; int ret; - if (invalid_item(key, key_len, val_len)) + if (invalid_item(val_len)) return -EINVAL; - ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, - key, key_len, val_len, &bl, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (cmp == 0) { delete_item(bt, pos); - create_item(bt, pos, key, key_len, val, val_len); + create_item(bt, pos, key, val, val_len); ret = 0; } else { ret = -ENOENT; @@ -1136,7 +1089,7 @@ int scoutfs_btree_force(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len) { struct scoutfs_btree_block *bt; @@ -1145,17 +1098,17 @@ int scoutfs_btree_force(struct super_block *sb, int cmp; int ret; - if (invalid_item(key, key_len, val_len)) + if (invalid_item(val_len)) return -EINVAL; - ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, - key, key_len, val_len, &bl, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, key, + val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (cmp == 0) delete_item(bt, pos); - create_item(bt, pos, key, key_len, val, val_len); + create_item(bt, pos, key, val, val_len); scoutfs_block_put(sb, bl); } @@ -1170,7 +1123,7 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len) + struct scoutfs_key *key) { struct scoutfs_btree_block *bt; struct scoutfs_block *bl; @@ -1178,11 +1131,11 @@ int scoutfs_btree_delete(struct super_block *sb, int cmp; int ret; - ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, - key, key_len, 0, &bl, NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, key, + 0, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); if (cmp == 0) { if (le32_to_cpu(bt->nr_items) == 1) { /* remove final empty block */ @@ -1220,16 +1173,14 @@ int scoutfs_btree_delete(struct super_block *sb, * blocks. */ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, - int flags, void *key, unsigned key_len, + int flags, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - unsigned iter_len; - unsigned walk_len; - void *iter_key; - void *walk_key; + struct scoutfs_key iter_key; + struct scoutfs_key walk_key; int pos; int cmp; int ret; @@ -1238,24 +1189,17 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, WARN_ON_ONCE(iref->key)) return -EINVAL; - walk_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); - iter_key = kmalloc(SCOUTFS_BTREE_MAX_KEY_LEN, GFP_NOFS); - if (!walk_key || !iter_key) { - ret = -ENOMEM; - goto out; - } - - memcpy(walk_key, key, key_len); - walk_len = key_len; + walk_key = *key; for (;;) { - ret = btree_walk(sb, NULL, NULL, root, flags, walk_key, - walk_len, 0, &bl, iter_key, &iter_len); + scoutfs_key_set_zeros(&iter_key); + ret = btree_walk(sb, NULL, NULL, root, flags, &walk_key, + 0, &bl, &iter_key); if (ret < 0) break; bt = bl->data; - pos = find_pos(bt, key, key_len, &cmp); + pos = find_pos(bt, key, &cmp); /* point pos towards iteration, find_pos already for _NEXT */ if ((flags & BTW_AFTER) && cmp == 0) @@ -1276,9 +1220,8 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, scoutfs_block_put(sb, bl); /* nothing in this leaf, walk gave us a key */ - if (iter_len > 0) { - memcpy(walk_key, iter_key, iter_len); - walk_len = iter_len; + if (!scoutfs_key_is_zeros(&iter_key)) { + walk_key = iter_key; continue; } @@ -1286,39 +1229,36 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, break; } -out: - kfree(walk_key); - kfree(iter_key); - return ret; } int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { - return btree_iter(sb, root, BTW_NEXT, key, key_len, iref); + return btree_iter(sb, root, BTW_NEXT, key, iref); } int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { - return btree_iter(sb, root, BTW_NEXT | BTW_AFTER, key, key_len, iref); + return btree_iter(sb, root, BTW_NEXT | BTW_AFTER, key, iref); } int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { - return btree_iter(sb, root, BTW_PREV, key, key_len, iref); + return btree_iter(sb, root, BTW_PREV, key, iref); } -int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, +int scoutfs_btree_before(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { - return btree_iter(sb, root, BTW_PREV | BTW_BEFORE, key, key_len, iref); + return btree_iter(sb, root, BTW_PREV | BTW_BEFORE, key, iref); } /* @@ -1332,18 +1272,17 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len) + struct scoutfs_key *key) { struct scoutfs_btree_block *bt; struct scoutfs_block *bl; int cmp; int ret; - ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, key_len, 0, &bl, - NULL, NULL); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - find_pos(bt, key, key_len, &cmp); + find_pos(bt, key, &cmp); if (cmp == 0) ret = 0; else diff --git a/kmod/src/btree.h b/kmod/src/btree.h index e37ab023..133832ba 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -10,8 +10,7 @@ struct scoutfs_block; struct scoutfs_btree_item_ref { struct super_block *sb; struct scoutfs_block *bl; - void *key; - unsigned key_len; + struct scoutfs_key *key; void *val; unsigned val_len; }; @@ -20,49 +19,51 @@ struct scoutfs_btree_item_ref { struct scoutfs_btree_item_ref name = {NULL,} -int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, +int scoutfs_btree_lookup(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_update(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_force(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len); + struct scoutfs_key *key); int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); -int scoutfs_btree_before(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, +int scoutfs_btree_before(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - void *key, unsigned key_len); + struct scoutfs_key *key); void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref); diff --git a/kmod/src/forest.c b/kmod/src/forest.c index bcb7b809..af44204e 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -301,7 +301,6 @@ static int refresh_bloom_roots(struct super_block *sb, { DECLARE_FOREST_INFO(sb, finf); struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); - struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct forest_bloom_nrs bloom; @@ -309,6 +308,7 @@ static int refresh_bloom_roots(struct super_block *sb, struct forest_root *fr = NULL; struct scoutfs_bloom_block *bb; struct scoutfs_block *bl; + struct scoutfs_key key; int ret; int i; @@ -328,11 +328,10 @@ static int refresh_bloom_roots(struct super_block *sb, calc_bloom_nrs(&bloom, &lock->start); - memset(<k, 0, sizeof(ltk)); - for (;; be64_add_cpu(<k.nr, 1)) { + scoutfs_key_init_log_trees(&key, 0, 0); + for (;; scoutfs_key_inc(&key)) { - ret = scoutfs_btree_next(sb, &super.logs_root, - <k, sizeof(ltk), &iref); + ret = scoutfs_btree_next(sb, &super.logs_root, &key, &iref); if (ret == -ENOENT) { ret = 0; break; @@ -340,9 +339,8 @@ static int refresh_bloom_roots(struct super_block *sb, if (ret < 0) goto out; - if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && - iref.val_len == sizeof(struct scoutfs_log_trees_val)) { - memcpy(<k, iref.key, iref.key_len); + if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + key = *iref.key; memcpy(<v, iref.val, iref.val_len); } else { ret = -EIO; @@ -369,8 +367,8 @@ static int refresh_bloom_roots(struct super_block *sb, scoutfs_block_put(sb, bl); trace_scoutfs_forest_bloom_search(sb, &lock->start, - be64_to_cpu(ltk.rid), - be64_to_cpu(ltk.nr), + le64_to_cpu(key.sklt_rid), + le64_to_cpu(key.sklt_nr), le64_to_cpu(ltv.bloom_ref.blkno), le64_to_cpu(ltv.bloom_ref.seq), i); @@ -380,8 +378,8 @@ static int refresh_bloom_roots(struct super_block *sb, continue; /* use our dirty log instead of the old committed version */ - if (be64_to_cpu(ltk.rid) == le64_to_cpu(finf->our_log.rid) && - be64_to_cpu(ltk.nr) == le64_to_cpu(finf->our_log.nr)) { + if (key.sklt_rid == finf->our_log.rid && + key.sklt_nr == finf->our_log.nr) { add_our_log_root(finf, lpriv); continue; } @@ -394,8 +392,8 @@ static int refresh_bloom_roots(struct super_block *sb, } fr->item_root = ltv.item_root; - fr->rid = be64_to_cpu(ltk.rid); - fr->nr = be64_to_cpu(ltk.nr); + fr->rid = le64_to_cpu(key.sklt_rid); + fr->nr = le64_to_cpu(key.sklt_nr); list_add_tail(&fr->entry, &lpriv->roots); @@ -568,7 +566,6 @@ int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); struct forest_lock_private *lpriv; SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key_be kbe; struct forest_root *fr; u64 found_vers; u64 vers; @@ -584,8 +581,6 @@ int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, goto out; } - scoutfs_key_to_be(&kbe, key); - retry: down_read(&lpriv->rwsem); @@ -600,8 +595,7 @@ retry: break; read_lock_forest_root(finf, lpriv, fr); - err = scoutfs_btree_lookup(sb, &fr->item_root, - &kbe, sizeof(kbe), &iref); + err = scoutfs_btree_lookup(sb, &fr->item_root, key, &iref); if (err < 0) read_unlock_forest_root(finf, lpriv, fr); if (err == -ENOENT) @@ -705,14 +699,14 @@ static inline bool forest_iter_key_within(struct scoutfs_key *a, static inline int forest_iter_btree_search(struct super_block *sb, struct scoutfs_btree_root *root, - void *key, unsigned key_len, + struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref, bool forward) { if (forward) - return scoutfs_btree_next(sb, root, key, key_len, iref); + return scoutfs_btree_next(sb, root, key, iref); else - return scoutfs_btree_prev(sb, root, key, key_len, iref); + return scoutfs_btree_prev(sb, root, key, iref); } struct forest_iter_pos { @@ -835,7 +829,6 @@ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, SCOUTFS_BTREE_ITEM_REF(iref); struct rb_root iter_root = RB_ROOT; struct scoutfs_key found_key; - struct scoutfs_key_be kbe; struct forest_iter_pos *nip; struct forest_iter_pos *ip; struct forest_root *fr; @@ -896,12 +889,9 @@ retry: /* search for the next item in the root */ if (ip->vers == 0) { - scoutfs_key_to_be(&kbe, &ip->key); - read_lock_forest_root(finf, lpriv, fr); ret = forest_iter_btree_search(sb, &fr->item_root, - &kbe, sizeof(kbe), - &iref, fwd); + &ip->key, &iref, fwd); if (ret < 0) read_unlock_forest_root(finf, lpriv, fr); if (ret == -ENOENT) { @@ -911,7 +901,7 @@ retry: if (ret < 0) goto unlock; - scoutfs_key_from_be(&ip->key, iref.key); + ip->key = *iref.key; ip->vers = item_vers(lpriv, fr, iref.val); ip->deletion = item_is_deletion(lpriv, fr, iref.val); @@ -1026,11 +1016,10 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, { DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); struct scoutfs_super_block super; - struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_key_be kbe; struct scoutfs_key found; + struct scoutfs_key ltk; bool have_next; int ret; @@ -1042,13 +1031,12 @@ retry: srefs.fs_ref = super.fs_root.ref; srefs.logs_ref = super.logs_root.ref; - memset(<k, 0, sizeof(ltk)); + scoutfs_key_init_log_trees(<k, 0, 0); have_next = false; - for (;; be64_add_cpu(<k.nr, 1)) { + for (;; scoutfs_key_inc(<k)) { - ret = scoutfs_btree_next(sb, &super.logs_root, - <k, sizeof(ltk), &iref); + ret = scoutfs_btree_next(sb, &super.logs_root, <k, &iref); if (ret == -ENOENT) { if (have_next) ret = 0; @@ -1059,9 +1047,8 @@ retry: if (ret < 0) goto out; - if (iref.key_len == sizeof(ltk) && - iref.val_len == sizeof(ltv)) { - memcpy(<k, iref.key, iref.key_len); + if (iref.val_len == sizeof(ltv)) { + ltk = *iref.key; memcpy(<v, iref.val, iref.val_len); } else { ret = -EIO; @@ -1070,9 +1057,7 @@ retry: if (ret < 0) goto out; - scoutfs_key_to_be(&kbe, key); - ret = scoutfs_btree_next(sb, <v.item_root, - &kbe, sizeof(kbe), &iref); + ret = scoutfs_btree_next(sb, <v.item_root, key, &iref); if (ret == -ENOENT) continue; if (ret == -ESTALE) @@ -1080,13 +1065,8 @@ retry: if (ret < 0) goto out; - if (iref.key_len == sizeof(kbe)) - scoutfs_key_from_be(&found, iref.key); - else - ret = -EIO; + found = *iref.key; scoutfs_btree_put_iref(&iref); - if (ret < 0) - goto out; if (!have_next || scoutfs_key_compare(&found, next) < 0) { have_next = true; @@ -1274,7 +1254,6 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, bool check_eexist, bool check_enoent) { DECLARE_FOREST_INFO(sb, finf); - struct scoutfs_key_be kbe; struct kvec *iv = NULL; int ret; @@ -1303,11 +1282,9 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, goto out; } - scoutfs_key_to_be(&kbe, key); - down_write(&finf->rwsem); ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, - &finf->our_log.item_root, &kbe, sizeof(kbe), + &finf->our_log.item_root, key, iv->iov_base, iv->iov_len); up_write(&finf->rwsem); kfree(iv); @@ -1372,7 +1349,6 @@ static int forest_delete(struct super_block *sb, struct scoutfs_key *key, { DECLARE_FOREST_INFO(sb, finf); struct scoutfs_log_item_value liv; - struct scoutfs_key_be kbe; int ret; if (check_enoent) { @@ -1385,14 +1361,13 @@ static int forest_delete(struct super_block *sb, struct scoutfs_key *key, if (ret < 0) goto out; - scoutfs_key_to_be(&kbe, key); liv.vers = cpu_to_le64(lock->write_version); liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; down_write(&finf->rwsem); ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, - &finf->our_log.item_root, - &kbe, sizeof(kbe), &liv, sizeof(liv)); + &finf->our_log.item_root, key, &liv, + sizeof(liv)); up_write(&finf->rwsem); out: return ret; diff --git a/kmod/src/format.h b/kmod/src/format.h index 493a4ae3..740a64fb 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -133,6 +133,20 @@ struct scoutfs_key { #define skpe_base _sk_second #define skpe_part _sk_fourth +/* log trees */ +#define sklt_rid _sk_first +#define sklt_nr _sk_second + +/* lock clients */ +#define sklc_rid _sk_first + +/* seqs */ +#define skts_trans_seq _sk_first +#define skts_rid _sk_second + +/* mounted clients */ +#define skmc_rid _sk_first + struct scoutfs_radix_block { struct scoutfs_block_header hdr; __le32 sm_first; @@ -170,34 +184,17 @@ struct scoutfs_radix_root { ~(__u64)SCOUTFS_RADIX_LG_MASK) #define SCOUTFS_RADIX_BITS_BYTES (SCOUTFS_RADIX_BITS / 8) -/* - * The btree still uses memcmp() to compare keys. We should fix that - * before too long. - */ -struct scoutfs_key_be { - __u8 sk_zone; - __be64 _sk_first; - __u8 sk_type; - __be64 _sk_second; - __be64 _sk_third; - __u8 _sk_fourth; -}__packed; - -/* chose reasonable max key lens that have room for some u64s */ -#define SCOUTFS_BTREE_MAX_KEY_LEN 40 /* when we split we want to have multiple items on each side */ #define SCOUTFS_BTREE_MAX_VAL_LEN (SCOUTFS_BLOCK_SIZE / 8) /* * The min number of free bytes we must leave in a parent as we descend - * to modify. This leaves enough free bytes to insert a possibly maximal - * sized key as a seperator for a child block. Fewer bytes then this - * and split/merge might try to insert a max child item in the parent - * that wouldn't fit. + * to modify. This guarantees enough free bytes in a parent to insert a + * new child reference item as a child block splits. */ #define SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES \ (sizeof(struct scoutfs_btree_item_header) + \ - sizeof(struct scoutfs_btree_item) + SCOUTFS_BTREE_MAX_KEY_LEN +\ + sizeof(struct scoutfs_btree_item) + \ sizeof(struct scoutfs_btree_ref)) /* @@ -233,9 +230,9 @@ struct scoutfs_btree_item_header { } __packed; struct scoutfs_btree_item { - __le16 key_len; + struct scoutfs_key key; __le16 val_len; - __u8 data[0]; + __u8 val[0]; } __packed; struct scoutfs_btree_block { @@ -246,30 +243,6 @@ struct scoutfs_btree_block { struct scoutfs_btree_item_header item_hdrs[0]; } __packed; -/* - * The lock server keeps a persistent record of connected clients so that - * server failover knows who to wait for before resuming operations. - */ -struct scoutfs_lock_client_btree_key { - __be64 rid; -} __packed; - -/* - * The server tracks transaction sequence numbers that clients have - * open. This limits results that can be returned from the seq indices. - */ -struct scoutfs_trans_seq_btree_key { - __be64 trans_seq; - __be64 rid; -} __packed; - -/* - * The server keeps a persistent record of mounted clients. - */ -struct scoutfs_mounted_client_btree_key { - __be64 rid; -} __packed; - struct scoutfs_mounted_client_btree_val { __u8 flags; } __packed; @@ -292,11 +265,6 @@ struct scoutfs_log_trees { __le64 nr; } __packed; -struct scoutfs_log_trees_key { - __be64 rid; - __be64 nr; -} __packed; - struct scoutfs_log_trees_val { struct scoutfs_radix_root meta_avail; struct scoutfs_radix_root meta_freed; @@ -348,6 +316,11 @@ struct scoutfs_bloom_block { #define SCOUTFS_RID_ZONE 3 #define SCOUTFS_FS_ZONE 4 #define SCOUTFS_LOCK_ZONE 5 +/* Items only stored in server btrees */ +#define SCOUTFS_LOG_TREES_ZONE 6 +#define SCOUTFS_LOCK_CLIENTS_ZONE 7 +#define SCOUTFS_TRANS_SEQ_ZONE 8 +#define SCOUTFS_MOUNTED_CLIENT_ZONE 9 /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 diff --git a/kmod/src/key.h b/kmod/src/key.h index 9427f2e4..76b245c9 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -186,29 +186,19 @@ static inline void scoutfs_key_dec(struct scoutfs_key *key) key->sk_zone--; } -static inline void scoutfs_key_to_be(struct scoutfs_key_be *be, - struct scoutfs_key *key) -{ - BUILD_BUG_ON(sizeof(struct scoutfs_key_be) != - sizeof(struct scoutfs_key)); +/* + * Some key types are used by multiple subsystems and shouldn't have + * duplicate private key init functions. + */ - be->sk_zone = key->sk_zone; - be->_sk_first = le64_to_be64(key->_sk_first); - be->sk_type = key->sk_type; - be->_sk_second = le64_to_be64(key->_sk_second); - be->_sk_third = le64_to_be64(key->_sk_third); - be->_sk_fourth = key->_sk_fourth; -} - -static inline void scoutfs_key_from_be(struct scoutfs_key *key, - struct scoutfs_key_be *be) +static inline void scoutfs_key_init_log_trees(struct scoutfs_key *key, + u64 rid, u64 nr) { - key->sk_zone = be->sk_zone; - key->_sk_first = be64_to_le64(be->_sk_first); - key->sk_type = be->sk_type; - key->_sk_second = be64_to_le64(be->_sk_second); - key->_sk_third = be64_to_le64(be->_sk_third); - key->_sk_fourth = be->_sk_fourth; + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_LOG_TREES_ZONE, + .sklt_rid = cpu_to_le64(rid), + .sklt_nr = cpu_to_le64(nr), + }; } #endif diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 03460ef3..117d163e 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -575,6 +575,14 @@ out: return ret; } +static void init_lock_clients_key(struct scoutfs_key *key, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_LOCK_CLIENTS_ZONE, + .sklc_rid = cpu_to_le64(rid), + }; +} + /* * The server received a greeting from a client for the first time. If * the client had already talked to the server then we must find an @@ -589,23 +597,22 @@ int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid, { DECLARE_LOCK_SERVER_INFO(sb, inf); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_lock_client_btree_key cbk; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; - cbk.rid = cpu_to_be64(rid); + init_lock_clients_key(&key, rid); mutex_lock(&inf->mutex); if (should_exist) { - ret = scoutfs_btree_lookup(sb, &super->lock_clients, - &cbk, sizeof(cbk), &iref); + ret = scoutfs_btree_lookup(sb, &super->lock_clients, &key, + &iref); if (ret == 0) scoutfs_btree_put_iref(&iref); } else { ret = scoutfs_btree_insert(sb, inf->alloc, inf->wri, &super->lock_clients, - &cbk, sizeof(cbk), NULL, 0); + &key, NULL, 0); } mutex_unlock(&inf->mutex); @@ -738,15 +745,12 @@ out: return ret; } -static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, - u64 *rid) +static int get_rid_and_put_ref(struct scoutfs_btree_item_ref *iref, u64 *rid) { - struct scoutfs_lock_client_btree_key *cbk; int ret; - if (iref->key_len == sizeof(*cbk) && iref->val_len == 0) { - cbk = iref->key; - *rid = be64_to_cpu(cbk->rid); + if (iref->val_len == 0) { + *rid = le64_to_cpu(iref->key->sklc_rid); ret = 0; } else { ret = -EIO; @@ -767,8 +771,8 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) recovery_dwork.work); struct super_block *sb = inf->sb; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_lock_client_btree_key cbk; SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; bool timed_out; u64 rid; int ret; @@ -779,9 +783,8 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) /* we enter recovery if there are any client records */ for (rid = 0; ; rid++) { - cbk.rid = cpu_to_be64(rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, - &cbk, sizeof(cbk), &iref); + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); if (ret == -ENOENT) { ret = 0; break; @@ -806,10 +809,9 @@ static void scoutfs_lock_server_recovery_timeout(struct work_struct *work) scoutfs_err(sb, "client rid %016llx lock recovery timed out", rid); - cbk.rid = cpu_to_be64(rid); + init_lock_clients_key(&key, rid); ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, - &cbk, sizeof(cbk)); + &super->lock_clients, &key); if (ret) break; } @@ -838,7 +840,6 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) { DECLARE_LOCK_SERVER_INFO(sb, inf); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_lock_client_btree_key cli; struct client_lock_entry *clent; struct client_lock_entry *tmp; struct server_lock_node *snode; @@ -847,10 +848,10 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid) bool freed; int ret = 0; - cli.rid = cpu_to_be64(rid); mutex_lock(&inf->mutex); + init_lock_clients_key(&key, rid); ret = scoutfs_btree_delete(sb, inf->alloc, inf->wri, - &super->lock_clients, &cli, sizeof(cli)); + &super->lock_clients, &key); mutex_unlock(&inf->mutex); if (ret == -ENOENT) { ret = 0; @@ -958,7 +959,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct lock_server_info *inf; SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_lock_client_btree_key cbk; + struct scoutfs_key key; unsigned int nr; u64 rid; int ret; @@ -990,9 +991,8 @@ int scoutfs_lock_server_setup(struct super_block *sb, /* we enter recovery if there are any client records */ nr = 0; for (rid = 0; ; rid++) { - cbk.rid = cpu_to_be64(rid); - ret = scoutfs_btree_next(sb, &super->lock_clients, - &cbk, sizeof(cbk), &iref); + init_lock_clients_key(&key, rid); + ret = scoutfs_btree_next(sb, &super->lock_clients, &key, &iref); if (ret == -ENOENT) break; if (ret == 0) diff --git a/kmod/src/server.c b/kmod/src/server.c index 7cd47f18..f6db3179 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -340,9 +340,9 @@ static int server_get_log_trees(struct super_block *sb, u64 rid = scoutfs_net_client_rid(conn); DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees lt; + struct scoutfs_key key; u64 count; u64 target; int ret; @@ -358,20 +358,16 @@ static int server_get_log_trees(struct super_block *sb, mutex_lock(&server->logs_mutex); - memset(<k, 0, sizeof(ltk)); - ltk.rid = cpu_to_be64(rid); - ltk.nr = cpu_to_be64(U64_MAX); + scoutfs_key_init_log_trees(&key, rid, U64_MAX); - ret = scoutfs_btree_prev(sb, &super->logs_root, - <k, sizeof(ltk), &iref); + ret = scoutfs_btree_prev(sb, &super->logs_root, &key, &iref); if (ret < 0 && ret != -ENOENT) goto unlock; if (ret == 0) { - if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && - iref.val_len == sizeof(struct scoutfs_log_trees_val)) { - memcpy(<k, iref.key, iref.key_len); + if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + key = *iref.key; memcpy(<v, iref.val, iref.val_len); - if (be64_to_cpu(ltk.rid) != rid) + if (le64_to_cpu(key.sklt_rid) != rid) ret = -ENOENT; } else { ret = -EIO; @@ -383,8 +379,8 @@ static int server_get_log_trees(struct super_block *sb, /* initialize new roots if we don't have any */ if (ret == -ENOENT) { - ltk.rid = cpu_to_be64(rid); - ltk.nr = cpu_to_be64(1); + key.sklt_rid = cpu_to_le64(rid); + key.sklt_nr = cpu_to_le64(1); memset(<v, 0, sizeof(ltv)); scoutfs_radix_root_init(sb, <v.meta_avail, true); scoutfs_radix_root_init(sb, <v.meta_freed, true); @@ -432,8 +428,7 @@ static int server_get_log_trees(struct super_block *sb, /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, - &super->logs_root, <k, sizeof(ltk), - <v, sizeof(ltv)); + &super->logs_root, &key, <v, sizeof(ltv)); unlock: mutex_unlock(&server->logs_mutex); @@ -445,8 +440,8 @@ unlock: lt.bloom_ref = ltv.bloom_ref; lt.data_avail = ltv.data_avail; lt.data_freed = ltv.data_freed; - lt.rid = be64_to_le64(ltk.rid); - lt.nr = be64_to_le64(ltk.nr); + lt.rid = key.sklt_rid; + lt.nr = key.sklt_nr; } out: @@ -467,9 +462,9 @@ static int server_commit_log_trees(struct super_block *sb, struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees *lt; + struct scoutfs_key key; int ret; if (arg_len != sizeof(struct scoutfs_log_trees)) { @@ -487,11 +482,9 @@ static int server_commit_log_trees(struct super_block *sb, mutex_lock(&server->logs_mutex); /* find the client's existing item */ - memset(<k, 0, sizeof(ltk)); - ltk.rid = le64_to_be64(lt->rid); - ltk.nr = le64_to_be64(lt->nr); - ret = scoutfs_btree_lookup(sb, &super->logs_root, - <k, sizeof(ltk), &iref); + scoutfs_key_init_log_trees(&key, le64_to_cpu(lt->rid), + le64_to_cpu(lt->nr)); + ret = scoutfs_btree_lookup(sb, &super->logs_root, &key, &iref); if (ret < 0 && ret != -ENOENT) { scoutfs_err(sb, "server error finding client logs: %d", ret); goto unlock; @@ -526,8 +519,7 @@ static int server_commit_log_trees(struct super_block *sb, ltv.data_freed = lt->data_freed; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, - &super->logs_root, <k, sizeof(ltk), - <v, sizeof(ltv)); + &super->logs_root, &key, <v, sizeof(ltv)); if (ret < 0) scoutfs_err(sb, "server error updating client logs: %d", ret); @@ -564,8 +556,8 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_key ltk; struct scoutfs_log_trees_val ltv; + struct scoutfs_key key; int ret; int err; @@ -573,16 +565,13 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) down_write(&server->alloc_rwsem); /* find the client's existing item */ - ltk.rid = cpu_to_be64(rid); - ltk.nr = 0; - ret = scoutfs_btree_next(sb, &super->logs_root, - <k, sizeof(ltk), &iref); + scoutfs_key_init_log_trees(&key, rid, 0); + ret = scoutfs_btree_next(sb, &super->logs_root, &key, &iref); if (ret == 0) { - if (iref.key_len == sizeof(struct scoutfs_log_trees_key) && - iref.val_len == sizeof(struct scoutfs_log_trees_val)) { - memcpy(<k, iref.key, iref.key_len); + if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + key = *iref.key; memcpy(<v, iref.val, iref.val_len); - if (be64_to_cpu(ltk.rid) != rid) + if (le64_to_cpu(key.sklt_rid) != rid) ret = -ENOENT; } else { ret = -EIO; @@ -618,8 +607,7 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) le64_to_cpu(ltv.data_freed.ref.sm_total)); err = scoutfs_btree_update(sb, &server->alloc, &server->wri, - &super->logs_root, <k, sizeof(ltk), - <v, sizeof(ltv)); + &super->logs_root, &key, <v, sizeof(ltv)); BUG_ON(err != 0); /* alloc and log item roots out of sync */ out: @@ -629,6 +617,15 @@ out: return ret; } +static void init_trans_seq_key(struct scoutfs_key *key, u64 seq, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_TRANS_SEQ_ZONE, + .skts_trans_seq = cpu_to_le64(seq), + .skts_rid = cpu_to_le64(rid), + }; +} + /* * Give the client the next sequence number for their transaction. They * provide their previous transaction sequence number that they've @@ -653,8 +650,8 @@ static int server_advance_seq(struct super_block *sb, struct scoutfs_super_block *super = &sbi->super; __le64 their_seq; __le64 next_seq; - struct scoutfs_trans_seq_btree_key tsk; u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_key key; int ret; if (arg_len != sizeof(__le64)) { @@ -670,12 +667,9 @@ static int server_advance_seq(struct super_block *sb, down_write(&server->seq_rwsem); if (their_seq != 0) { - tsk.trans_seq = le64_to_be64(their_seq); - tsk.rid = cpu_to_be64(rid); - + init_trans_seq_key(&key, le64_to_cpu(their_seq), rid); ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, - &super->trans_seqs, - &tsk, sizeof(tsk)); + &super->trans_seqs, &key); if (ret < 0 && ret != -ENOENT) goto out; } @@ -686,12 +680,9 @@ static int server_advance_seq(struct super_block *sb, trace_scoutfs_trans_seq_advance(sb, rid, le64_to_cpu(their_seq), le64_to_cpu(next_seq)); - tsk.trans_seq = le64_to_be64(next_seq); - tsk.rid = cpu_to_be64(rid); - + init_trans_seq_key(&key, le64_to_cpu(next_seq), rid); ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, - &super->trans_seqs, - &tsk, sizeof(tsk), NULL, 0); + &super->trans_seqs, &key, NULL, 0); out: up_write(&server->seq_rwsem); ret = scoutfs_server_apply_commit(sb, ret); @@ -712,39 +703,35 @@ static int remove_trans_seq(struct super_block *sb, u64 rid) DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_trans_seq_btree_key tsk; SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; int ret = 0; down_write(&server->seq_rwsem); - tsk.trans_seq = 0; - tsk.rid = 0; + init_trans_seq_key(&key, 0, 0); for (;;) { - ret = scoutfs_btree_next(sb, &super->trans_seqs, - &tsk, sizeof(tsk), &iref); + ret = scoutfs_btree_next(sb, &super->trans_seqs, &key, &iref); if (ret < 0) { if (ret == -ENOENT) ret = 0; break; } - memcpy(&tsk, iref.key, iref.key_len); + key = *iref.key; scoutfs_btree_put_iref(&iref); - if (be64_to_cpu(tsk.rid) == rid) { + if (le64_to_cpu(key.skts_rid) == rid) { trace_scoutfs_trans_seq_farewell(sb, rid, - be64_to_cpu(tsk.trans_seq)); + le64_to_cpu(key.skts_trans_seq)); ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, - &super->trans_seqs, - &tsk, sizeof(tsk)); + &super->trans_seqs, &key); break; } - be64_add_cpu(&tsk.trans_seq, 1); - tsk.rid = 0; + scoutfs_key_inc(&key); } up_write(&server->seq_rwsem); @@ -767,9 +754,9 @@ static int server_get_last_seq(struct super_block *sb, DECLARE_SERVER_INFO(sb, server); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_trans_seq_btree_key tsk; SCOUTFS_BTREE_ITEM_REF(iref); u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_key key; __le64 last_seq = 0; int ret; @@ -780,26 +767,19 @@ static int server_get_last_seq(struct super_block *sb, down_read(&server->seq_rwsem); - tsk.trans_seq = 0; - tsk.rid = 0; - - ret = scoutfs_btree_next(sb, &super->trans_seqs, - &tsk, sizeof(tsk), &iref); + init_trans_seq_key(&key, 0, 0); + ret = scoutfs_btree_next(sb, &super->trans_seqs, &key, &iref); if (ret == 0) { - if (iref.key_len != sizeof(tsk)) { - ret = -EINVAL; - } else { - memcpy(&tsk, iref.key, iref.key_len); - last_seq = cpu_to_le64(be64_to_cpu(tsk.trans_seq) - 1); - } + key = *iref.key; scoutfs_btree_put_iref(&iref); + last_seq = key.skts_trans_seq; } else if (ret == -ENOENT) { last_seq = super->next_trans_seq; - le64_add_cpu(&last_seq, -1ULL); ret = 0; } + le64_add_cpu(&last_seq, -1ULL); trace_scoutfs_trans_seq_last(sb, rid, le64_to_cpu(last_seq)); up_read(&server->seq_rwsem); @@ -926,22 +906,30 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, NULL, NULL); } +static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_MOUNTED_CLIENT_ZONE, + .skmc_rid = cpu_to_le64(rid), + }; +} + static int insert_mounted_client(struct super_block *sb, u64 rid, u64 gr_flags) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val mcv; + struct scoutfs_key key; - mck.rid = cpu_to_be64(rid); + init_mounted_client_key(&key, rid); mcv.flags = 0; if (gr_flags & SCOUTFS_NET_GREETING_FLAG_VOTER) mcv.flags |= SCOUTFS_MOUNTED_CLIENT_VOTER; return scoutfs_btree_insert(sb, &server->alloc, &server->wri, - &super->mounted_clients, - &mck, sizeof(mck), &mcv, sizeof(mcv)); + &super->mounted_clients, &key, &mcv, + sizeof(mcv)); } /* @@ -958,14 +946,13 @@ static int delete_mounted_client(struct super_block *sb, u64 rid) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_mounted_client_btree_key mck; + struct scoutfs_key key; int ret; - mck.rid = cpu_to_be64(rid); + init_mounted_client_key(&key, rid); ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, - &super->mounted_clients, - &mck, sizeof(mck)); + &super->mounted_clients, &key); if (ret == -ENOENT) ret = 0; @@ -1101,9 +1088,7 @@ struct farewell_request { static bool invalid_mounted_client_item(struct scoutfs_btree_item_ref *iref) { - return (iref->key_len != - sizeof(struct scoutfs_mounted_client_btree_key)) || - (iref->val_len != + return (iref->val_len != sizeof(struct scoutfs_mounted_client_btree_val)); } @@ -1139,13 +1124,13 @@ static void farewell_worker(struct work_struct *work) farewell_work); struct super_block *sb = server->sb; struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_mounted_client_btree_key mck; struct scoutfs_mounted_client_btree_val *mcv; struct farewell_request *tmp; struct farewell_request *fw; SCOUTFS_BTREE_ITEM_REF(iref); unsigned int nr_unmounting = 0; unsigned int nr_mounted = 0; + struct scoutfs_key key; LIST_HEAD(reqs); LIST_HEAD(send); bool deleted = false; @@ -1161,9 +1146,9 @@ static void farewell_worker(struct work_struct *work) /* count how many reqs requests are from voting clients */ nr_unmounting = 0; list_for_each_entry_safe(fw, tmp, &reqs, entry) { - mck.rid = cpu_to_be64(fw->rid); - ret = scoutfs_btree_lookup(sb, &super->mounted_clients, - &mck, sizeof(mck), &iref); + init_mounted_client_key(&key, fw->rid); + ret = scoutfs_btree_lookup(sb, &super->mounted_clients, &key, + &iref); if (ret == 0 && invalid_mounted_client_item(&iref)) { scoutfs_btree_put_iref(&iref); ret = -EIO; @@ -1189,10 +1174,10 @@ static void farewell_worker(struct work_struct *work) } /* see how many mounted clients could vote for quorum */ - memset(&mck, 0, sizeof(mck)); + init_mounted_client_key(&key, 0); for (;;) { - ret = scoutfs_btree_next(sb, &super->mounted_clients, - &mck, sizeof(mck), &iref); + ret = scoutfs_btree_next(sb, &super->mounted_clients, &key, + &iref); if (ret == 0 && invalid_mounted_client_item(&iref)) { scoutfs_btree_put_iref(&iref); ret = -EIO; @@ -1203,15 +1188,14 @@ static void farewell_worker(struct work_struct *work) goto out; } - memcpy(&mck, iref.key, sizeof(mck)); + key = *iref.key; mcv = iref.val; if (mcv->flags & SCOUTFS_MOUNTED_CLIENT_VOTER) nr_mounted++; scoutfs_btree_put_iref(&iref); - be64_add_cpu(&mck.rid, 1); - + scoutfs_key_inc(&key); } /* send as many responses as we can to maintain quorum */ From f59336085dd1de4554ab7727dac3af0408c65d00 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 23 Apr 2020 14:39:37 -0700 Subject: [PATCH 824/920] scoutfs: add avl Add the little avl implementation that we're going to use for indexing items within the btree blocks. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/avl.c | 403 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/avl.h | 30 ++++ kmod/src/format.h | 11 ++ 4 files changed, 445 insertions(+) create mode 100644 kmod/src/avl.c create mode 100644 kmod/src/avl.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index f0c62bb4..dd3d3622 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -9,6 +9,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -include $(src)/Makefile.kernelcompat scoutfs-y += \ + avl.o \ block.o \ btree.o \ client.o \ diff --git a/kmod/src/avl.c b/kmod/src/avl.c new file mode 100644 index 00000000..f626e2a8 --- /dev/null +++ b/kmod/src/avl.c @@ -0,0 +1,403 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include + +#include "format.h" +#include "avl.h" + +/* + * We use a simple avl to index items in btree blocks. The interface + * looks a bit like the kernel rbtree interface in that the caller + * manages locking and storage for the nodes. Node references are + * stored as byte offsets from the root so that the implementation + * doesn't have to know anything about the caller's container. + * + * We store the full height in each node, rather than just 2 bits for + * the balance, so that we can use the extra redundancy to verify the + * integrity of the tree. + */ + +static struct scoutfs_avl_node *node_ptr(struct scoutfs_avl_root *root, + __le16 off) +{ + return off ? (void *)root + le16_to_cpu(off) : NULL; +} + +static __le16 node_off(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + return node ? cpu_to_le16((void *)node - (void *)root) : 0; +} + +static __u8 node_height(struct scoutfs_avl_node *node) +{ + return node ? node->height : 0; +} + +struct scoutfs_avl_node * +scoutfs_avl_search(struct scoutfs_avl_root *root, + scoutfs_avl_compare_t compare, void *arg, int *cmp_ret, + struct scoutfs_avl_node **par, + struct scoutfs_avl_node **next, + struct scoutfs_avl_node **prev) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + int cmp; + + if (cmp_ret) + *cmp_ret = -1; + if (par) + *par = NULL; + if (next) + *next = NULL; + if (prev) + *prev = NULL; + + while (node) { + cmp = compare(arg, node); + if (par) + *par = node; + if (cmp_ret) + *cmp_ret = cmp; + if (cmp < 0) { + if (next) + *next = node; + node = node_ptr(root, node->left); + } else if (cmp > 0) { + if (prev) + *prev = node; + node = node_ptr(root, node->right); + } else { + return node; + } + } + + return NULL; +} + +struct scoutfs_avl_node *scoutfs_avl_first(struct scoutfs_avl_root *root) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + + while (node && node->left) + node = node_ptr(root, node->left); + + return node; +} + +struct scoutfs_avl_node *scoutfs_avl_last(struct scoutfs_avl_root *root) +{ + struct scoutfs_avl_node *node = node_ptr(root, root->node); + + while (node && node->right) + node = node_ptr(root, node->right); + + return node; +} + +struct scoutfs_avl_node *scoutfs_avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + + if (node->right) { + node = node_ptr(root, node->right); + while (node->left) + node = node_ptr(root, node->left); + return node; + } + + while ((parent = node_ptr(root, node->parent)) && + node == node_ptr(root, parent->right)) + node = parent; + + return parent; +} + +struct scoutfs_avl_node *scoutfs_avl_prev(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + + if (node->left) { + node = node_ptr(root, node->left); + while (node->right) + node = node_ptr(root, node->right); + return node; + } + + while ((parent = node_ptr(root, node->parent)) && + node == node_ptr(root, parent->left)) + node = parent; + + return parent; +} + +static void set_parent_left_right(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *old, + struct scoutfs_avl_node *new) +{ + __le16 *off; + + if (parent == NULL) + off = &root->node; + else if (parent->left == node_off(root, old)) + off = &parent->left; + else + off = &parent->right; + + *off = node_off(root, new); +} + +static void set_height(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *left = node_ptr(root, node->left); + struct scoutfs_avl_node *right = node_ptr(root, node->right); + + node->height = 1 + max(node_height(left), node_height(right)); +} + +static int node_balance(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + if (node == NULL) + return 0; + + return (int)node_height(node_ptr(root, node->right)) - + (int)node_height(node_ptr(root, node->left)); +} + +/* + * d b + * / \ rotate right -> / \ + * b e a d + * / \ <- rotate left / \ + * a c c e + * + * The rotate functions are always called with the higher node as the + * earlier argument. Links to a and e are constant. We have to update + * the forward and back refs between parents and nodes for the three links + * along root->[db]->[bd]->c. + */ +static void rotate_right(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *d) +{ + struct scoutfs_avl_node *gpa = node_ptr(root, d->parent); + struct scoutfs_avl_node *b = node_ptr(root, d->left); + struct scoutfs_avl_node *c = node_ptr(root, b->right); + + set_parent_left_right(root, gpa, d, b); + b->parent = node_off(root, gpa); + + b->right = node_off(root, d); + d->parent = node_off(root, b); + + d->left = node_off(root, c); + if (c) + c->parent = node_off(root, d); + + set_height(root, d); + set_height(root, b); +} + +static void rotate_left(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *b) +{ + struct scoutfs_avl_node *gpa = node_ptr(root, b->parent); + struct scoutfs_avl_node *d = node_ptr(root, b->right); + struct scoutfs_avl_node *c = node_ptr(root, d->left); + + set_parent_left_right(root, gpa, b, d); + d->parent = node_off(root, gpa); + + d->left = node_off(root, b); + b->parent = node_off(root, d); + + b->right = node_off(root, c); + if (c) + c->parent = node_off(root, b); + + set_height(root, b); + set_height(root, d); +} + +/* + * Check the balance factor for the given node and perform rotations if + * its two child subtrees are too far out of balance. Return either the + * node again or the root of the newly balanced subtree. + */ +static struct scoutfs_avl_node * +rotate_imbalance(struct scoutfs_avl_root *root, struct scoutfs_avl_node *node) +{ + int bal = node_balance(root, node); + struct scoutfs_avl_node *child; + + if (bal >= -1 && bal <= 1) + return node; + + if (bal > 0) { + /* turn right-left case into right-right */ + child = node_ptr(root, node->right); + if (node_balance(root, child) < 0) + rotate_right(root, child); + /* rotate left to address right-right */ + rotate_left(root, node); + + } else { + /* or do the mirror for the left- cases */ + child = node_ptr(root, node->left); + if (node_balance(root, child) > 0) + rotate_left(root, child); + rotate_right(root, node); + } + + return node_ptr(root, node->parent); +} + +void scoutfs_avl_insert(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *node, int cmp) +{ + node->parent = 0; + node->left = 0; + node->right = 0; + set_height(root, node); + + if (parent == NULL) { + root->node = node_off(root, node); + node->parent = 0; + return; + } + + if (cmp < 0) + parent->left = node_off(root, node); + else + parent->right = node_off(root, node); + node->parent = node_off(root, parent); + + while (parent) { + set_height(root, parent); + parent = rotate_imbalance(root, parent); + parent = node_ptr(root, parent->parent); + } +} + +static struct scoutfs_avl_node *avl_successor(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + node = node_ptr(root, node->right); + while (node->left) + node = node_ptr(root, node->left); + + return node; +} + +/* + * Find a node next successor and then swap the positions of the two + * nodes with each other in the tree. This is only tricky because the + * successor can be a direct child of the node and if we weren't careful + * we'd be modifying each of the nodes through the pointers between + * them. + */ +static void swap_with_successor(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *succ = avl_successor(root, node); + struct scoutfs_avl_node *succ_par = node_ptr(root, succ->parent); + struct scoutfs_avl_node *succ_right = node_ptr(root, succ->right); + struct scoutfs_avl_node *parent; + struct scoutfs_avl_node *left; + struct scoutfs_avl_node *right; + + /* Link old node's parent and left child with the successor */ + succ->parent = node->parent; + parent = node_ptr(root, succ->parent); + set_parent_left_right(root, parent, node, succ); + succ->left = node->left; + left = node_ptr(root, succ->left); + if (left) + left->parent = node_off(root, succ); + + /* + * Link the old node's right with successor and the old + * successor's parent with the node, they could have pointed to + * each other. + */ + if (succ_par == node) { + succ->right = node_off(root, node); + node->parent = node_off(root, succ); + } else { + succ->right = node->right; + right = node_ptr(root, succ->right); + if (right) + right->parent = node_off(root, succ); + set_parent_left_right(root, succ_par, succ, node); + node->parent = node_off(root, succ_par); + } + + /* Link the old successor's right with the node, it can't have left */ + node->right = node_off(root, succ_right); + if (succ_right) + succ_right->parent = node_off(root, node); + node->left = 0; + + swap(node->height, succ->height); +} + +void scoutfs_avl_delete(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node) +{ + struct scoutfs_avl_node *parent; + struct scoutfs_avl_node *child; + + if (node->left && node->right) + swap_with_successor(root, node); + + parent = node_ptr(root, node->parent); + child = node_ptr(root, node->left ?: node->right); + + set_parent_left_right(root, parent, node, child); + if (child) + child->parent = node->parent; + + while (parent) { + set_height(root, parent); + parent = rotate_imbalance(root, parent); + parent = node_ptr(root, parent->parent); + } +} + +/* + * Move the contents of a node to a new node location in memory. The + * logical position of the node in the tree does not change. + */ +void scoutfs_avl_relocate(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *to, + struct scoutfs_avl_node *from) +{ + struct scoutfs_avl_node *parent = node_ptr(root, from->parent); + struct scoutfs_avl_node *left = node_ptr(root, from->left); + struct scoutfs_avl_node *right = node_ptr(root, from->right); + + set_parent_left_right(root, parent, from, to); + to->parent = from->parent; + to->left = from->left; + if (left) + left->parent = node_off(root, to); + to->right = from->right; + if (right) + right->parent = node_off(root, to); + to->height = from->height; +} diff --git a/kmod/src/avl.h b/kmod/src/avl.h new file mode 100644 index 00000000..f50ca423 --- /dev/null +++ b/kmod/src/avl.h @@ -0,0 +1,30 @@ +#ifndef _SCOUTFS_AVL_H_ +#define _SCOUTFS_AVL_H_ + +#include "format.h" + +typedef int (*scoutfs_avl_compare_t)(void *arg, + struct scoutfs_avl_node *node); + +struct scoutfs_avl_node * +scoutfs_avl_search(struct scoutfs_avl_root *root, + scoutfs_avl_compare_t compare, void *arg, int *cmp_ret, + struct scoutfs_avl_node **par, + struct scoutfs_avl_node **next, + struct scoutfs_avl_node **prev); +struct scoutfs_avl_node *scoutfs_avl_first(struct scoutfs_avl_root *root); +struct scoutfs_avl_node *scoutfs_avl_last(struct scoutfs_avl_root *root); +struct scoutfs_avl_node *scoutfs_avl_next(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +struct scoutfs_avl_node *scoutfs_avl_prev(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +void scoutfs_avl_insert(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *parent, + struct scoutfs_avl_node *node, int cmp); +void scoutfs_avl_delete(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *node); +void scoutfs_avl_relocate(struct scoutfs_avl_root *root, + struct scoutfs_avl_node *to, + struct scoutfs_avl_node *from); + +#endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 740a64fb..413774da 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -184,6 +184,17 @@ struct scoutfs_radix_root { ~(__u64)SCOUTFS_RADIX_LG_MASK) #define SCOUTFS_RADIX_BITS_BYTES (SCOUTFS_RADIX_BITS / 8) +struct scoutfs_avl_root { + __le16 node; +} __packed; + +struct scoutfs_avl_node { + __le16 parent; + __le16 left; + __le16 right; + __u8 height; +} __packed; + /* when we split we want to have multiple items on each side */ #define SCOUTFS_BTREE_MAX_VAL_LEN (SCOUTFS_BLOCK_SIZE / 8) From efd97633553bc1af9c356397b067bb24f743dd86 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Apr 2020 14:50:11 -0700 Subject: [PATCH 825/920] scoutfs: use efficient btree block structures This btree implementation was first built for the relatively light duty of indexing segments in the LSM item implementation. We're now using it as the core metadata index. It's already using a lot of cpu to do its job with small blocks and it only gets more expensive as the block size increases. These changes reduce the CPU use of working with the btree block structures. We use a balanced binary tree to index items by key in the block. This gives us rare tree balancing cost on insertion and deletion instead of the memmove overhead of maintaining a dense array of item offsets sorted by key. The keys are stored in the item struct which are stored in an array at the front of the block so searching for an item uses contiguous cachelines. We add a trailing owner offset to values so that we can iterate through them. This is used to track space freed up by values instead of paying the memmove cost of keeping all the values at the end of the block. We occasionally reclaim the fragmented value free space instead of splitting the block. Direct item lookups use a small hash table at the end of the block which maps offsets to items. It uses linear probing and is guaranteed to have a light load factor so lookups are very likely to only need a single cache lookup. We adjust the watermark for triggering a join from half of a block down to a quarter. This results in less utilized blocks on average. But it creates distance between the join and split thresholds so we get less cpu use from constantly joining and splitting if item populations happen to hover around the previously shared threshold. While shifting the implementation we choose not to add support for some features that no longer make sense. There are no longer callers of _before and _after, and having synthetic tests to use small btree blocks no longer makes ense when we can easily create very tall trees. Both those btree interfaces and the tiny btree block support will be removed. Signed-off-by: Zach Brown --- kmod/src/btree.c | 1076 ++++++++++++++++++++++++++++----------------- kmod/src/format.h | 42 +- 2 files changed, 693 insertions(+), 425 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 0aa2edd6..a62535f5 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -28,48 +28,51 @@ #include "msg.h" #include "block.h" #include "radix.h" +#include "avl.h" #include "scoutfs_trace.h" /* * scoutfs uses a cow btree to index fs metadata. * - * Using a cow btree lets nodes determine the validity of cached blocks - * based on a single root ref (blkno, seq) that is communicated through - * locking and messaging. As long as their cached blocks aren't - * overwritten in the ring they can continue to use those cached blocks - * as the newer cowed blocks continue to reference them. - * * Today callers provide all the locking. They serialize readers and * writers and writers and committing all the dirty blocks. * - * Btree items are stored in each block as a small header with the key - * followed by the value. New items are allocated from the back of the - * block towards the front. + * Block reference have sufficient metadata to discover corrupt + * references. If a reader encounters a bad block it backs off which + * gives the caller the opportunity to resample the root in case it was + * reading through a stale btree that has been overwritten. This lets + * mounts read trees that are modified by other mounts without exclusive + * locking. * - * A dense array of item headers after the btree block header stores the - * offsets of the items and is kept sorted by the item's keys. The - * array is small enough that keeping it sorted with memmove() involves - * a few cache lines at most. + * Btree items are stored as a dense array of structs at the front of + * each block. New items are allocated at the end of the array. + * Deleted items are swapped with the last item to maintain the dense + * array. The items are indexed by a balanced binary tree with parent + * pointers so the relocated item can have references to it updated. * - * Parent blocks in the btree have the same format as leaf blocks. - * There's one key for every child reference instead of having separator - * keys between child references. The key in a child reference contains - * the largest key that may be found in the child subtree. The right - * spine of the tree has maximal keys so that they don't have to be - * updated if we insert an item with a key greater than everything in - * the tree. - */ - -/* - * XXX: - * - counters and tracing - * - could issue read-ahead around reads up to dirty blkno - * - have barrier as we cross to prevent refreshing clobbering stale reads - * - audit/comment that dirty blknos can wrap around ring - * - figure out some max transaction size so ring won't wrap in one - * - update the world of comments - * - validate structures on read? + * Values are allocated from the end of the block towards the front, + * consuming the end of free space in the center of the block. Deleted + * values can be merged with this free space, but more likely they'll + * create fragmented free space amongst other existing values. All + * values are stored with an offset at the end which contains either the + * offset of their item or the offset of the start of their free space. + * This lets an infrequent compaction process move items towards the + * back of the block to reclaim free space. + * + * Exact item searches are only performed on leaf blocks. Leaf blocks + * have a hash table at the end of the block which is used to find items + * with a specific key. It uses linear probing and maintains a low load + * factor so any given search will most likely only need a single + * cacheline. + * + * Parent block reference items are stored as items with a block + * reference as a value. There's an item with a key for every child + * reference instead of having separator keys between child references. + * The key in a child reference contains the largest key that may be + * found in the child subtree. The right spine of the tree has maximal + * keys so that they don't have to be updated if we insert an item with + * a key greater than everything in the tree. */ /* btree walking has a bunch of behavioural bit flags */ @@ -81,88 +84,75 @@ enum { BTW_DIRTY = (1 << 4), /* cow stable blocks */ BTW_ALLOC = (1 << 5), /* allocate a new block for 0 ref */ BTW_INSERT = (1 << 6), /* walking to insert, try splitting */ - BTW_DELETE = (1 << 7), /* walking to delete, try merging */ + BTW_DELETE = (1 << 7), /* walking to delete, try joining */ }; -/* number of contiguous bytes used by the item and it's value */ -static inline unsigned int len_bytes(unsigned val_len) +/* total length of the value payload */ +static inline unsigned int val_bytes(unsigned val_len) { - return sizeof(struct scoutfs_btree_item) + val_len; + return val_len + (val_len ? SCOUTFS_BTREE_VAL_OWNER_BYTES : 0); } -/* number of contiguous bytes used an existing item */ +/* number of bytes in a block used by an item with the given value length */ +static inline unsigned int item_len_bytes(unsigned val_len) +{ + return sizeof(struct scoutfs_btree_item) + val_bytes(val_len); +} + +/* number of bytes used by an existing item */ static inline unsigned int item_bytes(struct scoutfs_btree_item *item) { - return len_bytes(le16_to_cpu(item->val_len)); -} - -/* total block bytes used by an item: header, item, key, value */ -static inline unsigned int all_len_bytes(unsigned val_len) -{ - return sizeof(struct scoutfs_btree_item_header) + len_bytes(val_len); + return item_len_bytes(le16_to_cpu(item->val_len)); } /* - * The minimum number of bytes we allow in a block. During descent to - * modify if we see a block with fewer used bytes then we'll try to - * merge items from neighbours. If the neighbour also has less than the - * min bytes then the two blocks are merged. - * - * This is carefully calculated so that if two blocks are merged the - * resulting block will have at least parent min free bytes free so - * that it's not immediately split again. - * - * new_used = min_used + min_used - hdr - * new_used <= (bs - parent_min_free) - * - * min_used + min_used - hdr <= (bs - parent_min_free) - * 2 * min_used <= (bs - parent_min_free - hdr) - * min_used <= (bs - parent_min_free - hdr) / 2 + * Join blocks when they both are 1/4 full. This puts some distance + * between the join threshold and the full threshold for splitting. + * Blocks that just split or joined need to undergo a reasonable amount + * of item modification before they'll split or join again. */ -static inline int min_used_bytes(int block_size) +static unsigned int join_low_watermark(void) { - return (block_size - sizeof(struct scoutfs_btree_block) - - SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES) / 2; + return (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 4; } -/* total block bytes used by an existing item */ -static inline unsigned int all_item_bytes(struct scoutfs_btree_item *item) +/* + * return the integer percentages of total space the block could have + * consumed by items that is currently consumed. + */ +static unsigned int item_full_pct(struct scoutfs_btree_block *bt) { - return all_len_bytes(le16_to_cpu(item->val_len)); + return (int)le16_to_cpu(bt->total_item_bytes) * 100 / + (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)); } -/* number of free bytes between last item header and first item */ -static inline unsigned int free_bytes(struct scoutfs_btree_block *bt) +static inline __le16 ptr_off(struct scoutfs_btree_block *bt, void *ptr) { - unsigned int nr = le32_to_cpu(bt->nr_items); - - return le32_to_cpu(bt->free_end) - - offsetof(struct scoutfs_btree_block, item_hdrs[nr]); + return cpu_to_le16(ptr - (void *)bt); } -/* all bytes used by item offsets, headers, and values */ -static inline unsigned int used_total(struct scoutfs_btree_block *bt) +static inline void *off_ptr(struct scoutfs_btree_block *bt, u16 off) { - return SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block) - - free_bytes(bt); + return (void *)bt + off; } static inline struct scoutfs_btree_item * -off_item(struct scoutfs_btree_block *bt, __le32 off) +off_item(struct scoutfs_btree_block *bt, __le16 off) { - return (void *)bt + le32_to_cpu(off); + return (void *)bt + le16_to_cpu(off); } -static inline struct scoutfs_btree_item * -pos_item(struct scoutfs_btree_block *bt, unsigned int pos) +static struct scoutfs_btree_item *last_item(struct scoutfs_btree_block *bt) { - return off_item(bt, bt->item_hdrs[pos].off); + BUG_ON(bt->nr_items == 0); + + return &bt->items[le16_to_cpu(bt->nr_items) - 1]; } -static inline struct scoutfs_btree_item * -last_item(struct scoutfs_btree_block *bt) +/* offset of the start of the free range in the middle of the block */ +static inline unsigned int mid_free_off(struct scoutfs_btree_block *bt) { - return pos_item(bt, le32_to_cpu(bt->nr_items) - 1); + return le16_to_cpu(ptr_off(bt, &bt->items[le16_to_cpu(bt->nr_items)])); } static inline struct scoutfs_key *item_key(struct scoutfs_btree_item *item) @@ -170,9 +160,10 @@ static inline struct scoutfs_key *item_key(struct scoutfs_btree_item *item) return &item->key; } -static inline void *item_val(struct scoutfs_btree_item *item) +static inline void *item_val(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) { - return item->val; + return off_ptr(bt, le16_to_cpu(item->val_off)); } static inline unsigned item_val_len(struct scoutfs_btree_item *item) @@ -180,173 +171,482 @@ static inline unsigned item_val_len(struct scoutfs_btree_item *item) return le16_to_cpu(item->val_len); } -/* - * Returns the sorted item position that an item with the given key - * should occupy. - * - * It sets *cmp to the final comparison of the given key and the - * position's item key. This can only be -1 or 0 because we bias - * towards returning the pos that a key should occupy. - * - * If the given key is greater then all items' keys then the number of - * items can be returned. - */ -static int find_pos(struct scoutfs_btree_block *bt, struct scoutfs_key *key, - int *cmp) +static struct scoutfs_btree_item *node_item(struct scoutfs_avl_node *node) { + if (node == NULL) + return NULL; + return container_of(node, struct scoutfs_btree_item, node); +} + +static struct scoutfs_btree_item *prev_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + if (item == NULL) + return NULL; + return node_item(scoutfs_avl_prev(&bt->item_root, &item->node)); +} + +static struct scoutfs_btree_item *next_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item) +{ + if (item == NULL) + return NULL; + return node_item(scoutfs_avl_next(&bt->item_root, &item->node)); +} + +static int cmp_key_item(void *arg, struct scoutfs_avl_node *node) +{ + struct scoutfs_key *key = arg; + struct scoutfs_btree_item *item = node_item(node); + + return scoutfs_key_compare(key, item_key(item)); +} + +/* + * We have a small fixed-size linearly probed hash table at the end of + * leaf blocks which is used for direct item lookups (as opposed to + * iterators). The hash table only stores non-zero offsets to the + * items. If an item is moved then its offset is updated. The hash + * table is sized to allow a max load of 75%, but most items are larger + * and most blocks aren't full. + */ +static int leaf_item_hash_ind(struct scoutfs_key *key) +{ + return crc32c(~0, key, sizeof(struct scoutfs_key)) % + SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; +} + +static __le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt) +{ + return (void *)bt + SCOUTFS_BLOCK_SIZE - + SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; +} + +static inline int leaf_item_hash_next_bucket(int i) +{ + if (++i >= SCOUTFS_BTREE_LEAF_ITEM_HASH_NR) + i = 0; + return i; +} + +#define foreach_leaf_item_hash_bucket(i, nr, key) \ + for (i = leaf_item_hash_ind(key), nr = SCOUTFS_BTREE_LEAF_ITEM_HASH_NR;\ + nr-- > 0; \ + i = leaf_item_hash_next_bucket(i)) + +static struct scoutfs_btree_item * +leaf_item_hash_search(struct scoutfs_btree_block *bt, struct scoutfs_key *key) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); struct scoutfs_btree_item *item; - unsigned int start = 0; - unsigned int end = le32_to_cpu(bt->nr_items); - unsigned int pos = 0; + __le16 off; + int nr; + int i; - *cmp = -1; + if (WARN_ON_ONCE(bt->level > 0)) + return NULL; - while (start < end) { - pos = start + (end - start) / 2; + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + if (off == 0) + return NULL; - item = pos_item(bt, pos); - *cmp = scoutfs_key_compare(key, item_key(item)); - if (*cmp < 0) { - end = pos; - } else if (*cmp > 0) { - start = ++pos; - *cmp = -1; - } else { + item = off_item(bt, off); + if (scoutfs_key_compare(key, item_key(item)) == 0) + return item; + } + + return NULL; +} + +static void leaf_item_hash_insert(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 off) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + if (buckets[i] == 0) { + buckets[i] = off; + return; + } + } + + /* table should have been been enough for all items */ + BUG(); +} + +/* + * Deletion clears the offset in a bucket. That could create a + * discontinuity that would stop a search from seeing colliding + * insertions that were pushed into further buckets. Each time we zero + * a bucket we rehash all the populated buckets following it. There + * won't be many in our light load tables and this works reliably as the + * contiguous population wraps past the end of table. Comparing hashed + * bucket positions to find candidates to relocate after the wrap is + * tricky. + */ +static void leaf_item_hash_delete(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 del_off) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + __le16 off; + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + /* we must find the item we're trying to delete */ + BUG_ON(off == 0); + + if (off == del_off) { + buckets[i] = 0; break; } } - return pos; + while ((i = leaf_item_hash_next_bucket(i)), buckets[i] != 0) { + off = buckets[i]; + buckets[i] = 0; + leaf_item_hash_insert(bt, item_key(off_item(bt, off)), off); + } } -/* move a number of contigous elements from the src index to the dst index */ -#define memmove_arr(arr, dst, src, nr) \ - memmove(&(arr)[dst], &(arr)[src], (nr) * sizeof(*(arr))) +static void leaf_item_hash_change(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, __le16 to, + __le16 from) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + __le16 off; + int nr; + int i; + + if (bt->level > 0) + return; + + foreach_leaf_item_hash_bucket(i, nr, key) { + off = buckets[i]; + /* we must find the item we're trying to change */ + BUG_ON(off == 0); + + if (off == from) { + buckets[i] = to; + return; + } + } +} + +/* + * Given an offset to the start of a value, return info describing the + * previous value in the block. Each value ends with an owner offset + * which points to either the value's item if it's in use or to the + * start of the value if it's been freed. Either the item is returned + * or the length of the previous value is set. + */ +static struct scoutfs_btree_item * +get_prev_val_owner(struct scoutfs_btree_block *bt, unsigned int off, + unsigned int *prev_val_bytes) +{ + __le16 *owner = off_ptr(bt, off - sizeof(*owner)); + unsigned int own = get_unaligned_le16(owner); + + if (own >= mid_free_off(bt)) { + *prev_val_bytes = off - own; + return NULL; + } else { + *prev_val_bytes = 0; + return off_ptr(bt, own); + } +} + +/* + * Set the owner offset at the end of a full value, the given length includes + * the offset. + */ +static void set_val_owner(struct scoutfs_btree_block *bt, unsigned int val_off, + unsigned int vb, __le16 item_off) +{ + __le16 *owner = off_ptr(bt, val_off + vb - sizeof(*owner)); + + put_unaligned_le16(le16_to_cpu(item_off) ?: val_off, owner); +} + +/* + * As values are freed they can leave fragmented free space amongst + * other values. This is called when we can't insert because there + * isn't enough free space but we know that there's sufficient free + * space amongst the values for the new insertion. + * + * But we only want to do this when there is enough free space to + * justify the cost of the compaction. We don't want to bother + * compacting if the block is almost full and we just be split in a few + * more operations. The split heuristic requires a generous amount of + * fragmented free space that will avoid a split. + */ +static void compact_values(struct scoutfs_btree_block *bt) +{ + struct scoutfs_btree_item *item; + unsigned int free_off; + unsigned int free_len; + unsigned int to_off; + unsigned int end; + unsigned int vb; + void *from; + void *to; + + if (bt->last_free_off == 0) + return; + + free_off = le16_to_cpu(bt->last_free_off); + free_len = le16_to_cpu(bt->last_free_len); + end = mid_free_off(bt) + le16_to_cpu(bt->mid_free_len); + + while (free_off > end) { + item = get_prev_val_owner(bt, free_off, &vb); + if (item == NULL) { + free_off -= vb; + free_len += vb; + continue; + } + + from = off_ptr(bt, le16_to_cpu(item->val_off)); + vb = val_bytes(le16_to_cpu(item->val_len)); + to_off = free_off + free_len - vb; + to = off_ptr(bt, to_off); + if (to >= from + vb) + memcpy(to, from, vb); + else + memmove(to, from, vb); + + free_off = le16_to_cpu(item->val_off); + item->val_off = cpu_to_le16(to_off); + } + + le16_add_cpu(&bt->mid_free_len, free_len); + bt->last_free_off = 0; + bt->last_free_len = 0; +} + +/* + * Insert an item's value into the block. The caller has made sure + * there's free space. We store the value at the end of free space in + * the block and point its final offset at its owning item, and copy the + * value into place. + */ +static __le16 insert_value(struct scoutfs_btree_block *bt, __le16 item_off, + void *val, unsigned val_len) +{ + unsigned int val_off; + unsigned int vb; + + if (val_len == 0) + return 0; + + BUG_ON(le16_to_cpu(bt->mid_free_len) < val_bytes(val_len)); + + vb = val_bytes(val_len); + val_off = mid_free_off(bt) + le16_to_cpu(bt->mid_free_len) - vb; + le16_add_cpu(&bt->mid_free_len, -vb); + + memcpy(off_ptr(bt, val_off), val, val_len); + set_val_owner(bt, val_off, vb, item_off); + + return cpu_to_le16(val_off); +} + +/* + * Delete an item's value from the block. The caller has updated the + * item. We leave behind a free region whose owner offset indicates + * that the value isn't in use. It might merge with the central free + * region or the final freed value, and might become the final freed + * value. + */ +static void delete_value(struct scoutfs_btree_block *bt, + unsigned int val_off, unsigned int val_len) +{ + unsigned int free_off; + unsigned int free_len; + bool is_last; + + if (val_len == 0) + return; + + free_off = val_off; + free_len = val_bytes(val_len); + is_last = false; + + /* see if we can merge with mid free region */ + if (mid_free_off(bt) + le16_to_cpu(bt->mid_free_len) == free_off) { + le16_add_cpu(&bt->mid_free_len, free_len); + return; + } + + if (free_off + free_len == le16_to_cpu(bt->last_free_off)) { + /* merge with front of last free */ + free_len += le16_to_cpu(bt->last_free_len); + is_last = true; + + } else if ((le16_to_cpu(bt->last_free_off) + + le16_to_cpu(bt->last_free_len)) == free_off) { + /* merge with end of last free */ + free_off = le16_to_cpu(bt->last_free_off); + free_len += le16_to_cpu(bt->last_free_len); + is_last = true; + + } else if (free_off > le16_to_cpu(bt->last_free_off)) { + /* become new last */ + is_last = true; + } + + set_val_owner(bt, free_off, free_len, 0); + if (is_last) { + bt->last_free_off = cpu_to_le16(free_off); + bt->last_free_len = cpu_to_le16(free_len); + } +} /* * Insert a new item into the block. The caller has made sure that - * there's space for the item and its metadata. + * there is sufficient free space in block for the new item. We might + * have to compact the values to the end of the block to reclaim + * fragmented free space between values. + * + * This only consumes free space. It's safe to use references to block + * structures after this call. */ -static void create_item(struct scoutfs_btree_block *bt, unsigned int pos, - struct scoutfs_key *key, void *val, unsigned val_len) +static void create_item(struct scoutfs_btree_block *bt, + struct scoutfs_key *key, void *val, unsigned val_len, + struct scoutfs_avl_node *parent, int cmp) { - unsigned int nr = le32_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; - unsigned all_bytes; - all_bytes = all_len_bytes(val_len); - BUG_ON(free_bytes(bt) < all_bytes); + BUG_ON(le16_to_cpu(bt->mid_free_len) < item_len_bytes(val_len)); - if (pos < nr) - memmove_arr(bt->item_hdrs, pos + 1, pos, nr - pos); + le16_add_cpu(&bt->mid_free_len, + -(u16)sizeof(struct scoutfs_btree_item)); + le16_add_cpu(&bt->nr_items, 1); + item = last_item(bt); - le32_add_cpu(&bt->free_end, -len_bytes(val_len)); - bt->item_hdrs[pos].off = bt->free_end; - nr++; - bt->nr_items = cpu_to_le32(nr); + item->key = *key; - BUG_ON(le32_to_cpu(bt->free_end) < - offsetof(struct scoutfs_btree_block, item_hdrs[nr])); + scoutfs_avl_insert(&bt->item_root, parent, &item->node, cmp); + leaf_item_hash_insert(bt, item_key(item), ptr_off(bt, item)); - item = pos_item(bt, pos); - *item_key(item) = *key; + item->val_off = insert_value(bt, ptr_off(bt, item), val, val_len); item->val_len = cpu_to_le16(val_len); - if (val_len) - memcpy(item_val(item), val, val_len); + le16_add_cpu(&bt->total_item_bytes, item_bytes(item)); } /* * Delete an item from a btree block. * - * This moves all the headers after the item (in sort order) towards the - * start of the header array. It moves all the items before the removed - * item towards the end of the block. The items that have to be moved - * can be anywhere in the sort order. We first move the item region - * and then walk the headers looking for offsets that need to be updated. - * - * The item motion means that callers can not hold item references - * across item deletion. + * As we delete the item we can relocate an unrelated item to maintain + * the dense array of items. The caller can use another single item + * after this call if they give us the opportunity to let them know if + * we move it. */ -static void delete_item(struct scoutfs_btree_block *bt, unsigned int pos) +static void delete_item(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item, + struct scoutfs_btree_item **use_after) { - unsigned int nr = le32_to_cpu(bt->nr_items); - unsigned int updated; - unsigned int total; - unsigned int first; - unsigned int bytes; - unsigned int last; - unsigned int off; - int i; + struct scoutfs_btree_item *last; + unsigned int val_off; + unsigned int val_len; - /* calculate region of items to move */ - first = le32_to_cpu(bt->free_end); - last = le32_to_cpu(bt->item_hdrs[pos].off); - total = last - first; - bytes = item_bytes(pos_item(bt, pos)); + /* save some values before we delete the item */ + val_off = le16_to_cpu(item->val_off); + val_len = le16_to_cpu(item->val_len); + last = last_item(bt); - /* move items before deleted to the back of the block */ - if (total > 0) { - /* update headers before memove overwrites deleted item */ - for (i = 0, updated = 0; i < nr && updated < total; i++) { - off = le32_to_cpu(bt->item_hdrs[i].off); - if (off >= first && off < last) { - updated += item_bytes(pos_item(bt, i)); - le32_add_cpu(&bt->item_hdrs[i].off, bytes); - } - } - BUG_ON(updated != total); - - memmove(off_item(bt, cpu_to_le32(first + bytes)), - off_item(bt, cpu_to_le32(first)), total); + /* delete the item */ + scoutfs_avl_delete(&bt->item_root, &item->node); + leaf_item_hash_delete(bt, item_key(item), ptr_off(bt, item)); + le16_add_cpu(&bt->nr_items, -1); + le16_add_cpu(&bt->mid_free_len, sizeof(struct scoutfs_btree_item)); + le16_add_cpu(&bt->total_item_bytes, -item_bytes(item)); + /* move the final item into the deleted space */ + if (last != item) { + item->key = last->key; + item->val_off = last->val_off; + item->val_len = last->val_len; + if (last->val_len) + set_val_owner(bt, le16_to_cpu(last->val_off), + val_bytes(le16_to_cpu(last->val_len)), + ptr_off(bt, item)); + leaf_item_hash_change(bt, &last->key, ptr_off(bt, item), + ptr_off(bt, last)); + scoutfs_avl_relocate(&bt->item_root, &item->node,&last->node); + if (use_after && *use_after == last) + *use_after = item; } - /* wipe deleted bytes to avoid leaking data */ - memset(off_item(bt, cpu_to_le32(first)), 0, bytes); - - if (pos < (nr - 1)) - memmove_arr(bt->item_hdrs, pos, pos + 1, nr - 1 - pos); - - le32_add_cpu(&bt->free_end, bytes); - le32_add_cpu(&bt->nr_items, -1); + delete_value(bt, val_off, val_len); } /* * Move items from a source block to a destination block. The caller - * tells us if we're moving from the tail of the source block right to - * the head of the destination block, or vice versa. We stop moving - * once we've moved enough bytes of items. + * has made sure there's sufficient free space in the destination block, + * though item creation may need to compact values. The caller tells us + * if we're moving from the tail of the source block right to the head + * of the destination block, or vice versa. We're always adding the + * first or last item to the avl, so the parent is always the previous + * first or last node. */ static void move_items(struct scoutfs_btree_block *dst, struct scoutfs_btree_block *src, bool move_right, int to_move) { + struct scoutfs_avl_node *par; + struct scoutfs_avl_node *node; struct scoutfs_btree_item *from; - unsigned int t; - unsigned int f; + struct scoutfs_btree_item *next; + int cmp; if (move_right) { - f = le32_to_cpu(src->nr_items) - 1; - t = 0; + node = scoutfs_avl_last(&src->item_root); + par = scoutfs_avl_first(&dst->item_root); + cmp = -1; } else { - f = 0; - t = le32_to_cpu(dst->nr_items); + node = scoutfs_avl_first(&src->item_root); + par = scoutfs_avl_last(&dst->item_root); + cmp = 1; } + from = node_item(node); - while (f < le32_to_cpu(src->nr_items) && to_move > 0) { - from = pos_item(src, f); + while (to_move > 0 && from != NULL) { + to_move -= item_bytes(from); - create_item(dst, t, item_key(from), item_val(from), - item_val_len(from)); - - to_move -= all_item_bytes(from); - - delete_item(src, f); if (move_right) - f--; + next = prev_item(src, from); else - t++; + next = next_item(src, from); + + create_item(dst, item_key(from), item_val(src, from), + item_val_len(from), par, cmp); + + if (move_right) { + if (par) + par = scoutfs_avl_prev(&dst->item_root, par); + else + par = scoutfs_avl_first(&dst->item_root); + } else { + if (par) + par = scoutfs_avl_next(&dst->item_root, par); + else + par = scoutfs_avl_last(&dst->item_root); + } + + delete_item(src, from, &next); + from = next; } } @@ -468,7 +768,6 @@ retry: /* returning a newly allocated block */ memset(new, 0, SCOUTFS_BLOCK_SIZE); new->hdr.fsid = super->hdr.fsid; - new->free_end = cpu_to_le32(SCOUTFS_BLOCK_SIZE); } bl = new_bl; bt = new; @@ -497,35 +796,52 @@ out: * specifies the key in the item that describes the items in the child. */ static void create_parent_item(struct scoutfs_btree_block *parent, - unsigned pos, struct scoutfs_btree_block *child, + struct scoutfs_btree_block *child, struct scoutfs_key *key) { + struct scoutfs_avl_node *par; + int cmp; struct scoutfs_btree_ref ref = { .blkno = child->hdr.blkno, .seq = child->hdr.seq, }; - create_item(parent, pos, key, &ref, sizeof(ref)); + scoutfs_avl_search(&parent->item_root, cmp_key_item, key, &cmp, &par, + NULL, NULL); + create_item(parent, key, &ref, sizeof(ref), par, cmp); } /* - * Update the parent item that refers to a child by deleting and - * recreating it. Descent should have ensured that there was always - * room for a maximal key in parents. + * Update an existing parent item reference to a child who may be new or + * may have had its last item changed. */ static void update_parent_item(struct scoutfs_btree_block *parent, - unsigned pos, struct scoutfs_btree_block *child) + struct scoutfs_btree_item *par_item, + struct scoutfs_btree_block *child) { - struct scoutfs_btree_item *item = last_item(child); + struct scoutfs_btree_ref *ref = item_val(parent, par_item); - delete_item(parent, pos); - create_parent_item(parent, pos, child, item_key(item)); + par_item->key = *item_key(last_item(child)); + ref->blkno = child->hdr.blkno; + ref->seq = child->hdr.seq; +} + +static void init_btree_block(struct scoutfs_btree_block *bt, int level) +{ + int free; + + free = SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block); + if (level == 0) + free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + + bt->level = level; + bt->mid_free_len = cpu_to_le16(free); } /* * See if we need to split this block while descending for insertion so * that we have enough space to insert. Parent blocks need enough space - * for a new item and child ref if a child block splits. Leaf blocks + * to insert a new parent item if a child block splits. Leaf blocks * need enough space to insert the new item with its value. * * We split to the left so that the greatest key in the existing block @@ -538,35 +854,36 @@ static int try_split(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, unsigned val_len, - struct scoutfs_btree_block *parent, unsigned pos, + struct scoutfs_btree_block *parent, struct scoutfs_btree_block *right) { struct scoutfs_block *left_bl = NULL; struct scoutfs_block *par_bl = NULL; struct scoutfs_btree_block *left; - struct scoutfs_btree_item *item; struct scoutfs_key max_key; - unsigned int all_bytes; int ret; int err; - if (scoutfs_option_bool(sb, Opt_btree_force_tiny_blocks)) - all_bytes = SCOUTFS_BLOCK_SIZE - SCOUTFS_BTREE_TINY_BLOCK_SIZE; - else if (right->level) - all_bytes = SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES; - else - all_bytes = all_len_bytes(val_len); + /* parents need to leave room for child references */ + if (right->level) + val_len = sizeof(struct scoutfs_btree_ref); - if (free_bytes(right) >= all_bytes) + /* don't need to split if there's enough space for the item */ + if (le16_to_cpu(right->mid_free_len) >= item_len_bytes(val_len)) return 0; + if (item_full_pct(right) < 80) { + compact_values(right); + return 0; + } + /* alloc split neighbour first to avoid unwinding tree growth */ ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &left_bl); if (ret) return ret; left = left_bl->data; - left->level = right->level; + init_btree_block(left, right->level); if (!parent) { ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl); @@ -579,21 +896,19 @@ static int try_split(struct super_block *sb, } parent = par_bl->data; - parent->level = root->height; + init_btree_block(parent, root->height); root->height++; root->ref.blkno = parent->hdr.blkno; root->ref.seq = parent->hdr.seq; scoutfs_key_set_ones(&max_key); - - pos = 0; - create_parent_item(parent, pos, right, &max_key); + create_parent_item(parent, right, &max_key); } - move_items(left, right, false, used_total(right) / 2); + move_items(left, right, false, + le16_to_cpu(right->total_item_bytes) / 2); - item = last_item(left); - create_parent_item(parent, pos, left, item_key(item)); + create_parent_item(parent, left, item_key(last_item(left))); scoutfs_block_put(sb, left_bl); scoutfs_block_put(sb, par_bl); @@ -603,78 +918,73 @@ static int try_split(struct super_block *sb, /* * This is called during descent for deletion when we have a parent and - * might need to merge items from a sibling block if this block has too - * much free space. Eventually we'll be able to fit all of the + * might need to join this block with a sibling block if this block has + * too much free space. Eventually we'll be able to fit all of the * sibling's items in our free space which lets us delete the sibling * block. - * - * XXX this could more cleverly chose a merge candidate sibling */ -static int try_merge(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, - struct scoutfs_btree_block *parent, unsigned pos, - struct scoutfs_btree_block *bt) +static int try_join(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_block *parent, + struct scoutfs_btree_item *par_item, + struct scoutfs_btree_block *bt) { + struct scoutfs_btree_item *sib_par_item; struct scoutfs_btree_block *sib; struct scoutfs_block *sib_bl; struct scoutfs_btree_ref *ref; - unsigned int min_used; - unsigned int sib_pos; + unsigned int sib_tot; bool move_right; int to_move; int ret; - BUILD_BUG_ON(min_used_bytes(SCOUTFS_BTREE_TINY_BLOCK_SIZE) < 0); - - if (scoutfs_option_bool(sb, Opt_btree_force_tiny_blocks)) - min_used = min_used_bytes(SCOUTFS_BTREE_TINY_BLOCK_SIZE); - else - min_used = min_used_bytes(SCOUTFS_BLOCK_SIZE); - - if (used_total(bt) >= min_used) + if (le16_to_cpu(bt->total_item_bytes) >= join_low_watermark()) return 0; /* move items right into our block if we have a left sibling */ - if (pos) { - sib_pos = pos - 1; + sib_par_item = prev_item(parent, par_item); + if (sib_par_item) { move_right = true; } else { - sib_pos = pos + 1; + sib_par_item = next_item(parent, par_item); move_right = false; } - ref = item_val(pos_item(parent, sib_pos)); + ref = item_val(parent, sib_par_item); ret = get_ref_block(sb, alloc, wri, BTW_DIRTY, ref, &sib_bl); if (ret) return ret; sib = sib_bl->data; - if (used_total(sib) < min_used) - to_move = used_total(sib); + sib_tot = le16_to_cpu(bt->total_item_bytes); + if (sib_tot < join_low_watermark()) + to_move = sib_tot; else - to_move = min_used - used_total(bt); + to_move = sib_tot - join_low_watermark(); + if (le16_to_cpu(bt->mid_free_len) < to_move) + compact_values(bt); move_items(bt, sib, move_right, to_move); /* update our parent's item */ if (!move_right) - update_parent_item(parent, pos, bt); + update_parent_item(parent, par_item, bt); /* update or delete sibling's parent item */ - if (le32_to_cpu(sib->nr_items) == 0) { - delete_item(parent, sib_pos); + if (le16_to_cpu(sib->nr_items) == 0) { + delete_item(parent, sib_par_item, NULL); ret = scoutfs_radix_free(sb, alloc, wri, le64_to_cpu(sib->hdr.blkno)); BUG_ON(ret); /* could have dirtied alloc to avoid error */ } else if (move_right) { - update_parent_item(parent, sib_pos, sib); + update_parent_item(parent, sib_par_item, sib); } /* and finally shrink the tree if our parent is the root with 1 */ - if (le32_to_cpu(parent->nr_items) == 1) { + if (le16_to_cpu(parent->nr_items) == 1) { root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; @@ -688,77 +998,6 @@ static int try_merge(struct super_block *sb, return 1; } -/* - * A quick and dirty verification of the btree block. We could add a - * lot more checks and make it only verified on read or after - * significant events like splitting and merging. - */ -static int verify_btree_block(struct scoutfs_btree_block *bt, int level) -{ - struct scoutfs_btree_item *item; - struct scoutfs_btree_item *prev = NULL; - unsigned int bytes = 0; - unsigned int after_off = sizeof(struct scoutfs_btree_block); - unsigned int first_off; - unsigned int off; - unsigned int nr; - unsigned int i = 0; - int bad = 1; - - nr = le32_to_cpu(bt->nr_items); - if (nr == 0) - goto out; - - after_off = offsetof(struct scoutfs_btree_block, item_hdrs[nr]); - first_off = SCOUTFS_BLOCK_SIZE; - - if (after_off > SCOUTFS_BLOCK_SIZE) { - nr = 0; - goto out; - } - - for (i = 0; i < nr; i++) { - off = le32_to_cpu(bt->item_hdrs[i].off); - if (off >= SCOUTFS_BLOCK_SIZE || off < after_off) - goto out; - - first_off = min(first_off, off); - - item = pos_item(bt, i); - bytes += item_bytes(item); - - if (i > 0 && scoutfs_key_compare(item_key(item), - item_key(prev)) <= 0) - goto out; - - prev = item; - } - - if (first_off < le32_to_cpu(bt->free_end)) - goto out; - - if ((le32_to_cpu(bt->free_end) + bytes) != SCOUTFS_BLOCK_SIZE) - goto out; - - bad = 0; -out: - if (bad) { - printk("bt %p blkno %llu level %d end %u nr %u (after %u bytes %u)\n", - bt, le64_to_cpu(bt->hdr.blkno), level, - le32_to_cpu(bt->free_end), le32_to_cpu(bt->nr_items), - after_off, bytes); - for (i = 0; i < nr; i++) { - item = pos_item(bt, i); - printk(" [%u] off %u val_len %u\n", - i, le32_to_cpu(bt->item_hdrs[i].off), - item_val_len(item)); - } - BUG_ON(bad); - } - - return 0; -} - /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their @@ -788,12 +1027,14 @@ static int btree_walk(struct super_block *sb, struct scoutfs_block *bl = NULL; struct scoutfs_btree_block *parent = NULL; struct scoutfs_btree_block *bt; + struct scoutfs_btree_item *par_item; struct scoutfs_btree_item *item; + struct scoutfs_btree_item *prev; + struct scoutfs_avl_node *next_node; + struct scoutfs_avl_node *node; struct scoutfs_btree_ref *ref; unsigned int level; - unsigned int pos; unsigned int nr; - int cmp; int ret; if (WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL) || @@ -804,11 +1045,11 @@ restart: scoutfs_block_put(sb, par_bl); par_bl = NULL; parent = NULL; + par_item = NULL; scoutfs_block_put(sb, bl); bl = NULL; bt = NULL; level = root->height; - pos = 0; ret = 0; if (!root->height) { @@ -819,7 +1060,7 @@ restart: &root->ref, &bl); if (ret == 0) { bt = bl->data; - bt->level = 0; + init_btree_block(bt, 0); root->height = 1; } } @@ -834,11 +1075,6 @@ restart: break; bt = bl->data; - /* XXX it'd be nice to make this tunable */ - ret = 0 && verify_btree_block(bt, level); - if (ret) - break; - /* XXX more aggressive block verification, before ref updates? */ if (bt->level != level) { scoutfs_corruption(sb, SC_BTREE_BLOCK_LEVEL, @@ -855,19 +1091,19 @@ restart: } /* - * Splitting and merging can add or remove parents or - * change the pos we take through parents to reach the + * Splitting and joining can add or remove parents or + * change the parent item we use to reach the child * block with the search key. In the rare case that we - * split or merge we simply restart the walk rather than - * try and special case modifying the path to reflect - * the tree changes. + * split or join we simply restart the walk instead of + * update our state to reflect the tree changes. */ ret = 0; if (flags & (BTW_INSERT | BTW_DELETE)) ret = try_split(sb, alloc, wri, root, key, val_len, - parent, pos, bt); + parent, bt); if (ret == 0 && (flags & BTW_DELETE) && parent) - ret = try_merge(sb, alloc, wri, root, parent, pos, bt); + ret = try_join(sb, alloc, wri, root, parent, par_item, + bt); if (ret > 0) goto restart; else if (ret < 0) @@ -877,33 +1113,33 @@ restart: if (level == 0) break; - nr = le32_to_cpu(bt->nr_items); - + nr = le16_to_cpu(bt->nr_items); /* Find the next child block for the search key. */ - pos = find_pos(bt, key, &cmp); - if (pos >= nr) { + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + NULL, NULL, &next_node, NULL); + item = node_item(node ?: next_node); + if (item == NULL) { scoutfs_corruption(sb, SC_BTREE_NO_CHILD_REF, corrupt_btree_block_level, - "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u nr %u pos %u cmp %d", + "root_height %u root_blkno %llu root_seq %llu blkno %llu seq %llu level %u nr %u", root->height, le64_to_cpu(root->ref.blkno), le64_to_cpu(root->ref.seq), le64_to_cpu(bt->hdr.blkno), le64_to_cpu(bt->hdr.seq), bt->level, - nr, pos, cmp); + nr); ret = -EIO; break; } /* give the caller the next key to iterate towards */ - if (iter_key && (flags & BTW_NEXT) && (pos < (nr - 1))) { - item = pos_item(bt, pos); + if (iter_key && (flags & BTW_NEXT) && next_item(bt, item)) { *iter_key = *item_key(item); scoutfs_key_inc(iter_key); - } else if (iter_key && (flags & BTW_PREV) && (pos > 0)) { - item = pos_item(bt, pos - 1); - *iter_key = *item_key(item); + } else if (iter_key && (flags & BTW_PREV) && + (prev = prev_item(bt, item))) { + *iter_key = *item_key(prev); } scoutfs_block_put(sb, par_bl); @@ -912,7 +1148,8 @@ restart: bl = NULL; bt = NULL; - ref = item_val(pos_item(parent, pos)); + par_item = item; + ref = item_val(parent, par_item); } out: @@ -935,10 +1172,12 @@ static void init_item_ref(struct scoutfs_btree_item_ref *iref, struct scoutfs_block *bl, struct scoutfs_btree_item *item) { + struct scoutfs_btree_block *bt = bl->data; + iref->sb = sb; iref->bl = bl; iref->key = item_key(item); - iref->val = item_val(item); + iref->val = item_val(bt, item); iref->val_len = le16_to_cpu(item->val_len); } @@ -963,8 +1202,6 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - unsigned int pos; - int cmp; int ret; if (WARN_ON_ONCE(iref->key)) @@ -973,16 +1210,15 @@ int scoutfs_btree_lookup(struct super_block *sb, ret = btree_walk(sb, NULL, NULL, root, 0, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - item = pos_item(bt, pos); + + item = leaf_item_hash_search(bt, key); + if (item) { init_item_ref(iref, sb, bl, item); ret = 0; } else { scoutfs_block_put(sb, bl); ret = -ENOENT; } - } return ret; @@ -1009,9 +1245,11 @@ int scoutfs_btree_insert(struct super_block *sb, struct scoutfs_key *key, void *val, unsigned val_len) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *node; + struct scoutfs_avl_node *par; struct scoutfs_block *bl; - int pos; int cmp; int ret; @@ -1022,12 +1260,19 @@ int scoutfs_btree_insert(struct super_block *sb, val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, &cmp); - if (cmp) { - create_item(bt, pos, key, val, val_len); - ret = 0; - } else { + + item = leaf_item_hash_search(bt, key); + if (item) { ret = -EEXIST; + } else { + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, + key, &cmp, &par, NULL, NULL); + if (node) { + ret = -EEXIST; + } else { + create_item(bt, key, val, val_len, par, cmp); + ret = 0; + } } scoutfs_block_put(sb, bl); @@ -1036,6 +1281,18 @@ int scoutfs_btree_insert(struct super_block *sb, return ret; } +static void update_item_value(struct scoutfs_btree_block *bt, + struct scoutfs_btree_item *item, + void *val, unsigned val_len) +{ + le16_add_cpu(&bt->total_item_bytes, val_bytes(val_len) - + val_bytes(le16_to_cpu(item->val_len))); + delete_value(bt, le16_to_cpu(item->val_off), + le16_to_cpu(item->val_len)); + item->val_off = insert_value(bt, ptr_off(bt, item), val, val_len); + item->val_len = cpu_to_le16(val_len); +} + /* * Update a btree item. -ENOENT is returned if the item didn't exist. * @@ -1053,10 +1310,9 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_key *key, void *val, unsigned val_len) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - int pos; - int cmp; int ret; if (invalid_item(val_len)) @@ -1066,10 +1322,10 @@ int scoutfs_btree_update(struct super_block *sb, val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - delete_item(bt, pos); - create_item(bt, pos, key, val, val_len); + + item = leaf_item_hash_search(bt, key); + if (item) { + update_item_value(bt, item, val, val_len); ret = 0; } else { ret = -ENOENT; @@ -1092,9 +1348,10 @@ int scoutfs_btree_force(struct super_block *sb, struct scoutfs_key *key, void *val, unsigned val_len) { + struct scoutfs_btree_item *item; + struct scoutfs_avl_node *par; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - int pos; int cmp; int ret; @@ -1105,10 +1362,17 @@ int scoutfs_btree_force(struct super_block *sb, val_len, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, &cmp); - if (cmp == 0) - delete_item(bt, pos); - create_item(bt, pos, key, val, val_len); + + item = leaf_item_hash_search(bt, key); + if (item) { + update_item_value(bt, item, val, val_len); + } else { + scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + &cmp, &par, NULL, NULL); + create_item(bt, key, val, val_len, par, cmp); + } + ret = 0; + scoutfs_block_put(sb, bl); } @@ -1125,19 +1389,19 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - int pos; - int cmp; int ret; ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - pos = find_pos(bt, key, &cmp); - if (cmp == 0) { - if (le32_to_cpu(bt->nr_items) == 1) { + + item = leaf_item_hash_search(bt, key); + if (item) { + if (le16_to_cpu(bt->nr_items) == 1) { /* remove final empty block */ ret = scoutfs_radix_free(sb, alloc, wri, bl->blkno); @@ -1147,7 +1411,7 @@ int scoutfs_btree_delete(struct super_block *sb, root->ref.seq = 0; } } else { - delete_item(bt, pos); + delete_item(bt, item, NULL); ret = 0; } } else { @@ -1162,8 +1426,8 @@ int scoutfs_btree_delete(struct super_block *sb, /* * Iterate from a key value to the next item in the direction of - * iteration. Callers set flags to tell which way to iterate and - * whether the search key is inclusive, or not. + * iteration. Callers set flags to tell which way to iterate. The + * first key is always inclusive. * * Walking can land in a leaf that doesn't contain any items in the * direction of the iteration. Walking gives us the next key to walk @@ -1176,13 +1440,14 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, int flags, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { + struct scoutfs_avl_node *node; + struct scoutfs_avl_node *next; + struct scoutfs_avl_node *prev; struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct scoutfs_block *bl; struct scoutfs_key iter_key; struct scoutfs_key walk_key; - int pos; - int cmp; + struct scoutfs_block *bl; int ret; if (WARN_ON_ONCE(flags & BTW_DIRTY) || @@ -1199,19 +1464,15 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, break; bt = bl->data; - pos = find_pos(bt, key, &cmp); + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, key, + NULL, NULL, &next, &prev); - /* point pos towards iteration, find_pos already for _NEXT */ - if ((flags & BTW_AFTER) && cmp == 0) - pos++; - else if ((flags & BTW_PREV) && cmp < 0) - pos--; - else if ((flags & BTW_BEFORE) && cmp == 0) - pos--; - - /* found the next item in this leaf */ - if (pos >= 0 && pos < le32_to_cpu(bt->nr_items)) { - item = pos_item(bt, pos); + if (node == NULL && (flags & BTW_NEXT)) + node = next; + else if (node == NULL && (flags & BTW_PREV)) + node = prev; + item = node_item(node); + if (item) { init_item_ref(iref, sb, bl, item); ret = 0; break; @@ -1274,16 +1535,17 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key) { + struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; struct scoutfs_block *bl; - int cmp; int ret; ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - find_pos(bt, key, &cmp); - if (cmp == 0) + + item = leaf_item_hash_search(bt, key); + if (item) ret = 0; else ret = -ENOENT; diff --git a/kmod/src/format.h b/kmod/src/format.h index 413774da..e8b4ec86 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -196,17 +196,10 @@ struct scoutfs_avl_node { } __packed; /* when we split we want to have multiple items on each side */ -#define SCOUTFS_BTREE_MAX_VAL_LEN (SCOUTFS_BLOCK_SIZE / 8) +#define SCOUTFS_BTREE_MAX_VAL_LEN 512 -/* - * The min number of free bytes we must leave in a parent as we descend - * to modify. This guarantees enough free bytes in a parent to insert a - * new child reference item as a child block splits. - */ -#define SCOUTFS_BTREE_PARENT_MIN_FREE_BYTES \ - (sizeof(struct scoutfs_btree_item_header) + \ - sizeof(struct scoutfs_btree_item) + \ - sizeof(struct scoutfs_btree_ref)) +/* each value ends with an offset which lets compaction iterate over values */ +#define SCOUTFS_BTREE_VAL_OWNER_BYTES sizeof(__le16) /* * When debugging we can tune the splitting and merging thresholds to @@ -236,24 +229,37 @@ struct scoutfs_btree_root { __u8 height; } __packed; -struct scoutfs_btree_item_header { - __le32 off; -} __packed; - struct scoutfs_btree_item { + struct scoutfs_avl_node node; struct scoutfs_key key; + __le16 val_off; __le16 val_len; - __u8 val[0]; } __packed; struct scoutfs_btree_block { struct scoutfs_block_header hdr; - __le32 free_end; - __le32 nr_items; + struct scoutfs_avl_root item_root; + __le16 nr_items; + __le16 total_item_bytes; + __le16 mid_free_len; + __le16 last_free_off; + __le16 last_free_len; __u8 level; - struct scoutfs_btree_item_header item_hdrs[0]; + struct scoutfs_btree_item items[0]; + /* leaf blocks have a fixed size item offset hash table at the end */ } __packed; +/* + * Try to aim for a 75% load in a leaf full of items with no value. + * We'll almost never see this because most items have values and most + * blocks aren't full. + */ +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ + ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / \ + (sizeof(struct scoutfs_btree_item) + (sizeof(__le16))) * 100 / 75) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ + (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) + struct scoutfs_mounted_client_btree_val { __u8 flags; } __packed; From ac0e58839de40ee5d9d139be68dedabb4f54d334 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Apr 2020 14:51:11 -0700 Subject: [PATCH 826/920] scoutfs: remove btree _before and _after There's no users of these variants of _prev and _next so they can be removed. Support for them was also dropped in the previous reworking of the internal structure of the btree blocks. Signed-off-by: Zach Brown --- kmod/src/btree.c | 27 +++++---------------------- kmod/src/btree.h | 7 ------- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index a62535f5..7edee5b6 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -78,13 +78,11 @@ /* btree walking has a bunch of behavioural bit flags */ enum { BTW_NEXT = (1 << 0), /* return >= key */ - BTW_AFTER = (1 << 1), /* return > key */ - BTW_PREV = (1 << 2), /* return <= key */ - BTW_BEFORE = (1 << 3), /* return < key */ - BTW_DIRTY = (1 << 4), /* cow stable blocks */ - BTW_ALLOC = (1 << 5), /* allocate a new block for 0 ref */ - BTW_INSERT = (1 << 6), /* walking to insert, try splitting */ - BTW_DELETE = (1 << 7), /* walking to delete, try joining */ + BTW_PREV = (1 << 1), /* return <= key */ + BTW_DIRTY = (1 << 2), /* cow stable blocks */ + BTW_ALLOC = (1 << 3), /* allocate a new block for 0 ref */ + BTW_INSERT = (1 << 4), /* walking to insert, try splitting */ + BTW_DELETE = (1 << 5), /* walking to delete, try joining */ }; /* total length of the value payload */ @@ -1500,13 +1498,6 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, return btree_iter(sb, root, BTW_NEXT, key, iref); } -int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_item_ref *iref) -{ - return btree_iter(sb, root, BTW_NEXT | BTW_AFTER, key, iref); -} - int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) @@ -1514,14 +1505,6 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, return btree_iter(sb, root, BTW_PREV, key, iref); } -int scoutfs_btree_before(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_item_ref *iref) -{ - return btree_iter(sb, root, BTW_PREV | BTW_BEFORE, key, iref); -} - /* * Ensure that the blocks that lead to the item with the given key are * dirty. caller can hold a transaction to pin the dirty blocks and diff --git a/kmod/src/btree.h b/kmod/src/btree.h index 133832ba..e86396ae 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -49,16 +49,9 @@ int scoutfs_btree_delete(struct super_block *sb, int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); -int scoutfs_btree_after(struct super_block *sb, struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_item_ref *iref); int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); -int scoutfs_btree_before(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_item_ref *iref); int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, From 99bc710f03b7f625b055c0934e66d85a5415e1a3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 29 Apr 2020 14:54:59 -0700 Subject: [PATCH 827/920] scoutfs: remove tiny btree block option It used to take significant effort to create very tall btrees because they only stored small references to large LSM segments. Now they store all file system metadata and we can easily create sufficiently large btrees for testing. We don't need the tiny btree option. Signed-off-by: Zach Brown --- kmod/src/format.h | 8 -------- kmod/src/options.c | 16 ---------------- kmod/src/options.h | 5 ----- 3 files changed, 29 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index e8b4ec86..822675e6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -201,14 +201,6 @@ struct scoutfs_avl_node { /* each value ends with an offset which lets compaction iterate over values */ #define SCOUTFS_BTREE_VAL_OWNER_BYTES sizeof(__le16) -/* - * When debugging we can tune the splitting and merging thresholds to - * create much larger trees by having blocks with many fewer items. We - * implement this by pretending the blocks are tiny. They're still - * large enough for a handful of items. - */ -#define SCOUTFS_BTREE_TINY_BLOCK_SIZE 512 - /* * A 4EB test image measured a worst case height of 17. This is plenty * generous. diff --git a/kmod/src/options.c b/kmod/src/options.c index 332b1cda..e7fe1843 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -33,19 +33,10 @@ static const match_table_t tokens = { struct options_sb_info { struct dentry *debugfs_dir; - u32 btree_force_tiny_blocks; }; u32 scoutfs_option_u32(struct super_block *sb, int token) { - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct options_sb_info *osi = sbi->options; - - switch(token) { - case Opt_btree_force_tiny_blocks: - return osi->btree_force_tiny_blocks; - } - WARN_ON_ONCE(1); return 0; } @@ -143,13 +134,6 @@ int scoutfs_options_setup(struct super_block *sb) goto out; } - if (!debugfs_create_bool("btree_force_tiny_blocks", 0644, - osi->debugfs_dir, - &osi->btree_force_tiny_blocks)) { - ret = -ENOMEM; - goto out; - } - ret = 0; out: if (ret) diff --git a/kmod/src/options.h b/kmod/src/options.h index 0078dca2..d02b40d6 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -6,11 +6,6 @@ #include "format.h" enum { - /* - * For debugging we can quickly create huge trees by limiting - * the number of items in each block as though the blocks were tiny. - */ - Opt_btree_force_tiny_blocks, Opt_server_addr, Opt_err, }; From 177af7f746a9c3b411e38d45dc3db6e23c3d3544 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 1 May 2020 10:29:09 -0700 Subject: [PATCH 828/920] scoutfs: use larger metadata blocks Introduce different constants for small and large metadata block sizes. The small 4KB size is used for the super block, quorum blocks, and as the granularity of file data block allocation. The larger 64KB size is used for the radix, btree, and forest bloom metadata block structures. The bulk of this are obvious transitions from the old single constant to the appropriate new constant. But there are a few more involved changes, though just barely. The block crc calculation now needs the caller to pass in the size of the block. The radix function to return free bytes instead returns free blocks and the caller is responsible for knowing how big its managed blocks are. Signed-off-by: Zach Brown --- kmod/src/block.c | 55 ++++++++++++++++++----------------- kmod/src/block.h | 4 +-- kmod/src/btree.c | 12 ++++---- kmod/src/count.h | 6 ++-- kmod/src/data.c | 31 ++++++++++---------- kmod/src/forest.c | 4 +-- kmod/src/format.h | 73 +++++++++++++++++++++++++++++------------------ kmod/src/inode.c | 7 +++-- kmod/src/ioctl.c | 16 +++++------ kmod/src/quorum.c | 18 ++++++------ kmod/src/radix.c | 14 ++++----- kmod/src/radix.h | 4 +-- kmod/src/server.c | 4 +-- kmod/src/super.c | 8 +++--- 14 files changed, 138 insertions(+), 118 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 8c8f1b5b..b7da3950 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -108,18 +108,18 @@ do { \ * be refactored away. */ -__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr) +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size) { int off = offsetof(struct scoutfs_block_header, crc) + FIELD_SIZEOF(struct scoutfs_block_header, crc); - u32 calc = crc32c(~0, (char *)hdr + off, SCOUTFS_BLOCK_SIZE - off); + u32 calc = crc32c(~0, (char *)hdr + off, size - off); return cpu_to_le32(calc); } -bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr) +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size) { - return hdr->crc == scoutfs_block_calc_crc(hdr); + return hdr->crc == scoutfs_block_calc_crc(hdr, size); } bool scoutfs_block_valid_ref(struct super_block *sb, @@ -157,19 +157,19 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) * more careful with a partial page allocator when allocating * blocks and would make the lru per-page instead of per-block. */ - BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_SIZE); + BUILD_BUG_ON(PAGE_SIZE > SCOUTFS_BLOCK_LG_SIZE); bp = kzalloc(sizeof(struct block_private), GFP_NOFS); if (!bp) goto out; - bp->page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_PAGE_ORDER); + bp->page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_LG_PAGE_ORDER); if (bp->page) { scoutfs_inc_counter(sb, block_cache_alloc_page_order); set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); bp->bl.data = page_address(bp->page); } else { - bp->virt = __vmalloc(SCOUTFS_BLOCK_SIZE, + bp->virt = __vmalloc(SCOUTFS_BLOCK_LG_SIZE, GFP_NOFS | __GFP_HIGHMEM, PAGE_KERNEL); if (!bp->virt) { kfree(bp); @@ -206,7 +206,7 @@ static void block_free(struct super_block *sb, struct block_private *bp) TRACE_BLOCK(free, bp); if (test_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits)) - __free_pages(bp->page, SCOUTFS_BLOCK_PAGE_ORDER); + __free_pages(bp->page, SCOUTFS_BLOCK_LG_PAGE_ORDER); else if (test_bit(BLOCK_BIT_VIRT, &bp->bits)) vfree(bp->virt); else @@ -441,7 +441,7 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, sector_t sector; int ret = 0; - sector = bp->bl.blkno << (SCOUTFS_BLOCK_SHIFT - 9); + sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9); WARN_ON_ONCE(bp->bl.blkno == U64_MAX); WARN_ON_ONCE(sector == U64_MAX || sector == 0); @@ -453,9 +453,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, blk_start_plug(&plug); - for (off = 0; off < SCOUTFS_BLOCK_SIZE; off += PAGE_SIZE) { + for (off = 0; off < SCOUTFS_BLOCK_LG_SIZE; off += PAGE_SIZE) { if (!bio) { - bio = bio_alloc(GFP_NOFS, SCOUTFS_PAGES_PER_BLOCK); + bio = bio_alloc(GFP_NOFS, SCOUTFS_BLOCK_LG_PAGES_PER); if (!bio) { ret = -ENOMEM; break; @@ -634,6 +634,7 @@ void scoutfs_block_invalidate(struct super_block *sb, struct scoutfs_block *bl) } } +/* This is only used for large metadata blocks */ bool scoutfs_block_consistent_ref(struct super_block *sb, struct scoutfs_block *bl, __le64 seq, __le64 blkno, u32 magic) @@ -643,7 +644,8 @@ bool scoutfs_block_consistent_ref(struct super_block *sb, struct scoutfs_block_header *hdr = bl->data; if (!test_bit(BLOCK_BIT_CRC_VALID, &bp->bits)) { - if (hdr->crc != scoutfs_block_calc_crc(hdr)) + if (hdr->crc != + scoutfs_block_calc_crc(hdr, SCOUTFS_BLOCK_LG_SIZE)) return false; set_bit(BLOCK_BIT_CRC_VALID, &bp->bits); } @@ -722,7 +724,7 @@ int scoutfs_block_writer_write(struct super_block *sb, /* checksum everything to reduce time between io submission merging */ list_for_each_entry(bp, &wri->dirty_list, dirty_entry) { hdr = bp->bl.data; - hdr->crc = scoutfs_block_calc_crc(hdr); + hdr->crc = scoutfs_block_calc_crc(hdr, SCOUTFS_BLOCK_LG_SIZE); } blk_start_plug(&plug); @@ -866,7 +868,7 @@ bool scoutfs_block_writer_has_dirty(struct super_block *sb, u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, struct scoutfs_block_writer *wri) { - return wri->nr_dirty_blocks * SCOUTFS_BLOCK_SIZE; + return wri->nr_dirty_blocks * SCOUTFS_BLOCK_LG_SIZE; } /* @@ -916,12 +918,9 @@ static int block_shrink(struct shrinker *shrink, struct shrink_control *sc) spin_unlock(&binf->lock); out: - return min_t(u64, binf->lru_nr * SCOUTFS_PAGES_PER_BLOCK, INT_MAX); + return min_t(u64, binf->lru_nr * SCOUTFS_BLOCK_LG_PAGES_PER, INT_MAX); } -#define SCOUTFS_SM_BLOCK_SHIFT 12 -#define SCOUTFS_SM_BLOCK_SIZE (1 << SCOUTFS_SM_BLOCK_SHIFT) - struct sm_block_completion { struct completion comp; int err; @@ -956,11 +955,9 @@ static int sm_block_io(struct super_block *sb, int rw, u64 blkno, struct bio *bio; int ret; - BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_SM_BLOCK_SIZE); - /* block calc crc is assuming block size, they'll be different later */ - BUILD_BUG_ON(SCOUTFS_SM_BLOCK_SIZE != SCOUTFS_BLOCK_SIZE); + BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE); - if (WARN_ON_ONCE(len > SCOUTFS_SM_BLOCK_SIZE) || + if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) || WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) return -EINVAL; @@ -972,10 +969,11 @@ static int sm_block_io(struct super_block *sb, int rw, u64 blkno, if (rw & WRITE) { memcpy(pg_hdr, hdr, len); - if (len < SCOUTFS_SM_BLOCK_SIZE) + if (len < SCOUTFS_BLOCK_SM_SIZE) memset((char *)pg_hdr + len, 0, - SCOUTFS_SM_BLOCK_SIZE - len); - pg_hdr->crc = scoutfs_block_calc_crc(pg_hdr); + SCOUTFS_BLOCK_SM_SIZE - len); + pg_hdr->crc = scoutfs_block_calc_crc(pg_hdr, + SCOUTFS_BLOCK_SM_SIZE); } bio = bio_alloc(GFP_NOFS, 1); @@ -984,11 +982,11 @@ static int sm_block_io(struct super_block *sb, int rw, u64 blkno, goto out; } - bio->bi_sector = blkno << (SCOUTFS_SM_BLOCK_SHIFT - 9); + bio->bi_sector = blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9); bio->bi_bdev = sb->s_bdev; bio->bi_end_io = sm_block_bio_end_io; bio->bi_private = &sbc; - bio_add_page(bio, page, SCOUTFS_SM_BLOCK_SIZE, 0); + bio_add_page(bio, page, SCOUTFS_BLOCK_SM_SIZE, 0); init_completion(&sbc.comp); sbc.err = 0; @@ -1000,7 +998,8 @@ static int sm_block_io(struct super_block *sb, int rw, u64 blkno, if (ret == 0 && !(rw & WRITE)) { memcpy(hdr, pg_hdr, len); - *blk_crc = scoutfs_block_calc_crc(pg_hdr); + *blk_crc = scoutfs_block_calc_crc(pg_hdr, + SCOUTFS_BLOCK_SM_SIZE); } out: __free_page(page); diff --git a/kmod/src/block.h b/kmod/src/block.h index dc62bf77..57e849a5 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -12,8 +12,8 @@ struct scoutfs_block { void *data; }; -__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr); -bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr); +__le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size); +bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size); bool scoutfs_block_valid_ref(struct super_block *sb, struct scoutfs_block_header *hdr, __le64 seq, __le64 blkno); diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 7edee5b6..969b831a 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -111,7 +111,7 @@ static inline unsigned int item_bytes(struct scoutfs_btree_item *item) */ static unsigned int join_low_watermark(void) { - return (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / 4; + return (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / 4; } /* @@ -121,7 +121,7 @@ static unsigned int join_low_watermark(void) static unsigned int item_full_pct(struct scoutfs_btree_block *bt) { return (int)le16_to_cpu(bt->total_item_bytes) * 100 / - (SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)); + (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)); } static inline __le16 ptr_off(struct scoutfs_btree_block *bt, void *ptr) @@ -216,7 +216,7 @@ static int leaf_item_hash_ind(struct scoutfs_key *key) static __le16 *leaf_item_hash_buckets(struct scoutfs_btree_block *bt) { - return (void *)bt + SCOUTFS_BLOCK_SIZE - + return (void *)bt + SCOUTFS_BLOCK_LG_SIZE - SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; } @@ -760,11 +760,11 @@ retry: if (bt) { /* returning a cow of an existing block */ - memcpy(new, bt, SCOUTFS_BLOCK_SIZE); + memcpy(new, bt, SCOUTFS_BLOCK_LG_SIZE); scoutfs_block_put(sb, bl); } else { /* returning a newly allocated block */ - memset(new, 0, SCOUTFS_BLOCK_SIZE); + memset(new, 0, SCOUTFS_BLOCK_LG_SIZE); new->hdr.fsid = super->hdr.fsid; } bl = new_bl; @@ -828,7 +828,7 @@ static void init_btree_block(struct scoutfs_btree_block *bt, int level) { int free; - free = SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block); + free = SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block); if (level == 0) free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; diff --git a/kmod/src/count.h b/kmod/src/count.h index b2dd8152..176321d0 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -245,9 +245,9 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, static inline const struct scoutfs_item_count SIC_WRITE_BEGIN(void) { struct scoutfs_item_count cnt = {0,}; - unsigned nr_free = (1 + SCOUTFS_BLOCKS_PER_PAGE) * 3; - unsigned nr_file = (DIV_ROUND_UP(SCOUTFS_BLOCKS_PER_PAGE, 2) + - SCOUTFS_BLOCKS_PER_PAGE) * 3; + unsigned nr_free = (1 + SCOUTFS_BLOCK_SM_PER_PAGE) * 3; + unsigned nr_file = (DIV_ROUND_UP(SCOUTFS_BLOCK_SM_PER_PAGE, 2) + + SCOUTFS_BLOCK_SM_PER_PAGE) * 3; __count_dirty_inode(&cnt); diff --git a/kmod/src/data.c b/kmod/src/data.c index b129aa42..ed42bb8a 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -803,8 +803,8 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, WARN_ON_ONCE(inode && !mutex_is_locked(&inode->i_mutex)); /* clamp last to the last possible block? */ - if (last > SCOUTFS_BLOCK_MAX) - last = SCOUTFS_BLOCK_MAX; + if (last > SCOUTFS_BLOCK_SM_MAX) + last = SCOUTFS_BLOCK_SM_MAX; trace_scoutfs_data_truncate_items(sb, iblock, last, offline); @@ -1060,7 +1060,7 @@ out: offset = iblock - ext->iblock; map_bh(bh, inode->i_sb, ext->blkno + offset); bh->b_size = min_t(u64, bh->b_size, - (ext->count - offset) << SCOUTFS_BLOCK_SHIFT); + (ext->count - offset) << SCOUTFS_BLOCK_SM_SHIFT); } if (ext) @@ -1483,8 +1483,8 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) goto out; } - iblock = offset >> SCOUTFS_BLOCK_SHIFT; - last = (offset + len - 1) >> SCOUTFS_BLOCK_SHIFT; + iblock = offset >> SCOUTFS_BLOCK_SM_SHIFT; + last = (offset + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; while(iblock <= last) { @@ -1496,7 +1496,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) ret = fallocate_extents(sb, inode, iblock, last, lock); if (ret >= 0 && !(mode & FALLOC_FL_KEEP_SIZE)) { - end = (iblock + ret) << SCOUTFS_BLOCK_SHIFT; + end = (iblock + ret) << SCOUTFS_BLOCK_SM_SHIFT; if (end > offset + len) end = offset + len; if (end > i_size_read(inode)) @@ -1549,7 +1549,7 @@ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, u64 off; int ret; - blocks = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SIZE); + blocks = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SM_SIZE); scoutfs_inode_get_onoff(inode, &on, &off); iblock = off; @@ -1622,9 +1622,9 @@ static int fill_extent(struct fiemap_extent_info *fieinfo, flags |= FIEMAP_EXTENT_UNWRITTEN; return fiemap_fill_next_extent(fieinfo, - ext->iblock << SCOUTFS_BLOCK_SHIFT, - ext->blkno << SCOUTFS_BLOCK_SHIFT, - ext->count << SCOUTFS_BLOCK_SHIFT, + ext->iblock << SCOUTFS_BLOCK_SM_SHIFT, + ext->blkno << SCOUTFS_BLOCK_SM_SHIFT, + ext->count << SCOUTFS_BLOCK_SM_SHIFT, flags); } @@ -1666,8 +1666,8 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, memset(&cur, 0, sizeof(cur)); last_flags = 0; - iblock = start >> SCOUTFS_BLOCK_SHIFT; - last = (start + len - 1) >> SCOUTFS_BLOCK_SHIFT; + iblock = start >> SCOUTFS_BLOCK_SM_SHIFT; + last = (start + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; for (;;) { ret = load_unpacked_extents(sb, ino, iblock, last, false, @@ -1831,8 +1831,8 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, } } - iblock = pos >> SCOUTFS_BLOCK_SHIFT; - last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SHIFT; + iblock = pos >> SCOUTFS_BLOCK_SM_SHIFT; + last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; while(iblock <= last_block) { @@ -2056,7 +2056,8 @@ u64 scoutfs_data_alloc_free_bytes(struct super_block *sb) { DECLARE_DATA_INFO(sb, datinf); - return scoutfs_radix_root_free_bytes(sb, &datinf->data_avail); + return scoutfs_radix_root_free_blocks(sb, &datinf->data_avail) << + SCOUTFS_BLOCK_SM_SHIFT; } int scoutfs_data_setup(struct super_block *sb) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index af44204e..9bad162e 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -1163,9 +1163,9 @@ static int set_lock_bloom_bits(struct super_block *sb, err = scoutfs_radix_free(sb, finf->alloc, finf->wri, le64_to_cpu(ref->blkno)); BUG_ON(err); /* could have dirtied */ - memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_SIZE); + memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); } else { - memset(new_bl->data, 0, SCOUTFS_BLOCK_SIZE); + memset(new_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); } scoutfs_block_writer_mark_dirty(sb, finf->wri, new_bl); diff --git a/kmod/src/format.h b/kmod/src/format.h index 822675e6..6c667c97 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -11,24 +11,42 @@ #define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 /* - * The super block and btree blocks are fixed 4k. + * The super block, quorum block, and file data allocation granularity + * use the smaller 4KB block. */ -#define SCOUTFS_BLOCK_SHIFT 12 -#define SCOUTFS_BLOCK_SIZE (1 << SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_BLOCK_MASK (SCOUTFS_BLOCK_SIZE - 1) -#define SCOUTFS_BLOCKS_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SIZE) -#define SCOUTFS_BLOCK_SECTOR_SHIFT (SCOUTFS_BLOCK_SHIFT - 9) -#define SCOUTFS_BLOCK_SECTORS (1 << SCOUTFS_BLOCK_SECTOR_SHIFT) -#define SCOUTFS_BLOCK_MAX (U64_MAX >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_BLOCK_SM_SHIFT 12 +#define SCOUTFS_BLOCK_SM_SIZE (1 << SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_MASK (SCOUTFS_BLOCK_SM_SIZE - 1) +#define SCOUTFS_BLOCK_SM_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_SM_SIZE) +#define SCOUTFS_BLOCK_SM_SECTOR_SHIFT (SCOUTFS_BLOCK_SM_SHIFT - 9) +#define SCOUTFS_BLOCK_SM_SECTORS (1 << SCOUTFS_BLOCK_SM_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_SM_MAX (U64_MAX >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_BLOCK_SM_PAGES_PER (SCOUTFS_BLOCK_SM_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_SM_PAGE_ORDER (SCOUTFS_BLOCK_SM_SHIFT - PAGE_SHIFT) + +/* + * The radix and btree structures, and the forest bloom block, use the + * larger 64KB metadata block size. + */ +#define SCOUTFS_BLOCK_LG_SHIFT 16 +#define SCOUTFS_BLOCK_LG_SIZE (1 << SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_MASK (SCOUTFS_BLOCK_LG_SIZE - 1) +#define SCOUTFS_BLOCK_LG_PER_PAGE (PAGE_SIZE / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_BLOCK_LG_SECTOR_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - 9) +#define SCOUTFS_BLOCK_LG_SECTORS (1 << SCOUTFS_BLOCK_LG_SECTOR_SHIFT) +#define SCOUTFS_BLOCK_LG_MAX (U64_MAX >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_BLOCK_LG_PAGES_PER (SCOUTFS_BLOCK_LG_SIZE / PAGE_SIZE) +#define SCOUTFS_BLOCK_LG_PAGE_ORDER (SCOUTFS_BLOCK_LG_SHIFT - PAGE_SHIFT) + +#define SCOUTFS_BLOCK_SM_LG_SHIFT (SCOUTFS_BLOCK_LG_SHIFT - \ + SCOUTFS_BLOCK_SM_SHIFT) -#define SCOUTFS_PAGES_PER_BLOCK (SCOUTFS_BLOCK_SIZE / PAGE_SIZE) -#define SCOUTFS_BLOCK_PAGE_ORDER (SCOUTFS_BLOCK_SHIFT - PAGE_SHIFT) /* * The super block leaves some room before the first block for platform * structures like boot loaders. */ -#define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_SUPER_BLKNO ((64ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) /* * A reasonably large region of aligned quorum blocks follow the super @@ -38,8 +56,8 @@ * mounts that have a reasonable probability of not overwriting each * other's random block locations. */ -#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) -#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SHIFT) +#define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ @@ -168,8 +186,9 @@ struct scoutfs_radix_root { struct scoutfs_radix_ref ref; } __packed; -#define SCOUTFS_RADIX_REFS \ - ((SCOUTFS_BLOCK_SIZE - offsetof(struct scoutfs_radix_block, refs[0])) /\ +#define SCOUTFS_RADIX_REFS \ + ((SCOUTFS_BLOCK_LG_SIZE - \ + offsetof(struct scoutfs_radix_block, refs[0])) / \ sizeof(struct scoutfs_radix_ref)) /* 8 meg regions with 4k data blocks */ @@ -178,8 +197,8 @@ struct scoutfs_radix_root { #define SCOUTFS_RADIX_LG_MASK (SCOUTFS_RADIX_LG_BITS - 1) /* round block bits down to a multiple of large ranges */ -#define SCOUTFS_RADIX_BITS \ - (((SCOUTFS_BLOCK_SIZE - \ +#define SCOUTFS_RADIX_BITS \ + (((SCOUTFS_BLOCK_LG_SIZE - \ offsetof(struct scoutfs_radix_block, bits[0])) * 8) & \ ~(__u64)SCOUTFS_RADIX_LG_MASK) #define SCOUTFS_RADIX_BITS_BYTES (SCOUTFS_RADIX_BITS / 8) @@ -247,7 +266,7 @@ struct scoutfs_btree_block { * blocks aren't full. */ #define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_btree_block)) / \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / \ (sizeof(struct scoutfs_btree_item) + (sizeof(__le16))) * 100 / 75) #define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) @@ -313,9 +332,9 @@ struct scoutfs_bloom_block { */ #define SCOUTFS_FOREST_BLOOM_NRS 7 #define SCOUTFS_FOREST_BLOOM_BITS \ - (((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_bloom_block)) / \ - member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ - member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) \ + (((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_bloom_block)) / \ + member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ + member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) \ /* * Keys are first sorted by major key zones. @@ -380,10 +399,10 @@ struct scoutfs_packed_extent { __u8 le_blkno_diff[0]; } __packed; -#define SCOUTFS_PACKEXT_BLOCKS (8 * 1024 * 1024 / SCOUTFS_BLOCK_SIZE) -#define SCOUTFS_PACKEXT_BASE_SHIFT (ilog2(SCOUTFS_PACKEXT_BLOCKS)) -#define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) -#define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE +#define SCOUTFS_PACKEXT_BLOCKS (8 * 1024 * 1024 / SCOUTFS_BLOCK_SM_SIZE) +#define SCOUTFS_PACKEXT_BASE_SHIFT (ilog2(SCOUTFS_PACKEXT_BLOCKS)) +#define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) +#define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE #define SEF_OFFLINE (1 << 0) #define SEF_UNWRITTEN (1 << 1) @@ -445,8 +464,8 @@ struct scoutfs_quorum_block { } __packed log[0]; } __packed; -#define SCOUTFS_QUORUM_LOG_MAX \ - ((SCOUTFS_BLOCK_SIZE - sizeof(struct scoutfs_quorum_block)) / \ +#define SCOUTFS_QUORUM_LOG_MAX \ + ((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \ sizeof(struct scoutfs_quorum_log)) struct scoutfs_super_block { diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 32ed084e..124d9375 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -244,7 +244,7 @@ static void load_inode(struct inode *inode, struct scoutfs_inode *cinode) * maintained as blocks come and go. */ inode->i_blocks = (ci->online_blocks + ci->offline_blocks) - << SCOUTFS_BLOCK_SECTOR_SHIFT; + << SCOUTFS_BLOCK_SM_SECTOR_SHIFT; set_item_info(ci, cinode); } @@ -390,7 +390,8 @@ int scoutfs_complete_truncate(struct inode *inode, struct scoutfs_lock *lock) if (!(ci->flags & SCOUTFS_INO_FLAG_TRUNCATE)) return 0; - start = (i_size_read(inode) + SCOUTFS_BLOCK_SIZE - 1) >> SCOUTFS_BLOCK_SHIFT; + start = (i_size_read(inode) + SCOUTFS_BLOCK_SM_SIZE - 1) >> + SCOUTFS_BLOCK_SM_SHIFT; ret = scoutfs_data_truncate_items(inode->i_sb, inode, scoutfs_ino(inode), start, ~0ULL, false, lock); @@ -573,7 +574,7 @@ void scoutfs_inode_add_onoff(struct inode *inode, s64 on, s64 off) si->online_blocks += on; si->offline_blocks += off; /* XXX not sure if this is right */ - inode->i_blocks += (on + off) * SCOUTFS_BLOCK_SECTORS; + inode->i_blocks += (on + off) * SCOUTFS_BLOCK_SM_SECTORS; trace_scoutfs_online_offline_blocks(inode, on, off, si->online_blocks, diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index a321a1db..c0dac183 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -318,8 +318,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) inode_dio_wait(inode); /* drop all clean and dirty cached blocks in the range */ - start = args.block << SCOUTFS_BLOCK_SHIFT; - end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SHIFT) - 1; + start = args.block << SCOUTFS_BLOCK_SM_SHIFT; + end_inc = ((args.block + args.count) << SCOUTFS_BLOCK_SM_SHIFT) - 1; truncate_inode_pages_range(&inode->i_data, start, end_inc); ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), @@ -330,8 +330,8 @@ static long scoutfs_ioc_release(struct file *file, unsigned long arg) scoutfs_inode_get_onoff(inode, &online, &offline); isize = i_size_read(inode); if (online == 0 && isize) { - start = (isize + SCOUTFS_BLOCK_SIZE - 1) - >> SCOUTFS_BLOCK_SHIFT; + start = (isize + SCOUTFS_BLOCK_SM_SIZE - 1) + >> SCOUTFS_BLOCK_SM_SHIFT; ret = scoutfs_data_truncate_items(sb, inode, scoutfs_ino(inode), start, U64_MAX, @@ -371,8 +371,8 @@ static long scoutfs_ioc_data_wait_err(struct file *file, unsigned long arg) trace_scoutfs_ioc_data_wait_err(sb, &args); - sblock = args.offset >> SCOUTFS_BLOCK_SHIFT; - eblock = (args.offset + args.count - 1) >> SCOUTFS_BLOCK_SHIFT; + sblock = args.offset >> SCOUTFS_BLOCK_SM_SHIFT; + eblock = (args.offset + args.count - 1) >> SCOUTFS_BLOCK_SM_SHIFT; if (sblock > eblock) return -EINVAL; @@ -460,7 +460,7 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) /* verify arg constraints that aren't dependent on file */ if (args.count < 0 || (end_size < args.offset) || - args.offset & SCOUTFS_BLOCK_MASK) + args.offset & SCOUTFS_BLOCK_SM_MASK) return -EINVAL; if (args.count == 0) @@ -494,7 +494,7 @@ static long scoutfs_ioc_stage(struct file *file, unsigned long arg) (file->f_flags & (O_APPEND | O_DIRECT | O_DSYNC)) || IS_SYNC(file->f_mapping->host) || (end_size > isize) || - ((end_size & SCOUTFS_BLOCK_MASK) && (end_size != isize))) { + ((end_size & SCOUTFS_BLOCK_SM_MASK) && (end_size != isize))) { ret = -EINVAL; goto out; } diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index a3e7e6d2..e1960fed 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -144,7 +144,7 @@ struct quorum_block_head { struct list_head head; union { struct scoutfs_quorum_block blk; - u8 bytes[SCOUTFS_BLOCK_SIZE]; + u8 bytes[SCOUTFS_BLOCK_SM_SIZE]; }; }; @@ -184,13 +184,13 @@ static size_t quorum_block_bytes(struct scoutfs_quorum_block *blk) static bool invalid_quorum_block(struct buffer_head *bh, struct scoutfs_quorum_block *blk) { - return bh->b_size != SCOUTFS_BLOCK_SIZE || - sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE || + return bh->b_size != SCOUTFS_BLOCK_SM_SIZE || + sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SM_SIZE || quorum_block_crc(blk) != blk->crc || le64_to_cpu(blk->blkno) != bh->b_blocknr || blk->term == 0 || blk->log_nr > SCOUTFS_QUORUM_LOG_MAX || - quorum_block_bytes(blk) > SCOUTFS_BLOCK_SIZE; + quorum_block_bytes(blk) > SCOUTFS_BLOCK_SM_SIZE; } /* true if a is stale and should be ignored */ @@ -296,7 +296,8 @@ static int write_quorum_block(struct super_block *sb, size_t size; int ret; - BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SIZE); + BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > + SCOUTFS_BLOCK_SM_SIZE); bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS)); @@ -306,8 +307,7 @@ static int write_quorum_block(struct super_block *sb, } size = quorum_block_bytes(our_blk); - if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SIZE || - size > bh->b_size)) { + if (WARN_ON_ONCE(size > SCOUTFS_BLOCK_SM_SIZE || size > bh->b_size)) { ret = -EIO; goto out; } @@ -530,7 +530,7 @@ int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, trace_scoutfs_quorum_election(sb, prev_term); super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); - our_blk = kmalloc(SCOUTFS_BLOCK_SIZE, GFP_NOFS); + our_blk = kmalloc(SCOUTFS_BLOCK_SM_SIZE, GFP_NOFS); if (!super || !our_blk) { ret = -ENOMEM; goto out; @@ -548,7 +548,7 @@ int scoutfs_quorum_election(struct super_block *sb, ktime_t timeout_abs, SCOUTFS_QUORUM_TERM_HI_MS); for (;;) { - memset(our_blk, 0, SCOUTFS_BLOCK_SIZE); + memset(our_blk, 0, SCOUTFS_BLOCK_SM_SIZE); scoutfs_inc_counter(sb, quorum_cycle); diff --git a/kmod/src/radix.c b/kmod/src/radix.c index e8e39b42..f7b61483 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -83,7 +83,7 @@ * stubbed out refs that reference entirely empty or full subtrees. * They're moved to properly allocated blknos. */ -#define RADIX_SYNTH_BLKNO (SCOUTFS_BLOCK_MAX + 1) +#define RADIX_SYNTH_BLKNO (SCOUTFS_BLOCK_LG_MAX + 1) struct radix_path { struct rb_node node; @@ -763,7 +763,7 @@ static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, else memset(rdx->bits, 0, SCOUTFS_RADIX_BITS_BYTES); - tail = SCOUTFS_BLOCK_SIZE - + tail = SCOUTFS_BLOCK_LG_SIZE - offsetof(struct scoutfs_radix_block, bits) - SCOUTFS_RADIX_BITS_BYTES; } else { @@ -772,14 +772,14 @@ static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, for (i = 0; i < SCOUTFS_RADIX_REFS; i++) memcpy(&rdx->refs[i], &ref, sizeof(ref)); - tail = SCOUTFS_BLOCK_SIZE - + tail = SCOUTFS_BLOCK_LG_SIZE - offsetof(struct scoutfs_radix_block, refs[SCOUTFS_RADIX_REFS]); } /* make sure we don't write uninitialized tail kernel memory to disk */ if (tail) - memset((void *)rdx + SCOUTFS_BLOCK_SIZE - tail, 0, tail); + memset((void *)rdx + SCOUTFS_BLOCK_LG_SIZE - tail, 0, tail); } /* get path flags */ @@ -1529,10 +1529,10 @@ void scoutfs_radix_root_init(struct super_block *sb, init_ref(&root->ref, 0, false); } -u64 scoutfs_radix_root_free_bytes(struct super_block *sb, - struct scoutfs_radix_root *root) +u64 scoutfs_radix_root_free_blocks(struct super_block *sb, + struct scoutfs_radix_root *root) { - return le64_to_cpu(root->ref.sm_total) << SCOUTFS_BLOCK_SHIFT; + return le64_to_cpu(root->ref.sm_total); } /* diff --git a/kmod/src/radix.h b/kmod/src/radix.h index 0ca79431..729e810b 100644 --- a/kmod/src/radix.h +++ b/kmod/src/radix.h @@ -38,8 +38,8 @@ void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, struct scoutfs_radix_root *freed); void scoutfs_radix_root_init(struct super_block *sb, struct scoutfs_radix_root *root, bool meta); -u64 scoutfs_radix_root_free_bytes(struct super_block *sb, - struct scoutfs_radix_root *root); +u64 scoutfs_radix_root_free_blocks(struct super_block *sb, + struct scoutfs_radix_root *root); u64 scoutfs_radix_bit_leaf_nr(u64 bit); #endif diff --git a/kmod/src/server.c b/kmod/src/server.c index f6db3179..b4097c65 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -400,7 +400,7 @@ static int server_get_log_trees(struct super_block *sb, goto unlock; /* ensure client has enough free metadata blocks for a transaction */ - target = (64*1024*1024) / SCOUTFS_BLOCK_SIZE; + target = (64*1024*1024) / SCOUTFS_BLOCK_LG_SIZE; if (le64_to_cpu(ltv.meta_avail.ref.sm_total) < target) { count = target - le64_to_cpu(ltv.meta_avail.ref.sm_total); @@ -413,7 +413,7 @@ static int server_get_log_trees(struct super_block *sb, } /* ensure client has enough free data blocks for a transaction */ - target = SCOUTFS_TRANS_DATA_ALLOC_HWM / SCOUTFS_BLOCK_SIZE; + target = SCOUTFS_TRANS_DATA_ALLOC_HWM / SCOUTFS_BLOCK_SM_SIZE; if (le64_to_cpu(ltv.data_avail.ref.sm_total) < target) { count = target - le64_to_cpu(ltv.data_avail.ref.sm_total); diff --git a/kmod/src/super.c b/kmod/src/super.c index 30cabd73..dc3ff01b 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -103,7 +103,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_bfree = le64_to_cpu(nstatfs.bfree); kst->f_type = SCOUTFS_SUPER_MAGIC; - kst->f_bsize = SCOUTFS_BLOCK_SIZE; + kst->f_bsize = SCOUTFS_BLOCK_SM_SIZE; kst->f_blocks = le64_to_cpu(nstatfs.total_blocks); kst->f_bavail = kst->f_bfree; @@ -115,7 +115,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); kst->f_namelen = SCOUTFS_NAME_LEN; - kst->f_frsize = SCOUTFS_BLOCK_SIZE; + kst->f_frsize = SCOUTFS_BLOCK_SM_SIZE; /* the vfs fills f_flags */ /* @@ -379,8 +379,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) sbi->opts = opts; - ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SIZE); - if (ret != SCOUTFS_BLOCK_SIZE) { + ret = sb_set_blocksize(sb, SCOUTFS_BLOCK_SM_SIZE); + if (ret != SCOUTFS_BLOCK_SM_SIZE) { scoutfs_err(sb, "failed to set blocksize, returned %d", ret); ret = -EIO; goto out; From 304dbbbafab11b97c5b7a998e523391624e03d55 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 1 May 2020 16:20:45 -0700 Subject: [PATCH 829/920] scoutfs: merge partial allocator blocks The server fills radix allocators for the client to consume while allocating during a transaction. The radix merge function used to move an entire radix block at a time. With larger blocks this becomes much too coarse and can move way too much in one call. This moves allocator bits a word at a time and more precisely moves the amount that the caller asked for. Signed-off-by: Zach Brown --- kmod/src/radix.c | 69 ++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index f7b61483..a3d10272 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -247,20 +247,34 @@ static void bitmap_set_le(__le64 *map, int ind, int nbits) } /* - * xor the destination bitmap with the source. bitmap_xor() requires 2 - * const inputs so I'm not comfortable giving it the changing - * destination pointer as one of the const input pointers. + * xor at least nbits total dst bits with set src bits, a full word at a + * time, starting around the given starting index. The src and dst + * pointers can be to the same bitmap. We might xor bits before the + * starting index and might xor a bit more than nbits because we're + * working an __le64 at a time. Return the total amount xored and + * set the caller's size that includes the last word we modified. */ -static void bitmap_xor_bitmap_le(__le64 *dst, __le64 *src, int nbits) +static int bitmap_xor_bitmap_le(__le64 *dst, __le64 *src, int ind, int nbits, + int *size) { + int xored = 0; int i; BUG_ON((unsigned long)src & 7); BUG_ON((unsigned long)dst & 7); - BUG_ON(nbits & 63); - for (i = 0; i < nbits; i += 64) - *(dst++) ^= *(src++); + while (xored < nbits && + (ind = find_next_bit_le(src, SCOUTFS_RADIX_BITS, ind)) < + SCOUTFS_RADIX_BITS) { + i = ind / 64; + xored += hweight64((u64 __force)src[i]); + dst[i] = dst[i] ^ src[i]; + ind = round_up(ind + 1, 64); + if (size) + *size = ind; + } + + return xored; } static void bitmap_clear_le(__le64 *map, int ind, int nbits) @@ -334,13 +348,11 @@ static u64 count_lg_bits(void *bits, int ind, int nbits) * count the number of bits in corresponding large regions that are * fully set in the result bitmap. */ -static u64 count_lg_bitmap(void *result, void *input) +static u64 count_lg_from_set(void *result, void *input, int ind, int size) { u64 count = 0; - int ind = 0; - while ((ind = find_next_bit_le(input, SCOUTFS_RADIX_BITS, ind)) - < SCOUTFS_RADIX_BITS) { + while ((ind = find_next_bit_le(input, size, ind)) < size) { if (lg_is_full(result, ind)) count += SCOUTFS_RADIX_LG_BITS; ind = round_up(ind + 1, SCOUTFS_RADIX_LG_BITS); @@ -1347,8 +1359,8 @@ int scoutfs_radix_free_data(struct super_block *sb, * * The caller specifies the minimum count to move. -ENOENT will be * returned if the source tree runs out of bits, potentially after - * having already moved bits. More than the minimum can be moved - * because whole leaves worth of bits are moved. + * having already moved bits. Up to 63 bits more than the minimum can + * be moved because bits are manipulated in chunks of 64 bits. * * This is pretty expensive because it fully references full leaf blocks * a few times. It could be more efficient if it short circuited walks @@ -1371,8 +1383,10 @@ int scoutfs_radix_merge(struct super_block *sb, struct radix_path *dst_path; s64 src_lg_delta; s64 dst_lg_delta; - s64 sm_delta; u64 bit; + int merge_size; + int merged; + int inp_sm; int lg_ind; int ind; int ret; @@ -1432,13 +1446,13 @@ wrapped: src_rdx = src_path->bls[0]->data; dst_rdx = dst_path->bls[0]->data; - sm_delta = le64_to_cpu(path_ref(inp_path, 0)->sm_total); + inp_sm = le64_to_cpu(path_ref(inp_path, 0)->sm_total); ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, le32_to_cpu(inp_rdx->sm_first)); lg_ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); /* back out and retry if no input left, or inp not ro */ - if (sm_delta == 0 || + if (inp_sm == 0 || (inp != src && paths_share_blocks(inp_path, src_path))) { free_path(sb, inp_path); inp_path = NULL; @@ -1465,13 +1479,16 @@ wrapped: } /* carefully modify src last, it might also be inp */ - bitmap_xor_bitmap_le(dst_rdx->bits, inp_rdx->bits, - SCOUTFS_RADIX_BITS); - dst_lg_delta = count_lg_bitmap(dst_rdx->bits, inp_rdx->bits); + merged = bitmap_xor_bitmap_le(dst_rdx->bits, inp_rdx->bits, + ind, min_t(u64, inp_sm, count), + &merge_size); + dst_lg_delta = count_lg_from_set(dst_rdx->bits, inp_rdx->bits, + ind, merge_size); - src_lg_delta = count_lg_bitmap(src_rdx->bits, inp_rdx->bits); - bitmap_xor_bitmap_le(src_rdx->bits, inp_rdx->bits, - SCOUTFS_RADIX_BITS); + src_lg_delta = count_lg_from_set(src_rdx->bits, inp_rdx->bits, + ind, merge_size); + bitmap_xor_bitmap_le(src_rdx->bits, inp_rdx->bits, ind, merged, + NULL); if (ind < le32_to_cpu(dst_rdx->sm_first)) dst_rdx->sm_first = cpu_to_le32(ind); @@ -1479,13 +1496,13 @@ wrapped: if (lg_ind < le32_to_cpu(dst_rdx->lg_first)) dst_rdx->lg_first = cpu_to_le32(lg_ind); - fixup_parent_refs(src_path, -sm_delta, -src_lg_delta); - fixup_parent_refs(dst_path, sm_delta, dst_lg_delta); + fixup_parent_refs(src_path, -merged, -src_lg_delta); + fixup_parent_refs(dst_path, merged, dst_lg_delta); trace_scoutfs_radix_merge(sb, inp, inp_path->bls[0]->blkno, src, src_path->bls[0]->blkno, dst, dst_path->bls[0]->blkno, count, - bit, ind, sm_delta, src_lg_delta, + bit, ind, merged, src_lg_delta, dst_lg_delta); free_path(sb, inp_path); @@ -1494,7 +1511,7 @@ wrapped: chg = NULL; store_next_find_bit(sb, meta, src, bit + SCOUTFS_RADIX_BITS); - count -= min_t(u64, count, sm_delta); + count -= min_t(u64, count, merged); } ret = 0; From b7943c5412ba7995f6205fac07cedf36a1fc652f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 May 2020 11:21:27 -0700 Subject: [PATCH 830/920] scoutfs: avoid reading forest roots with block IO The forest item operations were reading the super block to find the roots that it should read items from. This was easiest to implement to start, but it is too expensive. We have to find the roots for every newly acquired lock and every call to walk the inode seq indexes. To avoid all these reads we first send the current stable versions of the fs and logs btrees roots along with root grants. Then we add a net command to get the current stable roots from the server. This is used to refresh the roots if stale blocks are encountered and on the seq index queries. Signed-off-by: Zach Brown --- kmod/src/client.c | 20 ++++++- kmod/src/client.h | 3 ++ kmod/src/counters.h | 3 ++ kmod/src/forest.c | 114 ++++++++++++++++++++++----------------- kmod/src/format.h | 11 ++++ kmod/src/lock.c | 5 +- kmod/src/lock.h | 4 +- kmod/src/lock_server.c | 7 ++- kmod/src/scoutfs_trace.h | 23 ++++---- kmod/src/server.c | 64 ++++++++++++++++++++-- kmod/src/server.h | 7 ++- 11 files changed, 190 insertions(+), 71 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 83dbacab..53d47942 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -108,6 +108,24 @@ int scoutfs_client_commit_log_trees(struct super_block *sb, lt, sizeof(*lt), NULL, 0); } +int scoutfs_client_get_fs_roots(struct super_block *sb, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + struct scoutfs_net_fs_roots nfr; + int ret; + + ret = scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_FS_ROOTS, + NULL, 0, &nfr, sizeof(nfr)); + if (ret == 0) { + *fs_root = nfr.fs_root; + *logs_root = nfr.logs_root; + } + return 0; +} + int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) { struct client_info *client = SCOUTFS_SB(sb)->client_info; @@ -157,7 +175,7 @@ static int client_lock_response(struct super_block *sb, void *resp, unsigned int resp_len, int error, void *data) { - if (resp_len != sizeof(struct scoutfs_net_lock)) + if (resp_len != sizeof(struct scoutfs_net_lock_grant_response)) return -EINVAL; /* XXX error? */ diff --git a/kmod/src/client.h b/kmod/src/client.h index cb77c30c..cfc5e482 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -7,6 +7,9 @@ int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); int scoutfs_client_commit_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); +int scoutfs_client_get_fs_roots(struct super_block *sb, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root); u64 *scoutfs_client_bulk_alloc(struct super_block *sb); int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index ab4a9c15..74ec5147 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -42,6 +42,9 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ + EXPAND_COUNTER(forest_roots_next_hint) \ + EXPAND_COUNTER(forest_roots_lock) \ + EXPAND_COUNTER(forest_roots_server) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grace_elapsed) \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 9bad162e..5adf66e5 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -24,6 +24,7 @@ #include "radix.h" #include "block.h" #include "forest.h" +#include "counters.h" #include "scoutfs_trace.h" /* @@ -76,7 +77,7 @@ struct forest_root { u64 nr; }; -struct forest_super_refs { +struct forest_refs { struct scoutfs_btree_ref fs_ref; struct scoutfs_btree_ref logs_ref; } __packed; @@ -93,6 +94,7 @@ struct forest_bloom_nrs { struct forest_lock_private { u64 last_refreshed; struct rw_semaphore rwsem; + unsigned int used_lock_roots:1; struct list_head roots; struct forest_root fs_root; struct forest_root our_log_root; @@ -283,28 +285,29 @@ static struct scoutfs_block *read_bloom_ref(struct super_block *sb, * * This doesn't deal with rereading stale blocks itself.. it returns * ESTALE to the caller who already has to deal with retrying stale - * blocks from their btree reads. We give them the super refs we read - * so that they can identify persistent stale block errors that come - * from corruption. + * blocks from their btree reads. We give them the refs we read so that + * they can identify persistent stale block errors that come from + * corruption. * - * Because we're starting all the reads from a stable read super this - * will not see any dirty blocks we have in memory. We don't have to - * lock any of the btree reads. It also won't find the currently dirty - * version of our log btree. Writers mark our static log btree in lpriv - * to indicate that we should include our dirty log btree in reads. - * We'll also naturally add it if we see a persistent version on disk - * with all of the bloom bits set. + * Because we're starting all the reads from stable refs from the + * server, this will not see any dirty blocks we have in memory. We + * don't have to lock any of the btree reads. It also won't find the + * currently dirty version of our log btree. Writers mark our static + * log btree in lpriv to indicate that we should include our dirty log + * btree in reads. We'll also naturally add it if we see a persistent + * version on disk with all of the bloom bits set. */ static int refresh_bloom_roots(struct super_block *sb, struct scoutfs_lock *lock, - struct forest_super_refs *srefs) + struct forest_refs *refs) { DECLARE_FOREST_INFO(sb, finf); struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct forest_bloom_nrs bloom; - struct scoutfs_super_block super; struct forest_root *fr = NULL; struct scoutfs_bloom_block *bb; struct scoutfs_block *bl; @@ -312,26 +315,36 @@ static int refresh_bloom_roots(struct super_block *sb, int ret; int i; - memset(srefs, 0, sizeof(*srefs)); + memset(refs, 0, sizeof(*refs)); + + down_write(&lpriv->rwsem); /* empty the list so no one iterates until someone's added */ clear_roots(lpriv); - ret = scoutfs_read_super(sb, &super); - if (ret) - goto out; + /* first use the lock's constant roots, then sample newer roots */ + if (!lpriv->used_lock_roots) { + lpriv->used_lock_roots = 1; + fs_root = lock->fs_root; + logs_root = lock->logs_root; + scoutfs_inc_counter(sb, forest_roots_lock); + } else { + ret = scoutfs_client_get_fs_roots(sb, &fs_root, &logs_root); + if (ret) + goto out; + scoutfs_inc_counter(sb, forest_roots_server); + } - trace_scoutfs_forest_read_super(sb, &super); - - srefs->fs_ref = super.fs_root.ref; - srefs->logs_ref = super.logs_root.ref; + trace_scoutfs_forest_using_roots(sb, &fs_root, &logs_root); + refs->fs_ref = fs_root.ref; + refs->logs_ref = logs_root.ref; calc_bloom_nrs(&bloom, &lock->start); scoutfs_key_init_log_trees(&key, 0, 0); for (;; scoutfs_key_inc(&key)) { - ret = scoutfs_btree_next(sb, &super.logs_root, &key, &iref); + ret = scoutfs_btree_next(sb, &logs_root, &key, &iref); if (ret == -ENOENT) { ret = 0; break; @@ -408,7 +421,7 @@ static int refresh_bloom_roots(struct super_block *sb, /* always add the fs root at the tail */ fr = &lpriv->fs_root; - fr->item_root = super.fs_root; + fr->item_root = fs_root; fr->rid = 0; fr->nr = 0; list_add_tail(&fr->entry, &lpriv->roots); @@ -420,14 +433,15 @@ static int refresh_bloom_roots(struct super_block *sb, out: if (ret < 0) clear_roots(lpriv); + + up_write(&lpriv->rwsem); return ret; } -/* initialize some super refs that initially aren't equal */ -#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ - struct forest_super_refs a = {{cpu_to_le64(0),}}; \ - struct forest_super_refs b = {{cpu_to_le64(1),}} - +/* initialize some refs that initially aren't equal */ +#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ + struct forest_refs a = {{cpu_to_le64(0),}}; \ + struct forest_refs b = {{cpu_to_le64(1),}} /* * The caller saw stale blocks. If they're seeing the same root refs @@ -439,19 +453,16 @@ out: */ static int refresh_check_stale(struct super_block *sb, struct scoutfs_lock *lock, - struct forest_super_refs *prev_srefs, - struct forest_super_refs *srefs) + struct forest_refs *prev_refs, + struct forest_refs *refs) { - struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); int ret; - if (memcmp(prev_srefs, srefs, sizeof(*srefs)) == 0) + if (memcmp(prev_refs, refs, sizeof(*refs)) == 0) return -EIO; - *prev_srefs = *srefs; + *prev_refs = *refs; - down_write(&lpriv->rwsem); - ret = refresh_bloom_roots(sb, lock, srefs); - up_write(&lpriv->rwsem); + ret = refresh_bloom_roots(sb, lock, refs); if (ret == -ESTALE) ret = 0; @@ -563,7 +574,7 @@ int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock) { DECLARE_FOREST_INFO(sb, finf); - DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); struct forest_lock_private *lpriv; SCOUTFS_BTREE_ITEM_REF(iref); struct forest_root *fr; @@ -625,7 +636,7 @@ retry: up_read(&lpriv->rwsem); if (err == -ESTALE) { - err = refresh_check_stale(sb, lock, &prev_srefs, &srefs); + err = refresh_check_stale(sb, lock, &prev_refs, &refs); if (err == 0) goto retry; ret = err; @@ -823,7 +834,7 @@ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *end, struct kvec *val, struct scoutfs_lock *lock, bool fwd) { - DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); struct forest_lock_private *lpriv; DECLARE_FOREST_INFO(sb, finf); SCOUTFS_BTREE_ITEM_REF(iref); @@ -964,7 +975,7 @@ unlock: } if (ret == -ESTALE) { - ret = refresh_check_stale(sb, lock, &prev_srefs, &srefs); + ret = refresh_check_stale(sb, lock, &prev_refs, &refs); if (ret == 0) goto retry; } @@ -1001,8 +1012,8 @@ int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, * This is an unlocked iteration across all the btrees to find a hint at * the next key that the caller could read. It's used to find out what * next key range to lock, presuming you're allowed to only see items - * that have been synced. We read the super every time to get the most - * recent trees. + * that have been synced. We ask the server for the current roots to + * check. * * We don't bother skipping deletion or reservation items here. They're * unlikely. The caller will iterate them over safely and call again to @@ -1014,8 +1025,9 @@ int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next) { - DECLARE_STALE_TRACKING_SUPER_REFS(prev_srefs, srefs); - struct scoutfs_super_block super; + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key found; @@ -1024,19 +1036,21 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, int ret; retry: - ret = scoutfs_read_super(sb, &super); + scoutfs_inc_counter(sb, forest_roots_next_hint); + ret = scoutfs_client_get_fs_roots(sb, &fs_root, &logs_root); if (ret) goto out; - srefs.fs_ref = super.fs_root.ref; - srefs.logs_ref = super.logs_root.ref; + trace_scoutfs_forest_using_roots(sb, &fs_root, &logs_root); + refs.fs_ref = fs_root.ref; + refs.logs_ref = logs_root.ref; scoutfs_key_init_log_trees(<k, 0, 0); have_next = false; for (;; scoutfs_key_inc(<k)) { - ret = scoutfs_btree_next(sb, &super.logs_root, <k, &iref); + ret = scoutfs_btree_next(sb, &logs_root, <k, &iref); if (ret == -ENOENT) { if (have_next) ret = 0; @@ -1075,9 +1089,9 @@ retry: } if (ret == -ESTALE) { - if (memcmp(&prev_srefs, &srefs, sizeof(srefs)) == 0) + if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0) return -EIO; - prev_srefs = srefs; + prev_refs = refs; goto retry; } out: diff --git a/kmod/src/format.h b/kmod/src/format.h index 6c667c97..dc83c506 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -683,6 +683,7 @@ enum { SCOUTFS_NET_CMD_ALLOC_INODES, SCOUTFS_NET_CMD_GET_LOG_TREES, SCOUTFS_NET_CMD_COMMIT_LOG_TREES, + SCOUTFS_NET_CMD_GET_FS_ROOTS, SCOUTFS_NET_CMD_ADVANCE_SEQ, SCOUTFS_NET_CMD_GET_LAST_SEQ, SCOUTFS_NET_CMD_STATFS, @@ -731,6 +732,11 @@ struct scoutfs_net_statfs { __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; +struct scoutfs_net_fs_roots { + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; +} __packed; + struct scoutfs_net_lock { struct scoutfs_key key; __le64 write_version; @@ -738,6 +744,11 @@ struct scoutfs_net_lock { __u8 new_mode; } __packed; +struct scoutfs_net_lock_grant_response { + struct scoutfs_net_lock nl; + struct scoutfs_net_fs_roots nfr; +} __packed; + struct scoutfs_net_lock_recover { __le16 nr; struct scoutfs_net_lock locks[0]; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 1faa8df0..69a593e2 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -556,8 +556,9 @@ static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) * period anyway as they unlock. */ int scoutfs_lock_grant_response(struct super_block *sb, - struct scoutfs_net_lock *nl) + struct scoutfs_net_lock_grant_response *gr) { + struct scoutfs_net_lock *nl = &gr->nl; DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; @@ -589,6 +590,8 @@ int scoutfs_lock_grant_response(struct super_block *sb, lock->request_pending = 0; lock->mode = nl->new_mode; lock->write_version = le64_to_cpu(nl->write_version); + lock->fs_root = gr->nfr.fs_root; + lock->logs_root = gr->nfr.logs_root; if (lock_count_match_exists(nl->new_mode, lock->waiters)) extend_grace(sb, lock); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index a6f8a688..971a12ff 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -22,6 +22,8 @@ struct scoutfs_lock { struct rb_node range_node; u64 refresh_gen; u64 write_version; + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; struct list_head lru_head; wait_queue_head_t waitq; struct work_struct shrink_work; @@ -49,7 +51,7 @@ struct scoutfs_lock_coverage { }; int scoutfs_lock_grant_response(struct super_block *sb, - struct scoutfs_net_lock *nl); + struct scoutfs_net_lock_grant_response *gr); int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, struct scoutfs_net_lock *nl); int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 117d163e..3d4cbeea 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -489,6 +489,7 @@ static int process_waiting_requests(struct super_block *sb, struct server_lock_node *snode) { DECLARE_LOCK_SERVER_INFO(sb, inf); + struct scoutfs_net_lock_grant_response gres; struct scoutfs_net_lock nl; struct client_lock_entry *req; struct client_lock_entry *req_tmp; @@ -552,8 +553,12 @@ static int process_waiting_requests(struct super_block *sb, nl.write_version = cpu_to_le64(wv); } + gres.nl = nl; + scoutfs_server_get_fs_roots(sb, &gres.nfr.fs_root, + &gres.nfr.logs_root); + ret = scoutfs_server_lock_response(sb, req->rid, - req->net_id, &nl); + req->net_id, &gres); if (ret) goto out; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index b87a5443..0fad61cb 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2044,12 +2044,12 @@ TRACE_EVENT(scoutfs_forest_prepare_commit, __entry->bloom_blkno, __entry->bloom_seq) ); -TRACE_EVENT(scoutfs_forest_read_super, - TP_PROTO(struct super_block *sb, struct scoutfs_super_block *super), - TP_ARGS(sb, super), +TRACE_EVENT(scoutfs_forest_using_roots, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root), + TP_ARGS(sb, fs_root, logs_root), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, hdr_seq) __field(__u64, fs_blkno) __field(__u64, fs_seq) __field(__u64, logs_blkno) @@ -2057,15 +2057,14 @@ TRACE_EVENT(scoutfs_forest_read_super, ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->hdr_seq = le64_to_cpu(super->hdr.seq); - __entry->fs_blkno = le64_to_cpu(super->fs_root.ref.blkno); - __entry->fs_seq = le64_to_cpu(super->fs_root.ref.seq); - __entry->logs_blkno = le64_to_cpu(super->logs_root.ref.blkno); - __entry->logs_seq = le64_to_cpu(super->logs_root.ref.seq); + __entry->fs_blkno = le64_to_cpu(fs_root->ref.blkno); + __entry->fs_seq = le64_to_cpu(fs_root->ref.seq); + __entry->logs_blkno = le64_to_cpu(logs_root->ref.blkno); + __entry->logs_seq = le64_to_cpu(logs_root->ref.seq); ), - TP_printk(SCSBF" hdr seq %llu fs blkno %llu seq %llu logs blkno %llu seq %llu", - SCSB_TRACE_ARGS, __entry->hdr_seq, __entry->fs_blkno, - __entry->fs_seq, __entry->logs_blkno, __entry->logs_seq) + TP_printk(SCSBF" fs blkno %llu seq %llu logs blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->fs_blkno, __entry->fs_seq, + __entry->logs_blkno, __entry->logs_seq) ); TRACE_EVENT(scoutfs_forest_add_root, diff --git a/kmod/src/server.c b/kmod/src/server.c index b4097c65..4ff46fc6 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -84,6 +84,11 @@ struct server_info { struct scoutfs_block_writer wri; struct mutex logs_mutex; + + /* stable versions stored from commits, given in locks and rpcs */ + seqcount_t fs_roots_seqcount; + struct scoutfs_btree_root fs_root; + struct scoutfs_btree_root logs_root; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -216,6 +221,32 @@ static void update_free_blocks(__le64 *blocks, struct scoutfs_radix_root *prev, le64_to_cpu(prev->ref.sm_total)); } +void scoutfs_server_get_fs_roots(struct super_block *sb, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root) +{ + DECLARE_SERVER_INFO(sb, server); + unsigned int seq; + + do { + seq = read_seqcount_begin(&server->fs_roots_seqcount); + *fs_root = server->fs_root; + *logs_root = server->logs_root; + } while (read_seqcount_retry(&server->fs_roots_seqcount, seq)); +} + +static void set_fs_roots(struct server_info *server, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root) +{ + preempt_disable(); + write_seqcount_begin(&server->fs_roots_seqcount); + server->fs_root = *fs_root; + server->logs_root = *logs_root; + write_seqcount_end(&server->fs_roots_seqcount); + preempt_enable(); +} + /* * Concurrent request processing dirties blocks in a commit and makes * the modifications persistent before replying. We'd like to batch @@ -270,6 +301,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) } server->prepared_commit = false; + set_fs_roots(server, &super->fs_root, &super->logs_root); ret = 0; out: node = llist_del_all(&server->commit_waiters); @@ -534,6 +566,29 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); } +/* + * Give the client the most recent version of the fs btrees that are + * visible in persistent storage. We don't want to accidentally give + * them our in-memory dirty version. This can be racing with commits. + */ +static int server_get_fs_roots(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + struct scoutfs_net_fs_roots nfr; + int ret; + + if (arg_len != 0) { + memset(&nfr, 0, sizeof(nfr)); + ret = -EINVAL; + } else { + scoutfs_server_get_fs_roots(sb, &nfr.fs_root, &nfr.logs_root); + ret = 0; + } + + return scoutfs_net_response(sb, conn, cmd, id, 0, &nfr, sizeof(nfr)); +} + /* * A client is being evicted so we want to reclaim resources from their * log tree items. The item trees and bloom refs stay around to be read @@ -863,14 +918,14 @@ int scoutfs_server_lock_request(struct super_block *sb, u64 rid, lock_response, NULL, NULL); } -int scoutfs_server_lock_response(struct super_block *sb, u64 rid, - u64 id, struct scoutfs_net_lock *nl) +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_net_lock_grant_response *gr) { struct server_info *server = SCOUTFS_SB(sb)->server_info; return scoutfs_net_response_node(sb, server->conn, rid, SCOUTFS_NET_CMD_LOCK, id, 0, - nl, sizeof(*nl)); + gr, sizeof(*gr)); } static bool invalid_recover(struct scoutfs_net_lock_recover *nlr, @@ -1328,6 +1383,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, + [SCOUTFS_NET_CMD_GET_FS_ROOTS] = server_get_fs_roots, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, [SCOUTFS_NET_CMD_STATFS] = server_statfs, @@ -1418,6 +1474,7 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret < 0) goto shutdown; + set_fs_roots(server, &super->fs_root, &super->logs_root); scoutfs_radix_init_alloc(&server->alloc, &super->core_meta_avail, &super->core_meta_freed); scoutfs_block_writer_init(sb, &server->wri); @@ -1557,6 +1614,7 @@ int scoutfs_server_setup(struct super_block *sb) INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); mutex_init(&server->logs_mutex); + seqcount_init(&server->fs_roots_seqcount); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/server.h b/kmod/src/server.h index 0bc92ab8..07e95606 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -58,12 +58,15 @@ do { \ int scoutfs_server_lock_request(struct super_block *sb, u64 rid, struct scoutfs_net_lock *nl); -int scoutfs_server_lock_response(struct super_block *sb, u64 rid, - u64 id, struct scoutfs_net_lock *nl); +int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, + struct scoutfs_net_lock_grant_response *gr); int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); int scoutfs_server_hold_commit(struct super_block *sb); int scoutfs_server_apply_commit(struct super_block *sb, int err); +void scoutfs_server_get_fs_roots(struct super_block *sb, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root); struct sockaddr_in; struct scoutfs_quorum_elected_info; From ca8abeebb186e316f60364d0c99eb25dbd3fa740 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 May 2020 11:31:20 -0700 Subject: [PATCH 831/920] scoutfs: check fs root in forest hint The forst code has a hint call to gives iterators a place to start reading from before they acquire locks. It was checking all the log trees but it wasn't checking the main fs tree. This happened to be OK today because we're not yet merging items from the log trees into the main fs tree, but we don't want to miss them once we do start merging the trees. Signed-off-by: Zach Brown --- kmod/src/forest.c | 64 ++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 5adf66e5..7dc863ad 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -1015,9 +1015,9 @@ int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, * that have been synced. We ask the server for the current roots to * check. * - * We don't bother skipping deletion or reservation items here. They're - * unlikely. The caller will iterate them over safely and call again to - * find the next hint after them. + * We don't bother skipping deletion items here. The caller will safely + * skip over them when really reading from their locked region and will + * call again after them to find the next hint. * * We're reading from stable persistent trees so we don't need to lock * against writers, their writes are cow into free blocks. @@ -1028,10 +1028,12 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; - struct scoutfs_log_trees_val ltv; + struct scoutfs_btree_root item_root; + struct scoutfs_log_trees_val *ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key found; struct scoutfs_key ltk; + bool checked_fs; bool have_next; int ret; @@ -1046,32 +1048,42 @@ retry: refs.logs_ref = logs_root.ref; scoutfs_key_init_log_trees(<k, 0, 0); + checked_fs = false; have_next = false; - for (;; scoutfs_key_inc(<k)) { - - ret = scoutfs_btree_next(sb, &logs_root, <k, &iref); - if (ret == -ENOENT) { - if (have_next) - ret = 0; - break; - } - if (ret == -ESTALE) - break; - if (ret < 0) - goto out; - - if (iref.val_len == sizeof(ltv)) { - ltk = *iref.key; - memcpy(<v, iref.val, iref.val_len); + for (;;) { + if (!checked_fs) { + checked_fs = true; + item_root = fs_root; } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); - if (ret < 0) - goto out; + ret = scoutfs_btree_next(sb, &logs_root, <k, &iref); + if (ret == -ENOENT) { + if (have_next) + ret = 0; + break; + } + if (ret == -ESTALE) + break; + if (ret < 0) + goto out; - ret = scoutfs_btree_next(sb, <v.item_root, key, &iref); + if (iref.val_len == sizeof(*ltv)) { + ltk = *iref.key; + scoutfs_key_inc(<k); + ltv = iref.val; + item_root = ltv->item_root; + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + if (item_root.ref.blkno == 0) + continue; + } + + ret = scoutfs_btree_next(sb, &item_root, key, &iref); if (ret == -ENOENT) continue; if (ret == -ESTALE) From 26ccaca80bcbb1916d2135e93e0c4e7693177062 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 May 2020 13:30:10 -0700 Subject: [PATCH 832/920] scoutfs: add commit written counter Signed-off-by: Zach Brown --- kmod/src/counters.h | 5 +++-- kmod/src/trans.c | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 74ec5147..acfe5c7a 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -95,10 +95,11 @@ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ EXPAND_COUNTER(trans_commit_sync_fs) \ - EXPAND_COUNTER(trans_commit_timer) + EXPAND_COUNTER(trans_commit_timer) \ + EXPAND_COUNTER(trans_commit_written) #define FIRST_COUNTER block_cache_access -#define LAST_COUNTER trans_commit_timer +#define LAST_COUNTER trans_commit_written #undef EXPAND_COUNTER #define EXPAND_COUNTER(which) struct percpu_counter which; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index a3467aa6..11522c0a 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -169,6 +169,8 @@ void scoutfs_trans_write_func(struct work_struct *work) if (sbi->trans_deadline_expired) scoutfs_inc_counter(sb, trans_commit_timer); + scoutfs_inc_counter(sb, trans_commit_written); + ret = scoutfs_inode_walk_writeback(sb, true) ?: scoutfs_block_writer_write(sb, &tri->wri) ?: scoutfs_inode_walk_writeback(sb, false) ?: From 6d7b8233c628ad6b6a3746a76b4dcbef65b17637 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 7 May 2020 13:49:44 -0700 Subject: [PATCH 833/920] scoutfs: add radix merge retry counter Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/radix.c | 1 + 2 files changed, 2 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index acfe5c7a..06314b40 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -91,6 +91,7 @@ EXPAND_COUNTER(radix_enospc_data) \ EXPAND_COUNTER(radix_enospc_paths) \ EXPAND_COUNTER(radix_enospc_synth) \ + EXPAND_COUNTER(radix_merge_retry) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ diff --git a/kmod/src/radix.c b/kmod/src/radix.c index a3d10272..c8940d3f 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1454,6 +1454,7 @@ wrapped: /* back out and retry if no input left, or inp not ro */ if (inp_sm == 0 || (inp != src && paths_share_blocks(inp_path, src_path))) { + scoutfs_inc_counter(sb, radix_merge_retry); free_path(sb, inp_path); inp_path = NULL; free_change(sb, chg); From 8fe683dab8990f14b23f3f57b91caf1a6bbcb4fa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 May 2020 15:56:34 -0700 Subject: [PATCH 834/920] scoutfs: cow dirty radix blocks instead of moving The radix allocator has to be careful to not get lost in recursion trying to allocate metadata blocks for its dirty radix blocks while allocating metadata blocks for others. The first pass had used path data structures to record the references to all the blocks we'd need to modify to reflect the frees and allocations performed while dirtying radix blocks. Once it had all the path blocks it moved the old clean blocks into new dirty locations so that the dirtying couldn't fail. This had two very bad performance implications. First, it meant that trying to read clean versions of dirtied trees would always read the old blocks again because their clean version had been moved to the dirty version. Typically this wouldn't happen but the server does exactly this every time it tries to merge freed blocks back into its avail allocator. This created a significant IO load on the server. Secondly, that block cache move not being allowed to fail motivated us to move to a locked rbtree for the block cache instead of the lockless rcu radix_tree. This changes the recursion avoidance to use per-block private metadata to track every block that we allocate and cow rather than move. Each dirty block knows its parent ref and the blknos it would clear and set. If dirtying fails we can walk back through all the blocks we dirty and restore their original references before dropping all the dirty blocks and returning an error. This lets us get rid of the path structure entirely and results in a much cleaner system. This change meant tracking free blocks without clearing them as they're used to satisfy dirty block allocations. The change now has a cursor that walks the avail metadata tree without modifying it. While building this it became clear that tracking the first set bits of refs doesn't provide any value if we're always searching from a cursor. The cursor ends up providing the same value of avoiding constantly searching empty initial bits and refs. Maintaining the first metadata was just overhead. Signed-off-by: Zach Brown --- kmod/src/block.h | 1 + kmod/src/counters.h | 17 +- kmod/src/format.h | 2 - kmod/src/radix.c | 1467 ++++++++++++++++++-------------------- kmod/src/scoutfs_trace.h | 102 +-- 5 files changed, 743 insertions(+), 846 deletions(-) diff --git a/kmod/src/block.h b/kmod/src/block.h index 57e849a5..22e2437d 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -10,6 +10,7 @@ struct scoutfs_block_writer { struct scoutfs_block { u64 blkno; void *data; + void *priv; }; __le32 scoutfs_block_calc_crc(struct scoutfs_block_header *hdr, u32 size); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 06314b40..79da35cc 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -88,10 +88,23 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ + EXPAND_COUNTER(radix_alloc) \ + EXPAND_COUNTER(radix_alloc_data) \ + EXPAND_COUNTER(radix_block_cow) \ + EXPAND_COUNTER(radix_block_read) \ + EXPAND_COUNTER(radix_complete_dirty_block) \ + EXPAND_COUNTER(radix_create_synth) \ + EXPAND_COUNTER(radix_free) \ + EXPAND_COUNTER(radix_free_data) \ EXPAND_COUNTER(radix_enospc_data) \ - EXPAND_COUNTER(radix_enospc_paths) \ + EXPAND_COUNTER(radix_enospc_meta) \ EXPAND_COUNTER(radix_enospc_synth) \ - EXPAND_COUNTER(radix_merge_retry) \ + EXPAND_COUNTER(radix_inconsistent_eio) \ + EXPAND_COUNTER(radix_inconsistent_ref) \ + EXPAND_COUNTER(radix_merge) \ + EXPAND_COUNTER(radix_merge_empty) \ + EXPAND_COUNTER(radix_undo_ref) \ + EXPAND_COUNTER(radix_walk) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index dc83c506..8418a638 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -167,8 +167,6 @@ struct scoutfs_key { struct scoutfs_radix_block { struct scoutfs_block_header hdr; - __le32 sm_first; - __le32 lg_first; union { struct scoutfs_radix_ref { __le64 blkno; diff --git a/kmod/src/radix.c b/kmod/src/radix.c index c8940d3f..5d233c2a 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -52,9 +52,7 @@ * merge process. * * Allocations search for the next free bit from a cursor that's stored - * in the root of each tree. We track the first set parent ref or leaf - * bit in references to blocks to avoid searching entire blocks every - * time. + * in the root of each tree. * * The radix isn't always fully populated. References can contain * blknos with 0 or ~0 to indicate that its referenced subtree is either @@ -64,10 +62,14 @@ * descends. This lets mkfs initialize a tree with a large contigious * set region without having to populate all its blocks. * - * The radix is used to allocate and free blocks when performing cow - * updates of the blocks that make up radix itself. Recursion is - * carefully avoided by building up references to all the blocks needed - * for the operation and then dirtying and modifying them all at once. + * The metadata allocator radix tree itself is used to allocate and free + * its own blocks as it makes cow updates to itself. Recursion is + * avoided by tracking all the blocks we dirty with their parents, + * making sure we have dirty leaves to record frees and allocs for all + * the dirtied blocks, and using a read-only cursor to find blknos for + * each new dirty block. This lets us either atomically set and clear + * all the leaf bits once we have all the dirty blocks or unwind all the + * dirty blocks and restore their parent references. * * Radix block references contain totals of bits set in its referenced * subtree. This helps us balance the number of free bits stored across @@ -78,6 +80,16 @@ * tracked in the metadata allocator trees but aren't used. */ +/* + * This is just a sanity test at run time. It's log base + * SCOUTFS_RADIX_BITS of SCOUTFS_BLOCK_SM_MAX, but we can come close by + * dividing bit widths by shifts if we under-estimate the number of bits + * in a leaf by rounding it down to a power of two. In practice the + * trees are sized for the capacity of the device and are very short. + */ +#define RADIX_MAX_HEIGHT (((64 - SCOUTFS_BLOCK_SM_SHIFT) % \ + (SCOUTFS_BLOCK_LG_SHIFT + 2)) + 2) + /* * We create temporary synthetic blocks past possible blocks to populate * stubbed out refs that reference entirely empty or full subtrees. @@ -85,141 +97,49 @@ */ #define RADIX_SYNTH_BLKNO (SCOUTFS_BLOCK_LG_MAX + 1) -struct radix_path { - struct rb_node node; - struct list_head head; - struct list_head alloc_head; - u8 height; - struct scoutfs_radix_root *root; - u64 leaf_bit; - /* path and index arrays indexed by level, [0] is leaf */ - struct scoutfs_block **bls; - unsigned int *inds; +static bool is_synth(u64 blkno) +{ + return blkno >= RADIX_SYNTH_BLKNO; +} + +/* we use fake blknos to indicate subtrees either entirely empty or full */ +static bool is_stub(u64 blkno) +{ + return blkno == 0 || blkno == U64_MAX; +} + +struct radix_block_private { + struct scoutfs_block *bl; + struct list_head entry; + struct list_head dirtied_entry; + struct scoutfs_block *parent; + struct scoutfs_radix_ref *ref; + struct scoutfs_radix_ref orig_ref; + struct scoutfs_block *blkno_bl; + struct scoutfs_block *old_blkno_bl; + int blkno_ind; + int old_blkno_ind; }; +static bool was_dirtied(struct radix_block_private *priv) +{ + return !list_empty(&priv->dirtied_entry); +} + struct radix_change { - struct list_head paths; - struct list_head new_paths; - struct list_head alloc_paths; - struct rb_root rbroot; - u64 block_allocs; - u64 caller_allocs; - u64 alloc_bits; + struct scoutfs_radix_root *avail; + struct list_head blocks; + struct list_head dirtied_blocks; u64 next_synth; + u64 next_find_bit; + u64 first_free; + struct scoutfs_block *free_bl; + u64 free_leaf_bit; + unsigned int free_ind; }; -static struct radix_path *alloc_path(struct scoutfs_radix_root *root) -{ - struct radix_path *path; - u8 height = root->height; - - path = kzalloc(sizeof(struct radix_path) + - (member_sizeof(struct radix_path, inds[0]) * height) + - (member_sizeof(struct radix_path, bls[0]) * height), - GFP_NOFS); - if (path) { - RB_CLEAR_NODE(&path->node); - INIT_LIST_HEAD(&path->head); - INIT_LIST_HEAD(&path->alloc_head); - path->height = root->height; - path->root = root; - path->bls = (void *)(path + 1); - path->inds = (void *)(&path->bls[height]); - } - return path; -} - -/* Return a pointer to a reference in the path to a block at the given level. */ -static struct scoutfs_radix_ref *path_ref(struct radix_path *path, int level) -{ - struct scoutfs_radix_block *rdx; - - BUG_ON(level < 0 || level >= path->height); - - if (level == path->height - 1) { - return &path->root->ref; - } else { - rdx = path->bls[level + 1]->data; - return &rdx->refs[path->inds[level + 1]]; - } -} - -static bool paths_share_blocks(struct radix_path *a, struct radix_path *b) -{ - int i; - - for (i = 0; i < min(a->height, b->height); i++) { - if (a->bls[i] == b->bls[i]) - return true; - } - - return false; -} - -/* - * Drop a path's reference to blocks and free its memory. If we still - * have synthetic blocks then we reset their references to the original - * empty or full blknos. Ref sequence numbers aren't updated when we - * initially reference synthetic blocks. - */ -static void free_path(struct super_block *sb, struct radix_path *path) -{ - struct scoutfs_radix_ref *ref; - struct scoutfs_block *bl; - __le64 orig; - int i; - - if (!IS_ERR_OR_NULL(path)) { - for (i = 0; i < path->height; i++) { - bl = path->bls[i]; - if (bl == NULL) - continue; - - if (bl->blkno >= RADIX_SYNTH_BLKNO) { - ref = path_ref(path, i); - if (bl->blkno & 1) - orig = cpu_to_le64(U64_MAX); - else - orig = 0; - - if (ref->blkno != orig) - ref->blkno = orig; - } - scoutfs_block_put(sb, bl); - } - kfree(path); - } -} - -static struct radix_change *alloc_change(void) -{ - struct radix_change *chg; - - chg = kzalloc(sizeof(struct radix_change), GFP_NOFS); - if (chg) { - INIT_LIST_HEAD(&chg->paths); - INIT_LIST_HEAD(&chg->new_paths); - INIT_LIST_HEAD(&chg->alloc_paths); - chg->rbroot = RB_ROOT; - chg->next_synth = RADIX_SYNTH_BLKNO; - } - return chg; -} - -static void free_change(struct super_block *sb, struct radix_change *chg) -{ - struct radix_path *path; - struct radix_path *tmp; - - if (!IS_ERR_OR_NULL(chg)) { - list_splice_init(&chg->new_paths, &chg->paths); - list_for_each_entry_safe(path, tmp, &chg->paths, head) { - list_del_init(&path->head); - free_path(sb, path); - } - kfree(chg); - } -} +#define DECLARE_RADIX_CHANGE(a) \ + struct radix_change a = {NULL, } /* * We can use native longs to set full aligned regions, but we have to @@ -375,14 +295,14 @@ static int find_next_lg(__le64 *map, int ind) return SCOUTFS_RADIX_BITS; } -static u64 bit_from_inds(struct radix_path *path) +static u64 bit_from_inds(u32 *level_inds, u8 height) { - u64 bit = path->inds[0]; + u64 bit = level_inds[0]; u64 mult = SCOUTFS_RADIX_BITS; int i; - for (i = 1; i < path->height; i++) { - bit += (u64)path->inds[i] * mult; + for (i = 1; i < height; i++) { + bit += (u64)level_inds[i] * mult; mult *= SCOUTFS_RADIX_REFS; } @@ -428,17 +348,17 @@ static u64 full_subtree_total(int level) return total; } -static void calc_level_inds(struct radix_path *path, u64 bit) +static void calc_level_inds(u32 *level_inds, u8 height, u64 bit) { u32 ind; int i; bit = div_u64_rem(bit, SCOUTFS_RADIX_BITS, &ind); - path->inds[0] = ind; + level_inds[0] = ind; - for (i = 1; i < path->height; i++) { + for (i = 1; i < height; i++) { bit = div_u64_rem(bit, SCOUTFS_RADIX_REFS, &ind); - path->inds[i] = ind; + level_inds[i] = ind; } } @@ -450,279 +370,127 @@ static u64 calc_leaf_bit(u64 bit) return bit - ind; } -static int compare_path(struct scoutfs_radix_root *root, u64 leaf_bit, - struct radix_path *path) -{ - return scoutfs_cmp((unsigned long)root, (unsigned long)path->root) ?: - scoutfs_cmp(leaf_bit, path->leaf_bit); -} - -static struct radix_path *walk_paths(struct rb_root *rbroot, - struct scoutfs_radix_root *root, - u64 leaf_bit, struct radix_path *ins) -{ - struct rb_node **node = &rbroot->rb_node; - struct rb_node *parent = NULL; - struct radix_path *path; - int cmp; - - while (*node) { - parent = *node; - path = container_of(*node, struct radix_path, node); - - cmp = compare_path(root, leaf_bit, path); - if (cmp < 0) - node = &(*node)->rb_left; - else if (cmp > 0) - node = &(*node)->rb_right; - else - return path; - } - - if (ins) { - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, rbroot); - return ins; - } - - return NULL; -} - /* - * Make sure radix metadata is consistent. + * Make sure ref total tracking is correct after having modified a leaf + * and updated all the parent refs. */ -static void check_first_total(struct radix_path *path) +static void check_totals(struct scoutfs_block *leaf) { + struct radix_block_private *priv; + struct scoutfs_block *bl = leaf; struct scoutfs_radix_block *rdx; struct scoutfs_radix_ref *ref; int level; u64 st; u64 lt; - u32 sf; - u32 lf; int i; - for (level = 0; level < path->height; level++) { - rdx = path->bls[level]->data; - ref = path_ref(path, level); + for (level = 0; bl; level++, bl = priv->parent) { + priv = bl->priv; + rdx = bl->data; + ref = priv->ref; if (level == 0) { st = bitmap_weight((long *)rdx->bits, SCOUTFS_RADIX_BITS); lt = count_lg_bits(rdx->bits, 0, SCOUTFS_RADIX_BITS); - - sf = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, 0); - lf = find_next_lg(rdx->bits, 0); } else { st = 0; lt = 0; - sf = SCOUTFS_RADIX_REFS; - lf = SCOUTFS_RADIX_REFS; for (i = 0; i < SCOUTFS_RADIX_REFS; i++) { st += le64_to_cpu(rdx->refs[i].sm_total); lt += le64_to_cpu(rdx->refs[i].lg_total); - if (rdx->refs[i].sm_total != 0 && i < sf) - sf = i; - if (rdx->refs[i].lg_total != 0 && i < lf) - lf = i; } } if (le64_to_cpu(ref->sm_total) != st || - le64_to_cpu(ref->lg_total) != lt || - le32_to_cpu(rdx->sm_first) > sf || - le32_to_cpu(rdx->lg_first) > lf) { - printk("radix inconsistency: level %u calced sf %u st %llu lf %u lt %llu, stored sf %u st %llu lf %u lt %llu\n", - level, sf, st, lf, lt, - le32_to_cpu(rdx->sm_first), + le64_to_cpu(ref->lg_total) != lt) { + printk("radix inconsistency: level %u calced st %llu lt %llu, stored st %llu lt %llu\n", + level, st, lt, le64_to_cpu(ref->sm_total), - le32_to_cpu(rdx->lg_first), le64_to_cpu(ref->lg_total)); BUG(); } + + bl = priv->parent; } } -#define set_first_nonzero_ref(rdx, ind, first, total) \ -do { \ - int _ind = min_t(u32, le32_to_cpu(rdx->first), (ind)); \ - \ - while (_ind < SCOUTFS_RADIX_REFS && rdx->refs[_ind].total == 0) \ - _ind++; \ - \ - rdx->first = cpu_to_le32(_ind); \ -} while (0) - /* - * The caller has changed bits in a leaf block and updated the block's - * first tracking. We update the first tracking and totals in parent - * blocks and refs up to the root ref. We do this after modifying - * leaves, instead of during descent, because we descend through clean - * blocks and then dirty all he blocks in all the paths before modifying - * leaves. + * The caller has changed bits in a leaf block. We update the totals in + * rers up to the root ref. */ -static void fixup_parent_refs(struct radix_path *path, +static void fixup_parent_refs(struct super_block *sb, + struct scoutfs_block *leaf, s64 sm_delta, s64 lg_delta) { - struct scoutfs_radix_block *rdx; + struct radix_block_private *priv; struct scoutfs_radix_ref *ref; - int level; - int ind; + struct scoutfs_block *bl; - for (level = 0; level < path->height; level++) { - rdx = path->bls[level]->data; - ref = path_ref(path, level); + for (bl = leaf; bl; bl = priv->parent) { + priv = bl->priv; + ref = priv->ref; le64_add_cpu(&ref->sm_total, sm_delta); le64_add_cpu(&ref->lg_total, lg_delta); - if (level > 0) { - ind = path->inds[level]; - set_first_nonzero_ref(rdx, ind, sm_first, sm_total); - set_first_nonzero_ref(rdx, ind, lg_first, lg_total); - } } if (0) /* expensive, would be nice to make conditional */ - check_first_total(path); + check_totals(leaf); +} + +/* return 0 if the bit is past the last bit for the device */ +static u64 wrap_bit(struct super_block *sb, bool meta, u64 bit) +{ + return bit > last_from_super(sb, meta) ? 0 : bit; } static void store_next_find_bit(struct super_block *sb, bool meta, struct scoutfs_radix_root *root, u64 bit) { - if (bit > last_from_super(sb, meta)) - bit = 0; - root->next_find_bit = cpu_to_le64(bit); + root->next_find_bit = cpu_to_le64(wrap_bit(sb, meta, bit)); } -/* - * Allocate (clear and return) a region of bits from the leaf block of a - * path. The leaf walk has ensured that we have at least one block free. - * - * We always try to allocate smaller multi-block allocations from the - * start of the small region. This at least gets a single task extending - * a file one large extent. Multiple tasks extending writes will interleave. - * It'll do for now. - * - * We always search for free bits from the start of the leaf. - * This means that we can return recently freed blocks just behind the - * next free cursor. I'm not sure if that's much of a problem. - */ -static void alloc_leaf_bits(struct super_block *sb, bool meta, - struct radix_path *path, - int nbits, u64 *bit_ret, int *nbits_ret) +static void bug_on_bad_bits(int ind, int nbits) { - struct scoutfs_radix_block *rdx = path->bls[0]->data; - struct scoutfs_radix_ref *ref = path_ref(path, 0); - u32 sm_first; - u32 lg_first; + BUG_ON(ind < 0 || ind > SCOUTFS_RADIX_BITS); + BUG_ON(nbits < 0 || nbits > SCOUTFS_RADIX_BITS); + BUG_ON(ind + nbits > SCOUTFS_RADIX_BITS); +} + +static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, + int ind, int nbits) +{ + struct scoutfs_radix_block *rdx = bl->data; int lg_nbits; - int ind; - int end; - if (nbits >= SCOUTFS_RADIX_LG_BITS && ref->lg_total != 0) { - /* always allocate large allocs from full large regions */ - ind = le32_to_cpu(rdx->lg_first); - ind = find_next_lg(rdx->bits, ind); - sm_first = le32_to_cpu(rdx->sm_first); - lg_first = round_up(ind + nbits, SCOUTFS_RADIX_LG_BITS); + bug_on_bad_bits(ind, nbits); - } else { - /* otherwise alloc as much as we can from the next small */ - ind = le32_to_cpu(rdx->sm_first); - ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); + /* must never double-free bits */ + BUG_ON(!bitmap_empty_region_le(rdx->bits, ind, nbits)); + bitmap_set_le(rdx->bits, ind, nbits); + lg_nbits = count_lg_bits(rdx->bits, ind, nbits); - if (nbits > 1) { - end = find_next_zero_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); - nbits = min(nbits, end - ind); - } + fixup_parent_refs(sb, bl, nbits, lg_nbits); + trace_scoutfs_radix_set_bits(sb, bl->blkno, ind, nbits); +} - sm_first = ind + nbits; - lg_first = le32_to_cpu(rdx->lg_first); - } +static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, + int ind, int nbits) +{ + struct scoutfs_radix_block *rdx = bl->data; + int lg_nbits; - /* callers and structures should have ensured success */ - BUG_ON(ind >= SCOUTFS_RADIX_BITS); + bug_on_bad_bits(ind, nbits); + /* must never alloc in-use bits */ + BUG_ON(!bitmap_full_region_le(rdx->bits, ind, nbits)); lg_nbits = count_lg_bits(rdx->bits, ind, nbits); bitmap_clear_le(rdx->bits, ind, nbits); - /* always update the first we searched through */ - rdx->sm_first = cpu_to_le32(sm_first); - rdx->lg_first = cpu_to_le32(lg_first); - fixup_parent_refs(path, -nbits, -lg_nbits); - - *bit_ret = path->leaf_bit + ind; - *nbits_ret = nbits; - - store_next_find_bit(sb, meta, path->root, path->leaf_bit + ind + nbits); -} - -/* - * Allocate a metadata blkno for the caller from the leaves of paths - * which were stored in the change for metadata allocation. - */ -static u64 change_alloc_meta(struct super_block *sb, struct radix_change *chg) -{ - struct scoutfs_radix_ref *ref; - struct radix_path *path; - int nbits_ret; - u64 bit; - - path = list_first_entry_or_null(&chg->alloc_paths, struct radix_path, - alloc_head); - BUG_ON(!path); /* shouldn't be possible */ - - alloc_leaf_bits(sb, true, path, 1, &bit, &nbits_ret); - - /* remove the path from the alloc list once its empty */ - ref = path_ref(path, 0); - if (ref->sm_total == 0) - list_del_init(&path->alloc_head); - - return bit; -} - -static void set_path_leaf_bits(struct super_block *sb, struct radix_path *path, - u64 bit, int nbits) -{ - struct scoutfs_radix_block *rdx; - int lg_ind; - int ind; - - BUG_ON(nbits <= 0); - BUG_ON(calc_leaf_bit(bit) != calc_leaf_bit(bit + nbits - 1)); - BUG_ON(calc_leaf_bit(bit) != path->leaf_bit); - - rdx = path->bls[0]->data; - ind = bit - path->leaf_bit; - lg_ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); - - /* should have returned an error if it was set while we got paths */ - BUG_ON(!bitmap_empty_region_le(rdx->bits, ind, nbits)); - bitmap_set_le(rdx->bits, ind, nbits); - - if (ind < le32_to_cpu(rdx->sm_first)) - rdx->sm_first = cpu_to_le32(ind); - if (lg_ind < le32_to_cpu(rdx->lg_first) && - lg_is_full(rdx->bits, lg_ind)) - rdx->lg_first = cpu_to_le32(lg_ind); - fixup_parent_refs(path, nbits, count_lg_bits(rdx->bits, ind, nbits)); - - trace_scoutfs_radix_set(sb, path->root, path->bls[0]->blkno, - bit, ind, nbits); -} - -/* Find the path for the root and bit in the change and set the region */ -static void set_change_leaf_bits(struct super_block *sb, - struct radix_change *chg, - struct scoutfs_radix_root *root, - u64 bit, int nbits) -{ - struct radix_path *path; - - path = walk_paths(&chg->rbroot, root, calc_leaf_bit(bit), NULL); - BUG_ON(!path); /* should have gotten paths for all leaves to set */ - set_path_leaf_bits(sb, path, bit, nbits); + fixup_parent_refs(sb, bl, -nbits, -lg_nbits); + trace_scoutfs_radix_clear_bits(sb, bl->blkno, ind, nbits); } /* @@ -754,7 +522,6 @@ static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; struct scoutfs_radix_ref ref; - u32 first = full ? 0 : level ? SCOUTFS_RADIX_REFS : SCOUTFS_RADIX_BITS; int tail; int i; @@ -766,8 +533,6 @@ static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, rdx->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_RADIX); rdx->hdr.blkno = cpu_to_le64(blkno); rdx->hdr.seq = seq; - rdx->sm_first = cpu_to_le32(first); - rdx->lg_first = cpu_to_le32(first); if (level == 0) { if (full) @@ -794,76 +559,62 @@ static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, memset((void *)rdx + SCOUTFS_BLOCK_LG_SIZE - tail, 0, tail); } -/* get path flags */ +static int find_next_change_blkno(struct super_block *sb, + struct radix_change *chg, + u64 *blkno); + enum { - GPF_NEXT_SM = (1 << 0), - GPF_NEXT_LG = (1 << 1), + GLF_NEXT_SM = (1 << 0), + GLF_NEXT_LG = (1 << 1), + GLF_DIRTY = (1 << 2), }; + /* - * Give the caller an allocated path that holds references to the blocks - * traversed to the leaf of the given root. + * Get the caller their block for walking down the radix. We can have + * to populate synthetic blocks, read existing blocks, and cow new dirty + * copies of either of those for callers who need to modify. We update + * references and record the blocks and references in the change for + * callers to further build atomic changes with. */ -static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, - struct radix_change *chg, int gpf, u64 bit, - struct radix_path **path_ret) +static int get_radix_block(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct radix_change *chg, + struct scoutfs_radix_root *root, int glf, + struct scoutfs_block *parent, + struct scoutfs_radix_ref *ref, int level, + struct scoutfs_block **bl_ret) { - struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *ref; - struct radix_path *path = NULL; - struct scoutfs_block *bl; + struct radix_block_private *priv = NULL; bool saw_inconsistent = false; + struct scoutfs_radix_block *rdx; + struct scoutfs_block *bl = NULL; + struct scoutfs_block *dirty; + bool put_block = true; u64 blkno; u64 synth; - int level; - int ind; int ret; - int i; - /* can't operate outside radix until we support growing devices */ - if (WARN_ON_ONCE(root->height < height_from_last(bit)) || - WARN_ON_ONCE((gpf & GPF_NEXT_SM) && (gpf & GPF_NEXT_LG))) - return -EINVAL; - - path = alloc_path(root); - if (!path) { - ret = -ENOMEM; - goto out; - } - - /* switch to searching for small bits if no large found */ - if ((gpf & GPF_NEXT_LG) && le64_to_cpu(root->ref.lg_total) == 0) - gpf ^= GPF_NEXT_LG | GPF_NEXT_SM; - - calc_level_inds(path, bit); - - for (level = root->height - 1; level >= 0; level--) { - ref = path_ref(path, level); - - blkno = le64_to_cpu(ref->blkno); - if (blkno == U64_MAX || blkno == 0) { - synth = chg->next_synth++; - if ((blkno & 1) != (synth & 1)) - synth = chg->next_synth++; - /* careful not to go too high or wrap */ - if (synth == U64_MAX || synth < RADIX_SYNTH_BLKNO) { - scoutfs_inc_counter(sb, radix_enospc_synth); - ret = -ENOSPC; - goto out; - } - bl = scoutfs_block_create(sb, synth); - if (!IS_ERR_OR_NULL(bl)) { - init_block(sb, bl->data, synth, ref->seq, level, - blkno == U64_MAX); - ref->blkno = cpu_to_le64(bl->blkno); - - } - } else { - bl = scoutfs_block_read(sb, blkno); - } - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); + /* create a synthetic block or read an existing block */ + blkno = le64_to_cpu(ref->blkno); + if (is_stub(blkno)) { + synth = chg->next_synth++; + /* don't create synth mistaken for all-full */ + if (synth == U64_MAX) { + scoutfs_inc_counter(sb, radix_enospc_synth); + ret = -ENOSPC; goto out; } + bl = scoutfs_block_create(sb, synth); + if (!IS_ERR_OR_NULL(bl)) { + init_block(sb, bl->data, synth, ref->seq, level, + blkno == U64_MAX); + scoutfs_inc_counter(sb, radix_create_synth); + } + } else { + bl = scoutfs_block_read(sb, blkno); + if (!IS_ERR_OR_NULL(bl)) + scoutfs_inc_counter(sb, radix_block_read); /* * We can have a stale block in the cache but the tree @@ -872,46 +623,153 @@ static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, * consistent block after reading from the device then * we've found corruption. */ - if (!scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + while (!IS_ERR(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, + ref->blkno, SCOUTFS_BLOCK_MAGIC_RADIX)) { + scoutfs_inc_counter(sb, radix_inconsistent_ref); + scoutfs_block_writer_forget(sb, wri, bl); + scoutfs_block_invalidate(sb, bl); + BUG_ON(bl->priv != NULL); + scoutfs_block_put(sb, bl); + bl = NULL; if (!saw_inconsistent) { - scoutfs_block_invalidate(sb, bl); - scoutfs_block_put(sb, bl); saw_inconsistent = true; - level++; - continue; + bl = scoutfs_block_read(sb, blkno); + } else { + bl = ERR_PTR(-EIO); + scoutfs_inc_counter(sb, radix_inconsistent_eio); } - ret = -EIO; - goto out; } saw_inconsistent = false; + } + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + + if ((glf & GLF_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl)) { + /* make a cow copy for the caller that needs a dirty block */ + ret = find_next_change_blkno(sb, chg, &blkno); + if (ret < 0) + goto out; + + dirty = scoutfs_block_create(sb, blkno); + if (IS_ERR(dirty)) { + ret = PTR_ERR(dirty); + goto out; + } + + memcpy(dirty->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = dirty; + scoutfs_inc_counter(sb, radix_block_cow); + } + + priv = bl->priv; + if (!priv) { + priv = kzalloc(sizeof(struct radix_block_private), GFP_NOFS); + if (!priv) { + ret = -ENOMEM; + goto out; + } + + bl->priv = priv; + priv->bl = bl; + INIT_LIST_HEAD(&priv->dirtied_entry); + priv->parent = parent; + priv->ref = ref; + priv->orig_ref = *ref; + /* put at head so for_each restores refs in reverse */ + list_add(&priv->entry, &chg->blocks); + /* priv holds bl get, put as change is completed */ + put_block = false; + } + + if ((glf & GLF_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl)) { + scoutfs_block_writer_mark_dirty(sb, wri, bl); + list_add(&priv->dirtied_entry, &chg->dirtied_blocks); + } + + trace_scoutfs_radix_get_block(sb, root, glf, level, + parent ? parent->blkno : 0, + le64_to_cpu(ref->blkno), bl->blkno); + + /* update refs to new synth or dirty blocks */ + if (le64_to_cpu(ref->blkno) != bl->blkno) { + rdx = bl->data; + rdx->hdr.blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&rdx->hdr.seq, sizeof(rdx->hdr.seq)); + ref->blkno = rdx->hdr.blkno; + ref->seq = rdx->hdr.seq; + } + + ret = 0; +out: + if (put_block) + scoutfs_block_put(sb, bl); + if (ret < 0) + bl = NULL; + + *bl_ret = bl; + return ret; +} + +static int get_leaf_walk(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct radix_change *chg, + struct scoutfs_radix_root *root, + int glf, u64 bit, u64 *leaf_bit_ret, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_radix_ref *ref; + struct scoutfs_block *parent = NULL; + struct scoutfs_block *bl; + u32 level_inds[RADIX_MAX_HEIGHT]; + int level; + int ind = 0; + int ret; + int i; + + /* can't operate outside radix until we support growing devices */ + if (WARN_ON_ONCE(root->height < height_from_last(bit)) || + WARN_ON_ONCE(root->height > RADIX_MAX_HEIGHT) || + WARN_ON_ONCE((glf & GLF_NEXT_SM) && (glf & GLF_NEXT_LG))) + return -EINVAL; + + calc_level_inds(level_inds, root->height, bit); + ref = &root->ref; + + for (level = root->height - 1; level >= 0; level--) { + ret = get_radix_block(sb, alloc, wri, chg, root, glf, parent, + ref, level, &bl); + if (ret) + goto out; + + trace_scoutfs_radix_walk(sb, root, glf, level, bl->blkno, ind, + bit); - path->bls[level] = bl; if (level == 0) { - /* path's leaf_bit is first in the leaf block */ - path->inds[0] = 0; + /* returned leaf_bit is first in the leaf block */ + level_inds[0] = 0; break; } rdx = bl->data; - ind = path->inds[level]; + ind = level_inds[level]; - /* search for a path to a leaf with a set large region */ - while ((gpf & GPF_NEXT_LG) && ind < SCOUTFS_RADIX_REFS && + /* search for a ref to a child with a set large region */ + while ((glf & GLF_NEXT_LG) && ind < SCOUTFS_RADIX_REFS && le64_to_cpu(rdx->refs[ind].lg_total) == 0) { - if (ind < le32_to_cpu(rdx->lg_first)) - ind = le32_to_cpu(rdx->lg_first); - else - ind++; + ind++; } - /* search for a path to a leaf with a any bits set */ - while ((gpf & GPF_NEXT_SM) && ind < SCOUTFS_RADIX_REFS && + /* search for a ref to a child with any bits set */ + while ((glf & GLF_NEXT_SM) && ind < SCOUTFS_RADIX_REFS && le64_to_cpu(rdx->refs[ind].sm_total) == 0) { - if (ind < le32_to_cpu(rdx->sm_first)) - ind = le32_to_cpu(rdx->sm_first); - else - ind++; + ind++; } /* @@ -928,224 +786,332 @@ static int get_path(struct super_block *sb, struct scoutfs_radix_root *root, ret = -ENOENT; goto out; } - path->inds[level + 1]++; + level_inds[level + 1]++; for (i = level; i >= 0; i--) - path->inds[i] = 0; - for (i = level; i <= level + 1; i++) { - scoutfs_block_put(sb, path->bls[i]); - path->bls[i] = NULL; - } + level_inds[i] = 0; level += 2; continue; } /* reset all lower indices if we searched */ - if (ind != path->inds[level]) { + if (ind != level_inds[level]) { for (i = level - 1; i >= 0; i--) - path->inds[i] = 0; - path->inds[level] = ind; + level_inds[i] = 0; + level_inds[level] = ind; + } + + parent = bl; + ref = &rdx->refs[ind]; + } + + *leaf_bit_ret = bit_from_inds(level_inds, root->height); + ret = 0; + scoutfs_inc_counter(sb, radix_walk); +out: + if (ret < 0) + *bl_ret = NULL; + else + *bl_ret = bl; + return ret; +} + +/* + * Get the caller their leaf block in which they'll set or clear bits. + * If they're asking for a dirty block then the leaf walk might dirty + * blocks. For each newly dirtied block we also make sure we have dirty + * blocks for the leaves that contain the bits for each newly dirtied + * block's old blkno and new blkno. + */ +static int get_leaf(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, struct radix_change *chg, + struct scoutfs_radix_root *root, int glf, u64 bit, + u64 *leaf_bit_ret, struct scoutfs_block **bl_ret) +{ + struct radix_block_private *priv; + struct scoutfs_block *bl; + u64 leaf_bit; + u64 old_blkno; + int ret; + + ret = get_leaf_walk(sb, alloc, wri, chg, root, glf, bit, leaf_bit_ret, + bl_ret); + if (ret < 0 || !(glf & GLF_DIRTY)) + goto out; + + /* walk to leaves containing bits of newly dirtied block's blknos */ + while ((priv = list_first_entry_or_null(&chg->dirtied_blocks, + struct radix_block_private, + dirtied_entry))) { + /* done when we see tail blocks with their blkno_bl set */ + if (priv->blkno_bl != NULL) + break; + + old_blkno = le64_to_cpu(priv->orig_ref.blkno); + if (!is_stub(old_blkno) && !is_synth(old_blkno)) { + ret = get_leaf_walk(sb, alloc, wri, chg, &alloc->freed, + GLF_DIRTY, old_blkno, &leaf_bit, + &bl); + if (ret < 0) + break; + priv->old_blkno_ind = old_blkno - leaf_bit; + priv->old_blkno_bl = bl; + } + + ret = get_leaf_walk(sb, alloc, wri, chg, &alloc->avail, + GLF_DIRTY, priv->bl->blkno, &leaf_bit, + &bl); + if (ret < 0) + break; + + priv->blkno_ind = priv->bl->blkno - leaf_bit; + priv->blkno_bl = bl; + + list_move_tail(&priv->dirtied_entry, &chg->dirtied_blocks); + } +out: + return ret; +} + +/* + * Find the next region of set bits of the given size starting from the + * given bit. This only finds the bits, it doesn't change anything. We + * always try to return regions past the starting bit. We can search to + * a leaf that has bits that are all past the starting bit and we'll + * retry. This will wrap around to the start of the tree and fall back + * to satisfying large regions with small regions. + */ +static int find_next_set_bits(struct super_block *sb, struct radix_change *chg, + struct scoutfs_radix_root *root, bool meta, + u64 start, int nbits, u64 *bit_ret, + int *nbits_ret, struct scoutfs_block **bl_ret) +{ + struct scoutfs_radix_block *rdx; + struct scoutfs_block *bl; + u64 leaf_bit; + u64 bit; + int end; + int ind; + int glf; + int ret; + + bit = start; + glf = nbits > 1 ? GLF_NEXT_LG : GLF_NEXT_SM; +retry: + ret = get_leaf(sb, NULL, NULL, chg, root, glf, bit, &leaf_bit, &bl); + if (ret == -ENOENT) { + if (bit != 0) { + bit = 0; + goto retry; + } + + /* switch to searching for small bits if no large found */ + if (glf == GLF_NEXT_LG) { + glf = GLF_NEXT_SM; + bit = start; + goto retry; + } + ret = -ENOSPC; + goto out; + } + rdx = bl->data; + + /* start from search bit if it's in the leaf, otherwise 0 */ + if (leaf_bit < bit && ((bit - leaf_bit) < SCOUTFS_RADIX_BITS)) + ind = bit - leaf_bit; + else + ind = 0; + + /* large allocs are always aligned from large regions */ + if (nbits >= SCOUTFS_RADIX_LG_BITS && (glf == GLF_NEXT_LG)) { + ind = find_next_lg(rdx->bits, ind); + if (ind == SCOUTFS_RADIX_BITS) { + bit = wrap_bit(sb, meta, leaf_bit + SCOUTFS_RADIX_BITS); + goto retry; + } + nbits = SCOUTFS_RADIX_LG_BITS; + ret = 0; + goto out; + } + + /* otherwise use as much of the next set region as we can */ + ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); + if (ind == SCOUTFS_RADIX_BITS) { + bit = wrap_bit(sb, meta, leaf_bit + SCOUTFS_RADIX_BITS); + goto retry; + } + + if (nbits > 1) { + end = find_next_zero_bit_le(rdx->bits, min_t(int, ind + nbits, + SCOUTFS_RADIX_BITS), ind); + nbits = end - ind; + } + ret = 0; + +out: + *bit_ret = leaf_bit + ind; + *nbits_ret = nbits; + if (bl_ret) + *bl_ret = bl; + + return ret; +} + +static void prepare_change(struct radix_change *chg, + struct scoutfs_radix_root *avail) +{ + memset(chg, 0, sizeof(struct radix_change)); + chg->avail = avail; + INIT_LIST_HEAD(&chg->blocks); + INIT_LIST_HEAD(&chg->dirtied_blocks); + chg->next_synth = RADIX_SYNTH_BLKNO; + chg->next_find_bit = le64_to_cpu(avail->next_find_bit); +} + +/* + * We successfully got all the dirty block references we need to make + * the change. Set their old blkno's freed bits and clear all their new + * dirty blkno's avail bits. We drop the blocks from the dirtied_blocks + * list here as we go so we won't attempt to do this all over again + * as we complete the change. + */ +static void apply_change_bits(struct super_block *sb, struct radix_change *chg) +{ + struct radix_block_private *priv; + struct scoutfs_block *bl; + + /* first update the contents of the blocks */ + list_for_each_entry(priv, &chg->blocks, entry) { + bl = priv->bl; + + /* complete cow allocations for dirtied blocks */ + if (was_dirtied(priv)) { + /* can't try to write to synth blknos */ + BUG_ON(is_synth(bl->blkno)); + + clear_leaf_bits(sb, priv->blkno_bl, priv->blkno_ind, 1); + if (priv->old_blkno_bl) { + set_leaf_bits(sb, priv->old_blkno_bl, + priv->old_blkno_ind, 1); + } + scoutfs_inc_counter(sb, radix_complete_dirty_block); + + list_del_init(&priv->dirtied_entry); + } + } +} + +/* + * Drop all references to the blocks that we held as we worked with the + * radix blocks. + * + * If the operation failed then we drop the blocks we dirtied during + * this change and restore their refs. Nothing can update a ref to a + * dirty block so these will always be current. + * + * We always drop synthetic blocks. They could been cowed so they might + * not be currently referenced. Blocks are added to the head of the + * blocks list as they're first used so we're undoing ref changes in + * reverse order. This means that the error case will always first + * unwind synthetic cows then the synthetic source block itself. + */ +static void complete_change(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct radix_change *chg, int err) +{ + struct radix_block_private *priv; + struct radix_block_private *tmp; + struct scoutfs_block *bl; + + /* only complete once for each call to prepare */ + if (!chg->avail) + return; + + /* finish dirty block frees and allocs on success */ + if (err == 0 && !list_empty(&chg->dirtied_blocks)) + apply_change_bits(sb, chg); + + /* replace refs and remove blocks from the cache */ + list_for_each_entry(priv, &chg->blocks, entry) { + bl = priv->bl; + + if (is_synth(bl->blkno) || (err < 0 && was_dirtied(priv))) { + if (le64_to_cpu(priv->ref->blkno) == bl->blkno) { + *priv->ref = priv->orig_ref; + scoutfs_inc_counter(sb, radix_undo_ref); + } + scoutfs_block_writer_forget(sb, wri, bl); + scoutfs_block_invalidate(sb, bl); } } - path->leaf_bit = bit_from_inds(path); + /* finally put all blocks now that were done with contents */ + list_for_each_entry_safe(priv, tmp, &chg->blocks, entry) { + bl = priv->bl; + + bl->priv = NULL; + scoutfs_block_put(sb, bl); + list_del(&priv->entry); + kfree(priv); + } + + if (err == 0) + store_next_find_bit(sb, true, chg->avail, chg->next_find_bit); + chg->avail = NULL; +} + +/* + * Find the next free metadata blkno from the metadata allocator that + * the change is tracking. This is used to find the next free blkno for + * the next cowed block without modifying the allocator. Because it's + * not modifying the allocator it can wrap and find the same block + * twice, we watch for that. + */ +static int find_next_change_blkno(struct super_block *sb, + struct radix_change *chg, u64 *blkno) +{ + struct scoutfs_radix_block *rdx; + u64 bit; + int nbits; + int ret; + + if (chg->free_bl == NULL) { + ret = find_next_set_bits(sb, chg, chg->avail, true, + chg->next_find_bit, 1, &bit, &nbits, + &chg->free_bl); + if (ret < 0) + goto out; + chg->free_leaf_bit = calc_leaf_bit(bit); + chg->free_ind = bit - chg->free_leaf_bit; + } + + bit = chg->free_leaf_bit + chg->free_ind; + if (chg->first_free == 0) { + chg->first_free = bit; + } else if (chg->first_free == bit) { + ret = -ENOSPC; + goto out; + } + + *blkno = bit; + + rdx = chg->free_bl->data; + chg->free_ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, + chg->free_ind + 1); + if (chg->free_ind >= SCOUTFS_RADIX_BITS) { + chg->free_ind = SCOUTFS_RADIX_BITS; + chg->free_bl = NULL; + } + chg->next_find_bit = wrap_bit(sb, true, + chg->free_leaf_bit + chg->free_ind); + ret = 0; out: - if (ret < 0) { - free_path(sb, path); - path = NULL; - } - - *path_ret = path; + if (ret == -ENOSPC) + scoutfs_inc_counter(sb, radix_enospc_meta); return ret; } -/* - * Get all the paths we're going to need to dirty all the blocks in all - * the paths in the change. The caller has added their path to the leaf - * that they want to change to start the process off. - * - * For every clean block in paths we can have to set a bit in a leaf to - * free the old blkno and clear a bit in a leaf to allocate a new dirty - * blkno. We keep checking new paths for clean blocks until eventually - * all the paths only contain blocks whose blknos are in leaves that we - * already have paths to. - */ -static int get_all_paths(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct radix_change *chg) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *ref; - struct scoutfs_block *bl; - struct radix_path *path; - struct radix_path *adding; - struct radix_path *found; - bool meta_wrapped; - bool stable; - u64 start_meta; - u64 next_meta; - u64 last_meta; - u64 leaf_bit; - int ind; - int ret; - int i; - - start_meta = calc_leaf_bit(le64_to_cpu(alloc->avail.next_find_bit)); - next_meta = start_meta; - last_meta = le64_to_cpu(super->last_meta_blkno); - meta_wrapped = false; - - do { - stable = true; - - /* get paths to leaves to allocate dirty blknos from */ - if (chg->alloc_bits < chg->block_allocs + chg->caller_allocs) { - stable = false; - - /* we're not modifying as we go, check for wrapping */ - if (next_meta >= start_meta && meta_wrapped) { - scoutfs_inc_counter(sb, radix_enospc_paths); - ret = -ENOSPC; - break; - } - - ret = get_path(sb, &alloc->avail, chg, GPF_NEXT_SM, - next_meta, &adding); - if (ret < 0) { - if (ret == -ENOENT) { - meta_wrapped = true; - next_meta = 0; - continue; - } - break; - } - - next_meta = adding->leaf_bit + SCOUTFS_RADIX_BITS; - if (next_meta > last_meta) { - meta_wrapped = true; - next_meta = 0; - } - - /* might already have path, maybe add it to alloc */ - found = walk_paths(&chg->rbroot, adding->root, - adding->leaf_bit, adding); - if (found != adding) { - free_path(sb, adding); - adding = found; - } else { - list_add_tail(&adding->head, &chg->new_paths); - } - if (list_empty(&adding->alloc_head)) { - ref = path_ref(adding, 0); - chg->alloc_bits += le64_to_cpu(ref->sm_total); - list_add_tail(&adding->alloc_head, - &chg->alloc_paths); - } - } - - if ((path = list_first_entry_or_null(&chg->new_paths, - struct radix_path, - head))) { - list_move_tail(&path->head, &chg->paths); - stable = false; - - /* check all the blocks in all new paths */ - for (i = path->height - 1; i >= 0; i--) { - bl = path->bls[i]; - - /* dirty are done, only visit each block once */ - if (scoutfs_block_writer_is_dirty(sb, bl) || - scoutfs_block_tas_visited(sb, bl)) - continue; - - /* record the number of allocs we'll need */ - chg->block_allocs++; - - /* don't need to free synth blknos */ - if (bl->blkno >= RADIX_SYNTH_BLKNO) - continue; - - /* see if we already a path to this leaf */ - leaf_bit = calc_leaf_bit(bl->blkno); - if (walk_paths(&chg->rbroot, &alloc->freed, - leaf_bit, NULL)) - continue; - - /* get a new path to freed leaf to set */ - ret = get_path(sb, &alloc->freed, chg, 0, - bl->blkno, &adding); - if (ret < 0) - break; - - rdx = adding->bls[0]->data; - ind = bl->blkno - adding->leaf_bit; - if (test_bit_le(ind, rdx->bits)) { - /* XXX corruption, bit already set? */ - ret = -EIO; - break; - } - - walk_paths(&chg->rbroot, adding->root, - adding->leaf_bit, adding); - list_add_tail(&adding->head, &chg->new_paths); - } - } - - ret = 0; - } while (!stable); - - return ret; -} - -/* - * We have pinned blocks in paths to all the leaves that we need to - * modify to make a change to radix trees. Walk through the paths - * moving blocks to their new allocated blknos, freeing the old stable - * blknos. - */ -static void dirty_all_path_blocks(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct radix_change *chg) -{ - struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *ref; - struct scoutfs_block *bl; - struct radix_path *path; - u64 blkno; - int level; - - BUG_ON(!list_empty(&chg->new_paths)); - - list_for_each_entry(path, &chg->paths, head) { - - for (level = path->height - 1; level >= 0; level--) { - bl = path->bls[level]; - - if (scoutfs_block_writer_is_dirty(sb, bl)) - continue; - - if (bl->blkno < RADIX_SYNTH_BLKNO) - set_change_leaf_bits(sb, chg, &alloc->freed, - bl->blkno, 1); - - blkno = change_alloc_meta(sb, chg); - scoutfs_block_clear_visited(sb, bl); - scoutfs_block_move(sb, wri, bl, blkno); - scoutfs_block_writer_mark_dirty(sb, wri, bl); - - rdx = bl->data; - rdx->hdr.blkno = cpu_to_le64(bl->blkno); - prandom_bytes(&rdx->hdr.seq, sizeof(rdx->hdr.seq)); - - ref = path_ref(path, level); - ref->blkno = rdx->hdr.blkno; - ref->seq = rdx->hdr.seq; - } - } -} - static bool valid_free_bit_range(struct super_block *sb, bool meta, u64 bit, int nbits) { @@ -1166,9 +1132,9 @@ static int radix_free(struct super_block *sb, struct scoutfs_radix_root *root, bool meta, u64 bit, int nbits) { - struct scoutfs_radix_block *rdx; - struct radix_change *chg; - struct radix_path *path; + struct scoutfs_block *bl; + DECLARE_RADIX_CHANGE(chg); + u64 leaf_bit; int ind; int ret; @@ -1178,36 +1144,19 @@ static int radix_free(struct super_block *sb, return -EINVAL; mutex_lock(&alloc->mutex); + prepare_change(&chg, &alloc->avail); - chg = alloc_change(); - if (!chg) { - ret = -ENOMEM; - goto out; - } - - ret = get_path(sb, root, chg, 0, bit, &path); - if (ret < 0) - goto out; - list_add_tail(&path->head, &chg->new_paths); - - ind = bit - path->leaf_bit; - rdx = path->bls[0]->data; - if (!bitmap_empty_region_le(rdx->bits, ind, nbits)) { - /* XXX corruption, trying to free set bits */ - ret = -EIO; - goto out; - } - - ret = get_all_paths(sb, alloc, chg); + ret = get_leaf(sb, alloc, wri, &chg, root, GLF_DIRTY, bit, + &leaf_bit, &bl); if (ret < 0) goto out; - dirty_all_path_blocks(sb, alloc, wri, chg); - set_path_leaf_bits(sb, path, bit, nbits); - ret = 0; + ind = bit - leaf_bit; + set_leaf_bits(sb, bl, ind, nbits); out: - free_change(sb, chg); + complete_change(sb, wri, &chg, ret); mutex_unlock(&alloc->mutex); + return ret; } @@ -1219,27 +1168,33 @@ int scoutfs_radix_alloc(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, u64 *blkno) { - struct radix_change *chg; + struct scoutfs_block *bl; + DECLARE_RADIX_CHANGE(chg); + u64 leaf_bit; + u64 bit; + int ind; int ret; + scoutfs_inc_counter(sb, radix_alloc); + mutex_lock(&alloc->mutex); + prepare_change(&chg, &alloc->avail); - chg = alloc_change(); - if (!chg) { - ret = -ENOMEM; - goto out; - } - - chg->caller_allocs = 1; - ret = get_all_paths(sb, alloc, chg); + ret = find_next_change_blkno(sb, &chg, &bit); if (ret < 0) goto out; - dirty_all_path_blocks(sb, alloc, wri, chg); - *blkno = change_alloc_meta(sb, chg); + ret = get_leaf(sb, alloc, wri, &chg, &alloc->avail, GLF_DIRTY, bit, + &leaf_bit, &bl); + if (ret < 0) + goto out; + + ind = bit - leaf_bit; + clear_leaf_bits(sb, bl, ind, 1); + *blkno = bit; ret = 0; out: - free_change(sb, chg); + complete_change(sb, wri, &chg, ret); mutex_unlock(&alloc->mutex); return ret; @@ -1257,13 +1212,16 @@ int scoutfs_radix_alloc_data(struct super_block *sb, struct scoutfs_radix_root *root, int count, u64 *blkno_ret, int *count_ret) { - struct radix_change *chg; - struct radix_path *path; + struct scoutfs_block *bl; + DECLARE_RADIX_CHANGE(chg); + u64 leaf_bit; u64 bit; int nbits; - int gpf; + int ind; int ret; + scoutfs_inc_counter(sb, radix_alloc_data); + *blkno_ret = 0; *count_ret = 0; @@ -1271,41 +1229,32 @@ int scoutfs_radix_alloc_data(struct super_block *sb, return -EINVAL; nbits = min(count, SCOUTFS_RADIX_LG_BITS); - gpf = nbits > 1 ? GPF_NEXT_LG : GPF_NEXT_SM; mutex_lock(&alloc->mutex); + prepare_change(&chg, &alloc->avail); - chg = alloc_change(); - if (!chg) { - ret = -ENOMEM; - goto out; - } - -find_next: - bit = le64_to_cpu(root->next_find_bit); - ret = get_path(sb, root, chg, gpf, bit, &path); - if (ret) { - if (ret == -ENOENT) { - if (root->next_find_bit != 0) { - root->next_find_bit = 0; - goto find_next; - } + ret = find_next_set_bits(sb, &chg, root, false, + le64_to_cpu(root->next_find_bit), nbits, + &bit, &nbits, NULL); + if (ret < 0) { + if (ret == -ENOSPC) scoutfs_inc_counter(sb, radix_enospc_data); - ret = -ENOSPC; - } goto out; } - list_add_tail(&path->head, &chg->new_paths); - ret = get_all_paths(sb, alloc, chg); + ret = get_leaf(sb, alloc, wri, &chg, root, GLF_DIRTY, bit, + &leaf_bit, &bl); if (ret < 0) goto out; - dirty_all_path_blocks(sb, alloc, wri, chg); - alloc_leaf_bits(sb, false, path, nbits, blkno_ret, count_ret); + ind = bit - leaf_bit; + clear_leaf_bits(sb, bl, ind, nbits); + *blkno_ret = bit; + *count_ret = nbits; + store_next_find_bit(sb, false, root, bit + nbits); ret = 0; out: - free_change(sb, chg); + complete_change(sb, wri, &chg, ret); mutex_unlock(&alloc->mutex); return ret; @@ -1319,6 +1268,7 @@ int scoutfs_radix_free(struct super_block *sb, struct scoutfs_radix_allocator *alloc, struct scoutfs_block_writer *wri, u64 blkno) { + scoutfs_inc_counter(sb, radix_free); return radix_free(sb, alloc, wri, &alloc->freed, true, blkno, 1); } @@ -1332,6 +1282,7 @@ int scoutfs_radix_free_data(struct super_block *sb, struct scoutfs_radix_root *root, u64 blkno, int count) { + scoutfs_inc_counter(sb, radix_free_data); return radix_free(sb, alloc, wri, root, false, blkno, count); } @@ -1353,9 +1304,9 @@ int scoutfs_radix_free_data(struct super_block *sb, * read the old blocks. * * We can also be called with a src tree that is the current allocator - * avail tree. In this case dirtying the blocks in all the paths can - * consume bits in the source tree. We notice when dirtying allocation - * empties the src block and we retry finding a new leaf to merge. + * avail tree. In this case dirtying the leaf blocks can consume bits + * in the source tree. We notice when dirtying the src block and we + * retry finding a new leaf to merge. * * The caller specifies the minimum count to move. -ENOENT will be * returned if the source tree runs out of bits, potentially after @@ -1377,20 +1328,21 @@ int scoutfs_radix_merge(struct super_block *sb, struct scoutfs_radix_block *inp_rdx; struct scoutfs_radix_block *src_rdx; struct scoutfs_radix_block *dst_rdx; - struct radix_change *chg = NULL; - struct radix_path *inp_path = NULL; - struct radix_path *src_path; - struct radix_path *dst_path; + struct scoutfs_block *inp_bl; + struct scoutfs_block *src_bl; + struct scoutfs_block *dst_bl; + DECLARE_RADIX_CHANGE(chg); s64 src_lg_delta; s64 dst_lg_delta; + u64 leaf_bit; u64 bit; int merge_size; int merged; - int inp_sm; - int lg_ind; int ind; int ret; + scoutfs_inc_counter(sb, radix_merge); + mutex_lock(&alloc->mutex); /* can't try to free too much when inp is read-only */ @@ -1402,15 +1354,11 @@ int scoutfs_radix_merge(struct super_block *sb, while (count > 0) { - chg = alloc_change(); - if (!chg) { - ret = -ENOMEM; - goto out; - } - + prepare_change(&chg, &alloc->avail); bit = le64_to_cpu(src->next_find_bit); wrapped: - ret = get_path(sb, inp, chg, GPF_NEXT_SM, bit, &inp_path); + ret = get_leaf(sb, NULL, NULL, &chg, inp, GLF_NEXT_SM, bit, + &leaf_bit, &inp_bl); if (ret < 0) { if (ret == -ENOENT) { if (bit != 0) { @@ -1422,43 +1370,28 @@ wrapped: } goto out; } - /* unique input is not modified, not stored in the change */ - bit = inp_path->leaf_bit; + bit = leaf_bit; + inp_rdx = inp_bl->data; - ret = get_path(sb, src, chg, 0, bit, &src_path); + ret = get_leaf(sb, alloc, wri, &chg, src, GLF_DIRTY, bit, + &leaf_bit, &src_bl); if (ret < 0) goto out; - list_add_tail(&src_path->head, &chg->new_paths); + src_rdx = src_bl->data; - ret = get_path(sb, dst, chg, 0, bit, &dst_path); + ret = get_leaf(sb, alloc, wri, &chg, dst, GLF_DIRTY, bit, + &leaf_bit, &dst_bl); if (ret < 0) goto out; - list_add_tail(&dst_path->head, &chg->new_paths); + dst_rdx = dst_bl->data; - ret = get_all_paths(sb, alloc, chg); - if (ret < 0) - goto out; + apply_change_bits(sb, &chg); - /* this can modify src/dst when they're alloc trees */ - dirty_all_path_blocks(sb, alloc, wri, chg); - - inp_rdx = inp_path->bls[0]->data; - src_rdx = src_path->bls[0]->data; - dst_rdx = dst_path->bls[0]->data; - - inp_sm = le64_to_cpu(path_ref(inp_path, 0)->sm_total); - ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, - le32_to_cpu(inp_rdx->sm_first)); - lg_ind = round_down(ind, SCOUTFS_RADIX_LG_BITS); - - /* back out and retry if no input left, or inp not ro */ - if (inp_sm == 0 || - (inp != src && paths_share_blocks(inp_path, src_path))) { - scoutfs_inc_counter(sb, radix_merge_retry); - free_path(sb, inp_path); - inp_path = NULL; - free_change(sb, chg); - chg = NULL; + /* change allocs could have cleared all of inp if its avail */ + ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, 0); + if (ind == SCOUTFS_RADIX_BITS) { + scoutfs_inc_counter(sb, radix_merge_empty); + complete_change(sb, wri, &chg, -EAGAIN); continue; } @@ -1481,8 +1414,7 @@ wrapped: /* carefully modify src last, it might also be inp */ merged = bitmap_xor_bitmap_le(dst_rdx->bits, inp_rdx->bits, - ind, min_t(u64, inp_sm, count), - &merge_size); + ind, count, &merge_size); dst_lg_delta = count_lg_from_set(dst_rdx->bits, inp_rdx->bits, ind, merge_size); @@ -1491,25 +1423,15 @@ wrapped: bitmap_xor_bitmap_le(src_rdx->bits, inp_rdx->bits, ind, merged, NULL); - if (ind < le32_to_cpu(dst_rdx->sm_first)) - dst_rdx->sm_first = cpu_to_le32(ind); - /* first doesn't have to be precise, search will cleanup */ - if (lg_ind < le32_to_cpu(dst_rdx->lg_first)) - dst_rdx->lg_first = cpu_to_le32(lg_ind); + fixup_parent_refs(sb, src_bl, -merged, -src_lg_delta); + fixup_parent_refs(sb, dst_bl, merged, dst_lg_delta); - fixup_parent_refs(src_path, -merged, -src_lg_delta); - fixup_parent_refs(dst_path, merged, dst_lg_delta); + trace_scoutfs_radix_merge(sb, inp, inp_bl->blkno, src, + src_bl->blkno, dst, dst_bl->blkno, + count, bit, ind, merged, + src_lg_delta, dst_lg_delta); - trace_scoutfs_radix_merge(sb, inp, inp_path->bls[0]->blkno, - src, src_path->bls[0]->blkno, - dst, dst_path->bls[0]->blkno, count, - bit, ind, merged, src_lg_delta, - dst_lg_delta); - - free_path(sb, inp_path); - inp_path = NULL; - free_change(sb, chg); - chg = NULL; + complete_change(sb, wri, &chg, 0); store_next_find_bit(sb, meta, src, bit + SCOUTFS_RADIX_BITS); count -= min_t(u64, count, merged); @@ -1517,8 +1439,7 @@ wrapped: ret = 0; out: - free_path(sb, inp_path); - free_change(sb, chg); + complete_change(sb, wri, &chg, ret); mutex_unlock(&alloc->mutex); return ret; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0fad61cb..05c1427b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2226,123 +2226,87 @@ DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink, TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) ); -TRACE_EVENT(scoutfs_radix_dirty, +TRACE_EVENT(scoutfs_radix_get_block, TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - u64 orig_blkno, u64 dirty_blkno, u64 par_blkno), - TP_ARGS(sb, root, orig_blkno, dirty_blkno, par_blkno), + int glf, int level, u64 par_blkno, u64 ref_blkno, u64 blkno), + TP_ARGS(sb, root, glf, level, par_blkno, ref_blkno, blkno), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, root_blkno) - __field(__u64, orig_blkno) - __field(__u64, dirty_blkno) + __field(int, glf) + __field(int, level) __field(__u64, par_blkno) + __field(__u64, ref_blkno) + __field(__u64, blkno) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->root_blkno = le64_to_cpu(root->ref.blkno); - __entry->orig_blkno = orig_blkno; - __entry->dirty_blkno = dirty_blkno; + __entry->glf = glf; + __entry->level = level; __entry->par_blkno = par_blkno; + __entry->ref_blkno = ref_blkno; + __entry->blkno = blkno; ), - TP_printk(SCSBF" root_blkno %llu orig_blkno %llu dirty_blkno %llu par_blkno %llu", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->orig_blkno, - __entry->dirty_blkno, __entry->par_blkno) + TP_printk(SCSBF" root_blkno %llu glf 0x%x level %u par_blkno %llu ref_blkno %llu blkno %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->glf, + __entry->level, __entry->par_blkno, __entry->ref_blkno, + __entry->blkno) ); TRACE_EVENT(scoutfs_radix_walk, TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - int grl, int level, u64 blkno, int ind, u64 bit, u64 next), - TP_ARGS(sb, root, grl, level, blkno, ind, bit, next), + int glf, int level, u64 blkno, int ind, u64 bit), + TP_ARGS(sb, root, glf, level, blkno, ind, bit), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, root_blkno) - __field(unsigned int, grl) + __field(unsigned int, glf) __field(__u64, blkno) __field(int, level) __field(int, ind) __field(__u64, bit) - __field(__u64, next) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->root_blkno = le64_to_cpu(root->ref.blkno); - __entry->grl = grl; + __entry->glf = glf; __entry->blkno = blkno; __entry->level = level; __entry->ind = ind; __entry->bit = bit; - __entry->next = next; ), - TP_printk(SCSBF" root_blkno %llu grl 0x%x blkno %llu level %d ind %d bit %llu next %llu", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->grl, - __entry->blkno, __entry->level, __entry->ind, __entry->bit, - __entry->next) -); - -TRACE_EVENT(scoutfs_radix_fixup_refs, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - u32 sm_first, u64 sm_total, u16 lg_first, u64 lg_total, - u64 blkno, int level), - TP_ARGS(sb, root, sm_first, sm_total, lg_first, lg_total, blkno, level), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(__u32, sm_first) - __field(__u64, sm_total) - __field(__u16, lg_first) - __field(__u64, lg_total) - __field(__u64, blkno) - __field(int, level) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(root->ref.blkno); - __entry->sm_first = sm_first; - __entry->sm_total = sm_total; - __entry->lg_first = lg_first; - __entry->lg_total = lg_total; - __entry->blkno = blkno; - __entry->level = level; - ), - TP_printk(SCSBF" root_blkno %llu sm_first %u sm_total %llu lg_first %u lg_total %llu blkno %llu level %u", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->sm_first, - __entry->sm_total, __entry->lg_first, __entry->lg_total, - __entry->blkno, __entry->level) + TP_printk(SCSBF" root_blkno %llu glf 0x%x blkno %llu level %d par_ind %d bit %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->glf, + __entry->blkno, __entry->level, __entry->ind, __entry->bit) ); DECLARE_EVENT_CLASS(scoutfs_radix_bitop, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - u64 blkno, u64 bit, int ind, int nbits), - TP_ARGS(sb, root, blkno, bit, ind, nbits), + TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), + TP_ARGS(sb, blkno, ind, nbits), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, root_blkno) __field(__u64, blkno) - __field(__u64, bit) __field(int, ind) __field(int, nbits) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(root->ref.blkno); __entry->blkno = blkno; - __entry->bit = bit; __entry->ind = ind; __entry->nbits = nbits; ), - TP_printk(SCSBF" root_blkno %llu blkno %llu bit %llu ind %d nbits %d", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->blkno, - __entry->bit, __entry->ind, __entry->nbits) + TP_printk(SCSBF" blkno %llu ind %d nbits %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ind, + __entry->nbits) ); -DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_clear, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - u64 blkno, u64 bit, int ind, int nbits), - TP_ARGS(sb, root, blkno, bit, ind, nbits) +DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_clear_bits, + TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), + TP_ARGS(sb, blkno, ind, nbits) ); -DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - u64 blkno, u64 bit, int ind, int nbits), - TP_ARGS(sb, root, blkno, bit, ind, nbits) +DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set_bits, + TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), + TP_ARGS(sb, blkno, ind, nbits) ); TRACE_EVENT(scoutfs_radix_merge, From e5f5ee2679dff7486d892c295f36344d70870989 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 May 2020 16:08:52 -0700 Subject: [PATCH 835/920] Revert "scoutfs: add scoutfs_block_move" We add _block_move for the radix allocator, but it no longer needs it. This reverts commit 6bb0726689981eb9699296ae2cb4c8599add5b90. --- kmod/src/block.c | 38 -------------------------------------- kmod/src/block.h | 3 --- kmod/src/scoutfs_trace.h | 5 ----- 3 files changed, 46 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index b7da3950..ed5359ab 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -812,44 +812,6 @@ void scoutfs_block_writer_forget(struct super_block *sb, } } -/* - * Change a cached block's location. We're careful to only change its - * position in the rbtree. If we find another block existing at the new - * location then we remove it from the cache and forget it if it was - * dirty. - */ -void scoutfs_block_move(struct super_block *sb, - struct scoutfs_block_writer *wri, - struct scoutfs_block *bl, u64 blkno) -{ - DECLARE_BLOCK_INFO(sb, binf); - struct block_private *bp = BLOCK_PRIVATE(bl); - struct block_private *existing = NULL; - - spin_lock(&binf->lock); - - existing = walk_block_rbtree(&binf->root, blkno, NULL); - if (existing) { - /* only nesting of binf and wri locks */ - if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) { - spin_lock(&wri->lock); - if (test_bit(BLOCK_BIT_DIRTY, &bp->bits)) - block_forget(sb, wri, bp); - spin_unlock(&wri->lock); - } - block_remove(sb, existing); - } - - rb_erase(&bp->node, &binf->root); - RB_CLEAR_NODE(&bp->node); - bp->bl.blkno = blkno; - walk_block_rbtree(&binf->root, blkno, bp); - - TRACE_BLOCK(move, bp); - - spin_unlock(&binf->lock); -} - /* * The caller has ensured that no more dirtying will take place. This * helps the caller avoid doing a bunch of work before calling into the diff --git a/kmod/src/block.h b/kmod/src/block.h index 22e2437d..6f73176f 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -45,9 +45,6 @@ void scoutfs_block_writer_forget_all(struct super_block *sb, void scoutfs_block_writer_forget(struct super_block *sb, struct scoutfs_block_writer *wri, struct scoutfs_block *bl); -void scoutfs_block_move(struct super_block *sb, - struct scoutfs_block_writer *wri, - struct scoutfs_block *bl, u64 blkno); bool scoutfs_block_writer_has_dirty(struct super_block *sb, struct scoutfs_block_writer *wri); u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 05c1427b..47238c9d 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2205,11 +2205,6 @@ DEFINE_EVENT(scoutfs_block_class, scoutfs_block_invalidate, int refcount, int io_count, unsigned long bits, u64 lru_moved), TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) ); -DEFINE_EVENT(scoutfs_block_class, scoutfs_block_move, - TP_PROTO(struct super_block *sb, void *bp, u64 blkno, - int refcount, int io_count, unsigned long bits, u64 lru_moved), - TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) -); DEFINE_EVENT(scoutfs_block_class, scoutfs_block_mark_dirty, TP_PROTO(struct super_block *sb, void *bp, u64 blkno, int refcount, int io_count, unsigned long bits, u64 lru_moved), From e6ae397d1215c577452d3bc7f4694f13acefc98b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 11 May 2020 16:14:29 -0700 Subject: [PATCH 836/920] Revert "scoutfs: switch block cache to rbtree" We had switched away from the radix_tree because we were adding a _block_move call which couldn't fail. We no longer need that call, so we can go back to storing cached blocks in the radix tree which can use RCU lookups. This revert has some conflict resolution around recent commits to add the IO_BUSY block flag and the switch to _LG_ blocks. This reverts commit 10205a5670dd96af350cf481a3336817871a9a5b. Signed-off-by: Zach Brown --- kmod/src/block.c | 90 +++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 54 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index ed5359ab..1ed7d0d7 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -19,7 +19,6 @@ #include #include #include -#include #include "format.h" #include "super.h" @@ -46,7 +45,7 @@ struct block_info { struct super_block *sb; spinlock_t lock; - struct rb_root root; + struct radix_tree_root radix; struct list_head lru_list; u64 lru_nr; u64 lru_move_counter; @@ -65,7 +64,7 @@ enum { BLOCK_BIT_DIRTY, /* dirty, writer will write */ BLOCK_BIT_IO_BUSY, /* bios are in flight */ BLOCK_BIT_ERROR, /* saw IO error */ - BLOCK_BIT_DELETED, /* has been deleted from rbtree */ + BLOCK_BIT_DELETED, /* has been deleted from radix tree */ BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ BLOCK_BIT_VIRT, /* mapped virt allocation */ BLOCK_BIT_CRC_VALID, /* crc has been verified */ @@ -74,7 +73,6 @@ enum { struct block_private { struct scoutfs_block bl; - struct rb_node node; struct super_block *sb; atomic_t refcount; union { @@ -183,7 +181,6 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) } bp->bl.blkno = blkno; - RB_CLEAR_NODE(&bp->node); bp->sb = sb; atomic_set(&bp->refcount, 1); INIT_LIST_HEAD(&bp->lru_entry); @@ -253,39 +250,9 @@ static void block_put(struct super_block *sb, struct block_private *bp) } } -static struct block_private *walk_block_rbtree(struct rb_root *root, - u64 blkno, - struct block_private *ins) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct block_private *bp; - int cmp; - - while (*node) { - parent = *node; - bp = container_of(*node, struct block_private, node); - - cmp = scoutfs_cmp_u64s(bp->bl.blkno, blkno); - if (cmp == 0) - return bp; - else if (cmp < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - if (ins) { - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); - return ins; - } - - return NULL; -} - /* - * Add a new block into the cache. The caller holds the lock. + * Add a new block into the cache. The caller holds the lock and has + * preloaded the radix. */ static void block_insert(struct super_block *sb, struct block_private *bp, u64 blkno) @@ -294,10 +261,9 @@ static void block_insert(struct super_block *sb, struct block_private *bp, assert_spin_locked(&binf->lock); BUG_ON(!list_empty(&bp->lru_entry)); - BUG_ON(!RB_EMPTY_NODE(&bp->node)); atomic_inc(&bp->refcount); - walk_block_rbtree(&binf->root, blkno, bp); + radix_tree_insert(&binf->radix, blkno, bp); list_add_tail(&bp->lru_entry, &binf->lru_list); bp->lru_moved = ++binf->lru_move_counter; binf->lru_nr++; @@ -345,10 +311,11 @@ static void block_remove(struct super_block *sb, struct block_private *bp) { DECLARE_BLOCK_INFO(sb, binf); + assert_spin_locked(&binf->lock); + if (!test_and_set_bit(BLOCK_BIT_DELETED, &bp->bits)) { BUG_ON(list_empty(&bp->lru_entry)); - rb_erase(&bp->node, &binf->root); - RB_CLEAR_NODE(&bp->node); + radix_tree_delete(&binf->radix, bp->bl.blkno); list_del_init(&bp->lru_entry); binf->lru_nr--; block_put(sb, bp); @@ -368,18 +335,19 @@ static void block_remove_all(struct super_block *sb) { DECLARE_BLOCK_INFO(sb, binf); struct block_private *bp; - struct rb_node *node; - for (node = rb_first(&binf->root); node; ) { - bp = container_of(node, struct block_private, node); - node = rb_next(node); + spin_lock(&binf->lock); + + while (radix_tree_gang_lookup(&binf->radix, (void **)&bp, 0, 1) == 1) { wait_event(binf->waitq, !io_busy(bp)); block_remove(sb, bp); } + spin_unlock(&binf->lock); + WARN_ON_ONCE(!list_empty(&binf->lru_list)); WARN_ON_ONCE(binf->lru_nr != 0); - WARN_ON_ONCE(!RB_EMPTY_ROOT(&binf->root)); + WARN_ON_ONCE(binf->radix.rnode != NULL); } /* @@ -497,8 +465,8 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, /* * Return a reference to a cached block in the system, allocating a new - * block if one isn't found in the rbtree. Its contents are undefined - * if it's newly allocated. + * block if one isn't found in the radix. Its contents are undefined if + * it's newly allocated. */ static struct block_private *block_get(struct super_block *sb, u64 blkno) { @@ -507,11 +475,11 @@ static struct block_private *block_get(struct super_block *sb, u64 blkno) struct block_private *bp; int ret; - spin_lock(&binf->lock); - bp = walk_block_rbtree(&binf->root, blkno, NULL); + rcu_read_lock(); + bp = radix_tree_lookup(&binf->radix, blkno); if (bp) atomic_inc(&bp->refcount); - spin_unlock(&binf->lock); + rcu_read_unlock(); /* drop failed reads that interrupted waiters abandoned */ if (bp && (test_bit(BLOCK_BIT_ERROR, &bp->bits) && @@ -530,15 +498,20 @@ static struct block_private *block_get(struct super_block *sb, u64 blkno) goto out; } - /* could refactor to insert in one walk */ + ret = radix_tree_preload(GFP_NOFS); + if (ret) + goto out; + + /* could use slot instead of lookup/insert */ spin_lock(&binf->lock); - found = walk_block_rbtree(&binf->root, blkno, NULL); + found = radix_tree_lookup(&binf->radix, blkno); if (found) { atomic_inc(&found->refcount); } else { block_insert(sb, bp, blkno); } spin_unlock(&binf->lock); + radix_tree_preload_end(); if (found) { block_put(sb, bp); @@ -985,8 +958,17 @@ int scoutfs_block_setup(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct block_info *binf; + loff_t size; int ret; + /* we store blknos in longs in the radix */ + size = i_size_read(sb->s_bdev->bd_inode); + if ((size >> SCOUTFS_BLOCK_LG_SHIFT) >= LONG_MAX) { + scoutfs_err(sb, "Cant reference all blocks in %llu byte device with %u bit long radix tree indexes", + size, BITS_PER_LONG); + return -EINVAL; + } + binf = kzalloc(sizeof(struct block_info), GFP_KERNEL); if (!binf) { ret = -ENOMEM; @@ -995,7 +977,7 @@ int scoutfs_block_setup(struct super_block *sb) binf->sb = sb; spin_lock_init(&binf->lock); - binf->root = RB_ROOT; + INIT_RADIX_TREE(&binf->radix, GFP_ATOMIC); /* insertion preloads */ INIT_LIST_HEAD(&binf->lru_list); init_waitqueue_head(&binf->waitq); binf->shrinker.shrink = block_shrink; From 4d0b78f5cbe5249c2e07cdfff38f83554188fd3b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 May 2020 16:56:06 -0700 Subject: [PATCH 837/920] scoutfs: add counters for server commits Add some counters for server commits. Signed-off-by: Zach Brown --- kmod/src/counters.h | 4 ++++ kmod/src/server.c | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 79da35cc..97b06dd1 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -105,6 +105,10 @@ EXPAND_COUNTER(radix_merge_empty) \ EXPAND_COUNTER(radix_undo_ref) \ EXPAND_COUNTER(radix_walk) \ + EXPAND_COUNTER(server_commit_hold) \ + EXPAND_COUNTER(server_commit_prepare) \ + EXPAND_COUNTER(server_commit_queue) \ + EXPAND_COUNTER(server_commit_worker) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ diff --git a/kmod/src/server.c b/kmod/src/server.c index 4ff46fc6..cb85fb24 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -139,6 +139,8 @@ int scoutfs_server_hold_commit(struct super_block *sb) u64 tot; int ret = 0; + scoutfs_inc_counter(sb, server_commit_hold); + down_read(&server->commit_rwsem); while (!server->prepared_commit) { @@ -146,6 +148,7 @@ int scoutfs_server_hold_commit(struct super_block *sb) down_write(&server->commit_rwsem); if (!server->prepared_commit) { + scoutfs_inc_counter(sb, server_commit_prepare); BUG_ON(scoutfs_block_writer_dirty_bytes(sb, &server->wri)); tot = le64_to_cpu(super->core_meta_freed.ref.sm_total); @@ -196,6 +199,7 @@ int scoutfs_server_apply_commit(struct super_block *sb, int err) cw.ret = 0; init_completion(&cw.comp); llist_add(&cw.node, &server->commit_waiters); + scoutfs_inc_counter(sb, server_commit_queue); queue_work(server->wq, &server->commit_work); } @@ -277,9 +281,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) int ret; trace_scoutfs_server_commit_work_enter(sb, 0, 0); + scoutfs_inc_counter(sb, server_commit_worker); down_write(&server->commit_rwsem); + ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { scoutfs_err(sb, "server error writing btree blocks: %d", ret); From 3a82090ab158c80f06aea26fbba4e1544ea2b33d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 12 May 2020 18:17:14 -0700 Subject: [PATCH 838/920] scoutfs: have per-fs inode nr allocators We had previously seen lock contention between mounts that were either resolving paths by looking up entries in directories or writing xattrs in file inodes as they did archiving work. The previous attempt to avoid this contention was to give each directory its own inode number allocator which ensured that inodes created for entries in the directory wouldn't share lock groups with inodes in other directories. But this creates the problem of operating on few files per lock for reasonably small directories. It also creates more server commits as each new directory gets its inode allocation reservation. The fix is to have mount-wide seperate allocators for directories and for everything else. This puts directories and files in seperate groups and locks, regardless of directory population. Signed-off-by: Zach Brown --- kmod/src/dir.c | 2 +- kmod/src/inode.c | 23 +++++++++++++++-------- kmod/src/inode.h | 11 +---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 18164384..12cd42ab 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -642,7 +642,7 @@ static struct inode *lock_hold_create(struct inode *dir, struct dentry *dentry, if (ret) return ERR_PTR(ret); - ret = scoutfs_alloc_ino(dir, &ino); + ret = scoutfs_alloc_ino(sb, S_ISDIR(mode), &ino); if (ret) return ERR_PTR(ret); diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 124d9375..efab16fd 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -47,9 +47,17 @@ * - describe data locking size problems */ +struct inode_allocator { + spinlock_t lock; + u64 ino; + u64 nr; +}; + struct inode_sb_info { spinlock_t writeback_lock; struct rb_root writeback_inodes; + struct inode_allocator dir_ino_alloc; + struct inode_allocator ino_alloc; }; #define DECLARE_INODE_SB_INFO(sb, name) \ @@ -74,7 +82,6 @@ static void scoutfs_inode_ctor(void *obj) init_waitqueue_head(&ci->data_waitq.waitq); init_rwsem(&ci->xattr_rwsem); RB_CLEAR_NODE(&ci->writeback_node); - spin_lock_init(&ci->ino_alloc.lock); inode_init_once(&ci->inode); } @@ -682,8 +689,6 @@ struct inode *scoutfs_iget(struct super_block *sb, u64 ino) /* XXX ensure refresh, instead clear in drop_inode? */ si = SCOUTFS_I(inode); atomic64_set(&si->last_refreshed, 0); - si->ino_alloc.ino = 0; - si->ino_alloc.nr = 0; ret = scoutfs_inode_refresh(inode, lock, 0); if (ret) { @@ -1322,14 +1327,16 @@ u64 scoutfs_last_ino(struct super_block *sb) * minimize that loss while still being large enough for typical * directory file counts. */ -int scoutfs_alloc_ino(struct inode *parent, u64 *ino_ret) +int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret) { - struct scoutfs_inode_allocator *ia = &SCOUTFS_I(parent)->ino_alloc; - struct super_block *sb = parent->i_sb; + DECLARE_INODE_SB_INFO(sb, inf); + struct inode_allocator *ia; u64 ino; u64 nr; int ret; + ia = is_dir ? &inf->dir_ino_alloc : &inf->ino_alloc; + spin_lock(&ia->lock); if (ia->nr == 0) { @@ -1385,8 +1392,6 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, ci->have_item = false; atomic64_set(&ci->last_refreshed, lock->refresh_gen); ci->flags = 0; - ci->ino_alloc.ino = 0; - ci->ino_alloc.nr = 0; scoutfs_inode_set_meta_seq(inode); scoutfs_inode_set_data_seq(inode); @@ -1725,6 +1730,8 @@ int scoutfs_inode_setup(struct super_block *sb) spin_lock_init(&inf->writeback_lock); inf->writeback_inodes = RB_ROOT; + spin_lock_init(&inf->dir_ino_alloc.lock); + spin_lock_init(&inf->ino_alloc.lock); sbi->inode_sb_info = inf; diff --git a/kmod/src/inode.h b/kmod/src/inode.h index 719fb391..9034aef4 100644 --- a/kmod/src/inode.h +++ b/kmod/src/inode.h @@ -10,12 +10,6 @@ struct scoutfs_lock; -struct scoutfs_inode_allocator { - spinlock_t lock; - u64 ino; - u64 nr; -}; - struct scoutfs_inode_info { /* read or initialized for each inode instance */ u64 ino; @@ -42,9 +36,6 @@ struct scoutfs_inode_info { /* updated at on each new lock acquisition */ atomic64_t last_refreshed; - /* reset for every new inode instance */ - struct scoutfs_inode_allocator ino_alloc; - /* initialized once for slab object */ seqcount_t seqcount; bool staging; /* holder of i_mutex is staging */ @@ -95,7 +86,7 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock); void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, struct list_head *ind_locks); -int scoutfs_alloc_ino(struct inode *parent, u64 *ino); +int scoutfs_alloc_ino(struct super_block *sb, bool is_dir, u64 *ino_ret); struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t rdev, u64 ino, struct scoutfs_lock *lock); From 0a47e8f936a6c1cd377ac37a547266dfcca07ba3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 May 2020 11:07:45 -0700 Subject: [PATCH 839/920] Revert "scoutfs: add block visited bit" The radix allocator no longer uses the block visited bit because it maintains its own much richer private per-block data stored off the priv pointer. Signed-off-by: Zach Brown This reverts commit 294b6d1f79e6d00ba60e26960c764d10c7f4b8a5. --- kmod/src/block.c | 17 ----------------- kmod/src/block.h | 4 ---- 2 files changed, 21 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index 1ed7d0d7..b14cb9fb 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -68,7 +68,6 @@ enum { BLOCK_BIT_PAGE_ALLOC, /* page (possibly high order) allocation */ BLOCK_BIT_VIRT, /* mapped virt allocation */ BLOCK_BIT_CRC_VALID, /* crc has been verified */ - BLOCK_BIT_VISITED, /* used by callers to track blocks */ }; struct block_private { @@ -130,22 +129,6 @@ bool scoutfs_block_valid_ref(struct super_block *sb, hdr->blkno == blkno; } -bool scoutfs_block_tas_visited(struct super_block *sb, - struct scoutfs_block *bl) -{ - struct block_private *bp = BLOCK_PRIVATE(bl); - - return test_bit(BLOCK_BIT_VISITED, &bp->bits) != 0; -} - -void scoutfs_block_clear_visited(struct super_block *sb, - struct scoutfs_block *bl) -{ - struct block_private *bp = BLOCK_PRIVATE(bl); - - clear_bit(BLOCK_BIT_VISITED, &bp->bits); -} - static struct block_private *block_alloc(struct super_block *sb, u64 blkno) { struct block_private *bp; diff --git a/kmod/src/block.h b/kmod/src/block.h index 6f73176f..e1b85359 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -18,10 +18,6 @@ bool scoutfs_block_valid_crc(struct scoutfs_block_header *hdr, u32 size); bool scoutfs_block_valid_ref(struct super_block *sb, struct scoutfs_block_header *hdr, __le64 seq, __le64 blkno); -bool scoutfs_block_tas_visited(struct super_block *sb, - struct scoutfs_block *bl); -void scoutfs_block_clear_visited(struct super_block *sb, - struct scoutfs_block *bl); struct scoutfs_block *scoutfs_block_create(struct super_block *sb, u64 blkno); struct scoutfs_block *scoutfs_block_read(struct super_block *sb, u64 blkno); From f9ff25db231fe29b415822667847c28db30f14fa Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 14 May 2020 15:07:52 -0700 Subject: [PATCH 840/920] scoutfs: add dirent name fingerprint Entries in a directory are indexed by the hash of their name. This introduces a perfectly random access pattern. And this results in a cow storm as directories get large enough such that the leaf blocks that store their entries are larger than our commits. Each commit ends up being full of cowed leaf blocks that contain a single new entry. The dirent name fingerprints change the dirent key to first start with a fingerprint of the name. This reduces the scope of hash randomization from the entire directory to entries with the same fingerprint. On real customer dir sizes and file names we saw roughly 3x create rate improvements from being able to create more entries in leaf blocks within a commit. Signed-off-by: Zach Brown --- kmod/src/dir.c | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 12cd42ab..a84d2134 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -213,12 +213,44 @@ static struct scoutfs_dirent *alloc_dirent(unsigned int name_len) return kmalloc(dirent_bytes(name_len), GFP_NOFS); } +/* + * Test a bit number as though an array of bytes is a large len-bit + * big-endian value. nr 0 is the LSB of the final byte, nr (len - 1) is + * the MSB of the first byte. + */ +static int test_be_bytes_bit(int nr, const char *bytes, int len) +{ + return bytes[(len - 1 - nr) >> 3] & (1 << (nr & 7)); +} + +/* + * Generate a 32bit "fingerprint" of the name by extracting 32 evenly + * distributed bits from the name. The intent is to have the sort order + * of the fingerprints reflect the memcmp() sort order of the names + * while mapping large names down to small fs keys. + * + * Names that are smaller than 32bits are biased towards the high bits + * of the fingerprint so that most significant bits of the fingerprints + * consistently reflect the initial characters of the names. + */ +static u32 dirent_name_fingerprint(const char *name, unsigned int name_len) +{ + int name_bits = name_len * 8; + int skip = max(name_bits / 32, 1); + u32 fp = 0; + int f; + int n; + + for (f = 31, n = name_bits - 1; f >= 0 && n >= 0; f--, n -= skip) + fp |= !!test_be_bytes_bit(n, name, name_bits) << f; + + return fp; +} + static u64 dirent_name_hash(const char *name, unsigned int name_len) { - unsigned int half = (name_len + 1) / 2; - - return crc32c(~0, name, half) | - ((u64)crc32c(~0, name + name_len - half, half) << 32); + return crc32c(~0, name, name_len) | + ((u64)dirent_name_fingerprint(name, name_len) << 32); } static u64 dirent_names_equal(const char *a_name, unsigned int a_len, From 2980edac536440874bcad5156583d552172b45a3 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 22 May 2020 11:54:16 -0700 Subject: [PATCH 841/920] scoutfs: restore btree block verification Signed-off-by: Zach Brown --- kmod/src/btree.c | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 969b831a..99c4ddfb 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -996,6 +996,186 @@ static int try_join(struct super_block *sb, return 1; } +static bool bad_item_off(int off, int nr) +{ + return (off < offsetof(struct scoutfs_btree_block, items[0])) || + (off >= offsetof(struct scoutfs_btree_block, items[nr])) || + ((off - offsetof(struct scoutfs_btree_block, items[0])) + % sizeof(struct scoutfs_btree_item)); +} + +static bool bad_avl_node_off(__le16 node_off, int nr) +{ + int item_off; + + if (node_off == 0) + return false; + + item_off = (int)le16_to_cpu(node_off) + + offsetof(struct scoutfs_btree_block, item_root) - + offsetof(struct scoutfs_btree_item, node); + + return bad_item_off(item_off, nr); +} + +/* + * XXX: + * - values don't overlap items + * - values don't overlap each other + * - last_free_offset is in fact last free region + * - call after leaf modification + */ +static void verify_btree_block(struct scoutfs_btree_block *bt, int level, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + __le16 *buckets = leaf_item_hash_buckets(bt); + struct scoutfs_btree_item *item; + char *reason = NULL; + int first_val = 0; + int hashed = 0; + __le16 *owner; + int end_off; + int tot = 0; + int i = 0; + int nr; + + if (bt->level != level) { + reason = "unexpected level"; + goto out; + } + + end_off = SCOUTFS_BLOCK_LG_SIZE - + (level ? 0 : SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES); + + /* can have 0 item blocks during first insertion into a tree */ + nr = le16_to_cpu(bt->nr_items); + if (nr < 0 || nr > SCOUTFS_BLOCK_LG_SIZE || + offsetof(struct scoutfs_btree_block, items[nr]) > end_off) { + reason = "nr_items out of range"; + goto out; + } + + if (bad_avl_node_off(bt->item_root.node, nr)) { + reason = "item_root node off"; + goto out; + } + + tot = 0; + first_val = end_off; + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + item = &bt->items[i]; + + if (bad_avl_node_off(item->node.parent, nr) || + bad_avl_node_off(item->node.left, nr) || + bad_avl_node_off(item->node.right, nr)) { + reason = "item node off"; + goto out; + } + + if (scoutfs_key_compare(&item->key, start) < 0 || + scoutfs_key_compare(&item->key, end) > 0) { + reason = "item key out of parent range"; + goto out; + } + + if (level == 0 && + leaf_item_hash_search(bt, &item->key) != item) { + reason = "item not found in hash"; + goto out; + } + + if (le16_to_cpu(item->val_len) > SCOUTFS_BTREE_MAX_VAL_LEN) { + reason = "bad item val len"; + goto out; + } + + if (((int)le16_to_cpu(item->val_off) + + le16_to_cpu(item->val_len) + + SCOUTFS_BTREE_VAL_OWNER_BYTES) > end_off) { + reason = "item value outside valid"; + goto out; + } + + tot += sizeof(struct scoutfs_btree_item) + + le16_to_cpu(item->val_len); + + if (item->val_len != 0) { + owner = off_ptr(bt, le16_to_cpu(item->val_off) + + le16_to_cpu(item->val_len)); + if (get_unaligned_le16(owner) != + offsetof(struct scoutfs_btree_block, items[i])) { + reason = "item value owner not item off"; + goto out; + } + + tot += SCOUTFS_BTREE_VAL_OWNER_BYTES; + first_val = min_t(int, first_val, + le16_to_cpu(item->val_off)); + } + } + + for (i = 0; level == 0 && i < SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; i++) { + if (buckets[i] == 0) + continue; + + if (bad_item_off(le16_to_cpu(buckets[i]), nr)) { + reason = "bad item hash offset"; + goto out; + } + + hashed++; + } + + if (level == 0 && hashed != nr) { + reason = "set hash buckets not nr"; + goto out; + } + + if (le16_to_cpu(bt->total_item_bytes) != tot) { + reason = "total_item_bytes not sum of items"; + goto out; + } + + /* value deletion doesn't merge with adjacent fragmented freed vals */ + if (le16_to_cpu(bt->mid_free_len) > + (first_val - offsetof(struct scoutfs_btree_block, items[nr]))) { + reason = "mid_free_len too large"; + goto out; + } +out: + if (!reason) + return; + + printk("found btree block inconsistency: %s\n", reason); + printk("start "SK_FMT" end "SK_FMT"\n", SK_ARG(start), SK_ARG(end)); + printk("calced: i %u tot %u hashed %u fv %u\n", + i, tot, hashed, first_val); + + printk("hdr: crc %x magic %x fsid %llx seq %llx blkno %llu\n", + le32_to_cpu(bt->hdr.crc), le32_to_cpu(bt->hdr.magic), + le64_to_cpu(bt->hdr.fsid), le64_to_cpu(bt->hdr.seq), + le64_to_cpu(bt->hdr.blkno)); + printk("item_root: node %u\n", le16_to_cpu(bt->item_root.node)); + printk("nr %u tib %u mfl %u lfo %u lfl %u lvl %u\n", + le16_to_cpu(bt->nr_items), le16_to_cpu(bt->total_item_bytes), + le16_to_cpu(bt->mid_free_len), le16_to_cpu(bt->last_free_off), + le16_to_cpu(bt->last_free_len), bt->level); + + for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { + item = &bt->items[i]; + printk(" %u: n %u,%u,%u,%u k "SK_FMT" vo %u vl %u\n", + i, le16_to_cpu(item->node.parent), + le16_to_cpu(item->node.left), + le16_to_cpu(item->node.right), item->node.height, + SK_ARG(&item->key), le16_to_cpu(item->val_off), + le16_to_cpu(item->val_len)); + } + + BUG(); +} + /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their @@ -1031,6 +1211,8 @@ static int btree_walk(struct super_block *sb, struct scoutfs_avl_node *next_node; struct scoutfs_avl_node *node; struct scoutfs_btree_ref *ref; + struct scoutfs_key start; + struct scoutfs_key end; unsigned int level; unsigned int nr; int ret; @@ -1047,6 +1229,8 @@ restart: scoutfs_block_put(sb, bl); bl = NULL; bt = NULL; + scoutfs_key_set_zeros(&start); + scoutfs_key_set_ones(&end); level = root->height; ret = 0; @@ -1073,6 +1257,9 @@ restart: break; bt = bl->data; + if (0) + verify_btree_block(bt, level, &start, &end); + /* XXX more aggressive block verification, before ref updates? */ if (bt->level != level) { scoutfs_corruption(sb, SC_BTREE_BLOCK_LEVEL, @@ -1140,6 +1327,13 @@ restart: *iter_key = *item_key(prev); } + /* possible range of keys in referenced child block */ + if ((prev = prev_item(bt, item))) { + start = *item_key(prev); + scoutfs_key_inc(&start); + } + end = *item_key(item); + scoutfs_block_put(sb, par_bl); par_bl = bl; parent = bt; From 69e5f5ae5fdd3b1376f7ced3fd6a10f99fcdfa83 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 27 May 2020 10:37:02 -0700 Subject: [PATCH 842/920] scoutfs: add btree walk trace point Signed-off-by: Zach Brown --- kmod/src/btree.c | 3 +++ kmod/src/scoutfs_trace.h | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 99c4ddfb..5bf6f23d 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1252,6 +1252,9 @@ restart: ref = &root->ref; while(level-- > 0) { + + trace_scoutfs_btree_walk(sb, root, key, flags, level, ref); + ret = get_ref_block(sb, alloc, wri, flags, ref, &bl); if (ret) break; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 47238c9d..d96a836e 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -1675,6 +1675,43 @@ TRACE_EVENT(scoutfs_btree_dirty_block, __entry->bt_blkno, __entry->bt_seq) ); +TRACE_EVENT(scoutfs_btree_walk, + TP_PROTO(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *key, int flags, int level, + struct scoutfs_btree_ref *ref), + + TP_ARGS(sb, root, key, flags, level, ref), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, root_blkno) + __field(__u64, root_seq) + __field(__u8, root_height) + sk_trace_define(key) + __field(int, flags) + __field(int, level) + __field(__u64, ref_blkno) + __field(__u64, ref_seq) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->root_blkno = le64_to_cpu(root->ref.blkno); + __entry->root_seq = le64_to_cpu(root->ref.seq); + __entry->root_height = root->height; + sk_trace_assign(key, key); + __entry->flags = flags; + __entry->level = level; + __entry->ref_blkno = le64_to_cpu(ref->blkno); + __entry->ref_seq = le64_to_cpu(ref->seq); + ), + + TP_printk(SCSBF" root blkno %llu seq %llu height %u key "SK_FMT" flags 0x%x level %d ref blkno %llu seq %llu", + SCSB_TRACE_ARGS, __entry->root_blkno, __entry->root_seq, + __entry->root_height, sk_trace_args(key), __entry->flags, + __entry->level, __entry->ref_blkno, __entry->ref_seq) +); + TRACE_EVENT(scoutfs_online_offline_blocks, TP_PROTO(struct inode *inode, s64 on_delta, s64 off_delta, u64 on_now, u64 off_now), From 07ba053021c154ce6af046b0bb9bfad68c393c2a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 1 Jun 2020 16:04:13 -0700 Subject: [PATCH 843/920] scoutfs: check super blkno fields We had a bug where mkfs would set a free data blkno allocator bit past the end of the device. (Just at it, in fact. Those fenceposts.) Add some checks at mount to make sure that the allocator blkno ranges in the super don't have obvious mistakes. Signed-off-by: Zach Brown --- kmod/src/super.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index dc3ff01b..efc7f1f7 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -242,6 +242,7 @@ int scoutfs_read_super(struct super_block *sb, { struct scoutfs_super_block *super; __le32 calc; + u64 blkno; int ret; super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); @@ -294,6 +295,51 @@ int scoutfs_read_super(struct super_block *sb, goto out; } + blkno = (SCOUTFS_QUORUM_BLKNO + SCOUTFS_QUORUM_BLOCKS) >> + SCOUTFS_BLOCK_SM_LG_SHIFT; + if (le64_to_cpu(super->first_meta_blkno) < blkno) { + scoutfs_err(sb, "super block first meta blkno %llu is within quorum blocks", + le64_to_cpu(super->first_meta_blkno)); + ret = -EINVAL; + goto out; + } + + if (le64_to_cpu(super->first_meta_blkno) > + le64_to_cpu(super->last_meta_blkno)) { + scoutfs_err(sb, "super block first meta blkno %llu is greater than last meta blkno %llu", + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno)); + ret = -EINVAL; + goto out; + } + + blkno = (le64_to_cpu(super->last_meta_blkno) + 1) << + SCOUTFS_BLOCK_SM_LG_SHIFT; + if (le64_to_cpu(super->first_data_blkno) < blkno) { + scoutfs_err(sb, "super block first data blkno %llu is within last meta blkno %llu", + le64_to_cpu(super->first_data_blkno), blkno); + ret = -EINVAL; + goto out; + } + + if (le64_to_cpu(super->first_data_blkno) > + le64_to_cpu(super->last_data_blkno)) { + scoutfs_err(sb, "super block first data blkno %llu is greater than last data blkno %llu", + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno)); + ret = -EINVAL; + goto out; + } + + blkno = (i_size_read(sb->s_bdev->bd_inode) >> + SCOUTFS_BLOCK_SM_SHIFT) - 1; + if (le64_to_cpu(super->last_data_blkno) > blkno) { + scoutfs_err(sb, "super block last data blkno %llu is outsite device size last blkno %llu", + le64_to_cpu(super->last_data_blkno), blkno); + ret = -EINVAL; + goto out; + } + *super_res = *super; ret = 0; out: From f48112e2a775f8a282ad3967f420011a64908db2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 3 Jun 2020 09:40:00 -0700 Subject: [PATCH 844/920] scoutfs: allocate contig block pages with nowarn We first attempt to allocate our large logically contiguous cached blocks with physically contiguous pages to minimize the impact on the tlb. When that fails we fall back to vmalloc()ed blocks. Sadly, high-order page allocation failure is expected and we forgot to provide the flag that suppresses the page allocation failure message. Signed-off-by: Zach Brown --- kmod/src/block.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index b14cb9fb..e730f7e5 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -144,7 +144,8 @@ static struct block_private *block_alloc(struct super_block *sb, u64 blkno) if (!bp) goto out; - bp->page = alloc_pages(GFP_NOFS, SCOUTFS_BLOCK_LG_PAGE_ORDER); + bp->page = alloc_pages(GFP_NOFS | __GFP_NOWARN, + SCOUTFS_BLOCK_LG_PAGE_ORDER); if (bp->page) { scoutfs_inc_counter(sb, block_cache_alloc_page_order); set_bit(BLOCK_BIT_PAGE_ALLOC, &bp->bits); From 42e7fbb4f7ee18265624b49b3870934b3a2b560e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 2 Jul 2020 15:10:53 -0700 Subject: [PATCH 845/920] scoutfs: switch to using fnv1a for hashing We had a few uses of crc for hashing. That was fine enough for initial testing but the huge number of xattrs that srch is recording was seeing very bad collisions from the clumsy combination of crc32c into a 64bit hash. Replace it with FNV for now. This also takes the opportunity to use 3 hash functions in the forest bloom filter so that we can extract them from the 64bit hash of the key rather than iterating and recalculating hashes for each function. Signed-off-by: Zach Brown --- kmod/src/btree.c | 4 ++-- kmod/src/dir.c | 4 ++-- kmod/src/forest.c | 16 +++++++++------- kmod/src/format.h | 5 +++-- kmod/src/hash.h | 46 ++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 56 insertions(+), 19 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 5bf6f23d..e162cc40 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -29,6 +28,7 @@ #include "block.h" #include "radix.h" #include "avl.h" +#include "hash.h" #include "scoutfs_trace.h" @@ -210,7 +210,7 @@ static int cmp_key_item(void *arg, struct scoutfs_avl_node *node) */ static int leaf_item_hash_ind(struct scoutfs_key *key) { - return crc32c(~0, key, sizeof(struct scoutfs_key)) % + return scoutfs_hash32(key, sizeof(struct scoutfs_key)) % SCOUTFS_BTREE_LEAF_ITEM_HASH_NR; } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index a84d2134..89fda146 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +30,7 @@ #include "kvec.h" #include "forest.h" #include "lock.h" +#include "hash.h" #include "counters.h" #include "scoutfs_trace.h" @@ -249,7 +249,7 @@ static u32 dirent_name_fingerprint(const char *name, unsigned int name_len) static u64 dirent_name_hash(const char *name, unsigned int name_len) { - return crc32c(~0, name, name_len) | + return scoutfs_hash32(name, name_len) | ((u64)dirent_name_fingerprint(name, name_len) << 32); } diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 7dc863ad..6fe22d0d 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -14,7 +14,6 @@ #include #include #include -#include #include "super.h" #include "format.h" @@ -24,6 +23,7 @@ #include "radix.h" #include "block.h" #include "forest.h" +#include "hash.h" #include "counters.h" #include "scoutfs_trace.h" @@ -240,18 +240,20 @@ static void read_unlock_forest_root(struct forest_info *finf, } } -/* - * XXX need something better. - */ static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, struct scoutfs_key *key) { - u32 crc = ~0; + u64 hash; int i; + BUILD_BUG_ON((SCOUTFS_FOREST_BLOOM_FUNC_BITS * + SCOUTFS_FOREST_BLOOM_NRS) > 64); + + hash = scoutfs_hash64(key, sizeof(struct scoutfs_key)); + for (i = 0; i < ARRAY_SIZE(bloom->nrs); i++) { - crc = crc32c(crc, key, sizeof(struct scoutfs_key)); - bloom->nrs[i] = crc % SCOUTFS_FOREST_BLOOM_BITS; + bloom->nrs[i] = (u32)hash % SCOUTFS_FOREST_BLOOM_BITS; + hash >>= SCOUTFS_FOREST_BLOOM_FUNC_BITS; } } diff --git a/kmod/src/format.h b/kmod/src/format.h index 8418a638..ac05fecf 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -328,11 +328,12 @@ struct scoutfs_bloom_block { * before the bloom filters fill up and start returning excessive false * positives. */ -#define SCOUTFS_FOREST_BLOOM_NRS 7 +#define SCOUTFS_FOREST_BLOOM_NRS 3 #define SCOUTFS_FOREST_BLOOM_BITS \ (((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_bloom_block)) / \ member_sizeof(struct scoutfs_bloom_block, bits[0])) * \ - member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) \ + member_sizeof(struct scoutfs_bloom_block, bits[0]) * 8) +#define SCOUTFS_FOREST_BLOOM_FUNC_BITS (SCOUTFS_BLOCK_LG_SHIFT + 3) /* * Keys are first sorted by major key zones. diff --git a/kmod/src/hash.h b/kmod/src/hash.h index 3ad3de09..9b169877 100644 --- a/kmod/src/hash.h +++ b/kmod/src/hash.h @@ -1,15 +1,49 @@ #ifndef _SCOUTFS_HASH_H_ #define _SCOUTFS_HASH_H_ -#include +/* + * We're using FNV1a for now. It's fine. Ish. + * + * The longer term plan is xxh3 but it looks like it'll take just a bit + * more time to be declared stable and then it needs to be ported to the + * kernel. + * + * - https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * - https://github.com/Cyan4973/xxHash/releases/tag/v0.7.4 + */ + +static inline u32 fnv1a32(const void *data, unsigned int len) +{ + u32 hash = 0x811c9dc5; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x01000193; + } + + return hash; +} + +static inline u64 fnv1a64(const void *data, unsigned int len) +{ + u64 hash = 0xcbf29ce484222325; + + while (len--) { + hash ^= *(u8 *)(data++); + hash *= 0x100000001b3; + } + + return hash; +} + +static inline u32 scoutfs_hash32(const void *data, unsigned int len) +{ + return fnv1a32(data, len); +} -/* XXX replace with xxhash */ static inline u64 scoutfs_hash64(const void *data, unsigned int len) { - unsigned int half = (len + 1) / 2; - - return crc32c(~0, data, half) | - ((u64)crc32c(~0, data + len - half, half) << 32); + return fnv1a64(data, len); } #endif From ab271f4682af8279ddf4a772810f0f225eb4ff12 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 3 Jul 2020 16:17:47 -0700 Subject: [PATCH 846/920] scoutfs: report sm metadata blocks in statfs The conversion of the super block metadata block counters to units of large metadata blocks forgot to scale back to the small block size when filling out the block count fields in the statfs rpc. This resulted in the free and total metadata use being off by the factor of large to small block size (default of ~16x at the moment). Signed-off-by: Zach Brown --- kmod/src/server.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index cb85fb24..ad21587f 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -849,6 +849,11 @@ out: &last_seq, sizeof(last_seq)); } +static inline __le64 le64_lg_to_sm(__le64 lg) +{ + return cpu_to_le64(le64_to_cpu(lg) << SCOUTFS_BLOCK_SM_LG_SHIFT); +} + /* * Sample the super stats that the client wants for statfs by serializing * with each component. @@ -872,10 +877,10 @@ static int server_statfs(struct super_block *sb, spin_unlock(&sbi->next_ino_lock); down_read(&server->alloc_rwsem); - nstatfs.total_blocks = super->total_meta_blocks; + nstatfs.total_blocks = le64_lg_to_sm(super->total_meta_blocks); le64_add_cpu(&nstatfs.total_blocks, le64_to_cpu(super->total_data_blocks)); - nstatfs.bfree = super->free_meta_blocks; + nstatfs.bfree = le64_lg_to_sm(super->free_meta_blocks); le64_add_cpu(&nstatfs.bfree, le64_to_cpu(super->free_data_blocks)); up_read(&server->alloc_rwsem); From 8c114ddb87d609930957a97e2097b1614c123809 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 29 Jun 2020 10:40:23 -0700 Subject: [PATCH 847/920] scoutfs: increase max btree item size Now that we have larger blocks we can have a larger max item. This was increased to make room for the srch compaction items which store a good number of srch files in their value. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index ac05fecf..3d78dc8d 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -213,7 +213,7 @@ struct scoutfs_avl_node { } __packed; /* when we split we want to have multiple items on each side */ -#define SCOUTFS_BTREE_MAX_VAL_LEN 512 +#define SCOUTFS_BTREE_MAX_VAL_LEN 896 /* each value ends with an offset which lets compaction iterate over values */ #define SCOUTFS_BTREE_VAL_OWNER_BYTES sizeof(__le16) From cca83b1758a7ae0a8d2da45e860ebbb75c5ed352 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 30 Jun 2020 10:33:31 -0700 Subject: [PATCH 848/920] scoutfs: rework get_fs_roots to get_roots The get_fs_roots rpc and server interfaces were built around individual roots. Rebuild it around passing around a struct so that we can add roots without impacting all the current users. Signed-off-by: Zach Brown --- kmod/src/client.c | 18 ++++---------- kmod/src/client.h | 5 ++-- kmod/src/forest.c | 34 +++++++++++++------------- kmod/src/format.h | 6 ++--- kmod/src/lock.c | 3 +-- kmod/src/lock.h | 3 +-- kmod/src/lock_server.c | 3 +-- kmod/src/server.c | 54 ++++++++++++++++++++---------------------- kmod/src/server.h | 5 ++-- 9 files changed, 57 insertions(+), 74 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 53d47942..fd118832 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -108,22 +108,14 @@ int scoutfs_client_commit_log_trees(struct super_block *sb, lt, sizeof(*lt), NULL, 0); } -int scoutfs_client_get_fs_roots(struct super_block *sb, - struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root) +int scoutfs_client_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots) { struct client_info *client = SCOUTFS_SB(sb)->client_info; - struct scoutfs_net_fs_roots nfr; - int ret; - ret = scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_GET_FS_ROOTS, - NULL, 0, &nfr, sizeof(nfr)); - if (ret == 0) { - *fs_root = nfr.fs_root; - *logs_root = nfr.logs_root; - } - return 0; + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_GET_ROOTS, + NULL, 0, roots, sizeof(*roots)); } int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq) diff --git a/kmod/src/client.h b/kmod/src/client.h index cfc5e482..8ad2b676 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -7,9 +7,8 @@ int scoutfs_client_get_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); int scoutfs_client_commit_log_trees(struct super_block *sb, struct scoutfs_log_trees *lt); -int scoutfs_client_get_fs_roots(struct super_block *sb, - struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root); +int scoutfs_client_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots); u64 *scoutfs_client_bulk_alloc(struct super_block *sb); int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 6fe22d0d..ff23f553 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -305,8 +305,7 @@ static int refresh_bloom_roots(struct super_block *sb, { DECLARE_FOREST_INFO(sb, finf); struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); - struct scoutfs_btree_root fs_root; - struct scoutfs_btree_root logs_root; + struct scoutfs_net_roots roots; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct forest_bloom_nrs bloom; @@ -327,26 +326,25 @@ static int refresh_bloom_roots(struct super_block *sb, /* first use the lock's constant roots, then sample newer roots */ if (!lpriv->used_lock_roots) { lpriv->used_lock_roots = 1; - fs_root = lock->fs_root; - logs_root = lock->logs_root; + roots = lock->roots; scoutfs_inc_counter(sb, forest_roots_lock); } else { - ret = scoutfs_client_get_fs_roots(sb, &fs_root, &logs_root); + ret = scoutfs_client_get_roots(sb, &roots); if (ret) goto out; scoutfs_inc_counter(sb, forest_roots_server); } - trace_scoutfs_forest_using_roots(sb, &fs_root, &logs_root); - refs->fs_ref = fs_root.ref; - refs->logs_ref = logs_root.ref; + trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); + refs->fs_ref = roots.fs_root.ref; + refs->logs_ref = roots.logs_root.ref; calc_bloom_nrs(&bloom, &lock->start); scoutfs_key_init_log_trees(&key, 0, 0); for (;; scoutfs_key_inc(&key)) { - ret = scoutfs_btree_next(sb, &logs_root, &key, &iref); + ret = scoutfs_btree_next(sb, &roots.logs_root, &key, &iref); if (ret == -ENOENT) { ret = 0; break; @@ -423,7 +421,7 @@ static int refresh_bloom_roots(struct super_block *sb, /* always add the fs root at the tail */ fr = &lpriv->fs_root; - fr->item_root = fs_root; + fr->item_root = roots.fs_root; fr->rid = 0; fr->nr = 0; list_add_tail(&fr->entry, &lpriv->roots); @@ -1028,8 +1026,7 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next) { DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); - struct scoutfs_btree_root fs_root; - struct scoutfs_btree_root logs_root; + struct scoutfs_net_roots roots; struct scoutfs_btree_root item_root; struct scoutfs_log_trees_val *ltv; SCOUTFS_BTREE_ITEM_REF(iref); @@ -1041,13 +1038,13 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, retry: scoutfs_inc_counter(sb, forest_roots_next_hint); - ret = scoutfs_client_get_fs_roots(sb, &fs_root, &logs_root); + ret = scoutfs_client_get_roots(sb, &roots); if (ret) goto out; - trace_scoutfs_forest_using_roots(sb, &fs_root, &logs_root); - refs.fs_ref = fs_root.ref; - refs.logs_ref = logs_root.ref; + trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); + refs.fs_ref = roots.fs_root.ref; + refs.logs_ref = roots.logs_root.ref; scoutfs_key_init_log_trees(<k, 0, 0); checked_fs = false; @@ -1056,9 +1053,10 @@ retry: for (;;) { if (!checked_fs) { checked_fs = true; - item_root = fs_root; + item_root = roots.fs_root; } else { - ret = scoutfs_btree_next(sb, &logs_root, <k, &iref); + ret = scoutfs_btree_next(sb, &roots.logs_root, <k, + &iref); if (ret == -ENOENT) { if (have_next) ret = 0; diff --git a/kmod/src/format.h b/kmod/src/format.h index 3d78dc8d..66222e8e 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -682,7 +682,7 @@ enum { SCOUTFS_NET_CMD_ALLOC_INODES, SCOUTFS_NET_CMD_GET_LOG_TREES, SCOUTFS_NET_CMD_COMMIT_LOG_TREES, - SCOUTFS_NET_CMD_GET_FS_ROOTS, + SCOUTFS_NET_CMD_GET_ROOTS, SCOUTFS_NET_CMD_ADVANCE_SEQ, SCOUTFS_NET_CMD_GET_LAST_SEQ, SCOUTFS_NET_CMD_STATFS, @@ -731,7 +731,7 @@ struct scoutfs_net_statfs { __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ } __packed; -struct scoutfs_net_fs_roots { +struct scoutfs_net_roots { struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; } __packed; @@ -745,7 +745,7 @@ struct scoutfs_net_lock { struct scoutfs_net_lock_grant_response { struct scoutfs_net_lock nl; - struct scoutfs_net_fs_roots nfr; + struct scoutfs_net_roots roots; } __packed; struct scoutfs_net_lock_recover { diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 69a593e2..e86d5b22 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -590,8 +590,7 @@ int scoutfs_lock_grant_response(struct super_block *sb, lock->request_pending = 0; lock->mode = nl->new_mode; lock->write_version = le64_to_cpu(nl->write_version); - lock->fs_root = gr->nfr.fs_root; - lock->logs_root = gr->nfr.logs_root; + lock->roots = gr->roots; if (lock_count_match_exists(nl->new_mode, lock->waiters)) extend_grace(sb, lock); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 971a12ff..f77f4ed0 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -22,8 +22,7 @@ struct scoutfs_lock { struct rb_node range_node; u64 refresh_gen; u64 write_version; - struct scoutfs_btree_root fs_root; - struct scoutfs_btree_root logs_root; + struct scoutfs_net_roots roots; struct list_head lru_head; wait_queue_head_t waitq; struct work_struct shrink_work; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 3d4cbeea..5ef53cdd 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -554,8 +554,7 @@ static int process_waiting_requests(struct super_block *sb, } gres.nl = nl; - scoutfs_server_get_fs_roots(sb, &gres.nfr.fs_root, - &gres.nfr.logs_root); + scoutfs_server_get_roots(sb, &gres.roots); ret = scoutfs_server_lock_response(sb, req->rid, req->net_id, &gres); diff --git a/kmod/src/server.c b/kmod/src/server.c index ad21587f..22798411 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -86,9 +86,8 @@ struct server_info { struct mutex logs_mutex; /* stable versions stored from commits, given in locks and rpcs */ - seqcount_t fs_roots_seqcount; - struct scoutfs_btree_root fs_root; - struct scoutfs_btree_root logs_root; + seqcount_t roots_seqcount; + struct scoutfs_net_roots roots; }; #define DECLARE_SERVER_INFO(sb, name) \ @@ -225,29 +224,27 @@ static void update_free_blocks(__le64 *blocks, struct scoutfs_radix_root *prev, le64_to_cpu(prev->ref.sm_total)); } -void scoutfs_server_get_fs_roots(struct super_block *sb, - struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root) +void scoutfs_server_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots) { DECLARE_SERVER_INFO(sb, server); unsigned int seq; do { - seq = read_seqcount_begin(&server->fs_roots_seqcount); - *fs_root = server->fs_root; - *logs_root = server->logs_root; - } while (read_seqcount_retry(&server->fs_roots_seqcount, seq)); + seq = read_seqcount_begin(&server->roots_seqcount); + *roots = server->roots; + } while (read_seqcount_retry(&server->roots_seqcount, seq)); } -static void set_fs_roots(struct server_info *server, - struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root) +static void set_roots(struct server_info *server, + struct scoutfs_btree_root *fs_root, + struct scoutfs_btree_root *logs_root) { preempt_disable(); - write_seqcount_begin(&server->fs_roots_seqcount); - server->fs_root = *fs_root; - server->logs_root = *logs_root; - write_seqcount_end(&server->fs_roots_seqcount); + write_seqcount_begin(&server->roots_seqcount); + server->roots.fs_root = *fs_root; + server->roots.logs_root = *logs_root; + write_seqcount_end(&server->roots_seqcount); preempt_enable(); } @@ -307,7 +304,7 @@ static void scoutfs_server_commit_func(struct work_struct *work) } server->prepared_commit = false; - set_fs_roots(server, &super->fs_root, &super->logs_root); + set_roots(server, &super->fs_root, &super->logs_root); ret = 0; out: node = llist_del_all(&server->commit_waiters); @@ -577,22 +574,23 @@ out: * visible in persistent storage. We don't want to accidentally give * them our in-memory dirty version. This can be racing with commits. */ -static int server_get_fs_roots(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) +static int server_get_roots(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) { - struct scoutfs_net_fs_roots nfr; + struct scoutfs_net_roots roots; int ret; if (arg_len != 0) { - memset(&nfr, 0, sizeof(nfr)); + memset(&roots, 0, sizeof(roots)); ret = -EINVAL; } else { - scoutfs_server_get_fs_roots(sb, &nfr.fs_root, &nfr.logs_root); + scoutfs_server_get_roots(sb, &roots); ret = 0; } - return scoutfs_net_response(sb, conn, cmd, id, 0, &nfr, sizeof(nfr)); + return scoutfs_net_response(sb, conn, cmd, id, 0, + &roots, sizeof(roots)); } /* @@ -1394,7 +1392,7 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_ALLOC_INODES] = server_alloc_inodes, [SCOUTFS_NET_CMD_GET_LOG_TREES] = server_get_log_trees, [SCOUTFS_NET_CMD_COMMIT_LOG_TREES] = server_commit_log_trees, - [SCOUTFS_NET_CMD_GET_FS_ROOTS] = server_get_fs_roots, + [SCOUTFS_NET_CMD_GET_ROOTS] = server_get_roots, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, [SCOUTFS_NET_CMD_STATFS] = server_statfs, @@ -1485,7 +1483,7 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret < 0) goto shutdown; - set_fs_roots(server, &super->fs_root, &super->logs_root); + set_roots(server, &super->fs_root, &super->logs_root); scoutfs_radix_init_alloc(&server->alloc, &super->core_meta_avail, &super->core_meta_freed); scoutfs_block_writer_init(sb, &server->wri); @@ -1625,7 +1623,7 @@ int scoutfs_server_setup(struct super_block *sb) INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); mutex_init(&server->logs_mutex); - seqcount_init(&server->fs_roots_seqcount); + seqcount_init(&server->roots_seqcount); server->wq = alloc_workqueue("scoutfs_server", WQ_UNBOUND | WQ_NON_REENTRANT, 0); diff --git a/kmod/src/server.h b/kmod/src/server.h index 07e95606..274a66ea 100644 --- a/kmod/src/server.h +++ b/kmod/src/server.h @@ -62,11 +62,10 @@ int scoutfs_server_lock_response(struct super_block *sb, u64 rid, u64 id, struct scoutfs_net_lock_grant_response *gr); int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, struct scoutfs_key *key); +void scoutfs_server_get_roots(struct super_block *sb, + struct scoutfs_net_roots *roots); int scoutfs_server_hold_commit(struct super_block *sb); int scoutfs_server_apply_commit(struct super_block *sb, int err); -void scoutfs_server_get_fs_roots(struct super_block *sb, - struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root); struct sockaddr_in; struct scoutfs_quorum_elected_info; From f8e181228825857e0a737ff75101ec2d6cdb5597 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Jun 2020 09:48:08 -0700 Subject: [PATCH 849/920] scoutfs: add srch infrastructure This introduces the srch mechanism that we'll use to accelerate finding files based on the presence of a given named xattr. This is an optimized version of the initial prototype that was using locked btree items for .indx. xattrs. This is built around specific compressed data structures, having the operation cost match the reality of orders of magnitude more writers than readers, and adopting a relaxed locking model. Combine all of this and maintaining the xattrs no longer tanks creation rates while maintaining excellent search latencies, given that searches are defined as rare and relatively expensive. The core data type is the srch entry which maps a hashed name to an inode number. Mounts can append entries to the end of unsorted log files during their transaction. The server tracks these files and rotates them into a list of files as they get large enough. Mounts have compaction work that regularly asks the server for a set of files to read and combine into a single sorted output file. The server only initiates compactions when it sees a number of files of roughly the same size. Searches then walk all the commited srch files, both log files and sorted compacted files, looking for entries that associate an xattr name with an inode number. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/client.c | 22 + kmod/src/client.h | 4 + kmod/src/counters.h | 18 + kmod/src/forest.c | 35 + kmod/src/forest.h | 1 + kmod/src/format.h | 101 +++ kmod/src/server.c | 174 +++- kmod/src/srch.c | 1974 +++++++++++++++++++++++++++++++++++++++++++ kmod/src/srch.h | 69 ++ kmod/src/super.c | 5 +- kmod/src/super.h | 2 + 12 files changed, 2401 insertions(+), 5 deletions(-) create mode 100644 kmod/src/srch.c create mode 100644 kmod/src/srch.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index dd3d3622..cf2c39ee 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -32,6 +32,7 @@ scoutfs-y += \ scoutfs_trace.o \ server.o \ spbm.o \ + srch.o \ super.o \ sysfs.o \ trans.o \ diff --git a/kmod/src/client.c b/kmod/src/client.c index fd118832..6809fde8 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -210,6 +210,28 @@ int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, net_id, 0, nlr, bytes); } +/* Find srch files that need to be compacted. */ +int scoutfs_client_srch_get_compact(struct super_block *sb, + struct scoutfs_srch_compact_input *scin) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_SRCH_GET_COMPACT, + NULL, 0, scin, sizeof(*scin)); +} + +/* Commit the result of a srch file compaction. */ +int scoutfs_client_srch_commit_compact(struct super_block *sb, + struct scoutfs_srch_compact_result *scres) +{ + struct client_info *client = SCOUTFS_SB(sb)->client_info; + + return scoutfs_net_sync_request(sb, client->conn, + SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, + scres, sizeof(*scres), NULL, 0); +} + /* The client is receiving a invalidation request from the server */ static int client_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, diff --git a/kmod/src/client.h b/kmod/src/client.h index 8ad2b676..21bf7a39 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -20,6 +20,10 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock *nl); int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock_recover *nlr); +int scoutfs_client_srch_get_compact(struct super_block *sb, + struct scoutfs_srch_compact_input *scin); +int scoutfs_client_srch_commit_compact(struct super_block *sb, + struct scoutfs_srch_compact_result *scres); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 97b06dd1..e96c01b0 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -109,6 +109,24 @@ EXPAND_COUNTER(server_commit_prepare) \ EXPAND_COUNTER(server_commit_queue) \ EXPAND_COUNTER(server_commit_worker) \ + EXPAND_COUNTER(srch_add_entry) \ + EXPAND_COUNTER(srch_compact_dirty_block) \ + EXPAND_COUNTER(srch_compact_entry) \ + EXPAND_COUNTER(srch_compact_flush) \ + EXPAND_COUNTER(srch_compact_free_block) \ + EXPAND_COUNTER(srch_compact_log_page) \ + EXPAND_COUNTER(srch_compact_removed_entry) \ + EXPAND_COUNTER(srch_inconsistent_ref) \ + EXPAND_COUNTER(srch_rotate_log) \ + EXPAND_COUNTER(srch_search_log) \ + EXPAND_COUNTER(srch_search_log_block) \ + EXPAND_COUNTER(srch_search_retry_empty) \ + EXPAND_COUNTER(srch_search_sorted) \ + EXPAND_COUNTER(srch_search_sorted_block) \ + EXPAND_COUNTER(srch_search_stale_eio) \ + EXPAND_COUNTER(srch_search_stale_retry) \ + EXPAND_COUNTER(srch_search_xattrs) \ + EXPAND_COUNTER(srch_read_stale) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_fsync) \ EXPAND_COUNTER(trans_commit_full) \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c index ff23f553..319ec906 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -24,6 +24,7 @@ #include "block.h" #include "forest.h" #include "hash.h" +#include "srch.h" #include "counters.h" #include "scoutfs_trace.h" @@ -65,6 +66,10 @@ struct forest_info { struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; + + struct mutex srch_mutex; + struct scoutfs_srch_file srch_file; + struct scoutfs_block *srch_bl; }; #define DECLARE_FOREST_INFO(sb, name) \ @@ -1457,6 +1462,27 @@ void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) { } +/* + * Add a srch entry to the current transaction's log file. It will be + * committed in a transaction along with the dirty btree blocks that + * hold dirty items. The srch entries aren't governed by lock + * consistency. + * + * We lock here because of the shared file and block reference. + * Typically these calls are a quick appending to the end of the block, + * but they will allocate or cow blocks every few thousand calls. + */ +int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id) +{ + DECLARE_FOREST_INFO(sb, finf); + int ret; + + mutex_lock(&finf->srch_mutex); + ret = scoutfs_srch_add(sb, finf->alloc, finf->wri, &finf->srch_file, + &finf->srch_bl, hash, ino, id); + mutex_unlock(&finf->srch_mutex); + return ret; +} /* * This is called from transactions as a new transaction opens and is @@ -1480,6 +1506,9 @@ void scoutfs_forest_init_btrees(struct super_block *sb, finf->our_log.bloom_ref = lt->bloom_ref; finf->our_log.rid = lt->rid; finf->our_log.nr = lt->nr; + finf->srch_file = lt->srch_file; + WARN_ON_ONCE(finf->srch_bl); /* commiting should have put the block */ + finf->srch_bl = NULL; up_write(&finf->rwsem); } @@ -1497,6 +1526,10 @@ void scoutfs_forest_get_btrees(struct super_block *sb, lt->item_root = finf->our_log.item_root; lt->bloom_ref = finf->our_log.bloom_ref; + lt->srch_file = finf->srch_file; + + scoutfs_block_put(sb, finf->srch_bl); + finf->srch_bl = NULL; trace_scoutfs_forest_prepare_commit(sb, <->item_root.ref, <->bloom_ref); @@ -1516,6 +1549,7 @@ int scoutfs_forest_setup(struct super_block *sb) /* the finf fields will be setup as we open a transaction */ init_rwsem(&finf->rwsem); + mutex_init(&finf->srch_mutex); sbi->forest_info = finf; ret = 0; @@ -1532,6 +1566,7 @@ void scoutfs_forest_destroy(struct super_block *sb) struct forest_info *finf = SCOUTFS_SB(sb)->forest_info; if (finf) { + scoutfs_block_put(sb, finf->srch_bl); kfree(finf); sbi->forest_info = NULL; } diff --git a/kmod/src/forest.h b/kmod/src/forest.h index c82f1a2b..e1411ad5 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -38,6 +38,7 @@ int scoutfs_forest_delete_save(struct super_block *sb, int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, struct scoutfs_lock *lock); void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); +int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); void scoutfs_forest_init_btrees(struct super_block *sb, struct scoutfs_radix_allocator *alloc, diff --git a/kmod/src/format.h b/kmod/src/format.h index 66222e8e..1e004fbe 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -9,6 +9,8 @@ #define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d #define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 #define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 +#define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d +#define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 /* * The super block, quorum block, and file data allocation granularity @@ -275,6 +277,93 @@ struct scoutfs_mounted_client_btree_val { #define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) +/* + * srch files are a contiguous run of blocks with compressed entries + * described by a dense parent radix. The files can be stored in + * log_tree items when the files contain unsorted entries written by + * mounts during their transactions. Sorted files of increasing size + * are kept in a btree off the super for searching and further + * compacting. + */ +struct scoutfs_srch_entry { + __le64 hash; + __le64 ino; + __le64 id; +} __packed; + +#define SCOUTFS_SRCH_ENTRY_MAX_BYTES (2 + (sizeof(__u64) * 3)) + +struct scoutfs_srch_ref { + __le64 blkno; + __le64 seq; +} __packed; + +struct scoutfs_srch_file { + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + __le64 blocks; + __le64 entries; + struct scoutfs_srch_ref ref; + __u8 height; +} __packed; + +struct scoutfs_srch_parent { + struct scoutfs_block_header hdr; + struct scoutfs_srch_ref refs[0]; +} __packed; + +#define SCOUTFS_SRCH_PARENT_REFS \ + ((SCOUTFS_BLOCK_LG_SIZE - \ + offsetof(struct scoutfs_srch_parent, refs)) / \ + sizeof(struct scoutfs_srch_ref)) + +struct scoutfs_srch_block { + struct scoutfs_block_header hdr; + struct scoutfs_srch_entry first; + struct scoutfs_srch_entry last; + struct scoutfs_srch_entry tail; + __le32 entry_nr; + __le32 entry_bytes; + __u8 entries[0]; +} __packed; + +/* + * Decoding loads final small deltas with full __u64 loads. Rather than + * check the size before each load we stop coding entries past the point + * where a full size entry could overflow the block. A final entry can + * start at this byte count and consume the rest of the block, though + * its unlikely. + */ +#define SCOUTFS_SRCH_BLOCK_SAFE_BYTES \ + (SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_srch_block) - \ + SCOUTFS_SRCH_ENTRY_MAX_BYTES) + +#define SCOUTFS_SRCH_LOG_BLOCK_LIMIT (1024 * 1024 / SCOUTFS_BLOCK_LG_SIZE) +#define SCOUTFS_SRCH_COMPACT_ORDER 3 +#define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) + +struct scoutfs_srch_compact_input { + struct scoutfs_radix_root meta_avail; + struct scoutfs_radix_root meta_freed; + __le64 id; + __u8 nr; + __u8 flags; + struct scoutfs_srch_file sfl[SCOUTFS_SRCH_COMPACT_NR]; +} __packed; + +struct scoutfs_srch_compact_result { + struct scoutfs_radix_root meta_avail; + struct scoutfs_radix_root meta_freed; + __le64 id; + __u8 flags; + struct scoutfs_srch_file sfl; +} __packed; + +/* files are insorted logs */ +#define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) +/* compaction failed, release inputs */ +#define SCOUTFS_SRCH_COMPACT_FLAG_ERROR (1 << 1) + /* * XXX I imagine we should rename these now that they've evolved to track * all the btrees that clients use during a transaction. It's not just @@ -287,6 +376,7 @@ struct scoutfs_log_trees { struct scoutfs_btree_ref bloom_ref; struct scoutfs_radix_root data_avail; struct scoutfs_radix_root data_freed; + struct scoutfs_srch_file srch_file; __le64 rid; __le64 nr; } __packed; @@ -298,6 +388,7 @@ struct scoutfs_log_trees_val { struct scoutfs_btree_ref bloom_ref; struct scoutfs_radix_root data_avail; struct scoutfs_radix_root data_freed; + struct scoutfs_srch_file srch_file; } __packed; struct scoutfs_log_item_value { @@ -348,6 +439,7 @@ struct scoutfs_bloom_block { #define SCOUTFS_LOCK_CLIENTS_ZONE 7 #define SCOUTFS_TRANS_SEQ_ZONE 8 #define SCOUTFS_MOUNTED_CLIENT_ZONE 9 +#define SCOUTFS_SRCH_ZONE 10 /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -372,6 +464,11 @@ struct scoutfs_bloom_block { /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 +/* srch zone, only in server btrees */ +#define SCOUTFS_SRCH_LOG_TYPE 1 +#define SCOUTFS_SRCH_BLOCKS_TYPE 2 +#define SCOUTFS_SRCH_BUSY_TYPE 3 + /* * The extents that map blocks in a fixed-size logical region of a file * are packed and stored in item values. The packed extents are @@ -496,6 +593,7 @@ struct scoutfs_super_block { struct scoutfs_btree_root lock_clients; struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; + struct scoutfs_btree_root srch_root; } __packed; #define SCOUTFS_ROOT_INO 1 @@ -688,6 +786,8 @@ enum { SCOUTFS_NET_CMD_STATFS, SCOUTFS_NET_CMD_LOCK, SCOUTFS_NET_CMD_LOCK_RECOVER, + SCOUTFS_NET_CMD_SRCH_GET_COMPACT, + SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, SCOUTFS_NET_CMD_FAREWELL, SCOUTFS_NET_CMD_UNKNOWN, }; @@ -734,6 +834,7 @@ struct scoutfs_net_statfs { struct scoutfs_net_roots { struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; + struct scoutfs_btree_root srch_root; } __packed; struct scoutfs_net_lock { diff --git a/kmod/src/server.c b/kmod/src/server.c index 22798411..e90e87ad 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -36,6 +36,7 @@ #include "endian_swap.h" #include "quorum.h" #include "trans.h" +#include "srch.h" /* * Every active mount can act as the server that listens on a net @@ -84,6 +85,7 @@ struct server_info { struct scoutfs_block_writer wri; struct mutex logs_mutex; + struct mutex srch_mutex; /* stable versions stored from commits, given in locks and rpcs */ seqcount_t roots_seqcount; @@ -238,12 +240,14 @@ void scoutfs_server_get_roots(struct super_block *sb, static void set_roots(struct server_info *server, struct scoutfs_btree_root *fs_root, - struct scoutfs_btree_root *logs_root) + struct scoutfs_btree_root *logs_root, + struct scoutfs_btree_root *srch_root) { preempt_disable(); write_seqcount_begin(&server->roots_seqcount); server->roots.fs_root = *fs_root; server->roots.logs_root = *logs_root; + server->roots.srch_root = *srch_root; write_seqcount_end(&server->roots_seqcount); preempt_enable(); } @@ -304,7 +308,8 @@ static void scoutfs_server_commit_func(struct work_struct *work) } server->prepared_commit = false; - set_roots(server, &super->fs_root, &super->logs_root); + set_roots(server, &super->fs_root, &super->logs_root, + &super->srch_root); ret = 0; out: node = llist_del_all(&server->commit_waiters); @@ -475,6 +480,7 @@ unlock: lt.bloom_ref = ltv.bloom_ref; lt.data_avail = ltv.data_avail; lt.data_freed = ltv.data_freed; + lt.srch_file = ltv.srch_file; lt.rid = key.sklt_rid; lt.nr = key.sklt_nr; } @@ -537,6 +543,16 @@ static int server_commit_log_trees(struct super_block *sb, goto unlock; } + /* try to rotate the srch log when big enough */ + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_rotate_log(sb, &server->alloc, &server->wri, + &super->srch_root, <->srch_file); + mutex_unlock(&server->srch_mutex); + if (ret < 0) { + scoutfs_err(sb, "server error, rotating srch log: %d", ret); + goto unlock; + } + update_free_blocks(&super->free_meta_blocks, <v.meta_avail, <->meta_avail); update_free_blocks(&super->free_meta_blocks, <v.meta_freed, @@ -552,6 +568,7 @@ static int server_commit_log_trees(struct super_block *sb, ltv.bloom_ref = lt->bloom_ref; ltv.data_avail = lt->data_avail; ltv.data_freed = lt->data_freed; + ltv.srch_file = lt->srch_file; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, &key, <v, sizeof(ltv)); @@ -970,6 +987,112 @@ int scoutfs_server_lock_recover_request(struct super_block *sb, u64 rid, NULL, NULL); } +static int server_srch_get_compact(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_srch_compact_input scin; + u64 blocks; + int ret; + int i; + + memset(&scin, 0, sizeof(scin)); + scoutfs_radix_root_init(sb, &scin.meta_avail, true); + scoutfs_radix_root_init(sb, &scin.meta_freed, true); + + if (arg_len != 0) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_get_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, &scin); + mutex_unlock(&server->srch_mutex); + if (ret == 0 && scin.nr == 0) + ret = -ENOENT; + if (ret < 0) + goto apply; + + /* provide ~3x input blocks to allocate, write+delete+cow */ + blocks = 0; + for (i = 0; i < scin.nr; i++) + blocks += le64_to_cpu(scin.sfl[i].blocks); + blocks *= 3; + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &scin.meta_avail, &server->alloc.avail, + &server->alloc.avail, true, blocks); + if (ret < 0) + goto apply; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_update_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, &scin); + mutex_unlock(&server->srch_mutex); + +apply: + ret = scoutfs_server_apply_commit(sb, ret); + WARN_ON_ONCE(ret < 0 && ret != -ENOENT); /* XXX leaked busy item */ +out: + return scoutfs_net_response(sb, conn, cmd, id, ret, + &scin, sizeof(scin)); +} + +static int server_srch_commit_compact(struct super_block *sb, + struct scoutfs_net_connection *conn, + u8 cmd, u64 id, void *arg, u16 arg_len) +{ + DECLARE_SERVER_INFO(sb, server); + u64 rid = scoutfs_net_client_rid(conn); + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_super_block *super = &sbi->super; + struct scoutfs_srch_compact_result *scres; + struct scoutfs_radix_root av; + struct scoutfs_radix_root fr; + int ret; + + scres = arg; + if (arg_len != sizeof(*scres)) { + ret = -EINVAL; + goto out; + } + + ret = scoutfs_server_hold_commit(sb); + if (ret) + goto out; + + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_commit_compact(sb, &server->alloc, &server->wri, + &super->srch_root, rid, scres, + &av, &fr); + mutex_unlock(&server->srch_mutex); + if (ret < 0) /* XXX very bad, leaks allocators */ + goto apply; + + /* XXX like all merges, doesn't reclaim allocator blocks themselves */ + + /* merge the client's allocators into freed, commit before reuse */ + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.freed, &av, &av, true, + le64_to_cpu(av.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.freed, &fr, &fr, true, + le64_to_cpu(fr.ref.sm_total)); +apply: + ret = scoutfs_server_apply_commit(sb, ret); +out: + WARN_ON(ret < 0); /* XXX leaks allocators */ + return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0); +} + static void init_mounted_client_key(struct scoutfs_key *key, u64 rid) { *key = (struct scoutfs_key) { @@ -1023,6 +1146,44 @@ static int delete_mounted_client(struct super_block *sb, u64 rid) return ret; } +/* + * Remove all the busy items for srch compactions that the mount might + * have been responsible for and reclaim all their allocators. + */ +static int cancel_srch_compact(struct super_block *sb, u64 rid) +{ + DECLARE_SERVER_INFO(sb, server); + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_radix_root av; + struct scoutfs_radix_root fr; + int ret; + + for (;;) { + mutex_lock(&server->srch_mutex); + ret = scoutfs_srch_cancel_compact(sb, &server->alloc, + &server->wri, + &super->srch_root, rid, + &av, &fr); + mutex_unlock(&server->srch_mutex); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.freed, &av, &av, true, + le64_to_cpu(av.ref.sm_total)) ?: + scoutfs_radix_merge(sb, &server->alloc, &server->wri, + &server->alloc.freed, &fr, &fr, true, + le64_to_cpu(fr.ref.sm_total)); + if (WARN_ON_ONCE(ret < 0)) + break; + } + + return ret; +} + /* * Process an incoming greeting request in the server from the client. * We try to send responses to failed greetings so that the sender can @@ -1283,7 +1444,8 @@ static void farewell_worker(struct work_struct *work) ret = scoutfs_lock_server_farewell(sb, fw->rid) ?: remove_trans_seq(sb, fw->rid) ?: reclaim_log_trees(sb, fw->rid) ?: - delete_mounted_client(sb, fw->rid); + delete_mounted_client(sb, fw->rid) ?: + cancel_srch_compact(sb, fw->rid); ret = scoutfs_server_apply_commit(sb, ret); if (ret) @@ -1397,6 +1559,8 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, [SCOUTFS_NET_CMD_STATFS] = server_statfs, [SCOUTFS_NET_CMD_LOCK] = server_lock, + [SCOUTFS_NET_CMD_SRCH_GET_COMPACT] = server_srch_get_compact, + [SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT] = server_srch_commit_compact, [SCOUTFS_NET_CMD_FAREWELL] = server_farewell, }; @@ -1483,7 +1647,8 @@ static void scoutfs_server_worker(struct work_struct *work) if (ret < 0) goto shutdown; - set_roots(server, &super->fs_root, &super->logs_root); + set_roots(server, &super->fs_root, &super->logs_root, + &super->srch_root); scoutfs_radix_init_alloc(&server->alloc, &super->core_meta_avail, &super->core_meta_freed); scoutfs_block_writer_init(sb, &server->wri); @@ -1623,6 +1788,7 @@ int scoutfs_server_setup(struct super_block *sb) INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); mutex_init(&server->logs_mutex); + mutex_init(&server->srch_mutex); seqcount_init(&server->roots_seqcount); server->wq = alloc_workqueue("scoutfs_server", diff --git a/kmod/src/srch.c b/kmod/src/srch.c new file mode 100644 index 00000000..db45fba8 --- /dev/null +++ b/kmod/src/srch.c @@ -0,0 +1,1974 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "format.h" +#include "counters.h" +#include "block.h" +#include "radix.h" +#include "srch.h" +#include "btree.h" +#include "spbm.h" +#include "client.h" +#include "scoutfs_trace.h" + +/* + * This srch subsystem gives us a way to find inodes that have a given + * tagged xattr set. It's designed for an xattr population that is + * orders of magnitudes larger than the file population, is updated much + * more frequently than it is searched, and can have slightly relaxed + * consistency requirements so that searches don't have to serialize + * with updates through locking. + * + * A srch entry is logged every time a .srch. xattr is created or + * deleted. Commits append entries to a growing srch log file along + * with the item btree and allocator block structures they're modifying. + * + * The server regularly rotates these growing log files so that they + * don't exceed a given size. Once there are enough log files they're + * all read and their sorted entries are written to a larger sorted + * file. Once there are enough sorted files they're all read and their + * combined sorted entries are written to a larger file, and so on. + * + * Searches combine all the entries read from unsorted log files and + * binary searches of larger sorted files to come up with the candidate + * inodes that probably contain the given named .srch. xattr. + * + * Searches read rotated log files and sorted files which have been + * committed. There is nothing protecting their blocks from being + * re-allocated and re-written. Search can restart by checking the + * btree for the current set of files. Compaction reads log files which + * are protected from other compactions by the persistent busy items + * created by the server. Compaction won't see it's blocks reused out + * from under it, but it can encounter stale cached blocks that need to + * be invalidated. + */ + +struct srch_info { + struct super_block *sb; + atomic_t shutdown; + struct workqueue_struct *workq; + struct delayed_work compact_dwork; +}; + +#define DECLARE_SRCH_INFO(sb, name) \ + struct srch_info *name = SCOUTFS_SB(sb)->srch_info + +#define SRE_FMT "%016llx.%llu.%llu" +#define SRE_ARG(sre) \ + le64_to_cpu((sre)->hash), le64_to_cpu((sre)->ino), \ + le64_to_cpu((sre)->id) + +/* + * Compactions dirty radix allocator blocks, file radix parent blocks, + * and especially srch file blocks. The files can get enormous and we + * can't have compactions OOM the box but they're meant to be large + * streaming operations, so we only stop and write out dirty blocks in + * large chunks. + */ +#define SRCH_COMPACT_DIRTY_LIMIT_BYTES (32 * 1024 * 1024) + +static int sre_cmp(const struct scoutfs_srch_entry *a, + const struct scoutfs_srch_entry *b) +{ + return scoutfs_cmp_u64s(le64_to_cpu(a->hash), le64_to_cpu(b->hash)) ?: + scoutfs_cmp_u64s(le64_to_cpu(a->ino), le64_to_cpu(b->ino)) ?: + scoutfs_cmp_u64s(le64_to_cpu(a->id), le64_to_cpu(b->id)); +} + +/* + * srch items are first grouped by type and we have log files, sorted + * files, and busy compactions. + */ +static void init_srch_key(struct scoutfs_key *key, int type, + u64 major, u64 minor) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_SRCH_ZONE, + .sk_type = type, + ._sk_second = cpu_to_le64(major), + ._sk_third = cpu_to_le64(minor), + }; +} + +/* + * The caller has ensured that there is space for a full word at the + * buf. Only the set low order bytes will be used. The clear high + * order bytes will be overwritten in the future and ignored in the + * final encoding in the block. + */ +static int encode_u64(__le64 *buf, u64 val) +{ + int bytes; + + val = (val << 1) ^ ((s64)val >> 63); /* shift sign extend */ + bytes = (fls64(val) + 7) >> 3; + + put_unaligned_le64(val, buf); + return bytes; +} + +/* shifting by width is undefined :/ */ +#define BYTE_MASK(b) ((1ULL << (b << 3)) - 1) +static u64 byte_masks[] = { + 0, BYTE_MASK(1), BYTE_MASK(2), BYTE_MASK(3), + BYTE_MASK(4), BYTE_MASK(5), BYTE_MASK(6), BYTE_MASK(7), U64_MAX, +}; + +static u64 decode_u64(void *buf, int bytes) +{ + u64 val = get_unaligned_le64(buf) & byte_masks[bytes]; + + return (val >> 1) ^ (-(val & 1)); +} + +/* + * Encode an entry at the offset in the block. Leave room for the + * lengths short, encode the diff of the encoded entry from the + * previous, then update the length short with the length of each + * encoded diff. The caller ensures that there's room for a full size + * entry at position in the block. + */ +static int encode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev) +{ + u64 diffs[] = { + le64_to_cpu(sre->hash) - le64_to_cpu(prev->hash), + le64_to_cpu(sre->ino) - le64_to_cpu(prev->ino), + le64_to_cpu(sre->id) - le64_to_cpu(prev->id), + }; + u16 lengths = 0; + int bytes; + int tot = 2; + int i; + + for (i = 0; i < ARRAY_SIZE(diffs); i++) { + bytes = encode_u64(buf + tot, diffs[i]); + lengths |= bytes << (i << 2); + tot += bytes; + } + + put_unaligned_le16(lengths, buf); + + return tot; +} + +/* + * Decode an entry from the offset of the block. Load the length short + * and decode the bytes of diffs and apply them to the previous entry. + * The caller ensures that we won't read off the end of block if we were + * to try and decode a full size set of diffs. + */ +static int decode_entry(void *buf, struct scoutfs_srch_entry *sre, + struct scoutfs_srch_entry *prev) +{ + u64 diffs[3]; + u16 lengths; + int bytes; + int tot; + int i; + + lengths = get_unaligned_le16(buf); + tot = 2; + + for (i = 0; i < ARRAY_SIZE(diffs); i++) { + bytes = min_t(int, 8, lengths & 15); + diffs[i] = decode_u64(buf + tot, bytes); + tot += bytes; + lengths >>= 4; + } + + sre->hash = cpu_to_le64(le64_to_cpu(prev->hash) + diffs[0]); + sre->ino = cpu_to_le64(le64_to_cpu(prev->ino) + diffs[1]); + sre->id = cpu_to_le64(le64_to_cpu(prev->id) + diffs[2]); + + return tot; +} + +/* return refs ind to traverse through parent at level to blk */ +static int calc_ref_ind(u64 blk, int level) +{ + int ind; + int i; + + BUG_ON(level < 1); + + for (i = 1; i <= level; i++) + blk = div_u64_rem(blk, SCOUTFS_SRCH_PARENT_REFS, &ind); + + return ind; +} + +static u8 height_for_blk(u64 blk) +{ + u64 total = SCOUTFS_SRCH_PARENT_REFS; + int hei = 2; + + if (blk == 0) + return 1; + + while (blk >= total) { + hei++; + total *= SCOUTFS_SRCH_PARENT_REFS; + } + + return hei; +} + +static void init_file_block(struct super_block *sb, struct scoutfs_block *bl, + int level) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_block_header *hdr; + + /* don't leak uninit kernel mem.. block should do this for us? */ + memset(bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + + hdr = bl->data; + hdr->fsid = super->hdr.fsid; + hdr->blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&hdr->seq, sizeof(hdr->seq)); + + if (level) + hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_PARENT); + else + hdr->magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK); +} + +/* + * This is operating on behalf of writers writing into private files and + * readers who could see stale blocks. We can find stale cached blocks + * and should retry the read ourselves after invalidating, but if we hit + * stale blocks on disk then we have to return to the caller who can + * decide to return errors or retry. + */ +static int read_srch_block(struct super_block *sb, + struct scoutfs_block_writer *wri, int level, + struct scoutfs_srch_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl; + int retries = 0; + int ret = 0; + int mag; + + mag = level ? SCOUTFS_BLOCK_MAGIC_SRCH_PARENT : + SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK; +retry: + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR_OR_NULL(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, mag)) { + + scoutfs_inc_counter(sb, srch_inconsistent_ref); + scoutfs_block_writer_forget(sb, wri, bl); + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = NULL; + + if (retries++ == 0) + goto retry; + + bl = ERR_PTR(-ESTALE); + scoutfs_inc_counter(sb, srch_read_stale); + } + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + bl = NULL; + } + + *bl_ret = bl; + return ret; +} + +/* + * Walk radix blocks to find the logical file block and return the + * reference to the caller. Flags determine if we cow new dirty blocks, + * allocate new blocks, or return errors for missing blocks (files are + * never sparse, this won't happen). + */ +enum { + GFB_INSERT = (1 << 0), + GFB_DIRTY = (1 << 1), +}; +static int get_file_block(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + int gfb, u64 blk, struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *parent = NULL; + struct scoutfs_block_header *hdr; + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_parent *srp; + struct scoutfs_block *new_bl; + struct scoutfs_srch_ref *ref; + u64 blkno = 0; + int level; + int ind; + int err; + int ret; + u8 hei; + + /* see if we need to grow to insert a new largest blk */ + hei = height_for_blk(blk); + while (sfl->height < hei) { + if (!(gfb & GFB_INSERT)) { + ret = -ENOENT; + goto out; + } + + ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; + + bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + blkno = 0; + + scoutfs_block_writer_mark_dirty(sb, wri, bl); + + init_file_block(sb, bl, sfl->height); + if (sfl->height) { + srp = bl->data; + srp->refs[0].blkno = sfl->ref.blkno; + srp->refs[0].seq = sfl->ref.seq; + } + + hdr = bl->data; + sfl->ref.blkno = hdr->blkno; + sfl->ref.seq = hdr->seq; + sfl->height++; + scoutfs_block_put(sb, bl); + bl = NULL; + } + + /* walk file and parent block references to the leaf blocks */ + level = sfl->height; + ref = &sfl->ref; + while (level--) { + /* searchin an unused part of the tree */ + if (!ref->blkno && !(gfb & GFB_INSERT)) { + ret = -ENOENT; + goto out; + } + + /* read an existing block */ + if (ref->blkno) { + ret = read_srch_block(sb, wri, level, ref, &bl); + if (ret < 0) + goto out; + } + + /* allocate a new block if we need it */ + if (!ref->blkno || ((gfb & GFB_DIRTY) && + !scoutfs_block_writer_is_dirty(sb, bl))) { + ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); + if (ret < 0) + goto out; + + new_bl = scoutfs_block_create(sb, blkno); + if (IS_ERR(new_bl)) { + ret = PTR_ERR(new_bl); + goto out; + } + + if (bl) { + /* cow old block if we have one */ + ret = scoutfs_radix_free(sb, alloc, wri, + bl->blkno); + if (ret) + goto out; + + memcpy(new_bl->data, bl->data, + SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = new_bl; + hdr = bl->data; + hdr->blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&hdr->seq, sizeof(hdr->seq)); + } else { + /* init new allocated block */ + bl = new_bl; + init_file_block(sb, bl, level); + } + + blkno = 0; + scoutfs_block_writer_mark_dirty(sb, wri, bl); + + /* update file or parent block ref */ + hdr = bl->data; + ref->blkno = hdr->blkno; + ref->seq = hdr->seq; + } + + if (level == 0) { + ret = 0; + break; + } + + srp = bl->data; + ind = calc_ref_ind(blk, level); + ref = &srp->refs[ind]; + + scoutfs_block_put(sb, parent); + parent = bl; + bl = NULL; + } + ret = 0; + +out: + scoutfs_block_put(sb, parent); + + /* return allocated blkno on error */ + if (blkno > 0) { + err = scoutfs_radix_free(sb, alloc, wri, blkno); + BUG_ON(err); /* radix should have been dirty */ + } + + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + + /* record that we successfully grew the file */ + if (ret == 0 && (gfb & GFB_INSERT) && blk >= le64_to_cpu(sfl->blocks)) + sfl->blocks = cpu_to_le64(blk + 1); + + *bl_ret = bl; + return ret; +} + +int scoutfs_srch_add(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + struct scoutfs_block **bl_ret, + u64 hash, u64 ino, u64 id) +{ + struct scoutfs_srch_block *srb; + struct scoutfs_block *bl = NULL; + u64 blk; + int ret; + struct scoutfs_srch_entry sre = { + .hash = cpu_to_le64(hash), + .ino = cpu_to_le64(ino), + .id = cpu_to_le64(id), + }; + + /* start with a new block or the last existing block */ + if (le64_to_cpu(sfl->blocks) > 1) + blk = le64_to_cpu(sfl->blocks) - 1; + else + blk = 0; + + bl = *bl_ret; +get_last_block: + if (bl == NULL) { + ret = get_file_block(sb, alloc, wri, sfl, + GFB_INSERT | GFB_DIRTY, blk, &bl); + if (ret < 0) { + /* writing into a private file, shouldn't happen */ + WARN_ON_ONCE(ret == -ESTALE); + goto out; + } + } + srb = bl->data; + + /* stop encoding once we might overflow the block */ + if (le32_to_cpu(srb->entry_bytes) > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + scoutfs_block_put(sb, bl); + bl = NULL; + blk++; + goto get_last_block; + } + + ret = encode_entry(srb->entries + le32_to_cpu(srb->entry_bytes), + &sre, &srb->tail); + if (ret > 0) { + if (srb->entry_bytes == 0) { + if (blk == 0) { + sfl->first = sre; + sfl->last = sre; + } + srb->first = sre; + srb->last = sre; + } else { + if (sre_cmp(&sre, &sfl->first) < 0) + sfl->first = sre; + else if (sre_cmp(&sre, &sfl->last) > 0) + sfl->last = sre; + if (sre_cmp(&sre, &srb->first) < 0) + srb->first = sre; + else if (sre_cmp(&sre, &srb->last) > 0) + srb->last = sre; + } + srb->tail = sre; + le32_add_cpu(&srb->entry_nr, 1); + le32_add_cpu(&srb->entry_bytes, ret); + le64_add_cpu(&sfl->entries, 1); + ret = 0; + scoutfs_inc_counter(sb, srch_add_entry); + } + +out: + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + *bl_ret = bl; + + return ret; +} + +/* + * Track an inode and id of an xattr hash that we found while searching. + * We'll return inos from the nodes in order to userspace when we're + * done searching. The first time we see the entry we track it, the + * second time must be a deletion so we remove it + * + * We track the size of the pool of tracked inodes here. Once its full + * we're still able to replace greater inodes with earlier ones. We do + * that work here because we can minimize the number of traversals and + * comparisons that the caller would otherwise have to make. + */ +static int track_found(struct scoutfs_srch_rb_root *sroot, u64 ino, u64 id, + unsigned long limit) +{ + struct rb_node **node = &sroot->root.rb_node; + struct rb_node *parent = NULL; + struct scoutfs_srch_rb_node *snode; + int cmp = 1; /* set last for first insertion */ + + while (*node) { + parent = *node; + snode = container_of(*node, struct scoutfs_srch_rb_node, node); + + cmp = scoutfs_cmp(ino, snode->ino) ?: + scoutfs_cmp(id, snode->id); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + /* update last if removed as a dupe */ + if (sroot->last == &snode->node) + sroot->last = rb_prev(sroot->last); + rb_erase(&snode->node, &sroot->root); + kfree(snode); + sroot->nr--; + return 0; + } + } + + /* can't track greater while we're at the limit */ + if (sroot->nr >= limit && cmp > 0 && parent == sroot->last) + return -ENOSPC; + + snode = kzalloc(sizeof(*snode), GFP_NOFS); + if (!snode) + return -ENOMEM; + + rb_link_node(&snode->node, parent, node); + rb_insert_color(&snode->node, &sroot->root); + + /* track a newly inserted last item */ + if (cmp > 0 && parent == sroot->last) + sroot->last = &snode->node; + + snode->ino = ino; + snode->id = id; + sroot->nr++; + + /* remove and update last if we inserted earlier at limit */ + if (sroot->nr > limit && sroot->last != &snode->node) { + snode = container_of(sroot->last, struct scoutfs_srch_rb_node, + node); + sroot->last = rb_prev(sroot->last); + rb_erase(&snode->node, &sroot->root); + kfree(snode); + sroot->nr--; + } + + return 0; +} + +/* + * Sweep all the unsorted entries of a log file looking for hash matches + * and tracking their xattr inos and ids. If the tracking sroot fills + * we update end but keep searching because we might find earlier + * entries. + */ +static int search_log_file(struct super_block *sb, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, + unsigned long limit) +{ + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_entry sre; + struct scoutfs_srch_entry prev; + struct scoutfs_srch_block *srb; + int ret = 0; + u64 blk; + int pos; + int i; + + for (blk = 0; blk < le64_to_cpu(sfl->blocks); blk++) { + scoutfs_block_put(sb, bl); + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + break; + srb = bl->data; + + memset(&prev, 0, sizeof(prev)); + pos = 0; + scoutfs_inc_counter(sb, srch_search_log_block); + + for (i = 0; i < le32_to_cpu(srb->entry_nr); i++) { + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, &sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + pos += ret; + prev = sre; + + if (sre_cmp(start, &sre) > 0 || + sre_cmp(&sre, end) > 0) + continue; + + ret = track_found(sroot, le64_to_cpu(sre.ino), + le64_to_cpu(sre.id), limit); + if (ret < 0) { + /* have to keep searching */ + if (ret == -ENOSPC) { + if (sre_cmp(&sre, end) < 0) + *end = sre; + ret = 0; + } else { + break; + } + } + } + } + + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Search a sorted file for entries for inodes that could contain the + * xattr hash that we're looking for. The caller has checked that the + * start entry is contained in the file. We find the first block that + * could contain it and stream entries from there until we fill the + * rbtree or arrive at the end entry. + */ +static int search_sorted_file(struct super_block *sb, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, + unsigned long limit) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry sre; + struct scoutfs_srch_entry prev; + struct scoutfs_block *bl = NULL; + int ret = 0; + int pos = 0; + u64 left; + u64 right; + u64 blk; + + /* binary search for the block that contains the start */ + blk = 0; + left = 0; + right = le64_to_cpu(sfl->blocks) - 1; + while (left != right) { + blk = (left + right) >> 1; + + scoutfs_block_put(sb, bl); + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + + if (sre_cmp(start, &srb->first) < 0) + right = --blk; + else if (sre_cmp(start, &srb->last) > 0) + left = ++blk; + else + break; + } + + /* blk is the result of the search */ + scoutfs_block_put(sb, bl); + bl = NULL; + + /* stream entries until end or we're past the full tracking rb_root */ + for (;;) { + if (bl == NULL) { + /* only check on each new input block */ + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + + memset(&prev, 0, sizeof(prev)); + pos = 0; + scoutfs_inc_counter(sb, srch_search_sorted_block); + } + + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, &sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + pos += ret; + prev = sre; + + + if (sre_cmp(start, &sre) > 0) + continue; + if (sre_cmp(&sre, end) > 0) + break; + + ret = track_found(sroot, le64_to_cpu(sre.ino), + le64_to_cpu(sre.id), limit); + if (ret < 0) { + if (ret == -ENOSPC) { + ret = 0; + /* done when we're past full rb_root */ + if (sre_cmp(&sre, end) < 0) + *end = sre; + break; + } + goto out; + } + + if (pos >= le32_to_cpu(srb->entry_bytes)) { + scoutfs_block_put(sb, bl); + bl = NULL; + if (++blk == le64_to_cpu(sfl->blocks)) + break; + } + } + ret = 0; +out: + scoutfs_block_put(sb, bl); + return ret; +} + +static int search_file(struct super_block *sb, int type, + struct scoutfs_srch_file *sfl, + struct scoutfs_srch_rb_root *sroot, + struct scoutfs_srch_entry *start, + struct scoutfs_srch_entry *end, unsigned long limit) +{ + + /* ignore files that don't have our hash */ + if (sre_cmp(start, &sfl->last) > 0 || + sre_cmp(end, &sfl->first) < 0) + return 0; + + if (type == SCOUTFS_SRCH_LOG_TYPE) { + scoutfs_inc_counter(sb, srch_search_log); + return search_log_file(sb, sfl, sroot, start, end, limit); + } else { + scoutfs_inc_counter(sb, srch_search_sorted); + return search_sorted_file(sb, sfl, sroot, start, end, limit); + } +} + +static void srch_init_rb_root(struct scoutfs_srch_rb_root *sroot) +{ + sroot->root = RB_ROOT; + sroot->last = NULL; + sroot->nr = 0; +} + +void scoutfs_srch_destroy_rb_root(struct scoutfs_srch_rb_root *sroot) +{ + struct scoutfs_srch_rb_node *snode; + struct scoutfs_srch_rb_node *pos; + + rbtree_postorder_for_each_entry_safe(snode, pos, &sroot->root, node) + kfree(snode); + + srch_init_rb_root(sroot); +} + +/* + * There are no constraints on the distribution of entries in log or + * sorted srch files. We limit the number of entries we track to avoid + * consuming absurd amounts of memory for very large searches. The + * larger the limit the more memory each search will take. The smaller + * this is the more searches will be necessary to find all the entries. + */ +#define SRCH_LIMIT 1000000 + +/* + * Search all the srch files for entries recording that inodes might + * have a given xattr. + * + * Advancing from an inode number that was returned is the only way the + * caller can make forward progress between searches. We might not find + * any inodes if we have the bad luck of pruning all the entries we + * tracked with deletions. We'll restart the search ourselves in this + * case to see if we can find an inode to return to the caller. + */ +int scoutfs_srch_search_xattrs(struct super_block *sb, + struct scoutfs_srch_rb_root *sroot, + u64 hash, u64 ino, u64 last_ino, bool *done) +{ + struct scoutfs_net_roots prev_roots; + struct scoutfs_net_roots roots; + struct scoutfs_srch_entry start; + struct scoutfs_srch_entry end; + struct scoutfs_srch_entry final; + struct scoutfs_log_trees_val ltv; + struct scoutfs_srch_file sfl; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + unsigned long limit = SRCH_LIMIT; + int ret; + + scoutfs_inc_counter(sb, srch_search_xattrs); + + *done = false; + srch_init_rb_root(sroot); + memset(&prev_roots, 0, sizeof(prev_roots)); + + start.hash = cpu_to_le64(hash); + start.ino = cpu_to_le64(ino); + start.id = 0; + final.hash = cpu_to_le64(hash); + final.ino = cpu_to_le64(last_ino); + final.id = cpu_to_le64(U64_MAX); + +retry: + scoutfs_srch_destroy_rb_root(sroot); + + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + memset(&roots.fs_root, 0, sizeof(roots.fs_root)); + + end = final; + + /* search intersecting sorted files, then logs */ + init_srch_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &roots.srch_root, &key, &iref); + if (ret == 0) { + if (iref.key->sk_type != key.sk_type) { + ret = -ENOENT; + } else if (iref.val_len == sizeof(sfl)) { + key = *iref.key; + scoutfs_key_inc(&key); + memcpy(&sfl, iref.val, iref.val_len); + } else { + ret = -EIO; + } + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) { + if (ret == -ENOENT) { + if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { + init_srch_key(&key, + SCOUTFS_SRCH_LOG_TYPE, 0, 0); + continue; + } else { + break; + } + } + goto out; + } + + ret = search_file(sb, key.sk_type, &sfl, sroot, + &start, &end, limit); + if (ret < 0) + goto out; + } + + /* search all the log files being written by mounts */ + scoutfs_key_init_log_trees(&key, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &roots.logs_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(ltv)) { + key = *iref.key; + scoutfs_key_inc(&key); + memcpy(<v, iref.val, iref.val_len); + } else { + ret = -EIO; + } + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + ret = search_file(sb, SCOUTFS_SRCH_LOG_TYPE, <v.srch_file, + sroot, &start, &end, limit); + if (ret < 0) + goto out; + } + + /* keep searching if we didn't find any entries in the limit */ + if (sroot->nr == 0 && sre_cmp(&end, &final) < 0) { + start = end; + scoutfs_inc_counter(sb, srch_search_retry_empty); + goto retry; + } + + /* let the caller know our search was exhaustive */ + *done = sre_cmp(&end, &final) == 0; + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&prev_roots, &roots, sizeof(roots)) == 0) { + scoutfs_inc_counter(sb, srch_search_stale_eio); + ret = -EIO; + } else { + scoutfs_inc_counter(sb, srch_search_stale_retry); + prev_roots = roots; + goto retry; + } + } + + return ret; +} + +/* + * Running in the server, rotate the client's log file as they commit if + * it's large enough. + */ +int scoutfs_srch_rotate_log(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_file *sfl) +{ + struct scoutfs_key key; + int ret; + + if (le64_to_cpu(sfl->blocks) < SCOUTFS_SRCH_LOG_BLOCK_LIMIT) + return 0; + + init_srch_key(&key, SCOUTFS_SRCH_LOG_TYPE, + le64_to_cpu(sfl->ref.blkno), 0); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sfl, sizeof(*sfl)); + if (ret == 0) { + memset(sfl, 0, sizeof(*sfl)); + scoutfs_inc_counter(sb, srch_rotate_log); + } + return ret; +} + +/* + * Running in the server, find candidates for a compaction operation. + * We see if any tier has enough files waiting for a compaction. We + * first search log files and then each greater size tier. We skip any + * files which are currently referenced by existing compaction busy + * items. + */ +int scoutfs_srch_get_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + u64 rid, + struct scoutfs_srch_compact_input *scin) +{ + struct scoutfs_srch_compact_input busy_scin = {{0,}}; + struct scoutfs_srch_file sfl; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_spbm busy; + struct scoutfs_key key; + int cur_order = -1; + int order; + int type; + int ret; + int i; + + /* build up a bitmap of file files already being compacted */ + scoutfs_spbm_init(&busy); + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, 0, 0); + + for (;;) { + /* _BUSY_ is last type, _next won't see other types */ + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(busy_scin)) { + key = *iref.key; + scoutfs_key_inc(&key); + memcpy(&busy_scin, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + for (i = 0; i < busy_scin.nr; i++) { + ret = scoutfs_spbm_set(&busy, + le64_to_cpu(busy_scin.sfl[i].ref.blkno)); + if (ret < 0) + goto out; + } + } + + /* first look for unsorted log files */ + type = SCOUTFS_SRCH_LOG_TYPE; + init_srch_key(&key, type, 0, 0); + + scin->nr = 0; + for (;;scoutfs_key_inc(&key)) { + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == -ENOENT) { + ret = 0; + scin->nr = 0; + goto out; + } + + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_srch_file)) { + key = *iref.key; + memcpy(&sfl, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + /* skip any files already being compacted */ + if (scoutfs_spbm_test(&busy, le64_to_cpu(sfl.ref.blkno))) + continue; + + /* see if we ran out of log files or files entirely */ + if (key.sk_type != type) { + scin->nr = 0; + if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { + type = SCOUTFS_SRCH_BLOCKS_TYPE; + } else { + ret = 0; + goto out; + } + } + + /* reset if we iterated into the next size category */ + if (type == SCOUTFS_SRCH_BLOCKS_TYPE) { + order = fls64(le64_to_cpu(sfl.blocks)) / + SCOUTFS_SRCH_COMPACT_ORDER; + if (order != cur_order) { + cur_order = order; + scin->nr = 0; + } + } + + scin->sfl[scin->nr++] = sfl; + if (scin->nr == SCOUTFS_SRCH_COMPACT_NR) + break; + + scoutfs_key_inc(&key); + } + + if (type == SCOUTFS_SRCH_LOG_TYPE) + scin->flags = SCOUTFS_SRCH_COMPACT_FLAG_LOG; + + /* record that our client has a compaction in process */ + scin->id = scin->sfl[0].ref.blkno; + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(scin->id)); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + scin, sizeof(*scin)); +out: + scoutfs_spbm_destroy(&busy); + if (ret < 0) + scin->nr = 0; + if (scin->nr < SCOUTFS_SRCH_COMPACT_NR) + memset(&scin->sfl[scin->nr], 0, + (SCOUTFS_SRCH_COMPACT_NR - scin->nr) * + sizeof(scin->sfl[0])); + return ret; +} + +/* + * get_ previously created a busy item to reserve the files for a compaction. + * The caller has finished the input struct and we can update the persistent + * copy. + */ +int scoutfs_srch_update_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact_input *scin) +{ + struct scoutfs_key key; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(scin->id)); + return scoutfs_btree_update(sb, alloc, wri, root, &key, + scin, sizeof(*scin)); +} + +static int mod_srch_items(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u8 scom_flags, + bool ins, struct scoutfs_srch_file *sfls, int nr) +{ + struct scoutfs_srch_file *sfl; + struct scoutfs_key key; + int ret = 0; + int type; + int i; + + if (nr <= 0) + return 0; + + if (scom_flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) + type = SCOUTFS_SRCH_LOG_TYPE; + else + type = SCOUTFS_SRCH_BLOCKS_TYPE; + + for (i = 0; i < nr; i++) { + sfl = &sfls[i]; + + /* don't bother inserting empty files */ + if (ins && sfl->entries == 0) + continue; + + if (type == SCOUTFS_SRCH_LOG_TYPE) + init_srch_key(&key, type, + le64_to_cpu(sfl->ref.blkno), 0); + else + init_srch_key(&key, type, + le64_to_cpu(sfl->blocks), + le64_to_cpu(sfl->ref.blkno)); + + if (ins) + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sfl, sizeof(*sfl)); + else + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0) + break; + } + + return ret; +} + +/* + * Running in the server: commit the result of a compaction. Given the + * response id, find the input files in the compact's busy item. Remove + * the input files, add the new sorted file, and remove the busy item. + * We give the caller the allocator trees to merge if we return success. + */ +int scoutfs_srch_commit_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact_result *scres, + struct scoutfs_radix_root *av, + struct scoutfs_radix_root *fr) +{ + struct scoutfs_srch_compact_input scin; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + /* find the record of our compaction */ + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, + le64_to_cpu(scres->id)); + ret = scoutfs_btree_lookup(sb, root, &key, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(scin)) + memcpy(&scin, iref.val, iref.val_len); + else + ret = -EIO; + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) /* XXX leaks allocators */ + goto out; + + if (!(scres->flags & SCOUTFS_SRCH_COMPACT_FLAG_ERROR)) { + /* delete old items and insert new file items */ + ret = mod_srch_items(sb, alloc, wri, root, scin.flags, false, + scin.sfl, scin.nr) ?: + mod_srch_items(sb, alloc, wri, root, 0, true, + &scres->sfl, 1); + if (ret < 0) + goto out; + + *av = scres->meta_avail; + *fr = scres->meta_freed; + } else { + /* reclaim input allocators on error */ + *av = scin.meta_avail; + *fr = scin.meta_freed; + } + + /* delete the record of our compaction */ + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); +out: + WARN_ON_ONCE(ret < 0); /* XXX inconsistency */ + return ret; +} + +/* + * Remove a busy item for the given client and give the caller its + * allocators. Returns -ENOENT when there are no more items. + */ +int scoutfs_srch_cancel_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_radix_root *av, + struct scoutfs_radix_root *fr) +{ + struct scoutfs_srch_compact_input scin; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + struct scoutfs_key last; + int ret; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, 0); + init_srch_key(&last, SCOUTFS_SRCH_BUSY_TYPE, rid, U64_MAX); + + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == 0) { + if (scoutfs_key_compare(iref.key, &last) > 0) { + ret = -ENOENT; + } else if (iref.val_len != sizeof(scin)) { + ret = -EIO; + } else { + key = *iref.key; + memcpy(&scin, iref.val, iref.val_len); + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + *av = scin.meta_avail; + *fr = scin.meta_freed; + + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); +out: + return ret; +} + +struct tourn_node { + struct scoutfs_srch_entry sre; + int ind; +}; + +static void tourn_update(struct tourn_node *tnodes, struct tourn_node *tn) +{ + struct tourn_node *sib; + struct tourn_node *par; + size_t ind; + + /* root is at [1] */ + while (tn != &tnodes[1]) { + ind = tn - tnodes; + sib = &tnodes[ind ^ 1]; + par = &tnodes[ind >> 1]; + *par = sre_cmp(&tn->sre, &sib->sre) < 0 ? *tn : *sib; + tn = par; + } +} + +typedef int (*kway_next_func_t)(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg); + +static int kway_merge(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + kway_next_func_t kway_next, void **args, int nr) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_block *bl = NULL; + struct tourn_node *tnodes; + struct tourn_node *leaves; + struct tourn_node *root; + struct tourn_node *tn; + int nr_parents; + int nr_nodes; + int ret = 0; + u64 blk; + int ind; + int i; + + if (WARN_ON_ONCE(nr <= 1)) + return -EINVAL; + + nr_parents = roundup_pow_of_two(nr) - 1; + /* root at [1] for easy sib/parent index calc, final pad for odd sib */ + nr_nodes = 1 + nr_parents + nr + 1; + tnodes = __vmalloc(nr_nodes * sizeof(struct tourn_node), + GFP_NOFS, PAGE_KERNEL); + if (!tnodes) + return -ENOMEM; + + memset(tnodes, 0xff, nr_nodes * sizeof(struct tourn_node)); + root = &tnodes[1]; + leaves = &root[nr_parents]; + + /* initialize tournament leaves */ + for (i = 0; i < nr; i++) { + tn = &leaves[i]; + tn->ind = i; + ret = kway_next(sb, &tn->sre, args[i]); + if (ret < 0) + goto out; + } + + /* prepare parents.. not optimal, but not a big deal either */ + for (i = 0; i < nr; i += 2) + tourn_update(tnodes, &leaves[i]); + + blk = 0; + while (nr > 0) { + if (bl == NULL) { + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + /* check dirty limit before each block creation */ + if (scoutfs_block_writer_dirty_bytes(sb, wri) >= + SRCH_COMPACT_DIRTY_LIMIT_BYTES) { + scoutfs_inc_counter(sb, srch_compact_flush); + ret = scoutfs_block_writer_write(sb, wri); + if (ret < 0) + goto out; + } + + ret = get_file_block(sb, alloc, wri, sfl, + GFB_INSERT | GFB_DIRTY, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + scoutfs_inc_counter(sb, srch_compact_dirty_block); + } + + if (sre_cmp(&root->sre, &sfl->last) != 0) { + ret = encode_entry(srb->entries + + le32_to_cpu(srb->entry_bytes), + &root->sre, &srb->tail); + if (WARN_ON_ONCE(ret <= 0)) { + /* shouldn't happen */ + ret = -EIO; + goto out; + } + + if (srb->entry_bytes == 0) { + if (blk == 0) + sfl->first = root->sre; + srb->first = root->sre; + } + le32_add_cpu(&srb->entry_nr, 1); + le32_add_cpu(&srb->entry_bytes, ret); + srb->last = root->sre; + srb->tail = root->sre; + sfl->last = root->sre; + le64_add_cpu(&sfl->entries, 1); + ret = 0; + + if (le32_to_cpu(srb->entry_bytes) > + SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + scoutfs_block_put(sb, bl); + bl = NULL; + blk++; + } + + scoutfs_inc_counter(sb, srch_compact_entry); + + } else { + scoutfs_inc_counter(sb, srch_compact_removed_entry); + } + + /* get the next */ + ind = root->ind; + tn = &leaves[ind]; + ret = kway_next(sb, &tn->sre, args[ind]); + if (ret == -ENOENT) { + /* this index is done */ + memset(&tn->sre, 0xff, sizeof(tn->sre)); + nr--; + ret = 0; + } else if (ret < 0) { + goto out; + } + + /* update the tourney and carry on */ + tourn_update(tnodes, tn); +#if 0 + /* would be worth it if we have uneven key distribution */ + if (ind < nr - 1) { + /* order doesn't matter, fill hole */ + swap(args[ind], args[nr - 1]); + swap(tn->sre, leaves[nr - 1].sre); + } + /* drop a level of the tree when we shrink to a power of 2 */ + if (nr > 0 && is_power_of_two(nr)) { + memcpy(leaves - nr, leaves, nr * sizeof(*tn)); + leaves -= nr; + for (i = 0; i < nr; i += 2) + tourn_update(least, leaves[i]); + } +#endif + } + + /* could stream a final index.. arguably a small portion of work */ + +out: + scoutfs_block_put(sb, bl); + vfree(tnodes); + return ret; +} + +#define SRES_PER_PAGE (PAGE_SIZE / sizeof(struct scoutfs_srch_entry)) + +static struct scoutfs_srch_entry *page_priv_sre(struct page *page) +{ + return (struct scoutfs_srch_entry *)page_address(page) + page->private; +} + +static int kway_next_page(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) +{ + struct page *page = arg; + struct scoutfs_srch_entry *sre = page_priv_sre(page); + + if (page->private >= SRES_PER_PAGE || sre->ino == 0) + return -ENOENT; + + *sre_ret = *sre; + page->private++; + return 0; +} + +static int cmp_page_sre(const void *A, const void *B) +{ + const struct scoutfs_srch_entry *a = A; + const struct scoutfs_srch_entry *b = B; + + return sre_cmp(a, b); +} + +static void swap_page_sre(void *A, void *B, int size) +{ + struct scoutfs_srch_entry *a = A; + struct scoutfs_srch_entry *b = B; + + swap(*a, *b); +} + +/* + * Compact a set of log files by sorting all their entries and writing + * them to a sorted output file. We decode all the file's entries into + * pages, sort the contents of each page, and then stream a k-way merge + * of the entries in the pages into an output file. While not sorted, + * the input log files entries are encoded so we can allocate quite a + * bit more memory in pages than the files took in blocks on disk (~2x + * typically, ~10x worst case). + */ +static int compact_logs(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl_out, + struct scoutfs_srch_file *sfls, int nr_sfls) +{ + DECLARE_SRCH_INFO(sb, srinf); + struct scoutfs_srch_file *sfl_end = sfls + nr_sfls; + struct scoutfs_srch_file *sfl = &sfls[0]; + struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry *sre; + struct scoutfs_srch_entry prev; + struct scoutfs_block *bl = NULL; + struct page *page = NULL; + struct page *tmp; + void **args = NULL; + int nr_pages = 0; + LIST_HEAD(pages); + u64 blk = 0; + int pos = 0; + int ret; + int i; + + if (WARN_ON_ONCE(nr_sfls <= 1)) + return -EINVAL; + + memset(&prev, 0, sizeof(prev)); + + /* decode all the log file's block's entries into pages */ + while (sfl < sfl_end) { + if (bl == NULL) { + /* only check on each new input block */ + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); + if (ret < 0) + goto out; + srb = bl->data; + } + + if (page == NULL) { + page = alloc_page(GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + page->private = 0; + list_add_tail(&page->list, &pages); + nr_pages++; + scoutfs_inc_counter(sb, srch_compact_log_page); + } + + sre = page_priv_sre(page); + + if (pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* can only be inconsistency :/ */ + ret = EIO; + break; + } + + ret = decode_entry(srb->entries + pos, sre, &prev); + if (ret <= 0) { + /* can only be inconsistency :/ */ + ret = EIO; + goto out; + } + prev = *sre; + + pos += ret; + if (pos >= le32_to_cpu(srb->entry_bytes)) { + scoutfs_block_put(sb, bl); + bl = NULL; + memset(&prev, 0, sizeof(prev)); + pos = 0; + if (++blk == le64_to_cpu(sfl->blocks)) { + blk = 0; + sfl++; + } + } + + if (++page->private == SRES_PER_PAGE) + page = NULL; + } + + /* add a terminal entry to the last partial page */ + if (page) { + sre = page_priv_sre(page); + sre->ino = 0; + } + + /* allocate args array for k-way merge */ + args = vmalloc(nr_pages * sizeof(struct page *)); + if (!args) { + ret = -ENOMEM; + goto out; + } + + /* sort page entries and reset private for _next */ + i = 0; + list_for_each_entry(page, &pages, list) { + args[i++] = page; + + if (atomic_read(&srinf->shutdown)) { + ret = -ESHUTDOWN; + goto out; + } + + sort(page_address(page), page->private, + sizeof(struct scoutfs_srch_entry), cmp_page_sre, + swap_page_sre); + page->private = 0; + + } + + ret = kway_merge(sb, alloc, wri, sfl_out, kway_next_page, args, + nr_pages); +out: + scoutfs_block_put(sb, bl); + vfree(args); + list_for_each_entry_safe(page, tmp, &pages, list) { + list_del(&page->list); + __free_page(page); + } + + return ret; +} + +struct kway_file_reader { + struct scoutfs_srch_file *sfl; + struct scoutfs_block *bl; + struct scoutfs_srch_entry prev; + u64 blk; + u32 pos; +}; + +static int kway_next_file_reader(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) +{ + struct kway_file_reader *rdr = arg; + struct scoutfs_srch_block *srb; + int ret; + + if (rdr->sfl == NULL) + return -ENOENT; + + if (rdr->bl == NULL) { + ret = get_file_block(sb, NULL, NULL, rdr->sfl, 0, rdr->blk, + &rdr->bl); + if (ret < 0) + goto out; + memset(&rdr->prev, 0, sizeof(rdr->prev)); + rdr->pos = 0; + } + srb = rdr->bl->data; + + if (rdr->pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + /* XXX inconsistency */ + return -EIO; + } + + ret = decode_entry(srb->entries + rdr->pos, sre_ret, &rdr->prev); + if (ret <= 0) { + /* XXX inconsistency */ + return -EIO; + } + + rdr->prev = *sre_ret; + rdr->pos += ret; + + if (rdr->pos >= le32_to_cpu(srb->entry_bytes)) { + scoutfs_block_put(sb, rdr->bl); + rdr->bl = NULL; + if (++rdr->blk == le64_to_cpu(rdr->sfl->blocks)) + rdr->sfl = NULL; + } + + ret = 0; +out: + return ret; +} + +/* + * Compact a set of sorted files by performing a k-way merge of the files + * into an output sorted file. The k-way merge works with an iterator + * which reads blocks and decodes entries. + */ +static int compact_sorted(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl_out, + struct scoutfs_srch_file *sfls, int nr) +{ + struct kway_file_reader *rdrs = NULL; + void **args = NULL; + int ret; + int i; + + if (WARN_ON_ONCE(nr <= 1)) + return -EINVAL; + + /* allocate args array for k-way merge */ + rdrs = kmalloc_array(nr, sizeof(rdrs[0]), __GFP_ZERO | GFP_NOFS); + args = kmalloc_array(nr, sizeof(args[0]), GFP_NOFS); + if (!rdrs || !args) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < nr; i++) { + rdrs[i].sfl = &sfls[i]; + args[i] = &rdrs[i]; + } + + ret = kway_merge(sb, alloc, wri, sfl_out, kway_next_file_reader, + args, nr); +out: + for (i = 0; rdrs && i < nr; i++) + scoutfs_block_put(sb, rdrs[i].bl); + kfree(rdrs); + kfree(args); + + return ret; +} + +/* + * Perform a depth-first walk of the file's parent blocks, freeing all + * the blocks that were allocated to the file. This is working with a + * read-only file in the block cache that can also be currently read by + * searchers. If we return an error then the server is going to clean + * up our entire operation, partial state doesn't matter. + */ +static int free_file(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl) +{ + struct scoutfs_block **bls = NULL; + struct scoutfs_srch_parent *srp; + struct scoutfs_srch_ref *ref; + unsigned int *inds = NULL; + u64 blkno; + u8 height; + int level; + int ret; + int i; + + if (sfl->ref.blkno == 0) + return 0; + + height = height_for_blk(le64_to_cpu(sfl->blocks) - 1); + if (height == 1) + goto free_root; + + bls = kmalloc_array(height, sizeof(bls[0]), __GFP_ZERO | GFP_NOFS); + inds = kmalloc_array(height, sizeof(inds[0]), __GFP_ZERO | GFP_NOFS); + if (!bls || !inds) { + ret = -ENOMEM; + goto out; + } + + ref = &sfl->ref; + level = height - 1; + while (level < height) { + if (bls[level] == NULL) { + ret = read_srch_block(sb, wri, level, ref, &bls[level]); + if (ret < 0) + goto out; + } + srp = bls[level]->data; + + /* find a parent to descend to, remembering where we were */ + ref = NULL; + for (i = inds[level]; level >= 2 && + i < SCOUTFS_SRCH_PARENT_REFS; i++) { + if (srp->refs[i].blkno) { + inds[level] = i + 1; + ref = &srp->refs[i]; + level--; + break; + } + } + if (ref) + continue; + + /* free all our referenced blocks */ + for (i = 0; i < SCOUTFS_SRCH_PARENT_REFS; i++) { + blkno = le64_to_cpu(srp->refs[i].blkno); + if (blkno == 0) + continue; + + ret = scoutfs_radix_free(sb, alloc, wri, blkno); + if (ret < 0) + goto out; + scoutfs_inc_counter(sb, srch_compact_free_block); + } + + scoutfs_block_put(sb, bls[level]); + bls[level] = NULL; + level++; + } + +free_root: + ret = scoutfs_radix_free(sb, alloc, wri, le64_to_cpu(sfl->ref.blkno)); + if (ret < 0) + goto out; + +out: + for (i = 0; bls && i < height; i++) + scoutfs_block_put(sb, bls[i]); + kfree(bls); + kfree(inds); + return ret; +} + +/* wait 10s between compact attempts on error, immediate after success */ +#define SRCH_COMPACT_DELAY_MS (10 * MSEC_PER_SEC) + +/* + * Get a compaction operation from the server, sort the entries from the + * input files as they're read, and stream the remaining sorted entries + * into a newly written output file. The server is protecting the input + * files from other compactions, they will be stable. The server gives + * us a populated allocator that should be enough to write a new file + * and delete the old file blocks. We'll regularly write out dirty + * blocks as we hit a dirty limit threshold so there will be some cow + * overhead of repeatedly dirtying, say, parent allocator and file radix + * blocks. We don't reclaim freed blocks in the allocator after each + * write so the initial allocator pool has to account for that cow + * overhead. + * + * All of our modifications are written into free blocks from the + * filesystem's perspective. If anything goes wrong we return an error + * and the server will ignore all our work and reclaim the initial + * allocator they gave us. + */ +static void scoutfs_srch_compact_worker(struct work_struct *work) +{ + struct srch_info *srinf = container_of(work, struct srch_info, + compact_dwork.work); + struct super_block *sb = srinf->sb; + struct scoutfs_radix_allocator alloc; + struct scoutfs_srch_compact_result scres; + struct scoutfs_srch_compact_input scin; + struct scoutfs_block_writer wri; + unsigned long delay; + int ret; + int i; + + scoutfs_block_writer_init(sb, &wri); + memset(&scres, 0, sizeof(scres)); + + ret = scoutfs_client_srch_get_compact(sb, &scin); + if (ret < 0 || scin.nr == 0) + goto out; + + scoutfs_radix_init_alloc(&alloc, &scin.meta_avail, &scin.meta_freed); + + if (scin.flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) + ret = compact_logs(sb, &alloc, &wri, &scres.sfl, + scin.sfl, scin.nr); + else + ret = compact_sorted(sb, &alloc, &wri, &scres.sfl, + scin.sfl, scin.nr); + if (ret < 0) + goto commit; + + for (i = 0; i < scin.nr; i++) { + ret = free_file(sb, &alloc, &wri, &scin.sfl[i]); + if (ret < 0) + goto commit; + } + + ret = scoutfs_block_writer_write(sb, &wri); +commit: + scres.meta_avail = alloc.avail; + scres.meta_freed = alloc.freed; + scres.id = scin.id; + scres.flags = ret < 0 ? SCOUTFS_SRCH_COMPACT_FLAG_ERROR : 0; + + ret = scoutfs_client_srch_commit_compact(sb, &scres); +out: + /* our allocators and files should be stable */ + WARN_ON_ONCE(ret == -ESTALE); + + scoutfs_block_writer_forget_all(sb, &wri); + if (!atomic_read(&srinf->shutdown)) { + delay = ret == 0 ? 0 : msecs_to_jiffies(SRCH_COMPACT_DELAY_MS); + queue_delayed_work(srinf->workq, &srinf->compact_dwork, delay); + } +} + +void scoutfs_srch_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + DECLARE_SRCH_INFO(sb, srinf); + + if (!srinf) + return; + + if (srinf->workq) { + /* pending grace work queues normal work */ + atomic_set(&srinf->shutdown, 1); + cancel_delayed_work_sync(&srinf->compact_dwork); + flush_workqueue(srinf->workq); + destroy_workqueue(srinf->workq); + } + + kfree(srinf); + sbi->srch_info = NULL; +} + +int scoutfs_srch_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct srch_info *srinf; + int ret; + + srinf = kzalloc(sizeof(struct srch_info), GFP_KERNEL); + if (!srinf) + return -ENOMEM; + + srinf->sb = sb; + atomic_set(&srinf->shutdown, 0); + INIT_DELAYED_WORK(&srinf->compact_dwork, scoutfs_srch_compact_worker); + sbi->srch_info = srinf; + + srinf->workq = alloc_workqueue("scoutfs_srch_compact", + WQ_NON_REENTRANT | WQ_UNBOUND | + WQ_HIGHPRI, 0); + if (!srinf->workq) { + ret = -ENOMEM; + goto out; + } + + queue_delayed_work(srinf->workq, &srinf->compact_dwork, + msecs_to_jiffies(SRCH_COMPACT_DELAY_MS)); + + ret = 0; +out: + if (ret) + scoutfs_srch_destroy(sb); + + return ret; +} diff --git a/kmod/src/srch.h b/kmod/src/srch.h new file mode 100644 index 00000000..937692e9 --- /dev/null +++ b/kmod/src/srch.h @@ -0,0 +1,69 @@ +#ifndef _SCOUTFS_SRCH_H_ +#define _SCOUTFS_SRCH_H_ + +struct scoutfs_block; + +struct scoutfs_srch_rb_root { + struct rb_root root; + struct rb_node *last; + unsigned long nr; +}; + +struct scoutfs_srch_rb_node { + struct rb_node node; + u64 ino; + u64 id; +}; + +#define scoutfs_srch_foreach_rb_node(snode, node, sroot) \ + for (node = rb_first(&(sroot)->root); \ + node && (snode = container_of(node, struct scoutfs_srch_rb_node, \ + node), 1); \ + node = rb_next(node)) + +int scoutfs_srch_add(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + struct scoutfs_block **bl_ret, + u64 hash, u64 ino, u64 id); + +void scoutfs_srch_destroy_rb_root(struct scoutfs_srch_rb_root *sroot); +int scoutfs_srch_search_xattrs(struct super_block *sb, + struct scoutfs_srch_rb_root *sroot, + u64 hash, u64 ino, u64 last_ino, bool *done); + +int scoutfs_srch_rotate_log(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_file *sfl); +int scoutfs_srch_get_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + u64 rid, + struct scoutfs_srch_compact_input *scin_ret); +int scoutfs_srch_update_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact_input *scin); +int scoutfs_srch_commit_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_srch_compact_result *scres, + struct scoutfs_radix_root *av, + struct scoutfs_radix_root *fr); +int scoutfs_srch_cancel_compact(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, u64 rid, + struct scoutfs_radix_root *av, + struct scoutfs_radix_root *fr); + +void scoutfs_srch_destroy(struct super_block *sb); +int scoutfs_srch_setup(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index efc7f1f7..c479b484 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -41,6 +41,7 @@ #include "sysfs.h" #include "quorum.h" #include "forest.h" +#include "srch.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -178,6 +179,7 @@ static void scoutfs_put_super(struct super_block *sb) sbi->shutdown = true; scoutfs_data_destroy(sb); + scoutfs_srch_destroy(sb); scoutfs_unlock(sb, sbi->rid_lock, SCOUTFS_LOCK_WRITE); sbi->rid_lock = NULL; @@ -452,7 +454,8 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_client_setup(sb) ?: scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid, &sbi->rid_lock) ?: - scoutfs_trans_get_log_trees(sb); + scoutfs_trans_get_log_trees(sb) ?: + scoutfs_srch_setup(sb); if (ret) goto out; diff --git a/kmod/src/super.h b/kmod/src/super.h index 07d6653b..1f9776b9 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -25,6 +25,7 @@ struct options_sb_info; struct net_info; struct block_info; struct forest_info; +struct srch_info; struct scoutfs_sb_info { struct super_block *sb; @@ -44,6 +45,7 @@ struct scoutfs_sb_info { struct quorum_info *quorum_info; struct block_info *block_info; struct forest_info *forest_info; + struct srch_info *srch_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From c415cab1e9575b1eeff6670d1eef766034e0f94e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 23 Jun 2020 09:49:06 -0700 Subject: [PATCH 850/920] scoutfs: use srch to track .srch. xattrs Using strictly coherent btree items to map the hash of xattr names to inode numbers proved the value of the functionality, but it was too expensive. We now have the more efficient srch infrastructure to use. We change from the .indx. to the .srch. tag, and change the ioctl from find_xattr to search_xattrs. The idea is to communicate that these are accelerated searches, not precise index lookups and are relatively expensive. Rather than maintaining btree items, xattr setting and deleting emits srch entries which either tracks the xattr or combines with the previous tracker and removes the entry. These are done under the lock that protects the main xattr item, we can remove the separate locking of the previous index items. The semantics of the search ioctl needs to change a bit. Because searches are so expensive we now return a flag to indicate that the search completed. While we're there, we also allow a last_ino parameter so that searches can be divided up and run in parallel. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 83 ++++++++++++++++++++++-------------------------- kmod/src/ioctl.h | 49 ++++++++++++++++++++-------- kmod/src/xattr.c | 72 ++++++++++------------------------------- 3 files changed, 91 insertions(+), 113 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index c0dac183..fc35dd34 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -34,6 +34,7 @@ #include "trans.h" #include "xattr.h" #include "hash.h" +#include "srch.h" #include "scoutfs_trace.h" /* @@ -759,18 +760,18 @@ out: * but we don't check that the callers xattr name contains the tag and * search for it regardless. */ -static long scoutfs_ioc_find_xattrs(struct file *file, unsigned long arg) +static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) { struct super_block *sb = file_inode(file)->i_sb; - struct scoutfs_ioctl_find_xattrs __user *ufx = (void __user *)arg; - struct scoutfs_ioctl_find_xattrs fx; - struct scoutfs_lock *lock = NULL; - struct scoutfs_key last; - struct scoutfs_key key; + struct scoutfs_ioctl_search_xattrs __user *usx = (void __user *)arg; + struct scoutfs_ioctl_search_xattrs sx; + struct scoutfs_srch_rb_root sroot; + struct scoutfs_srch_rb_node *snode; + u64 __user *uinos; + struct rb_node *node; char *name = NULL; - int total = 0; - u64 hash; - u64 ino; + bool done = false; + u64 total = 0; int ret; if (!(file->f_mode & FMODE_READ)) { @@ -783,67 +784,59 @@ static long scoutfs_ioc_find_xattrs(struct file *file, unsigned long arg) goto out; } - if (copy_from_user(&fx, ufx, sizeof(fx))) { + if (copy_from_user(&sx, usx, sizeof(sx))) { ret = -EFAULT; goto out; } + uinos = (u64 __user *)sx.inodes_ptr; - if (fx.name_bytes > SCOUTFS_XATTR_MAX_NAME_LEN) { + if (sx.name_bytes > SCOUTFS_XATTR_MAX_NAME_LEN) { ret = -EINVAL; goto out; } - name = kmalloc(fx.name_bytes, GFP_KERNEL); + if (sx.nr_inodes == 0 || sx.last_ino < sx.next_ino) { + ret = 0; + goto out; + } + + name = kmalloc(sx.name_bytes, GFP_KERNEL); if (!name) { ret = -ENOMEM; goto out; } - if (copy_from_user(name, (void __user *)fx.name_ptr, fx.name_bytes)) { + if (copy_from_user(name, (void __user *)sx.name_ptr, sx.name_bytes)) { ret = -EFAULT; goto out; } - hash = scoutfs_hash64(name, fx.name_bytes); - scoutfs_xattr_index_key(&key, hash, fx.next_ino, 0); - scoutfs_xattr_index_key(&last, hash, U64_MAX, U64_MAX); - ino = 0; - - ret = scoutfs_lock_xattr_index(sb, SCOUTFS_LOCK_READ, 0, hash, &lock); + ret = scoutfs_srch_search_xattrs(sb, &sroot, + scoutfs_hash64(name, sx.name_bytes), + sx.next_ino, sx.last_ino, &done); if (ret < 0) goto out; - while (fx.nr_inodes) { - - ret = scoutfs_forest_next(sb, &key, &last, NULL, lock); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; + scoutfs_srch_foreach_rb_node(snode, node, &sroot) { + if (put_user(snode->ino, uinos + total)) { + ret = -EFAULT; break; } - - /* xattrs hashes can collide and add multiple entries */ - if (le64_to_cpu(key.skxi_ino) != ino) { - ino = le64_to_cpu(key.skxi_ino); - if (put_user(ino, (u64 __user *)fx.inodes_ptr)) { - ret = -EFAULT; - break; - } - - fx.inodes_ptr += sizeof(u64); - fx.nr_inodes--; - total++; - ret = 0; - } - - scoutfs_key_inc(&key); + if (++total == sx.nr_inodes) + break; } - scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); + sx.output_flags = 0; + if (done && total == sroot.nr) + sx.output_flags |= SCOUTFS_SEARCH_XATTRS_OFLAG_END; + if (put_user(sx.output_flags, &usx->output_flags)) + ret = -EFAULT; + else + ret = 0; out: + scoutfs_srch_destroy_rb_root(&sroot); kfree(name); - return ret ?: total; } @@ -887,8 +880,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_setattr_more(file, arg); case SCOUTFS_IOC_LISTXATTR_HIDDEN: return scoutfs_ioc_listxattr_hidden(file, arg); - case SCOUTFS_IOC_FIND_XATTRS: - return scoutfs_ioc_find_xattrs(file, arg); + case SCOUTFS_IOC_SEARCH_XATTRS: + return scoutfs_ioc_search_xattrs(file, arg); case SCOUTFS_IOC_STATFS_MORE: return scoutfs_ioc_statfs_more(file, arg); case SCOUTFS_IOC_DATA_WAIT_ERR: diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 4b635f88..2f861a4d 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -296,34 +296,57 @@ struct scoutfs_ioctl_listxattr_hidden { /* * Return the inode numbers of inodes which might contain the given - * named xattr. The inode may not have a set xattr with that name, the - * caller must check the returned inodes to see if they match. + * xattr. The inode may not have a set xattr with that name, the caller + * must check the returned inodes to see if they match. * * @next_ino: The next inode number that could be returned. Initialized * to 0 when first searching and set to one past the last inode number * returned to continue searching. - * @name_ptr: The address of the name of the xattr to search for. It does - * not need to be null terminated. - * @inodes_ptr: The address of the array of uint64_t inode numbers in which - * to store inode numbers that may contain the xattr. EFAULT may be returned - * if this address is not naturally aligned. - * @name_bytes: The number of non-null bytes found in the name at name_ptr. + * @last_ino: The last inode number that could be returned. U64_MAX to + * find all inodes. + * @name_ptr: The address of the name of the xattr to search for. It is + * not null terminated. + * @inodes_ptr: The address of the array of uint64_t inode numbers in + * which to store inode numbers that may contain the xattr. EFAULT may + * be returned if this address is not naturally aligned. + * @output_flags: Set as success is returned. If an error is returned + * then this field is undefined and should not be read. * @nr_inodes: The number of elements in the array found at inodes_ptr. + * @name_bytes: The number of non-null bytes found in the name at + * name_ptr. * * This requires the CAP_SYS_ADMIN capability and will return -EPERM if * it's not granted. + * + * The number of inode numbers stored in the inodes_ptr array is + * returned. If nr_inodes is 0 or last_ino is less than next_ino then 0 + * will be immediately returned. + * + * Partial progress can be returned if an error is hit or if nr_inodes + * was larger than the internal limit on the number of inodes returned + * in a search pass. The _END output flag is set if all the results + * including last_ino were searched in this pass. + * + * It's valuable to provide a large inodes array so that all the results + * can be found in one search pass and _END can be set. There are + * significant constant costs for performing each search pass. */ -struct scoutfs_ioctl_find_xattrs { +struct scoutfs_ioctl_search_xattrs { __u64 next_ino; + __u64 last_ino; __u64 name_ptr; __u64 inodes_ptr; + __u64 output_flags; + __u64 nr_inodes; __u16 name_bytes; - __u16 nr_inodes; - __u8 _pad[4]; + __u8 _pad[6]; }; -#define SCOUTFS_IOC_FIND_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ - struct scoutfs_ioctl_find_xattrs) +/* set in output_flags if returned inodes reached last_ino */ +#define SCOUTFS_SEARCH_XATTRS_OFLAG_END (1ULL << 0) + +#define SCOUTFS_IOC_SEARCH_XATTRS _IOR(SCOUTFS_IOCTL_MAGIC, 9, \ + struct scoutfs_ioctl_search_xattrs) /* * Give the user information about the filesystem. diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 4dbe9900..03aa27f9 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -96,11 +96,11 @@ static int unknown_prefix(const char *name) struct prefix_tags { unsigned long hide:1, - indx:1; + srch:1; }; #define HIDE_TAG "hide." -#define INDX_TAG "indx." +#define SRCH_TAG "srch." #define TAG_LEN (sizeof(HIDE_TAG) - 1) static int parse_tags(const char *name, unsigned int name_len, @@ -120,8 +120,8 @@ static int parse_tags(const char *name, unsigned int name_len, if (!strncmp(name, HIDE_TAG, TAG_LEN)) { if (++tgs->hide == 0) return -EINVAL; - } else if (!strncmp(name, INDX_TAG, TAG_LEN)) { - if (++tgs->indx == 0) + } else if (!strncmp(name, SRCH_TAG, TAG_LEN)) { + if (++tgs->srch == 0) return -EINVAL; } else { /* only reason to use scoutfs. is tags */ @@ -412,19 +412,17 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); struct scoutfs_xattr *xat = NULL; - struct scoutfs_lock *indx_lock = NULL; struct scoutfs_lock *lck = NULL; size_t name_len = strlen(name); - struct scoutfs_key indx_key; struct scoutfs_key key; struct prefix_tags tgs; - bool undo_indx = false; + bool undo_srch = false; LIST_HEAD(ind_locks); LIST_HEAD(saved); u8 found_parts; unsigned int bytes; u64 ind_seq; - u64 hash; + u64 hash = 0; u64 id = 0; int ret; int err; @@ -447,7 +445,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (parse_tags(name, name_len, &tgs) != 0) return -EINVAL; - if ((tgs.hide || tgs.indx) && !capable(CAP_SYS_ADMIN)) + if ((tgs.hide || tgs.srch) && !capable(CAP_SYS_ADMIN)) return -EPERM; bytes = sizeof(struct scoutfs_xattr) + name_len + size; @@ -498,14 +496,6 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, memcpy(&xat->name[xat->name_len], value, size); } - if (tgs.indx && !(found_parts && value)) { - hash = scoutfs_hash64(name, name_len); - ret = scoutfs_lock_xattr_index(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, - hash, &indx_lock); - if (ret < 0) - goto unlock; - } - retry: ret = scoutfs_inode_index_start(sb, &ind_seq) ?: scoutfs_inode_index_prepare(sb, &ind_locks, inode, false) ?: @@ -513,7 +503,7 @@ retry: SIC_XATTR_SET(found_parts, value != NULL, name_len, size, - tgs.indx)); + tgs.srch)); if (ret > 0) goto retry; if (ret) @@ -523,20 +513,14 @@ retry: if (ret < 0) goto release; - if (tgs.indx && !(found_parts && value)) { + if (tgs.srch && !(found_parts && value)) { if (found_parts) id = le64_to_cpu(key.skx_id); hash = scoutfs_hash64(name, name_len); - scoutfs_xattr_index_key(&indx_key, hash, ino, id); - if (value) - ret = scoutfs_forest_create_force(sb, &indx_key, NULL, - indx_lock); - else - ret = scoutfs_forest_delete_force(sb, &indx_key, - indx_lock); + ret = scoutfs_forest_srch_add(sb, hash, ino, id); if (ret < 0) goto release; - undo_indx = true; + undo_srch = true; } ret = 0; @@ -559,13 +543,8 @@ retry: ret = 0; release: - if (ret < 0 && undo_indx) { - if (value) - err = scoutfs_forest_delete_force(sb, &indx_key, - indx_lock); - else - err = scoutfs_forest_create_force(sb, &indx_key, NULL, - indx_lock); + if (ret < 0 && undo_srch) { + err = scoutfs_forest_srch_add(sb, hash, ino, id); BUG_ON(err); } @@ -573,7 +552,6 @@ release: scoutfs_inode_index_unlock(sb, &ind_locks); unlock: up_write(&si->xattr_rwsem); - scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE); out: kfree(xat); @@ -693,9 +671,7 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock) { - struct scoutfs_lock *indx_lock = NULL; struct scoutfs_xattr *xat = NULL; - struct scoutfs_key indx_key; struct scoutfs_key last; struct scoutfs_key key; struct prefix_tags tgs; @@ -729,17 +705,6 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, parse_tags(xat->name, xat->name_len, &tgs) != 0) memset(&tgs, 0, sizeof(tgs)); - if (tgs.indx) { - hash = scoutfs_hash64(xat->name, xat->name_len); - scoutfs_xattr_index_key(&indx_key, hash, ino, - le64_to_cpu(key.skx_id)); - ret = scoutfs_lock_xattr_index(sb, - SCOUTFS_LOCK_WRITE_ONLY, - 0, hash, &indx_lock); - if (ret < 0) - break; - } - ret = scoutfs_hold_trans(sb, SIC_EXACT(2, 0)); if (ret < 0) break; @@ -749,9 +714,10 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, if (ret < 0) break; - if (tgs.indx) { - ret = scoutfs_forest_delete_force(sb, &indx_key, - indx_lock); + if (tgs.srch) { + hash = scoutfs_hash64(xat->name, xat->name_len); + ret = scoutfs_forest_srch_add(sb, hash, ino, + le64_to_cpu(key.skx_id)); if (ret < 0) break; } @@ -759,15 +725,11 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, scoutfs_release_trans(sb); release = false; - scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); - indx_lock = NULL; - /* don't need to inc, next won't see deleted item */ } if (release) scoutfs_release_trans(sb); - scoutfs_unlock(sb, indx_lock, SCOUTFS_LOCK_WRITE_ONLY); kfree(xat); out: return ret; From f8bf1718a0454cc4c8211250d8dcb346abffd16b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 7 Jul 2020 10:59:11 -0700 Subject: [PATCH 851/920] scoutfs: add a bunch of btree counters Add some counters for the most basic btree events. Signed-off-by: Zach Brown --- kmod/src/btree.c | 59 ++++++++++++++++++++++++++++++++++----------- kmod/src/counters.h | 14 +++++++++++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index e162cc40..59cde151 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -233,7 +233,8 @@ static inline int leaf_item_hash_next_bucket(int i) i = leaf_item_hash_next_bucket(i)) static struct scoutfs_btree_item * -leaf_item_hash_search(struct scoutfs_btree_block *bt, struct scoutfs_key *key) +leaf_item_hash_search(struct super_block *sb, struct scoutfs_btree_block *bt, + struct scoutfs_key *key) { __le16 *buckets = leaf_item_hash_buckets(bt); struct scoutfs_btree_item *item; @@ -241,6 +242,8 @@ leaf_item_hash_search(struct scoutfs_btree_block *bt, struct scoutfs_key *key) int nr; int i; + scoutfs_inc_counter(sb, btree_leaf_item_hash_search); + if (WARN_ON_ONCE(bt->level > 0)) return NULL; @@ -388,7 +391,8 @@ static void set_val_owner(struct scoutfs_btree_block *bt, unsigned int val_off, * more operations. The split heuristic requires a generous amount of * fragmented free space that will avoid a split. */ -static void compact_values(struct scoutfs_btree_block *bt) +static void compact_values(struct super_block *sb, + struct scoutfs_btree_block *bt) { struct scoutfs_btree_item *item; unsigned int free_off; @@ -399,6 +403,8 @@ static void compact_values(struct scoutfs_btree_block *bt) void *from; void *to; + scoutfs_inc_counter(sb, btree_compact_values); + if (bt->last_free_off == 0) return; @@ -871,10 +877,12 @@ static int try_split(struct super_block *sb, return 0; if (item_full_pct(right) < 80) { - compact_values(right); + compact_values(sb, right); return 0; } + scoutfs_inc_counter(sb, btree_split); + /* alloc split neighbour first to avoid unwinding tree growth */ ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &left_bl); if (ret) @@ -941,6 +949,8 @@ static int try_join(struct super_block *sb, if (le16_to_cpu(bt->total_item_bytes) >= join_low_watermark()) return 0; + scoutfs_inc_counter(sb, btree_join); + /* move items right into our block if we have a left sibling */ sib_par_item = prev_item(parent, par_item); if (sib_par_item) { @@ -963,7 +973,7 @@ static int try_join(struct super_block *sb, to_move = sib_tot - join_low_watermark(); if (le16_to_cpu(bt->mid_free_len) < to_move) - compact_values(bt); + compact_values(sb, bt); move_items(bt, sib, move_right, to_move); /* update our parent's item */ @@ -1025,7 +1035,8 @@ static bool bad_avl_node_off(__le16 node_off, int nr) * - last_free_offset is in fact last free region * - call after leaf modification */ -static void verify_btree_block(struct scoutfs_btree_block *bt, int level, +static void verify_btree_block(struct super_block *sb, + struct scoutfs_btree_block *bt, int level, struct scoutfs_key *start, struct scoutfs_key *end) { @@ -1081,7 +1092,7 @@ static void verify_btree_block(struct scoutfs_btree_block *bt, int level, } if (level == 0 && - leaf_item_hash_search(bt, &item->key) != item) { + leaf_item_hash_search(sb, bt, &item->key) != item) { reason = "item not found in hash"; goto out; } @@ -1221,6 +1232,8 @@ static int btree_walk(struct super_block *sb, WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; + scoutfs_inc_counter(sb, btree_walk); + restart: scoutfs_block_put(sb, par_bl); par_bl = NULL; @@ -1261,7 +1274,7 @@ restart: bt = bl->data; if (0) - verify_btree_block(bt, level, &start, &end); + verify_btree_block(sb, bt, level, &start, &end); /* XXX more aggressive block verification, before ref updates? */ if (bt->level != level) { @@ -1292,8 +1305,10 @@ restart: if (ret == 0 && (flags & BTW_DELETE) && parent) ret = try_join(sb, alloc, wri, root, parent, par_item, bt); - if (ret > 0) + if (ret > 0) { + scoutfs_inc_counter(sb, btree_walk_restart); goto restart; + } else if (ret < 0) break; @@ -1399,6 +1414,8 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_block *bl; int ret; + scoutfs_inc_counter(sb, btree_lookup); + if (WARN_ON_ONCE(iref->key)) return -EINVAL; @@ -1406,7 +1423,7 @@ int scoutfs_btree_lookup(struct super_block *sb, if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) { init_item_ref(iref, sb, bl, item); ret = 0; @@ -1448,6 +1465,8 @@ int scoutfs_btree_insert(struct super_block *sb, int cmp; int ret; + scoutfs_inc_counter(sb, btree_insert); + if (invalid_item(val_len)) return -EINVAL; @@ -1456,7 +1475,7 @@ int scoutfs_btree_insert(struct super_block *sb, if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) { ret = -EEXIST; } else { @@ -1510,6 +1529,8 @@ int scoutfs_btree_update(struct super_block *sb, struct scoutfs_block *bl; int ret; + scoutfs_inc_counter(sb, btree_update); + if (invalid_item(val_len)) return -EINVAL; @@ -1518,7 +1539,7 @@ int scoutfs_btree_update(struct super_block *sb, if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) { update_item_value(bt, item, val, val_len); ret = 0; @@ -1550,6 +1571,8 @@ int scoutfs_btree_force(struct super_block *sb, int cmp; int ret; + scoutfs_inc_counter(sb, btree_force); + if (invalid_item(val_len)) return -EINVAL; @@ -1558,7 +1581,7 @@ int scoutfs_btree_force(struct super_block *sb, if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) { update_item_value(bt, item, val, val_len); } else { @@ -1589,12 +1612,14 @@ int scoutfs_btree_delete(struct super_block *sb, struct scoutfs_block *bl; int ret; + scoutfs_inc_counter(sb, btree_delete); + ret = btree_walk(sb, alloc, wri, root, BTW_DELETE | BTW_DIRTY, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) { if (le16_to_cpu(bt->nr_items) == 1) { /* remove final empty block */ @@ -1692,6 +1717,8 @@ int scoutfs_btree_next(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { + scoutfs_inc_counter(sb, btree_next); + return btree_iter(sb, root, BTW_NEXT, key, iref); } @@ -1699,6 +1726,8 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref) { + scoutfs_inc_counter(sb, btree_prev); + return btree_iter(sb, root, BTW_PREV, key, iref); } @@ -1720,11 +1749,13 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_block *bl; int ret; + scoutfs_inc_counter(sb, btree_dirty); + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY, key, 0, &bl, NULL); if (ret == 0) { bt = bl->data; - item = leaf_item_hash_search(bt, key); + item = leaf_item_hash_search(sb, bt, key); if (item) ret = 0; else diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e96c01b0..db2785f6 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -22,8 +22,22 @@ EXPAND_COUNTER(block_cache_invalidate) \ EXPAND_COUNTER(block_cache_lru_move) \ EXPAND_COUNTER(block_cache_shrink) \ + EXPAND_COUNTER(btree_compact_values) \ + EXPAND_COUNTER(btree_delete) \ + EXPAND_COUNTER(btree_dirty) \ + EXPAND_COUNTER(btree_force) \ + EXPAND_COUNTER(btree_join) \ + EXPAND_COUNTER(btree_insert) \ + EXPAND_COUNTER(btree_leaf_item_hash_search) \ + EXPAND_COUNTER(btree_lookup) \ + EXPAND_COUNTER(btree_next) \ + EXPAND_COUNTER(btree_prev) \ EXPAND_COUNTER(btree_read_error) \ + EXPAND_COUNTER(btree_split) \ EXPAND_COUNTER(btree_stale_read) \ + EXPAND_COUNTER(btree_update) \ + EXPAND_COUNTER(btree_walk) \ + EXPAND_COUNTER(btree_walk_restart) \ EXPAND_COUNTER(client_farewell_error) \ EXPAND_COUNTER(corrupt_btree_block_level) \ EXPAND_COUNTER(corrupt_btree_no_child_ref) \ From 57c7caf3487242f72b298d48214b662da157f356 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 9 Jul 2020 15:01:06 -0700 Subject: [PATCH 852/920] scoutfs: fix forest dirty log tracking The forest code is responsible for constructing a consistent fs image out of the items spread across all the btrees written by mounts in the system. Usually readers walk a btree looking for log trees that they should read. As a mount modifies items in its dirty log tree, readers need to be sure to check that in-memory dirty log tree even though it isn't present in the btree that records persistent log trees. The code did this by setting a flag to indicate that readers using a lock should check the dirty log tree. But the flag usage wasn't properly locked and left a race where a reader and writer could race, leaving future readers to not know that they should check the dirty log tree. When we rarely hit that race we'd see item errors that made no sense, like not being able to find an inode item to update after having just created it in the current transaction. To fix this, we clean up the tree tracking in the forest code. We get rid of the static forest_root structs in the lock_private that were used to track the two special-case roots that aren't found in log tree items: the in-memory dirty log root and the final fs root. All roots are now dynamically allocated. We use a flag in the root to identify it as the dirty log root, and identify the fs root by its rid/nr. This results in a bunch of caller churn as we remove lpriv from root identifying functions. We get rid of the idea of the writer adding a static root to the list as well as marking the log as needing to read the root. Instead we make all root management happen as we refresh the list. The forest maintains a commit sequence and writers set state in the lock to indicate that the lock has dirty items in the log during this transaction. Iteration then compares the state set by the commit, writer, and the last refresh to determine if a new refresh needs to happen. Properly tracking the presence of dirty items lets us recognize when the lock no longer has dirty items in the log and we can stop locking and reading the dirty log and fall back to reading the committed stable version. The previous code didn't do that, it would lock and read the dirty root forever. While we're in here, we fix the locking around setting bloom bits and have it track the version of the log tree that was set so that we don't have to clear set bits as the log version is rotated out by the server. There was also a subtle bug where we could hit to stale errors for the same root and return -EIO because we triggering refresh returned stale. We rework the retrying logic to use a separate error code to force refreshing so that we can't accidentally trigger eio by conflating reading stale blocks and forcing refreshing. And finally, we no longer record that we need the dirty log tree in a root if we have a lock that could never read. It's a minor optimization that doesn't change functional behaviour. Signed-off-by: Zach Brown --- kmod/src/forest.c | 559 ++++++++++++++++++++++----------------- kmod/src/scoutfs_trace.h | 121 +++++++++ 2 files changed, 442 insertions(+), 238 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 319ec906..712aeb84 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -51,13 +51,20 @@ * readers to read every log btree looking for an item. Each log btree * contains a bloom filter keyed on the starting key of locks. This * lets lock holders quickly eliminate log trees that cannot contain - * keys protected by their lock and it caches the btrees to search in - * the lock for the duration of its use. + * keys protected by their lock. Since reads have to be done under + * locks, we cache the list of trees that could contain items in the + * lock. + * + * The list of roots in the locks can get out of date. Item + * modification in the current transactoin requires that the list + * contain the dirty log tree. Transaction commits mean that we can + * read from the stale log tree instead of the dirty one. And getting + * stale block reads from any of the trees means we need to rebuild the + * list from scratch. */ /* * todo: - * - when we adopt a new bloom root we'd need to reset bloom bits in locks * - add a bunch of counters so we can see bloom/tree ops/etc */ @@ -66,6 +73,7 @@ struct forest_info { struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; + atomic64_t commit_seq; struct mutex srch_mutex; struct scoutfs_srch_file srch_file; @@ -80,6 +88,7 @@ struct forest_root { struct scoutfs_btree_root item_root; u64 rid; u64 nr; + u8 our_dirty:1; }; struct forest_refs { @@ -91,46 +100,17 @@ struct forest_bloom_nrs { unsigned int nrs[SCOUTFS_FOREST_BLOOM_NRS]; }; -/* - * We have static forest_root entries for the fs and our log btrees so - * that we can iterate over them along with all the discovered and - * allocated log btrees. - */ struct forest_lock_private { u64 last_refreshed; struct rw_semaphore rwsem; unsigned int used_lock_roots:1; struct list_head roots; - struct forest_root fs_root; - struct forest_root our_log_root; - unsigned long flags; + u64 set_bloom_nr; + atomic64_t dirtied_cseq; + u64 refreshed_cseq; + u64 refreshed_dirtied; }; -enum { - LPRIV_FLAG_ALL_BLOOM_BITS = 0, -}; - -static inline void set_lpriv_flag(struct forest_lock_private *lpriv, int flag) -{ - set_bit(flag, &lpriv->flags); -} -static inline int test_lpriv_flag(struct forest_lock_private *lpriv, int flag) -{ - return test_bit(flag, &lpriv->flags); -} - -static bool is_fs_root(struct forest_lock_private *lpriv, - struct forest_root *fr) -{ - return fr == &lpriv->fs_root; -} - -static bool is_our_log_root(struct forest_lock_private *lpriv, - struct forest_root *fr) -{ - return fr == &lpriv->our_log_root; -} - static struct forest_lock_private *get_lock_private(struct scoutfs_lock *lock) { struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); @@ -140,8 +120,7 @@ static struct forest_lock_private *get_lock_private(struct scoutfs_lock *lock) if (lpriv) { init_rwsem(&lpriv->rwsem); INIT_LIST_HEAD(&lpriv->roots); - INIT_LIST_HEAD(&lpriv->fs_root.entry); - INIT_LIST_HEAD(&lpriv->our_log_root.entry); + atomic64_set(&lpriv->dirtied_cseq, 0); if (cmpxchg(&lock->forest_private, NULL, lpriv) != NULL) kfree(lpriv); @@ -152,51 +131,87 @@ static struct forest_lock_private *get_lock_private(struct scoutfs_lock *lock) return lpriv; } -/* - * We can tell if an item is currently dirty in our transaction's log - * root if its lock is held for writing and the item's version matches - * the lock's write version. - */ -static bool is_our_dirty_item(struct scoutfs_lock *lock, - struct forest_root *fr, u64 vers) +static bool is_fs_root(struct forest_root *fr) { - struct forest_lock_private *lpriv = get_lock_private(lock); + return fr->rid == 0 && fr->nr == 0; +} - return is_our_log_root(lpriv, fr) && - lock->mode == SCOUTFS_LOCK_WRITE && +/* + * We can be sure that we have the most recent version of an item if we + * have it write locked with the version of the lock. There can be no + * greater versions of the item in the system. + */ +static bool is_write_locked_version(struct scoutfs_lock *lock, u64 vers) +{ + return lock->mode == SCOUTFS_LOCK_WRITE && vers == lock->write_version; } -static void clear_roots(struct forest_lock_private *lpriv) +static void free_roots(struct forest_lock_private *lpriv) { struct forest_root *fr; struct forest_root *tmp; list_for_each_entry_safe(fr, tmp, &lpriv->roots, entry) { list_del_init(&fr->entry); - if (!is_fs_root(lpriv, fr) && !is_our_log_root(lpriv, fr)) - kfree(fr); + kfree(fr); } } /* - * Make sure that our log btree will be at the head of the list of - * btrees to read. We update the forest_root to refer to the most - * recent version of our log root before we try and use it instead of - * updating every instance of the forest_roots on locks as commits give - * us new versions of the same log tree. + * Add a *copy* of the root to the list of roots to read. If our_dirty + * is set then later readers will acquire the lock to serialize writers + * and update the root from the current dirty version. */ -static void add_our_log_root(struct forest_info *finf, - struct forest_lock_private *lpriv) +static int add_root(struct super_block *sb, struct scoutfs_lock *lock, + struct forest_lock_private *lpriv, + struct scoutfs_btree_root *item_root, u64 rid, u64 nr, + bool our_dirty) { - struct forest_root *fr = &lpriv->our_log_root; + struct forest_root *fr; BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); - if (list_empty(&fr->entry)) { - fr->rid = le64_to_cpu(finf->our_log.rid); - fr->nr = le64_to_cpu(finf->our_log.nr); - list_add(&fr->entry, &lpriv->roots); + fr = kmalloc(sizeof(struct forest_root), GFP_NOFS); + if (!fr) + return -ENOMEM; + + fr->item_root = *item_root; + fr->rid = rid; + fr->nr = nr; + fr->our_dirty = !!our_dirty; + list_add_tail(&fr->entry, &lpriv->roots); + + trace_scoutfs_forest_add_root(sb, &lock->start, fr->rid, fr->nr, + le64_to_cpu(fr->item_root.ref.blkno), + le64_to_cpu(fr->item_root.ref.seq)); + + return 0; +} + +/* + * The caller has dirtied the current log tree and still holds the + * transaction. We need to make sure that future reads know to check + * this dirty tree in particular. The tree can be committed (and + * rotated out!) before the next refresh so we use a commit sequence + * which will identify that it can find this tree either still dirty or + * can trust that it will find an item for it. + */ +static void set_dirtied_cseq(struct super_block *sb, struct forest_info *finf, + struct scoutfs_lock *lock, + struct forest_lock_private *lpriv) +{ + u64 cseq = atomic64_read(&finf->commit_seq); + + BUG_ON(!rwsem_is_locked(&finf->rwsem)); + + if (atomic64_read(&lpriv->dirtied_cseq) != cseq) { + atomic64_set(&lpriv->dirtied_cseq, cseq); + + trace_scoutfs_forest_set_dirtied(sb, &lock->start, + le64_to_cpu(finf->our_log.rid), + le64_to_cpu(finf->our_log.nr), + cseq); } } @@ -210,39 +225,48 @@ void scoutfs_forest_clear_lock(struct super_block *sb, struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); if (lpriv) { - clear_roots(lpriv); + free_roots(lpriv); kfree(lpriv); + lock->forest_private = NULL; } } /* - * All the btrees we read are stable and read-only except for our log - * btree which is being actively modified in memory by locked writers. - * Once we lock it we need to get the current version of the root. - * - * The finf rwsem protects updates of the finf root fields, the first - * caller here will change the fr fields and the rest will overwrite - * them with the same values. + * Usually we're reading from persistent btrees that won't be changing. + * But refresh can add a root that references the current dirty log root + * so that readers can see items which haven't yet been committed. Once + * we get the lock we make sure to give the forest root the current + * version of the tree which could have changed since it was added. + * Acquiring the lock also serializes commit responses updating the log + * and we can see if a commit has rotated in a new tree and we need to + * refresh the list. */ -static void read_lock_forest_root(struct forest_info *finf, - struct forest_lock_private *lpriv, - struct forest_root *fr) +static int read_lock_forest_root(struct forest_info *finf, + struct forest_lock_private *lpriv, + struct forest_root *fr) { - if (is_our_log_root(lpriv, fr)) { + int ret = 0; + + BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); + + if (fr->our_dirty) { down_read(&finf->rwsem); - fr->item_root = finf->our_log.item_root; - fr->rid = le64_to_cpu(finf->our_log.rid); - fr->nr = le64_to_cpu(finf->our_log.nr); + if (fr->nr == le64_to_cpu(finf->our_log.nr)) { + fr->item_root = finf->our_log.item_root; + } else { + up_read(&finf->rwsem); + ret = -EUCLEAN; + } } + + return ret; } static void read_unlock_forest_root(struct forest_info *finf, - struct forest_lock_private *lpriv, struct forest_root *fr) { - if (is_our_log_root(lpriv, fr)) { + if (fr->our_dirty) up_read(&finf->rwsem); - } } static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, @@ -299,10 +323,9 @@ static struct scoutfs_block *read_bloom_ref(struct super_block *sb, * Because we're starting all the reads from stable refs from the * server, this will not see any dirty blocks we have in memory. We * don't have to lock any of the btree reads. It also won't find the - * currently dirty version of our log btree. Writers mark our static - * log btree in lpriv to indicate that we should include our dirty log - * btree in reads. We'll also naturally add it if we see a persistent - * version on disk with all of the bloom bits set. + * currently dirty version of our log btree. Writers record the version + * of the current dirty log tree that must be added if it's still dirty + * when we refresh. */ static int refresh_bloom_roots(struct super_block *sb, struct scoutfs_lock *lock, @@ -312,12 +335,16 @@ static int refresh_bloom_roots(struct super_block *sb, struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); struct scoutfs_net_roots roots; struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees *lt; SCOUTFS_BTREE_ITEM_REF(iref); struct forest_bloom_nrs bloom; - struct forest_root *fr = NULL; struct scoutfs_bloom_block *bb; struct scoutfs_block *bl; struct scoutfs_key key; + u64 our_rid = 0; + u64 our_nr = 0; + u64 dirtied; + u64 cseq; int ret; int i; @@ -326,7 +353,36 @@ static int refresh_bloom_roots(struct super_block *sb, down_write(&lpriv->rwsem); /* empty the list so no one iterates until someone's added */ - clear_roots(lpriv); + free_roots(lpriv); + + /* make sure readers see writer's in-memory dirty items */ + cseq = atomic64_read(&finf->commit_seq); + dirtied = atomic64_read(&lpriv->dirtied_cseq); + + if (dirtied == cseq) { + down_read(&finf->rwsem); + cseq = atomic64_read(&finf->commit_seq); + dirtied = atomic64_read(&lpriv->dirtied_cseq); + if (dirtied == cseq) { + lt = &finf->our_log; + our_rid = le64_to_cpu(lt->rid); + our_nr = le64_to_cpu(lt->nr); + /* root be updated before reads, but nice to trace */ + ret = add_root(sb, lock, lpriv, <->item_root, + our_rid, our_nr, true); + } else { + ret = 0; + /* must get roots from network to see committed */ + lpriv->used_lock_roots = 1; + } + up_read(&finf->rwsem); + if (ret < 0) + goto out; + } + + trace_scoutfs_forest_refresh_seqs(sb, &lock->start, our_rid, our_nr, + dirtied, lpriv->refreshed_dirtied, + cseq, lpriv->refreshed_cseq); /* first use the lock's constant roots, then sample newer roots */ if (!lpriv->used_lock_roots) { @@ -350,22 +406,22 @@ static int refresh_bloom_roots(struct super_block *sb, for (;; scoutfs_key_inc(&key)) { ret = scoutfs_btree_next(sb, &roots.logs_root, &key, &iref); - if (ret == -ENOENT) { - ret = 0; - break; + if (ret == 0) { + if (iref.val_len == sizeof(ltv)) { + key = *iref.key; + memcpy(<v, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); } - if (ret < 0) + if (ret < 0) { + if (ret == -ENOENT) { + ret = 0; + break; + } goto out; - - if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { - key = *iref.key; - memcpy(<v, iref.val, iref.val_len); - } else { - ret = -EIO; } - scoutfs_btree_put_iref(&iref); - if (ret < 0) - goto out; if (ltv.bloom_ref.blkno == 0) continue; @@ -395,49 +451,32 @@ static int refresh_bloom_roots(struct super_block *sb, if (i != ARRAY_SIZE(bloom.nrs)) continue; - /* use our dirty log instead of the old committed version */ - if (key.sklt_rid == finf->our_log.rid && - key.sklt_nr == finf->our_log.nr) { - add_our_log_root(finf, lpriv); + /* we've added our dirty log, skip old committed versions */ + if (le64_to_cpu(key.sklt_rid) == our_rid && + le64_to_cpu(key.sklt_nr) == our_nr) continue; - } - /* all bloom bits set, add to the list */ - fr = kzalloc(sizeof(struct forest_root), GFP_NOFS); - if (fr == NULL) { - ret = -ENOMEM; + ret = add_root(sb, lock, lpriv, <v.item_root, + le64_to_cpu(key.sklt_rid), + le64_to_cpu(key.sklt_nr), false); + if (ret < 0) goto out; - } - - fr->item_root = ltv.item_root; - fr->rid = le64_to_cpu(key.sklt_rid); - fr->nr = le64_to_cpu(key.sklt_nr); - - list_add_tail(&fr->entry, &lpriv->roots); - - trace_scoutfs_forest_add_root(sb, &lock->start, fr->rid, - fr->nr, le64_to_cpu(fr->item_root.ref.blkno), - le64_to_cpu(fr->item_root.ref.seq)); } - /* make sure readers search our dirty log after writers set bloom */ - if (test_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS)) - add_our_log_root(finf, lpriv); - - /* always add the fs root at the tail */ - fr = &lpriv->fs_root; - fr->item_root = roots.fs_root; - fr->rid = 0; - fr->nr = 0; - list_add_tail(&fr->entry, &lpriv->roots); + /* always add final fs tree last */ + ret = add_root(sb, lock, lpriv, &roots.fs_root, 0, 0, false); + if (ret < 0) + goto out; + lpriv->refreshed_cseq = cseq; + lpriv->refreshed_dirtied = dirtied; lpriv->last_refreshed = lock->refresh_gen; ret = 0; out: if (ret < 0) - clear_roots(lpriv); + free_roots(lpriv); up_write(&lpriv->rwsem); return ret; @@ -449,27 +488,34 @@ out: struct forest_refs b = {{cpu_to_le64(1),}} /* - * The caller saw stale blocks. If they're seeing the same root refs - * and are still getting stale then it's consistent corruption and we - * return an error. Otherwise we refresh the bloom roots and try again. - * If this returns 0 then the caller is going to retry. If *we* saw - * stale blocks trying to refresh the bloom then we return 0 to have the - * caller remember the root refs and try again. + * If the caller got our magic errnos we refresh the roots and return + * -EAGAIN so they retry. If we get -ESTALE from block reference + * inconsistency with the same root refs then it's consistent corruption + * and we return an error. We pass through all other errnos that aren't + * our magic retry errnos. */ -static int refresh_check_stale(struct super_block *sb, - struct scoutfs_lock *lock, - struct forest_refs *prev_refs, - struct forest_refs *refs) +static int refresh_check(struct super_block *sb, struct scoutfs_lock *lock, + struct forest_refs *prev_refs, + struct forest_refs *refs, int err) { int ret; - if (memcmp(prev_refs, refs, sizeof(*refs)) == 0) - return -EIO; + /* don't want to get in a loop passing eagain through, not expected */ + if (WARN_ON_ONCE(err == -EAGAIN)) + return -EINVAL; + + if (!(err == -ESTALE || err == -EUCLEAN)) + return err; + + if (err == -ESTALE) { + if (memcmp(prev_refs, refs, sizeof(*refs)) == 0) + return -EIO; + } *prev_refs = *refs; ret = refresh_bloom_roots(sb, lock, refs); - if (ret == -ESTALE) - ret = 0; + if (ret == 0 || ret == -ESTALE) + ret = -EAGAIN; return ret; } @@ -477,20 +523,41 @@ static int refresh_check_stale(struct super_block *sb, /* * Iterate over all the roots that could contain items covered by the * caller's lock. The caller starts iteration by passing in a NULL fr. - * We return -ESTALE if the caller needs to refresh the bloom roots. We - * use the lock's refresh gen to find out when the lock was invalidated - * and the contents of the trees could have changed. + * We return -EUCLEAN if the caller needs to refresh the bloom roots. + * We use the lock's refresh gen to find out when the lock was + * invalidated and the contents of the trees could have changed. + * + * The commit_seqs are keeping the list of roots in sync with our log + * root. As writers modify it we make sure we have a root that will + * lock and check our in-memory dirty log tre. Once that's committed we + * refresh again so we read the stable committed version without locks. */ -static int for_each_forest_root(struct scoutfs_lock *lock, +static int for_each_forest_root(struct super_block *sb, + struct scoutfs_lock *lock, + struct forest_info *finf, struct forest_lock_private *lpriv, struct forest_root **fr) { + u64 cseq = atomic64_read(&finf->commit_seq); + u64 dirtied = atomic64_read(&lpriv->dirtied_cseq); + if (WARN_ON_ONCE(!rwsem_is_locked(&lpriv->rwsem))) return -EIO; if (list_empty(&lpriv->roots) || - lock->refresh_gen != lpriv->last_refreshed) - return -ESTALE; + lock->refresh_gen != lpriv->last_refreshed || + dirtied > lpriv->refreshed_dirtied || + (dirtied == lpriv->refreshed_cseq && + cseq > lpriv->refreshed_cseq)) { + trace_scoutfs_forest_trigger_refresh(sb, + &lock->start, + !!list_empty(&lpriv->roots), + lock->refresh_gen, + lpriv->last_refreshed, + dirtied, lpriv->refreshed_dirtied, + cseq, lpriv->refreshed_cseq); + return -EUCLEAN; + } if (*fr == NULL) *fr = list_prepare_entry((*fr), &lpriv->roots, entry); @@ -507,34 +574,31 @@ static int for_each_forest_root(struct scoutfs_lock *lock, * version is also 1, but we guarantee that we check the log trees first * so they'll always be found before the fs items. */ -static u64 item_vers(struct forest_lock_private *lpriv, - struct forest_root *fr, void *val) +static u64 item_vers(struct forest_root *fr, void *val) { struct scoutfs_log_item_value *liv; - if (is_fs_root(lpriv, fr)) + if (is_fs_root(fr)) return 1; liv = val; return le64_to_cpu(liv->vers); } -static bool item_flags(struct forest_lock_private *lpriv, - struct forest_root *fr, void *val) +static bool item_flags(struct forest_root *fr, void *val) { struct scoutfs_log_item_value *liv; - if (is_fs_root(lpriv, fr)) + if (is_fs_root(fr)) return 0; liv = val; return liv->flags; } -static bool item_is_deletion(struct forest_lock_private *lpriv, - struct forest_root *fr, void *val) +static bool item_is_deletion(struct forest_root *fr, void *val) { - return item_flags(lpriv, fr, val) & SCOUTFS_LOG_ITEM_FLAG_DELETION; + return item_flags(fr, val) & SCOUTFS_LOG_ITEM_FLAG_DELETION; } /* just a little helper to slim down all the call sites */ @@ -553,14 +617,14 @@ static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, * A null val returns 0. Items in log trees have a value header that * needs to be skipped. */ -static int copy_val(struct forest_lock_private *lpriv, struct forest_root *fr, - struct kvec *val, void *item_val, int item_val_len) +static int copy_val(struct forest_root *fr, struct kvec *val, void *item_val, + int item_val_len) { void *val_start = item_val; unsigned int val_len = item_val_len; int ret; - if (!is_fs_root(lpriv, fr)) { + if (!is_fs_root(fr)) { val_start += sizeof(struct scoutfs_log_item_value); val_len -= sizeof(struct scoutfs_log_item_value); } @@ -604,48 +668,48 @@ retry: ret = -ENOENT; fr = NULL; - while (!(err = for_each_forest_root(lock, lpriv, &fr)) && fr) { + while (!(err = for_each_forest_root(sb, lock, finf, lpriv, &fr)) && fr){ /* done if we found log items before fs root */ - if (found_vers > 0 && is_fs_root(lpriv, fr)) + if (found_vers > 0 && is_fs_root(fr)) break; - read_lock_forest_root(finf, lpriv, fr); + err = read_lock_forest_root(finf, lpriv, fr); + if (err < 0) + break; err = scoutfs_btree_lookup(sb, &fr->item_root, key, &iref); if (err < 0) - read_unlock_forest_root(finf, lpriv, fr); + read_unlock_forest_root(finf, fr); if (err == -ENOENT) continue; if (err < 0) break; - vers = item_vers(lpriv, fr, iref.val); + vers = item_vers(fr, iref.val); if (vers > found_vers) { found_vers = vers; - if (item_is_deletion(lpriv, fr, iref.val)) + if (item_is_deletion(fr, iref.val)) ret = -ENOENT; else - ret = copy_val(lpriv, fr, val, - iref.val, iref.val_len); + ret = copy_val(fr, val, iref.val, iref.val_len); } scoutfs_btree_put_iref(&iref); - read_unlock_forest_root(finf, lpriv, fr); + read_unlock_forest_root(finf, fr); /* done if we have the most recent locked dirty version */ - if (is_our_dirty_item(lock, fr, vers)) + if (is_write_locked_version(lock, vers)) break; } up_read(&lpriv->rwsem); - if (err == -ESTALE) { - err = refresh_check_stale(sb, lock, &prev_refs, &refs); - if (err == 0) - goto retry; + err = refresh_check(sb, lock, &prev_refs, &refs, err); + if (err == -EAGAIN) + goto retry; + if (err < 0) ret = err; - } out: return ret; } @@ -878,7 +942,7 @@ retry: /* initialize iter position for each tree */ fr = NULL; - while (!(ret = for_each_forest_root(lock, lpriv, &fr)) && fr) { + while (!(ret = for_each_forest_root(sb, lock, finf, lpriv, &fr)) && fr){ ip = kmalloc(sizeof(struct forest_iter_pos), GFP_NOFS); if (!ip) { ret = -ENOMEM; @@ -905,11 +969,13 @@ retry: /* search for the next item in the root */ if (ip->vers == 0) { - read_lock_forest_root(finf, lpriv, fr); + ret = read_lock_forest_root(finf, lpriv, fr); + if (ret < 0) + goto unlock; ret = forest_iter_btree_search(sb, &fr->item_root, &ip->key, &iref, fwd); if (ret < 0) - read_unlock_forest_root(finf, lpriv, fr); + read_unlock_forest_root(finf, fr); if (ret == -ENOENT) { destroy_iter_pos(ip, &iter_root); continue; @@ -918,12 +984,12 @@ retry: goto unlock; ip->key = *iref.key; - ip->vers = item_vers(lpriv, fr, iref.val); - ip->deletion = item_is_deletion(lpriv, fr, iref.val); + ip->vers = item_vers(fr, iref.val); + ip->deletion = item_is_deletion(fr, iref.val); trace_scoutfs_forest_iter_search(sb, fr->rid, fr->nr, ip->vers, - item_flags(lpriv, fr, iref.val), + item_flags(fr, iref.val), &ip->key); if (!forest_iter_key_within(&ip->key, end, fwd)) { @@ -944,7 +1010,7 @@ retry: } scoutfs_btree_put_iref(&iref); - read_unlock_forest_root(finf, lpriv, fr); + read_unlock_forest_root(finf, fr); if (ret < 0) goto unlock; @@ -964,7 +1030,7 @@ retry: /* use the first non-deletion across all roots */ found_key = ip->key; found_vers = ip->vers; - found_ret = copy_val(lpriv, ip->fr, val, ip->val, ip->val_len); + found_ret = copy_val(ip->fr, val, ip->val, ip->val_len); break; } @@ -979,11 +1045,9 @@ unlock: destroy_iter_pos(ip, &iter_root); } - if (ret == -ESTALE) { - ret = refresh_check_stale(sb, lock, &prev_refs, &refs); - if (ret == 0) - goto retry; - } + ret = refresh_check(sb, lock, &prev_refs, &refs, ret); + if (ret == -EAGAIN) + goto retry; out: trace_scoutfs_forest_iter_ret(sb, key, end, fwd, ret, @@ -1116,24 +1180,18 @@ out: return ret; } - /* - * Make sure that the bloom bits for the lock's start value are all set - * in the bloom block. We record the bits being set in the lock so that - * we only dirty the bloom block once per lock acquisition per log - * btree. + * Make sure that the bloom bits for the lock's start key are all set in + * the current log's bloom block. We record the nr of our log tree in + * the lock so that we only try to cow and set the bits once per tree. * - * If all the bloom bits weren't set then our log btree won't have been - * found by the search for log btrees to read under the lock. The - * caller is about to insert an item into the log tree that future - * readers must find so we make sure that the log root is added to the - * lock's list of roots. - * - * This can be racing with itself and readers in any stages of checking - * the forest trees and bloom blocks. + * The caller already gets the big finf write rwsem lock to modify the + * dirty log btree, might as well use it to protect the bloom ref and + * the lpriv field. We'll need finer grained locking once the btrees + * get block locks. */ static int set_lock_bloom_bits(struct super_block *sb, - struct scoutfs_lock *lock) + struct scoutfs_lock *lock, u64 nr) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_FOREST_INFO(sb, finf); @@ -1149,28 +1207,29 @@ static int set_lock_bloom_bits(struct super_block *sb, int err; int i; + BUG_ON(!rwsem_is_locked(&finf->rwsem)); + lpriv = get_lock_private(lock); if (!lpriv) { ret = -ENOMEM; goto out; } - if (test_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS)) { + /* our rid is constant */ + if (lpriv->set_bloom_nr == nr) { ret = 0; goto out; } calc_bloom_nrs(&bloom, &lock->start); - down_write(&finf->rwsem); - ref = &finf->our_log.bloom_ref; if (ref->blkno) { bl = read_bloom_ref(sb, ref); if (IS_ERR(bl)) { ret = PTR_ERR(bl); - goto unlock; + goto out; } bb = bl->data; } @@ -1179,7 +1238,7 @@ static int set_lock_bloom_bits(struct super_block *sb, ret = scoutfs_radix_alloc(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) - goto unlock; + goto out; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { @@ -1187,7 +1246,7 @@ static int set_lock_bloom_bits(struct super_block *sb, blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); - goto unlock; + goto out; } if (bl) { @@ -1228,17 +1287,8 @@ static int set_lock_bloom_bits(struct super_block *sb, le64_to_cpu(finf->our_log.bloom_ref.seq), nr_set); + lpriv->set_bloom_nr = nr; ret = 0; -unlock: - up_write(&finf->rwsem); - - if (ret == 0) { - down_write(&lpriv->rwsem); - add_our_log_root(finf, lpriv); - up_write(&lpriv->rwsem); - set_lpriv_flag(lpriv, LPRIV_FLAG_ALL_BLOOM_BITS); - } - out: scoutfs_block_put(sb, bl); return ret; @@ -1282,12 +1332,19 @@ static struct kvec *alloc_log_item_value(struct kvec *val, __u8 flags, */ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock, - bool check_eexist, bool check_enoent) + bool check_eexist, bool check_enoent, bool could_read) { DECLARE_FOREST_INFO(sb, finf); + struct forest_lock_private *lpriv; struct kvec *iv = NULL; int ret; + lpriv = get_lock_private(lock); + if (!lpriv) { + ret = -ENOMEM; + goto out; + } + if (check_eexist || check_enoent) { ret = scoutfs_forest_lookup(sb, key, NULL, lock); if (ret == 0 && check_eexist) { @@ -1301,11 +1358,8 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, } if (ret < 0) goto out; - } - ret = set_lock_bloom_bits(sb, lock); - if (ret < 0) - goto out; + } iv = alloc_log_item_value(val, 0, lock); if (iv == NULL) { @@ -1314,11 +1368,20 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, } down_write(&finf->rwsem); + + ret = set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); + if (ret < 0) + goto unlock; + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, &finf->our_log.item_root, key, iv->iov_base, iv->iov_len); + if (ret == 0 && could_read) + set_dirtied_cseq(sb, finf, lock, lpriv); +unlock: up_write(&finf->rwsem); kfree(iv); + out: return ret; } @@ -1334,7 +1397,7 @@ int scoutfs_forest_create(struct super_block *sb, struct scoutfs_key *key, if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) return ret; - return forest_insert(sb, key, val, lock, true, false); + return forest_insert(sb, key, val, lock, true, false, true); } /* @@ -1349,7 +1412,7 @@ int scoutfs_forest_create_force(struct super_block *sb, if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) return ret; - return forest_insert(sb, key, val, lock, false, false); + return forest_insert(sb, key, val, lock, false, false, false); } /* @@ -1364,7 +1427,7 @@ int scoutfs_forest_update(struct super_block *sb, struct scoutfs_key *key, if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) return ret; - return forest_insert(sb, key, val, lock, false, true); + return forest_insert(sb, key, val, lock, false, true, true); } /* XXX not yet supported, idea is btree op that only uses dirty blocks */ @@ -1376,29 +1439,41 @@ int scoutfs_forest_delete_dirty(struct super_block *sb, } static int forest_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock, bool check_enoent) + struct scoutfs_lock *lock, bool check_enoent, + bool could_read) { DECLARE_FOREST_INFO(sb, finf); + struct forest_lock_private *lpriv; struct scoutfs_log_item_value liv; int ret; + lpriv = get_lock_private(lock); + if (!lpriv) { + ret = -ENOMEM; + goto out; + } + if (check_enoent) { ret = scoutfs_forest_lookup(sb, key, NULL, lock); if (ret < 0) goto out; } - ret = set_lock_bloom_bits(sb, lock); - if (ret < 0) - goto out; - liv.vers = cpu_to_le64(lock->write_version); liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; down_write(&finf->rwsem); + + ret = set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); + if (ret < 0) + goto unlock; + ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, - &finf->our_log.item_root, key, &liv, - sizeof(liv)); + &finf->our_log.item_root, + key, &liv, sizeof(liv)); + if (ret == 0 && could_read) + set_dirtied_cseq(sb, finf, lock, lpriv); +unlock: up_write(&finf->rwsem); out: return ret; @@ -1419,7 +1494,7 @@ int scoutfs_forest_delete(struct super_block *sb, struct scoutfs_key *key, if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) return ret; - return forest_delete(sb, key, lock, true); + return forest_delete(sb, key, lock, true, true); } /* @@ -1435,7 +1510,7 @@ int scoutfs_forest_delete_force(struct super_block *sb, if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) return ret; - return forest_delete(sb, key, lock, false); + return forest_delete(sb, key, lock, false, false); } /* XXX not supported, just for initial demo */ @@ -1506,10 +1581,17 @@ void scoutfs_forest_init_btrees(struct super_block *sb, finf->our_log.bloom_ref = lt->bloom_ref; finf->our_log.rid = lt->rid; finf->our_log.nr = lt->nr; + atomic64_inc(&finf->commit_seq); finf->srch_file = lt->srch_file; WARN_ON_ONCE(finf->srch_bl); /* commiting should have put the block */ finf->srch_bl = NULL; + trace_scoutfs_forest_init_our_log(sb, le64_to_cpu(lt->rid), + le64_to_cpu(lt->nr), + le64_to_cpu(lt->item_root.ref.blkno), + le64_to_cpu(lt->item_root.ref.seq), + atomic64_read(&finf->commit_seq)); + up_write(&finf->rwsem); } @@ -1550,6 +1632,7 @@ int scoutfs_forest_setup(struct super_block *sb) /* the finf fields will be setup as we open a transaction */ init_rwsem(&finf->rwsem); mutex_init(&finf->srch_mutex); + atomic64_set(&finf->commit_seq, 0); sbi->forest_info = finf; ret = 0; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index d96a836e..f44d0241 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2129,6 +2129,127 @@ TRACE_EVENT(scoutfs_forest_add_root, __entry->b_rid, __entry->nr, __entry->blkno, __entry->seq) ); +TRACE_EVENT(scoutfs_forest_set_dirtied, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, + u64 nr, u64 cseq), + TP_ARGS(sb, key, rid, nr, cseq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, cseq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->cseq = cseq; + ), + TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu cseq %llu", + SCSB_TRACE_ARGS, sk_trace_args(key), + __entry->b_rid, __entry->nr, __entry->cseq) +); + +TRACE_EVENT(scoutfs_forest_trigger_refresh, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, + bool empty_roots, u64 refresh_gen, u64 last_refreshed, + u64 dirtied_cseq, u64 refreshed_dirtied, + u64 commit_seq, u64 refreshed_cseq), + TP_ARGS(sb, key, empty_roots, refresh_gen, last_refreshed, + dirtied_cseq, refreshed_dirtied, commit_seq, refreshed_cseq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(int, empty_roots) + __field(__u64, refresh_gen) + __field(__u64, last_refreshed) + __field(__u64, dirtied_cseq) + __field(__u64, refreshed_dirtied) + __field(__u64, commit_seq) + __field(__u64, refreshed_cseq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->empty_roots = !!empty_roots; + __entry->refresh_gen = refresh_gen; + __entry->last_refreshed = last_refreshed; + __entry->dirtied_cseq = dirtied_cseq; + __entry->refreshed_dirtied = refreshed_dirtied; + __entry->commit_seq = commit_seq; + __entry->refreshed_cseq = refreshed_cseq; + ), + TP_printk(SCSBF" key "SK_FMT" empty %u refg %llu last_refg %llu dirt %llu refdir %llu cseq %llu refcseq %llu", + SCSB_TRACE_ARGS, sk_trace_args(key), + __entry->empty_roots, + __entry->refresh_gen, + __entry->last_refreshed, + __entry->dirtied_cseq, + __entry->refreshed_dirtied, + __entry->commit_seq, + __entry->refreshed_cseq) +); + +TRACE_EVENT(scoutfs_forest_refresh_seqs, + TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, + u64 nr, u64 dirtied_cseq, u64 refreshed_dirtied, + u64 commit_seq, u64 refreshed_cseq), + TP_ARGS(sb, key, rid, nr, dirtied_cseq, refreshed_dirtied, commit_seq, + refreshed_cseq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + sk_trace_define(key) + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, dirtied_cseq) + __field(__u64, refreshed_dirtied) + __field(__u64, commit_seq) + __field(__u64, refreshed_cseq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + sk_trace_assign(key, key); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->dirtied_cseq = dirtied_cseq; + __entry->refreshed_dirtied = refreshed_dirtied; + __entry->commit_seq = commit_seq; + __entry->refreshed_cseq = refreshed_cseq; + ), + TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu dirt %llu refdir %llu cseq %llu refcseq %llu", + SCSB_TRACE_ARGS, sk_trace_args(key), __entry->b_rid, + __entry->nr, __entry->dirtied_cseq, + __entry->refreshed_dirtied, __entry->commit_seq, + __entry->refreshed_cseq) +); + +TRACE_EVENT(scoutfs_forest_init_our_log, + TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 blkno, u64 seq, + u64 cseq), + TP_ARGS(sb, rid, nr, blkno, seq, cseq), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, b_rid) + __field(__u64, nr) + __field(__u64, blkno) + __field(__u64, seq) + __field(__u64, cseq) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->b_rid = rid; + __entry->nr = nr; + __entry->blkno = blkno; + __entry->seq = seq; + __entry->cseq = cseq; + ), + TP_printk(SCSBF" rid %016llx nr %llu blkno %llu seq %llx cseq %llu", + SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, + __entry->blkno, __entry->seq, __entry->cseq) +); + TRACE_EVENT(scoutfs_forest_iter_search, TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 vers, u8 flags, struct scoutfs_key *key), From 9658412d09ffe352a5367306e6e93e26f99bf71a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Jul 2020 14:54:17 -0700 Subject: [PATCH 853/920] scoutfs: add forest counters Add a bunch of counters to track significant events in the forest. Signed-off-by: Zach Brown --- kmod/src/counters.h | 17 +++++++++++++++++ kmod/src/forest.c | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index db2785f6..af1d86a5 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -56,9 +56,26 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ + EXPAND_COUNTER(forest_add_root) \ + EXPAND_COUNTER(forest_bloom_fail) \ + EXPAND_COUNTER(forest_bloom_pass) \ + EXPAND_COUNTER(forest_clear_lock) \ + EXPAND_COUNTER(forest_delete) \ + EXPAND_COUNTER(forest_insert) \ + EXPAND_COUNTER(forest_iter) \ + EXPAND_COUNTER(forest_lookup) \ + EXPAND_COUNTER(forest_read_lock_log) \ + EXPAND_COUNTER(forest_read_lock_rotated) \ + EXPAND_COUNTER(forest_refresh_bloom_roots) \ + EXPAND_COUNTER(forest_refresh_dirty_log) \ + EXPAND_COUNTER(forest_refresh_skip_log) \ EXPAND_COUNTER(forest_roots_next_hint) \ EXPAND_COUNTER(forest_roots_lock) \ EXPAND_COUNTER(forest_roots_server) \ + EXPAND_COUNTER(forest_saw_stale) \ + EXPAND_COUNTER(forest_set_bloom_bits) \ + EXPAND_COUNTER(forest_set_dirtied) \ + EXPAND_COUNTER(forest_trigger_refresh) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grace_elapsed) \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 712aeb84..e37490bd 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -63,11 +63,6 @@ * list from scratch. */ -/* - * todo: - * - add a bunch of counters so we can see bloom/tree ops/etc - */ - struct forest_info { struct rw_semaphore rwsem; struct scoutfs_radix_allocator *alloc; @@ -182,6 +177,7 @@ static int add_root(struct super_block *sb, struct scoutfs_lock *lock, fr->our_dirty = !!our_dirty; list_add_tail(&fr->entry, &lpriv->roots); + scoutfs_inc_counter(sb, forest_add_root); trace_scoutfs_forest_add_root(sb, &lock->start, fr->rid, fr->nr, le64_to_cpu(fr->item_root.ref.blkno), le64_to_cpu(fr->item_root.ref.seq)); @@ -207,6 +203,7 @@ static void set_dirtied_cseq(struct super_block *sb, struct forest_info *finf, if (atomic64_read(&lpriv->dirtied_cseq) != cseq) { atomic64_set(&lpriv->dirtied_cseq, cseq); + scoutfs_inc_counter(sb, forest_set_dirtied); trace_scoutfs_forest_set_dirtied(sb, &lock->start, le64_to_cpu(finf->our_log.rid), @@ -225,6 +222,7 @@ void scoutfs_forest_clear_lock(struct super_block *sb, struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); if (lpriv) { + scoutfs_inc_counter(sb, forest_clear_lock); free_roots(lpriv); kfree(lpriv); lock->forest_private = NULL; @@ -241,7 +239,8 @@ void scoutfs_forest_clear_lock(struct super_block *sb, * and we can see if a commit has rotated in a new tree and we need to * refresh the list. */ -static int read_lock_forest_root(struct forest_info *finf, +static int read_lock_forest_root(struct super_block *sb, + struct forest_info *finf, struct forest_lock_private *lpriv, struct forest_root *fr) { @@ -252,8 +251,10 @@ static int read_lock_forest_root(struct forest_info *finf, if (fr->our_dirty) { down_read(&finf->rwsem); if (fr->nr == le64_to_cpu(finf->our_log.nr)) { + scoutfs_inc_counter(sb, forest_read_lock_log); fr->item_root = finf->our_log.item_root; } else { + scoutfs_inc_counter(sb, forest_read_lock_rotated); up_read(&finf->rwsem); ret = -EUCLEAN; } @@ -348,6 +349,8 @@ static int refresh_bloom_roots(struct super_block *sb, int ret; int i; + scoutfs_inc_counter(sb, forest_refresh_bloom_roots); + memset(refs, 0, sizeof(*refs)); down_write(&lpriv->rwsem); @@ -364,6 +367,7 @@ static int refresh_bloom_roots(struct super_block *sb, cseq = atomic64_read(&finf->commit_seq); dirtied = atomic64_read(&lpriv->dirtied_cseq); if (dirtied == cseq) { + scoutfs_inc_counter(sb, forest_refresh_dirty_log); lt = &finf->our_log; our_rid = le64_to_cpu(lt->rid); our_nr = le64_to_cpu(lt->nr); @@ -448,13 +452,19 @@ static int refresh_bloom_roots(struct super_block *sb, i); /* one of the bloom bits wasn't set */ - if (i != ARRAY_SIZE(bloom.nrs)) + if (i != ARRAY_SIZE(bloom.nrs)) { + scoutfs_inc_counter(sb, forest_bloom_fail); continue; + } + + scoutfs_inc_counter(sb, forest_bloom_pass); /* we've added our dirty log, skip old committed versions */ if (le64_to_cpu(key.sklt_rid) == our_rid && - le64_to_cpu(key.sklt_nr) == our_nr) + le64_to_cpu(key.sklt_nr) == our_nr) { + scoutfs_inc_counter(sb, forest_refresh_skip_log); continue; + } ret = add_root(sb, lock, lpriv, <v.item_root, le64_to_cpu(key.sklt_rid), @@ -508,6 +518,7 @@ static int refresh_check(struct super_block *sb, struct scoutfs_lock *lock, return err; if (err == -ESTALE) { + scoutfs_inc_counter(sb, forest_saw_stale); if (memcmp(prev_refs, refs, sizeof(*refs)) == 0) return -EIO; } @@ -549,6 +560,7 @@ static int for_each_forest_root(struct super_block *sb, dirtied > lpriv->refreshed_dirtied || (dirtied == lpriv->refreshed_cseq && cseq > lpriv->refreshed_cseq)) { + scoutfs_inc_counter(sb, forest_trigger_refresh); trace_scoutfs_forest_trigger_refresh(sb, &lock->start, !!list_empty(&lpriv->roots), @@ -652,6 +664,8 @@ int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, int ret; int err; + scoutfs_inc_counter(sb, forest_lookup); + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) goto out; @@ -674,7 +688,7 @@ retry: if (found_vers > 0 && is_fs_root(fr)) break; - err = read_lock_forest_root(finf, lpriv, fr); + err = read_lock_forest_root(sb, finf, lpriv, fr); if (err < 0) break; err = scoutfs_btree_lookup(sb, &fr->item_root, key, &iref); @@ -916,6 +930,7 @@ static int forest_iter(struct super_block *sb, struct scoutfs_key *key, int found_ret = 0; int ret; + scoutfs_inc_counter(sb, forest_iter); scoutfs_key_set_zeros(&found_key); if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) @@ -969,7 +984,7 @@ retry: /* search for the next item in the root */ if (ip->vers == 0) { - ret = read_lock_forest_root(finf, lpriv, fr); + ret = read_lock_forest_root(sb, finf, lpriv, fr); if (ret < 0) goto unlock; ret = forest_iter_btree_search(sb, &fr->item_root, @@ -1221,6 +1236,7 @@ static int set_lock_bloom_bits(struct super_block *sb, goto out; } + scoutfs_inc_counter(sb, forest_set_bloom_bits); calc_bloom_nrs(&bloom, &lock->start); ref = &finf->our_log.bloom_ref; @@ -1339,6 +1355,8 @@ static int forest_insert(struct super_block *sb, struct scoutfs_key *key, struct kvec *iv = NULL; int ret; + scoutfs_inc_counter(sb, forest_insert); + lpriv = get_lock_private(lock); if (!lpriv) { ret = -ENOMEM; @@ -1447,6 +1465,8 @@ static int forest_delete(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_log_item_value liv; int ret; + scoutfs_inc_counter(sb, forest_delete); + lpriv = get_lock_private(lock); if (!lpriv) { ret = -ENOMEM; From 63564400737f4662497e89363be39ab7b87c6b38 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Jul 2020 16:40:34 -0700 Subject: [PATCH 854/920] scoutfs: add error message for client commit error We had a debugging WARN_ON that warns when a client has an error commiting their transaction. Let's add a bit more detail and promote it to a proper error. These should not happen. Signed-off-by: Zach Brown --- kmod/src/trans.c | 59 +++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 11522c0a..6001724e 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -27,6 +27,7 @@ #include "inode.h" #include "radix.h" #include "block.h" +#include "msg.h" #include "scoutfs_trace.h" /* @@ -126,6 +127,7 @@ bool scoutfs_trans_has_dirty(struct super_block *sb) return scoutfs_block_writer_has_dirty(sb, &tri->wri); } + /* * This work func is responsible for writing out all the dirty blocks * that make up the current dirty transaction. It prevents writers from @@ -156,6 +158,7 @@ void scoutfs_trans_write_func(struct work_struct *work) trans_write_work.work); struct super_block *sb = sbi->sb; DECLARE_TRANS_INFO(sb, tri); + char *s = NULL; int ret = 0; sbi->trans_task = current; @@ -165,35 +168,39 @@ void scoutfs_trans_write_func(struct work_struct *work) trace_scoutfs_trans_write_func(sb, scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); - if (scoutfs_block_writer_has_dirty(sb, &tri->wri)) { - if (sbi->trans_deadline_expired) - scoutfs_inc_counter(sb, trans_commit_timer); - - scoutfs_inc_counter(sb, trans_commit_written); - - ret = scoutfs_inode_walk_writeback(sb, true) ?: - scoutfs_block_writer_write(sb, &tri->wri) ?: - scoutfs_inode_walk_writeback(sb, false) ?: - commit_btrees(sb) ?: - scoutfs_client_advance_seq(sb, &sbi->trans_seq) ?: - scoutfs_trans_get_log_trees(sb); - if (ret) - goto out; - - } else if (sbi->trans_deadline_expired) { - /* - * If we're not writing data then we only advance the - * seq at the sync deadline interval. This keeps idle - * mounts from pinning a seq and stopping readers of the - * seq indices but doesn't send a message for every sync - * syscall. - */ - ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); + if (!scoutfs_block_writer_has_dirty(sb, &tri->wri)) { + if (sbi->trans_deadline_expired) { + /* + * If we're not writing data then we only advance the + * seq at the sync deadline interval. This keeps idle + * mounts from pinning a seq and stopping readers of the + * seq indices but doesn't send a message for every sync + * syscall. + */ + ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); + if (ret < 0) + s = "clean advance seq"; + } + goto out; } -out: + if (sbi->trans_deadline_expired) + scoutfs_inc_counter(sb, trans_commit_timer); + + scoutfs_inc_counter(sb, trans_commit_written); + /* XXX this all needs serious work for dealing with errors */ - WARN_ON_ONCE(ret); + ret = (s = "data submit", scoutfs_inode_walk_writeback(sb, true)) ?: + (s = "meta write", scoutfs_block_writer_write(sb, &tri->wri)) ?: + (s = "data wait", scoutfs_inode_walk_writeback(sb, false)) ?: + (s = "commit log trees", commit_btrees(sb)) ?: + (s = "advance seq", scoutfs_client_advance_seq(sb, + &sbi->trans_seq))?: + (s = "get log trees", scoutfs_trans_get_log_trees(sb)); +out: + if (ret < 0) + scoutfs_err(sb, "critical transaction commit failure: %s, %d", + s, ret); spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; From 4b9c02ba3273407c3e894fe97ecf3de583ab75b6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 15 Jul 2020 17:00:36 -0700 Subject: [PATCH 855/920] scoutfs: add committed_seq to statfs_more Add the committed_seq to statfs_more which gives the greatest seq which has been committed. This lets callers disocover that a seq for a change they made has been committed. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 5 +++++ kmod/src/ioctl.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index fc35dd34..2bff5c48 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -846,6 +846,7 @@ static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_ioctl_statfs_more sfm; + int ret; if (get_user(sfm.valid_bytes, (__u64 __user *)arg)) return -EFAULT; @@ -855,6 +856,10 @@ static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) sfm.fsid = le64_to_cpu(super->hdr.fsid); sfm.rid = sbi->rid; + ret = scoutfs_client_get_last_seq(sb, &sfm.committed_seq); + if (ret) + return ret; + if (copy_to_user((void __user *)arg, &sfm, sfm.valid_bytes)) return -EFAULT; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 2f861a4d..1ef0aa36 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -358,12 +358,17 @@ struct scoutfs_ioctl_search_xattrs { * field is set if all of its bytes are within the valid_bytes that the * kernel set on return. * + * @committed_seq: All seqs up to and including this seq have been + * committed. Can be compared with meta_seq and data_seq from inodes in + * stat_more to discover if changes have been committed to disk. + * * New fields are only added to the end of the struct. */ struct scoutfs_ioctl_statfs_more { __u64 valid_bytes; __u64 fsid; __u64 rid; + __u64 committed_seq; } __packed; #define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ From f4db553c28b6b7acc9bb8a305913a0f910d74e60 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 16 Jul 2020 11:07:14 -0700 Subject: [PATCH 856/920] scoutfs: fix error unwinding in server advance_seq While checking for lost server commit holds, I noticed that the advance_seq request path had obviously incorrect unwinding after getting an error. Fix it up so that it always unlocks and applies its commit. Signed-off-by: Zach Brown --- kmod/src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index e90e87ad..751ea274 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -747,7 +747,7 @@ static int server_advance_seq(struct super_block *sb, ret = scoutfs_btree_delete(sb, &server->alloc, &server->wri, &super->trans_seqs, &key); if (ret < 0 && ret != -ENOENT) - goto out; + goto unlock; } next_seq = super->next_trans_seq; @@ -759,10 +759,11 @@ static int server_advance_seq(struct super_block *sb, init_trans_seq_key(&key, le64_to_cpu(next_seq), rid); ret = scoutfs_btree_insert(sb, &server->alloc, &server->wri, &super->trans_seqs, &key, NULL, 0); -out: +unlock: up_write(&server->seq_rwsem); ret = scoutfs_server_apply_commit(sb, ret); +out: return scoutfs_net_response(sb, conn, cmd, id, ret, &next_seq, sizeof(next_seq)); } From 55dde87bb17cf62f5ef2c7a943ded0960a504727 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 23 Jul 2020 11:16:08 -0700 Subject: [PATCH 857/920] scoutfs: fix lock invalidation work deadlock The client lock network message processing callbacks were built to simply perform the processing work for the message in the networking work context that it was called in. This particularly makes sense for invalidation because it has to interact with other components that require blocking contexts (syncing commits, invalidating inodes, truncating pages, etc). The problem is that these messages are per-lock. With the right workloads we can use all the capacity for executing work just in lock invalidation work. There is no more work execution available for other network processing. Critically, the blocked invalidation work is waiting for the commit thread to get its network responses before invalidation can make forward progress. I was easily reproducing deadlocks by leaving behind a lot of locks and then triggering a flood of invalidation requests on behalf of shrinking due to memory pressure. The fix is to put locks on lists and have a small fixed number of work contexts process all the locks pending for each message type. The network callbacks don't block, they just put the lock on the list and queue the work that will walk the lists. Invalidation now blocks one work context, not the number of incoming requests. There were some wait conditions in work that used to use the lock workq. Other paths that change those conditions now have to know to queue the work specifically, not just wake tasks which included blocked work executors. The other subtle impact of the change is that we can no longer rely on networking to shutdown message processing work that was happening in its callbacks. We have to specifically stop our work queues in _shutdown. Signed-off-by: Zach Brown --- kmod/src/counters.h | 8 +- kmod/src/lock.c | 424 +++++++++++++++++++++++++++++--------------- kmod/src/lock.h | 8 +- 3 files changed, 295 insertions(+), 145 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index af1d86a5..01bbdfc3 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -78,23 +78,25 @@ EXPAND_COUNTER(forest_trigger_refresh) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ - EXPAND_COUNTER(lock_grace_elapsed) \ EXPAND_COUNTER(lock_grace_extended) \ EXPAND_COUNTER(lock_grace_set) \ EXPAND_COUNTER(lock_grace_wait) \ EXPAND_COUNTER(lock_grant_request) \ EXPAND_COUNTER(lock_grant_response) \ + EXPAND_COUNTER(lock_grant_work) \ EXPAND_COUNTER(lock_invalidate_commit) \ EXPAND_COUNTER(lock_invalidate_coverage) \ EXPAND_COUNTER(lock_invalidate_inode) \ EXPAND_COUNTER(lock_invalidate_request) \ EXPAND_COUNTER(lock_invalidate_response) \ + EXPAND_COUNTER(lock_invalidate_work) \ EXPAND_COUNTER(lock_lock) \ EXPAND_COUNTER(lock_lock_error) \ EXPAND_COUNTER(lock_nonblock_eagain) \ EXPAND_COUNTER(lock_recover_request) \ - EXPAND_COUNTER(lock_shrink_queued) \ - EXPAND_COUNTER(lock_shrink_request_aborted) \ + EXPAND_COUNTER(lock_shrink_attempted) \ + EXPAND_COUNTER(lock_shrink_aborted) \ + EXPAND_COUNTER(lock_shrink_work) \ EXPAND_COUNTER(lock_unlock) \ EXPAND_COUNTER(lock_wait) \ EXPAND_COUNTER(net_dropped_response) \ diff --git a/kmod/src/lock.c b/kmod/src/lock.c index e86d5b22..806debe4 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -80,6 +80,12 @@ struct lock_info { struct list_head lru_list; unsigned long long lru_nr; struct workqueue_struct *workq; + struct work_struct grant_work; + struct list_head grant_list; + struct delayed_work inv_dwork; + struct list_head inv_list; + struct work_struct shrink_work; + struct list_head shrink_list; atomic64_t next_refresh_gen; struct dentry *tseq_dentry; struct scoutfs_tseq_tree tseq_tree; @@ -88,8 +94,6 @@ struct lock_info { #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info -static void scoutfs_lock_shrink_worker(struct work_struct *work); - static bool lock_mode_invalid(int mode) { return (unsigned)mode >= SCOUTFS_LOCK_INVALID; @@ -220,6 +224,9 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!RB_EMPTY_NODE(&lock->node)); BUG_ON(!RB_EMPTY_NODE(&lock->range_node)); BUG_ON(!list_empty(&lock->lru_head)); + BUG_ON(!list_empty(&lock->grant_head)); + BUG_ON(!list_empty(&lock->inv_head)); + BUG_ON(!list_empty(&lock->shrink_head)); BUG_ON(!list_empty(&lock->cov_list)); scoutfs_forest_clear_lock(sb, lock); @@ -245,7 +252,9 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, RB_CLEAR_NODE(&lock->node); RB_CLEAR_NODE(&lock->range_node); INIT_LIST_HEAD(&lock->lru_head); - + INIT_LIST_HEAD(&lock->grant_head); + INIT_LIST_HEAD(&lock->inv_head); + INIT_LIST_HEAD(&lock->shrink_head); spin_lock_init(&lock->cov_list_lock); INIT_LIST_HEAD(&lock->cov_list); @@ -253,7 +262,6 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, lock->end = *end; lock->sb = sb; init_waitqueue_head(&lock->waitq); - INIT_WORK(&lock->shrink_work, scoutfs_lock_shrink_worker); lock->mode = SCOUTFS_LOCK_NULL; trace_scoutfs_lock_alloc(sb, lock); @@ -540,11 +548,36 @@ static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) lock->grace_deadline = ktime_add(now, GRACE_PERIOD_KT); } +static void queue_grant_work(struct lock_info *linfo) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&linfo->grant_list) && !linfo->shutdown) + queue_work(linfo->workq, &linfo->grant_work); +} + /* - * The client is receiving a lock response message from the server. - * This can be reordered with incoming invlidation requests from the - * server so we have to be careful to only set the new mode once the old - * mode matches. + * We immediately queue work on the assumption that the caller might + * have made a change (set a lock mode) which can let one of the + * invalidating locks make forward progress, even if other locks are + * waiting for their grace period to elapse. It's a trade-off between + * invalidation latency and burning cpu repeatedly finding that locks + * are still in their grace period. + */ +static void queue_inv_work(struct lock_info *linfo) +{ + assert_spin_locked(&linfo->lock); + + if (!list_empty(&linfo->inv_list) && !linfo->shutdown) + mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0); +} + +/* + * Each lock has received a grant response message from the server. + * + * Grant responses can be reordered with incoming invalidation requests + * from the server so we have to be careful to only set the new mode + * once the old mode matches. * * We extend the grace period as we grant the lock if there is a waiting * locker who can use the lock. This stops invalidation from pulling @@ -555,6 +588,58 @@ static void extend_grace(struct super_block *sb, struct scoutfs_lock *lock) * against the invalidation. In that case they'd extend the grace * period anyway as they unlock. */ +static void lock_grant_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, + grant_work); + struct super_block *sb = linfo->sb; + struct scoutfs_net_lock_grant_response *gr; + struct scoutfs_net_lock *nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + + scoutfs_inc_counter(sb, lock_grant_work); + + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &linfo->grant_list, grant_head) { + gr = &lock->grant_resp; + nl = &lock->grant_resp.nl; + + /* wait for reordered invalidation to finish */ + if (lock->mode != nl->old_mode) + continue; + + if (!lock_mode_can_read(nl->old_mode) && + lock_mode_can_read(nl->new_mode)) { + lock->refresh_gen = + atomic64_inc_return(&linfo->next_refresh_gen); + } + + lock->request_pending = 0; + lock->mode = nl->new_mode; + lock->write_version = le64_to_cpu(nl->write_version); + lock->roots = gr->roots; + + if (lock_count_match_exists(nl->new_mode, lock->waiters)) + extend_grace(sb, lock); + + trace_scoutfs_lock_granted(sb, lock); + list_del_init(&lock->grant_head); + wake_up(&lock->waitq); + put_lock(linfo, lock); + } + + /* invalidations might be waiting for our reordered grant */ + queue_inv_work(linfo); + spin_unlock(&linfo->lock); +} + +/* + * The client is receiving a grant response message from the server. We + * find the lock, record the response, and add it to the list for grant + * work to process. + */ int scoutfs_lock_grant_response(struct super_block *sb, struct scoutfs_net_lock_grant_response *gr) { @@ -569,35 +654,12 @@ int scoutfs_lock_grant_response(struct super_block *sb, /* lock must already be busy with request_pending */ lock = lock_lookup(sb, &nl->key, NULL); BUG_ON(!lock); + trace_scoutfs_lock_grant_response(sb, lock); BUG_ON(!lock->request_pending); - trace_scoutfs_lock_grant_response(sb, lock); - - /* resolve unlikely work reordering with invalidation request */ - while (lock->mode != nl->old_mode) { - spin_unlock(&linfo->lock); - /* implicit read barrier from waitq locks */ - wait_event(lock->waitq, lock->mode == nl->old_mode); - spin_lock(&linfo->lock); - } - - if (!lock_mode_can_read(nl->old_mode) && - lock_mode_can_read(nl->new_mode)) { - lock->refresh_gen = - atomic64_inc_return(&linfo->next_refresh_gen); - } - - lock->request_pending = 0; - lock->mode = nl->new_mode; - lock->write_version = le64_to_cpu(nl->write_version); - lock->roots = gr->roots; - - if (lock_count_match_exists(nl->new_mode, lock->waiters)) - extend_grace(sb, lock); - - trace_scoutfs_lock_granted(sb, lock); - wake_up(&lock->waitq); - put_lock(linfo, lock); + lock->grant_resp = *gr; + list_add_tail(&lock->grant_head, &linfo->grant_list); + queue_grant_work(linfo); spin_unlock(&linfo->lock); @@ -605,34 +667,9 @@ int scoutfs_lock_grant_response(struct super_block *sb, } /* - * Invalidation waits until the old mode indicates that we've resolved - * unlikely races with reordered grant responses from the server and - * until the new mode satisfies active users. - * - * Once it's safe to proceed we set the lock mode here under the lock to - * prevent additional users of the old mode while we're invalidating. - */ -static bool lock_invalidate_safe(struct lock_info *linfo, - struct scoutfs_lock *lock, - int old_mode, int new_mode) -{ - bool safe; - - spin_lock(&linfo->lock); - safe = (lock->mode == old_mode) && - lock_counts_match(new_mode, lock->users); - if (safe) - lock->mode = new_mode; - spin_unlock(&linfo->lock); - - return safe; -} - -/* - * The client is receiving a lock invalidation request from the server + * Each lock has received a lock invalidation request from the server * which specifies a new mode for the lock. The server will only send - * one invalidation request at a time. This is executing in a blocking - * net receive work context. + * one invalidation request at a time for each lock. * * This is an unsolicited request from the server so it can arrive at * any time after we make the server aware of the lock by initially @@ -649,70 +686,135 @@ static bool lock_invalidate_safe(struct lock_info *linfo, * invalidate once the lock mode matches what the server told us to * invalidate. * - * We delay invalidation processing until a grace period has elapsed since - * the last unlock. The intent is to let users do a reasonable batch of - * work before dropping the lock. Continuous unlocking can continuously - * extend the deadline. + * We delay invalidation processing until a grace period has elapsed + * since the last unlock. The intent is to let users do a reasonable + * batch of work before dropping the lock. Continuous unlocking can + * continuously extend the deadline. + * + * Before we start invalidating the lock we set the lock to the new + * mode, preventing further incompatible users of the old mode from + * using the lock while we're invalidating. + * + * This does a lot of serialized inode invalidation in one context and + * performs a lot of repeated calls to sync. It would be nice to get + * some concurrent inode invalidation and to more carefully only call + * sync when needed. + */ +static void lock_invalidate_worker(struct work_struct *work) +{ + struct lock_info *linfo = container_of(work, struct lock_info, + inv_dwork.work); + struct super_block *sb = linfo->sb; + struct scoutfs_net_lock *nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + unsigned long delay = MAX_JIFFY_OFFSET; + ktime_t now = ktime_get(); + ktime_t deadline; + LIST_HEAD(ready); + u64 net_id; + int ret; + + scoutfs_inc_counter(sb, lock_invalidate_work); + + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &linfo->inv_list, inv_head) { + nl = &lock->inv_nl; + + /* skip if grace hasn't elapsed, record earliest */ + deadline = lock->grace_deadline; + if (ktime_before(now, deadline)) { + delay = min(delay, + nsecs_to_jiffies(ktime_to_ns( + ktime_sub(deadline, now)))); + scoutfs_inc_counter(linfo->sb, lock_grace_wait); + continue; + } + + /* wait for reordered grant to finish */ + if (lock->mode != nl->old_mode) + continue; + + /* wait until incompatible holders unlock */ + if (!lock_counts_match(nl->new_mode, lock->users)) + continue; + + /* set the new mode, no incompatible users during inval */ + lock->mode = nl->new_mode; + + /* move everyone that's ready to our private list */ + list_move_tail(&lock->inv_head, &ready); + } + + spin_unlock(&linfo->lock); + + if (list_empty(&ready)) + goto out; + + /* invalidate once the lock is read */ + list_for_each_entry(lock, &ready, inv_head) { + nl = &lock->inv_nl; + net_id = lock->inv_net_id; + + ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); + BUG_ON(ret); + + /* respond with the key and modes from the request */ + ret = scoutfs_client_lock_response(sb, net_id, nl); + BUG_ON(ret); + + scoutfs_inc_counter(sb, lock_invalidate_response); + } + + /* and finish all the invalidated locks */ + spin_lock(&linfo->lock); + + list_for_each_entry_safe(lock, tmp, &ready, inv_head) { + list_del_init(&lock->inv_head); + + lock->invalidate_pending = 0; + trace_scoutfs_lock_invalidated(sb, lock); + wake_up(&lock->waitq); + put_lock(linfo, lock); + } + + /* grant might have been waiting for invalidate request */ + queue_grant_work(linfo); + spin_unlock(&linfo->lock); + +out: + /* queue delayed work if invalidations waiting on grace deadline */ + if (delay != MAX_JIFFY_OFFSET) + queue_delayed_work(linfo->workq, &linfo->inv_dwork, delay); +} + +/* + * Record an incoming invalidate request from the server and add its lock + * to the list for processing. + * + * This is trusting the server and will crash if it's sent bad requests :/ */ int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, struct scoutfs_net_lock *nl) { DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; - ktime_t deadline; - bool grace_waited = false; - int ret; scoutfs_inc_counter(sb, lock_invalidate_request); spin_lock(&linfo->lock); lock = get_lock(sb, &nl->key); - if (lock) { - BUG_ON(lock->invalidate_pending); /* XXX trusting server :/ */ - lock->invalidate_pending = 1; - deadline = lock->grace_deadline; - trace_scoutfs_lock_invalidate_request(sb, lock); - } - spin_unlock(&linfo->lock); - BUG_ON(!lock); - - /* wait for a grace period after the most recent unlock */ - while (ktime_before(ktime_get(), deadline)) { - grace_waited = true; - scoutfs_inc_counter(linfo->sb, lock_grace_wait); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout(&deadline, HRTIMER_MODE_ABS); - - spin_lock(&linfo->lock); - deadline = lock->grace_deadline; - spin_unlock(&linfo->lock); + if (lock) { + BUG_ON(lock->invalidate_pending); + lock->invalidate_pending = 1; + lock->inv_nl = *nl; + lock->inv_net_id = net_id; + list_add_tail(&lock->inv_head, &linfo->inv_list); + trace_scoutfs_lock_invalidate_request(sb, lock); + queue_inv_work(linfo); } - - if (grace_waited) - scoutfs_inc_counter(linfo->sb, lock_grace_elapsed); - - /* sets the lock mode to prevent use of old mode during invalidate */ - wait_event(lock->waitq, lock_invalidate_safe(linfo, lock, nl->old_mode, - nl->new_mode)); - - ret = lock_invalidate(sb, lock, nl->old_mode, nl->new_mode); - BUG_ON(ret); - - /* respond with the key and modes from the request */ - ret = scoutfs_client_lock_response(sb, net_id, nl); - BUG_ON(ret); - - scoutfs_inc_counter(sb, lock_invalidate_response); - - spin_lock(&linfo->lock); - - lock->invalidate_pending = 0; - - trace_scoutfs_lock_invalidated(sb, lock); - wake_up(&lock->waitq); - put_lock(linfo, lock); - spin_unlock(&linfo->lock); return 0; @@ -1174,6 +1276,7 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) trace_scoutfs_lock_unlock(sb, lock); wake_up(&lock->waitq); + queue_inv_work(linfo); put_lock(linfo, lock); spin_unlock(&linfo->lock); @@ -1258,38 +1361,50 @@ bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, } /* - * The shrink callback got the lock, marked it request_pending, and - * handed it off to us. We kick off a null request and the lock will - * be freed by the response once all users drain. If this races with + * The shrink callback got the lock, marked it request_pending, and put + * it on the shrink list. We send a null request and the lock will be + * freed by the response once all users drain. If this races with * invalidation then the server will only send the grant response once * the invalidation is finished. */ -static void scoutfs_lock_shrink_worker(struct work_struct *work) +static void lock_shrink_worker(struct work_struct *work) { - struct scoutfs_lock *lock = container_of(work, struct scoutfs_lock, - shrink_work); - struct super_block *sb = lock->sb; - DECLARE_LOCK_INFO(sb, linfo); + struct lock_info *linfo = container_of(work, struct lock_info, + shrink_work); + struct super_block *sb = linfo->sb; struct scoutfs_net_lock nl; + struct scoutfs_lock *lock; + struct scoutfs_lock *tmp; + LIST_HEAD(list); int ret; - /* unlocked lock access, but should be stable since we queued */ - nl.key = lock->start; - nl.old_mode = lock->mode; - nl.new_mode = SCOUTFS_LOCK_NULL; + scoutfs_inc_counter(sb, lock_shrink_work); - ret = scoutfs_client_lock_request(sb, &nl); - if (ret) { - /* oh well, not freeing */ - scoutfs_inc_counter(sb, lock_shrink_request_aborted); + spin_lock(&linfo->lock); + list_splice_init(&linfo->shrink_list, &list); + spin_unlock(&linfo->lock); - spin_lock(&linfo->lock); + list_for_each_entry_safe(lock, tmp, &list, shrink_head) { + list_del_init(&lock->shrink_head); - lock->request_pending = 0; - wake_up(&lock->waitq); - put_lock(linfo, lock); + /* unlocked lock access, but should be stable since we queued */ + nl.key = lock->start; + nl.old_mode = lock->mode; + nl.new_mode = SCOUTFS_LOCK_NULL; - spin_unlock(&linfo->lock); + ret = scoutfs_client_lock_request(sb, &nl); + if (ret) { + /* oh well, not freeing */ + scoutfs_inc_counter(sb, lock_shrink_aborted); + + spin_lock(&linfo->lock); + + lock->request_pending = 0; + wake_up(&lock->waitq); + put_lock(linfo, lock); + + spin_unlock(&linfo->lock); + } } } @@ -1314,6 +1429,7 @@ static int scoutfs_lock_shrink(struct shrinker *shrink, struct scoutfs_lock *lock; struct scoutfs_lock *tmp; unsigned long nr; + bool added = false; int ret; nr = sc->nr_to_scan; @@ -1327,15 +1443,17 @@ restart: BUG_ON(!lock_idle(lock)); BUG_ON(lock->mode == SCOUTFS_LOCK_NULL); + BUG_ON(!list_empty(&lock->shrink_head)); - if (nr-- == 0) + if (linfo->shutdown || nr-- == 0) break; __lock_del_lru(linfo, lock); lock->request_pending = 1; - queue_work(linfo->workq, &lock->shrink_work); + list_add_tail(&lock->shrink_head, &linfo->shrink_list); + added = true; - scoutfs_inc_counter(sb, lock_shrink_queued); + scoutfs_inc_counter(sb, lock_shrink_attempted); trace_scoutfs_lock_shrink(sb, lock); /* could have bazillions of idle locks */ @@ -1345,6 +1463,9 @@ restart: spin_unlock(&linfo->lock); + if (added) + queue_work(linfo->workq, &linfo->shrink_work); + out: ret = min_t(unsigned long, linfo->lru_nr, INT_MAX); trace_scoutfs_lock_shrink_exit(sb, sc->nr_to_scan, ret); @@ -1379,10 +1500,15 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent) } /* - * We're going to be destroying the locks soon. We shouldn't have any - * normal task holders that would have prevented unmount. We can have - * internal threads blocked in locks. We force all currently blocked - * and future lock calls to return -ESHUTDOWN. + * The caller is going to be calling _destroy soon and, critically, is + * about to shutdown networking before calling us so that we don't get + * any callbacks while we're destroying. We have to ensure that we + * won't call networking after this returns. + * + * Internal fs threads can be using locking, and locking can have async + * work pending. We use ->shutdown to force callers to return + * -ESHUTDOWN and to prevent the future queueing of work that could call + * networking. Locks whose work is stopped will be torn down by _destroy. */ void scoutfs_lock_shutdown(struct super_block *sb) { @@ -1404,6 +1530,10 @@ void scoutfs_lock_shutdown(struct super_block *sb) } spin_unlock(&linfo->lock); + + flush_work(&linfo->grant_work); + flush_delayed_work(&linfo->inv_dwork); + flush_work(&linfo->shrink_work); } /* @@ -1476,6 +1606,12 @@ void scoutfs_lock_destroy(struct super_block *sb) lock->request_pending = 0; if (!list_empty(&lock->lru_head)) __lock_del_lru(linfo, lock); + if (!list_empty(&lock->grant_head)) + list_del_init(&lock->grant_head); + if (!list_empty(&lock->inv_head)) + list_del_init(&lock->inv_head); + if (!list_empty(&lock->shrink_head)) + list_del_init(&lock->shrink_head); lock_remove(linfo, lock); lock_free(linfo, lock); } @@ -1503,6 +1639,12 @@ int scoutfs_lock_setup(struct super_block *sb) linfo->shrinker.seeks = DEFAULT_SEEKS; register_shrinker(&linfo->shrinker); INIT_LIST_HEAD(&linfo->lru_list); + INIT_WORK(&linfo->grant_work, lock_grant_worker); + INIT_LIST_HEAD(&linfo->grant_list); + INIT_DELAYED_WORK(&linfo->inv_dwork, lock_invalidate_worker); + INIT_LIST_HEAD(&linfo->inv_list); + INIT_WORK(&linfo->shrink_work, lock_shrink_worker); + INIT_LIST_HEAD(&linfo->shrink_list); atomic64_set(&linfo->next_refresh_gen, 0); scoutfs_tseq_tree_init(&linfo->tseq_tree, lock_tseq_show); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index f77f4ed0..f63a4838 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -25,11 +25,17 @@ struct scoutfs_lock { struct scoutfs_net_roots roots; struct list_head lru_head; wait_queue_head_t waitq; - struct work_struct shrink_work; ktime_t grace_deadline; unsigned long request_pending:1, invalidate_pending:1; + struct list_head grant_head; + struct scoutfs_net_lock_grant_response grant_resp; + struct list_head inv_head; + struct scoutfs_net_lock inv_nl; + u64 inv_net_id; + struct list_head shrink_head; + spinlock_t cov_list_lock; struct list_head cov_list; From ca6b7f1e6defc4bdfda2c3fdcab3c15bd062ea0b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 23 Jul 2020 16:14:17 -0700 Subject: [PATCH 858/920] scoutfs: lock invalidate only syncs dirty Lock invalidation has to make sure that changes are visible to future readers. It was syncing if the current transaction is dirty. This was never optimal, but it wasn't catastrophic when concurrent invalidation work could all block on one sync in progress. With the move to a single invalidation worker serially invalidating locks it became unacceptable. Invalidation happening in the presence of writers would constantly sync the current transaction while very old unused write locks were invalidated. Their changes had long since been committed in previous transactions. We add a lock field to remember the transaction sequence which could have been dirtied under the lock. If that transaction has already been comitted by the time we invalidate the lock it doesn't have to sync. Signed-off-by: Zach Brown --- kmod/src/counters.h | 2 +- kmod/src/lock.c | 12 ++++++------ kmod/src/lock.h | 1 + kmod/src/trans.c | 24 +++++++++++++++++++++--- kmod/src/trans.h | 1 + 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 01bbdfc3..d3e1b175 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -84,11 +84,11 @@ EXPAND_COUNTER(lock_grant_request) \ EXPAND_COUNTER(lock_grant_response) \ EXPAND_COUNTER(lock_grant_work) \ - EXPAND_COUNTER(lock_invalidate_commit) \ EXPAND_COUNTER(lock_invalidate_coverage) \ EXPAND_COUNTER(lock_invalidate_inode) \ EXPAND_COUNTER(lock_invalidate_request) \ EXPAND_COUNTER(lock_invalidate_response) \ + EXPAND_COUNTER(lock_invalidate_sync) \ EXPAND_COUNTER(lock_invalidate_work) \ EXPAND_COUNTER(lock_lock) \ EXPAND_COUNTER(lock_lock_error) \ diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 806debe4..14b8122c 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -160,15 +160,13 @@ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, BUG_ON(!(prev == SCOUTFS_LOCK_WRITE && mode == SCOUTFS_LOCK_READ) && mode != SCOUTFS_LOCK_NULL); - /* any transition from a mode allowed to dirty items has to write */ - if (lock_mode_can_write(prev) && scoutfs_trans_has_dirty(sb)) { + /* sync when a write lock could have dirtied the current transaction */ + if (lock_mode_can_write(prev) && + (lock->dirty_trans_seq == scoutfs_trans_sample_seq(sb))) { + scoutfs_inc_counter(sb, lock_invalidate_sync); ret = scoutfs_trans_sync(sb, 1); if (ret < 0) return ret; - if (ret > 0) { - scoutfs_add_counter(sb, lock_invalidate_commit, ret); - ret = 0; - } } /* have to invalidate if we're not in the only usable case */ @@ -1273,6 +1271,8 @@ void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) lock_dec_count(lock->users, mode); extend_grace(sb, lock); + if (lock_mode_can_write(mode)) + lock->dirty_trans_seq = scoutfs_trans_sample_seq(sb); trace_scoutfs_lock_unlock(sb, lock); wake_up(&lock->waitq); diff --git a/kmod/src/lock.h b/kmod/src/lock.h index f63a4838..3b16db03 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -22,6 +22,7 @@ struct scoutfs_lock { struct rb_node range_node; u64 refresh_gen; u64 write_version; + u64 dirty_trans_seq; struct scoutfs_net_roots roots; struct list_head lru_head; wait_queue_head_t waitq; diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 6001724e..bd06503a 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -158,6 +158,7 @@ void scoutfs_trans_write_func(struct work_struct *work) trans_write_work.work); struct super_block *sb = sbi->sb; DECLARE_TRANS_INFO(sb, tri); + u64 trans_seq = sbi->trans_seq; char *s = NULL; int ret = 0; @@ -177,7 +178,7 @@ void scoutfs_trans_write_func(struct work_struct *work) * seq indices but doesn't send a message for every sync * syscall. */ - ret = scoutfs_client_advance_seq(sb, &sbi->trans_seq); + ret = scoutfs_client_advance_seq(sb, &trans_seq); if (ret < 0) s = "clean advance seq"; } @@ -194,8 +195,7 @@ void scoutfs_trans_write_func(struct work_struct *work) (s = "meta write", scoutfs_block_writer_write(sb, &tri->wri)) ?: (s = "data wait", scoutfs_inode_walk_writeback(sb, false)) ?: (s = "commit log trees", commit_btrees(sb)) ?: - (s = "advance seq", scoutfs_client_advance_seq(sb, - &sbi->trans_seq))?: + (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: (s = "get log trees", scoutfs_trans_get_log_trees(sb)); out: if (ret < 0) @@ -205,6 +205,7 @@ out: spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; sbi->trans_write_ret = ret; + sbi->trans_seq = trans_seq; spin_unlock(&sbi->trans_write_lock); wake_up(&sbi->trans_write_wq); @@ -522,6 +523,23 @@ void scoutfs_release_trans(struct super_block *sb) wake_up(&sbi->trans_hold_wq); } +/* + * Return the current transaction sequence. Whether this is racing with + * the transaction write thread is entirely dependent on the caller's + * context. + */ +u64 scoutfs_trans_sample_seq(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + u64 ret; + + spin_lock(&sbi->trans_write_lock); + ret = sbi->trans_seq; + spin_unlock(&sbi->trans_write_lock); + + return ret; +} + int scoutfs_setup_trans(struct super_block *sb) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/trans.h b/kmod/src/trans.h index 014a35e8..f1b50f8f 100644 --- a/kmod/src/trans.h +++ b/kmod/src/trans.h @@ -18,6 +18,7 @@ int scoutfs_hold_trans(struct super_block *sb, const struct scoutfs_item_count cnt); bool scoutfs_trans_held(void); void scoutfs_release_trans(struct super_block *sb); +u64 scoutfs_trans_sample_seq(struct super_block *sb); void scoutfs_trans_track_item(struct super_block *sb, signed items, signed vals); From 5c6b263d97715c1723700bb2f2a4c2c98d366c58 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 24 Jul 2020 14:38:19 -0700 Subject: [PATCH 859/920] scoutfs: trace radix bit ops before assertions Trace operations before they can trigger assertions so we can see the violating operation in the traces. Signed-off-by: Zach Brown --- kmod/src/radix.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 5d233c2a..c508054e 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -465,6 +465,7 @@ static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, struct scoutfs_radix_block *rdx = bl->data; int lg_nbits; + trace_scoutfs_radix_set_bits(sb, bl->blkno, ind, nbits); bug_on_bad_bits(ind, nbits); /* must never double-free bits */ @@ -473,7 +474,6 @@ static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, lg_nbits = count_lg_bits(rdx->bits, ind, nbits); fixup_parent_refs(sb, bl, nbits, lg_nbits); - trace_scoutfs_radix_set_bits(sb, bl->blkno, ind, nbits); } static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, @@ -482,6 +482,7 @@ static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, struct scoutfs_radix_block *rdx = bl->data; int lg_nbits; + trace_scoutfs_radix_clear_bits(sb, bl->blkno, ind, nbits); bug_on_bad_bits(ind, nbits); /* must never alloc in-use bits */ @@ -490,7 +491,6 @@ static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, bitmap_clear_le(rdx->bits, ind, nbits); fixup_parent_refs(sb, bl, -nbits, -lg_nbits); - trace_scoutfs_radix_clear_bits(sb, bl->blkno, ind, nbits); } /* From ba879b977ab7ad7f924dd48ba08916c414418008 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 25 Jul 2020 11:36:49 -0700 Subject: [PATCH 860/920] scoutfs: expand radix merge tracing Add a trace event for entering _radix_merge() and rename the current per-merge trace event. Signed-off-by: Zach Brown --- kmod/src/radix.c | 15 +++++++++++---- kmod/src/scoutfs_trace.h | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index c508054e..a973ab50 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1341,6 +1341,12 @@ int scoutfs_radix_merge(struct super_block *sb, int ind; int ret; + trace_scoutfs_radix_merge(sb, le64_to_cpu(dst->ref.blkno), + le64_to_cpu(dst->ref.sm_total), + le64_to_cpu(src->ref.blkno), + le64_to_cpu(src->ref.sm_total), + le64_to_cpu(inp->ref.blkno), + le64_to_cpu(inp->ref.sm_total), count); scoutfs_inc_counter(sb, radix_merge); mutex_lock(&alloc->mutex); @@ -1426,10 +1432,11 @@ wrapped: fixup_parent_refs(sb, src_bl, -merged, -src_lg_delta); fixup_parent_refs(sb, dst_bl, merged, dst_lg_delta); - trace_scoutfs_radix_merge(sb, inp, inp_bl->blkno, src, - src_bl->blkno, dst, dst_bl->blkno, - count, bit, ind, merged, - src_lg_delta, dst_lg_delta); + trace_scoutfs_radix_merged_blocks(sb, inp, inp_bl->blkno, src, + src_bl->blkno, dst, + dst_bl->blkno, count, bit, + ind, merged, src_lg_delta, + dst_lg_delta); complete_change(sb, wri, &chg, 0); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index f44d0241..5abe16fc 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2463,6 +2463,38 @@ DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set_bits, ); TRACE_EVENT(scoutfs_radix_merge, + TP_PROTO(struct super_block *sb, u64 dst_blkno, u64 dst_sm_tot, + u64 src_blkno, u64 src_sm_tot, u64 inp_blkno, u64 inp_sm_tot, + u64 count), + TP_ARGS(sb, dst_blkno, dst_sm_tot, src_blkno, src_sm_tot, inp_blkno, + inp_sm_tot, count), + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, dst_blkno) + __field(__u64, dst_sm_tot) + __field(__u64, src_blkno) + __field(__u64, src_sm_tot) + __field(__u64, inp_blkno) + __field(__u64, inp_sm_tot) + __field(__u64, count) + ), + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->dst_blkno = dst_blkno; + __entry->dst_sm_tot = dst_sm_tot; + __entry->src_blkno = src_blkno; + __entry->src_sm_tot = src_sm_tot; + __entry->inp_blkno = inp_blkno; + __entry->inp_sm_tot = inp_sm_tot; + __entry->count = count; + ), + TP_printk(SCSBF" d_blkno %llu d_sm_tot %llu s_blkno %llu s_sm_tot %llu i_blkno %llu i_sm_tot %llu count %llu", + SCSB_TRACE_ARGS, __entry->dst_blkno, __entry->dst_sm_tot, + __entry->src_blkno, __entry->src_sm_tot, __entry->inp_blkno, + __entry->inp_sm_tot, __entry->count) +); + +TRACE_EVENT(scoutfs_radix_merged_blocks, TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *inp, u64 inp_blkno, struct scoutfs_radix_root *src, u64 src_blkno, From 289caeb3536c5f8e08477de0d0615dee64e76a4f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 27 Jul 2020 13:57:14 -0700 Subject: [PATCH 861/920] scoutfs: trace leaf_bit of modified radix bits Signed-off-by: Zach Brown --- kmod/src/radix.c | 23 +++++++++++++++-------- kmod/src/scoutfs_trace.h | 23 ++++++++++++++--------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/kmod/src/radix.c b/kmod/src/radix.c index a973ab50..4f636bf7 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -119,6 +119,8 @@ struct radix_block_private { struct scoutfs_block *old_blkno_bl; int blkno_ind; int old_blkno_ind; + u64 blkno_leaf_bit; + u64 old_blkno_leaf_bit; }; static bool was_dirtied(struct radix_block_private *priv) @@ -460,12 +462,12 @@ static void bug_on_bad_bits(int ind, int nbits) } static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, - int ind, int nbits) + u64 leaf_bit, int ind, int nbits) { struct scoutfs_radix_block *rdx = bl->data; int lg_nbits; - trace_scoutfs_radix_set_bits(sb, bl->blkno, ind, nbits); + trace_scoutfs_radix_set_bits(sb, bl->blkno, leaf_bit, ind, nbits); bug_on_bad_bits(ind, nbits); /* must never double-free bits */ @@ -477,12 +479,12 @@ static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, } static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, - int ind, int nbits) + u64 leaf_bit, int ind, int nbits) { struct scoutfs_radix_block *rdx = bl->data; int lg_nbits; - trace_scoutfs_radix_clear_bits(sb, bl->blkno, ind, nbits); + trace_scoutfs_radix_clear_bits(sb, bl->blkno, leaf_bit, ind, nbits); bug_on_bad_bits(ind, nbits); /* must never alloc in-use bits */ @@ -854,6 +856,7 @@ static int get_leaf(struct super_block *sb, &bl); if (ret < 0) break; + priv->old_blkno_leaf_bit = leaf_bit; priv->old_blkno_ind = old_blkno - leaf_bit; priv->old_blkno_bl = bl; } @@ -864,6 +867,7 @@ static int get_leaf(struct super_block *sb, if (ret < 0) break; + priv->blkno_leaf_bit = leaf_bit; priv->blkno_ind = priv->bl->blkno - leaf_bit; priv->blkno_bl = bl; @@ -989,9 +993,12 @@ static void apply_change_bits(struct super_block *sb, struct radix_change *chg) /* can't try to write to synth blknos */ BUG_ON(is_synth(bl->blkno)); - clear_leaf_bits(sb, priv->blkno_bl, priv->blkno_ind, 1); + clear_leaf_bits(sb, priv->blkno_bl, + priv->blkno_leaf_bit, + priv->blkno_ind, 1); if (priv->old_blkno_bl) { set_leaf_bits(sb, priv->old_blkno_bl, + priv->old_blkno_leaf_bit, priv->old_blkno_ind, 1); } scoutfs_inc_counter(sb, radix_complete_dirty_block); @@ -1152,7 +1159,7 @@ static int radix_free(struct super_block *sb, goto out; ind = bit - leaf_bit; - set_leaf_bits(sb, bl, ind, nbits); + set_leaf_bits(sb, bl, leaf_bit, ind, nbits); out: complete_change(sb, wri, &chg, ret); mutex_unlock(&alloc->mutex); @@ -1190,7 +1197,7 @@ int scoutfs_radix_alloc(struct super_block *sb, goto out; ind = bit - leaf_bit; - clear_leaf_bits(sb, bl, ind, 1); + clear_leaf_bits(sb, bl, leaf_bit, ind, 1); *blkno = bit; ret = 0; out: @@ -1248,7 +1255,7 @@ int scoutfs_radix_alloc_data(struct super_block *sb, goto out; ind = bit - leaf_bit; - clear_leaf_bits(sb, bl, ind, nbits); + clear_leaf_bits(sb, bl, leaf_bit, ind, nbits); *blkno_ret = bit; *count_ret = nbits; store_next_find_bit(sb, false, root, bit + nbits); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5abe16fc..eeafd395 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2435,31 +2435,36 @@ TRACE_EVENT(scoutfs_radix_walk, ); DECLARE_EVENT_CLASS(scoutfs_radix_bitop, - TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), - TP_ARGS(sb, blkno, ind, nbits), + TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, + int nbits), + TP_ARGS(sb, blkno, leaf_bit, ind, nbits), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, blkno) + __field(__u64, leaf_bit) __field(int, ind) __field(int, nbits) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->blkno = blkno; + __entry->leaf_bit = leaf_bit; __entry->ind = ind; __entry->nbits = nbits; ), - TP_printk(SCSBF" blkno %llu ind %d nbits %d", - SCSB_TRACE_ARGS, __entry->blkno, __entry->ind, - __entry->nbits) + TP_printk(SCSBF" blkno %llu leaf_bit %llu ind %d nbits %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->leaf_bit, + __entry->ind, __entry->nbits) ); DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_clear_bits, - TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), - TP_ARGS(sb, blkno, ind, nbits) + TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, + int nbits), + TP_ARGS(sb, blkno, leaf_bit, ind, nbits) ); DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set_bits, - TP_PROTO(struct super_block *sb, u64 blkno, int ind, int nbits), - TP_ARGS(sb, blkno, ind, nbits) + TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, + int nbits), + TP_ARGS(sb, blkno, leaf_bit, ind, nbits) ); TRACE_EVENT(scoutfs_radix_merge, From d1e62a43c932120b0d92ee40498f4c65d132da03 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 28 Jul 2020 16:32:59 -0700 Subject: [PATCH 862/920] scoutfs: fix leaking alloc bits in merge In a merge where the input and source trees are the same, the input block can be an initial pre-cow version of the dirty source block. Dirtying blocks in the change will clear allocations in the dirty source block but they will remain in the pre-cow input block. The merge can then set these blocks in the dst, even though they were also used by allocation, because they're still set in the pre-cow input block. This fix is clumsy, but minimal and specific to this problem. A more thorough fix is being worked on which introduces more staging more allocator trees and should stop calls that are modifying the current active avail or free trees. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/radix.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index d3e1b175..bce86418 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -135,6 +135,7 @@ EXPAND_COUNTER(radix_inconsistent_eio) \ EXPAND_COUNTER(radix_inconsistent_ref) \ EXPAND_COUNTER(radix_merge) \ + EXPAND_COUNTER(radix_merge_bad_clean_input) \ EXPAND_COUNTER(radix_merge_empty) \ EXPAND_COUNTER(radix_undo_ref) \ EXPAND_COUNTER(radix_walk) \ diff --git a/kmod/src/radix.c b/kmod/src/radix.c index 4f636bf7..0e734ff5 100644 --- a/kmod/src/radix.c +++ b/kmod/src/radix.c @@ -1392,6 +1392,21 @@ wrapped: goto out; src_rdx = src_bl->data; + /* + * If we're searching the avail allocator tree then we + * must be sure that we copy leaves after change + * allocations have been applied. If we had a read-only + * copy of the allocator leaf before it was cowed we + * could merge bits that were used for dirty block + * allocations by the change. By not resetting the + * change the repeated lookup will find the current + * dirty leaf block. + */ + if (src == inp && inp_bl != src_bl) { + scoutfs_inc_counter(sb, radix_merge_bad_clean_input); + goto wrapped; + } + ret = get_leaf(sb, alloc, wri, &chg, dst, GLF_DIRTY, bit, &leaf_bit, &dst_bl); if (ret < 0) From d440056e6fd3ad7ee0a28653cd55bba1b97133ec Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 30 Jul 2020 11:30:30 -0700 Subject: [PATCH 863/920] scoutfs: remove unused xattr index code Remove the last remnants of the indexed xattrs which used fs items. This makes the significant change of renumbering the key zones so I wanted it in its own commit. Signed-off-by: Zach Brown --- kmod/src/count.h | 8 ++------ kmod/src/format.h | 15 +++------------ kmod/src/lock.c | 18 ------------------ kmod/src/lock.h | 2 -- kmod/src/xattr.c | 14 +------------- kmod/src/xattr.h | 3 --- 6 files changed, 6 insertions(+), 54 deletions(-) diff --git a/kmod/src/count.h b/kmod/src/count.h index 176321d0..5756f407 100644 --- a/kmod/src/count.h +++ b/kmod/src/count.h @@ -205,14 +205,12 @@ static inline const struct scoutfs_item_count SIC_RENAME(unsigned old_len, * item with the header and name. Any previously existing items are * deleted which dirties their key but removes their value. The two * sets of items are indexed by different ids so their items don't - * overlap. If the xattr name is indexed then we modify one xattr index - * item. + * overlap. */ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, bool creating, unsigned name_len, - unsigned size, - bool indexed) + unsigned size) { struct scoutfs_item_count cnt = {0,}; unsigned int new_parts; @@ -221,8 +219,6 @@ static inline const struct scoutfs_item_count SIC_XATTR_SET(unsigned old_parts, if (old_parts) cnt.items += old_parts; - if (indexed) - cnt.items++; if (creating) { new_parts = SCOUTFS_XATTR_NR_PARTS(name_len, size); diff --git a/kmod/src/format.h b/kmod/src/format.h index 1e004fbe..cd33a7c6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -121,11 +121,6 @@ struct scoutfs_key { #define skii_major _sk_second #define skii_ino _sk_third -/* xattr index */ -#define skxi_hash _sk_first -#define skxi_ino _sk_second -#define skxi_id _sk_third - /* node orphan inode */ #define sko_rid _sk_first #define sko_ino _sk_second @@ -430,10 +425,9 @@ struct scoutfs_bloom_block { * Keys are first sorted by major key zones. */ #define SCOUTFS_INODE_INDEX_ZONE 1 -#define SCOUTFS_XATTR_INDEX_ZONE 2 -#define SCOUTFS_RID_ZONE 3 -#define SCOUTFS_FS_ZONE 4 -#define SCOUTFS_LOCK_ZONE 5 +#define SCOUTFS_RID_ZONE 2 +#define SCOUTFS_FS_ZONE 3 +#define SCOUTFS_LOCK_ZONE 4 /* Items only stored in server btrees */ #define SCOUTFS_LOG_TREES_ZONE 6 #define SCOUTFS_LOCK_CLIENTS_ZONE 7 @@ -446,9 +440,6 @@ struct scoutfs_bloom_block { #define SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE 2 #define SCOUTFS_INODE_INDEX_NR 3 /* don't forget to update */ -/* xattr index zone */ -#define SCOUTFS_XATTR_INDEX_NAME_TYPE 1 - /* rid zone (also used in server alloc btree) */ #define SCOUTFS_ORPHAN_TYPE 1 diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 14b8122c..fba8fef5 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -1208,24 +1208,6 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, return lock_key_range(sb, mode, 0, &start, &end, ret_lock); } -/* - * Today we lock a hash value entirely. If we went to finer grained ino - * locking as well we'd need to check the manifest to find the next - * possible ino to lock so that we didn't try to iterate over all of - * them. - */ -int scoutfs_lock_xattr_index(struct super_block *sb, int mode, int flags, - u64 hash, struct scoutfs_lock **ret_lock) -{ - struct scoutfs_key start; - struct scoutfs_key end; - - scoutfs_xattr_index_key(&start, hash, 0, 0); - scoutfs_xattr_index_key(&end, hash, U64_MAX, U64_MAX); - - return lock_key_range(sb, mode, flags, &start, &end, ret_lock); -} - /* * The rid lock protects a mount's private persistent items in the rid * zone. It's held for the duration of the mount. It lets the mount diff --git a/kmod/src/lock.h b/kmod/src/lock.h index 3b16db03..f659e50b 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -73,8 +73,6 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, int scoutfs_lock_inode_index(struct super_block *sb, int mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); -int scoutfs_lock_xattr_index(struct super_block *sb, int mode, int flags, - u64 hash, struct scoutfs_lock **ret_lock); int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, struct inode *a, struct scoutfs_lock **a_lock, struct inode *b, struct scoutfs_lock **b_lock, diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 03aa27f9..d7c9d112 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -136,17 +136,6 @@ static int parse_tags(const char *name, unsigned int name_len, return 0; } -void scoutfs_xattr_index_key(struct scoutfs_key *key, - u64 hash, u64 ino, u64 id) -{ - scoutfs_key_set_zeros(key); - key->sk_zone = SCOUTFS_XATTR_INDEX_ZONE; - key->skxi_hash = cpu_to_le64(hash); - key->sk_type = SCOUTFS_XATTR_INDEX_NAME_TYPE; - key->skxi_ino = cpu_to_le64(ino); - key->skxi_id = cpu_to_le64(id); -} - /* * Find the next xattr and copy the key, xattr header, and as much of * the name and value into the callers buffer as we can. Returns the @@ -502,8 +491,7 @@ retry: scoutfs_inode_index_try_lock_hold(sb, &ind_locks, ind_seq, SIC_XATTR_SET(found_parts, value != NULL, - name_len, size, - tgs.srch)); + name_len, size)); if (ret > 0) goto retry; if (ret) diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index 4c78e323..8af0026f 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -14,7 +14,4 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock); -void scoutfs_xattr_index_key(struct scoutfs_key *key, - u64 hash, u64 ino, u64 id); - #endif From 9e975dffe104a48dce50f0b5344d0316fa1fd310 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Aug 2020 10:14:12 -0700 Subject: [PATCH 864/920] scoutfs: refactor btree split condition Btree traversal doesn't split a block if it has room for the caller's item. Extract this test into a function so that an upcoming btree call can test that each of multiple insertions into a leaf will fit. Signed-off-by: Zach Brown --- kmod/src/btree.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 59cde151..9d79335c 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -153,6 +153,13 @@ static inline unsigned int mid_free_off(struct scoutfs_btree_block *bt) return le16_to_cpu(ptr_off(bt, &bt->items[le16_to_cpu(bt->nr_items)])); } +/* true if the mid free region has room for an item struct and its value */ +static inline bool mid_free_item_room(struct scoutfs_btree_block *bt, + int val_len) +{ + return le16_to_cpu(bt->mid_free_len) >= item_len_bytes(val_len); +} + static inline struct scoutfs_key *item_key(struct scoutfs_btree_item *item) { return &item->key; @@ -873,7 +880,7 @@ static int try_split(struct super_block *sb, val_len = sizeof(struct scoutfs_btree_ref); /* don't need to split if there's enough space for the item */ - if (le16_to_cpu(right->mid_free_len) >= item_len_bytes(val_len)) + if (mid_free_item_room(right, val_len)) return 0; if (item_full_pct(right) < 80) { From 57af2bd34b77b3eb7da9ea8259b767ced4034881 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Aug 2020 10:17:44 -0700 Subject: [PATCH 865/920] scoutfs: give btree walk callers more keys The current btree walk recorded the start and end of child subtrees as it walked, and it could give the caller the next key to iterate towards after the block it returned. Future methods want to get at the key bounds of child subtrees, so we add a key range struct that all walk callers provide and fill it with all the interesting keys calculated during the walk. Signed-off-by: Zach Brown --- kmod/src/btree.c | 65 +++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 9d79335c..99dede60 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1194,6 +1194,14 @@ out: BUG(); } +struct btree_walk_key_range { + struct scoutfs_key start; + struct scoutfs_key end; + /* zero if no remaining blocks outside our walk in that direction */ + struct scoutfs_key iter_prev; + struct scoutfs_key iter_next; +}; + /* * Return the leaf block that should contain the given key. The caller * is responsible for searching the leaf block and performing their @@ -1217,7 +1225,7 @@ static int btree_walk(struct super_block *sb, int flags, struct scoutfs_key *key, unsigned int val_len, struct scoutfs_block **bl_ret, - struct scoutfs_key *iter_key) + struct btree_walk_key_range *kr) { struct scoutfs_block *par_bl = NULL; struct scoutfs_block *bl = NULL; @@ -1229,14 +1237,11 @@ static int btree_walk(struct super_block *sb, struct scoutfs_avl_node *next_node; struct scoutfs_avl_node *node; struct scoutfs_btree_ref *ref; - struct scoutfs_key start; - struct scoutfs_key end; unsigned int level; unsigned int nr; int ret; - if (WARN_ON_ONCE((flags & (BTW_NEXT|BTW_PREV)) && iter_key == NULL) || - WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) + if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; scoutfs_inc_counter(sb, btree_walk); @@ -1249,8 +1254,12 @@ restart: scoutfs_block_put(sb, bl); bl = NULL; bt = NULL; - scoutfs_key_set_zeros(&start); - scoutfs_key_set_ones(&end); + if (kr) { + scoutfs_key_set_zeros(&kr->start); + scoutfs_key_set_ones(&kr->end); + scoutfs_key_set_zeros(&kr->iter_prev); + scoutfs_key_set_zeros(&kr->iter_next); + } level = root->height; ret = 0; @@ -1280,8 +1289,8 @@ restart: break; bt = bl->data; - if (0) - verify_btree_block(sb, bt, level, &start, &end); + if (0 && kr) + verify_btree_block(sb, bt, level, &kr->start, &kr->end); /* XXX more aggressive block verification, before ref updates? */ if (bt->level != level) { @@ -1342,23 +1351,20 @@ restart: break; } - /* give the caller the next key to iterate towards */ - if (iter_key && (flags & BTW_NEXT) && next_item(bt, item)) { - *iter_key = *item_key(item); - scoutfs_key_inc(iter_key); - - } else if (iter_key && (flags & BTW_PREV) && - (prev = prev_item(bt, item))) { - *iter_key = *item_key(prev); + if (kr) { + /* update keys for walk bounds and next iteration */ + if ((prev = prev_item(bt, item))) { + kr->start = *item_key(prev); + scoutfs_key_inc(&kr->start); + kr->iter_prev = *item_key(prev); + } + kr->end = *item_key(item); + if (next_item(bt, item)) { + kr->iter_next = *item_key(item); + scoutfs_key_inc(&kr->iter_next); + } } - /* possible range of keys in referenced child block */ - if ((prev = prev_item(bt, item))) { - start = *item_key(prev); - scoutfs_key_inc(&start); - } - end = *item_key(item); - scoutfs_block_put(sb, par_bl); par_bl = bl; parent = bt; @@ -1672,8 +1678,9 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, struct scoutfs_avl_node *prev; struct scoutfs_btree_item *item; struct scoutfs_btree_block *bt; - struct scoutfs_key iter_key; + struct btree_walk_key_range kr; struct scoutfs_key walk_key; + struct scoutfs_key *iter_key; struct scoutfs_block *bl; int ret; @@ -1684,9 +1691,8 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, walk_key = *key; for (;;) { - scoutfs_key_set_zeros(&iter_key); ret = btree_walk(sb, NULL, NULL, root, flags, &walk_key, - 0, &bl, &iter_key); + 0, &bl, &kr); if (ret < 0) break; bt = bl->data; @@ -1708,8 +1714,9 @@ static int btree_iter(struct super_block *sb,struct scoutfs_btree_root *root, scoutfs_block_put(sb, bl); /* nothing in this leaf, walk gave us a key */ - if (!scoutfs_key_is_zeros(&iter_key)) { - walk_key = iter_key; + iter_key = (flags & BTW_NEXT) ? &kr.iter_next : &kr.iter_prev; + if (!scoutfs_key_is_zeros(iter_key)) { + walk_key = *iter_key; continue; } From 1a994137f40c0c0bd9e2987efdadb2494fb33868 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Aug 2020 10:20:50 -0700 Subject: [PATCH 866/920] scoutfs: add btree methods for item cache Add btree calls to call a callback for all items in a leaf, and to insert a list of items into their leaf blocks. These will be used by the item cache to populate the cache and to write dirty items into dirty btree blocks. Signed-off-by: Zach Brown --- kmod/src/btree.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/btree.h | 24 +++++++++++ 2 files changed, 128 insertions(+) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 99dede60..5c97e2e0 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1780,3 +1780,107 @@ int scoutfs_btree_dirty(struct super_block *sb, return ret; } + +/* + * Call the users callback on all the items in the leaf that we find. + * We also set the caller's keys for the first and last possible keys + * that could exist in the leaf block. + */ +int scoutfs_btree_read_items(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_btree_item_cb cb, void *arg) +{ + struct scoutfs_btree_item *item; + struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *next_node; + struct scoutfs_avl_node *node; + struct btree_walk_key_range kr; + struct scoutfs_block *bl; + int ret; + + ret = btree_walk(sb, NULL, NULL, root, 0, key, 0, &bl, &kr); + if (ret < 0) + goto out; + bt = bl->data; + + if (scoutfs_key_compare(&kr.start, start) > 0) + *start = kr.start; + if (scoutfs_key_compare(&kr.end, end) < 0) + *end = kr.end; + + node = scoutfs_avl_search(&bt->item_root, cmp_key_item, start, NULL, + NULL, &next_node, NULL) ?: next_node; + while (node) { + item = node_item(node); + if (scoutfs_key_compare(&item->key, end) > 0) + break; + + ret = cb(sb, item_key(item), item_val(bt, item), + item_val_len(item), arg); + if (ret < 0) + break; + + node = scoutfs_avl_next(&bt->item_root, node); + } + + scoutfs_block_put(sb, bl); +out: + return ret; +} + +/* + * The caller has a sorted list of items to insert. We find the leaf + * block that contains each item and either overwrite or insert the + * caller's item. This has no mechanism for deleting items. + * + * This can make partial progress before returning an error, leaving + * dirty btree blocks with only some of the caller's items. It's up to + * the caller to resolve this. + */ +int scoutfs_btree_insert_list(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_item_list *lst) +{ + struct scoutfs_btree_item *item; + struct btree_walk_key_range kr; + struct scoutfs_btree_block *bt; + struct scoutfs_avl_node *par; + struct scoutfs_block *bl; + int cmp; + int ret = 0; + + while (lst) { + ret = btree_walk(sb, alloc, wri, root, BTW_DIRTY | BTW_INSERT, + &lst->key, lst->val_len, &bl, &kr); + if (ret < 0) + goto out; + bt = bl->data; + + do { + item = leaf_item_hash_search(sb, bt, &lst->key); + if (item) { + update_item_value(bt, item, lst->val, + lst->val_len); + } else { + scoutfs_avl_search(&bt->item_root, + cmp_key_item, &lst->key, + &cmp, &par, NULL, NULL); + create_item(bt, &lst->key, lst->val, + lst->val_len, par, cmp); + } + + lst = lst->next; + } while (lst && scoutfs_key_compare(&lst->key, &kr.end) <= 0 && + mid_free_item_room(bt, lst->val_len)); + + scoutfs_block_put(sb, bl); + } + +out: + return ret; +} diff --git a/kmod/src/btree.h b/kmod/src/btree.h index e86396ae..c9bd6478 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -18,6 +18,18 @@ struct scoutfs_btree_item_ref { #define SCOUTFS_BTREE_ITEM_REF(name) \ struct scoutfs_btree_item_ref name = {NULL,} +/* caller gives an item to the callback */ +typedef int (*scoutfs_btree_item_cb)(struct super_block *sb, + struct scoutfs_key *key, + void *val, int val_len, void *arg); + +/* simple singly-linked list of items */ +struct scoutfs_btree_item_list { + struct scoutfs_btree_item_list *next; + struct scoutfs_key key; + int val_len; + u8 val[0]; +}; int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_btree_root *root, @@ -58,6 +70,18 @@ int scoutfs_btree_dirty(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key); +int scoutfs_btree_read_items(struct super_block *sb, + struct scoutfs_btree_root *root, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_btree_item_cb cb, void *arg); +int scoutfs_btree_insert_list(struct super_block *sb, + struct scoutfs_radix_allocator *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_btree_item_list *lst); + void scoutfs_btree_put_iref(struct scoutfs_btree_item_ref *iref); #endif From b1757a061ec8c00c23cbfdb07a55d1705a298571 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Aug 2020 10:22:46 -0700 Subject: [PATCH 867/920] scoutfs: add forest methods for item cache Add forest calls that the item cache will use. It needs to read all the items in the leaf blocks of forest btree which could contain the key, write dirty items to the log btree, and dirty bits in the bloom block as items are dirtied. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/forest.c | 166 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/forest.h | 20 ++++++ 3 files changed, 187 insertions(+) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index bce86418..a29ea29f 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -69,6 +69,7 @@ EXPAND_COUNTER(forest_refresh_bloom_roots) \ EXPAND_COUNTER(forest_refresh_dirty_log) \ EXPAND_COUNTER(forest_refresh_skip_log) \ + EXPAND_COUNTER(forest_read_items) \ EXPAND_COUNTER(forest_roots_next_hint) \ EXPAND_COUNTER(forest_roots_lock) \ EXPAND_COUNTER(forest_roots_server) \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c index e37490bd..284f1f1c 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -1195,6 +1195,155 @@ out: return ret; } +struct forest_read_items_data { + bool is_fs; + scoutfs_forest_item_cb cb; + void *cb_arg; +}; + +static int forest_read_items(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, void *arg) +{ + struct forest_read_items_data *rid = arg; + struct scoutfs_log_item_value _liv = {0,}; + struct scoutfs_log_item_value *liv = &_liv; + + if (!rid->is_fs) { + liv = val; + val += sizeof(struct scoutfs_log_item_value); + val_len -= sizeof(struct scoutfs_log_item_value); + } + + return rid->cb(sb, key, liv, val, val_len, rid->cb_arg); +} + +/* + * For each forest btree whose bloom block indicates that the lock might + * have items stored, call the caller's callback for every item in the + * leaf block in each tree which contains the key. + * + * The btree iter calls clamp the caller's range to the tightest range + * that covers all the blocks. Any keys outside of this range can't be + * trusted because we didn't visit all the trees to check their items. + * + * If we hit stale blocks and retry we can call the callback for + * duplicate items. This is harmless because the items are stable while + * the caller holds their cluster lock and the caller has to filter out + * item versions anyway. + */ +int scoutfs_forest_read_items(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_forest_item_cb cb, void *arg) +{ + DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); + struct forest_read_items_data rid = { + .cb = cb, + .cb_arg = arg, + }; + struct scoutfs_log_trees_val ltv; + struct scoutfs_net_roots roots; + struct scoutfs_bloom_block *bb; + struct forest_bloom_nrs bloom; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_block *bl; + struct scoutfs_key ltk; + int ret; + int i; + + calc_bloom_nrs(&bloom, &lock->start); + + roots = lock->roots; +retry: + scoutfs_inc_counter(sb, forest_read_items); + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + + trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); + refs.fs_ref = roots.fs_root.ref; + refs.logs_ref = roots.logs_root.ref; + + *start = lock->start; + *end = lock->end; + + /* start with fs root items */ + rid.is_fs = true; + ret = scoutfs_btree_read_items(sb, &roots.fs_root, key, start, end, + forest_read_items, &rid); + if (ret < 0) + goto out; + rid.is_fs = false; + + scoutfs_key_init_log_trees(<k, 0, 0); + for (;; scoutfs_key_inc(<k)) { + ret = scoutfs_btree_next(sb, &roots.logs_root, <k, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(ltv)) { + ltk = *iref.key; + memcpy(<v, iref.val, sizeof(ltv)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + break; + goto out; /* including stale */ + } + + if (ltv.bloom_ref.blkno == 0) + continue; + + bl = read_bloom_ref(sb, <v.bloom_ref); + if (IS_ERR(bl)) { + ret = PTR_ERR(bl); + goto out; + } + bb = bl->data; + + for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { + if (!test_bit_le(bloom.nrs[i], bb->bits)) + break; + } + + scoutfs_block_put(sb, bl); + + /* one of the bloom bits wasn't set */ + if (i != ARRAY_SIZE(bloom.nrs)) { + scoutfs_inc_counter(sb, forest_bloom_fail); + continue; + } + + scoutfs_inc_counter(sb, forest_bloom_pass); + + ret = scoutfs_btree_read_items(sb, <v.item_root, key, start, + end, forest_read_items, &rid); + if (ret < 0) + goto out; + } + + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&prev_refs, &refs, sizeof(refs)) == 0) { + ret = -EIO; + goto out; + } + prev_refs = refs; + + ret = scoutfs_client_get_roots(sb, &roots); + if (ret) + goto out; + goto retry; + } + + return ret; +} + /* * Make sure that the bloom bits for the lock's start key are all set in * the current log's bloom block. We record the nr of our log tree in @@ -1310,6 +1459,23 @@ out: return ret; } +int scoutfs_forest_set_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock) +{ + DECLARE_FOREST_INFO(sb, finf); + + return set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); +} + +int scoutfs_forest_insert_list(struct super_block *sb, + struct scoutfs_btree_item_list *lst) +{ + DECLARE_FOREST_INFO(sb, finf); + + return scoutfs_btree_insert_list(sb, finf->alloc, finf->wri, + &finf->our_log.item_root, lst); +} + /* * The btree code takes a single value buffer. When we're working with * the log btrees we want to add a log item value metadata header. In diff --git a/kmod/src/forest.h b/kmod/src/forest.h index e1411ad5..b480c971 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -3,6 +3,15 @@ struct scoutfs_radix_allocator; struct scoutfs_block_writer; +struct scoutfs_block; + +#include "btree.h" + +/* caller gives an item to the callback */ +typedef int (*scoutfs_forest_item_cb)(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, + void *val, int val_len, void *arg); int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, struct kvec *val, struct scoutfs_lock *lock); @@ -38,6 +47,17 @@ int scoutfs_forest_delete_save(struct super_block *sb, int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, struct scoutfs_lock *lock); void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); + +int scoutfs_forest_read_items(struct super_block *sb, + struct scoutfs_lock *lock, + struct scoutfs_key *key, + struct scoutfs_key *start, + struct scoutfs_key *end, + scoutfs_forest_item_cb cb, void *arg); +int scoutfs_forest_set_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock); +int scoutfs_forest_insert_list(struct super_block *sb, + struct scoutfs_btree_item_list *lst); int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); void scoutfs_forest_init_btrees(struct super_block *sb, From 45e594396f5e64bbb955d50bccd0d1487e83ec98 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 18 Aug 2020 16:49:18 -0700 Subject: [PATCH 868/920] scoutfs: add an item cache above the btrees Add an item cache between fs callers and the forest of btrees. Calling out to the btrees for every item operation was far too expensive. This gives us a flexible in-memory structure for working with items that isn't bound by the constrants of persistent block IO. We can rarely stream large groups of items to and from the btrees and then use efficient kernel memory structures for more frequent item operations. This adds the infrastructure, nothing is calling it yet. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/counters.h | 29 + kmod/src/item.c | 2325 +++++++++++++++++++++++++++++++++++++++++++ kmod/src/item.h | 39 + kmod/src/super.c | 3 + kmod/src/super.h | 1 + 6 files changed, 2398 insertions(+) create mode 100644 kmod/src/item.c create mode 100644 kmod/src/item.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index cf2c39ee..53ce0a5b 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -21,6 +21,7 @@ scoutfs-y += \ forest.o \ inode.o \ ioctl.o \ + item.o \ lock.o \ lock_server.o \ msg.o \ diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a29ea29f..6c97923c 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -77,6 +77,35 @@ EXPAND_COUNTER(forest_set_bloom_bits) \ EXPAND_COUNTER(forest_set_dirtied) \ EXPAND_COUNTER(forest_trigger_refresh) \ + EXPAND_COUNTER(item_clear_dirty) \ + EXPAND_COUNTER(item_create) \ + EXPAND_COUNTER(item_delete) \ + EXPAND_COUNTER(item_dirty) \ + EXPAND_COUNTER(item_invalidate) \ + EXPAND_COUNTER(item_invalidate_page) \ + EXPAND_COUNTER(item_lookup) \ + EXPAND_COUNTER(item_mark_dirty) \ + EXPAND_COUNTER(item_next) \ + EXPAND_COUNTER(item_page_accessed) \ + EXPAND_COUNTER(item_page_alloc) \ + EXPAND_COUNTER(item_page_clear_dirty) \ + EXPAND_COUNTER(item_page_free) \ + EXPAND_COUNTER(item_page_lru_add) \ + EXPAND_COUNTER(item_page_lru_remove) \ + EXPAND_COUNTER(item_page_mark_dirty) \ + EXPAND_COUNTER(item_page_rbtree_walk) \ + EXPAND_COUNTER(item_page_split) \ + EXPAND_COUNTER(item_pcpu_add_replaced) \ + EXPAND_COUNTER(item_pcpu_page_hit) \ + EXPAND_COUNTER(item_pcpu_page_miss) \ + EXPAND_COUNTER(item_pcpu_page_miss_keys) \ + EXPAND_COUNTER(item_read_pages_split) \ + EXPAND_COUNTER(item_shrink_page) \ + EXPAND_COUNTER(item_shrink_page_dirty) \ + EXPAND_COUNTER(item_shrink_page_reader) \ + EXPAND_COUNTER(item_shrink_page_trylock) \ + EXPAND_COUNTER(item_update) \ + EXPAND_COUNTER(item_write_dirty) \ EXPAND_COUNTER(lock_alloc) \ EXPAND_COUNTER(lock_free) \ EXPAND_COUNTER(lock_grace_extended) \ diff --git a/kmod/src/item.c b/kmod/src/item.c new file mode 100644 index 00000000..7062f079 --- /dev/null +++ b/kmod/src/item.c @@ -0,0 +1,2325 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "item.h" +#include "forest.h" +#include "block.h" +#include "trans.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * The item cache maintains a consistent view of items that are read + * from and written to the forest of btrees under the protection of + * cluster locks. + * + * The cache is built around pages of items. A page has the range of + * keys that it caches and the items that are present in that range. + * Pages are non-overlapping, there is only one page that can contain a + * given key at a time. The pages are tracked by an rbtree, and each + * page has an rbtree of items. + * + * The cache is populated by reading items from the forest of btrees + * into a private set of pages. The regions of those pages which + * weren't already cached are then inserted into the cache. + * + * CPUs can concurrently modify items that are in different pages. The + * page rbtree can be read locked to find a page, and then the page is + * locked to work with its items. We then add per-cpu references to + * recently used pages so that the global page rbtree can be skipped in + * the typical case of repeated calls to localized portions of the key + * space. + * + * Dirty items are kept in a per-page dirty list, and pages with dirty + * items are kept in a global dirty list. This reduces contention on + * the global list by accessing it at page granularity instead of every + * time an item is dirtied. The dirty items are not sorted until it + * comes time to commit them to the btrees. This reduces the cost of + * tracking dirty items during the transaction, particularly moving them + * between pages as pages are split to make room for new items. + * + * The size of the cache is only limited by memory reclaim. Pages are + * kept in a very coarse lru. Dirtying doesn't remove pages from the + * lru, and is operating against lock ordering with trylocks, so + * shrinking can rarely have to skip pages in the LRU. + * + * The locking is built around the fast path of everyone checking the + * the page rbtree, then locking pages, and then adding or removing + * pages from the lru or dirty lists. Writing and the shrinker work + * work in reverse, starting with the dirty or lru lists and have to use + * trylock to lock the pages. When we split we have to lock multiple + * pages and we use trylock which is guaranteed to succeed because the + * pages are private. + */ + +struct item_cache_info { + /* almost always read, barely written */ + struct super_block *sb; + struct item_percpu_pages __percpu *pcpu_pages; + struct shrinker shrinker; + struct notifier_block notifier; + + /* often walked, but per-cpu refs are fast path */ + rwlock_t rwlock; + struct rb_root pg_root; + + /* page-granular modification by writers, then exclusive to commit */ + spinlock_t dirty_lock; + struct list_head dirty_list; + atomic_t dirty_pages; + + /* page-granular modification by readers */ + spinlock_t lru_lock; + struct list_head lru_list; + unsigned long lru_pages; + + /* written by page readers, read by shrink */ + spinlock_t active_lock; + struct rb_root active_root; +}; + +#define DECLARE_ITEM_CACHE_INFO(sb, name) \ + struct item_cache_info *name = SCOUTFS_SB(sb)->item_cache_info + +#define PG_PER_CPU 32 +struct item_percpu_pages { + struct rb_root root; + struct list_head list; + struct pcpu_page_ref { + struct scoutfs_key start; + struct scoutfs_key end; + struct cached_page *pg; + struct rb_node node; + struct list_head head; + } refs[PG_PER_CPU]; +}; + +struct cached_page { + /* often read by concurrent rbtree walks */ + struct rb_node node; + struct scoutfs_key start; + struct scoutfs_key end; + + /* often modified by page rwlock holder */ + rwlock_t rwlock; + struct rb_root item_root; + struct list_head lru_head; + unsigned long lru_time; + struct list_head dirty_list; + struct list_head dirty_head; + struct page *page; + unsigned int page_off; + atomic_t refcount; +}; + +struct cached_item { + struct rb_node node; + struct list_head dirty_head; + unsigned int dirty:1, /* needs to be written */ + persistent:1, /* in btrees, needs deletion item */ + deletion:1; /* negative del item for writing */ + unsigned int val_len; + struct scoutfs_key key; + struct scoutfs_log_item_value liv; + char val[0]; +}; + +#define CACHED_ITEM_ALIGN 16 + +static int item_val_bytes(int val_len) +{ + return offsetof(struct cached_item, val[val_len]); +} + +/* + * Return if the page has room to allocate an item with the given value + * length at its free page offset. This must be called with the page + * writelock held because it can modify the page to reclaim free space + * to mkae room for the allocation. Today all it does is recognize that + * the page is empty and reset the page_off. + */ +static bool page_has_room(struct cached_page *pg, int val_len) +{ + if (RB_EMPTY_ROOT(&pg->item_root)) + pg->page_off = 0; + + return pg->page_off + item_val_bytes(val_len) <= PAGE_SIZE; +} + +static struct cached_page *first_page(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_first(root))) + return NULL; + + return rb_entry(node, struct cached_page, node); +} + +static struct cached_item *first_item(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_first(root))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *last_item(struct rb_root *root) +{ + struct rb_node *node; + + if (!root || !(node = rb_last(root))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *next_item(struct cached_item *item) +{ + struct rb_node *node; + + if (!item || !(node = rb_next(&item->node))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static struct cached_item *prev_item(struct cached_item *item) +{ + struct rb_node *node; + + if (!item || !(node = rb_prev(&item->node))) + return NULL; + + return rb_entry(node, struct cached_item, node); +} + +static void rbtree_insert(struct rb_node *node, struct rb_node *par, + struct rb_node **pnode, struct rb_root *root) +{ + rb_link_node(node, par, pnode); + rb_insert_color(node, root); +} + +static void rbtree_erase(struct rb_node *node, struct rb_root *root) +{ + rb_erase(node, root); + RB_CLEAR_NODE(node); +} + +static void rbtree_replace_node(struct rb_node *victim, struct rb_node *new, + struct rb_root *root) +{ + rb_replace_node(victim, new, root); + RB_CLEAR_NODE(victim); +} + +/* + * This lets us lock newly allocated pages without having to add nesting + * annotation. The non-acquired path is never executed. + */ +static void write_trylock_will_succeed(rwlock_t *rwlock) +__acquires(rwlock) +{ + while (!write_trylock(rwlock)) + BUG(); +} + +static struct cached_page *alloc_pg(struct super_block *sb, gfp_t gfp) +{ + struct cached_page *pg; + struct page *page; + + pg = kzalloc(sizeof(struct cached_page), GFP_NOFS | gfp); + page = alloc_page(GFP_NOFS | gfp); + if (!page || !pg) { + kfree(pg); + __free_page(page); + return NULL; + } + + scoutfs_inc_counter(sb, item_page_alloc); + + RB_CLEAR_NODE(&pg->node); + rwlock_init(&pg->rwlock); + pg->item_root = RB_ROOT; + INIT_LIST_HEAD(&pg->lru_head); + INIT_LIST_HEAD(&pg->dirty_list); + INIT_LIST_HEAD(&pg->dirty_head); + pg->page = page; + atomic_set(&pg->refcount, 1); + + return pg; +} + +static void get_pg(struct cached_page *pg) +{ + atomic_inc(&pg->refcount); +} + +static void put_pg(struct super_block *sb, struct cached_page *pg) +{ + if (pg && atomic_dec_and_test(&pg->refcount)) { + scoutfs_inc_counter(sb, item_page_free); + + BUG_ON(!RB_EMPTY_NODE(&pg->node)); + BUG_ON(!list_empty(&pg->lru_head)); + BUG_ON(!list_empty(&pg->dirty_list)); + BUG_ON(!list_empty(&pg->dirty_head)); + + __free_page(pg->page); + kfree(pg); + } +} + +/* + * Allocate space for a new item from the free offset at the end of a + * cached page. This isn't a blocking allocation, and it's likely that + * the caller has ensured it will succeed by allocating from a new empty + * page or checking the free space first. + */ +static struct cached_item *alloc_item(struct cached_page *pg, + struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, + void *val, int val_len) +{ + struct cached_item *item; + + if (!page_has_room(pg, val_len)) + return NULL; + + item = page_address(pg->page) + pg->page_off; + pg->page_off += round_up(item_val_bytes(val_len), CACHED_ITEM_ALIGN); + + RB_CLEAR_NODE(&item->node); + INIT_LIST_HEAD(&item->dirty_head); + item->dirty = 0; + item->persistent = 0; + item->deletion = !!(liv->flags & SCOUTFS_LOG_ITEM_FLAG_DELETION); + item->val_len = val_len; + item->key = *key; + item->liv = *liv; + + if (val_len) + memcpy(item->val, val, val_len); + + return item; +} + +static void lru_add(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + spin_lock(&cinf->lru_lock); + if (list_empty(&pg->lru_head)) { + scoutfs_inc_counter(sb, item_page_lru_add); + list_add_tail(&pg->lru_head, &cinf->lru_list); + cinf->lru_pages++; + } + spin_unlock(&cinf->lru_lock); +} + +static void __lru_remove(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + if (!list_empty(&pg->lru_head)) { + scoutfs_inc_counter(sb, item_page_lru_remove); + list_del_init(&pg->lru_head); + cinf->lru_pages--; + } +} + +static void lru_remove(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + spin_lock(&cinf->lru_lock); + __lru_remove(sb, cinf, pg); + spin_unlock(&cinf->lru_lock); +} + +/* + * Make sure that the page the caller just accessed is reasonably close + * to the tail of the lru so it will be less likely to be reclaimed by + * the shrinker. + * + * We want to quickly determine that the page is close enough to the + * tail by only looking at the page. We use a coarse clock tick to + * determine if we've already moved the head to the tail sufficiently + * recently. We can't differentiate shrinking priority amongst the + * number of pages that the cpu can access within given chunk of time. + * + * We don't care that the lru_time accessed aren't locked and could see + * rare corruption. It's just a shrink priority heuristic. + */ +static void lru_accessed(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + unsigned long time = jiffies_to_msecs(jiffies); + + scoutfs_inc_counter(sb, item_page_accessed); + + if (pg->lru_time != time) { + lru_remove(sb, cinf, pg); + pg->lru_time = time; + lru_add(sb, cinf, pg); + } +} + +/* + * Return the pg that contains the key and set the parent nodes for insertion. + * When we find the pg we go right so that the caller can insert a new + * page to the right of the found page if it had to split the page. + */ +static struct cached_page *page_rbtree_walk(struct super_block *sb, + struct rb_root *root, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct cached_page **prev, + struct cached_page **next, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct cached_page *ret = NULL; + struct cached_page *pg; + int cmp; + + scoutfs_inc_counter(sb, item_page_rbtree_walk); + + if (next) + *next = NULL; + if (prev) + *prev = NULL; + + while (*node) { + parent = *node; + pg = container_of(*node, struct cached_page, node); + + cmp = scoutfs_key_compare_ranges(start, end, &pg->start, + &pg->end); + if (cmp < 0) { + if (next) + *next = pg; + node = &(*node)->rb_left; + } else if (cmp > 0) { + if (prev) + *prev = pg; + node = &(*node)->rb_right; + } else { + ret = pg; + node = &(*node)->rb_right; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +#define for_each_page_safe(root, pg, tmp) \ + for (tmp = rb_first(root); \ + tmp && (pg = container_of(tmp, struct cached_page, node)) && \ + ((tmp = rb_next(tmp)), 1); ) + +static struct cached_item *item_rbtree_walk(struct rb_root *root, + struct scoutfs_key *key, + struct cached_item **next, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct cached_item *ret = NULL; + struct cached_item *item; + int cmp; + + if (next) + *next = NULL; + + while (*node) { + parent = *node; + item = container_of(*node, struct cached_item, node); + + cmp = scoutfs_key_compare(key, &item->key); + if (cmp < 0) { + if (next) + *next = item; + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = item; + node = &(*node)->rb_left; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +#define for_each_item_from_safe(root, item, tmp, key) \ + for (item = item_rbtree_walk(root, key, &tmp, NULL, NULL) ?: tmp; \ + item && ((tmp = next_item(item)), 1); \ + item = tmp) + +#define for_each_item_safe(root, item, tmp) \ + for (tmp = rb_first(root); \ + tmp && (item = container_of(tmp, struct cached_item, node)) && \ + ((tmp = rb_next(tmp)), 1); ) + +/* + * As we mark the first and clear the last items in a page, we add or + * delete the page from the dirty list. The caller can give us a page + * to add the newly dirtied page after, rather than at the tail of the + * list. + */ +static void mark_item_dirty(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_page *after, + struct cached_item *item) +{ + if (!item->dirty) { + if (list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_page_mark_dirty); + spin_lock(&cinf->dirty_lock); + if (after) + list_add(&pg->dirty_head, &after->dirty_head); + else + list_add_tail(&pg->dirty_head, + &cinf->dirty_list); + atomic_inc(&cinf->dirty_pages); + spin_unlock(&cinf->dirty_lock); + } + + scoutfs_inc_counter(sb, item_mark_dirty); + list_add_tail(&item->dirty_head, &pg->dirty_list); + item->dirty = 1; + } +} + +static void clear_item_dirty(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_item *item) +{ + if (item->dirty) { + scoutfs_inc_counter(sb, item_clear_dirty); + item->dirty = 0; + list_del_init(&item->dirty_head); + + if (list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_page_clear_dirty); + spin_lock(&cinf->dirty_lock); + list_del_init(&pg->dirty_head); + atomic_dec(&cinf->dirty_pages); + spin_unlock(&cinf->dirty_lock); + } + } +} + +static void erase_page_items(struct cached_page *pg, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + struct cached_item *item; + struct cached_item *tmp; + + for_each_item_from_safe(&pg->item_root, item, tmp, start) { + + /* only called in unused read regions or read_pages pages */ + BUG_ON(item->dirty); + + if (scoutfs_key_compare(&item->key, end) > 0) + break; + rbtree_erase(&item->node, &pg->item_root); + } +} + +/* + * Move all the items starting from the key and stopping before moving + * the stop key. The right destination page must be empty. Items are + * copied in tree order which lets us easily insert after each previous + * item. + * + * This preserves dirty page and item ordering by adding the right page + * to the dirty list after the left page, and by adding items to the + * tail of right's dirty list in key sort order. + * + * The caller is responsible for page locking and managing the lru. + */ +static void move_page_items(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *left, + struct cached_page *right, + struct scoutfs_key *key, + struct scoutfs_key *stop) +{ + struct cached_item *from; + struct cached_item *to; + struct cached_item *tmp; + struct rb_node **pnode; + struct rb_node *par; + + /* really empty right destination? */ + BUG_ON(!RB_EMPTY_ROOT(&right->item_root)); + par = NULL; + pnode = &right->item_root.rb_node; + + for_each_item_from_safe(&left->item_root, from, tmp, key) { + + if (stop && scoutfs_key_compare(&from->key, stop) >= 0) + break; + + to = alloc_item(right, &from->key, &from->liv, from->val, + from->val_len); + rbtree_insert(&to->node, par, pnode, &right->item_root); + par = &to->node; + pnode = &to->node.rb_right; + + if (from->dirty) { + mark_item_dirty(sb, cinf, right, left, to); + clear_item_dirty(sb, cinf, left, from); + } + + to->persistent = from->persistent; + to->deletion = from->deletion; + + rbtree_erase(&from->node, &left->item_root); + } +} + +enum { + PGI_DISJOINT, + PGI_INSIDE, + PGI_START_OLAP, + PGI_END_OLAP, + PGI_BISECT_NEEDED, + PGI_BISECT, +}; + +/* + * Remove items from the page with intersect with the range. We return + * a code to indicate which kind of intersection occurred. The caller + * provides the right page to move items to if the page is bisected by + * the range. + * + * This modifies the page keys so it needs to be held with a write page + * rbtree lock if the page is in the page rbtree. + */ +static int trim_page_intersection(struct super_block *sb, + struct item_cache_info *cinf, + struct cached_page *pg, + struct cached_page *right, + struct scoutfs_key *start, + struct scoutfs_key *end) +{ + if (scoutfs_key_compare(&pg->start, end) > 0 || + scoutfs_key_compare(&pg->end, start) < 0) { + /* page and range don't intersect */ + return PGI_DISJOINT; + } + + if (scoutfs_key_compare(&pg->start, start) >= 0 && + scoutfs_key_compare(&pg->end, end) <= 0) { + /* page entirely inside range */ + return PGI_INSIDE; + } + + if (scoutfs_key_compare(&pg->start, end) <= 0 && + scoutfs_key_compare(&pg->end, end) > 0) { + /* start of page intersects with range */ + pg->start = *end; + scoutfs_key_inc(&pg->start); + erase_page_items(pg, start, end); + return PGI_START_OLAP; + } + + if (scoutfs_key_compare(&pg->end, start) >= 0 && + scoutfs_key_compare(&pg->start, start) < 0) { + /* end of page intersects with range */ + pg->end = *start; + scoutfs_key_dec(&pg->end); + erase_page_items(pg, start, end); + return PGI_END_OLAP; + } + + /* page surrounds range, and is bisected by it */ + if (!right) + return PGI_BISECT_NEEDED; + + right->start = *end; + scoutfs_key_inc(&right->start); + right->end = pg->end; + pg->end = *start; + scoutfs_key_dec(&pg->end); + erase_page_items(pg, start, end); + move_page_items(sb, cinf, pg, right, &right->start, NULL); + return PGI_BISECT; +} + +/* + * This behaves a little differently than the other walks because we + * want to minimize compares and there are only simple searching and + * inserting callers. + */ +static struct pcpu_page_ref *pcpu_page_rbtree_walk(struct rb_root *root, + struct scoutfs_key *key, + struct pcpu_page_ref *ins) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct pcpu_page_ref *ret = NULL; + struct pcpu_page_ref *ref; + int cmp; + + while (*node) { + parent = *node; + ref = container_of(*node, struct pcpu_page_ref, node); + + cmp = scoutfs_key_compare_ranges(key, key, + &ref->start, &ref->end); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = ref; + if (!ins) + return ret; + node = &(*node)->rb_right; + } + } + + if (ins) + rbtree_insert(&ins->node, parent, node, root); + + return ret; +} + +/* + * Search the per-cpu page references for a page that contains the key + * the caller needs. These lookups are very frequent and key + * comparisons are relatively expensive, so we use an rbtree to decrease + * the comparison costs, particularly of misses. + * + * All the references in all the cpus go stale as page key boundaries + * are modified by reading, insertion, and invalidation. If we find a + * stale ref we will drop it, but otherwise we let stale refs age out as + * new refs are inserted. + */ +static struct cached_page *get_pcpu_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_key *key, + bool write) +{ + struct item_percpu_pages *pages = get_cpu_ptr(cinf->pcpu_pages); + struct cached_page *pg = NULL; + struct pcpu_page_ref *ref; + + ref = pcpu_page_rbtree_walk(&pages->root, key, NULL); + if (ref) { + pg = ref->pg; + if (write) + write_lock(&pg->rwlock); + else + read_lock(&pg->rwlock); + + if (scoutfs_key_compare_ranges(key, key, + &pg->start, &pg->end)) { + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + + scoutfs_inc_counter(sb, item_pcpu_page_miss_keys); + rbtree_erase(&ref->node, &pages->root); + list_move_tail(&ref->head, &pages->list); + put_pg(sb, pg); + ref->pg = NULL; + pg = NULL; + } else { + if (pages->list.next != &ref->head) + list_move(&ref->head, &pages->list); + __release(pg_rwlock); + } + } + + put_cpu_ptr(cinf->pcpu_pages); + + if (pg) + scoutfs_inc_counter(sb, item_pcpu_page_hit); + else + scoutfs_inc_counter(sb, item_pcpu_page_miss); + + return pg; +} + +/* + * The caller has a locked page that it knows is authoritative for its + * range of keys. Add it to this cpu's cache and remove any other page + * in the pool which intersects with its range. + */ +static void add_pcpu_page(struct super_block *sb, struct item_cache_info *cinf, + struct cached_page *pg) +{ + struct item_percpu_pages *pages = get_cpu_ptr(cinf->pcpu_pages); + struct pcpu_page_ref *old; + struct pcpu_page_ref *ref; + + ref = list_last_entry(&pages->list, struct pcpu_page_ref, head); + if (ref->pg) { + rbtree_erase(&ref->node, &pages->root); + put_pg(sb, ref->pg); + } + ref->start = pg->start; + ref->end = pg->end; + ref->pg = pg; + get_pg(pg); + + list_move(&ref->head, &pages->list); + + old = pcpu_page_rbtree_walk(&pages->root, &ref->end, ref); + if (old) { + scoutfs_inc_counter(sb, item_pcpu_add_replaced); + rbtree_erase(&old->node, &pages->root); + list_move_tail(&old->head, &pages->list); + put_pg(sb, old->pg); + old->pg = NULL; + } + + put_cpu_ptr(cinf->pcpu_pages); +} + +/* + * If a page is removed from the page rbtree we clear its keys so that percpu + * references won't use the page and will drop their reference. Must be + * called with a write page rwlock. + */ +static void invalidate_pcpu_page(struct cached_page *pg) +{ + scoutfs_key_set_zeros(&pg->start); + scoutfs_key_set_zeros(&pg->end); +} + +static void init_pcpu_pages(struct item_cache_info *cinf, int cpu) +{ + struct item_percpu_pages *pages = per_cpu_ptr(cinf->pcpu_pages, cpu); + struct pcpu_page_ref *ref; + int i; + + pages->root = RB_ROOT; + INIT_LIST_HEAD(&pages->list); + + for (i = 0; i < ARRAY_SIZE(pages->refs); i++) { + ref = &pages->refs[i]; + + ref->pg = NULL; + list_add_tail(&ref->head, &pages->list); + } +} + +static void drop_pcpu_pages(struct super_block *sb, + struct item_cache_info *cinf, int cpu) +{ + struct item_percpu_pages *pages = per_cpu_ptr(cinf->pcpu_pages, cpu); + struct pcpu_page_ref *ref; + int i; + + for (i = 0; i < ARRAY_SIZE(pages->refs); i++) { + ref = &pages->refs[i]; + + if (ref->pg) + put_pg(sb, ref->pg); + ref->pg = NULL; + } + + pages->root = RB_ROOT; +} + +/* + * We're about to move all the items between to a pair of new pages. + * Find the item that balances the space consumed by items in either + * page. We move the mid (and possibly only) item to the right page. + */ +static void set_split_keys(struct cached_page *pg, struct cached_page *left, + struct cached_page *right) +{ + struct cached_item *left_item = first_item(&pg->item_root); + struct cached_item *right_item = last_item(&pg->item_root); + struct cached_item *mid; + int left_tot = 0; + int right_tot = 0; + + while (left_item && right_item && left_item != right_item) { + if (left_tot < right_tot) { + left_tot += item_val_bytes(left_item->val_len); + left_item = next_item(left_item); + } else { + right_tot += item_val_bytes(right_item->val_len); + right_item = prev_item(right_item); + } + } + + mid = left_item ?: right_item; + + left->start = pg->start; + left->end = mid->key; + scoutfs_key_dec(&left->end); + right->start = mid->key; + right->end = pg->end; +} + +/* + * The caller found a page that didn't have room for the item they + * wanted to allocate. We allocate pages for the split and see if the + * page still needs splitting once we've locked it. + * + * To modify page keys we need a write lock on the page rbtree, which + * globally prevents reads from finding pages. We want to minimize this + * so we add empty pages with the split ranges to the rbtree and then + * perform the item motion only with the page locks held. This will + * exclude any users of the items in the affected range. + */ +static int try_split_page(struct super_block *sb, struct item_cache_info *cinf, + struct scoutfs_key *key, int val_len) +{ + struct cached_page *right; + struct cached_page *left; + struct cached_page *pg; + struct cached_item *item; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + left = alloc_pg(sb, 0); + right = alloc_pg(sb, 0); + if (!left || !right) { + ret = -ENOMEM; + goto out; + } + + write_lock(&cinf->rwlock); + + pg = page_rbtree_walk(sb, &cinf->pg_root, key, key, NULL, NULL, + &par, &pnode); + if (pg == NULL) { + write_unlock(&cinf->rwlock); + ret = 0; + goto out; + } + + write_lock(&pg->rwlock); + + if (page_has_room(pg, val_len)) { + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + ret = 0; + goto out; + } + + /* special case adding an empty page when key is after the last item */ + item = last_item(&pg->item_root); + if (scoutfs_key_compare(key, &item->key) > 0) { + right->start = *key; + right->end = pg->end; + pg->end = *key; + scoutfs_key_dec(&pg->end); + + write_trylock_will_succeed(&right->rwlock); + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, right); + + /* adding right first removes pg */ + add_pcpu_page(sb, cinf, right); + add_pcpu_page(sb, cinf, pg); + + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + write_unlock(&right->rwlock); + right = NULL; + ret = 0; + goto out; + } + + scoutfs_inc_counter(sb, item_page_split); + + /* pages are still private, tylock will succeed */ + write_trylock_will_succeed(&left->rwlock); + write_trylock_will_succeed(&right->rwlock); + + set_split_keys(pg, left, right); + + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + rbtree_replace_node(&pg->node, &left->node, &cinf->pg_root); + lru_remove(sb, cinf, pg); + + write_unlock(&cinf->rwlock); + + /* move items while only holding page locks, visible once unlocked */ + move_page_items(sb, cinf, pg, left, &left->start, &right->start); + lru_accessed(sb, cinf, left); + add_pcpu_page(sb, cinf, left); + write_unlock(&left->rwlock); + left = NULL; + + move_page_items(sb, cinf, pg, right, &right->start, NULL); + lru_accessed(sb, cinf, right); + add_pcpu_page(sb, cinf, right); + write_unlock(&right->rwlock); + right = NULL; + + /* and drop the source page, it was replaced above */ + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + put_pg(sb, pg); + + ret = 0; +out: + put_pg(sb, left); + put_pg(sb, right); + return ret; +} + +/* + * The caller has a write-only cluster lock and wants to populate the + * cache so that it can insert an item without reading. They found a + * hole but unlocked so we check again under the lock after allocating. + * We insert an empty page that covers the key and extends to either the + * neighbours or the caller's (lock's) range. + */ +static int cache_empty_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_key *key, struct scoutfs_key *start, + struct scoutfs_key *end) +{ + struct cached_page *prev; + struct cached_page *next; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + + pg = alloc_pg(sb, 0); + if (!pg) + return -ENOMEM; + + write_lock(&cinf->rwlock); + + if (!page_rbtree_walk(sb, &cinf->pg_root, key, key, &prev, &next, + &par, &pnode)) { + pg->start = *start; + if (prev && scoutfs_key_compare(&prev->end, start) > 0) { + pg->start = prev->end; + scoutfs_key_inc(&pg->start); + } + + pg->end = *end; + if (next && scoutfs_key_compare(&next->start, end) < 0) { + pg->end = next->start; + scoutfs_key_dec(&pg->end); + } + + rbtree_insert(&pg->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, pg); + pg = NULL; + } + + write_unlock(&cinf->rwlock); + + put_pg(sb, pg); + + return 0; +} + +struct active_reader { + struct rb_node node; + struct scoutfs_key start; + struct scoutfs_key end; +}; + +static struct active_reader *active_rbtree_walk(struct rb_root *root, + struct scoutfs_key *start, + struct scoutfs_key *end, + struct rb_node **par, + struct rb_node ***pnode) +{ + struct rb_node **node = &root->rb_node; + struct rb_node *parent = NULL; + struct active_reader *ret = NULL; + struct active_reader *active; + int cmp; + + while (*node) { + parent = *node; + active = container_of(*node, struct active_reader, node); + + cmp = scoutfs_key_compare_ranges(start, end, &active->start, + &active->end); + if (cmp < 0) { + node = &(*node)->rb_left; + } else if (cmp > 0) { + node = &(*node)->rb_right; + } else { + ret = active; + node = &(*node)->rb_left; + } + } + + if (par) + *par = parent; + if (pnode) + *pnode = node; + + return ret; +} + +/* + * Add a newly read item to the pages that we're assembling for + * insertion into the cache. These pages are private, they only exist + * on our root and aren't in dirty or lru lists. + * + * We need to store deletion items here as we read items from all the + * btrees so that they can override older versions of the items. The + * deletion items will be deleted before we insert the pages into the + * cache. We don't insert old versions of items into the tree here so + * that the trees don't have to compare versions. + */ +static int read_page_item(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_log_item_value *liv, void *val, + int val_len, void *arg) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct rb_root *root = arg; + struct cached_page *right; + struct cached_page *left; + struct cached_page *pg; + struct cached_item *found; + struct cached_item *item; + struct rb_node *p_par; + struct rb_node *par; + struct rb_node **p_pnode; + struct rb_node **pnode; + + pg = page_rbtree_walk(sb, root, key, key, NULL, NULL, &p_par, &p_pnode); + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (found && (le64_to_cpu(found->liv.vers) >= le64_to_cpu(liv->vers))) + return 0; + + item = alloc_item(pg, key, liv, val, val_len); + if (!item) { + /* simpler split of private pages, no locking/dirty/lru */ + left = alloc_pg(sb, 0); + right = alloc_pg(sb, 0); + if (!left || !right) { + put_pg(sb, left); + put_pg(sb, right); + return -ENOMEM; + } + + scoutfs_inc_counter(sb, item_read_pages_split); + + set_split_keys(pg, left, right); + rbtree_insert(&right->node, p_par, p_pnode, root); + rbtree_replace_node(&pg->node, &left->node, root); + move_page_items(sb, cinf, pg, left, + &left->start, &right->start); + move_page_items(sb, cinf, pg, right, &right->start, NULL); + put_pg(sb, pg); + + pg = scoutfs_key_compare(key, &left->end) <= 0 ? left : right; + item = alloc_item(pg, key, liv, val, val_len); + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, + &pnode); + } + + /* if deleted a deletion item will be required */ + item->persistent = 1; + + rbtree_insert(&item->node, par, pnode, &pg->item_root); + if (found) + rbtree_erase(&found->node, &pg->item_root); + return 0; +} + +/* + * The caller couldn't find a page that contains the key we're looking + * for. We combine a block's worth of items around the key in all the + * forest btrees and store them in pages. After filtering out deletions + * and duplicates, we insert any resulting pages which don't overlap + * with existing cached pages. + * + * We only insert uncached regions because this is called with cluster + * locks held, but without locking the cache. The regions we read can + * be stale with respect to the current cache, which can be read and + * dirtied by other cluster lock holders on our node, but the cluster + * locks protect the stable items we read. + * + * There's also the exciting case where a reader can populate the cache + * with stale old persistent data which was read before another local + * cluster lock holder was able to read, dirty, write, and then shrink + * the cache. In this case the cache couldn't be cleared by lock + * invalidation because the caller is actively holding the lock. But + * shrinking could evict the cache within the held lock. So we record + * that we're an active reader in the range covered by the lock and + * shrink will refuse to reclaim any pages that intersect with our read. + */ +static int read_pages(struct super_block *sb, struct item_cache_info *cinf, + struct scoutfs_key *key, struct scoutfs_lock *lock) +{ + struct rb_root root = RB_ROOT; + struct active_reader active; + struct cached_page *right = NULL; + struct cached_page *pg; + struct cached_page *rd; + struct cached_item *item; + struct scoutfs_key start; + struct scoutfs_key end; + struct scoutfs_key inf; + struct scoutfs_key edge; + struct rb_node **pnode; + struct rb_node *par; + struct rb_node *pg_tmp; + struct rb_node *item_tmp; + int pgi; + int ret; + + /* stop shrink from freeing new clean data, would let us cache stale */ + active.start = lock->start; + active.end = lock->end; + spin_lock(&cinf->active_lock); + active_rbtree_walk(&cinf->active_root, &active.start, &active.end, + &par, &pnode); + rbtree_insert(&active.node, par, pnode, &cinf->active_root); + spin_unlock(&cinf->active_lock); + + /* start with an empty page that covers the whole lock */ + pg = alloc_pg(sb, 0); + if (!pg) { + ret = -ENOMEM; + goto out; + } + pg->start = lock->start; + pg->end = lock->end; + rbtree_insert(&pg->node, NULL, &root.rb_node, &root); + + ret = scoutfs_forest_read_items(sb, lock, key, &start, &end, + read_page_item, &root); + if (ret < 0) + goto out; + + /* clean up our read items and pages before locking */ + for_each_page_safe(&root, pg, pg_tmp) { + + /* trim any items we read outside the read range */ + scoutfs_key_set_zeros(&inf); + edge = start; + scoutfs_key_dec(&edge); + pgi = trim_page_intersection(sb, cinf, pg, NULL, &inf, &edge); + if (pgi != PGI_INSIDE) { + scoutfs_key_set_ones(&inf); + edge = end; + scoutfs_key_inc(&edge); + pgi = trim_page_intersection(sb, cinf, pg, NULL, &edge, + &inf); + } + if (pgi == PGI_INSIDE) { + rbtree_erase(&pg->node, &root); + put_pg(sb, pg); + continue; + } + + /* drop deletion items, we don't need them in the cache */ + for_each_item_safe(&pg->item_root, item, item_tmp) { + if (item->deletion) + rbtree_erase(&item->node, &pg->item_root); + } + } + +retry: + write_lock(&cinf->rwlock); + + while ((rd = first_page(&root))) { + + pg = page_rbtree_walk(sb, &cinf->pg_root, &rd->start, &rd->end, + NULL, NULL, &par, &pnode); + if (!pg) { + /* insert read pages that don't intersect */ + rbtree_erase(&rd->node, &root); + rbtree_insert(&rd->node, par, pnode, &cinf->pg_root); + lru_accessed(sb, cinf, rd); + continue; + } + + pgi = trim_page_intersection(sb, cinf, rd, right, &pg->start, + &pg->end); + if (pgi == PGI_INSIDE) { + rbtree_erase(&rd->node, &root); + put_pg(sb, rd); + + } else if (pgi == PGI_BISECT_NEEDED) { + write_unlock(&cinf->rwlock); + right = alloc_pg(sb, 0); + if (!right) { + ret = -ENOMEM; + goto out; + } + goto retry; + + } else if (pgi == PGI_BISECT) { + page_rbtree_walk(sb, &root, &right->start, &right->end, + NULL, NULL, &par, &pnode); + rbtree_insert(&right->node, par, pnode, &root); + right = NULL; + } + } + + write_unlock(&cinf->rwlock); + + ret = 0; +out: + spin_lock(&cinf->active_lock); + rbtree_erase(&active.node, &cinf->active_root); + spin_unlock(&cinf->active_lock); + + /* free any pages we left dangling on error */ + for_each_page_safe(&root, rd, pg_tmp) { + rbtree_erase(&rd->node, &root); + put_pg(sb, rd); + } + + put_pg(sb, right); + + return ret; +} + +/* + * Get a locked cached page for the caller to work with. This populates + * the cache on misses and can ensure that the locked page has enough + * room for an item allocation for the caller. Unfortunately, sparse + * doesn't seem to deal very well with the pattern of conditional lock + * acquisition. Callers manually add __acquire. + */ +static int get_cached_page(struct super_block *sb, + struct item_cache_info *cinf, + struct scoutfs_lock *lock, struct scoutfs_key *key, + bool write, bool alloc, int val_len, + struct cached_page **pg_ret) +{ + struct cached_page *pg = NULL; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + if (WARN_ON_ONCE(alloc && !write)) + return -EINVAL; + + pg = get_pcpu_page(sb, cinf, key, write); + if (pg) { + __acquire(pg->rwlock); + if (!alloc || page_has_room(pg, val_len)) + goto found; + + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + pg = NULL; + } + +retry: + read_lock(&cinf->rwlock); + + pg = page_rbtree_walk(sb, &cinf->pg_root, key, key, NULL, NULL, + &par, &pnode); + if (pg == NULL) { + read_unlock(&cinf->rwlock); + if (lock->mode == SCOUTFS_LOCK_WRITE_ONLY) + ret = cache_empty_page(sb, cinf, key, &lock->start, + &lock->end); + else + ret = read_pages(sb, cinf, key, lock); + if (ret < 0) + goto out; + goto retry; + } + + if (write) + write_lock(&pg->rwlock); + else + read_lock(&pg->rwlock); + + if (alloc && !page_has_room(pg, val_len)) { + read_unlock(&cinf->rwlock); + if (write) + write_unlock(&pg->rwlock); + else + read_unlock(&pg->rwlock); + + ret = try_split_page(sb, cinf, key, val_len); + if (ret < 0) + goto out; + goto retry; + } + + read_unlock(&cinf->rwlock); + + add_pcpu_page(sb, cinf, pg); +found: + __release(pg_rwlock); + lru_accessed(sb, cinf, pg); + ret = 0; +out: + if (ret < 0) + *pg_ret = NULL; + else + *pg_ret = pg; + return ret; +} + +static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, + int mode) +{ + if (WARN_ON_ONCE(!scoutfs_lock_protected(lock, key, mode))) + return -EINVAL; + else + return 0; +} + +/* + * Copy the cached item's value into the caller's value. The number of + * bytes copied is returned. A null val returns 0. + */ +static int copy_val(void *dst, int dst_len, void *src, int src_len) +{ + int ret; + + BUG_ON(dst_len < 0 || src_len < 0); + + ret = min(dst_len, src_len); + if (ret) + memcpy(dst, src, ret); + return ret; +} + +/* + * Find an item with the given key and copy its value to the caller. + * The amount of bytes copied is returned which can be 0 or truncated if + * the caller's buffer isn't big enough. + */ +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + int ret; + + scoutfs_inc_counter(sb, item_lookup); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ))) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, false, false, 0, &pg); + if (ret < 0) + goto out; + __acquire(&pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, NULL, NULL); + if (!item || item->deletion) + ret = -ENOENT; + else + ret = copy_val(val, val_len, item->val, item->val_len); + + read_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock) +{ + int ret; + + ret = scoutfs_item_lookup(sb, key, val, val_len, lock); + if (ret == val_len) + ret = 0; + else if (ret >= 0) + ret = -EIO; + + return ret; +} + +/* + * Return the next item starting with the given key and returning the + * last key at most. + * + * The range covered by the lock also limits the last item that can be + * returned. -ENOENT can be returned when there are no next items + * covered by the lock but there are still items before the last key + * outside of the lock. The caller needs to know to reacquire the next + * lock to continue iteration. + * + * -ENOENT is returned if there are no items between the given and last + * keys inside the range covered by the lock. + * + * The next item's key is copied to the caller's key. + * + * The next item's value is copied into the callers value. The number + * of value bytes copied is returned. The copied value can be truncated + * by the caller's value buffer length. + */ +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, void *val, int val_len, + struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_item *next; + struct cached_page *pg = NULL; + struct scoutfs_key pos; + int ret; + + scoutfs_inc_counter(sb, item_next); + + /* use the end key as the last key if it's closer */ + if (scoutfs_key_compare(&lock->end, last) < 0) + last = &lock->end; + + if (scoutfs_key_compare(key, last) > 0) { + ret = -ENOENT; + goto out; + } + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ))) + goto out; + + pos = *key; + + for (;;) { + ret = get_cached_page(sb, cinf, lock, &pos, false, false, 0, + &pg); + if (ret < 0) + goto out; + __acquire(&pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, &pos, &next, + NULL, NULL) ?: next; + while (item && scoutfs_key_compare(&item->key, last) <= 0) { + if (!item->deletion) { + *key = item->key; + ret = copy_val(val, val_len, item->val, + item->val_len); + goto unlock; + } + + item = next_item(item); + } + + if (scoutfs_key_compare(&pg->end, last) >= 0) { + ret = -ENOENT; + goto unlock; + } + + pos = pg->end; + read_unlock(&pg->rwlock); + + scoutfs_key_inc(&pos); + } + +unlock: + read_unlock(&pg->rwlock); +out: + + return ret; +} + +/* + * Mark the item dirty. Dirtying while holding a transaction pins the + * page holding the item and guarantees that the item can be deleted or + * updated (without increasing the value length) during the transaction + * without errors. + */ +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + int ret; + + scoutfs_inc_counter(sb, item_dirty); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, false, 0, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, NULL, NULL); + if (!item || item->deletion) { + ret = -ENOENT; + } else { + mark_item_dirty(sb, cinf, pg, NULL, item); + item->liv.vers = cpu_to_le64(lock->write_version); + ret = 0; + } + + write_unlock(&pg->rwlock); +out: + return ret; +} + +/* + * Create a new cached item with the given value. -EEXIST is returned + * if the item already exists. Forcing creates the item without knowldge + * of any existing items.. it doesn't read and can't return -EEXIST. + */ +static int item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock, + int mode, bool force) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *found; + struct cached_item *item; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_create); + + if ((ret = lock_safe(lock, key, mode))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, true, val_len, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!force && found && !found->deletion) { + ret = -EEXIST; + goto unlock; + } + + item = alloc_item(pg, key, &liv, val, val_len); + rbtree_insert(&item->node, par, pnode, &pg->item_root); + mark_item_dirty(sb, cinf, pg, NULL, item); + + if (found) { + item->persistent = found->persistent; + clear_item_dirty(sb, cinf, pg, found); + rbtree_erase(&found->node, &pg->item_root); + } + + if (force) + item->persistent = 1; + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + return item_create(sb, key, val, val_len, lock, + SCOUTFS_LOCK_READ, false); +} + +int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock) +{ + return item_create(sb, key, val, val_len, lock, + SCOUTFS_LOCK_WRITE_ONLY, true); +} + +/* + * Update an item with a new value. If the new value is smaller and the + * item is dirty then this is guaranteed to succeed. It can fail if the + * item doesn't exist or it gets errors reading or allocating new pages + * for a larger value. + */ +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *item; + struct cached_item *found; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_update); + + if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, true, val_len, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!found || found->deletion) { + ret = -ENOENT; + goto unlock; + } + + if (val_len <= found->val_len) { + if (val_len) + memcpy(found->val, val, val_len); + found->val_len = val_len; + found->liv.vers = liv.vers; + mark_item_dirty(sb, cinf, pg, NULL, found); + } else { + item = alloc_item(pg, key, &liv, val, val_len); + item->persistent = found->persistent; + rbtree_insert(&item->node, par, pnode, &pg->item_root); + mark_item_dirty(sb, cinf, pg, NULL, item); + + clear_item_dirty(sb, cinf, pg, found); + rbtree_erase(&found->node, &pg->item_root); + } + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +/* + * Delete an item from the cache. We can leave behind a dirty deletion + * item if there is a persistent item that needs to be overwritten. + * This can't fail if the caller knows that the item exists and it has + * been dirtied during the transaction it holds. If we're forcing then + * we're not reading the old state of the item and have to create a + * deletion item if there isn't one already cached. + */ +static int item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock, int mode, bool force) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_log_item_value liv = { + .vers = cpu_to_le64(lock->write_version), + }; + struct cached_item *item; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int ret; + + scoutfs_inc_counter(sb, item_delete); + + if ((ret = lock_safe(lock, key, mode))) + goto out; + + ret = scoutfs_forest_set_bloom_bits(sb, lock); + if (ret < 0) + goto out; + + ret = get_cached_page(sb, cinf, lock, key, true, force, 0, &pg); + if (ret < 0) + goto out; + __acquire(pg->rwlock); + + item = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + if (!force && (!item || item->deletion)) { + ret = -ENOENT; + goto unlock; + } + + if (!item) { + item = alloc_item(pg, key, &liv, NULL, 0); + rbtree_insert(&item->node, par, pnode, &pg->item_root); + } + + if (force) + item->persistent = 1; + + if (!item->persistent) { + /* can just forget items that aren't yet persistent */ + clear_item_dirty(sb, cinf, pg, item); + rbtree_erase(&item->node, &pg->item_root); + } else { + /* must emit deletion to clobber old persistent item */ + item->liv.vers = cpu_to_le64(lock->write_version); + item->liv.flags |= SCOUTFS_LOG_ITEM_FLAG_DELETION; + item->deletion = 1; + item->val_len = 0; + mark_item_dirty(sb, cinf, pg, NULL, item); + } + + ret = 0; +unlock: + write_unlock(&pg->rwlock); +out: + return ret; +} + +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + return item_delete(sb, key, lock, SCOUTFS_LOCK_WRITE, false); +} + +int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock) +{ + return item_delete(sb, key, lock, SCOUTFS_LOCK_WRITE_ONLY, true); +} + +/* + * Give a rough idea of the number of bytes that would need to be + * written to commit the current dirty items. Reporting the total item + * dirty bytes wouldn't be accurate because they're written into btree + * pages. The number of dirty pages holding the dirty items is + * comparable. This could probably use some tuning. + */ +u64 scoutfs_item_dirty_bytes(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + + return (u64)atomic_read(&cinf->dirty_pages) << PAGE_SHIFT; +} + +static int cmp_pg_start(void *priv, struct list_head *A, struct list_head *B) +{ + struct cached_page *a = list_entry(A, struct cached_page, dirty_head); + struct cached_page *b = list_entry(B, struct cached_page, dirty_head); + + return scoutfs_key_compare(&a->start, &b->start); +} + +static int cmp_item_key(void *priv, struct list_head *A, struct list_head *B) +{ + struct cached_item *a = list_entry(A, struct cached_item, dirty_head); + struct cached_item *b = list_entry(B, struct cached_item, dirty_head); + + return scoutfs_key_compare(&a->key, &b->key); +} + +/* + * Write all the dirty items into dirty blocks in the forest of btrees. + * If this succeeds then the dirty blocks can be submitted to commit + * their transaction. If this returns an error then the dirty blocks + * could have a partial set of the dirty items and result in an + * inconsistent state. The blocks should only be committed once all the + * dirty items have been written. + * + * This is called during transaction commit which prevents item writers + * from entering a transaction and dirtying items. The set of dirty + * items will be constant. + * + * But the pages that contain the dirty items can be changing. A + * neighbouring read lock can be invalidated and require bisecting a + * page, moving dirty items to a new page. That new page will be put + * after the original page on the dirty list. This will be done under + * the page rwlock and the global dirty_lock. + * + * We first sort the pages by their keys, then lock each page and copy + * its items into a private allocated singly-linked list of the items to + * dirty. Once we have that we can hand it off to the forest of btrees + * to write into items without causing any contention with other page + * users. + */ +int scoutfs_item_write_dirty(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct scoutfs_btree_item_list *first; + struct scoutfs_btree_item_list **prev; + struct scoutfs_btree_item_list *lst; + struct cached_item *item; + struct cached_page *pg; + struct page *second = NULL; + struct page *page; + LIST_HEAD(pages); + LIST_HEAD(pos); + int val_len; + int bytes; + int off; + int ret; + + /* we're relying on struct layout to prepend item value headers */ + BUILD_BUG_ON(offsetof(struct cached_item, val) != + (offsetof(struct cached_item, liv) + + member_sizeof(struct cached_item, liv))); + + if (atomic_read(&cinf->dirty_pages) == 0) + return 0; + + scoutfs_inc_counter(sb, item_write_dirty); + + /* sort page dirty list by keys */ + read_lock(&cinf->rwlock); + spin_lock(&cinf->dirty_lock); + + /* sort cached pages by key, add our pos head */ + list_sort(NULL, &cinf->dirty_list, cmp_pg_start); + list_add(&pos, &cinf->dirty_list); + + read_unlock(&cinf->rwlock); + spin_unlock(&cinf->dirty_lock); + + page = alloc_page(GFP_NOFS); + if (!page) { + ret = -ENOMEM; + goto out; + } + list_add(&page->list, &pages); + + first = NULL; + prev = &first; + off = 0; + + while (!list_empty_careful(&pos)) { + if (!second) { + second = alloc_page(GFP_NOFS); + if (!second) { + ret = -ENOMEM; + goto out; + } + list_add(&second->list, &pages); + } + + /* read lock next sorted page, we're only dirty_list user */ + + spin_lock(&cinf->dirty_lock); + pg = list_entry(pos.next, struct cached_page, dirty_head); + if (!read_trylock(&pg->rwlock)) { + spin_unlock(&cinf->dirty_lock); + cpu_relax(); + continue; + } + spin_unlock(&cinf->dirty_lock); + + list_sort(NULL, &pg->dirty_list, cmp_item_key); + + list_for_each_entry(item, &pg->dirty_list, dirty_head) { + val_len = sizeof(item->liv) + item->val_len; + bytes = offsetof(struct scoutfs_btree_item_list, + val[val_len]); + + if (off + bytes > PAGE_SIZE) { + page = second; + second = NULL; + off = 0; + } + + lst = (void *)page_address(page) + off; + off += round_up(bytes, CACHED_ITEM_ALIGN); + + lst->next = NULL; + *prev = lst; + prev = &lst->next; + + lst->key = item->key; + lst->val_len = val_len; + memcpy(lst->val, &item->liv, val_len); + } + + spin_lock(&cinf->dirty_lock); + if (pg->dirty_head.next == &cinf->dirty_list) + list_del_init(&pos); + else + list_move(&pos, &pg->dirty_head); + spin_unlock(&cinf->dirty_lock); + + read_unlock(&pg->rwlock); + } + + /* write all the dirty items into log btree blocks */ + ret = scoutfs_forest_insert_list(sb, first); +out: + list_for_each_entry_safe(page, second, &pages, list) { + list_del_init(&page->list); + __free_page(page); + } + + return ret; +} + +/* + * The caller has successfully committed all the dirty btree blocks that + * contained the currently dirty items. Clear all the dirty items and + * pages. + */ +int scoutfs_item_write_done(struct super_block *sb) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_item *tmp; + struct cached_page *pg; + +retry: + spin_lock(&cinf->dirty_lock); + + while ((pg = list_first_entry_or_null(&cinf->dirty_list, + struct cached_page, + dirty_head))) { + + if (!write_trylock(&pg->rwlock)) { + spin_unlock(&cinf->dirty_lock); + cpu_relax(); + goto retry; + } + + spin_unlock(&cinf->dirty_lock); + + list_for_each_entry_safe(item, tmp, &pg->dirty_list, + dirty_head) { + clear_item_dirty(sb, cinf, pg, item); + + /* free deletion items */ + if (item->deletion) + rbtree_erase(&item->node, &pg->item_root); + else + item->persistent = 1; + } + + write_unlock(&pg->rwlock); + + spin_lock(&cinf->dirty_lock); + } + + spin_unlock(&cinf->dirty_lock); + + return 0; +} + +/* + * Return true if the item cache covers the given range and set *dirty + * to true if any items in the cached range are dirty. + * + * This is relatively rarely called as locks are granted to make sure + * that we *don't* have existing cache covered by the lock which then + * must be inconsistent. Finding pages is the critical error case, + * under correct operation this will be a read locked walk of the page + * rbtree that doesn't find anything. + */ +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end, bool *dirty) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_item *item; + struct cached_page *pg; + struct scoutfs_key pos; + bool cached; + + cached = false; + *dirty = false; + pos = *start; + + read_lock(&cinf->rwlock); + + while (!(*dirty) && scoutfs_key_compare(&pos, end) <= 0 && + (pg = page_rbtree_walk(sb, &cinf->pg_root, &pos, end, NULL, NULL, + NULL, NULL))) { + cached = true; + + read_lock(&pg->rwlock); + read_unlock(&cinf->rwlock); + + /* the dirty list isn't sorted :/ */ + list_for_each_entry(item, &pg->dirty_list, dirty_head) { + if (!scoutfs_key_compare_ranges(&item->key, &item->key, + start, end)) { + *dirty = true; + break; + } + } + + pos = pg->end; + scoutfs_key_inc(&pos); + + read_unlock(&pg->rwlock); + read_lock(&cinf->rwlock); + } + + read_unlock(&cinf->rwlock); + + return cached; +} + +/* + * Remove the cached items in the given range. We drop pages that are + * fully inside the range and trim any pages that intersect it. This is + * being by locking for a lock that can't be used so there can't be item + * calls within the range. It can race with all our other page uses. + */ +void scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end) +{ + DECLARE_ITEM_CACHE_INFO(sb, cinf); + struct cached_page *right = NULL; + struct cached_page *pg; + struct rb_node **pnode; + struct rb_node *par; + int pgi; + + scoutfs_inc_counter(sb, item_invalidate); + +retry: + write_lock(&cinf->rwlock); + + while ((pg = page_rbtree_walk(sb, &cinf->pg_root, start, end, NULL, + NULL, &par, &pnode))) { + + scoutfs_inc_counter(sb, item_invalidate_page); + + write_lock(&pg->rwlock); + + pgi = trim_page_intersection(sb, cinf, pg, right, start, end); + BUG_ON(pgi == PGI_DISJOINT); /* walk wouldn't ret disjoint */ + + if (pgi == PGI_INSIDE) { + /* free entirely invalidated page */ + lru_remove(sb, cinf, pg); + rbtree_erase(&pg->node, &cinf->pg_root); + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + put_pg(sb, pg); + continue; + + } else if (pgi == PGI_BISECT_NEEDED) { + /* allocate so we can bisect a larger page */ + write_unlock(&cinf->rwlock); + write_unlock(&pg->rwlock); + right = alloc_pg(sb, __GFP_NOFAIL); + goto retry; + + } else if (pgi == PGI_BISECT) { + /* inv was entirely inside page, done after bisect */ + write_trylock_will_succeed(&right->rwlock); + rbtree_insert(&right->node, par, pnode, &cinf->pg_root); + write_unlock(&right->rwlock); + write_unlock(&pg->rwlock); + lru_accessed(sb, cinf, right); + right = NULL; + break; + } + + /* OLAP trimmed edge, keep searching */ + write_unlock(&pg->rwlock); + } + + write_unlock(&cinf->rwlock); + + put_pg(sb, right); +} + +/* + * Shrink the size the item cache. We're operating against the fast + * path lock ordering and we skip pages if we can't acquire locks. + * Similarly, we can run into dirty pages or pages which intersect with + * active readers that we can't shrink and also choose to skip. + */ +static int item_lru_shrink(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct item_cache_info *cinf = container_of(shrink, + struct item_cache_info, + shrinker); + struct super_block *sb = cinf->sb; + struct active_reader *active; + struct cached_page *tmp; + struct cached_page *pg; + LIST_HEAD(list); + int nr; + + if (sc->nr_to_scan == 0) + goto out; + nr = sc->nr_to_scan; + + write_lock(&cinf->rwlock); + spin_lock(&cinf->lru_lock); + + list_for_each_entry_safe(pg, tmp, &cinf->lru_list, lru_head) { + + /* can't invalidate ranges being read, reader might be stale */ + spin_lock(&cinf->active_lock); + active = active_rbtree_walk(&cinf->active_root, &pg->start, + &pg->end, NULL, NULL); + spin_unlock(&cinf->active_lock); + if (active) { + scoutfs_inc_counter(sb, item_shrink_page_reader); + continue; + } + + if (!write_trylock(&pg->rwlock)) { + scoutfs_inc_counter(sb, item_shrink_page_trylock); + continue; + } + + if (!list_empty(&pg->dirty_list)) { + scoutfs_inc_counter(sb, item_shrink_page_dirty); + write_unlock(&pg->rwlock); + continue; + } + + scoutfs_inc_counter(sb, item_shrink_page); + + __lru_remove(sb, cinf, pg); + rbtree_erase(&pg->node, &cinf->pg_root); + list_move_tail(&pg->lru_head, &list); + invalidate_pcpu_page(pg); + write_unlock(&pg->rwlock); + + if (--nr == 0) + break; + } + + write_unlock(&cinf->rwlock); + spin_unlock(&cinf->lru_lock); + + list_for_each_entry_safe(pg, tmp, &list, lru_head) { + list_del_init(&pg->lru_head); + put_pg(sb, pg); + } +out: + return min_t(unsigned long, cinf->lru_pages, INT_MAX); +} + +static int item_cpu_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + struct item_cache_info *cinf = container_of(nfb, + struct item_cache_info, + notifier); + struct super_block *sb = cinf->sb; + unsigned long cpu = (unsigned long)hcpu; + + if (action == CPU_DEAD) + drop_pcpu_pages(sb, cinf, cpu); + + return NOTIFY_OK; +} + +int scoutfs_item_setup(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache_info *cinf; + int cpu; + + cinf = kzalloc(sizeof(struct item_cache_info), GFP_KERNEL); + if (!cinf) + return -ENOMEM; + + cinf->sb = sb; + rwlock_init(&cinf->rwlock); + cinf->pg_root = RB_ROOT; + spin_lock_init(&cinf->dirty_lock); + INIT_LIST_HEAD(&cinf->dirty_list); + atomic_set(&cinf->dirty_pages, 0); + spin_lock_init(&cinf->lru_lock); + INIT_LIST_HEAD(&cinf->lru_list); + spin_lock_init(&cinf->active_lock); + cinf->active_root = RB_ROOT; + + cinf->pcpu_pages = alloc_percpu(struct item_percpu_pages); + if (!cinf->pcpu_pages) + return -ENOMEM; + + for_each_possible_cpu(cpu) + init_pcpu_pages(cinf, cpu); + + cinf->shrinker.shrink = item_lru_shrink; + cinf->shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&cinf->shrinker); + cinf->notifier.notifier_call = item_cpu_callback; + register_hotcpu_notifier(&cinf->notifier); + + sbi->item_cache_info = cinf; + return 0; +} + +/* + * There must be no more item callers at this point. + */ +void scoutfs_item_destroy(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct item_cache_info *cinf = sbi->item_cache_info; + struct cached_page *tmp; + struct cached_page *pg; + int cpu; + + if (cinf) { + BUG_ON(!RB_EMPTY_ROOT(&cinf->active_root)); + + unregister_hotcpu_notifier(&cinf->notifier); + unregister_shrinker(&cinf->shrinker); + + for_each_possible_cpu(cpu) + drop_pcpu_pages(sb, cinf, cpu); + free_percpu(cinf->pcpu_pages); + + rbtree_postorder_for_each_entry_safe(pg, tmp, &cinf->pg_root, + node) { + RB_CLEAR_NODE(&pg->node); + INIT_LIST_HEAD(&pg->lru_head); + INIT_LIST_HEAD(&pg->dirty_list); + INIT_LIST_HEAD(&pg->dirty_head); + put_pg(sb, pg); + } + + kfree(cinf); + sbi->item_cache_info = NULL; + } +} diff --git a/kmod/src/item.h b/kmod/src/item.h new file mode 100644 index 00000000..726e7bdd --- /dev/null +++ b/kmod/src/item.h @@ -0,0 +1,39 @@ +#ifndef _SCOUTFS_ITEM_H_ +#define _SCOUTFS_ITEM_H_ + +int scoutfs_item_lookup(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_lookup_exact(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_next(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_key *last, void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_dirty(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_item_create(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_create_force(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, + struct scoutfs_lock *lock); +int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, + void *val, int val_len, struct scoutfs_lock *lock); +int scoutfs_item_delete(struct super_block *sb, struct scoutfs_key *key, + struct scoutfs_lock *lock); +int scoutfs_item_delete_force(struct super_block *sb, + struct scoutfs_key *key, + struct scoutfs_lock *lock); + +u64 scoutfs_item_dirty_bytes(struct super_block *sb); +int scoutfs_item_write_dirty(struct super_block *sb); +int scoutfs_item_write_done(struct super_block *sb); +bool scoutfs_item_range_cached(struct super_block *sb, + struct scoutfs_key *start, + struct scoutfs_key *end, bool *dirty); +void scoutfs_item_invalidate(struct super_block *sb, struct scoutfs_key *start, + struct scoutfs_key *end); + +int scoutfs_item_setup(struct super_block *sb); +void scoutfs_item_destroy(struct super_block *sb); + +#endif diff --git a/kmod/src/super.c b/kmod/src/super.c index c479b484..5c722787 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -42,6 +42,7 @@ #include "quorum.h" #include "forest.h" #include "srch.h" +#include "item.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -187,6 +188,7 @@ static void scoutfs_put_super(struct super_block *sb) scoutfs_shutdown_trans(sb); scoutfs_client_destroy(sb); scoutfs_inode_destroy(sb); + scoutfs_item_destroy(sb); scoutfs_forest_destroy(sb); /* the server locks the listen address and compacts */ @@ -444,6 +446,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) scoutfs_setup_triggers(sb) ?: scoutfs_block_setup(sb) ?: scoutfs_forest_setup(sb) ?: + scoutfs_item_setup(sb) ?: scoutfs_inode_setup(sb) ?: scoutfs_data_setup(sb) ?: scoutfs_setup_trans(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 1f9776b9..8160a583 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -46,6 +46,7 @@ struct scoutfs_sb_info { struct block_info *block_info; struct forest_info *forest_info; struct srch_info *srch_info; + struct item_cache_info *item_cache_info; wait_queue_head_t trans_hold_wq; struct task_struct *trans_task; From 6bacd95aeac1a02f3f4e1f7b34386ac41997875e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 19 Aug 2020 09:28:56 -0700 Subject: [PATCH 869/920] scoutfs: fs uses item cache instead of forest Use the new item cache for all the item work in the fs instead of calling into the forest of btrees. Most of this is mechanical conversion from the _forest calls to the _item calls. The item cache no longer supports the kvec argument for describing values so all the callers pass in the value pointer and length directly. The item cache doesn't support saving items as they're deleted and later restoring them from an error unwinding path. There were only two users of this. Directory entries can easily guarantee that deletion won't fail by dirtying the items first in the item cache. Xattr updates were a little trickier. They can combine dirtying, creating, updating, and deleting to atomically switch between items that describe different versions of a multi-item value. This also fixed a bug in the srch xattrs where replacing an xattr would create a new id for the xattr and leave existing srch items referencing a now deleted id. Replacing now reuses the old id. And finally we add back in the locking and transaction item cache integration. Signed-off-by: Zach Brown --- kmod/src/data.c | 23 ++++--- kmod/src/dir.c | 64 ++++++++++---------- kmod/src/inode.c | 41 +++++-------- kmod/src/ioctl.c | 3 +- kmod/src/lock.c | 50 ++++++++++++++++ kmod/src/trans.c | 9 ++- kmod/src/xattr.c | 152 +++++++++++++++++++++++++++++++++++++---------- 7 files changed, 236 insertions(+), 106 deletions(-) diff --git a/kmod/src/data.c b/kmod/src/data.c index ed42bb8a..9360b481 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -27,11 +27,10 @@ #include "inode.h" #include "key.h" #include "data.h" -#include "kvec.h" #include "trans.h" #include "counters.h" #include "scoutfs_trace.h" -#include "forest.h" +#include "item.h" #include "ioctl.h" #include "btree.h" #include "lock.h" @@ -323,7 +322,6 @@ static int load_unpacked_extents(struct super_block *sb, u64 ino, struct rb_node *parent; struct rb_node **node; void *buf = NULL; - struct kvec val; u64 prev_blkno; bool saw_final; int size; @@ -359,13 +357,16 @@ static int load_unpacked_extents(struct super_block *sb, u64 ino, for (p = 0; !saw_final; p++) { init_packed_extent_key(&key, ino, iblock, p); - kvec_init(&val, buf, SCOUTFS_PACKEXT_MAX_BYTES); /* maybe search for next initial item, lookup more parts */ if (p == 0 && last > iblock) - ret = scoutfs_forest_next(sb, &key, &end, &val, lock); + ret = scoutfs_item_next(sb, &key, &end, buf, + SCOUTFS_PACKEXT_MAX_BYTES, + lock); else - ret = scoutfs_forest_lookup(sb, &key, &val, lock); + ret = scoutfs_item_lookup(sb, &key, buf, + SCOUTFS_PACKEXT_MAX_BYTES, + lock); if (ret < 0) { if (p == 0 && ret == -ENOENT && empty_enoent) ret = 0; @@ -475,7 +476,6 @@ static int store_packed_extents(struct super_block *sb, u64 ino, struct unpacked_extent *final; struct unpacked_extent *ext; struct scoutfs_key key; - struct kvec val; void *buf = NULL; u64 prev_blkno; u64 iblock; @@ -491,7 +491,7 @@ static int store_packed_extents(struct super_block *sb, u64 ino, if (RB_EMPTY_ROOT(&unpe->extents)) { for (p = 0; p < unpe->existing_parts; p++) { init_packed_extent_key(&key, ino, unpe->iblock, p); - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); BUG_ON(ret); /* XXX inconsistent between parts */ } unpe->existing_parts = 0; @@ -544,11 +544,10 @@ static int store_packed_extents(struct super_block *sb, u64 ino, /* store full item or after packing final extent */ init_packed_extent_key(&key, ino, unpe->iblock, p); - kvec_init(&val, buf, size); if (p < unpe->existing_parts) - ret = scoutfs_forest_update(sb, &key, &val, lock); + ret = scoutfs_item_update(sb, &key, buf, size, lock); else - ret = scoutfs_forest_create(sb, &key, &val, lock); + ret = scoutfs_item_create(sb, &key, buf, size, lock); BUG_ON(ret); /* XXX inconsistent between parts */ pe = buf; @@ -560,7 +559,7 @@ static int store_packed_extents(struct super_block *sb, u64 ino, /* delete any remaining previous part items */ for (i = p; i < unpe->existing_parts; i++) { init_packed_extent_key(&key, ino, unpe->iblock, i); - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); BUG_ON(ret); /* XXX inconsistent between parts */ } diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 89fda146..83ab48c2 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -27,8 +27,7 @@ #include "super.h" #include "trans.h" #include "xattr.h" -#include "kvec.h" -#include "forest.h" +#include "item.h" #include "lock.h" #include "hash.h" #include "counters.h" @@ -271,7 +270,6 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, struct scoutfs_key last_key; struct scoutfs_key key; struct scoutfs_dirent *dent = NULL; - struct kvec val; int ret; dent = alloc_dirent(SCOUTFS_NAME_LEN); @@ -282,10 +280,10 @@ static int lookup_dirent(struct super_block *sb, u64 dir_ino, const char *name, init_dirent_key(&key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, 0); init_dirent_key(&last_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, U64_MAX); - kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); for (;;) { - ret = scoutfs_forest_next(sb, &key, &last_key, &val, lock); + ret = scoutfs_item_next(sb, &key, &last_key, dent, + dirent_bytes(SCOUTFS_NAME_LEN), lock); if (ret < 0) break; @@ -484,7 +482,6 @@ static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, struct scoutfs_key key; struct scoutfs_key last_key; struct scoutfs_lock *dir_lock; - struct kvec val; int name_len; u64 pos; int ret; @@ -500,7 +497,6 @@ static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, init_dirent_key(&last_key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), SCOUTFS_DIRENT_LAST_POS, 0); - kvec_init(&val, dent, dirent_bytes(SCOUTFS_NAME_LEN)); ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &dir_lock); if (ret) @@ -510,7 +506,9 @@ static int KC_DECLARE_READDIR(scoutfs_readdir, struct file *file, init_dirent_key(&key, SCOUTFS_READDIR_TYPE, scoutfs_ino(inode), kc_readdir_pos(file, ctx), 0); - ret = scoutfs_forest_next(sb, &key, &last_key, &val, dir_lock); + ret = scoutfs_item_next(sb, &key, &last_key, dent, + dirent_bytes(SCOUTFS_NAME_LEN), + dir_lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -567,7 +565,6 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, struct scoutfs_dirent *dent; bool del_ent = false; bool del_rdir = false; - struct kvec val; int ret; dent = alloc_dirent(name_len); @@ -586,25 +583,27 @@ static int add_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); - kvec_init(&val, dent, dirent_bytes(name_len)); - ret = scoutfs_forest_create(sb, &ent_key, &val, dir_lock); + ret = scoutfs_item_create(sb, &ent_key, dent, dirent_bytes(name_len), + dir_lock); if (ret) goto out; del_ent = true; - ret = scoutfs_forest_create(sb, &rdir_key, &val, dir_lock); + ret = scoutfs_item_create(sb, &rdir_key, dent, dirent_bytes(name_len), + dir_lock); if (ret) goto out; del_rdir = true; - ret = scoutfs_forest_create(sb, &lb_key, &val, inode_lock); + ret = scoutfs_item_create(sb, &lb_key, dent, dirent_bytes(name_len), + inode_lock); out: if (ret < 0) { if (del_ent) - scoutfs_forest_delete_dirty(sb, &ent_key); + scoutfs_item_delete(sb, &ent_key, dir_lock); if (del_rdir) - scoutfs_forest_delete_dirty(sb, &rdir_key); + scoutfs_item_delete(sb, &rdir_key, dir_lock); } kfree(dent); @@ -626,23 +625,20 @@ static int del_entry_items(struct super_block *sb, u64 dir_ino, u64 hash, struct scoutfs_key rdir_key; struct scoutfs_key ent_key; struct scoutfs_key lb_key; - LIST_HEAD(dir_saved); - LIST_HEAD(inode_saved); int ret; init_dirent_key(&ent_key, SCOUTFS_DIRENT_TYPE, dir_ino, hash, pos); init_dirent_key(&rdir_key, SCOUTFS_READDIR_TYPE, dir_ino, pos, 0); init_dirent_key(&lb_key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, pos); - ret = scoutfs_forest_delete_save(sb, &ent_key, &dir_saved, dir_lock) ?: - scoutfs_forest_delete_save(sb, &rdir_key, &dir_saved, dir_lock) ?: - scoutfs_forest_delete_save(sb, &lb_key, &inode_saved, inode_lock); - if (ret < 0) { - scoutfs_forest_restore(sb, &dir_saved, dir_lock); - scoutfs_forest_restore(sb, &inode_saved, inode_lock); - } else { - scoutfs_forest_free_batch(sb, &dir_saved); - scoutfs_forest_free_batch(sb, &inode_saved); + ret = scoutfs_item_dirty(sb, &ent_key, dir_lock) ?: + scoutfs_item_dirty(sb, &rdir_key, dir_lock) ?: + scoutfs_item_dirty(sb, &lb_key, inode_lock); + if (ret == 0) { + ret = scoutfs_item_delete(sb, &ent_key, dir_lock) ?: + scoutfs_item_delete(sb, &rdir_key, dir_lock) ?: + scoutfs_item_delete(sb, &lb_key, inode_lock); + BUG_ON(ret); /* _dirty should have guaranteed success */ } return ret; @@ -1002,7 +998,6 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, size_t size) { struct scoutfs_key key; - struct kvec val; unsigned bytes; unsigned nr; int ret; @@ -1017,14 +1012,16 @@ static int symlink_item_ops(struct super_block *sb, int op, u64 ino, init_symlink_key(&key, ino, i); bytes = min_t(u64, size, SCOUTFS_MAX_VAL_SIZE); - kvec_init(&val, (void *)target, bytes); if (op == SYM_CREATE) - ret = scoutfs_forest_create(sb, &key, &val, lock); + ret = scoutfs_item_create(sb, &key, (void *)target, + bytes, lock); else if (op == SYM_LOOKUP) - ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, + (void *)target, bytes, + lock); else if (op == SYM_DELETE) - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); if (ret) break; @@ -1239,7 +1236,6 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, struct scoutfs_key last_key; struct scoutfs_key key; struct scoutfs_lock *lock = NULL; - struct kvec val; int len; int ret; @@ -1255,13 +1251,13 @@ int scoutfs_dir_add_next_linkref(struct super_block *sb, u64 ino, init_dirent_key(&key, SCOUTFS_LINK_BACKREF_TYPE, ino, dir_ino, dir_pos); init_dirent_key(&last_key, SCOUTFS_LINK_BACKREF_TYPE, ino, U64_MAX, U64_MAX); - kvec_init(&val, &ent->dent, dirent_bytes(SCOUTFS_NAME_LEN)); ret = scoutfs_lock_ino(sb, SCOUTFS_LOCK_READ, 0, ino, &lock); if (ret) goto out; - ret = scoutfs_forest_next(sb, &key, &last_key, &val, lock); + ret = scoutfs_item_next(sb, &key, &last_key, &ent->dent, + dirent_bytes(SCOUTFS_NAME_LEN), lock); scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); lock = NULL; if (ret < 0) diff --git a/kmod/src/inode.c b/kmod/src/inode.c index efab16fd..5d914159 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -30,8 +30,7 @@ #include "xattr.h" #include "trans.h" #include "msg.h" -#include "kvec.h" -#include "forest.h" +#include "item.h" #include "client.h" #include "cmp.h" @@ -283,7 +282,6 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, struct super_block *sb = inode->i_sb; struct scoutfs_key key; struct scoutfs_inode sinode; - struct kvec val; const u64 refresh_gen = lock->refresh_gen; int ret; @@ -299,11 +297,11 @@ int scoutfs_inode_refresh(struct inode *inode, struct scoutfs_lock *lock, return 0; init_inode_key(&key, scoutfs_ino(inode)); - kvec_init(&val, &sinode, sizeof(sinode)); mutex_lock(&si->item_mutex); if (atomic64_read(&si->last_refreshed) < refresh_gen) { - ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &sinode, + sizeof(sinode), lock); if (ret == 0) { load_inode(inode, &sinode); atomic64_set(&si->last_refreshed, refresh_gen); @@ -759,15 +757,13 @@ int scoutfs_dirty_inode_item(struct inode *inode, struct scoutfs_lock *lock) struct super_block *sb = inode->i_sb; struct scoutfs_inode sinode; struct scoutfs_key key; - struct kvec val; int ret; store_inode(&sinode, inode); - kvec_init(&val, &sinode, sizeof(sinode)); init_inode_key(&key, scoutfs_ino(inode)); - ret = scoutfs_forest_update(sb, &key, &val, lock); + ret = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock); if (!ret) trace_scoutfs_dirty_inode(inode); return ret; @@ -899,7 +895,7 @@ static int update_index_items(struct super_block *sb, scoutfs_inode_init_index_key(&ins, type, major, minor, ino); ins_lock = find_index_lock(lock_list, type, major, minor, ino); - ret = scoutfs_forest_create_force(sb, &ins, NULL, ins_lock); + ret = scoutfs_item_create_force(sb, &ins, NULL, 0, ins_lock); if (ret || !will_del_index(si, type, major, minor)) return ret; @@ -911,9 +907,9 @@ static int update_index_items(struct super_block *sb, del_lock = find_index_lock(lock_list, type, si->item_majors[type], si->item_minors[type], ino); - ret = scoutfs_forest_delete_force(sb, &del, del_lock); + ret = scoutfs_item_delete_force(sb, &del, del_lock); if (ret) { - err = scoutfs_forest_delete(sb, &ins, ins_lock); + err = scoutfs_item_delete(sb, &ins, ins_lock); BUG_ON(err); } @@ -972,7 +968,6 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, const u64 ino = scoutfs_ino(inode); struct scoutfs_key key; struct scoutfs_inode sinode; - struct kvec val; int ret; int err; @@ -988,9 +983,8 @@ void scoutfs_update_inode_item(struct inode *inode, struct scoutfs_lock *lock, BUG_ON(ret); init_inode_key(&key, ino); - kvec_init(&val, &sinode, sizeof(sinode)); - err = scoutfs_forest_update(sb, &key, &val, lock); + err = scoutfs_item_update(sb, &key, &sinode, sizeof(sinode), lock); if (err) { scoutfs_err(sb, "inode %llu update err %d", ino, err); BUG_ON(err); @@ -1265,7 +1259,7 @@ static int remove_index(struct super_block *sb, u64 ino, u8 type, u64 major, scoutfs_inode_init_index_key(&key, type, major, minor, ino); lock = find_index_lock(ind_locks, type, major, minor, ino); - ret = scoutfs_forest_delete_force(sb, &key, lock); + ret = scoutfs_item_delete_force(sb, &key, lock); if (ret == -ENOENT) ret = 0; return ret; @@ -1375,7 +1369,6 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, struct scoutfs_key key; struct scoutfs_inode sinode; struct inode *inode; - struct kvec val; int ret; inode = new_inode(sb); @@ -1405,9 +1398,8 @@ struct inode *scoutfs_new_inode(struct super_block *sb, struct inode *dir, store_inode(&sinode, inode); init_inode_key(&key, scoutfs_ino(inode)); - kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_forest_create(sb, &key, &val, lock); + ret = scoutfs_item_create(sb, &key, &sinode, sizeof(sinode), lock); if (ret) { iput(inode); return ERR_PTR(ret); @@ -1435,7 +1427,7 @@ static int remove_orphan_item(struct super_block *sb, u64 ino) init_orphan_key(&key, sbi->rid, ino); - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); if (ret == -ENOENT) ret = 0; @@ -1457,7 +1449,6 @@ static int delete_inode_items(struct super_block *sb, u64 ino) struct scoutfs_key key; LIST_HEAD(ind_locks); bool release = false; - struct kvec val; umode_t mode; u64 ind_seq; u64 size; @@ -1468,9 +1459,9 @@ static int delete_inode_items(struct super_block *sb, u64 ino) return ret; init_inode_key(&key, ino); - kvec_init(&val, &sinode, sizeof(sinode)); - ret = scoutfs_forest_lookup_exact(sb, &key, &val, lock); + ret = scoutfs_item_lookup_exact(sb, &key, &sinode, sizeof(sinode), + lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1523,7 +1514,7 @@ retry: goto out; } - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); if (ret) goto out; @@ -1592,7 +1583,7 @@ int scoutfs_scan_orphans(struct super_block *sb) init_orphan_key(&last, sbi->rid, ~0ULL); while (1) { - ret = scoutfs_forest_next(sb, &key, &last, NULL, lock); + ret = scoutfs_item_next(sb, &key, &last, NULL, 0, lock); if (ret == -ENOENT) /* No more orphan items */ break; if (ret < 0) @@ -1626,7 +1617,7 @@ int scoutfs_orphan_inode(struct inode *inode) init_orphan_key(&key, sbi->rid, scoutfs_ino(inode)); - ret = scoutfs_forest_create(sb, &key, NULL, lock); + ret = scoutfs_item_create(sb, &key, NULL, 0, lock); return ret; } diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 2bff5c48..932c4ce3 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -27,6 +27,7 @@ #include "ioctl.h" #include "super.h" #include "inode.h" +#include "item.h" #include "forest.h" #include "data.h" #include "client.h" @@ -110,7 +111,7 @@ static long scoutfs_ioc_walk_inodes(struct file *file, unsigned long arg) for (nr = 0; nr < walk.nr_entries; ) { - ret = scoutfs_forest_next(sb, &key, &last_key, NULL, lock); + ret = scoutfs_item_next(sb, &key, &last_key, NULL, 0, lock); if (ret < 0 && ret != -ENOENT) break; diff --git a/kmod/src/lock.c b/kmod/src/lock.c index fba8fef5..19413458 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -34,6 +34,7 @@ #include "client.h" #include "data.h" #include "xattr.h" +#include "item.h" /* * scoutfs uses a lock service to manage item cache consistency between @@ -195,6 +196,8 @@ retry: ino++; } } + + scoutfs_item_invalidate(sb, &lock->start, &lock->end); } return ret; @@ -570,6 +573,50 @@ static void queue_inv_work(struct lock_info *linfo) mod_delayed_work(linfo->workq, &linfo->inv_dwork, 0); } +/* + * The given lock is processing a received a grant response. Trigger a + * bug if the cache is inconsistent. + * + * We only have two modes that can create dirty items. We can't have + * dirty items when transitioning from write_only to write because the + * writer can't trust the cached items in the cache for reading. And we + * don't currently transition directly from write to write_only, we + * first go through null. So if we have dirty items as we're granted a + * mode it's always incorrect. + * + * And we can't have cached items that we're going to use for reading if + * the previous mode didn't allow reading. + * + * Inconsistencies have come from all sorts of bugs: invalidation missed + * items, the cache was populated outside of locking coverage, lock + * holders performed the wrong item operations under their lock, + * overlapping locks, out of order granting or invalidating, etc. + */ +static void bug_on_inconsistent_grant_cache(struct super_block *sb, + struct scoutfs_lock *lock, + int old_mode, int new_mode) +{ + bool cached; + bool dirty; + + cached = scoutfs_item_range_cached(sb, &lock->start, &lock->end, + &dirty); + if (dirty || + (cached && (!lock_mode_can_read(old_mode) || + !lock_mode_can_read(new_mode)))) { + scoutfs_err(sb, "granted lock item cache inconsistency, cached %u dirty %u old_mode %d new_mode %d: start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %u waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u", + cached, dirty, old_mode, new_mode, SK_ARG(&lock->start), + SK_ARG(&lock->end), lock->refresh_gen, lock->mode, + lock->waiters[SCOUTFS_LOCK_READ], + lock->waiters[SCOUTFS_LOCK_WRITE], + lock->waiters[SCOUTFS_LOCK_WRITE_ONLY], + lock->users[SCOUTFS_LOCK_READ], + lock->users[SCOUTFS_LOCK_WRITE], + lock->users[SCOUTFS_LOCK_WRITE_ONLY]); + BUG(); + } +} + /* * Each lock has received a grant response message from the server. * @@ -608,6 +655,9 @@ static void lock_grant_worker(struct work_struct *work) if (lock->mode != nl->old_mode) continue; + bug_on_inconsistent_grant_cache(sb, lock, nl->old_mode, + nl->new_mode); + if (!lock_mode_can_read(nl->old_mode) && lock_mode_can_read(nl->new_mode)) { lock->refresh_gen = diff --git a/kmod/src/trans.c b/kmod/src/trans.c index bd06503a..af659bd9 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -28,6 +28,7 @@ #include "radix.h" #include "block.h" #include "msg.h" +#include "item.h" #include "scoutfs_trace.h" /* @@ -169,7 +170,8 @@ void scoutfs_trans_write_func(struct work_struct *work) trace_scoutfs_trans_write_func(sb, scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); - if (!scoutfs_block_writer_has_dirty(sb, &tri->wri)) { + if (!scoutfs_block_writer_has_dirty(sb, &tri->wri) && + !scoutfs_item_dirty_bytes(sb)) { if (sbi->trans_deadline_expired) { /* * If we're not writing data then we only advance the @@ -192,9 +194,11 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX this all needs serious work for dealing with errors */ ret = (s = "data submit", scoutfs_inode_walk_writeback(sb, true)) ?: + (s = "item dirty", scoutfs_item_write_dirty(sb)) ?: (s = "meta write", scoutfs_block_writer_write(sb, &tri->wri)) ?: (s = "data wait", scoutfs_inode_walk_writeback(sb, false)) ?: (s = "commit log trees", commit_btrees(sb)) ?: + scoutfs_item_write_done(sb) ?: (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: (s = "get log trees", scoutfs_trans_get_log_trees(sb)); out: @@ -364,8 +368,7 @@ static bool acquired_hold(struct super_block *sb, vals = tri->reserved_vals + cnt->vals; /* XXX arbitrarily limit to 8 meg transactions */ - if (scoutfs_block_writer_dirty_bytes(sb, &tri->wri) >= - (8 * 1024 * 1024)) { + if (scoutfs_item_dirty_bytes(sb) >= (8 * 1024 * 1024)) { scoutfs_inc_counter(sb, trans_commit_full); queue_trans_work(sbi); goto out; diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index d7c9d112..4666eecd 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -20,7 +20,7 @@ #include "inode.h" #include "key.h" #include "super.h" -#include "kvec.h" +#include "item.h" #include "forest.h" #include "trans.h" #include "xattr.h" @@ -160,7 +160,6 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key, { struct super_block *sb = inode->i_sb; struct scoutfs_key last; - struct kvec val; u8 last_part; int total; u8 part; @@ -183,8 +182,9 @@ static int get_next_xattr(struct inode *inode, struct scoutfs_key *key, for (;;) { key->skx_part = part; - kvec_init(&val, (void *)xat + total, bytes - total); - ret = scoutfs_forest_next(sb, key, &last, &val, lock); + ret = scoutfs_item_next(sb, key, &last, + (void *)xat + total, bytes - total, + lock); if (ret < 0) { /* XXX corruption, ran out of parts */ if (ret == -ENOENT && part > 0) @@ -260,7 +260,6 @@ static int create_xattr_items(struct inode *inode, u64 id, struct scoutfs_key key; unsigned int part_bytes; unsigned int total; - struct kvec val; int ret; init_xattr_key(&key, scoutfs_ino(inode), @@ -271,12 +270,13 @@ static int create_xattr_items(struct inode *inode, u64 id, while (total < bytes) { part_bytes = min_t(unsigned int, bytes - total, SCOUTFS_XATTR_MAX_PART_SIZE); - kvec_init(&val, (void *)xat + total, part_bytes); - ret = scoutfs_forest_create(sb, &key, &val, lock); + ret = scoutfs_item_create(sb, &key, + (void *)xat + total, part_bytes, + lock); if (ret) { while (key.skx_part-- > 0) - scoutfs_forest_delete_dirty(sb, &key); + scoutfs_item_delete(sb, &key, lock); break; } @@ -288,24 +288,114 @@ static int create_xattr_items(struct inode *inode, u64 id, } /* - * Delete and save the items that make up the given xattr. If this - * returns an error then the deleted and saved items are left on the - * list for the caller to restore. + * Delete the items that make up the given xattr. If this returns an + * error then no items have been deleted. */ static int delete_xattr_items(struct inode *inode, u32 name_hash, u64 id, - u8 nr_parts, struct list_head *list, - struct scoutfs_lock *lock) + u8 nr_parts, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; struct scoutfs_key key; - int ret; + int ret = 0; + int i; init_xattr_key(&key, scoutfs_ino(inode), name_hash, id); - do { - ret = scoutfs_forest_delete_save(sb, &key, list, lock); - } while (ret == 0 && ++key.skx_part < nr_parts); + /* dirty additional existing old items */ + for (i = 1; i < nr_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_dirty(sb, &key, lock); + if (ret) + goto out; + } + for (i = 0; i < nr_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_delete(sb, &key, lock); + if (ret) + break; + } +out: + return ret; +} + +/* + * The caller needs to overwrite existing old xattr items with new + * items. We carefully stage the changes so that we can always unwind + * to the original items if we return an error. Both items have at + * least one part. Either the old or new can have more parts. We dirty + * and create first because we can always unwind those. We delete last + * after dirtying so that it can't fail and we don't have to restore the + * deleted items. + */ +static int change_xattr_items(struct inode *inode, u64 id, + struct scoutfs_xattr *new_xat, + unsigned int new_bytes, u8 new_parts, + u8 old_parts, struct scoutfs_lock *lock) +{ + struct super_block *sb = inode->i_sb; + struct scoutfs_key key; + int last_created = -1; + int bytes; + int off; + int i; + int ret; + + init_xattr_key(&key, scoutfs_ino(inode), + xattr_name_hash(new_xat->name, new_xat->name_len), id); + + /* dirty existing old items */ + for (i = 0; i < old_parts; i++) { + key.skx_part = i; + ret = scoutfs_item_dirty(sb, &key, lock); + if (ret) + goto out; + } + + /* create any new items past the old */ + for (i = old_parts; i < new_parts; i++) { + off = i * SCOUTFS_XATTR_MAX_PART_SIZE; + bytes = min_t(unsigned int, new_bytes - off, + SCOUTFS_XATTR_MAX_PART_SIZE); + + key.skx_part = i; + ret = scoutfs_item_create(sb, &key, (void *)new_xat + off, + bytes, lock); + if (ret) + goto out; + + last_created = i; + } + + /* update dirtied overlapping existing items, last partial first */ + for (i = old_parts - 1; i >= 0; i--) { + off = i * SCOUTFS_XATTR_MAX_PART_SIZE; + bytes = min_t(unsigned int, new_bytes - off, + SCOUTFS_XATTR_MAX_PART_SIZE); + + key.skx_part = i; + ret = scoutfs_item_update(sb, &key, (void *)new_xat + off, + bytes, lock); + /* only last partial can fail, then we unwind created */ + if (ret < 0) + goto out; + } + + /* delete any dirtied old items past new */ + for (i = new_parts; i < old_parts; i++) { + key.skx_part = i; + scoutfs_item_delete(sb, &key, lock); + } + + ret = 0; +out: + if (ret < 0) { + /* delete any newly created items */ + for (i = old_parts; i <= last_created; i++) { + key.skx_part = i; + scoutfs_item_delete(sb, &key, lock); + } + } return ret; } @@ -407,7 +497,6 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct prefix_tags tgs; bool undo_srch = false; LIST_HEAD(ind_locks); - LIST_HEAD(saved); u8 found_parts; unsigned int bytes; u64 ind_seq; @@ -478,7 +567,10 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, /* prepare our xattr */ if (value) { - id = si->next_xattr_id++; + if (found_parts) + id = le64_to_cpu(key.skx_id); + else + id = si->next_xattr_id++; xat->name_len = name_len; xat->val_len = cpu_to_le16(size); memcpy(xat->name, name, name_len); @@ -511,18 +603,17 @@ retry: undo_srch = true; } - ret = 0; - if (found_parts) + if (found_parts && value) + ret = change_xattr_items(inode, id, xat, bytes, + xattr_nr_parts(xat), found_parts, lck); + else if (found_parts) ret = delete_xattr_items(inode, le64_to_cpu(key.skx_name_hash), le64_to_cpu(key.skx_id), found_parts, - &saved, lck); - if (value && ret == 0) + lck); + else ret = create_xattr_items(inode, id, xat, bytes, lck); - if (ret < 0) { - scoutfs_forest_restore(sb, &saved, lck); + if (ret < 0) goto release; - } - scoutfs_forest_free_batch(sb, &saved); /* XXX do these want i_mutex or anything? */ inode_inc_iversion(inode); @@ -665,7 +756,6 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct prefix_tags tgs; bool release = false; unsigned int bytes; - struct kvec val; u64 hash; int ret; @@ -681,8 +771,8 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, init_xattr_key(&last, ino, U32_MAX, U64_MAX); for (;;) { - kvec_init(&val, (void *)xat, bytes); - ret = scoutfs_forest_next(sb, &key, &last, &val, lock); + ret = scoutfs_item_next(sb, &key, &last, (void *)xat, bytes, + lock); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -698,7 +788,7 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, break; release = true; - ret = scoutfs_forest_delete(sb, &key, lock); + ret = scoutfs_item_delete(sb, &key, lock); if (ret < 0) break; From 12067e99aba28c63774ded94ea54442bbe9b3b0c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 19 Aug 2020 10:02:33 -0700 Subject: [PATCH 870/920] scoutfs: remove item granular work from forest Now that the item cache is bearing the load of high frequency item calls, we can remove all the item granular work that the forest was trying to do. The item cache amortizes the cost of the forest so its remaining methods can go straight to the btrees and don't need complicated state to reduce the overhead of item calls. Signed-off-by: Zach Brown --- kmod/src/counters.h | 16 - kmod/src/forest.c | 1306 +------------------------------------- kmod/src/forest.h | 36 -- kmod/src/lock.c | 4 +- kmod/src/lock.h | 4 +- kmod/src/scoutfs_trace.h | 188 +----- 6 files changed, 43 insertions(+), 1511 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 6c97923c..82d34e46 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -56,27 +56,11 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ - EXPAND_COUNTER(forest_add_root) \ EXPAND_COUNTER(forest_bloom_fail) \ EXPAND_COUNTER(forest_bloom_pass) \ - EXPAND_COUNTER(forest_clear_lock) \ - EXPAND_COUNTER(forest_delete) \ - EXPAND_COUNTER(forest_insert) \ - EXPAND_COUNTER(forest_iter) \ - EXPAND_COUNTER(forest_lookup) \ - EXPAND_COUNTER(forest_read_lock_log) \ - EXPAND_COUNTER(forest_read_lock_rotated) \ - EXPAND_COUNTER(forest_refresh_bloom_roots) \ - EXPAND_COUNTER(forest_refresh_dirty_log) \ - EXPAND_COUNTER(forest_refresh_skip_log) \ EXPAND_COUNTER(forest_read_items) \ EXPAND_COUNTER(forest_roots_next_hint) \ - EXPAND_COUNTER(forest_roots_lock) \ - EXPAND_COUNTER(forest_roots_server) \ - EXPAND_COUNTER(forest_saw_stale) \ EXPAND_COUNTER(forest_set_bloom_bits) \ - EXPAND_COUNTER(forest_set_dirtied) \ - EXPAND_COUNTER(forest_trigger_refresh) \ EXPAND_COUNTER(item_clear_dirty) \ EXPAND_COUNTER(item_create) \ EXPAND_COUNTER(item_delete) \ diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 284f1f1c..2d53a1d9 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -12,7 +12,7 @@ */ #include #include -#include +#include #include #include "super.h" @@ -41,34 +41,21 @@ * for the item. Readers check log btrees for the most recent version * that it should use. * - * From a mount's perspective, the only btree whose blocks are actively - * changing is the mount's own log btree in memory. Every other btree - * it reads is stable (but could be stale) on disk. They don't need to - * be locked, but we might have to retry reads if we hit blocks that - * have been overwritten. + * The item cache reads items in bulk from stable btrees, and writes a + * transaction's worth of dirty items into the item log btree. * * Log btrees are typically very sparse. It would be wasteful for * readers to read every log btree looking for an item. Each log btree * contains a bloom filter keyed on the starting key of locks. This * lets lock holders quickly eliminate log trees that cannot contain - * keys protected by their lock. Since reads have to be done under - * locks, we cache the list of trees that could contain items in the - * lock. - * - * The list of roots in the locks can get out of date. Item - * modification in the current transactoin requires that the list - * contain the dirty log tree. Transaction commits mean that we can - * read from the stale log tree instead of the dirty one. And getting - * stale block reads from any of the trees means we need to rebuild the - * list from scratch. + * keys protected by their lock. */ struct forest_info { - struct rw_semaphore rwsem; + struct mutex mutex; struct scoutfs_radix_allocator *alloc; struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; - atomic64_t commit_seq; struct mutex srch_mutex; struct scoutfs_srch_file srch_file; @@ -78,198 +65,20 @@ struct forest_info { #define DECLARE_FOREST_INFO(sb, name) \ struct forest_info *name = SCOUTFS_SB(sb)->forest_info -struct forest_root { - struct list_head entry; - struct scoutfs_btree_root item_root; - u64 rid; - u64 nr; - u8 our_dirty:1; -}; - struct forest_refs { struct scoutfs_btree_ref fs_ref; struct scoutfs_btree_ref logs_ref; } __packed; +/* initialize some refs that initially aren't equal */ +#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ + struct forest_refs a = {{cpu_to_le64(0),}}; \ + struct forest_refs b = {{cpu_to_le64(1),}} + struct forest_bloom_nrs { unsigned int nrs[SCOUTFS_FOREST_BLOOM_NRS]; }; -struct forest_lock_private { - u64 last_refreshed; - struct rw_semaphore rwsem; - unsigned int used_lock_roots:1; - struct list_head roots; - u64 set_bloom_nr; - atomic64_t dirtied_cseq; - u64 refreshed_cseq; - u64 refreshed_dirtied; -}; - -static struct forest_lock_private *get_lock_private(struct scoutfs_lock *lock) -{ - struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); - - if (lpriv == NULL) { - lpriv = kzalloc(sizeof(struct forest_lock_private), GFP_NOFS); - if (lpriv) { - init_rwsem(&lpriv->rwsem); - INIT_LIST_HEAD(&lpriv->roots); - atomic64_set(&lpriv->dirtied_cseq, 0); - - if (cmpxchg(&lock->forest_private, NULL, lpriv) != NULL) - kfree(lpriv); - lpriv = lock->forest_private; - } - } - - return lpriv; -} - -static bool is_fs_root(struct forest_root *fr) -{ - return fr->rid == 0 && fr->nr == 0; -} - -/* - * We can be sure that we have the most recent version of an item if we - * have it write locked with the version of the lock. There can be no - * greater versions of the item in the system. - */ -static bool is_write_locked_version(struct scoutfs_lock *lock, u64 vers) -{ - return lock->mode == SCOUTFS_LOCK_WRITE && - vers == lock->write_version; -} - -static void free_roots(struct forest_lock_private *lpriv) -{ - struct forest_root *fr; - struct forest_root *tmp; - - list_for_each_entry_safe(fr, tmp, &lpriv->roots, entry) { - list_del_init(&fr->entry); - kfree(fr); - } -} - -/* - * Add a *copy* of the root to the list of roots to read. If our_dirty - * is set then later readers will acquire the lock to serialize writers - * and update the root from the current dirty version. - */ -static int add_root(struct super_block *sb, struct scoutfs_lock *lock, - struct forest_lock_private *lpriv, - struct scoutfs_btree_root *item_root, u64 rid, u64 nr, - bool our_dirty) -{ - struct forest_root *fr; - - BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); - - fr = kmalloc(sizeof(struct forest_root), GFP_NOFS); - if (!fr) - return -ENOMEM; - - fr->item_root = *item_root; - fr->rid = rid; - fr->nr = nr; - fr->our_dirty = !!our_dirty; - list_add_tail(&fr->entry, &lpriv->roots); - - scoutfs_inc_counter(sb, forest_add_root); - trace_scoutfs_forest_add_root(sb, &lock->start, fr->rid, fr->nr, - le64_to_cpu(fr->item_root.ref.blkno), - le64_to_cpu(fr->item_root.ref.seq)); - - return 0; -} - -/* - * The caller has dirtied the current log tree and still holds the - * transaction. We need to make sure that future reads know to check - * this dirty tree in particular. The tree can be committed (and - * rotated out!) before the next refresh so we use a commit sequence - * which will identify that it can find this tree either still dirty or - * can trust that it will find an item for it. - */ -static void set_dirtied_cseq(struct super_block *sb, struct forest_info *finf, - struct scoutfs_lock *lock, - struct forest_lock_private *lpriv) -{ - u64 cseq = atomic64_read(&finf->commit_seq); - - BUG_ON(!rwsem_is_locked(&finf->rwsem)); - - if (atomic64_read(&lpriv->dirtied_cseq) != cseq) { - atomic64_set(&lpriv->dirtied_cseq, cseq); - scoutfs_inc_counter(sb, forest_set_dirtied); - - trace_scoutfs_forest_set_dirtied(sb, &lock->start, - le64_to_cpu(finf->our_log.rid), - le64_to_cpu(finf->our_log.nr), - cseq); - } -} - -/* - * This is called by the locking code while it's excluding users of the - * lock. - */ -void scoutfs_forest_clear_lock(struct super_block *sb, - struct scoutfs_lock *lock) -{ - struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); - - if (lpriv) { - scoutfs_inc_counter(sb, forest_clear_lock); - free_roots(lpriv); - kfree(lpriv); - lock->forest_private = NULL; - } -} - -/* - * Usually we're reading from persistent btrees that won't be changing. - * But refresh can add a root that references the current dirty log root - * so that readers can see items which haven't yet been committed. Once - * we get the lock we make sure to give the forest root the current - * version of the tree which could have changed since it was added. - * Acquiring the lock also serializes commit responses updating the log - * and we can see if a commit has rotated in a new tree and we need to - * refresh the list. - */ -static int read_lock_forest_root(struct super_block *sb, - struct forest_info *finf, - struct forest_lock_private *lpriv, - struct forest_root *fr) -{ - int ret = 0; - - BUG_ON(!rwsem_is_locked(&lpriv->rwsem)); - - if (fr->our_dirty) { - down_read(&finf->rwsem); - if (fr->nr == le64_to_cpu(finf->our_log.nr)) { - scoutfs_inc_counter(sb, forest_read_lock_log); - fr->item_root = finf->our_log.item_root; - } else { - scoutfs_inc_counter(sb, forest_read_lock_rotated); - up_read(&finf->rwsem); - ret = -EUCLEAN; - } - } - - return ret; -} - -static void read_unlock_forest_root(struct forest_info *finf, - struct forest_root *fr) -{ - if (fr->our_dirty) - up_read(&finf->rwsem); -} - static void calc_bloom_nrs(struct forest_bloom_nrs *bloom, struct scoutfs_key *key) { @@ -306,792 +115,6 @@ static struct scoutfs_block *read_bloom_ref(struct super_block *sb, return bl; } -/* - * Empty the list of btrees currently stored in the lock and walk the - * current fs image looking for btrees whose bloom filters indicate that - * the btree may contain items covered by the lock. - * - * We ensure that the our log btree is always first and that the fs - * btree is always last because those positions offer short-circuiting - * optimizations. - * - * This doesn't deal with rereading stale blocks itself.. it returns - * ESTALE to the caller who already has to deal with retrying stale - * blocks from their btree reads. We give them the refs we read so that - * they can identify persistent stale block errors that come from - * corruption. - * - * Because we're starting all the reads from stable refs from the - * server, this will not see any dirty blocks we have in memory. We - * don't have to lock any of the btree reads. It also won't find the - * currently dirty version of our log btree. Writers record the version - * of the current dirty log tree that must be added if it's still dirty - * when we refresh. - */ -static int refresh_bloom_roots(struct super_block *sb, - struct scoutfs_lock *lock, - struct forest_refs *refs) -{ - DECLARE_FOREST_INFO(sb, finf); - struct forest_lock_private *lpriv = ACCESS_ONCE(lock->forest_private); - struct scoutfs_net_roots roots; - struct scoutfs_log_trees_val ltv; - struct scoutfs_log_trees *lt; - SCOUTFS_BTREE_ITEM_REF(iref); - struct forest_bloom_nrs bloom; - struct scoutfs_bloom_block *bb; - struct scoutfs_block *bl; - struct scoutfs_key key; - u64 our_rid = 0; - u64 our_nr = 0; - u64 dirtied; - u64 cseq; - int ret; - int i; - - scoutfs_inc_counter(sb, forest_refresh_bloom_roots); - - memset(refs, 0, sizeof(*refs)); - - down_write(&lpriv->rwsem); - - /* empty the list so no one iterates until someone's added */ - free_roots(lpriv); - - /* make sure readers see writer's in-memory dirty items */ - cseq = atomic64_read(&finf->commit_seq); - dirtied = atomic64_read(&lpriv->dirtied_cseq); - - if (dirtied == cseq) { - down_read(&finf->rwsem); - cseq = atomic64_read(&finf->commit_seq); - dirtied = atomic64_read(&lpriv->dirtied_cseq); - if (dirtied == cseq) { - scoutfs_inc_counter(sb, forest_refresh_dirty_log); - lt = &finf->our_log; - our_rid = le64_to_cpu(lt->rid); - our_nr = le64_to_cpu(lt->nr); - /* root be updated before reads, but nice to trace */ - ret = add_root(sb, lock, lpriv, <->item_root, - our_rid, our_nr, true); - } else { - ret = 0; - /* must get roots from network to see committed */ - lpriv->used_lock_roots = 1; - } - up_read(&finf->rwsem); - if (ret < 0) - goto out; - } - - trace_scoutfs_forest_refresh_seqs(sb, &lock->start, our_rid, our_nr, - dirtied, lpriv->refreshed_dirtied, - cseq, lpriv->refreshed_cseq); - - /* first use the lock's constant roots, then sample newer roots */ - if (!lpriv->used_lock_roots) { - lpriv->used_lock_roots = 1; - roots = lock->roots; - scoutfs_inc_counter(sb, forest_roots_lock); - } else { - ret = scoutfs_client_get_roots(sb, &roots); - if (ret) - goto out; - scoutfs_inc_counter(sb, forest_roots_server); - } - - trace_scoutfs_forest_using_roots(sb, &roots.fs_root, &roots.logs_root); - refs->fs_ref = roots.fs_root.ref; - refs->logs_ref = roots.logs_root.ref; - - calc_bloom_nrs(&bloom, &lock->start); - - scoutfs_key_init_log_trees(&key, 0, 0); - for (;; scoutfs_key_inc(&key)) { - - ret = scoutfs_btree_next(sb, &roots.logs_root, &key, &iref); - if (ret == 0) { - if (iref.val_len == sizeof(ltv)) { - key = *iref.key; - memcpy(<v, iref.val, iref.val_len); - } else { - ret = -EIO; - } - scoutfs_btree_put_iref(&iref); - } - if (ret < 0) { - if (ret == -ENOENT) { - ret = 0; - break; - } - goto out; - } - - if (ltv.bloom_ref.blkno == 0) - continue; - - bl = read_bloom_ref(sb, <v.bloom_ref); - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - goto out; - } - bb = bl->data; - - for (i = 0; i < ARRAY_SIZE(bloom.nrs); i++) { - if (!test_bit_le(bloom.nrs[i], bb->bits)) - break; - } - - scoutfs_block_put(sb, bl); - - trace_scoutfs_forest_bloom_search(sb, &lock->start, - le64_to_cpu(key.sklt_rid), - le64_to_cpu(key.sklt_nr), - le64_to_cpu(ltv.bloom_ref.blkno), - le64_to_cpu(ltv.bloom_ref.seq), - i); - - /* one of the bloom bits wasn't set */ - if (i != ARRAY_SIZE(bloom.nrs)) { - scoutfs_inc_counter(sb, forest_bloom_fail); - continue; - } - - scoutfs_inc_counter(sb, forest_bloom_pass); - - /* we've added our dirty log, skip old committed versions */ - if (le64_to_cpu(key.sklt_rid) == our_rid && - le64_to_cpu(key.sklt_nr) == our_nr) { - scoutfs_inc_counter(sb, forest_refresh_skip_log); - continue; - } - - ret = add_root(sb, lock, lpriv, <v.item_root, - le64_to_cpu(key.sklt_rid), - le64_to_cpu(key.sklt_nr), false); - if (ret < 0) - goto out; - } - - /* always add final fs tree last */ - ret = add_root(sb, lock, lpriv, &roots.fs_root, 0, 0, false); - if (ret < 0) - goto out; - - lpriv->refreshed_cseq = cseq; - lpriv->refreshed_dirtied = dirtied; - lpriv->last_refreshed = lock->refresh_gen; - - ret = 0; - -out: - if (ret < 0) - free_roots(lpriv); - - up_write(&lpriv->rwsem); - return ret; -} - -/* initialize some refs that initially aren't equal */ -#define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ - struct forest_refs a = {{cpu_to_le64(0),}}; \ - struct forest_refs b = {{cpu_to_le64(1),}} - -/* - * If the caller got our magic errnos we refresh the roots and return - * -EAGAIN so they retry. If we get -ESTALE from block reference - * inconsistency with the same root refs then it's consistent corruption - * and we return an error. We pass through all other errnos that aren't - * our magic retry errnos. - */ -static int refresh_check(struct super_block *sb, struct scoutfs_lock *lock, - struct forest_refs *prev_refs, - struct forest_refs *refs, int err) -{ - int ret; - - /* don't want to get in a loop passing eagain through, not expected */ - if (WARN_ON_ONCE(err == -EAGAIN)) - return -EINVAL; - - if (!(err == -ESTALE || err == -EUCLEAN)) - return err; - - if (err == -ESTALE) { - scoutfs_inc_counter(sb, forest_saw_stale); - if (memcmp(prev_refs, refs, sizeof(*refs)) == 0) - return -EIO; - } - *prev_refs = *refs; - - ret = refresh_bloom_roots(sb, lock, refs); - if (ret == 0 || ret == -ESTALE) - ret = -EAGAIN; - - return ret; -} - -/* - * Iterate over all the roots that could contain items covered by the - * caller's lock. The caller starts iteration by passing in a NULL fr. - * We return -EUCLEAN if the caller needs to refresh the bloom roots. - * We use the lock's refresh gen to find out when the lock was - * invalidated and the contents of the trees could have changed. - * - * The commit_seqs are keeping the list of roots in sync with our log - * root. As writers modify it we make sure we have a root that will - * lock and check our in-memory dirty log tre. Once that's committed we - * refresh again so we read the stable committed version without locks. - */ -static int for_each_forest_root(struct super_block *sb, - struct scoutfs_lock *lock, - struct forest_info *finf, - struct forest_lock_private *lpriv, - struct forest_root **fr) -{ - u64 cseq = atomic64_read(&finf->commit_seq); - u64 dirtied = atomic64_read(&lpriv->dirtied_cseq); - - if (WARN_ON_ONCE(!rwsem_is_locked(&lpriv->rwsem))) - return -EIO; - - if (list_empty(&lpriv->roots) || - lock->refresh_gen != lpriv->last_refreshed || - dirtied > lpriv->refreshed_dirtied || - (dirtied == lpriv->refreshed_cseq && - cseq > lpriv->refreshed_cseq)) { - scoutfs_inc_counter(sb, forest_trigger_refresh); - trace_scoutfs_forest_trigger_refresh(sb, - &lock->start, - !!list_empty(&lpriv->roots), - lock->refresh_gen, - lpriv->last_refreshed, - dirtied, lpriv->refreshed_dirtied, - cseq, lpriv->refreshed_cseq); - return -EUCLEAN; - } - - if (*fr == NULL) - *fr = list_prepare_entry((*fr), &lpriv->roots, entry); - - list_for_each_entry_continue((*fr), &lpriv->roots, entry) - return 0; - - *fr = NULL; - return 0; -} - -/* - * We fake 1 as the version for the fs items. The least valid log item - * version is also 1, but we guarantee that we check the log trees first - * so they'll always be found before the fs items. - */ -static u64 item_vers(struct forest_root *fr, void *val) -{ - struct scoutfs_log_item_value *liv; - - if (is_fs_root(fr)) - return 1; - - liv = val; - return le64_to_cpu(liv->vers); -} - -static bool item_flags(struct forest_root *fr, void *val) -{ - struct scoutfs_log_item_value *liv; - - if (is_fs_root(fr)) - return 0; - - liv = val; - return liv->flags; -} - -static bool item_is_deletion(struct forest_root *fr, void *val) -{ - return item_flags(fr, val) & SCOUTFS_LOG_ITEM_FLAG_DELETION; -} - -/* just a little helper to slim down all the call sites */ -static int lock_safe(struct scoutfs_lock *lock, struct scoutfs_key *key, - int mode) -{ - if (WARN_ON_ONCE(!scoutfs_lock_protected(lock, key, mode))) - return -EINVAL; - else - return 0; -} - -/* - * Copy the cached item's value into the caller's single value vector. - * The number of bytes that fit in the vec and were copied is returned. - * A null val returns 0. Items in log trees have a value header that - * needs to be skipped. - */ -static int copy_val(struct forest_root *fr, struct kvec *val, void *item_val, - int item_val_len) -{ - void *val_start = item_val; - unsigned int val_len = item_val_len; - int ret; - - if (!is_fs_root(fr)) { - val_start += sizeof(struct scoutfs_log_item_value); - val_len -= sizeof(struct scoutfs_log_item_value); - } - - if (val) { - ret = min_t(size_t, val_len, val->iov_len); - memcpy(val->iov_base, val_start, ret); - } else { - ret = 0; - } - - return ret; -} - -int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - DECLARE_FOREST_INFO(sb, finf); - DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); - struct forest_lock_private *lpriv; - SCOUTFS_BTREE_ITEM_REF(iref); - struct forest_root *fr; - u64 found_vers; - u64 vers; - int ret; - int err; - - scoutfs_inc_counter(sb, forest_lookup); - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) - goto out; - - lpriv = get_lock_private(lock); - if (!lpriv) { - ret = -ENOMEM; - goto out; - } - -retry: - down_read(&lpriv->rwsem); - - found_vers = 0; - ret = -ENOENT; - fr = NULL; - - while (!(err = for_each_forest_root(sb, lock, finf, lpriv, &fr)) && fr){ - - /* done if we found log items before fs root */ - if (found_vers > 0 && is_fs_root(fr)) - break; - - err = read_lock_forest_root(sb, finf, lpriv, fr); - if (err < 0) - break; - err = scoutfs_btree_lookup(sb, &fr->item_root, key, &iref); - if (err < 0) - read_unlock_forest_root(finf, fr); - if (err == -ENOENT) - continue; - if (err < 0) - break; - - vers = item_vers(fr, iref.val); - - if (vers > found_vers) { - found_vers = vers; - - if (item_is_deletion(fr, iref.val)) - ret = -ENOENT; - else - ret = copy_val(fr, val, iref.val, iref.val_len); - } - scoutfs_btree_put_iref(&iref); - read_unlock_forest_root(finf, fr); - - /* done if we have the most recent locked dirty version */ - if (is_write_locked_version(lock, vers)) - break; - } - - up_read(&lpriv->rwsem); - - err = refresh_check(sb, lock, &prev_refs, &refs, err); - if (err == -EAGAIN) - goto retry; - if (err < 0) - ret = err; -out: - return ret; -} - -int scoutfs_forest_lookup_exact(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock) -{ - int ret; - - ret = scoutfs_forest_lookup(sb, key, val, lock); - if (ret == val->iov_len) - ret = 0; - else if (ret >= 0) - ret = -EIO; - - return ret; -} - -static inline void forest_iter_set_max(struct scoutfs_key *key, bool forward) -{ - if (forward) - scoutfs_key_set_ones(key); - else - scoutfs_key_set_zeros(key); -} - -static inline void forest_iter_set_min(struct scoutfs_key *key, bool forward) -{ - return forest_iter_set_max(key, !forward); -} - -static inline void forest_iter_key_advance(struct scoutfs_key *key, bool forward) -{ - if (forward) - scoutfs_key_inc(key); - else - scoutfs_key_dec(key); -} - -static inline int forest_iter_key_cmp(struct scoutfs_key *a, - struct scoutfs_key *b, bool forward) -{ - int cmp = scoutfs_key_compare(a, b); - if (cmp == 0 || forward) - return cmp; - return -cmp; -} - -/* returns true if a is before b in the direction of iteration */ -static inline bool forest_iter_key_before(struct scoutfs_key *a, - struct scoutfs_key *b, bool forward) -{ - int cmp = scoutfs_key_compare(a, b); - - return forward ? cmp < 0 : cmp > 0; -} - -/* returns true if a is before or equal to b in the direction of iteration */ -static inline bool forest_iter_key_within(struct scoutfs_key *a, - struct scoutfs_key *b, bool forward) -{ - int cmp = scoutfs_key_compare(a, b); - - return forward ? cmp <= 0 : cmp >= 0; -} - -static inline int forest_iter_btree_search(struct super_block *sb, - struct scoutfs_btree_root *root, - struct scoutfs_key *key, - struct scoutfs_btree_item_ref *iref, - bool forward) -{ - if (forward) - return scoutfs_btree_next(sb, root, key, iref); - else - return scoutfs_btree_prev(sb, root, key, iref); -} - -struct forest_iter_pos { - struct rb_node node; - struct forest_root *fr; - struct scoutfs_key key; - u64 vers; - bool deletion; - void *val; - int val_len; -}; - -static struct forest_iter_pos *first_iter_pos(struct rb_root *root) -{ - return rb_entry_safe(rb_first(root), struct forest_iter_pos, node); -} - -static struct forest_iter_pos *next_iter_pos(struct forest_iter_pos *ip) -{ - return rb_entry_safe(rb_next(&ip->node), struct forest_iter_pos, node); -} - -/* - * Sort root iter positions first by missing items, then by key in the - * direction if iteration, and then by reverse version. Thus the first - * iter_pos in the rbtree is either a root that needs to check the next - * item, a deletion that removes all older versions of the key, or is - * the item that iteration should return. - */ -static int cmp_iter_pos(struct forest_iter_pos *a, struct forest_iter_pos *b, - bool fwd) -{ - int cmp; - - if (a->vers == 0) - return -1; - if (b->vers == 0) - return 1; - - cmp = forest_iter_key_cmp(&a->key, &b->key, fwd); - if (cmp) - return cmp; - - return scoutfs_cmp_u64s(b->vers, a->vers); -} - -/* - * There's a sneaky subtlety here. The fs items have a fake verison of - * 1 which can equal a log tree version of 1. We always iterate over - * the fs root last so we try to insert the fake fs item last. It will - * compare equal to the version and will be inserted to the right of the - * existing log item. - */ -static void insert_iter_pos(struct forest_iter_pos *ins, struct rb_root *root, - bool fwd) -{ - struct rb_node **node = &root->rb_node; - struct rb_node *parent = NULL; - struct forest_iter_pos *ip; - int cmp; - - while (*node) { - parent = *node; - ip = container_of(*node, struct forest_iter_pos, node); - - cmp = cmp_iter_pos(ins, ip, fwd); - if (cmp < 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, root); -} - -/* - * clear the version and re-insert the iter_pos so that the next - * iteration will search for the next item in the root. - */ -static void advance_iter_pos(struct forest_iter_pos *ip, struct rb_root *root, - bool fwd) -{ - ip->vers = 0; - forest_iter_key_advance(&ip->key, fwd); - kfree(ip->val); - ip->val = NULL; - rb_erase(&ip->node, root); - insert_iter_pos(ip, root, fwd); -} - -static void destroy_iter_pos(struct forest_iter_pos *ip, struct rb_root *root) -{ - kfree(ip->val); - rb_erase(&ip->node, root); - kfree(ip); -} - -/* - * Iterate over items in all the roots looking for the next least - * non-deletion item in the direction of iteration. The roots can have - * any combination of item keys, versions, and deletions so we have to - * be very careful. - * - * We store the next item in each root in a node in an rbtree. The - * nodes are sorted by needing to be read, key, then reverse version. - * The first node in the rbtree is always a root to search, a deletion - * item to remove, or the item that iteration should return. - * - * btree locking prevents us from holding references to the items in all - * the roots so we store copies of the items in the nodes. - */ -static int forest_iter(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *end, struct kvec *val, - struct scoutfs_lock *lock, bool fwd) -{ - DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); - struct forest_lock_private *lpriv; - DECLARE_FOREST_INFO(sb, finf); - SCOUTFS_BTREE_ITEM_REF(iref); - struct rb_root iter_root = RB_ROOT; - struct scoutfs_key found_key; - struct forest_iter_pos *nip; - struct forest_iter_pos *ip; - struct forest_root *fr; - u64 found_vers = 0; - int found_ret = 0; - int ret; - - scoutfs_inc_counter(sb, forest_iter); - scoutfs_key_set_zeros(&found_key); - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_READ)) < 0) - goto out; - - /* use the end key as the end key if it's closer to reduce compares */ - if (forest_iter_key_before(&lock->end, end, fwd)) - end = &lock->end; - - /* convenience to avoid searching if caller iterates past their end */ - if (!forest_iter_key_within(key, end, fwd)) { - ret = -ENOENT; - goto out; - } - - lpriv = get_lock_private(lock); - if (!lpriv) { - ret = -ENOMEM; - goto out; - } - -retry: - down_read(&lpriv->rwsem); - - /* initialize iter position for each tree */ - fr = NULL; - while (!(ret = for_each_forest_root(sb, lock, finf, lpriv, &fr)) && fr){ - ip = kmalloc(sizeof(struct forest_iter_pos), GFP_NOFS); - if (!ip) { - ret = -ENOMEM; - goto unlock; - } - - ip->fr = fr; - ip->key = *key; - ip->vers = 0; - ip->deletion = false; - ip->val = NULL; - insert_iter_pos(ip, &iter_root, fwd); - } - if (ret < 0) - goto unlock; - - scoutfs_key_set_zeros(&found_key); - found_vers = 0; - found_ret = -ENOENT; - - /* search until we hit the end key on all roots */ - while ((ip = first_iter_pos(&iter_root))) { - fr = ip->fr; - - /* search for the next item in the root */ - if (ip->vers == 0) { - ret = read_lock_forest_root(sb, finf, lpriv, fr); - if (ret < 0) - goto unlock; - ret = forest_iter_btree_search(sb, &fr->item_root, - &ip->key, &iref, fwd); - if (ret < 0) - read_unlock_forest_root(finf, fr); - if (ret == -ENOENT) { - destroy_iter_pos(ip, &iter_root); - continue; - } - if (ret < 0) - goto unlock; - - ip->key = *iref.key; - ip->vers = item_vers(fr, iref.val); - ip->deletion = item_is_deletion(fr, iref.val); - - trace_scoutfs_forest_iter_search(sb, fr->rid, fr->nr, - ip->vers, - item_flags(fr, iref.val), - &ip->key); - - if (!forest_iter_key_within(&ip->key, end, fwd)) { - /* root is done if next is past end */ - destroy_iter_pos(ip, &iter_root); - } else { - kfree(ip->val); - ip->val = kmalloc(iref.val_len, GFP_NOFS); - if (!ip->val) { - ret = -ENOMEM; - } else { - /* copy item and re-sort its node */ - memcpy(ip->val, iref.val, iref.val_len); - ip->val_len = iref.val_len; - rb_erase(&ip->node, &iter_root); - insert_iter_pos(ip, &iter_root, fwd); - } - } - - scoutfs_btree_put_iref(&iref); - read_unlock_forest_root(finf, fr); - - if (ret < 0) - goto unlock; - continue; - } - - /* deletions remove all earlier versions and themselves */ - if (ip->deletion) { - while ((nip = next_iter_pos(ip)) && - !scoutfs_key_compare(&ip->key, &nip->key)) { - advance_iter_pos(nip, &iter_root, fwd); - } - advance_iter_pos(ip, &iter_root, fwd); - continue; - } - - /* use the first non-deletion across all roots */ - found_key = ip->key; - found_vers = ip->vers; - found_ret = copy_val(ip->fr, val, ip->val, ip->val_len); - break; - } - - ret = 0; -unlock: - up_read(&lpriv->rwsem); - - /* destroy_ rebalances so postorder traversal could skip nodes */ - for (ip = first_iter_pos(&iter_root); - ip && (nip = next_iter_pos(ip), 1); - ip = nip) { - destroy_iter_pos(ip, &iter_root); - } - - ret = refresh_check(sb, lock, &prev_refs, &refs, ret); - if (ret == -EAGAIN) - goto retry; - -out: - trace_scoutfs_forest_iter_ret(sb, key, end, fwd, ret, - found_vers, found_ret, &found_key); - - if (ret == 0) { - ret = found_ret; - /* _next/_prev interfaces modify caller's key :/ */ - if (ret >= 0) - *key = found_key; - } - - return ret; -} - -int scoutfs_forest_next(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *last, struct kvec *val, - struct scoutfs_lock *lock) -{ - return forest_iter(sb, key, last, val, lock, true); -} - -int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *first, struct kvec *val, - struct scoutfs_lock *lock) -{ - return forest_iter(sb, key, first, val, lock, false); -} - /* * This is an unlocked iteration across all the btrees to find a hint at * the next key that the caller could read. It's used to find out what @@ -1120,8 +143,9 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, bool have_next; int ret; -retry: scoutfs_inc_counter(sb, forest_roots_next_hint); + +retry: ret = scoutfs_client_get_roots(sb, &roots); if (ret) goto out; @@ -1253,11 +277,11 @@ int scoutfs_forest_read_items(struct super_block *sb, int ret; int i; + scoutfs_inc_counter(sb, forest_read_items); calc_bloom_nrs(&bloom, &lock->start); roots = lock->roots; retry: - scoutfs_inc_counter(sb, forest_read_items); ret = scoutfs_client_get_roots(sb, &roots); if (ret) goto out; @@ -1347,19 +371,18 @@ out: /* * Make sure that the bloom bits for the lock's start key are all set in * the current log's bloom block. We record the nr of our log tree in - * the lock so that we only try to cow and set the bits once per tree. + * the lock so that we only try to cow and set the bits once per tree + * across multiple commits as long as the lock isn't purged. * - * The caller already gets the big finf write rwsem lock to modify the - * dirty log btree, might as well use it to protect the bloom ref and - * the lpriv field. We'll need finer grained locking once the btrees - * get block locks. + * This is using a coarse mutex to serialize cowing the block. It could + * be much finer grained, but it's infrequent. We'll keep an eye on if + * it gets expensive enough to warrant fixing. */ -static int set_lock_bloom_bits(struct super_block *sb, - struct scoutfs_lock *lock, u64 nr) +int scoutfs_forest_set_bloom_bits(struct super_block *sb, + struct scoutfs_lock *lock) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_FOREST_INFO(sb, finf); - struct forest_lock_private *lpriv; struct scoutfs_block *new_bl = NULL; struct scoutfs_block *bl = NULL; struct scoutfs_bloom_block *bb; @@ -1367,24 +390,21 @@ static int set_lock_bloom_bits(struct super_block *sb, struct forest_bloom_nrs bloom; int nr_set = 0; u64 blkno; + u64 nr; int ret; int err; int i; - BUG_ON(!rwsem_is_locked(&finf->rwsem)); - - lpriv = get_lock_private(lock); - if (!lpriv) { - ret = -ENOMEM; - goto out; - } + nr = le64_to_cpu(finf->our_log.nr); /* our rid is constant */ - if (lpriv->set_bloom_nr == nr) { + if (atomic64_read(&lock->forest_bloom_nr) == nr) { ret = 0; goto out; } + mutex_lock(&finf->mutex); + scoutfs_inc_counter(sb, forest_set_bloom_bits); calc_bloom_nrs(&bloom, &lock->start); @@ -1394,7 +414,7 @@ static int set_lock_bloom_bits(struct super_block *sb, bl = read_bloom_ref(sb, ref); if (IS_ERR(bl)) { ret = PTR_ERR(bl); - goto out; + goto unlock; } bb = bl->data; } @@ -1403,7 +423,7 @@ static int set_lock_bloom_bits(struct super_block *sb, ret = scoutfs_radix_alloc(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) - goto out; + goto unlock; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { @@ -1411,7 +431,7 @@ static int set_lock_bloom_bits(struct super_block *sb, blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); - goto out; + goto unlock; } if (bl) { @@ -1452,21 +472,15 @@ static int set_lock_bloom_bits(struct super_block *sb, le64_to_cpu(finf->our_log.bloom_ref.seq), nr_set); - lpriv->set_bloom_nr = nr; + atomic64_set(&lock->forest_bloom_nr, nr); ret = 0; +unlock: + mutex_unlock(&finf->mutex); out: scoutfs_block_put(sb, bl); return ret; } -int scoutfs_forest_set_bloom_bits(struct super_block *sb, - struct scoutfs_lock *lock) -{ - DECLARE_FOREST_INFO(sb, finf); - - return set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); -} - int scoutfs_forest_insert_list(struct super_block *sb, struct scoutfs_btree_item_list *lst) { @@ -1476,253 +490,6 @@ int scoutfs_forest_insert_list(struct super_block *sb, &finf->our_log.item_root, lst); } -/* - * The btree code takes a single value buffer. When we're working with - * the log btrees we want to add a log item value metadata header. In - * the interest of expedience we're just allocating a new contiguous - * buffer that prepends the header. We could make the btree ops take - * vectored values or we could make all btree items have the metadata. - */ -static struct kvec *alloc_log_item_value(struct kvec *val, __u8 flags, - struct scoutfs_lock *lock) -{ - struct scoutfs_log_item_value *liv; - struct kvec *kv; - unsigned int val_len = val ? val->iov_len : 0; - - kv = kmalloc(sizeof(*kv) + sizeof(*liv) + val_len, GFP_NOFS); - if (kv) { - liv = (void *)kv + sizeof(*kv); - - kv->iov_base = liv; - kv->iov_len = sizeof(*liv) + val_len; - - liv->vers = cpu_to_le64(lock->write_version); - liv->flags = flags; - if (val) - memcpy(liv->data, val->iov_base, val->iov_len); - } - - return kv; -} - -/* - * Create a new dirty item. Can return -EEXIST if the item already - * exists or will just force createion the caller's item, overwriting - * any existing item. We can be overwriting an existing deletion item - * in our log root. - */ -static int forest_insert(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock, - bool check_eexist, bool check_enoent, bool could_read) -{ - DECLARE_FOREST_INFO(sb, finf); - struct forest_lock_private *lpriv; - struct kvec *iv = NULL; - int ret; - - scoutfs_inc_counter(sb, forest_insert); - - lpriv = get_lock_private(lock); - if (!lpriv) { - ret = -ENOMEM; - goto out; - } - - if (check_eexist || check_enoent) { - ret = scoutfs_forest_lookup(sb, key, NULL, lock); - if (ret == 0 && check_eexist) { - ret = -EEXIST; - goto out; - } - if (ret == -ENOENT) { - if (check_enoent) - goto out; - ret = 0; - } - if (ret < 0) - goto out; - - } - - iv = alloc_log_item_value(val, 0, lock); - if (iv == NULL) { - ret = -ENOMEM; - goto out; - } - - down_write(&finf->rwsem); - - ret = set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); - if (ret < 0) - goto unlock; - - ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, - &finf->our_log.item_root, key, - iv->iov_base, iv->iov_len); - if (ret == 0 && could_read) - set_dirtied_cseq(sb, finf, lock, lpriv); -unlock: - up_write(&finf->rwsem); - kfree(iv); - -out: - return ret; -} - -/* - * Insert an item, returning -EEXIST if it already exists. - */ -int scoutfs_forest_create(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - int ret; - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) - return ret; - - return forest_insert(sb, key, val, lock, true, false, true); -} - -/* - * Insert an item, ignoring whether it exists or not. - */ -int scoutfs_forest_create_force(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock) -{ - int ret; - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) - return ret; - - return forest_insert(sb, key, val, lock, false, false, false); -} - -/* - * Overwrite an existing item, possibly changing its value length, - * returning -ENOENT if it didn't already exist. - */ -int scoutfs_forest_update(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock) -{ - int ret; - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) - return ret; - - return forest_insert(sb, key, val, lock, false, true, true); -} - -/* XXX not yet supported, idea is btree op that only uses dirty blocks */ -int scoutfs_forest_delete_dirty(struct super_block *sb, - struct scoutfs_key *key) -{ - BUG(); - return 0; -} - -static int forest_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock, bool check_enoent, - bool could_read) -{ - DECLARE_FOREST_INFO(sb, finf); - struct forest_lock_private *lpriv; - struct scoutfs_log_item_value liv; - int ret; - - scoutfs_inc_counter(sb, forest_delete); - - lpriv = get_lock_private(lock); - if (!lpriv) { - ret = -ENOMEM; - goto out; - } - - if (check_enoent) { - ret = scoutfs_forest_lookup(sb, key, NULL, lock); - if (ret < 0) - goto out; - } - - liv.vers = cpu_to_le64(lock->write_version); - liv.flags = SCOUTFS_LOG_ITEM_FLAG_DELETION; - - down_write(&finf->rwsem); - - ret = set_lock_bloom_bits(sb, lock, le64_to_cpu(finf->our_log.nr)); - if (ret < 0) - goto unlock; - - ret = scoutfs_btree_force(sb, finf->alloc, finf->wri, - &finf->our_log.item_root, - key, &liv, sizeof(liv)); - if (ret == 0 && could_read) - set_dirtied_cseq(sb, finf, lock, lpriv); -unlock: - up_write(&finf->rwsem); -out: - return ret; -} - -/* - * Delete an item from the forest of btrees. This interface returns - * -ENOENT if the item doesn't exist (may already be deleted). We have - * to first read from the forest to see if it exists. If we get -ENOENT - * it might be because it exists in our log tree. We force our deletion - * item regardless of the current state of the item in our log tree. - */ -int scoutfs_forest_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock) -{ - int ret; - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE)) < 0) - return ret; - - return forest_delete(sb, key, lock, true, true); -} - -/* - * Like deletion, but we don't have to read the current item to return - * -ENOENT. We just force a deletion item. - */ -int scoutfs_forest_delete_force(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_lock *lock) -{ - int ret; - - if ((ret = lock_safe(lock, key, SCOUTFS_LOCK_WRITE_ONLY)) < 0) - return ret; - - return forest_delete(sb, key, lock, false, false); -} - -/* XXX not supported, just for initial demo */ -int scoutfs_forest_delete_save(struct super_block *sb, - struct scoutfs_key *key, - struct list_head *list, - struct scoutfs_lock *lock) -{ - int ret = scoutfs_forest_delete(sb, key, lock); - BUG_ON(ret != 0); - return ret; -} - -/* XXX not supported, just for initial demo */ -int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, - struct scoutfs_lock *lock) -{ - BUG(); - return 0; -} - -/* XXX not supported, just for initial demo */ -void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list) -{ -} - /* * Add a srch entry to the current transaction's log file. It will be * committed in a transaction along with the dirty btree blocks that @@ -1756,7 +523,7 @@ void scoutfs_forest_init_btrees(struct super_block *sb, { DECLARE_FOREST_INFO(sb, finf); - down_write(&finf->rwsem); + mutex_lock(&finf->mutex); finf->alloc = alloc; finf->wri = wri; @@ -1767,7 +534,6 @@ void scoutfs_forest_init_btrees(struct super_block *sb, finf->our_log.bloom_ref = lt->bloom_ref; finf->our_log.rid = lt->rid; finf->our_log.nr = lt->nr; - atomic64_inc(&finf->commit_seq); finf->srch_file = lt->srch_file; WARN_ON_ONCE(finf->srch_bl); /* commiting should have put the block */ finf->srch_bl = NULL; @@ -1775,10 +541,9 @@ void scoutfs_forest_init_btrees(struct super_block *sb, trace_scoutfs_forest_init_our_log(sb, le64_to_cpu(lt->rid), le64_to_cpu(lt->nr), le64_to_cpu(lt->item_root.ref.blkno), - le64_to_cpu(lt->item_root.ref.seq), - atomic64_read(&finf->commit_seq)); + le64_to_cpu(lt->item_root.ref.seq)); - up_write(&finf->rwsem); + mutex_unlock(&finf->mutex); } /* @@ -1816,9 +581,8 @@ int scoutfs_forest_setup(struct super_block *sb) } /* the finf fields will be setup as we open a transaction */ - init_rwsem(&finf->rwsem); + mutex_init(&finf->mutex); mutex_init(&finf->srch_mutex); - atomic64_set(&finf->commit_seq, 0); sbi->forest_info = finf; ret = 0; diff --git a/kmod/src/forest.h b/kmod/src/forest.h index b480c971..6d0c0c8c 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -13,41 +13,8 @@ typedef int (*scoutfs_forest_item_cb)(struct super_block *sb, struct scoutfs_log_item_value *liv, void *val, int val_len, void *arg); -int scoutfs_forest_lookup(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_forest_lookup_exact(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_forest_next(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *last, struct kvec *val, - struct scoutfs_lock *lock); int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_key *next); -int scoutfs_forest_prev(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *first, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_forest_create(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_forest_create_force(struct super_block *sb, - struct scoutfs_key *key, struct kvec *val, - struct scoutfs_lock *lock); -int scoutfs_forest_update(struct super_block *sb, struct scoutfs_key *key, - struct kvec *val, struct scoutfs_lock *lock); -int scoutfs_forest_delete_dirty(struct super_block *sb, - struct scoutfs_key *key); -int scoutfs_forest_delete(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_lock *lock); -int scoutfs_forest_delete_force(struct super_block *sb, - struct scoutfs_key *key, - struct scoutfs_lock *lock); -int scoutfs_forest_delete_save(struct super_block *sb, - struct scoutfs_key *key, - struct list_head *list, - struct scoutfs_lock *lock); -int scoutfs_forest_restore(struct super_block *sb, struct list_head *list, - struct scoutfs_lock *lock); -void scoutfs_forest_free_batch(struct super_block *sb, struct list_head *list); - int scoutfs_forest_read_items(struct super_block *sb, struct scoutfs_lock *lock, struct scoutfs_key *key, @@ -67,9 +34,6 @@ void scoutfs_forest_init_btrees(struct super_block *sb, void scoutfs_forest_get_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); -void scoutfs_forest_clear_lock(struct super_block *sb, - struct scoutfs_lock *lock); - int scoutfs_forest_setup(struct super_block *sb); void scoutfs_forest_destroy(struct super_block *sb); diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 19413458..7775b1ad 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -21,7 +21,6 @@ #include "super.h" #include "lock.h" -#include "forest.h" #include "scoutfs_trace.h" #include "msg.h" #include "cmp.h" @@ -230,7 +229,6 @@ static void lock_free(struct lock_info *linfo, struct scoutfs_lock *lock) BUG_ON(!list_empty(&lock->shrink_head)); BUG_ON(!list_empty(&lock->cov_list)); - scoutfs_forest_clear_lock(sb, lock); kfree(lock); } @@ -265,6 +263,8 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, init_waitqueue_head(&lock->waitq); lock->mode = SCOUTFS_LOCK_NULL; + atomic64_set(&lock->forest_bloom_nr, 0); + trace_scoutfs_lock_alloc(sb, lock); return lock; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index f659e50b..cb9bb3d6 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -46,8 +46,8 @@ struct scoutfs_lock { struct scoutfs_tseq_entry tseq_entry; - /* the forest btree code stores data per lock */ - struct forest_lock_private *forest_private; + /* the forest tracks which log tree last saw bloom bit updates */ + atomic64_t forest_bloom_nr; }; struct scoutfs_lock_coverage { diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index eeafd395..f26525bd 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2104,138 +2104,15 @@ TRACE_EVENT(scoutfs_forest_using_roots, __entry->logs_blkno, __entry->logs_seq) ); -TRACE_EVENT(scoutfs_forest_add_root, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, - u64 nr, u64 blkno, u64 seq), - TP_ARGS(sb, key, rid, nr, blkno, seq), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - __field(__u64, b_rid) - __field(__u64, nr) - __field(__u64, blkno) - __field(__u64, seq) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - __entry->b_rid = rid; - __entry->nr = nr; - __entry->blkno = blkno; - __entry->seq = seq; - ), - TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu blkno %llu seq %llx", - SCSB_TRACE_ARGS, sk_trace_args(key), - __entry->b_rid, __entry->nr, __entry->blkno, __entry->seq) -); - -TRACE_EVENT(scoutfs_forest_set_dirtied, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, - u64 nr, u64 cseq), - TP_ARGS(sb, key, rid, nr, cseq), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - __field(__u64, b_rid) - __field(__u64, nr) - __field(__u64, cseq) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - __entry->b_rid = rid; - __entry->nr = nr; - __entry->cseq = cseq; - ), - TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu cseq %llu", - SCSB_TRACE_ARGS, sk_trace_args(key), - __entry->b_rid, __entry->nr, __entry->cseq) -); - -TRACE_EVENT(scoutfs_forest_trigger_refresh, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, - bool empty_roots, u64 refresh_gen, u64 last_refreshed, - u64 dirtied_cseq, u64 refreshed_dirtied, - u64 commit_seq, u64 refreshed_cseq), - TP_ARGS(sb, key, empty_roots, refresh_gen, last_refreshed, - dirtied_cseq, refreshed_dirtied, commit_seq, refreshed_cseq), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - __field(int, empty_roots) - __field(__u64, refresh_gen) - __field(__u64, last_refreshed) - __field(__u64, dirtied_cseq) - __field(__u64, refreshed_dirtied) - __field(__u64, commit_seq) - __field(__u64, refreshed_cseq) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - __entry->empty_roots = !!empty_roots; - __entry->refresh_gen = refresh_gen; - __entry->last_refreshed = last_refreshed; - __entry->dirtied_cseq = dirtied_cseq; - __entry->refreshed_dirtied = refreshed_dirtied; - __entry->commit_seq = commit_seq; - __entry->refreshed_cseq = refreshed_cseq; - ), - TP_printk(SCSBF" key "SK_FMT" empty %u refg %llu last_refg %llu dirt %llu refdir %llu cseq %llu refcseq %llu", - SCSB_TRACE_ARGS, sk_trace_args(key), - __entry->empty_roots, - __entry->refresh_gen, - __entry->last_refreshed, - __entry->dirtied_cseq, - __entry->refreshed_dirtied, - __entry->commit_seq, - __entry->refreshed_cseq) -); - -TRACE_EVENT(scoutfs_forest_refresh_seqs, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, u64 rid, - u64 nr, u64 dirtied_cseq, u64 refreshed_dirtied, - u64 commit_seq, u64 refreshed_cseq), - TP_ARGS(sb, key, rid, nr, dirtied_cseq, refreshed_dirtied, commit_seq, - refreshed_cseq), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - __field(__u64, b_rid) - __field(__u64, nr) - __field(__u64, dirtied_cseq) - __field(__u64, refreshed_dirtied) - __field(__u64, commit_seq) - __field(__u64, refreshed_cseq) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - __entry->b_rid = rid; - __entry->nr = nr; - __entry->dirtied_cseq = dirtied_cseq; - __entry->refreshed_dirtied = refreshed_dirtied; - __entry->commit_seq = commit_seq; - __entry->refreshed_cseq = refreshed_cseq; - ), - TP_printk(SCSBF" key "SK_FMT" rid %016llx nr %llu dirt %llu refdir %llu cseq %llu refcseq %llu", - SCSB_TRACE_ARGS, sk_trace_args(key), __entry->b_rid, - __entry->nr, __entry->dirtied_cseq, - __entry->refreshed_dirtied, __entry->commit_seq, - __entry->refreshed_cseq) -); - TRACE_EVENT(scoutfs_forest_init_our_log, - TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 blkno, u64 seq, - u64 cseq), - TP_ARGS(sb, rid, nr, blkno, seq, cseq), + TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 blkno, u64 seq), + TP_ARGS(sb, rid, nr, blkno, seq), TP_STRUCT__entry( SCSB_TRACE_FIELDS __field(__u64, b_rid) __field(__u64, nr) __field(__u64, blkno) __field(__u64, seq) - __field(__u64, cseq) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); @@ -2243,67 +2120,10 @@ TRACE_EVENT(scoutfs_forest_init_our_log, __entry->nr = nr; __entry->blkno = blkno; __entry->seq = seq; - __entry->cseq = cseq; ), - TP_printk(SCSBF" rid %016llx nr %llu blkno %llu seq %llx cseq %llu", + TP_printk(SCSBF" rid %016llx nr %llu blkno %llu seq %llx", SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, - __entry->blkno, __entry->seq, __entry->cseq) -); - -TRACE_EVENT(scoutfs_forest_iter_search, - TP_PROTO(struct super_block *sb, u64 rid, u64 nr, u64 vers, - u8 flags, struct scoutfs_key *key), - TP_ARGS(sb, rid, nr, vers, flags, key), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, b_rid) - __field(__u64, nr) - __field(__u64, vers) - __field(__u8, flags) - sk_trace_define(key) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->b_rid = rid; - __entry->nr = nr; - __entry->vers = vers; - __entry->flags = flags; - sk_trace_assign(key, key); - ), - TP_printk(SCSBF" rid %016llx nr %llu vers %llu flags %x key "SK_FMT, - SCSB_TRACE_ARGS, __entry->b_rid, __entry->nr, - __entry->vers, __entry->flags, sk_trace_args(key)) -); - -TRACE_EVENT(scoutfs_forest_iter_ret, - TP_PROTO(struct super_block *sb, struct scoutfs_key *key, - struct scoutfs_key *end, bool forward, int ret, - u64 found_vers, int found_ret, struct scoutfs_key *found), - TP_ARGS(sb, key, end, forward, ret, found_vers, found_ret, found), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - sk_trace_define(key) - sk_trace_define(end) - __field(char, forward) - __field(int, ret) - __field(__u64, found_vers) - __field(int, found_ret) - sk_trace_define(found) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - sk_trace_assign(key, key); - sk_trace_assign(end, end); - __entry->forward = !!forward; - __entry->ret = ret; - __entry->found_vers = found_vers; - __entry->found_ret = found_ret; - sk_trace_assign(found, found); - ), - TP_printk(SCSBF" key "SK_FMT" end "SK_FMT" fwd %u ret %d fv %llu fc %d f "SK_FMT, - SCSB_TRACE_ARGS, sk_trace_args(key), sk_trace_args(end), - __entry->forward, __entry->ret, __entry->found_vers, - __entry->found_ret, sk_trace_args(found)) + __entry->blkno, __entry->seq) ); DECLARE_EVENT_CLASS(scoutfs_block_class, From ae97ffd6fc447414c0995df29d06293e5aae77b2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 19 Aug 2020 10:04:26 -0700 Subject: [PATCH 871/920] scoutfs: remove unused kvec.h We've removed the last use of kvecs to describe item values. Signed-off-by: Zach Brown --- kmod/src/kvec.h | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 kmod/src/kvec.h diff --git a/kmod/src/kvec.h b/kmod/src/kvec.h deleted file mode 100644 index 9341f724..00000000 --- a/kmod/src/kvec.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _SCOUTFS_KVEC_H_ -#define _SCOUTFS_KVEC_H_ - -#include - -static inline void kvec_init(struct kvec *kv, void *base, size_t len) -{ - kv->iov_base = base; - kv->iov_len = len; -} - -#endif From b28acdf9042c803e1e79cf93bc2417a3431fa0ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 21 Aug 2020 15:40:02 -0700 Subject: [PATCH 872/920] scoutfs: use larger percpu_counter batch The percpu_counter library merges the per-cpu counters with a shared count when the per-cpu counter gets larger than a certain value. The default is very small, so we often end up taking a shared lock to update the count. Use a larger batch so that we take the lock less often. Signed-off-by: Zach Brown --- kmod/src/counters.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 82d34e46..a9658bf3 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -201,11 +201,21 @@ struct scoutfs_counters { pcpu <= &SCOUTFS_SB(sb)->counters->LAST_COUNTER; \ pcpu++) -#define scoutfs_inc_counter(sb, which) \ - percpu_counter_inc(&SCOUTFS_SB(sb)->counters->which) +/* + * We always read with _sum, we have no use for the shared count and + * certainly don't want to pay the cost of a shared lock to update it. + * The default batch of 32 make counter increments show up significantly + * in profiles. + */ +#define SCOUTFS_PCPU_COUNTER_BATCH (1 << 30) -#define scoutfs_add_counter(sb, which, cnt) \ - percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, cnt) +#define scoutfs_inc_counter(sb, which) \ + __percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, 1, \ + SCOUTFS_PCPU_COUNTER_BATCH) + +#define scoutfs_add_counter(sb, which, cnt) \ + __percpu_counter_add(&SCOUTFS_SB(sb)->counters->which, cnt, \ + SCOUTFS_PCPU_COUNTER_BATCH) void __init scoutfs_init_counters(void); int scoutfs_setup_counters(struct super_block *sb); From b605407c29e6bf243e53932ff909535ea996752e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Sep 2020 11:34:17 -0700 Subject: [PATCH 873/920] scoutfs: add extent layer Add infrastructure for working with extents. Callers provide callbacks which operate on their extent storage while this code performs the fiddly splitting and merging of extents. This layer doesn't have any persitent structures itself, it only operates on native structs in memory. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/counters.h | 3 + kmod/src/ext.c | 394 +++++++++++++++++++++++++++++++++++++++ kmod/src/ext.h | 35 ++++ kmod/src/scoutfs_trace.h | 143 +++++++++++++- 5 files changed, 566 insertions(+), 10 deletions(-) create mode 100644 kmod/src/ext.c create mode 100644 kmod/src/ext.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 53ce0a5b..9c299c5d 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -17,6 +17,7 @@ scoutfs-y += \ data.o \ dir.o \ export.o \ + ext.o \ file.o \ forest.o \ inode.o \ diff --git a/kmod/src/counters.h b/kmod/src/counters.h index a9658bf3..2230ced2 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -56,6 +56,9 @@ EXPAND_COUNTER(dentry_revalidate_root) \ EXPAND_COUNTER(dentry_revalidate_valid) \ EXPAND_COUNTER(dir_backref_excessive_retries) \ + EXPAND_COUNTER(ext_op_insert) \ + EXPAND_COUNTER(ext_op_next) \ + EXPAND_COUNTER(ext_op_remove) \ EXPAND_COUNTER(forest_bloom_fail) \ EXPAND_COUNTER(forest_bloom_pass) \ EXPAND_COUNTER(forest_read_items) \ diff --git a/kmod/src/ext.c b/kmod/src/ext.c new file mode 100644 index 00000000..f87c064d --- /dev/null +++ b/kmod/src/ext.c @@ -0,0 +1,394 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include + +#include "ext.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * Extents are used to track free block regions and to map logical file + * regions to device blocks. Extents can be split and merged as + * they're modified. These helpers implement all the fiddly extent + * manipulations. Callers provide callbacks which implement the actual + * storage of extents in either the item cache or btree items. + */ + +static void ext_zero(struct scoutfs_extent *ext) +{ + memset(ext, 0, sizeof(struct scoutfs_extent)); +} + +static bool ext_overlap(struct scoutfs_extent *ext, u64 start, u64 len) +{ + u64 e_end = ext->start + ext->len - 1; + u64 end = start + len - 1; + + return !(e_end < start || ext->start > end); +} + +static bool ext_inside(u64 start, u64 len, struct scoutfs_extent *out) +{ + u64 in_end = start + len - 1; + u64 out_end = out->start + out->len - 1; + + return out->start <= start && out_end >= in_end; +} + +/* we only translate mappings when they exist */ +static inline u64 ext_map_add(u64 map, u64 diff) +{ + return map ? map + diff : 0; +} + +/* + * Extents can merge if they're logically contiguous, both don't have + * mappings or have mappings which are also contiguous, and have + * matching flags. + */ +bool scoutfs_ext_can_merge(struct scoutfs_extent *left, + struct scoutfs_extent *right) +{ + return (left->start + left->len == right->start) && + ((!left->map && !right->map) || + (left->map + left->len == right->map)) && + (left->flags == right->flags); +} + +/* + * Split an existing extent in to left and right extents by removing + * an interior range. The split extents are all zeros if the range + * extends to their end of the extent. + */ +static void ext_split(struct scoutfs_extent *ext, u64 start, u64 len, + struct scoutfs_extent *left, + struct scoutfs_extent *right) +{ + if (ext->start < start) { + left->start = ext->start; + left->len = start - ext->start; + left->map = ext->map; + left->flags = ext->flags; + } else { + ext_zero(left); + } + + if (ext->start + ext->len > start + len) { + right->start = start + len; + right->len = ext->start + ext->len - right->start; + right->map = ext_map_add(ext->map, right->start - ext->start); + right->flags = ext->flags; + } else { + ext_zero(right); + } +} + +#define op_call(sb, ops, arg, which, args...) \ +({ \ + int _ret; \ + _ret = ops->which(sb, arg, ##args); \ + scoutfs_inc_counter(sb, ext_op_##which); \ + trace_scoutfs_ext_op_##which(sb, ##args, _ret); \ + _ret; \ +}) + +struct extent_changes { + struct scoutfs_extent exts[4]; + bool ins[4]; + u8 nr; +}; + +static void add_change(struct extent_changes *chg, + struct scoutfs_extent *ext, bool ins) +{ + BUILD_BUG_ON(ARRAY_SIZE(chg->ins) != ARRAY_SIZE(chg->exts)); + + if (ext->len) { + BUG_ON(chg->nr == ARRAY_SIZE(chg->exts)); + chg->exts[chg->nr] = *ext; + chg->ins[chg->nr] = !!ins; + chg->nr++; + } +} + +static int apply_changes(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, struct extent_changes *chg) +{ + int ret = 0; + int err; + int i; + + for (i = 0; i < chg->nr; i++) { + if (chg->ins[i]) + ret = op_call(sb, ops, arg, insert, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + else + ret = op_call(sb, ops, arg, remove, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + if (ret < 0) + break; + } + + while (ret < 0 && --i >= 0) { + if (chg->ins[i]) + err = op_call(sb, ops, arg, remove, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + else + err = op_call(sb, ops, arg, insert, chg->exts[i].start, + chg->exts[i].len, chg->exts[i].map, + chg->exts[i].flags); + BUG_ON(err); /* inconsistent */ + } + + return ret; +} + +int scoutfs_ext_next(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, struct scoutfs_extent *ext) +{ + int ret; + + ret = op_call(sb, ops, arg, next, start, len, ext); + trace_scoutfs_ext_next(sb, start, len, ext, ret); + return ret; +} + +/* + * Insert the given extent. EINVAL is returned if there's already an existing + * overlapping extent. This can merge with its neighbours. + */ +int scoutfs_ext_insert(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent ins; + int ret; + + ins.start = start; + ins.len = len; + ins.map = map; + ins.flags = flags; + + /* find right neighbour and check for overlap */ + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + /* inserting extent must not overlap */ + if (found.len && ext_overlap(&ins, found.start, found.len)) { + ret = -EINVAL; + goto out; + } + + /* merge with right if we can */ + if (found.len && scoutfs_ext_can_merge(&ins, &found)) { + ins.len += found.len; + add_change(&chg, &found, false); + } + + /* see if we can merge with a left neighbour */ + if (start > 0) { + ret = op_call(sb, ops, arg, next, start - 1, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == 0 && scoutfs_ext_can_merge(&found, &ins)) { + ins.start = found.start; + ins.map = found.map; + ins.len += found.len; + add_change(&chg, &found, false); + } + } + + add_change(&chg, &ins, true); + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_insert(sb, start, len, map, flags, ret); + return ret; +} + +/* + * Remove the given extent. The extent to remove must be found entirely + * in an existing extent. If the existing extent is larger then we leave + * behind the remaining extent. The existing extent can be split. + */ +int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent left; + struct scoutfs_extent right; + int ret; + + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0) + goto out; + + /* removed extent must be entirely within found */ + if (!ext_inside(start, len, &found)) { + ret = -EINVAL; + goto out; + } + + ext_split(&found, start, len, &left, &right); + + add_change(&chg, &found, false); + add_change(&chg, &left, true); + add_change(&chg, &right, true); + + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_remove(sb, start, len, 0, 0, ret); + return ret; +} + +/* + * Find and remove the next extent, removing only a portion if the + * extent is larger than the count. Returns ENOENT if it didn't + * find any extents. + * + * This does not search for merge candidates so it's safe to call with + * extents indexed by length. + */ +int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 count, + struct scoutfs_extent *ext) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent ins; + int ret; + + ret = op_call(sb, ops, arg, next, start, len, &found); + if (ret < 0) + goto out; + + add_change(&chg, &found, false); + + if (found.len > count) { + ins.start = found.start + count; + ins.len = found.len - count; + ins.map = ext_map_add(found.map, count); + ins.flags = found.flags; + + add_change(&chg, &ins, true); + } + + ret = apply_changes(sb, ops, arg, &chg); +out: + if (ret == 0) { + ext->start = found.start; + ext->len = min(found.len, count); + ext->map = found.map; + ext->flags = found.flags; + } else { + ext_zero(ext); + } + + trace_scoutfs_ext_alloc(sb, start, len, count, ext, ret); + return ret; +} + +/* + * Set the map and flags for an extent region, with the magical property + * that extents with map and flags set to 0 are removed. + * + * If we're modifying an existing extent then the modification must be + * fully inside the existing extent. The modification can leave edges + * of the extent which need to be inserted. If the modification extends + * to the end of the existing extent then we need to check for adjacent + * neighbouring extents which might now be able to be merged. + * + * Inserting a new extent is like the case of modifying the entire + * existing extent. We need to check neighbours of the inserted extent + * to see if they can be merged. + */ +int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags) +{ + struct extent_changes chg = { .nr = 0 }; + struct scoutfs_extent found; + struct scoutfs_extent left; + struct scoutfs_extent right; + struct scoutfs_extent set; + int ret; + + set.start = start; + set.len = len; + set.map = map; + set.flags = flags; + + /* find extent to remove */ + ret = op_call(sb, ops, arg, next, start, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + + if (ret == 0 && ext_overlap(&found, start, len)) { + /* set extent must be entirely within found */ + if (!ext_inside(start, len, &found)) { + ret = -EINVAL; + goto out; + } + + add_change(&chg, &found, false); + ext_split(&found, start, len, &left, &right); + } else { + ext_zero(&found); + ext_zero(&left); + ext_zero(&right); + } + + if (left.len) { + /* inserting split left, won't merge */ + add_change(&chg, &left, true); + } else if (start > 0) { + ret = op_call(sb, ops, arg, next, start - 1, 1, &left); + if (ret < 0 && ret != -ENOENT) + goto out; + else if (ret == 0 && scoutfs_ext_can_merge(&left, &set)) { + /* remove found left, merging */ + set.start = left.start; + set.map = left.map; + set.len += left.len; + add_change(&chg, &left, false); + } + } + + if (right.len) { + /* inserting split right, won't merge */ + add_change(&chg, &right, true); + } else { + ret = op_call(sb, ops, arg, next, start + len, 1, &right); + if (ret < 0 && ret != -ENOENT) + goto out; + else if (ret == 0 && scoutfs_ext_can_merge(&set, &right)) { + /* remove found right, merging */ + set.len += right.len; + add_change(&chg, &right, false); + } + } + + if (set.flags || set.map) + add_change(&chg, &set, true); + + ret = apply_changes(sb, ops, arg, &chg); +out: + trace_scoutfs_ext_set(sb, start, len, map, flags, ret); + return ret; +} diff --git a/kmod/src/ext.h b/kmod/src/ext.h new file mode 100644 index 00000000..31dbd57a --- /dev/null +++ b/kmod/src/ext.h @@ -0,0 +1,35 @@ +#ifndef _SCOUTFS_EXT_H_ +#define _SCOUTFS_EXT_H_ + +struct scoutfs_extent { + u64 start; + u64 len; + u64 map; + u8 flags; +}; + +struct scoutfs_ext_ops { + int (*next)(struct super_block *sb, void *arg, + u64 start, u64 len, struct scoutfs_extent *ext); + int (*insert)(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags); + int (*remove)(struct super_block *sb, void *arg, u64 start, u64 len, + u64 map, u8 flags); +}; + +bool scoutfs_ext_can_merge(struct scoutfs_extent *left, + struct scoutfs_extent *right); + +int scoutfs_ext_next(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, struct scoutfs_extent *ext); +int scoutfs_ext_insert(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags); +int scoutfs_ext_remove(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len); +int scoutfs_ext_alloc(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 limit, + struct scoutfs_extent *ext); +int scoutfs_ext_set(struct super_block *sb, struct scoutfs_ext_ops *ops, + void *arg, u64 start, u64 len, u64 map, u8 flags); + +#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index f26525bd..5ecb3a3b 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -37,25 +37,26 @@ #include "server.h" #include "net.h" #include "data.h" +#include "ext.h" struct lock_info; #define STE_FMT "[%llu %llu %llu 0x%x]" -#define STE_ARGS(te) (te)->iblock, (te)->count, (te)->blkno, (te)->flags +#define STE_ARGS(te) (te)->start, (te)->len, (te)->map, (te)->flags #define STE_FIELDS(pref) \ - __field(__u64, pref##_iblock) \ - __field(__u64, pref##_count) \ - __field(__u64, pref##_blkno) \ + __field(__u64, pref##_start) \ + __field(__u64, pref##_len) \ + __field(__u64, pref##_map) \ __field(__u8, pref##_flags) #define STE_ASSIGN(pref, te) \ - __entry->pref##_iblock = (te)->iblock; \ - __entry->pref##_count = (te)->count; \ - __entry->pref##_blkno = (te)->blkno; \ + __entry->pref##_start = (te)->start; \ + __entry->pref##_len = (te)->len; \ + __entry->pref##_map = (te)->map; \ __entry->pref##_flags = (te)->flags; #define STE_ENTRY_ARGS(pref) \ - __entry->pref##_iblock, \ - __entry->pref##_count, \ - __entry->pref##_blkno, \ + __entry->pref##_start, \ + __entry->pref##_len, \ + __entry->pref##_map, \ __entry->pref##_flags #define DECLARE_TRACED_EXTENT(name) \ @@ -2367,6 +2368,128 @@ TRACE_EVENT(scoutfs_radix_merged_blocks, __entry->dst_lg_delta) ); +DECLARE_EVENT_CLASS(scoutfs_ext_next_class, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + + TP_ARGS(sb, start, len, ext, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + STE_FIELDS(ext) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu ext "STE_FMT" ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, + STE_ENTRY_ARGS(ext), __entry->ret) +); + +DEFINE_EVENT(scoutfs_ext_next_class, scoutfs_ext_op_next, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + TP_ARGS(sb, start, len, ext, ret) +); +DEFINE_EVENT(scoutfs_ext_next_class, scoutfs_ext_next, + TP_PROTO(struct super_block *sb, u64 start, u64 len, + struct scoutfs_extent *ext, int ret), + TP_ARGS(sb, start, len, ext, ret) +); + +DECLARE_EVENT_CLASS(scoutfs_ext_typical_class, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + + TP_ARGS(sb, start, len, map, flags, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + __field(__u64, map) + __field(__u8, flags) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + __entry->map = map; + __entry->flags = flags; + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu map %llu flags %u ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, __entry->map, + __entry->flags, __entry->ret) +); + +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_op_insert, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_insert, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_op_remove, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_remove, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); +DEFINE_EVENT(scoutfs_ext_typical_class, scoutfs_ext_set, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 map, u8 flags, + int ret), + TP_ARGS(sb, start, len, map, flags, ret) +); + +TRACE_EVENT(scoutfs_ext_alloc, + TP_PROTO(struct super_block *sb, u64 start, u64 len, u64 count, + struct scoutfs_extent *ext, int ret), + + TP_ARGS(sb, start, len, count, ext, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, start) + __field(__u64, len) + __field(__u64, count) + STE_FIELDS(ext) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->start = start; + __entry->len = len; + __entry->count = count; + STE_ASSIGN(ext, ext) + __entry->ret = ret; + ), + + TP_printk(SCSBF" start %llu len %llu count %llu ext "STE_FMT" ret %d", + SCSB_TRACE_ARGS, __entry->start, __entry->len, __entry->count, + STE_ENTRY_ARGS(ext), __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From 8f946aa478a412cd271aa39229d765f80d8c0ba6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Sep 2020 11:39:55 -0700 Subject: [PATCH 874/920] scoutfs: add btree item extent allocator Add an allocator which uses btree items to store extents. Both the client and server will use this for btree blocks, the client will use it for srch blocks and data extents, and the server will move extents between the core fs allocator btree roots and the clients' roots. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/alloc.c | 1112 ++++++++++++++++++++++++++++++++++++++ kmod/src/alloc.h | 124 +++++ kmod/src/counters.h | 11 +- kmod/src/format.h | 48 ++ kmod/src/scoutfs_trace.h | 117 ++++ 6 files changed, 1412 insertions(+), 1 deletion(-) create mode 100644 kmod/src/alloc.c create mode 100644 kmod/src/alloc.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 9c299c5d..bfd8f38a 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -10,6 +10,7 @@ CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include scoutfs-y += \ avl.o \ + alloc.o \ block.o \ btree.o \ client.o \ diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c new file mode 100644 index 00000000..ca088c89 --- /dev/null +++ b/kmod/src/alloc.c @@ -0,0 +1,1112 @@ +/* + * Copyright (C) 2020 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "block.h" +#include "btree.h" +#include "trans.h" +#include "alloc.h" +#include "counters.h" +#include "scoutfs_trace.h" + +/* + * The core allocator uses extent items in btrees rooted in the super. + * Each free extent is stored in two items. The first item is indexed + * by block location and is used to merge adjacent extents when freeing. + * The second item is indexed by length and is used to find large + * extents to allocate from. + * + * Free extent always consumes the front of the largest extent. This + * attempts to discourage fragmentation by given smaller freed extents + * time for an adjacent free to merge before we attempt to re-use them. + * + * The metadata btrees that store extents are updated with cow. This + * requires allocation during extent item modification on behalf of + * allocation. Avoiding this recursion introduces the second structure, + * persistent singly linked lists of individual blknos. + * + * The alloc lists are used for metadata allocation during a + * transaction. Before each transaction lists of blknos are prepared + * for use during the transaction. This ensures a small predictable + * number of cows needed to fully dirty the metadata allocator + * structures during the transaction. As the transaction proceeds + * allocations are made from a list of available meta blknos, and frees + * are performed by adding blknos to another list of freed blknos. + * After transactions these lists are merged back in to extents. + * + * Data allocations are performed directly on a btree of extent items, + * with a bit of caching to stream small file data allocations from + * memory instead of performing multiple btree calls per block + * allocation. + * + * Every transaction has exclusive access to its metadata list blocks + * and data extent trees which are prepared by the server. For client + * metadata and srch transactions the server moved extents and blocks + * into persistent items that are communicated with the server. For + * server transactions metadata the server has to prepare structures for + * itself. To avoid modifying the same structure both explicitly + * (refilling an allocator) and implicitly (using the current allocator + * for cow allocations), it double buffers list blocks. It uses current + * blocks to modify the next blocks, and swaps them at each transaction. + */ + +/* + * Free extents don't have flags and are stored in two indexes sorted by + * block location and by length, largest first. The block location key + * is set to the final block in the extent so that we can find + * intersections by calling _next() iterators starting with the block + * we're searching for. + */ +static void init_ext_key(struct scoutfs_key *key, int type, u64 start, u64 len) +{ + *key = (struct scoutfs_key) { + .sk_zone = SCOUTFS_FREE_EXTENT_ZONE, + .sk_type = type, + }; + + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + key->skfb_end = cpu_to_le64(start + len - 1); + key->skfb_len = cpu_to_le64(len); + } else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) { + key->skfl_neglen = cpu_to_le64(-len); + key->skfl_blkno = cpu_to_le64(start); + } else { + BUG(); + } +} + +static void ext_from_key(struct scoutfs_extent *ext, struct scoutfs_key *key) +{ + if (key->sk_type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) { + ext->start = le64_to_cpu(key->skfb_end) - + le64_to_cpu(key->skfb_len) + 1; + ext->len = le64_to_cpu(key->skfb_len); + } else { + ext->start = le64_to_cpu(key->skfl_blkno); + ext->len = -le64_to_cpu(key->skfl_neglen); + } + ext->map = 0; + ext->flags = 0; +} + +struct alloc_ext_args { + struct scoutfs_alloc *alloc; + struct scoutfs_block_writer *wri; + struct scoutfs_alloc_root *root; + int type; +}; + +static int alloc_ext_next(struct super_block *sb, void *arg, + u64 start, u64 len, struct scoutfs_extent *ext) +{ + struct alloc_ext_args *args = arg; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + init_ext_key(&key, args->type, start, len); + + ret = scoutfs_btree_next(sb, &args->root->root, &key, &iref); + if (ret == 0) { + if (iref.val_len != 0) + ret = -EIO; + else if (iref.key->sk_type != args->type) + ret = -ENOENT; + else + ext_from_key(ext, iref.key); + scoutfs_btree_put_iref(&iref); + } + + if (ret < 0) + memset(ext, 0, sizeof(struct scoutfs_extent)); + + return ret; +} + +static int other_type(int type) +{ + if (type == SCOUTFS_FREE_EXTENT_BLKNO_TYPE) + return SCOUTFS_FREE_EXTENT_LEN_TYPE; + else if (type == SCOUTFS_FREE_EXTENT_LEN_TYPE) + return SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + else + BUG(); +} + +/* + * Insert an extent along with its matching item which is indexed by + * opposite of its len or blkno. If we succeed we update the root's + * record of the total length of all the stored extents. + */ +static int alloc_ext_insert(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + /* allocator extents don't have mappings or flags */ + if (WARN_ON_ONCE(map || flags)) + return -EINVAL; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, NULL, 0); + if (ret == 0) { + ret = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &other, NULL, 0); + if (ret < 0) { + err = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, len); + } + } + + return ret; +} + +static int alloc_ext_remove(struct super_block *sb, void *arg, + u64 start, u64 len, u64 map, u8 flags) +{ + struct alloc_ext_args *args = arg; + struct scoutfs_key other; + struct scoutfs_key key; + int ret; + int err; + + init_ext_key(&key, args->type, start, len); + init_ext_key(&other, other_type(args->type), start, len); + + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &key); + if (ret == 0) { + ret = scoutfs_btree_delete(sb, args->alloc, args->wri, + &args->root->root, &other); + if (ret < 0) { + err = scoutfs_btree_insert(sb, args->alloc, args->wri, + &args->root->root, &key, + NULL, 0); + BUG_ON(err); + } else { + le64_add_cpu(&args->root->total_len, -len); + } + } + + return ret; +} + +static struct scoutfs_ext_ops alloc_ext_ops = { + .next = alloc_ext_next, + .insert = alloc_ext_insert, + .remove = alloc_ext_remove, +}; + +static bool invalid_extent(u64 start, u64 end, u64 first, u64 last) +{ + return start > end || start < first || end > last; +} + +static bool invalid_meta_blkno(struct super_block *sb, u64 blkno) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(blkno, blkno, + le64_to_cpu(super->first_meta_blkno), + le64_to_cpu(super->last_meta_blkno)); +} + +static bool invalid_data_extent(struct super_block *sb, u64 start, u64 len) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + + return invalid_extent(start, start + len - 1, + le64_to_cpu(super->first_data_blkno), + le64_to_cpu(super->last_data_blkno)); +} + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed) +{ + memset(alloc, 0, sizeof(struct scoutfs_alloc)); + + spin_lock_init(&alloc->lock); + mutex_init(&alloc->mutex); + alloc->avail = *avail; + alloc->freed = *freed; +} + +/* + * We're about to commit the transaction that used this allocator, drop + * its block references. + */ +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + scoutfs_block_put(sb, alloc->dirty_avail_bl); + alloc->dirty_avail_bl = NULL; + scoutfs_block_put(sb, alloc->dirty_freed_bl); + alloc->dirty_freed_bl = NULL; + + return 0; +} + +static u32 list_block_space(__le32 nr) +{ + return SCOUTFS_ALLOC_LIST_MAX_BLOCKS - le32_to_cpu(nr); +} + +static u64 list_block_peek(struct scoutfs_alloc_list_block *lblk, + unsigned int skip) +{ + BUG_ON(skip >= le32_to_cpu(lblk->nr)); + + return le64_to_cpu(lblk->blknos[le32_to_cpu(lblk->start) + skip]); +} + +/* + * Add a blkno to the array. Typically we append of the array. But we + * can also prepend once there's no more room at the end. Consumers of + * the blocks sort before removing them. + */ +static void list_block_add(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, u64 blkno) +{ + u32 start = le32_to_cpu(lblk->start); + u32 nr = le32_to_cpu(lblk->nr); + + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(list_block_space(lblk->nr) == 0); + + if (start + nr < SCOUTFS_ALLOC_LIST_MAX_BLOCKS) { + lblk->blknos[start + nr] = cpu_to_le64(blkno); + } else { + start--; + lblk->blknos[start] = cpu_to_le64(blkno); + lblk->start = cpu_to_le32(start); + } + + le32_add_cpu(&lblk->nr, 1); + le64_add_cpu(&lhead->total_nr, 1); + le32_add_cpu(&lhead->first_nr, 1); +} + +/* + * Remove blknos from the start of the array. + */ +static void list_block_remove(struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_list_block *lblk, + unsigned int count) +{ + BUG_ON(lhead->ref.blkno != lblk->hdr.blkno); + BUG_ON(count > SCOUTFS_ALLOC_LIST_MAX_BLOCKS); + BUG_ON(le32_to_cpu(lblk->nr) < count); + + le32_add_cpu(&lblk->nr, -count); + if (lblk->nr == 0) + lblk->start = 0; + else + le32_add_cpu(&lblk->start, count); + le64_add_cpu(&lhead->total_nr, -(u64)count); + le32_add_cpu(&lhead->first_nr, -count); +} + +static int cmp_le64(const void *A, const void *B) +{ + const __le64 *a = A; + const __le64 *b = B; + + return scoutfs_cmp_u64s(le64_to_cpu(*a), le64_to_cpu(*b)); +} + +static void swap_le64(void *A, void *B, int size) +{ + __le64 *a = A; + __le64 *b = B; + + swap(*a, *b); +} + +static void list_block_sort(struct scoutfs_alloc_list_block *lblk) +{ + sort(&lblk->blknos[le32_to_cpu(lblk->start)], le32_to_cpu(lblk->nr), + sizeof(lblk->blknos[0]), cmp_le64, swap_le64); +} + +/* + * We're always reading blocks that we own, so we shouldn't see stale + * references. But the cached block can be stale and we can need to + * invalidate it. + */ +static int read_list_block(struct super_block *sb, + struct scoutfs_alloc_list_ref *ref, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl = NULL; + + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + if (!IS_ERR_OR_NULL(bl) && + !scoutfs_block_consistent_ref(sb, bl, ref->seq, ref->blkno, + SCOUTFS_BLOCK_MAGIC_ALLOC_LIST)) { + scoutfs_inc_counter(sb, alloc_stale_cached_list_block); + scoutfs_block_invalidate(sb, bl); + scoutfs_block_put(sb, bl); + bl = scoutfs_block_read(sb, le64_to_cpu(ref->blkno)); + } + if (IS_ERR(bl)) { + *bl_ret = NULL; + return PTR_ERR(bl); + } + + *bl_ret = bl; + return 0; +} + +/* + * Give the caller a dirty list block, always allocating a new block if + * the ref is empty. + * + * If the caller gives us an allocated blkno for the cow then we know + * that they're taking care of allocating and freeing the blknos, if not + * we call meta alloc and free. + */ +static int dirty_list_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_ref *ref, + u64 dirty, u64 *old, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_block *cow_bl = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_alloc_list_block *lblk; + bool undo_alloc = false; + u64 blkno; + int ret; + int err; + + blkno = le64_to_cpu(ref->blkno); + if (blkno) { + ret = read_list_block(sb, ref, &bl); + if (ret < 0) + goto out; + + if (scoutfs_block_writer_is_dirty(sb, bl)) { + ret = 0; + goto out; + } + } + + if (dirty == 0) { + ret = scoutfs_alloc_meta(sb, alloc, wri, &dirty); + if (ret < 0) + goto out; + undo_alloc = true; + } + + cow_bl = scoutfs_block_create(sb, dirty); + if (IS_ERR(cow_bl)) { + ret = PTR_ERR(cow_bl); + goto out; + } + + if (old) { + *old = blkno; + } else if (blkno) { + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + if (ret < 0) + goto out; + } + + if (bl) + memcpy(cow_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); + else + memset(cow_bl->data, 0, SCOUTFS_BLOCK_LG_SIZE); + scoutfs_block_put(sb, bl); + bl = cow_bl; + cow_bl = NULL; + + lblk = bl->data; + lblk->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_ALLOC_LIST); + lblk->hdr.fsid = super->hdr.fsid; + lblk->hdr.blkno = cpu_to_le64(bl->blkno); + prandom_bytes(&lblk->hdr.seq, sizeof(lblk->hdr.seq)); + + ref->blkno = lblk->hdr.blkno; + ref->seq = lblk->hdr.seq; + + scoutfs_block_writer_mark_dirty(sb, wri, bl); + ret = 0; + +out: + scoutfs_block_put(sb, cow_bl); + if (ret < 0 && undo_alloc) { + err = scoutfs_free_meta(sb, alloc, wri, dirty); + BUG_ON(err); /* inconsistent */ + } + + if (ret < 0) { + scoutfs_block_put(sb, bl); + bl = NULL; + } + *bl_ret = bl; + + return ret; +} + +/* Allocate a new dirty list block if we fill up more than 3/4 of the block. */ +#define EMPTY_FREED_THRESH (SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 4) + +/* + * Get dirty avail and freed list blocks that will be used for meta + * allocations during our transaction. We peek at the next avail blknos + * for the cow allocations and manually record the cow frees rather than + * recursively calling into alloc_meta and free_meta. + * + * In the client the server will have emptied the freed list so it will + * always allocate a new first empty block for frees. But in the server + * it might have long lists of frees that it's trying to merge in to + * extents over multiple transactions. If the head of the freed list + * doesn't have room we add a new empty block. + */ +static int dirty_alloc_blocks(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + struct scoutfs_alloc_list_ref orig_freed; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *av_bl = NULL; + struct scoutfs_block *fr_bl = NULL; + struct scoutfs_block *bl; + bool link_orig = false; + u64 av_peek; + u64 av_old; + u64 fr_peek; + u64 fr_old; + int ret; + + if (alloc->dirty_avail_bl != NULL) + return 0; + + mutex_lock(&alloc->mutex); + + /* undo dirty freed if we get an error after */ + orig_freed = alloc->freed.ref; + + if (alloc->dirty_avail_bl != NULL) { + ret = 0; + goto out; + } + + /* caller must ensure that transactions commit before running out */ + if (WARN_ON_ONCE(alloc->avail.ref.blkno == 0) || + WARN_ON_ONCE(le32_to_cpu(alloc->avail.first_nr) < 2)) { + ret = -ENOSPC; + goto out; + } + + ret = read_list_block(sb, &alloc->avail.ref, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + av_peek = list_block_peek(lblk, 0); + fr_peek = list_block_peek(lblk, 1); + scoutfs_block_put(sb, bl); + lblk = NULL; + + if (alloc->freed.ref.blkno && + list_block_space(alloc->freed.first_nr) < EMPTY_FREED_THRESH) { + /* zero ref to force alloc of new block... */ + memset(&alloc->freed.ref, 0, sizeof(alloc->freed.ref)); + alloc->freed.first_nr = 0; + link_orig = true; + } + + /* dirty the first free block */ + ret = dirty_list_block(sb, alloc, wri, &alloc->freed.ref, + fr_peek, &fr_old, &fr_bl); + if (ret < 0) + goto out; + + if (link_orig) { + /* .. and point the new block at the rest of the list */ + lblk = fr_bl->data; + lblk->next = orig_freed; + lblk = NULL; + } + + ret = dirty_list_block(sb, alloc, wri, &alloc->avail.ref, + av_peek, &av_old, &av_bl); + if (ret < 0) + goto out; + + list_block_remove(&alloc->avail, av_bl->data, 2); + /* sort dirty avail to encourage contiguous sorted meta blocks */ + list_block_sort(av_bl->data); + + if (av_old) + list_block_add(&alloc->freed, fr_bl->data, av_old); + if (fr_old) + list_block_add(&alloc->freed, fr_bl->data, fr_old); + + alloc->dirty_avail_bl = av_bl; + av_bl = NULL; + alloc->dirty_freed_bl = fr_bl; + fr_bl = NULL; + ret = 0; + +out: + if (ret < 0 && alloc->freed.ref.blkno != orig_freed.blkno) { + if (fr_bl) + scoutfs_block_writer_forget(sb, wri, fr_bl); + alloc->freed.ref = orig_freed; + } + + mutex_unlock(&alloc->mutex); + scoutfs_block_put(sb, av_bl); + scoutfs_block_put(sb, fr_bl); + return ret; +} + +/* + * Alloc a metadata block for a transaction in either the client or the + * server. The list block in the allocator was prepared for the transaction. + */ +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_avail_bl->data; + if (WARN_ON_ONCE(lblk->nr == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -ENOSPC; + } else { + *blkno = list_block_peek(lblk, 0); + list_block_remove(&alloc->avail, lblk, 1); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + if (ret < 0) + *blkno = 0; + scoutfs_inc_counter(sb, alloc_alloc_meta); + trace_scoutfs_alloc_alloc_meta(sb, *blkno, ret); + return ret; +} + +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno) +{ + struct scoutfs_alloc_list_block *lblk; + int ret; + + if (WARN_ON_ONCE(invalid_meta_blkno(sb, blkno))) + return -EINVAL; + + ret = dirty_alloc_blocks(sb, alloc, wri); + if (ret < 0) + goto out; + + spin_lock(&alloc->lock); + lblk = alloc->dirty_freed_bl->data; + if (WARN_ON_ONCE(list_block_space(lblk->nr) == 0)) { + /* shouldn't happen, transaction should commit first */ + ret = -EIO; + } else { + list_block_add(&alloc->freed, lblk, blkno); + ret = 0; + } + spin_unlock(&alloc->lock); + +out: + scoutfs_inc_counter(sb, alloc_free_meta); + trace_scoutfs_alloc_free_meta(sb, blkno, ret); + return ret; +} + +/* + * Allocate a data extent. An extent that's smaller than the requested + * size can be returned. + * + * The caller can provide a cached extent that can satisfy allocations + * and will be refilled by allocations. The caller is responsible for + * freeing any remaining cached extent back into persistent items before + * committing. + * + * Unlike meta allocations, the caller is expected to serialize + * allocations from the root. + */ +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_extent ext; + u64 len; + int ret; + + /* large allocations come straight from the allocator */ + if (count >= SCOUTFS_ALLOC_DATA_LG_THRESH) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, count, &ext); + if (ret < 0) + goto out; + + *blkno_ret = ext.start; + *count_ret = ext.len; + ret = 0; + goto out; + } + + /* smaller allocations come from a cached extent */ + if (cached->len == 0) { + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + SCOUTFS_ALLOC_DATA_LG_THRESH, cached); + if (ret < 0) + goto out; + } + + len = min(count, cached->len); + + *blkno_ret = cached->start; + *count_ret = len; + + cached->start += len; + cached->len -= len; + ret = 0; +out: + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + *blkno_ret = 0; + *count_ret = 0; + } + + scoutfs_inc_counter(sb, alloc_alloc_data); + trace_scoutfs_alloc_alloc_data(sb, count, *blkno_ret, *count_ret, ret); + return ret; +} + +/* + * Free data extents into the freed tree that will be reclaimed by the + * server and made available for future allocators only if our + * transaction succeeds. We don't want to overwrite existing data if + * our transaction fails. + * + * Unlike meta allocations, the caller is expected to serialize data + * allocations. + */ +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + int ret; + + if (WARN_ON_ONCE(invalid_data_extent(sb, blkno, count))) + return -EINVAL; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, blkno, count, 0, 0); + scoutfs_inc_counter(sb, alloc_free_data); + trace_scoutfs_alloc_free_data(sb, blkno, count, ret); + return ret; +} + + +/* + * Move extent items adding up to the requested total length from the + * src to the dst tree. The caller is responsible for locking the + * trees, usually because they're also looking at total_len to decide + * how much to move. + * + * -ENOENT is returned if we run out of extents in the source tree + * before moving the total. + * + * This first pass is not optimal because it performs full btree walks + * per extent. We could optimize this with more clever btree item + * manipulation functions which can iterate through src and dst blocks + * and let callbacks indicate how to change items. + */ +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + }; + struct scoutfs_extent ext; + u64 moved = 0; + int ret = 0; + int err; + + while (moved < total) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_LEN_TYPE; + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, + 0, 0, total - moved, &ext); + if (ret < 0) + break; + + args.root = dst; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, ext.start, + ext.len, ext.map, ext.flags); + if (ret < 0) { + args.root = src; + args.type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE; + err = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, ext.map, + ext.flags); + BUG_ON(err); /* inconsistent */ + break; + } + + moved += ext.len; + scoutfs_inc_counter(sb, alloc_moved_extent); + } + + scoutfs_inc_counter(sb, alloc_move); + trace_scoutfs_alloc_move(sb, total, moved, ret); + + return ret; +} + +/* + * We only trim one block, instead of looping trimming all, because the + * caller is assuming that we do a fixed amount of work when they check + * that their allocator has enough remaining free blocks for us. + */ +static int trim_empty_first_block(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead) +{ + struct scoutfs_alloc_list_block *one = NULL; + struct scoutfs_alloc_list_block *two = NULL; + struct scoutfs_block *one_bl = NULL; + struct scoutfs_block *two_bl = NULL; + int ret; + + if (WARN_ON_ONCE(lhead->ref.blkno == 0) || + WARN_ON_ONCE(lhead->first_nr != 0)) + return 0; + + ret = read_list_block(sb, &lhead->ref, &one_bl); + if (ret < 0) + goto out; + one = one_bl->data; + + if (one->next.blkno) { + ret = read_list_block(sb, &one->next, &two_bl); + if (ret < 0) + goto out; + two = two_bl->data; + } + + ret = scoutfs_free_meta(sb, alloc, wri, le64_to_cpu(lhead->ref.blkno)); + if (ret < 0) + goto out; + + lhead->ref = one->next; + lhead->first_nr = two ? two->nr : 0; + ret = 0; +out: + scoutfs_block_put(sb, one_bl); + scoutfs_block_put(sb, two_bl); + return ret; +} + +/* + * True if the allocator has enough free blocks to cow (alloc and free) + * a list block and all the btree blocks that store extent items. + * + * At most, an extent operation can dirty down three paths of the tree + * to modify a blkno item and two distant len items. We can grow and + * split the root, and then those three paths could share blocks but each + * modify two leaf blocks. + */ +static bool list_can_cow(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_alloc_root *root) +{ + u32 most = 1 + (1 + 1 + (3 * (1 - root->root.height + 1))); + + if (le32_to_cpu(alloc->avail.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_avail_lo); + return false; + } + + if (list_block_space(alloc->freed.first_nr) < most) { + scoutfs_inc_counter(sb, alloc_list_freed_hi); + return false; + } + + return true; +} + +static bool lhead_in_alloc(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *lhead) +{ + return lhead == &alloc->avail || lhead == &alloc->freed; +} + +/* + * Move free blocks from extent items in the root into only the first + * block in the list towards the target if it's fallen below the lo + * threshold. This can return success without necessarily moving as + * much as was requested if its meta allocator runs low, the caller is + * expected to check the counts and act accordingly. + * + * -ENOSPC is returned if the root runs out of extents before the list + * reaches the target. + */ +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_LEN_TYPE, + }; + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + int i; + + if (WARN_ON_ONCE(target < lo) || + WARN_ON_ONCE(lo > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(target > SCOUTFS_ALLOC_LIST_MAX_BLOCKS) || + WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + if (le32_to_cpu(lhead->first_nr) >= lo) + return 0; + + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, 0, NULL, &bl); + if (ret < 0) + goto out; + lblk = bl->data; + + while (le32_to_cpu(lblk->nr) < target && + list_can_cow(sb, alloc, root)) { + + ret = scoutfs_ext_alloc(sb, &alloc_ext_ops, &args, 0, 0, + target - le32_to_cpu(lblk->nr), &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ENOSPC; + break; + } + + for (i = 0; i < ext.len; i++) + list_block_add(lhead, lblk, ext.start + i); + } + +out: + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Move blknos from all the blocks in the list into extents in the root, + * removing empty blocks as we go. This can return success and leave blocks + * on the list if its metadata alloc runs out of space. + */ +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead) +{ + struct alloc_ext_args args = { + .alloc = alloc, + .wri = wri, + .root = root, + .type = SCOUTFS_FREE_EXTENT_BLKNO_TYPE, + }; + struct scoutfs_alloc_list_block *lblk = NULL; + struct scoutfs_block *bl = NULL; + struct scoutfs_extent ext; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, lhead))) + return -EINVAL; + + while (lhead->ref.blkno && list_can_cow(sb, alloc, args.root)) { + + if (lhead->first_nr == 0) { + ret = trim_empty_first_block(sb, alloc, wri, lhead); + if (ret < 0) + break; + + scoutfs_block_put(sb, bl); + bl = NULL; + continue; + } + + if (bl == NULL) { + ret = dirty_list_block(sb, alloc, wri, &lhead->ref, + 0, NULL, &bl); + if (ret < 0) + break; + lblk = bl->data; + + /* sort to encourage forming extents */ + list_block_sort(lblk); + } + + /* combine free blknos into extents and insert them */ + ext.start = list_block_peek(lblk, 0); + ext.len = 1; + while ((le32_to_cpu(lblk->nr) > ext.len) && + (list_block_peek(lblk, ext.len) == ext.start + ext.len)) + ext.len++; + + ret = scoutfs_ext_insert(sb, &alloc_ext_ops, &args, + ext.start, ext.len, 0, 0); + if (ret < 0) + break; + + list_block_remove(lhead, lblk, ext.len); + } + + scoutfs_block_put(sb, bl); + + return ret; +} + +/* + * Insert the source list at the head of the destination list, leaving + * the source empty. + * + * This looks bad because the lists are singly-linked and we have to cow + * the entire src lsit to update its tail block next ref to the start of + * the dst list. + * + * In practice, this isn't a problem because the server only calls this + * with small lists that it's going to use soon. + */ +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src) +{ + struct scoutfs_alloc_list_block *lblk; + struct scoutfs_alloc_list_ref *ref; + struct scoutfs_block *prev = NULL; + struct scoutfs_block *bl = NULL; + int ret = 0; + + if (WARN_ON_ONCE(lhead_in_alloc(alloc, dst)) || + WARN_ON_ONCE(lhead_in_alloc(alloc, src))) + return -EINVAL; + + if (src->ref.blkno == 0) + return 0; + + ref = &src->ref; + while (ref->blkno) { + ret = dirty_list_block(sb, alloc, wri, ref, 0, NULL, &bl); + if (ret < 0) + goto out; + + lblk = bl->data; + ref = &lblk->next; + + scoutfs_block_put(sb, prev); + prev = bl; + bl = NULL; + } + + *ref = dst->ref; + dst->ref = src->ref; + dst->first_nr = src->first_nr; + le64_add_cpu(&dst->total_nr, le64_to_cpu(src->total_nr)); + + memset(src, 0, sizeof(struct scoutfs_alloc_list_head)); + ret = 0; +out: + scoutfs_block_put(sb, prev); + scoutfs_block_put(sb, bl); + return ret; +} + +/* + * Returns true if we're running low on avail blocks or running out of + * space for freed blocks. + * + * On the avail side, we're avoiding spurious enospc as our avail block + * runs low. If we commit it can be refilled by the server. + * + * On the freed side, we're avoiding getting errors in frees where they + * can't be recovered from. This is mostly in freeing cowed blocks in + * the data allocator btree which is related to its height. + * + * And both of these need to be mindful of multiple tasks entering the + * transaction. + */ +bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, + struct scoutfs_alloc *alloc) +{ + bool lo; + + spin_lock(&alloc->lock); + lo = le32_to_cpu(alloc->avail.first_nr) < 8 || + list_block_space(alloc->freed.first_nr) < 8; + spin_unlock(&alloc->lock); + + return lo; +} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h new file mode 100644 index 00000000..167725b0 --- /dev/null +++ b/kmod/src/alloc.h @@ -0,0 +1,124 @@ +#ifndef _SCOUTFS_ALLOC_H_ +#define _SCOUTFS_ALLOC_H_ + +#include "ext.h" + +/* + * These are implementation-specific metrics, they don't need to be + * consistent across implementations. They should probably be run-time + * knobs. + */ + +/* + * The largest extent that we'll try to allocate with fallocate. We're + * trying not to completely consume a transactions data allocation all + * at once. This is only allocation granularity, repeated allocations + * can produce large contiguous extents. + */ +#define SCOUTFS_FALLOCATE_ALLOC_LIMIT \ + (128ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * The largest aligned region that we'll try to allocate at the end of + * the file as it's extended. This is also limited to the current file + * size so we can only waste at most twice the total file size when + * files are less than this. We try to keep this around the point of + * diminishing returns in streaming performance of common data devices + * to limit waste. + */ +#define SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Small data allocations are satisfied by cached extents stored in + * the run-time alloc struct to minimize item operations for small + * block allocations. Large allocations come directly from btree + * extent items, and this defines the threshold beetwen them. + */ +#define SCOUTFS_ALLOC_DATA_LG_THRESH \ + (8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Fill client alloc roots to the target when they fall below the lo + * threshold. + */ +#define SCOUTFS_SERVER_META_FILL_TARGET \ + (256ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_SERVER_META_FILL_LO \ + (64ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) +#define SCOUTFS_SERVER_DATA_FILL_TARGET \ + (4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) +#define SCOUTFS_SERVER_DATA_FILL_LO \ + (1ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) + +/* + * Each of the server meta_alloc roots will try to keep a minimum amount + * of free blocks. The server will use the next root once its current + * root gets this low. It must have room for all the largest allocation + * attempted in a transaction on the server. + */ +#define SCOUTFS_SERVER_META_ALLOC_MIN \ + (SCOUTFS_SERVER_META_FILL_TARGET * 2) + +/* + * A run-time use of a pair of persistent avail/freed roots as a + * metadata allocator. It has the machinery needed to lock and avoid + * recursion when dirtying the list blocks that are used during the + * transaction. + */ +struct scoutfs_alloc { + spinlock_t lock; + struct mutex mutex; + struct scoutfs_block *dirty_avail_bl; + struct scoutfs_block *dirty_freed_bl; + struct scoutfs_alloc_list_head avail; + struct scoutfs_alloc_list_head freed; +}; + +void scoutfs_alloc_init(struct scoutfs_alloc *alloc, + struct scoutfs_alloc_list_head *avail, + struct scoutfs_alloc_list_head *freed); +int scoutfs_alloc_prepare_commit(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri); + +int scoutfs_alloc_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 *blkno); +int scoutfs_free_meta(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, u64 blkno); + +int scoutfs_alloc_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_extent *cached, u64 count, + u64 *blkno_ret, u64 *count_ret); +int scoutfs_free_data(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, u64 blkno, u64 count); + +int scoutfs_alloc_move(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 total); + +int scoutfs_alloc_fill_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *lhead, + struct scoutfs_alloc_root *root, + u64 lo, u64 target); +int scoutfs_alloc_empty_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_root *root, + struct scoutfs_alloc_list_head *lhead); +int scoutfs_alloc_splice_list(struct super_block *sb, + struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_alloc_list_head *dst, + struct scoutfs_alloc_list_head *src); + +bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, + struct scoutfs_alloc *alloc); + +#endif diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 2230ced2..b8686bc9 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -12,6 +12,15 @@ * other places by this macro. Don't forget to update LAST_COUNTER. */ #define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(alloc_alloc_data) \ + EXPAND_COUNTER(alloc_alloc_meta) \ + EXPAND_COUNTER(alloc_free_data) \ + EXPAND_COUNTER(alloc_free_meta) \ + EXPAND_COUNTER(alloc_list_avail_lo) \ + EXPAND_COUNTER(alloc_list_freed_hi) \ + EXPAND_COUNTER(alloc_move) \ + EXPAND_COUNTER(alloc_moved_extent) \ + EXPAND_COUNTER(alloc_stale_cached_list_block) \ EXPAND_COUNTER(block_cache_access) \ EXPAND_COUNTER(block_cache_alloc_failure) \ EXPAND_COUNTER(block_cache_alloc_page_order) \ @@ -185,7 +194,7 @@ EXPAND_COUNTER(trans_commit_timer) \ EXPAND_COUNTER(trans_commit_written) -#define FIRST_COUNTER block_cache_access +#define FIRST_COUNTER alloc_alloc_data #define LAST_COUNTER trans_commit_written #undef EXPAND_COUNTER diff --git a/kmod/src/format.h b/kmod/src/format.h index cd33a7c6..15bd7d92 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -11,6 +11,7 @@ #define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 #define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d #define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 +#define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST 0x8a93ac83 /* * The super block, quorum block, and file data allocation granularity @@ -266,6 +267,53 @@ struct scoutfs_btree_block { #define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) +struct scoutfs_alloc_list_ref { + __le64 blkno; + __le64 seq; +}__packed; + +/* + * first_nr tracks the nr of the first block in the list and is used for + * allocation sizing. total_nr is the sum of the nr of all the blocks in + * the list and is used for calculating total free block counts. + */ +struct scoutfs_alloc_list_head { + struct scoutfs_alloc_list_ref ref; + __le64 total_nr; + __le32 first_nr; +}__packed; + +/* + * While the main allocator uses extent items in btree blocks, metadata + * allocations for a single transaction are recorded in arrays in + * blocks. This limits the number of allocations and frees needed to + * cow and modify the structure. The blocks can be stored in a list + * which lets us create a persistent log of pending frees that are + * generated as we cow btree blocks to insert freed extents. + * + * The array floats in the block so that both adding and removing blknos + * only modifies an index. + */ +struct scoutfs_alloc_list_block { + struct scoutfs_block_header hdr; + struct scoutfs_alloc_list_ref next; + __le32 start; + __le32 nr; + __le64 blknos[0]; /* naturally aligned for sorting */ +}__packed; + +#define SCOUTFS_ALLOC_LIST_MAX_BLOCKS \ + ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_alloc_list_block)) / \ + (member_sizeof(struct scoutfs_alloc_list_block, blknos[0]))) + +/* + * These can safely be initialized to all-zeros. + */ +struct scoutfs_alloc_root { + __le64 total_len; + struct scoutfs_btree_root root; +}__packed; + struct scoutfs_mounted_client_btree_val { __u8 flags; } __packed; diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 5ecb3a3b..a4d58bca 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2490,6 +2490,123 @@ TRACE_EVENT(scoutfs_ext_alloc, STE_ENTRY_ARGS(ext), __entry->ret) ); +TRACE_EVENT(scoutfs_alloc_alloc_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_meta, + TP_PROTO(struct super_block *sb, u64 blkno, int ret), + + TP_ARGS(sb, blkno, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_alloc_data, + TP_PROTO(struct super_block *sb, u64 req, u64 blkno, u64 count, + int ret), + + TP_ARGS(sb, req, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, req) + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->req = req; + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" req %llu blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->req, __entry->blkno, + __entry->count, __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_free_data, + TP_PROTO(struct super_block *sb, u64 blkno, u64 count, int ret), + + TP_ARGS(sb, blkno, count, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, blkno) + __field(__u64, count) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->blkno = blkno; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk(SCSBF" blkno %llu count %llu ret %d", + SCSB_TRACE_ARGS, __entry->blkno, __entry->count, + __entry->ret) +); + +TRACE_EVENT(scoutfs_alloc_move, + TP_PROTO(struct super_block *sb, u64 total, u64 moved, int ret), + + TP_ARGS(sb, total, moved, ret), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, total) + __field(__u64, moved) + __field(int, ret) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->total = total; + __entry->moved = moved; + __entry->ret = ret; + ), + + TP_printk(SCSBF" total %llu moved %llu ret %d", + SCSB_TRACE_ARGS, __entry->total, __entry->moved, + __entry->ret) +); + #endif /* _TRACE_SCOUTFS_H */ /* This part must be outside protection */ From e60f4e7082c6f1431c2fdc2ba879b08348687e70 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Sep 2020 11:43:28 -0700 Subject: [PATCH 875/920] scoutfs: use full extents for data and alloc Previously we'd avoided full extents in file data mapping items because we were deleting items from forest btrees directly. That created deletion items for every version of file extents as they were modified. Now we have the item cache which can remove deleted items from memory when deletion items aren't necessary. By layering file data extents on an extent layer, we can also transition allocators to use extents and fix a lot of problems in the radix block allocator. Most of this change is churn from changing allocator function and struct names. File data extents no longer have to manage loading and storing from and to packed extent items at a fixed granularity. All those loops are torn out and data operations now call the extent layer with their callbacks instead of calling its packed item extent functions. This now means that fallocate and especially restoring offline extents can use larger extents. Small file block allocation now comes from a cached extent which reduces item calls for small file data streaming writes. The big change in the server is to use more root structures to manage recursive modification instead of relying on the allocator to notice and do the right thing. The radix allocator tried to notice when it was actively operating on a root that it was also using to allocate and free metadata blocks. This resulted in a lot of bugs. Instead we now double buffer the server's avail and freed roots so that the server fills and drains the stable roots from the previous transaction. We also double buffer the core fs metadata avail root so that we can increase the time to reuse freed metadata blocks. The server now only moves free extents into client allocators when they fall below a low threshold. This reduces the shared modification of the client's allocator roots which requires cold block reads on both the client and server. Signed-off-by: Zach Brown --- kmod/src/alloc.h | 7 +- kmod/src/btree.c | 56 +- kmod/src/btree.h | 14 +- kmod/src/counters.h | 4 +- kmod/src/data.c | 1380 ++++++++++++-------------------------- kmod/src/data.h | 5 +- kmod/src/forest.c | 16 +- kmod/src/forest.h | 4 +- kmod/src/format.h | 60 +- kmod/src/lock_server.c | 5 +- kmod/src/lock_server.h | 2 +- kmod/src/scoutfs_trace.h | 101 +-- kmod/src/server.c | 328 +++++---- kmod/src/srch.c | 54 +- kmod/src/srch.h | 20 +- kmod/src/trans.c | 18 +- 16 files changed, 789 insertions(+), 1285 deletions(-) diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index 167725b0..7b053756 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -53,9 +53,10 @@ /* * Each of the server meta_alloc roots will try to keep a minimum amount - * of free blocks. The server will use the next root once its current - * root gets this low. It must have room for all the largest allocation - * attempted in a transaction on the server. + * of free blocks. The server will swap roots when its current avail + * falls below the threshold while the freed root is still above it. It + * must have room for all the largest allocation attempted in a + * transaction on the server. */ #define SCOUTFS_SERVER_META_ALLOC_MIN \ (SCOUTFS_SERVER_META_FILL_TARGET * 2) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 5c97e2e0..1d249a3a 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -26,7 +26,7 @@ #include "options.h" #include "msg.h" #include "block.h" -#include "radix.h" +#include "alloc.h" #include "avl.h" #include "hash.h" @@ -674,7 +674,7 @@ static void move_items(struct scoutfs_btree_block *dst, * error. */ static int get_ref_block(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, int flags, struct scoutfs_btree_ref *ref, struct scoutfs_block **bl_ret) @@ -737,7 +737,7 @@ retry: goto out; } - ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); if (ret < 0) goto out; @@ -745,8 +745,8 @@ retry: new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - ret = scoutfs_radix_free(sb, alloc, wri, blkno); - BUG_ON(ret); /* radix should have been dirty */ + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + BUG_ON(ret); ret = PTR_ERR(new_bl); goto out; } @@ -754,11 +754,11 @@ retry: /* free old stable blkno we're about to overwrite */ if (ref && ref->blkno) { - ret = scoutfs_radix_free(sb, alloc, wri, - le64_to_cpu(ref->blkno)); + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(ref->blkno)); if (ret) { - ret = scoutfs_radix_free(sb, alloc, wri, blkno); - BUG_ON(ret); /* radix should have been dirty */ + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + BUG_ON(ret); scoutfs_block_put(sb, new_bl); new_bl = NULL; goto out; @@ -861,7 +861,7 @@ static void init_btree_block(struct scoutfs_btree_block *bt, int level) * Returns -errno, 0 if nothing done, or 1 if we split. */ static int try_split(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, unsigned val_len, @@ -901,8 +901,8 @@ static int try_split(struct super_block *sb, if (!parent) { ret = get_ref_block(sb, alloc, wri, BTW_ALLOC, NULL, &par_bl); if (ret) { - err = scoutfs_radix_free(sb, alloc, wri, - le64_to_cpu(left->hdr.blkno)); + err = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(left->hdr.blkno)); BUG_ON(err); /* radix should have been dirty */ scoutfs_block_put(sb, left_bl); return ret; @@ -937,7 +937,7 @@ static int try_split(struct super_block *sb, * block. */ static int try_join(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_btree_block *parent, @@ -990,9 +990,9 @@ static int try_join(struct super_block *sb, /* update or delete sibling's parent item */ if (le16_to_cpu(sib->nr_items) == 0) { delete_item(parent, sib_par_item, NULL); - ret = scoutfs_radix_free(sb, alloc, wri, - le64_to_cpu(sib->hdr.blkno)); - BUG_ON(ret); /* could have dirtied alloc to avoid error */ + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(sib->hdr.blkno)); + BUG_ON(ret); } else if (move_right) { update_parent_item(parent, sib_par_item, sib); @@ -1003,9 +1003,9 @@ static int try_join(struct super_block *sb, root->height--; root->ref.blkno = bt->hdr.blkno; root->ref.seq = bt->hdr.seq; - ret = scoutfs_radix_free(sb, alloc, wri, - le64_to_cpu(parent->hdr.blkno)); - BUG_ON(ret); /* could have dirtied alloc to avoid error */ + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(parent->hdr.blkno)); + BUG_ON(ret); } scoutfs_block_put(sb, sib_bl); @@ -1219,7 +1219,7 @@ struct btree_walk_key_range { * blocks themselves. */ static int btree_walk(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, int flags, struct scoutfs_key *key, @@ -1464,7 +1464,7 @@ static bool invalid_item(unsigned val_len) * length value. */ int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, @@ -1531,7 +1531,7 @@ static void update_item_value(struct scoutfs_btree_block *bt, * which doesn't fit. */ int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, @@ -1571,7 +1571,7 @@ int scoutfs_btree_update(struct super_block *sb, * which will insert instead of returning -ENOENT. */ int scoutfs_btree_force(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, @@ -1615,7 +1615,7 @@ int scoutfs_btree_force(struct super_block *sb, * found. */ int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key) @@ -1636,8 +1636,8 @@ int scoutfs_btree_delete(struct super_block *sb, if (item) { if (le16_to_cpu(bt->nr_items) == 1) { /* remove final empty block */ - ret = scoutfs_radix_free(sb, alloc, wri, - bl->blkno); + ret = scoutfs_free_meta(sb, alloc, wri, + bl->blkno); if (ret == 0) { root->height = 0; root->ref.blkno = 0; @@ -1753,7 +1753,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, * <0 is returned on error, including -ENOENT if the key isn't present. */ int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key) @@ -1841,7 +1841,7 @@ out: * the caller to resolve this. */ int scoutfs_btree_insert_list(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_btree_item_list *lst) diff --git a/kmod/src/btree.h b/kmod/src/btree.h index c9bd6478..79d4de58 100644 --- a/kmod/src/btree.h +++ b/kmod/src/btree.h @@ -3,7 +3,7 @@ #include -struct scoutfs_radix_allocator; +struct scoutfs_alloc; struct scoutfs_block_writer; struct scoutfs_block; @@ -36,25 +36,25 @@ int scoutfs_btree_lookup(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_insert(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_update(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_force(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key, void *val, unsigned val_len); int scoutfs_btree_delete(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key); @@ -65,7 +65,7 @@ int scoutfs_btree_prev(struct super_block *sb, struct scoutfs_btree_root *root, struct scoutfs_key *key, struct scoutfs_btree_item_ref *iref); int scoutfs_btree_dirty(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_key *key); @@ -77,7 +77,7 @@ int scoutfs_btree_read_items(struct super_block *sb, struct scoutfs_key *end, scoutfs_btree_item_cb cb, void *arg); int scoutfs_btree_insert_list(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_btree_item_list *lst); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index b8686bc9..e3c2e8ae 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -166,7 +166,6 @@ EXPAND_COUNTER(radix_undo_ref) \ EXPAND_COUNTER(radix_walk) \ EXPAND_COUNTER(server_commit_hold) \ - EXPAND_COUNTER(server_commit_prepare) \ EXPAND_COUNTER(server_commit_queue) \ EXPAND_COUNTER(server_commit_worker) \ EXPAND_COUNTER(srch_add_entry) \ @@ -188,8 +187,9 @@ EXPAND_COUNTER(srch_search_xattrs) \ EXPAND_COUNTER(srch_read_stale) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ + EXPAND_COUNTER(trans_commit_dirty_meta_full) \ EXPAND_COUNTER(trans_commit_fsync) \ - EXPAND_COUNTER(trans_commit_full) \ + EXPAND_COUNTER(trans_commit_meta_alloc_low) \ EXPAND_COUNTER(trans_commit_sync_fs) \ EXPAND_COUNTER(trans_commit_timer) \ EXPAND_COUNTER(trans_commit_written) diff --git a/kmod/src/data.c b/kmod/src/data.c index 9360b481..a8bca721 100644 --- a/kmod/src/data.c +++ b/kmod/src/data.c @@ -26,6 +26,7 @@ #include "super.h" #include "inode.h" #include "key.h" +#include "alloc.h" #include "data.h" #include "trans.h" #include "counters.h" @@ -37,657 +38,135 @@ #include "file.h" #include "msg.h" #include "count.h" -#include "radix.h" +#include "ext.h" /* - * Logical file blocks are mapped to device blocks with extents stored - * in items. Each extent item maps a fixed size logical region and can - * contain multiple extent records. Each extent record is packed to - * minimize the space it uses. The logical starting block is implicit - * so sparse extents are stored to skip unmapped blocks, and the mapped - * blkno is encoded as the difference from the previous extent and only - * its set bytes are stored. - * - * To operate on the extents we load their item and unpack them into an - * rbtree of full extent records in memory. Once the memory extents are - * modified they can be packed back into the item. Typically there are - * very few extents that cover the region. - * - * The client is given a radix allocator with trees for allocating - * blocks and recording frees at the start of each transaction. + * We want to amortize work done after dirtying the shared transaction + * accounting, but we don't want to blow out dirty allocator btree + * blocks. Each allocation can dirty quite a few allocator btree blocks + * so we check in pretty often. */ +#define EXTENTS_PER_HOLD 8 struct data_info { struct super_block *sb; - struct rw_semaphore alloc_rwsem; - struct scoutfs_radix_allocator *alloc; + struct mutex mutex; + struct scoutfs_alloc *alloc; struct scoutfs_block_writer *wri; - struct scoutfs_radix_root data_avail; - struct scoutfs_radix_root data_freed; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; + struct scoutfs_extent cached_ext; }; #define DECLARE_DATA_INFO(sb, name) \ struct data_info *name = SCOUTFS_SB(sb)->data_info -static void init_packed_extent_key(struct scoutfs_key *key, u64 ino, - u64 iblock, u8 part) +struct data_ext_args { + u64 ino; + struct inode *inode; + struct scoutfs_lock *lock; +}; + +static void item_from_extent(struct scoutfs_key *key, + struct scoutfs_data_extent_val *dv, u64 ino, + u64 start, u64 len, u64 map, u8 flags) { *key = (struct scoutfs_key) { .sk_zone = SCOUTFS_FS_ZONE, - .skpe_ino = cpu_to_le64(ino), - .sk_type = SCOUTFS_PACKED_EXTENT_TYPE, - .skpe_base = cpu_to_le64(iblock >> SCOUTFS_PACKEXT_BASE_SHIFT), - .skpe_part = part, + .skdx_ino = cpu_to_le64(ino), + .sk_type = SCOUTFS_DATA_EXTENT_TYPE, + .skdx_end = cpu_to_le64(start + len - 1), + .skdx_len = cpu_to_le64(len), }; + dv->blkno = cpu_to_le64(map); + dv->flags = flags; } -/* - * Packed extents are read from items and unpacked into this structure - * in memory so they can be easily manipulated before being packed and - * stored in items. - */ -struct unpacked_extents { - u64 iblock; - struct rb_root extents; - __u8 existing_parts; - bool changed; -}; - -struct unpacked_extent { - struct rb_node node; - u64 iblock; - u64 count; - u64 blkno; - u8 flags; -}; - -static void init_traced_extent(struct scoutfs_traced_extent *te, - u64 iblock, u64 count, u64 blkno, u8 flags) +static void ext_from_item(struct scoutfs_extent *ext, + struct scoutfs_key *key, + struct scoutfs_data_extent_val *dv) { - te->iblock = iblock; - te->count = count; - te->blkno = blkno; - te->flags = flags; + ext->start = le64_to_cpu(key->skdx_end) - + le64_to_cpu(key->skdx_len) + 1; + ext->len = le64_to_cpu(key->skdx_len); + ext->map = le64_to_cpu(dv->blkno); + ext->flags = dv->flags; } -static void copy_traced_extent(struct scoutfs_traced_extent *te, - struct unpacked_extent *ext) +static int data_ext_next(struct super_block *sb, void *arg, u64 start, u64 len, + struct scoutfs_extent *ext) { - te->iblock = ext->iblock; - te->count = ext->count; - te->blkno = ext->blkno; - te->flags = ext->flags; -} - -static u64 ext_last(struct unpacked_extent *ext) -{ - return ext->iblock + ext->count - 1; -} - -/* The first possible iblock in an item that contains the given iblock */ -static u64 first_iblock(u64 iblock) -{ - return iblock & SCOUTFS_PACKEXT_BASE_MASK; -} - -/* The last possible iblock in an item that contains the given iblock */ -static u64 last_iblock(u64 iblock) -{ - return iblock | ~SCOUTFS_PACKEXT_BASE_MASK; -} - -/* - * Extents can merge if they're logically contiguous, have block - * mappings or not which also must be contiguous, and have matching - * flags. - * - * We also require that a given extent's allocation be from only one - * radix bitmap leaf block because the radix freeing functions only - * operate on one leaf block. - */ -static bool extents_merge(struct unpacked_extent *left, - struct unpacked_extent *right) -{ - return (left->iblock + left->count == right->iblock) && - ((!left->blkno && !right->blkno) || - (left->blkno + left->count == right->blkno)) && - (left->flags == right->flags) && - (scoutfs_radix_bit_leaf_nr(left->blkno) == - scoutfs_radix_bit_leaf_nr(right->blkno + right->count - 1)); -} - -static struct unpacked_extent *first_extent(struct unpacked_extents *unpe) -{ - return rb_entry_safe(rb_first(&unpe->extents), - struct unpacked_extent, node); -} - -static struct unpacked_extent *last_extent(struct unpacked_extents *unpe) -{ - return rb_entry_safe(rb_last(&unpe->extents), - struct unpacked_extent, node); -} - -static struct unpacked_extent *next_extent(struct unpacked_extent *ext) -{ - return rb_entry_safe(rb_next(&ext->node), - struct unpacked_extent, node); -} - -static struct unpacked_extent *prev_extent(struct unpacked_extent *ext) -{ - return rb_entry_safe(rb_prev(&ext->node), - struct unpacked_extent, node); -} - -/* - * Find the first extent that intersects the requested range. NULL is - * returned if no extents intersect. - */ -static struct unpacked_extent *find_extent(struct unpacked_extents *unpe, - u64 iblock, u64 last) -{ - - struct rb_node *node = unpe->extents.rb_node; - struct unpacked_extent *ret = NULL; - struct unpacked_extent *ext; - - if (iblock > last) - return NULL; - - while (node) { - ext = rb_entry(node, struct unpacked_extent, node); - - if (last < ext->iblock) { - node = node->rb_left; - } else if (iblock > ext_last(ext)) { - node = node->rb_right; - } else { - ret = ext; - node = node->rb_left; - } - } - - return ret; -} - -static void track_blocks(struct unpacked_extent *ext, s64 delta, - s64 *on, s64 *off) -{ - if (ext->blkno && !(ext->flags & SEF_UNWRITTEN)) - *on += delta; - else if (ext->flags & SEF_OFFLINE) - *off += delta; -} - -static void modify_and_track_count(struct unpacked_extent *ext, u64 count, - s64 *on, s64 *off) -{ - track_blocks(ext, count - ext->count, on, off); - ext->count = count; -} - -/* - * Callers can temporarily insert extents with equal starting iblocks. - * We're careful to insert those to the left so that caller's can find - * these existing overlapping extents by iterating with next. - */ -static void insert_extent(struct unpacked_extents *unpe, - struct unpacked_extent *ins, s64 *on, s64 *off) -{ - struct rb_node **node = &unpe->extents.rb_node; - struct rb_node *parent = NULL; - struct unpacked_extent *ext; - int cmp; - - while (*node) { - parent = *node; - ext = rb_entry(*node, struct unpacked_extent, node); - - cmp = scoutfs_cmp_u64s(ins->iblock, ext->iblock); - if (cmp <= 0) - node = &(*node)->rb_left; - else - node = &(*node)->rb_right; - } - - rb_link_node(&ins->node, parent, node); - rb_insert_color(&ins->node, &unpe->extents); - - track_blocks(ins, ins->count, on, off); -} - -static void remove_extent(struct unpacked_extents *unpe, - struct unpacked_extent *ext, s64 *on, s64 *off) -{ - rb_erase(&ext->node, &unpe->extents); - track_blocks(ext, -ext->count, on, off); - kfree(ext); -} - -static void free_unpacked_extents(struct unpacked_extents *unpe) -{ - struct unpacked_extent *ext; - struct unpacked_extent *tmp; - - if (unpe) { - rbtree_postorder_for_each_entry_safe(ext, tmp, &unpe->extents, - node) { - kfree(ext); - } - kfree(unpe); - } -} - -static int unpack_extent(struct unpacked_extent *ext, u64 iblock, - struct scoutfs_packed_extent *pe, int size, - u64 prev_blkno) -{ - __le64 lediff; - u64 blkno; - u64 diff; - - if (size < sizeof(struct scoutfs_packed_extent) || - size < (sizeof(struct scoutfs_packed_extent) + pe->diff_bytes)) - return 0; - - if (pe->diff_bytes) { - lediff = 0; - memcpy(&lediff, pe->le_blkno_diff, pe->diff_bytes); - diff = le64_to_cpu(lediff); - diff = (diff >> 1) ^ (-(diff & 1)); - blkno = prev_blkno + diff; - } else { - blkno = 0; - } - - ext->iblock = iblock; - ext->blkno = blkno; - ext->count = le16_to_cpu(pe->count); - ext->flags = pe->flags; - - return sizeof(struct scoutfs_packed_extent) + pe->diff_bytes; -} - -static int load_unpacked_extents(struct super_block *sb, u64 ino, - u64 iblock, u64 last, bool empty_enoent, - struct unpacked_extents **unpe_ret, - struct scoutfs_lock *lock) -{ - struct unpacked_extents *unpe = NULL; - struct scoutfs_packed_extent *pe; - struct unpacked_extent *ext; + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; struct scoutfs_key key; - struct scoutfs_key end; - struct rb_node *parent; - struct rb_node **node; - void *buf = NULL; - u64 prev_blkno; - bool saw_final; - int size; + struct scoutfs_key last; int ret; - int p; - *unpe_ret = NULL; + item_from_extent(&last, &dv, args->ino, U64_MAX, 1, 0, 0); + item_from_extent(&key, &dv, args->ino, start, len, 0, 0); - unpe = kzalloc(sizeof(struct unpacked_extents), GFP_NOFS); - if (!unpe) { - ret = -ENOMEM; - goto out; + ret = scoutfs_item_next(sb, &key, &last, &dv, sizeof(dv), args->lock); + if (ret == sizeof(dv)) { + ext_from_item(ext, &key, &dv); + ret = 0; + } else if (ret >= 0) { + ret = -EIO; } - unpe->extents = RB_ROOT; - unpe->changed = true; - /* updated later if _next gives us a greater key */ - unpe->iblock = first_iblock(iblock); - - buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); - if (!buf) { - ret = -ENOMEM; - goto out; - } - - if (last > iblock) - init_packed_extent_key(&end, ino, last, 0); - - parent = NULL; - node = &unpe->extents.rb_node; - prev_blkno = 0; - saw_final = false; - - for (p = 0; !saw_final; p++) { - init_packed_extent_key(&key, ino, iblock, p); - - /* maybe search for next initial item, lookup more parts */ - if (p == 0 && last > iblock) - ret = scoutfs_item_next(sb, &key, &end, buf, - SCOUTFS_PACKEXT_MAX_BYTES, - lock); - else - ret = scoutfs_item_lookup(sb, &key, buf, - SCOUTFS_PACKEXT_MAX_BYTES, - lock); - if (ret < 0) { - if (p == 0 && ret == -ENOENT && empty_enoent) - ret = 0; - goto out; - } - - if (key.skpe_part != p) { - ret = -EIO; /* corruption */ - goto out; - } - - if (p == 0) { - iblock = le64_to_cpu(key.skpe_base) << - SCOUTFS_PACKEXT_BASE_SHIFT; - unpe->iblock = iblock; - } - pe = buf; - size = ret; - - while (size > 0) { - ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); - if (!ext) { - ret = -ENOMEM; - goto out; - } - - ret = unpack_extent(ext, iblock, pe, size, prev_blkno); - if (ret == 0) { /* XXX corruption? */ - kfree(ext); - ret = -EIO; - goto out; - } - - saw_final = pe->final; - pe = (void *)pe + ret; - size -= ret; - - /* sparse packed extents advance iblock */ - if (ext->flags == 0 && ext->blkno == 0) { - iblock += ext->count; - kfree(ext); - ext = NULL; - continue; - } - - iblock += ext->count; - prev_blkno = ext->blkno + ext->count - 1; - - /* building the rbtree from sorted nodes */ - rb_link_node(&ext->node, parent, node); - rb_insert_color(&ext->node, &unpe->extents); - parent = &ext->node; - node = &ext->node.rb_right; - - if (saw_final) - unpe->existing_parts = p + 1; - } - } - - ret = 0; -out: - kfree(buf); if (ret < 0) - free_unpacked_extents(unpe); - else - *unpe_ret = unpe; - + memset(ext, 0, sizeof(struct scoutfs_extent)); return ret; } -static int pack_extent(struct scoutfs_packed_extent *pe, int size, - struct unpacked_extent *ext, - u64 prev_blkno, bool final) +static void add_onoff(struct inode *inode, u64 map, u8 flags, s64 len) { - int diff_bytes; - __le64 lediff; - u64 diff; - int bytes; - int last; - - diff = ext->blkno - prev_blkno; - diff = (diff << 1) ^ ((s64)diff >> 63); /* shift sign extend */ - lediff = cpu_to_le64(diff); - last = fls64(diff); - diff_bytes = (last + 7) >> 3; - - bytes = offsetof(struct scoutfs_packed_extent, - le_blkno_diff[diff_bytes]); - if (size < bytes) - return 0; - - pe->count = cpu_to_le16(ext->count); - pe->diff_bytes = diff_bytes; - pe->flags = ext->flags; - pe->final = !!final; - if (diff_bytes) - memcpy(pe->le_blkno_diff, &lediff, diff_bytes); - - return bytes; -} - -static int store_packed_extents(struct super_block *sb, u64 ino, - struct unpacked_extents *unpe, - struct scoutfs_lock *lock) -{ - struct scoutfs_packed_extent *pe; - struct unpacked_extent *final; - struct unpacked_extent *ext; - struct scoutfs_key key; - void *buf = NULL; - u64 prev_blkno; - u64 iblock; - int space; - int size; - int ret; - int p; - int i; - - if (!unpe->changed) - return 0; - - if (RB_EMPTY_ROOT(&unpe->extents)) { - for (p = 0; p < unpe->existing_parts; p++) { - init_packed_extent_key(&key, ino, unpe->iblock, p); - ret = scoutfs_item_delete(sb, &key, lock); - BUG_ON(ret); /* XXX inconsistent between parts */ - } - unpe->existing_parts = 0; - unpe->changed = false; - return 0; - } - - buf = kmalloc(SCOUTFS_PACKEXT_MAX_BYTES, GFP_NOFS); - if (!buf) { - ret = -ENOMEM; - goto out; - } - - final = last_extent(unpe); - prev_blkno = 0; - - pe = buf; - space = SCOUTFS_PACKEXT_MAX_BYTES; - size = 0; - p = 0; - iblock = unpe->iblock; - - ext = first_extent(unpe); - while (ext) { - /* encode sparse extent to advance iblock */ - if (ext->iblock > iblock && space >= sizeof(*pe)) { - pe->count = cpu_to_le16(ext->iblock - iblock); - pe->diff_bytes = 0; - pe->flags = 0; - pe->final = 0; - pe++; - space -= sizeof(*pe); - size += sizeof(*pe); - iblock = ext->iblock; - } - - /* encode actual extent */ - if (ext->iblock == iblock && - (ret = pack_extent(pe, space, ext, prev_blkno, - ext == final)) > 0) { - pe = (void *)pe + ret; - space -= ret; - size += ret; - iblock += ext->count; - prev_blkno = ext->blkno + ext->count - 1; - ext = next_extent(ext); - if (ext) - continue; - } - - /* store full item or after packing final extent */ - init_packed_extent_key(&key, ino, unpe->iblock, p); - if (p < unpe->existing_parts) - ret = scoutfs_item_update(sb, &key, buf, size, lock); - else - ret = scoutfs_item_create(sb, &key, buf, size, lock); - BUG_ON(ret); /* XXX inconsistent between parts */ - - pe = buf; - space = SCOUTFS_PACKEXT_MAX_BYTES; - size = 0; - p++; - } - - /* delete any remaining previous part items */ - for (i = p; i < unpe->existing_parts; i++) { - init_packed_extent_key(&key, ino, unpe->iblock, i); - ret = scoutfs_item_delete(sb, &key, lock); - BUG_ON(ret); /* XXX inconsistent between parts */ - } - - /* the next store has to know our stored parts */ - unpe->existing_parts = p; - unpe->changed = false; - ret = 0; -out: - kfree(buf); - - return ret; -} - -/* - * Set a logical extent mapping in the unpacked extents for a region of - * a file. The caller's extent is authoritative, any existing - * overlapping extents are trimmed or removed. The new extent can be - * merged with remaining adjacent and compatible extents. - * - * If the caller provides an inode struct then we'll keep the inode - * block counts in sync with flagged extents because updating the inode - * counts won't fail. The caller is expected to keep all other state - * consistent with the extents (i_size, i_blocks, allocator bitmaps). - */ -static int set_extent(struct super_block *sb, struct inode *inode, - u64 ino, struct unpacked_extents *unpe, - u64 iblock, u64 blkno, u64 count, u8 flags) -{ - struct unpacked_extent *split; - struct unpacked_extent *next; - struct unpacked_extent *prev; - struct unpacked_extent *ext; - u64 offset; s64 on = 0; s64 off = 0; - /* make sure the given extent fits entirely within one item */ - if (WARN_ON_ONCE(first_iblock(iblock) != - first_iblock(iblock + count - 1))) - return -EINVAL; + if (map && !(flags & SEF_UNWRITTEN)) + on += len; + else if (flags & SEF_OFFLINE) + off += len; - ext = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); - split = kmalloc(sizeof(struct unpacked_extent), GFP_NOFS); - if (!ext || !split) { - kfree(ext); - kfree(split); - return -ENOMEM; - } - - unpe->changed = true; - - ext->iblock = iblock; - ext->blkno = blkno; - ext->count = count; - ext->flags = flags; - - insert_extent(unpe, ext, &on, &off); - - prev = prev_extent(ext); - - /* splitting an existing extent? */ - if (prev && ext_last(prev) > ext_last(ext)) { - split->iblock = ext_last(ext) + 1; - split->count = ext_last(prev) - split->iblock + 1; - split->blkno = prev->blkno ? - prev->blkno + prev->count - split->count : 0; - split->flags = prev->flags; - - modify_and_track_count(prev, ext->iblock - prev->iblock, - &on, &off); - - insert_extent(unpe, split, &on, &off); - next = split; - split = NULL; - } else { - next = NULL; - } - - /* trimming a prev extent? */ - if (prev && ext_last(prev) >= ext->iblock) { - modify_and_track_count(prev, ext->iblock - prev->iblock, - &on, &off); - } - - /* merging with a prev extent? */ - if (prev && extents_merge(prev, ext)) { - ext->iblock = prev->iblock; - ext->blkno = prev->blkno; - modify_and_track_count(ext, ext->count + prev->count, - &on, &off); - remove_extent(unpe, prev, &on, &off); - } - - /* if didn't split find next, removing any totally within ours */ - if (!next) { - while ((next = next_extent(ext)) && - ext_last(next) <= ext_last(ext)) { - remove_extent(unpe, next, &on, &off); - } - } - - /* trimming a next extent? */ - if (next && next->iblock <= ext_last(ext)) { - offset = (ext_last(ext) + 1) - next->iblock; - next->iblock += offset; - next->blkno = next->blkno ? next->blkno + offset : 0; - modify_and_track_count(next, next->count - offset, - &on, &off); - } - - /* merging with a next extent? */ - if (next && extents_merge(ext, next)) { - modify_and_track_count(ext, ext->count + next->count, - &on, &off); - remove_extent(unpe, next, &on, &off); - } - - /* and finally remove our extent if it was only removing others */ - if (ext->blkno == 0 && ext->flags == 0) - remove_extent(unpe, ext, &on, &off); - - if (inode) - scoutfs_inode_add_onoff(inode, on, off); - - kfree(split); - return 0; + scoutfs_inode_add_onoff(inode, on, off); } +static int data_ext_insert(struct super_block *sb, void *arg, u64 start, + u64 len, u64 map, u8 flags) +{ + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; + struct scoutfs_key key; + int ret; + + item_from_extent(&key, &dv, args->ino, start, len, map, flags); + ret = scoutfs_item_create(sb, &key, &dv, sizeof(dv), args->lock); + if (ret == 0 && args->inode) + add_onoff(args->inode, map, flags, len); + return ret; +} + +static int data_ext_remove(struct super_block *sb, void *arg, u64 start, + u64 len, u64 map, u8 flags) +{ + struct data_ext_args *args = arg; + struct scoutfs_data_extent_val dv; + struct scoutfs_key key; + int ret; + + item_from_extent(&key, &dv, args->ino, start, len, map, flags); + ret = scoutfs_item_delete(sb, &key, args->lock); + if (ret == 0 && args->inode) + add_onoff(args->inode, map, flags, -len); + return ret; +} + +static struct scoutfs_ext_ops data_ext_ops = { + .next = data_ext_next, + .insert = data_ext_insert, + .remove = data_ext_remove, +}; + /* * Find and remove or mark offline the block mappings that intersect * with the caller's range. The caller is responsible for transactions @@ -703,74 +182,75 @@ static s64 truncate_extents(struct super_block *sb, struct inode *inode, struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); - struct unpacked_extents *unpe = NULL; - struct unpacked_extent *ext; - struct scoutfs_traced_extent te; + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent ext; + struct scoutfs_extent tr; u64 offset; - u64 blkno; - u64 count; - u8 flags; s64 ret; - int err; - - ret = load_unpacked_extents(sb, ino, iblock, last, false, &unpe, lock); - if (ret < 0) { - if (ret == -ENOENT) - ret = 0; - goto out; - } + u8 flags; + int i; flags = offline ? SEF_OFFLINE : 0; - ret = 0; - ext = find_extent(unpe, iblock, last); - while (ext && ext->iblock <= last) { + + for (i = 0; iblock <= last; i++) { + if (i == EXTENTS_PER_HOLD) { + ret = iblock; + break; + } + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + break; + } + + /* done if we went past the region */ + if (ext.start > last) { + ret = 0; + break; + } /* nothing to do when already offline and unmapped */ - if ((offline && (ext->flags & SEF_OFFLINE)) && !ext->blkno) { - ext = next_extent(ext); + if ((offline && (ext.flags & SEF_OFFLINE)) && !ext.map) { + iblock = ext.start + ext.len; continue; } - iblock = max(ext->iblock, iblock); - offset = iblock - ext->iblock; - blkno = ext->blkno + offset; - count = min(ext->count - offset, last - iblock + 1); + iblock = max(ext.start, iblock); + offset = iblock - ext.start; - if (ext->blkno) { - down_write(&datinf->alloc_rwsem); - err = scoutfs_radix_free_data(sb, datinf->alloc, - datinf->wri, - &datinf->data_freed, - blkno, count); - up_write(&datinf->alloc_rwsem); - if (err < 0) { - ret = err; + tr.start = iblock; + tr.map = ext.map ? ext.map + offset : 0; + tr.len = min(ext.len - offset, last - iblock + 1); + tr.flags = ext.flags; + + if (tr.map) { + mutex_lock(&datinf->mutex); + ret = scoutfs_free_data(sb, datinf->alloc, + datinf->wri, + &datinf->data_freed, + tr.map, tr.len); + mutex_unlock(&datinf->mutex); + if (ret < 0) break; - } } - init_traced_extent(&te, iblock, count, 0, flags); - trace_scoutfs_data_extent_truncated(sb, ino, &te); + trace_scoutfs_data_extent_truncated(sb, ino, &tr); - err = set_extent(sb, inode, ino, unpe, iblock, 0, count, flags); - BUG_ON(err); /* inconsistent alloc and extents */ + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, + tr.start, tr.len, 0, flags); + BUG_ON(ret); /* inconsistent, could prealloc items */ - /* modifying could have merged and deleted ext, search again */ - iblock += count; - if (iblock > last) - break; - ext = find_extent(unpe, iblock, last); + iblock += tr.len; } - err = store_packed_extents(sb, ino, unpe, lock); - BUG_ON(err); /* inconsistent alloc and extents */ - - /* continue after the packed extent item if we exhausted extents */ - if (ret == 0) - ret = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; -out: - free_unpacked_extents(unpe); return ret; } @@ -844,6 +324,11 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, return ret; } +static inline u64 ext_last(struct scoutfs_extent *ext) +{ + return ext->start + ext->len - 1; +} + /* * The caller is writing to a logical iblock that doesn't have an * allocated extent. @@ -861,141 +346,111 @@ int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, * block. It doesn't work for concurrent stages, releasing behind * staging, sparse files, multi-node writes, etc. fallocate() is always * a better tool to use. - * - * We can mangle the extents so the caller is going to search for the - * intersecting extent again if we succeed. */ static int alloc_block(struct super_block *sb, struct inode *inode, - struct unpacked_extents *unpe, - struct unpacked_extent *ext, u64 iblock, + struct scoutfs_extent *ext, u64 iblock, struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); const u64 ino = scoutfs_ino(inode); - struct scoutfs_traced_extent te; + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent found; + struct scoutfs_extent pre; u64 blkno = 0; u64 online; u64 offline; - u64 last; u8 flags; - int count; + u64 count; int ret; int err; + trace_scoutfs_data_alloc_block_enter(sb, ino, iblock, ext); + /* can only allocate over existing unallocated offline extent */ - if (WARN_ON_ONCE(ext && - !(iblock >= ext->iblock && iblock <= ext_last(ext) && - ext->blkno == 0 && (ext->flags & SEF_OFFLINE)))) + if (WARN_ON_ONCE(ext->len && + !(iblock >= ext->start && iblock <= ext_last(ext) && + ext->map == 0 && (ext->flags & SEF_OFFLINE)))) return -EINVAL; - down_write(&datinf->alloc_rwsem); + mutex_lock(&datinf->mutex); scoutfs_inode_get_onoff(inode, &online, &offline); - if (ext) { + if (ext->len) { /* limit preallocation to remaining existing (offline) extent */ - count = ext->count - (iblock - ext->iblock); + count = ext->len - (iblock - ext->start); flags = ext->flags; } else { - /* otherwise alloc to next extent or end of packed item */ - last = last_iblock(iblock); - ext = find_extent(unpe, iblock, last); - if (ext) - count = ext->iblock - iblock; + /* otherwise alloc to next extent */ + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &found); + if (ret < 0 && ret != -ENOENT) + goto out; + if (found.len && found.start > iblock) + count = found.start - iblock; else - count = last - iblock + 1; + count = SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT; flags = 0; } + /* overall prealloc limit */ + count = min_t(u64, count, SCOUTFS_DATA_EXTEND_PREALLOC_LIMIT); + /* only strictly contiguous extending writes will try to preallocate */ if (iblock > 1 && iblock == online) - count = min_t(u64, iblock, count); + count = min(iblock, count); else count = 1; - ret = scoutfs_radix_alloc_data(sb, datinf->alloc, datinf->wri, - &datinf->data_avail, count, &blkno, - &count); + ret = scoutfs_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, &datinf->cached_ext, + count, &blkno, &count); if (ret < 0) goto out; - ret = set_extent(sb, inode, ino, unpe, iblock, blkno, 1, 0); + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, 1, blkno, 0); if (ret < 0) goto out; - init_traced_extent(&te, iblock, blkno, 1, 0); - trace_scoutfs_data_alloc_block(sb, ino, &te); - if (count > 1) { - ret = set_extent(sb, inode, ino, unpe, iblock + 1, - blkno + 1, count - 1, flags | SEF_UNWRITTEN); + pre.start = iblock + 1; + pre.len = count - 1; + pre.map = blkno + 1; + pre.flags = flags | SEF_UNWRITTEN; + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, pre.start, + pre.len, pre.map, pre.flags); if (ret < 0) { - err = set_extent(sb, inode, ino, unpe, iblock, 0, 1, - flags); + err = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, + 1, 0, flags); BUG_ON(err); /* couldn't restore original */ + goto out; } - - init_traced_extent(&te, iblock + 1, blkno + 1, count - 1, - flags | SEF_UNWRITTEN); - trace_scoutfs_data_prealloc_unwritten(sb, ino, &te); } - ret = store_packed_extents(sb, ino, unpe, lock); - BUG_ON(ret); /* inconsistent previous extent state */ - + /* tell the caller we have a single block, could check next? */ + ext->start = iblock; + ext->len = 1; + ext->map = blkno; + ext->flags = 0; + ret = 0; out: if (ret < 0 && blkno > 0) { - err = scoutfs_radix_free_data(sb, datinf->alloc, datinf->wri, - &datinf->data_freed, - blkno, count); + err = scoutfs_free_data(sb, datinf->alloc, datinf->wri, + &datinf->data_freed, blkno, count); BUG_ON(err); /* leaked free blocks */ } - up_write(&datinf->alloc_rwsem); - - return ret; -} - -/* - * A caller is writing into an unwritten block. This can also be called - * for staging writes so we clear both the unwritten and offline flags. - * - * We don't have to wait for dirty block IO to complete before clearing - * the unwritten flag in metadata because we have strict synchronization - * between data and metadata. All dirty data in the current transaction - * is written before the metadata in the transaction that references it - * is committed. - */ -static int convert_unwritten(struct super_block *sb, struct inode *inode, - struct unpacked_extents *unpe, - struct unpacked_extent *ext, u64 iblock, - struct scoutfs_lock *lock) -{ - struct scoutfs_traced_extent te; - u64 blkno; - u8 ext_fl; - int err; - int ret; - - blkno = ext->blkno + (iblock - ext->iblock); - ext_fl = ext->flags; - - init_traced_extent(&te, iblock, 1, blkno, ext_fl); - trace_scoutfs_data_convert_unwritten(sb, scoutfs_ino(inode), &te); - - ret = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, - blkno, 1, ext_fl & ~(SEF_OFFLINE|SEF_UNWRITTEN)); - if (ret < 0) - goto out; - - ret = store_packed_extents(sb, scoutfs_ino(inode), unpe, lock); - if (ret < 0) { - err = set_extent(sb, inode, scoutfs_ino(inode), unpe, iblock, - blkno, 1, ext_fl); - BUG_ON(err); /* packed and unpacked inconsistent */ + if (ret == 0) { + trace_scoutfs_data_alloc(sb, ino, ext); + trace_scoutfs_data_prealloc(sb, ino, &pre); } -out: + mutex_unlock(&datinf->mutex); + return ret; } @@ -1005,10 +460,10 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, struct scoutfs_inode_info *si = SCOUTFS_I(inode); const u64 ino = scoutfs_ino(inode); struct super_block *sb = inode->i_sb; + struct data_ext_args args; struct scoutfs_lock *lock = NULL; - struct unpacked_extents *unpe = NULL; - struct unpacked_extent *ext = NULL; - DECLARE_TRACED_EXTENT(te); + struct scoutfs_extent ext = {0,}; + struct scoutfs_extent un; u64 offset; int ret; @@ -1021,53 +476,60 @@ static int scoutfs_get_block(struct inode *inode, sector_t iblock, goto out; } - ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); - if (ret < 0) + args.ino = ino; + args.inode = inode; + args.lock = lock; + + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, iblock, 1, &ext); + if (ret == -ENOENT || (ret == 0 && ext.start > iblock)) + memset(&ext, 0, sizeof(ext)); + else if (ret < 0) goto out; - ext = find_extent(unpe, iblock, iblock); + if (ext.len) + trace_scoutfs_data_get_block_found(sb, ino, &ext); /* non-staging callers should have waited on offline blocks */ - if (WARN_ON_ONCE(ext && (ext->flags & SEF_OFFLINE) && !si->staging)) { + if (WARN_ON_ONCE(ext.map && (ext.flags & SEF_OFFLINE) && !si->staging)){ ret = -EIO; goto out; } - /* convert unwritten to written */ - if (create && ext && (ext->flags & SEF_UNWRITTEN)) { - ret = convert_unwritten(sb, inode, unpe, ext, iblock, lock); + /* convert unwritten to written, could be staging */ + if (create && ext.map && (ext.flags & SEF_UNWRITTEN)) { + un.start = iblock; + un.len = 1; + un.map = ext.map + (iblock - ext.start); + un.flags = ext.flags & ~(SEF_OFFLINE|SEF_UNWRITTEN); + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, + un.start, un.len, un.map, un.flags); if (ret == 0) { + ext = un; set_buffer_new(bh); - ext = find_extent(unpe, iblock, iblock); } goto out; } /* allocate and map blocks containing our logical block */ - if (create && (!ext || !ext->blkno)) { - ret = alloc_block(sb, inode, unpe, ext, iblock, lock); - if (ret == 0) { + if (create && !ext.map) { + ret = alloc_block(sb, inode, &ext, iblock, lock); + if (ret == 0) set_buffer_new(bh); - ext = find_extent(unpe, iblock, iblock); - } } else { ret = 0; } out: /* map usable extent, else leave bh unmapped for sparse reads */ - if (ret == 0 && ext && ext->blkno && !(ext->flags & SEF_UNWRITTEN)) { - offset = iblock - ext->iblock; - map_bh(bh, inode->i_sb, ext->blkno + offset); + if (ret == 0 && ext.map && !(ext.flags & SEF_UNWRITTEN)) { + offset = iblock - ext.start; + map_bh(bh, inode->i_sb, ext.map + offset); bh->b_size = min_t(u64, bh->b_size, - (ext->count - offset) << SCOUTFS_BLOCK_SM_SHIFT); + (ext.len - offset) << SCOUTFS_BLOCK_SM_SHIFT); + trace_scoutfs_data_get_block_mapped(sb, ino, &ext); } - if (ext) - copy_traced_extent(&te, ext); - trace_scoutfs_get_block(sb, scoutfs_ino(inode), iblock, create, - &te, ret, bh->b_blocknr, bh->b_size); - free_unpacked_extents(unpe); + &ext, ret, bh->b_blocknr, bh->b_size); return ret; } @@ -1330,74 +792,82 @@ static int scoutfs_write_end(struct file *file, struct address_space *mapping, /* * Try to allocate unwritten extents for any unallocated regions of the - * logical block extent from the caller. We work one packed extent item - * at a time. + * logical block extent from the caller. The caller manages locks and + * transactions. We limit ourselves to a reasonable number of extents + * before returning to open another transaction. * - * We return an error or the numbet of contiguous blocks starting at - * iblock that were successfully processed. + * We return an error or the number of blocks starting at iblock that + * were successfully processed. The caller will continue after those + * blocks until they reach last. */ -static int fallocate_extents(struct super_block *sb, struct inode *inode, +static s64 fallocate_extents(struct super_block *sb, struct inode *inode, u64 iblock, u64 last, struct scoutfs_lock *lock) { DECLARE_DATA_INFO(sb, datinf); - const u64 ino = scoutfs_ino(inode); - struct unpacked_extents *unpe = NULL; - struct unpacked_extent *ext; + struct data_ext_args args = { + .ino = scoutfs_ino(inode), + .inode = inode, + .lock = lock, + }; + struct scoutfs_extent ext; u8 ext_fl; u64 blkno; - int count; - int done; - int ret; + u64 count; + s64 done = 0; + int ret = 0; int err; + int i; - /* work with the extents in one item at a time */ - last = min(last, last_iblock(iblock)); - done = 0; + for (i = 0; iblock <= last && i < EXTENTS_PER_HOLD; i++) { - ret = load_unpacked_extents(sb, ino, iblock, iblock, true, &unpe, lock); - if (ret < 0) - goto out; - - ext = find_extent(unpe, iblock, last); - while (iblock <= last) { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); + if (ret == -ENOENT) + ret = 0; + else if (ret < 0) + break; /* default to allocate to end of region */ count = last - iblock + 1; ext_fl = 0; - if (!ext) { + if (!ext.len) { /* no extent, default alloc from above */ - } else if (ext->iblock <= iblock && ext->blkno) { + } else if (ext.start <= iblock && ext.map) { /* skip portion of allocated extent */ count = min_t(u64, count, - ext->count - (iblock - ext->iblock)); + ext.len - (iblock - ext.start)); iblock += count; done += count; - ext = next_extent(ext); continue; - } else if (ext->iblock <= iblock && !ext->blkno) { + } else if (ext.start <= iblock && !ext.map) { /* alloc portion of unallocated extent */ count = min_t(u64, count, - ext->count - (iblock - ext->iblock)); - ext_fl = ext->flags; + ext.len - (iblock - ext.start)); + ext_fl = ext.flags; - } else if (iblock < ext->iblock) { + } else if (iblock < ext.start) { /* alloc hole until next extent */ - count = min_t(u64, count, ext->iblock - iblock); + count = min_t(u64, count, ext.start - iblock); } - down_write(&datinf->alloc_rwsem); + /* limit allocation attempts */ + count = min_t(u64, count, SCOUTFS_FALLOCATE_ALLOC_LIMIT); - ret = scoutfs_radix_alloc_data(sb, datinf->alloc, datinf->wri, - &datinf->data_avail, count, - &blkno, &count); + mutex_lock(&datinf->mutex); + + ret = scoutfs_alloc_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, + &datinf->cached_ext, + count, &blkno, &count); if (ret == 0) { - ret = set_extent(sb, inode, ino, unpe, iblock, blkno, - count, ext_fl | SEF_UNWRITTEN); + ret = scoutfs_ext_set(sb, &data_ext_ops, &args, iblock, + count, blkno, + ext_fl | SEF_UNWRITTEN); if (ret < 0) { - err = scoutfs_radix_free_data(sb, datinf->alloc, + err = scoutfs_free_data(sb, datinf->alloc, datinf->wri, &datinf->data_avail, blkno, count); @@ -1405,25 +875,18 @@ static int fallocate_extents(struct super_block *sb, struct inode *inode, } } - up_write(&datinf->alloc_rwsem); + mutex_unlock(&datinf->mutex); if (ret < 0) break; iblock += count; done += count; - ext = find_extent(unpe, iblock, last); } - ret = store_packed_extents(sb, ino, unpe, lock); - BUG_ON(ret); /* inconsistent with unpacked and alloc */ - if (ret == 0) ret = done; -out: - free_unpacked_extents(unpe); - return ret; } @@ -1447,7 +910,7 @@ long scoutfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) loff_t end; u64 iblock; u64 last; - int ret; + s64 ret; mutex_lock(&inode->i_mutex); @@ -1527,79 +990,56 @@ out: * on regular files with no data extents. It's used to restore a file * with an offline extent which can then trigger staging. * - * The caller has taken care of locking. We're creating many packed - * extent items which may have to be written in multiple transactions. - * We create exetnts from the front of the file and use the offline - * block count to figure out where to continue from. + * The caller has taken care of locking the inode. We're updating the + * inode offline count as we create the offline extent so we take care + * of the index locking, updating, and transaction. */ int scoutfs_data_init_offline_extent(struct inode *inode, u64 size, struct scoutfs_lock *lock) { struct super_block *sb = inode->i_sb; - struct unpacked_extents *unpe = NULL; - u64 ino = scoutfs_ino(inode); + struct data_ext_args args = { + .ino = scoutfs_ino(inode), + .inode = inode, + .lock = lock, + }; + const u64 count = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SM_SIZE); LIST_HEAD(ind_locks); - bool held = false; - u64 blocks; - u64 iblock; - u64 count; u64 on; u64 off; int ret; - blocks = DIV_ROUND_UP(size, SCOUTFS_BLOCK_SM_SIZE); - scoutfs_inode_get_onoff(inode, &on, &off); - iblock = off; - while (iblock < blocks) { - /* we're updating meta_seq with offline block count */ - ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, - SIC_SETATTR_MORE()); - if (ret < 0) - goto out; - held = true; - - ret = scoutfs_dirty_inode_item(inode, lock); - if (ret < 0) - goto out; - - ret = load_unpacked_extents(sb, ino, iblock, iblock, true, - &unpe, lock); - if (ret < 0) - goto out; - - count = min(blocks - iblock, last_iblock(iblock) - iblock + 1); - - ret = set_extent(sb, inode, ino, unpe, iblock, 0, count, - SEF_OFFLINE); - if (ret < 0) - goto out; - - ret = store_packed_extents(sb, ino, unpe, lock); - if (ret < 0) - goto out; - - free_unpacked_extents(unpe); - unpe = NULL; - - scoutfs_update_inode_item(inode, lock, &ind_locks); - - scoutfs_release_trans(sb); - scoutfs_inode_index_unlock(sb, &ind_locks); - held = false; - - iblock += count; + /* caller should have checked */ + if (on > 0 || off > 0) { + ret = -EINVAL; + goto out; } + /* we're updating meta_seq with offline block count */ + ret = scoutfs_inode_index_lock_hold(inode, &ind_locks, false, + SIC_SETATTR_MORE()); + if (ret < 0) + goto out; + + ret = scoutfs_dirty_inode_item(inode, lock); + if (ret < 0) + goto unlock; + + ret = scoutfs_ext_insert(sb, &data_ext_ops, &args, + 0, count, 0, SEF_OFFLINE); + if (ret < 0) + goto unlock; + + scoutfs_update_inode_item(inode, lock, &ind_locks); + +unlock: + scoutfs_release_trans(sb); + scoutfs_inode_index_unlock(sb, &ind_locks); ret = 0; out: - if (held) { - scoutfs_release_trans(sb); - scoutfs_inode_index_unlock(sb, &ind_locks); - } - free_unpacked_extents(unpe); return ret; } @@ -1607,11 +1047,11 @@ out: * This copies to userspace :/ */ static int fill_extent(struct fiemap_extent_info *fieinfo, - struct unpacked_extent *ext, u32 fiemap_flags) + struct scoutfs_extent *ext, u32 fiemap_flags) { u32 flags; - if (ext->count == 0) + if (ext->len == 0) return 0; flags = fiemap_flags; @@ -1621,9 +1061,9 @@ static int fill_extent(struct fiemap_extent_info *fieinfo, flags |= FIEMAP_EXTENT_UNWRITTEN; return fiemap_fill_next_extent(fieinfo, - ext->iblock << SCOUTFS_BLOCK_SM_SHIFT, - ext->blkno << SCOUTFS_BLOCK_SM_SHIFT, - ext->count << SCOUTFS_BLOCK_SM_SHIFT, + ext->start << SCOUTFS_BLOCK_SM_SHIFT, + ext->map << SCOUTFS_BLOCK_SM_SHIFT, + ext->len << SCOUTFS_BLOCK_SM_SHIFT, flags); } @@ -1638,28 +1078,33 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); struct scoutfs_lock *lock = NULL; - struct unpacked_extents *unpe = NULL; - struct unpacked_extent *ext; - struct unpacked_extent cur; - struct scoutfs_traced_extent te; + struct scoutfs_extent ext; + struct scoutfs_extent cur; + struct data_ext_args args; u32 last_flags; u64 iblock; u64 last; int ret; - if (len == 0) - return 0; + if (len == 0) { + ret = 0; + goto out; + } ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); if (ret) - return ret; + goto out; /* XXX overkill? */ mutex_lock(&inode->i_mutex); ret = scoutfs_lock_inode(sb, SCOUTFS_LOCK_READ, 0, inode, &lock); if (ret) - goto out; + goto unlock; + + args.ino = ino; + args.inode = inode; + args.lock = lock; /* use a dummy extent to track */ memset(&cur, 0, sizeof(cur)); @@ -1668,9 +1113,9 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, iblock = start >> SCOUTFS_BLOCK_SM_SHIFT; last = (start + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; - for (;;) { - ret = load_unpacked_extents(sb, ino, iblock, last, false, - &unpe, lock); + while (iblock <= last) { + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -1678,45 +1123,39 @@ int scoutfs_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, break; } - for (ext = find_extent(unpe, iblock, last); ext; - ext = next_extent(ext)) { + trace_scoutfs_data_fiemap_extent(sb, ino, &ext); - copy_traced_extent(&te, ext); - trace_scoutfs_data_fiemap_extent(sb, ino, &te); - - if (ext->iblock > last) { - /* not setting _LAST, it's for end of file */ - ret = 0; - break; - } - - if (extents_merge(&cur, ext)) { - cur.count += ext->count; - continue; - } - - ret = fill_extent(fieinfo, &cur, 0); - if (ret != 0) - goto out; - cur = *ext; + if (ext.start > last) { + /* not setting _LAST, it's for end of file */ + ret = 0; + break; } - iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; - free_unpacked_extents(unpe); - unpe = NULL; + if (scoutfs_ext_can_merge(&cur, &ext)) { + /* merged extents could be greater than input len */ + cur.len += ext.len; + } else { + ret = fill_extent(fieinfo, &cur, 0); + if (ret != 0) + goto unlock; + cur = ext; + } + + iblock = ext.start + ext.len; } - if (cur.count) + if (cur.len) ret = fill_extent(fieinfo, &cur, last_flags); -out: +unlock: scoutfs_unlock(sb, lock, SCOUTFS_LOCK_READ); mutex_unlock(&inode->i_mutex); - free_unpacked_extents(unpe); - +out: if (ret == 1) ret = 0; + trace_scoutfs_data_fiemap(sb, start, len, ret); + return ret; } @@ -1803,11 +1242,14 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, { struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); + struct data_ext_args args = { + .ino = ino, + .inode = inode, + .lock = lock, + }; DECLARE_DATA_WAIT_ROOT(sb, rt); DECLARE_DATA_WAITQ(inode, wq); - struct unpacked_extents *unpe = NULL; - struct unpacked_extent *ext; - DECLARE_TRACED_EXTENT(te); + struct scoutfs_extent ext = {0,}; u64 iblock; u64 last_block; u64 on; @@ -1834,50 +1276,40 @@ int scoutfs_data_wait_check(struct inode *inode, loff_t pos, loff_t len, last_block = (pos + len - 1) >> SCOUTFS_BLOCK_SM_SHIFT; while(iblock <= last_block) { - - free_unpacked_extents(unpe); - ret = load_unpacked_extents(sb, ino, iblock, last_block, false, - &unpe, lock); + ret = scoutfs_ext_next(sb, &data_ext_ops, &args, + iblock, 1, &ext); if (ret < 0) { if (ret == -ENOENT) ret = 0; - goto out; + break; } - for (ext = find_extent(unpe, iblock, last_block); ext; - ext = next_extent(ext)) { - - if (ext->iblock > last_block) { - ret = 0; - goto out; - } - - if (sef & ext->flags) { - if (dw) { - dw->chg = atomic64_read(&wq->changed); - dw->ino = ino; - dw->iblock = max(iblock, ext->iblock); - dw->op = op; - - spin_lock(&rt->lock); - insert_offline_waiting(&rt->root, dw); - spin_unlock(&rt->lock); - } - - copy_traced_extent(&te, ext); - ret = 1; - goto out; - } - + if (ext.start > last_block) { + ret = 0; + break; } - iblock = unpe->iblock + SCOUTFS_PACKEXT_BLOCKS; + if (sef & ext.flags) { + if (dw) { + dw->chg = atomic64_read(&wq->changed); + dw->ino = ino; + dw->iblock = max(iblock, ext.start); + dw->op = op; + + spin_lock(&rt->lock); + insert_offline_waiting(&rt->root, dw); + spin_unlock(&rt->lock); + } + + ret = 1; + break; + } + + iblock = ext.start + ext.len; } out: - trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &te, ret); - - free_unpacked_extents(unpe); + trace_scoutfs_data_wait_check(sb, ino, pos, len, sef, op, &ext, ret); return ret; } @@ -2019,20 +1451,20 @@ const struct file_operations scoutfs_file_fops = { }; void scoutfs_data_init_btrees(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt) { DECLARE_DATA_INFO(sb, datinf); - down_write(&datinf->alloc_rwsem); + mutex_lock(&datinf->mutex); datinf->alloc = alloc; datinf->wri = wri; datinf->data_avail = lt->data_avail; datinf->data_freed = lt->data_freed; - up_write(&datinf->alloc_rwsem); + mutex_unlock(&datinf->mutex); } void scoutfs_data_get_btrees(struct super_block *sb, @@ -2040,12 +1472,38 @@ void scoutfs_data_get_btrees(struct super_block *sb, { DECLARE_DATA_INFO(sb, datinf); - down_read(&datinf->alloc_rwsem); + mutex_lock(&datinf->mutex); lt->data_avail = datinf->data_avail; lt->data_freed = datinf->data_freed; - up_read(&datinf->alloc_rwsem); + mutex_unlock(&datinf->mutex); +} + +/* + * This should be called before preparing the allocators for the commit + * because it can allocate and free btree blocks in the data allocator. + */ +int scoutfs_data_prepare_commit(struct super_block *sb) +{ + DECLARE_DATA_INFO(sb, datinf); + int ret; + + mutex_lock(&datinf->mutex); + if (datinf->cached_ext.len) { + ret = scoutfs_free_data(sb, datinf->alloc, datinf->wri, + &datinf->data_avail, + datinf->cached_ext.start, + datinf->cached_ext.len); + if (ret == 0) + memset(&datinf->cached_ext, 0, + sizeof(datinf->cached_ext)); + } else { + ret = 0; + } + mutex_unlock(&datinf->mutex); + + return ret; } /* @@ -2055,8 +1513,8 @@ u64 scoutfs_data_alloc_free_bytes(struct super_block *sb) { DECLARE_DATA_INFO(sb, datinf); - return scoutfs_radix_root_free_blocks(sb, &datinf->data_avail) << - SCOUTFS_BLOCK_SM_SHIFT; + return le64_to_cpu(datinf->data_avail.total_len) << + SCOUTFS_BLOCK_SM_SHIFT; } int scoutfs_data_setup(struct super_block *sb) @@ -2069,7 +1527,7 @@ int scoutfs_data_setup(struct super_block *sb) return -ENOMEM; datinf->sb = sb; - init_rwsem(&datinf->alloc_rwsem); + mutex_init(&datinf->mutex); sbi->data_info = datinf; return 0; diff --git a/kmod/src/data.h b/kmod/src/data.h index b4ee7344..09a64fe7 100644 --- a/kmod/src/data.h +++ b/kmod/src/data.h @@ -47,7 +47,7 @@ struct scoutfs_traced_extent { extern const struct address_space_operations scoutfs_file_aops; extern const struct file_operations scoutfs_file_fops; -struct scoutfs_radix_allocator; +struct scoutfs_alloc; struct scoutfs_block_writer; int scoutfs_data_truncate_items(struct super_block *sb, struct inode *inode, @@ -77,11 +77,12 @@ int scoutfs_data_waiting(struct super_block *sb, u64 ino, u64 iblock, unsigned int nr); void scoutfs_data_init_btrees(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt); void scoutfs_data_get_btrees(struct super_block *sb, struct scoutfs_log_trees *lt); +int scoutfs_data_prepare_commit(struct super_block *sb); u64 scoutfs_data_alloc_free_bytes(struct super_block *sb); int scoutfs_data_setup(struct super_block *sb); diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 2d53a1d9..49a255e7 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -20,7 +20,7 @@ #include "lock.h" #include "btree.h" #include "client.h" -#include "radix.h" +#include "alloc.h" #include "block.h" #include "forest.h" #include "hash.h" @@ -53,7 +53,7 @@ struct forest_info { struct mutex mutex; - struct scoutfs_radix_allocator *alloc; + struct scoutfs_alloc *alloc; struct scoutfs_block_writer *wri; struct scoutfs_log_trees our_log; @@ -421,22 +421,22 @@ int scoutfs_forest_set_bloom_bits(struct super_block *sb, if (!ref->blkno || !scoutfs_block_writer_is_dirty(sb, bl)) { - ret = scoutfs_radix_alloc(sb, finf->alloc, finf->wri, &blkno); + ret = scoutfs_alloc_meta(sb, finf->alloc, finf->wri, &blkno); if (ret < 0) goto unlock; new_bl = scoutfs_block_create(sb, blkno); if (IS_ERR(new_bl)) { - err = scoutfs_radix_free(sb, finf->alloc, finf->wri, - blkno); + err = scoutfs_free_meta(sb, finf->alloc, finf->wri, + blkno); BUG_ON(err); /* could have dirtied */ ret = PTR_ERR(new_bl); goto unlock; } if (bl) { - err = scoutfs_radix_free(sb, finf->alloc, finf->wri, - le64_to_cpu(ref->blkno)); + err = scoutfs_free_meta(sb, finf->alloc, finf->wri, + le64_to_cpu(ref->blkno)); BUG_ON(err); /* could have dirtied */ memcpy(new_bl->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); } else { @@ -517,7 +517,7 @@ int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id) * serialized with all writers. */ void scoutfs_forest_init_btrees(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt) { diff --git a/kmod/src/forest.h b/kmod/src/forest.h index 6d0c0c8c..e6e72a4a 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -1,7 +1,7 @@ #ifndef _SCOUTFS_FOREST_H_ #define _SCOUTFS_FOREST_H_ -struct scoutfs_radix_allocator; +struct scoutfs_alloc; struct scoutfs_block_writer; struct scoutfs_block; @@ -28,7 +28,7 @@ int scoutfs_forest_insert_list(struct super_block *sb, int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); void scoutfs_forest_init_btrees(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_log_trees *lt); void scoutfs_forest_get_btrees(struct super_block *sb, diff --git a/kmod/src/format.h b/kmod/src/format.h index 15bd7d92..d5a78ade 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -144,10 +144,10 @@ struct scoutfs_key { #define sks_ino _sk_first #define sks_nr _sk_second -/* packed extents */ -#define skpe_ino _sk_first -#define skpe_base _sk_second -#define skpe_part _sk_fourth +/* data extents */ +#define skdx_ino _sk_first +#define skdx_end _sk_second +#define skdx_len _sk_third /* log trees */ #define sklt_rid _sk_first @@ -163,6 +163,13 @@ struct scoutfs_key { /* mounted clients */ #define skmc_rid _sk_first +/* free extents by blkno */ +#define skfb_end _sk_second +#define skfb_len _sk_third +/* free extents by len */ +#define skfl_neglen _sk_second +#define skfl_blkno _sk_third + struct scoutfs_radix_block { struct scoutfs_block_header hdr; union { @@ -386,8 +393,8 @@ struct scoutfs_srch_block { #define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) struct scoutfs_srch_compact_input { - struct scoutfs_radix_root meta_avail; - struct scoutfs_radix_root meta_freed; + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; __le64 id; __u8 nr; __u8 flags; @@ -395,8 +402,8 @@ struct scoutfs_srch_compact_input { } __packed; struct scoutfs_srch_compact_result { - struct scoutfs_radix_root meta_avail; - struct scoutfs_radix_root meta_freed; + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; __le64 id; __u8 flags; struct scoutfs_srch_file sfl; @@ -413,24 +420,24 @@ struct scoutfs_srch_compact_result { * about item logs, it's about clients making changes to trees. */ struct scoutfs_log_trees { - struct scoutfs_radix_root meta_avail; - struct scoutfs_radix_root meta_freed; + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; - struct scoutfs_radix_root data_avail; - struct scoutfs_radix_root data_freed; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; struct scoutfs_srch_file srch_file; __le64 rid; __le64 nr; } __packed; struct scoutfs_log_trees_val { - struct scoutfs_radix_root meta_avail; - struct scoutfs_radix_root meta_freed; + struct scoutfs_alloc_list_head meta_avail; + struct scoutfs_alloc_list_head meta_freed; struct scoutfs_btree_root item_root; struct scoutfs_btree_ref bloom_ref; - struct scoutfs_radix_root data_avail; - struct scoutfs_radix_root data_freed; + struct scoutfs_alloc_root data_avail; + struct scoutfs_alloc_root data_freed; struct scoutfs_srch_file srch_file; } __packed; @@ -482,6 +489,7 @@ struct scoutfs_bloom_block { #define SCOUTFS_TRANS_SEQ_ZONE 8 #define SCOUTFS_MOUNTED_CLIENT_ZONE 9 #define SCOUTFS_SRCH_ZONE 10 +#define SCOUTFS_FREE_EXTENT_ZONE 11 /* inode index zone */ #define SCOUTFS_INODE_INDEX_META_SEQ_TYPE 1 @@ -498,7 +506,7 @@ struct scoutfs_bloom_block { #define SCOUTFS_READDIR_TYPE 4 #define SCOUTFS_LINK_BACKREF_TYPE 5 #define SCOUTFS_SYMLINK_TYPE 6 -#define SCOUTFS_PACKED_EXTENT_TYPE 7 +#define SCOUTFS_DATA_EXTENT_TYPE 7 /* lock zone, only ever found in lock ranges, never in persistent items */ #define SCOUTFS_RENAME_TYPE 1 @@ -508,6 +516,10 @@ struct scoutfs_bloom_block { #define SCOUTFS_SRCH_BLOCKS_TYPE 2 #define SCOUTFS_SRCH_BUSY_TYPE 3 +/* free extents in allocator btrees in client and server, by blkno or len */ +#define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 +#define SCOUTFS_FREE_EXTENT_LEN_TYPE 2 + /* * The extents that map blocks in a fixed-size logical region of a file * are packed and stored in item values. The packed extents are @@ -539,6 +551,12 @@ struct scoutfs_packed_extent { #define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) #define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE +/* file data extents have start and len in key */ +struct scoutfs_data_extent_val { + __le64 blkno; + __u8 flags; +} __packed; + #define SEF_OFFLINE (1 << 0) #define SEF_UNWRITTEN (1 << 1) #define SEF_UNKNOWN (U8_MAX << 2) @@ -623,10 +641,10 @@ struct scoutfs_super_block { __le64 unmount_barrier; __u8 quorum_count; struct scoutfs_inet_addr server_addr; - struct scoutfs_radix_root core_meta_avail; - struct scoutfs_radix_root core_meta_freed; - struct scoutfs_radix_root core_data_avail; - struct scoutfs_radix_root core_data_freed; + struct scoutfs_alloc_root meta_alloc[2]; + struct scoutfs_alloc_root data_alloc; + struct scoutfs_alloc_list_head server_meta_avail[2]; + struct scoutfs_alloc_list_head server_meta_freed[2]; struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root lock_clients; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index 5ef53cdd..ca590635 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -20,7 +20,6 @@ #include "tseq.h" #include "spbm.h" #include "block.h" -#include "radix.h" #include "btree.h" #include "msg.h" #include "scoutfs_trace.h" @@ -87,7 +86,7 @@ struct lock_server_info { struct scoutfs_tseq_tree tseq_tree; struct dentry *tseq_dentry; - struct scoutfs_radix_allocator *alloc; + struct scoutfs_alloc *alloc; struct scoutfs_block_writer *wri; }; @@ -956,7 +955,7 @@ static void lock_server_tseq_show(struct seq_file *m, * we time them out. */ int scoutfs_lock_server_setup(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index 99c82b8d..c4fe5621 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -12,7 +12,7 @@ int scoutfs_lock_server_response(struct super_block *sb, u64 rid, int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); int scoutfs_lock_server_setup(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri); void scoutfs_lock_server_destroy(struct super_block *sb); diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index a4d58bca..0465dd13 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -170,35 +170,35 @@ TRACE_EVENT(scoutfs_data_fallocate, ); TRACE_EVENT(scoutfs_data_fiemap, - TP_PROTO(struct super_block *sb, __u64 off, int i, __u64 blkno), + TP_PROTO(struct super_block *sb, __u64 start, __u64 len, int ret), - TP_ARGS(sb, off, i, blkno), + TP_ARGS(sb, start, len, ret), TP_STRUCT__entry( SCSB_TRACE_FIELDS - __field(__u64, off) - __field(int, i) - __field(__u64, blkno) + __field(__u64, start) + __field(__u64, len) + __field(int, ret) ), TP_fast_assign( SCSB_TRACE_ASSIGN(sb); - __entry->off = off; - __entry->i = i; - __entry->blkno = blkno; + __entry->start = start; + __entry->len = len; + __entry->ret = ret; ), - TP_printk(SCSBF" blk_off %llu i %u blkno %llu", SCSB_TRACE_ARGS, - __entry->off, __entry->i, __entry->blkno) + TP_printk(SCSBF" start %llu len %llu ret %d", SCSB_TRACE_ARGS, + __entry->start, __entry->len, __entry->ret) ); TRACE_EVENT(scoutfs_get_block, TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, - int create, struct scoutfs_traced_extent *te, + int create, struct scoutfs_extent *ext, int ret, __u64 blkno, size_t size), - TP_ARGS(sb, ino, iblock, create, te, ret, blkno, size), + TP_ARGS(sb, ino, iblock, create, ext, ret, blkno, size), TP_STRUCT__entry( SCSB_TRACE_FIELDS @@ -216,7 +216,7 @@ TRACE_EVENT(scoutfs_get_block, __entry->ino = ino; __entry->iblock = iblock; __entry->create = create; - STE_ASSIGN(ext, te) + STE_ASSIGN(ext, ext) __entry->ret = ret; __entry->blkno = blkno; __entry->size = size; @@ -228,11 +228,35 @@ TRACE_EVENT(scoutfs_get_block, __entry->blkno, __entry->size) ); -TRACE_EVENT(scoutfs_data_file_extent_class, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), +TRACE_EVENT(scoutfs_data_alloc_block_enter, + TP_PROTO(struct super_block *sb, __u64 ino, __u64 iblock, + struct scoutfs_extent *ext), - TP_ARGS(sb, ino, te), + TP_ARGS(sb, ino, iblock, ext), + + TP_STRUCT__entry( + SCSB_TRACE_FIELDS + __field(__u64, ino) + __field(__u64, iblock) + STE_FIELDS(ext) + ), + + TP_fast_assign( + SCSB_TRACE_ASSIGN(sb); + __entry->ino = ino; + __entry->iblock = iblock; + STE_ASSIGN(ext, ext) + ), + + TP_printk(SCSBF" ino %llu iblock %llu ext "STE_FMT, + SCSB_TRACE_ARGS, __entry->ino, __entry->iblock, + STE_ENTRY_ARGS(ext)) +); + +DECLARE_EVENT_CLASS(scoutfs_data_file_extent_class, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + + TP_ARGS(sb, ino, ext), TP_STRUCT__entry( SCSB_TRACE_FIELDS @@ -243,36 +267,35 @@ TRACE_EVENT(scoutfs_data_file_extent_class, TP_fast_assign( SCSB_TRACE_ASSIGN(sb); __entry->ino = ino; - STE_ASSIGN(ext, te) + STE_ASSIGN(ext, ext) ), TP_printk(SCSBF" ino %llu ext "STE_FMT, SCSB_TRACE_ARGS, __entry->ino, STE_ENTRY_ARGS(ext)) ); -DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc_block, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), - TP_ARGS(sb, ino, te) +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_alloc, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) ); -DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_convert_unwritten, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), - TP_ARGS(sb, ino, te) +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) ); -DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_prealloc_unwritten, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), - TP_ARGS(sb, ino, te) +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_get_block_found, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) +); +DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_get_block_mapped, + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) ); DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_extent_truncated, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), - TP_ARGS(sb, ino, te) + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) ); DEFINE_EVENT(scoutfs_data_file_extent_class, scoutfs_data_fiemap_extent, - TP_PROTO(struct super_block *sb, __u64 ino, - struct scoutfs_traced_extent *te), - TP_ARGS(sb, ino, te) + TP_PROTO(struct super_block *sb, __u64 ino, struct scoutfs_extent *ext), + TP_ARGS(sb, ino, ext) ); TRACE_EVENT(scoutfs_data_truncate_items, @@ -300,9 +323,9 @@ TRACE_EVENT(scoutfs_data_truncate_items, TRACE_EVENT(scoutfs_data_wait_check, TP_PROTO(struct super_block *sb, __u64 ino, __u64 pos, __u64 len, - __u8 sef, __u8 op, struct scoutfs_traced_extent *te, int ret), + __u8 sef, __u8 op, struct scoutfs_extent *ext, int ret), - TP_ARGS(sb, ino, pos, len, sef, op, te, ret), + TP_ARGS(sb, ino, pos, len, sef, op, ext, ret), TP_STRUCT__entry( SCSB_TRACE_FIELDS @@ -322,7 +345,7 @@ TRACE_EVENT(scoutfs_data_wait_check, __entry->len = len; __entry->sef = sef; __entry->op = op; - STE_ASSIGN(ext, te) + STE_ASSIGN(ext, ext) __entry->ret = ret; ), diff --git a/kmod/src/server.c b/kmod/src/server.c index 751ea274..48e107ce 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -26,7 +26,6 @@ #include "counters.h" #include "inode.h" #include "block.h" -#include "radix.h" #include "btree.h" #include "scoutfs_trace.h" #include "msg.h" @@ -37,6 +36,7 @@ #include "quorum.h" #include "trans.h" #include "srch.h" +#include "alloc.h" /* * Every active mount can act as the server that listens on a net @@ -66,13 +66,10 @@ struct server_info { struct rw_semaphore commit_rwsem; struct llist_head commit_waiters; struct work_struct commit_work; - bool prepared_commit; /* server tracks seq use */ struct rw_semaphore seq_rwsem; - struct rw_semaphore alloc_rwsem; - struct list_head clients; unsigned long nr_clients; @@ -81,7 +78,15 @@ struct server_info { struct list_head farewell_requests; struct work_struct farewell_work; - struct scoutfs_radix_allocator alloc; + struct mutex alloc_mutex; + /* swap between two fs meta roots to increase time to reuse */ + struct scoutfs_alloc_root *meta_avail; + struct scoutfs_alloc_root *meta_freed; + /* server's meta allocators alternate between persistent heads */ + struct scoutfs_alloc alloc; + int other_ind; + struct scoutfs_alloc_list_head *other_avail; + struct scoutfs_alloc_list_head *other_freed; struct scoutfs_block_writer wri; struct mutex logs_mutex; @@ -119,15 +124,7 @@ static void stop_server(struct server_info *server) /* * Hold the shared rwsem that lets multiple holders modify blocks in the * current commit and prevents the commit worker from acquiring the - * exclusive write lock to write the commit. This can fail for the - * first holder failing to prepare a new commit. - * - * We reclaim the server's stable meta_freed blocks. This is run before - * anything has modified allocators in the server. We know that the - * stable meta_freed tree in the super contains all the stable free - * blocks which can be merged back into avail. We reference the stable - * freed tree in the super because the server allocator's freed tree is - * going to be added to as blocks are freed during the merge. + * exclusive write lock to write the commit. * * This is exported for server components isolated in their own files * (lock_server) and which are not called directly by the server core @@ -135,43 +132,13 @@ static void stop_server(struct server_info *server) */ int scoutfs_server_hold_commit(struct super_block *sb) { - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); - u64 tot; - int ret = 0; scoutfs_inc_counter(sb, server_commit_hold); down_read(&server->commit_rwsem); - while (!server->prepared_commit) { - up_read(&server->commit_rwsem); - down_write(&server->commit_rwsem); - - if (!server->prepared_commit) { - scoutfs_inc_counter(sb, server_commit_prepare); - BUG_ON(scoutfs_block_writer_dirty_bytes(sb, - &server->wri)); - tot = le64_to_cpu(super->core_meta_freed.ref.sm_total); - - ret = scoutfs_radix_merge(sb, &server->alloc, - &server->wri, - &server->alloc.avail, - &server->alloc.freed, - &super->core_meta_freed, - true, tot); - if (ret == 0) - server->prepared_commit = true; - } - - up_write(&server->commit_rwsem); - if (ret < 0) - break; - - down_read(&server->commit_rwsem); - } - - return ret; + return 0; } /* @@ -214,18 +181,6 @@ int scoutfs_server_apply_commit(struct super_block *sb, int err) return err; } -/* - * The caller is about to overwrite a ref to an alloc tree. As we do - * so we update the given super free block counter with the difference - * between the old and new allocator roots. - */ -static void update_free_blocks(__le64 *blocks, struct scoutfs_radix_root *prev, - struct scoutfs_radix_root *next) -{ - le64_add_cpu(blocks, le64_to_cpu(next->ref.sm_total) - - le64_to_cpu(prev->ref.sm_total)); -} - void scoutfs_server_get_roots(struct super_block *sb, struct scoutfs_net_roots *roots) { @@ -286,6 +241,31 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); + /* make sure next avail has sufficient blocks */ + ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + server->other_avail, + server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET); + if (ret) { + scoutfs_err(sb, "server error refilling avail: %d", ret); + goto out; + } + + /* merge freed blocks into extents, might be partial */ + ret = scoutfs_alloc_empty_list(sb, &server->alloc, &server->wri, + server->meta_freed, + server->other_freed); + if (ret) { + scoutfs_err(sb, "server error emptying freed: %d", ret); + goto out; + } + + ret = scoutfs_alloc_prepare_commit(sb, &server->alloc, &server->wri); + if (ret < 0) { + scoutfs_err(sb, "server error prepare alloc commit: %d", ret); + goto out; + } ret = scoutfs_block_writer_write(sb, &server->wri); if (ret) { @@ -293,13 +273,8 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - update_free_blocks(&super->free_meta_blocks, &super->core_meta_avail, - &server->alloc.avail); - update_free_blocks(&super->free_meta_blocks, &super->core_meta_freed, - &server->alloc.freed); - - super->core_meta_avail = server->alloc.avail; - super->core_meta_freed = server->alloc.freed; + super->server_meta_avail[server->other_ind ^ 1] = server->alloc.avail; + super->server_meta_freed[server->other_ind ^ 1] = server->alloc.freed; ret = scoutfs_write_super(sb, super); if (ret) { @@ -307,9 +282,23 @@ static void scoutfs_server_commit_func(struct work_struct *work) goto out; } - server->prepared_commit = false; set_roots(server, &super->fs_root, &super->logs_root, &super->srch_root); + + /* swizzle the active and idle server alloc/freed heads */ + server->other_ind ^= 1; + server->alloc.avail = super->server_meta_avail[server->other_ind ^ 1]; + server->alloc.freed = super->server_meta_freed[server->other_ind ^ 1]; + server->other_avail = &super->server_meta_avail[server->other_ind]; + server->other_freed = &super->server_meta_freed[server->other_ind]; + + /* swap avail/free if avail gets low and freed is high */ + if (le64_to_cpu(server->meta_avail->total_len) <= + SCOUTFS_SERVER_META_ALLOC_MIN && + le64_to_cpu(server->meta_freed->total_len) > + SCOUTFS_SERVER_META_ALLOC_MIN) + swap(server->meta_avail, server->meta_freed); + ret = 0; out: node = llist_del_all(&server->commit_waiters); @@ -362,6 +351,34 @@ out: return scoutfs_net_response(sb, conn, cmd, id, ret, &ial, sizeof(ial)); } +/* + * Refill the destination root if it's fallen below the lo threshold by + * moving from the src root to bring it up to the target. + */ +static int alloc_move_refill(struct super_block *sb, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src, u64 lo, u64 target) +{ + DECLARE_SERVER_INFO(sb, server); + + if (le64_to_cpu(dst->total_len) >= lo) + return 0; + + return scoutfs_alloc_move(sb, &server->alloc, &server->wri, dst, src, + min(target - le64_to_cpu(dst->total_len), + le64_to_cpu(src->total_len))); +} + +static int alloc_move_empty(struct super_block *sb, + struct scoutfs_alloc_root *dst, + struct scoutfs_alloc_root *src) +{ + DECLARE_SERVER_INFO(sb, server); + + return scoutfs_alloc_move(sb, &server->alloc, &server->wri, + dst, src, le64_to_cpu(src->total_len)); +} + /* * Give the client roots to all the trees that they'll use to build * their transaction. @@ -383,8 +400,6 @@ static int server_get_log_trees(struct super_block *sb, struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees lt; struct scoutfs_key key; - u64 count; - u64 target; int ret; if (arg_len != 0) { @@ -422,50 +437,25 @@ static int server_get_log_trees(struct super_block *sb, key.sklt_rid = cpu_to_le64(rid); key.sklt_nr = cpu_to_le64(1); memset(<v, 0, sizeof(ltv)); - scoutfs_radix_root_init(sb, <v.meta_avail, true); - scoutfs_radix_root_init(sb, <v.meta_freed, true); - scoutfs_radix_root_init(sb, <v.data_avail, false); - scoutfs_radix_root_init(sb, <v.data_freed, false); } - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.avail, - <v.meta_freed, <v.meta_freed, true, - le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &super->core_data_avail, - <v.data_freed, <v.data_freed, false, - le64_to_cpu(ltv.data_freed.ref.sm_total)); + /* return freed to server for emptying, refill avail */ + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <v.meta_freed) ?: + alloc_move_empty(sb, &super->data_alloc, <v.data_freed) ?: + scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + <v.meta_avail, server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET) ?: + alloc_move_refill(sb, <v.data_avail, &super->data_alloc, + SCOUTFS_SERVER_DATA_FILL_LO, + SCOUTFS_SERVER_DATA_FILL_TARGET); + mutex_unlock(&server->alloc_mutex); if (ret < 0) goto unlock; - /* ensure client has enough free metadata blocks for a transaction */ - target = (64*1024*1024) / SCOUTFS_BLOCK_LG_SIZE; - if (le64_to_cpu(ltv.meta_avail.ref.sm_total) < target) { - count = target - le64_to_cpu(ltv.meta_avail.ref.sm_total); - - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - <v.meta_avail, - &server->alloc.avail, - &server->alloc.avail, true, count); - if (ret < 0) - goto unlock; - } - - /* ensure client has enough free data blocks for a transaction */ - target = SCOUTFS_TRANS_DATA_ALLOC_HWM / SCOUTFS_BLOCK_SM_SIZE; - if (le64_to_cpu(ltv.data_avail.ref.sm_total) < target) { - count = target - le64_to_cpu(ltv.data_avail.ref.sm_total); - - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - <v.data_avail, - &super->core_data_avail, - &super->core_data_avail, false, - count); - if (ret < 0) - goto unlock; - } - /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, &super->logs_root, &key, <v, sizeof(ltv)); @@ -553,21 +543,12 @@ static int server_commit_log_trees(struct super_block *sb, goto unlock; } - update_free_blocks(&super->free_meta_blocks, <v.meta_avail, - <->meta_avail); - update_free_blocks(&super->free_meta_blocks, <v.meta_freed, - <->meta_freed); - update_free_blocks(&super->free_data_blocks, <v.data_avail, - <->data_avail); - update_free_blocks(&super->free_data_blocks, <v.data_freed, - <->data_freed); - ltv.meta_avail = lt->meta_avail; ltv.meta_freed = lt->meta_freed; - ltv.item_root = lt->item_root; - ltv.bloom_ref = lt->bloom_ref; ltv.data_avail = lt->data_avail; ltv.data_freed = lt->data_freed; + ltv.item_root = lt->item_root; + ltv.bloom_ref = lt->bloom_ref; ltv.srch_file = lt->srch_file; ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, @@ -638,7 +619,6 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) int err; mutex_lock(&server->logs_mutex); - down_write(&server->alloc_rwsem); /* find the client's existing item */ scoutfs_key_init_log_trees(&key, rid, 0); @@ -662,32 +642,25 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) /* * All of these can return errors after having modified the - * radix trees. We have to try and update the roots in the + * allocator trees. We have to try and update the roots in the * log item. */ - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.avail, - <v.meta_avail, <v.meta_avail, true, - le64_to_cpu(ltv.meta_avail.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.avail, - <v.meta_freed, <v.meta_freed, true, - le64_to_cpu(ltv.meta_freed.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &super->core_data_avail, - <v.data_avail, <v.data_avail, false, - le64_to_cpu(ltv.data_avail.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &super->core_data_avail, - <v.data_freed, <v.data_freed, false, - le64_to_cpu(ltv.data_freed.ref.sm_total)); + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <v.meta_freed) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, + <v.meta_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <v.data_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <v.data_freed); + mutex_unlock(&server->alloc_mutex); err = scoutfs_btree_update(sb, &server->alloc, &server->wri, &super->logs_root, &key, <v, sizeof(ltv)); BUG_ON(err != 0); /* alloc and log item roots out of sync */ out: - up_write(&server->alloc_rwsem); mutex_unlock(&server->logs_mutex); return ret; @@ -892,14 +865,14 @@ static int server_statfs(struct super_block *sb, nstatfs.next_ino = super->next_ino; spin_unlock(&sbi->next_ino_lock); - down_read(&server->alloc_rwsem); + mutex_lock(&server->alloc_mutex); nstatfs.total_blocks = le64_lg_to_sm(super->total_meta_blocks); le64_add_cpu(&nstatfs.total_blocks, le64_to_cpu(super->total_data_blocks)); nstatfs.bfree = le64_lg_to_sm(super->free_meta_blocks); le64_add_cpu(&nstatfs.bfree, le64_to_cpu(super->free_data_blocks)); - up_read(&server->alloc_rwsem); + mutex_unlock(&server->alloc_mutex); ret = 0; } else { ret = -EINVAL; @@ -1002,8 +975,6 @@ static int server_srch_get_compact(struct super_block *sb, int i; memset(&scin, 0, sizeof(scin)); - scoutfs_radix_root_init(sb, &scin.meta_avail, true); - scoutfs_radix_root_init(sb, &scin.meta_freed, true); if (arg_len != 0) { ret = -EINVAL; @@ -1028,9 +999,11 @@ static int server_srch_get_compact(struct super_block *sb, for (i = 0; i < scin.nr; i++) blocks += le64_to_cpu(scin.sfl[i].blocks); blocks *= 3; - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &scin.meta_avail, &server->alloc.avail, - &server->alloc.avail, true, blocks); + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, + &scin.meta_avail, server->meta_avail, + blocks, blocks); + mutex_unlock(&server->alloc_mutex); if (ret < 0) goto apply; @@ -1047,6 +1020,12 @@ out: &scin, sizeof(scin)); } +/* + * Commit the client's compaction. Their freed allocator contains the + * source srch files blocks that are currently in use which can't be + * available for allocation until after the commit. We move them into + * freed so they won't satisfy allocations. + */ static int server_srch_commit_compact(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) @@ -1056,8 +1035,8 @@ static int server_srch_commit_compact(struct super_block *sb, struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; struct scoutfs_srch_compact_result *scres; - struct scoutfs_radix_root av; - struct scoutfs_radix_root fr; + struct scoutfs_alloc_list_head av; + struct scoutfs_alloc_list_head fr; int ret; scres = arg; @@ -1078,15 +1057,12 @@ static int server_srch_commit_compact(struct super_block *sb, if (ret < 0) /* XXX very bad, leaks allocators */ goto apply; - /* XXX like all merges, doesn't reclaim allocator blocks themselves */ - - /* merge the client's allocators into freed, commit before reuse */ - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.freed, &av, &av, true, - le64_to_cpu(av.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.freed, &fr, &fr, true, - le64_to_cpu(fr.ref.sm_total)); + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &av) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &fr); + mutex_unlock(&server->alloc_mutex); apply: ret = scoutfs_server_apply_commit(sb, ret); out: @@ -1149,14 +1125,15 @@ static int delete_mounted_client(struct super_block *sb, u64 rid) /* * Remove all the busy items for srch compactions that the mount might - * have been responsible for and reclaim all their allocators. + * have been responsible for and reclaim all their allocators. The freed + * allocator could still contain stable srch file blknos. */ static int cancel_srch_compact(struct super_block *sb, u64 rid) { DECLARE_SERVER_INFO(sb, server); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_radix_root av; - struct scoutfs_radix_root fr; + struct scoutfs_alloc_list_head av; + struct scoutfs_alloc_list_head fr; int ret; for (;;) { @@ -1172,12 +1149,14 @@ static int cancel_srch_compact(struct super_block *sb, u64 rid) break; } - ret = scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.freed, &av, &av, true, - le64_to_cpu(av.ref.sm_total)) ?: - scoutfs_radix_merge(sb, &server->alloc, &server->wri, - &server->alloc.freed, &fr, &fr, true, - le64_to_cpu(fr.ref.sm_total)); + mutex_lock(&server->alloc_mutex); + ret = scoutfs_alloc_splice_list(sb, &server->alloc, + &server->wri, + server->other_freed, &av) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, + &server->wri, + server->other_freed, &fr); + mutex_unlock(&server->alloc_mutex); if (WARN_ON_ONCE(ret < 0)) break; } @@ -1650,10 +1629,27 @@ static void scoutfs_server_worker(struct work_struct *work) set_roots(server, &super->fs_root, &super->logs_root, &super->srch_root); - scoutfs_radix_init_alloc(&server->alloc, &super->core_meta_avail, - &super->core_meta_freed); scoutfs_block_writer_init(sb, &server->wri); + /* prepare server alloc for this transaction, larger first */ + if (le64_to_cpu(super->server_meta_avail[0].total_nr) < + le64_to_cpu(super->server_meta_avail[1].total_nr)) + server->other_ind = 0; + else + server->other_ind = 1; + scoutfs_alloc_init(&server->alloc, + &super->server_meta_avail[server->other_ind ^ 1], + &super->server_meta_freed[server->other_ind ^ 1]); + server->other_avail = &super->server_meta_avail[server->other_ind]; + server->other_freed = &super->server_meta_freed[server->other_ind]; + + /* use largest meta_alloc to start */ + server->meta_avail = &super->meta_alloc[0]; + server->meta_freed = &super->meta_alloc[1]; + if (le64_to_cpu(server->meta_freed->total_len) > + le64_to_cpu(server->meta_avail->total_len)) + swap(server->meta_avail, server->meta_freed); + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); if (ret) goto shutdown; @@ -1783,11 +1779,11 @@ int scoutfs_server_setup(struct super_block *sb) init_llist_head(&server->commit_waiters); INIT_WORK(&server->commit_work, scoutfs_server_commit_func); init_rwsem(&server->seq_rwsem); - init_rwsem(&server->alloc_rwsem); INIT_LIST_HEAD(&server->clients); mutex_init(&server->farewell_mutex); INIT_LIST_HEAD(&server->farewell_requests); INIT_WORK(&server->farewell_work, farewell_worker); + mutex_init(&server->alloc_mutex); mutex_init(&server->logs_mutex); mutex_init(&server->srch_mutex); seqcount_init(&server->roots_seqcount); diff --git a/kmod/src/srch.c b/kmod/src/srch.c index db45fba8..2ecae4fb 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -23,7 +23,7 @@ #include "format.h" #include "counters.h" #include "block.h" -#include "radix.h" +#include "alloc.h" #include "srch.h" #include "btree.h" #include "spbm.h" @@ -309,7 +309,7 @@ enum { GFB_DIRTY = (1 << 1), }; static int get_file_block(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, int gfb, u64 blk, struct scoutfs_block **bl_ret) @@ -335,7 +335,7 @@ static int get_file_block(struct super_block *sb, goto out; } - ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); if (ret < 0) goto out; @@ -383,7 +383,7 @@ static int get_file_block(struct super_block *sb, /* allocate a new block if we need it */ if (!ref->blkno || ((gfb & GFB_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl))) { - ret = scoutfs_radix_alloc(sb, alloc, wri, &blkno); + ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); if (ret < 0) goto out; @@ -395,8 +395,8 @@ static int get_file_block(struct super_block *sb, if (bl) { /* cow old block if we have one */ - ret = scoutfs_radix_free(sb, alloc, wri, - bl->blkno); + ret = scoutfs_free_meta(sb, alloc, wri, + bl->blkno); if (ret) goto out; @@ -442,7 +442,7 @@ out: /* return allocated blkno on error */ if (blkno > 0) { - err = scoutfs_radix_free(sb, alloc, wri, blkno); + err = scoutfs_free_meta(sb, alloc, wri, blkno); BUG_ON(err); /* radix should have been dirty */ } @@ -460,7 +460,7 @@ out: } int scoutfs_srch_add(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, struct scoutfs_block **bl_ret, @@ -988,7 +988,7 @@ out: * it's large enough. */ int scoutfs_srch_rotate_log(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_srch_file *sfl) @@ -1018,13 +1018,13 @@ int scoutfs_srch_rotate_log(struct super_block *sb, * items. */ int scoutfs_srch_get_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_input *scin) { - struct scoutfs_srch_compact_input busy_scin = {{0,}}; + struct scoutfs_srch_compact_input busy_scin = {{{0,}}}; struct scoutfs_srch_file sfl; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_spbm busy; @@ -1147,7 +1147,7 @@ out: * copy. */ int scoutfs_srch_update_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_input *scin) @@ -1160,7 +1160,7 @@ int scoutfs_srch_update_compact(struct super_block *sb, } static int mod_srch_items(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u8 scom_flags, bool ins, struct scoutfs_srch_file *sfls, int nr) @@ -1213,12 +1213,12 @@ static int mod_srch_items(struct super_block *sb, * We give the caller the allocator trees to merge if we return success. */ int scoutfs_srch_commit_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_result *scres, - struct scoutfs_radix_root *av, - struct scoutfs_radix_root *fr) + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr) { struct scoutfs_srch_compact_input scin; SCOUTFS_BTREE_ITEM_REF(iref); @@ -1268,11 +1268,11 @@ out: * allocators. Returns -ENOENT when there are no more items. */ int scoutfs_srch_cancel_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_radix_root *av, - struct scoutfs_radix_root *fr) + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr) { struct scoutfs_srch_compact_input scin; SCOUTFS_BTREE_ITEM_REF(iref); @@ -1331,7 +1331,7 @@ typedef int (*kway_next_func_t)(struct super_block *sb, struct scoutfs_srch_entry *sre_ret, void *arg); static int kway_merge(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, kway_next_func_t kway_next, void **args, int nr) @@ -1526,7 +1526,7 @@ static void swap_page_sre(void *A, void *B, int size) * typically, ~10x worst case). */ static int compact_logs(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl_out, struct scoutfs_srch_file *sfls, int nr_sfls) @@ -1715,7 +1715,7 @@ out: * which reads blocks and decodes entries. */ static int compact_sorted(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl_out, struct scoutfs_srch_file *sfls, int nr) @@ -1760,7 +1760,7 @@ out: * up our entire operation, partial state doesn't matter. */ static int free_file(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl) { @@ -1818,7 +1818,7 @@ static int free_file(struct super_block *sb, if (blkno == 0) continue; - ret = scoutfs_radix_free(sb, alloc, wri, blkno); + ret = scoutfs_free_meta(sb, alloc, wri, blkno); if (ret < 0) goto out; scoutfs_inc_counter(sb, srch_compact_free_block); @@ -1830,7 +1830,7 @@ static int free_file(struct super_block *sb, } free_root: - ret = scoutfs_radix_free(sb, alloc, wri, le64_to_cpu(sfl->ref.blkno)); + ret = scoutfs_free_meta(sb, alloc, wri, le64_to_cpu(sfl->ref.blkno)); if (ret < 0) goto out; @@ -1868,7 +1868,7 @@ static void scoutfs_srch_compact_worker(struct work_struct *work) struct srch_info *srinf = container_of(work, struct srch_info, compact_dwork.work); struct super_block *sb = srinf->sb; - struct scoutfs_radix_allocator alloc; + struct scoutfs_alloc alloc; struct scoutfs_srch_compact_result scres; struct scoutfs_srch_compact_input scin; struct scoutfs_block_writer wri; @@ -1883,7 +1883,7 @@ static void scoutfs_srch_compact_worker(struct work_struct *work) if (ret < 0 || scin.nr == 0) goto out; - scoutfs_radix_init_alloc(&alloc, &scin.meta_avail, &scin.meta_freed); + scoutfs_alloc_init(&alloc, &scin.meta_avail, &scin.meta_freed); if (scin.flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) ret = compact_logs(sb, &alloc, &wri, &scres.sfl, diff --git a/kmod/src/srch.h b/kmod/src/srch.h index 937692e9..97604bd6 100644 --- a/kmod/src/srch.h +++ b/kmod/src/srch.h @@ -22,7 +22,7 @@ struct scoutfs_srch_rb_node { node = rb_next(node)) int scoutfs_srch_add(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, struct scoutfs_block **bl_ret, @@ -34,34 +34,34 @@ int scoutfs_srch_search_xattrs(struct super_block *sb, u64 hash, u64 ino, u64 last_ino, bool *done); int scoutfs_srch_rotate_log(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, struct scoutfs_srch_file *sfl); int scoutfs_srch_get_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_input *scin_ret); int scoutfs_srch_update_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_input *scin); int scoutfs_srch_commit_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, struct scoutfs_srch_compact_result *scres, - struct scoutfs_radix_root *av, - struct scoutfs_radix_root *fr); + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr); int scoutfs_srch_cancel_compact(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, + struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_radix_root *av, - struct scoutfs_radix_root *fr); + struct scoutfs_alloc_list_head *av, + struct scoutfs_alloc_list_head *fr); void scoutfs_srch_destroy(struct super_block *sb); int scoutfs_srch_setup(struct super_block *sb); diff --git a/kmod/src/trans.c b/kmod/src/trans.c index af659bd9..9f36a19d 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -25,7 +25,7 @@ #include "counters.h" #include "client.h" #include "inode.h" -#include "radix.h" +#include "alloc.h" #include "block.h" #include "msg.h" #include "item.h" @@ -66,7 +66,7 @@ struct trans_info { bool writing; struct scoutfs_log_trees lt; - struct scoutfs_radix_allocator alloc; + struct scoutfs_alloc alloc; struct scoutfs_block_writer wri; }; @@ -112,8 +112,7 @@ int scoutfs_trans_get_log_trees(struct super_block *sb) ret = scoutfs_client_get_log_trees(sb, <); if (ret == 0) { tri->lt = lt; - scoutfs_radix_init_alloc(&tri->alloc, <.meta_avail, - <.meta_freed); + scoutfs_alloc_init(&tri->alloc, <.meta_avail, <.meta_freed); scoutfs_block_writer_init(sb, &tri->wri); scoutfs_forest_init_btrees(sb, &tri->alloc, &tri->wri, <); @@ -195,6 +194,9 @@ void scoutfs_trans_write_func(struct work_struct *work) /* XXX this all needs serious work for dealing with errors */ ret = (s = "data submit", scoutfs_inode_walk_writeback(sb, true)) ?: (s = "item dirty", scoutfs_item_write_dirty(sb)) ?: + (s = "data prepare", scoutfs_data_prepare_commit(sb)) ?: + (s = "alloc prepare", scoutfs_alloc_prepare_commit(sb, + &tri->alloc, &tri->wri)) ?: (s = "meta write", scoutfs_block_writer_write(sb, &tri->wri)) ?: (s = "data wait", scoutfs_inode_walk_writeback(sb, false)) ?: (s = "commit log trees", commit_btrees(sb)) ?: @@ -369,7 +371,13 @@ static bool acquired_hold(struct super_block *sb, /* XXX arbitrarily limit to 8 meg transactions */ if (scoutfs_item_dirty_bytes(sb) >= (8 * 1024 * 1024)) { - scoutfs_inc_counter(sb, trans_commit_full); + scoutfs_inc_counter(sb, trans_commit_dirty_meta_full); + queue_trans_work(sbi); + goto out; + } + + if (scoutfs_alloc_meta_lo_thresh(sb, &tri->alloc)) { + scoutfs_inc_counter(sb, trans_commit_meta_alloc_low); queue_trans_work(sbi); goto out; } From c61175e79678506f2a31ec7b90d1ad8e370f15fe Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 21 Sep 2020 14:28:28 -0700 Subject: [PATCH 876/920] scoutfs: remove unused radix code Remove the radix allocator that was added as we expermented with packed extent items. It didn't work out. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 - kmod/src/counters.h | 18 - kmod/src/format.h | 24 - kmod/src/radix.c | 1514 -------------------------------------- kmod/src/radix.h | 45 -- kmod/src/scoutfs_trace.h | 168 ----- 6 files changed, 1770 deletions(-) delete mode 100644 kmod/src/radix.c delete mode 100644 kmod/src/radix.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index bfd8f38a..5af7fdd0 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -31,7 +31,6 @@ scoutfs-y += \ options.o \ per_task.o \ quorum.o \ - radix.o \ scoutfs_trace.o \ server.o \ spbm.o \ diff --git a/kmod/src/counters.h b/kmod/src/counters.h index e3c2e8ae..140a07f5 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -147,24 +147,6 @@ EXPAND_COUNTER(quorum_write_block) \ EXPAND_COUNTER(quorum_write_block_error) \ EXPAND_COUNTER(quorum_fenced) \ - EXPAND_COUNTER(radix_alloc) \ - EXPAND_COUNTER(radix_alloc_data) \ - EXPAND_COUNTER(radix_block_cow) \ - EXPAND_COUNTER(radix_block_read) \ - EXPAND_COUNTER(radix_complete_dirty_block) \ - EXPAND_COUNTER(radix_create_synth) \ - EXPAND_COUNTER(radix_free) \ - EXPAND_COUNTER(radix_free_data) \ - EXPAND_COUNTER(radix_enospc_data) \ - EXPAND_COUNTER(radix_enospc_meta) \ - EXPAND_COUNTER(radix_enospc_synth) \ - EXPAND_COUNTER(radix_inconsistent_eio) \ - EXPAND_COUNTER(radix_inconsistent_ref) \ - EXPAND_COUNTER(radix_merge) \ - EXPAND_COUNTER(radix_merge_bad_clean_input) \ - EXPAND_COUNTER(radix_merge_empty) \ - EXPAND_COUNTER(radix_undo_ref) \ - EXPAND_COUNTER(radix_walk) \ EXPAND_COUNTER(server_commit_hold) \ EXPAND_COUNTER(server_commit_queue) \ EXPAND_COUNTER(server_commit_worker) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index d5a78ade..428c94e6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -8,7 +8,6 @@ #define SCOUTFS_BLOCK_MAGIC_SUPER 0x103c428b #define SCOUTFS_BLOCK_MAGIC_BTREE 0xe597f96d #define SCOUTFS_BLOCK_MAGIC_BLOOM 0x31995604 -#define SCOUTFS_BLOCK_MAGIC_RADIX 0xebeb5e65 #define SCOUTFS_BLOCK_MAGIC_SRCH_BLOCK 0x897e4a7d #define SCOUTFS_BLOCK_MAGIC_SRCH_PARENT 0xb23a2a05 #define SCOUTFS_BLOCK_MAGIC_ALLOC_LIST 0x8a93ac83 @@ -183,29 +182,6 @@ struct scoutfs_radix_block { } __packed; } __packed; -struct scoutfs_radix_root { - __u8 height; - __le64 next_find_bit; - struct scoutfs_radix_ref ref; -} __packed; - -#define SCOUTFS_RADIX_REFS \ - ((SCOUTFS_BLOCK_LG_SIZE - \ - offsetof(struct scoutfs_radix_block, refs[0])) / \ - sizeof(struct scoutfs_radix_ref)) - -/* 8 meg regions with 4k data blocks */ -#define SCOUTFS_RADIX_LG_SHIFT 11 -#define SCOUTFS_RADIX_LG_BITS (1 << SCOUTFS_RADIX_LG_SHIFT) -#define SCOUTFS_RADIX_LG_MASK (SCOUTFS_RADIX_LG_BITS - 1) - -/* round block bits down to a multiple of large ranges */ -#define SCOUTFS_RADIX_BITS \ - (((SCOUTFS_BLOCK_LG_SIZE - \ - offsetof(struct scoutfs_radix_block, bits[0])) * 8) & \ - ~(__u64)SCOUTFS_RADIX_LG_MASK) -#define SCOUTFS_RADIX_BITS_BYTES (SCOUTFS_RADIX_BITS / 8) - struct scoutfs_avl_root { __le16 node; } __packed; diff --git a/kmod/src/radix.c b/kmod/src/radix.c deleted file mode 100644 index 0e734ff5..00000000 --- a/kmod/src/radix.c +++ /dev/null @@ -1,1514 +0,0 @@ -/* - * Copyright (C) 2020 Versity Software, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License v2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ -#include -#include -#include -#include -#include - -#include "super.h" -#include "format.h" -#include "counters.h" -#include "block.h" -#include "radix.h" -#include "scoutfs_trace.h" - -/* - * scoutfs uses bitmap blocks in cow radix trees to allocate free - * blocks. We like the radix trees because their stable structure lets - * us easily build up the resources to make atomic changes, splice trees - * around, and they have a respectable storage overhead when they're - * highly fragmented. - * - * An allocator itself contains two trees: one for bits that were stable - * at the start of a transaction and are available to satisfy - * allocation, and one for bits that were freed during this transaction - * which describe stable referenced blocks and can't be re-used to - * satisfy allocations until the transaction is committed. - * - * Each allocator contains a mutex that protects its two trees. It's - * typical for callers to allocate by calling radix ops on one of the - * alloc trees while providing the allocator to manage allocation of the - * radix blocks themselves. This is safe. If the caller is operating - * on other radix trees outside of the allocator struct (data - * allocations, server manipulating client trees) it is responsible for - * locking these external trees. - * - * The trees are updated by making cow copies of modified blocks and - * writing them into free space. The system does have a use for walking - * the old stable version of a tree -- the server needs to merge stable - * freed space back into its dirty available allocator tree while - * avoiding new frees that are arriving as it cows blocks during the - * merge process. - * - * Allocations search for the next free bit from a cursor that's stored - * in the root of each tree. - * - * The radix isn't always fully populated. References can contain - * blknos with 0 or ~0 to indicate that its referenced subtree is either - * entirely empty or full. The counters that describe these stubbed out - * subtrees will be correct as though all the blocks were populated. - * Traversal instantiates initialized empty or full blocks as it - * descends. This lets mkfs initialize a tree with a large contigious - * set region without having to populate all its blocks. - * - * The metadata allocator radix tree itself is used to allocate and free - * its own blocks as it makes cow updates to itself. Recursion is - * avoided by tracking all the blocks we dirty with their parents, - * making sure we have dirty leaves to record frees and allocs for all - * the dirtied blocks, and using a read-only cursor to find blknos for - * each new dirty block. This lets us either atomically set and clear - * all the leaf bits once we have all the dirty blocks or unwind all the - * dirty blocks and restore their parent references. - * - * Radix block references contain totals of bits set in its referenced - * subtree. This helps us balance the number of free bits stored across - * multiple trees. - * - * The radix tracks large aligned regions of set bits that are used to - * satisfy larger data extent allocations. These large regions are also - * tracked in the metadata allocator trees but aren't used. - */ - -/* - * This is just a sanity test at run time. It's log base - * SCOUTFS_RADIX_BITS of SCOUTFS_BLOCK_SM_MAX, but we can come close by - * dividing bit widths by shifts if we under-estimate the number of bits - * in a leaf by rounding it down to a power of two. In practice the - * trees are sized for the capacity of the device and are very short. - */ -#define RADIX_MAX_HEIGHT (((64 - SCOUTFS_BLOCK_SM_SHIFT) % \ - (SCOUTFS_BLOCK_LG_SHIFT + 2)) + 2) - -/* - * We create temporary synthetic blocks past possible blocks to populate - * stubbed out refs that reference entirely empty or full subtrees. - * They're moved to properly allocated blknos. - */ -#define RADIX_SYNTH_BLKNO (SCOUTFS_BLOCK_LG_MAX + 1) - -static bool is_synth(u64 blkno) -{ - return blkno >= RADIX_SYNTH_BLKNO; -} - -/* we use fake blknos to indicate subtrees either entirely empty or full */ -static bool is_stub(u64 blkno) -{ - return blkno == 0 || blkno == U64_MAX; -} - -struct radix_block_private { - struct scoutfs_block *bl; - struct list_head entry; - struct list_head dirtied_entry; - struct scoutfs_block *parent; - struct scoutfs_radix_ref *ref; - struct scoutfs_radix_ref orig_ref; - struct scoutfs_block *blkno_bl; - struct scoutfs_block *old_blkno_bl; - int blkno_ind; - int old_blkno_ind; - u64 blkno_leaf_bit; - u64 old_blkno_leaf_bit; -}; - -static bool was_dirtied(struct radix_block_private *priv) -{ - return !list_empty(&priv->dirtied_entry); -} - -struct radix_change { - struct scoutfs_radix_root *avail; - struct list_head blocks; - struct list_head dirtied_blocks; - u64 next_synth; - u64 next_find_bit; - u64 first_free; - struct scoutfs_block *free_bl; - u64 free_leaf_bit; - unsigned int free_ind; -}; - -#define DECLARE_RADIX_CHANGE(a) \ - struct radix_change a = {NULL, } - -/* - * We can use native longs to set full aligned regions, but we have to - * use individual _le bit calls on leading and trailing partial regions. - * - * XXX these would be more efficient if we calculated masks for the - * initial and final partial regions. - */ -static void bitmap_set_le(__le64 *map, int ind, int nbits) -{ - unsigned int full; - - while (ind & (BITS_PER_LONG - 1) && nbits-- > 0) - set_bit_le(ind++, map); - - if (nbits >= BITS_PER_LONG) { - full = round_down(nbits, BITS_PER_LONG); - bitmap_set((long *)map, ind, full); - ind += full; - nbits -= full; - } - - while (nbits-- > 0) - set_bit_le(ind++, map); -} - -/* - * xor at least nbits total dst bits with set src bits, a full word at a - * time, starting around the given starting index. The src and dst - * pointers can be to the same bitmap. We might xor bits before the - * starting index and might xor a bit more than nbits because we're - * working an __le64 at a time. Return the total amount xored and - * set the caller's size that includes the last word we modified. - */ -static int bitmap_xor_bitmap_le(__le64 *dst, __le64 *src, int ind, int nbits, - int *size) -{ - int xored = 0; - int i; - - BUG_ON((unsigned long)src & 7); - BUG_ON((unsigned long)dst & 7); - - while (xored < nbits && - (ind = find_next_bit_le(src, SCOUTFS_RADIX_BITS, ind)) < - SCOUTFS_RADIX_BITS) { - i = ind / 64; - xored += hweight64((u64 __force)src[i]); - dst[i] = dst[i] ^ src[i]; - ind = round_up(ind + 1, 64); - if (size) - *size = ind; - } - - return xored; -} - -static void bitmap_clear_le(__le64 *map, int ind, int nbits) -{ - unsigned int full; - - while (ind & (BITS_PER_LONG - 1) && nbits-- > 0) - clear_bit_le(ind++, map); - - if (nbits >= BITS_PER_LONG) { - full = round_down(nbits, BITS_PER_LONG); - bitmap_clear((long *)map, ind, full); - ind += full; - nbits -= full; - } - - while (nbits-- > 0) - clear_bit_le(ind++, map); -} - -/* Returns true if the given region is all 0. */ -static bool bitmap_empty_region_le(__le64 *map, int ind, int nbits) -{ - unsigned long size = ind + nbits; - - return find_next_bit_le(map, size, ind) >= size; -} - -/* Returns true if the given region is all set. */ -static bool bitmap_full_region_le(__le64 *map, int ind, int nbits) -{ - unsigned long size = ind + nbits; - - return find_next_zero_bit_le(map, size, ind) >= size; -} - -/* - * Return true if the large region containing the full precision small bit - * index is full. - */ -static bool lg_is_full(__le64 *map, int ind) -{ - return bitmap_full_region_le(map, ind & ~SCOUTFS_RADIX_LG_MASK, - SCOUTFS_RADIX_LG_BITS); -} - -/* - * Count the number of bits set in the large regions that contain the input - * bits. - */ -static u64 count_lg_bits(void *bits, int ind, int nbits) -{ - u64 count = 0; - int end; - int i; - - i = round_down(ind, SCOUTFS_RADIX_LG_BITS); - end = round_up(ind + nbits, SCOUTFS_RADIX_LG_BITS); - - while (i < end) { - if (lg_is_full(bits, i)) - count += SCOUTFS_RADIX_LG_BITS; - i += SCOUTFS_RADIX_LG_BITS; - } - - return count; -} - -/* - * For each of the large bit regions with bits set in the input bitmap, - * count the number of bits in corresponding large regions that are - * fully set in the result bitmap. - */ -static u64 count_lg_from_set(void *result, void *input, int ind, int size) -{ - u64 count = 0; - - while ((ind = find_next_bit_le(input, size, ind)) < size) { - if (lg_is_full(result, ind)) - count += SCOUTFS_RADIX_LG_BITS; - ind = round_up(ind + 1, SCOUTFS_RADIX_LG_BITS); - } - - return count; -} - - -/* ind is a small full precision bit index, not in units of large regions */ -static int find_next_lg(__le64 *map, int ind) -{ - for (ind = round_up(ind, SCOUTFS_RADIX_LG_BITS); - ind <= (SCOUTFS_RADIX_BITS - SCOUTFS_RADIX_LG_BITS); - ind += SCOUTFS_RADIX_LG_BITS) { - if (test_bit_le(ind, map) && lg_is_full(map, ind)) - return ind; - } - - return SCOUTFS_RADIX_BITS; -} - -static u64 bit_from_inds(u32 *level_inds, u8 height) -{ - u64 bit = level_inds[0]; - u64 mult = SCOUTFS_RADIX_BITS; - int i; - - for (i = 1; i < height; i++) { - bit += (u64)level_inds[i] * mult; - mult *= SCOUTFS_RADIX_REFS; - } - - return bit; -} - -static u64 last_from_super(struct super_block *sb, bool meta) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - - if (meta) - return le64_to_cpu(super->last_meta_blkno); - else - return le64_to_cpu(super->last_data_blkno); -} - -static u8 height_from_last(u64 last) -{ - u64 bit = SCOUTFS_RADIX_BITS - 1; - u64 mult = SCOUTFS_RADIX_BITS; - int i; - - for (i = 1; i <= U8_MAX; i++) { - if (bit >= last) - return i; - - bit += (u64)(SCOUTFS_RADIX_REFS - 1) * mult; - mult *= SCOUTFS_RADIX_REFS; - } - - return U8_MAX; -} - -/* total number of bits set in a full subtree with first block at level */ -static u64 full_subtree_total(int level) -{ - u64 total = SCOUTFS_RADIX_BITS; - int i; - - for (i = 1; i <= level; i++) - total *= SCOUTFS_RADIX_REFS; - - return total; -} - -static void calc_level_inds(u32 *level_inds, u8 height, u64 bit) -{ - u32 ind; - int i; - - bit = div_u64_rem(bit, SCOUTFS_RADIX_BITS, &ind); - level_inds[0] = ind; - - for (i = 1; i < height; i++) { - bit = div_u64_rem(bit, SCOUTFS_RADIX_REFS, &ind); - level_inds[i] = ind; - } -} - -static u64 calc_leaf_bit(u64 bit) -{ - u32 ind; - div_u64_rem(bit, SCOUTFS_RADIX_BITS, &ind); - - return bit - ind; -} - -/* - * Make sure ref total tracking is correct after having modified a leaf - * and updated all the parent refs. - */ -static void check_totals(struct scoutfs_block *leaf) -{ - struct radix_block_private *priv; - struct scoutfs_block *bl = leaf; - struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *ref; - int level; - u64 st; - u64 lt; - int i; - - for (level = 0; bl; level++, bl = priv->parent) { - priv = bl->priv; - rdx = bl->data; - ref = priv->ref; - - if (level == 0) { - st = bitmap_weight((long *)rdx->bits, - SCOUTFS_RADIX_BITS); - lt = count_lg_bits(rdx->bits, 0, SCOUTFS_RADIX_BITS); - } else { - st = 0; - lt = 0; - for (i = 0; i < SCOUTFS_RADIX_REFS; i++) { - st += le64_to_cpu(rdx->refs[i].sm_total); - lt += le64_to_cpu(rdx->refs[i].lg_total); - } - } - - if (le64_to_cpu(ref->sm_total) != st || - le64_to_cpu(ref->lg_total) != lt) { - printk("radix inconsistency: level %u calced st %llu lt %llu, stored st %llu lt %llu\n", - level, st, lt, - le64_to_cpu(ref->sm_total), - le64_to_cpu(ref->lg_total)); - BUG(); - } - - bl = priv->parent; - } -} - -/* - * The caller has changed bits in a leaf block. We update the totals in - * rers up to the root ref. - */ -static void fixup_parent_refs(struct super_block *sb, - struct scoutfs_block *leaf, - s64 sm_delta, s64 lg_delta) -{ - struct radix_block_private *priv; - struct scoutfs_radix_ref *ref; - struct scoutfs_block *bl; - - for (bl = leaf; bl; bl = priv->parent) { - priv = bl->priv; - ref = priv->ref; - - le64_add_cpu(&ref->sm_total, sm_delta); - le64_add_cpu(&ref->lg_total, lg_delta); - } - - if (0) /* expensive, would be nice to make conditional */ - check_totals(leaf); -} - -/* return 0 if the bit is past the last bit for the device */ -static u64 wrap_bit(struct super_block *sb, bool meta, u64 bit) -{ - return bit > last_from_super(sb, meta) ? 0 : bit; -} - -static void store_next_find_bit(struct super_block *sb, bool meta, - struct scoutfs_radix_root *root, u64 bit) -{ - root->next_find_bit = cpu_to_le64(wrap_bit(sb, meta, bit)); -} - -static void bug_on_bad_bits(int ind, int nbits) -{ - BUG_ON(ind < 0 || ind > SCOUTFS_RADIX_BITS); - BUG_ON(nbits < 0 || nbits > SCOUTFS_RADIX_BITS); - BUG_ON(ind + nbits > SCOUTFS_RADIX_BITS); -} - -static void set_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, - u64 leaf_bit, int ind, int nbits) -{ - struct scoutfs_radix_block *rdx = bl->data; - int lg_nbits; - - trace_scoutfs_radix_set_bits(sb, bl->blkno, leaf_bit, ind, nbits); - bug_on_bad_bits(ind, nbits); - - /* must never double-free bits */ - BUG_ON(!bitmap_empty_region_le(rdx->bits, ind, nbits)); - bitmap_set_le(rdx->bits, ind, nbits); - lg_nbits = count_lg_bits(rdx->bits, ind, nbits); - - fixup_parent_refs(sb, bl, nbits, lg_nbits); -} - -static void clear_leaf_bits(struct super_block *sb, struct scoutfs_block *bl, - u64 leaf_bit, int ind, int nbits) -{ - struct scoutfs_radix_block *rdx = bl->data; - int lg_nbits; - - trace_scoutfs_radix_clear_bits(sb, bl->blkno, leaf_bit, ind, nbits); - bug_on_bad_bits(ind, nbits); - - /* must never alloc in-use bits */ - BUG_ON(!bitmap_full_region_le(rdx->bits, ind, nbits)); - lg_nbits = count_lg_bits(rdx->bits, ind, nbits); - bitmap_clear_le(rdx->bits, ind, nbits); - - fixup_parent_refs(sb, bl, -nbits, -lg_nbits); -} - -/* - * Initialize a reference to a block at the given level. - */ -static void init_ref(struct scoutfs_radix_ref *ref, int level, bool full) -{ - u64 tot; - - if (full) { - tot = full_subtree_total(level); - - ref->blkno = cpu_to_le64(U64_MAX); - ref->seq = cpu_to_le64(0); - ref->sm_total = cpu_to_le64(tot); - ref->lg_total = cpu_to_le64(tot); - } else { - - ref->blkno = cpu_to_le64(0); - ref->seq = cpu_to_le64(0); - ref->sm_total = cpu_to_le64(0); - ref->lg_total = cpu_to_le64(0); - } -} - -/* Initialize a new empty or full block at a given level. */ -static void init_block(struct super_block *sb, struct scoutfs_radix_block *rdx, - u64 blkno, __le64 seq, int level, bool full) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - struct scoutfs_radix_ref ref; - int tail; - int i; - - /* we use native long bitmap functions on the block bitmaps */ - BUILD_BUG_ON(offsetof(struct scoutfs_radix_block, bits) & - (sizeof(long) - 1)); - - rdx->hdr.fsid = super->hdr.fsid; - rdx->hdr.magic = cpu_to_le32(SCOUTFS_BLOCK_MAGIC_RADIX); - rdx->hdr.blkno = cpu_to_le64(blkno); - rdx->hdr.seq = seq; - - if (level == 0) { - if (full) - memset(rdx->bits, 0xff, SCOUTFS_RADIX_BITS_BYTES); - else - memset(rdx->bits, 0, SCOUTFS_RADIX_BITS_BYTES); - - tail = SCOUTFS_BLOCK_LG_SIZE - - offsetof(struct scoutfs_radix_block, bits) - - SCOUTFS_RADIX_BITS_BYTES; - } else { - init_ref(&ref, level - 1, full); - - for (i = 0; i < SCOUTFS_RADIX_REFS; i++) - memcpy(&rdx->refs[i], &ref, sizeof(ref)); - - tail = SCOUTFS_BLOCK_LG_SIZE - - offsetof(struct scoutfs_radix_block, - refs[SCOUTFS_RADIX_REFS]); - } - - /* make sure we don't write uninitialized tail kernel memory to disk */ - if (tail) - memset((void *)rdx + SCOUTFS_BLOCK_LG_SIZE - tail, 0, tail); -} - -static int find_next_change_blkno(struct super_block *sb, - struct radix_change *chg, - u64 *blkno); - -enum { - GLF_NEXT_SM = (1 << 0), - GLF_NEXT_LG = (1 << 1), - GLF_DIRTY = (1 << 2), -}; - -/* - * Get the caller their block for walking down the radix. We can have - * to populate synthetic blocks, read existing blocks, and cow new dirty - * copies of either of those for callers who need to modify. We update - * references and record the blocks and references in the change for - * callers to further build atomic changes with. - */ -static int get_radix_block(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct radix_change *chg, - struct scoutfs_radix_root *root, int glf, - struct scoutfs_block *parent, - struct scoutfs_radix_ref *ref, int level, - struct scoutfs_block **bl_ret) -{ - struct radix_block_private *priv = NULL; - bool saw_inconsistent = false; - struct scoutfs_radix_block *rdx; - struct scoutfs_block *bl = NULL; - struct scoutfs_block *dirty; - bool put_block = true; - u64 blkno; - u64 synth; - int ret; - - /* create a synthetic block or read an existing block */ - blkno = le64_to_cpu(ref->blkno); - if (is_stub(blkno)) { - synth = chg->next_synth++; - /* don't create synth mistaken for all-full */ - if (synth == U64_MAX) { - scoutfs_inc_counter(sb, radix_enospc_synth); - ret = -ENOSPC; - goto out; - } - bl = scoutfs_block_create(sb, synth); - if (!IS_ERR_OR_NULL(bl)) { - init_block(sb, bl->data, synth, ref->seq, level, - blkno == U64_MAX); - scoutfs_inc_counter(sb, radix_create_synth); - } - } else { - bl = scoutfs_block_read(sb, blkno); - if (!IS_ERR_OR_NULL(bl)) - scoutfs_inc_counter(sb, radix_block_read); - - /* - * We can have a stale block in the cache but the tree - * shouldn't be changing under us. We don't have to - * reread a root and restart descent. If we don't get a - * consistent block after reading from the device then - * we've found corruption. - */ - while (!IS_ERR(bl) && - !scoutfs_block_consistent_ref(sb, bl, ref->seq, - ref->blkno, - SCOUTFS_BLOCK_MAGIC_RADIX)) { - scoutfs_inc_counter(sb, radix_inconsistent_ref); - scoutfs_block_writer_forget(sb, wri, bl); - scoutfs_block_invalidate(sb, bl); - BUG_ON(bl->priv != NULL); - scoutfs_block_put(sb, bl); - bl = NULL; - if (!saw_inconsistent) { - saw_inconsistent = true; - bl = scoutfs_block_read(sb, blkno); - } else { - bl = ERR_PTR(-EIO); - scoutfs_inc_counter(sb, radix_inconsistent_eio); - } - } - saw_inconsistent = false; - } - if (IS_ERR(bl)) { - ret = PTR_ERR(bl); - goto out; - } - - if ((glf & GLF_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl)) { - /* make a cow copy for the caller that needs a dirty block */ - ret = find_next_change_blkno(sb, chg, &blkno); - if (ret < 0) - goto out; - - dirty = scoutfs_block_create(sb, blkno); - if (IS_ERR(dirty)) { - ret = PTR_ERR(dirty); - goto out; - } - - memcpy(dirty->data, bl->data, SCOUTFS_BLOCK_LG_SIZE); - scoutfs_block_put(sb, bl); - bl = dirty; - scoutfs_inc_counter(sb, radix_block_cow); - } - - priv = bl->priv; - if (!priv) { - priv = kzalloc(sizeof(struct radix_block_private), GFP_NOFS); - if (!priv) { - ret = -ENOMEM; - goto out; - } - - bl->priv = priv; - priv->bl = bl; - INIT_LIST_HEAD(&priv->dirtied_entry); - priv->parent = parent; - priv->ref = ref; - priv->orig_ref = *ref; - /* put at head so for_each restores refs in reverse */ - list_add(&priv->entry, &chg->blocks); - /* priv holds bl get, put as change is completed */ - put_block = false; - } - - if ((glf & GLF_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl)) { - scoutfs_block_writer_mark_dirty(sb, wri, bl); - list_add(&priv->dirtied_entry, &chg->dirtied_blocks); - } - - trace_scoutfs_radix_get_block(sb, root, glf, level, - parent ? parent->blkno : 0, - le64_to_cpu(ref->blkno), bl->blkno); - - /* update refs to new synth or dirty blocks */ - if (le64_to_cpu(ref->blkno) != bl->blkno) { - rdx = bl->data; - rdx->hdr.blkno = cpu_to_le64(bl->blkno); - prandom_bytes(&rdx->hdr.seq, sizeof(rdx->hdr.seq)); - ref->blkno = rdx->hdr.blkno; - ref->seq = rdx->hdr.seq; - } - - ret = 0; -out: - if (put_block) - scoutfs_block_put(sb, bl); - if (ret < 0) - bl = NULL; - - *bl_ret = bl; - return ret; -} - -static int get_leaf_walk(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct radix_change *chg, - struct scoutfs_radix_root *root, - int glf, u64 bit, u64 *leaf_bit_ret, - struct scoutfs_block **bl_ret) -{ - struct scoutfs_radix_block *rdx; - struct scoutfs_radix_ref *ref; - struct scoutfs_block *parent = NULL; - struct scoutfs_block *bl; - u32 level_inds[RADIX_MAX_HEIGHT]; - int level; - int ind = 0; - int ret; - int i; - - /* can't operate outside radix until we support growing devices */ - if (WARN_ON_ONCE(root->height < height_from_last(bit)) || - WARN_ON_ONCE(root->height > RADIX_MAX_HEIGHT) || - WARN_ON_ONCE((glf & GLF_NEXT_SM) && (glf & GLF_NEXT_LG))) - return -EINVAL; - - calc_level_inds(level_inds, root->height, bit); - ref = &root->ref; - - for (level = root->height - 1; level >= 0; level--) { - ret = get_radix_block(sb, alloc, wri, chg, root, glf, parent, - ref, level, &bl); - if (ret) - goto out; - - trace_scoutfs_radix_walk(sb, root, glf, level, bl->blkno, ind, - bit); - - if (level == 0) { - /* returned leaf_bit is first in the leaf block */ - level_inds[0] = 0; - break; - } - - rdx = bl->data; - ind = level_inds[level]; - - /* search for a ref to a child with a set large region */ - while ((glf & GLF_NEXT_LG) && ind < SCOUTFS_RADIX_REFS && - le64_to_cpu(rdx->refs[ind].lg_total) == 0) { - ind++; - } - - /* search for a ref to a child with any bits set */ - while ((glf & GLF_NEXT_SM) && ind < SCOUTFS_RADIX_REFS && - le64_to_cpu(rdx->refs[ind].sm_total) == 0) { - ind++; - } - - /* - * Didn't find a ref in the rest of the block at - * this level. If we're the root block there's no - * more next bits to return. If we're further down - * we bubble up a level and continue on a depth-first - * search. We check the next ref from our parent and reset - * all the child inds to the left spine of the new - * subtree. - */ - if (ind >= SCOUTFS_RADIX_REFS) { - if (level == root->height - 1) { - ret = -ENOENT; - goto out; - } - level_inds[level + 1]++; - for (i = level; i >= 0; i--) - level_inds[i] = 0; - level += 2; - continue; - } - - /* reset all lower indices if we searched */ - if (ind != level_inds[level]) { - for (i = level - 1; i >= 0; i--) - level_inds[i] = 0; - level_inds[level] = ind; - } - - parent = bl; - ref = &rdx->refs[ind]; - } - - *leaf_bit_ret = bit_from_inds(level_inds, root->height); - ret = 0; - scoutfs_inc_counter(sb, radix_walk); -out: - if (ret < 0) - *bl_ret = NULL; - else - *bl_ret = bl; - return ret; -} - -/* - * Get the caller their leaf block in which they'll set or clear bits. - * If they're asking for a dirty block then the leaf walk might dirty - * blocks. For each newly dirtied block we also make sure we have dirty - * blocks for the leaves that contain the bits for each newly dirtied - * block's old blkno and new blkno. - */ -static int get_leaf(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, struct radix_change *chg, - struct scoutfs_radix_root *root, int glf, u64 bit, - u64 *leaf_bit_ret, struct scoutfs_block **bl_ret) -{ - struct radix_block_private *priv; - struct scoutfs_block *bl; - u64 leaf_bit; - u64 old_blkno; - int ret; - - ret = get_leaf_walk(sb, alloc, wri, chg, root, glf, bit, leaf_bit_ret, - bl_ret); - if (ret < 0 || !(glf & GLF_DIRTY)) - goto out; - - /* walk to leaves containing bits of newly dirtied block's blknos */ - while ((priv = list_first_entry_or_null(&chg->dirtied_blocks, - struct radix_block_private, - dirtied_entry))) { - /* done when we see tail blocks with their blkno_bl set */ - if (priv->blkno_bl != NULL) - break; - - old_blkno = le64_to_cpu(priv->orig_ref.blkno); - if (!is_stub(old_blkno) && !is_synth(old_blkno)) { - ret = get_leaf_walk(sb, alloc, wri, chg, &alloc->freed, - GLF_DIRTY, old_blkno, &leaf_bit, - &bl); - if (ret < 0) - break; - priv->old_blkno_leaf_bit = leaf_bit; - priv->old_blkno_ind = old_blkno - leaf_bit; - priv->old_blkno_bl = bl; - } - - ret = get_leaf_walk(sb, alloc, wri, chg, &alloc->avail, - GLF_DIRTY, priv->bl->blkno, &leaf_bit, - &bl); - if (ret < 0) - break; - - priv->blkno_leaf_bit = leaf_bit; - priv->blkno_ind = priv->bl->blkno - leaf_bit; - priv->blkno_bl = bl; - - list_move_tail(&priv->dirtied_entry, &chg->dirtied_blocks); - } -out: - return ret; -} - -/* - * Find the next region of set bits of the given size starting from the - * given bit. This only finds the bits, it doesn't change anything. We - * always try to return regions past the starting bit. We can search to - * a leaf that has bits that are all past the starting bit and we'll - * retry. This will wrap around to the start of the tree and fall back - * to satisfying large regions with small regions. - */ -static int find_next_set_bits(struct super_block *sb, struct radix_change *chg, - struct scoutfs_radix_root *root, bool meta, - u64 start, int nbits, u64 *bit_ret, - int *nbits_ret, struct scoutfs_block **bl_ret) -{ - struct scoutfs_radix_block *rdx; - struct scoutfs_block *bl; - u64 leaf_bit; - u64 bit; - int end; - int ind; - int glf; - int ret; - - bit = start; - glf = nbits > 1 ? GLF_NEXT_LG : GLF_NEXT_SM; -retry: - ret = get_leaf(sb, NULL, NULL, chg, root, glf, bit, &leaf_bit, &bl); - if (ret == -ENOENT) { - if (bit != 0) { - bit = 0; - goto retry; - } - - /* switch to searching for small bits if no large found */ - if (glf == GLF_NEXT_LG) { - glf = GLF_NEXT_SM; - bit = start; - goto retry; - } - ret = -ENOSPC; - goto out; - } - rdx = bl->data; - - /* start from search bit if it's in the leaf, otherwise 0 */ - if (leaf_bit < bit && ((bit - leaf_bit) < SCOUTFS_RADIX_BITS)) - ind = bit - leaf_bit; - else - ind = 0; - - /* large allocs are always aligned from large regions */ - if (nbits >= SCOUTFS_RADIX_LG_BITS && (glf == GLF_NEXT_LG)) { - ind = find_next_lg(rdx->bits, ind); - if (ind == SCOUTFS_RADIX_BITS) { - bit = wrap_bit(sb, meta, leaf_bit + SCOUTFS_RADIX_BITS); - goto retry; - } - nbits = SCOUTFS_RADIX_LG_BITS; - ret = 0; - goto out; - } - - /* otherwise use as much of the next set region as we can */ - ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, ind); - if (ind == SCOUTFS_RADIX_BITS) { - bit = wrap_bit(sb, meta, leaf_bit + SCOUTFS_RADIX_BITS); - goto retry; - } - - if (nbits > 1) { - end = find_next_zero_bit_le(rdx->bits, min_t(int, ind + nbits, - SCOUTFS_RADIX_BITS), ind); - nbits = end - ind; - } - ret = 0; - -out: - *bit_ret = leaf_bit + ind; - *nbits_ret = nbits; - if (bl_ret) - *bl_ret = bl; - - return ret; -} - -static void prepare_change(struct radix_change *chg, - struct scoutfs_radix_root *avail) -{ - memset(chg, 0, sizeof(struct radix_change)); - chg->avail = avail; - INIT_LIST_HEAD(&chg->blocks); - INIT_LIST_HEAD(&chg->dirtied_blocks); - chg->next_synth = RADIX_SYNTH_BLKNO; - chg->next_find_bit = le64_to_cpu(avail->next_find_bit); -} - -/* - * We successfully got all the dirty block references we need to make - * the change. Set their old blkno's freed bits and clear all their new - * dirty blkno's avail bits. We drop the blocks from the dirtied_blocks - * list here as we go so we won't attempt to do this all over again - * as we complete the change. - */ -static void apply_change_bits(struct super_block *sb, struct radix_change *chg) -{ - struct radix_block_private *priv; - struct scoutfs_block *bl; - - /* first update the contents of the blocks */ - list_for_each_entry(priv, &chg->blocks, entry) { - bl = priv->bl; - - /* complete cow allocations for dirtied blocks */ - if (was_dirtied(priv)) { - /* can't try to write to synth blknos */ - BUG_ON(is_synth(bl->blkno)); - - clear_leaf_bits(sb, priv->blkno_bl, - priv->blkno_leaf_bit, - priv->blkno_ind, 1); - if (priv->old_blkno_bl) { - set_leaf_bits(sb, priv->old_blkno_bl, - priv->old_blkno_leaf_bit, - priv->old_blkno_ind, 1); - } - scoutfs_inc_counter(sb, radix_complete_dirty_block); - - list_del_init(&priv->dirtied_entry); - } - } -} - -/* - * Drop all references to the blocks that we held as we worked with the - * radix blocks. - * - * If the operation failed then we drop the blocks we dirtied during - * this change and restore their refs. Nothing can update a ref to a - * dirty block so these will always be current. - * - * We always drop synthetic blocks. They could been cowed so they might - * not be currently referenced. Blocks are added to the head of the - * blocks list as they're first used so we're undoing ref changes in - * reverse order. This means that the error case will always first - * unwind synthetic cows then the synthetic source block itself. - */ -static void complete_change(struct super_block *sb, - struct scoutfs_block_writer *wri, - struct radix_change *chg, int err) -{ - struct radix_block_private *priv; - struct radix_block_private *tmp; - struct scoutfs_block *bl; - - /* only complete once for each call to prepare */ - if (!chg->avail) - return; - - /* finish dirty block frees and allocs on success */ - if (err == 0 && !list_empty(&chg->dirtied_blocks)) - apply_change_bits(sb, chg); - - /* replace refs and remove blocks from the cache */ - list_for_each_entry(priv, &chg->blocks, entry) { - bl = priv->bl; - - if (is_synth(bl->blkno) || (err < 0 && was_dirtied(priv))) { - if (le64_to_cpu(priv->ref->blkno) == bl->blkno) { - *priv->ref = priv->orig_ref; - scoutfs_inc_counter(sb, radix_undo_ref); - } - scoutfs_block_writer_forget(sb, wri, bl); - scoutfs_block_invalidate(sb, bl); - } - } - - /* finally put all blocks now that were done with contents */ - list_for_each_entry_safe(priv, tmp, &chg->blocks, entry) { - bl = priv->bl; - - bl->priv = NULL; - scoutfs_block_put(sb, bl); - list_del(&priv->entry); - kfree(priv); - } - - if (err == 0) - store_next_find_bit(sb, true, chg->avail, chg->next_find_bit); - chg->avail = NULL; -} - -/* - * Find the next free metadata blkno from the metadata allocator that - * the change is tracking. This is used to find the next free blkno for - * the next cowed block without modifying the allocator. Because it's - * not modifying the allocator it can wrap and find the same block - * twice, we watch for that. - */ -static int find_next_change_blkno(struct super_block *sb, - struct radix_change *chg, u64 *blkno) -{ - struct scoutfs_radix_block *rdx; - u64 bit; - int nbits; - int ret; - - if (chg->free_bl == NULL) { - ret = find_next_set_bits(sb, chg, chg->avail, true, - chg->next_find_bit, 1, &bit, &nbits, - &chg->free_bl); - if (ret < 0) - goto out; - chg->free_leaf_bit = calc_leaf_bit(bit); - chg->free_ind = bit - chg->free_leaf_bit; - } - - bit = chg->free_leaf_bit + chg->free_ind; - if (chg->first_free == 0) { - chg->first_free = bit; - } else if (chg->first_free == bit) { - ret = -ENOSPC; - goto out; - } - - *blkno = bit; - - rdx = chg->free_bl->data; - chg->free_ind = find_next_bit_le(rdx->bits, SCOUTFS_RADIX_BITS, - chg->free_ind + 1); - if (chg->free_ind >= SCOUTFS_RADIX_BITS) { - chg->free_ind = SCOUTFS_RADIX_BITS; - chg->free_bl = NULL; - } - chg->next_find_bit = wrap_bit(sb, true, - chg->free_leaf_bit + chg->free_ind); - - ret = 0; -out: - if (ret == -ENOSPC) - scoutfs_inc_counter(sb, radix_enospc_meta); - return ret; -} - -static bool valid_free_bit_range(struct super_block *sb, bool meta, - u64 bit, int nbits) -{ - struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; - u64 last = bit + nbits - 1; - - return (nbits > 0) && - (last >= bit) && - (!meta || (bit >= le64_to_cpu(super->first_meta_blkno) && - last <= le64_to_cpu(super->last_meta_blkno))) && - (meta || (bit >= le64_to_cpu(super->first_data_blkno) && - last <= le64_to_cpu(super->last_data_blkno))); -} - -static int radix_free(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *root, bool meta, - u64 bit, int nbits) -{ - struct scoutfs_block *bl; - DECLARE_RADIX_CHANGE(chg); - u64 leaf_bit; - int ind; - int ret; - - /* we only operate on one leaf */ - if (WARN_ON_ONCE(!valid_free_bit_range(sb, meta, bit, nbits)) || - WARN_ON_ONCE(calc_leaf_bit(bit) != calc_leaf_bit(bit + nbits - 1))) - return -EINVAL; - - mutex_lock(&alloc->mutex); - prepare_change(&chg, &alloc->avail); - - ret = get_leaf(sb, alloc, wri, &chg, root, GLF_DIRTY, bit, - &leaf_bit, &bl); - if (ret < 0) - goto out; - - ind = bit - leaf_bit; - set_leaf_bits(sb, bl, leaf_bit, ind, nbits); -out: - complete_change(sb, wri, &chg, ret); - mutex_unlock(&alloc->mutex); - - return ret; -} - -/* - * Return a single allocated metadata block for the caller. We let the change - * find a leaf in the metadata allocator for us. - */ -int scoutfs_radix_alloc(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno) -{ - struct scoutfs_block *bl; - DECLARE_RADIX_CHANGE(chg); - u64 leaf_bit; - u64 bit; - int ind; - int ret; - - scoutfs_inc_counter(sb, radix_alloc); - - mutex_lock(&alloc->mutex); - prepare_change(&chg, &alloc->avail); - - ret = find_next_change_blkno(sb, &chg, &bit); - if (ret < 0) - goto out; - - ret = get_leaf(sb, alloc, wri, &chg, &alloc->avail, GLF_DIRTY, bit, - &leaf_bit, &bl); - if (ret < 0) - goto out; - - ind = bit - leaf_bit; - clear_leaf_bits(sb, bl, leaf_bit, ind, 1); - *blkno = bit; - ret = 0; -out: - complete_change(sb, wri, &chg, ret); - mutex_unlock(&alloc->mutex); - - return ret; -} - -/* - * Return an allocated data block extent by finding and clearing it from - * the caller's tree. The caller must protect access to their tree. We - * have to search in and allocate from the separate data allocator tree - * ourselves. - */ -int scoutfs_radix_alloc_data(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *root, - int count, u64 *blkno_ret, int *count_ret) -{ - struct scoutfs_block *bl; - DECLARE_RADIX_CHANGE(chg); - u64 leaf_bit; - u64 bit; - int nbits; - int ind; - int ret; - - scoutfs_inc_counter(sb, radix_alloc_data); - - *blkno_ret = 0; - *count_ret = 0; - - if (WARN_ON_ONCE(count <= 0 || blkno_ret == NULL || count_ret == NULL)) - return -EINVAL; - - nbits = min(count, SCOUTFS_RADIX_LG_BITS); - - mutex_lock(&alloc->mutex); - prepare_change(&chg, &alloc->avail); - - ret = find_next_set_bits(sb, &chg, root, false, - le64_to_cpu(root->next_find_bit), nbits, - &bit, &nbits, NULL); - if (ret < 0) { - if (ret == -ENOSPC) - scoutfs_inc_counter(sb, radix_enospc_data); - goto out; - } - - ret = get_leaf(sb, alloc, wri, &chg, root, GLF_DIRTY, bit, - &leaf_bit, &bl); - if (ret < 0) - goto out; - - ind = bit - leaf_bit; - clear_leaf_bits(sb, bl, leaf_bit, ind, nbits); - *blkno_ret = bit; - *count_ret = nbits; - store_next_find_bit(sb, false, root, bit + nbits); - ret = 0; -out: - complete_change(sb, wri, &chg, ret); - mutex_unlock(&alloc->mutex); - - return ret; -} - -/* - * Free a single metadata block by adding it to the allocator's freed - * tree. Callers can trust our allocator to lock. - */ -int scoutfs_radix_free(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, u64 blkno) -{ - scoutfs_inc_counter(sb, radix_free); - return radix_free(sb, alloc, wri, &alloc->freed, true, blkno, 1); -} - -/* - * Free a data block extent by setting it in the caller's tree. The - * caller must protect access to their tree. - */ -int scoutfs_radix_free_data(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *root, - u64 blkno, int count) -{ - scoutfs_inc_counter(sb, radix_free_data); - return radix_free(sb, alloc, wri, root, false, blkno, count); -} - -/* - * Move bits between the source and destination trees. The bits to move - * are found in the input tree. - * - * Typically the input and source trees are the same. We're careful to - * modify the dst first because modifying src might also be modifying - * inp. - * - * The input and source trees aren't the same when the caller is being - * careful to use a read-only input tree because the source tree is - * changing during the merge. This happens when the server tries to - * reclaim its freed tree by moving it into its avail. Because our - * dirtying actually moves clean blocks we need to be careful to not - * reference dirty blocks from the input tree walk. This is discovered - * after dirtying the blocks. The additional input walk will this time - * read the old blocks. - * - * We can also be called with a src tree that is the current allocator - * avail tree. In this case dirtying the leaf blocks can consume bits - * in the source tree. We notice when dirtying the src block and we - * retry finding a new leaf to merge. - * - * The caller specifies the minimum count to move. -ENOENT will be - * returned if the source tree runs out of bits, potentially after - * having already moved bits. Up to 63 bits more than the minimum can - * be moved because bits are manipulated in chunks of 64 bits. - * - * This is pretty expensive because it fully references full leaf blocks - * a few times. It could be more efficient if it short circuited walks - * and spliced refs in parents when it finds that subtrees don't - * intersect. - */ -int scoutfs_radix_merge(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *dst, - struct scoutfs_radix_root *src, - struct scoutfs_radix_root *inp, bool meta, u64 count) -{ - struct scoutfs_radix_block *inp_rdx; - struct scoutfs_radix_block *src_rdx; - struct scoutfs_radix_block *dst_rdx; - struct scoutfs_block *inp_bl; - struct scoutfs_block *src_bl; - struct scoutfs_block *dst_bl; - DECLARE_RADIX_CHANGE(chg); - s64 src_lg_delta; - s64 dst_lg_delta; - u64 leaf_bit; - u64 bit; - int merge_size; - int merged; - int ind; - int ret; - - trace_scoutfs_radix_merge(sb, le64_to_cpu(dst->ref.blkno), - le64_to_cpu(dst->ref.sm_total), - le64_to_cpu(src->ref.blkno), - le64_to_cpu(src->ref.sm_total), - le64_to_cpu(inp->ref.blkno), - le64_to_cpu(inp->ref.sm_total), count); - scoutfs_inc_counter(sb, radix_merge); - - mutex_lock(&alloc->mutex); - - /* can't try to free too much when inp is read-only */ - if (inp != src && - WARN_ON_ONCE(count > le64_to_cpu(inp->ref.sm_total))) { - ret = -EINVAL; - goto out; - } - - while (count > 0) { - - prepare_change(&chg, &alloc->avail); - bit = le64_to_cpu(src->next_find_bit); -wrapped: - ret = get_leaf(sb, NULL, NULL, &chg, inp, GLF_NEXT_SM, bit, - &leaf_bit, &inp_bl); - if (ret < 0) { - if (ret == -ENOENT) { - if (bit != 0) { - bit = 0; - goto wrapped; - } else { - ret = -ENOSPC; - } - } - goto out; - } - bit = leaf_bit; - inp_rdx = inp_bl->data; - - ret = get_leaf(sb, alloc, wri, &chg, src, GLF_DIRTY, bit, - &leaf_bit, &src_bl); - if (ret < 0) - goto out; - src_rdx = src_bl->data; - - /* - * If we're searching the avail allocator tree then we - * must be sure that we copy leaves after change - * allocations have been applied. If we had a read-only - * copy of the allocator leaf before it was cowed we - * could merge bits that were used for dirty block - * allocations by the change. By not resetting the - * change the repeated lookup will find the current - * dirty leaf block. - */ - if (src == inp && inp_bl != src_bl) { - scoutfs_inc_counter(sb, radix_merge_bad_clean_input); - goto wrapped; - } - - ret = get_leaf(sb, alloc, wri, &chg, dst, GLF_DIRTY, bit, - &leaf_bit, &dst_bl); - if (ret < 0) - goto out; - dst_rdx = dst_bl->data; - - apply_change_bits(sb, &chg); - - /* change allocs could have cleared all of inp if its avail */ - ind = find_next_bit_le(inp_rdx->bits, SCOUTFS_RADIX_BITS, 0); - if (ind == SCOUTFS_RADIX_BITS) { - scoutfs_inc_counter(sb, radix_merge_empty); - complete_change(sb, wri, &chg, -EAGAIN); - continue; - } - - /* make sure all input bits are set in src */ - if (inp != src && - !bitmap_subset((void *)inp_rdx->bits, - (void *)src_rdx->bits, - SCOUTFS_RADIX_BITS)) { - ret = -EIO; - goto out; - } - - /* make sure all input bits are clear in dst */ - if (bitmap_intersects((void *)dst_rdx->bits, - (void *)inp_rdx->bits, - SCOUTFS_RADIX_BITS)) { - ret = -EIO; - goto out; - } - - /* carefully modify src last, it might also be inp */ - merged = bitmap_xor_bitmap_le(dst_rdx->bits, inp_rdx->bits, - ind, count, &merge_size); - dst_lg_delta = count_lg_from_set(dst_rdx->bits, inp_rdx->bits, - ind, merge_size); - - src_lg_delta = count_lg_from_set(src_rdx->bits, inp_rdx->bits, - ind, merge_size); - bitmap_xor_bitmap_le(src_rdx->bits, inp_rdx->bits, ind, merged, - NULL); - - fixup_parent_refs(sb, src_bl, -merged, -src_lg_delta); - fixup_parent_refs(sb, dst_bl, merged, dst_lg_delta); - - trace_scoutfs_radix_merged_blocks(sb, inp, inp_bl->blkno, src, - src_bl->blkno, dst, - dst_bl->blkno, count, bit, - ind, merged, src_lg_delta, - dst_lg_delta); - - complete_change(sb, wri, &chg, 0); - - store_next_find_bit(sb, meta, src, bit + SCOUTFS_RADIX_BITS); - count -= min_t(u64, count, merged); - } - - ret = 0; -out: - complete_change(sb, wri, &chg, ret); - mutex_unlock(&alloc->mutex); - - return ret; -} - -void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, - struct scoutfs_radix_root *avail, - struct scoutfs_radix_root *freed) -{ - mutex_init(&alloc->mutex); - alloc->avail = *avail; - alloc->freed = *freed; -} - -/* - * Initialize a root with an empty ref. We set the height to the size - * of the device and descent will fill in blocks. - */ -void scoutfs_radix_root_init(struct super_block *sb, - struct scoutfs_radix_root *root, bool meta) -{ - u64 last = last_from_super(sb, meta); - - root->height = height_from_last(last); - root->next_find_bit = cpu_to_le64(0); - init_ref(&root->ref, 0, false); -} - -u64 scoutfs_radix_root_free_blocks(struct super_block *sb, - struct scoutfs_radix_root *root) -{ - return le64_to_cpu(root->ref.sm_total); -} - -/* - * The first bit nr in a leaf containing the bit, used by callers to - * identify regions that span leafs and would need to be freed in - * multiple calls. - */ -u64 scoutfs_radix_bit_leaf_nr(u64 bit) -{ - return calc_leaf_bit(bit); -} diff --git a/kmod/src/radix.h b/kmod/src/radix.h deleted file mode 100644 index 729e810b..00000000 --- a/kmod/src/radix.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef _SCOUTFS_RADIX_H_ -#define _SCOUTFS_RADIX_H_ - -#include "per_task.h" - -struct scoutfs_block_writer; - -struct scoutfs_radix_allocator { - struct mutex mutex; - struct scoutfs_radix_root avail; - struct scoutfs_radix_root freed; -}; - -int scoutfs_radix_alloc(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, u64 *blkno); -int scoutfs_radix_alloc_data(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *root, - int count, u64 *blkno_ret, int *count_ret); -int scoutfs_radix_free(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, u64 blkno); -int scoutfs_radix_free_data(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *root, - u64 blkno, int count); -int scoutfs_radix_merge(struct super_block *sb, - struct scoutfs_radix_allocator *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_radix_root *dst, - struct scoutfs_radix_root *src, - struct scoutfs_radix_root *inp, bool meta, u64 count); -void scoutfs_radix_init_alloc(struct scoutfs_radix_allocator *alloc, - struct scoutfs_radix_root *avail, - struct scoutfs_radix_root *freed); -void scoutfs_radix_root_init(struct super_block *sb, - struct scoutfs_radix_root *root, bool meta); -u64 scoutfs_radix_root_free_blocks(struct super_block *sb, - struct scoutfs_radix_root *root); -u64 scoutfs_radix_bit_leaf_nr(u64 bit); - -#endif diff --git a/kmod/src/scoutfs_trace.h b/kmod/src/scoutfs_trace.h index 0465dd13..aa376dab 100644 --- a/kmod/src/scoutfs_trace.h +++ b/kmod/src/scoutfs_trace.h @@ -2223,174 +2223,6 @@ DEFINE_EVENT(scoutfs_block_class, scoutfs_block_shrink, TP_ARGS(sb, bp, blkno, refcount, io_count, bits, lru_moved) ); -TRACE_EVENT(scoutfs_radix_get_block, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - int glf, int level, u64 par_blkno, u64 ref_blkno, u64 blkno), - TP_ARGS(sb, root, glf, level, par_blkno, ref_blkno, blkno), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(int, glf) - __field(int, level) - __field(__u64, par_blkno) - __field(__u64, ref_blkno) - __field(__u64, blkno) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(root->ref.blkno); - __entry->glf = glf; - __entry->level = level; - __entry->par_blkno = par_blkno; - __entry->ref_blkno = ref_blkno; - __entry->blkno = blkno; - ), - TP_printk(SCSBF" root_blkno %llu glf 0x%x level %u par_blkno %llu ref_blkno %llu blkno %llu", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->glf, - __entry->level, __entry->par_blkno, __entry->ref_blkno, - __entry->blkno) -); - -TRACE_EVENT(scoutfs_radix_walk, - TP_PROTO(struct super_block *sb, struct scoutfs_radix_root *root, - int glf, int level, u64 blkno, int ind, u64 bit), - TP_ARGS(sb, root, glf, level, blkno, ind, bit), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, root_blkno) - __field(unsigned int, glf) - __field(__u64, blkno) - __field(int, level) - __field(int, ind) - __field(__u64, bit) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->root_blkno = le64_to_cpu(root->ref.blkno); - __entry->glf = glf; - __entry->blkno = blkno; - __entry->level = level; - __entry->ind = ind; - __entry->bit = bit; - ), - TP_printk(SCSBF" root_blkno %llu glf 0x%x blkno %llu level %d par_ind %d bit %llu", - SCSB_TRACE_ARGS, __entry->root_blkno, __entry->glf, - __entry->blkno, __entry->level, __entry->ind, __entry->bit) -); - -DECLARE_EVENT_CLASS(scoutfs_radix_bitop, - TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, - int nbits), - TP_ARGS(sb, blkno, leaf_bit, ind, nbits), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, blkno) - __field(__u64, leaf_bit) - __field(int, ind) - __field(int, nbits) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->blkno = blkno; - __entry->leaf_bit = leaf_bit; - __entry->ind = ind; - __entry->nbits = nbits; - ), - TP_printk(SCSBF" blkno %llu leaf_bit %llu ind %d nbits %d", - SCSB_TRACE_ARGS, __entry->blkno, __entry->leaf_bit, - __entry->ind, __entry->nbits) -); -DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_clear_bits, - TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, - int nbits), - TP_ARGS(sb, blkno, leaf_bit, ind, nbits) -); -DEFINE_EVENT(scoutfs_radix_bitop, scoutfs_radix_set_bits, - TP_PROTO(struct super_block *sb, u64 blkno, u64 leaf_bit, int ind, - int nbits), - TP_ARGS(sb, blkno, leaf_bit, ind, nbits) -); - -TRACE_EVENT(scoutfs_radix_merge, - TP_PROTO(struct super_block *sb, u64 dst_blkno, u64 dst_sm_tot, - u64 src_blkno, u64 src_sm_tot, u64 inp_blkno, u64 inp_sm_tot, - u64 count), - TP_ARGS(sb, dst_blkno, dst_sm_tot, src_blkno, src_sm_tot, inp_blkno, - inp_sm_tot, count), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, dst_blkno) - __field(__u64, dst_sm_tot) - __field(__u64, src_blkno) - __field(__u64, src_sm_tot) - __field(__u64, inp_blkno) - __field(__u64, inp_sm_tot) - __field(__u64, count) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->dst_blkno = dst_blkno; - __entry->dst_sm_tot = dst_sm_tot; - __entry->src_blkno = src_blkno; - __entry->src_sm_tot = src_sm_tot; - __entry->inp_blkno = inp_blkno; - __entry->inp_sm_tot = inp_sm_tot; - __entry->count = count; - ), - TP_printk(SCSBF" d_blkno %llu d_sm_tot %llu s_blkno %llu s_sm_tot %llu i_blkno %llu i_sm_tot %llu count %llu", - SCSB_TRACE_ARGS, __entry->dst_blkno, __entry->dst_sm_tot, - __entry->src_blkno, __entry->src_sm_tot, __entry->inp_blkno, - __entry->inp_sm_tot, __entry->count) -); - -TRACE_EVENT(scoutfs_radix_merged_blocks, - TP_PROTO(struct super_block *sb, - struct scoutfs_radix_root *inp, u64 inp_blkno, - struct scoutfs_radix_root *src, u64 src_blkno, - struct scoutfs_radix_root *dst, u64 dst_blkno, - u64 count, u64 leaf_bit, int ind, int sm_delta, - int src_lg_delta, int dst_lg_delta), - TP_ARGS(sb, inp, inp_blkno, src, src_blkno, dst, dst_blkno, count, - leaf_bit, ind, sm_delta, src_lg_delta, dst_lg_delta), - TP_STRUCT__entry( - SCSB_TRACE_FIELDS - __field(__u64, inp_root_blkno) - __field(__u64, inp_blkno) - __field(__u64, src_root_blkno) - __field(__u64, src_blkno) - __field(__u64, dst_root_blkno) - __field(__u64, dst_blkno) - __field(__u64, count) - __field(__u64, leaf_bit) - __field(int, ind) - __field(int, sm_delta) - __field(int, src_lg_delta) - __field(int, dst_lg_delta) - ), - TP_fast_assign( - SCSB_TRACE_ASSIGN(sb); - __entry->inp_root_blkno = le64_to_cpu(inp->ref.blkno); - __entry->inp_blkno = inp_blkno; - __entry->src_root_blkno = le64_to_cpu(src->ref.blkno); - __entry->src_blkno = src_blkno; - __entry->dst_root_blkno = le64_to_cpu(dst->ref.blkno); - __entry->dst_blkno = dst_blkno; - __entry->count = count; - __entry->leaf_bit = leaf_bit; - __entry->ind = ind; - __entry->sm_delta = sm_delta; - __entry->src_lg_delta = src_lg_delta; - __entry->dst_lg_delta = dst_lg_delta; - ), - TP_printk(SCSBF" irb %llu ib %llu srb %llu sb %llu drb %llu db %llu cnt %llu lb %llu ind %u smd %d sld %d dld %d", - SCSB_TRACE_ARGS, __entry->inp_root_blkno, __entry->inp_blkno, - __entry->src_root_blkno, __entry->src_blkno, - __entry->dst_root_blkno, __entry->dst_blkno, - __entry->count, __entry->leaf_bit, __entry->ind, - __entry->sm_delta, __entry->src_lg_delta, - __entry->dst_lg_delta) -); - DECLARE_EVENT_CLASS(scoutfs_ext_next_class, TP_PROTO(struct super_block *sb, u64 start, u64 len, struct scoutfs_extent *ext, int ret), From 005cf99f42a6796e338f4e5063255d5fb895f57e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 5 Oct 2020 09:42:04 -0700 Subject: [PATCH 877/920] scoutfs: use vmalloc for high order xattr allocs The xattr item stream is constructred from a large contiguous region that contains the struct header, the key, and the value. The value can be larger than a page so kmalloc is likely to fail as the system gets fragmented. Our recent move to the item cache added a significant source of page allocation churn which moved the system towards fragmentation much more quickly and was causing high-order allocation failures in testing. Signed-off-by: Zach Brown --- kmod/src/xattr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 4666eecd..921eb447 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -425,7 +425,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, /* only need enough for caller's name and value sizes */ bytes = sizeof(struct scoutfs_xattr) + name_len + size; - xat = kmalloc(bytes, GFP_NOFS); + xat = __vmalloc(bytes, GFP_NOFS, PAGE_KERNEL); if (!xat) return -ENOMEM; @@ -468,7 +468,7 @@ ssize_t scoutfs_getxattr(struct dentry *dentry, const char *name, void *buffer, ret = le16_to_cpu(xat->val_len); memcpy(buffer, &xat->name[xat->name_len], ret); out: - kfree(xat); + vfree(xat); return ret; } @@ -527,7 +527,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, return -EPERM; bytes = sizeof(struct scoutfs_xattr) + name_len + size; - xat = kmalloc(bytes, GFP_NOFS); + xat = __vmalloc(bytes, GFP_NOFS, PAGE_KERNEL); if (!xat) { ret = -ENOMEM; goto out; @@ -633,7 +633,7 @@ unlock: up_write(&si->xattr_rwsem); scoutfs_unlock(sb, lck, SCOUTFS_LOCK_WRITE); out: - kfree(xat); + vfree(xat); return ret; } From e347ca360606e63564a43b3d9ba4114ec21b4563 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Oct 2020 10:57:20 -0700 Subject: [PATCH 878/920] scoutfs: add unused item page rbtree verification Add a quick function that walks the rbtree and makes sure it doesn't see any obvious key errors. This is far too expensive to use regularly but it's handy to have around and add calls to when debugging. Signed-off-by: Zach Brown --- kmod/src/item.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/kmod/src/item.c b/kmod/src/item.c index 7062f079..fade9084 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -236,6 +236,94 @@ static void rbtree_replace_node(struct rb_node *victim, struct rb_node *new, RB_CLEAR_NODE(victim); } +/* + * This is far too expensive to use regularly, but it's very helpful for + * discovering corruption after modifications to cached pages. + */ +static __attribute__((unused)) void verify_page_rbtree(struct rb_root *root) +{ + struct cached_item *item; + struct cached_page *par; + struct cached_page *pg; + struct cached_page *n; + char *reason = NULL; + struct rb_node *p; + int cmp; + + rbtree_postorder_for_each_entry_safe(pg, n, root, node) { + + item = NULL; + par = NULL; + + if (scoutfs_key_compare(&pg->start, &pg->end) > 0) { + reason = "start > end"; + break; + } + + item = first_item(&pg->item_root); + if (item && scoutfs_key_compare(&item->key, &pg->start) < 0) { + reason = "first item < start"; + break; + } + + item = last_item(&pg->item_root); + if (item && scoutfs_key_compare(&item->key, &pg->end) > 0) { + reason = "last item > end"; + break; + } + + p = rb_parent(&pg->node); + if (!p) + continue; + par = rb_entry(p, struct cached_page, node); + + cmp = scoutfs_key_compare_ranges(&pg->start, &pg->end, + &par->start, &par->end); + if (cmp == 0) { + reason = "parent and child overlap"; + break; + } + + if (par->node.rb_right == &pg->node && cmp < 0) { + reason = "right child < parent"; + break; + } + + if (par->node.rb_left == &pg->node && cmp > 0) { + reason = "left child > parent"; + break; + } + } + + if (!reason) + return; + + printk("bad item page rbtree: %s\n", reason); + printk("pg %p start "SK_FMT" end "SK_FMT"\n", + pg, SK_ARG(&pg->start), SK_ARG(&pg->end)); + if (par) + printk("par %p start "SK_FMT" end "SK_FMT"\n", + par, SK_ARG(&par->start), SK_ARG(&par->end)); + if (item) + printk("item %p key "SK_FMT"\n", item, SK_ARG(&item->key)); + + rbtree_postorder_for_each_entry_safe(pg, n, root, node) { + printk(" pg %p left %p right %p start "SK_FMT" end "SK_FMT"\n", + pg, + pg->node.rb_left ? rb_entry(pg->node.rb_left, + struct cached_page, node) : + NULL, + pg->node.rb_right ? rb_entry(pg->node.rb_right, + struct cached_page, node) : + NULL, + SK_ARG(&pg->start), + SK_ARG(&pg->end)); + } + + BUG(); +} + + /* * This lets us lock newly allocated pages without having to add nesting * annotation. The non-acquired path is never executed. From c4663ea1a17f847fc857e4ecb5d2fe77ad5f89cd Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 6 Oct 2020 15:10:00 -0700 Subject: [PATCH 879/920] scoutfs: compact items in item cache pages The first pass of the item cache didn't try to reclaim freed space at all. It would leave behind very sparse pages. The oldest of which would be reclaimed by memory pressure. While this worked, it created much more stress on the system than is necessary. Splitting a page with one key also makes it hard to calculate the boundaries of the split pages, given that the start and end keys could be the single item. This adds a header field which tracks the free space in item cache pgaes. Free space is created before the alloc offset by removing items from the rbtree, but also from shrinking item values when updating or deleting items. If we try to split a page with sufficient free space to insert the largest possible item then we compact the page instead of splitting it. We copy the items into the front of an unused page and swap the pages. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/item.c | 110 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 140a07f5..2f4098cc 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -85,6 +85,7 @@ EXPAND_COUNTER(item_page_accessed) \ EXPAND_COUNTER(item_page_alloc) \ EXPAND_COUNTER(item_page_clear_dirty) \ + EXPAND_COUNTER(item_page_compact) \ EXPAND_COUNTER(item_page_free) \ EXPAND_COUNTER(item_page_lru_add) \ EXPAND_COUNTER(item_page_lru_remove) \ diff --git a/kmod/src/item.c b/kmod/src/item.c index fade9084..120c07c1 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -129,6 +129,7 @@ struct cached_page { struct list_head dirty_head; struct page *page; unsigned int page_off; + unsigned int erased_bytes; atomic_t refcount; }; @@ -416,6 +417,13 @@ static struct cached_item *alloc_item(struct cached_page *pg, return item; } +static void erase_item(struct cached_page *pg, struct cached_item *item) +{ + rbtree_erase(&item->node, &pg->item_root); + pg->erased_bytes += round_up(item_val_bytes(item->val_len), + CACHED_ITEM_ALIGN); +} + static void lru_add(struct super_block *sb, struct item_cache_info *cinf, struct cached_page *pg) { @@ -649,7 +657,8 @@ static void erase_page_items(struct cached_page *pg, if (scoutfs_key_compare(&item->key, end) > 0) break; - rbtree_erase(&item->node, &pg->item_root); + + erase_item(pg, item); } } @@ -702,7 +711,7 @@ static void move_page_items(struct super_block *sb, to->persistent = from->persistent; to->deletion = from->deletion; - rbtree_erase(&from->node, &left->item_root); + erase_item(left, from); } } @@ -775,6 +784,64 @@ static int trim_page_intersection(struct super_block *sb, return PGI_BISECT; } +/* + * The caller wants to allocate an item in the page but there isn't room + * at the page_off. If erasing items has left sufficient internal free + * space we can pack the existing items to the start of the page to make + * room for the insertion. + * + * The caller's empty pg is only used for its page struct, which we swap + * with our old empty page. We don't touch its pg struct. + * + * This is a coarse bulk way of dealing with free space, as opposed to + * specifically tracking internal free regions and using them to satisfy + * item allocations. + */ +static void compact_page_items(struct super_block *sb, + struct cached_page *pg, + struct cached_page *empty) +{ + struct cached_item *from; + struct cached_item *to; + struct rb_root item_root = RB_ROOT; + struct rb_node *par = NULL; + struct rb_node **pnode = &item_root.rb_node; + unsigned int page_off = 0; + LIST_HEAD(dirty_list); + + if (pg->erased_bytes < item_val_bytes(SCOUTFS_MAX_VAL_SIZE)) + return; + + if (WARN_ON_ONCE(empty->page_off != 0) || + WARN_ON_ONCE(!RB_EMPTY_ROOT(&empty->item_root)) || + WARN_ON_ONCE(!list_empty(&empty->dirty_list))) + return; + + scoutfs_inc_counter(sb, item_page_compact); + + for (from = first_item(&pg->item_root); from; from = next_item(from)) { + to = page_address(empty->page) + page_off; + page_off += round_up(item_val_bytes(from->val_len), + CACHED_ITEM_ALIGN); + + /* copy the entire item, struct members and all */ + memcpy(to, from, item_val_bytes(from->val_len)); + + rbtree_insert(&to->node, par, pnode, &item_root); + par = &to->node; + pnode = &to->node.rb_right; + + if (to->dirty) + list_add_tail(&to->dirty_head, &dirty_list); + } + + pg->item_root = item_root; + list_replace(&dirty_list, &pg->dirty_list); + swap(pg->page, empty->page); + pg->page_off = page_off; + pg->erased_bytes = 0; +} + /* * This behaves a little differently than the other walks because we * want to minimize compares and there are only simple searching and @@ -1028,6 +1095,9 @@ static int try_split_page(struct super_block *sb, struct item_cache_info *cinf, write_lock(&pg->rwlock); + if (!page_has_room(pg, val_len)) + compact_page_items(sb, pg, left); + if (page_has_room(pg, val_len)) { write_unlock(&cinf->rwlock); write_unlock(&pg->rwlock); @@ -1207,8 +1277,8 @@ static int read_page_item(struct super_block *sb, struct scoutfs_key *key, { DECLARE_ITEM_CACHE_INFO(sb, cinf); struct rb_root *root = arg; - struct cached_page *right; - struct cached_page *left; + struct cached_page *right = NULL; + struct cached_page *left = NULL; struct cached_page *pg; struct cached_item *found; struct cached_item *item; @@ -1222,10 +1292,19 @@ static int read_page_item(struct super_block *sb, struct scoutfs_key *key, if (found && (le64_to_cpu(found->liv.vers) >= le64_to_cpu(liv->vers))) return 0; + if (!page_has_room(pg, val_len)) { + left = alloc_pg(sb, 0); + /* split needs multiple items, sparse may not have enough */ + if (!left) + return -ENOMEM; + compact_page_items(sb, pg, left); + } + item = alloc_item(pg, key, liv, val, val_len); if (!item) { /* simpler split of private pages, no locking/dirty/lru */ - left = alloc_pg(sb, 0); + if (!left) + left = alloc_pg(sb, 0); right = alloc_pg(sb, 0); if (!left || !right) { put_pg(sb, left); @@ -1247,6 +1326,9 @@ static int read_page_item(struct super_block *sb, struct scoutfs_key *key, item = alloc_item(pg, key, liv, val, val_len); found = item_rbtree_walk(&pg->item_root, key, NULL, &par, &pnode); + + left = NULL; + right = NULL; } /* if deleted a deletion item will be required */ @@ -1254,7 +1336,10 @@ static int read_page_item(struct super_block *sb, struct scoutfs_key *key, rbtree_insert(&item->node, par, pnode, &pg->item_root); if (found) - rbtree_erase(&found->node, &pg->item_root); + erase_item(pg, found); + + put_pg(sb, left); + put_pg(sb, right); return 0; } @@ -1348,7 +1433,7 @@ static int read_pages(struct super_block *sb, struct item_cache_info *cinf, /* drop deletion items, we don't need them in the cache */ for_each_item_safe(&pg->item_root, item, item_tmp) { if (item->deletion) - rbtree_erase(&item->node, &pg->item_root); + erase_item(pg, item); } } @@ -1740,7 +1825,7 @@ static int item_create(struct super_block *sb, struct scoutfs_key *key, if (found) { item->persistent = found->persistent; clear_item_dirty(sb, cinf, pg, found); - rbtree_erase(&found->node, &pg->item_root); + erase_item(pg, found); } if (force) @@ -1811,6 +1896,8 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, if (val_len <= found->val_len) { if (val_len) memcpy(found->val, val, val_len); + if (val_len < found->val_len) + pg->erased_bytes += found->val_len - val_len; found->val_len = val_len; found->liv.vers = liv.vers; mark_item_dirty(sb, cinf, pg, NULL, found); @@ -1821,7 +1908,7 @@ int scoutfs_item_update(struct super_block *sb, struct scoutfs_key *key, mark_item_dirty(sb, cinf, pg, NULL, item); clear_item_dirty(sb, cinf, pg, found); - rbtree_erase(&found->node, &pg->item_root); + erase_item(pg, found); } ret = 0; @@ -1883,12 +1970,13 @@ static int item_delete(struct super_block *sb, struct scoutfs_key *key, if (!item->persistent) { /* can just forget items that aren't yet persistent */ clear_item_dirty(sb, cinf, pg, item); - rbtree_erase(&item->node, &pg->item_root); + erase_item(pg, item); } else { /* must emit deletion to clobber old persistent item */ item->liv.vers = cpu_to_le64(lock->write_version); item->liv.flags |= SCOUTFS_LOG_ITEM_FLAG_DELETION; item->deletion = 1; + pg->erased_bytes += item->val_len; item->val_len = 0; mark_item_dirty(sb, cinf, pg, NULL, item); } @@ -2115,7 +2203,7 @@ retry: /* free deletion items */ if (item->deletion) - rbtree_erase(&item->node, &pg->item_root); + erase_item(pg, item); else item->persistent = 1; } From 27bc0ef095e76ca65f2c67629b902a970786ee0d Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 7 Oct 2020 15:38:12 -0700 Subject: [PATCH 880/920] scoutfs: fix item cache page trim The tests for the various page range intersections were out of order. The edge overlap case could trigger before the bisection case and we'd fail to remove the initial items in the page. That would leave items before the start key which would later be used as a midpoint for a split, causing all kinds of chaos. Rework the cases so that the overlap cases are last. The unique bisect case will be caught before we can mistake it for an edge overlap case. And minimize the number of comparisons we calculate by storing the handful that all the cases need. Signed-off-by: Zach Brown --- kmod/src/item.c | 87 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 120c07c1..15c7a1aa 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -740,48 +740,79 @@ static int trim_page_intersection(struct super_block *sb, struct scoutfs_key *start, struct scoutfs_key *end) { - if (scoutfs_key_compare(&pg->start, end) > 0 || - scoutfs_key_compare(&pg->end, start) < 0) { - /* page and range don't intersect */ + int ps_e = scoutfs_key_compare(&pg->start, end); + int pe_s = scoutfs_key_compare(&pg->end, start); + int ps_s; + int pe_e; + + /* + * page and range don't intersect + * + * ps |----------| pe + * s |----------| e + * (or) + * ps |----------| pe + * s |----------| e + */ + if (ps_e > 0 || pe_s < 0) return PGI_DISJOINT; - } - if (scoutfs_key_compare(&pg->start, start) >= 0 && - scoutfs_key_compare(&pg->end, end) <= 0) { - /* page entirely inside range */ + ps_s = scoutfs_key_compare(&pg->start, start); + pe_e = scoutfs_key_compare(&pg->end, end); + + /* + * page entirely inside range + * + * ps |----------| pe + * s |----------| e + */ + if (ps_s >= 0 && pe_e <= 0) return PGI_INSIDE; + + /* + * page surrounds range, and is bisected by it + * + * ps |----------| pe + * s |------| e + */ + if (ps_s < 0 && pe_e > 0) { + if (!right) + return PGI_BISECT_NEEDED; + + right->start = *end; + scoutfs_key_inc(&right->start); + right->end = pg->end; + pg->end = *start; + scoutfs_key_dec(&pg->end); + erase_page_items(pg, start, end); + move_page_items(sb, cinf, pg, right, &right->start, NULL); + return PGI_BISECT; } - if (scoutfs_key_compare(&pg->start, end) <= 0 && - scoutfs_key_compare(&pg->end, end) > 0) { - /* start of page intersects with range */ + /* + * start of page overlaps with range + * + * ps |----------| pe + * s |----------| e + */ + if (pe_e > 0) { + /* start of page overlaps range */ pg->start = *end; scoutfs_key_inc(&pg->start); erase_page_items(pg, start, end); return PGI_START_OLAP; } - if (scoutfs_key_compare(&pg->end, start) >= 0 && - scoutfs_key_compare(&pg->start, start) < 0) { - /* end of page intersects with range */ - pg->end = *start; - scoutfs_key_dec(&pg->end); - erase_page_items(pg, start, end); - return PGI_END_OLAP; - } - - /* page surrounds range, and is bisected by it */ - if (!right) - return PGI_BISECT_NEEDED; - - right->start = *end; - scoutfs_key_inc(&right->start); - right->end = pg->end; + /* + * end of page overlaps with range + * + * ps |----------| pe + * s |----------| e + */ pg->end = *start; scoutfs_key_dec(&pg->end); erase_page_items(pg, start, end); - move_page_items(sb, cinf, pg, right, &right->start, NULL); - return PGI_BISECT; + return PGI_END_OLAP; } /* From 8bf4c078df282bd54db4934ea37016c763d662ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 8 Oct 2020 12:15:11 -0700 Subject: [PATCH 881/920] scoutfs: fix item cache page split key choice The algorithm for choosing the split key assumed that there were multiple items in the page. That wasn't always true and it could result in choosing the first item as the split key, which could end up decrementing the left page's end key before it's start key. We've since added compaction to the paths that split pages so we now guarantee that we have at least two items in the page being split. With that we can be sure to use the second item's key and ensure that we're never creating invalid keys for the pages created by the split. Signed-off-by: Zach Brown --- kmod/src/item.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 15c7a1aa..06104ad4 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -1053,9 +1053,14 @@ static void drop_pcpu_pages(struct super_block *sb, } /* - * We're about to move all the items between to a pair of new pages. - * Find the item that balances the space consumed by items in either - * page. We move the mid (and possibly only) item to the right page. + * Set the keys of the destination pages of a split. We try to find the + * key which balances the space consumed by items in the resulting split + * pages. We move the split key to the right, setting the left end by + * decrementing that key. We bias towards advancing the left item first + * so that we don't use it and possibly decrementing the starting page + * key. We can't have a page that covers a single key. Callers of + * split should have tried compacting which ensures that if we split we + * must have multiple items, even if they all have the max value length. */ static void set_split_keys(struct cached_page *pg, struct cached_page *left, struct cached_page *right) @@ -1066,8 +1071,14 @@ static void set_split_keys(struct cached_page *pg, struct cached_page *left, int left_tot = 0; int right_tot = 0; + BUILD_BUG_ON((PAGE_SIZE / SCOUTFS_MAX_VAL_SIZE) < 4); + BUG_ON(scoutfs_key_compare(&pg->start, &pg->end) > 0); + BUG_ON(left_item == NULL); + BUG_ON(right_item == NULL); + BUG_ON(left_item == right_item); + while (left_item && right_item && left_item != right_item) { - if (left_tot < right_tot) { + if (left_tot <= right_tot) { left_tot += item_val_bytes(left_item->val_len); left_item = next_item(left_item); } else { From fb66372988d6ee160fe5ea7087e8c8e866bf8685 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Oct 2020 10:46:16 -0700 Subject: [PATCH 882/920] scoutfs: add alloc foreach cb iterator Add an alloc call which reads all the persistent allocators and calls a callback for each. This is going to be used to calculate free blocks in clients for df, and in an ioctl to give a more detailed view of allocators. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/alloc.h | 6 ++ kmod/src/format.h | 6 ++ 3 files changed, 149 insertions(+) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index ca088c89..a7298e1e 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -1110,3 +1110,140 @@ bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, return lo; } + +/* + * Call the callers callback for every persistent allocator structure + * we can find. + */ +int scoutfs_alloc_foreach(struct super_block *sb, + scoutfs_alloc_foreach_cb_t cb, void *arg) +{ + struct scoutfs_btree_ref stale_refs[2] = {{0,}}; + struct scoutfs_btree_ref refs[2] = {{0,}}; + struct scoutfs_super_block *super = NULL; + struct scoutfs_srch_compact_input *scin; + struct scoutfs_log_trees_val ltv; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + scin = kmalloc(sizeof(struct scoutfs_srch_compact_input), GFP_NOFS); + if (!super || !scin) { + ret = -ENOMEM; + goto out; + } + +retry: + ret = scoutfs_read_super(sb, super); + if (ret < 0) + goto out; + + refs[0] = super->logs_root.ref; + refs[1] = super->srch_root.ref; + + /* all the server allocators */ + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, true, true, + le64_to_cpu(super->meta_alloc[0].total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, true, true, + le64_to_cpu(super->meta_alloc[1].total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 0, false, true, + le64_to_cpu(super->data_alloc.total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, true, + le64_to_cpu(super->server_meta_avail[0].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, true, + le64_to_cpu(super->server_meta_avail[1].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, false, + le64_to_cpu(super->server_meta_freed[0].total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SERVER, 1, true, false, + le64_to_cpu(super->server_meta_freed[1].total_nr)); + if (ret < 0) + goto out; + + /* mount fs transaction allocators */ + scoutfs_key_init_log_trees(&key, 0, 0); + for (;;) { + ret = scoutfs_btree_next(sb, &super->logs_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret < 0) + goto out; + + if (iref.val_len == sizeof(ltv)) { + key = *iref.key; + memcpy(<v, iref.val, sizeof(ltv)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + if (ret < 0) + goto out; + + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), true, true, + le64_to_cpu(ltv.meta_avail.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), true, false, + le64_to_cpu(ltv.meta_freed.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), false, true, + le64_to_cpu(ltv.data_avail.total_len)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, + le64_to_cpu(key.sklt_rid), false, false, + le64_to_cpu(ltv.data_freed.total_len)); + if (ret < 0) + goto out; + + scoutfs_key_inc(&key); + } + + /* srch compaction allocators */ + memset(&key, 0, sizeof(key)); + key.sk_zone = SCOUTFS_SRCH_ZONE; + key.sk_type = SCOUTFS_SRCH_BUSY_TYPE; + + for (;;) { + /* _BUSY_ is last type, _next won't see other types */ + ret = scoutfs_btree_next(sb, &super->srch_root, &key, &iref); + if (ret == -ENOENT) + break; + if (ret == 0) { + if (iref.val_len == sizeof(scin)) { + key = *iref.key; + memcpy(scin, iref.val, iref.val_len); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) + goto out; + + ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, + le64_to_cpu(scin->id), true, true, + le64_to_cpu(scin->meta_avail.total_nr)) ?: + cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, + le64_to_cpu(scin->id), true, false, + le64_to_cpu(scin->meta_freed.total_nr)); + if (ret < 0) + goto out; + + scoutfs_key_inc(&key); + } + + ret = 0; +out: + if (ret == -ESTALE) { + if (memcmp(&stale_refs, &refs, sizeof(refs)) == 0) { + ret = -EIO; + } else { + BUILD_BUG_ON(sizeof(stale_refs) != sizeof(refs)); + memcpy(stale_refs, refs, sizeof(stale_refs)); + goto retry; + } + } + + kfree(super); + kfree(scin); + return ret; +} diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index 7b053756..d2cc1f58 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -122,4 +122,10 @@ int scoutfs_alloc_splice_list(struct super_block *sb, bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, struct scoutfs_alloc *alloc); +typedef int (*scoutfs_alloc_foreach_cb_t)(struct super_block *sb, void *arg, + int owner, u64 id, + bool meta, bool avail, u64 blocks); +int scoutfs_alloc_foreach(struct super_block *sb, + scoutfs_alloc_foreach_cb_t cb, void *arg); + #endif diff --git a/kmod/src/format.h b/kmod/src/format.h index 428c94e6..2bc0d010 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -297,6 +297,12 @@ struct scoutfs_alloc_root { struct scoutfs_btree_root root; }__packed; +/* types of allocators, exposed to alloc_detail ioctl */ +#define SCOUTFS_ALLOC_OWNER_NONE 0 +#define SCOUTFS_ALLOC_OWNER_SERVER 1 +#define SCOUTFS_ALLOC_OWNER_MOUNT 2 +#define SCOUTFS_ALLOC_OWNER_SRCH 3 + struct scoutfs_mounted_client_btree_val { __u8 flags; } __packed; From 3d790b24d58e8a6d0dc3f6fd7c403578d0d30b2e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Oct 2020 10:48:05 -0700 Subject: [PATCH 883/920] scoutfs: add alloc_detail ioctl An an ioctl which copies details of each persistent allocator to userspace. This will be used by a scoutfs command to give information about the allocators in the system. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ kmod/src/ioctl.h | 17 ++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 932c4ce3..462619a0 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -36,6 +36,7 @@ #include "xattr.h" #include "hash.h" #include "srch.h" +#include "alloc.h" #include "scoutfs_trace.h" /* @@ -867,6 +868,53 @@ static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) return 0; } +struct copy_alloc_detail_args { + struct scoutfs_ioctl_alloc_detail_entry __user *uade; + u64 nr; + u64 copied; +}; + +static int copy_alloc_detail_to_user(struct super_block *sb, void *arg, + int owner, u64 id, bool meta, bool avail, + u64 blocks) +{ + struct copy_alloc_detail_args *args = arg; + struct scoutfs_ioctl_alloc_detail_entry ade; + + if (args->copied == args->nr) + return -EOVERFLOW; + + ade.blocks = blocks; + ade.id = id; + ade.meta = !!meta; + ade.avail = !!avail; + + if (copy_to_user(&args->uade[args->copied], &ade, sizeof(ade))) + return -EFAULT; + + args->copied++; + return 0; +} + +static long scoutfs_ioc_alloc_detail(struct file *file, unsigned long arg) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct scoutfs_ioctl_alloc_detail __user *uad = (void __user *)arg; + struct scoutfs_ioctl_alloc_detail ad; + struct copy_alloc_detail_args args; + + if (copy_from_user(&ad, uad, sizeof(ad))) + return -EFAULT; + + args.uade = (struct scoutfs_ioctl_alloc_detail_entry __user *) + (uintptr_t)ad.entries_ptr; + args.nr = ad.entries_nr; + args.copied = 0; + + return scoutfs_alloc_foreach(sb, copy_alloc_detail_to_user, &args) ?: + args.copied; +} + long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { @@ -892,6 +940,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return scoutfs_ioc_statfs_more(file, arg); case SCOUTFS_IOC_DATA_WAIT_ERR: return scoutfs_ioc_data_wait_err(file, arg); + case SCOUTFS_IOC_ALLOC_DETAIL: + return scoutfs_ioc_alloc_detail(file, arg); } return -ENOTTY; diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 1ef0aa36..ec0875b7 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -392,4 +392,21 @@ struct scoutfs_ioctl_data_wait_err { #define SCOUTFS_IOC_DATA_WAIT_ERR _IOR(SCOUTFS_IOCTL_MAGIC, 11, \ struct scoutfs_ioctl_data_wait_err) + +#define SCOUTFS_IOC_ALLOC_DETAIL _IOR(SCOUTFS_IOCTL_MAGIC, 12, \ + struct scoutfs_ioctl_alloc_detail) + +struct scoutfs_ioctl_alloc_detail { + __u64 entries_ptr; + __u64 entries_nr; +}; + +struct scoutfs_ioctl_alloc_detail_entry { + __u64 id; + __u64 blocks; + __u8 type; + __u8 meta:1, + avail:1; +}; + #endif From 33374d8fe6c208f1b17425ee7bd8a9fefcec2f61 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Oct 2020 10:50:31 -0700 Subject: [PATCH 884/920] scoutfs: get statfs free blocks with alloc_foreach Use alloc_foreach to count the free blocks in all the allocators instead of sending an RPC to the server. We cache the results so that constant df calls don't generate a constant stream of IO. Signed-off-by: Zach Brown --- kmod/src/counters.h | 1 + kmod/src/super.c | 70 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 2f4098cc..287e8cad 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -169,6 +169,7 @@ EXPAND_COUNTER(srch_search_stale_retry) \ EXPAND_COUNTER(srch_search_xattrs) \ EXPAND_COUNTER(srch_read_stale) \ + EXPAND_COUNTER(statfs) \ EXPAND_COUNTER(trans_commit_data_alloc_low) \ EXPAND_COUNTER(trans_commit_dirty_meta_full) \ EXPAND_COUNTER(trans_commit_fsync) \ diff --git a/kmod/src/super.c b/kmod/src/super.c index 5c722787..415ef4b8 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -43,6 +43,7 @@ #include "forest.h" #include "srch.h" #include "item.h" +#include "alloc.h" #include "scoutfs_trace.h" static struct dentry *scoutfs_debugfs_root; @@ -78,11 +79,30 @@ retry: return cpu_to_le64(ret); } +struct statfs_free_blocks { + u64 meta; + u64 data; +}; + +static int count_free_blocks(struct super_block *sb, void *arg, int owner, + u64 id, bool meta, bool avail, u64 blocks) +{ + struct statfs_free_blocks *sfb = arg; + + if (meta) + sfb->meta += blocks; + else + sfb->data += blocks; + + return 0; +} + /* - * Ask the server for the current statfs fields. The message is very - * cheap so we're not worrying about spinning in statfs flooding the - * server with requests. We can add a cache and stale results if that - * becomes a problem. + * Build the free block counts by having alloc read all the persistent + * blocks which contain allocators and calling us for each of them. + * Only the super block reads aren't cached so repeatedly calling statfs + * is like repeated O_DIRECT IO. We can add a cache and stale results + * if that IO becomes a problem. * * We fake the number of free inodes value by assuming that we can fill * free blocks with a certain number of inodes. We then the number of @@ -95,30 +115,50 @@ retry: static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) { struct super_block *sb = dentry->d_inode->i_sb; - struct scoutfs_net_statfs nstatfs; + struct scoutfs_super_block *super = NULL; + struct statfs_free_blocks sfb = {0,}; __le32 uuid[4]; int ret; - ret = scoutfs_client_statfs(sb, &nstatfs); - if (ret) - return ret; + scoutfs_inc_counter(sb, statfs); - kst->f_bfree = le64_to_cpu(nstatfs.bfree); + super = kzalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!super) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super(sb, super); + if (ret) + goto out; + + ret = scoutfs_alloc_foreach(sb, count_free_blocks, &sfb); + if (ret < 0) + goto out; + + kst->f_bfree = (sfb.meta << SCOUTFS_BLOCK_SM_LG_SHIFT) + sfb.data; kst->f_type = SCOUTFS_SUPER_MAGIC; kst->f_bsize = SCOUTFS_BLOCK_SM_SIZE; - kst->f_blocks = le64_to_cpu(nstatfs.total_blocks); + kst->f_blocks = (le64_to_cpu(super->total_meta_blocks) << + SCOUTFS_BLOCK_SM_LG_SHIFT) + + le64_to_cpu(super->total_data_blocks); kst->f_bavail = kst->f_bfree; - kst->f_ffree = kst->f_bfree * 16; - kst->f_files = kst->f_ffree + le64_to_cpu(nstatfs.next_ino); + /* arbitrarily assume ~1K / empty file */ + kst->f_ffree = sfb.meta * (SCOUTFS_BLOCK_LG_SIZE / 1024); + kst->f_files = kst->f_ffree + le64_to_cpu(super->next_ino); - BUILD_BUG_ON(sizeof(uuid) != sizeof(nstatfs.uuid)); - memcpy(uuid, &nstatfs, sizeof(uuid)); + BUILD_BUG_ON(sizeof(uuid) != sizeof(super->uuid)); + memcpy(uuid, super->uuid, sizeof(uuid)); kst->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[1]); kst->f_fsid.val[1] = le32_to_cpu(uuid[2]) ^ le32_to_cpu(uuid[3]); kst->f_namelen = SCOUTFS_NAME_LEN; kst->f_frsize = SCOUTFS_BLOCK_SM_SIZE; + /* the vfs fills f_flags */ + ret = 0; +out: + kfree(super); /* * We don't take cluster locks in statfs which makes it a very @@ -128,7 +168,7 @@ static int scoutfs_statfs(struct dentry *dentry, struct kstatfs *kst) if (scoutfs_trigger(sb, STATFS_LOCK_PURGE)) scoutfs_free_unused_locks(sb, -1UL); - return 0; + return ret; } static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) From 2073a672a0d81f90e73a43db1e084ae587f149ac Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Oct 2020 10:51:56 -0700 Subject: [PATCH 885/920] scoutfs: remove unused statfs RPC Remove the statfs RPC from the client and server now that we're using allocator iteration to calculate free blocks. Signed-off-by: Zach Brown --- kmod/src/client.c | 11 ----------- kmod/src/client.h | 2 -- kmod/src/format.h | 8 -------- kmod/src/server.c | 45 --------------------------------------------- 4 files changed, 66 deletions(-) diff --git a/kmod/src/client.c b/kmod/src/client.c index 6809fde8..50d8d2e0 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -150,17 +150,6 @@ int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq) return ret; } -int scoutfs_client_statfs(struct super_block *sb, - struct scoutfs_net_statfs *nstatfs) -{ - struct client_info *client = SCOUTFS_SB(sb)->client_info; - - return scoutfs_net_sync_request(sb, client->conn, - SCOUTFS_NET_CMD_STATFS, NULL, 0, - nstatfs, - sizeof(struct scoutfs_net_statfs)); -} - /* process an incoming grant response from the server */ static int client_lock_response(struct super_block *sb, struct scoutfs_net_connection *conn, diff --git a/kmod/src/client.h b/kmod/src/client.h index 21bf7a39..04bc1b48 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -12,8 +12,6 @@ int scoutfs_client_get_roots(struct super_block *sb, u64 *scoutfs_client_bulk_alloc(struct super_block *sb); int scoutfs_client_advance_seq(struct super_block *sb, u64 *seq); int scoutfs_client_get_last_seq(struct super_block *sb, u64 *seq); -int scoutfs_client_statfs(struct super_block *sb, - struct scoutfs_net_statfs *nstatfs); int scoutfs_client_lock_request(struct super_block *sb, struct scoutfs_net_lock *nl); int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, diff --git a/kmod/src/format.h b/kmod/src/format.h index 2bc0d010..ee0e0d69 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -822,7 +822,6 @@ enum { SCOUTFS_NET_CMD_GET_ROOTS, SCOUTFS_NET_CMD_ADVANCE_SEQ, SCOUTFS_NET_CMD_GET_LAST_SEQ, - SCOUTFS_NET_CMD_STATFS, SCOUTFS_NET_CMD_LOCK, SCOUTFS_NET_CMD_LOCK_RECOVER, SCOUTFS_NET_CMD_SRCH_GET_COMPACT, @@ -863,13 +862,6 @@ struct scoutfs_net_inode_alloc { __le64 nr; } __packed; -struct scoutfs_net_statfs { - __le64 total_blocks; /* total blocks in device */ - __le64 next_ino; /* next unused inode number */ - __le64 bfree; /* free blocks */ - __u8 uuid[SCOUTFS_UUID_BYTES]; /* logical volume uuid */ -} __packed; - struct scoutfs_net_roots { struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; diff --git a/kmod/src/server.c b/kmod/src/server.c index 48e107ce..938438a2 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -838,50 +838,6 @@ out: &last_seq, sizeof(last_seq)); } -static inline __le64 le64_lg_to_sm(__le64 lg) -{ - return cpu_to_le64(le64_to_cpu(lg) << SCOUTFS_BLOCK_SM_LG_SHIFT); -} - -/* - * Sample the super stats that the client wants for statfs by serializing - * with each component. - */ -static int server_statfs(struct super_block *sb, - struct scoutfs_net_connection *conn, - u8 cmd, u64 id, void *arg, u16 arg_len) -{ - DECLARE_SERVER_INFO(sb, server); - struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); - struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_net_statfs nstatfs; - int ret; - - if (arg_len == 0) { - /* uuid and total_segs are constant, so far */ - memcpy(nstatfs.uuid, super->uuid, sizeof(nstatfs.uuid)); - - spin_lock(&sbi->next_ino_lock); - nstatfs.next_ino = super->next_ino; - spin_unlock(&sbi->next_ino_lock); - - mutex_lock(&server->alloc_mutex); - nstatfs.total_blocks = le64_lg_to_sm(super->total_meta_blocks); - le64_add_cpu(&nstatfs.total_blocks, - le64_to_cpu(super->total_data_blocks)); - nstatfs.bfree = le64_lg_to_sm(super->free_meta_blocks); - le64_add_cpu(&nstatfs.bfree, - le64_to_cpu(super->free_data_blocks)); - mutex_unlock(&server->alloc_mutex); - ret = 0; - } else { - ret = -EINVAL; - } - - return scoutfs_net_response(sb, conn, cmd, id, ret, - &nstatfs, sizeof(nstatfs)); -} - static int server_lock(struct super_block *sb, struct scoutfs_net_connection *conn, u8 cmd, u64 id, void *arg, u16 arg_len) @@ -1537,7 +1493,6 @@ static scoutfs_net_request_t server_req_funcs[] = { [SCOUTFS_NET_CMD_GET_ROOTS] = server_get_roots, [SCOUTFS_NET_CMD_ADVANCE_SEQ] = server_advance_seq, [SCOUTFS_NET_CMD_GET_LAST_SEQ] = server_get_last_seq, - [SCOUTFS_NET_CMD_STATFS] = server_statfs, [SCOUTFS_NET_CMD_LOCK] = server_lock, [SCOUTFS_NET_CMD_SRCH_GET_COMPACT] = server_srch_get_compact, [SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT] = server_srch_commit_compact, From d589881855a31bb85e456de29ad441811f963cd5 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 12 Oct 2020 13:24:29 -0700 Subject: [PATCH 886/920] scoutfs: add tot m/d device blocks to statfs_more The total_{meta,data}_blocks scoutfs_super_block fields initialized by mkfs aren't visible to userspace anywhere. Add them to statfs_more so that tools can get the totals (and use them for df, in this particular case). Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 2 ++ kmod/src/ioctl.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 462619a0..34d75200 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -857,6 +857,8 @@ static long scoutfs_ioc_statfs_more(struct file *file, unsigned long arg) sizeof(struct scoutfs_ioctl_statfs_more)); sfm.fsid = le64_to_cpu(super->hdr.fsid); sfm.rid = sbi->rid; + sfm.total_meta_blocks = le64_to_cpu(super->total_meta_blocks); + sfm.total_data_blocks = le64_to_cpu(super->total_data_blocks); ret = scoutfs_client_get_last_seq(sb, &sfm.committed_seq); if (ret) diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index ec0875b7..f871d37e 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -369,6 +369,8 @@ struct scoutfs_ioctl_statfs_more { __u64 fsid; __u64 rid; __u64 committed_seq; + __u64 total_meta_blocks; + __u64 total_data_blocks; } __packed; #define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ From 7a3749d5913efb4b1911ecd1277bccec8d3dc9f8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 14 Oct 2020 14:10:50 -0700 Subject: [PATCH 887/920] scoutfs: incremental srch compaction Previously the srch compaction work would output the entire compacted file and delete the input files in one atomic commit. The server would send the input files and an allocator to the client, and the client would send back an output file and an allocator that included the deletion of the input files. The server would merge in the allocator and replace the input file items with the output file item. Doing it this way required giving an enormous allocation pool to the client in a radix, which would deal with recursive operations (allocating from and freeing to the radix that is being modified). We no longer have the radix allocator, and we use single block avail/free lists instead of recursively modifying the btrees with free extent items. The compaction RPC needs to work with a finite amount of allocator resources that can be stored in an alloc list block. The compaction work now does a fixed amount of work and a compaction operation spans multiple work iterations. A single compaction struct is now sent between the client and server in the get_compact and commit_compact messages. The client records any partial progress in the struct. The server writes that position into PENDING items. It first searchs for pending items to give to clients before searching for files to start a new compaction operation. The compact struct has flags to indicate whether the output file is being written or the input files are being deleted. The server manages the flags and sets the input file deletion flag only once the result of the compaction has been reflected in the btree items which record srch files. We added the progress fields to the compaction struct, making it even bigger than it already was, so we take the time to allocate them rather than declaring them on the stack. It's worth mentioning that each operation now takes a reasonably bounded amount of time will make it feasible to decide that it has failed and needs to be fenced. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 24 +- kmod/src/client.c | 8 +- kmod/src/client.h | 4 +- kmod/src/counters.h | 1 - kmod/src/format.h | 43 ++- kmod/src/server.c | 45 +-- kmod/src/srch.c | 712 ++++++++++++++++++++++++++++++-------------- kmod/src/srch.h | 7 +- 8 files changed, 555 insertions(+), 289 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index a7298e1e..29607898 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -1121,15 +1121,15 @@ int scoutfs_alloc_foreach(struct super_block *sb, struct scoutfs_btree_ref stale_refs[2] = {{0,}}; struct scoutfs_btree_ref refs[2] = {{0,}}; struct scoutfs_super_block *super = NULL; - struct scoutfs_srch_compact_input *scin; + struct scoutfs_srch_compact *sc; struct scoutfs_log_trees_val ltv; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); - scin = kmalloc(sizeof(struct scoutfs_srch_compact_input), GFP_NOFS); - if (!super || !scin) { + sc = kmalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (!super || !sc) { ret = -ENOMEM; goto out; } @@ -1200,17 +1200,17 @@ retry: /* srch compaction allocators */ memset(&key, 0, sizeof(key)); key.sk_zone = SCOUTFS_SRCH_ZONE; - key.sk_type = SCOUTFS_SRCH_BUSY_TYPE; + key.sk_type = SCOUTFS_SRCH_PENDING_TYPE; for (;;) { - /* _BUSY_ is last type, _next won't see other types */ + /* _PENDING_ and _BUSY_ are last, _next won't see other types */ ret = scoutfs_btree_next(sb, &super->srch_root, &key, &iref); if (ret == -ENOENT) break; if (ret == 0) { - if (iref.val_len == sizeof(scin)) { + if (iref.val_len == sizeof(*sc)) { key = *iref.key; - memcpy(scin, iref.val, iref.val_len); + memcpy(sc, iref.val, iref.val_len); } else { ret = -EIO; } @@ -1220,11 +1220,11 @@ retry: goto out; ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, - le64_to_cpu(scin->id), true, true, - le64_to_cpu(scin->meta_avail.total_nr)) ?: + le64_to_cpu(sc->id), true, true, + le64_to_cpu(sc->meta_avail.total_nr)) ?: cb(sb, arg, SCOUTFS_ALLOC_OWNER_SRCH, - le64_to_cpu(scin->id), true, false, - le64_to_cpu(scin->meta_freed.total_nr)); + le64_to_cpu(sc->id), true, false, + le64_to_cpu(sc->meta_freed.total_nr)); if (ret < 0) goto out; @@ -1244,6 +1244,6 @@ out: } kfree(super); - kfree(scin); + kfree(sc); return ret; } diff --git a/kmod/src/client.c b/kmod/src/client.c index 50d8d2e0..305794ca 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -201,24 +201,24 @@ int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, /* Find srch files that need to be compacted. */ int scoutfs_client_srch_get_compact(struct super_block *sb, - struct scoutfs_srch_compact_input *scin) + struct scoutfs_srch_compact *sc) { struct client_info *client = SCOUTFS_SB(sb)->client_info; return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_SRCH_GET_COMPACT, - NULL, 0, scin, sizeof(*scin)); + NULL, 0, sc, sizeof(*sc)); } /* Commit the result of a srch file compaction. */ int scoutfs_client_srch_commit_compact(struct super_block *sb, - struct scoutfs_srch_compact_result *scres) + struct scoutfs_srch_compact *res) { struct client_info *client = SCOUTFS_SB(sb)->client_info; return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT, - scres, sizeof(*scres), NULL, 0); + res, sizeof(*res), NULL, 0); } /* The client is receiving a invalidation request from the server */ diff --git a/kmod/src/client.h b/kmod/src/client.h index 04bc1b48..ae830ef8 100644 --- a/kmod/src/client.h +++ b/kmod/src/client.h @@ -19,9 +19,9 @@ int scoutfs_client_lock_response(struct super_block *sb, u64 net_id, int scoutfs_client_lock_recover_response(struct super_block *sb, u64 net_id, struct scoutfs_net_lock_recover *nlr); int scoutfs_client_srch_get_compact(struct super_block *sb, - struct scoutfs_srch_compact_input *scin); + struct scoutfs_srch_compact *sc); int scoutfs_client_srch_commit_compact(struct super_block *sb, - struct scoutfs_srch_compact_result *scres); + struct scoutfs_srch_compact *res); int scoutfs_client_setup(struct super_block *sb); void scoutfs_client_destroy(struct super_block *sb); diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 287e8cad..93515df0 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -155,7 +155,6 @@ EXPAND_COUNTER(srch_compact_dirty_block) \ EXPAND_COUNTER(srch_compact_entry) \ EXPAND_COUNTER(srch_compact_flush) \ - EXPAND_COUNTER(srch_compact_free_block) \ EXPAND_COUNTER(srch_compact_log_page) \ EXPAND_COUNTER(srch_compact_removed_entry) \ EXPAND_COUNTER(srch_inconsistent_ref) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index ee0e0d69..6dafb382 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -374,27 +374,39 @@ struct scoutfs_srch_block { #define SCOUTFS_SRCH_COMPACT_ORDER 3 #define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) -struct scoutfs_srch_compact_input { +/* + * A persistent record of a srch file compaction operation in progress. + * + * When compacting log files blk and pos aren't used. When compacting + * sorted files blk is the logical block number and pos is the byte + * offset of the next entry. When deleting files pos is the height of + * the level that we're deleting, and blk is the logical block offset of + * the next parent ref array index to descend through. + */ +struct scoutfs_srch_compact { struct scoutfs_alloc_list_head meta_avail; struct scoutfs_alloc_list_head meta_freed; __le64 id; __u8 nr; __u8 flags; - struct scoutfs_srch_file sfl[SCOUTFS_SRCH_COMPACT_NR]; + struct scoutfs_srch_file out; + struct scoutfs_srch_compact_input { + struct scoutfs_srch_file sfl; + __le64 blk; + __le64 pos; + } in[SCOUTFS_SRCH_COMPACT_NR] __packed; } __packed; -struct scoutfs_srch_compact_result { - struct scoutfs_alloc_list_head meta_avail; - struct scoutfs_alloc_list_head meta_freed; - __le64 id; - __u8 flags; - struct scoutfs_srch_file sfl; -} __packed; - -/* files are insorted logs */ -#define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) -/* compaction failed, release inputs */ -#define SCOUTFS_SRCH_COMPACT_FLAG_ERROR (1 << 1) +/* server -> client: combine input log file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) +/* server -> client: combine input sorted file entries into output file */ +#define SCOUTFS_SRCH_COMPACT_FLAG_SORTED (1 << 1) +/* server -> client: delete input files */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DELETE (1 << 2) +/* client -> server: compaction phase (LOG,SORTED,DELETE) done */ +#define SCOUTFS_SRCH_COMPACT_FLAG_DONE (1 << 4) +/* client -> server: compaction failed */ +#define SCOUTFS_SRCH_COMPACT_FLAG_ERROR (1 << 5) /* * XXX I imagine we should rename these now that they've evolved to track @@ -496,7 +508,8 @@ struct scoutfs_bloom_block { /* srch zone, only in server btrees */ #define SCOUTFS_SRCH_LOG_TYPE 1 #define SCOUTFS_SRCH_BLOCKS_TYPE 2 -#define SCOUTFS_SRCH_BUSY_TYPE 3 +#define SCOUTFS_SRCH_PENDING_TYPE 3 +#define SCOUTFS_SRCH_BUSY_TYPE 4 /* free extents in allocator btrees in client and server, by blkno or len */ #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 diff --git a/kmod/src/server.c b/kmod/src/server.c index 938438a2..7a72b3f0 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -925,55 +925,57 @@ static int server_srch_get_compact(struct super_block *sb, u64 rid = scoutfs_net_client_rid(conn); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_srch_compact_input scin; - u64 blocks; + struct scoutfs_srch_compact *sc = NULL; int ret; - int i; - - memset(&scin, 0, sizeof(scin)); if (arg_len != 0) { ret = -EINVAL; goto out; } + sc = kzalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (sc == NULL) { + ret = -ENOMEM; + goto out; + } + ret = scoutfs_server_hold_commit(sb); if (ret) goto out; mutex_lock(&server->srch_mutex); ret = scoutfs_srch_get_compact(sb, &server->alloc, &server->wri, - &super->srch_root, rid, &scin); + &super->srch_root, rid, sc); mutex_unlock(&server->srch_mutex); - if (ret == 0 && scin.nr == 0) + if (ret == 0 && sc->nr == 0) ret = -ENOENT; if (ret < 0) goto apply; - /* provide ~3x input blocks to allocate, write+delete+cow */ - blocks = 0; - for (i = 0; i < scin.nr; i++) - blocks += le64_to_cpu(scin.sfl[i].blocks); - blocks *= 3; mutex_lock(&server->alloc_mutex); ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, - &scin.meta_avail, server->meta_avail, - blocks, blocks); + &sc->meta_avail, server->meta_avail, + SCOUTFS_SERVER_META_FILL_LO, + SCOUTFS_SERVER_META_FILL_TARGET) ?: + scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, + server->other_freed, &sc->meta_freed); mutex_unlock(&server->alloc_mutex); if (ret < 0) goto apply; mutex_lock(&server->srch_mutex); ret = scoutfs_srch_update_compact(sb, &server->alloc, &server->wri, - &super->srch_root, rid, &scin); + &super->srch_root, rid, sc); mutex_unlock(&server->srch_mutex); apply: ret = scoutfs_server_apply_commit(sb, ret); WARN_ON_ONCE(ret < 0 && ret != -ENOENT); /* XXX leaked busy item */ out: - return scoutfs_net_response(sb, conn, cmd, id, ret, - &scin, sizeof(scin)); + ret = scoutfs_net_response(sb, conn, cmd, id, ret, + sc, sizeof(struct scoutfs_srch_compact)); + kfree(sc); + return ret; } /* @@ -990,16 +992,16 @@ static int server_srch_commit_compact(struct super_block *sb, u64 rid = scoutfs_net_client_rid(conn); struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &sbi->super; - struct scoutfs_srch_compact_result *scres; + struct scoutfs_srch_compact *sc; struct scoutfs_alloc_list_head av; struct scoutfs_alloc_list_head fr; int ret; - scres = arg; - if (arg_len != sizeof(*scres)) { + if (arg_len != sizeof(struct scoutfs_srch_compact)) { ret = -EINVAL; goto out; } + sc = arg; ret = scoutfs_server_hold_commit(sb); if (ret) @@ -1007,12 +1009,13 @@ static int server_srch_commit_compact(struct super_block *sb, mutex_lock(&server->srch_mutex); ret = scoutfs_srch_commit_compact(sb, &server->alloc, &server->wri, - &super->srch_root, rid, scres, + &super->srch_root, rid, sc, &av, &fr); mutex_unlock(&server->srch_mutex); if (ret < 0) /* XXX very bad, leaks allocators */ goto apply; + /* reclaim allocators if they were set by _srch_commit_ */ mutex_lock(&server->alloc_mutex); ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, server->other_freed, &av) ?: diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 2ecae4fb..283ba208 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -298,6 +298,58 @@ retry: return ret; } +/* + * Give the caller a read-only reference to the block along the path to + * the logical block at the given level. This shouldn't be called on an + * empty root. + */ +static int read_path_block(struct super_block *sb, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_file *sfl, + u64 blk, int at_level, + struct scoutfs_block **bl_ret) +{ + struct scoutfs_block *bl = NULL; + struct scoutfs_srch_parent *srp; + struct scoutfs_srch_ref ref; + int level; + int ind; + int ret; + + if (WARN_ON_ONCE(at_level < 0 || at_level >= sfl->height)) + return -EINVAL; + + level = sfl->height; + ref = sfl->ref; + while (level--) { + if (ref.blkno == 0) { + ret = -ENOENT; + break; + } + + ret = read_srch_block(sb, wri, level, &ref, &bl); + if (ret < 0) + break; + + if (level == at_level) { + ret = 0; + break; + } + + srp = bl->data; + ind = calc_ref_ind(blk, level); + ref = srp->refs[ind]; + scoutfs_block_put(sb, bl); + bl = NULL; + } + + if (ret < 0) + scoutfs_block_put(sb, bl); + else + *bl_ret = bl; + return ret; +} + /* * Walk radix blocks to find the logical file block and return the * reference to the caller. Flags determine if we cow new dirty blocks, @@ -1011,20 +1063,19 @@ int scoutfs_srch_rotate_log(struct super_block *sb, } /* - * Running in the server, find candidates for a compaction operation. - * We see if any tier has enough files waiting for a compaction. We - * first search log files and then each greater size tier. We skip any - * files which are currently referenced by existing compaction busy - * items. + * Running in the server, get a compaction operation to send to the + * client. We first see if there are any pending operations to continue + * working on. If not, we see if any tier has enough files waiting for + * a compaction. We first search log files and then each greater size + * tier. We skip input files which are currently being read by busy + * compaction items. */ int scoutfs_srch_get_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - u64 rid, - struct scoutfs_srch_compact_input *scin) + u64 rid, struct scoutfs_srch_compact *sc) { - struct scoutfs_srch_compact_input busy_scin = {{{0,}}}; struct scoutfs_srch_file sfl; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_spbm busy; @@ -1033,22 +1084,26 @@ int scoutfs_srch_get_compact(struct super_block *sb, int order; int type; int ret; + int err; int i; - /* build up a bitmap of file files already being compacted */ + /* + * Search for pending or busy items. If we find a pending item + * we move it to busy and return it. We build up a bitmap of + * input files which are in busy items. + */ scoutfs_spbm_init(&busy); - init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, 0, 0); + for (init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, 0, 0); ; + scoutfs_key_inc(&key)) { - for (;;) { - /* _BUSY_ is last type, _next won't see other types */ + /* _PENDING_ and _BUSY_ are last, _next won't see other types */ ret = scoutfs_btree_next(sb, root, &key, &iref); if (ret == -ENOENT) break; if (ret == 0) { - if (iref.val_len == sizeof(busy_scin)) { + if (iref.val_len == sizeof(*sc)) { key = *iref.key; - scoutfs_key_inc(&key); - memcpy(&busy_scin, iref.val, iref.val_len); + memcpy(sc, iref.val, iref.val_len); } else { ret = -EIO; } @@ -1057,24 +1112,53 @@ int scoutfs_srch_get_compact(struct super_block *sb, if (ret < 0) goto out; - for (i = 0; i < busy_scin.nr; i++) { - ret = scoutfs_spbm_set(&busy, - le64_to_cpu(busy_scin.sfl[i].ref.blkno)); - if (ret < 0) - goto out; + /* record all the busy input files */ + if (key.sk_type == SCOUTFS_SRCH_BUSY_TYPE) { + for (i = 0; i < sc->nr; i++) { + ret = scoutfs_spbm_set(&busy, + le64_to_cpu(sc->in[i].sfl.ref.blkno)); + if (ret < 0) + goto out; + } + continue; } + + /* or move the first pending to busy and return it */ + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, + le64_to_cpu(sc->id)); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sc, sizeof(*sc)); + if (ret < 0) + goto out; + + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(sc->id), 0); + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0) { + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, + le64_to_cpu(sc->id)); + err = scoutfs_btree_delete(sb, alloc, wri, root, &key); + BUG_ON(err); /* XXX both pending and busy :/ */ + goto out; + } + + /* found one */ + ret = 0; + goto out; } + /* no pending, look for sufficient files to start a new compaction */ + memset(sc, 0, sizeof(struct scoutfs_srch_compact)); + /* first look for unsorted log files */ type = SCOUTFS_SRCH_LOG_TYPE; init_srch_key(&key, type, 0, 0); - scin->nr = 0; for (;;scoutfs_key_inc(&key)) { ret = scoutfs_btree_next(sb, root, &key, &iref); if (ret == -ENOENT) { ret = 0; - scin->nr = 0; + sc->nr = 0; goto out; } @@ -1096,7 +1180,7 @@ int scoutfs_srch_get_compact(struct super_block *sb, /* see if we ran out of log files or files entirely */ if (key.sk_type != type) { - scin->nr = 0; + sc->nr = 0; if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { type = SCOUTFS_SRCH_BLOCKS_TYPE; } else { @@ -1111,33 +1195,35 @@ int scoutfs_srch_get_compact(struct super_block *sb, SCOUTFS_SRCH_COMPACT_ORDER; if (order != cur_order) { cur_order = order; - scin->nr = 0; + sc->nr = 0; } } - scin->sfl[scin->nr++] = sfl; - if (scin->nr == SCOUTFS_SRCH_COMPACT_NR) + sc->in[sc->nr++].sfl = sfl; + if (sc->nr == SCOUTFS_SRCH_COMPACT_NR) break; scoutfs_key_inc(&key); } if (type == SCOUTFS_SRCH_LOG_TYPE) - scin->flags = SCOUTFS_SRCH_COMPACT_FLAG_LOG; + sc->flags = SCOUTFS_SRCH_COMPACT_FLAG_LOG; + else + sc->flags = SCOUTFS_SRCH_COMPACT_FLAG_SORTED; /* record that our client has a compaction in process */ - scin->id = scin->sfl[0].ref.blkno; - init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(scin->id)); + sc->id = sc->in[0].sfl.ref.blkno; + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(sc->id)); ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, - scin, sizeof(*scin)); + sc, sizeof(*sc)); out: scoutfs_spbm_destroy(&busy); if (ret < 0) - scin->nr = 0; - if (scin->nr < SCOUTFS_SRCH_COMPACT_NR) - memset(&scin->sfl[scin->nr], 0, - (SCOUTFS_SRCH_COMPACT_NR - scin->nr) * - sizeof(scin->sfl[0])); + sc->nr = 0; + if (sc->nr < SCOUTFS_SRCH_COMPACT_NR) + memset(&sc->in[sc->nr], 0, + (SCOUTFS_SRCH_COMPACT_NR - sc->nr) * sizeof(sc->in[0])); return ret; } @@ -1150,88 +1236,129 @@ int scoutfs_srch_update_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_srch_compact_input *scin) + struct scoutfs_srch_compact *sc) { struct scoutfs_key key; - init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(scin->id)); + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(sc->id)); return scoutfs_btree_update(sb, alloc, wri, root, &key, - scin, sizeof(*scin)); + sc, sizeof(struct scoutfs_srch_compact)); } -static int mod_srch_items(struct super_block *sb, - struct scoutfs_alloc *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_btree_root *root, u8 scom_flags, - bool ins, struct scoutfs_srch_file *sfls, int nr) +static void init_file_key(struct scoutfs_key *key, int type, + struct scoutfs_srch_file *sfl) +{ + if (type == SCOUTFS_SRCH_LOG_TYPE) + init_srch_key(key, type, le64_to_cpu(sfl->ref.blkno), 0); + else + init_srch_key(key, type, le64_to_cpu(sfl->blocks), + le64_to_cpu(sfl->ref.blkno)); +} + +/* + * A compaction has completed so we remove the input file reference + * items and add the output file, if it has contents. If this returns + * an error then the file items were not changed. + */ +static int commit_files(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_btree_root *root, + struct scoutfs_srch_compact *sc) { struct scoutfs_srch_file *sfl; struct scoutfs_key key; - int ret = 0; int type; + int ret; + int err; int i; - if (nr <= 0) - return 0; - - if (scom_flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) + if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) type = SCOUTFS_SRCH_LOG_TYPE; else type = SCOUTFS_SRCH_BLOCKS_TYPE; - for (i = 0; i < nr; i++) { - sfl = &sfls[i]; - - /* don't bother inserting empty files */ - if (ins && sfl->entries == 0) - continue; - - if (type == SCOUTFS_SRCH_LOG_TYPE) - init_srch_key(&key, type, - le64_to_cpu(sfl->ref.blkno), 0); - else - init_srch_key(&key, type, - le64_to_cpu(sfl->blocks), - le64_to_cpu(sfl->ref.blkno)); - - if (ins) - ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, - sfl, sizeof(*sfl)); - else - ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (sc->out.blocks != 0) { + sfl = &sc->out; + init_file_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, sfl); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + sfl, sizeof(*sfl)); if (ret < 0) - break; + goto out; } + for (i = 0; i < sc->nr; i++) { + sfl = &sc->in[i].sfl; + init_file_key(&key, type, sfl); + + ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0) { + while (--i >= 0) { + sfl = &sc->in[i].sfl; + init_file_key(&key, type, sfl); + + err = scoutfs_btree_insert(sb, alloc, wri, + root, &key, + sfl, sizeof(*sfl)); + BUG_ON(err); /* lost srch file */ + } + + if (sc->out.blocks != 0) { + sfl = &sc->out; + init_file_key(&key, SCOUTFS_SRCH_BLOCKS_TYPE, + sfl); + err = scoutfs_btree_delete(sb, alloc, wri, + root, &key); + BUG_ON(err); /* duplicate srch files data */ + } + goto out; + } + } + + ret = 0; +out: return ret; } /* * Running in the server: commit the result of a compaction. Given the - * response id, find the input files in the compact's busy item. Remove - * the input files, add the new sorted file, and remove the busy item. - * We give the caller the allocator trees to merge if we return success. + * response id, find the compaction's busy item. The busy item is + * returned to a pending item or is advanced depending on the result. + * If the compaction completed then we replace the input files with the + * output files and transition the compaction to delete the input files. + * Once the input files are deleted we can remove the compaction item. */ int scoutfs_srch_commit_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_srch_compact_result *scres, + struct scoutfs_srch_compact *res, struct scoutfs_alloc_list_head *av, struct scoutfs_alloc_list_head *fr) { - struct scoutfs_srch_compact_input scin; + struct scoutfs_srch_compact *pending = NULL; + struct scoutfs_srch_compact *busy; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; + int err; + int i; + + /* only free allocators when we finish deleting */ + memset(av, 0, sizeof(struct scoutfs_alloc_list_head)); + memset(fr, 0, sizeof(struct scoutfs_alloc_list_head)); + + busy = kzalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (busy == NULL) { + ret = -ENOMEM; + goto out; + } /* find the record of our compaction */ - init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, - le64_to_cpu(scres->id)); + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(res->id)); ret = scoutfs_btree_lookup(sb, root, &key, &iref); if (ret == 0) { - if (iref.val_len == sizeof(scin)) - memcpy(&scin, iref.val, iref.val_len); + if (iref.val_len == sizeof(struct scoutfs_srch_compact)) + memcpy(busy, iref.val, iref.val_len); else ret = -EIO; scoutfs_btree_put_iref(&iref); @@ -1239,27 +1366,68 @@ int scoutfs_srch_commit_compact(struct super_block *sb, if (ret < 0) /* XXX leaks allocators */ goto out; - if (!(scres->flags & SCOUTFS_SRCH_COMPACT_FLAG_ERROR)) { - /* delete old items and insert new file items */ - ret = mod_srch_items(sb, alloc, wri, root, scin.flags, false, - scin.sfl, scin.nr) ?: - mod_srch_items(sb, alloc, wri, root, 0, true, - &scres->sfl, 1); - if (ret < 0) - goto out; - - *av = scres->meta_avail; - *fr = scres->meta_freed; - } else { - /* reclaim input allocators on error */ - *av = scin.meta_avail; - *fr = scin.meta_freed; + /* restore busy to pending if the operation failed */ + if (res->flags & SCOUTFS_SRCH_COMPACT_FLAG_ERROR) { + pending = busy; + ret = 0; + goto update; } - /* delete the record of our compaction */ + /* store result as pending if it isn't done */ + if (!(res->flags & SCOUTFS_SRCH_COMPACT_FLAG_DONE)) { + pending = res; + ret = 0; + goto update; + } + + /* update file references if we finished compaction (!deleting) */ + if (!(res->flags & SCOUTFS_SRCH_COMPACT_FLAG_DELETE)) { + ret = commit_files(sb, alloc, wri, root, res); + if (ret < 0) { + /* XXX we can't commit, shutdown? */ + goto out; + } + + /* transition flags for deleting input files */ + for (i = 0; i < res->nr; i++) { + res->in[i].blk = 0; + res->in[i].pos = 0; + } + res->flags &= ~(SCOUTFS_SRCH_COMPACT_FLAG_DONE | + SCOUTFS_SRCH_COMPACT_FLAG_LOG | + SCOUTFS_SRCH_COMPACT_FLAG_SORTED); + res->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DELETE; + pending = res; + ret = 0; + goto update; + } + + /* ok, finished deleting, reclaim allocs and delete busy */ + *av = res->meta_avail; + *fr = res->meta_freed; + pending = NULL; + ret = 0; +update: + if (pending) { + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(pending->id), 0); + ret = scoutfs_btree_insert(sb, alloc, wri, root, &key, + pending, sizeof(*pending)); + if (ret < 0) + goto out; + } + + init_srch_key(&key, SCOUTFS_SRCH_BUSY_TYPE, rid, le64_to_cpu(res->id)); ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); + if (ret < 0 && pending) { + init_srch_key(&key, SCOUTFS_SRCH_PENDING_TYPE, + le64_to_cpu(pending->id), 0); + err = scoutfs_btree_delete(sb, alloc, wri, root, &key); + BUG_ON(err); /* both busy and pending present */ + } out: WARN_ON_ONCE(ret < 0); /* XXX inconsistency */ + kfree(busy); return ret; } @@ -1274,7 +1442,7 @@ int scoutfs_srch_cancel_compact(struct super_block *sb, struct scoutfs_alloc_list_head *av, struct scoutfs_alloc_list_head *fr) { - struct scoutfs_srch_compact_input scin; + struct scoutfs_srch_compact *sc; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; struct scoutfs_key last; @@ -1287,25 +1455,36 @@ int scoutfs_srch_cancel_compact(struct super_block *sb, if (ret == 0) { if (scoutfs_key_compare(iref.key, &last) > 0) { ret = -ENOENT; - } else if (iref.val_len != sizeof(scin)) { + } else if (iref.val_len != sizeof(*sc)) { ret = -EIO; } else { key = *iref.key; - memcpy(&scin, iref.val, iref.val_len); + sc = iref.val; + *av = sc->meta_avail; + *fr = sc->meta_freed; } scoutfs_btree_put_iref(&iref); } if (ret < 0) goto out; - *av = scin.meta_avail; - *fr = scin.meta_freed; - ret = scoutfs_btree_delete(sb, alloc, wri, root, &key); out: return ret; } +/* + * We're done with an operation when we have sufficient dirty blocks or + * run out of avail or freed allocator space. + */ +static bool should_commit(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri) +{ + return (scoutfs_block_writer_dirty_bytes(sb, wri) >= + SRCH_COMPACT_DIRTY_LIMIT_BYTES) || + scoutfs_alloc_meta_lo_thresh(sb, alloc); +} + struct tourn_node { struct scoutfs_srch_entry sre; int ind; @@ -1345,6 +1524,7 @@ static int kway_merge(struct super_block *sb, struct tourn_node *tn; int nr_parents; int nr_nodes; + int empty = 0; int ret = 0; u64 blk; int ind; @@ -1370,29 +1550,29 @@ static int kway_merge(struct super_block *sb, tn = &leaves[i]; tn->ind = i; ret = kway_next(sb, &tn->sre, args[i]); - if (ret < 0) + if (ret == 0) { + tourn_update(tnodes, &leaves[i]); + } else if (ret == -ENOENT) { + memset(&tn->sre, 0xff, sizeof(tn->sre)); + empty++; + } else { goto out; + } } - /* prepare parents.. not optimal, but not a big deal either */ - for (i = 0; i < nr; i += 2) - tourn_update(tnodes, &leaves[i]); - - blk = 0; - while (nr > 0) { + /* always append new blocks */ + blk = le64_to_cpu(sfl->blocks); + while (empty < nr) { if (bl == NULL) { if (atomic_read(&srinf->shutdown)) { ret = -ESHUTDOWN; goto out; } - /* check dirty limit before each block creation */ - if (scoutfs_block_writer_dirty_bytes(sb, wri) >= - SRCH_COMPACT_DIRTY_LIMIT_BYTES) { - scoutfs_inc_counter(sb, srch_compact_flush); - ret = scoutfs_block_writer_write(sb, wri); - if (ret < 0) - goto out; + /* check for committing before dirtying blocks */ + if (should_commit(sb, alloc, wri)) { + ret = 0; + goto out; } ret = get_file_block(sb, alloc, wri, sfl, @@ -1446,7 +1626,7 @@ static int kway_merge(struct super_block *sb, if (ret == -ENOENT) { /* this index is done */ memset(&tn->sre, 0xff, sizeof(tn->sre)); - nr--; + empty++; ret = 0; } else if (ret < 0) { goto out; @@ -1524,37 +1704,43 @@ static void swap_page_sre(void *A, void *B, int size) * the input log files entries are encoded so we can allocate quite a * bit more memory in pages than the files took in blocks on disk (~2x * typically, ~10x worst case). + * + * Because we read and sort all the input files we must perform the full + * compaction in one operation. The server must have given us a + * sufficiently large avail/freed lists, otherwise we'll return ENOSPC. */ static int compact_logs(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, - struct scoutfs_srch_file *sfl_out, - struct scoutfs_srch_file *sfls, int nr_sfls) + struct scoutfs_srch_compact *sc) { DECLARE_SRCH_INFO(sb, srinf); - struct scoutfs_srch_file *sfl_end = sfls + nr_sfls; - struct scoutfs_srch_file *sfl = &sfls[0]; struct scoutfs_srch_block *srb = NULL; struct scoutfs_srch_entry *sre; struct scoutfs_srch_entry prev; struct scoutfs_block *bl = NULL; + struct scoutfs_srch_file *sfl; struct page *page = NULL; struct page *tmp; void **args = NULL; int nr_pages = 0; LIST_HEAD(pages); + int sfl_ind; u64 blk = 0; int pos = 0; int ret; int i; - if (WARN_ON_ONCE(nr_sfls <= 1)) - return -EINVAL; + if (sc->nr <= 1) { + ret = -EINVAL; + goto out; + } memset(&prev, 0, sizeof(prev)); /* decode all the log file's block's entries into pages */ - while (sfl < sfl_end) { + for (sfl_ind = 0, sfl = &sc->in[0].sfl; sfl_ind < sc->nr; ) { + if (bl == NULL) { /* only check on each new input block */ if (atomic_read(&srinf->shutdown)) { @@ -1604,7 +1790,8 @@ static int compact_logs(struct super_block *sb, pos = 0; if (++blk == le64_to_cpu(sfl->blocks)) { blk = 0; - sfl++; + sfl_ind++; + sfl = &sc->in[sfl_ind].sfl; } } @@ -1642,8 +1829,22 @@ static int compact_logs(struct super_block *sb, } - ret = kway_merge(sb, alloc, wri, sfl_out, kway_next_page, args, + ret = kway_merge(sb, alloc, wri, &sc->out, kway_next_page, args, nr_pages); + if (ret < 0) + goto out; + + /* make sure we finished all the pages */ + list_for_each_entry(page, &pages, list) { + sre = page_priv_sre(page); + if (page->private < SRES_PER_PAGE && sre->ino != 0) { + ret = -ENOSPC; + goto out; + } + } + + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + ret = 0; out: scoutfs_block_put(sb, bl); vfree(args); @@ -1660,6 +1861,7 @@ struct kway_file_reader { struct scoutfs_block *bl; struct scoutfs_srch_entry prev; u64 blk; + u32 skip; u32 pos; }; @@ -1670,7 +1872,7 @@ static int kway_next_file_reader(struct super_block *sb, struct scoutfs_srch_block *srb; int ret; - if (rdr->sfl == NULL) + if (rdr->blk == le64_to_cpu(rdr->sfl->blocks)) return -ENOENT; if (rdr->bl == NULL) { @@ -1678,30 +1880,37 @@ static int kway_next_file_reader(struct super_block *sb, &rdr->bl); if (ret < 0) goto out; + memset(&rdr->prev, 0, sizeof(rdr->prev)); - rdr->pos = 0; } srb = rdr->bl->data; - if (rdr->pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES) { + if (rdr->pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES || + rdr->skip > SCOUTFS_SRCH_BLOCK_SAFE_BYTES || + rdr->skip >= le32_to_cpu(srb->entry_bytes)) { /* XXX inconsistency */ return -EIO; } - ret = decode_entry(srb->entries + rdr->pos, sre_ret, &rdr->prev); - if (ret <= 0) { - /* XXX inconsistency */ - return -EIO; - } + /* decode entry, possibly skipping start of the block */ + do { + ret = decode_entry(srb->entries + rdr->pos, sre_ret, + &rdr->prev); + if (ret <= 0) { + /* XXX inconsistency */ + return -EIO; + } - rdr->prev = *sre_ret; - rdr->pos += ret; + rdr->prev = *sre_ret; + rdr->pos += ret; + } while (rdr->pos <= rdr->skip); + rdr->skip = 0; if (rdr->pos >= le32_to_cpu(srb->entry_bytes)) { + rdr->pos = 0; scoutfs_block_put(sb, rdr->bl); rdr->bl = NULL; - if (++rdr->blk == le64_to_cpu(rdr->sfl->blocks)) - rdr->sfl = NULL; + rdr->blk++; } ret = 0; @@ -1717,17 +1926,19 @@ out: static int compact_sorted(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, - struct scoutfs_srch_file *sfl_out, - struct scoutfs_srch_file *sfls, int nr) + struct scoutfs_srch_compact *sc) { struct kway_file_reader *rdrs = NULL; void **args = NULL; int ret; + int nr; int i; - if (WARN_ON_ONCE(nr <= 1)) + if (WARN_ON_ONCE(sc->nr <= 1)) return -EINVAL; + nr = sc->nr; + /* allocate args array for k-way merge */ rdrs = kmalloc_array(nr, sizeof(rdrs[0]), __GFP_ZERO | GFP_NOFS); args = kmalloc_array(nr, sizeof(args[0]), GFP_NOFS); @@ -1737,12 +1948,29 @@ static int compact_sorted(struct super_block *sb, } for (i = 0; i < nr; i++) { - rdrs[i].sfl = &sfls[i]; + if (le64_to_cpu(sc->in[i].blk) > + le64_to_cpu(sc->in[i].sfl.blocks)) { + ret = -EINVAL; + goto out; + } + + rdrs[i].sfl = &sc->in[i].sfl; + rdrs[i].blk = le64_to_cpu(sc->in[i].blk); + rdrs[i].skip = le64_to_cpu(sc->in[i].pos); args[i] = &rdrs[i]; } - ret = kway_merge(sb, alloc, wri, sfl_out, kway_next_file_reader, + ret = kway_merge(sb, alloc, wri, &sc->out, kway_next_file_reader, args, nr); + + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + for (i = 0; i < nr; i++) { + sc->in[i].blk = cpu_to_le64(rdrs[i].blk); + sc->in[i].pos = cpu_to_le64(rdrs[i].pos); + + if (rdrs[i].blk < le64_to_cpu(sc->in[i].sfl.blocks)) + sc->flags &= ~SCOUTFS_SRCH_COMPACT_FLAG_DONE; + } out: for (i = 0; rdrs && i < nr; i++) scoutfs_block_put(sb, rdrs[i].bl); @@ -1753,92 +1981,111 @@ out: } /* - * Perform a depth-first walk of the file's parent blocks, freeing all - * the blocks that were allocated to the file. This is working with a - * read-only file in the block cache that can also be currently read by - * searchers. If we return an error then the server is going to clean - * up our entire operation, partial state doesn't matter. + * Delete a file that has been compacted and is no longer referenced by + * items in the srch_root. The server protects the input file from + * other compactions while we're working, but other readers could be + * still trying to read it while searching. + * + * We don't modify the blocks to avoid the cost of allocating and + * freeing dirty parent metadata blocks, and we want to avoid triggering + * stale reads in racing readers. We free blocks from leaf parents + * upwards and from left to right. Once we've freed a block we never + * visit it again. We store our walk position in each file's compact + * input so that it can be stored in pending items as progress is made + * over multiple operations. */ -static int free_file(struct super_block *sb, - struct scoutfs_alloc *alloc, - struct scoutfs_block_writer *wri, - struct scoutfs_srch_file *sfl) +static int delete_file(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact_input *in) { - struct scoutfs_block **bls = NULL; + struct scoutfs_block *bl = NULL; struct scoutfs_srch_parent *srp; - struct scoutfs_srch_ref *ref; - unsigned int *inds = NULL; u64 blkno; - u8 height; + u64 blk; + u64 inc; int level; int ret; int i; - if (sfl->ref.blkno == 0) - return 0; + blk = le64_to_cpu(in->blk); + level = max(le64_to_cpu(in->pos), 1ULL); - height = height_for_blk(le64_to_cpu(sfl->blocks) - 1); - if (height == 1) - goto free_root; - - bls = kmalloc_array(height, sizeof(bls[0]), __GFP_ZERO | GFP_NOFS); - inds = kmalloc_array(height, sizeof(inds[0]), __GFP_ZERO | GFP_NOFS); - if (!bls || !inds) { - ret = -ENOMEM; + if (level > in->sfl.height) { + ret = 0; goto out; } - ref = &sfl->ref; - level = height - 1; - while (level < height) { - if (bls[level] == NULL) { - ret = read_srch_block(sb, wri, level, ref, &bls[level]); + for (; level < in->sfl.height; level++) { + + for (inc = 1, i = 2; i <= level; i++) + inc *= SCOUTFS_SRCH_PARENT_REFS; + + while (blk < le64_to_cpu(in->sfl.blocks)) { + + ret = read_path_block(sb, wri, &in->sfl, blk, level, + &bl); if (ret < 0) goto out; - } - srp = bls[level]->data; + srp = bl->data; - /* find a parent to descend to, remembering where we were */ - ref = NULL; - for (i = inds[level]; level >= 2 && - i < SCOUTFS_SRCH_PARENT_REFS; i++) { - if (srp->refs[i].blkno) { - inds[level] = i + 1; - ref = &srp->refs[i]; - level--; - break; + for (i = calc_ref_ind(blk, level); + i < SCOUTFS_SRCH_PARENT_REFS && + blk < le64_to_cpu(in->sfl.blocks); + i++, blk += inc) { + + blkno = le64_to_cpu(srp->refs[i].blkno); + if (!blkno) + continue; + + if (should_commit(sb, alloc, wri)) { + ret = 0; + goto out; + } + + ret = scoutfs_free_meta(sb, alloc, wri, blkno); + if (ret < 0) + goto out; } + + scoutfs_block_put(sb, bl); + bl = NULL; } - if (ref) - continue; + blk = 0; + } - /* free all our referenced blocks */ - for (i = 0; i < SCOUTFS_SRCH_PARENT_REFS; i++) { - blkno = le64_to_cpu(srp->refs[i].blkno); - if (blkno == 0) - continue; - - ret = scoutfs_free_meta(sb, alloc, wri, blkno); - if (ret < 0) - goto out; - scoutfs_inc_counter(sb, srch_compact_free_block); - } - - scoutfs_block_put(sb, bls[level]); - bls[level] = NULL; + if (level == in->sfl.height) { + ret = scoutfs_free_meta(sb, alloc, wri, + le64_to_cpu(in->sfl.ref.blkno)); + if (ret < 0) + goto out; level++; } -free_root: - ret = scoutfs_free_meta(sb, alloc, wri, le64_to_cpu(sfl->ref.blkno)); - if (ret < 0) - goto out; - + ret = 0; out: - for (i = 0; bls && i < height; i++) - scoutfs_block_put(sb, bls[i]); - kfree(bls); - kfree(inds); + in->blk = cpu_to_le64(blk); + in->pos = cpu_to_le64(level); + + scoutfs_block_put(sb, bl); + return ret; +} + +static int delete_files(struct super_block *sb, struct scoutfs_alloc *alloc, + struct scoutfs_block_writer *wri, + struct scoutfs_srch_compact *sc) +{ + int ret; + int i; + + for (i = 0; i < sc->nr; i++) { + ret = delete_file(sb, alloc, wri, &sc->in[i]); + if (ret < 0 || + (le64_to_cpu(sc->in[i].pos) <= sc->in[i].sfl.height)) + break; + } + if (i == sc->nr) + sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; + return ret; } @@ -1867,47 +2114,50 @@ static void scoutfs_srch_compact_worker(struct work_struct *work) { struct srch_info *srinf = container_of(work, struct srch_info, compact_dwork.work); + struct scoutfs_srch_compact *sc = NULL; struct super_block *sb = srinf->sb; - struct scoutfs_alloc alloc; - struct scoutfs_srch_compact_result scres; - struct scoutfs_srch_compact_input scin; struct scoutfs_block_writer wri; + struct scoutfs_alloc alloc; unsigned long delay; int ret; - int i; + + sc = kmalloc(sizeof(struct scoutfs_srch_compact), GFP_NOFS); + if (sc == NULL) { + ret = -ENOMEM; + goto out; + } scoutfs_block_writer_init(sb, &wri); - memset(&scres, 0, sizeof(scres)); - ret = scoutfs_client_srch_get_compact(sb, &scin); - if (ret < 0 || scin.nr == 0) + ret = scoutfs_client_srch_get_compact(sb, sc); + if (ret < 0 || sc->nr == 0) goto out; - scoutfs_alloc_init(&alloc, &scin.meta_avail, &scin.meta_freed); + scoutfs_alloc_init(&alloc, &sc->meta_avail, &sc->meta_freed); - if (scin.flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) - ret = compact_logs(sb, &alloc, &wri, &scres.sfl, - scin.sfl, scin.nr); - else - ret = compact_sorted(sb, &alloc, &wri, &scres.sfl, - scin.sfl, scin.nr); + if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_LOG) { + ret = compact_logs(sb, &alloc, &wri, sc); + + } else if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_SORTED) { + ret = compact_sorted(sb, &alloc, &wri, sc); + + } else if (sc->flags & SCOUTFS_SRCH_COMPACT_FLAG_DELETE) { + ret = delete_files(sb, &alloc, &wri, sc); + + } else { + ret = -EINVAL; + } if (ret < 0) goto commit; - for (i = 0; i < scin.nr; i++) { - ret = free_file(sb, &alloc, &wri, &scin.sfl[i]); - if (ret < 0) - goto commit; - } - ret = scoutfs_block_writer_write(sb, &wri); commit: - scres.meta_avail = alloc.avail; - scres.meta_freed = alloc.freed; - scres.id = scin.id; - scres.flags = ret < 0 ? SCOUTFS_SRCH_COMPACT_FLAG_ERROR : 0; + /* the server won't use our partial compact if _ERROR is set */ + sc->meta_avail = alloc.avail; + sc->meta_freed = alloc.freed; + sc->flags |= ret < 0 ? SCOUTFS_SRCH_COMPACT_FLAG_ERROR : 0; - ret = scoutfs_client_srch_commit_compact(sb, &scres); + ret = scoutfs_client_srch_commit_compact(sb, sc); out: /* our allocators and files should be stable */ WARN_ON_ONCE(ret == -ESTALE); @@ -1917,6 +2167,8 @@ out: delay = ret == 0 ? 0 : msecs_to_jiffies(SRCH_COMPACT_DELAY_MS); queue_delayed_work(srinf->workq, &srinf->compact_dwork, delay); } + + kfree(sc); } void scoutfs_srch_destroy(struct super_block *sb) diff --git a/kmod/src/srch.h b/kmod/src/srch.h index 97604bd6..69448ab3 100644 --- a/kmod/src/srch.h +++ b/kmod/src/srch.h @@ -42,18 +42,17 @@ int scoutfs_srch_get_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, - u64 rid, - struct scoutfs_srch_compact_input *scin_ret); + u64 rid, struct scoutfs_srch_compact *sc); int scoutfs_srch_update_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_srch_compact_input *scin); + struct scoutfs_srch_compact *sc); int scoutfs_srch_commit_compact(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_btree_root *root, u64 rid, - struct scoutfs_srch_compact_result *scres, + struct scoutfs_srch_compact *res, struct scoutfs_alloc_list_head *av, struct scoutfs_alloc_list_head *fr); int scoutfs_srch_cancel_compact(struct super_block *sb, From b094b186180e480557d5f6b07b168ccd5e8d97f2 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 16 Oct 2020 15:20:49 -0700 Subject: [PATCH 888/920] scoutfs: compact fewer srch files each time With the introduction of incremental srch file compaction we added some fields to the srch_compact struct to record the position of compaction in each file. This increased the size of the struct past the limit the btree places on the size of item values. We decrease the number of files per compaction from 8 to 4 to cut the size of the srch_compcat struct in half. This compacts twice as often, but still relatively infrequently, and it uses half the space for srch files waiting to hit the compaction threshold. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 6dafb382..09092b2a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -371,7 +371,7 @@ struct scoutfs_srch_block { SCOUTFS_SRCH_ENTRY_MAX_BYTES) #define SCOUTFS_SRCH_LOG_BLOCK_LIMIT (1024 * 1024 / SCOUTFS_BLOCK_LG_SIZE) -#define SCOUTFS_SRCH_COMPACT_ORDER 3 +#define SCOUTFS_SRCH_COMPACT_ORDER 2 #define SCOUTFS_SRCH_COMPACT_NR (1 << SCOUTFS_SRCH_COMPACT_ORDER) /* From a848477e644c4a8c34bcce576a8468461ee1eaa1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 21 Oct 2020 11:01:41 -0700 Subject: [PATCH 889/920] scoutfs: remove unused packed exents We use full data extent items now, we don't need the packed extent structures. Signed-off-by: Zach Brown --- kmod/src/format.h | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 09092b2a..800d9fab 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -515,37 +515,6 @@ struct scoutfs_bloom_block { #define SCOUTFS_FREE_EXTENT_BLKNO_TYPE 1 #define SCOUTFS_FREE_EXTENT_LEN_TYPE 2 -/* - * The extents that map blocks in a fixed-size logical region of a file - * are packed and stored in item values. The packed extents are - * contiguous so the starting logical block is implicit from the length - * of previous extents. Sparse regions are represented by 0 flags and - * blkno. The blkno of a packed extent is encoded as the zigzag (lsb is - * sign bit) difference from the last blkno of the previous extent. - * This guarantees that non-sparse extents must have a blkno delta of at - * least -1/1. High zero byte aren't stored. - */ -struct scoutfs_packed_extent { - __le16 count; -#if defined(__LITTLE_ENDIAN_BITFIELD) - __u8 diff_bytes:4, - flags:3, - final:1; -#elif defined(__BIG_ENDIAN_BITFIELD) - __u8 final:1, - flags:3, - diff_bytes:4; -#else -#error "no {BIG,LITTLE}_ENDIAN_BITFIELD defined?" -#endif - __u8 le_blkno_diff[0]; -} __packed; - -#define SCOUTFS_PACKEXT_BLOCKS (8 * 1024 * 1024 / SCOUTFS_BLOCK_SM_SIZE) -#define SCOUTFS_PACKEXT_BASE_SHIFT (ilog2(SCOUTFS_PACKEXT_BLOCKS)) -#define SCOUTFS_PACKEXT_BASE_MASK (~((__u64)SCOUTFS_PACKEXT_BLOCKS - 1)) -#define SCOUTFS_PACKEXT_MAX_BYTES SCOUTFS_MAX_VAL_SIZE - /* file data extents have start and len in key */ struct scoutfs_data_extent_val { __le64 blkno; From 735c2c6905ccb4ea8119d44ad4d1f33431c93e21 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 23 Oct 2020 15:51:39 -0700 Subject: [PATCH 890/920] scoutfs: fix btree split/join setting parent keys Before the introduction of the AVL tree to sort btree items, the items were sorted by sorting a small packed array of offsets. The final offset in that array pointed to the item in the block with the greatest key. With the move to sorting items in an AVL tree by nodes embedded in item structs, we now don't have the array of offsets and instead have a dense array of items. Creation and deletion of items always works with the final item in the array. last_item() used to return the item with the greatest key by returning the item pointed to by the final entry in the sorted offset array, then it returned the final entry in the item array for creation and deletion but that was no longer the item with the greatest key. But spliting and joining still used last_item() to find the item in the block with the greatest key for updating references to blocks in parents. Since the introduction of the AVL tree splitting and joining has been corrrupting the tree by setting parent block reference keys to whatever item happened to be at the end of the array, not the item with the greatest key. The extent code recently pushed hard enough to hit this by working with relatively random extent items in the core allocation btrees. Eventually the parent block reference keys got out of sync and we'd fail to find items by descending into the wrong children when looking for them. Extent deletion hit this during allocation, returned -ENOENT, and the allocator turned that into -ENOSPC. With this fixed we can repetedly create and delte millions of files with heavily fragmented extents in a tiny metadata device. Eventually it actually runs out of space instead of spuriously returning ENOSPC in a matter of minutes. Signed-off-by: Zach Brown --- kmod/src/btree.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 1d249a3a..d395216e 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -140,7 +140,12 @@ off_item(struct scoutfs_btree_block *bt, __le16 off) return (void *)bt + le16_to_cpu(off); } -static struct scoutfs_btree_item *last_item(struct scoutfs_btree_block *bt) + +/* + * The item at the end of the item array. This is *not* the item in the + * block with the greatest key. + */ +static struct scoutfs_btree_item *end_item(struct scoutfs_btree_block *bt) { BUG_ON(bt->nr_items == 0); @@ -183,6 +188,11 @@ static struct scoutfs_btree_item *node_item(struct scoutfs_avl_node *node) return container_of(node, struct scoutfs_btree_item, node); } +static struct scoutfs_btree_item *last_item(struct scoutfs_btree_block *bt) +{ + return node_item(scoutfs_avl_last(&bt->item_root)); +} + static struct scoutfs_btree_item *prev_item(struct scoutfs_btree_block *bt, struct scoutfs_btree_item *item) { @@ -543,7 +553,7 @@ static void create_item(struct scoutfs_btree_block *bt, le16_add_cpu(&bt->mid_free_len, -(u16)sizeof(struct scoutfs_btree_item)); le16_add_cpu(&bt->nr_items, 1); - item = last_item(bt); + item = end_item(bt); item->key = *key; @@ -568,14 +578,14 @@ static void delete_item(struct scoutfs_btree_block *bt, struct scoutfs_btree_item *item, struct scoutfs_btree_item **use_after) { - struct scoutfs_btree_item *last; + struct scoutfs_btree_item *end; unsigned int val_off; unsigned int val_len; /* save some values before we delete the item */ val_off = le16_to_cpu(item->val_off); val_len = le16_to_cpu(item->val_len); - last = last_item(bt); + end = end_item(bt); /* delete the item */ scoutfs_avl_delete(&bt->item_root, &item->node); @@ -585,18 +595,18 @@ static void delete_item(struct scoutfs_btree_block *bt, le16_add_cpu(&bt->total_item_bytes, -item_bytes(item)); /* move the final item into the deleted space */ - if (last != item) { - item->key = last->key; - item->val_off = last->val_off; - item->val_len = last->val_len; - if (last->val_len) - set_val_owner(bt, le16_to_cpu(last->val_off), - val_bytes(le16_to_cpu(last->val_len)), + if (end != item) { + item->key = end->key; + item->val_off = end->val_off; + item->val_len = end->val_len; + if (end->val_len) + set_val_owner(bt, le16_to_cpu(end->val_off), + val_bytes(le16_to_cpu(end->val_len)), ptr_off(bt, item)); - leaf_item_hash_change(bt, &last->key, ptr_off(bt, item), - ptr_off(bt, last)); - scoutfs_avl_relocate(&bt->item_root, &item->node,&last->node); - if (use_after && *use_after == last) + leaf_item_hash_change(bt, &end->key, ptr_off(bt, item), + ptr_off(bt, end)); + scoutfs_avl_relocate(&bt->item_root, &item->node,&end->node); + if (use_after && *use_after == end) *use_after = item; } From 2e7053497efc8ad46edb928b6cf193c2f9886801 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 26 Oct 2020 10:49:59 -0700 Subject: [PATCH 891/920] scoutfs: remove free_*_blocks super fields Remove the old superblock fields which were used to track free blocks found in the radix allocators. We now walk all the allocators when we need to know the free totals, rather than trying to keep fields in sync. Signed-off-by: Zach Brown --- kmod/src/format.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index 800d9fab..34ebed1a 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -595,11 +595,9 @@ struct scoutfs_super_block { __le64 total_meta_blocks; /* both static and dynamic */ __le64 first_meta_blkno; /* first dynamically allocated */ __le64 last_meta_blkno; - __le64 free_meta_blocks; __le64 total_data_blocks; __le64 first_data_blkno; __le64 last_data_blkno; - __le64 free_data_blocks; __le64 quorum_fenced_term; __le64 quorum_server_term; __le64 unmount_barrier; From dbea353b920b4ba2c7adb9d1f6c809c24b35b5ae Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 13 Oct 2020 11:34:52 -0700 Subject: [PATCH 892/920] scoutfs: bring back sort_priv Bring back sort_priv, we have need for sorting with a caller argument. Signed-off-by: Zach Brown --- kmod/src/Makefile | 1 + kmod/src/sort_priv.c | 71 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/sort_priv.h | 8 +++++ 3 files changed, 80 insertions(+) create mode 100644 kmod/src/sort_priv.c create mode 100644 kmod/src/sort_priv.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 5af7fdd0..5bbee931 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -33,6 +33,7 @@ scoutfs-y += \ quorum.o \ scoutfs_trace.o \ server.o \ + sort_priv.o \ spbm.o \ srch.o \ super.o \ diff --git a/kmod/src/sort_priv.c b/kmod/src/sort_priv.c new file mode 100644 index 00000000..2acc0802 --- /dev/null +++ b/kmod/src/sort_priv.c @@ -0,0 +1,71 @@ +/* + * A copy of sort() from upstream with a priv argument that's passed + * to comparison, like list_sort(). + */ + +/* ------------------------ */ + +/* + * A fast, small, non-recursive O(nlog n) sort for the Linux kernel + * + * Jan 23 2005 Matt Mackall + */ + +#include +#include +#include +#include +#include "sort_priv.h" + +/** + * sort - sort an array of elements + * @priv: caller's pointer to pass to comparison and swap functions + * @base: pointer to data to sort + * @num: number of elements + * @size: size of each element + * @cmp_func: pointer to comparison function + * @swap_func: pointer to swap function or NULL + * + * This function does a heapsort on the given array. You may provide a + * swap_func function optimized to your element type. + * + * Sorting time is O(n log n) both on average and worst-case. While + * qsort is about 20% faster on average, it suffers from exploitable + * O(n*n) worst-case behavior and extra memory requirements that make + * it less suitable for kernel use. + */ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)) +{ + /* pre-scale counters for performance */ + int i = (num/2 - 1) * size, n = num * size, c, r; + + /* heapify */ + for ( ; i >= 0; i -= size) { + for (r = i; r * 2 + size < n; r = c) { + c = r * 2 + size; + if (c < n - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } + + /* sort */ + for (i = n - size; i > 0; i -= size) { + swap_func(priv, base, base + i, size); + for (r = 0; r * 2 + size < i; r = c) { + c = r * 2 + size; + if (c < i - size && + cmp_func(priv, base + c, base + c + size) < 0) + c += size; + if (cmp_func(priv, base + r, base + c) >= 0) + break; + swap_func(priv, base + r, base + c, size); + } + } +} diff --git a/kmod/src/sort_priv.h b/kmod/src/sort_priv.h new file mode 100644 index 00000000..c5fde547 --- /dev/null +++ b/kmod/src/sort_priv.h @@ -0,0 +1,8 @@ +#ifndef _SCOUTFS_SORT_PRIV_H_ +#define _SCOUTFS_SORT_PRIV_H_ + +void sort_priv(void *priv, void *base, size_t num, size_t size, + int (*cmp_func)(void *priv, const void *, const void *), + void (*swap_func)(void *priv, void *, void *, int size)); + +#endif From dc47ec65e4a2db94bfb9c750b223a5a29fe223e6 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 13 Oct 2020 11:41:24 -0700 Subject: [PATCH 893/920] scoutfs: remove btree value owner footer offset We were using a trailing owner offset to iterate over btree item values from the back of the block towards the front. We did this to reclaim fragmented free space in a block to satisfy an allocation instead of having to split the block, which is expensive mostly because it has to allocate and free metadata blocks. In the before times, we used to compact items by sorting items by their offset, moving them, and then sorting them by their keys again. The sorting by keys was expensive so we added these owner offsets to be able to compact without sorting. But the complexity of maintaining the owner metadata is not worth it. We can avoid the expensive sorting by keys by allocating a temporary array of item offsets and sorting only it by the value offset. That's nice and quick, it was the key comparisons that were expensive. Then we can remove the owner offset entirely, as well as the block header final free region that compaction needed. And we also don't compact as often in the modern era because we do the bulk of our work in the item cache instead of in the btree, and we've changed the split/merge/compaction heuristics to avoid constantly splitting/merging/comapcting and an item population happens to hover right around a shared threshold. Signed-off-by: Zach Brown --- kmod/src/btree.c | 252 +++++++++++++++++--------------------------- kmod/src/counters.h | 1 + kmod/src/format.h | 5 - 3 files changed, 96 insertions(+), 162 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index d395216e..28644c98 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -29,6 +29,7 @@ #include "alloc.h" #include "avl.h" #include "hash.h" +#include "sort_priv.h" #include "scoutfs_trace.h" @@ -53,12 +54,11 @@ * * Values are allocated from the end of the block towards the front, * consuming the end of free space in the center of the block. Deleted - * values can be merged with this free space, but more likely they'll - * create fragmented free space amongst other existing values. All - * values are stored with an offset at the end which contains either the - * offset of their item or the offset of the start of their free space. - * This lets an infrequent compaction process move items towards the - * back of the block to reclaim free space. + * values create fragmented free space in other existing values. Rather + * than tracking free space specifically, we compact values in bulk to + * defragment free space if there is enough of to be worth the cost of + * compaction. When there's only a little bit of fragmented free space + * we split the block as usual. * * Exact item searches are only performed on leaf blocks. Leaf blocks * have a hash table at the end of the block which is used to find items @@ -88,7 +88,7 @@ enum { /* total length of the value payload */ static inline unsigned int val_bytes(unsigned val_len) { - return val_len + (val_len ? SCOUTFS_BTREE_VAL_OWNER_BYTES : 0); + return val_len; } /* number of bytes in a block used by an item with the given value length */ @@ -361,98 +361,100 @@ static void leaf_item_hash_change(struct scoutfs_btree_block *bt, } } -/* - * Given an offset to the start of a value, return info describing the - * previous value in the block. Each value ends with an owner offset - * which points to either the value's item if it's in use or to the - * start of the value if it's been freed. Either the item is returned - * or the length of the previous value is set. - */ -static struct scoutfs_btree_item * -get_prev_val_owner(struct scoutfs_btree_block *bt, unsigned int off, - unsigned int *prev_val_bytes) +static int cmp_sorted(void *priv, const void *A, const void *B) { - __le16 *owner = off_ptr(bt, off - sizeof(*owner)); - unsigned int own = get_unaligned_le16(owner); + struct scoutfs_btree_block *bt = priv; + const unsigned short *a = A; + const unsigned short *b = B; + struct scoutfs_btree_item *item_a = &bt->items[*a]; + struct scoutfs_btree_item *item_b = &bt->items[*b]; - if (own >= mid_free_off(bt)) { - *prev_val_bytes = off - own; - return NULL; - } else { - *prev_val_bytes = 0; - return off_ptr(bt, own); - } + return scoutfs_cmp(le16_to_cpu(item_a->val_off), + le16_to_cpu(item_b->val_off)); } -/* - * Set the owner offset at the end of a full value, the given length includes - * the offset. - */ -static void set_val_owner(struct scoutfs_btree_block *bt, unsigned int val_off, - unsigned int vb, __le16 item_off) +static void swap_sorted(void *priv, void *A, void *B, int size) { - __le16 *owner = off_ptr(bt, val_off + vb - sizeof(*owner)); + unsigned short *a = A; + unsigned short *b = B; - put_unaligned_le16(le16_to_cpu(item_off) ?: val_off, owner); + swap(*a, *b); } /* * As values are freed they can leave fragmented free space amongst - * other values. This is called when we can't insert because there - * isn't enough free space but we know that there's sufficient free - * space amongst the values for the new insertion. + * other values. We compact the values by sorting an array of item + * indices by the offset of the item's values. We can then walk values + * from the back of the block and pack them into contiguous space, + * bubbling any fragmented free space towards the middle. * - * But we only want to do this when there is enough free space to - * justify the cost of the compaction. We don't want to bother - * compacting if the block is almost full and we just be split in a few - * more operations. The split heuristic requires a generous amount of + * This is called when we can't insert because there isn't enough + * available free space in the middle of the block but we know that + * there's sufficient free fragmented space in the values. + * + * We only want to compact when there is enough free space to justify + * the cost of the compaction. We don't want to bother compacting if + * the block is almost full and we just be split in a few more + * operations. The split heuristic requires a generous amount of * fragmented free space that will avoid a split. */ -static void compact_values(struct super_block *sb, - struct scoutfs_btree_block *bt) +static int compact_values(struct super_block *sb, + struct scoutfs_btree_block *bt) { + const int nr = le16_to_cpu(bt->nr_items); struct scoutfs_btree_item *item; - unsigned int free_off; - unsigned int free_len; + unsigned short *sorted = NULL; unsigned int to_off; - unsigned int end; unsigned int vb; void *from; void *to; + int i; scoutfs_inc_counter(sb, btree_compact_values); - if (bt->last_free_off == 0) - return; + BUILD_BUG_ON(sizeof(sorted[0]) != sizeof(bt->nr_items)); - free_off = le16_to_cpu(bt->last_free_off); - free_len = le16_to_cpu(bt->last_free_len); - end = mid_free_off(bt) + le16_to_cpu(bt->mid_free_len); - - while (free_off > end) { - item = get_prev_val_owner(bt, free_off, &vb); - if (item == NULL) { - free_off -= vb; - free_len += vb; - continue; - } - - from = off_ptr(bt, le16_to_cpu(item->val_off)); - vb = val_bytes(le16_to_cpu(item->val_len)); - to_off = free_off + free_len - vb; - to = off_ptr(bt, to_off); - if (to >= from + vb) - memcpy(to, from, vb); - else - memmove(to, from, vb); - - free_off = le16_to_cpu(item->val_off); - item->val_off = cpu_to_le16(to_off); + sorted = kmalloc_array(le16_to_cpu(bt->nr_items), sizeof(sorted[0]), + GFP_NOFS); + if (!sorted) { + scoutfs_inc_counter(sb, btree_compact_values_enomem); + return -ENOMEM; } - le16_add_cpu(&bt->mid_free_len, free_len); - bt->last_free_off = 0; - bt->last_free_len = 0; + /* sort the sorted array of item indices by their value offset */ + for (i = 0; i < nr; i++) + sorted[i] = i; + sort_priv(bt, sorted, nr, sizeof(sorted[0]), cmp_sorted, swap_sorted); + + to_off = SCOUTFS_BLOCK_LG_SIZE; + if (bt->level == 0) + to_off -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + + /* move values towards the back of the block */ + for (i = nr - 1; i >= 0; i--) { + item = &bt->items[sorted[i]]; + if (item->val_len == 0) + continue; + + vb = val_bytes(le16_to_cpu(item->val_len)); + to_off -= vb; + from = off_ptr(bt, le16_to_cpu(item->val_off)); + to = off_ptr(bt, to_off); + + if (from != to) { + if (to >= from + vb) + memcpy(to, from, vb); + else + memmove(to, from, vb); + + item->val_off = cpu_to_le16(to_off); + } + } + + bt->mid_free_len = cpu_to_le16(to_off - mid_free_off(bt)); + + kfree(sorted); + return 0; } /* @@ -477,62 +479,10 @@ static __le16 insert_value(struct scoutfs_btree_block *bt, __le16 item_off, le16_add_cpu(&bt->mid_free_len, -vb); memcpy(off_ptr(bt, val_off), val, val_len); - set_val_owner(bt, val_off, vb, item_off); return cpu_to_le16(val_off); } -/* - * Delete an item's value from the block. The caller has updated the - * item. We leave behind a free region whose owner offset indicates - * that the value isn't in use. It might merge with the central free - * region or the final freed value, and might become the final freed - * value. - */ -static void delete_value(struct scoutfs_btree_block *bt, - unsigned int val_off, unsigned int val_len) -{ - unsigned int free_off; - unsigned int free_len; - bool is_last; - - if (val_len == 0) - return; - - free_off = val_off; - free_len = val_bytes(val_len); - is_last = false; - - /* see if we can merge with mid free region */ - if (mid_free_off(bt) + le16_to_cpu(bt->mid_free_len) == free_off) { - le16_add_cpu(&bt->mid_free_len, free_len); - return; - } - - if (free_off + free_len == le16_to_cpu(bt->last_free_off)) { - /* merge with front of last free */ - free_len += le16_to_cpu(bt->last_free_len); - is_last = true; - - } else if ((le16_to_cpu(bt->last_free_off) + - le16_to_cpu(bt->last_free_len)) == free_off) { - /* merge with end of last free */ - free_off = le16_to_cpu(bt->last_free_off); - free_len += le16_to_cpu(bt->last_free_len); - is_last = true; - - } else if (free_off > le16_to_cpu(bt->last_free_off)) { - /* become new last */ - is_last = true; - } - - set_val_owner(bt, free_off, free_len, 0); - if (is_last) { - bt->last_free_off = cpu_to_le16(free_off); - bt->last_free_len = cpu_to_le16(free_len); - } -} - /* * Insert a new item into the block. The caller has made sure that * there is sufficient free space in block for the new item. We might @@ -599,18 +549,12 @@ static void delete_item(struct scoutfs_btree_block *bt, item->key = end->key; item->val_off = end->val_off; item->val_len = end->val_len; - if (end->val_len) - set_val_owner(bt, le16_to_cpu(end->val_off), - val_bytes(le16_to_cpu(end->val_len)), - ptr_off(bt, item)); leaf_item_hash_change(bt, &end->key, ptr_off(bt, item), ptr_off(bt, end)); scoutfs_avl_relocate(&bt->item_root, &item->node,&end->node); if (use_after && *use_after == end) *use_after = item; } - - delete_value(bt, val_off, val_len); } /* @@ -847,7 +791,7 @@ static void update_parent_item(struct scoutfs_btree_block *parent, ref->seq = child->hdr.seq; } -static void init_btree_block(struct scoutfs_btree_block *bt, int level) +static __le16 init_mid_free_len(int level) { int free; @@ -855,8 +799,14 @@ static void init_btree_block(struct scoutfs_btree_block *bt, int level) if (level == 0) free -= SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES; + return cpu_to_le16(free); +} + +static void init_btree_block(struct scoutfs_btree_block *bt, int level) +{ + bt->level = level; - bt->mid_free_len = cpu_to_le16(free); + bt->mid_free_len = init_mid_free_len(level); } /* @@ -893,10 +843,8 @@ static int try_split(struct super_block *sb, if (mid_free_item_room(right, val_len)) return 0; - if (item_full_pct(right) < 80) { - compact_values(sb, right); - return 0; - } + if (item_full_pct(right) < 80) + return compact_values(sb, right); scoutfs_inc_counter(sb, btree_split); @@ -989,8 +937,12 @@ static int try_join(struct super_block *sb, else to_move = sib_tot - join_low_watermark(); - if (le16_to_cpu(bt->mid_free_len) < to_move) - compact_values(sb, bt); + if (le16_to_cpu(bt->mid_free_len) < to_move) { + ret = compact_values(sb, bt); + if (ret < 0) + scoutfs_block_put(sb, sib_bl); + return ret; + } move_items(bt, sib, move_right, to_move); /* update our parent's item */ @@ -1062,7 +1014,6 @@ static void verify_btree_block(struct super_block *sb, char *reason = NULL; int first_val = 0; int hashed = 0; - __le16 *owner; int end_off; int tot = 0; int i = 0; @@ -1120,8 +1071,7 @@ static void verify_btree_block(struct super_block *sb, } if (((int)le16_to_cpu(item->val_off) + - le16_to_cpu(item->val_len) + - SCOUTFS_BTREE_VAL_OWNER_BYTES) > end_off) { + le16_to_cpu(item->val_len)) > end_off) { reason = "item value outside valid"; goto out; } @@ -1130,15 +1080,6 @@ static void verify_btree_block(struct super_block *sb, le16_to_cpu(item->val_len); if (item->val_len != 0) { - owner = off_ptr(bt, le16_to_cpu(item->val_off) + - le16_to_cpu(item->val_len)); - if (get_unaligned_le16(owner) != - offsetof(struct scoutfs_btree_block, items[i])) { - reason = "item value owner not item off"; - goto out; - } - - tot += SCOUTFS_BTREE_VAL_OWNER_BYTES; first_val = min_t(int, first_val, le16_to_cpu(item->val_off)); } @@ -1186,10 +1127,9 @@ out: le64_to_cpu(bt->hdr.fsid), le64_to_cpu(bt->hdr.seq), le64_to_cpu(bt->hdr.blkno)); printk("item_root: node %u\n", le16_to_cpu(bt->item_root.node)); - printk("nr %u tib %u mfl %u lfo %u lfl %u lvl %u\n", + printk("nr %u tib %u mfl %u lvl %u\n", le16_to_cpu(bt->nr_items), le16_to_cpu(bt->total_item_bytes), - le16_to_cpu(bt->mid_free_len), le16_to_cpu(bt->last_free_off), - le16_to_cpu(bt->last_free_len), bt->level); + le16_to_cpu(bt->mid_free_len), bt->level); for (i = 0; i < le16_to_cpu(bt->nr_items); i++) { item = &bt->items[i]; @@ -1524,8 +1464,6 @@ static void update_item_value(struct scoutfs_btree_block *bt, { le16_add_cpu(&bt->total_item_bytes, val_bytes(val_len) - val_bytes(le16_to_cpu(item->val_len))); - delete_value(bt, le16_to_cpu(item->val_off), - le16_to_cpu(item->val_len)); item->val_off = insert_value(bt, ptr_off(bt, item), val, val_len); item->val_len = cpu_to_le16(val_len); } diff --git a/kmod/src/counters.h b/kmod/src/counters.h index 93515df0..6c470e58 100644 --- a/kmod/src/counters.h +++ b/kmod/src/counters.h @@ -32,6 +32,7 @@ EXPAND_COUNTER(block_cache_lru_move) \ EXPAND_COUNTER(block_cache_shrink) \ EXPAND_COUNTER(btree_compact_values) \ + EXPAND_COUNTER(btree_compact_values_enomem) \ EXPAND_COUNTER(btree_delete) \ EXPAND_COUNTER(btree_dirty) \ EXPAND_COUNTER(btree_force) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 34ebed1a..0ccf78ca 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -196,9 +196,6 @@ struct scoutfs_avl_node { /* when we split we want to have multiple items on each side */ #define SCOUTFS_BTREE_MAX_VAL_LEN 896 -/* each value ends with an offset which lets compaction iterate over values */ -#define SCOUTFS_BTREE_VAL_OWNER_BYTES sizeof(__le16) - /* * A 4EB test image measured a worst case height of 17. This is plenty * generous. @@ -232,8 +229,6 @@ struct scoutfs_btree_block { __le16 nr_items; __le16 total_item_bytes; __le16 mid_free_len; - __le16 last_free_off; - __le16 last_free_len; __u8 level; struct scoutfs_btree_item items[0]; /* leaf blocks have a fixed size item offset hash table at the end */ From 87cb971630b10f869b14988ff8dbbedc9c401f7f Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 8 Oct 2020 10:42:01 -0700 Subject: [PATCH 894/920] scoutfs: fix hash compiler warnings Signed-off-by: Andy Grover --- kmod/src/hash.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/hash.h b/kmod/src/hash.h index 9b169877..cb50b99c 100644 --- a/kmod/src/hash.h +++ b/kmod/src/hash.h @@ -26,11 +26,11 @@ static inline u32 fnv1a32(const void *data, unsigned int len) static inline u64 fnv1a64(const void *data, unsigned int len) { - u64 hash = 0xcbf29ce484222325; + u64 hash = 0xcbf29ce484222325ULL; while (len--) { hash ^= *(u8 *)(data++); - hash *= 0x100000001b3; + hash *= 0x100000001b3ULL; } return hash; From 68d7a2e2cb21d34f10b75ac49913123a714f8205 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Thu, 8 Oct 2020 10:44:17 -0700 Subject: [PATCH 895/920] scoutfs: align items in item cache to 8 bytes This will ensure structs, which are internally 8 byte aligned, will remain so when in the item cache. 16 bytes alignment doesn't seem like it's needed so just do 8. Signed-off-by: Andy Grover --- kmod/src/item.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kmod/src/item.c b/kmod/src/item.c index 06104ad4..c25e74a5 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -145,11 +145,11 @@ struct cached_item { char val[0]; }; -#define CACHED_ITEM_ALIGN 16 +#define CACHED_ITEM_ALIGN 8 static int item_val_bytes(int val_len) { - return offsetof(struct cached_item, val[val_len]); + return round_up(offsetof(struct cached_item, val[val_len]), CACHED_ITEM_ALIGN); } /* @@ -400,7 +400,7 @@ static struct cached_item *alloc_item(struct cached_page *pg, return NULL; item = page_address(pg->page) + pg->page_off; - pg->page_off += round_up(item_val_bytes(val_len), CACHED_ITEM_ALIGN); + pg->page_off += item_val_bytes(val_len); RB_CLEAR_NODE(&item->node); INIT_LIST_HEAD(&item->dirty_head); From 5e1c8586ccb947ffbe87592413f75006fd854518 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 14 Oct 2020 09:23:50 -0700 Subject: [PATCH 896/920] scoutfs: ensure btree values end on 8-byte-alignment boundary Round val_len up to BTREE_VALUE_ALIGN (8), to keep mid_free_len aligned. Signed-off-by: Andy Grover --- kmod/src/btree.c | 4 +++- kmod/src/format.h | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 28644c98..92a4d4da 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -88,7 +88,7 @@ enum { /* total length of the value payload */ static inline unsigned int val_bytes(unsigned val_len) { - return val_len; + return round_up(val_len, SCOUTFS_BTREE_VALUE_ALIGN); } /* number of bytes in a block used by an item with the given value length */ @@ -1024,6 +1024,8 @@ static void verify_btree_block(struct super_block *sb, goto out; } + BUILD_BUG_ON(SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES % SCOUTFS_BTREE_VALUE_ALIGN != 0); + end_off = SCOUTFS_BLOCK_LG_SIZE - (level ? 0 : SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES); diff --git a/kmod/src/format.h b/kmod/src/format.h index 0ccf78ca..1dcadbe2 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -234,14 +234,19 @@ struct scoutfs_btree_block { /* leaf blocks have a fixed size item offset hash table at the end */ } __packed; +#define SCOUTFS_BTREE_VALUE_ALIGN 8 + /* * Try to aim for a 75% load in a leaf full of items with no value. * We'll almost never see this because most items have values and most * blocks aren't full. */ -#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED \ ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_btree_block)) / \ (sizeof(struct scoutfs_btree_item) + (sizeof(__le16))) * 100 / 75) +#define SCOUTFS_BTREE_LEAF_ITEM_HASH_NR \ + (round_up(SCOUTFS_BTREE_LEAF_ITEM_HASH_NR_UNALIGNED, \ + SCOUTFS_BTREE_VALUE_ALIGN)) #define SCOUTFS_BTREE_LEAF_ITEM_HASH_BYTES \ (SCOUTFS_BTREE_LEAF_ITEM_HASH_NR * sizeof(__le16)) From d9d9b65f147e699a8630f7dcc3a221f4fd856838 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 9 Oct 2020 16:18:52 -0700 Subject: [PATCH 897/920] scoutfs: remove __packed from all struct definitions Instead, explicitly add padding field, and adjust member ordering to eliminate compiler-added padding between members, and at the end of the struct (if possible: some structs end in a u8[0] array.) This should prevent unaligned accesses. Not a big deal on x86_64, but other archs like aarch64 really want this. Signed-off-by: Andy Grover --- kmod/src/Makefile | 4 ++ kmod/src/forest.c | 2 +- kmod/src/format.h | 118 +++++++++++++++++++++++++++------------------- kmod/src/ioctl.h | 4 +- 4 files changed, 77 insertions(+), 51 deletions(-) diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 5bbee931..bd50ec38 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -55,5 +55,9 @@ $(src)/check_exported_types: echo "no raw types in exported headers, preface with __"; \ exit 1; \ fi + @if egrep '\<__packed\>' $(src)/format.h $(src)/ioctl.h; then \ + echo "no __packed allowed in exported headers"; \ + exit 1; \ + fi extra-y += check_exported_types diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 49a255e7..bac52c92 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -68,7 +68,7 @@ struct forest_info { struct forest_refs { struct scoutfs_btree_ref fs_ref; struct scoutfs_btree_ref logs_ref; -} __packed; +}; /* initialize some refs that initially aren't equal */ #define DECLARE_STALE_TRACKING_SUPER_REFS(a, b) \ diff --git a/kmod/src/format.h b/kmod/src/format.h index 1dcadbe2..ce472dd6 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -69,18 +69,21 @@ struct scoutfs_timespec { __le64 sec; __le32 nsec; -} __packed; + __u8 __pad[4]; +}; struct scoutfs_betimespec { __be64 sec; __be32 nsec; -} __packed; + __u8 __pad[4]; +}; /* XXX ipv6 */ struct scoutfs_inet_addr { __le32 addr; __le16 port; -} __packed; + __u8 __pad[2]; +}; /* * This header is stored at the start of btree blocks and the super @@ -93,7 +96,7 @@ struct scoutfs_block_header { __le64 fsid; __le64 seq; __le64 blkno; -} __packed; +}; /* * scoutfs identifies all file system metadata items by a small key @@ -109,13 +112,14 @@ struct scoutfs_block_header { * increment them, subtract them from each other, etc. */ struct scoutfs_key { - __u8 sk_zone; __le64 _sk_first; - __u8 sk_type; __le64 _sk_second; __le64 _sk_third; __u8 _sk_fourth; -}__packed; + __u8 sk_zone; + __u8 sk_type; + __u8 __pad[5]; +}; /* inode index */ #define skii_major _sk_second @@ -177,21 +181,22 @@ struct scoutfs_radix_block { __le64 seq; __le64 sm_total; __le64 lg_total; - } __packed refs[0]; + } refs[0]; __le64 bits[0]; - } __packed; -} __packed; + }; +}; struct scoutfs_avl_root { __le16 node; -} __packed; +}; struct scoutfs_avl_node { __le16 parent; __le16 left; __le16 right; __u8 height; -} __packed; + __u8 __pad[1]; +}; /* when we split we want to have multiple items on each side */ #define SCOUTFS_BTREE_MAX_VAL_LEN 896 @@ -205,7 +210,7 @@ struct scoutfs_avl_node { struct scoutfs_btree_ref { __le64 blkno; __le64 seq; -} __packed; +}; /* * A height of X means that the first block read will have level X-1 and @@ -214,14 +219,16 @@ struct scoutfs_btree_ref { struct scoutfs_btree_root { struct scoutfs_btree_ref ref; __u8 height; -} __packed; + __u8 __pad[7]; +}; struct scoutfs_btree_item { struct scoutfs_avl_node node; struct scoutfs_key key; __le16 val_off; __le16 val_len; -} __packed; + __u8 __pad[4]; +}; struct scoutfs_btree_block { struct scoutfs_block_header hdr; @@ -230,9 +237,10 @@ struct scoutfs_btree_block { __le16 total_item_bytes; __le16 mid_free_len; __u8 level; + __u8 __pad[7]; struct scoutfs_btree_item items[0]; /* leaf blocks have a fixed size item offset hash table at the end */ -} __packed; +}; #define SCOUTFS_BTREE_VALUE_ALIGN 8 @@ -253,7 +261,7 @@ struct scoutfs_btree_block { struct scoutfs_alloc_list_ref { __le64 blkno; __le64 seq; -}__packed; +}; /* * first_nr tracks the nr of the first block in the list and is used for @@ -264,7 +272,8 @@ struct scoutfs_alloc_list_head { struct scoutfs_alloc_list_ref ref; __le64 total_nr; __le32 first_nr; -}__packed; + __u8 __pad[4]; +}; /* * While the main allocator uses extent items in btree blocks, metadata @@ -283,7 +292,7 @@ struct scoutfs_alloc_list_block { __le32 start; __le32 nr; __le64 blknos[0]; /* naturally aligned for sorting */ -}__packed; +}; #define SCOUTFS_ALLOC_LIST_MAX_BLOCKS \ ((SCOUTFS_BLOCK_LG_SIZE - sizeof(struct scoutfs_alloc_list_block)) / \ @@ -295,7 +304,7 @@ struct scoutfs_alloc_list_block { struct scoutfs_alloc_root { __le64 total_len; struct scoutfs_btree_root root; -}__packed; +}; /* types of allocators, exposed to alloc_detail ioctl */ #define SCOUTFS_ALLOC_OWNER_NONE 0 @@ -305,7 +314,7 @@ struct scoutfs_alloc_root { struct scoutfs_mounted_client_btree_val { __u8 flags; -} __packed; +}; #define SCOUTFS_MOUNTED_CLIENT_VOTER (1 << 0) @@ -321,28 +330,29 @@ struct scoutfs_srch_entry { __le64 hash; __le64 ino; __le64 id; -} __packed; +}; #define SCOUTFS_SRCH_ENTRY_MAX_BYTES (2 + (sizeof(__u64) * 3)) struct scoutfs_srch_ref { __le64 blkno; __le64 seq; -} __packed; +}; struct scoutfs_srch_file { struct scoutfs_srch_entry first; struct scoutfs_srch_entry last; + struct scoutfs_srch_ref ref; __le64 blocks; __le64 entries; - struct scoutfs_srch_ref ref; __u8 height; -} __packed; + __u8 __pad[7]; +}; struct scoutfs_srch_parent { struct scoutfs_block_header hdr; struct scoutfs_srch_ref refs[0]; -} __packed; +}; #define SCOUTFS_SRCH_PARENT_REFS \ ((SCOUTFS_BLOCK_LG_SIZE - \ @@ -357,7 +367,7 @@ struct scoutfs_srch_block { __le32 entry_nr; __le32 entry_bytes; __u8 entries[0]; -} __packed; +}; /* * Decoding loads final small deltas with full __u64 loads. Rather than @@ -389,13 +399,14 @@ struct scoutfs_srch_compact { __le64 id; __u8 nr; __u8 flags; + __u8 __pad[6]; struct scoutfs_srch_file out; struct scoutfs_srch_compact_input { struct scoutfs_srch_file sfl; __le64 blk; __le64 pos; - } in[SCOUTFS_SRCH_COMPACT_NR] __packed; -} __packed; + } in[SCOUTFS_SRCH_COMPACT_NR]; +}; /* server -> client: combine input log file entries into output file */ #define SCOUTFS_SRCH_COMPACT_FLAG_LOG (1 << 0) @@ -423,7 +434,7 @@ struct scoutfs_log_trees { struct scoutfs_srch_file srch_file; __le64 rid; __le64 nr; -} __packed; +}; struct scoutfs_log_trees_val { struct scoutfs_alloc_list_head meta_avail; @@ -433,13 +444,14 @@ struct scoutfs_log_trees_val { struct scoutfs_alloc_root data_avail; struct scoutfs_alloc_root data_freed; struct scoutfs_srch_file srch_file; -} __packed; +}; struct scoutfs_log_item_value { __le64 vers; __u8 flags; + __u8 __pad[7]; __u8 data[0]; -} __packed; +}; /* * FS items are limited by the max btree value length with the log item @@ -454,7 +466,7 @@ struct scoutfs_bloom_block { struct scoutfs_block_header hdr; __le64 total_set; __le64 bits[0]; -} __packed; +}; /* * Item log trees are accompanied by a block of bits that make up a @@ -519,7 +531,8 @@ struct scoutfs_bloom_block { struct scoutfs_data_extent_val { __le64 blkno; __u8 flags; -} __packed; + __u8 __pad[7]; +}; #define SEF_OFFLINE (1 << 0) #define SEF_UNWRITTEN (1 << 1) @@ -531,10 +544,11 @@ struct scoutfs_data_extent_val { * part item and overflow into the values of the rest of the part items. */ struct scoutfs_xattr { - __u8 name_len; __le16 val_len; + __u8 name_len; + __u8 __pad[5]; __u8 name[0]; -} __packed; +}; /* XXX does this exist upstream somewhere? */ @@ -574,12 +588,13 @@ struct scoutfs_quorum_block { __le64 vote_for_rid; __le32 crc; __u8 log_nr; + __u8 __pad[3]; struct scoutfs_quorum_log { __le64 term; __le64 rid; struct scoutfs_inet_addr addr; - } __packed log[0]; -} __packed; + } log[0]; +}; #define SCOUTFS_QUORUM_LOG_MAX \ ((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \ @@ -602,6 +617,7 @@ struct scoutfs_super_block { __le64 quorum_server_term; __le64 unmount_barrier; __u8 quorum_count; + __u8 __pad[7]; struct scoutfs_inet_addr server_addr; struct scoutfs_alloc_root meta_alloc[2]; struct scoutfs_alloc_root data_alloc; @@ -613,7 +629,7 @@ struct scoutfs_super_block { struct scoutfs_btree_root trans_seqs; struct scoutfs_btree_root mounted_clients; struct scoutfs_btree_root srch_root; -} __packed; +}; #define SCOUTFS_ROOT_INO 1 @@ -662,7 +678,7 @@ struct scoutfs_inode { struct scoutfs_timespec atime; struct scoutfs_timespec ctime; struct scoutfs_timespec mtime; -} __packed; +}; #define SCOUTFS_INO_FLAG_TRUNCATE 0x1 @@ -684,8 +700,9 @@ struct scoutfs_dirent { __le64 hash; __le64 pos; __u8 type; + __u8 __pad[7]; __u8 name[0]; -} __packed; +}; #define SCOUTFS_NAME_LEN 255 @@ -753,7 +770,7 @@ struct scoutfs_net_greeting { __le64 unmount_barrier; __le64 rid; __le64 flags; -} __packed; +}; #define SCOUTFS_NET_GREETING_FLAG_FAREWELL (1 << 0) #define SCOUTFS_NET_GREETING_FLAG_VOTER (1 << 1) @@ -788,8 +805,9 @@ struct scoutfs_net_header { __u8 cmd; __u8 flags; __u8 error; + __u8 __pad[3]; __u8 data[0]; -} __packed; +}; #define SCOUTFS_NET_FLAG_RESPONSE (1 << 0) #define SCOUTFS_NET_FLAGS_UNKNOWN (U8_MAX << 1) @@ -840,30 +858,32 @@ enum { struct scoutfs_net_inode_alloc { __le64 ino; __le64 nr; -} __packed; +}; struct scoutfs_net_roots { struct scoutfs_btree_root fs_root; struct scoutfs_btree_root logs_root; struct scoutfs_btree_root srch_root; -} __packed; +}; struct scoutfs_net_lock { struct scoutfs_key key; __le64 write_version; __u8 old_mode; __u8 new_mode; -} __packed; + __u8 __pad[6]; +}; struct scoutfs_net_lock_grant_response { struct scoutfs_net_lock nl; struct scoutfs_net_roots roots; -} __packed; +}; struct scoutfs_net_lock_recover { __le16 nr; + __u8 __pad[6]; struct scoutfs_net_lock locks[0]; -} __packed; +}; #define SCOUTFS_NET_LOCK_MAX_RECOVER_NR \ ((SCOUTFS_NET_MAX_DATA_LEN - sizeof(struct scoutfs_net_lock_recover)) /\ @@ -906,7 +926,7 @@ enum { struct scoutfs_fid { __le64 ino; __le64 parent_ino; -} __packed; +}; #define FILEID_SCOUTFS 0x81 #define FILEID_SCOUTFS_WITH_PARENT 0x82 diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index f871d37e..8dc9d93c 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -371,7 +371,7 @@ struct scoutfs_ioctl_statfs_more { __u64 committed_seq; __u64 total_meta_blocks; __u64 total_data_blocks; -} __packed; +}; #define SCOUTFS_IOC_STATFS_MORE _IOR(SCOUTFS_IOCTL_MAGIC, 10, \ struct scoutfs_ioctl_statfs_more) @@ -409,6 +409,8 @@ struct scoutfs_ioctl_alloc_detail_entry { __u8 type; __u8 meta:1, avail:1; + __u8 __bit_pad:6; + __u8 __pad[6]; }; #endif From 13438c8f5df64ea8edb4761d45129f1da046d0f5 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 23 Oct 2020 14:18:52 -0700 Subject: [PATCH 898/920] scoutfs: Remove struct scoutfs_betimespec Unused. Signed-off-by: Andy Grover --- kmod/src/format.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index ce472dd6..941620b8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -72,12 +72,6 @@ struct scoutfs_timespec { __u8 __pad[4]; }; -struct scoutfs_betimespec { - __be64 sec; - __be32 nsec; - __u8 __pad[4]; -}; - /* XXX ipv6 */ struct scoutfs_inet_addr { __le32 addr; From e6228ead73ed197f9a37fd0cb6686b3b84ca1c00 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Fri, 23 Oct 2020 15:30:53 -0700 Subject: [PATCH 899/920] scoutfs: Ensure padding in structs remains zeroed Audit code for structs allocated on stack without initialization, or using kmalloc() instead of kzalloc(). - avl.c: zero padding in avl_node on insert. - btree.c: Verify item padding is zero, or WARN_ONCE. - inode.c: scoutfs_inode contains scoutfs_timespecs, which have padding. - net.c: zero pad in net header. - net.h: scoutfs_net_addr has padding, zero it in scoutfs_addr_from_sin(). - xattr.c: scoutfs_xattr has padding, zero it. - forest.c: item_root in forest_next_hint() appears to either be assigned-to or unused, so no need to zero it. - key.h: Ensure padding is zeroed in scoutfs_key_set_{zeros,ones} Signed-off-by: Andy Grover --- kmod/src/avl.c | 2 ++ kmod/src/btree.c | 6 ++++++ kmod/src/inode.c | 3 +++ kmod/src/key.h | 2 ++ kmod/src/net.c | 1 + kmod/src/net.h | 1 + kmod/src/xattr.c | 1 + 7 files changed, 16 insertions(+) diff --git a/kmod/src/avl.c b/kmod/src/avl.c index f626e2a8..98c4a0a5 100644 --- a/kmod/src/avl.c +++ b/kmod/src/avl.c @@ -11,6 +11,7 @@ * General Public License for more details. */ #include +#include #include "format.h" #include "avl.h" @@ -274,6 +275,7 @@ void scoutfs_avl_insert(struct scoutfs_avl_root *root, node->left = 0; node->right = 0; set_height(root, node); + memset(node->__pad, 0, sizeof(node->__pad)); if (parent == NULL) { root->node = node_off(root, node); diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 92a4d4da..654b7dc7 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1003,6 +1003,7 @@ static bool bad_avl_node_off(__le16 node_off, int nr) * - values don't overlap each other * - last_free_offset is in fact last free region * - call after leaf modification + * - padding is zero */ static void verify_btree_block(struct super_block *sb, struct scoutfs_btree_block *bt, int level, @@ -1017,6 +1018,7 @@ static void verify_btree_block(struct super_block *sb, int end_off; int tot = 0; int i = 0; + int j = 0; int nr; if (bt->level != level) { @@ -1055,6 +1057,10 @@ static void verify_btree_block(struct super_block *sb, goto out; } + for (j = 0; j < sizeof(item->__pad); j++) { + WARN_ON_ONCE(item->__pad[j] != 0); + } + if (scoutfs_key_compare(&item->key, start) < 0 || scoutfs_key_compare(&item->key, end) > 0) { reason = "item key out of parent range"; diff --git a/kmod/src/inode.c b/kmod/src/inode.c index 5d914159..63aa70b9 100644 --- a/kmod/src/inode.c +++ b/kmod/src/inode.c @@ -719,10 +719,13 @@ static void store_inode(struct scoutfs_inode *cinode, struct inode *inode) cinode->rdev = cpu_to_le32(inode->i_rdev); cinode->atime.sec = cpu_to_le64(inode->i_atime.tv_sec); cinode->atime.nsec = cpu_to_le32(inode->i_atime.tv_nsec); + memset(cinode->atime.__pad, 0, sizeof(cinode->atime.__pad)); cinode->ctime.sec = cpu_to_le64(inode->i_ctime.tv_sec); cinode->ctime.nsec = cpu_to_le32(inode->i_ctime.tv_nsec); + memset(cinode->ctime.__pad, 0, sizeof(cinode->ctime.__pad)); cinode->mtime.sec = cpu_to_le64(inode->i_mtime.tv_sec); cinode->mtime.nsec = cpu_to_le32(inode->i_mtime.tv_nsec); + memset(cinode->mtime.__pad, 0, sizeof(cinode->mtime.__pad)); cinode->meta_seq = cpu_to_le64(scoutfs_inode_meta_seq(inode)); cinode->data_seq = cpu_to_le64(scoutfs_inode_data_seq(inode)); diff --git a/kmod/src/key.h b/kmod/src/key.h index 76b245c9..5ea4dd4c 100644 --- a/kmod/src/key.h +++ b/kmod/src/key.h @@ -78,6 +78,7 @@ static inline void scoutfs_key_set_zeros(struct scoutfs_key *key) key->_sk_second = 0; key->_sk_third = 0; key->_sk_fourth = 0; + memset(key->__pad, 0, sizeof(key->__pad)); } static inline bool scoutfs_key_is_zeros(struct scoutfs_key *key) @@ -104,6 +105,7 @@ static inline void scoutfs_key_set_ones(struct scoutfs_key *key) key->_sk_second = cpu_to_le64(U64_MAX); key->_sk_third = cpu_to_le64(U64_MAX); key->_sk_fourth = U8_MAX; + memset(key->__pad, 0, sizeof(key->__pad)); } /* diff --git a/kmod/src/net.c b/kmod/src/net.c index 9d9d9145..0db03705 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -369,6 +369,7 @@ static int submit_send(struct super_block *sb, msend->nh.cmd = cmd; msend->nh.flags = flags; msend->nh.error = net_err; + memset(msend->nh.__pad, 0, sizeof(msend->nh.__pad)); msend->nh.data_len = cpu_to_le16(data_len); if (data_len) memcpy(msend->nh.data, data, data_len); diff --git a/kmod/src/net.h b/kmod/src/net.h index 4e2312f9..2d8ef91d 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -102,6 +102,7 @@ static inline void scoutfs_addr_from_sin(struct scoutfs_inet_addr *addr, { addr->addr = be32_to_le32(sin->sin_addr.s_addr); addr->port = be16_to_le16(sin->sin_port); + memset(addr->__pad, 0, sizeof(addr->__pad)); } struct scoutfs_net_connection * diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 921eb447..1b579132 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -573,6 +573,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, id = si->next_xattr_id++; xat->name_len = name_len; xat->val_len = cpu_to_le16(size); + memset(xat->__pad, 0, sizeof(xat->__pad)); memcpy(xat->name, name, name_len); memcpy(&xat->name[xat->name_len], value, size); } From 736d9d7df89c1bb34a77ef83f029a1de37f4d39a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 28 Oct 2020 15:58:33 -0700 Subject: [PATCH 900/920] scoutfs: remove struct scoutfs_log_trees_val The log_trees structs store the data that is used by client commits. The primary struct is communicated over the wire so it includes the rid and nr that identify the log. The _val struct was stored in btree item values and was missing the rid and nr because those were stored in the item's key. It's madness to duplicate the entire struct just to shave off those two fields. We can remove the _val struct and store the main struct in item values, including the rid and nr. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 14 ++++---- kmod/src/forest.c | 20 +++++------ kmod/src/format.h | 10 ------ kmod/src/server.c | 84 ++++++++++++++++------------------------------- kmod/src/srch.c | 8 ++--- 5 files changed, 49 insertions(+), 87 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 29607898..2eb855b3 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -1122,7 +1122,7 @@ int scoutfs_alloc_foreach(struct super_block *sb, struct scoutfs_btree_ref refs[2] = {{0,}}; struct scoutfs_super_block *super = NULL; struct scoutfs_srch_compact *sc; - struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees lt; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; int ret; @@ -1169,9 +1169,9 @@ retry: if (ret < 0) goto out; - if (iref.val_len == sizeof(ltv)) { + if (iref.val_len == sizeof(lt)) { key = *iref.key; - memcpy(<v, iref.val, sizeof(ltv)); + memcpy(<, iref.val, sizeof(lt)); } else { ret = -EIO; } @@ -1181,16 +1181,16 @@ retry: ret = cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, le64_to_cpu(key.sklt_rid), true, true, - le64_to_cpu(ltv.meta_avail.total_nr)) ?: + le64_to_cpu(lt.meta_avail.total_nr)) ?: cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, le64_to_cpu(key.sklt_rid), true, false, - le64_to_cpu(ltv.meta_freed.total_nr)) ?: + le64_to_cpu(lt.meta_freed.total_nr)) ?: cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, le64_to_cpu(key.sklt_rid), false, true, - le64_to_cpu(ltv.data_avail.total_len)) ?: + le64_to_cpu(lt.data_avail.total_len)) ?: cb(sb, arg, SCOUTFS_ALLOC_OWNER_MOUNT, le64_to_cpu(key.sklt_rid), false, false, - le64_to_cpu(ltv.data_freed.total_len)); + le64_to_cpu(lt.data_freed.total_len)); if (ret < 0) goto out; diff --git a/kmod/src/forest.c b/kmod/src/forest.c index bac52c92..2915ac47 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -135,7 +135,7 @@ int scoutfs_forest_next_hint(struct super_block *sb, struct scoutfs_key *key, DECLARE_STALE_TRACKING_SUPER_REFS(prev_refs, refs); struct scoutfs_net_roots roots; struct scoutfs_btree_root item_root; - struct scoutfs_log_trees_val *ltv; + struct scoutfs_log_trees *lt; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key found; struct scoutfs_key ltk; @@ -175,11 +175,11 @@ retry: if (ret < 0) goto out; - if (iref.val_len == sizeof(*ltv)) { + if (iref.val_len == sizeof(*lt)) { ltk = *iref.key; scoutfs_key_inc(<k); - ltv = iref.val; - item_root = ltv->item_root; + lt = iref.val; + item_root = lt->item_root; } else { ret = -EIO; } @@ -267,7 +267,7 @@ int scoutfs_forest_read_items(struct super_block *sb, .cb = cb, .cb_arg = arg, }; - struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees lt; struct scoutfs_net_roots roots; struct scoutfs_bloom_block *bb; struct forest_bloom_nrs bloom; @@ -305,9 +305,9 @@ retry: for (;; scoutfs_key_inc(<k)) { ret = scoutfs_btree_next(sb, &roots.logs_root, <k, &iref); if (ret == 0) { - if (iref.val_len == sizeof(ltv)) { + if (iref.val_len == sizeof(lt)) { ltk = *iref.key; - memcpy(<v, iref.val, sizeof(ltv)); + memcpy(<, iref.val, sizeof(lt)); } else { ret = -EIO; } @@ -319,10 +319,10 @@ retry: goto out; /* including stale */ } - if (ltv.bloom_ref.blkno == 0) + if (lt.bloom_ref.blkno == 0) continue; - bl = read_bloom_ref(sb, <v.bloom_ref); + bl = read_bloom_ref(sb, <.bloom_ref); if (IS_ERR(bl)) { ret = PTR_ERR(bl); goto out; @@ -344,7 +344,7 @@ retry: scoutfs_inc_counter(sb, forest_bloom_pass); - ret = scoutfs_btree_read_items(sb, <v.item_root, key, start, + ret = scoutfs_btree_read_items(sb, <.item_root, key, start, end, forest_read_items, &rid); if (ret < 0) goto out; diff --git a/kmod/src/format.h b/kmod/src/format.h index 941620b8..d418828b 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -430,16 +430,6 @@ struct scoutfs_log_trees { __le64 nr; }; -struct scoutfs_log_trees_val { - struct scoutfs_alloc_list_head meta_avail; - struct scoutfs_alloc_list_head meta_freed; - struct scoutfs_btree_root item_root; - struct scoutfs_btree_ref bloom_ref; - struct scoutfs_alloc_root data_avail; - struct scoutfs_alloc_root data_freed; - struct scoutfs_srch_file srch_file; -}; - struct scoutfs_log_item_value { __le64 vers; __u8 flags; diff --git a/kmod/src/server.c b/kmod/src/server.c index 7a72b3f0..3f5eb4b5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -397,7 +397,6 @@ static int server_get_log_trees(struct super_block *sb, u64 rid = scoutfs_net_client_rid(conn); DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_val ltv; struct scoutfs_log_trees lt; struct scoutfs_key key; int ret; @@ -419,9 +418,9 @@ static int server_get_log_trees(struct super_block *sb, if (ret < 0 && ret != -ENOENT) goto unlock; if (ret == 0) { - if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { key = *iref.key; - memcpy(<v, iref.val, iref.val_len); + memcpy(<, iref.val, iref.val_len); if (le64_to_cpu(key.sklt_rid) != rid) ret = -ENOENT; } else { @@ -436,20 +435,22 @@ static int server_get_log_trees(struct super_block *sb, if (ret == -ENOENT) { key.sklt_rid = cpu_to_le64(rid); key.sklt_nr = cpu_to_le64(1); - memset(<v, 0, sizeof(ltv)); + memset(<, 0, sizeof(lt)); + lt.rid = key.sklt_rid; + lt.nr = key.sklt_nr; } /* return freed to server for emptying, refill avail */ mutex_lock(&server->alloc_mutex); ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, server->other_freed, - <v.meta_freed) ?: - alloc_move_empty(sb, &super->data_alloc, <v.data_freed) ?: + <.meta_freed) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_freed) ?: scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, - <v.meta_avail, server->meta_avail, + <.meta_avail, server->meta_avail, SCOUTFS_SERVER_META_FILL_LO, SCOUTFS_SERVER_META_FILL_TARGET) ?: - alloc_move_refill(sb, <v.data_avail, &super->data_alloc, + alloc_move_refill(sb, <.data_avail, &super->data_alloc, SCOUTFS_SERVER_DATA_FILL_LO, SCOUTFS_SERVER_DATA_FILL_TARGET); mutex_unlock(&server->alloc_mutex); @@ -458,23 +459,11 @@ static int server_get_log_trees(struct super_block *sb, /* update client's log tree's item */ ret = scoutfs_btree_force(sb, &server->alloc, &server->wri, - &super->logs_root, &key, <v, sizeof(ltv)); + &super->logs_root, &key, <, sizeof(lt)); unlock: mutex_unlock(&server->logs_mutex); ret = scoutfs_server_apply_commit(sb, ret); - if (ret == 0) { - lt.meta_avail = ltv.meta_avail; - lt.meta_freed = ltv.meta_freed; - lt.item_root = ltv.item_root; - lt.bloom_ref = ltv.bloom_ref; - lt.data_avail = ltv.data_avail; - lt.data_freed = ltv.data_freed; - lt.srch_file = ltv.srch_file; - lt.rid = key.sklt_rid; - lt.nr = key.sklt_nr; - } - out: WARN_ON_ONCE(ret < 0); return scoutfs_net_response(sb, conn, cmd, id, ret, <, sizeof(lt)); @@ -493,8 +482,7 @@ static int server_commit_log_trees(struct super_block *sb, struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_val ltv; - struct scoutfs_log_trees *lt; + struct scoutfs_log_trees lt; struct scoutfs_key key; int ret; @@ -502,7 +490,9 @@ static int server_commit_log_trees(struct super_block *sb, ret = -EINVAL; goto out; } - lt = arg; + + /* don't modify the caller's log_trees */ + memcpy(<, arg, sizeof(struct scoutfs_log_trees)); ret = scoutfs_server_hold_commit(sb); if (ret < 0) { @@ -513,46 +503,28 @@ static int server_commit_log_trees(struct super_block *sb, mutex_lock(&server->logs_mutex); /* find the client's existing item */ - scoutfs_key_init_log_trees(&key, le64_to_cpu(lt->rid), - le64_to_cpu(lt->nr)); + scoutfs_key_init_log_trees(&key, le64_to_cpu(lt.rid), + le64_to_cpu(lt.nr)); ret = scoutfs_btree_lookup(sb, &super->logs_root, &key, &iref); - if (ret < 0 && ret != -ENOENT) { + if (ret < 0) { scoutfs_err(sb, "server error finding client logs: %d", ret); goto unlock; } - if (ret == 0) { - if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { - memcpy(<v, iref.val, iref.val_len); - } else { - ret = -EIO; - scoutfs_err(sb, "server error, invalid log item: %d", - ret); - } + if (ret == 0) scoutfs_btree_put_iref(&iref); - if (ret < 0) - goto unlock; - } /* try to rotate the srch log when big enough */ mutex_lock(&server->srch_mutex); ret = scoutfs_srch_rotate_log(sb, &server->alloc, &server->wri, - &super->srch_root, <->srch_file); + &super->srch_root, <.srch_file); mutex_unlock(&server->srch_mutex); if (ret < 0) { scoutfs_err(sb, "server error, rotating srch log: %d", ret); goto unlock; } - ltv.meta_avail = lt->meta_avail; - ltv.meta_freed = lt->meta_freed; - ltv.data_avail = lt->data_avail; - ltv.data_freed = lt->data_freed; - ltv.item_root = lt->item_root; - ltv.bloom_ref = lt->bloom_ref; - ltv.srch_file = lt->srch_file; - ret = scoutfs_btree_update(sb, &server->alloc, &server->wri, - &super->logs_root, &key, <v, sizeof(ltv)); + &super->logs_root, &key, <, sizeof(lt)); if (ret < 0) scoutfs_err(sb, "server error updating client logs: %d", ret); @@ -613,7 +585,7 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; DECLARE_SERVER_INFO(sb, server); SCOUTFS_BTREE_ITEM_REF(iref); - struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees lt; struct scoutfs_key key; int ret; int err; @@ -624,9 +596,9 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) scoutfs_key_init_log_trees(&key, rid, 0); ret = scoutfs_btree_next(sb, &super->logs_root, &key, &iref); if (ret == 0) { - if (iref.val_len == sizeof(struct scoutfs_log_trees_val)) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { key = *iref.key; - memcpy(<v, iref.val, iref.val_len); + memcpy(<, iref.val, iref.val_len); if (le64_to_cpu(key.sklt_rid) != rid) ret = -ENOENT; } else { @@ -648,16 +620,16 @@ static int reclaim_log_trees(struct super_block *sb, u64 rid) mutex_lock(&server->alloc_mutex); ret = scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, server->other_freed, - <v.meta_freed) ?: + <.meta_freed) ?: scoutfs_alloc_splice_list(sb, &server->alloc, &server->wri, server->other_freed, - <v.meta_avail) ?: - alloc_move_empty(sb, &super->data_alloc, <v.data_avail) ?: - alloc_move_empty(sb, &super->data_alloc, <v.data_freed); + <.meta_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_avail) ?: + alloc_move_empty(sb, &super->data_alloc, <.data_freed); mutex_unlock(&server->alloc_mutex); err = scoutfs_btree_update(sb, &server->alloc, &server->wri, - &super->logs_root, &key, <v, sizeof(ltv)); + &super->logs_root, &key, <, sizeof(lt)); BUG_ON(err != 0); /* alloc and log item roots out of sync */ out: diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 283ba208..585a2a42 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -920,7 +920,7 @@ int scoutfs_srch_search_xattrs(struct super_block *sb, struct scoutfs_srch_entry start; struct scoutfs_srch_entry end; struct scoutfs_srch_entry final; - struct scoutfs_log_trees_val ltv; + struct scoutfs_log_trees lt; struct scoutfs_srch_file sfl; SCOUTFS_BTREE_ITEM_REF(iref); struct scoutfs_key key; @@ -992,10 +992,10 @@ retry: if (ret == -ENOENT) break; if (ret == 0) { - if (iref.val_len == sizeof(ltv)) { + if (iref.val_len == sizeof(lt)) { key = *iref.key; scoutfs_key_inc(&key); - memcpy(<v, iref.val, iref.val_len); + memcpy(<, iref.val, iref.val_len); } else { ret = -EIO; } @@ -1004,7 +1004,7 @@ retry: if (ret < 0) goto out; - ret = search_file(sb, SCOUTFS_SRCH_LOG_TYPE, <v.srch_file, + ret = search_file(sb, SCOUTFS_SRCH_LOG_TYPE, <.srch_file, sroot, &start, &end, limit); if (ret < 0) goto out; From ff532eba757f952c1c1425d6da6c761657cfcf3e Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 29 Oct 2020 11:24:47 -0700 Subject: [PATCH 901/920] scoutfs: recover max lock write_version Write locks are given an increasing version number as they're granted which makes its way into items in the log btrees and is used to find the most recent version of an item. The initialization of the lock server's next write_version for granted locks dates back to the initial prototype of the forest of log btrees. It is only initialized to zero as the module is loaded. This means that reloading the module, perhaps by rebooting, resets all the item versions to 0 and can lead to newly written items being ignored in favour of older existing items with greater versions from a previous mount. To fix this we initialize the lock server's write_version to the greatest of all the versions in items in log btrees. We add a field to the log_trees struct which records the greatest version which is maintained as we write out items in transactions. These are read by the server as it starts. Then lock recovery needs to include the write_version so that the lock_server can be sure to set the next write_version past the greatest version in the currently granted locks. Signed-off-by: Zach Brown --- kmod/src/forest.c | 62 ++++++++++++++++++++++++++++++++++++++++++ kmod/src/forest.h | 4 +++ kmod/src/format.h | 1 + kmod/src/item.c | 5 ++++ kmod/src/lock.c | 1 + kmod/src/lock_server.c | 20 ++++++++++++-- kmod/src/lock_server.h | 2 +- kmod/src/server.c | 11 +++++++- 8 files changed, 101 insertions(+), 5 deletions(-) diff --git a/kmod/src/forest.c b/kmod/src/forest.c index 2915ac47..f5f259c0 100644 --- a/kmod/src/forest.c +++ b/kmod/src/forest.c @@ -481,6 +481,65 @@ out: return ret; } +/* + * The caller is commiting items in the transaction and has found the + * greatest item version amongst them. We store it in the log_trees root + * to send to the server. + */ +void scoutfs_forest_set_max_vers(struct super_block *sb, u64 max_vers) +{ + DECLARE_FOREST_INFO(sb, finf); + + finf->our_log.max_item_vers = cpu_to_le64(max_vers); +} + +/* + * The server is calling during setup to find the greatest item version + * amongst all the log tree roots. They have the authoritative current + * super. + * + * Item versions are only used to compare items in log trees, not in the + * main fs tree. All we have to do is find the greatest version amongst + * the log_trees so that new locks will have a write_version greater + * than all the items in the log_trees. + */ +int scoutfs_forest_get_max_vers(struct super_block *sb, + struct scoutfs_super_block *super, + u64 *vers) +{ + struct scoutfs_log_trees *lt; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key ltk; + int ret; + + scoutfs_key_init_log_trees(<k, 0, 0); + *vers = 0; + + for (;; scoutfs_key_inc(<k)) { + ret = scoutfs_btree_next(sb, &super->logs_root, <k, &iref); + if (ret == 0) { + if (iref.val_len == sizeof(struct scoutfs_log_trees)) { + ltk = *iref.key; + lt = iref.val; + *vers = max(*vers, + le64_to_cpu(lt->max_item_vers)); + } else { + ret = -EIO; + } + scoutfs_btree_put_iref(&iref); + } + if (ret < 0) { + if (ret == -ENOENT) + break; + goto out; + } + } + + ret = 0; +out: + return ret; +} + int scoutfs_forest_insert_list(struct super_block *sb, struct scoutfs_btree_item_list *lst) { @@ -532,9 +591,11 @@ void scoutfs_forest_init_btrees(struct super_block *sb, memset(&finf->our_log, 0, sizeof(finf->our_log)); finf->our_log.item_root = lt->item_root; finf->our_log.bloom_ref = lt->bloom_ref; + finf->our_log.max_item_vers = lt->max_item_vers; finf->our_log.rid = lt->rid; finf->our_log.nr = lt->nr; finf->srch_file = lt->srch_file; + WARN_ON_ONCE(finf->srch_bl); /* commiting should have put the block */ finf->srch_bl = NULL; @@ -560,6 +621,7 @@ void scoutfs_forest_get_btrees(struct super_block *sb, lt->item_root = finf->our_log.item_root; lt->bloom_ref = finf->our_log.bloom_ref; lt->srch_file = finf->srch_file; + lt->max_item_vers = finf->our_log.max_item_vers; scoutfs_block_put(sb, finf->srch_bl); finf->srch_bl = NULL; diff --git a/kmod/src/forest.h b/kmod/src/forest.h index e6e72a4a..b73ea7a4 100644 --- a/kmod/src/forest.h +++ b/kmod/src/forest.h @@ -23,6 +23,10 @@ int scoutfs_forest_read_items(struct super_block *sb, scoutfs_forest_item_cb cb, void *arg); int scoutfs_forest_set_bloom_bits(struct super_block *sb, struct scoutfs_lock *lock); +void scoutfs_forest_set_max_vers(struct super_block *sb, u64 max_vers); +int scoutfs_forest_get_max_vers(struct super_block *sb, + struct scoutfs_super_block *super, + u64 *vers); int scoutfs_forest_insert_list(struct super_block *sb, struct scoutfs_btree_item_list *lst); int scoutfs_forest_srch_add(struct super_block *sb, u64 hash, u64 ino, u64 id); diff --git a/kmod/src/format.h b/kmod/src/format.h index d418828b..7be325bc 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -426,6 +426,7 @@ struct scoutfs_log_trees { struct scoutfs_alloc_root data_avail; struct scoutfs_alloc_root data_freed; struct scoutfs_srch_file srch_file; + __le64 max_item_vers; __le64 rid; __le64 nr; }; diff --git a/kmod/src/item.c b/kmod/src/item.c index c25e74a5..72051828 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -2108,6 +2108,7 @@ int scoutfs_item_write_dirty(struct super_block *sb) struct page *page; LIST_HEAD(pages); LIST_HEAD(pos); + u64 max_vers = 0; int val_len; int bytes; int off; @@ -2172,6 +2173,7 @@ int scoutfs_item_write_dirty(struct super_block *sb) val_len = sizeof(item->liv) + item->val_len; bytes = offsetof(struct scoutfs_btree_item_list, val[val_len]); + max_vers = max(max_vers, le64_to_cpu(item->liv.vers)); if (off + bytes > PAGE_SIZE) { page = second; @@ -2201,6 +2203,9 @@ int scoutfs_item_write_dirty(struct super_block *sb) read_unlock(&pg->rwlock); } + /* store max item vers in forest's log_trees */ + scoutfs_forest_set_max_vers(sb, max_vers); + /* write all the dirty items into log btree blocks */ ret = scoutfs_forest_insert_list(sb, first); out: diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 7775b1ad..8f068ed7 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -901,6 +901,7 @@ int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, for (i = 0; lock && i < SCOUTFS_NET_LOCK_MAX_RECOVER_NR; i++) { nlr->locks[i].key = lock->start; + nlr->locks[i].write_version = cpu_to_le64(lock->write_version); nlr->locks[i].old_mode = lock->mode; nlr->locks[i].new_mode = lock->mode; diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index ca590635..af01f553 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -88,6 +88,8 @@ struct lock_server_info { struct scoutfs_alloc *alloc; struct scoutfs_block_writer *wri; + + atomic64_t write_version; }; #define DECLARE_LOCK_SERVER_INFO(sb, name) \ @@ -494,7 +496,6 @@ static int process_waiting_requests(struct super_block *sb, struct client_lock_entry *req_tmp; struct client_lock_entry *gr; struct client_lock_entry *gr_tmp; - static atomic64_t write_version = ATOMIC64_INIT(0); u64 wv; int ret; @@ -548,7 +549,7 @@ static int process_waiting_requests(struct super_block *sb, if (nl.new_mode == SCOUTFS_LOCK_WRITE || nl.new_mode == SCOUTFS_LOCK_WRITE_ONLY) { - wv = atomic64_inc_return(&write_version); + wv = atomic64_inc_return(&inf->write_version); nl.write_version = cpu_to_le64(wv); } @@ -674,6 +675,14 @@ static int finished_recovery(struct super_block *sb, u64 rid, bool cancel) return ret; } +static void set_max_write_version(struct lock_server_info *inf, u64 new) +{ + u64 old; + + while (new > (old = atomic64_read(&inf->write_version)) && + (atomic64_cmpxchg(&inf->write_version, old, new) != old)); +} + /* * We sent a lock recover request to the client when we received its * greeting while in recovery. Here we instantiate all the locks it @@ -737,6 +746,10 @@ int scoutfs_lock_server_recover_response(struct super_block *sb, u64 rid, scoutfs_tseq_add(&inf->tseq_tree, &clent->tseq_entry); put_server_lock(inf, snode); + + /* make sure next write lock is greater than all recovered */ + set_max_write_version(inf, + le64_to_cpu(nlr->locks[i].write_version)); } /* send request for next batch of keys */ @@ -956,7 +969,7 @@ static void lock_server_tseq_show(struct seq_file *m, */ int scoutfs_lock_server_setup(struct super_block *sb, struct scoutfs_alloc *alloc, - struct scoutfs_block_writer *wri) + struct scoutfs_block_writer *wri, u64 max_vers) { struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; @@ -981,6 +994,7 @@ int scoutfs_lock_server_setup(struct super_block *sb, scoutfs_tseq_tree_init(&inf->tseq_tree, lock_server_tseq_show); inf->alloc = alloc; inf->wri = wri; + atomic64_set(&inf->write_version, max_vers); /* inc_return gives +1 */ inf->tseq_dentry = scoutfs_tseq_create("server_locks", sbi->debug_root, &inf->tseq_tree); diff --git a/kmod/src/lock_server.h b/kmod/src/lock_server.h index c4fe5621..357fd5af 100644 --- a/kmod/src/lock_server.h +++ b/kmod/src/lock_server.h @@ -13,7 +13,7 @@ int scoutfs_lock_server_farewell(struct super_block *sb, u64 rid); int scoutfs_lock_server_setup(struct super_block *sb, struct scoutfs_alloc *alloc, - struct scoutfs_block_writer *wri); + struct scoutfs_block_writer *wri, u64 max_vers); void scoutfs_lock_server_destroy(struct super_block *sb); #endif diff --git a/kmod/src/server.c b/kmod/src/server.c index 3f5eb4b5..57a7e8d5 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -37,6 +37,7 @@ #include "trans.h" #include "srch.h" #include "alloc.h" +#include "forest.h" /* * Every active mount can act as the server that listens on a net @@ -1523,6 +1524,7 @@ static void scoutfs_server_worker(struct work_struct *work) DECLARE_WAIT_QUEUE_HEAD(waitq); struct sockaddr_in sin; LIST_HEAD(conn_list); + u64 max_vers; int ret; int err; @@ -1580,7 +1582,14 @@ static void scoutfs_server_worker(struct work_struct *work) le64_to_cpu(server->meta_avail->total_len)) swap(server->meta_avail, server->meta_freed); - ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri); + ret = scoutfs_forest_get_max_vers(sb, super, &max_vers); + if (ret) { + scoutfs_err(sb, "server couldn't find max item vers: %d", ret); + goto shutdown; + } + + ret = scoutfs_lock_server_setup(sb, &server->alloc, &server->wri, + max_vers); if (ret) goto shutdown; From 9f151fde926e59a9f09c137732bc2d84e003c0d5 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 21 Oct 2020 10:30:37 -0700 Subject: [PATCH 902/920] scoutfs: Use separate block devices for metadata and data Require a second path to metadata bdev be given via mount option. Verify meta sb matches sb also written to data sb. Change code as needed in super.c to allow both to be read. Remove check for overlapping meta and data blknos, since they are now on entirely separate bdevs. Use meta_bdev for superblock, quorum, and block.c reads and writes. Signed-off-by: Andy Grover --- kmod/src/block.c | 17 +++--- kmod/src/block.h | 6 +- kmod/src/format.h | 9 +++ kmod/src/options.c | 60 ++++++++++++++++++++ kmod/src/options.h | 2 + kmod/src/quorum.c | 16 ++++-- kmod/src/super.c | 136 ++++++++++++++++++++++++++++++++++++++------- kmod/src/super.h | 9 +++ 8 files changed, 221 insertions(+), 34 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index e730f7e5..ce79f608 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -386,6 +386,7 @@ static void block_bio_end_io(struct bio *bio, int err) static int block_submit_bio(struct super_block *sb, struct block_private *bp, int rw) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct bio *bio = NULL; struct blk_plug plug; struct page *page; @@ -414,7 +415,7 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, } bio->bi_sector = sector + (off >> 9); - bio->bi_bdev = sb->s_bdev; + bio->bi_bdev = sbi->meta_bdev; bio->bi_end_io = block_bio_end_io; bio->bi_private = bp; @@ -864,7 +865,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err) * only layer that sees the full block buffer so we pass the calculated * crc to the caller for them to check in their context. */ -static int sm_block_io(struct super_block *sb, int rw, u64 blkno, +static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { @@ -902,7 +903,7 @@ static int sm_block_io(struct super_block *sb, int rw, u64 blkno, } bio->bi_sector = blkno << (SCOUTFS_BLOCK_SM_SHIFT - 9); - bio->bi_bdev = sb->s_bdev; + bio->bi_bdev = bdev; bio->bi_end_io = sm_block_bio_end_io; bio->bi_private = &sbc; bio_add_page(bio, page, SCOUTFS_BLOCK_SM_SIZE, 0); @@ -925,17 +926,19 @@ out: return ret; } -int scoutfs_block_read_sm(struct super_block *sb, u64 blkno, +int scoutfs_block_read_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { - return sm_block_io(sb, READ, blkno, hdr, len, blk_crc); + return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc); } -int scoutfs_block_write_sm(struct super_block *sb, u64 blkno, +int scoutfs_block_write_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len) { - return sm_block_io(sb, WRITE, blkno, hdr, len, NULL); + return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL); } int scoutfs_block_setup(struct super_block *sb) diff --git a/kmod/src/block.h b/kmod/src/block.h index e1b85359..79a859d7 100644 --- a/kmod/src/block.h +++ b/kmod/src/block.h @@ -46,10 +46,12 @@ bool scoutfs_block_writer_has_dirty(struct super_block *sb, u64 scoutfs_block_writer_dirty_bytes(struct super_block *sb, struct scoutfs_block_writer *wri); -int scoutfs_block_read_sm(struct super_block *sb, u64 blkno, +int scoutfs_block_read_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc); -int scoutfs_block_write_sm(struct super_block *sb, u64 blkno, +int scoutfs_block_write_sm(struct super_block *sb, + struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len); int scoutfs_block_setup(struct super_block *sb); diff --git a/kmod/src/format.h b/kmod/src/format.h index 7be325bc..b38a55a8 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -61,6 +61,12 @@ #define SCOUTFS_QUORUM_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) #define SCOUTFS_QUORUM_BLOCKS ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) +/* + * Start data on the data device aligned as well. + */ +#define SCOUTFS_DATA_DEV_START_BLKNO ((256ULL * 1024) >> SCOUTFS_BLOCK_SM_SHIFT) + + #define SCOUTFS_UNIQUE_NAME_MAX_BYTES 64 /* includes null */ /* @@ -585,10 +591,13 @@ struct scoutfs_quorum_block { ((SCOUTFS_BLOCK_SM_SIZE - sizeof(struct scoutfs_quorum_block)) / \ sizeof(struct scoutfs_quorum_log)) +#define SCOUTFS_FLAG_IS_META_BDEV 0x01 + struct scoutfs_super_block { struct scoutfs_block_header hdr; __le64 id; __le64 format_hash; + __le64 flags; __u8 uuid[SCOUTFS_UUID_BYTES]; __le64 next_ino; __le64 next_trans_seq; diff --git a/kmod/src/options.c b/kmod/src/options.c index e7fe1843..4d698b36 100644 --- a/kmod/src/options.c +++ b/kmod/src/options.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,7 @@ static const match_table_t tokens = { {Opt_server_addr, "server_addr=%s"}, + {Opt_metadev_path, "metadev_path=%s"}, {Opt_err, NULL} }; @@ -81,6 +83,52 @@ static int parse_ipv4(struct super_block *sb, char *str, return 0; } +static int parse_bdev_path(struct super_block *sb, substring_t *substr, + char **bdev_path_ret) +{ + char *bdev_path; + struct inode *bdev_inode; + struct path path; + bool got_path = false; + int ret; + + bdev_path = match_strdup(substr); + if (!bdev_path) { + scoutfs_err(sb, "bdev string dup failed"); + ret = -ENOMEM; + goto out; + } + + ret = kern_path(bdev_path, LOOKUP_FOLLOW, &path); + if (ret) { + scoutfs_err(sb, "path %s not found for bdev: error %d", + bdev_path, ret); + goto out; + } + got_path = true; + + bdev_inode = d_inode(path.dentry); + if (!S_ISBLK(bdev_inode->i_mode)) { + scoutfs_err(sb, "path %s for bdev is not a block device", + bdev_path); + ret = -ENOTBLK; + goto out; + } + +out: + if (got_path) { + path_put(&path); + } + + if (ret < 0) { + kfree(bdev_path); + } else { + *bdev_path_ret = bdev_path; + } + + return ret; +} + int scoutfs_parse_options(struct super_block *sb, char *options, struct mount_options *parsed) { @@ -106,6 +154,13 @@ int scoutfs_parse_options(struct super_block *sb, char *options, if (ret < 0) return ret; break; + case Opt_metadev_path: + + ret = parse_bdev_path(sb, &args[0], + &parsed->metadev_path); + if (ret < 0) + return ret; + break; default: scoutfs_err(sb, "Unknown or malformed option, \"%s\"", p); @@ -113,6 +168,11 @@ int scoutfs_parse_options(struct super_block *sb, char *options, } } + if (!parsed->metadev_path) { + scoutfs_err(sb, "Required mount option \"metadev_path\" not found"); + return -EINVAL; + } + return 0; } diff --git a/kmod/src/options.h b/kmod/src/options.h index d02b40d6..aab863f8 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -7,11 +7,13 @@ enum { Opt_server_addr, + Opt_metadev_path, Opt_err, }; struct mount_options { struct sockaddr_in server_addr; + char *metadev_path; }; int scoutfs_parse_options(struct super_block *sb, char *options, diff --git a/kmod/src/quorum.c b/kmod/src/quorum.c index e1960fed..43c398d9 100644 --- a/kmod/src/quorum.c +++ b/kmod/src/quorum.c @@ -112,12 +112,13 @@ static ktime_t random_to(u32 lo, u32 hi) /* * The caller is about to read all the quorum blocks. We invalidate any * cached blocks and issue one large contiguous read to repopulate the - * cache. The caller then uses normal sb_bread to read each block. I'm + * cache. The caller then uses normal __bread to read each block. I'm * not a huge fan of the plug but I couldn't get the individual * readahead requests merged without it. */ static void readahead_quorum_blocks(struct super_block *sb) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct buffer_head *bh; struct blk_plug plug; int i; @@ -125,7 +126,8 @@ static void readahead_quorum_blocks(struct super_block *sb) blk_start_plug(&plug); for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { - bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + i); + bh = __getblk(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SIZE); if (!bh) continue; @@ -215,6 +217,7 @@ static bool stale_quorum_block(struct scoutfs_quorum_block *a, static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_quorum_block *blk; struct quorum_block_head *qbh; struct quorum_block_head *tmp; @@ -227,7 +230,8 @@ static int read_quorum_blocks(struct super_block *sb, struct list_head *blocks) for (i = 0; i < SCOUTFS_QUORUM_BLOCKS; i++) { brelse(bh); - bh = sb_bread(sb, SCOUTFS_QUORUM_BLKNO + i); + bh = __bread(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + i, + SCOUTFS_BLOCK_SM_SIZE); if (!bh) { scoutfs_inc_counter(sb, quorum_read_block_error); ret = -EIO; @@ -291,6 +295,7 @@ static int write_quorum_block(struct super_block *sb, struct scoutfs_quorum_block *our_blk) { struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); struct scoutfs_quorum_block *blk; struct buffer_head *bh = NULL; size_t size; @@ -299,8 +304,9 @@ static int write_quorum_block(struct super_block *sb, BUILD_BUG_ON(sizeof(struct scoutfs_quorum_block) > SCOUTFS_BLOCK_SM_SIZE); - bh = sb_getblk(sb, SCOUTFS_QUORUM_BLKNO + - prandom_u32_max(SCOUTFS_QUORUM_BLOCKS)); + bh = __getblk(sbi->meta_bdev, SCOUTFS_QUORUM_BLKNO + + prandom_u32_max(SCOUTFS_QUORUM_BLOCKS), + SCOUTFS_BLOCK_SM_SIZE); if (bh == NULL) { ret = -EIO; goto out; diff --git a/kmod/src/super.c b/kmod/src/super.c index 415ef4b8..23df589e 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -177,6 +177,7 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) struct mount_options *opts = &SCOUTFS_SB(sb)->opts; seq_printf(seq, ",server_addr="SIN_FMT, SIN_ARG(&opts->server_addr)); + seq_printf(seq, ",metadev_path=%s", opts->metadev_path); return 0; } @@ -205,6 +206,20 @@ static int scoutfs_sync_fs(struct super_block *sb, int wait) return scoutfs_trans_sync(sb, wait); } +/* + * Data dev is closed by generic code, but we have to explicitly close the meta + * dev. + */ +static void scoutfs_metadev_close(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + if (sbi->meta_bdev) { + blkdev_put(sbi->meta_bdev, SCOUTFS_META_BDEV_MODE); + sbi->meta_bdev = NULL; + } +} + /* * This destroys all the state that's built up in the sb info during * mount. It's called by us on errors during mount if we haven't set @@ -247,6 +262,9 @@ static void scoutfs_put_super(struct super_block *sb) debugfs_remove(sbi->debug_root); scoutfs_destroy_counters(sb); scoutfs_destroy_sysfs(sb); + scoutfs_metadev_close(sb); + + kfree(sbi->opts.metadev_path); kfree(sbi); sb->s_fs_info = NULL; @@ -271,18 +289,21 @@ static const struct super_operations scoutfs_super_ops = { int scoutfs_write_super(struct super_block *sb, struct scoutfs_super_block *super) { + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + le64_add_cpu(&super->hdr.seq, 1); - return scoutfs_block_write_sm(sb, SCOUTFS_SUPER_BLKNO, &super->hdr, + return scoutfs_block_write_sm(sb, sbi->meta_bdev, SCOUTFS_SUPER_BLKNO, + &super->hdr, sizeof(struct scoutfs_super_block)); } /* - * Read the super block. If it's valid store it in the caller's super - * struct. + * Read super, specifying bdev. */ -int scoutfs_read_super(struct super_block *sb, - struct scoutfs_super_block *super_res) +static int scoutfs_read_super_from_bdev(struct super_block *sb, + struct block_device *bdev, + struct scoutfs_super_block *super_res) { struct scoutfs_super_block *super; __le32 calc; @@ -293,9 +314,8 @@ int scoutfs_read_super(struct super_block *sb, if (!super) return -ENOMEM; - ret = scoutfs_block_read_sm(sb, SCOUTFS_SUPER_BLKNO, &super->hdr, - sizeof(struct scoutfs_super_block), - &calc); + ret = scoutfs_block_read_sm(sb, bdev, SCOUTFS_SUPER_BLKNO, &super->hdr, + sizeof(struct scoutfs_super_block), &calc); if (ret < 0) goto out; @@ -357,15 +377,6 @@ int scoutfs_read_super(struct super_block *sb, goto out; } - blkno = (le64_to_cpu(super->last_meta_blkno) + 1) << - SCOUTFS_BLOCK_SM_LG_SHIFT; - if (le64_to_cpu(super->first_data_blkno) < blkno) { - scoutfs_err(sb, "super block first data blkno %llu is within last meta blkno %llu", - le64_to_cpu(super->first_data_blkno), blkno); - ret = -EINVAL; - goto out; - } - if (le64_to_cpu(super->first_data_blkno) > le64_to_cpu(super->last_data_blkno)) { scoutfs_err(sb, "super block first data blkno %llu is greater than last data blkno %llu", @@ -384,13 +395,25 @@ int scoutfs_read_super(struct super_block *sb, goto out; } - *super_res = *super; - ret = 0; out: + if (ret == 0) + *super_res = *super; kfree(super); + return ret; } +/* + * Read the super block from meta dev. + */ +int scoutfs_read_super(struct super_block *sb, + struct scoutfs_super_block *super_res) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return scoutfs_read_super_from_bdev(sb, sbi->meta_bdev, super_res); +} + /* * This needs to be setup after reading the super because it uses the * fsid found in the super block. @@ -427,10 +450,66 @@ static int assign_random_id(struct scoutfs_sb_info *sbi) return 0; } +/* + * Ensure superblock copies in metadata and data block devices are valid, and + * fill in in-memory superblock if so. + */ +static int scoutfs_read_supers(struct super_block *sb) +{ + struct scoutfs_super_block *meta_super = NULL; + struct scoutfs_super_block *data_super = NULL; + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + int ret = 0; + + meta_super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + data_super = kmalloc(sizeof(struct scoutfs_super_block), GFP_NOFS); + if (!meta_super || !data_super) { + ret = -ENOMEM; + goto out; + } + + ret = scoutfs_read_super_from_bdev(sb, sbi->meta_bdev, meta_super); + if (ret < 0) { + scoutfs_err(sb, "could not get meta_super: error %d", ret); + goto out; + } + + ret = scoutfs_read_super_from_bdev(sb, sb->s_bdev, data_super); + if (ret < 0) { + scoutfs_err(sb, "could not get data_super: error %d", ret); + goto out; + } + + if (!SCOUTFS_IS_META_BDEV(meta_super)) { + scoutfs_err(sb, "meta_super META flag not set"); + ret = -EINVAL; + goto out; + } + + if (SCOUTFS_IS_META_BDEV(data_super)) { + scoutfs_err(sb, "data_super META flag set"); + ret = -EINVAL; + goto out; + } + + if (memcmp(meta_super->uuid, data_super->uuid, SCOUTFS_UUID_BYTES)) { + scoutfs_err(sb, "superblock UUID mismatch"); + ret = -EINVAL; + goto out; + } + + sbi->super = *meta_super; +out: + kfree(meta_super); + kfree(data_super); + return ret; +} + static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) { struct scoutfs_sb_info *sbi; struct mount_options opts; + struct block_device *meta_bdev; struct inode *inode; int ret; @@ -476,7 +555,24 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) goto out; } - ret = scoutfs_read_super(sb, &SCOUTFS_SB(sb)->super) ?: + meta_bdev = + blkdev_get_by_path(sbi->opts.metadev_path, + SCOUTFS_META_BDEV_MODE, sb); + if (IS_ERR(meta_bdev)) { + scoutfs_err(sb, "could not open metadev: error %ld", + PTR_ERR(meta_bdev)); + ret = PTR_ERR(meta_bdev); + goto out; + } + sbi->meta_bdev = meta_bdev; + ret = set_blocksize(sbi->meta_bdev, SCOUTFS_BLOCK_SM_SIZE); + if (ret != 0) { + scoutfs_err(sb, "failed to set metadev blocksize, returned %d", + ret); + goto out; + } + + ret = scoutfs_read_supers(sb) ?: scoutfs_debugfs_setup(sb) ?: scoutfs_setup_sysfs(sb) ?: scoutfs_setup_counters(sb) ?: diff --git a/kmod/src/super.h b/kmod/src/super.h index 8160a583..c5453b69 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -36,6 +36,8 @@ struct scoutfs_sb_info { struct scoutfs_super_block super; + struct block_device *meta_bdev; + spinlock_t next_ino_lock; struct data_info *data_info; @@ -94,6 +96,13 @@ static inline bool SCOUTFS_HAS_SBI(struct super_block *sb) return (sb != NULL) && (SCOUTFS_SB(sb) != NULL); } +static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) +{ + return !!(super_block->flags & SCOUTFS_FLAG_IS_META_BDEV); +} + +#define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) + /* * A small string embedded in messages that's used to identify a * specific mount. It's the three most significant bytes of the fsid From 08eb75c508ac6743a075266396e167219c8d0a84 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 19 Nov 2020 11:20:13 -0800 Subject: [PATCH 903/920] scoutfs: update README.md for metadev_path Update the README.md introduction to scoutfs to mention the need for and use of metadata and data block devices. Signed-off-by: Zach Brown --- kmod/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kmod/README.md b/kmod/README.md index 2b249c6f..0fa85e3e 100644 --- a/kmod/README.md +++ b/kmod/README.md @@ -62,17 +62,17 @@ help on the mailing list.** The requirements for running scoutfs on a small cluster are: 1. One or more nodes running x86-64 CentOS/RHEL 7.4 (or 7.3) - 2. Access to a single shared block device + 2. Access to two shared block devices 3. IPv4 connectivity between the nodes The steps for getting scoutfs mounted and operational are: 1. Get the kernel module running on the nodes - 2. Make a new filesystem on the device with the userspace utilities - 3. Mount the device on all the nodes + 2. Make a new filesystem on the devices with the userspace utilities + 3. Mount the devices on all the nodes -In this example we run all of these commands on three nodes. The block -device name is the same on all the nodes. +In this example we run all of these commands on three nodes. The names +of the block devices are the same on all the nodes. 1. Get the Kernel Module and Userspace Binaries @@ -103,7 +103,7 @@ device name is the same on all the nodes. quorum for the system to function. ```shell - scoutfs mkfs -Q 2 /dev/shared_block_device + scoutfs mkfs -Q 2 /dev/meta_dev /dev/data_dev ``` 3. Mount the Filesystem @@ -114,7 +114,7 @@ device name is the same on all the nodes. ```shell mkdir /mnt/scoutfs - mount -t scoutfs -o server_addr=$NODE_ADDR /dev/shared_block_device /mnt/scoutfs + mount -t scoutfs -o server_addr=$NODE_ADDR,metadev_path=/dev/meta_dev /dev/data_dev /mnt/scoutfs ``` 4. For Kicks, Observe the Metadata Change Index From 222e5f1b9df32c7aee959c40f44477ec9b9f47ad Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Nov 2020 12:15:13 -0800 Subject: [PATCH 904/920] scoutfs: convert endian in SCOUTFS_IS_META_BDEV We missed that flags is le64. Signed-off-by: Zach Brown --- kmod/src/super.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/super.h b/kmod/src/super.h index c5453b69..13912bdc 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -98,7 +98,7 @@ static inline bool SCOUTFS_HAS_SBI(struct super_block *sb) static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) { - return !!(super_block->flags & SCOUTFS_FLAG_IS_META_BDEV); + return !!(le64_to_cpu(super_block->flags) & SCOUTFS_FLAG_IS_META_BDEV); } #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) From 2f3d1c395e9d6feea06e7fa6117cb77df5b044e9 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 24 Nov 2020 12:15:41 -0800 Subject: [PATCH 905/920] scoutfs: show metadev_path in sysfs/mount_options We forgot to add metadev_path to the options that are found in the mount_options sysfs directory. Signed-off-by: Zach Brown --- kmod/src/super.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kmod/src/super.c b/kmod/src/super.c index 23df589e..926531d6 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -182,6 +182,16 @@ static int scoutfs_show_options(struct seq_file *seq, struct dentry *root) return 0; } +static ssize_t metadev_path_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj); + struct mount_options *opts = &SCOUTFS_SB(sb)->opts; + + return snprintf(buf, PAGE_SIZE, "%s", opts->metadev_path); +} +SCOUTFS_ATTR_RO(metadev_path); + static ssize_t server_addr_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -194,6 +204,7 @@ static ssize_t server_addr_show(struct kobject *kobj, SCOUTFS_ATTR_RO(server_addr); static struct attribute *mount_options_attrs[] = { + SCOUTFS_ATTR_PTR(metadev_path), SCOUTFS_ATTR_PTR(server_addr), NULL, }; From 73333af364458df1a441d49a8898225678e7550e Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 9 Nov 2020 09:51:57 -0800 Subject: [PATCH 906/920] scoutfs: Use enum for lock mode Signed-off-by: Andy Grover --- kmod/src/format.h | 2 +- kmod/src/lock.c | 40 ++++++++++++++++++++-------------------- kmod/src/lock.h | 18 +++++++++--------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/kmod/src/format.h b/kmod/src/format.h index b38a55a8..cf601885 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -905,7 +905,7 @@ enum { * * The null mode provides no access and is used to destroy locks. */ -enum { +enum scoutfs_lock_mode { SCOUTFS_LOCK_NULL = 0, SCOUTFS_LOCK_READ, SCOUTFS_LOCK_WRITE, diff --git a/kmod/src/lock.c b/kmod/src/lock.c index 8f068ed7..309a1152 100644 --- a/kmod/src/lock.c +++ b/kmod/src/lock.c @@ -94,17 +94,17 @@ struct lock_info { #define DECLARE_LOCK_INFO(sb, name) \ struct lock_info *name = SCOUTFS_SB(sb)->lock_info -static bool lock_mode_invalid(int mode) +static bool lock_mode_invalid(enum scoutfs_lock_mode mode) { return (unsigned)mode >= SCOUTFS_LOCK_INVALID; } -static bool lock_mode_can_read(int mode) +static bool lock_mode_can_read(enum scoutfs_lock_mode mode) { return mode == SCOUTFS_LOCK_READ || mode == SCOUTFS_LOCK_WRITE; } -static bool lock_mode_can_write(int mode) +static bool lock_mode_can_write(enum scoutfs_lock_mode mode) { return mode == SCOUTFS_LOCK_WRITE || mode == SCOUTFS_LOCK_WRITE_ONLY; } @@ -147,7 +147,7 @@ static void invalidate_inode(struct super_block *sb, u64 ino) * leave cached items behind in the case of invalidating to a read lock. */ static int lock_invalidate(struct super_block *sb, struct scoutfs_lock *lock, - int prev, int mode) + enum scoutfs_lock_mode prev, enum scoutfs_lock_mode mode) { struct scoutfs_lock_coverage *cov; struct scoutfs_lock_coverage *tmp; @@ -270,13 +270,13 @@ static struct scoutfs_lock *lock_alloc(struct super_block *sb, return lock; } -static void lock_inc_count(unsigned int *counts, int mode) +static void lock_inc_count(unsigned int *counts, enum scoutfs_lock_mode mode) { BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); counts[mode]++; } -static void lock_dec_count(unsigned int *counts, int mode) +static void lock_dec_count(unsigned int *counts, enum scoutfs_lock_mode mode) { BUG_ON(mode < 0 || mode >= SCOUTFS_LOCK_NR_MODES); counts[mode]--; @@ -288,7 +288,7 @@ static void lock_dec_count(unsigned int *counts, int mode) */ static bool lock_counts_match(int granted, unsigned int *counts) { - int mode; + enum scoutfs_lock_mode mode; for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { if (counts[mode] && !lock_modes_match(granted, mode)) @@ -305,7 +305,7 @@ static bool lock_counts_match(int granted, unsigned int *counts) */ static bool lock_count_match_exists(int desired, unsigned int *counts) { - int mode; + enum scoutfs_lock_mode mode; for (mode = 0; mode < SCOUTFS_LOCK_NR_MODES; mode++) { if (counts[mode] && lock_modes_match(desired, mode)) @@ -321,7 +321,7 @@ static bool lock_count_match_exists(int desired, unsigned int *counts) */ static bool lock_idle(struct scoutfs_lock *lock) { - int mode; + enum scoutfs_lock_mode mode; if (lock->request_pending || lock->invalidate_pending) return false; @@ -922,7 +922,7 @@ int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, } static bool lock_wait_cond(struct super_block *sb, struct scoutfs_lock *lock, - int mode) + enum scoutfs_lock_mode mode) { DECLARE_LOCK_INFO(sb, linfo); bool wake; @@ -956,7 +956,7 @@ static bool lock_flags_invalid(int flags) * won't process our request until it receives our invalidation * response. */ -static int lock_key_range(struct super_block *sb, int mode, int flags, +static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct scoutfs_key *start, struct scoutfs_key *end, struct scoutfs_lock **ret_lock) { @@ -1064,7 +1064,7 @@ out_unlock: return ret; } -int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, +int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 ino, struct scoutfs_lock **ret_lock) { struct scoutfs_key start; @@ -1089,7 +1089,7 @@ int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, * is incremented as new locks are acquired and then indicates that an * old inode with a smaller refresh_gen needs to be refreshed. */ -int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, +int scoutfs_lock_inode(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct inode *inode, struct scoutfs_lock **lock) { int ret; @@ -1152,7 +1152,7 @@ static void swap_arg(void *A, void *B, int size) * * (pretty great collision with d_lock() here) */ -int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, +int scoutfs_lock_inodes(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct inode *a, struct scoutfs_lock **a_lock, struct inode *b, struct scoutfs_lock **b_lock, struct inode *c, struct scoutfs_lock **c_lock, @@ -1200,7 +1200,7 @@ int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, /* * The rename lock is magical because it's global. */ -int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, +int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct scoutfs_lock **lock) { struct scoutfs_key key = { @@ -1247,7 +1247,7 @@ void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, * Lock the given index item. We use the index masks to calculate the * start and end key values that are covered by the lock. */ -int scoutfs_lock_inode_index(struct super_block *sb, int mode, +int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock) { @@ -1270,7 +1270,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, int mode, * able to. Maybe we have a bunch free and they're trying to allocate * and are getting ENOSPC. */ -int scoutfs_lock_rid(struct super_block *sb, int mode, int flags, +int scoutfs_lock_rid(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 rid, struct scoutfs_lock **lock) { struct scoutfs_key start; @@ -1291,7 +1291,7 @@ int scoutfs_lock_rid(struct super_block *sb, int mode, int flags, * As we unlock we always extend the grace period to give the caller * another pass at the lock before its invalidated. */ -void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, int mode) +void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scoutfs_lock_mode mode) { DECLARE_LOCK_INFO(sb, linfo); @@ -1384,7 +1384,7 @@ void scoutfs_lock_del_coverage(struct super_block *sb, * the mode and keys from changing. */ bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, - int mode) + enum scoutfs_lock_mode mode) { signed char lock_mode = ACCESS_ONCE(lock->mode); @@ -1587,7 +1587,7 @@ void scoutfs_lock_destroy(struct super_block *sb) DECLARE_LOCK_INFO(sb, linfo); struct scoutfs_lock *lock; struct rb_node *node; - int mode; + enum scoutfs_lock_mode mode; if (!linfo) return; diff --git a/kmod/src/lock.h b/kmod/src/lock.h index cb9bb3d6..b447df54 100644 --- a/kmod/src/lock.h +++ b/kmod/src/lock.h @@ -40,7 +40,7 @@ struct scoutfs_lock { spinlock_t cov_list_lock; struct list_head cov_list; - int mode; + enum scoutfs_lock_mode mode; unsigned int waiters[SCOUTFS_LOCK_NR_MODES]; unsigned int users[SCOUTFS_LOCK_NR_MODES]; @@ -63,27 +63,27 @@ int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id, int scoutfs_lock_recover_request(struct super_block *sb, u64 net_id, struct scoutfs_key *key); -int scoutfs_lock_inode(struct super_block *sb, int mode, int flags, +int scoutfs_lock_inode(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct inode *inode, struct scoutfs_lock **ret_lock); -int scoutfs_lock_ino(struct super_block *sb, int mode, int flags, u64 ino, +int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 ino, struct scoutfs_lock **ret_lock); void scoutfs_lock_get_index_item_range(u8 type, u64 major, u64 ino, struct scoutfs_key *start, struct scoutfs_key *end); -int scoutfs_lock_inode_index(struct super_block *sb, int mode, +int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode, u8 type, u64 major, u64 ino, struct scoutfs_lock **ret_lock); -int scoutfs_lock_inodes(struct super_block *sb, int mode, int flags, +int scoutfs_lock_inodes(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct inode *a, struct scoutfs_lock **a_lock, struct inode *b, struct scoutfs_lock **b_lock, struct inode *c, struct scoutfs_lock **c_lock, struct inode *d, struct scoutfs_lock **D_lock); -int scoutfs_lock_rename(struct super_block *sb, int mode, int flags, +int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, struct scoutfs_lock **lock); -int scoutfs_lock_rid(struct super_block *sb, int mode, int flags, +int scoutfs_lock_rid(struct super_block *sb, enum scoutfs_lock_mode mode, int flags, u64 rid, struct scoutfs_lock **lock); void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, - int level); + enum scoutfs_lock_mode mode); void scoutfs_lock_init_coverage(struct scoutfs_lock_coverage *cov); void scoutfs_lock_add_coverage(struct super_block *sb, @@ -94,7 +94,7 @@ bool scoutfs_lock_is_covered(struct super_block *sb, void scoutfs_lock_del_coverage(struct super_block *sb, struct scoutfs_lock_coverage *cov); bool scoutfs_lock_protected(struct scoutfs_lock *lock, struct scoutfs_key *key, - int mode); + enum scoutfs_lock_mode mode); void scoutfs_free_unused_locks(struct super_block *sb, unsigned long nr); From cf278f5fa0ff7d21e300aeae4eaa78fca6199dba Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 9 Nov 2020 10:57:19 -0800 Subject: [PATCH 907/920] scoutfs: Tidy some enum usage Prefer named to anonymous enums. This helps readability a little. Use enum as param type if possible (a couple spots). Remove unused enum in lock_server.c. Define enum spbm_flags using shift notation for consistency. Rename get_file_block()'s "gfb" parameter to "flags" for consistency. Signed-off-by: Andy Grover --- kmod/src/block.c | 2 +- kmod/src/btree.c | 2 +- kmod/src/dir.c | 6 +++--- kmod/src/format.h | 10 +++++----- kmod/src/ioctl.h | 2 +- kmod/src/item.c | 2 +- kmod/src/lock_server.c | 6 ------ kmod/src/net.c | 2 +- kmod/src/net.h | 2 +- kmod/src/options.h | 2 +- kmod/src/spbm.c | 4 ++-- kmod/src/srch.c | 14 +++++++------- kmod/src/triggers.h | 2 +- 13 files changed, 25 insertions(+), 31 deletions(-) diff --git a/kmod/src/block.c b/kmod/src/block.c index ce79f608..146925cb 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -58,7 +58,7 @@ struct block_info { #define DECLARE_BLOCK_INFO(sb, name) \ struct block_info *name = SCOUTFS_SB(sb)->block_info -enum { +enum block_status_bits { BLOCK_BIT_UPTODATE = 0, /* contents consistent with media */ BLOCK_BIT_NEW, /* newly allocated, contents undefined */ BLOCK_BIT_DIRTY, /* dirty, writer will write */ diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 654b7dc7..d4eee1cc 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -76,7 +76,7 @@ */ /* btree walking has a bunch of behavioural bit flags */ -enum { +enum btree_walk_flags { BTW_NEXT = (1 << 0), /* return >= key */ BTW_PREV = (1 << 1), /* return <= key */ BTW_DIRTY = (1 << 2), /* cow stable blocks */ diff --git a/kmod/src/dir.c b/kmod/src/dir.c index 83ab48c2..8cbc20f0 100644 --- a/kmod/src/dir.c +++ b/kmod/src/dir.c @@ -78,7 +78,7 @@ static unsigned int mode_to_type(umode_t mode) #undef S_SHIFT } -static unsigned int dentry_type(unsigned int type) +static unsigned int dentry_type(enum scoutfs_dentry_type type) { static unsigned char types[] = { [SCOUTFS_DT_FIFO] = DT_FIFO, @@ -988,12 +988,12 @@ static void init_symlink_key(struct scoutfs_key *key, u64 ino, u8 nr) * The target name can be null for deletion when val isn't used. Size * still has to be provided to determine the number of items. */ -enum { +enum symlink_ops { SYM_CREATE = 0, SYM_LOOKUP, SYM_DELETE, }; -static int symlink_item_ops(struct super_block *sb, int op, u64 ino, +static int symlink_item_ops(struct super_block *sb, enum symlink_ops op, u64 ino, struct scoutfs_lock *lock, const char *target, size_t size) { diff --git a/kmod/src/format.h b/kmod/src/format.h index cf601885..033552bf 100644 --- a/kmod/src/format.h +++ b/kmod/src/format.h @@ -708,7 +708,7 @@ struct scoutfs_dirent { /* getdents returns next pos with an entry, no entry at (f_pos)~0 */ #define SCOUTFS_DIRENT_LAST_POS (U64_MAX - 1) -enum { +enum scoutfs_dentry_type { SCOUTFS_DT_FIFO = 0, SCOUTFS_DT_CHR, SCOUTFS_DT_DIR, @@ -806,7 +806,7 @@ struct scoutfs_net_header { #define SCOUTFS_NET_FLAG_RESPONSE (1 << 0) #define SCOUTFS_NET_FLAGS_UNKNOWN (U8_MAX << 1) -enum { +enum scoutfs_net_cmd { SCOUTFS_NET_CMD_GREETING = 0, SCOUTFS_NET_CMD_ALLOC_INODES, SCOUTFS_NET_CMD_GET_LOG_TREES, @@ -836,7 +836,7 @@ enum { #undef EXPAND_NET_ERRNO #define EXPAND_NET_ERRNO(which) SCOUTFS_NET_ERR_##which, -enum { +enum scoutfs_net_errors { SCOUTFS_NET_ERR_NONE = 0, EXPAND_EACH_NET_ERRNO SCOUTFS_NET_ERR_UNKNOWN, @@ -884,7 +884,7 @@ struct scoutfs_net_lock_recover { sizeof(struct scoutfs_net_lock)) /* some enums for tracing */ -enum { +enum scoutfs_lock_trace { SLT_CLIENT, SLT_SERVER, SLT_GRANT, @@ -928,7 +928,7 @@ struct scoutfs_fid { /* * Identifiers for sources of corruption that can generate messages. */ -enum { +enum scoutfs_corruption_sources { SC_DIRENT_NAME_LEN = 0, SC_DIRENT_BACKREF_NAME_LEN, SC_DIRENT_READDIR_NAME_LEN, diff --git a/kmod/src/ioctl.h b/kmod/src/ioctl.h index 8dc9d93c..a53626a0 100644 --- a/kmod/src/ioctl.h +++ b/kmod/src/ioctl.h @@ -78,7 +78,7 @@ struct scoutfs_ioctl_walk_inodes { __u8 _pad[11]; /* padded to align walk_inodes_entry total size */ }; -enum { +enum scoutfs_ino_walk_seq_type { SCOUTFS_IOC_WALK_INODES_META_SEQ = 0, SCOUTFS_IOC_WALK_INODES_DATA_SEQ, SCOUTFS_IOC_WALK_INODES_UNKNOWN, diff --git a/kmod/src/item.c b/kmod/src/item.c index 72051828..7afd34cf 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -715,7 +715,7 @@ static void move_page_items(struct super_block *sb, } } -enum { +enum page_intersection_type { PGI_DISJOINT, PGI_INSIDE, PGI_START_OLAP, diff --git a/kmod/src/lock_server.c b/kmod/src/lock_server.c index af01f553..036bfcc4 100644 --- a/kmod/src/lock_server.c +++ b/kmod/src/lock_server.c @@ -118,12 +118,6 @@ struct server_lock_node { struct list_head invalidated; }; -enum { - CLE_GRANTED, - CLE_REQUESTED, - CLE_INVALIDATED, -}; - /* * Interactions with the client are tracked with these little mode * wrappers. diff --git a/kmod/src/net.c b/kmod/src/net.c index 0db03705..c885d2bd 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -100,7 +100,7 @@ do { \ } while (0) /* listening and their accepting sockets have a fixed locking order */ -enum { +enum spin_lock_subtype { CONN_LOCK_LISTENER, CONN_LOCK_ACCEPTED, }; diff --git a/kmod/src/net.h b/kmod/src/net.h index 2d8ef91d..05e9c3be 100644 --- a/kmod/src/net.h +++ b/kmod/src/net.h @@ -76,7 +76,7 @@ struct scoutfs_net_connection { void *info; }; -enum { +enum conn_flags { CONN_FL_valid_greeting = (1UL << 0), /* other commands can proceed */ CONN_FL_established = (1UL << 1), /* added sends queue send work */ CONN_FL_shutting_down = (1UL << 2), /* shutdown work was queued */ diff --git a/kmod/src/options.h b/kmod/src/options.h index aab863f8..b62be4d3 100644 --- a/kmod/src/options.h +++ b/kmod/src/options.h @@ -5,7 +5,7 @@ #include #include "format.h" -enum { +enum scoutfs_mount_options { Opt_server_addr, Opt_metadev_path, Opt_err, diff --git a/kmod/src/spbm.c b/kmod/src/spbm.c index 6960f65b..d2ff89eb 100644 --- a/kmod/src/spbm.c +++ b/kmod/src/spbm.c @@ -47,9 +47,9 @@ bool scoutfs_spbm_empty(struct scoutfs_spbm *spbm) return RB_EMPTY_ROOT(&spbm->root); } -enum { +enum spbm_flags { /* if a node isn't found then return an allocated new node */ - SPBM_FIND_ALLOC = 0x1, + SPBM_FIND_ALLOC = (1 << 0), }; static struct spbm_node *find_node(struct scoutfs_spbm *spbm, u64 index, int flags) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 585a2a42..bfe7a567 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -356,7 +356,7 @@ static int read_path_block(struct super_block *sb, * allocate new blocks, or return errors for missing blocks (files are * never sparse, this won't happen). */ -enum { +enum gfb_flags { GFB_INSERT = (1 << 0), GFB_DIRTY = (1 << 1), }; @@ -364,7 +364,7 @@ static int get_file_block(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, - int gfb, u64 blk, struct scoutfs_block **bl_ret) + int flags, u64 blk, struct scoutfs_block **bl_ret) { struct scoutfs_block *parent = NULL; struct scoutfs_block_header *hdr; @@ -382,7 +382,7 @@ static int get_file_block(struct super_block *sb, /* see if we need to grow to insert a new largest blk */ hei = height_for_blk(blk); while (sfl->height < hei) { - if (!(gfb & GFB_INSERT)) { + if (!(flags & GFB_INSERT)) { ret = -ENOENT; goto out; } @@ -419,8 +419,8 @@ static int get_file_block(struct super_block *sb, level = sfl->height; ref = &sfl->ref; while (level--) { - /* searchin an unused part of the tree */ - if (!ref->blkno && !(gfb & GFB_INSERT)) { + /* searching an unused part of the tree */ + if (!ref->blkno && !(flags & GFB_INSERT)) { ret = -ENOENT; goto out; } @@ -433,7 +433,7 @@ static int get_file_block(struct super_block *sb, } /* allocate a new block if we need it */ - if (!ref->blkno || ((gfb & GFB_DIRTY) && + if (!ref->blkno || ((flags & GFB_DIRTY) && !scoutfs_block_writer_is_dirty(sb, bl))) { ret = scoutfs_alloc_meta(sb, alloc, wri, &blkno); if (ret < 0) @@ -504,7 +504,7 @@ out: } /* record that we successfully grew the file */ - if (ret == 0 && (gfb & GFB_INSERT) && blk >= le64_to_cpu(sfl->blocks)) + if (ret == 0 && (flags & GFB_INSERT) && blk >= le64_to_cpu(sfl->blocks)) sfl->blocks = cpu_to_le64(blk + 1); *bl_ret = bl; diff --git a/kmod/src/triggers.h b/kmod/src/triggers.h index d1fa9e71..8796cd18 100644 --- a/kmod/src/triggers.h +++ b/kmod/src/triggers.h @@ -1,7 +1,7 @@ #ifndef _SCOUTFS_TRIGGERS_H_ #define _SCOUTFS_TRIGGERS_H_ -enum { +enum scoutfs_trigger { SCOUTFS_TRIGGER_BTREE_STALE_READ, SCOUTFS_TRIGGER_BTREE_ADVANCE_RING_HALF, SCOUTFS_TRIGGER_HARD_STALE_ERROR, From a5d9ac551499321aa00b52ad996fa6c7c05656f7 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Nov 2020 11:15:30 -0800 Subject: [PATCH 908/920] scoutfs: rework scoutfs_alloc_meta_low, takes arg Previously, scoutfs_alloc_meta_lo_thresh() returned true when a small static number of metadata blocks were either available to allocate or had space for freeing. This didn't make a lot of sense as the correct number depends on how many allocations each caller will make during their atomic transaction. Rework the call to take an argument for the number of avail or freed blocks available to test. This first pass just uses the existing number, we'll get to the callers. Signed-off-by: Zach Brown --- kmod/src/alloc.c | 22 ++++++---------------- kmod/src/alloc.h | 4 ++-- kmod/src/srch.c | 2 +- kmod/src/trans.c | 2 +- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/kmod/src/alloc.c b/kmod/src/alloc.c index 2eb855b3..3f0c6aaa 100644 --- a/kmod/src/alloc.c +++ b/kmod/src/alloc.c @@ -1085,27 +1085,17 @@ out: } /* - * Returns true if we're running low on avail blocks or running out of - * space for freed blocks. - * - * On the avail side, we're avoiding spurious enospc as our avail block - * runs low. If we commit it can be refilled by the server. - * - * On the freed side, we're avoiding getting errors in frees where they - * can't be recovered from. This is mostly in freeing cowed blocks in - * the data allocator btree which is related to its height. - * - * And both of these need to be mindful of multiple tasks entering the - * transaction. + * Returns true if meta avail and free don't have room for the given + * number of alloctions or frees. */ -bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, - struct scoutfs_alloc *alloc) +bool scoutfs_alloc_meta_low(struct super_block *sb, + struct scoutfs_alloc *alloc, u32 nr) { bool lo; spin_lock(&alloc->lock); - lo = le32_to_cpu(alloc->avail.first_nr) < 8 || - list_block_space(alloc->freed.first_nr) < 8; + lo = le32_to_cpu(alloc->avail.first_nr) < nr || + list_block_space(alloc->freed.first_nr) < nr; spin_unlock(&alloc->lock); return lo; diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index d2cc1f58..d7cffb49 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -119,8 +119,8 @@ int scoutfs_alloc_splice_list(struct super_block *sb, struct scoutfs_alloc_list_head *dst, struct scoutfs_alloc_list_head *src); -bool scoutfs_alloc_meta_lo_thresh(struct super_block *sb, - struct scoutfs_alloc *alloc); +bool scoutfs_alloc_meta_low(struct super_block *sb, + struct scoutfs_alloc *alloc, u32 nr); typedef int (*scoutfs_alloc_foreach_cb_t)(struct super_block *sb, void *arg, int owner, u64 id, diff --git a/kmod/src/srch.c b/kmod/src/srch.c index bfe7a567..2b8569bb 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1482,7 +1482,7 @@ static bool should_commit(struct super_block *sb, struct scoutfs_alloc *alloc, { return (scoutfs_block_writer_dirty_bytes(sb, wri) >= SRCH_COMPACT_DIRTY_LIMIT_BYTES) || - scoutfs_alloc_meta_lo_thresh(sb, alloc); + scoutfs_alloc_meta_low(sb, alloc, 8); } struct tourn_node { diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 9f36a19d..d029fb19 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -376,7 +376,7 @@ static bool acquired_hold(struct super_block *sb, goto out; } - if (scoutfs_alloc_meta_lo_thresh(sb, &tri->alloc)) { + if (scoutfs_alloc_meta_low(sb, &tri->alloc, 8)) { scoutfs_inc_counter(sb, trans_commit_meta_alloc_low); queue_trans_work(sbi); goto out; From ae286bf83720c4188a56dac184f2da6669b2084b Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Nov 2020 13:26:15 -0800 Subject: [PATCH 909/920] scoutfs: update srch _alloc_meta_low callers The srch system checks that is has allocator space while deleting srch files and while merging them and dirtying output blocks. Update the callers to check for the correct number of avail or freed blocks that it needs between each check. Signed-off-by: Zach Brown --- kmod/src/srch.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 2b8569bb..856d0efd 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1474,15 +1474,15 @@ out: } /* - * We're done with an operation when we have sufficient dirty blocks or - * run out of avail or freed allocator space. + * We should commit our progress when we have sufficient dirty blocks or + * don't have enough metadata alloc space for our caller's operations. */ static bool should_commit(struct super_block *sb, struct scoutfs_alloc *alloc, - struct scoutfs_block_writer *wri) + struct scoutfs_block_writer *wri, u32 nr) { return (scoutfs_block_writer_dirty_bytes(sb, wri) >= SRCH_COMPACT_DIRTY_LIMIT_BYTES) || - scoutfs_alloc_meta_low(sb, alloc, 8); + scoutfs_alloc_meta_low(sb, alloc, nr); } struct tourn_node { @@ -1569,8 +1569,8 @@ static int kway_merge(struct super_block *sb, goto out; } - /* check for committing before dirtying blocks */ - if (should_commit(sb, alloc, wri)) { + /* could grow and dirty to a leaf */ + if (should_commit(sb, alloc, wri, sfl->height + 1)) { ret = 0; goto out; } @@ -2037,7 +2037,8 @@ static int delete_file(struct super_block *sb, struct scoutfs_alloc *alloc, if (!blkno) continue; - if (should_commit(sb, alloc, wri)) { + /* free below, then final root block */ + if (should_commit(sb, alloc, wri, 2)) { ret = 0; goto out; } From 9375b9d3b7c402e3f640e9952a38336ab2547c23 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 13 Nov 2020 13:39:13 -0800 Subject: [PATCH 910/920] scoutfs: commit while enough meta for dirty items Dirty items in a client transaction are stored in OS pages. When the transaction is committed each item is stored in its position in a dirty btree block in the client's existing log btree. Allocators are refilled between transaction commits so a given commit must have sufficient meta allocator space (avail blocks and unused freed entries) for all the btree blocks that are dirtied. The number of btree blocks that are written, thus the number of cow allocations and frees, depends on the number of blocks in the log btree and the distribution of dirty items amongst those blocks. In a typical load items will be near each other and many dirty items in smaller kernel pages will be stored in fewer larger btree blocks. But with the right circumstances, the ratio of dirty pages to dirty blocks can be much smaller. With a very large directory and random entry renames you can easily have 1 btree block dirtied for every page of dirty items. Our existing allocator meta allocator fill targets and the number of dirty item cache pages we allowed did not properly take this in to account. It was possible (and, it turned out, relatively easy to test for with a hgue directory and random renames) to run out of meta avail blocks while storing dirty items in dirtied btree blocks. This rebalances our targets and thresholds to make it more likely that we'll have enough allocator resources to commit dirty items. Instead of having an arbitrary limit on the number of dirty item cache pages, we require that a given number of dirty item cache pages have a given number of allocator blocks available. We require a decent number of avialable blocks for each dirty page, so we increase the server's target number of blocks to give the client so that it can still build large transactions. This code is conservative and should not be a problem in practice, but it's theoretically possible to build a log btree and set of dirty items that would dirty more blocks that this code assumes. We will probably revisit this as we add proper support for ENOSPC. Signed-off-by: Zach Brown --- kmod/src/alloc.h | 8 ++++++-- kmod/src/item.c | 11 ++--------- kmod/src/item.h | 2 +- kmod/src/trans.c | 24 ++++++++++++++++++++---- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/kmod/src/alloc.h b/kmod/src/alloc.h index d7cffb49..da8686f7 100644 --- a/kmod/src/alloc.h +++ b/kmod/src/alloc.h @@ -41,11 +41,15 @@ /* * Fill client alloc roots to the target when they fall below the lo * threshold. + * + * We're giving the client the most available meta blocks we can so that + * it has the freedom to build large transactions before worrying that + * it might run out of meta allocs during commits. */ #define SCOUTFS_SERVER_META_FILL_TARGET \ - (256ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) + SCOUTFS_ALLOC_LIST_MAX_BLOCKS #define SCOUTFS_SERVER_META_FILL_LO \ - (64ULL * 1024 * 1024 >> SCOUTFS_BLOCK_LG_SHIFT) + (SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 2) #define SCOUTFS_SERVER_DATA_FILL_TARGET \ (4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT) #define SCOUTFS_SERVER_DATA_FILL_LO \ diff --git a/kmod/src/item.c b/kmod/src/item.c index 7afd34cf..2684a881 100644 --- a/kmod/src/item.c +++ b/kmod/src/item.c @@ -2042,18 +2042,11 @@ int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, return item_delete(sb, key, lock, SCOUTFS_LOCK_WRITE_ONLY, true); } -/* - * Give a rough idea of the number of bytes that would need to be - * written to commit the current dirty items. Reporting the total item - * dirty bytes wouldn't be accurate because they're written into btree - * pages. The number of dirty pages holding the dirty items is - * comparable. This could probably use some tuning. - */ -u64 scoutfs_item_dirty_bytes(struct super_block *sb) +u64 scoutfs_item_dirty_pages(struct super_block *sb) { DECLARE_ITEM_CACHE_INFO(sb, cinf); - return (u64)atomic_read(&cinf->dirty_pages) << PAGE_SHIFT; + return (u64)atomic_read(&cinf->dirty_pages); } static int cmp_pg_start(void *priv, struct list_head *A, struct list_head *B) diff --git a/kmod/src/item.h b/kmod/src/item.h index 726e7bdd..ae4046e7 100644 --- a/kmod/src/item.h +++ b/kmod/src/item.h @@ -24,7 +24,7 @@ int scoutfs_item_delete_force(struct super_block *sb, struct scoutfs_key *key, struct scoutfs_lock *lock); -u64 scoutfs_item_dirty_bytes(struct super_block *sb); +u64 scoutfs_item_dirty_pages(struct super_block *sb); int scoutfs_item_write_dirty(struct super_block *sb); int scoutfs_item_write_done(struct super_block *sb); bool scoutfs_item_range_cached(struct super_block *sb, diff --git a/kmod/src/trans.c b/kmod/src/trans.c index d029fb19..e024c244 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -170,7 +170,7 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); if (!scoutfs_block_writer_has_dirty(sb, &tri->wri) && - !scoutfs_item_dirty_bytes(sb)) { + !scoutfs_item_dirty_pages(sb)) { if (sbi->trans_deadline_expired) { /* * If we're not writing data then we only advance the @@ -369,14 +369,30 @@ static bool acquired_hold(struct super_block *sb, items = tri->reserved_items + cnt->items; vals = tri->reserved_vals + cnt->vals; - /* XXX arbitrarily limit to 8 meg transactions */ - if (scoutfs_item_dirty_bytes(sb) >= (8 * 1024 * 1024)) { + /* + * In theory each dirty item page could be straddling two full + * blocks, requiring 4 allocations for each item cache page. + * That's much too conservative, typically many dirty item cache + * pages that are near each other all land in one block. This + * rough estimate is still so far beyond what typically happens + * that it accounts for having to dirty parent blocks and + * whatever dirtying is done during the transaction hold. + */ + if (scoutfs_alloc_meta_low(sb, &tri->alloc, + scoutfs_item_dirty_pages(sb) * 2)) { scoutfs_inc_counter(sb, trans_commit_dirty_meta_full); queue_trans_work(sbi); goto out; } - if (scoutfs_alloc_meta_low(sb, &tri->alloc, 8)) { + /* + * Extent modifications can use meta allocators without creating + * dirty items so we have to check the meta alloc specifically. + * The size of the client's avail and freed roots are bound so + * we're unlikely to need very many block allocations per + * transaction hold. XXX This should be more precisely tuned. + */ + if (scoutfs_alloc_meta_low(sb, &tri->alloc, 16)) { scoutfs_inc_counter(sb, trans_commit_meta_alloc_low); queue_trans_work(sbi); goto out; From 1bef610416f70782e668ad3e84e8ecb2d5d309f9 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Wed, 2 Dec 2020 10:11:22 -0800 Subject: [PATCH 911/920] scoutfs: Don't destroy sroot unless srch_search_xattrs() was called Until then, sroot is uninitialized so it's not safe to call destroy_rb_root(). Signed-off-by: Andy Grover --- kmod/src/ioctl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 34d75200..7b1bb620 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -836,8 +836,10 @@ static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) ret = -EFAULT; else ret = 0; -out: + scoutfs_srch_destroy_rb_root(&sroot); + +out: kfree(name); return ret ?: total; } From 4647a6ccb20a31cf1900f5313110659664ab94ba Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 16 Nov 2020 15:44:53 -0800 Subject: [PATCH 912/920] scoutfs: fix srch btree iref puts The srch code was putting btree item refs outside of success. This is fine, but they only need to be put when btree ops return success and have set the reference. Signed-off-by: Zach Brown --- kmod/src/srch.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 856d0efd..efe66021 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -964,8 +964,8 @@ retry: } else { ret = -EIO; } + scoutfs_btree_put_iref(&iref); } - scoutfs_btree_put_iref(&iref); if (ret < 0) { if (ret == -ENOENT) { if (key.sk_type == SCOUTFS_SRCH_BLOCKS_TYPE) { @@ -999,8 +999,8 @@ retry: } else { ret = -EIO; } + scoutfs_btree_put_iref(&iref); } - scoutfs_btree_put_iref(&iref); if (ret < 0) goto out; From 560c91a0e46d7d17655145a65c4c6a327582b3de Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 17 Nov 2020 09:41:06 -0800 Subject: [PATCH 913/920] scoutfs: fix binary search for sorted srch block The search_xattrs ioctl looks for srch entries in srch files that map the caller's hashed xattr name to inodes. As it searches it maintains a range of entries that it is looking for. When it searches sorted srch files for entries it first performs a binary search for the start of the range and then iterates over the blocks until it reaches the end of its range. The binary search for the start of the range was a bit wrong. If the start of the range was less than all the blocks then the binary search could wrap the left index, try to get a file block at a negative index, and return an error for the search. This is relatively hard to hit in practice. You have to search for the xattr name with the smallest hashed value and have a sorted srch file that's just the right size so that blk offset 0 is the last block compared in the binary search, which sets the right index to -1. If there are lots of xattrs, or sorted files of the wrong length, it'll work. This fixes the binary search so that it specifically records the first block offset that intersects with the range and tests that the left and right offsets haven't been inverted. Now that we're not breaking out of the binary search loop we can more obviously put each block reference that we get. Signed-off-by: Zach Brown --- kmod/src/srch.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index efe66021..66346ea6 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -758,34 +758,45 @@ static int search_sorted_file(struct super_block *sb, struct scoutfs_block *bl = NULL; int ret = 0; int pos = 0; - u64 left; - u64 right; + s64 left; + s64 right; + u64 first; u64 blk; - /* binary search for the block that contains the start */ - blk = 0; + if (sfl->blocks == 0) + return 0; + + /* binary search for first block in the range */ + first = U64_MAX; left = 0; right = le64_to_cpu(sfl->blocks) - 1; - while (left != right) { + while (left <= right) { blk = (left + right) >> 1; - scoutfs_block_put(sb, bl); ret = get_file_block(sb, NULL, NULL, sfl, 0, blk, &bl); if (ret < 0) goto out; srb = bl->data; - if (sre_cmp(start, &srb->first) < 0) - right = --blk; - else if (sre_cmp(start, &srb->last) > 0) - left = ++blk; - else - break; + if (sre_cmp(end, &srb->first) < 0) { + right = blk - 1; + } else if (sre_cmp(start, &srb->last) > 0) { + left = blk + 1; + } else { + first = min(blk, first); + right = blk - 1; + } + + scoutfs_block_put(sb, bl); + bl = NULL; } - /* blk is the result of the search */ - scoutfs_block_put(sb, bl); - bl = NULL; + /* no blocks in range */ + if (first == U64_MAX) { + ret = 0; + goto out; + } + blk = first; /* stream entries until end or we're past the full tracking rb_root */ for (;;) { From 7c5823ad12a24f77263bd738365fd83637f25e59 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Mon, 30 Nov 2020 11:04:19 -0800 Subject: [PATCH 914/920] scoutfs: drop duplicate compacted srch entries The k-way merge used by srch file compaction only dropped the second entry in a pair of duplicate entries. Duplicate entries are both supposed to be removed so that entries for removed xattrs don't take up space in the files. This both drops the second entry and removes the first encoded entry. As we encode entries we rememeber their starting offset and the previous entry that they were encoded from. When we hit a duplicate entry we undo the encoding of the previous entry. This only works wihin srch file blocks. We can still have duplicate entries that span blocks but that's unlikely and relatively harmless. Signed-off-by: Zach Brown --- kmod/src/srch.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 66346ea6..aac56dc6 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1528,15 +1528,18 @@ static int kway_merge(struct super_block *sb, { DECLARE_SRCH_INFO(sb, srinf); struct scoutfs_srch_block *srb = NULL; + struct scoutfs_srch_entry last_tail; struct scoutfs_block *bl = NULL; struct tourn_node *tnodes; struct tourn_node *leaves; struct tourn_node *root; struct tourn_node *tn; + int last_bytes = 0; int nr_parents; int nr_nodes; int empty = 0; int ret = 0; + int diff; u64 blk; int ind; int i; @@ -1594,7 +1597,9 @@ static int kway_merge(struct super_block *sb, scoutfs_inc_counter(sb, srch_compact_dirty_block); } - if (sre_cmp(&root->sre, &sfl->last) != 0) { + if (sre_cmp(&root->sre, &srb->last) != 0) { + last_bytes = le32_to_cpu(srb->entry_bytes); + last_tail = srb->last; ret = encode_entry(srb->entries + le32_to_cpu(srb->entry_bytes), &root->sre, &srb->tail); @@ -1627,6 +1632,31 @@ static int kway_merge(struct super_block *sb, scoutfs_inc_counter(sb, srch_compact_entry); } else { + /* + * Duplicate entries indicate deletion so we + * undo the previously encoded entry and ignore + * this entry. This only happens within each + * block. Deletions can span block boundaries + * and will be filtered out by search and + * hopefully removed in future compactions. + */ + diff = le32_to_cpu(srb->entry_bytes) - last_bytes; + if (diff) { + memset(srb->entries + last_bytes, 0, diff); + if (srb->entry_bytes == 0) { + /* last_tail will be 0 */ + if (blk == 0) + sfl->first = last_tail; + srb->first = last_tail; + } + le32_add_cpu(&srb->entry_nr, -1); + srb->entry_bytes = cpu_to_le32(last_bytes); + srb->last = last_tail; + srb->tail = last_tail; + sfl->last = last_tail; + le64_add_cpu(&sfl->entries, -1); + } + scoutfs_inc_counter(sb, srch_compact_removed_entry); } From 9395360324be5fdf1a632c613ad5d1da8cc7371f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Dec 2020 09:56:45 -0800 Subject: [PATCH 915/920] scoutfs: add srch entry inc/dec We're going to need to increment and decrement srch entries in coming fixes. Signed-off-by: Zach Brown --- kmod/src/srch.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index aac56dc6..cb25c5b5 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -94,6 +94,28 @@ static int sre_cmp(const struct scoutfs_srch_entry *a, scoutfs_cmp_u64s(le64_to_cpu(a->id), le64_to_cpu(b->id)); } +static void sre_inc(struct scoutfs_srch_entry *sre) +{ + le64_add_cpu(&sre->id, 1); + if (sre->id != 0) + return; + le64_add_cpu(&sre->ino, 1); + if (sre->ino != 0) + return; + le64_add_cpu(&sre->hash, 1); +} + +static void sre_dec(struct scoutfs_srch_entry *sre) +{ + le64_add_cpu(&sre->id, -1); + if (sre->id != cpu_to_le64(U64_MAX)) + return; + le64_add_cpu(&sre->ino, -1); + if (sre->ino != cpu_to_le64(U64_MAX)) + return; + le64_add_cpu(&sre->hash, -1); +} + /* * srch items are first grouped by type and we have log files, sorted * files, and busy compactions. From 6770a3168310e67cad6a421ccdb8758865d782bf Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Dec 2020 09:47:44 -0800 Subject: [PATCH 916/920] scoutfs: consistently trim srch entry range We have to limit the number of srch entries that we'll track while performing a search for all the inodes that contain xattrs that match the search hash value. As we hit the limit on the number of entries to track we have to drop entries. As we drop entries we can't return any inodes for entries past the dropped entries. We were updating the end point of the search as we dropped entries past the tracked set, but we weren't updating the search end point if we dropped the last currently tracked entry. And we were setting the end point to the dropped entry, not to the entry before it. This could lead us to spuriously returning deleted entries if we drop the creation entry and then allow tracking its deletion later. This fixes both those problems. We now properly set the end point to just before the dropped entry for all entries that we drop. Signed-off-by: Zach Brown --- kmod/src/srch.c | 66 +++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index cb25c5b5..461ffd6f 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -615,19 +615,40 @@ out: return ret; } +/* + * The caller is dropping an ino/id because the tracking rbtree is full. + * This loses information so we can't return any entries at or after the + * one that we dropped. Update end to the entry before the dropped + * entry if it's less than the current end. + */ +static void set_end_before(struct scoutfs_srch_entry *end, u64 ino, u64 id) +{ + struct scoutfs_srch_entry sre; + + sre.hash = end->hash; + sre.ino = cpu_to_le64(ino); + sre.id = cpu_to_le64(id); + sre_dec(&sre); + if (sre_cmp(&sre, end) < 0) + *end = sre; +} + /* * Track an inode and id of an xattr hash that we found while searching. * We'll return inos from the nodes in order to userspace when we're * done searching. The first time we see the entry we track it, the - * second time must be a deletion so we remove it + * second time must be a deletion so we remove it. * - * We track the size of the pool of tracked inodes here. Once its full - * we're still able to replace greater inodes with earlier ones. We do - * that work here because we can minimize the number of traversals and - * comparisons that the caller would otherwise have to make. + * We count the number of tracked entries here. Once we hit the limit + * we drop entries which are greater than what's tracked. If we track + * new entries which are within the set then we drop the last entry. + * When we drop entries we have to trim the range of entries that we'll + * return because we've lost data. The caller will perform the search + * again from that point, giving them another window of tracked entries + * to fill from that entry. */ static int track_found(struct scoutfs_srch_rb_root *sroot, u64 ino, u64 id, - unsigned long limit) + unsigned long limit, struct scoutfs_srch_entry *end) { struct rb_node **node = &sroot->root.rb_node; struct rb_node *parent = NULL; @@ -656,8 +677,10 @@ static int track_found(struct scoutfs_srch_rb_root *sroot, u64 ino, u64 id, } /* can't track greater while we're at the limit */ - if (sroot->nr >= limit && cmp > 0 && parent == sroot->last) - return -ENOSPC; + if (sroot->nr >= limit && cmp > 0 && parent == sroot->last) { + set_end_before(end, ino, id); + return 0; + } snode = kzalloc(sizeof(*snode), GFP_NOFS); if (!snode) @@ -679,6 +702,7 @@ static int track_found(struct scoutfs_srch_rb_root *sroot, u64 ino, u64 id, snode = container_of(sroot->last, struct scoutfs_srch_rb_node, node); sroot->last = rb_prev(sroot->last); + set_end_before(end, snode->ino, snode->id); rb_erase(&snode->node, &sroot->root); kfree(snode); sroot->nr--; @@ -741,17 +765,9 @@ static int search_log_file(struct super_block *sb, continue; ret = track_found(sroot, le64_to_cpu(sre.ino), - le64_to_cpu(sre.id), limit); - if (ret < 0) { - /* have to keep searching */ - if (ret == -ENOSPC) { - if (sre_cmp(&sre, end) < 0) - *end = sre; - ret = 0; - } else { - break; - } - } + le64_to_cpu(sre.id), limit, end); + if (ret < 0) + break; } } @@ -861,17 +877,9 @@ static int search_sorted_file(struct super_block *sb, break; ret = track_found(sroot, le64_to_cpu(sre.ino), - le64_to_cpu(sre.id), limit); - if (ret < 0) { - if (ret == -ENOSPC) { - ret = 0; - /* done when we're past full rb_root */ - if (sre_cmp(&sre, end) < 0) - *end = sre; - break; - } + le64_to_cpu(sre.id), limit, end); + if (ret < 0) goto out; - } if (pos >= le32_to_cpu(srb->entry_bytes)) { scoutfs_block_put(sb, bl); From c35f1ff324588ff6dbadc3411fac74ee93c9604a Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Dec 2020 10:00:36 -0800 Subject: [PATCH 917/920] scoutfs: inc end when search xattrs retries In the rare case that searching for xattrs only finds deletions within its window it retries the search past the window. The end entry is inclusive and is the last entry that can be returned. When retrying the search we need to start from the entry after that to ensure forward progress. Signed-off-by: Zach Brown --- kmod/src/srch.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 461ffd6f..4f7116a5 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1054,6 +1054,7 @@ retry: /* keep searching if we didn't find any entries in the limit */ if (sroot->nr == 0 && sre_cmp(&end, &final) < 0) { start = end; + sre_inc(&start); scoutfs_inc_counter(sb, srch_search_retry_empty); goto retry; } From 18aee0ebbd712e5b9d7e602bc803d3182d265061 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Dec 2020 11:34:29 -0800 Subject: [PATCH 918/920] scoutfs: fix lost entries in resumed srch compact Compacting very large srch files can use all of a given operation's metadata allocator. When this happens we record the position in the srch files of the compcation in the pending item. We could lose entries when this happens because the kway_next callback would advance the srch file position as it read entries and put them in the tournament tree leaves, not as it put them in the output file. We'd continue from the entries that were next to go in the tournament leaves, not from what was in the leaves. This refactors the kway merge callbacks to differentiate between getting entries at the position and advancing the positions. We initialize the tournament leaves by getting entries at the positions and only advance the position as entries leave the tournament tree and are either stored in the output srch files or are dropped. Signed-off-by: Zach Brown --- kmod/src/srch.c | 88 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/kmod/src/srch.c b/kmod/src/srch.c index 4f7116a5..4c361dd1 100644 --- a/kmod/src/srch.c +++ b/kmod/src/srch.c @@ -1548,14 +1548,18 @@ static void tourn_update(struct tourn_node *tnodes, struct tourn_node *tn) } } -typedef int (*kway_next_func_t)(struct super_block *sb, - struct scoutfs_srch_entry *sre_ret, void *arg); +/* return the entry at the current position, can return enoent if done */ +typedef int (*kway_get_t)(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg); +/* only called after _get returns 0, advances to next entry for _get */ +typedef void (*kway_advance_t)(struct super_block *sb, void *arg); static int kway_merge(struct super_block *sb, struct scoutfs_alloc *alloc, struct scoutfs_block_writer *wri, struct scoutfs_srch_file *sfl, - kway_next_func_t kway_next, void **args, int nr) + kway_get_t kway_get, kway_advance_t kway_adv, + void **args, int nr) { DECLARE_SRCH_INFO(sb, srinf); struct scoutfs_srch_block *srb = NULL; @@ -1594,11 +1598,10 @@ static int kway_merge(struct super_block *sb, for (i = 0; i < nr; i++) { tn = &leaves[i]; tn->ind = i; - ret = kway_next(sb, &tn->sre, args[i]); + ret = kway_get(sb, &tn->sre, args[i]); if (ret == 0) { tourn_update(tnodes, &leaves[i]); } else if (ret == -ENOENT) { - memset(&tn->sre, 0xff, sizeof(tn->sre)); empty++; } else { goto out; @@ -1694,7 +1697,8 @@ static int kway_merge(struct super_block *sb, /* get the next */ ind = root->ind; tn = &leaves[ind]; - ret = kway_next(sb, &tn->sre, args[ind]); + kway_adv(sb, args[ind]); + ret = kway_get(sb, &tn->sre, args[ind]); if (ret == -ENOENT) { /* this index is done */ memset(&tn->sre, 0xff, sizeof(tn->sre)); @@ -1738,8 +1742,8 @@ static struct scoutfs_srch_entry *page_priv_sre(struct page *page) return (struct scoutfs_srch_entry *)page_address(page) + page->private; } -static int kway_next_page(struct super_block *sb, - struct scoutfs_srch_entry *sre_ret, void *arg) +static int kway_get_page(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) { struct page *page = arg; struct scoutfs_srch_entry *sre = page_priv_sre(page); @@ -1748,10 +1752,16 @@ static int kway_next_page(struct super_block *sb, return -ENOENT; *sre_ret = *sre; - page->private++; return 0; } +static void kway_adv_page(struct super_block *sb, void *arg) +{ + struct page *page = arg; + + page->private++; +} + static int cmp_page_sre(const void *A, const void *B) { const struct scoutfs_srch_entry *a = A; @@ -1901,8 +1911,8 @@ static int compact_logs(struct super_block *sb, } - ret = kway_merge(sb, alloc, wri, &sc->out, kway_next_page, args, - nr_pages); + ret = kway_merge(sb, alloc, wri, &sc->out, kway_get_page, kway_adv_page, + args, nr_pages); if (ret < 0) goto out; @@ -1932,13 +1942,15 @@ struct kway_file_reader { struct scoutfs_srch_file *sfl; struct scoutfs_block *bl; struct scoutfs_srch_entry prev; + struct scoutfs_srch_entry decoded_sre; u64 blk; u32 skip; u32 pos; + int decoded_bytes; }; -static int kway_next_file_reader(struct super_block *sb, - struct scoutfs_srch_entry *sre_ret, void *arg) +static int kway_get_reader(struct super_block *sb, + struct scoutfs_srch_entry *sre_ret, void *arg) { struct kway_file_reader *rdr = arg; struct scoutfs_srch_block *srb; @@ -1951,43 +1963,63 @@ static int kway_next_file_reader(struct super_block *sb, ret = get_file_block(sb, NULL, NULL, rdr->sfl, 0, rdr->blk, &rdr->bl); if (ret < 0) - goto out; + return ret; memset(&rdr->prev, 0, sizeof(rdr->prev)); } srb = rdr->bl->data; if (rdr->pos > SCOUTFS_SRCH_BLOCK_SAFE_BYTES || - rdr->skip > SCOUTFS_SRCH_BLOCK_SAFE_BYTES || + rdr->skip >= SCOUTFS_SRCH_BLOCK_SAFE_BYTES || rdr->skip >= le32_to_cpu(srb->entry_bytes)) { /* XXX inconsistency */ return -EIO; } /* decode entry, possibly skipping start of the block */ - do { - ret = decode_entry(srb->entries + rdr->pos, sre_ret, - &rdr->prev); + while (rdr->decoded_bytes == 0 || rdr->pos < rdr->skip) { + ret = decode_entry(srb->entries + rdr->pos, + &rdr->decoded_sre, &rdr->prev); if (ret <= 0) { /* XXX inconsistency */ return -EIO; } - rdr->prev = *sre_ret; - rdr->pos += ret; - } while (rdr->pos <= rdr->skip); - rdr->skip = 0; + rdr->decoded_bytes = ret; + if (rdr->pos < rdr->skip) { + rdr->prev = rdr->decoded_sre; + rdr->pos += ret; + if (rdr->pos >= rdr->skip) + rdr->skip = 0; + rdr->decoded_bytes = 0; + } + } + + *sre_ret = rdr->decoded_sre; + return 0; +} + +static void kway_adv_reader(struct super_block *sb, void *arg) +{ + struct kway_file_reader *rdr = arg; + struct scoutfs_srch_block *srb; + + /* _get must have set */ + BUG_ON(rdr->bl == NULL); + BUG_ON(rdr->decoded_bytes == 0); + + rdr->prev = rdr->decoded_sre; + rdr->pos += rdr->decoded_bytes; + rdr->decoded_bytes = 0; + + srb = rdr->bl->data; if (rdr->pos >= le32_to_cpu(srb->entry_bytes)) { rdr->pos = 0; scoutfs_block_put(sb, rdr->bl); rdr->bl = NULL; rdr->blk++; } - - ret = 0; -out: - return ret; } /* @@ -2032,8 +2064,8 @@ static int compact_sorted(struct super_block *sb, args[i] = &rdrs[i]; } - ret = kway_merge(sb, alloc, wri, &sc->out, kway_next_file_reader, - args, nr); + ret = kway_merge(sb, alloc, wri, &sc->out, kway_get_reader, + kway_adv_reader, args, nr); sc->flags |= SCOUTFS_SRCH_COMPACT_FLAG_DONE; for (i = 0; i < nr; i++) { From f0ddf5ff041a5e2c38d85e66ba0465a695c9d687 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 1 Dec 2020 16:34:48 -0800 Subject: [PATCH 919/920] scoutfs: search_xattrs returns each ino once Hash collisions can lead to multiple xattr ids in an inode being found for a given name hash value. If this happens we only want to return the inode number once. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index 7b1bb620..accff1ee 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -773,6 +773,7 @@ static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) struct rb_node *node; char *name = NULL; bool done = false; + u64 prev_ino = 0; u64 total = 0; int ret; @@ -819,11 +820,17 @@ static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) if (ret < 0) goto out; + prev_ino = 0; scoutfs_srch_foreach_rb_node(snode, node, &sroot) { + if (prev_ino == snode->ino) + continue; + if (put_user(snode->ino, uinos + total)) { ret = -EFAULT; break; } + prev_ino = snode->ino; + if (++total == sx.nr_inodes) break; } From e2dfffcab9b8b438086f65e2b78492a2f17f8b20 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 2 Dec 2020 09:42:22 -0800 Subject: [PATCH 920/920] scoutfs: search_xattrs name requires srch tag The search_xattrs ioctl is only going to find entries for xattrs with the .srch. tag which create srch entries as they're created and destroyed. Export the xattr tag parsing so that the ioctl can return -EINVAL for xattrs which don't have the scoutfs prefix and the .srch. tag. Signed-off-by: Zach Brown --- kmod/src/ioctl.c | 7 +++++++ kmod/src/xattr.c | 25 +++++++++++-------------- kmod/src/xattr.h | 8 ++++++++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/kmod/src/ioctl.c b/kmod/src/ioctl.c index accff1ee..cfb06462 100644 --- a/kmod/src/ioctl.c +++ b/kmod/src/ioctl.c @@ -767,6 +767,7 @@ static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) struct super_block *sb = file_inode(file)->i_sb; struct scoutfs_ioctl_search_xattrs __user *usx = (void __user *)arg; struct scoutfs_ioctl_search_xattrs sx; + struct scoutfs_xattr_prefix_tags tgs; struct scoutfs_srch_rb_root sroot; struct scoutfs_srch_rb_node *snode; u64 __user *uinos; @@ -814,6 +815,12 @@ static long scoutfs_ioc_search_xattrs(struct file *file, unsigned long arg) goto out; } + if (scoutfs_xattr_parse_tags(name, sx.name_bytes, &tgs) < 0 || + !tgs.srch) { + ret = -EINVAL; + goto out; + } + ret = scoutfs_srch_search_xattrs(sb, &sroot, scoutfs_hash64(name, sx.name_bytes), sx.next_ino, sx.last_ino, &done); diff --git a/kmod/src/xattr.c b/kmod/src/xattr.c index 1b579132..824b549b 100644 --- a/kmod/src/xattr.c +++ b/kmod/src/xattr.c @@ -94,21 +94,17 @@ static int unknown_prefix(const char *name) strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN); } -struct prefix_tags { - unsigned long hide:1, - srch:1; -}; #define HIDE_TAG "hide." #define SRCH_TAG "srch." #define TAG_LEN (sizeof(HIDE_TAG) - 1) -static int parse_tags(const char *name, unsigned int name_len, - struct prefix_tags *tgs) +int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len, + struct scoutfs_xattr_prefix_tags *tgs) { bool found; - memset(tgs, 0, sizeof(struct prefix_tags)); + memset(tgs, 0, sizeof(struct scoutfs_xattr_prefix_tags)); if ((name_len < (SCOUTFS_XATTR_PREFIX_LEN + TAG_LEN + 1)) || strncmp(name, SCOUTFS_XATTR_PREFIX, SCOUTFS_XATTR_PREFIX_LEN)) @@ -490,11 +486,11 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; const u64 ino = scoutfs_ino(inode); + struct scoutfs_xattr_prefix_tags tgs; struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; size_t name_len = strlen(name); struct scoutfs_key key; - struct prefix_tags tgs; bool undo_srch = false; LIST_HEAD(ind_locks); u8 found_parts; @@ -520,7 +516,7 @@ static int scoutfs_xattr_set(struct dentry *dentry, const char *name, if (unknown_prefix(name)) return -EOPNOTSUPP; - if (parse_tags(name, name_len, &tgs) != 0) + if (scoutfs_xattr_parse_tags(name, name_len, &tgs) != 0) return -EINVAL; if ((tgs.hide || tgs.srch) && !capable(CAP_SYS_ADMIN)) @@ -659,10 +655,10 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, { struct scoutfs_inode_info *si = SCOUTFS_I(inode); struct super_block *sb = inode->i_sb; + struct scoutfs_xattr_prefix_tags tgs; struct scoutfs_xattr *xat = NULL; struct scoutfs_lock *lck = NULL; struct scoutfs_key key; - struct prefix_tags tgs; unsigned int bytes; ssize_t total = 0; u32 name_hash = 0; @@ -698,8 +694,8 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, break; } - is_hidden = parse_tags(xat->name, xat->name_len, &tgs) == 0 && - tgs.hide; + is_hidden = scoutfs_xattr_parse_tags(xat->name, xat->name_len, + &tgs) == 0 && tgs.hide; if (show_hidden == is_hidden) { if (size) { @@ -751,10 +747,10 @@ ssize_t scoutfs_listxattr(struct dentry *dentry, char *buffer, size_t size) int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock) { + struct scoutfs_xattr_prefix_tags tgs; struct scoutfs_xattr *xat = NULL; struct scoutfs_key last; struct scoutfs_key key; - struct prefix_tags tgs; bool release = false; unsigned int bytes; u64 hash; @@ -781,7 +777,8 @@ int scoutfs_xattr_drop(struct super_block *sb, u64 ino, } if (key.skx_part != 0 || - parse_tags(xat->name, xat->name_len, &tgs) != 0) + scoutfs_xattr_parse_tags(xat->name, xat->name_len, + &tgs) != 0) memset(&tgs, 0, sizeof(tgs)); ret = scoutfs_hold_trans(sb, SIC_EXACT(2, 0)); diff --git a/kmod/src/xattr.h b/kmod/src/xattr.h index 8af0026f..39313801 100644 --- a/kmod/src/xattr.h +++ b/kmod/src/xattr.h @@ -14,4 +14,12 @@ ssize_t scoutfs_list_xattrs(struct inode *inode, char *buffer, int scoutfs_xattr_drop(struct super_block *sb, u64 ino, struct scoutfs_lock *lock); +struct scoutfs_xattr_prefix_tags { + unsigned long hide:1, + srch:1; +}; + +int scoutfs_xattr_parse_tags(const char *name, unsigned int name_len, + struct scoutfs_xattr_prefix_tags *tgs); + #endif